import cheerio from "https://esm.sh/cheerio@1.0.0-rc.10";
import { fetch } from "https://esm.town/v/std/fetch";
async function getCsrfToken(url, cookies) {
try {
console.log("Fetching CSRF Token from URL:", url);
console.log("Using Cookies:", cookies);
const response = await fetch(url, {
headers: {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.113 Safari/537.36",
"Referer": "https://forums.spacebattles.com/",
"Accept-Language": "en-US,en;q=0.9",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Cookie": cookies,
},
});
console.log(`HTTP Response Status (CSRF Token Request): ${response.status}`);
if (!response.ok) {
throw new Error(`HTTP error! Status: ${response.status}`);
}
const html = await response.text();
console.log("Fetched HTML (CSRF Token Request):", html);
const $ = cheerio.load(html);
const csrfToken = $("meta[name=\"csrf-token\"]").attr("content");
console.log("Extracted CSRF Token:", csrfToken);
return csrfToken || "";
} catch (error) {
console.error("Error fetching CSRF token:", error);
return "";
}
}
async function titleExtractor(url, cookies, csrfToken) {
try {
console.log("Fetching URL:", url);
console.log("Using CSRF Token:", csrfToken);
const encodedUrl = encodeURI(url);
await new Promise(resolve => setTimeout(resolve, Math.floor(Math.random() * 1000) + 500));
const response = await fetch(encodedUrl, {
headers: {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.113 Safari/537.36",
"Referer": "https://forums.spacebattles.com/",
"Accept-Language": "en-US,en;q=0.9",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Cookie": cookies,
"X-CSRF-Token": csrfToken,
},
});
console.log(`HTTP Response Status: ${response.status}`);
if (!response.ok) {
throw new Error(`HTTP error! Status: ${response.status}`);
}
const html = await response.text();
console.log("Fetched HTML:", html);
const $ = cheerio.load(html);
const titleElements = $(".contentRow-title a[href*='threads/']");
const titles = titleElements.map((i, element) => $(element).attr("href").trim()).get();
console.log("Extracted Titles:", titles);
return titles;
} catch (error) {
console.error("Error fetching or parsing HTML:", error);
return [];
}
}
(async () => {
try {
const url = "https://forums.spacebattles.com/search/69363141/?t=post&c[content]=thread&c[users]=3ndless&o=date";
const cookies = "your_cookie_string_here";
const csrfToken = await getCsrfToken(url, cookies);
if (!csrfToken) {
throw new Error("Failed to retrieve CSRF token");
}
const titles = await titleExtractor(url, cookies, csrfToken);
console.log("Final Extracted Titles:", titles);
} catch (error) {
console.error("Error in example usage:", error);
}
})();