Public
Script
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
import cheerio from "https://esm.sh/cheerio@1.0.0-rc.10";
import { fetch } from "https://esm.town/v/std/fetch";
// Helper function to retrieve CSRF token if needed
async function getCsrfToken(url, cookies) {
try {
console.log("Fetching CSRF Token from URL:", url);
console.log("Using Cookies:", cookies);
const response = await fetch(url, {
headers: {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.113 Safari/537.36",
"Referer": "https://forums.spacebattles.com/",
"Accept-Language": "en-US,en;q=0.9",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Cookie": cookies,
},
});
console.log(`HTTP Response Status (CSRF Token Request): ${response.status}`);
if (!response.ok) {
throw new Error(`HTTP error! Status: ${response.status}`);
}
const html = await response.text();
console.log("Fetched HTML (CSRF Token Request):", html);
const $ = cheerio.load(html);
const csrfToken = $("meta[name=\"csrf-token\"]").attr("content");
console.log("Extracted CSRF Token:", csrfToken);
return csrfToken || "";
} catch (error) {
console.error("Error fetching CSRF token:", error);
return "";
}
}
// Function to fetch HTML content and extract titles
async function titleExtractor(url, cookies, csrfToken) {
try {
console.log("Fetching URL:", url);
console.log("Using CSRF Token:", csrfToken);
const encodedUrl = encodeURI(url);
await new Promise(resolve => setTimeout(resolve, Math.floor(Math.random() * 1000) + 500));
const response = await fetch(encodedUrl, {
headers: {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.113 Safari/537.36",
"Referer": "https://forums.spacebattles.com/",
"Accept-Language": "en-US,en;q=0.9",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Cookie": cookies,
"X-CSRF-Token": csrfToken,
},
});
console.log(`HTTP Response Status: ${response.status}`);
if (!response.ok) {
throw new Error(`HTTP error! Status: ${response.status}`);
}
const html = await response.text();
console.log("Fetched HTML:", html);
const $ = cheerio.load(html);
const titleElements = $(".contentRow-title a[href*='threads/']");
const titles = titleElements.map((i, element) => $(element).attr("href").trim()).get();
console.log("Extracted Titles:", titles);
return titles;
} catch (error) {
console.error("Error fetching or parsing HTML:", error);
return [];
}
}
// Example usage
(async () => {
try {
const url = "https://forums.spacebattles.com/search/69363141/?t=post&c[content]=thread&c[users]=3ndless&o=date";
const cookies = "your_cookie_string_here"; // Replace with actual cookies from an authenticated session
const csrfToken = await getCsrfToken(url, cookies); // Retrieve CSRF token if needed
if (!csrfToken) {
throw new Error("Failed to retrieve CSRF token");
}
const titles = await titleExtractor(url, cookies, csrfToken);
console.log("Final Extracted Titles:", titles);
} catch (error) {
console.error("Error in example usage:", error);
}
})();
Val Town is a social website to write and deploy JavaScript.
Build APIs and schedule functions from your browser.
Comments
Nobody has commented on this val yet: be the first!
May 29, 2024