[{"_1":2,"_309":-5,"_310":-5},"loaderData",{"_3":4,"_91":92,"_247":248,"_308":-5},"root",{"_5":6,"_32":33,"_58":59,"_89":90},"ctx",{"_7":8,"_18":19,"_20":-5,"_21":22,"_23":24,"_25":26,"_27":26,"_28":29,"_30":31},"auth",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5},"sessionClaims","sessionId","userId","actor","orgId","orgRole","orgSlug","orgPermissions","factorVerificationAge","environmentVariableKeys",[],"user","preferences_keybindings","default","preferences_theme","light","preferences_fixers",false,"preferences_copilot","preferences_model","claude-3-5-haiku-20241022","notification_count",0,"ENV",{"_34":35,"_36":37,"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":57},"REACT_APP_SENTRY_DSN","https://586020d86f05489db63a554690acf1e9@o1398359.ingest.sentry.io/6724499","VAL_RUNTIME_NAME","prod","RENDER_GIT_COMMIT","fa80b67e1f763bfb4b247c3f5ca4f0b0730f3f5c","RENDER_GIT_BRANCH","main","CLERK_PUBLISHABLE_KEY","pk_live_Y2xlcmsudmFsLnRvd24k","POSTHOG_API_TOKEN","phc_nmoyuAkStEZuUp69rYlg1DZgrIt19Gx0Y4Hwf8tGgJP","POSTHOG_API_HOST","https://wart.val.town","WEB_API_TEMPLATE","https://{val}.web.val.run","APP_URL","https://www.val.town","MODULE_URL","https://esm.town","API_URL","https://api.val.town","WORKER_FINGERPRINT","799d99f2805306c2714a7cfefbeae178","clerkState",{"_60":61},"__internal_clerk_state",{"_62":63,"_64":43,"_65":66,"_67":66,"_68":26,"_69":66,"_70":66,"_71":66,"_72":66,"_73":66,"_74":66,"_75":66,"_76":66,"_77":78,"_87":26,"_88":26},"__clerk_ssr_state",{"_9":-5,"_10":-5,"_11":-5,"_12":-5,"_13":-5,"_14":-5,"_15":-5,"_16":-5,"_17":-5},"__publishableKey","__proxyUrl","","__domain","__isSatellite","__signInUrl","__signUpUrl","__afterSignInUrl","__afterSignUpUrl","__signInForceRedirectUrl","__signUpForceRedirectUrl","__signInFallbackRedirectUrl","__signUpFallbackRedirectUrl","__clerk_debug",{"_79":26,"_80":66,"_81":82,"_83":66,"_84":43,"_85":26,"_86":66},"isSignedIn","proxyUrl","reason","session-token-and-uat-missing","message","publishableKey","isSatellite","domain","__telemetryDisabled","__telemetryDebug","remixVersion",2,"routes/_app.v.$handle.$name",{"_93":94,"_95":96,"_94":97,"_242":-5,"_243":-5,"_244":245,"_129":130,"_142":143,"_100":101,"_98":99,"_117":-7,"_50":51,"_246":26},"type","val","author","e5615e9f-07dd-4d28-aaa1-9ee23a41135a",{"_98":99,"_95":96,"_100":101,"_102":-5,"_103":-5,"_104":105,"_106":66,"_107":108,"_109":110,"_111":26,"_112":110,"_113":114,"_115":116,"_117":118,"_119":26,"_120":31,"_121":122,"_123":31,"_124":31,"_125":31,"_126":31,"_127":-5,"_20":128,"_133":134,"_139":-7,"_140":141,"_142":143,"_144":145},"name","broadRedMarmoset","valNameId","1b90ac98-79e5-11ef-8471-de64eea55b61","parentFolderId","isOwned","privacy","public","readme","valType","http","currentVersion","19587b0c-7a7d-11ef-947a-de64eea55b61","legacyPrivate","id","code","/** @jsxImportSource https://esm.sh/react */\nimport React, { useState } from \"https://esm.sh/react\";\nimport { createRoot } from \"https://esm.sh/react-dom/client\";\n// first stable version is v22\nfunction App() {\n const [url, setUrl] = useState(\"\");\n const [result, setResult] = useState(null);\n const [loading, setLoading] = useState(false);\n\n const handleSubmit = async (e) => {\n e.preventDefault();\n setLoading(true);\n console.log(`Submitting URL for scraping: ${url}`);\n try {\n const response = await fetch(\"/scrape\", {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify({ url }),\n });\n console.log(`Received response for URL: ${url}. Status: ${response.status}`);\n const data = await response.json();\n setResult(data);\n } catch (error) {\n console.log(`Error occurred while scraping URL: ${url}. Error details: ${error.message}`);\n setResult({ error: error.message });\n }\n setLoading(false);\n };\n\n return (\n

FanFic Scraper

\n );\n}\n\nfunction client() {\n console.log(\"Initializing client-side React application\");\n createRoot(document.getElementById(\"root\")).render();\n}\n\nif (typeof document !== \"undefined\") {\n client();\n}\n\nasync function scrapePage(url) {\n console.log(`Starting to scrape page: ${url}`);\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n\n // We'll keep this part as is\n const threadId = url.split(\"/\").pop().split(\".\")[0];\n const threadmarkCategories = [19, 10, 13, 16];\n const threadmarkResults = await Promise.all(threadmarkCategories.map(async (category) => {\n const threadmarkUrl = `${url}/threadmarks?hide_wrapper=1&display=embedded&threadmark_category=${category}`;\n const response = await fetch(\n `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(threadmarkUrl)\n }&render_js=true&premium_proxy=true&stealth_proxy=true`,\n );\n if (!response.ok) {\n console.log(`Error fetching threadmarks for category ${category}`);\n return null;\n }\n // We'll log the full response text here\n const responseText = await response.text();\n console.log(`Full response for category ${category}:`, responseText);\n try {\n return JSON.parse(responseText);\n } catch (error) {\n console.log(`Error parsing JSON for category ${category}:`, error);\n return null;\n }\n }));\n\n // The rest of the function remains the same\n const extractRules = {\n title: \"h1\",\n articles: {\n selector: \".message:has(.message-cell--threadmark-header)\",\n type: \"list\",\n output: {\n threadmark: \".threadmarkLabel\",\n date: {\n selector: \".message-attribution-main time\",\n output: \"text\",\n },\n author: {\n selector: \".message-attribution-main .username\",\n output: \"text\",\n },\n },\n },\n };\n\n console.log(`Sending request to ScrapingBee for URL: ${url} at ${new Date().toISOString()}`);\n const startTime = Date.now();\n try {\n const requestUrl = `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${\n encodeURIComponent(url)\n }&render_js=true&premium_proxy=true&stealth_proxy=true&extract_rules=${\n encodeURIComponent(JSON.stringify(extractRules))\n }`;\n\n const response = await fetch(requestUrl, {\n method: \"GET\",\n });\n\n const duration = Date.now() - startTime;\n console.log(`Received response from ScrapingBee at ${new Date().toISOString()}. Duration: ${duration}ms`);\n console.log(`Received response from ScrapingBee for URL: ${url}. Status: ${response.status}`);\n\n if (!response.ok) {\n console.log(`Error response from ScrapingBee. Status: ${response.status}`);\n const errorBody = await response.text();\n console.log(`Error response body: ${errorBody}`);\n console.log(`Request headers:`, Object.fromEntries(response.headers));\n throw new Error(`HTTP error! status: ${response.status}, body: ${errorBody}`);\n }\n\n const responseText = await response.text();\n console.log(`Response size (in characters): ${responseText.length}`);\n console.log(`Full response:`, responseText);\n\n let result;\n try {\n result = JSON.parse(responseText);\n } catch (error) {\n console.log(`Error parsing JSON:`, error);\n console.log(`Response was not valid JSON:`, responseText);\n throw new Error(`Failed to parse response as JSON`);\n }\n\n console.log(`Title scraped: ${result.title}`);\n console.log(`Number of articles scraped: ${result.articles ? result.articles.length : 0}`);\n\n if (result.articles && result.articles.length > 0) {\n console.log(`Detected ${result.articles.length} threadmarked articles on this page.`);\n result.articles.forEach((article, index) => {\n console.log(`Threadmark ${index + 1}: ${article.threadmark}, Date: ${article.date}, Author: ${article.author}`);\n });\n } else {\n console.log(`No articles were scraped or no threadmarks detected.`);\n }\n\n return {\n ...result,\n threadmarks: threadmarkResults.filter(Boolean),\n };\n } catch (error) {\n console.log(`Error occurred while scraping URL: ${url}`);\n console.log(`Error details: ${error.message}`);\n console.log(`Error stack: ${error.stack}`);\n throw error;\n }\n}\n\nasync function getApiKey() {\n const apiKey = Deno.env.get(\"ScrapingBeeAPIkey\");\n if (!apiKey) {\n console.log(\"ScrapingBee API key not found in environment variables\");\n throw new Error(\"ScrapingBee API key not found in environment variables\");\n }\n return apiKey;\n}\n\nasync function server(request) {\n console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`);\n if (request.method === \"POST\" && new URL(request.url).pathname === \"/scrape\") {\n try {\n const { url } = await request.json();\n console.log(`Processing scrape request for URL: ${url}`);\n const result = await scrapePage(url);\n console.log(`Successfully scraped URL: ${url}`);\n return new Response(JSON.stringify(result), {\n headers: { \"Content-Type\": \"application/json\" },\n });\n } catch (error) {\n console.log(`Error occurred while processing request. Error details: ${error.message}`);\n console.log(`Error stack trace: ${error.stack}`);\n return new Response(JSON.stringify({ error: error.message, stack: error.stack }), {\n status: 500,\n headers: { \"Content-Type\": \"application/json\" },\n });\n }\n }\n\n console.log(\"Serving HTML page\");\n return new Response(\n `\n \n \n \n \n \n Web Scraper\n \n \n \n

\n \n \n \n \n `,\n {\n headers: { \"Content-Type\": \"text/html\" },\n },\n );\n}\n\nconst css = `\nbody {\n font-family: Arial, sans-serif;\n max-width: 800px;\n margin: 0 auto;\n padding: 20px;\n}\ninput[type=\"url\"] {\n width: 70%;\n padding: 10px;\n margin-right: 10px;\n}\nbutton {\n padding: 10px 20px;\n}\npre {\n background-color: #f4f4f4;\n padding: 10px;\n border-radius: 5px;\n white-space: pre-wrap;\n word-wrap: break-word;\n}\n`;\n\nexport default server;","createdAt",["D",1727186318648],"version",24,"currentUserHasLiked","likeCount","forkCount",1,"referenceCount","dependencyCount","commentCount","prCount","mergedPr",{"_129":130,"_131":132},"handle","willthereader","profileImageUrl","https://img.clerk.com/eyJ0eXBlIjoiZGVmYXVsdCIsImlpZCI6Imluc18yTm10cGhQUklwU0tqZHNiOHNnNGd2ZjJURFUiLCJyaWQiOiJ1c2VyXzJObXZ0bE1rbVBEMkhHNzR3SmVmMHc3eldBQiJ9","forkedVal",{"_112":135,"_129":130,"_98":136,"_117":137,"_138":-5},"00242a20-7765-11ef-960c-de64eea55b61","FanFicScraper",140,"prDisabledHint","interval","slug","willthereader/broadRedMarmoset","description","@jsxImportSource https://esm.sh/react","versions",[146,149,153,157,161,165,169,173,177,181,185,189,193,197,201,205,209,213,217,221,225,229,233,236,239],{"_117":118,"_147":-5,"_115":148,"_112":110},"prId",["D",1727186318648],{"_117":150,"_147":-5,"_115":151,"_112":152},23,["D",1727185768515],"d170f2b6-7a7b-11ef-b6c1-de64eea55b61",{"_117":154,"_147":-5,"_115":155,"_112":156},22,["D",1727125944936],"87cf2762-79f0-11ef-b27b-de64eea55b61",{"_117":158,"_147":-5,"_115":159,"_112":160},21,["D",1727125644546],"d4c34086-79ef-11ef-a979-de64eea55b61",{"_117":162,"_147":-5,"_115":163,"_112":164},20,["D",1727125318474],"1268b5e8-79ef-11ef-b891-de64eea55b61",{"_117":166,"_147":-5,"_115":167,"_112":168},19,["D",1727125025455],"63c18f7e-79ee-11ef-a196-de64eea55b61",{"_117":170,"_147":-5,"_115":171,"_112":172},18,["D",1727124897064],"173a983a-79ee-11ef-a0f4-de64eea55b61",{"_117":174,"_147":-5,"_115":175,"_112":176},17,["D",1727124771318],"cc474b2a-79ed-11ef-9ccc-de64eea55b61",{"_117":178,"_147":-5,"_115":179,"_112":180},16,["D",1727124616268],"6fdc7630-79ed-11ef-8f91-de64eea55b61",{"_117":182,"_147":-5,"_115":183,"_112":184},15,["D",1727124566955],"5277f452-79ed-11ef-9352-de64eea55b61",{"_117":186,"_147":-5,"_115":187,"_112":188},14,["D",1727124512085],"31c376d2-79ed-11ef-8f85-de64eea55b61",{"_117":190,"_147":-5,"_115":191,"_112":192},13,["D",1727124498069],"2968c352-79ed-11ef-acc7-de64eea55b61",{"_117":194,"_147":-5,"_115":195,"_112":196},12,["D",1727124252620],"971c41d6-79ec-11ef-8a04-de64eea55b61",{"_117":198,"_147":-5,"_115":199,"_112":200},11,["D",1727124245374],"92caaa00-79ec-11ef-b9b1-de64eea55b61",{"_117":202,"_147":-5,"_115":203,"_112":204},10,["D",1727123960305],"e8e061ba-79eb-11ef-9656-de64eea55b61",{"_117":206,"_147":-5,"_115":207,"_112":208},9,["D",1727123479453],"ca445c08-79ea-11ef-bd36-de64eea55b61",{"_117":210,"_147":-5,"_115":211,"_112":212},8,["D",1727122471721],"719ca792-79e8-11ef-9fcd-de64eea55b61",{"_117":214,"_147":-5,"_115":215,"_112":216},7,["D",1727122433367],"5ac01ea0-79e8-11ef-8515-de64eea55b61",{"_117":218,"_147":-5,"_115":219,"_112":220},6,["D",1727122364027],"316bac36-79e8-11ef-815c-de64eea55b61",{"_117":222,"_147":-5,"_115":223,"_112":224},5,["D",1727122338208],"220826c0-79e8-11ef-b72b-de64eea55b61",{"_117":226,"_147":-5,"_115":227,"_112":228},4,["D",1727122249493],"ed2754a8-79e7-11ef-95ca-de64eea55b61",{"_117":230,"_147":-5,"_115":231,"_112":232},3,["D",1727121629887],"7bd6ce10-79e6-11ef-ae67-de64eea55b61",{"_117":90,"_147":-5,"_115":234,"_112":235},["D",1727121579761],"5df6429a-79e6-11ef-9548-de64eea55b61",{"_117":122,"_147":-5,"_115":237,"_112":238},["D",1727121264612],"a21e7b5a-79e5-11ef-9533-de64eea55b61",{"_117":31,"_147":-5,"_115":240,"_112":241},["D",1727121038894],"1b9476fc-79e5-11ef-8471-de64eea55b61","recentThread","breadcrumbs","comments",[],"isOwner","routes/_app.v.$handle.$name.pulls._index",{"_95":96,"_94":249,"_129":130,"_142":143,"_98":99,"_304":31,"_305":31,"_50":51,"_306":307},{"_98":99,"_95":96,"_100":101,"_102":-5,"_103":-5,"_104":105,"_106":66,"_107":108,"_109":110,"_111":26,"_112":110,"_113":114,"_115":250,"_117":118,"_119":26,"_120":31,"_121":122,"_123":31,"_124":31,"_125":31,"_126":31,"_127":-5,"_20":251,"_133":252,"_139":-7,"_140":141,"_142":143,"_144":253},["D",1727186318648],{"_129":130,"_131":132},{"_112":135,"_129":130,"_98":136,"_117":137,"_138":-5},[254,256,258,260,262,264,266,268,270,272,274,276,278,280,282,284,286,288,290,292,294,296,298,300,302],{"_117":118,"_147":-5,"_115":255,"_112":110},["D",1727186318648],{"_117":150,"_147":-5,"_115":257,"_112":152},["D",1727185768515],{"_117":154,"_147":-5,"_115":259,"_112":156},["D",1727125944936],{"_117":158,"_147":-5,"_115":261,"_112":160},["D",1727125644546],{"_117":162,"_147":-5,"_115":263,"_112":164},["D",1727125318474],{"_117":166,"_147":-5,"_115":265,"_112":168},["D",1727125025455],{"_117":170,"_147":-5,"_115":267,"_112":172},["D",1727124897064],{"_117":174,"_147":-5,"_115":269,"_112":176},["D",1727124771318],{"_117":178,"_147":-5,"_115":271,"_112":180},["D",1727124616268],{"_117":182,"_147":-5,"_115":273,"_112":184},["D",1727124566955],{"_117":186,"_147":-5,"_115":275,"_112":188},["D",1727124512085],{"_117":190,"_147":-5,"_115":277,"_112":192},["D",1727124498069],{"_117":194,"_147":-5,"_115":279,"_112":196},["D",1727124252620],{"_117":198,"_147":-5,"_115":281,"_112":200},["D",1727124245374],{"_117":202,"_147":-5,"_115":283,"_112":204},["D",1727123960305],{"_117":206,"_147":-5,"_115":285,"_112":208},["D",1727123479453],{"_117":210,"_147":-5,"_115":287,"_112":212},["D",1727122471721],{"_117":214,"_147":-5,"_115":289,"_112":216},["D",1727122433367],{"_117":218,"_147":-5,"_115":291,"_112":220},["D",1727122364027],{"_117":222,"_147":-5,"_115":293,"_112":224},["D",1727122338208],{"_117":226,"_147":-5,"_115":295,"_112":228},["D",1727122249493],{"_117":230,"_147":-5,"_115":297,"_112":232},["D",1727121629887],{"_117":90,"_147":-5,"_115":299,"_112":235},["D",1727121579761],{"_117":122,"_147":-5,"_115":301,"_112":238},["D",1727121264612],{"_117":31,"_147":-5,"_115":303,"_112":241},["D",1727121038894],"page","count","prs",[],"routes/_app","actionData","errors"]

FanFic Scraper

Result:

{result.title}

Articles:

Threadmarks:

Threadmark: {article.threadmark}