/** @jsxImportSource https://esm.sh/react */ import React, { useState } from "https://esm.sh/react"; import { createRoot } from "https://esm.sh/react-dom/client"; // last stable version is v138 function App() { const [url, setUrl] = useState(""); const [result, setResult] = useState(null); const [loading, setLoading] = useState(false); const handleSubmit = async (e) => { e.preventDefault(); setLoading(true); console.log(`Submitting URL for scraping: ${url}`); try { const response = await fetch("/scrape", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ url }), }); console.log(`Received response for URL: ${url}. Status: ${response.status}`); const data = await response.json(); setResult(data); } catch (error) { console.log(`Error occurred while scraping URL: ${url}. Error details: ${error.message}`); setResult({ error: error.message }); } setLoading(false); }; return (

FanFic Scraper

{result && (

Result:

{result.title}

Content:

Chapters:

Threadmark: {article.threadmark}

Date: {article.date}

Author: {article.author}

{result.error &&

Error: {result.error}

}

)}

); } function client() { console.log("Initializing client-side React application"); createRoot(document.getElementById("root")).render(); } if (typeof document !== "undefined") { client(); } async function scrapePage(url) { console.log(`Starting to scrape page: ${url}`); const apiKey = Deno.env.get("ScrapingBeeAPIkey"); if (!apiKey) { console.log("ScrapingBee API key not found in environment variables"); throw new Error("ScrapingBee API key not found in environment variables"); } const extractRules = { title: { selector: "h1.p-title-value", output: "text", }, prefixTags: { selector: "div.p-body-header a[href*='prefix_id']", type: "list", output: "text", }, firstPost: { selector: "article.message--post:not(.sticky-container)", output: { body: { selector: "article.message-body div.bbWrapper", output: "html", }, createdDate: { selector: "div.message-attribution-main time", output: "text", }, updatedDate: { selector: "div.message-lastEdit time", output: "text", }, }, }, threadmarks: { selector: "div.block-outer-recent-threadmarks a.menuTrigger[href*='threadmarks']", type: "list", output: { category: { selector: "a", output: "text", }, url: { selector: "a", output: "@href", }, }, }, threadmarksList: { selector: "div.structItemContainer, div.structItem--threadmark", type: "list", output: { title: { selector: "div.structItem-title a", output: "text", }, url: { selector: "div.structItem-title a", output: "@href", }, }, }, pagination: { selector: ".pageNav-main a:last-child", output: "@href", }, }; let allResults = []; let nextPageUrl = url; while (nextPageUrl) { console.log(`Scraping page: ${nextPageUrl}`); const response = await fetch( `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${encodeURIComponent(nextPageUrl)}&render_js=true&extract_rules=${encodeURIComponent(JSON.stringify(extractRules))}`, ); if (!response.ok) { console.log(`Error response from ScrapingBee. Status: ${response.status}`); throw new Error(`HTTP error! status: ${response.status}`); } const result = await response.json(); allResults.push(result); nextPageUrl = result.pagination ? new URL(result.pagination, url).href : null; } return processResults(allResults); } function processResults(results) { const processedResult = { title: results[0].title, prefixTags: results[0].prefixTags, firstPost: results[0].firstPost, threadmarks: results[0].threadmarks, articles: [], }; for (const result of results) { processedResult.articles = processedResult.articles.concat(result.threadmarksList); } return processedResult; } async function getApiKey() { const apiKey = Deno.env.get("ScrapingBeeAPIkey"); if (!apiKey) { console.log("ScrapingBee API key not found in environment variables"); throw new Error("ScrapingBee API key not found in environment variables"); } return apiKey; } async function server(request) { console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`); if (request.method === "POST" && new URL(request.url).pathname === "/scrape") { try { const { url } = await request.json(); console.log(`Processing scrape request for URL: ${url}`); const result = await scrapePage(url); console.log(`Successfully scraped URL: ${url}`); return new Response(JSON.stringify(result), { headers: { "Content-Type": "application/json" }, }); } catch (error) { console.log(`Error occurred while processing request. Error details: ${error.message}`); console.log(`Error stack trace: ${error.stack}`); return new Response(JSON.stringify({ error: error.message, stack: error.stack }), { status: 500, headers: { "Content-Type": "application/json" }, }); } } console.log("Serving HTML page"); return new Response( ` Web Scraper

`, { headers: { "Content-Type": "text/html" }, }, ); } const css = ` body { font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; } input[type="url"] { width: 70%; padding: 10px; margin-right: 10px; } button { padding: 10px 20px; } pre { background-color: #f4f4f4; padding: 10px; border-radius: 5px; white-space: pre-wrap; word-wrap: break-word; } `; export default server;

FanFicScraper