/** @jsxImportSource https://esm.sh/react */ import React, { useState } from "https://esm.sh/react"; import { createRoot } from "https://esm.sh/react-dom/client"; //last stable version is 47 function App() { const [url, setUrl] = useState(""); const [result, setResult] = useState(null); const [loading, setLoading] = useState(false); const handleSubmit = async (e) => { e.preventDefault(); setLoading(true); console.log(`Submitting URL for scraping: ${url}`); const maxRetries = 3; let retries = 0; while (retries < maxRetries) { try { const response = await fetch("/scrape", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ url }), }); console.log(`Received response for URL: ${url}. Status: ${response.status}`); if (!response.ok) { if (response.status === 404) { setResult({ error: "The requested URL was not found. Please check the URL and try again." }); } else if (response.status === 500) { setResult({ error: "The server encountered an error. Please try again later." }); } else { throw new Error(`HTTP error! status: ${response.status}`); } } else { const data = await response.json(); setResult(data); } break; // Exit the loop if successful } catch (error) { console.log(`Error occurred while scraping URL: ${url}. Error details: ${error.message}`); if (error.name === 'TypeError' && error.message === 'Failed to fetch') { setResult({ error: "Network error. Please check your internet connection and try again." }); } else if (retries === maxRetries - 1) { setResult({ error: `An unexpected error occurred: ${error.message}. Please try again later.` }); } retries++; if (retries < maxRetries) { console.log(`Retrying... Attempt ${retries + 1} of ${maxRetries}`); await new Promise(resolve => setTimeout(resolve, 1000 * retries)); // Wait before retrying } } } setLoading(false); }; return (

FanFic Scraper

{result && (

Result:

{result.title}

Content:

Chapters:

Threadmark: {article.threadmark}

Date: {article.date}

Author: {article.author}

{result.error &&

Error: {result.error}

}

)}

); } function client() { console.log("Initializing client-side React application"); createRoot(document.getElementById("root")).render(); } if (typeof document !== "undefined") { client(); } const _REGEX_THREAD_URL = /threads\/(?\d+)/; const _REGEX_POST_URL = /posts\/(?\d+)/; function normalizeUrl(url) { const threadMatch = url.match(_REGEX_THREAD_URL); const postMatch = url.match(_REGEX_POST_URL); if (postMatch) { return `https://forums.spacebattles.com/posts/${postMatch.groups.id}/`; } else if (threadMatch) { return `https://forums.spacebattles.com/threads/${threadMatch.groups.id}/`; } return url; } async function scrapePage(url) { console.log(`Starting to scrape page: ${url}`); const apiKey = await getApiKey(); url = normalizeUrl(url); console.log(`Normalized URL for scraping: ${url}`); const extractRules = { title: "h1.p-title-value", prefixTags: "div.p-body-header a[href*='prefix_id']", posts: { selector: "article.message--post", type: "list", output: { id: "@data-content", content: "article.message-body div.bbWrapper", author: "div.message-userDetails .username", date: "div.message-attribution-main time", lastEdit: "div.message-lastEdit time", isSticky: ".sticky-container", }, }, threadmarksTop: "div.block-outer-recent-threadmarks", threadmarks: { selector: "div.block-outer-recent-threadmarks a.menuTrigger[href*='threadmarks']", type: "list", output: { category: "@href", name: "text", }, }, }; const result = await scrapePageWithRules(url, apiKey, extractRules); return processResults(result); } function processThreadmarks(threadmarks) { return threadmarks.map(tm => { let category = "1"; const categoryMatch = tm.category.match(/threadmark_category=(\d+)/); if (categoryMatch) { category = categoryMatch[1]; } return { category: category, name: tm.name, }; }); } class NetworkError extends Error { constructor(message) { super(message); this.name = "NetworkError"; } } class APIError extends Error { constructor(message, status) { super(message); this.name = "APIError"; this.status = status; } } class ParsingError extends Error { constructor(message) { super(message); this.name = "ParsingError"; } } async function scrapePageWithRules(url, apiKey, extractRules) { console.log(`Sending request to ScrapingBee API for URL: ${url}`); try { const response = await fetch( `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${ encodeURIComponent(url) }&render_js=true&extract_rules=${encodeURIComponent(JSON.stringify(extractRules))}`, ); if (!response.ok) { throw new APIError(`HTTP error! status: ${response.status}`, response.status); } console.log('Response received from ScrapingBee API'); let result; try { result = await response.json(); } catch (jsonError) { throw new ParsingError("Failed to parse JSON response"); } console.log("ScrapingBee API response parsed successfully"); return result; } catch (error) { if (error instanceof TypeError) { console.error("Network error:", error.message); throw new NetworkError("Failed to connect to the API"); } else if (error instanceof APIError) { console.error("API error:", error.message, "Status:", error.status); throw error; } else if (error instanceof ParsingError) { console.error("Parsing error:", error.message); throw error; } else { console.error("Unexpected error:", error.message); throw new Error("An unexpected error occurred"); } } } function processResults(results) { console.log("Processing results:", JSON.stringify(results, null, 2)); if (!results || results.length === 0) { console.log("No results to process"); return { title: "No title found", prefixTags: [], firstPost: null, threadmarks: [], posts: [], }; } const processedResult = { title: results[0].title || "No title found", prefixTags: results[0].prefixTags || [], firstPost: null, threadmarks: results[0].threadmarks ? processThreadmarks(results[0].threadmarks) : [], posts: [], }; if (results[0].posts && Array.isArray(results[0].posts)) { processedResult.posts = results[0].posts.map(processPost); processedResult.firstPost = processedResult.posts.find(post => !post.isSticky) || processedResult.posts[0]; } console.log(`Final processed result: ${JSON.stringify(processedResult, null, 2)}`); return processedResult; } function processPost(post) { return { id: post.id, content: post.content, author: post.author, date: new Date(post.date), lastEdit: post.lastEdit ? new Date(post.lastEdit) : null, isSticky: !!post.isSticky, }; } async function getApiKey() { const apiKey = Deno.env.get("ScrapingBeeAPIkey"); if (!apiKey) { console.log("ScrapingBee API key not found in environment variables"); throw new Error("ScrapingBee API key not found in environment variables"); } return apiKey; } export default async function server(request) { console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`); if (request.method === "POST" && new URL(request.url).pathname === "/scrape") { try { const { url } = await request.json(); const result = await scrapePage(url); console.log(`Successfully scraped URL: ${url}`); console.log(`Scraping completed for URL: ${url} Result summary: Title: ${result.title} Number of posts: ${result.posts.length}`); return new Response(JSON.stringify(result), { headers: { "Content-Type": "application/json" }, }); } catch (error) { console.log(`Error occurred while processing request. Error details: ${error.message}`); console.log(`Error stack trace: ${error.stack}`); console.error(`Error occurred during scraping operation: Error message: ${error.message} Stack trace: ${error.stack}`); return new Response(JSON.stringify({ error: error.message, stack: error.stack }), { status: 500, headers: { "Content-Type": "application/json" }, }); } catch (error) { console.log(`Error occurred while processing request. Error details: ${error.message}`); console.log(`Error stack trace: ${error.stack}`); console.error(`Error occurred during scraping operation: Error message: ${error.message} Stack trace: ${error.stack}`); return new Response(JSON.stringify({ error: error.message, stack: error.stack }), { status: 500, headers: { "Content-Type": "application/json" }, }); } } console.log("Serving HTML page"); return new Response( ` Web Scraper

`, { headers: { "Content-Type": "text/html" }, }, ); } const css = ` body { font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; } input[type="url"] { width: 70%; padding: 10px; margin-right: 10px; } button { padding: 10px 20px; } pre { background-color: #f4f4f4; padding: 10px; border-radius: 5px; white-space: pre-wrap; word-wrap: break-word; } `;

beneficialCopperBovid