/** @jsxImportSource https://esm.sh/react */ import React, { useState } from "https://esm.sh/react"; import { createRoot } from "https://esm.sh/react-dom/client"; // last stable version is v138 function App() { const [url, setUrl] = useState(""); const [result, setResult] = useState(null); const [loading, setLoading] = useState(false); const handleSubmit = async (e) => { e.preventDefault(); setLoading(true); console.log(`Submitting URL for scraping: ${url}`); try { const response = await fetch("/scrape", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ url }), }); console.log(`Received response for URL: ${url}. Status: ${response.status}`); const data = await response.json(); setResult(data); } catch (error) { console.log(`Error occurred while scraping URL: ${url}. Error details: ${error.message}`); setResult({ error: error.message }); } setLoading(false); }; return (

FanFic Scraper

{result && (

Result:

{result.title}

Content:

Chapters:

Threadmark: {article.threadmark}

Date: {article.date}

Author: {article.author}

{result.error &&

Error: {result.error}

}

)}

); } function client() { console.log("Initializing client-side React application"); createRoot(document.getElementById("root")).render(); } if (typeof document !== "undefined") { client(); } async function scrapePage(url) { console.log(`Starting to scrape page: ${url}`); const apiKey = Deno.env.get("ScrapingBeeAPIkey"); if (!apiKey) { console.log("ScrapingBee API key not found in environment variables"); throw new Error("ScrapingBee API key not found in environment variables"); } const extractRules = { title: "h1", articles: { selector: ".message:has(.message-cell--threadmark-header)", type: "list", output: { threadmark: ".threadmarkLabel", date: { selector: ".message-attribution-main time", output: "text", }, author: { selector: ".message-attribution-main .username", output: "text", }, }, }, }; console.log(`Sending request to ScrapingBee for URL: ${url} at ${new Date().toISOString()}`); const startTime = Date.now(); try { const response = await fetch( `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${ encodeURIComponent(url) }&render_js=false&extract_rules=${encodeURIComponent(JSON.stringify(extractRules))}`, ); const duration = Date.now() - startTime; console.log(`Received response from ScrapingBee at ${new Date().toISOString()}. Duration: ${duration}ms`); console.log(`Received response from ScrapingBee for URL: ${url}. Status: ${response.status}`); if (!response.ok) { console.log(`Error response from ScrapingBee. Status: ${response.status}`); throw new Error(`HTTP error! status: ${response.status}`); } // Log response size before parsing const responseText = await response.text(); console.log(`Response size (in characters): ${responseText.length}`); const result = JSON.parse(responseText); console.log(`Title scraped: ${result.title}`); console.log(`Number of articles scraped: ${result.articles ? result.articles.length : 0}`); // Log all detected threadmarks if (result.articles && result.articles.length > 0) { console.log(`Detected ${result.articles.length} threadmarked articles on this page.`); result.articles.forEach((article, index) => { console.log(`Threadmark ${index + 1}: ${article.threadmark}, Date: ${article.date}, Author: ${article.author}`); }); } else { console.log(`No articles were scraped or no threadmarks detected.`); } // Log extracted data for edge cases if (result.articles && result.articles.length > 0) { result.articles.forEach((article, index) => { console.log(`Threadmark ${index + 1}: ${article.threadmark || "N/A"}`); console.log(`Date: ${article.date || "N/A"}`); console.log(`Author: ${article.author || "N/A"}`); if (!article.threadmark || !article.date || !article.author) { console.log(`Warning: Missing data in one or more fields for article ${index + 1}`); } }); } else { console.log(`No articles detected.`); } return result; } catch (error) { console.log(`Error occurred while scraping URL: ${url}`); console.log(`Error details: ${error.message}`); console.log(`Error stack: ${error.stack}`); throw error; } } async function getApiKey() { const apiKey = Deno.env.get("ScrapingBeeAPIkey"); if (!apiKey) { console.log("ScrapingBee API key not found in environment variables"); throw new Error("ScrapingBee API key not found in environment variables"); } return apiKey; } async function server(request) { console.log(`Received ${request.method} request for path: ${new URL(request.url).pathname}`); if (request.method === "POST" && new URL(request.url).pathname === "/scrape") { try { const { url } = await request.json(); console.log(`Processing scrape request for URL: ${url}`); const result = await scrapePage(url); console.log(`Successfully scraped URL: ${url}`); return new Response(JSON.stringify(result), { headers: { "Content-Type": "application/json" }, }); } catch (error) { console.log(`Error occurred while processing request. Error details: ${error.message}`); console.log(`Error stack trace: ${error.stack}`); return new Response(JSON.stringify({ error: error.message, stack: error.stack }), { status: 500, headers: { "Content-Type": "application/json" }, }); } } console.log("Serving HTML page"); return new Response( ` Web Scraper

`, { headers: { "Content-Type": "text/html" }, }, ); } const css = ` body { font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; } input[type="url"] { width: 70%; padding: 10px; margin-right: 10px; } button { padding: 10px 20px; } pre { background-color: #f4f4f4; padding: 10px; border-radius: 5px; white-space: pre-wrap; word-wrap: break-word; } `; export default server;

FanFicScraper

Version 45