/** @jsxImportSource https://esm.sh/react */ import * as cheerio from "https://esm.sh/cheerio"; import React, { useEffect, useState } from "https://esm.sh/react"; import { createRoot } from "https://esm.sh/react-dom/client"; console.log("Version: " + import.meta.url.match(/[?&]v=([^&]*)/)?.at(1)); // URL Processing Helper export class UrlProcessor { static validateUrl(url, baseUrl) { // console.log("Version: " + import.meta.url.match(/[?&]v=([^&]*)/)?.at(1)); if ( !url || url.startsWith("javascript:") || url.startsWith("mailto:") || url.startsWith("tel:") || url === "#" ) { return null; } return url; } static makeUrlAbsolute(url, baseUrl) { try { const baseUrlObj = new URL(baseUrl); const rootBaseUrl = baseUrlObj.origin; const absoluteUrl = new URL(url, rootBaseUrl).href; return absoluteUrl; } catch (error) { console.log(`Error in URL resolution:`); console.log(`Failed URL: ${url}`); console.log(`Base URL: ${baseUrl}`); console.log(`Error: ${error.message}`); return null; } } static classifyLinkType(url, baseUrl) { const validUrl = this.validateUrl(url, baseUrl); if (!validUrl) return null; const absoluteUrl = this.makeUrlAbsolute(validUrl, baseUrl); if (!absoluteUrl) return null; const type = absoluteUrl.startsWith(baseUrl) ? "internal" : "external"; return { url: absoluteUrl, type: type, }; } } // Link Fetching Helper export class LinkFetcher { static discoveredUrls = new Set(); static async fetchLinksFromWebsite(websiteUrl, maxDepth = 5, currentDepth = 0) { if (currentDepth >= maxDepth) { return []; } currentProgress = { phase: "fetching", processedPages: 0, totalLinks: 0, remainingPages: 1, depth: currentDepth, }; const allLinks = []; const pagesToProcess = [websiteUrl]; const processedPages = new Set(); while (pagesToProcess.length > 0) { currentProgress = { phase: "fetching", processedPages: processedPages.size, totalLinks: allLinks.length, remainingPages: pagesToProcess.length, depth: currentDepth, }; const currentBatch = pagesToProcess.splice(0, 10); const batchPromises = currentBatch.map(async (pageUrl) => { if (processedPages.has(pageUrl)) { return []; } try { const startTime = Date.now(); const response = await fetch(pageUrl, { headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", }, }); const elapsed = Date.now() - startTime; if (elapsed > 3000) { console.log(`Slow page fetch: ${pageUrl} took ${elapsed / 1000} seconds`); } if (!response.ok) { console.log(`Failed to fetch page: ${pageUrl}`); return []; } const html = await response.text(); const $ = cheerio.load(html); const pageLinks = []; $("a").each((index, element) => { const $element = $(element); const href = $element.attr("href"); const text = $element.text().trim(); if (href && href.trim() !== "") { try { const absoluteUrl = new URL(href, pageUrl).href; pageLinks.push({ link: absoluteUrl, text: text, foundOn: pageUrl, }); if ( absoluteUrl.startsWith(websiteUrl) && !processedPages.has(absoluteUrl) && !pagesToProcess.includes(absoluteUrl) ) { pagesToProcess.push(absoluteUrl); } } catch (error) { console.log(`Error processing URL: ${href}`, error); } } }); processedPages.add(pageUrl); const nestedLinks = await this.fetchLinksFromWebsite(pageUrl, maxDepth, currentDepth + 1); return [...pageLinks, ...nestedLinks]; } catch (error) { console.log(`Error processing page ${pageUrl}:`, error); return []; } }); const batchResults = await Promise.all(batchPromises); allLinks.push(...batchResults.flat()); if (pagesToProcess.length % 20 === 0) { console.log(`Checkpoint: ${processedPages.size} total pages found, ${allLinks.length} total links`); currentProgress = { phase: "fetching", processedPages: processedPages.size, totalLinks: allLinks.length, remainingPages: pagesToProcess.length, depth: currentDepth, }; } } return allLinks; } static parseLsdResponse(responseText) { return JSON.parse(responseText); } } // Link Checking Helper export class LinkChecker { constructor() { this.checkedLinks = new Set(); this.startTime = new Date(); this.batchSizeHistory = []; this.initialBatchSize = 10; this.minBatchSize = 5; this.maxBatchSize = 50; this.uncheckedLinks = []; this.skippedLinks = 0; // New property to keep track of skipped links } async checkSingleLink(linkData, baseUrl) { const processedUrl = UrlProcessor.classifyLinkType(linkData.link, baseUrl); if (!processedUrl || this.checkedLinks.has(processedUrl.url)) { return null; } this.checkedLinks.add(processedUrl.url); const status = await this.validateLinkStatus(processedUrl.url); if (status.unchecked) { this.uncheckedLinks.push({ url: processedUrl.url, text: linkData.text, type: processedUrl.type, reason: status.reason, foundOn: linkData.foundOn, }); return null; } if (!status.ok) { return { url: processedUrl.url, text: linkData.text, type: processedUrl.type, reason: status.reason, statusCode: status.status, foundOn: linkData.foundOn, }; } return null; } async validateLinkStatus(url, retries = 3) { const startTime = Date.now(); const isRedditUrl = url.includes("reddit.com"); const userAgent = isRedditUrl ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 BrokenLinkChecker/1.0" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"; for (let attempt = 0; attempt < retries; attempt++) { try { const response = await fetch(url, { method: "GET", headers: { "User-Agent": userAgent, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Referer": "https://www.google.com/", }, }); const elapsed = Date.now() - startTime; const elapsedTime = Date.now() - startTime; if (elapsedTime > 10000) { console.log(`Warning: Link ${url} took ${elapsedTime / 1000} seconds to check`); } if (response.ok) { return { ok: true, status: response.status }; } // If we've reached the last attempt and still haven't succeeded if (attempt === retries - 1) { return { ok: false, status: response.status, reason: this.guessErrorCause(response.status), }; } } catch (error) { // Handle SSL certificate errors if (error.message.includes("invalid peer certificate")) { return { ok: false, status: 0, reason: "Invalid SSL Certificate", }; } // Handle unsuccessful tunnel errors if (error.message.includes("unsuccessful tunnel")) { return { ok: false, status: 0, reason: "Connection Failed - Invalid URL or Server Unreachable", }; } // If we've reached the last attempt if (attempt === retries - 1) { return { ok: false, status: 0, reason: "Connection Error: " + error.message, }; } // Wait before retrying const delay = isRedditUrl ? 5000 : 1000 * (attempt + 1); await new Promise(resolve => setTimeout(resolve, delay)); } } } handleRepeatedForbiddenError(url) { console.log(`Repeated 403 Forbidden errors for ${url}`); return { ok: false, status: 403, reason: "Repeated Access Forbidden (Possible rate limiting or IP blocking)", }; } handleSSLError(url, error) { console.log(`SSL Certificate Error for ${url}: ${error.message}`); return { ok: false, status: 0, reason: "SSL Certificate Error: " + error.message, }; } handleUnsuccessfulTunnelError(url, error) { console.log(`Unsuccessful Tunnel Error for ${url}: ${error.message}`); return { ok: false, status: 0, reason: "Unsuccessful Tunnel Error: " + error.message, }; } guessErrorCause(statusCode) { switch (statusCode) { case 400: return "Bad Request"; case 401: return "Unauthorized"; case 403: return "Access Forbidden"; case 404: return "Page Not Found"; case 405: return "Method Not Allowed"; case 408: return "Request Timeout"; case 429: return "Too Many Requests"; case 500: return "Internal Server Error"; case 502: return "Bad Gateway"; case 503: return "Service Unavailable"; case 504: return "Gateway Timeout"; case 0: return "Network Error or Timeout"; default: return "Unknown Error"; } } calculateOptimalBatchSize(currentBatchSize, processingTime) { const targetProcessingTime = 5000; // 5 seconds let newBatchSize = Math.round(currentBatchSize * (targetProcessingTime / processingTime)); newBatchSize = Math.max(this.minBatchSize, Math.min(newBatchSize, this.maxBatchSize)); return newBatchSize; } async processBatchOfLinks(links, baseUrl) { const brokenLinks = []; let batchSize = this.initialBatchSize; const linksToProcess = links.slice(this.skippedLinks); for (let i = 0; i < linksToProcess.length;) { const batch = linksToProcess.slice(i, i + batchSize); const batchStartTime = Date.now(); const batchPromises = batch.map(link => this.checkSingleLink(link, baseUrl)); const batchResults = await Promise.all(batchPromises); const batchEndTime = Date.now(); const batchProcessingTime = batchEndTime - batchStartTime; // Get new broken links from this batch const newBrokenLinks = batchResults.filter(result => result !== null); newBrokenLinks.forEach(link => { console.log(`Found broken link: URL: ${link.url} Found on: ${link.foundOn} Text: ${link.text} Status: ${link.statusCode} Reason: ${link.reason} `); }); brokenLinks.push(...newBrokenLinks); // Update progress with current broken links currentProgress = { ...currentProgress, phase: "checking", checkedLinks: this.checkedLinks.size + this.skippedLinks, totalLinks: links.length, brokenLinks: brokenLinks, uncheckedLinks: this.uncheckedLinks, progress: ((this.checkedLinks.size + this.skippedLinks) / links.length) * 100, currentBrokenLinks: brokenLinks, }; batchSize = this.calculateOptimalBatchSize(batchSize, batchProcessingTime); this.batchSizeHistory.push({ size: batchSize, time: batchProcessingTime }); i += batch.length; } return { brokenLinks, uncheckedLinks: this.uncheckedLinks }; } getCheckedLinksCount() { return this.checkedLinks.size + this.skippedLinks; } getElapsedTime() { return (new Date() - this.startTime) / 1000; } } // Report Generation Helper export class ReportGenerator { static categorizeLinks(brokenLinks) { const byDepth = new Map(); const bySection = new Map(); brokenLinks.forEach(link => { const depth = link.url.split("/").length - 3; byDepth.set(depth, (byDepth.get(depth) || []).concat(link)); const urlParts = link.url.split("/"); const section = urlParts[3] || "root"; bySection.set(section, (bySection.get(section) || []).concat(link)); }); return { internal: brokenLinks.filter(link => link.type === "internal"), external: brokenLinks.filter(link => link.type === "external"), byDepth: Object.fromEntries(byDepth), bySection: Object.fromEntries(bySection), }; } static generateReport(brokenLinks, uncheckedLinks, totalChecked, elapsedTime, totalPages) { const categorized = this.categorizeLinks(brokenLinks); return { totalPages, totalLinks: totalChecked, uniqueLinks: totalChecked, elapsedTime: elapsedTime, brokenLinks: { total: brokenLinks.length, internal: categorized.internal, external: categorized.external, byDepth: categorized.byDepth, bySection: categorized.bySection, }, uncheckedLinks: uncheckedLinks, summary: `Found ${brokenLinks.length} broken links ` + `and ${uncheckedLinks.length} unchecked links ` + `across ${totalPages} pages ` + `(${categorized.internal.length} internal, ` + `${categorized.external.length} external) ` + `out of ${totalChecked} unique links checked ` + `in ${elapsedTime.toFixed(2)} seconds`, }; } } // Main Function export async function findBrokenLinks(websiteUrl, maxDepth = 5) { try { console.log(`Starting link check for ${websiteUrl}`); const links = await LinkFetcher.fetchLinksFromWebsite(websiteUrl); console.log(`Fetched ${links.length} links from ${websiteUrl}`); const totalPages = new Set(links.map(link => new URL(link.link, websiteUrl).origin + new URL(link.link, websiteUrl).pathname )).size; console.log(`Total pages found: ${totalPages}`); const linkChecker = new LinkChecker(); const { brokenLinks, uncheckedLinks } = await linkChecker.processBatchOfLinks(links, websiteUrl); console.log( `Finished checking links. Found ${brokenLinks.length} broken links and ${uncheckedLinks.length} unchecked links.`, ); return ReportGenerator.generateReport( brokenLinks, uncheckedLinks, linkChecker.getCheckedLinksCount(), linkChecker.getElapsedTime(), totalPages, ); } catch (error) { console.error(`Error in findBrokenLinks: ${error.message}`); console.error(`Stack trace: ${error.stack}`); throw error; } } function App() { const [url, setUrl] = useState(""); const [results, setResults] = useState(null); const [loading, setLoading] = useState(false); const [progress, setProgress] = useState(null); const [error, setError] = useState(null); useEffect(() => { let intervalId; if (loading) { intervalId = setInterval(async () => { try { const response = await fetch("/api/progress"); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } const text = await response.text(); const data = JSON.parse(text); setProgress(data); } catch (e) { console.error("Error fetching progress:", e); setError(`Error fetching progress: ${e.message}`); } }, 1000); } return () => clearInterval(intervalId); }, [loading]); const handleSubmit = async (e) => { e.preventDefault(); setLoading(true); setResults(null); setProgress(null); setError(null); try { const response = await fetch("/api/check-links", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ url }), }); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } const text = await response.text(); const data = JSON.parse(text); if (data.error) { throw new Error(data.error); } setResults(data); } catch (error) { console.error("Error:", error); setError(error.message || "An error occurred while checking links."); } setLoading(false); }; const renderBrokenLinks = (links) => { if (!Array.isArray(links)) { console.error("Invalid links data:", links); return null; } return links.map((link, index) => ( {`${index + 1}`} {link.foundOn} {link.url} {link.text} {link.statusCode} {link.reason} )); }; return (

Broken Link Checker

{error && (

Error

{error}

)} {progress && (

Progress

Phase: {progress.phase}

{progress.phase === "fetching" && (

Processed Pages: {progress.processedPages}
Total Links: {progress.totalLinks}
Remaining Pages: {progress.remainingPages}

)} {progress.phase === "checking" && (

Checked Links: {progress.checkedLinks} / {progress.totalLinks}
Broken Links: {progress.brokenLinks?.length || 0}
Unchecked Links: {progress.uncheckedLinks?.length || 0}
Progress: {progress.progress?.toFixed(2)}%

)}

)} {(progress?.currentBrokenLinks?.length > 0 || results) && (

Results

{progress?.phase === "checking" && (

Found {progress.currentBrokenLinks?.length || 0} broken links so far...

)} {results &&

{results.summary}

}

Broken Links

{renderBrokenLinks( progress?.currentBrokenLinks || (results?.brokenLinks?.internal?.concat(results?.brokenLinks?.external) || []), )}

#	Found On	Broken Link	Link Text	Status Code	Probable Cause

{(progress?.uncheckedLinks?.length > 0 || (results && results.uncheckedLinks)) && ( <>

Unchecked Links

{(progress?.uncheckedLinks || results?.uncheckedLinks || []).map((link, index) => ( ))}

#	Found On	Unchecked Link	Link Text	Reason
{`${index + 1}`}	{link.foundOn}	{link.url}	{link.text}	{link.reason}

)}

); } function client() { createRoot(document.getElementById("root")).render(); } if (typeof document !== "undefined") { client(); } let currentProgress = null; let lastProgressUpdate = null; let progressUpdateCount = 0; export default async function server(request: Request): Promise { const url = new URL(request.url); if (url.pathname === "/api/progress" && request.method === "GET") { const now = new Date(); if (lastProgressUpdate && currentProgress) { const timeSinceLastUpdate = now.getTime() - lastProgressUpdate.getTime(); if (timeSinceLastUpdate < 1000) { progressUpdateCount++; } else { progressUpdateCount = 0; } } lastProgressUpdate = now; return new Response(JSON.stringify(currentProgress), { headers: { "Content-Type": "application/json" }, }); } if (url.pathname === "/api/check-links" && request.method === "POST") { let requestBody; try { requestBody = await request.json(); } catch (error) { console.error("Error parsing request body:", error); return new Response(JSON.stringify({ error: "Invalid request body" }), { status: 400, headers: { "Content-Type": "application/json" }, }); } const { url: websiteUrl } = requestBody; if (!websiteUrl) { return new Response(JSON.stringify({ error: "Missing URL in request body" }), { status: 400, headers: { "Content-Type": "application/json" }, }); } try { const maxDepth = 5; currentProgress = { phase: "fetching", processedPages: 0, totalLinks: 0, remainingPages: 1, depth: 0, }; const result = await findBrokenLinks(websiteUrl, maxDepth); return new Response(JSON.stringify(result), { headers: { "Content-Type": "application/json" }, }); } catch (error) { console.log(`\n=== Detailed Error Information ===`); console.log(`Error name: ${error.name}`); console.log(`Error message: ${error.message}`); console.log(`Error stack: ${error.stack}`); console.log(`Current progress state: ${JSON.stringify(currentProgress, null, 2)}`); console.log(`Last progress update: ${lastProgressUpdate}`); console.log(`Progress update count: ${progressUpdateCount}`); console.log(`=== End Error Information ===\n`); currentProgress = null; lastProgressUpdate = null; progressUpdateCount = 0; return new Response( JSON.stringify({ error: error.message, errorName: error.name, errorStack: error.stack, lastKnownProgress: currentProgress, }), { status: 500, headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*", }, }, ); } } return new Response( ` Broken Link Checker

`, { headers: { "content-type": "text/html", }, }, ); } const css = ` body { font-family: Arial, sans-serif; line-height: 1.6; margin: 0; padding: 20px; background-color: #f4f4f4; } h1, h2, h3 { color: #333; } form { margin-bottom: 20px; } input[type="url"] { width: 300px; padding: 5px; margin-right: 10px; } button { padding: 5px 10px; background-color: #333; color: #fff; border: none; cursor: pointer; } button:disabled { background-color: #999; } table { width: 100%; border-collapse: collapse; margin-top: 20px; } th, td { padding: 10px; border: 1px solid #ddd; text-align: left; } th { background-color: #f2f2f2; font-weight: bold; } tr:nth-child(even) { background-color: #f9f9f9; } a { color: #0066cc; text-decoration: none; } a:hover { text-decoration: underline; } `;

fullWebsiteChecker

Version 43