\n \n \n URL Checker\n \n \n \n\n {error && (\n \n {error}\n \n )}\n\n {isLoading && progress && (\n

URLs Found: {progress.urlsFound}

URLs Processed: {progress.urlsProcessed}

Broken URLs Found: {progress.brokenUrlsFound}

\n )}\n\n {results && (\n

Total URLs Checked

{results.totalURLsChecked}

Broken URLs Found

{results.brokenURLsFound}

\n\n {results.brokenURLs.length > 0 && (\n

Broken URLs

\n {results.brokenURLs.map((brokenUrl, index) => (\n

{brokenUrl.url}

\n {brokenUrl.sourceUrl && (\n

\n Found on: {brokenUrl.sourceUrl}\n

\n )}\n

\n {brokenUrl.possibleCause}\n

\n ))}\n

\n )}\n\n

\n Scan completed in {(results.scanDuration / 1000).toFixed(2)} seconds\n

\n )}\n \n \n

\n );\n};\n\nexport default URLCheckerInterface;",[],[194],{"_104":203,"_95":204,"_140":205,"_142":-5,"_143":131,"_144":206,"_146":26,"_106":207,"_100":159,"_148":149,"_150":208,"_152":26,"_153":209,"_98":210},"d4d77d10-dd97-11ef-b241-e6cdfca9ef9f","summaryTest","b0e422ba-dd96-11ef-b8cf-e6cdfca9ef9f",40,["D",1738082965240],"import { expect } from \"jsr:@std/expect\";\nimport { SummaryGenerator } from \"./summaryGenerator\";\n\nconsole.log(\"Starting Summary Generator test execution\");\n\nconst generator = new SummaryGenerator();\n\n// Sample test URLs\nconst TEST_URLS = [\n \"https://willkrouse.com\",\n \"https://dateme.directory\",\n];\n\n// Sample test results\nconst sampleResults = [\n {\n url: TEST_URLS[0],\n sourceUrl: null,\n status: 200,\n error: null,\n timestamp: new Date(),\n },\n {\n url: TEST_URLS[1],\n sourceUrl: TEST_URLS[0],\n status: 200,\n error: null,\n timestamp: new Date(),\n },\n {\n url: TEST_URLS[2],\n sourceUrl: TEST_URLS[0],\n status: 404,\n error: \"Not Found\",\n timestamp: new Date(),\n },\n {\n url: TEST_URLS[3],\n sourceUrl: TEST_URLS[1],\n status: 403,\n error: \"Forbidden\",\n timestamp: new Date(),\n },\n];\n\ninterface TestCase {\n description: string;\n function: Function;\n expectedBehavior: string;\n}\n\nconst tests: TestCase[] = [\n {\n description: \"initialize generator and start new scan\",\n function: () => {\n generator.reset();\n generator.startNewScan();\n const summary = generator.generateSummary();\n expect(summary.totalURLsChecked).toEqual(0);\n expect(summary.brokenURLsFound).toEqual(0);\n expect(summary.brokenURLs).toEqual([]);\n expect(summary.scanDuration).toBeGreaterThanOrEqual(0);\n },\n expectedBehavior: \"should reset all counters and start timing\",\n },\n {\n description: \"add and process sample results\",\n function: () => {\n generator.startNewScan();\n\n // Add all sample results\n sampleResults.forEach(result => generator.addResult(result));\n\n const summary = generator.generateSummary();\n\n // Verify totals\n expect(summary.totalURLsChecked).toEqual(4);\n expect(summary.brokenURLsFound).toEqual(2);\n\n // Verify broken URLs are correctly identified\n expect(summary.brokenURLs.length).toEqual(2);\n\n // Check first broken URL (404)\n const notFoundUrl = summary.brokenURLs.find(url => url.statusCode === 404);\n expect(notFoundUrl).toBeTruthy();\n expect(notFoundUrl?.url).toEqual(TEST_URLS[2]);\n expect(notFoundUrl?.possibleCause).toInclude(\"Page not found\");\n\n // Check second broken URL (403)\n const forbiddenUrl = summary.brokenURLs.find(url => url.statusCode === 403);\n expect(forbiddenUrl).toBeTruthy();\n expect(forbiddenUrl?.url).toEqual(TEST_URLS[3]);\n expect(forbiddenUrl?.possibleCause).toInclude(\"Access forbidden\");\n },\n expectedBehavior: \"should correctly identify and report broken URLs\",\n },\n {\n description: \"test error cause determination\",\n function: () => {\n generator.startNewScan();\n\n const timeoutResult = {\n url: \"https://example.com/timeout\",\n sourceUrl: null,\n status: null,\n error: \"ETIMEDOUT\",\n timestamp: new Date(),\n };\n\n const certErrorResult = {\n url: \"https://example.com/cert\",\n sourceUrl: null,\n status: null,\n error: \"CERT_HAS_EXPIRED\",\n timestamp: new Date(),\n };\n\n generator.addResult(timeoutResult);\n generator.addResult(certErrorResult);\n\n const summary = generator.generateSummary();\n\n // Find timeout error report\n const timeoutReport = summary.brokenURLs.find(url => url.errorMessage.includes(\"ETIMEDOUT\"));\n expect(timeoutReport).toBeTruthy();\n expect(timeoutReport?.possibleCause).toInclude(\"Connection timed out\");\n\n // Find certificate error report\n const certReport = summary.brokenURLs.find(url => url.errorMessage.includes(\"CERT\"));\n expect(certReport).toBeTruthy();\n expect(certReport?.possibleCause).toInclude(\"SSL/TLS certificate issue\");\n },\n expectedBehavior: \"should provide appropriate error causes for different types of errors\",\n },\n {\n description: \"test scan duration tracking\",\n function: async () => {\n generator.startNewScan();\n\n // Simulate some processing time\n await new Promise(resolve => setTimeout(resolve, 100));\n\n const summary = generator.generateSummary();\n expect(summary.scanDuration).toBeGreaterThanOrEqual(100);\n },\n expectedBehavior: \"should accurately track scan duration\",\n },\n];\n\n// Run all tests\nfor (const testCase of tests) {\n console.log(`Running test: ${testCase.description}`);\n console.log(`Expected behavior: ${testCase.expectedBehavior}`);\n\n try {\n await testCase.function();\n console.log(\"Test passed\\n\");\n } catch (error) {\n console.error(`Test failed: ${error}\\n`);\n throw error;\n }\n}\n\nconsole.log(\"All Summary Generator tests completed\");",[],[204],{"_104":212,"_95":99,"_140":213,"_142":-5,"_143":131,"_144":214,"_146":26,"_106":215,"_100":216,"_148":149,"_150":217,"_152":26,"_153":218,"_98":219},"e8846222-dd94-11ef-985c-e6cdfca9ef9f","6707eeaa-dd8d-11ef-a134-e6cdfca9ef9f",32,["D",1738081709761],10,"// last good version of was v19\ntype CrawlResult = {\n url: string;\n sourceUrl: string | null;\n depth: number;\n status: number | null;\n error: string | null;\n};\n\ntype CrawlProgress = {\n urlsFound: number;\n urlsProcessed: number;\n brokenUrlsFound: number;\n inProgress: boolean;\n};\n\ntype URLCheckResult = {\n url: string;\n sourceUrl: string | null;\n status: number | null;\n error: string | null;\n timestamp: Date;\n};\n\ntype BrokenURLReport = {\n url: string;\n sourceUrl: string | null;\n statusCode: number | null;\n errorMessage: string;\n timestamp: Date;\n possibleCause: string;\n};\n\ntype CrawlSummary = {\n totalURLsChecked: number;\n brokenURLsFound: number;\n brokenURLs: BrokenURLReport[];\n scanDuration: number;\n};\n\nexport class URLCrawler {\n private visitedUrls: Set = new Set();\n private urlQueue: { url: string; sourceUrl: string | null; depth: number }[] = [];\n private brokenUrls: URLCheckResult[] = [];\n private maxDepth: number;\n private baseUrl: string;\n private startTime: number = 0;\n private progress: CrawlProgress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n };\n\n constructor(maxDepth: number = 3) {\n this.maxDepth = maxDepth;\n }\n\n private determineErrorCause(status: number | null, error: string | null): string {\n if (status) {\n switch (status) {\n case 404:\n return \"Page not found - The URL may have been moved or deleted\";\n case 403:\n return \"Access forbidden - Authentication may be required\";\n case 500:\n return \"Server error - The website's server encountered an error\";\n case 502:\n return \"Bad gateway - There may be an issue with the server configuration\";\n case 503:\n return \"Service unavailable - The server may be overloaded or down for maintenance\";\n default:\n return `Server returned status code ${status}`;\n }\n }\n\n if (error) {\n if (error.includes(\"ETIMEDOUT\")) return \"Connection timed out - The server took too long to respond\";\n if (error.includes(\"ECONNREFUSED\")) return \"Connection refused - The server may be down\";\n if (error.includes(\"CERT_\")) return \"SSL/TLS certificate issue - There may be a security configuration problem\";\n return error;\n }\n\n return \"Unknown error occurred\";\n }\n\n async crawl(startUrl: string): Promise {\n this.reset();\n this.startTime = Date.now();\n this.baseUrl = new URL(startUrl).origin;\n this.urlQueue = [{ url: startUrl, sourceUrl: null, depth: 0 }];\n this.progress.inProgress = true;\n\n while (this.urlQueue.length > 0) {\n const current = this.urlQueue.shift()!;\n\n if (this.visitedUrls.has(current.url)) {\n continue;\n }\n\n this.visitedUrls.add(current.url);\n const checkResult = await this.checkUrl(current.url, current.sourceUrl);\n\n if (checkResult.error || (checkResult.status && checkResult.status >= 400)) {\n this.brokenUrls.push(checkResult);\n this.progress.brokenUrlsFound++;\n } else if (current.depth < this.maxDepth) {\n const newUrls = await this.extractUrlsFromPage(current.url);\n this.processNewUrls(newUrls, current.url, current.depth + 1);\n }\n\n this.progress.urlsProcessed++;\n }\n\n this.progress.inProgress = false;\n return this.generateSummary();\n }\n\n private async checkUrl(url: string, sourceUrl: string | null): Promise {\n try {\n const response = await fetch(url);\n return {\n url,\n sourceUrl,\n status: response.status,\n error: response.ok ? null : `HTTP ${response.status} - ${response.statusText}`,\n timestamp: new Date(),\n };\n } catch (error) {\n return {\n url,\n sourceUrl,\n status: null,\n error: error instanceof Error ? error.message : String(error),\n timestamp: new Date(),\n };\n }\n }\n\n private async extractUrlsFromPage(url: string): Promise {\n try {\n const response = await fetch(url);\n if (!response.ok) return [];\n\n const html = await response.text();\n const hrefRegex = /href=[\"'](https?:\\/\\/[^\"']+)[\"']/g;\n const matches = [...html.matchAll(hrefRegex)];\n\n return matches\n .map(match => match[1])\n .filter(href => href && this.isValidUrl(href))\n .map(href => this.normalizeUrl(href));\n } catch (error) {\n console.error(`Error fetching ${url}:`, error);\n return [];\n }\n }\n\n private processNewUrls(urls: string[], sourceUrl: string, depth: number): void {\n for (const url of urls) {\n if (!this.visitedUrls.has(url) && this.shouldCrawl(url)) {\n this.urlQueue.push({ url, sourceUrl, depth });\n this.progress.urlsFound++;\n }\n }\n }\n\n private shouldCrawl(url: string): boolean {\n try {\n const urlObj = new URL(url);\n return urlObj.origin === this.baseUrl;\n } catch {\n return false;\n }\n }\n\n private isValidUrl(url: string): boolean {\n try {\n new URL(url);\n return true;\n } catch {\n return false;\n }\n }\n\n private normalizeUrl(url: string): string {\n return url.split(\"#\")[0].replace(/\\/+$/, \"\");\n }\n\n private generateSummary(): CrawlSummary {\n const brokenURLs: BrokenURLReport[] = this.brokenUrls.map(result => ({\n url: result.url,\n sourceUrl: result.sourceUrl,\n statusCode: result.status,\n errorMessage: result.error || \"Unknown error\",\n timestamp: result.timestamp,\n possibleCause: this.determineErrorCause(result.status, result.error),\n }));\n\n return {\n totalURLsChecked: this.progress.urlsProcessed,\n brokenURLsFound: this.progress.brokenUrlsFound,\n brokenURLs,\n scanDuration: Date.now() - this.startTime,\n };\n }\n\n getProgress(): CrawlProgress {\n return { ...this.progress };\n }\n\n reset(): void {\n this.visitedUrls.clear();\n this.urlQueue = [];\n this.brokenUrls = [];\n this.startTime = 0;\n this.progress = {\n urlsFound: 0,\n urlsProcessed: 0,\n brokenUrlsFound: 0,\n inProgress: false,\n };\n }\n}",[],[99],{"_104":221,"_95":222,"_140":223,"_142":-5,"_143":131,"_144":224,"_146":26,"_106":225,"_100":226,"_148":149,"_150":227,"_152":26,"_153":228,"_98":229},"fac9ab86-dd94-11ef-a3b4-e6cdfca9ef9f","urlCrawlerTests","d97f1968-dd8d-11ef-8fa1-e6cdfca9ef9f",33,["D",1738081740413],15,"// last good version of was v19\nimport { expect } from \"jsr:@std/expect\";\nimport { URLCrawler } from \"./urlCrawlerComponent\";\n\nconsole.log(\"Starting URL Crawler test execution\");\n\nconst TEST_URLS = [\n \"https://willkrouse.com\",\n \"https://dateme.directory\",\n \"https://nonexistent-test-url-123.com\", // This URL should fail\n];\n\ninterface TestCase {\n description: string;\n inputs: Record;\n expectedBehavior: string;\n function: Function;\n}\n\nconst crawler = new URLCrawler(2);\n\nconst tests: TestCase[] = [\n {\n description: \"initialize crawler with default settings\",\n inputs: {},\n expectedBehavior: \"should return initial progress state\",\n function: () => {\n crawler.reset();\n const progress = crawler.getProgress();\n expect(progress.urlsFound).toEqual(0);\n expect(progress.urlsProcessed).toEqual(0);\n expect(progress.brokenUrlsFound).toEqual(0);\n expect(progress.inProgress).toEqual(false);\n },\n },\n {\n description: \"crawl valid single page\",\n inputs: { url: TEST_URLS[0] },\n expectedBehavior: \"should return crawl summary\",\n function: async (url: string) => {\n const summary = await crawler.crawl(url);\n expect(summary.totalURLsChecked).toBeGreaterThan(0);\n expect(summary.scanDuration).toBeGreaterThan(0);\n expect(Array.isArray(summary.brokenURLs)).toBe(true);\n },\n },\n {\n description: \"handle nonexistent URL\",\n inputs: { url: TEST_URLS[2] },\n expectedBehavior: \"should report broken URL in summary\",\n function: async (url: string) => {\n const summary = await crawler.crawl(url);\n expect(summary.brokenURLsFound).toBeGreaterThan(0);\n expect(summary.brokenURLs.length).toBeGreaterThan(0);\n expect(summary.brokenURLs[0].url).toEqual(url);\n expect(summary.brokenURLs[0].possibleCause).toBeTruthy();\n },\n },\n {\n description: \"check progress tracking\",\n inputs: { url: TEST_URLS[0] },\n expectedBehavior: \"should update progress during crawl\",\n function: async (url: string) => {\n crawler.reset();\n const crawlPromise = crawler.crawl(url);\n\n // Check initial progress\n let progress = crawler.getProgress();\n expect(progress.inProgress).toBe(true);\n\n // Wait for crawl to complete\n await crawlPromise;\n\n // Check final progress\n progress = crawler.getProgress();\n expect(progress.inProgress).toBe(false);\n expect(progress.urlsProcessed).toBeGreaterThan(0);\n },\n },\n];\n\nfor (const testCase of tests) {\n console.log(`Running test: ${testCase.description}`);\n console.log(`Expected behavior: ${testCase.expectedBehavior}`);\n\n try {\n if (\n testCase.description.includes(\"crawl\") || testCase.description.includes(\"handle\")\n || testCase.description.includes(\"check\")\n ) {\n await testCase.function(testCase.inputs.url);\n } else {\n testCase.function();\n }\n console.log(\"Test passed\\n\");\n } catch (error) {\n console.error(`Test failed: ${error}\\n`);\n throw error;\n }\n}\n\nconsole.log(\"All tests completed\");",[],[222],"counts",{"_232":31,"_233":90,"_234":90,"_235":31},"pulls","branches","forks","envVars","contents",[],"file",{"_104":212,"_95":99,"_140":213,"_142":-5,"_143":131,"_144":214,"_146":26,"_106":240,"_100":216,"_148":149,"_150":217,"_152":26,"_98":241},["D",1738081709761],[99],"upstream","routes/_app.x.$handle.$name.branch.$branch.version.$version.code.$",{"_93":94,"_95":96,"_97":41,"_100":101,"_98":99,"_102":245,"_238":249,"_236":251,"_242":-5,"_252":-5},{"_104":105,"_95":96,"_106":246,"_108":-5,"_109":110,"_111":-5,"_112":-5,"_11":113,"_114":-5,"_115":26,"_116":117,"_118":117,"_119":120,"_121":120,"_122":120,"_123":117,"_124":26,"_20":247,"_128":248},["D",1737670113028],{"_93":94,"_126":127},{"_100":130,"_95":41,"_104":131,"_132":105,"_133":-5,"_134":-5},{"_104":212,"_95":99,"_140":213,"_142":-5,"_143":131,"_144":214,"_146":26,"_106":240,"_100":216,"_148":149,"_150":217,"_152":26,"_98":250},[99],[],"pull","routes/_app","actionData","errors"]