import { Hono } from "https://esm.sh/hono@3.10.2"; import { encode } from "https://esm.sh/js-base64@3.7.5"; import { parse } from "https://esm.sh/node-html-parser@6.1.5"; const app = new Hono(); // Expanded stopwords list with additional navigation-related terms const stopwords = new Set([ 'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'could', 'did', 'do', 'does', 'doing', 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'has', 'have', 'having', 'he', 'her', 'here', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'its', 'itself', 'just', 'me', 'more', 'most', 'my', 'myself', 'no', 'nor', 'not', 'now', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same', 'she', 'should', 'so', 'some', 'such', 'than', 'that', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'these', 'they', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which', 'while', 'who', 'whom', 'why', 'with', 'would', 'you', 'your', 'yours', 'yourself', 'yourselves', // Additional navigation-related terms 'back', 'contents', 'menu', 'home', 'next', 'previous', 'top', 'bottom', 'navigation', 'nav', 'sidebar', 'footer', 'header', 'main', 'article', 'section', 'page', 'skip', 'jump', 'return' ]); // Expanded Synonym dictionary (unchanged) const synonyms = { "important": ["significant", "crucial", "vital", "essential", "key"], // ... (rest of the synonyms dictionary remains unchanged) }; // Simple sentiment analysis function (unchanged) function analyseSentiment(text) { const positiveWords = new Set([ 'good', 'great', 'excellent', 'wonderful', 'amazing', 'fantastic', 'awesome', 'happy', 'joy', 'love', 'like', 'beautiful', 'nice', 'pleasant', 'delightful' ]); const negativeWords = new Set([ 'bad', 'terrible', 'awful', 'horrible', 'dreadful', 'poor', 'disappointing', 'sad', 'unhappy', 'hate', 'dislike', 'ugly', 'unpleasant', 'nasty', 'cruel' ]); const words = text.toLowerCase().match(/\b\w+\b/g) || []; let score = 0; words.forEach(word => { if (positiveWords.has(word)) score++; if (negativeWords.has(word)) score--; }); let sentiment; if (score > 0) { sentiment = "positive"; } else if (score < 0) { sentiment = "negative"; } else { sentiment = "neutral"; } return { score, sentiment }; } // Improved TF-IDF implementation (unchanged) function calculateTFIDF(text, outputSize = 7) { const words = text.toLowerCase().match(/\b(?!(?:\d+|[a-z])\b)[a-z']{2,}(?:[\s-][a-z']{2,}){0,2}\b/g) || []; const wordCount = words.length; const termFrequency = {}; const documentFrequency = {}; // Calculate term frequency and document frequency for (let i = 0; i < words.length; i++) { for (let j = 1; j <= 3 && i + j <= words.length; j++) { const phrase = words.slice(i, i + j).join(' '); if (!phrase.split(' ').every(word => stopwords.has(word))) { termFrequency[phrase] = (termFrequency[phrase] || 0) + 1; documentFrequency[phrase] = 1; } } } // Calculate TF-IDF with improved scoring const tfidf = {}; Object.keys(termFrequency).forEach(term => { const tf = termFrequency[term] / wordCount; const idf = Math.log(wordCount / (documentFrequency[term] || 1)) + 1; const termWordCount = term.split(' ').length; // Boost score for multi-word phrases, less common terms, and longer phrases tfidf[term] = tf * idf * (termWordCount * 0.5) * (1 / Math.sqrt(termFrequency[term])) * (term.length / 10); }); // Sort by score (descending) and get top unique phrases const uniquePhrases = new Set(); return Object.entries(tfidf) .sort((a, b) => b[1] - a[1]) .filter(([phrase, _]) => { if (uniquePhrases.size >= outputSize) return false; const words = phrase.split(' '); if (words.length > 3) return false; if (words.some(word => word.length === 1)) return false; if (words.every(word => stopwords.has(word))) return false; // Check if the phrase is a subset of an already included phrase if (Array.from(uniquePhrases).some(existingPhrase => existingPhrase.includes(phrase))) return false; // Additional filtering to remove unwanted terms if (phrase.includes('http') || phrase.includes('www') || phrase.includes('com')) return false; uniquePhrases.add(phrase); return true; }) .slice(0, outputSize); } // Updated function to calculate word frequency function calculateWordFrequency(text, topN = 30) { const words = text.toLowerCase().match(/\b[a-z]{2,}\b/g) || []; const frequency = {}; words.forEach(word => { if (!stopwords.has(word) && !word.includes('http') && !word.includes('www') && !word.includes('com')) { frequency[word] = (frequency[word] || 0) + 1; } }); return Object.entries(frequency) .sort((a, b) => b[1] - a[1]) .slice(0, topN); } // Function to generate a fun fact (unchanged) function generateFunFact(text) { const wordCount = text.trim().split(/\s+/).length; const charCount = text.length; const sentences = text.split(/[.!?]+/).length; const avgWordLength = charCount / wordCount; const facts = [ `Your text has an average word length of ${avgWordLength.toFixed(2)} characters. Shakespeare's average was about 3.99!`, `You've written ${sentences} sentence${sentences !== 1 ? 's' : ''}. A typical novel has about 50-60 sentences per page.`, `Your text is ${wordCount} words long. "War and Peace" by Leo Tolstoy is about 587,287 words!`, `Your longest word is "${text.match(/\b\w+\b/g).reduce((a, b) => a.length > b.length ? a : b)}". The longest word in English is 189,819 letters long!`, `You've used ${charCount} characters. That's about ${(charCount / 280).toFixed(2)} tweets' worth of content!` ]; return facts[Math.floor(Math.random() * facts.length)]; } // Function to generate a simple summary (unchanged) function generateSummary(text, sentenceCount = 3) { const sentences = text.match(/[^\.!\?]+[\.!\?]+/g) || []; return sentences.slice(0, sentenceCount).join(' '); } // Updated function to clean the extracted text function cleanText(text) { return text .replace(/<[^>]*>/g, '') // Remove HTML tags .replace(/\s+/g, ' ') // Replace multiple spaces with a single space .replace(/[^\w\s.,!?]/g, '') // Remove special characters except punctuation .replace(/\b(https?:\/\/)?[^\s]+\.(com|org|net|edu|gov)\b/g, '') // Remove URLs .replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, '') // Remove email addresses .replace(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, '') // Remove IP addresses .replace(/\b\w+ing\b|\b\w+ed\b|\b\w+s\b/g, match => match.toLowerCase()) // Convert common word endings to lowercase .replace(/\b(back to (contents|top)|table of contents|skip to (main content|navigation))\b/gi, '') // Remove common navigation phrases .replace(/\b(menu|home|next|previous|top|bottom|navigation|nav|sidebar|footer|header|main|article|section|page)\b/gi, '') // Remove more navigation-related words .trim(); // Remove leading and trailing whitespace } // Updated function to fetch and parse URL content async function fetchUrlContent(url) { try { const response = await fetch(url); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } const html = await response.text(); const root = parse(html); if (!root || typeof root.querySelectorAll !== 'function') { throw new Error('Failed to parse HTML content'); } // Remove script, style, nav, header, footer, and other navigation-related elements const elementsToRemove = root.querySelectorAll('script, style, nav, header, footer, aside, .navigation, .menu, .sidebar, [role="navigation"]'); elementsToRemove.forEach(el => el.remove()); // Extract content from the main content area if possible, otherwise use body const mainContent = root.querySelector('main, #main, #content, .content, article') || root.querySelector('body'); const content = mainContent ? mainContent.textContent : ''; if (!content) { throw new Error('No content found in the parsed HTML'); } // Clean the extracted text return cleanText(content); } catch (error) { console.error('Error fetching URL:', error); throw new Error(`Failed to fetch URL content: ${error.message}`); } } app.get("/", (c) => { return c.html(` Radical Text Analyser

Welcome to the Radical Text Analyser, mate!

Radical Text Analyser

Enter your text or a URL below to get a totally brilliant analysis, mate!

Analysing... Hang tight, mate!

This site is always under construction!

TF-IDF (Term Frequency-Inverse Document Frequency) is a brilliant way to figure out how important a word or phrase is in a document. Higher scores mean more important words or phrases!

We show the top 7 TF-IDF scores because it's a good balance between providing enough information and not overwhelming the user. It's also considered a "lucky" number in many cultures!

View Source

`); }); app.post('/analyse', async (c) => { const { text } = await c.req.json(); if (text.length > 100000) { return c.json({ error: "Text too long. Please limit to 100,000 characters." }, 400); } const words = text.trim().split(/\s+/).length; const characters = text.length; // Calculate TF-IDF const tfidfScores = calculateTFIDF(text, 7); // Calculate word frequency const frequency = calculateWordFrequency(text, 30); // Generate fun fact const funFact = generateFunFact(text); // Perform sentiment analysis const sentimentAnalysis = analyseSentiment(text); // Generate summary const summary = generateSummary(text); return c.json({ words, characters, tfidf: tfidfScores, frequency, funFact, sentiment: sentimentAnalysis, summary }); }); app.post('/fetch-url', async (c) => { const { url } = await c.req.json(); try { const content = await fetchUrlContent(url); if (content.length > 100000) { return c.json({ error:"Content from URL is too long. Please try a different URL with less content." }, 400); } return c.json({ content }); } catch (error) { console.error('Error fetching URL:', error); return c.json({ error: error.message }, 400); } }); export default async function(request: Request): Promise { return app.fetch(request); }

sanguineCyanMastodon

Version 61