import { Hono } from "https://esm.sh/hono@3.10.2"; import { encode } from "https://esm.sh/js-base64@3.7.5"; import { parse } from "https://esm.sh/node-html-parser@6.1.5"; const app = new Hono(); // Expanded stopwords list with additional web-specific terms const stopwords = new Set([ 'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', 'could', 'did', 'do', 'does', 'doing', 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', 'has', 'have', 'having', 'he', 'her', 'here', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'its', 'itself', 'just', 'me', 'more', 'most', 'my', 'myself', 'no', 'nor', 'not', 'now', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'same', 'she', 'should', 'so', 'some', 'such', 'than', 'that', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'these', 'they', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', 'we', 'were', 'what', 'when', 'where', 'which', 'while', 'who', 'whom', 'why', 'with', 'would', 'you', 'your', 'yours', 'yourself', 'yourselves', // Additional navigation-related terms 'back', 'contents', 'menu', 'home', 'next', 'previous', 'top', 'bottom', 'navigation', 'nav', 'sidebar', 'footer', 'header', 'main', 'article', 'section', 'page', 'skip', 'jump', 'return', // Web-specific terms 'image', 'photo', 'photograph', 'picture', 'fullscreen', 'share', 'reuse', 'features', 'topics', 'observer', 'getty', 'rex', 'shutterstock', 'ap', 'afp', 'reuters', 'epa', 'pa', 'alamy', 'getty images', 'associated press', 'agence france-presse', 'press association', 'copyright', 'rights reserved', 'all rights reserved', 'click', 'tap', 'swipe', 'scroll', 'zoom', 'enlarge', 'expand', 'collapse', 'hide', 'show', 'login', 'sign in', 'register', 'subscribe', 'newsletter', 'email', 'follow', 'like', 'comment', 'share', 'twitter', 'facebook', 'instagram', 'linkedin', 'pinterest', 'youtube', 'google', 'advertisement', 'sponsored', 'promoted', 'partner content', 'paid content', 'cookie', 'privacy', 'terms', 'conditions', 'policy', 'agreement', 'mobile', 'desktop', 'tablet', 'app', 'website', 'browser', 'search', 'find', 'results', 'query', 'keyword', 'loading', 'please wait', 'error', 'not found', '404', 'server error', '500', 'menu', 'navigation', 'sitemap', 'archive', 'category', 'tag', 'related', 'popular', 'trending', 'breaking news', 'latest', 'update', 'live', 'developing story', 'read more', 'continue reading', 'full article', 'paywall', 'subscribe now', 'comments', 'discussion', 'forum', 'community', 'join the conversation', 'share your thoughts', 'leave a comment', 'what do you think', 'fact check', 'analysis', 'opinion', 'editorial', 'column', 'blog post', 'video', 'audio', 'podcast', 'gallery', 'slideshow', 'interactive', 'download', 'upload', 'attachment', 'file', 'document', 'print', 'email', 'save', 'bookmark', 'favorite', 'font size', 'text size', 'dark mode', 'light mode', 'accessibility', 'support', 'help', 'faq', 'contact us', 'about us', 'team', 'careers', 'advertise', 'partnerships', 'sponsorships', 'media kit', 'rss feed', 'xml', 'api', 'widget', 'embed', 'copyright', 'trademark', 'legal', 'disclaimer', 'terms of use', 'privacy policy' ]); // Expanded Synonym dictionary (unchanged) const synonyms = { "important": ["significant", "crucial", "vital", "essential", "key"], // ... (rest of the synonyms dictionary remains unchanged) }; // Simple sentiment analysis function (unchanged) function analyseSentiment(text) { const positiveWords = new Set([ 'good', 'great', 'excellent', 'wonderful', 'amazing', 'fantastic', 'awesome', 'happy', 'joy', 'love', 'like', 'beautiful', 'nice', 'pleasant', 'delightful' ]); const negativeWords = new Set([ 'bad', 'terrible', 'awful', 'horrible', 'dreadful', 'poor', 'disappointing', 'sad', 'unhappy', 'hate', 'dislike', 'ugly', 'unpleasant', 'nasty', 'cruel' ]); const words = text.toLowerCase().match(/\b\w+\b/g) || []; let score = 0; words.forEach(word => { if (positiveWords.has(word)) score++; if (negativeWords.has(word)) score--; }); let sentiment; if (score > 0) { sentiment = "positive"; } else if (score < 0) { sentiment = "negative"; } else { sentiment = "neutral"; } return { score, sentiment }; } // Improved TF-IDF implementation with additional filtering function calculateTFIDF(text, outputSize = 7) { const words = text.toLowerCase().match(/\b[a-z]{2,}\b/g) || []; const wordCount = words.length; const termFrequency = {}; const documentFrequency = {}; // Calculate term frequency and document frequency for (let i = 0; i < words.length; i++) { for (let j = 1; j <= 3 && i + j <= words.length; j++) { const phrase = words.slice(i, i + j).join(' '); if (!phrase.split(' ').every(word => stopwords.has(word))) { termFrequency[phrase] = (termFrequency[phrase] || 0) + 1; documentFrequency[phrase] = 1; } } } // Calculate TF-IDF with improved scoring const tfidf = {}; Object.keys(termFrequency).forEach(term => { const tf = termFrequency[term] / wordCount; const idf = Math.log(wordCount / (documentFrequency[term] || 1)) + 1; const termWordCount = term.split(' ').length; // Boost score for multi-word phrases, less common terms, and longer phrases tfidf[term] = tf * idf * (termWordCount * 0.5) * (1 / Math.sqrt(termFrequency[term])) * (term.length / 10); }); // Sort by score (descending) and get top unique phrases const uniquePhrases = new Set(); return Object.entries(tfidf) .sort((a, b) => b[1] - a[1]) .filter(([phrase, _]) => { if (uniquePhrases.size >= outputSize) return false; const words = phrase.split(' '); if (words.length > 3) return false; if (words.some(word => word.length < 3)) return false; if (words.every(word => stopwords.has(word))) return false; // Check if the phrase is a subset of an already included phrase if (Array.from(uniquePhrases).some(existingPhrase => existingPhrase.includes(phrase))) return false; // Additional filtering to remove unwanted terms if (phrase.includes('http') || phrase.includes('www') || phrase.includes('com')) return false; if (/\d/.test(phrase)) return false; // Exclude phrases containing numbers if (phrase.length < 4) return false; // Exclude very short phrases uniquePhrases.add(phrase); return true; }) .slice(0, outputSize); } // Updated function to calculate word frequency function calculateWordFrequency(text, topN = 30) { const words = text.toLowerCase().match(/\b[a-z]{3,}\b/g) || []; const frequency = {}; words.forEach(word => { if (!stopwords.has(word) && !word.includes('http') && !word.includes('www') && !word.includes('com')) { frequency[word] = (frequency[word] || 0) + 1; } }); return Object.entries(frequency) .sort((a, b) => b[1] - a[1]) .slice(0, topN); } // Updated function to generate a fun fact function generateFunFact(text) { const wordCount = text.trim().split(/\s+/).length; const charCount = text.length; const sentences = text.split(/[.!?]+/).length; const avgWordLength = wordCount > 0 ? charCount / wordCount : 0; const words = text.match(/\b\w+\b/g) || []; const longestWord = words.length > 0 ? words.reduce((a, b) => a.length > b.length ? a : b) : 'N/A'; const facts = [ `Your text has an average word length of ${avgWordLength.toFixed(2)} characters. Shakespeare's average was about 3.99!`, `You've written ${sentences} sentence${sentences !== 1 ? 's' : ''}. A typical novel has about 50-60 sentences per page.`, `Your text is ${wordCount} words long. "War and Peace" by Leo Tolstoy is about 587,287 words!`, `Your longest word is "${longestWord}". The longest word in English is 189,819 letters long!`, `You've used ${charCount} characters. That's about ${(charCount / 280).toFixed(2)} tweets' worth of content!` ]; return facts[Math.floor(Math.random() * facts.length)]; } // Function to generate a simple summary (unchanged) function generateSummary(text, sentenceCount = 3) { const sentences = text.match(/[^\.!\?]+[\.!\?]+/g) || []; return sentences.slice(0, sentenceCount).join(' '); } // Updated function to clean the extracted text function cleanText(text) { return text .replace(/<[^>]*>/g, '') // Remove HTML tags .replace(/\s+/g, ' ') // Replace multiple spaces with a single space .replace(/[^\w\s.,!?-]/g, ' ') // Replace special characters with spaces, except for some punctuation .replace(/\b(https?:\/\/)?[^\s]+\.(com|org|net|edu|gov)\b/g, '') // Remove URLs .replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, '') // Remove email addresses .replace(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, '') // Remove IP addresses .replace(/\b(back to (contents|top)|table of contents|skip to (main content|navigation))\b/gi, '') // Remove common navigation phrases .trim(); // Remove leading and trailing whitespace } // Updated function to fetch and parse URL content async function fetchUrlContent(url) { try { console.log('Fetching URL:', url); const response = await fetch(url); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } const html = await response.text(); console.log('Fetched HTML length:', html.length); const root = parse(html); if (!root || typeof root.querySelectorAll !== 'function') { throw new Error('Failed to parse HTML content'); } // Remove script, style, nav, header, footer, and other non-content elements const elementsToRemove = root.querySelectorAll('script, style, nav, header, footer, aside, .navigation, .menu, .sidebar, [role="navigation"], .ad, .advertisement, .social-share, .related-articles, .comments, .author-bio'); elementsToRemove.forEach(el => el.remove()); // Extract content from the main content area if possible, otherwise use body const mainContent = root.querySelector('main, #main, #content, .content, article, [role="main"]') || root.querySelector('body'); let content = mainContent ? mainContent.textContent : ''; if (!content) { console.log('No content found in parsed HTML, falling back to full body content'); content = root.querySelector('body').textContent; } console.log('Extracted content length:', content.length); // Clean the extracted text content = cleanText(content); console.log('Cleaned content length:', content.length); if (content.trim().length === 0) { throw new Error('Cleaned content is empty'); } return content; } catch (error) { console.error('Error fetching URL:', error); throw new Error(`Failed to fetch URL content: ${error.message}`); } } app.get("/", (c) => { return c.html(` Radical Text Analyser

Welcome to the Radical Text Analyser, mate!

Radical Text Analyser

Enter your text or a URL below to get a totally brilliant analysis, mate!

Analysing... Hang tight, mate!

This site is always under construction!

TF-IDF (Term Frequency-Inverse Document Frequency) is a brilliant way to figure out how important a word or phrase is in a document. Higher scores mean more important words or phrases!

We show the top 7 TF-IDF scores because it's a good balance between providing enough information and not overwhelming the user. It's also considered a "lucky" number in many cultures!

View Source

`); }); app.post('/analyse', async (c) => { const { text } = await c.req.json(); if (text.length > 100000) { return c.json({ error: "Text too long. Please limit to 100,000 characters." }, 400); } const words = text.trim().split(/\s+/).length; const characters = text.length; // Calculate TF-IDF const tfidfScores = calculateTFIDF(text, 7); // Calculate word frequency const frequency = calculateWordFrequency(text, 30); // Generate fun fact const funFact = generateFunFact(text); // Perform sentiment analysis const sentimentAnalysis = analyseSentiment(text); // Generate summary const summary = generateSummary(text); return c.json({ words, characters, tfidf: tfidfScores, frequency, funFact, sentiment: sentimentAnalysis, summary }); }); app.post('/fetch-url', async (c) => { const { url } = await c.req.json(); try { console.log('Fetching URL:', url); const content = await fetchUrlContent(url); console.log('Fetched content length:', content.length); if (content.length > 100000) { return c.json({ error: "Content from URL is too long. Please try a different URL with less content." }, 400); } if (content.trim().length === 0) { return c.json({ error: "The fetched content is empty. Please try a different URL." }, 400); } return c.json({ content }); } catch (error) { console.error('Error fetching URL:', error); return c.json({ error: error.message }, 400); } }); export default async function(request: Request): Promise { return app.fetch(request); }

lovelyYellowXerinae

Version 3