/** @jsx jsx */ import { jsx } from "https://deno.land/x/hono@v3.11.7/middleware.ts"; import { Hono } from "https://deno.land/x/hono@v3.11.7/mod.ts"; import { ai } from "https://esm.town/v/yawnxyz/ai"; import { getUrlMetadata } from "https://esm.town/v/yawnxyz/urlMetadata"; import { getHtmlMetadata } from "https://esm.town/v/yawnxyz/htmlMetadata"; import { getCitation } from "https://esm.town/v/yawnxyz/citation"; import { blobby } from "https://esm.town/v/yawnxyz/blobby"; import { transcribeAudio } from "https://esm.town/v/yawnxyz/stt"; import stringHash from 'npm:string-hash'; const app = new Hono(); // https://www.crossref.org/blog/dois-and-matching-regular-expressions/ const DOI_REGEX = /\b(10\.\d{4,9}\/[-._;()\/:\w]+)\b/i; export const getJinaContent = async (url, opts = {}) => { const baseUrl = 'https://r.jina.ai/'; if (!url.includes('r.jina.ai')) { url = baseUrl + url; } const fullUrl = new URL(url); const headers = { ...(opts.withImagesSummary && { 'X-With-Images-Summary': 'true' }), ...(opts.withGeneratedAlt && { 'X-With-Generated-Alt': 'true' }), ...(opts.withLinksSummary && { 'X-With-Links-Summary': 'true' }), ...(opts.noCache && { 'X-No-Cache': 'true' }), ...(opts.accept && { 'Accept': opts.accept }), ...(opts.targetSelector && { 'X-Target-Selector': opts.targetSelector }), ...(opts.timeout && { 'X-Timeout': opts.timeout.toString() }), ...(opts.waitForSelector && { 'X-Wait-For-Selector': opts.waitForSelector }), ...(opts.returnFormat && { 'X-Return-Format': opts.returnFormat }), }; console.log('[getJinaContent] Fetching:', fullUrl.toString(), headers); try { const response = await fetch(fullUrl.toString(), { method: 'GET', headers: headers, }); if (!response.ok) { throw new Error(`HTTP error! Status: ${response.status}`); } if (headers['Accept'] === 'text/event-stream') { return response.body; } const text = await response.text(); try { return JSON.parse(text); } catch { if (headers['X-Return-Format'] === 'screenshot') { return JSON.parse(text); } else { return text; } } } catch (error) { console.error('Error fetching from Jina:', error); throw error; } } export const getFirecrawlContent = async (url, opts = {}) => { const apiKey = Deno.env.get("FIRECRAWL_API_KEY"); if (!apiKey) { throw new Error("API key not found. Please set FIRECRAWL_API_KEY in your environment."); } const body = { url: url, pageOptions: { onlyMainContent: opts.onlyMainContent || true, includeHtml: opts.includeHtml || false, includeRawHtml: opts.includeRawHtml || false, screenshot: opts.screenshot || false, waitFor: opts.waitFor || 0, removeTags: opts.removeTags || [], onlyIncludeTags: opts.onlyIncludeTags || [], replaceAllPathsWithAbsolutePaths: opts.replaceAllPathsWithAbsolutePaths || true, parsePDF: opts.parsePDF || false }, extractorOptions: { // mode: opts.mode || "llm-extraction", mode: opts.mode, extractionPrompt: opts.extractionPrompt || "", extractionSchema: opts.extractionSchema || {} }, timeout: opts.timeout || 30000 }; console.log('[getFirecrawlContent] Fetching:', url, body); try { const response = await fetch("https://api.firecrawl.dev/v0/scrape", { method: 'POST', headers: { "Authorization": `Bearer ${apiKey}`, "Content-Type": "application/json" }, body: JSON.stringify(body) }); if (!response.ok) { throw new Error(`HTTP error! Status: ${response.status}`); } const result = await response.json(); let resultText = JSON.stringify(result); console.log('[getFirecrawlContent] Result:', resultText); return resultText; } catch (error) { console.error('Error fetching from Firecrawl:', error); throw error; } }; export const getContentFromUrl_v1 = async (url: string) => { try { let ogUrl = url; if (url.includes("youtube") || url.includes("youtu.be")) { url = `https://val.markdown.download/?url=${url}`; } url = "https://r.jina.ai/" + url; let [response, metadata] = await Promise.all([ fetch(url), getUrlMetadata(ogUrl) ]); const contentTextData = await response.text(); if (!metadata) { const html = await getJinaContent(url, { returnFormat: 'html' }); metadata = await getHtmlMetadata(html); } // console.log('---> metadata?? ', metadata, ' url:' + url) let doi if (metadata) { const doiRegex = /10.\d{4,9}\/[-._;()\/:A-Z0-9]+/gi; const doiMatch = JSON.stringify(metadata).match(doiRegex); doi = doiMatch ? doiMatch[0] : null; if (doi) { doi = doi.replace(/^doi[:\/]/i, ''); } } console.log('DOI found?:', doi); let citation if (doi) { citation = await getCitation(doi); } else { } if (!response.ok) { throw new Error(`HTTP error! Status: ${response.status}`); } let output = "" if(doi) { output = output + "\n " + doi + "\n\n\n" } if(citation) { output = output + "\n " + citation + "\n\n\n" } if(contentTextData) { output = output + "\n " + contentTextData + "\n\n\n" } if(metadata) { output = output + "\n " + JSON.stringify(metadata, null, 2) + "\n\n\n" } return output } catch (error) { console.error('Error fetching the URL:', error); return 'Error fetching the content: ', error.message || error || 'Unknown error.'; } }; export const extractDOI = (content) => { if(!content) return null; if(!content.match) return null; const doiRegex = DOI_REGEX; const doiMatch = content.match(doiRegex); let doi = doiMatch ? doiMatch[0] : null; if (doi) { doi = doi.replace(/^doi[:\/]/i, ''); } return doi; }; export const getContentFromUrl = async ({ url, optionStr = 'content', crawlMode = 'jina', useCache = true }) => { try { // Check if the input is a pure DOI const doiRegex = DOI_REGEX; if (doiRegex.test(url) && !url.includes("://")) { url = `https://doi.org/${url}`; } else if (!url.startsWith("http")) { return c.text("URL query parameter, valid path, or DOI is required."); } // Check if the URL is an MP3 file if (url.endsWith('.mp3')) { const transcript = await transcribeAudio(url); // Get transcript for MP3 const output = `\n ${transcript}\n\n\n`; const cacheId = `cache:${stringHash(`${url}_content_mp3`)}`; // Create a cache ID for the MP3 if (useCache) { await blobby.set(cacheId, output); // Cache the output console.log('Cache set:', cacheId); } return `${url} ${cacheId}${output}`; // Return source and cache ID } let ogUrl = url; let urlObject = new URL(url); if (urlObject.href.includes('&opts=')) { // Check if the URL contains '&opts=' // Remove everything from '&opts=' to the end of the string // *** this might be brittle!! urlObject.href = urlObject.href.split('&opts=')[0]; } url = urlObject.toString(); let str = `${url}_${optionStr}_${crawlMode}`; const cacheId = `cache:${stringHash(str)}`; console.log('[gCFU] useCache:', useCache, 'cacheId:', cacheId) if (useCache) { const cachedContent = await blobby.get(cacheId); if (cachedContent) { console.log('[gCFU] cache hit:', cacheId); return cachedContent; } } if (url.includes("youtube") || url.includes("youtu.be")) { url = `https://val.markdown.download/?url=${url}`; } const serviceUrl = crawlMode == 'fire' ? url : "https://r.jina.ai/" + url; console.log('getting content from url', serviceUrl); const optsArray = optionStr.split(','); const requestOptions = { returnFormat: optsArray.includes('html') ? 'html' : optsArray.includes('json') ? 'json' : undefined, withImagesSummary: optsArray.includes('images'), withLinksSummary: optsArray.includes('links'), withGeneratedAlt: optsArray.includes('caption') }; const fetchMetadata = optsArray.includes('metadata') || optsArray.includes('doi') || optsArray.includes('citation'); let metadata; if (fetchMetadata) { metadata = await getUrlMetadata(ogUrl); } const content = crawlMode === 'fire' ? await getFirecrawlContent(serviceUrl, requestOptions) : await getJinaContent(serviceUrl, requestOptions); if (fetchMetadata && !metadata) { const html = crawlMode === 'fire' ? await getFirecrawlContent(serviceUrl, { returnFormat: 'html' }) : await getJinaContent(serviceUrl, { returnFormat: 'html' }); metadata = await getHtmlMetadata(html); } if(!metadata || Object.keys(metadata).length === 0) { metadata = content; // for pdfs, it'll just have to be the content } let doi; if (optsArray.includes('doi') || optsArray.includes('citation')) { if (metadata) { doi = extractDOI(JSON.stringify(metadata)); } if (!doi) { doi = extractDOI(content); } if (!doi) { doi = extractDOI(metadata); } console.log('DOI found?:', doi); } let citation; if (optsArray.includes('citation') && doi) { citation = await getCitation(doi); } let output = ""; output += "" + serviceUrl + "" + " "; output += "" + cacheId + ""; output += "" + new Date().toLocaleString() + ""; if (optsArray.includes('doi') && doi) { output += "\n " + doi + "\n\n\n"; } if (optsArray.includes('citation') && citation) { output += "\n " + citation + "\n\n\n"; } if ((optsArray.includes('content')||optsArray.includes('html')) && content) { const contentString = typeof content === 'object' ? JSON.stringify(content, null, 2) : content; output += "\n " + contentString + "\n\n\n"; } if (optsArray.includes('metadata') && metadata) { output += "\n " + JSON.stringify(metadata, null, 2) + "\n\n\n"; } if (useCache) { await blobby.set(cacheId, output); console.log('Cache set:', cacheId); } if (output.length === 0) { output = '(This area intentionally left blank)'; } return output; } catch (error) { console.error('Error fetching the URL:', error); throw new Error('Error fetching the content: ' + error.message || error || 'Unknown error.'); // return 'Error fetching the content: ' + error.message || error || 'Unknown error.'; } }; app.get("/*", async (c) => { let url = c.req.query("url"); let opts = c.req.query("opts") || 'content'; let crawlMode = c.req.query("crawlMode") || 'jina'; let useCache = typeof c.req.query("useCache") !== 'undefined' ? c.req.query("useCache") : true; // let noCache = c.req.query("noCache") === 'true'; // let useCache = !noCache; if (!url) { const rawUrl = c.req.raw.url; const pathStartIndex = rawUrl.indexOf('/', rawUrl.indexOf('://') + 3); url = rawUrl.slice(pathStartIndex + 1); // Get the full path after the initial '/' } // Check if the input is a pure DOI const doiRegex = DOI_REGEX; if (doiRegex.test(url) && !url.includes("://")) { url = `https://doi.org/${url}`; } else if (!url.startsWith("http")) { return c.text("URL query parameter, valid path, or DOI is required."); } try { const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache }); return c.text(content); } catch (error) { return c.text(error.message || error || 'Unknown error.'); } }); app.post("/", async (c) => { const body = await c.req.json(); const { url, opts = 'content', crawlMode = 'jina', useCache, getTags, getTagsPrompt, getSummary, getSummaryPrompt, getEmbeddings } = body; try { const content = await getContentFromUrl({ url, optionStr: opts, crawlMode, useCache }); if (!getSummary && !getTags) { return c.text(content); } const [summary, tags, embeddings] = await Promise.all([ getSummary ? getSummaryFn(getSummaryPrompt, content) : null, getTags ? getTagsFn(getTagsPrompt, content) : null, getEmbeddings ? getEmbeddingsFn(content) : null ]); return c.json({ summary: summary?.text, tags: tags?.text, content, embeddings }); } catch (error) { return c.json({ error: error.message || error || "Unknown error."}) } }); export const getBlurbFromUrl = async (url: string, { content, getSummary = true, getSummaryPrompt, getTags = true, getTagsPrompt, getEmbeddings, model, provider } = {}) => { content = content || await getContentFromUrl_v1(url); if (!getSummary || !getTags) { return content; } let summary, tags, embeddings if (getSummary) { summary = await getSummaryFn(getSummaryPrompt, content, { model, provider }); } if (getTags) { tags = await getTagsFn(getTagsPrompt, content, { model, provider }); } if (getEmbeddings) { embeddings = await getEmbeddingsFn(content + " " + summary + " " + tags); } return { content, summary: summary.text, tags: tags.text, embeddings } }; export const getSummaryFn = async (prompt, content, { provider, model, } = {}) => { prompt = prompt || "Summarize this article in 30 words, focusing on the primary author. Do not respond with JSON or XML or stage instructions. Only reply in text. Text:" let result = await ai({ provider: provider || "openai", model: model || "gpt-3.5-turbo", prompt: prompt + content }) return result } export const getTagsFn = async (prompt, content, { provider, model, } = {}) => { prompt = prompt || "Return a comma separated list of 3-4 tags that broadly describes the text. Do not repeat the given text. Just return text and commas, no other symbols. Text to categorize:" console.log("prompt", prompt) let result = await ai({ provider: provider || "openai", model: model || "gpt-3.5-turbo", prompt: prompt + content }) return result } export const getEmbeddingsFn = async (content) => { let result = await ai({ provider: "openai", embed: true, value: content }) return result } export default app.fetch;