import { redditNew } from "https://esm.town/v/substrate/redditNew"; import { ComputeJSON, ComputeText, sb, Substrate } from "npm:substrate"; import { z } from "npm:zod"; import { zodToJsonSchema } from "npm:zod-to-json-schema"; const substrate = new Substrate({ apiKey: Deno.env.get("SUBSTRATE_API_KEY") }); const subreddit = "langchain"; const url = `https://www.reddit.com/r/${subreddit}/new/.json`; const response = await fetch(url, { headers: { "User-Agent": "Mozilla/5.0 (compatible; YourBot/1.0; +http://yourwebsite.com)", }, }); if (!response.ok) { console.log("Response status:", response.status); console.log("Response text:", await response.text()); throw new Error(`HTTP error! status: ${response.status}`); } const contentType = response.headers.get("content-type"); if (!contentType || !contentType.includes("application/json")) { console.log("Unexpected content type:", contentType); console.log("Response text:", await response.text()); throw new Error("Oops! We haven't gotten JSON!"); } const searchResults = await response.json(); // Extract summary, sentiment analysis, and other info for each tweet const extractedTweet = z.object({ summary: z.string().describe( "Summarize in a couple sentences: what is the text about and how it is related to the topic.", ), isProfile: z.boolean().describe("True if the link is a twitter profile, not a post."), sentiment: z.enum(["positive", "neutral", "negative"]), url: z.string().describe("url"), }); let summaries = []; for (const result of searchResults.results) { summaries.push( new ComputeJSON({ prompt: `Summarize this tweet and how it relates to the topic: TODO COMMENT: ${JSON.stringify(result)}`, json_schema: zodToJsonSchema(extractedTweet), }, { cache_age: 60 * 60 * 24 }), ); } // Generate markdown summary const markdown = new ComputeText({ prompt: sb.concat( `Below is a list of summarized posts about TODO. Generate concise markdown summarizing the results. DO NOT INCLUDE a title or introduction. Include the summary of the tweet and sentiment. DO NOT INCLUDE tweets that are twitter profiles (isProfile = true). DO NOT INCLUDE tweets that are updates and tutorials from LangChainAI. After the summary add a link to the tweet in this format: [full url](https://twitter.com//status/) Just return the markdown. Do not introduce with "Here is" or explain the output. RESULTS:\n`, // @ts-ignore ...summaries.map((s) => sb.jq(s.future.json_object, "@json")), ), model: "Llama3Instruct70B", }); const stream = await substrate.stream(markdown); // Render streaming markdown export default async function handler(req: Request): Promise { let { readable, writable } = new TransformStream(); let writer = writable.getWriter(); const textEncoder = new TextEncoder(); writer.write(textEncoder.encode(`

`)); new Promise(async () => { // for await (const event of stream.get(markdown)) { // if (event.object === "node.delta") { // const chunk = event.data.text || ""; // writer.write(textEncoder.encode(``)); // } // } writer.write(textEncoder.encode(`

${JSON.stringify(searchResults, null, 2)}

`)); writer.write(textEncoder.encode(``)); writer.close(); }); return new Response(readable, { headers: { "Content-Type": "text/html", }, }); }

redditAPIRAG