Unlisted
HTTP
Forked from substrate/twitterRAG
substrate-redditapirag.web.val.run
July 14, 2024
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import { redditNew } from "https://esm.town/v/substrate/redditNew";
import { ComputeJSON, ComputeText, sb, Substrate } from "npm:substrate";
import { z } from "npm:zod";
import { zodToJsonSchema } from "npm:zod-to-json-schema";
const substrate = new Substrate({ apiKey: Deno.env.get("SUBSTRATE_API_KEY") });
const subreddit = "langchain";
const url = `https://www.reddit.com/r/${subreddit}/new/.json`;
const response = await fetch(url, {
headers: {
"User-Agent": "Mozilla/5.0 (compatible; YourBot/1.0; +http://yourwebsite.com)",
},
});
if (!response.ok) {
console.log("Response status:", response.status);
console.log("Response text:", await response.text());
throw new Error(`HTTP error! status: ${response.status}`);
}
const contentType = response.headers.get("content-type");
if (!contentType || !contentType.includes("application/json")) {
console.log("Unexpected content type:", contentType);
console.log("Response text:", await response.text());
throw new Error("Oops! We haven't gotten JSON!");
}
const searchResults = await response.json();
// Extract summary, sentiment analysis, and other info for each tweet
const extractedTweet = z.object({
summary: z.string().describe(
"Summarize in a couple sentences: what is the text about and how it is related to the topic.",
),
isProfile: z.boolean().describe("True if the link is a twitter profile, not a post."),
sentiment: z.enum(["positive", "neutral", "negative"]),
url: z.string().describe("url"),
});
let summaries = [];
for (const result of searchResults.results) {
summaries.push(
new ComputeJSON({
prompt: `Summarize this tweet and how it relates to the topic: TODO
COMMENT: ${JSON.stringify(result)}`,
json_schema: zodToJsonSchema(extractedTweet),
}, { cache_age: 60 * 60 * 24 }),
);
}
// Generate markdown summary
const markdown = new ComputeText({
prompt: sb.concat(
`Below is a list of summarized posts about TODO.
Generate concise markdown summarizing the results.
DO NOT INCLUDE a title or introduction.
Include the summary of the tweet and sentiment.
DO NOT INCLUDE tweets that are twitter profiles (isProfile = true).
DO NOT INCLUDE tweets that are updates and tutorials from LangChainAI.
After the summary add a link to the tweet in this format: [full url](https://twitter.com/<handle>/status/<id>)
Just return the markdown. Do not introduce with "Here is" or explain the output.
RESULTS:\n`, // @ts-ignore
...summaries.map((s) => sb.jq(s.future.json_object, "@json")),
),
model: "Llama3Instruct70B",
});
const stream = await substrate.stream(markdown);
// Render streaming markdown
export default async function handler(req: Request): Promise<Response> {
let { readable, writable } = new TransformStream();
let writer = writable.getWriter();
const textEncoder = new TextEncoder();
writer.write(textEncoder.encode(`<!DOCTYPE html>
<html>
<head>
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
<style>
.link-pair {
display: inline-block;
margin-right: 10px;
}
.link-pair a {
display: inline-block;
margin-right: 5px;
}
</style>
</head>
<body>
<div id="content"></div>
<script>
let content = document.getElementById('content');
let markdown = '';
</script>`));
new Promise(async () => {
// for await (const event of stream.get(markdown)) {
// if (event.object === "node.delta") {
// const chunk = event.data.text || "";
// writer.write(textEncoder.encode(`<script>
// markdown += ${JSON.stringify(chunk)};
Val Town is a social website to write and deploy JavaScript.
Build APIs and schedule functions from your browser.
Comments
Nobody has commented on this val yet: be the first!
July 14, 2024