Public
Script
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
import process from "node:process";
export let generateEmbeddings = async (
req: express.Request,
res: express.Response,
) => {
await import("npm:cheerio");
const url = req?.body?.url;
if (!url) {
res.status(400)
.json({
message: "The URL parameter is required for this end point",
});
}
const { OpenAIEmbeddings } = await import("npm:langchain/embeddings");
const { createClient } = await import(
"https://esm.sh/@supabase/supabase-js@2"
);
const { SupabaseVectorStore } = await import("npm:langchain/vectorstores");
const { RecursiveCharacterTextSplitter } = await import(
"npm:langchain/text_splitter"
);
const { CheerioWebBaseLoader } = await import(
"npm:langchain/document_loaders/web/cheerio"
);
const client = createClient(
process.env.supabaseURL,
process.env.supabaseKey,
);
const loader = new CheerioWebBaseLoader(url);
const docs = await loader.load();
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize: 1000,
chunkOverlap: 200,
});
// Add URL metadata to each document
const splittedDocs = await textSplitter.splitDocuments(docs);
// Create a vector store from the documents.
const vectorStore = await SupabaseVectorStore.fromDocuments(
splittedDocs,
new OpenAIEmbeddings({
openAIApiKey: process.env.OPEN_API_KEY,
}),
{
client,
tableName: "documents",
queryName: "match_documents",
},
);
res.status(200).json({
message: "Embedding generated",
});
};
Val Town is a social website to write and deploy JavaScript.
Build APIs and schedule functions from your browser.
Comments
Nobody has commented on this val yet: be the first!
October 23, 2023