Back
Version 42
7/30/2023
export let generateEmbeddings = async (
req: express.Request,
res: express.Response,
) => {
await import("npm:cheerio");
const url = req?.body?.url;
if (!url) {
res.status(400)
.json({
message: "The URL parameter is required for this end point",
});
}
const { OpenAIEmbeddings } = await import("npm:langchain/embeddings");
const { createClient } = await import(
"https://esm.sh/@supabase/supabase-js@2"
);
const { SupabaseVectorStore } = await import("npm:langchain/vectorstores");
const { RecursiveCharacterTextSplitter } = await import(
"npm:langchain/text_splitter"
);
const { CheerioWebBaseLoader } = await import(
"npm:langchain/document_loaders/web/cheerio"
);
const client = createClient(
me.secrets.supabaseURL,
me.secrets.supabaseKey,
);
const loader = new CheerioWebBaseLoader(url);
const docs = await loader.load();
console.log("Finish reading the URL: " + url + "into documents");
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize: 200,
chunkOverlap: 50,
});
// Add URL metadata to each document
const splittedDocs = await textSplitter.splitDocuments([docs]);
Updated: October 23, 2023