import process from "node:process";
export async function getWebLoaderBuilder(
url: string,
type: "webpage" | "github" | "audio" = "webpage",
options?: any,
) {
const { cond, matches } = await import("npm:lodash-es");
const setup = cond([
[
matches({ type: "webpage" }),
async () => {
await import("npm:cheerio");
const { CheerioWebBaseLoader } = await import(
"npm:langchain/document_loaders/web/cheerio"
);
return new CheerioWebBaseLoader(url);
},
],
[
matches({ type: "github" }),
async () => {
await import("npm:ignore");
const { GithubRepoLoader } = await import(
"npm:langchain/document_loaders/web/github"
);
return new GithubRepoLoader(url, options);
},
],
[
matches({ type: "audio" }),
async () => {
const { AudioTranscriptLoader } = await import(
"npm:langchain/document_loaders/web/assemblyai"
);
return new AudioTranscriptLoader({
audio_url: url,
...options,
}, {
apiKey: process.env.ASSEMBLYAI,
});
},
],
]);
return () => setup({ type });
}