import { esm } from "https://esm.sh/v240/std/esm"; import { blob } from "https://esm.town/v/std/blob?v=12"; esm({ "together-ai": "https://esm.sh/together-ai", // Map together-ai }); import Together from "together-ai"; const togetherAI = new Together({ auth: process.env.c092457be8a67cacaa6f088294a2eaca96064e9b962589c035be669d9e7f31ff, }); const getCacheKey = (url: string): string => { return `webgen-url-cache-v2:${url}`; }; export default async (req) => { let { readable, writable } = new TransformStream(); let writer = writable.getWriter(); const textEncoder = new TextEncoder(); const loadingSpinnerHTMLStart = `

  Loading...
  `;
  const loadingSpinnerHTMLEnd = `

`; const loadingSpinnerHide = ``; const linkFixerScript = ` `; // 1. get the pathname of the URL and use it as the "fake URL " to generate from const promptURL = new URL(req.url).pathname.slice(1); const cachedResult = await blob.getJSON(getCacheKey(promptURL)); if (cachedResult) { return new Response(cachedResult, { headers: { "Content-Type": "text/html", }, }); } let pingInterval = setInterval(() => writer.write(textEncoder.encode("\n")), 1000); if (promptURL.trim() === "") { return new Response( `Please provide a URL!\nFor example, go to "https://thesephist-webgen.web.val.run/https://thesephist.com/".`, { headers: { "Content-Type": "text/plain", }, status: 400, }, ); } const promptURLHost = new URL(promptURL).hostname; let pageDescriptionInstructions = ""; let pageResult = ""; // // 2. Do one OpenAI inference to expand that URL to a longer page description const pageDescriptionStream = await togetherAI.inference("mistralai/Mixtral-8x7B-Instruct-v0.1", { prompt: ` You will be given a URL of an archived page whose contents have gone missing from our library. Please, to the best of your ability, write a detailed instruction about the likely contents of this page such that a competent web developer should be able to fully reconstruct its likely contents. Your description should include: - What content is on the page - Other elements like sidebars, links, images that may be in the page - The author and origin of the page - The stylistic aesthetic of the page Your description should be no longer than 300 words. Only respond with the description; do not add any introduction or additional commentary. `, max_tokens: 1024, stream: true, }); new Promise(async () => { // Stream back loading spinner and description writer.write(textEncoder.encode(loadingSpinnerHTMLStart)); for await (const part of pageDescriptionStream) { const text = part.choices[0]?.delta?.content || ""; writer.write(textEncoder.encode(text)); pageDescriptionInstructions += text; } writer.write(textEncoder.encode(loadingSpinnerHTMLEnd)); // 3. Generate the page const stream = await togetherAI.inference("mistralai/Mixtral-8x7B-Instruct-v0.1", { prompt: ` You are an expert programmer contracted to design and code beautiful simple webpages. Only respond with complete HTML pages, beginning with "". Do not include any other commentary. Do not use Markdown code fences. Use