sub.audio | animated subtitles for any audio file

import { ComputeJSON, sb, Substrate, TranscribeSpeech } from "npm:substrate"; import { extractValInfo } from "https://esm.town/v/pomdtr/extractValInfo?v=29"; export const listChapters = `Please provide a list of sections for the following transcript. Such a list might be used to define a table of contents for a podcast or video. Section titles should be short, rarely more than 5 words. If it's a short transcript, you might only have 1 or 2 sections and if it's a long transcript, limit it to 16. These sections should be the main topics or themes of the transcript. They should be chronologically ordered. They should be roughly evenly spaced throughout the transcript. Do not neglect the latter end of the transcript Do not include minutiae.`; export const timestampPrompt = `You will be provided with a list of sections as well as a transcript. The sections will be used as a table of contents for that transcript. Based on the content of the transcript, please provide the approximate timestamp in seconds where each section begins. Lines are prefixed with the timestamp in seconds surrounded by square brackets. e.g. [59.73] Hello I'm going to talk about some topic. Your job is to analyze the semantics of the text and provide the timestamp (in seconds) where each section begins. The first timestamp should be 0.`; async function processAudio(audio_uri) { const substrate = new Substrate({ apiKey: Deno.env.get("SUBSTRATE_API_KEY") }); const opts = { cache_age: 60 * 60 * 24 * 7 }; const transcribe = new TranscribeSpeech( { audio_uri, segment: true, align: true }, opts, ); const chaptersSchema = { type: "object", properties: { chapters: { type: "array", items: { type: "string" }, minItems: 1, maxItems: 16, }, }, }; const chapters = new ComputeJSON( { prompt: sb.concat( listChapters, "\n\nTRANSCRIPT:\n\n", transcribe.future.text, ), json_schema: chaptersSchema, model: "Mixtral8x7BInstruct", }, opts, ); const timestampedSchema = { type: "object", properties: { chapters: { type: "array", items: { type: "object", properties: { section: { type: "string" }, start: { type: "number" }, }, required: ["section", "start"], }, }, }, }; const timestamps = new ComputeJSON( { prompt: sb.concat( timestampPrompt, "SECTIONS: ", sb.jq<"string">(chapters.future.json_object, ".chapters | @json"), "\n\nTRANSCRIPT:\n\n", transcribe.future.text, ), json_schema: timestampedSchema, model: "Mixtral8x7BInstruct", }, opts, ); const res = await substrate.run(transcribe, chapters, timestamps); return { transcript: res.get(transcribe), timestampedChapters: res.get(timestamps).json_object?.chapters, }; } // Render site export default async function(req: Request): Promise { const url = new URL(req.url); const audio_uri = url.searchParams.get("url"); let html = ` sub.audio | animated subtitles for any audio file ${ !audio_uri ? `Or try an example: federer-dartmouth.m4a` : "" }

Running...

${ audio_uri ? `

` : "" }

made with substrate view source

`; if (!audio_uri) { html += ` `; return new Response(html, { headers: { "Content-Type": "text/html" } }); } const { transcript, timestampedChapters } = await processAudio(audio_uri); html += ` `; return new Response(html, { headers: { "Content-Type": "text/html", }, }); }

subaudio

Versions

v0