Scribe • Audio Recorder

import { Hono } from "https://deno.land/x/hono@v3.11.7/mod.ts"; import { useRecorderExport } from "https://esm.town/v/yawnxyz/scribeUseRecorderExport"; // Inlined content from ./middleware.ts import "jsr:@std/dotenv/load" import { Groq } from "npm:groq-sdk"; const groq = new Groq(); import { HfInference } from "npm:@huggingface/inference"; const hf = new HfInference(Deno.env.get("HUGGINGFACE_API_KEY")); const MAX_CHUNK_SIZE_BYTES = 25 * 1024 * 1024; // 25 MB in bytes const DEFAULT_WHISPER_MODEL = 'whisper-large-v3-turbo'; const MAX_TRANSCRIPT_CONTEXT = 3000; // Characters to keep from transcript history type WhisperModel = 'whisper-large-v3-turbo' | 'distil-whisper-large-v3-en' | 'whisper-large-v3'; type WhisperResponseFormat = 'json' | 'verbose_json' | 'text'; interface TranscriptionParams { model?: WhisperModel; prompt?: string; response_format?: WhisperResponseFormat; language?: string; temperature?: number; } interface Transcription { text: string; segments?: Array<{ start: number; end: number; text: string; }>; duration?: number; } // Helper function to normalize transcription response function normalizeTranscription(response: string | Transcription): Transcription { console.log('Normalized transcription:', response); if (typeof response === 'string') { return { text: response }; } return response; } // Fetch the size of the file using HTTP HEAD request export async function fetchFileSize(url: string): Promise { const response = await fetch(url, { method: 'HEAD' }); const contentLength = response.headers.get('content-length'); return contentLength ? parseInt(contentLength, 10) : 0; } // Transcribe a single chunk of audio data using Deno's ReadableStream async function transcribeChunk(buffer: Uint8Array, params: TranscriptionParams): Promise { console.log('Transcribing chunk with params:', { model: params.model, language: params.language, response_format: params.response_format }); const blob = new Blob([buffer], { type: 'audio/wav' }); const file = new File([blob], 'audio.wav', { type: 'audio/wav' }); const transcription = await groq.audio.transcriptions.create({ file, model: params?.model || DEFAULT_WHISPER_MODEL, prompt: params?.prompt, response_format: params?.response_format || 'text', language: params?.language || 'en', temperature: 0 // Set to 0 for consistency }); return normalizeTranscription(transcription); } // Add this interface interface AudioInput { type: 'url' | 'file'; content: string | File; } export async function transcribeAudio(input: AudioInput, params: TranscriptionParams): Promise { console.log('Transcribing audio with params:', params); let fileSize: number; let getBuffer: (start: number, end: number) => Promise; if (input.type === 'file') { const file = input.content as File; fileSize = file.size; // For files, create buffer getter that slices the file getBuffer = async (start: number, end: number) => { const slice = file.slice(start, end + 1); const arrayBuffer = await slice.arrayBuffer(); return new Uint8Array(arrayBuffer); }; } else { const url = input.content as string; fileSize = await fetchFileSize(url); // For URLs, create buffer getter that fetches ranges getBuffer = async (start: number, end: number) => { const response = await fetch(url, { headers: { Range: `bytes=${start}-${end}` }, }); if (!response.ok && response.status !== 206) { throw new Error(`Failed to fetch range ${start}-${end}: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); return new Uint8Array(arrayBuffer); }; } if (fileSize === 0) { throw new Error('Unable to determine the file size.'); } console.log('Check Upload File size:', fileSize, MAX_CHUNK_SIZE_BYTES); if (fileSize <= MAX_CHUNK_SIZE_BYTES) { // Handle small files directly if (input.type === 'file') { const file = input.content as File; const transcription = await groq.audio.transcriptions.create({ file, model: params?.model || DEFAULT_WHISPER_MODEL, prompt: params?.prompt, response_format: params?.response_format || 'verbose_json', language: params?.language || 'en', temperature: params?.temperature, }); return normalizeTranscription(transcription); } else { const url = input.content as string; const transcription = await groq.audio.transcriptions.create({ url, model: params?.model || DEFAULT_WHISPER_MODEL, prompt: params?.prompt, response_format: params?.response_format || 'text', language: params?.language || 'en', temperature: params?.temperature, }); return normalizeTranscription(transcription); } } // Handle large files by splitting into equal-sized chunks const numChunks = Math.ceil(fileSize / MAX_CHUNK_SIZE_BYTES); const chunkSize = Math.ceil(fileSize / numChunks); const transcriptions: (string | Transcription)[] = []; console.log('Handle large files by splitting into equal-sized chunks', numChunks); for (let i = 0; i < numChunks; i++) { const start = i * chunkSize; const end = Math.min((i + 1) * chunkSize - 1, fileSize - 1); console.log(`Processing chunk ${i + 1}/${numChunks} (bytes ${start}-${end})`); const buffer = await getBuffer(start, end); try { const transcription = await transcribeChunk(buffer, { ...params, model: params.model || DEFAULT_WHISPER_MODEL, language: params.language || 'en' }); transcriptions.push(transcription); } catch (error: any) { console.error(`Error transcribing chunk ${i + 1}:`, error); throw new Error(`Transcription failed for chunk ${i + 1}: ${error.message}`); } } // Handle joining based on response format if (params.response_format === 'text' || !params.response_format) { const combinedText = transcriptions.map(t => { const normalized = normalizeTranscription(t); return normalized.text; }).join(' '); return { text: combinedText }; } else { // For JSON responses, combine segments and adjust timestamps const firstChunk = normalizeTranscription(transcriptions[0]); const combinedResponse: Transcription = { text: firstChunk.text, segments: firstChunk.segments || [], duration: firstChunk.duration || 0 }; let timeOffset = combinedResponse.duration || 0; for (let i = 1; i < transcriptions.length; i++) { const currentChunk = normalizeTranscription(transcriptions[i]); if (currentChunk.segments && combinedResponse.segments) { const adjustedSegments = currentChunk.segments.map(segment => ({ ...segment, start: segment.start + timeOffset, end: segment.end + timeOffset })); combinedResponse.segments.push(...adjustedSegments); } combinedResponse.duration = (combinedResponse.duration || 0) + (currentChunk.duration || 0); combinedResponse.text += ' ' + currentChunk.text; timeOffset += currentChunk.duration || 0; } return combinedResponse; } } async function getTranscript(data) { try { console.log("Starting transcription..."); let input; let fileToTranscribe; // Extract parameters from FormData const language = data.get('language') || 'en'; const model = data.get('model') || DEFAULT_WHISPER_MODEL; console.log("Transcript params:", { language, model }); // Check if formData is a JSON object with a URL if (data.url) { console.log("Processing URL input:", data.url); input = { type: 'url' as const, content: data.url }; } else { // Handle file upload const file = data.get('file'); if (!file) { throw new Error('No file found in form data'); } console.log("Input file:", { name: file.name, size: file.size, type: file.type }); fileToTranscribe = file instanceof File ? file : new File([file], 'audio.wav', { type: file.type || 'audio/wav' }); input = { type: 'file' as const, content: fileToTranscribe }; } const params: TranscriptionParams = { model, response_format: 'text', language, temperature: 0 }; console.log("Using transcription params:", params); const transcription = await transcribeAudio(input, params); const transcriptText = typeof transcription === 'string' ? transcription : transcription.text; console.log("Transcription completed:", transcriptText); return transcriptText.trim() || null; } catch (error) { console.error("Transcription failed:", error); return null; } } // Keep existing transcriptHandler for SSE responses during recording export const transcriptHandler = async (c) => { c.header('Content-Type', 'text/event-stream'); console.log('Starting transcript handler'); return new Response( new ReadableStream({ async start(controller) { const encoder = new TextEncoder(); try { const formData = await c.req.formData(); const transcript = await getTranscript(formData); if (!transcript) { throw new Error('Invalid audio or transcription failed'); } const messageWithTimestamp = { timestamp: new Date().toLocaleString(), content: transcript }; controller.enqueue(encoder.encode(`data: ${JSON.stringify({ type: 'transcript', message: messageWithTimestamp })}\n\n`)); } catch (error) { controller.enqueue(encoder.encode(`data: ${JSON.stringify({ type: 'error', content: error.message, timestamp: new Date().toLocaleString() })}\n\n`)); } controller.close(); } }) ); }; // Add new transcribeHandler for file/URL uploads export const uploadTranscribeHandler = async (c) => { try { let file: File | null = null; let url: string | null = null; let params: TranscriptionParams = { model: DEFAULT_WHISPER_MODEL, response_format: 'text', language: 'en' }; const contentType = c.req.header('content-type'); console.log('starting uploadTranscribeHandler', contentType); if (contentType?.includes('multipart/form-data')) { try { console.log('[multipart/form-data] Received multipart/form-data'); const formData = await c.req.formData(); url = formData.get('url') as string | null; console.log('url', url); file = formData.get('file') as File | null; console.log('file', file); console.log('[multipart/form-data] Received language:', formData.get('language')); params = { model: (formData.get('model') as WhisperModel) || DEFAULT_WHISPER_MODEL, response_format: (formData.get('response_format') as WhisperResponseFormat) || 'text', language: formData.get('language')?.toString() || 'en', prompt: formData.get('prompt')?.toString(), temperature: formData.get('temperature') ? parseFloat(formData.get('temperature').toString()) : undefined, }; } catch(error) { console.error('Failed to parse multipart/form-data', error); }; } console.log('Using transcription params:::::', params); const input: AudioInput = file ? { type: 'file', content: file } : { type: 'url', content: url! }; const transcription = await transcribeAudio(input, params); return c.json({ transcript: params.response_format === 'text' ? transcription.text : { text: transcription.text, ...(transcription.segments && { segments: transcription.segments }) } }); } catch (error: any) { console.error("[uploadTranscribeHandler] Transcription failed:", error); return c.json({ error: `Transcription failed: ${error.message}` }, 500); } }; export const classifySpeechHandler = async (c) => { try { let file; const contentType = c.req.header('content-type'); if (contentType?.includes('multipart/form-data')) { const formData = await c.req.formData(); file = formData.get('file'); } else if (contentType?.includes('application/json')) { const body = await c.req.json(); if (body.url) { const response = await fetch(body.url); const blob = await response.blob(); file = new File([blob], 'audio.wav', { type: response.headers.get('content-type') || 'audio/wav' }); } } if (!file) { return c.json({ error: 'No audio file or URL provided' }, 400); } try { const result = await hf.audioClassification({ model: 'MIT/ast-finetuned-audioset-10-10-0.4593', data: file, }); console.log('HF classification result:', result); const speechResult = result.find(pred => pred.label === 'Speech'); const speechScore = speechResult?.score || 0; const SPEECH_THRESHOLD = 0.45; const isHumanSpeech = speechScore > SPEECH_THRESHOLD; return c.json({ isHumanSpeech, confidence: speechScore, details: result, shouldProcess: isHumanSpeech }); } catch (classificationError) { // console.warn('Classification failed, defaulting to yes [rate limit]', classificationError); console.warn('Classification failed, defaulting to yes [rate limit]',); // Default to "yes" for any classification errors (including rate limits) return c.json({ isHumanSpeech: true, confidence: 1, details: [], shouldProcess: true }); } } catch (error) { console.error('Speech classification failed:', error); return c.json({ error: error.message }, 500); } }; // Add new handler export const validateTranscriptHandler = async (c) => { try { const formData = await c.req.formData(); const messages = JSON.parse(formData.get('messages')); const model = formData.get('model') || 'mixtral-8x7b-32768'; const temperature = parseFloat(formData.get('temperature')) || 0.1; const maxTokens = parseInt(formData.get('max_tokens')) || 256; const completion = await groq.chat.completions.create({ messages, model, temperature, max_tokens: maxTokens, response_format: { type: "json_object" } // Enable JSON mode }); return c.json(completion); } catch (error) { console.error('Transcript validation failed:', error); return c.json({ error: error.message }, 500); } }; // Add new chat completion handler export const chatCompletionHandler = async (c) => { try { const formData = await c.req.formData(); const messages = JSON.parse(formData.get('messages')); const transcriptContext = formData.get('transcriptContext')?.toString() || ''; const temperature = parseFloat(formData.get('temperature')?.toString() || '0.2'); const model = formData.get('model')?.toString() || 'llama-3.1-8b-instant'; const systemPrompt = formData.get('systemPrompt')?.toString(); const isAutoChat = formData.get('isAutoChat') === 'true'; // Add different system prompt for auto-chat const autoChatPrompt = `You are a helpful AI assistant analyzing real-time transcripts. Your role is to: 1. Monitor for technical terms or jargon 2. Provide very brief, clear explanations when needed 3. Stay quiet (respond with "...") if no explanation is needed 4. Keep responses under 2 sentences 5. Focus only on technical terminology, ignore conversation context Example: Transcript: "We need to implement a REST API with proper CRUD operations" Response: "REST API: Style of web API using HTTP methods. CRUD: Create, Read, Update, Delete - basic data operations." Transcript: "Let's grab coffee later" Response: "..."`; const finalSystemPrompt = isAutoChat ? autoChatPrompt : systemPrompt; // Trim transcript context to max length from the end const trimmedContext = transcriptContext.slice(-MAX_TRANSCRIPT_CONTEXT); // Add system message with transcript context and custom prompt const systemMessage = { role: "system", content: `${finalSystemPrompt || ''}\n\nTranscript context:\n\n${trimmedContext}` }; const completion = await groq.chat.completions.create({ messages: [systemMessage, ...messages], model, temperature, max_tokens: 1024, top_p: 1, stream: true }); // Set up SSE response c.header('Content-Type', 'text/event-stream'); return new Response( new ReadableStream({ async start(controller) { const encoder = new TextEncoder(); for await (const chunk of completion) { const content = chunk.choices[0]?.delta?.content || ''; if (content) { controller.enqueue(encoder.encode(`data: ${JSON.stringify({ type: 'chat', content })}\n\n`)); } } controller.close(); } }) ); } catch (error: any) { console.error("Chat completion failed:", error); return c.json({ error: `Chat failed: ${error.message}` }, 500); } }; // Inlined content from ./styles.js export const getTailwindConfig = () => ({ theme: { extend: { colors: { 'primary': '#004699', 'accent': '#FFFBEB', 'background': '#F5F4F4', 'page-bg': '#EBF4FF', 'app-bg': '#FFFFFF', 'app-bg-alt': '#FFFFFF', 'app-black': '#004699', 'app-accent': '#FFF2C2', 'secondary': '#D9D5D4', 'text': '#100C04', 'button': '#004699', 'button-hover': '#0063CC', 'button-secondary': '#D6EAFF', 'button-secondary-hover': '#99CAFF', 'button-secondary-accent': '#161413', 'grad-start': '#FFF2C2', 'grad-end': '#D1F0DE', } } } }) export const getStyles = () => ` @tailwind base; @tailwind components; @tailwind utilities; body { @apply bg-gradient-to-b from-grad-start to-grad-end bg-grad-end min-h-screen; } .AppBody { @apply text-text py-4 min-h-screen; } .AppContainer { @apply container mx-auto max-w-2xl; } .App { @apply mx-2 md:mx-auto p-4 bg-app-bg border border-app-accent rounded-md shadow-lg; } .AppLogo { @apply mb-2; } h1 { @apply text-3xl font-semibold mb-4 text-primary; } textarea { @apply w-full mb-4 py-2 px-4 bg-app-bg-alt border border-app-accent rounded-md focus:outline-none focus:ring-1 focus:ring-secondary; } button.cta { @apply bg-button text-white px-2 py-2 rounded-md hover:bg-button-hover transition duration-300; } button:disabled { @apply opacity-50 cursor-not-allowed; } #textOutput { @apply text-sm my-0 w-full bg-accent border-0 rounded-md whitespace-pre-wrap; } .AppFooter { @apply mt-4 text-center text-sm; } .AppFooter a { @apply text-primary hover:text-button-hover underline; } select { @apply w-full py-2 px-4 pr-8 bg-app-bg-alt border border-app-accent rounded-md appearance-none focus:outline-none focus:ring-1 focus:ring-secondary; } .form-checkbox { @apply rounded border-gray-300 text-button focus:ring-button; } details summary { @apply mb-2; } textarea { @apply block text-primary w-full mt-2 p-2 bg-app-bg-alt border border-app-accent rounded-md focus:outline-none focus:ring-1 focus:ring-secondary; } /* Add history styles */ .history-item { @apply p-2 bg-accent hover:bg-app-bg-alt border border-app-accent rounded-md cursor-pointer transition-colors; } .delete-button { @apply self-end bg-button-secondary text-white px-2 py-1 rounded text-sm hover:bg-button-secondary-hover transition-colors; } button.text-sm { @apply text-xs bg-button-secondary border border-button-secondary-accent text-button-secondary-accent px-2 py-1 rounded-md hover:bg-button-secondary-hover transition duration-300; } .TextOutput { @apply mt-4; } .TextOutput-content { @apply p-4 bg-app-bg-alt border border-primary rounded-md; } .TextOutput-header { @apply mb-2 text-xs text-gray-600 flex justify-between items-center; } #textOutput { @apply resize-none focus:outline-none focus:ring-0; } .monaco-editor { @apply w-full border border-app-accent rounded-md; } .monaco-editor .monaco-scrollable-element > .scrollbar > .slider { background: rgba(100, 100, 100, 0.4) !important; } `; // Inlined content from ./auth.ts const AUTH_SERVICE_URL = "https://auth.labspace.ai"; // Auth middleware export const checkAuth = async (c, next) => { const cookies = c.req.header('Cookie'); const sessionMatch = cookies?.match(/auth_session=([^;]+)/); const sessionToken = sessionMatch?.[1]; if (sessionToken) { try { const response = await fetch(`${AUTH_SERVICE_URL}/api/auth/user`, { headers: { 'Cookie': cookies, 'Accept': 'application/json' }, mode: 'cors', credentials: 'include' }); if (response.ok) { const result = await response.json(); result.sessionId = sessionToken; c.set('user', result); } } catch (error) { console.error('[checkAuth] error:', error); } } await next(); }; // OAuth callback handler export const handleOAuthCallback = async (c) => { const searchParams = new URL(c.req.url).searchParams; const sessionToken = searchParams.get("session_token"); const error = searchParams.get("error"); if (error) { return c.html(` `); } if (!sessionToken) { return c.html(` `); } c.header("Set-Cookie", `auth_session=${sessionToken}; Path=/; SameSite=Lax`); return c.html(` `); }; // Inlined content from ./plugins/useAuth.js // Based on threadg1rl's auth system function useAuth(Alpine) { Alpine.data('authApp', () => ({ user: {}, isLoggingIn: false, init() { console.log('[initAuth] init'); if (window.initialUserData) { console.log('[initAuth] user:', window.initialUserData); this.user = window.initialUserData; // Set the session ID if available if (this.user.sessionId) { Alpine.store('recorder').sessionId = this.user.sessionId; } } }, loginWithGithub() { console.log('Logging in with GitHub'); this.isLoggingIn = true; window.location.href = "https://auth.labspace.ai/auth/github?redirect_uri=" + encodeURIComponent(window.location.origin + "/callback"); }, loginWithGoogle() { console.log('Logging in with Google'); this.isLoggingIn = true; window.location.href = "https://auth.labspace.ai/auth/google?redirect_uri=" + encodeURIComponent(window.location.origin + "/callback"); }, async handleLogout(e) { e.preventDefault(); try { await fetch("https://auth.labspace.ai/logout", { method: 'POST', credentials: 'include', mode: 'cors' }); // Clear cookie document.cookie = 'auth_session=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/'; // Reset user state and session ID this.user = {}; Alpine.store('recorder').sessionId = ''; // Reload page window.location.reload(); } catch (error) { console.error('[handleLogout] error:', error); } } })); } // Export the function properly export const useAuthExport = ` const useAuth = ${useAuth.toString()}; document.addEventListener('alpine:init', () => { useAuth(Alpine); }); `; // SVG icons for auth buttons const githubSvg = ``.replace(/"/g, '"'); const googleSvg = ``.replace(/"/g, '"'); // Auth UI template export const authTemplate = `

`; import { bodyLimit } from 'npm:hono/body-limit' const app = new Hono(); app.post("/transcript", transcriptHandler); app.post("/transcribe", bodyLimit({ maxSize: 100000 * 1024, // 100mb onError: (c) => { return c.text('too large :(', 413) }, }), uploadTranscribeHandler); app.post("/classify-speech", classifySpeechHandler); // app.post("/validate-transcript", validateTranscriptHandler); app.post("/chat", chatCompletionHandler); app.use('/*', checkAuth); app.get("/callback", handleOAuthCallback); const getHtml = () => ` Scribe • Audio Recorder

${authTemplate}

Upload

Auto-chat with transcripts

Temperature:

Model

Selected model affects response quality and speed

System Instructions

Sessions

A labspace.ai experiment

`; app.get("/", async (c) => { let html = getHtml(); const userData = c.get('user'); if (userData) { const initScript = ` `; return c.html(html.replace('', initScript + '')); } return c.html(html); }); // Export app.fetch for Val Town, otherwise export app export default (typeof Deno !== "undefined" && Deno.env.get("valtown")) ? app.fetch : app; // export default app;