/** # Email AI Assistant Chat with your favorite AI via email (with enhanced attachment and content support) ## What It Does This advanced email AI assistant allows you to: - Send emails to an AI for comprehensive analysis and responses - Automatically transform your queries into structured research objectives - Parse and analyze various types of content: - PDF attachments - Image attachments (using GPT-4 Vision) - Website content from links in your email - Get detailed, context-aware responses directly to your inbox ## Setup Guide 1. Copy this Val and save it as an Email Val (choose Val type in the top-right corner of the editor) 2. Set up the required environment variables: - OPENAI_API_KEY: Your OpenAI API key - MD_API_KEY: Your API key for the markdown extraction service (optional) You can set these using Val Town's environment variables: https://docs.val.town/reference/environment-variables/ 3. Copy the email address of the Val (click 3 dots in top-right > Copy > Copy email address) 4. Compose your email: - Write your query or request in the email body - Attach any relevant PDFs or images - Include links to websites you want analyzed - Send it to the Val email address 5. Wait for the AI's response, which will arrive in your inbox shortly ## How to Use Effectively - Be clear and specific in your queries - Provide context when necessary - Utilize attachments and links to give the AI more information to work with - The AI will transform your query into a structured research objective, so even simple questions may yield comprehensive answers ## Supported File Types and Limitations - PDFs: Text content will be extracted and analyzed - Images: Will be analyzed using GPT-4 Vision API - Websites: Content will be extracted and converted to markdown for analysis - Other file types are not currently supported and will be ignored Note: There may be size limitations for attachments and processing times may vary based on the complexity of the content. The AI uses advanced prompt transformation to enhance your queries, providing more detailed and structured responses. This process helps in generating comprehensive and relevant answers to your questions. */ import { encode } from "https://deno.land/std@0.203.0/encoding/base64.ts"; import { email } from "https://esm.town/v/std/email"; import { pdfText } from "jsr:@pdf/pdftext"; // Main controller function export default async function emailValHandler(receivedEmail) { const openaiUrl = "https://api.openai.com/v1/chat/completions"; const openaiKey = Deno.env.get("OPENAI_API_KEY"); const mdApiKey = Deno.env.get("MD_API_KEY"); const model = "o1-2024-12-17"; if (!openaiKey) { throw new Error("OPENAI_KEY environment variable is not set."); } console.log("Email received:", receivedEmail.from, receivedEmail.subject); // Step 1: Extract and process all attachments and links const attachments = await extractAttachments(receivedEmail); const links = extractLinks(receivedEmail.text); // Step 2: Transform the email prompt using the prompt transformer const transformedPrompt = await transformPrompt(receivedEmail.text, openaiUrl, openaiKey, model); // Step 3: Process different types of content const { pdfTexts, imageAnalysis } = await processAttachments(attachments, openaiKey, transformedPrompt); const websiteMarkdown = await extractWebsiteMarkdown(links, mdApiKey); // Step 4: Create final prompt with all context const finalPrompt = generateFinalPrompt( transformedPrompt, pdfTexts, imageAnalysis, websiteMarkdown, receivedEmail, ); // Step 5: Send to OpenAI and get response const openaiResponse = await sendRequestToOpenAI(finalPrompt, transformedPrompt, openaiUrl, openaiKey, model); // Step 6: Send response email await sendResponseByEmail(receivedEmail.from, openaiResponse); console.log("Response email sent."); } // Extract attachments (if any) async function extractAttachments(email) { if (!email.attachments || email.attachments.length === 0) { return []; } const attachments = []; for (const file of email.attachments) { attachments.push({ filename: file.name, content: encode(await file.arrayBuffer()), type: file.type, }); } return attachments; } // Extract links from email text function extractLinks(text) { const urlRegex = /(https?:\/\/[^\s]+)/g; return text.match(urlRegex) || []; } // Extract website markdown content async function extractWebsiteMarkdown(links, apiKey) { const markdownResults = []; const requestsPerMinute = 5; const requestInterval = 60000 / requestsPerMinute; // milliseconds between requests let lastRequestTime = 0; for (const url of links) { try { // Implement rate limiting const now = Date.now(); if (now - lastRequestTime < requestInterval) { await new Promise(resolve => setTimeout(resolve, requestInterval - (now - lastRequestTime))); } lastRequestTime = Date.now(); console.log(`Fetching markdown for ${url}`); console.log(`API Key present: ${!!apiKey}`); const fetchUrl = `https://md.dhr.wtf/?url=${encodeURIComponent(url)}`; console.log(`Fetch URL: ${fetchUrl}`); const fetchOptions: RequestInit = { method: "GET", }; // Only add headers if API key is available if (apiKey) { fetchOptions.headers = { "Authorization": `Bearer ${apiKey}`, }; } const response = await fetch(fetchUrl, fetchOptions); console.log(`Response status: ${response.status}`); if (!response.ok) { const responseText = await response.text(); console.error(`Error response body: ${responseText}`); throw new Error(`HTTP error! status: ${response.status}, body: ${responseText}`); } const markdown = await response.text(); console.log(`Successfully fetched markdown for ${url}`); console.log(`First 100 characters of markdown: ${markdown.substring(0, 100)}`); markdownResults.push({ url, markdown }); } catch (error) { console.error(`Error fetching markdown for ${url}:`, error); markdownResults.push({ url, markdown: `Error fetching content: ${error.message}` }); } } return markdownResults; } // Process image attachments with GPT-4V async function analyzeImage(imageAttachment, apiKey, transformedPrompt) { try { const response = await fetch("https://api.openai.com/v1/chat/completions", { method: "POST", headers: { "Content-Type": "application/json", "Authorization": `Bearer ${apiKey}`, }, body: JSON.stringify({ model: "gpt-4o", messages: [ { role: "system", content: `You are an AI assistant tasked with analyzing images in the context of a specific query. Use the following transformed prompt to guide your analysis and provide relevant context:\n\n${transformedPrompt}\n\nFocus your analysis on aspects that are most relevant to this prompt.`, }, { role: "user", content: [ { type: "text", text: "Analyze this image and provide relevant context based on the given prompt:${transformedPrompt}", }, { type: "image_url", image_url: { url: `data:${imageAttachment.type};base64,${imageAttachment.content}`, }, }, ], }, ], max_tokens: 300, }), }); const data = await response.json(); return data.choices[0]?.message?.content || "No analysis available"; } catch (error) { console.error("Error analyzing image:", error); return "Error analyzing image"; } } // Transform the original prompt using the prompt transformer async function transformPrompt(emailText, openaiUrl, apiKey, model) { const promptTransformerText = `You are an AI assistant tasked with transforming user queries into structured research or information requests. Your goal is to take a simple query and expand it into a comprehensive research objective with specific formatting requirements. Here's the user's query: ${emailText} Follow these steps to transform the query: 1. Analyze the user's query and think about how to expand it into a more comprehensive research objective. 2. Create a structured research request using the following format: Write the main objective or request, stated clearly and concisely in 1-2 sentences. Transform the user's basic query into a comprehensive research objective. Specify exactly how the information should be organized and presented using the following tags: List time-based elements that should be tracked or recorded List core information fields that must be included for each item/entry List quantifiable or measurable aspects to be included List deeper insights, impacts, or relationships to be explored Specify how many items/examples should be returned and any category organization needed. List critical considerations for accuracy, verification requirements, and potential limitations of the data or analysis. 3. Ensure that your transformed request is more detailed and comprehensive than the original query, providing a clear structure for research and analysis. 4. If the query is too vague or broad, make reasonable assumptions to narrow the focus and provide a more specific research direction. 5. Include all relevant sections (goal, return_format with timeline, key_details, metrics, and analysis, and warnings) in your response, even if some sections might have limited content based on the nature of the query. 6. Do not include example sections in your output. The examples provided in the original task description are for your reference only. Remember to think critically about the query and how to best structure the research request to provide the most valuable and comprehensive information.`; try { const messages = [ { role: "system", content: promptTransformerText, }, { role: "user", content: emailText, }, ]; const body = { model, messages, response_format: { type: "text" }, reasoning_effort: "medium", }; const response = await fetch(openaiUrl, { method: "POST", body: JSON.stringify(body), headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, }); const data = await response.json(); if (data.error) { throw new Error(`OpenAI API Error: ${data.error.message}`); } return data.choices[0]?.message?.content || emailText; } catch (err) { console.error("Error in transformPrompt:", err); return emailText; } } // Process all attachments (PDFs and Images) async function processAttachments(attachments, apiKey, transformedPrompt) { const pdfTexts = []; const imageAnalysis = []; for (const attachment of attachments) { if (attachment.type === "application/pdf") { const pdfText = await extractPdfText([attachment]); pdfTexts.push(...pdfText); } else if (attachment.type.startsWith("image/")) { const analysis = await analyzeImage(attachment, apiKey, transformedPrompt); imageAnalysis.push({ filename: attachment.filename, analysis, }); } } return { pdfTexts, imageAnalysis }; } async function extractPdfText(attachments) { const pdfTexts = []; for (const attachment of attachments) { if (attachment.type === "application/pdf") { console.log(`parsing PDF: ${attachment.filename}`); try { // decode Base64 content back to a buffer const pdfBuffer = Uint8Array.from(atob(attachment.content), (c) => c.charCodeAt(0)).buffer; // extract text from the PDF const pages = await pdfText(pdfBuffer); // combine all pages into a single string const fullText = Object.values(pages).join("\n"); pdfTexts.push({ filename: attachment.filename, text: fullText, }); } catch (err) { console.error(`Error parsing PDF (${attachment.filename}):`, err); pdfTexts.push({ filename: attachment.filename, text: "Error parsing PDF", }); } } } return pdfTexts; } // Generate the final prompt with all context function generateFinalPrompt(transformedPrompt, pdfTexts, imageAnalysis, websiteMarkdown, email) { let contextDump = []; // Add PDF content if (pdfTexts.length > 0) { contextDump.push("PDF Content:", ...pdfTexts.map(pdf => `${pdf.filename}: ${pdf.text}`)); } // Add image analysis if (imageAnalysis.length > 0) { contextDump.push("Image Analysis:", ...imageAnalysis.map(img => `${img.filename}: ${img.analysis}`)); } // Add website markdown if (websiteMarkdown.length > 0) { contextDump.push("Website Content:", ...websiteMarkdown.map(site => `${site.url}: ${site.markdown}`)); } // Combine transformed prompt with context dump return `${transformedPrompt}\n\n\n${contextDump.join("\n")}\n\n`; } // Helper function to send a request to OpenAI async function sendRequestToOpenAI(prompt, transformedPrompt, openaiUrl, apiKey, model) { try { // Debug logging for the prompt and transformed prompt console.log("Debug - Transformed Prompt:"); console.log(transformedPrompt); console.log("\nDebug - Final Prompt:"); console.log(prompt); // Prepare the OpenAI messages payload const messages = [ { role: "system", content: `You are a helpful AI that replies to emails. Address the sender by their first name if provided, and sign off as 'AI Assistant' in a friendly and professional tone. Be concise and thorough in your analysis. As you're replying via email, use email syntax and not markdown. Analyze all provided content, including PDFs, images, and website content, in the context of the email request. Follow the structured format provided in the transformed prompt to organize your response. Here's the transformed prompt to guide your response:\n\n${transformedPrompt}`, }, { role: "user", content: prompt, }, ]; // Setup request body const body = { model, messages, response_format: { type: "text" }, reasoning_effort: "medium", }; // Send the request to OpenAI const response = await fetch(openaiUrl, { method: "POST", body: JSON.stringify(body), headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, }); // Parse the response const data = await response.json(); if (data.error) { throw new Error(`OpenAI API Error: ${data.error.message}`); } return data.choices[0]?.message?.content || "No response from OpenAI."; } catch (err) { console.error("Error in sendRequestToOpenAI:", err); return "Error processing your request. Please try again later."; } } // Helper function to send a response back via email async function sendResponseByEmail(toEmail, responseContent) { const subject = "AI Response to Your Email"; const text = `${responseContent}`; await email({ subject, text, to: toEmail, }); }

emailValHandler

Version 0