GIF Story | Substrate

import { extractValInfo } from "https://esm.town/v/pomdtr/extractValInfo?v=29"; import { ComputeJSON, ComputeText, GenerateImage, sb, StableVideoDiffusion, Substrate } from "npm:substrate"; import { z } from "npm:zod"; import { zodToJsonSchema } from "npm:zod-to-json-schema"; const substrate = new Substrate({ apiKey: Deno.env.get("SUBSTRATE_API_KEY") }); const topic = "a traveler lost in a rainforest"; const story = new ComputeText({ prompt: `tell me a story about ${topic}`, }); story.cache_age = 60 * 60 * 24 * 7; const sentencesSchema = z.object({ sentences: z.array(z.string()).length(5).describe("5 short sentences condensing the story"), imageDescriptions: z .array(z.string()) .length(5) .describe( "5 descriptions of images to accompany each sentence. Describe simple images that depict the setting of each scene. Do not include characters, just the background. Return just the description, no prefix or explanation.", ), }); const sentences = new ComputeJSON({ prompt: sb .interpolate`We're making a storyboard. Generate 5 short sentences condensing the story about ${topic}, along with descriptions of images to accompany each sentence.`, json_schema: zodToJsonSchema(sentencesSchema), }); sentences.cache_age = 60 * 60 * 24 * 7; let videoNodes = []; for (let i = 0; i < 5; i++) { const image = new GenerateImage({ prompt: sentences.future.json_object.get("imageDescriptions").at(i), store: "hosted", }); image.cache_age = 60 * 60 * 24 * 7; const video = new StableVideoDiffusion({ image_uri: image.future.image_uri, store: "hosted", motion_bucket_id: 20, fps: 10, }); video.cache_age = 60 * 60 * 24 * 7; videoNodes.push(video); } export default async function(req: Request): Promise { let { readable, writable } = new TransformStream(); let writer = writable.getWriter(); const textEncoder = new TextEncoder(); new Promise(async () => { writer.write(textEncoder.encode(`

Takes about 1 minute

Generating... (0.0s)

`)); // Don't run the graph, we'll display the cached result // const res = await substrate.run(...videoNodes); // const texts = res.get(sentences).json_object.sentences; // let gifUrls = []; // for (let i = 0; i < 5; i++) { // gifUrls.push(res.get(videoNodes[i]).video_uri); // } // console.log(JSON.stringify(res.get(sentences).json_object, null, 2)); // console.log(JSON.stringify(gifUrls, null, 2)); // Display cached result const sentencesResult = { "sentences": [ "A traveler sets out on a journey through the dense rainforest.", "The air is thick with humidity and the canopy overhead blocks out most of the sunlight.", "The sound of rushing water grows louder as the traveler follows the winding path.", "A massive waterfall comes into view, cascading down a rocky slope.", "The roar of the falls subsides as the traveler rounds a bend in the path.", ], "imageDescriptions": [ "A dense jungle with towering trees and vines", "A misty forest with ferns and moss-covered tree trunks", "A narrow path leading to a fast-moving river", "A massive waterfall crashing over a rocky cliff", "A serene river flowing through a tranquil valley", ], }; const texts = sentencesResult.sentences; let gifUrls = [ "https://cdn.substrate.run/u/TdI00FeAB8nzr0Su/d73078011783f5ac37980e47df9d0ee7985fd2d476eed501_1722427247.gif", "https://cdn.substrate.run/u/TdI00FeAB8nzr0Su/69c79387d16391d3f8c246ccbd694bed55f39d3a60a66fde_1722427247.gif", "https://cdn.substrate.run/u/TdI00FeAB8nzr0Su/75db479fdc66d185edbf24f7b037257adb98ec7bfe7b10db_1722427248.gif", "https://cdn.substrate.run/u/TdI00FeAB8nzr0Su/bd26d54a083b540210ac23bc9fd296fc13db9a9edff6718b_1722427271.gif", "https://cdn.substrate.run/u/TdI00FeAB8nzr0Su/b14849ad8bb9af9c60567d85d37de79c1d634748ed6ad9d4_1722427265.gif", ]; // Render page writer.write(textEncoder.encode("")); const { htmlUrl } = extractValInfo(import.meta.url); let html = ` GIF Story | Substrate

made with Substrate view source on val.town

`; writer.write(textEncoder.encode(html)); writer.close(); }); return new Response(readable, { headers: { "Content-Type": "text/html", }, }); }