Public
Versions
- Open: VersionChanges from v36 to v37+2-16import { decode as base64Decode, encode as base64Encode } from "https://deno.land/std@0.166.0/encoding/base64.ts";
import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";import { blob } from "https://esm.town/v/std/blob";⦚ 39 unchanged lines ⦚await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];let code = undefined;for (let attempt = 1; true; attempt++) {try {code = (await allValsDb.execute({sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",args: val,})).rows[0][0];break;} catch (error) {if (attempt >= 10) {console.log(`FATAL after SQL failed with error ${error}`);throw error;}// console.log(`Retrying after SQL failed with error ${error}`);}}const embedding = await openai.embeddings.create({⦚ 28 unchanged lines ⦚import { decode as base64Decode, encode as base64Encode } from "https://deno.land/std@0.166.0/encoding/base64.ts";import getValCode from "https://esm.town/v/janpaul123/getValCode";import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";import { blob } from "https://esm.town/v/std/blob";⦚ 39 unchanged lines ⦚await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];const code = getValCode(val);const embedding = await openai.embeddings.create({⦚ 28 unchanged lines ⦚ - Open: VersionChanges from v35 to v36+16-4⦚ 43 unchanged lines ⦚await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];
const code = (await allValsDb.execute({sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",args: val,})).rows[0][0];const embedding = await openai.embeddings.create({⦚ 28 unchanged lines ⦚⦚ 43 unchanged lines ⦚await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];let code = undefined;for (let attempt = 1; true; attempt++) {try {code = (await allValsDb.execute({sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",args: val,})).rows[0][0];break;} catch (error) {if (attempt >= 10) {console.log(`FATAL after SQL failed with error ${error}`);throw error;}// console.log(`Retrying after SQL failed with error ${error}`);}}const embedding = await openai.embeddings.create({⦚ 28 unchanged lines ⦚ - Open: VersionChanges from v34 to v35+4-16⦚ 43 unchanged lines ⦚await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];
let code = undefined;for (let attempt = 1; true; attempt++) {try {code = (await allValsDb.execute({sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",args: val,})).rows[0][0];break;} catch (error) {if (attempt >= 10) {console.log(`FATAL after SQL failed with error ${error}`);throw error;}// console.log(`Retrying after SQL failed with error ${error}`);}}const embedding = await openai.embeddings.create({⦚ 28 unchanged lines ⦚⦚ 43 unchanged lines ⦚await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];const code = (await allValsDb.execute({sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",args: val,})).rows[0][0];const embedding = await openai.embeddings.create({⦚ 28 unchanged lines ⦚ - Open: VersionChanges from v33 to v34+1-1⦚ 40 unchanged lines ⦚for (const newValsBatch of newValsBatches) {const batchDataIndex = nextDataIndex;
const embeddingsBatch = new Float32Array(256 * newValsBatch.length);await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];⦚ 46 unchanged lines ⦚⦚ 40 unchanged lines ⦚for (const newValsBatch of newValsBatches) {const batchDataIndex = nextDataIndex;const embeddingsBatch = new Float32Array(dimensions * newValsBatch.length);await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {const val = newValsBatch[valIndex];⦚ 46 unchanged lines ⦚ - Open: VersionChanges from v32 to v33+1-1⦚ 26 unchanged lines ⦚currentBatch.push(val);}
if (currentBatch.length >= 1000) {currentBatch = [];newValsBatches.push(currentBatch);⦚ 60 unchanged lines ⦚⦚ 26 unchanged lines ⦚currentBatch.push(val);}if (currentBatch.length >= 500) {currentBatch = [];newValsBatches.push(currentBatch);⦚ 60 unchanged lines ⦚ - Open: VersionChanges from v31 to v32+1-1⦚ 64 unchanged lines ⦚input: truncateMessage(code, "text-embedding-3-small"),encoding_format: "base64",
dimensions: dimensions,});const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);⦚ 22 unchanged lines ⦚⦚ 64 unchanged lines ⦚input: truncateMessage(code, "text-embedding-3-small"),encoding_format: "base64",dimensions,});const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);⦚ 22 unchanged lines ⦚ - Open: VersionChanges from v30 to v31+8-6⦚ 5 unchanged lines ⦚import { truncateMessage } from "npm:openai-tokens";
const allVals = await sqlToJSON(await allValsDb.execute("SELECT author_username, name, version FROM vals WHERE LENGTH(code) > 10 ORDER BY name"),) as any;// const allValsBlobEmbeddingsMeta = (await blob.getJSON("allValsBlobEmbeddingsMeta")) ?? {};const allValsBlobEmbeddingsMeta = {};const existingEmbeddingsIds = new Set(Object.keys(allValsBlobEmbeddingsMeta));⦚ 48 unchanged lines ⦚input: truncateMessage(code, "text-embedding-3-small"),encoding_format: "base64",dimensions: 256,});const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);if (embeddingBinary.length != 256) {throw new Error(`Invalid embeddingBinary.length: ${embeddingBinary.length}`);}const id = idForVal(val);embeddingsBatch.set(embeddingBinary, 256 * valIndex);allValsBlobEmbeddingsMeta[id] = { batchDataIndex, valIndex };}));const embeddingsBatchBlobName = `allValsBlobEmbeddingsData_${batchDataIndex}`;await blob.set(embeddingsBatchBlobName, embeddingsBatch.buffer);await blob.setJSON("allValsBlobEmbeddingsMeta", allValsBlobEmbeddingsMeta);console.log(⦚ 8 unchanged lines ⦚⦚ 5 unchanged lines ⦚import { truncateMessage } from "npm:openai-tokens";const dimensions = 1536;const allVals = await sqlToJSON(await allValsDb.execute("SELECT author_username, name, version FROM vals WHERE LENGTH(code) > 10 ORDER BY name"),) as any;// const allValsBlobEmbeddingsMeta = (await blob.getJSON(`allValsBlob${dimensions}EmbeddingsMeta`)) ?? {};const allValsBlobEmbeddingsMeta = {};const existingEmbeddingsIds = new Set(Object.keys(allValsBlobEmbeddingsMeta));⦚ 48 unchanged lines ⦚input: truncateMessage(code, "text-embedding-3-small"),encoding_format: "base64",dimensions: dimensions,});const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);if (embeddingBinary.length != dimensions) {throw new Error(`Invalid embeddingBinary.length: ${embeddingBinary.length}`);}const id = idForVal(val);embeddingsBatch.set(embeddingBinary, dimensions * valIndex);allValsBlobEmbeddingsMeta[id] = { batchDataIndex, valIndex };}));const embeddingsBatchBlobName = `allValsBlob${dimensions}EmbeddingsData_${batchDataIndex}`;await blob.set(embeddingsBatchBlobName, embeddingsBatch.buffer);await blob.setJSON(`allValsBlob${dimensions}EmbeddingsMeta`, allValsBlobEmbeddingsMeta);console.log(⦚ 8 unchanged lines ⦚ - Open: VersionChanges from v29 to v30+1-1⦚ 50 unchanged lines ⦚break;} catch (error) {
if (attempt >= 5) {console.log(`FATAL after SQL failed with error ${error}`);throw error;⦚ 34 unchanged lines ⦚⦚ 50 unchanged lines ⦚break;} catch (error) {if (attempt >= 10) {console.log(`FATAL after SQL failed with error ${error}`);throw error;⦚ 34 unchanged lines ⦚ - Open: Version+89-0import { decode as base64Decode, encode as base64Encode } from "https://deno.land/std@0.166.0/encoding/base64.ts";import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";import { blob } from "https://esm.town/v/std/blob";import OpenAI from "npm:openai";import { truncateMessage } from "npm:openai-tokens";const allVals = await sqlToJSON(await allValsDb.execute("SELECT author_username, name, version FROM vals WHERE LENGTH(code) > 10 ORDER BY name"),) as any;// const allValsBlobEmbeddingsMeta = (await blob.getJSON("allValsBlobEmbeddingsMeta")) ?? {};const allValsBlobEmbeddingsMeta = {};const existingEmbeddingsIds = new Set(Object.keys(allValsBlobEmbeddingsMeta));function idForVal(val: any): string {return `${val.author_username}!!${val.name}!!${val.version}`;}const newValsBatches = [[]];let currentBatch = newValsBatches[0];for (const val of allVals) {const id = idForVal(val);if (!existingEmbeddingsIds.has(id)) {currentBatch.push(val);}if (currentBatch.length >= 1000) {currentBatch = [];newValsBatches.push(currentBatch);}}let nextDataIndex = Math.max(0,...Object.values(allValsBlobEmbeddingsMeta).map((item: any) => item.batchDataIndex + 1),);
Updated: June 17, 2024