Back
Version 28
5/28/2024
import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";
import { db } from "https://esm.town/v/sqlite/db?v=9";
import { sqlite } from "https://esm.town/v/std/sqlite?v=6";
import OpenAI from "npm:openai";
import { truncateMessage } from "npm:openai-tokens";
// sqlite.execute("CREATE TABLE vals_embeddings (id TEXT NOT NULL, embedding BLOB NOT NULL)");
const allVals = await sqlToJSON(await db.execute("SELECT author_username, name, version FROM vals")) as any;
// console.log(allVals);
const existingEmbeddingsIds = new Set(
(await sqlite.execute("SELECT id FROM vals_embeddings")).rows.map((row) => row[0]),
);
// console.log(existingEmbeddingsIds);
const newVals = [];
for (const val of allVals) {
const id = `${val.author_username}!!${val.name}!!${val.version}`;
if (!existingEmbeddingsIds.has(id)) {
newVals.push(val);
}
if (newVals.length >= 10) {
break;
}
}
const openai = new OpenAI();
for (const val of newVals) {
const code = (await db.execute({
sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",
args: val,
})).rows[0][0];
const embedding = await openai.embeddings.create({
model: "text-embedding-3-small",
Updated: May 29, 2024