Versions

  • v37

    6/17/2024
    Open: Version
    Changes from v36 to v37
    +2
    -16
    import { decode as base64Decode, encode as base64Encode } from "https://deno.land/std@0.166.0/encoding/base64.ts";
    import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";
    import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";
    import { blob } from "https://esm.town/v/std/blob";
    ⦚ 39 unchanged lines ⦚
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    let code = undefined;
    for (let attempt = 1; true; attempt++) {
    try {
    code = (await allValsDb.execute({
    sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",
    args: val,
    })).rows[0][0];
    break;
    } catch (error) {
    if (attempt >= 10) {
    console.log(`FATAL after SQL failed with error ${error}`);
    throw error;
    }
    // console.log(`Retrying after SQL failed with error ${error}`);
    }
    }

    const embedding = await openai.embeddings.create({
    ⦚ 28 unchanged lines ⦚
    import { decode as base64Decode, encode as base64Encode } from "https://deno.land/std@0.166.0/encoding/base64.ts";
    import getValCode from "https://esm.town/v/janpaul123/getValCode";
    import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";
    import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";
    import { blob } from "https://esm.town/v/std/blob";
    ⦚ 39 unchanged lines ⦚
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    const code = getValCode(val);

    const embedding = await openai.embeddings.create({
    ⦚ 28 unchanged lines ⦚
  • v36

    6/17/2024
    Open: Version
    Changes from v35 to v36
    +16
    -4
    ⦚ 43 unchanged lines ⦚
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    const code = (await allValsDb.execute({
    sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",
    args: val,
    })).rows[0][0];

    const embedding = await openai.embeddings.create({
    ⦚ 28 unchanged lines ⦚
    ⦚ 43 unchanged lines ⦚
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    let code = undefined;
    for (let attempt = 1; true; attempt++) {
    try {
    code = (await allValsDb.execute({
    sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",
    args: val,
    })).rows[0][0];
    break;
    } catch (error) {
    if (attempt >= 10) {
    console.log(`FATAL after SQL failed with error ${error}`);
    throw error;
    }
    // console.log(`Retrying after SQL failed with error ${error}`);
    }
    }

    const embedding = await openai.embeddings.create({
    ⦚ 28 unchanged lines ⦚
  • v35

    6/17/2024
    Open: Version
    Changes from v34 to v35
    +4
    -16
    ⦚ 43 unchanged lines ⦚
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    let code = undefined;
    for (let attempt = 1; true; attempt++) {
    try {
    code = (await allValsDb.execute({
    sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",
    args: val,
    })).rows[0][0];
    break;
    } catch (error) {
    if (attempt >= 10) {
    console.log(`FATAL after SQL failed with error ${error}`);
    throw error;
    }
    // console.log(`Retrying after SQL failed with error ${error}`);
    }
    }

    const embedding = await openai.embeddings.create({
    ⦚ 28 unchanged lines ⦚
    ⦚ 43 unchanged lines ⦚
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    const code = (await allValsDb.execute({
    sql: "SELECT code FROM vals WHERE author_username = :author_username AND name = :name AND version = :version",
    args: val,
    })).rows[0][0];

    const embedding = await openai.embeddings.create({
    ⦚ 28 unchanged lines ⦚
  • v34

    5/29/2024
    Open: Version
    Changes from v33 to v34
    +1
    -1
    ⦚ 40 unchanged lines ⦚
    for (const newValsBatch of newValsBatches) {
    const batchDataIndex = nextDataIndex;
    const embeddingsBatch = new Float32Array(256 * newValsBatch.length);
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    ⦚ 46 unchanged lines ⦚
    ⦚ 40 unchanged lines ⦚
    for (const newValsBatch of newValsBatches) {
    const batchDataIndex = nextDataIndex;
    const embeddingsBatch = new Float32Array(dimensions * newValsBatch.length);
    await Promise.all([...Array(newValsBatch.length).keys()].map(async (valIndex) => {
    const val = newValsBatch[valIndex];
    ⦚ 46 unchanged lines ⦚
  • v33

    5/29/2024
    Open: Version
    Changes from v32 to v33
    +1
    -1
    ⦚ 26 unchanged lines ⦚
    currentBatch.push(val);
    }
    if (currentBatch.length >= 1000) {
    currentBatch = [];
    newValsBatches.push(currentBatch);
    ⦚ 60 unchanged lines ⦚
    ⦚ 26 unchanged lines ⦚
    currentBatch.push(val);
    }
    if (currentBatch.length >= 500) {
    currentBatch = [];
    newValsBatches.push(currentBatch);
    ⦚ 60 unchanged lines ⦚
  • v32

    5/29/2024
    Open: Version
    Changes from v31 to v32
    +1
    -1
    ⦚ 64 unchanged lines ⦚
    input: truncateMessage(code, "text-embedding-3-small"),
    encoding_format: "base64",
    dimensions: dimensions,
    });
    const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);
    ⦚ 22 unchanged lines ⦚
    ⦚ 64 unchanged lines ⦚
    input: truncateMessage(code, "text-embedding-3-small"),
    encoding_format: "base64",
    dimensions,
    });
    const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);
    ⦚ 22 unchanged lines ⦚
  • v31

    5/29/2024
    Open: Version
    Changes from v30 to v31
    +8
    -6
    ⦚ 5 unchanged lines ⦚
    import { truncateMessage } from "npm:openai-tokens";

    const allVals = await sqlToJSON(
    await allValsDb.execute("SELECT author_username, name, version FROM vals WHERE LENGTH(code) > 10 ORDER BY name"),
    ) as any;

    // const allValsBlobEmbeddingsMeta = (await blob.getJSON("allValsBlobEmbeddingsMeta")) ?? {};
    const allValsBlobEmbeddingsMeta = {};
    const existingEmbeddingsIds = new Set(Object.keys(allValsBlobEmbeddingsMeta));
    ⦚ 48 unchanged lines ⦚
    input: truncateMessage(code, "text-embedding-3-small"),
    encoding_format: "base64",
    dimensions: 256,
    });
    const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);
    if (embeddingBinary.length != 256) {
    throw new Error(`Invalid embeddingBinary.length: ${embeddingBinary.length}`);
    }
    const id = idForVal(val);

    embeddingsBatch.set(embeddingBinary, 256 * valIndex);
    allValsBlobEmbeddingsMeta[id] = { batchDataIndex, valIndex };
    }));

    const embeddingsBatchBlobName = `allValsBlobEmbeddingsData_${batchDataIndex}`;
    await blob.set(embeddingsBatchBlobName, embeddingsBatch.buffer);
    await blob.setJSON("allValsBlobEmbeddingsMeta", allValsBlobEmbeddingsMeta);

    console.log(
    ⦚ 8 unchanged lines ⦚
    ⦚ 5 unchanged lines ⦚
    import { truncateMessage } from "npm:openai-tokens";

    const dimensions = 1536;

    const allVals = await sqlToJSON(
    await allValsDb.execute("SELECT author_username, name, version FROM vals WHERE LENGTH(code) > 10 ORDER BY name"),
    ) as any;

    // const allValsBlobEmbeddingsMeta = (await blob.getJSON(`allValsBlob${dimensions}EmbeddingsMeta`)) ?? {};
    const allValsBlobEmbeddingsMeta = {};
    const existingEmbeddingsIds = new Set(Object.keys(allValsBlobEmbeddingsMeta));
    ⦚ 48 unchanged lines ⦚
    input: truncateMessage(code, "text-embedding-3-small"),
    encoding_format: "base64",
    dimensions: dimensions,
    });
    const embeddingBinary = new Float32Array(base64Decode(embedding.data[0].embedding as any).buffer);
    if (embeddingBinary.length != dimensions) {
    throw new Error(`Invalid embeddingBinary.length: ${embeddingBinary.length}`);
    }
    const id = idForVal(val);

    embeddingsBatch.set(embeddingBinary, dimensions * valIndex);
    allValsBlobEmbeddingsMeta[id] = { batchDataIndex, valIndex };
    }));

    const embeddingsBatchBlobName = `allValsBlob${dimensions}EmbeddingsData_${batchDataIndex}`;
    await blob.set(embeddingsBatchBlobName, embeddingsBatch.buffer);
    await blob.setJSON(`allValsBlob${dimensions}EmbeddingsMeta`, allValsBlobEmbeddingsMeta);

    console.log(
    ⦚ 8 unchanged lines ⦚
  • v30

    5/29/2024
    Open: Version
    Changes from v29 to v30
    +1
    -1
    ⦚ 50 unchanged lines ⦚
    break;
    } catch (error) {
    if (attempt >= 5) {
    console.log(`FATAL after SQL failed with error ${error}`);
    throw error;
    ⦚ 34 unchanged lines ⦚
    ⦚ 50 unchanged lines ⦚
    break;
    } catch (error) {
    if (attempt >= 10) {
    console.log(`FATAL after SQL failed with error ${error}`);
    throw error;
    ⦚ 34 unchanged lines ⦚
  • v29

    5/29/2024
    Open: Version
    +89
    -0

    import { decode as base64Decode, encode as base64Encode } from "https://deno.land/std@0.166.0/encoding/base64.ts";
    import { sqlToJSON } from "https://esm.town/v/nbbaier/sqliteExportHelpers?v=22";
    import { db as allValsDb } from "https://esm.town/v/sqlite/db?v=9";
    import { blob } from "https://esm.town/v/std/blob";
    import OpenAI from "npm:openai";
    import { truncateMessage } from "npm:openai-tokens";

    const allVals = await sqlToJSON(
    await allValsDb.execute("SELECT author_username, name, version FROM vals WHERE LENGTH(code) > 10 ORDER BY name"),
    ) as any;

    // const allValsBlobEmbeddingsMeta = (await blob.getJSON("allValsBlobEmbeddingsMeta")) ?? {};
    const allValsBlobEmbeddingsMeta = {};
    const existingEmbeddingsIds = new Set(Object.keys(allValsBlobEmbeddingsMeta));

    function idForVal(val: any): string {
    return `${val.author_username}!!${val.name}!!${val.version}`;
    }

    const newValsBatches = [[]];
    let currentBatch = newValsBatches[0];
    for (const val of allVals) {
    const id = idForVal(val);
    if (!existingEmbeddingsIds.has(id)) {
    currentBatch.push(val);
    }
    if (currentBatch.length >= 1000) {
    currentBatch = [];
    newValsBatches.push(currentBatch);
    }
    }

    let nextDataIndex = Math.max(
    0,
    ...Object.values(allValsBlobEmbeddingsMeta).map((item: any) => item.batchDataIndex + 1),
    );
Updated: June 17, 2024