import { Hono } from "https://deno.land/x/hono@v3.11.7/mod.ts"; import { get_encoding } from "npm:tiktoken"; import "jsr:@std/dotenv/load"; // needed for deno run; not req for smallweb or valtown const app = new Hono(); const ignoredPatternsString = getIgnoredPatterns() .map(pattern => pattern.toString().slice(1, -1)) .join('\n'); const html = ` GitHub Repository Parser

GitHub Repository Parser

Parse and analyze GitHub repositories.

Sometimes GitHub will block your request (e.g. respond with "forbidden"), in which case you'll just have to try again later. Or clone this project and use your own API key ;)

Bytes

0 tokens (GPT-3.5/4 - cl100k_base)

0 tokens (GPT-3 - p50k_base)

0 tokens (GPT-2 - r50k_base)

0 tokens (4o - o200k_base)

Source code

A labspace.ai experiment

`; app.get("/", (c) => c.html(html)); app.post("/parse", async (c) => { try { console.log('Received parse request'); const body = await c.req.json(); console.log('Request body:', body); let { repoUrl, ignoredPatterns, selectedTypes } = body; if (!repoUrl) { throw new Error("Repository URL is missing"); } // Convert ignored patterns back to RegExp objects ignoredPatterns = ignoredPatterns.map(pattern => new RegExp(pattern)); console.log('Processing with selected types:', selectedTypes); // Extract branch if specified let branch = null; if (repoUrl.includes('#')) { [repoUrl, branch] = repoUrl.split('#'); } const [owner, repo] = repoUrl.split('/').slice(-2); if (!owner || !repo) { throw new Error("Invalid repository URL format"); } console.log(`Fetching content for ${owner}/${repo}`); const content = await fetchRepositoryContent(owner, repo, branch, ignoredPatterns, selectedTypes); const tokenCounts = await countTokens(content); console.log('Successfully processed repository'); return c.json({ content, tokenCounts }); } catch (error) { console.error(`Error parsing repository: ${error.message}`); return c.json({ error: `Failed to parse repository: ${error.message}` }, 400); } }); app.post("/tokenize", async (c) => { try { const body = await c.req.json(); const { text, model } = body; const encoder = get_encoding(model); const segments = getTiktokenSegments(encoder, text); encoder.free(); return c.json({ count: segments.reduce((acc, seg) => acc + seg.tokens.length, 0), segments }); } catch (error) { return c.json({ error: error.message }, 500); } }); function getTiktokenSegments(encoder, inputText) { try { const tokens = encoder.encode(inputText); const segments = []; let currentSegment = ''; let tokenAcc = []; for (let idx = 0; idx < tokens.length; idx++) { const token = tokens[idx]; const tokenBytes = encoder.decode_single_token_bytes(token); const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes)); currentSegment += tokenText; tokenAcc.push({ id: token, idx }); // Check if we've reached a valid segment boundary if (inputText.startsWith(currentSegment)) { segments.push({ text: currentSegment, tokens: tokenAcc }); // Remove the matched segment from the input text inputText = inputText.slice(currentSegment.length); // Reset for next segment currentSegment = ''; tokenAcc = []; } } return segments; } catch (error) { throw new Error(`Error in getTiktokenSegments: ${error.message}`); } } function getFileExtension(filename) { const parts = filename.split('.'); return parts.length > 1 ? '.' + parts.pop().toLowerCase() : 'no-extension'; } function getFileType(path) { // Handle directories if (path.endsWith('/')) { return path; } // Handle files const ext = getFileExtension(path); if (ext === 'no-extension') { // Return the filename for files without extensions return path.split('/').pop(); } return ext; } async function fetchRepositoryContent(owner, repo, specifiedBranch = null, ignoredPatterns, selectedTypes = null) { // First, try to get the default branch or use the specified branch const repoInfoUrl = `https://api.github.com/repos/${owner}/${repo}`; const repoInfoResponse = await fetch(repoInfoUrl, { headers: { 'Authorization': `token ${Deno.env.get("GITHUB_API_KEY")}`, 'Accept': 'application/vnd.github.v3+json' } }); if (!repoInfoResponse.ok) { if (repoInfoResponse.status === 404) { throw new Error(`Repository not found or not accessible: ${owner}/${repo}`); } throw new Error(`Failed to fetch repository info: ${repoInfoResponse.statusText}`); } const repoInfo = await repoInfoResponse.json(); const defaultBranch = specifiedBranch || repoInfo.default_branch; const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${defaultBranch}?recursive=1`; const response = await fetch(apiUrl, { headers: { 'Authorization': `token ${Deno.env.get("GITHUB_API_KEY")}`, 'Accept': 'application/vnd.github.v3+json' } }); if (!response.ok) { if (response.status === 404) { throw new Error(`Failed to fetch repository content. The ${defaultBranch} branch might not exist.`); } throw new Error(`GitHub API request failed: ${response.statusText}`); } const data = await response.json(); console.log('Selected types for filtering:', selectedTypes); const files = data.tree.filter(item => { if (item.type !== 'blob') return false; const fileType = getFileType(item.path); console.log(`Checking file ${item.path} with type ${fileType}`); // If no types are selected, include all non-ignored files if (!selectedTypes || selectedTypes.length === 0) { return !shouldIgnoreFile(item.path, ignoredPatterns); } // Otherwise, check if the file type is selected and not ignored return selectedTypes.includes(fileType) && !shouldIgnoreFile(item.path, ignoredPatterns); }); console.log('Filtered files:', files.map(f => f.path)); let content = ''; for (const file of files) { try { const fileContent = await fetchFileContent(owner, repo, file.path, defaultBranch); content += `File: ${file.path}\n\n${fileContent}\n\n`; } catch (error) { console.error(`Failed to fetch content for ${file.path}: ${error.message}`); content += `File: ${file.path}\n\nError: Failed to fetch content\n\n`; } } if (!content) { console.log('No content was gathered from any files'); throw new Error('No files were found matching the selected criteria'); } return content; } async function fetchFileContent(owner, repo, path, branch) { const apiUrl = `https://api.github.com/repos/${owner}/${repo}/contents/${path}`; const response = await fetch(apiUrl, { headers: { 'Authorization': `token ${Deno.env.get("GITHUB_API_KEY")}`, 'Accept': 'application/vnd.github.v3+json' } }); if (!response.ok) { throw new Error(`Failed to fetch file content: ${response.statusText}`); } const data = await response.json(); return atob(data.content); } function getIgnoredPatterns() { return [ // Folders /^\.github\//, /^dist\//, /^public\//, /^static\//, /^node_modules\//, /^vendor\//, /^test\//, /^\.git\//, // Files /\.npmrc$/, /\.gitignore$/, /\.prettierrc$/, /package-lock\.json$/, /yarn\.lock$/, /tsconfig\.json$/, /\.yaml$/, /\.yml$/, /^LICENSE$/, // Lint configuration files /\.eslintrc(\.json|\.js|\.yml|\.yaml)?$/, /\.prettierrc(\.json|\.js|\.yml|\.yaml)?$/, /\.stylelintrc(\.json|\.js|\.yml|\.yaml)?$/, /\.jshintrc$/, /\.jscsrc$/, /\.editorconfig$/, /\.babelrc$/, /\.flowconfig$/, /\.postcssrc$/, /\.htmlhintrc$/, /\.csslintrc$/, /\.sasslintrc$/, /\.lesshintrc$/, /\.markdownlintrc$/, /\.coffeelintrc$/, /\.eslintignore$/, /\.prettierignore$/, /\.stylelintignore$/, // Additional configuration files /\.env(\..+)?$/, /\.dockerignore$/, /Dockerfile$/, /docker-compose\.yml$/, /\.travis\.yml$/, /\.circleci\/config\.yml$/, /\.gitlab-ci\.yml$/, /\.github\/workflows\/.*\.yml$/, /\.vscode\/.*$/, /\.idea\/.*$/, // Additional ignored patterns from .npmignore /^samples\//, /^addons\//, /^libs\//, /\.njsproj$/, /^\..*$/, // This covers all dot files/folders /\.log$/, /^tools\//, /^bower\.json$/, /^\.lambdatest$/, /^tunnel\.pid$/, /^pnpm-workspace\.yaml$/, // File types /\.md$/, // Ignore all .md files except README.md /\.(pdf|png|jpg|jpeg|gif|ico|svg|woff|woff2|eot|ttf|otf)$/, /\.(mp4|avi|webm|mov|mp3|wav|flac|ogg|webp)$/, /(npm-debug|yarn-debug|yarn-error)\.log$/, ]; } function shouldIgnoreFile(path, ignoredPatterns) { // Special case for README.md if (path.toLowerCase() === 'readme.md') { return false; } return ignoredPatterns.some(pattern => pattern.test(path)); } async function countTokens(text) { const encoders = { cl100k_base: get_encoding("cl100k_base"), p50k_base: get_encoding("p50k_base"), r50k_base: get_encoding("r50k_base"), o200k_base: get_encoding("o200k_base"), }; const counts = {}; for (const [name, encoder] of Object.entries(encoders)) { counts[name] = encoder.encode(text).length; encoder.free(); } return counts; } async function getTokenizedText(text) { const encoder = get_encoding("cl100k_base"); const tokens = encoder.encode(text); const tokenizedHtml = tokens.map(token => { const tokenBytes = encoder.decode_single_token_bytes(token); const tokenText = new TextDecoder().decode(new Uint8Array(tokenBytes)); return `${tokenText === '\n' ? '↵' : tokenText}`; }).join(''); encoder.free(); return tokenizedHtml; } // Add new endpoint to handle file type fetching app.post("/file-types", async (c) => { try { const { repoUrl } = await c.req.json(); if (!repoUrl) { throw new Error("Repository URL is missing"); } let branch = null; if (repoUrl.includes('#')) { [repoUrl, branch] = repoUrl.split('#'); } const [owner, repo] = repoUrl.split('/').slice(-2); if (!owner || !repo) { throw new Error("Invalid repository URL format"); } const fileTypes = await getRepositoryFileTypes(owner, repo, branch); return c.json(fileTypes); } catch (error) { return c.json({ error: error.message }, 400); } }); // Add new function to get repository file types async function getRepositoryFileTypes(owner, repo, specifiedBranch = null) { const repoInfoUrl = `https://api.github.com/repos/${owner}/${repo}`; const repoInfoResponse = await fetch(repoInfoUrl, { headers: { 'Authorization': `token ${Deno.env.get("GITHUB_API_KEY")}`, 'Accept': 'application/vnd.github.v3+json' } }); if (!repoInfoResponse.ok) { throw new Error(`Failed to fetch repository info: ${repoInfoResponse.statusText}`); } const repoInfo = await repoInfoResponse.json(); const defaultBranch = specifiedBranch || repoInfo.default_branch; const apiUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${defaultBranch}?recursive=1`; const response = await fetch(apiUrl, { headers: { 'Authorization': `token ${Deno.env.get("GITHUB_API_KEY")}`, 'Accept': 'application/vnd.github.v3+json' } }); if (!response.ok) { throw new Error(`GitHub API request failed: ${response.statusText}`); } const data = await response.json(); // Count occurrences of each file type const typeCount = new Map(); data.tree.forEach(item => { const type = getFileType(item.path); typeCount.set(type, (typeCount.get(type) || 0) + 1); }); // Convert to array and sort by count return Array.from(typeCount.entries()) .map(([type, count]) => ({ type, count })) .sort((a, b) => b.count - a.count); } // Export app.fetch for Val Town, otherwise export app — this is only for hono apps export default (typeof Deno !== "undefined" && Deno.env.get("valtown")) ? app.fetch : app;

GitHub Repository Parser

Select File Types to Include:

githubParser

Version 3