import { readFileSync, statSync } from "fs"; import { createHash } from "crypto"; import { extname, basename } from "path"; import type { LLMClient } from "./llm-client.js"; import { getCached, setCached } from "./llm-cache.js"; import { LLMError } from "./llm-error.js"; interface LLMFileData { purpose: string; exports: string[]; deps: string[]; concepts: string[]; } interface LLMPackageData { role: string; arch: string; } const MAX_FILE_SIZE = 500 * 1024; // 500KB const CONTEXT_BUDGET = 4000; // tokens const CHARS_PER_TOKEN = 4; // approximate for ASCII // Known binary extensions — skip without reading content const BINARY_EXTENSIONS = new Set([ ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".ico", ".svgz", ".mp3", ".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv", ".wav", ".ogg", ".flac", ".aac", ".wma", ".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar", ".exe", ".dll", ".so", ".dylib", ".bin", ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx", ".wasm", ".class", ".jar", ".o", ".a", ".ttf", ".otf", ".woff", ".woff2", ".eot", ".db", ".sqlite", ".sqlite3", ]); function isBinaryFile(filePath: string): boolean { // Fast-path: check extension const ext = extname(filePath).toLowerCase(); if (BINARY_EXTENSIONS.has(ext)) return true; // Fallback: sniff first 8KB for null bytes or non-printable ratio try { const buf = readFileSync(filePath).subarray(0, 8192); let nonPrintable = 0; for (let i = 0; i < buf.length; i++) { const b = buf[i]; if (b === 0) return true; // null byte = definitely binary if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d) { nonPrintable++; } } // If >30% non-printable, treat as binary return nonPrintable / buf.length > 0.3; } catch { return false; } } function truncateForContext(content: string, promptLength: number): string { const maxChars = CONTEXT_BUDGET * CHARS_PER_TOKEN - promptLength; if (content.length <= maxChars) return content; return `${content.slice(0, maxChars - 20)}\n[...truncated]`; } function buildFilePrompt(filePath: string, content: string): string { const name = basename(filePath); return `Analyze this file. Respond in this exact format (one line each): PURPOSE: DEPS: CONCEPTS: File: ${name} \`\`\` ${content} \`\`\` `; } function parseFileResponse(response: string): { purpose: string; deps: string[]; concepts: string[] } { const lines = response.split("\n"); let purpose = ""; let deps: string[] = []; let concepts: string[] = []; for (const line of lines) { const trimmed = line.trim(); if (trimmed.startsWith("PURPOSE:")) { purpose = trimmed.slice("PURPOSE:".length).trim(); } else if (trimmed.startsWith("DEPS:")) { const depsStr = trimmed.slice("DEPS:".length).trim(); deps = depsStr === "none" ? [] : depsStr.split(",").map((s) => s.trim()).filter(Boolean); } else if (trimmed.startsWith("CONCEPTS:")) { const conceptsStr = trimmed.slice("CONCEPTS:".length).trim(); concepts = conceptsStr === "none" ? [] : conceptsStr.split(",").map((s) => s.trim()).filter(Boolean); } } return { purpose, deps, concepts }; } function buildPackagePrompt(relativePath: string, fileSummaries: { name: string; purpose: string }[]): string { const filesList = fileSummaries.map((f) => `- ${f.name}: ${f.purpose}`).join("\n"); return `Analyze this code package/directory. Respond in this exact format (one line each): ROLE: ARCH: Directory: ${relativePath} Files: ${filesList} `; } function parsePackageResponse(response: string): { role: string; arch: string } { const lines = response.split("\n"); let role = ""; let arch = ""; for (const line of lines) { const trimmed = line.trim(); if (trimmed.startsWith("ROLE:")) { role = trimmed.slice("ROLE:".length).trim(); } else if (trimmed.startsWith("ARCH:")) { arch = trimmed.slice("ARCH:".length).trim(); } } return { role, arch }; } export async function extractFileLLM( filePath: string, client?: LLMClient, cacheDir?: string, ): Promise { // LLM-only: require a client if (!client) { throw new LLMError( "No LLM client configured. " + "Set OPENAI_API_KEY / KIMI_API_KEY environment variable, " + "or run inside Pi with a configured model.", ); } // Skip binary files if (isBinaryFile(filePath)) { return { purpose: "Binary file", exports: [], deps: [], concepts: [], }; } const content = readFileSync(filePath, "utf8"); const hash = createHash("sha256").update(content).digest("hex"); // Check disk cache const cached = getCached(hash, cacheDir); if (cached) { const parsed = parseFileResponse(cached); return { purpose: parsed.purpose, exports: [], // AST provides precise exports deps: parsed.deps, concepts: parsed.concepts, }; } // Skip very large files const size = statSync(filePath).size; if (size > MAX_FILE_SIZE) { return { purpose: "Large file", exports: [], deps: [], concepts: [], }; } const prompt = buildFilePrompt(filePath, truncateForContext(content, 200)); const response = await client.complete(prompt); setCached(hash, response, cacheDir); const parsed = parseFileResponse(response); return { purpose: parsed.purpose, exports: [], // AST provides precise exports deps: parsed.deps, concepts: parsed.concepts, }; } export async function extractPackageLLM( relativePath: string, fileData: { name: string; purpose: string }[], client?: LLMClient, _cacheDir?: string, ): Promise { // LLM-only: require a client if (!client) { throw new LLMError( "No LLM client configured. " + "Set OPENAI_API_KEY / KIMI_API_KEY environment variable, " + "or run inside Pi with a configured model.", ); } const prompt = buildPackagePrompt(relativePath, fileData); const response = await client.complete(prompt); const parsed = parsePackageResponse(response); return { role: parsed.role || `Package ${basename(relativePath)}`, arch: parsed.arch || `Contains ${fileData.length} files.`, }; }