pi-map/src/llm/llm-extract.ts

import { readFileSync, statSync } from "fs";
import { createHash } from "crypto";
import { extname, basename } from "path";
import type { LLMClient } from "./llm-client.js";
import { getCached, setCached } from "./llm-cache.js";
import { LLMError } from "./llm-error.js";

interface LLMFileData {
	purpose: string;
	exports: string[];
	deps: string[];
	concepts: string[];
}

interface LLMPackageData {
	role: string;
	arch: string;
}

const MAX_FILE_SIZE = 500 * 1024; // 500KB
const CONTEXT_BUDGET = 4000; // tokens
const CHARS_PER_TOKEN = 4; // approximate for ASCII

// Known binary extensions — skip without reading content
const BINARY_EXTENSIONS = new Set([
	".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".ico", ".svgz",
	".mp3", ".mp4", ".avi", ".mov", ".mkv", ".flv", ".wmv",
	".wav", ".ogg", ".flac", ".aac", ".wma",
	".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar",
	".exe", ".dll", ".so", ".dylib", ".bin",
	".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
	".wasm", ".class", ".jar", ".o", ".a",
	".ttf", ".otf", ".woff", ".woff2", ".eot",
	".db", ".sqlite", ".sqlite3",
]);

function isBinaryFile(filePath: string): boolean {
	// Fast-path: check extension
	const ext = extname(filePath).toLowerCase();
	if (BINARY_EXTENSIONS.has(ext)) return true;

	// Fallback: sniff first 8KB for null bytes or non-printable ratio
	try {
		const buf = readFileSync(filePath).subarray(0, 8192);
		let nonPrintable = 0;
		for (let i = 0; i < buf.length; i++) {
			const b = buf[i];
			if (b === 0) return true; // null byte = definitely binary
			if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d) {
				nonPrintable++;
			}
		}
		// If >30% non-printable, treat as binary
		return nonPrintable / buf.length > 0.3;
	} catch {
		return false;
	}
}

function truncateForContext(content: string, promptLength: number): string {
	const maxChars = CONTEXT_BUDGET * CHARS_PER_TOKEN - promptLength;
	if (content.length <= maxChars) return content;
	return `${content.slice(0, maxChars - 20)}\n[...truncated]`;
}

function buildFilePrompt(filePath: string, content: string): string {
	const name = basename(filePath);
	return `Analyze this file. Respond in this exact format (one line each):
PURPOSE: <concise one-sentence description of what this file does>
DEPS: <comma-separated list of key dependencies/modules it relies on, or "none">
CONCEPTS: <comma-separated list of key concepts/patterns used, or "none">

File: ${name}
\`\`\`
${content}
\`\`\`
`;
}

function parseFileResponse(response: string): { purpose: string; deps: string[]; concepts: string[] } {
	const lines = response.split("\n");
	let purpose = "";
	let deps: string[] = [];
	let concepts: string[] = [];

	for (const line of lines) {
		const trimmed = line.trim();
		if (trimmed.startsWith("PURPOSE:")) {
			purpose = trimmed.slice("PURPOSE:".length).trim();
		} else if (trimmed.startsWith("DEPS:")) {
			const depsStr = trimmed.slice("DEPS:".length).trim();
			deps = depsStr === "none" ? [] : depsStr.split(",").map((s) => s.trim()).filter(Boolean);
		} else if (trimmed.startsWith("CONCEPTS:")) {
			const conceptsStr = trimmed.slice("CONCEPTS:".length).trim();
			concepts = conceptsStr === "none" ? [] : conceptsStr.split(",").map((s) => s.trim()).filter(Boolean);
		}
	}

	return { purpose, deps, concepts };
}

function buildPackagePrompt(relativePath: string, fileSummaries: { name: string; purpose: string }[]): string {
	const filesList = fileSummaries.map((f) => `- ${f.name}: ${f.purpose}`).join("\n");
	return `Analyze this code package/directory. Respond in this exact format (one line each):
ROLE: <concise one-sentence description of this package's role in the project>
ARCH: <concise description of architecture/patterns used in this package>

Directory: ${relativePath}
Files:
${filesList}
`;
}

function parsePackageResponse(response: string): { role: string; arch: string } {
	const lines = response.split("\n");
	let role = "";
	let arch = "";

	for (const line of lines) {
		const trimmed = line.trim();
		if (trimmed.startsWith("ROLE:")) {
			role = trimmed.slice("ROLE:".length).trim();
		} else if (trimmed.startsWith("ARCH:")) {
			arch = trimmed.slice("ARCH:".length).trim();
		}
	}

	return { role, arch };
}

export async function extractFileLLM(
	filePath: string,
	client?: LLMClient,
	cacheDir?: string,
): Promise<LLMFileData> {
	// LLM-only: require a client
	if (!client) {
		throw new LLMError(
			"No LLM client configured. " +
			"Set OPENAI_API_KEY / KIMI_API_KEY environment variable, " +
			"or run inside Pi with a configured model.",
		);
	}

	// Skip binary files
	if (isBinaryFile(filePath)) {
		return {
			purpose: "Binary file",
			exports: [],
			deps: [],
			concepts: [],
		};
	}

	const content = readFileSync(filePath, "utf8");
	const hash = createHash("sha256").update(content).digest("hex");

	// Check disk cache
	const cached = getCached(hash, cacheDir);
	if (cached) {
		const parsed = parseFileResponse(cached);
		return {
			purpose: parsed.purpose,
			exports: [], // AST provides precise exports
			deps: parsed.deps,
			concepts: parsed.concepts,
		};
	}

	// Skip very large files
	const size = statSync(filePath).size;
	if (size > MAX_FILE_SIZE) {
		return {
			purpose: "Large file",
			exports: [],
			deps: [],
			concepts: [],
		};
	}

	const prompt = buildFilePrompt(filePath, truncateForContext(content, 200));
	const response = await client.complete(prompt);
	setCached(hash, response, cacheDir);

	const parsed = parseFileResponse(response);
	return {
		purpose: parsed.purpose,
		exports: [], // AST provides precise exports
		deps: parsed.deps,
		concepts: parsed.concepts,
	};
}

export async function extractPackageLLM(
	relativePath: string,
	fileData: { name: string; purpose: string }[],
	client?: LLMClient,
	_cacheDir?: string,
): Promise<LLMPackageData> {
	// LLM-only: require a client
	if (!client) {
		throw new LLMError(
			"No LLM client configured. " +
			"Set OPENAI_API_KEY / KIMI_API_KEY environment variable, " +
			"or run inside Pi with a configured model.",
		);
	}

	const prompt = buildPackagePrompt(relativePath, fileData);
	const response = await client.complete(prompt);
	const parsed = parsePackageResponse(response);

	return {
		role: parsed.role || `Package ${basename(relativePath)}`,
		arch: parsed.arch || `Contains ${fileData.length} files.`,
	};
}