fix: parallelize file processing in init to prevent Pi timeout

- Use processFiles from llm-batch.ts in generateDirectoryMap
- Process files with 4 concurrent LLM calls + 100ms batch delays
- Reduces init time from O(n) sequential to O(n/4) parallel
- Prevents Pi tool timeout on projects with many files
This commit is contained in:
2026-06-10 16:27:05 +02:00
parent 553789a59f
commit bd5ba74aa2
2 changed files with 218 additions and 7 deletions
+12 -7
View File
@@ -7,6 +7,7 @@ import {
import { extractFileLLM, extractPackageLLM } from "./llm-extract.js";
import { extractFileAST } from "./ast-extract.js";
import { mergeFileData } from "./merge.js";
import { processFiles } from "./llm-batch.js";
import { writeFileSync } from "fs";
import { join } from "path";
import type { LLMClient } from "./llm-client.js";
@@ -37,13 +38,17 @@ export async function generateDirectoryMap(
llmClient?: LLMClient,
cacheDir?: string,
): Promise<FileEntry[]> {
const fileData: FileEntry[] = [];
for (const file of entry.files) {
const filePath = join(entry.dirPath, file);
const llmData = await extractFileLLM(filePath, llmClient, cacheDir);
const astData = await extractFileAST(filePath);
fileData.push(mergeFileData(file, llmData, astData));
}
// Process files in parallel (4 concurrent) with retry logic
const fileData = await processFiles(
entry.files,
async (file) => {
const filePath = join(entry.dirPath, file);
const llmData = await extractFileLLM(filePath, llmClient, cacheDir);
const astData = await extractFileAST(filePath);
return mergeFileData(file, llmData, astData);
},
{ concurrency: 4, batchDelayMs: 100, maxRetries: 2 },
);
const packageData = await extractPackageLLM(
entry.relativePath,