fix: parallelize file processing in init to prevent Pi timeout
- Use processFiles from llm-batch.ts in generateDirectoryMap - Process files with 4 concurrent LLM calls + 100ms batch delays - Reduces init time from O(n) sequential to O(n/4) parallel - Prevents Pi tool timeout on projects with many files
This commit is contained in:
+12
-7
@@ -7,6 +7,7 @@ import {
|
||||
import { extractFileLLM, extractPackageLLM } from "./llm-extract.js";
|
||||
import { extractFileAST } from "./ast-extract.js";
|
||||
import { mergeFileData } from "./merge.js";
|
||||
import { processFiles } from "./llm-batch.js";
|
||||
import { writeFileSync } from "fs";
|
||||
import { join } from "path";
|
||||
import type { LLMClient } from "./llm-client.js";
|
||||
@@ -37,13 +38,17 @@ export async function generateDirectoryMap(
|
||||
llmClient?: LLMClient,
|
||||
cacheDir?: string,
|
||||
): Promise<FileEntry[]> {
|
||||
const fileData: FileEntry[] = [];
|
||||
for (const file of entry.files) {
|
||||
const filePath = join(entry.dirPath, file);
|
||||
const llmData = await extractFileLLM(filePath, llmClient, cacheDir);
|
||||
const astData = await extractFileAST(filePath);
|
||||
fileData.push(mergeFileData(file, llmData, astData));
|
||||
}
|
||||
// Process files in parallel (4 concurrent) with retry logic
|
||||
const fileData = await processFiles(
|
||||
entry.files,
|
||||
async (file) => {
|
||||
const filePath = join(entry.dirPath, file);
|
||||
const llmData = await extractFileLLM(filePath, llmClient, cacheDir);
|
||||
const astData = await extractFileAST(filePath);
|
||||
return mergeFileData(file, llmData, astData);
|
||||
},
|
||||
{ concurrency: 4, batchDelayMs: 100, maxRetries: 2 },
|
||||
);
|
||||
|
||||
const packageData = await extractPackageLLM(
|
||||
entry.relativePath,
|
||||
|
||||
Reference in New Issue
Block a user