Llm Output Validation: Parse Trees Before Accepting Any Patches

// validate-patch.ts
import ts from "typescript";
import { applyPatch } from "./diff";
import { execa } from "execa";

export type Validation = { ok: true } | { ok: false; stage: string; detail: string };

export async function validatePatch(opts: {
  repoRoot: string;
  unifiedDiff: string;
  maxTypeErrors?: number;
}): Promise<Validation> {
  const tree = applyPatch(opts.repoRoot, opts.unifiedDiff, { dryRun: true });
  if (!tree.ok) return { ok: false, stage: "apply", detail: tree.error };

  for (const file of tree.files) {
    if (!/\.(ts|tsx|js|jsx)$/.test(file.path)) continue;
    const sf = ts.createSourceFile(file.path, file.content, ts.ScriptTarget.Latest, true);
    const parseDiags = (sf as any).parseDiagnostics ?? [];
    // ✅ Real parse tree — not a regex "looks like code"
    if (parseDiags.length) {
      return {
        ok: false,
        stage: "parse",
        detail: parseDiags.map((d: ts.Diagnostic) => ts.flattenDiagnosticMessageText(d.messageText, "\n")).join("\n"),
      };
    }
  }

  // Materialize to a temp worktree for typecheck
  const { exitCode, stderr } = await execa(
    "pnpm",
    ["exec", "tsc", "-p", "tsconfig.json", "--pretty", "false", "--noEmit"],
    { cwd: tree.tempRoot, reject: false },
  );
  if (exitCode !== 0) {
    const errors = stderr.split("\n").filter((l) => /error TS/.test(l));
    if (errors.length > (opts.maxTypeErrors ?? 0)) {
      return { ok: false, stage: "typecheck", detail: errors.slice(0, 40).join("\n") };
    }
  }
  return { ok: true };
}

async function generateUntilValid(agent: Agent, prompt: string, budget = 3) {
  let feedback = "";
  for (let i = 0; i < budget; i++) {
    const diff = await agent.proposePatch(prompt + feedback);
    const v = await validatePatch({ repoRoot: process.cwd(), unifiedDiff: diff });
    if (v.ok) return diff;
    // ✅ Feed machine-readable failure back — not "try again"
    feedback = `\n\nPREVIOUS_PATCH_REJECTED stage=${v.stage}\n${v.detail}\nEmit a corrected unified diff only.`;
  }
  throw new Error("validator_budget_exhausted");
}

const FORBIDDEN = [/^pnpm-lock\.yaml$/, /\/generated\//, /^\.env/];

function assertImportsResolve(file: string, content: string, resolver: Resolver) {
  for (const spec of extractImportSpecifiers(content)) {
    const resolved = resolver.resolve(file, spec);
    if (!resolved) throw new Error(`unresolved_import:${spec}`);
  }
}

function assertNotForbidden(paths: string[]) {
  for (const p of paths) {
    if (FORBIDDEN.some((re) => re.test(p))) throw new Error(`forbidden_path:${p}`);
  }
}