CI Failure Triage Bots: Separate Flaky Noise From Real Regressions

import { createHash } from "node:crypto";

export function fingerprint(fail: {
  testId: string;
  message: string;
  stack: string[];
}): string {
  const top = fail.stack.slice(0, 3).map(normalizeFrame).join("|");
  const msgClass = fail.message.replace(/\d+/g, "N").replace(/0x[0-9a-f]+/gi, "HEX");
  return createHash("sha256")
    .update(`${fail.testId}::${msgClass}::${top}`)
    .digest("hex")
    .slice(0, 16);
}

function normalizeFrame(f: string) {
  return f.replace(/:\d+:\d+/g, ":L:C").replace(/\/home\/runner\/.*/g, "REPO");
}

type Label = "flake" | "regression" | "infra" | "novel";

export async function classify(fp: string, ctx: {
  seenCount90d: number;
  flakeRatio90d: number;
  firstSeenAfterDeploy: boolean;
  matchedInfraSignature: boolean;
}): Promise<Label> {
  if (ctx.matchedInfraSignature) return "infra";
  if (ctx.seenCount90d === 0) return "novel";
  if (ctx.flakeRatio90d >= 0.6 && !ctx.firstSeenAfterDeploy) return "flake";
  if (ctx.firstSeenAfterDeploy && ctx.flakeRatio90d < 0.2) return "regression";
  // LLM assist for ambiguous mid-band — optional, never sole authority
  return "novel"; // fail closed toward human eyes
}

flake:
  action: rerun_job
  max_reruns: 1
  notify: silent_metric
regression:
  action: block_merge
  notify: pr_comment + slack_team
infra:
  action: rerun_on_new_runner
  notify: platform_oncall (if burst)
novel:
  action: block_merge
  notify: page_deploying_team
  require: human_label_within_24h

export async function handleFailure(label: Label, job: Job) {
  switch (label) {
    case "flake":
      if (job.reruns < 1) return rerun(job);
      return quarantineTest(job.testId, { slaHours: 72 });
    case "regression":
      return blockMerge(job.pr, `regression_fingerprint=${job.fp}`);
    case "infra":
      return rerun(job, { runnerLabel: "fresh" });
    case "novel":
      await page(job.deployingTeam, job);
      return blockMerge(job.pr, `novel_fingerprint=${job.fp}`);
  }
}