Ai Coding Tool Evaluation: Blind A/B Tests on Real Tickets

// eval_assign.ts
type Tool = "cursor" | "claude-code" | "copilot" | "control-no-ai";

export function assignTool(ticketId: string, salt: string): Tool {
  // ✅ stable hash assignment for reproducibility
  const h = hash(`${salt}:${ticketId}`);
  const tools: Tool[] = ["cursor", "claude-code", "copilot", "control-no-ai"];
  return tools[h % tools.length]!;
}

export type TicketOutcome = {
  ticketId: string;
  tool: Tool; // hidden from graders
  minutesToMerge: number;
  revertWithin7d: boolean;
  blockingReviewComments: number;
  escapedDefectSev: 0 | 1 | 2 | 3;
};

# summarize.py
import statistics as stats

def report(rows: list[dict]):
    for tool, group in groupby(rows, "tool"):
        print(tool, {
            "p50_ttm": stats.median(r["minutesToMerge"] for r in group),
            "revert_rate": mean(r["revertWithin7d"] for r in group),
            "p50_blocking": stats.median(r["blockingReviewComments"] for r in group),
        })

# trial protocol (illustrative)
tickets: 120  # >= 30 per arm after exclusions
duration: 4 weeks
exclude:
  - pure docs / typography
  - incidents (separate study)
fairness:
  - same CODEOWNERS rules
  - same CI gates
  - no tool-specific reviewer coaching mid-trial