Ai Differential Testing: AI Oracles Validated Against Shadow Traffic

// gateway/shadow.ts
export async function shadowCompare(
  req: Request,
  primary: Handler,
  candidate: Handler,
  opts: { sampleRate: number; compareWrites: false }
) {
  const primaryRes = await primary(req);
  if (Math.random() > opts.sampleRate) return primaryRes;
  if (req.method !== "GET" && !opts.compareWrites) {
    // ❌ Never shadow POST/PUT to a candidate that mutates shared DBs
    return primaryRes;
  }
  // fire-and-forget candidate
  void candidate(req.clone()).then((cand) =>
    enqueueDiff({ primary: primaryRes, cand, path: req.url })
  );
  return primaryRes;
}

import { createHash } from "node:crypto";

type Diff = { field: string; primary: unknown; candidate: unknown };

export function semanticDiff(a: unknown, b: unknown, path = "$"): Diff[] {
  const out: Diff[] = [];
  if (typeof a !== typeof b) {
    out.push({ field: path, primary: a, candidate: b });
    return out;
  }
  if (Array.isArray(a) && Array.isArray(b)) {
    // order-insensitive for unordered collections
    const sa = [...a].map(stable).sort();
    const sb = [...b].map(stable).sort();
    if (JSON.stringify(sa) !== JSON.stringify(sb)) {
      out.push({ field: path, primary: a, candidate: b });
    }
    return out;
  }
  if (a && typeof a === "object") {
    const ka = Object.keys(a as object).sort();
    const kb = Object.keys(b as object).sort();
    for (const k of new Set([...ka, ...kb])) {
      out.push(...semanticDiff((a as any)[k], (b as any)[k], `${path}.${k}`));
    }
    return out;
  }
  // money: compare cents; timestamps: ignore
  if (path.endsWith(".updatedAt")) return out;
  if (a !== b) out.push({ field: path, primary: a, candidate: b });
  return out;
}

function stable(v: unknown) {
  return createHash("sha256").update(JSON.stringify(v)).digest("hex");
}