Bedrock Model Distillation: Shrink Coding Assistants for Edge Latency

# collect_traces.py — illustrative
import json

def keep(example: dict) -> bool:
    # ✅ Only traces humans accepted AND that typecheck in context
    if not example.get("accepted"):
        return False
    if example.get("compile_errors"):
        return False
    if example.get("path", "").startswith("generated/"):
        return False
    return True

def to_distill_record(ex: dict) -> dict:
    return {
        "prompt": ex["prefix"],          # code before cursor
        "completion": ex["suggestion"],  # teacher or human-final text
        "meta": {
            "language": ex["language"],
            "repo": ex["repo"],
            "teacher_model": ex["teacher_model"],
        },
    }

with open("teacher_traces.jsonl") as f, open("distill.jsonl", "w") as out:
    for line in f:
        ex = json.loads(line)
        if keep(ex):
            out.write(json.dumps(to_distill_record(ex)) + "\n")

# Illustrative Bedrock / custom model flow — follow current console/API names
# 1) Upload distill.jsonl to S3 (CMK encrypted)
# 2) Create distillation / fine-tune job with teacher + student model IDs
# 3) Register model; create Provisioned Throughput if autocomplete QPS needs it
# 4) Pin application inference profile alias: coding-autocomplete-prod -> student-v3

type AutocompleteEval = {
  exactMatchRate: number;      // vs held-out human accepts
  compileRate: number;         // suggestion + suffix typechecks
  p95LatencyMs: number;        // IDE round-trip
  hallucinationRate: number;   // unknown symbols / APIs
};

export function shipGate(e: AutocompleteEval, baseline: AutocompleteEval) {
  if (e.p95LatencyMs > 150) throw new Error("latency_budget");
  if (e.compileRate < baseline.compileRate - 0.02) throw new Error("compile_regress");
  if (e.hallucinationRate > baseline.hallucinationRate + 0.01) throw new Error("hallucination_regress");
}