Rag Evaluation Aws: Hit Rate, Faithfulness, and Cost Curves

{
  "id": "q-042",
  "question": "Where do we enforce tenant isolation on order reads?",
  "expect_paths": ["packages/orders/src/getOrder.ts"],
  "expect_symbols": ["assertTenant"],
  "difficulty": "medium"
}

def hit_at_k(retrieved_paths: list[str], expect: list[str], k: int = 5) -> float:
    top = set(retrieved_paths[:k])
    return 1.0 if any(e in top for e in expect) else 0.0

def citation_precision(answer_cites: list[str], retrieved: list[str]) -> float:
    if not answer_cites:
        return 0.0
    ok = sum(1 for c in answer_cites if c in retrieved)
    return ok / len(answer_cites)

# Faithfulness: entailment model or Bedrock judge with rubric
FAITHFULNESS_RUBRIC = """
Score 1-5: does the answer only claim facts supported by the passages?
5 = fully supported; 1 = contradicts or invents APIs.
Return JSON {"score": int, "rationale": str}
"""

# batch/evaluate_job.py
import json, boto3, os
s3 = boto3.client("s3")
brt = boto3.client("bedrock-runtime")

def main():
    bucket, key = os.environ["GOLDEN_S3"].split("/", 3)[2], os.environ["GOLDEN_S3"].split("/", 3)[3]
    # Actually parse s3://bucket/key properly in prod
    questions = json.loads(s3.get_object(Bucket=os.environ["BUCKET"], Key=os.environ["GOLDEN_KEY"])["Body"].read())
    rows = []
    for q in questions:
        retrieved = retrieve(q["question"], k=8)  # OpenSearch or pgvector
        answer, usage = generate(q["question"], retrieved)
        rows.append({
            "id": q["id"],
            "hit@5": hit_at_k([r["path"] for r in retrieved], q["expect_paths"], 5),
            "cite_prec": citation_precision(answer["cites"], [r["path"] for r in retrieved]),
            "faithfulness": judge(answer["text"], retrieved),
            "input_tokens": usage["input"],
            "output_tokens": usage["output"],
            "cost_usd": estimate_cost(usage),
        })
    s3.put_object(Bucket=os.environ["BUCKET"], Key=os.environ["OUT_KEY"], Body=json.dumps(rows).encode())

# Illustrative planning numbers — label as such
# Haiku-class retrieve+answer: ~$0.002–0.008 / query
# Sonnet-class: ~$0.01–0.04 / query
# With prompt caching on large system+tool schemas: 30–70% input discount (see companion post)