Bedrock Knowledge Base Sync: Git Webhooks That Prevent Index Drift

// api/github-webhook.ts
import { createHmac, timingSafeEqual } from "crypto";
import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs";

const sqs = new SQSClient({});

export async function handlePush(req: Request) {
  const raw = Buffer.from(await req.arrayBuffer());
  verifySig(req.headers.get("x-hub-signature-256"), raw);

  const body = JSON.parse(raw.toString("utf8"));
  if (body.ref !== "refs/heads/main") return new Response("ignored");

  const commits = body.commits ?? [];
  const added = new Set<string>();
  const removed = new Set<string>();
  for (const c of commits) {
    for (const f of c.added ?? []) if (allow(f)) added.add(f);
    for (const f of c.modified ?? []) if (allow(f)) added.add(f);
    for (const f of c.removed ?? []) if (allow(f)) removed.add(f);
  }

  await sqs.send(new SendMessageCommand({
    QueueUrl: process.env.KB_SYNC_QUEUE!,
    MessageBody: JSON.stringify({
      sha: body.after,
      added: [...added],
      removed: [...removed],
    }),
  }));
  return new Response("ok");
}

function allow(path: string) {
  return /^(src|docs|packages)\//.test(path) && !path.includes("node_modules");
}

# kb_sync_worker.py
import hashlib, boto3

s3 = boto3.client("s3")
BUCKET = "kb-datasource-prod"

def sync_event(evt: dict):
    sha = evt["sha"]
    for path in evt["added"]:
        blob = git_show(sha, path)
        checksum = hashlib.sha256(blob).hexdigest()
        key = f"repo/{path}"
        s3.put_object(Bucket=BUCKET, Key=key, Body=blob,
                      Metadata={"checksum": checksum, "git_sha": sha})
        s3.put_object(
            Bucket=BUCKET,
            Key=f"{key}.metadata.json",
            Body=json_dumps({
                "metadataAttributes": {
                    "path": path,
                    "git_sha": sha,
                    "checksum": checksum,
                }
            }),
        )
    for path in evt["removed"]:
        key = f"repo/{path}"
        # ✅ Tombstone: delete object so next ingestion drops vectors
        s3.delete_object(Bucket=BUCKET, Key=key)
        s3.delete_object(Bucket=BUCKET, Key=f"{key}.metadata.json")
    start_kb_ingestion(job_name="incremental")

# fail CI/cron if deleted path still retrieves
aws bedrock-agent-runtime retrieve \
  --knowledge-base-id "$KB" \
  --retrieval-query text="path:src/legacy/Gone.ts" \
  | jq '.retrievalResults | length'   # expect 0