Cost-Aware Rag Caching: Reuse Answers for Identical Code Questions

import { createHash } from "node:crypto";

export function cacheKey(opts: {
  tenantId: string;
  repo: string;
  treeSha: string;
  question: string;
  modelId: string;
}) {
  const q = opts.question
    .toLowerCase()
    .replace(/[`*_]/g, "")
    .replace(/\s+/g, " ")
    .trim();
  const material = [opts.tenantId, opts.repo, opts.treeSha, opts.modelId, q].join("|");
  return createHash("sha256").update(material).digest("hex");
}

// ❌ Keying only on question → stale answers after refactors
// ❌ Omitting tenantId → cross-tenant leakage

type RagCacheItem = {
  pk: string; // tenant#repo
  sk: string; // key hash
  treeSha: string;
  answerMd: string;
  citations: { path: string; start: number; end: number }[];
  modelId: string;
  tokensSaved: number;
  expiresAt: number; // epoch seconds (TTL attribute)
};

export async function invalidateForPaths(repo: string, changed: string[]) {
  // Query GSIs of cached citations overlapping changed paths
  const items = await queryByCitedPaths(repo, changed);
  await Promise.all(items.map((i) => deleteItem(i)));
}

// Hook from git webhook after merge — same channel as KB sync

const inflight = new Map<string, Promise<RagAnswer>>();

export async function getOrRetrieve(key: string, produce: () => Promise<RagAnswer>) {
  const hit = await cache.get(key);
  if (hit) return hit;
  let p = inflight.get(key);
  if (!p) {
    p = produce().finally(() => inflight.delete(key));
    inflight.set(key, p);
  }
  const ans = await p;
  await cache.set(key, ans);
  return ans;
}