Bedrock Throughput Provisioning: Avoid Throttles During Sev-1 Incidents

// capacity planning — illustrative
type Workload = {
  name: string;
  peakConcurrentSevSessions: number;
  avgInputTokens: number;
  avgOutputTokens: number;
  tokensPerSecondNeeded: number;
};

export function estimateModelUnits(w: Workload): number {
  // Rough: peak sessions * (in+out) / seconds-per-answer
  const tokensPerMin =
    w.peakConcurrentSevSessions * (w.avgInputTokens + w.avgOutputTokens) *
    (60 / 20); // assume ~20s per answer
  // Map to model units per vendor docs; keep 2x headroom for sev-1
  return Math.ceil((tokensPerMin / 10_000) * 2);
}

const onCallAssistant: Workload = {
  name: "sev-copilot",
  peakConcurrentSevSessions: 25,
  avgInputTokens: 6_000,
  avgOutputTokens: 1_200,
  tokensPerSecondNeeded: 0, // derived
};

import {
  BedrockRuntimeClient,
  ConverseCommand,
  ThrottlingException,
} from "@aws-sdk/client-bedrock-runtime";

const PRIMARY = process.env.BEDROCK_PRIMARY_PROFILE!; // provisioned
const FALLBACK = process.env.BEDROCK_FALLBACK_MODEL!; // on-demand smaller

export async function converseResilient(input: {
  messages: any[];
  system: any[];
}) {
  const client = new BedrockRuntimeClient({});
  try {
    return await client.send(new ConverseCommand({
      modelId: PRIMARY,
      messages: input.messages,
      system: input.system,
    }));
  } catch (e) {
    if (!(e instanceof ThrottlingException)) throw e;
    metrics.increment("bedrock.throttle.primary");
    // ✅ Fallback only if prompt features intersect (tools, JSON mode)
    assertPromptCompatible(PRIMARY, FALLBACK);
    return await client.send(new ConverseCommand({
      modelId: FALLBACK,
      messages: input.messages,
      system: input.system,
      // optionally drop tools that fallback cannot support
    }));
  }
}

import { createHash } from "node:crypto";

function cacheKey(question: string, runbookSha: string) {
  return createHash("sha256")
    .update(normalize(question) + ":" + runbookSha)
    .digest("hex");
}

// During sev-1, 10 engineers ask "how do we failover payments redis?"
// Serve the cached grounded answer keyed by runbook SHA.

# CloudWatch — illustrative
AlarmName: bedrock-oncall-throttles
MetricName: ThrottlingException
Namespace: Bedrock/OnCallAssistant
Threshold: 5
Period: 60
EvaluationPeriods: 2
AlarmActions: [sns:sev-paging]