Ai Incident Chatops: Draft Change Tickets Without Auto-Remediating

// chatops/draft.ts
export async function draftFromAlarm(alarm: CloudWatchAlarmEvent) {
  const signals = await Promise.all([
    logsInsights(alarm.metricQuery),
    xrayTraceSummaries(alarm.service),
    recentDeploys(alarm.service),
  ]);
  const draft = await bedrock.converse({
    system: "Propose remediation; NEVER claim you executed anything. Cite signal IDs.",
    messages: [{ role: "user", content: JSON.stringify({ alarm, signals }) }],
  });
  return {
    ticket: {
      title: `[sev] ${alarm.alarmName}`,
      body: draft.markdown,
      citations: signals.citationIds,
    },
    proposedCommands: draft.commands.map(asDryRun),
    mutate: false,
  };
}

function asDryRun(cmd: string) {
  if (/\b(apply|delete|rm|put-item|update-function-code)\b/i.test(cmd)) {
    return `# DRY-RUN ONLY\n# ${cmd}`;
  }
  return cmd;
}

{
  "Version": "2012-10-17",
  "Statement": [{
    "Sid": "ReaderOnly",
    "Effect": "Allow",
    "Action": [
      "cloudwatch:GetMetricData",
      "logs:StartQuery",
      "logs:GetQueryResults",
      "xray:GetTraceSummaries",
      "ecs:Describe*"
    ],
    "Resource": "*"
  }]
}

// Slack interactive button
export async function onApprove(action: SlackAction) {
  assertOnCall(action.user);
  assertChangeTicketOpen(action.ticketId);
  // Dual control for prod-touching tools
  await requestSecondApprover(action);
  await enqueueMutatingTool(action.commandId); // still audited
}

## Proposed change
- Service: checkout-api
- Hypothesis: elevated DynamoDB throttles after deploy abc123
- Evidence: Logs Insights query Q-441, X-Ray trace 1-...
- Steps (dry-run):
  1. aws dynamodb describe-table ...
  2. # scale write capacity (APPROVAL REQUIRED)
- Rollback: redeploy previous alias weight 100%
- Blast radius: checkout write path only; reads unaffected

export async function ingress(evt: AlarmEvent) {
  const svc = await cmdb.lookup(evt.alarmArn);
  if (!svc?.runbookUrl) {
    await slack.post({ text: `No runbook for ${evt.alarmName}; human triage required.` });
    return;
  }
  return draftFromAlarm(evt);
}