AI On-Call Copilots: Suggest Runbooks Without Mutating Production

// oncall_tools.ts
const READ_ONLY = [
  /^kubectl (get|describe|logs|top) /,
  /^aws (logs|cloudwatch|xray|ecs describe|dynamodb describe)/,
  /^aws sts get-caller-identity$/,
];

const WRITEISH = [
  /kubectl (apply|delete|scale|rollout restart|exec)/,
  /aws (ecs update|autoscaling|s3 rm|dynamodb delete)/,
];

export function assertReadOnly(cmd: string) {
  if (WRITEISH.some((re) => re.test(cmd))) {
    throw new Error("write_requires_approval"); // ✅ hard stop
  }
  if (!READ_ONLY.some((re) => re.test(cmd))) {
    throw new Error("command_not_allowlisted");
  }
}

// ❌ Agent tool: run_shell(cmd) with no classifier

# suggest_remediation.py
def suggest(sev_context: dict) -> dict:
    evidence = {
        "logs": logs_insights(sev_context["query"]),
        "alarms": cloudwatch_alarms(sev_context["service"]),
        "trace": xray_trace(sev_context.get("traceId")),
    }
    draft = bedrock_converse(
        system="Propose remediations. Cite evidence IDs. Never claim you executed writes.",
        user=json.dumps(evidence),
    )
    return {
        "commands_dry_run": draft["commands"],
        "evidence_ids": [e["id"] for e in evidence["logs"][:5]],
        "change_ticket": open_ticket_draft(draft),  # ✅ human must submit
    }

# slack modal — illustrative
# ✅ "Approve apply of rollout restart on payments-api (change CHG-9182)"
# ❌ "Auto-remediate" toggle enabled by default in prod