Node Heap Snapshots in Production: Safe Capture Under Incident Load

// src/heap-snapshot-gate.ts
import { writeHeapSnapshot } from "node:v8";
import { randomUUID } from "node:crypto";
import { unlinkSync, statSync, createReadStream } from "node:fs";
import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";

const s3 = new S3Client({});
let inflight = false;
let lastAt = 0;
const MIN_INTERVAL_MS = 15 * 60_000;
const MAX_BYTES = 2 * 1024 * 1024 * 1024;

export async function captureHeap(reason: string) {
 const now = Date.now();
 if (inflight) throw new Error("snapshot_inflight");
 if (now - lastAt < MIN_INTERVAL_MS) throw new Error("snapshot_rate_limited");
 inflight = true;
 lastAt = now;
 const id = randomUUID();
 const path = `/tmp/heap-${id}.heapsnapshot`;
 try {
 // ✅ Advertise pause to metrics before the stop-the-world
 metrics.gauge("heap.snapshot.paused", 1);
 writeHeapSnapshot(path);
 const size = statSync(path).size;
 if (size > MAX_BYTES) throw new Error(`snapshot_too_large:${size}`);
 await s3.send(new PutObjectCommand({
 Bucket: process.env.HEAP_BUCKET!,
 Key: `heaps/${process.env.SERVICE}/${id}.heapsnapshot`,
 Body: createReadStream(path),
 ServerSideEncryption: "aws:kms",
 Metadata: { reason },
 }));
 return { id, size };
 } finally {
 try { unlinkSync(path); } catch {}
 metrics.gauge("heap.snapshot.paused", 0);
 inflight = false;
 }
}

// ❌ Free-for-all SIGUSR2 with no rate limit during a sev-1
process.on("SIGUSR2", () => writeHeapSnapshot());

// src/scrub-heapsnapshot.ts
import { Transform } from "node:stream";

const SECRET = /(api[_-]?key|password|authorization|Bearer\s+\S+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/gi;

export function scrubStrings() {
 return new Transform({
 transform(chunk, _enc, cb) {
 // ✅ Blank obvious secret-shaped strings in snapshot text
 cb(null, Buffer.from(String(chunk).replace(SECRET, "[REDACTED]")));
 },
 });
}

# ✅ Trigger on ONE ECS task, not the whole service
aws ecs execute-command --cluster prod --task $TASK \
 --interactive --command "node -e \"require('./dist/heap-snapshot-gate').captureHeap('sev1-rss')\""