OpenTelemetry Sampling for Node: Stay Useful at Fifty Thousand RPS

// src/otel.ts
import { NodeSDK } from "@opentelemetry/sdk-node";
import { ParentBasedSampler, TraceIdRatioBasedSampler, SamplingDecision } from "@opentelemetry/sdk-trace-base";
import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";

class ErrorAwareRatioSampler extends TraceIdRatioBasedSampler {
 // Ratio applied only when parent says "ask child"
 constructor(private ratio: number) { super(ratio); }
}

const exporter = new OTLPTraceExporter({ url: process.env.OTLP_URL });
const processor = new BatchSpanProcessor(exporter, {
 maxQueueSize: 2048, // ✅ bound memory
 scheduledDelayMillis: 5000,
 maxExportBatchSize: 512,
});

const sdk = new NodeSDK({
 sampler: new ParentBasedSampler({
 root: new TraceIdRatioBasedSampler(Number(process.env.TRACE_RATIO ?? "0.02")),
 }),
 spanProcessors: [processor],
});
sdk.start();

// ❌ AlwaysOnSampler in production at 50k RPS
import { AlwaysOnSampler } from "@opentelemetry/sdk-trace-base";

// src/http-hook.ts — after response, bump sampling bit for bad outcomes
import { trace, SpanStatusCode } from "@opentelemetry/api";

export function finalizeSpan(status: number, durationMs: number) {
 const span = trace.getActiveSpan();
 if (!span) return;
 if (status >= 500 || durationMs > 2000) {
 span.setStatus({ code: SpanStatusCode.ERROR });
 span.setAttribute("sampling.force", true);
 // Tail sampler / collector rule keys off this attribute or status
 }
}

# otel-collector-config.yaml (excerpt)
processors:
 tail_sampling:
 decision_wait: 10s
 policies:
 - name: errors
 type: status_code
 status_code: { status_codes: [ERROR] }
 - name: slow
 type: latency
 latency: { threshold_ms: 2000 }
 - name: probabilistic
 type: probabilistic
 probabilistic: { sampling_percentage: 2 }