OpenTelemetry for LLMs: Trace Prompt Latency Across Microservices

agent.turn (root)
  ├─ context.assemble
  ├─ retrieve.kb
  ├─ gen_ai.chat (Bedrock Converse)
  │    └─ gen_ai.tool.search_repo
  │         └─ aws.lambda.invoke
  └─ patch.validate

import { trace, SpanStatusCode, context, propagation } from "@opentelemetry/api";

const tracer = trace.getTracer("cheatcoders-agent");

export async function runTurn(userId: string, prompt: string) {
  return tracer.startActiveSpan("agent.turn", async (span) => {
    span.setAttribute("user.id", userId);
    span.setAttribute("agent.prompt_chars", prompt.length);
    try {
      const answer = await converseWithTools(prompt);
      span.setStatus({ code: SpanStatusCode.OK });
      return answer;
    } catch (err) {
      span.recordException(err as Error);
      span.setStatus({ code: SpanStatusCode.ERROR });
      throw err;
    } finally {
      span.end();
    }
  });
}

import {
  BedrockRuntimeClient,
  ConverseCommand,
} from "@aws-sdk/client-bedrock-runtime";

export async function converse(modelId: string, messages: any[]) {
  return tracer.startActiveSpan("gen_ai.chat", async (span) => {
    span.setAttribute("gen_ai.system", "aws.bedrock");
    span.setAttribute("gen_ai.request.model", modelId);
    const started = Date.now();
    const resp = await new BedrockRuntimeClient({}).send(
      new ConverseCommand({
        modelId,
        messages,
        inferenceConfig: { maxTokens: 2048 },
      })
    );
    span.setAttribute("gen_ai.response.model", modelId);
    span.setAttribute("gen_ai.usage.input_tokens", resp.usage?.inputTokens ?? 0);
    span.setAttribute("gen_ai.usage.output_tokens", resp.usage?.outputTokens ?? 0);
    span.setAttribute("gen_ai.latency_ms", Date.now() - started);
    span.end();
    return resp;
  });
}

// Inject W3C headers when enqueueing tool work
const carrier: Record<string, string> = {};
propagation.inject(context.active(), carrier);
await sqs.send({
  QueueUrl,
  MessageBody: JSON.stringify({ args }),
  MessageAttributes: {
    traceparent: { DataType: "String", StringValue: carrier["traceparent"] },
  },
});

# illustrative collector policy
policies:
  - name: errors
    type: status_code
    status_code: ERROR
  - name: slow-agent
    type: latency
    threshold_ms: 5000
  - name: baseline
    type: probabilistic
    sampling_percentage: 5

import { propagation, context } from "@opentelemetry/api";

export function withPrBaggage<T>(pr: number, fn: () => Promise<T>) {
  // Use baggage API in your SDK version; illustrative attribute mirror:
  return tracer.startActiveSpan("ci.agent.job", async (span) => {
    span.setAttribute("ci.pr", pr);
    span.setAttribute("ci.pipeline", process.env.GITHUB_WORKFLOW ?? "local");
    try {
      return await fn();
    } finally {
      span.end();
    }
  });
}