ADOT OpenTelemetry: Trace Multi-Hop Agent Tool Calls Across Lambda

// ❌ No context on nested Invoke — new trace every hop
await lambda.send(
  new InvokeCommand({
    FunctionName: "agent-tool-runner",
    Payload: Buffer.from(JSON.stringify({ tool: call })),
  })
);

// handler instrumented with OTel API (ADOT layer provides the SDK)
import { trace, context, propagation, SpanStatusCode } from "@opentelemetry/api";

const tracer = trace.getTracer("coding-agent", "1.0.0");

type ToolCall = {
  name: string;
  args: Record<string, unknown>;
  idempotencyKey: string;
  tenantId: string;
  tokensIn?: number;
  tokensOut?: number;
};

export async function runTool(call: ToolCall) {
  return tracer.startActiveSpan(
    `tool.${call.name}`,
    {
      attributes: {
        "faas.name": process.env.AWS_LAMBDA_FUNCTION_NAME!,
        "tool.name": call.name,
        "tool.idempotency_key": call.idempotencyKey,
        "tenant.id": call.tenantId,
        "llm.tokens.in": call.tokensIn ?? 0,
        "llm.tokens.out": call.tokensOut ?? 0,
      },
    },
    async (span) => {
      try {
        const result = await executeTool(call);
        span.setAttribute("tool.ok", true);
        return result;
      } catch (err) {
        span.recordException(err as Error);
        span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) });
        throw err;
      } finally {
        span.end();
      }
    }
  );
}

import { LambdaClient, InvokeCommand } from "@aws-sdk/client-lambda";
import { propagation, context as otelCtx } from "@opentelemetry/api";

const lambda = new LambdaClient({});

export async function invokeToolLambda(functionName: string, toolCall: ToolCall) {
  const carrier: Record<string, string> = {};
  propagation.inject(otelCtx.active(), carrier);

  const payload = {
    toolCall,
    _otel: {
      traceparent: carrier["traceparent"],
      tracestate: carrier["tracestate"],
    },
  };

  return lambda.send(
    new InvokeCommand({
      FunctionName: functionName,
      InvocationType: "RequestResponse",
      Payload: Buffer.from(JSON.stringify(payload)),
    })
  );
}

// Child handler extract
export function extractIncoming(event: { _otel?: { traceparent?: string; tracestate?: string } }) {
  const carrier: Record<string, string> = {};
  if (event._otel?.traceparent) carrier["traceparent"] = event._otel.traceparent;
  if (event._otel?.tracestate) carrier["tracestate"] = event._otel.tracestate;
  return propagation.extract(otelCtx.active(), carrier);
}

// ✅ Child wraps work in extracted context
export async function handler(event: any) {
  const ctx = extractIncoming(event);
  return otelCtx.with(ctx, () => runTool(event.toolCall));
}

// ✅ Annotate Map item with parent context from the state machine input
type MapItem = ToolCall & {
  _otel?: { traceparent?: string; tracestate?: string };
};

export function attachTraceToItems(
  items: ToolCall[],
  carrier: Record<string, string>
): MapItem[] {
  return items.map((item) => ({
    ...item,
    _otel: {
      traceparent: carrier["traceparent"],
      tracestate: carrier["tracestate"],
    },
  }));
}