Lambda Timeouts Retries: Idempotent Failure Handling

// handler.ts — explicit budgets beat hope
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
import { NodeHttpHandler } from "@smithy/node-http-handler";

// Function timeout in console/Terraform: 8s for this API path (illustrative)
const ddb = new DynamoDBClient({
  requestHandler: new NodeHttpHandler({
    connectionTimeout: 1_000,
    requestTimeout: 2_500, // ✅ well under 8s function timeout
  }),
  maxAttempts: 2, // ✅ finite; don’t stack SDK retries × async retries blindly
});

export async function handler(event: { orderId: string }) {
  const ac = new AbortController();
  const kill = setTimeout(() => ac.abort(), 6_000); // leave ~2s for cleanup/logging
  try {
    // pass ac.signal into fetch / ORM calls
    const res = await fetch(process.env.PRICING_URL! + "/" + event.orderId, {
      signal: ac.signal,
    });
    if (!res.ok) throw new Error(`pricing_${res.status}`);
    return { ok: true };
  } finally {
    clearTimeout(kill);
  }
}

// SQS partial batch failure — Node 20 / AWS Lambda
import type { SQSBatchResponse, SQSEvent } from "aws-lambda";

export async function handler(event: SQSEvent): Promise<SQSBatchResponse> {
  const batchItemFailures: { itemIdentifier: string }[] = [];

  await Promise.all(
    event.Records.map(async (record) => {
      try {
        await processOnce(record.body, record.messageId);
      } catch (err) {
        console.error("record_failed", { id: record.messageId, err: String(err) });
        // ✅ only failing ids retry; successes delete
        batchItemFailures.push({ itemIdentifier: record.messageId });
      }
    })
  );

  return { batchItemFailures };
}

// ❌ throw on first failure with ReportBatchItemFailures off → whole batch retries → dupes

resource "aws_sqs_queue" "orders" {
  name                      = "orders"
  visibility_timeout_seconds = 60 # ≥ 6× Lambda timeout is a common rule of thumb
  redrive_policy = jsonencode({
    deadLetterTargetArn = aws_sqs_queue.orders_dlq.arn
    maxReceiveCount     = 3
  })
}

resource "aws_sqs_queue" "orders_dlq" {
  name = "orders-dlq"
}

// idempotency with DynamoDB conditional put — unfair-advantage minimal version
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
import {
  DynamoDBDocumentClient,
  PutCommand,
  GetCommand,
} from "@aws-sdk/lib-dynamodb";

const doc = DynamoDBDocumentClient.from(new DynamoDBClient({}));
const TABLE = process.env.IDEMPOTENCY_TABLE!;

export async function processOnce(body: string, messageId: string) {
  const payload = JSON.parse(body) as { orderId: string; amountCents: number };
  const pk = `ORDER#${payload.orderId}`;

  try {
    await doc.send(
      new PutCommand({
        TableName: TABLE,
        Item: {
          pk,
          status: "IN_FLIGHT",
          messageId,
          ttl: Math.floor(Date.now() / 1000) + 86400,
        },
        ConditionExpression: "attribute_not_exists(pk) OR #s = :failed",
        ExpressionAttributeNames: { "#s": "status" },
        ExpressionAttributeValues: { ":failed": "FAILED" },
      })
    );
  } catch (e: any) {
    if (e?.name === "ConditionalCheckFailedException") {
      const existing = await doc.send(new GetCommand({ TableName: TABLE, Key: { pk } }));
      if (existing.Item?.status === "COMPLETED") {
        // ✅ safe replay
        return existing.Item.result;
      }
      throw e; // still in flight — let SQS retry with backoff
    }
    throw e;
  }

  const result = await chargeCard(payload); // side effect
  await doc.send(
    new PutCommand({
      TableName: TABLE,
      Item: { pk, status: "COMPLETED", result, messageId, ttl: Math.floor(Date.now() / 1000) + 86400 },
    })
  );
  return result;
}

// CloudWatch alarm sketch (conceptual)
// ApproximateNumberOfMessagesVisible > 0 for 5 minutes on orders-dlq → page

// Replay tool — intentional, rate-limited, idempotent handler still required
import { SQSClient, ReceiveMessageCommand, SendMessageCommand, DeleteMessageCommand } from "@aws-sdk/client-sqs";

const sqs = new SQSClient({});

export async function replayDlq(dlqUrl: string, mainUrl: string, max = 10) {
  const got = await sqs.send(new ReceiveMessageCommand({
    QueueUrl: dlqUrl,
    MaxNumberOfMessages: max,
    VisibilityTimeout: 30,
  }));
  for (const m of got.Messages ?? []) {
    await sqs.send(new SendMessageCommand({ QueueUrl: mainUrl, MessageBody: m.Body! }));
    await sqs.send(new DeleteMessageCommand({ QueueUrl: dlqUrl, ReceiptHandle: m.ReceiptHandle! }));
  }
}

aws lambda put-function-event-invoke-config \
  --function-name fulfill-order \
  --maximum-retry-attempts 2 \
  --destination-config '{"OnFailure":{"Destination":"arn:aws:sqs:REGION:ACCT:fulfill-dlq"}}'

import { Logger } from "@aws-lambda-powertools/logger";
import { Metrics, MetricUnit } from "@aws-lambda-powertools/metrics";

const logger = new Logger({ serviceName: "orders" });
const metrics = new Metrics({ namespace: "CheatCoders/Orders", serviceName: "orders" });

export async function guarded<T>(name: string, fn: () => Promise<T>): Promise<T> {
  const start = Date.now();
  try {
    const out = await fn();
    metrics.addMetric(`${name}Success`, MetricUnit.Count, 1);
    return out;
  } catch (err: any) {
    const timedOut = err?.name === "AbortError" || /Timeout/i.test(String(err));
    metrics.addMetric(timedOut ? `${name}Timeout` : `${name}Error`, MetricUnit.Count, 1);
    logger.error("downstream_failed", { name, timedOut, err: String(err), ms: Date.now() - start });
    throw err;
  } finally {
    metrics.publishStoredMetrics();
  }
}