RAG job autoscaling: Autoscaling Retrieval and Embed Jobs

docs → ingest API → SQS → embed workers → vector DB
user queries → retrieve API → (read replicas / separate pool)

# worker.py
import os, time, boto3
sqs = boto3.client("sqs")
MAX_INFLIGHT = int(os.environ.get("MAX_INFLIGHT", "8"))

def loop(queue_url: str):
    inflight = 0
    while True:
        if inflight >= MAX_INFLIGHT:
            time.sleep(0.2)
            continue
        msgs = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=1).get("Messages", [])
        for m in msgs:
            inflight += 1
            try:
                embed_and_upsert(m)
                sqs.delete_message(QueueUrl=queue_url, ReceiptHandle=m["ReceiptHandle"])
            finally:
                inflight -= 1

# target tracking on ApproximateNumberOfMessagesVisible
TargetValue: 50