Python Deadlock Debugging: faulthandler Plus gdb for Wedged Workers

# ✅ boot path in gunicorn/uvicorn worker
import faulthandler
import signal
import sys

faulthandler.enable(file=sys.stderr, all_threads=True)
# Dump all threads on SIGUSR1 without killing the process
faulthandler.register(signal.SIGUSR1, file=sys.stderr, all_threads=True)
# Optional: periodic dump every 5 minutes while debugging a fleet
# faulthandler.dump_traceback_later(300, repeat=True, file=sys.stderr)

# From the host / sidecar when readiness fails
kill -USR1 $(pgrep -f "uvicorn.*worker")
# stderr now has every Python thread stack

# ✅ gdb against the wedged PID (ECS exec / privileged debug sidecar)
gdb -p "$PID" -batch \
 -ex "set pagination off" \
 -ex "thread apply all bt" \
 -ex "thread apply all py-bt" \
 -ex "detach" -ex "quit"

# Reproduce classic extension + GIL hazard in staging
import threading
lock_a = threading.Lock()
lock_b = threading.Lock()

def t1():
 with lock_a:
 with lock_b:
 pass

def t2():
 with lock_b:
 with lock_a: # ❌ inverted order → deadlock under race
 pass