NumPy Operator Fusion: Stop Temporary Blowups in AWS Batch ETL

# ❌ Each arithmetic step may allocate a full temporary
import numpy as np

def normalize_bad(x: np.ndarray) -> np.ndarray:
    # temps: x.mean, x-mean, std, div, clip...
    return np.clip((x - x.mean(axis=0)) / (x.std(axis=0) + 1e-6), -5, 5)

# ✅ Preallocate + out= + chunking
def normalize(x: np.ndarray, out: np.ndarray | None = None, rows: int = 50_000) -> np.ndarray:
    if out is None:
        out = np.empty_like(x)
    mean = x.mean(axis=0)
    std = x.std(axis=0)
    std += 1e-6
    for i in range(0, len(x), rows):
        sl = slice(i, i + rows)
        np.subtract(x[sl], mean, out=out[sl])
        np.divide(out[sl], std, out=out[sl])
        np.clip(out[sl], -5, 5, out=out[sl])
    return out

# ✅ Allocation delta on a representative shard
import tracemalloc
import numpy as np

def measure(fn, x):
    tracemalloc.start()
    fn(x)
    current, peak = tracemalloc.get_traced_memory()
    tracemalloc.stop()
    return peak

x = np.random.rand(500_000, 256).astype(np.float32)
print("bad_peak", measure(normalize_bad, x))
print("good_peak", measure(lambda a: normalize(a), x))

# AWS Batch: set memory from measured peak * safety factor, not folklore
# ❌ ulimits that “usually work” on the laptop shard