Node Blob and File APIs: Efficient Multipart Upload Paths to S3

// src/upload-route.ts
import { S3Client } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import type { IncomingMessage } from "node:http";
import { Readable } from "node:stream";

const s3 = new S3Client({});

export async function uploadFromRequest(req: IncomingMessage, key: string) {
 const contentType = req.headers["content-type"] ?? "application/octet-stream";
 const upload = new Upload({
 client: s3,
 params: {
 Bucket: process.env.UPLOAD_BUCKET!,
 Key: key,
 Body: Readable.fromWeb(req as any), // or pass Node Readable req directly
 ContentType: contentType,
 },
 queueSize: 4,
 partSize: 8 * 1024 * 1024, // ✅ 8 MiB parts
 leavePartsOnError: false,
 });

 req.on("aborted", () => {
 void upload.abort(); // ✅ don't leak incomplete multiparts
 });

 const result = await upload.done();
 return { etag: result.ETag, key };
}

// ❌ Double-buffer the world
const buf = Buffer.concat(await collect(req)); // OOM
await s3.putObject({ Bucket, Key, Body: buf });

// Worker / edge adapter receiving a Web File
export async function uploadFile(file: File, key: string) {
 // ✅ file.stream() is a ReadableStream — don't arrayBuffer() first
 const upload = new Upload({
 client: s3,
 params: {
 Bucket: process.env.UPLOAD_BUCKET!,
 Key: key,
 Body: Readable.fromWeb(file.stream() as any),
 ContentType: file.type || "application/octet-stream",
 },
 partSize: 16 * 1024 * 1024,
 });
 return upload.done();
}

// Lifecycle rule: abort incomplete multipart after 7 days
// Plus metrics on AbortMultipartUpload count
upload.on("httpUploadProgress", (p) => {
 metrics.gauge("upload.bytes", p.loaded ?? 0);
});