github/openclaw
1
1
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-02-19 18:39:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor(media): share response size limiter

Browse Source
This commit is contained in:
Peter Steinberger
2026-02-15 05:01:05 +00:00
parent 7d89bebc4f
commit b289441e6f
3 changed files with 61 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/media/fetch.ts
View File

@@ -2,6 +2,7 @@ import path from "node:path";
import type { LookupFn, SsrFPolicy } from "../infra/net/ssrf.js";
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
import { detectMime, extensionForMime } from "./mime.js";
import { readResponseWithLimit } from "./read-response-with-limit.js";
type FetchMediaResult = {
buffer: Buffer;
@@ -129,7 +130,13 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise<Fetc
}
const buffer = maxBytes
? await readResponseWithLimit(res, maxBytes)
? await readResponseWithLimit(res, maxBytes, {
onOverflow: ({ maxBytes, res }) =>
new MediaFetchError(
"max_bytes",
`Failed to fetch media from ${res.url || url}: payload exceeds maxBytes ${maxBytes}`,
),
})
: Buffer.from(await res.arrayBuffer());
let fileNameFromUrl: string | undefined;
try {
@@ -169,51 +176,3 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise<Fetc
}
}
}
async function readResponseWithLimit(res: Response, maxBytes: number): Promise<Buffer> {
const body = res.body;
if (!body || typeof body.getReader !== "function") {
const fallback = Buffer.from(await res.arrayBuffer());
if (fallback.length > maxBytes) {
throw new MediaFetchError(
"max_bytes",
`Failed to fetch media from ${res.url || "response"}: payload exceeds maxBytes ${maxBytes}`,
);
}
return fallback;
}
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
if (value?.length) {
total += value.length;
if (total > maxBytes) {
try {
await reader.cancel();
} catch {}
throw new MediaFetchError(
"max_bytes",
`Failed to fetch media from ${res.url || "response"}: payload exceeds maxBytes ${maxBytes}`,
);
}
chunks.push(value);
}
}
} finally {
try {
reader.releaseLock();
} catch {}
}
return Buffer.concat(
chunks.map((chunk) => Buffer.from(chunk)),
total,
);
}

43
src/media/input-files.ts
View File

@@ -2,6 +2,7 @@ import type { SsrFPolicy } from "../infra/net/ssrf.js";
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
import { logWarn } from "../logger.js";
import { estimateBase64DecodedBytes } from "./base64.js";
import { readResponseWithLimit } from "./read-response-with-limit.js";
type CanvasModule = typeof import("@napi-rs/canvas");
type PdfJsModule = typeof import("pdfjs-dist/legacy/build/pdf.mjs");
@@ -194,48 +195,6 @@ export async function fetchWithGuard(params: {
}
}
async function readResponseWithLimit(res: Response, maxBytes: number): Promise<Buffer> {
const body = res.body;
if (!body || typeof body.getReader !== "function") {
const fallback = Buffer.from(await res.arrayBuffer());
if (fallback.byteLength > maxBytes) {
throw new Error(`Content too large: ${fallback.byteLength} bytes (limit: ${maxBytes} bytes)`);
}
return fallback;
}
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
if (value?.length) {
total += value.length;
if (total > maxBytes) {
try {
await reader.cancel();
} catch {}
throw new Error(`Content too large: ${total} bytes (limit: ${maxBytes} bytes)`);
}
chunks.push(value);
}
}
} finally {
try {
reader.releaseLock();
} catch {}
}
return Buffer.concat(
chunks.map((chunk) => Buffer.from(chunk)),
total,
);
}
function decodeTextContent(buffer: Buffer, charset: string | undefined): string {
const encoding = charset?.trim().toLowerCase() || "utf-8";
try {

52
src/media/read-response-with-limit.ts Normal file
View File

@@ -0,0 +1,52 @@
export async function readResponseWithLimit(
res: Response,
maxBytes: number,
opts?: {
onOverflow?: (params: { size: number; maxBytes: number; res: Response }) => Error;
},
): Promise<Buffer> {
const onOverflow =
opts?.onOverflow ??
((params: { size: number; maxBytes: number }) =>
new Error(`Content too large: ${params.size} bytes (limit: ${params.maxBytes} bytes)`));
const body = res.body;
if (!body || typeof body.getReader !== "function") {
const fallback = Buffer.from(await res.arrayBuffer());
if (fallback.length > maxBytes) {
throw onOverflow({ size: fallback.length, maxBytes, res });
}
return fallback;
}
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
if (value?.length) {
total += value.length;
if (total > maxBytes) {
try {
await reader.cancel();
} catch {}
throw onOverflow({ size: total, maxBytes, res });
}
chunks.push(value);
}
}
} finally {
try {
reader.releaseLock();
} catch {}
}
return Buffer.concat(
chunks.map((chunk) => Buffer.from(chunk)),
total,
);
}