mirror of
https://github.com/openclaw/openclaw.git
synced 2026-02-19 18:39:20 -05:00
fix: harden OpenResponses URL input fetching
This commit is contained in:
@@ -12,6 +12,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
### Fixes
|
||||
|
||||
- Gateway/OpenResponses: harden URL-based `input_file`/`input_image` handling with explicit SSRF deny policy, hostname allowlists (`files.urlAllowlist` / `images.urlAllowlist`), per-request URL input caps (`maxUrlParts`), blocked-fetch audit logging, and regression coverage/docs updates.
|
||||
- Security: fix unauthenticated Nostr profile API remote config tampering. (#13719) Thanks @coygeek.
|
||||
- Security: remove bundled soul-evil hook. (#14757) Thanks @Imccccc.
|
||||
- Security/Sandbox: confine mirrored skill sync destinations to the sandbox `skills/` root and stop using frontmatter-controlled skill names as filesystem destination paths. Thanks @1seal.
|
||||
|
||||
@@ -1934,6 +1934,10 @@ See [Plugins](/tools/plugin).
|
||||
|
||||
- Chat Completions: disabled by default. Enable with `gateway.http.endpoints.chatCompletions.enabled: true`.
|
||||
- Responses API: `gateway.http.endpoints.responses.enabled`.
|
||||
- Responses URL-input hardening:
|
||||
- `gateway.http.endpoints.responses.maxUrlParts`
|
||||
- `gateway.http.endpoints.responses.files.urlAllowlist`
|
||||
- `gateway.http.endpoints.responses.images.urlAllowlist`
|
||||
|
||||
### Multi-instance isolation
|
||||
|
||||
|
||||
@@ -186,7 +186,11 @@ URL fetch defaults:
|
||||
|
||||
- `files.allowUrl`: `true`
|
||||
- `images.allowUrl`: `true`
|
||||
- `maxUrlParts`: `8` (total URL-based `input_file` + `input_image` parts per request)
|
||||
- Requests are guarded (DNS resolution, private IP blocking, redirect caps, timeouts).
|
||||
- Optional hostname allowlists are supported per input type (`files.urlAllowlist`, `images.urlAllowlist`).
|
||||
- Exact host: `"cdn.example.com"`
|
||||
- Wildcard subdomains: `"*.assets.example.com"` (does not match apex)
|
||||
|
||||
## File + image limits (config)
|
||||
|
||||
@@ -200,8 +204,10 @@ Defaults can be tuned under `gateway.http.endpoints.responses`:
|
||||
responses: {
|
||||
enabled: true,
|
||||
maxBodyBytes: 20000000,
|
||||
maxUrlParts: 8,
|
||||
files: {
|
||||
allowUrl: true,
|
||||
urlAllowlist: ["cdn.example.com", "*.assets.example.com"],
|
||||
allowedMimes: [
|
||||
"text/plain",
|
||||
"text/markdown",
|
||||
@@ -222,6 +228,7 @@ Defaults can be tuned under `gateway.http.endpoints.responses`:
|
||||
},
|
||||
images: {
|
||||
allowUrl: true,
|
||||
urlAllowlist: ["images.example.com"],
|
||||
allowedMimes: ["image/jpeg", "image/png", "image/gif", "image/webp"],
|
||||
maxBytes: 10485760,
|
||||
maxRedirects: 3,
|
||||
@@ -237,6 +244,7 @@ Defaults can be tuned under `gateway.http.endpoints.responses`:
|
||||
Defaults when omitted:
|
||||
|
||||
- `maxBodyBytes`: 20MB
|
||||
- `maxUrlParts`: 8
|
||||
- `files.maxBytes`: 5MB
|
||||
- `files.maxChars`: 200k
|
||||
- `files.maxRedirects`: 3
|
||||
@@ -248,6 +256,13 @@ Defaults when omitted:
|
||||
- `images.maxRedirects`: 3
|
||||
- `images.timeoutMs`: 10s
|
||||
|
||||
Security note:
|
||||
|
||||
- URL allowlists are enforced before fetch and on redirect hops.
|
||||
- Allowlisting a hostname does not bypass private/internal IP blocking.
|
||||
- For internet-exposed gateways, apply network egress controls in addition to app-level guards.
|
||||
See [Security](/gateway/security).
|
||||
|
||||
## Streaming (SSE)
|
||||
|
||||
Set `stream: true` to receive Server-Sent Events (SSE):
|
||||
|
||||
@@ -265,6 +265,9 @@ tool calls. Reduce the blast radius by:
|
||||
- Using a read-only or tool-disabled **reader agent** to summarize untrusted content,
|
||||
then pass the summary to your main agent.
|
||||
- Keeping `web_search` / `web_fetch` / `browser` off for tool-enabled agents unless needed.
|
||||
- For OpenResponses URL inputs (`input_file` / `input_image`), set tight
|
||||
`gateway.http.endpoints.responses.files.urlAllowlist` and
|
||||
`gateway.http.endpoints.responses.images.urlAllowlist`, and keep `maxUrlParts` low.
|
||||
- Enabling sandboxing and strict tool allowlists for any agent that touches untrusted input.
|
||||
- Keeping secrets out of prompts; pass them via env/config on the gateway host instead.
|
||||
|
||||
|
||||
@@ -143,6 +143,11 @@ export type GatewayHttpResponsesConfig = {
|
||||
* Default: 20MB.
|
||||
*/
|
||||
maxBodyBytes?: number;
|
||||
/**
|
||||
* Max number of URL-based `input_file` + `input_image` parts per request.
|
||||
* Default: 8.
|
||||
*/
|
||||
maxUrlParts?: number;
|
||||
/** File inputs (input_file). */
|
||||
files?: GatewayHttpResponsesFilesConfig;
|
||||
/** Image inputs (input_image). */
|
||||
@@ -152,6 +157,11 @@ export type GatewayHttpResponsesConfig = {
|
||||
export type GatewayHttpResponsesFilesConfig = {
|
||||
/** Allow URL fetches for input_file. Default: true. */
|
||||
allowUrl?: boolean;
|
||||
/**
|
||||
* Optional hostname allowlist for URL fetches.
|
||||
* Supports exact hosts and `*.example.com` wildcards.
|
||||
*/
|
||||
urlAllowlist?: string[];
|
||||
/** Allowed MIME types (case-insensitive). */
|
||||
allowedMimes?: string[];
|
||||
/** Max bytes per file. Default: 5MB. */
|
||||
@@ -178,6 +188,11 @@ export type GatewayHttpResponsesPdfConfig = {
|
||||
export type GatewayHttpResponsesImagesConfig = {
|
||||
/** Allow URL fetches for input_image. Default: true. */
|
||||
allowUrl?: boolean;
|
||||
/**
|
||||
* Optional hostname allowlist for URL fetches.
|
||||
* Supports exact hosts and `*.example.com` wildcards.
|
||||
*/
|
||||
urlAllowlist?: string[];
|
||||
/** Allowed MIME types (case-insensitive). */
|
||||
allowedMimes?: string[];
|
||||
/** Max bytes per image. Default: 10MB. */
|
||||
|
||||
@@ -457,9 +457,11 @@ export const OpenClawSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
maxBodyBytes: z.number().int().positive().optional(),
|
||||
maxUrlParts: z.number().int().nonnegative().optional(),
|
||||
files: z
|
||||
.object({
|
||||
allowUrl: z.boolean().optional(),
|
||||
urlAllowlist: z.array(z.string()).optional(),
|
||||
allowedMimes: z.array(z.string()).optional(),
|
||||
maxBytes: z.number().int().positive().optional(),
|
||||
maxChars: z.number().int().positive().optional(),
|
||||
@@ -479,6 +481,7 @@ export const OpenClawSchema = z
|
||||
images: z
|
||||
.object({
|
||||
allowUrl: z.boolean().optional(),
|
||||
urlAllowlist: z.array(z.string()).optional(),
|
||||
allowedMimes: z.array(z.string()).optional(),
|
||||
maxBytes: z.number().int().positive().optional(),
|
||||
maxRedirects: z.number().int().nonnegative().optional(),
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { HISTORY_CONTEXT_MARKER } from "../auto-reply/reply/history.js";
|
||||
import { CURRENT_MESSAGE_MARKER } from "../auto-reply/reply/mentions.js";
|
||||
@@ -37,6 +39,15 @@ async function startServer(port: number, opts?: { openResponsesEnabled?: boolean
|
||||
});
|
||||
}
|
||||
|
||||
async function writeGatewayConfig(config: Record<string, unknown>) {
|
||||
const configPath = process.env.OPENCLAW_CONFIG_PATH;
|
||||
if (!configPath) {
|
||||
throw new Error("OPENCLAW_CONFIG_PATH is required for gateway config tests");
|
||||
}
|
||||
await fs.mkdir(path.dirname(configPath), { recursive: true });
|
||||
await fs.writeFile(configPath, JSON.stringify(config, null, 2), "utf-8");
|
||||
}
|
||||
|
||||
async function postResponses(port: number, body: unknown, headers?: Record<string, string>) {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/v1/responses`, {
|
||||
method: "POST",
|
||||
@@ -504,4 +515,187 @@ describe("OpenResponses HTTP API (e2e)", () => {
|
||||
// shared server
|
||||
}
|
||||
});
|
||||
|
||||
it("blocks unsafe URL-based file/image inputs", async () => {
|
||||
const port = enabledPort;
|
||||
agentCommand.mockReset();
|
||||
|
||||
const blockedPrivate = await postResponses(port, {
|
||||
model: "openclaw",
|
||||
input: [
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "input_text", text: "read this" },
|
||||
{
|
||||
type: "input_file",
|
||||
source: { type: "url", url: "http://127.0.0.1:6379/info" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(blockedPrivate.status).toBe(400);
|
||||
const blockedPrivateJson = (await blockedPrivate.json()) as {
|
||||
error?: { type?: string; message?: string };
|
||||
};
|
||||
expect(blockedPrivateJson.error?.type).toBe("invalid_request_error");
|
||||
expect(blockedPrivateJson.error?.message ?? "").toMatch(/private|internal|blocked/i);
|
||||
|
||||
const blockedMetadata = await postResponses(port, {
|
||||
model: "openclaw",
|
||||
input: [
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "input_text", text: "read this" },
|
||||
{
|
||||
type: "input_image",
|
||||
source: { type: "url", url: "http://metadata.google.internal/computeMetadata/v1" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(blockedMetadata.status).toBe(400);
|
||||
const blockedMetadataJson = (await blockedMetadata.json()) as {
|
||||
error?: { type?: string; message?: string };
|
||||
};
|
||||
expect(blockedMetadataJson.error?.type).toBe("invalid_request_error");
|
||||
expect(blockedMetadataJson.error?.message ?? "").toMatch(/blocked|metadata|internal/i);
|
||||
|
||||
const blockedScheme = await postResponses(port, {
|
||||
model: "openclaw",
|
||||
input: [
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "input_text", text: "read this" },
|
||||
{
|
||||
type: "input_file",
|
||||
source: { type: "url", url: "file:///etc/passwd" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(blockedScheme.status).toBe(400);
|
||||
const blockedSchemeJson = (await blockedScheme.json()) as {
|
||||
error?: { type?: string; message?: string };
|
||||
};
|
||||
expect(blockedSchemeJson.error?.type).toBe("invalid_request_error");
|
||||
expect(blockedSchemeJson.error?.message ?? "").toMatch(/http or https/i);
|
||||
expect(agentCommand).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("enforces URL allowlist and URL part cap for responses inputs", async () => {
|
||||
const allowlistConfig = {
|
||||
gateway: {
|
||||
http: {
|
||||
endpoints: {
|
||||
responses: {
|
||||
enabled: true,
|
||||
maxUrlParts: 1,
|
||||
files: {
|
||||
allowUrl: true,
|
||||
urlAllowlist: ["cdn.example.com", "*.assets.example.com"],
|
||||
},
|
||||
images: {
|
||||
allowUrl: true,
|
||||
urlAllowlist: ["images.example.com"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
await writeGatewayConfig(allowlistConfig);
|
||||
|
||||
const allowlistPort = await getFreePort();
|
||||
const allowlistServer = await startServer(allowlistPort, { openResponsesEnabled: true });
|
||||
try {
|
||||
agentCommand.mockReset();
|
||||
|
||||
const allowlistBlocked = await postResponses(allowlistPort, {
|
||||
model: "openclaw",
|
||||
input: [
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "input_text", text: "fetch this" },
|
||||
{
|
||||
type: "input_file",
|
||||
source: { type: "url", url: "https://evil.example.org/secret.txt" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(allowlistBlocked.status).toBe(400);
|
||||
const allowlistBlockedJson = (await allowlistBlocked.json()) as {
|
||||
error?: { type?: string; message?: string };
|
||||
};
|
||||
expect(allowlistBlockedJson.error?.type).toBe("invalid_request_error");
|
||||
expect(allowlistBlockedJson.error?.message ?? "").toMatch(/allowlist|blocked/i);
|
||||
} finally {
|
||||
await allowlistServer.close({ reason: "responses allowlist hardening test done" });
|
||||
}
|
||||
|
||||
const capConfig = {
|
||||
gateway: {
|
||||
http: {
|
||||
endpoints: {
|
||||
responses: {
|
||||
enabled: true,
|
||||
maxUrlParts: 0,
|
||||
files: {
|
||||
allowUrl: true,
|
||||
urlAllowlist: ["cdn.example.com", "*.assets.example.com"],
|
||||
},
|
||||
images: {
|
||||
allowUrl: true,
|
||||
urlAllowlist: ["images.example.com"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
await writeGatewayConfig(capConfig);
|
||||
|
||||
const capPort = await getFreePort();
|
||||
const capServer = await startServer(capPort, { openResponsesEnabled: true });
|
||||
try {
|
||||
agentCommand.mockReset();
|
||||
const maxUrlBlocked = await postResponses(capPort, {
|
||||
model: "openclaw",
|
||||
input: [
|
||||
{
|
||||
type: "message",
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "input_text", text: "fetch this" },
|
||||
{
|
||||
type: "input_file",
|
||||
source: { type: "url", url: "https://cdn.example.com/file-1.txt" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(maxUrlBlocked.status).toBe(400);
|
||||
const maxUrlBlockedJson = (await maxUrlBlocked.json()) as {
|
||||
error?: { type?: string; message?: string };
|
||||
};
|
||||
expect(maxUrlBlockedJson.error?.type).toBe("invalid_request_error");
|
||||
expect(maxUrlBlockedJson.error?.message ?? "").toMatch(/Too many URL-based input sources/i);
|
||||
expect(agentCommand).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await capServer.close({ reason: "responses url cap hardening test done" });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -63,6 +63,7 @@ type OpenResponsesHttpOptions = {
|
||||
};
|
||||
|
||||
const DEFAULT_BODY_BYTES = 20 * 1024 * 1024;
|
||||
const DEFAULT_MAX_URL_PARTS = 8;
|
||||
|
||||
function writeSseEvent(res: ServerResponse, event: StreamingEvent) {
|
||||
res.write(`event: ${event.type}\n`);
|
||||
@@ -89,10 +90,19 @@ function extractTextContent(content: string | ContentPart[]): string {
|
||||
|
||||
type ResolvedResponsesLimits = {
|
||||
maxBodyBytes: number;
|
||||
maxUrlParts: number;
|
||||
files: InputFileLimits;
|
||||
images: InputImageLimits;
|
||||
};
|
||||
|
||||
function normalizeHostnameAllowlist(values: string[] | undefined): string[] | undefined {
|
||||
if (!values || values.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = values.map((value) => value.trim()).filter((value) => value.length > 0);
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function resolveResponsesLimits(
|
||||
config: GatewayHttpResponsesConfig | undefined,
|
||||
): ResolvedResponsesLimits {
|
||||
@@ -100,8 +110,13 @@ function resolveResponsesLimits(
|
||||
const images = config?.images;
|
||||
return {
|
||||
maxBodyBytes: config?.maxBodyBytes ?? DEFAULT_BODY_BYTES,
|
||||
maxUrlParts:
|
||||
typeof config?.maxUrlParts === "number"
|
||||
? Math.max(0, Math.floor(config.maxUrlParts))
|
||||
: DEFAULT_MAX_URL_PARTS,
|
||||
files: {
|
||||
allowUrl: files?.allowUrl ?? true,
|
||||
urlAllowlist: normalizeHostnameAllowlist(files?.urlAllowlist),
|
||||
allowedMimes: normalizeMimeList(files?.allowedMimes, DEFAULT_INPUT_FILE_MIMES),
|
||||
maxBytes: files?.maxBytes ?? DEFAULT_INPUT_FILE_MAX_BYTES,
|
||||
maxChars: files?.maxChars ?? DEFAULT_INPUT_FILE_MAX_CHARS,
|
||||
@@ -115,6 +130,7 @@ function resolveResponsesLimits(
|
||||
},
|
||||
images: {
|
||||
allowUrl: images?.allowUrl ?? true,
|
||||
urlAllowlist: normalizeHostnameAllowlist(images?.urlAllowlist),
|
||||
allowedMimes: normalizeMimeList(images?.allowedMimes, DEFAULT_INPUT_IMAGE_MIMES),
|
||||
maxBytes: images?.maxBytes ?? DEFAULT_INPUT_IMAGE_MAX_BYTES,
|
||||
maxRedirects: images?.maxRedirects ?? DEFAULT_INPUT_MAX_REDIRECTS,
|
||||
@@ -384,6 +400,15 @@ export async function handleOpenResponsesHttpRequest(
|
||||
// Extract images + files from input (Phase 2)
|
||||
let images: ImageContent[] = [];
|
||||
let fileContexts: string[] = [];
|
||||
let urlParts = 0;
|
||||
const markUrlPart = () => {
|
||||
urlParts += 1;
|
||||
if (urlParts > limits.maxUrlParts) {
|
||||
throw new Error(
|
||||
`Too many URL-based input sources: ${urlParts} (limit: ${limits.maxUrlParts})`,
|
||||
);
|
||||
}
|
||||
};
|
||||
try {
|
||||
if (Array.isArray(payload.input)) {
|
||||
for (const item of payload.input) {
|
||||
@@ -401,6 +426,9 @@ export async function handleOpenResponsesHttpRequest(
|
||||
if (!sourceType) {
|
||||
throw new Error("input_image must have 'source.url' or 'source.data'");
|
||||
}
|
||||
if (sourceType === "url") {
|
||||
markUrlPart();
|
||||
}
|
||||
const imageSource: InputImageSource = {
|
||||
type: sourceType,
|
||||
url: source.url,
|
||||
@@ -425,6 +453,9 @@ export async function handleOpenResponsesHttpRequest(
|
||||
if (!sourceType) {
|
||||
throw new Error("input_file must have 'source.url' or 'source.data'");
|
||||
}
|
||||
if (sourceType === "url") {
|
||||
markUrlPart();
|
||||
}
|
||||
const file = await extractFileContentFromSource({
|
||||
source: {
|
||||
type: sourceType,
|
||||
|
||||
63
src/infra/net/fetch-guard.ssrf.test.ts
Normal file
63
src/infra/net/fetch-guard.ssrf.test.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fetchWithSsrFGuard } from "./fetch-guard.js";
|
||||
|
||||
function redirectResponse(location: string): Response {
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: { location },
|
||||
});
|
||||
}
|
||||
|
||||
describe("fetchWithSsrFGuard hardening", () => {
|
||||
it("blocks private IP literal URLs before fetch", async () => {
|
||||
const fetchImpl = vi.fn();
|
||||
await expect(
|
||||
fetchWithSsrFGuard({
|
||||
url: "http://127.0.0.1:8080/internal",
|
||||
fetchImpl,
|
||||
}),
|
||||
).rejects.toThrow(/private|internal|blocked/i);
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("blocks redirect chains that hop to private hosts", async () => {
|
||||
const lookupFn = vi.fn(async () => [{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchImpl = vi.fn().mockResolvedValueOnce(redirectResponse("http://127.0.0.1:6379/"));
|
||||
|
||||
await expect(
|
||||
fetchWithSsrFGuard({
|
||||
url: "https://public.example/start",
|
||||
fetchImpl,
|
||||
lookupFn,
|
||||
}),
|
||||
).rejects.toThrow(/private|internal|blocked/i);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("enforces hostname allowlist policies", async () => {
|
||||
const fetchImpl = vi.fn();
|
||||
await expect(
|
||||
fetchWithSsrFGuard({
|
||||
url: "https://evil.example.org/file.txt",
|
||||
fetchImpl,
|
||||
policy: { hostnameAllowlist: ["cdn.example.com", "*.assets.example.com"] },
|
||||
}),
|
||||
).rejects.toThrow(/allowlist/i);
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows wildcard allowlisted hosts", async () => {
|
||||
const lookupFn = vi.fn(async () => [{ address: "93.184.216.34", family: 4 }]);
|
||||
const fetchImpl = vi.fn(async () => new Response("ok", { status: 200 }));
|
||||
const result = await fetchWithSsrFGuard({
|
||||
url: "https://img.assets.example.com/pic.png",
|
||||
fetchImpl,
|
||||
lookupFn,
|
||||
policy: { hostnameAllowlist: ["*.assets.example.com"] },
|
||||
});
|
||||
|
||||
expect(result.response.status).toBe(200);
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(1);
|
||||
await result.release();
|
||||
});
|
||||
});
|
||||
@@ -1,10 +1,11 @@
|
||||
import type { Dispatcher } from "undici";
|
||||
import { logWarn } from "../../logger.js";
|
||||
import {
|
||||
closeDispatcher,
|
||||
createPinnedDispatcher,
|
||||
resolvePinnedHostname,
|
||||
resolvePinnedHostnameWithPolicy,
|
||||
type LookupFn,
|
||||
SsrFBlockedError,
|
||||
type SsrFPolicy,
|
||||
} from "./ssrf.js";
|
||||
|
||||
@@ -20,6 +21,7 @@ export type GuardedFetchOptions = {
|
||||
policy?: SsrFPolicy;
|
||||
lookupFn?: LookupFn;
|
||||
pinDns?: boolean;
|
||||
auditContext?: string;
|
||||
};
|
||||
|
||||
export type GuardedFetchResult = {
|
||||
@@ -113,15 +115,10 @@ export async function fetchWithSsrFGuard(params: GuardedFetchOptions): Promise<G
|
||||
|
||||
let dispatcher: Dispatcher | null = null;
|
||||
try {
|
||||
const usePolicy = Boolean(
|
||||
params.policy?.allowPrivateNetwork || params.policy?.allowedHostnames?.length,
|
||||
);
|
||||
const pinned = usePolicy
|
||||
? await resolvePinnedHostnameWithPolicy(parsedUrl.hostname, {
|
||||
lookupFn: params.lookupFn,
|
||||
policy: params.policy,
|
||||
})
|
||||
: await resolvePinnedHostname(parsedUrl.hostname, params.lookupFn);
|
||||
const pinned = await resolvePinnedHostnameWithPolicy(parsedUrl.hostname, {
|
||||
lookupFn: params.lookupFn,
|
||||
policy: params.policy,
|
||||
});
|
||||
if (params.pinDns !== false) {
|
||||
dispatcher = createPinnedDispatcher(pinned);
|
||||
}
|
||||
@@ -164,6 +161,12 @@ export async function fetchWithSsrFGuard(params: GuardedFetchOptions): Promise<G
|
||||
release: async () => release(dispatcher),
|
||||
};
|
||||
} catch (err) {
|
||||
if (err instanceof SsrFBlockedError) {
|
||||
const context = params.auditContext ?? "url-fetch";
|
||||
logWarn(
|
||||
`security: blocked URL fetch (${context}) target=${parsedUrl.origin}${parsedUrl.pathname} reason=${err.message}`,
|
||||
);
|
||||
}
|
||||
await release(dispatcher);
|
||||
throw err;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createPinnedLookup, resolvePinnedHostname } from "./ssrf.js";
|
||||
import {
|
||||
createPinnedLookup,
|
||||
resolvePinnedHostname,
|
||||
resolvePinnedHostnameWithPolicy,
|
||||
} from "./ssrf.js";
|
||||
|
||||
describe("ssrf pinning", () => {
|
||||
it("pins resolved addresses for the target hostname", async () => {
|
||||
@@ -68,4 +72,34 @@ describe("ssrf pinning", () => {
|
||||
expect(fallback).toHaveBeenCalledTimes(1);
|
||||
expect(result.address).toBe("1.2.3.4");
|
||||
});
|
||||
|
||||
it("enforces hostname allowlist when configured", async () => {
|
||||
const lookup = vi.fn(async () => [{ address: "93.184.216.34", family: 4 }]);
|
||||
|
||||
await expect(
|
||||
resolvePinnedHostnameWithPolicy("api.example.com", {
|
||||
lookupFn: lookup,
|
||||
policy: { hostnameAllowlist: ["cdn.example.com", "*.trusted.example"] },
|
||||
}),
|
||||
).rejects.toThrow(/allowlist/i);
|
||||
expect(lookup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("supports wildcard hostname allowlist patterns", async () => {
|
||||
const lookup = vi.fn(async () => [{ address: "93.184.216.34", family: 4 }]);
|
||||
|
||||
await expect(
|
||||
resolvePinnedHostnameWithPolicy("assets.example.com", {
|
||||
lookupFn: lookup,
|
||||
policy: { hostnameAllowlist: ["*.example.com"] },
|
||||
}),
|
||||
).resolves.toMatchObject({ hostname: "assets.example.com" });
|
||||
|
||||
await expect(
|
||||
resolvePinnedHostnameWithPolicy("example.com", {
|
||||
lookupFn: lookup,
|
||||
policy: { hostnameAllowlist: ["*.example.com"] },
|
||||
}),
|
||||
).rejects.toThrow(/allowlist/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@ export type LookupFn = typeof dnsLookup;
|
||||
export type SsrFPolicy = {
|
||||
allowPrivateNetwork?: boolean;
|
||||
allowedHostnames?: string[];
|
||||
hostnameAllowlist?: string[];
|
||||
};
|
||||
|
||||
const PRIVATE_IPV6_PREFIXES = ["fe80:", "fec0:", "fc", "fd"];
|
||||
@@ -40,6 +41,37 @@ function normalizeHostnameSet(values?: string[]): Set<string> {
|
||||
return new Set(values.map((value) => normalizeHostname(value)).filter(Boolean));
|
||||
}
|
||||
|
||||
function normalizeHostnameAllowlist(values?: string[]): string[] {
|
||||
if (!values || values.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return Array.from(
|
||||
new Set(
|
||||
values
|
||||
.map((value) => normalizeHostname(value))
|
||||
.filter((value) => value !== "*" && value !== "*." && value.length > 0),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function isHostnameAllowedByPattern(hostname: string, pattern: string): boolean {
|
||||
if (pattern.startsWith("*.")) {
|
||||
const suffix = pattern.slice(2);
|
||||
if (!suffix || hostname === suffix) {
|
||||
return false;
|
||||
}
|
||||
return hostname.endsWith(`.${suffix}`);
|
||||
}
|
||||
return hostname === pattern;
|
||||
}
|
||||
|
||||
function matchesHostnameAllowlist(hostname: string, allowlist: string[]): boolean {
|
||||
if (allowlist.length === 0) {
|
||||
return true;
|
||||
}
|
||||
return allowlist.some((pattern) => isHostnameAllowedByPattern(hostname, pattern));
|
||||
}
|
||||
|
||||
function parseIpv4(address: string): number[] | null {
|
||||
const parts = address.split(".");
|
||||
if (parts.length !== 4) {
|
||||
@@ -229,8 +261,13 @@ export async function resolvePinnedHostnameWithPolicy(
|
||||
|
||||
const allowPrivateNetwork = Boolean(params.policy?.allowPrivateNetwork);
|
||||
const allowedHostnames = normalizeHostnameSet(params.policy?.allowedHostnames);
|
||||
const hostnameAllowlist = normalizeHostnameAllowlist(params.policy?.hostnameAllowlist);
|
||||
const isExplicitAllowed = allowedHostnames.has(normalized);
|
||||
|
||||
if (!matchesHostnameAllowlist(normalized, hostnameAllowlist)) {
|
||||
throw new SsrFBlockedError(`Blocked hostname (not in allowlist): ${hostname}`);
|
||||
}
|
||||
|
||||
if (!allowPrivateNetwork && !isExplicitAllowed) {
|
||||
if (isBlockedHostname(normalized)) {
|
||||
throw new SsrFBlockedError(`Blocked hostname: ${hostname}`);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { SsrFPolicy } from "../infra/net/ssrf.js";
|
||||
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
|
||||
import { logWarn } from "../logger.js";
|
||||
|
||||
@@ -52,6 +53,7 @@ export type InputPdfLimits = {
|
||||
|
||||
export type InputFileLimits = {
|
||||
allowUrl: boolean;
|
||||
urlAllowlist?: string[];
|
||||
allowedMimes: Set<string>;
|
||||
maxBytes: number;
|
||||
maxChars: number;
|
||||
@@ -62,6 +64,7 @@ export type InputFileLimits = {
|
||||
|
||||
export type InputImageLimits = {
|
||||
allowUrl: boolean;
|
||||
urlAllowlist?: string[];
|
||||
allowedMimes: Set<string>;
|
||||
maxBytes: number;
|
||||
maxRedirects: number;
|
||||
@@ -141,11 +144,15 @@ export async function fetchWithGuard(params: {
|
||||
maxBytes: number;
|
||||
timeoutMs: number;
|
||||
maxRedirects: number;
|
||||
policy?: SsrFPolicy;
|
||||
auditContext?: string;
|
||||
}): Promise<InputFetchResult> {
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: params.url,
|
||||
maxRedirects: params.maxRedirects,
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
auditContext: params.auditContext,
|
||||
init: { headers: { "User-Agent": "OpenClaw-Gateway/1.0" } },
|
||||
});
|
||||
|
||||
@@ -283,6 +290,11 @@ export async function extractImageContentFromSource(
|
||||
maxBytes: limits.maxBytes,
|
||||
timeoutMs: limits.timeoutMs,
|
||||
maxRedirects: limits.maxRedirects,
|
||||
policy: {
|
||||
allowPrivateNetwork: false,
|
||||
hostnameAllowlist: limits.urlAllowlist,
|
||||
},
|
||||
auditContext: "openresponses.input_image",
|
||||
});
|
||||
if (!limits.allowedMimes.has(result.mimeType)) {
|
||||
throw new Error(`Unsupported image MIME type from URL: ${result.mimeType}`);
|
||||
@@ -321,6 +333,11 @@ export async function extractFileContentFromSource(params: {
|
||||
maxBytes: limits.maxBytes,
|
||||
timeoutMs: limits.timeoutMs,
|
||||
maxRedirects: limits.maxRedirects,
|
||||
policy: {
|
||||
allowPrivateNetwork: false,
|
||||
hostnameAllowlist: limits.urlAllowlist,
|
||||
},
|
||||
auditContext: "openresponses.input_file",
|
||||
});
|
||||
const parsed = parseContentType(result.contentType);
|
||||
mimeType = parsed.mimeType ?? normalizeMimeType(result.mimeType);
|
||||
|
||||
Reference in New Issue
Block a user