github/openclaw
1
1
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-03 03:03:24 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(test): share web fetch e2e setup helpers

Browse Source
This commit is contained in:
Peter Steinberger
2026-02-16 18:09:45 +00:00
parent 9c6e879a06
commit 74c49c943d
1 changed files with 80 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
src/agents/tools/web-tools.fetch.e2e.test.ts
View File

@@ -90,6 +90,41 @@ function requestUrl(input: RequestInfo): string {
return "";
}
function installMockFetch(impl: (input: RequestInfo) => Promise<Response>) {
const mockFetch = vi.fn(impl);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
return mockFetch;
}
function createFetchTool(fetchOverrides: Record<string, unknown> = {}) {
return createWebFetchTool({
config: {
tools: {
web: {
fetch: {
cacheTtlMinutes: 0,
...fetchOverrides,
},
},
},
},
sandboxed: false,
});
}
async function captureToolErrorMessage(params: {
tool: ReturnType<typeof createWebFetchTool>;
url: string;
}) {
try {
await params.tool?.execute?.("call", { url: params.url });
return "";
} catch (error) {
return (error as Error).message;
}
}
describe("web_fetch extraction fallbacks", () => {
const priorFetch = global.fetch;
@@ -112,7 +147,7 @@ describe("web_fetch extraction fallbacks", () => {
});
it("wraps fetched text with external content markers", async () => {
const mockFetch = vi.fn((input: RequestInfo) =>
installMockFetch((input: RequestInfo) =>
Promise.resolve({
ok: true,
status: 200,
@@ -121,19 +156,8 @@ describe("web_fetch extraction fallbacks", () => {
url: requestUrl(input),
} as Response),
);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } },
},
},
},
sandboxed: false,
});
const tool = createFetchTool({ firecrawl: { enabled: false } });
const result = await tool?.execute?.("call", { url: "https://example.com/plain" });
const details = result?.details as {
@@ -161,7 +185,7 @@ describe("web_fetch extraction fallbacks", () => {
it("enforces maxChars after wrapping", async () => {
const longText = "x".repeat(5_000);
const mockFetch = vi.fn((input: RequestInfo) =>
installMockFetch((input: RequestInfo) =>
Promise.resolve({
ok: true,
status: 200,
@@ -170,18 +194,10 @@ describe("web_fetch extraction fallbacks", () => {
url: requestUrl(input),
} as Response),
);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false }, maxChars: 2000 },
},
},
},
sandboxed: false,
const tool = createFetchTool({
firecrawl: { enabled: false },
maxChars: 2000,
});
const result = await tool?.execute?.("call", { url: "https://example.com/long" });
@@ -192,7 +208,7 @@ describe("web_fetch extraction fallbacks", () => {
});
it("honors maxChars even when wrapper overhead exceeds limit", async () => {
const mockFetch = vi.fn((input: RequestInfo) =>
installMockFetch((input: RequestInfo) =>
Promise.resolve({
ok: true,
status: 200,
@@ -201,18 +217,10 @@ describe("web_fetch extraction fallbacks", () => {
url: requestUrl(input),
} as Response),
);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false }, maxChars: 100 },
},
},
},
sandboxed: false,
const tool = createFetchTool({
firecrawl: { enabled: false },
maxChars: 100,
});
const result = await tool?.execute?.("call", { url: "https://example.com/short" });
@@ -226,7 +234,7 @@ describe("web_fetch extraction fallbacks", () => {
// The sanitization of these fields is verified by external-content.test.ts tests.
it("falls back to firecrawl when readability returns no content", async () => {
const mockFetch = vi.fn((input: RequestInfo) => {
installMockFetch((input: RequestInfo) => {
const url = requestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve(firecrawlResponse("firecrawl content")) as Promise<Response>;
@@ -235,21 +243,9 @@ describe("web_fetch extraction fallbacks", () => {
htmlResponse("<!doctype html><html><head></head><body></body></html>", url),
) as Promise<Response>;
});
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: {
cacheTtlMinutes: 0,
firecrawl: { apiKey: "firecrawl-test" },
},
},
},
},
sandboxed: false,
const tool = createFetchTool({
firecrawl: { apiKey: "firecrawl-test" },
});
const result = await tool?.execute?.("call", { url: "https://example.com/empty" });
@@ -259,21 +255,13 @@ describe("web_fetch extraction fallbacks", () => {
});
it("throws when readability is disabled and firecrawl is unavailable", async () => {
const mockFetch = vi.fn((input: RequestInfo) =>
installMockFetch((input: RequestInfo) =>
Promise.resolve(htmlResponse("<html><body>hi</body></html>", requestUrl(input))),
);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { readability: false, cacheTtlMinutes: 0, firecrawl: { enabled: false } },
},
},
},
sandboxed: false,
const tool = createFetchTool({
readability: false,
firecrawl: { enabled: false },
});
await expect(
@@ -282,7 +270,7 @@ describe("web_fetch extraction fallbacks", () => {
});
it("throws when readability is empty and firecrawl fails", async () => {
const mockFetch = vi.fn((input: RequestInfo) => {
installMockFetch((input: RequestInfo) => {
const url = requestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve(firecrawlError()) as Promise<Response>;
@@ -291,18 +279,9 @@ describe("web_fetch extraction fallbacks", () => {
htmlResponse("<!doctype html><html><head></head><body></body></html>", url),
) as Promise<Response>;
});
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { apiKey: "firecrawl-test" } },
},
},
},
sandboxed: false,
const tool = createFetchTool({
firecrawl: { apiKey: "firecrawl-test" },
});
await expect(
@@ -311,7 +290,7 @@ describe("web_fetch extraction fallbacks", () => {
});
it("uses firecrawl when direct fetch fails", async () => {
const mockFetch = vi.fn((input: RequestInfo) => {
installMockFetch((input: RequestInfo) => {
const url = requestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve(firecrawlResponse("firecrawl fallback", url)) as Promise<Response>;
@@ -323,18 +302,9 @@ describe("web_fetch extraction fallbacks", () => {
text: async () => "blocked",
} as Response);
});
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { apiKey: "firecrawl-test" } },
},
},
},
sandboxed: false,
const tool = createFetchTool({
firecrawl: { apiKey: "firecrawl-test" },
});
const result = await tool?.execute?.("call", { url: "https://example.com/blocked" });
@@ -345,22 +315,14 @@ describe("web_fetch extraction fallbacks", () => {
it("wraps external content and clamps oversized maxChars", async () => {
const large = "a".repeat(80_000);
const mockFetch = vi.fn(
installMockFetch(
(input: RequestInfo) =>
Promise.resolve(textResponse(large, requestUrl(input))) as Promise<Response>,
);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false }, maxCharsCap: 10_000 },
},
},
},
sandboxed: false,
const tool = createFetchTool({
firecrawl: { enabled: false },
maxCharsCap: 10_000,
});
const result = await tool?.execute?.("call", {
@@ -373,36 +335,23 @@ describe("web_fetch extraction fallbacks", () => {
expect(details.length).toBeLessThanOrEqual(10_000);
expect(details.truncated).toBe(true);
});
it("strips and truncates HTML from error responses", async () => {
const long = "x".repeat(12_000);
const html =
"<!doctype html><html><head><title>Not Found</title></head><body><h1>Not Found</h1><p>" +
long +
"</p></body></html>";
const mockFetch = vi.fn((input: RequestInfo) =>
installMockFetch((input: RequestInfo) =>
Promise.resolve(errorHtmlResponse(html, 404, requestUrl(input), "Text/HTML; charset=utf-8")),
);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } },
},
},
},
sandboxed: false,
const tool = createFetchTool({ firecrawl: { enabled: false } });
const message = await captureToolErrorMessage({
tool,
url: "https://example.com/missing",
});
let message = "";
try {
await tool?.execute?.("call", { url: "https://example.com/missing" });
} catch (error) {
message = (error as Error).message;
}
expect(message).toContain("Web fetch failed (404):");
expect(message).toContain("<<<EXTERNAL_UNTRUSTED_CONTENT>>>");
expect(message).toContain("SECURITY NOTICE");
@@ -414,37 +363,23 @@ describe("web_fetch extraction fallbacks", () => {
it("strips HTML errors when content-type is missing", async () => {
const html =
"<!DOCTYPE HTML><html><head><title>Oops</title></head><body><h1>Oops</h1></body></html>";
const mockFetch = vi.fn((input: RequestInfo) =>
installMockFetch((input: RequestInfo) =>
Promise.resolve(errorHtmlResponse(html, 500, requestUrl(input), null)),
);
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } },
},
},
},
sandboxed: false,
const tool = createFetchTool({ firecrawl: { enabled: false } });
const message = await captureToolErrorMessage({
tool,
url: "https://example.com/oops",
});
let message = "";
try {
await tool?.execute?.("call", { url: "https://example.com/oops" });
} catch (error) {
message = (error as Error).message;
}
expect(message).toContain("Web fetch failed (500):");
expect(message).toContain("<<<EXTERNAL_UNTRUSTED_CONTENT>>>");
expect(message).toContain("Oops");
});
it("wraps firecrawl error details", async () => {
const mockFetch = vi.fn((input: RequestInfo) => {
installMockFetch((input: RequestInfo) => {
const url = requestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve({
@@ -455,26 +390,15 @@ describe("web_fetch extraction fallbacks", () => {
}
return Promise.reject(new Error("network down"));
});
// @ts-expect-error mock fetch
global.fetch = mockFetch;
const tool = createWebFetchTool({
config: {
tools: {
web: {
fetch: { cacheTtlMinutes: 0, firecrawl: { apiKey: "firecrawl-test" } },
},
},
},
sandboxed: false,
const tool = createFetchTool({
firecrawl: { apiKey: "firecrawl-test" },
});
let message = "";
try {
await tool?.execute?.("call", { url: "https://example.com/firecrawl-error" });
} catch (error) {
message = (error as Error).message;
}
const message = await captureToolErrorMessage({
tool,
url: "https://example.com/firecrawl-error",
});
expect(message).toContain("Firecrawl fetch failed (403):");
expect(message).toContain("<<<EXTERNAL_UNTRUSTED_CONTENT>>>");