revert(tools): undo accidental merge of PR #18584

src/agents/tools/web-fetch.ts

@@ -1,16 +1,12 @@
 import { Type } from "@sinclair/typebox";
 import type { OpenClawConfig } from "../../config/config.js";
+import type { AnyAgentTool } from "./common.js";
 import { fetchWithSsrFGuard } from "../../infra/net/fetch-guard.js";
-import {
-  matchesHostnameAllowlist,
-  normalizeHostnameAllowlist,
-  SsrFBlockedError,
-} from "../../infra/net/ssrf.js";
+import { SsrFBlockedError } from "../../infra/net/ssrf.js";
 import { logDebug } from "../../logger.js";
 import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
 import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
 import { stringEnum } from "../schema/typebox.js";
-import type { AnyAgentTool } from "./common.js";
 import { jsonResult, readNumberParam, readStringParam } from "./common.js";
 import {
   extractReadableContent,
@@ -26,7 +22,6 @@ import {
   normalizeCacheKey,
   readCache,
   readResponseText,
-  resolveWebUrlAllowlist,
   resolveCacheTtlMs,
   resolveTimeoutSeconds,
   withTimeout,
@@ -73,22 +68,6 @@ type WebFetchConfig = NonNullable<OpenClawConfig["tools"]>["web"] extends infer
     : undefined
   : undefined;
 
-type WebConfig = NonNullable<OpenClawConfig["tools"]>["web"];
-
-export function resolveFetchUrlAllowlist(web?: WebConfig): string[] | undefined {
-  return resolveWebUrlAllowlist(web);
-}
-
-export function isUrlAllowedByAllowlist(url: string, allowlist: string[]): boolean {
-  try {
-    const hostname = new URL(url).hostname;
-    const normalizedAllowlist = normalizeHostnameAllowlist(allowlist);
-    return matchesHostnameAllowlist(hostname, normalizedAllowlist);
-  } catch {
-    return false;
-  }
-}
-
 type FirecrawlFetchConfig =
   | {
       enabled?: boolean;
@@ -753,7 +732,6 @@ export function createWebFetchTool(options?: {
     (fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) ||
     DEFAULT_FETCH_USER_AGENT;
   const maxResponseBytes = resolveFetchMaxResponseBytes(fetch);
-  const urlAllowlist = resolveFetchUrlAllowlist(options?.config?.tools?.web);
   return {
     label: "Web Fetch",
     name: "web_fetch",
@@ -763,25 +741,6 @@ export function createWebFetchTool(options?: {
     execute: async (_toolCallId, args) => {
       const params = args as Record<string, unknown>;
       const url = readStringParam(params, "url", { required: true });
-
-      // Check URL against allowlist if configured
-      if (urlAllowlist && urlAllowlist.length > 0) {
-        if (!isUrlAllowedByAllowlist(url, urlAllowlist)) {
-          let hostname: string;
-          try {
-            hostname = new URL(url).hostname;
-          } catch {
-            hostname = url;
-          }
-          return jsonResult({
-            error: "url_not_allowed",
-            message: `URL not in allowlist. Allowed domains: ${urlAllowlist.join(", ")}`,
-            blockedUrl: url,
-            blockedHostname: hostname,
-          });
-        }
-      }
-
       const extractMode = readStringParam(params, "extractMode") === "text" ? "text" : "markdown";
       const maxChars = readNumberParam(params, "maxChars", { integer: true });
       const maxCharsCap = resolveFetchMaxCharsCap(fetch);

src/agents/tools/web-search.ts

@@ -1,10 +1,9 @@
 import { Type } from "@sinclair/typebox";
-import { formatCliCommand } from "../../cli/command-format.js";
 import type { OpenClawConfig } from "../../config/config.js";
-import { matchesHostnameAllowlist, normalizeHostnameAllowlist } from "../../infra/net/ssrf.js";
+import type { AnyAgentTool } from "./common.js";
+import { formatCliCommand } from "../../cli/command-format.js";
 import { wrapWebContent } from "../../security/external-content.js";
 import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
-import type { AnyAgentTool } from "./common.js";
 import { jsonResult, readNumberParam, readStringParam } from "./common.js";
 import {
   CacheEntry,
@@ -13,7 +12,6 @@ import {
   normalizeCacheKey,
   readCache,
   readResponseText,
-  resolveWebUrlAllowlist,
   resolveCacheTtlMs,
   resolveTimeoutSeconds,
   withTimeout,
@@ -77,33 +75,6 @@ type WebSearchConfig = NonNullable<OpenClawConfig["tools"]>["web"] extends infer
     : undefined
   : undefined;
 
-type WebConfig = NonNullable<OpenClawConfig["tools"]>["web"];
-
-export function resolveUrlAllowlist(web?: WebConfig): string[] | undefined {
-  return resolveWebUrlAllowlist(web);
-}
-
-export function filterResultsByAllowlist(
-  results: Array<{ url?: string; siteName?: string }>,
-  allowlist: string[],
-): Array<{ url?: string; siteName?: string }> {
-  if (allowlist.length === 0) {
-    return results;
-  }
-  const normalizedAllowlist = normalizeHostnameAllowlist(allowlist);
-  return results.filter((result) => {
-    if (!result.url) {
-      return true; // Keep entries without URL
-    }
-    try {
-      const hostname = new URL(result.url).hostname;
-      return matchesHostnameAllowlist(hostname, normalizedAllowlist);
-    } catch {
-      return true; // Keep entries with invalid URLs (let them pass through)
-    }
-  });
-}
-
 type BraveSearchResult = {
   title?: string;
   url?: string;
@@ -595,7 +566,6 @@ async function runWebSearch(params: {
   perplexityModel?: string;
   grokModel?: string;
   grokInlineCitations?: boolean;
-  urlAllowlist?: string[];
 }): Promise<Record<string, unknown>> {
   const cacheKey = normalizeCacheKey(
     params.provider === "brave"
@@ -718,15 +688,10 @@ async function runWebSearch(params: {
     };
   });
 
-  // Filter results by urlAllowlist if configured
-  const filteredResults = params.urlAllowlist
-    ? filterResultsByAllowlist(mapped, params.urlAllowlist)
-    : mapped;
-
   const payload = {
     query: params.query,
     provider: params.provider,
-    count: filteredResults.length,
+    count: mapped.length,
     tookMs: Date.now() - start,
     externalContent: {
       untrusted: true,
@@ -734,7 +699,7 @@ async function runWebSearch(params: {
       provider: params.provider,
       wrapped: true,
     },
-    results: filteredResults,
+    results: mapped,
   };
   writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs);
   return payload;
@@ -752,7 +717,6 @@ export function createWebSearchTool(options?: {
   const provider = resolveSearchProvider(search);
   const perplexityConfig = resolvePerplexityConfig(search);
   const grokConfig = resolveGrokConfig(search);
-  const urlAllowlist = resolveUrlAllowlist(options?.config?.tools?.web);
 
   const description =
     provider === "perplexity"
@@ -822,7 +786,6 @@ export function createWebSearchTool(options?: {
         perplexityModel: resolvePerplexityModel(perplexityConfig),
         grokModel: resolveGrokModel(grokConfig),
         grokInlineCitations: resolveGrokInlineCitations(grokConfig),
-        urlAllowlist,
       });
       return jsonResult(result);
     },
@@ -840,6 +803,4 @@ export const __testing = {
   resolveGrokModel,
   resolveGrokInlineCitations,
   extractGrokContent,
-  resolveUrlAllowlist,
-  filterResultsByAllowlist,
 } as const;

src/agents/tools/web-shared.ts

@@ -8,20 +8,6 @@ export const DEFAULT_TIMEOUT_SECONDS = 30;
 export const DEFAULT_CACHE_TTL_MINUTES = 15;
 const DEFAULT_CACHE_MAX_ENTRIES = 100;
 
-export function resolveWebUrlAllowlist(web: unknown): string[] | undefined {
-  if (!web || typeof web !== "object") {
-    return undefined;
-  }
-  if (!("urlAllowlist" in web)) {
-    return undefined;
-  }
-  const allowlist = (web as { urlAllowlist?: unknown }).urlAllowlist;
-  if (!Array.isArray(allowlist)) {
-    return undefined;
-  }
-  return allowlist.length > 0 ? allowlist : undefined;
-}
-
 export function resolveTimeoutSeconds(value: unknown, fallback: number): number {
   const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
   return Math.max(1, Math.floor(parsed));

src/config/schema.help.ts

@@ -116,8 +116,6 @@ export const FIELD_HELP: Record<string, string> = {
     "Perplexity base URL override (default: https://openrouter.ai/api/v1 or https://api.perplexity.ai).",
   "tools.web.search.perplexity.model":
     'Perplexity model override (default: "perplexity/sonar-pro").',
-  "tools.web.urlAllowlist":
-    "Optional URL/domain allowlist shared by web_search and web_fetch. Accepts domain patterns like 'example.com', '*.github.com'. When configured, only matching URLs are allowed.",
   "tools.web.fetch.enabled": "Enable the web_fetch tool (lightweight HTTP fetch).",
   "tools.web.fetch.maxChars": "Max characters returned by web_fetch (truncated).",
   "tools.web.fetch.maxCharsCap":

src/config/types.tools.ts

@@ -395,8 +395,6 @@ export type ToolsConfig = {
   /** Optional tool policy overrides keyed by provider id or "provider/model". */
   byProvider?: Record<string, ToolPolicyConfig>;
   web?: {
-    /** Optional URL/domain allowlist for web tools. When configured, only URLs matching these patterns are allowed. */
-    urlAllowlist?: string[];
     search?: {
       /** Enable web search tool (default: true when API key is present). */
       enabled?: boolean;

src/config/zod-schema.agent-runtime.ts

@@ -267,7 +267,6 @@ export const ToolsWebFetchSchema = z
 
 export const ToolsWebSchema = z
   .object({
-    urlAllowlist: z.array(z.string()).optional(),
     search: ToolsWebSearchSchema,
     fetch: ToolsWebFetchSchema,
   })

src/infra/net/ssrf.ts

@@ -33,7 +33,7 @@ function normalizeHostnameSet(values?: string[]): Set<string> {
   return new Set(values.map((value) => normalizeHostname(value)).filter(Boolean));
 }
 
-export function normalizeHostnameAllowlist(values?: string[]): string[] {
+function normalizeHostnameAllowlist(values?: string[]): string[] {
   if (!values || values.length === 0) {
     return [];
   }
@@ -57,7 +57,7 @@ function isHostnameAllowedByPattern(hostname: string, pattern: string): boolean
   return hostname === pattern;
 }
 
-export function matchesHostnameAllowlist(hostname: string, allowlist: string[]): boolean {
+function matchesHostnameAllowlist(hostname: string, allowlist: string[]): boolean {
   if (allowlist.length === 0) {
     return true;
   }