improvement(browser-use): added secrets to browser use agent (#375)

* updated husky * added secrets to browser use, fixed gitignore * fixed prettier ignore * fixed param * fixed PR comments
2026-01-08 22:48:14 -05:00 · 2025-05-18 19:56:42 -07:00
parent c7866c8388
commit 18ce26616e
12 changed files with 164 additions and 118 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -7,8 +7,6 @@
 /scripts/node_modules

 # bun specific
-.bun
-bun.lock
 bun-debug.log*

 # testing
--- a/.husky/pre-commit
+++ b/.husky/pre-commit
@@ -1 +1 @@
-cd apps/sim && npx lint-staged 
+bunx lint-staged
--- a/.prettierignore
+++ b/.prettierignore
@@ -33,6 +33,3 @@ coverage
 **/public/workbox-*.js
 **/public/worker-*.js
 **/public/fallback-*.js
-
-# Documentation
-apps/docs/**/*.mdx
--- a/apps/docs/content/docs/tools/browser_use.mdx
+++ b/apps/docs/content/docs/tools/browser_use.mdx
@@ -67,22 +67,21 @@ Runs a browser automation task using BrowserUse
 #### Input

 | Parameter           | Type    | Required | Description                                                     |
-| -------------- | ------ | -------- | ---------------------------------------------------------------------------------------- |
+| ------------------- | ------- | -------- | --------------------------------------------------------------- |
 | `task`              | string  | Yes      | What should the browser agent do                                |
+| `variables`         | json    | No       | Optional variables to use as secrets \(format: \{key: value\}\) |
+| `save_browser_data` | boolean | No       | Whether to save browser data                                    |
+| `model`             | string  | No       | LLM model to use \(default: gpt-4o\)                            |
 | `apiKey`            | string  | Yes      | API key for BrowserUse API                                      |
-| `pollInterval` | number | No       | Interval between polling requests in milliseconds \(default: 5000\)                      |
-| `maxPollTime`  | number | No       | Maximum time to poll for task completion in milliseconds \(default: 300000 - 5 minutes\) |

 #### Output

 | Parameter | Type   |
-| ---------- | ------ |
+| --------- | ------ |
 | `id`      | string |
-| `task`     | string |
+| `success` | string |
 | `output`  | string |
-| `status`   | string |
 | `steps`   | string |
-| `live_url` | string |

 ## Block Configuration

@@ -95,14 +94,12 @@ Runs a browser automation task using BrowserUse
 ### Outputs

 | Output      | Type    | Description             |
-| ------------ | ------ | ------------------------ |
+| ----------- | ------- | ----------------------- |
 | `response`  | object  | Output from response    |
 | ↳ `id`      | string  | id of the response      |
-| ↳ `task`     | string | task of the response     |
+| ↳ `success` | boolean | success of the response |
 | ↳ `output`  | any     | output of the response  |
-| ↳ `status`   | string | status of the response   |
 | ↳ `steps`   | json    | steps of the response   |
-| ↳ `live_url` | any    | live_url of the response |

 ## Notes

--- a/apps/docs/content/docs/tools/clay.mdx
+++ b/apps/docs/content/docs/tools/clay.mdx
@@ -1,6 +1,6 @@
 ---
 title: Clay
-description: Populate Clay workbook with data
+description: Populate Clay workbook
 ---

 import { BlockInfoCard } from '@/components/ui/block-info-card'
--- a/apps/docs/content/docs/tools/discord.mdx
+++ b/apps/docs/content/docs/tools/discord.mdx
@@ -9,11 +9,22 @@ import { BlockInfoCard } from '@/components/ui/block-info-card'
  type="discord"
  color="#E0E0E0"
  icon={true}
-  iconSvg={`<svg className="block-icon"    viewBox="0 -28.5 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlnsXlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
-    <g>
-        <path d="M216.856339,16.5966031 C200.285002,8.84328665 182.566144,3.2084988 164.041564,0 C161.766523,4.11318106 159.108624,9.64549908 157.276099,14.0464379 C137.583995,11.0849896 118.072967,11.0849896 98.7430163,14.0464379 C96.9108417,9.64549908 94.1925838,4.11318106 91.8971895,0 C73.3526068,3.2084988 55.6133949,8.86399117 39.0420583,16.6376612 C5.61752293,67.146514 -3.4433191,116.400813 1.08711069,164.955721 C23.2560196,181.510915 44.7403634,191.567697 65.8621325,198.148576 C71.0772151,190.971126 75.7283628,183.341335 79.7352139,175.300261 C72.104019,172.400575 64.7949724,168.822202 57.8887866,164.667963 C59.7209612,163.310589 61.5131304,161.891452 63.2445898,160.431257 C105.36741,180.133187 151.134928,180.133187 192.754523,160.431257 C194.506336,161.891452 196.298154,163.310589 198.110326,164.667963 C191.183787,168.842556 183.854737,172.420929 176.223542,175.320965 C180.230393,183.341335 184.861538,190.991831 190.096624,198.16893 C211.238746,191.588051 232.743023,181.531619 254.911949,164.955721 C260.227747,108.668201 245.831087,59.8662432 216.856339,16.5966031 Z M85.4738752,135.09489 C72.8290281,135.09489 62.4592217,123.290155 62.4592217,108.914901 C62.4592217,94.5396472 72.607595,82.7145587 85.4738752,82.7145587 C98.3405064,82.7145587 108.709962,94.5189427 108.488529,108.914901 C108.508531,123.290155 98.3405064,135.09489 85.4738752,135.09489 Z M170.525237,135.09489 C157.88039,135.09489 147.510584,123.290155 147.510584,108.914901 C147.510584,94.5396472 157.658606,82.7145587 170.525237,82.7145587 C183.391518,82.7145587 193.761324,94.5189427 193.539891,108.914901 C193.539891,123.290155 183.391518,135.09489 170.525237,135.09489 Z" fill="#5865F2" fillRule="nonzero">
+  iconSvg={`<svg className="block-icon"
      
-</path>
+      
+      
+      viewBox="0 -28.5 256 256"
+      version="1.1"
+      xmlns="http://www.w3.org/2000/svg"
+      xmlnsXlink="http://www.w3.org/1999/xlink"
+      preserveAspectRatio="xMidYMid"
+    >
+      <g>
+        <path
+          d="M216.856339,16.5966031 C200.285002,8.84328665 182.566144,3.2084988 164.041564,0 C161.766523,4.11318106 159.108624,9.64549908 157.276099,14.0464379 C137.583995,11.0849896 118.072967,11.0849896 98.7430163,14.0464379 C96.9108417,9.64549908 94.1925838,4.11318106 91.8971895,0 C73.3526068,3.2084988 55.6133949,8.86399117 39.0420583,16.6376612 C5.61752293,67.146514 -3.4433191,116.400813 1.08711069,164.955721 C23.2560196,181.510915 44.7403634,191.567697 65.8621325,198.148576 C71.0772151,190.971126 75.7283628,183.341335 79.7352139,175.300261 C72.104019,172.400575 64.7949724,168.822202 57.8887866,164.667963 C59.7209612,163.310589 61.5131304,161.891452 63.2445898,160.431257 C105.36741,180.133187 151.134928,180.133187 192.754523,160.431257 C194.506336,161.891452 196.298154,163.310589 198.110326,164.667963 C191.183787,168.842556 183.854737,172.420929 176.223542,175.320965 C180.230393,183.341335 184.861538,190.991831 190.096624,198.16893 C211.238746,191.588051 232.743023,181.531619 254.911949,164.955721 C260.227747,108.668201 245.831087,59.8662432 216.856339,16.5966031 Z M85.4738752,135.09489 C72.8290281,135.09489 62.4592217,123.290155 62.4592217,108.914901 C62.4592217,94.5396472 72.607595,82.7145587 85.4738752,82.7145587 C98.3405064,82.7145587 108.709962,94.5189427 108.488529,108.914901 C108.508531,123.290155 98.3405064,135.09489 85.4738752,135.09489 Z M170.525237,135.09489 C157.88039,135.09489 147.510584,123.290155 147.510584,108.914901 C147.510584,94.5396472 157.658606,82.7145587 170.525237,82.7145587 C183.391518,82.7145587 193.761324,94.5189427 193.539891,108.914901 C193.539891,123.290155 183.391518,135.09489 170.525237,135.09489 Z"
+          fill="#5865F2"
+          fillRule="nonzero"
+        ></path>
      </g>
    </svg>`}
 />
--- a/apps/sim/blocks/blocks/browser_use.ts
+++ b/apps/sim/blocks/blocks/browser_use.ts
@@ -5,11 +5,9 @@ import { BlockConfig } from '../types'
 interface BrowserUseResponse extends ToolResponse {
  output: {
    id: string
-    task: string
-    output: string | null
-    status: string
+    success: boolean
+    output: any
    steps: any[]
-    live_url: string | null
  }
 }

@@ -31,6 +29,34 @@ export const BrowserUseBlock: BlockConfig<BrowserUseResponse> = {
      layout: 'full',
      placeholder: 'Describe what the browser agent should do...',
    },
+    {
+      id: 'variables',
+      title: 'Variables (Secrets)',
+      type: 'table',
+      layout: 'full',
+      columns: ['Key', 'Value'],
+    },
+    {
+      id: 'model',
+      title: 'Model',
+      type: 'dropdown',
+      layout: 'half',
+      options: [
+        { label: 'gpt-4o', id: 'gpt-4o' },
+        { label: 'gpt-4.1', id: 'gpt-4.1' },
+        { label: 'gemini-2.0-flash', id: 'gemini-2.0-flash' },
+        { label: 'gemini-2.0-flash-lite', id: 'gemini-2.0-flash-lite' },
+        { label: 'claude-3-7-sonnet-20250219', id: 'claude-3-7-sonnet-20250219' },
+        { label: 'llama-4-maverick-17b-128e-instruct', id: 'llama-4-maverick-17b-128e-instruct' },
+      ],
+    },
+    {
+      id: 'save_browser_data',
+      title: 'Save Browser Data',
+      type: 'switch',
+      layout: 'half',
+      placeholder: 'Save browser data',
+    },
    {
      id: 'apiKey',
      title: 'API Key',
@@ -39,20 +65,6 @@ export const BrowserUseBlock: BlockConfig<BrowserUseResponse> = {
      password: true,
      placeholder: 'Enter your BrowserUse API key',
    },
-    {
-      id: 'pollInterval',
-      title: 'Poll Interval (ms)',
-      type: 'short-input',
-      layout: 'half',
-      placeholder: '5000',
-    },
-    {
-      id: 'maxPollTime',
-      title: 'Max Poll Time (ms)',
-      type: 'short-input',
-      layout: 'half',
-      placeholder: '300000',
-    },
  ],
  tools: {
    access: ['browser_use_run_task'],
@@ -60,18 +72,17 @@ export const BrowserUseBlock: BlockConfig<BrowserUseResponse> = {
  inputs: {
    task: { type: 'string', required: true },
    apiKey: { type: 'string', required: true },
-    pollInterval: { type: 'number', required: false },
-    maxPollTime: { type: 'number', required: false },
+    variables: { type: 'json', required: false },
+    model: { type: 'string', required: false },
+    save_browser_data: { type: 'boolean', required: false },
  },
  outputs: {
    response: {
      type: {
        id: 'string',
-        task: 'string',
+        success: 'boolean',
        output: 'any',
-        status: 'string',
        steps: 'json',
-        live_url: 'any',
      },
    },
  },
--- a/apps/sim/package.json
+++ b/apps/sim/package.json
@@ -24,13 +24,6 @@
    "test:watch": "vitest",
    "test:coverage": "vitest run --coverage",
    "email:dev": "email dev --dir components/emails",
-    "cli:build": "bun run build -w packages/simstudio",
-    "cli:dev": "bun run build -w packages/simstudio && cd packages/simstudio && bun ./dist/index.js",
-    "cli:publish": "cd packages/simstudio && bun publish",
-    "cli:start": "cd packages/simstudio && bun ./dist/index.js start",
-    "build:standalone": "bun scripts/build-standalone.js",
-    "build:cli": "bun run cli:build && bun run build:standalone",
-    "publish:cli": "bun run build:cli && bun run cli:publish",
    "check-types": "tsc --noEmit"
  },
  "dependencies": {
--- a/apps/sim/tools/browser_use/run_task.ts
+++ b/apps/sim/tools/browser_use/run_task.ts
@@ -1,9 +1,12 @@
 import { createLogger } from '@/lib/logs/console-logger'
 import { ToolConfig } from '../types'
-import { BrowserUseRunTaskParams, BrowserUseRunTaskResponse, BrowserUseTaskOutput } from './types'
+import { BrowserUseRunTaskParams, BrowserUseRunTaskResponse } from './types'

 const logger = createLogger('BrowserUseTool')

+const POLL_INTERVAL_MS = 5000 // 5 seconds between polls
+const MAX_POLL_TIME_MS = 180000 // 3 minutes maximum polling time
+
 export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskResponse> = {
  id: 'browser_use_run_task',
  name: 'Browser Use',
@@ -16,24 +19,27 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
      required: true,
      description: 'What should the browser agent do',
    },
+    variables: {
+      type: 'json',
+      required: false,
+      description: 'Optional variables to use as secrets (format: {key: value})',
+    },
+    save_browser_data: {
+      type: 'boolean',
+      required: false,
+      description: 'Whether to save browser data',
+    },
+    model: {
+      type: 'string',
+      required: false,
+      description: 'LLM model to use (default: gpt-4o)',
+    },
    apiKey: {
      type: 'string',
      required: true,
+      requiredForToolCall: true,
      description: 'API key for BrowserUse API',
    },
-    pollInterval: {
-      type: 'number',
-      required: false,
-      default: 5000,
-      description: 'Interval between polling requests in milliseconds (default: 5000)',
-    },
-    maxPollTime: {
-      type: 'number',
-      required: false,
-      default: 300000,
-      description:
-        'Maximum time to poll for task completion in milliseconds (default: 300000 - 5 minutes)',
-    },
  },

  request: {
@@ -43,9 +49,51 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
      'Content-Type': 'application/json',
      Authorization: `Bearer ${params.apiKey}`,
    }),
-    body: (params) => ({
+    body: (params) => {
+      const requestBody: Record<string, any> = {
        task: params.task,
-    }),
+      }
+
+      if (params.variables) {
+        let secrets: Record<string, string> = {}
+
+        if (Array.isArray(params.variables)) {
+          logger.info('Converting variables array to dictionary format')
+          params.variables.forEach((row) => {
+            if (row.cells && row.cells.Key && row.cells.Value !== undefined) {
+              secrets[row.cells.Key] = row.cells.Value
+              logger.info(`Added secret for key: ${row.cells.Key}`)
+            } else if (row.Key && row.Value !== undefined) {
+              secrets[row.Key] = row.Value
+              logger.info(`Added secret for key: ${row.Key}`)
+            }
+          })
+        } else if (typeof params.variables === 'object' && params.variables !== null) {
+          logger.info('Using variables object directly')
+          secrets = params.variables
+        }
+
+        if (Object.keys(secrets).length > 0) {
+          logger.info(`Found ${Object.keys(secrets).length} secrets to include`)
+          requestBody.secrets = secrets
+        } else {
+          logger.warn('No usable secrets found in variables')
+        }
+      }
+
+      if (params.model) {
+        requestBody.llm_model = params.model
+      }
+
+      if (params.save_browser_data) {
+        requestBody.save_browser_data = params.save_browser_data
+      }
+
+      requestBody.use_adblock = true
+      requestBody.highlight_elements = true
+
+      return requestBody
+    },
  },

  transformResponse: async (response: Response) => {
@@ -57,16 +105,14 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
      throw new Error(`Request failed with status ${response.status}`)
    }

-    const data = (await response.json()) as BrowserUseTaskOutput
+    const data = (await response.json()) as { id: string }
    return {
      success: true,
      output: {
        id: data.id,
-        task: '',
+        success: true,
        output: null,
-        status: 'created',
        steps: [],
-        live_url: null,
      },
    }
  },
@@ -93,7 +139,6 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
          logger.info(
            `BrowserUse task ${taskId} launched with live URL: ${initialTaskData.live_url}`
          )
-          result.output.live_url = initialTaskData.live_url
          liveUrlLogged = true
        }
      }
@@ -101,19 +146,9 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
      logger.warn(`Failed to get initial task details for ${taskId}:`, error)
    }

-    const pollInterval =
-      typeof params.pollInterval === 'number' && params.pollInterval >= 1000
-        ? params.pollInterval
-        : 5000
-
-    const maxPollTime =
-      typeof params.maxPollTime === 'number' && params.maxPollTime >= 5000
-        ? params.maxPollTime
-        : 300000
-
    let elapsedTime = 0

-    while (elapsedTime < maxPollTime) {
+    while (elapsedTime < MAX_POLL_TIME_MS) {
      try {
        const statusResponse = await fetch(
          `https://api.browser-use.com/api/v1/task/${taskId}/status`,
@@ -130,7 +165,6 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
        }

        const status = await statusResponse.json()
-        result.output.status = status

        logger.info(`BrowserUse task ${taskId} status: ${status}`)

@@ -144,7 +178,12 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR

          if (taskResponse.ok) {
            const taskData = await taskResponse.json()
-            result.output = taskData as BrowserUseTaskOutput
+            result.output = {
+              id: taskId,
+              success: status === 'finished' ? true : false,
+              output: taskData.output,
+              steps: taskData.steps || [],
+            }
          }

          return result
@@ -162,14 +201,13 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
            const taskData = await taskResponse.json()
            if (taskData.live_url) {
              logger.info(`BrowserUse task ${taskId} running with live URL: ${taskData.live_url}`)
-              result.output.live_url = taskData.live_url
              liveUrlLogged = true
            }
          }
        }

-        await new Promise((resolve) => setTimeout(resolve, pollInterval))
-        elapsedTime += pollInterval
+        await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS))
+        elapsedTime += POLL_INTERVAL_MS
      } catch (error: any) {
        logger.error('Error polling for task status:', {
          message: error.message || 'Unknown error',
@@ -184,17 +222,16 @@ export const runTaskTool: ToolConfig<BrowserUseRunTaskParams, BrowserUseRunTaskR
    }

    logger.warn(
-      `Task ${taskId} did not complete within the maximum polling time (${maxPollTime / 1000}s)`
+      `Task ${taskId} did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`
    )
    return {
      ...result,
-      error: `Task did not complete within the maximum polling time (${maxPollTime / 1000}s)`,
+      error: `Task did not complete within the maximum polling time (${MAX_POLL_TIME_MS / 1000}s)`,
    }
  },

  transformError: (error) => {
    try {
-      // Check if error message contains a JSON string (from 422 errors)
      const errorData = JSON.parse(error.message)
      if (errorData.detail && Array.isArray(errorData.detail)) {
        const formattedError = errorData.detail
--- a/apps/sim/tools/browser_use/types.ts
+++ b/apps/sim/tools/browser_use/types.ts
@@ -3,8 +3,9 @@ import { ToolResponse } from '../types'
 export interface BrowserUseRunTaskParams {
  task: string
  apiKey: string
-  pollInterval?: number
-  maxPollTime?: number
+  variables?: Record<string, string>
+  model?: string
+  save_browser_data?: boolean
 }

 export interface BrowserUseTaskStep {
@@ -12,15 +13,15 @@ export interface BrowserUseTaskStep {
  step: number
  evaluation_previous_goal: string
  next_goal: string
+  url?: string
+  extracted_data?: Record<string, any>
 }

 export interface BrowserUseTaskOutput {
  id: string
-  task: string
-  output: string | null
-  status: 'created' | 'running' | 'finished' | 'stopped' | 'paused' | 'failed'
+  success: boolean
+  output: any
  steps: BrowserUseTaskStep[]
-  live_url: string | null
 }

 export interface BrowserUseRunTaskResponse extends ToolResponse {
--- a/package.json
+++ b/package.json
@@ -22,7 +22,8 @@
    "test": "turbo run test",
    "format": "prettier --write .",
    "format:check": "prettier --check .",
-    "lint": "turbo run lint"
+    "lint": "turbo run lint",
+    "prepare": "bun husky"
  },
  "overrides": {
    "react": "19.1.0",