From dc64f0b38898484047d2a36866091578768de372 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 29 Jan 2025 22:27:41 -0800 Subject: [PATCH] Added Firecrawl web scrape tool/block and switch toggle sub-block --- .../sub-block/components/switch.tsx | 18 ++++ .../components/sub-block/sub-block.tsx | 10 +- app/w/page.tsx | 66 ++++++++++++++ blocks/blocks/firecrawl.ts | 49 ++++++++++ blocks/index.ts | 6 +- blocks/types.ts | 2 +- components/icons.tsx | 20 +++- components/ui/switch.tsx | 26 ++++++ package-lock.json | 30 ++++++ package.json | 1 + tools/firecrawl/scrape.ts | 91 +++++++++++++++++++ tools/index.ts | 17 ++-- 12 files changed, 323 insertions(+), 13 deletions(-) create mode 100644 app/w/components/workflow-block/components/sub-block/components/switch.tsx create mode 100644 app/w/page.tsx create mode 100644 blocks/blocks/firecrawl.ts create mode 100644 components/ui/switch.tsx create mode 100644 tools/firecrawl/scrape.ts diff --git a/app/w/components/workflow-block/components/sub-block/components/switch.tsx b/app/w/components/workflow-block/components/sub-block/components/switch.tsx new file mode 100644 index 000000000..d6c917b9c --- /dev/null +++ b/app/w/components/workflow-block/components/sub-block/components/switch.tsx @@ -0,0 +1,18 @@ +import { Switch as UISwitch } from '@/components/ui/switch' +import { useSubBlockValue } from '../hooks/use-sub-block-value' + +interface SwitchProps { + blockId: string + subBlockId: string +} + +export function Switch({ blockId, subBlockId }: SwitchProps) { + const [value, setValue] = useSubBlockValue(blockId, subBlockId) + + return ( + setValue(checked)} + /> + ) +} diff --git a/app/w/components/workflow-block/components/sub-block/sub-block.tsx b/app/w/components/workflow-block/components/sub-block/sub-block.tsx index 0a2e290bf..75c8b9e15 100644 --- a/app/w/components/workflow-block/components/sub-block/sub-block.tsx +++ b/app/w/components/workflow-block/components/sub-block/sub-block.tsx @@ -6,6 +6,7 @@ import { Dropdown } from './components/dropdown' import { SliderInput } from './components/slider-input' import { Table } from './components/table' import { Code } from './components/code' +import { Switch } from './components/switch' interface SubBlockProps { blockId: string @@ -70,6 +71,13 @@ export function SubBlock({ blockId, config, isConnecting }: SubBlockProps) { ) case 'code': return + case 'switch': + return ( +
+ + +
+ ) default: return null } @@ -77,7 +85,7 @@ export function SubBlock({ blockId, config, isConnecting }: SubBlockProps) { return (
- + {config.type !== 'switch' && } {renderInput()}
) diff --git a/app/w/page.tsx b/app/w/page.tsx new file mode 100644 index 000000000..3c0b44168 --- /dev/null +++ b/app/w/page.tsx @@ -0,0 +1,66 @@ +"use client" + +import { useState } from "react" +import { WorkflowBlock } from "./components/workflow-block/workflow-block" +import { BlockConfig, BlockCategory, BlockIcon } from "@/blocks/types" +import { LayoutDashboard } from "lucide-react" + +const WorkflowIcon: BlockIcon = (props) => + +interface WorkflowBlockState { + id: string + type: string + position: { x: number; y: number } + config: BlockConfig + name: string +} + +export default function Page() { + const [workflows, setWorkflows] = useState([]) + + const createNewWorkflow = () => { + const newWorkflow: WorkflowBlockState = { + id: `workflow-${workflows.length + 1}`, + type: "default", + position: { x: 0, y: 0 }, + config: { + type: "default", + toolbar: { + title: "New Workflow", + description: "Empty workflow", + bgColor: "#808080", + icon: WorkflowIcon, + category: "basic" as BlockCategory + }, + tools: { + access: [] + }, + workflow: { + subBlocks: [], + inputs: {}, + outputs: {} + } + }, + name: `New Workflow ${workflows.length + 1}` + } + setWorkflows([...workflows, newWorkflow]) + } + + return ( +
+ +
+ {workflows.map((workflow) => ( + + ))} +
+
+ ) +} diff --git a/blocks/blocks/firecrawl.ts b/blocks/blocks/firecrawl.ts new file mode 100644 index 000000000..33011e19f --- /dev/null +++ b/blocks/blocks/firecrawl.ts @@ -0,0 +1,49 @@ +import { BlockConfig } from '../types' +import { FirecrawlIcon } from '@/components/icons' + +export const FirecrawlScrapeBlock: BlockConfig = { + type: 'firecrawlscrape', + toolbar: { + title: 'Firecrawl Scraper', + description: 'Extract clean content from any webpage', + bgColor: '#FF6B6B', + icon: FirecrawlIcon, + category: 'advanced' + }, + tools: { + access: ['firecrawl.scrape'] + }, + workflow: { + inputs: { + apiKey: { type: 'string', required: true }, + url: { type: 'string', required: true }, + scrapeOptions: { type: 'json', required: false } + }, + outputs: { + response: 'any' + }, + subBlocks: [ + { + id: 'apiKey', + title: 'API Key', + type: 'short-input', + layout: 'full', + placeholder: 'Enter your Firecrawl API key', + password: true + }, + { + id: 'url', + title: 'Website URL', + type: 'short-input', + layout: 'full', + placeholder: 'Enter the webpage URL to scrape' + }, + { + id: 'onlyMainContent', + title: 'Only Main Content', + type: 'switch', + layout: 'half' + } + ] + } +} diff --git a/blocks/index.ts b/blocks/index.ts index 8ecf4fef8..2c3de8194 100644 --- a/blocks/index.ts +++ b/blocks/index.ts @@ -5,16 +5,18 @@ import { AgentBlock } from './blocks/agent' import { ApiBlock } from './blocks/api' import { FunctionBlock } from './blocks/function' import { CrewAIVisionBlock } from './blocks/crewai' +import { FirecrawlScrapeBlock } from './blocks/firecrawl' // Export blocks for ease of use -export { AgentBlock, ApiBlock, FunctionBlock, CrewAIVisionBlock } +export { AgentBlock, ApiBlock, FunctionBlock, CrewAIVisionBlock, FirecrawlScrapeBlock } // Registry of all block configurations const blocks: Record = { agent: AgentBlock, api: ApiBlock, function: FunctionBlock, - crewaivision: CrewAIVisionBlock + crewaivision: CrewAIVisionBlock, + firecrawlscrape: FirecrawlScrapeBlock } // Build a reverse mapping of tools to block types diff --git a/blocks/types.ts b/blocks/types.ts index f8724fd6d..9c5933f36 100644 --- a/blocks/types.ts +++ b/blocks/types.ts @@ -6,7 +6,7 @@ export type BlockCategory = 'basic' | 'advanced' export type OutputType = 'string' | 'number' | 'json' | 'boolean' | 'any' export type ParamType = 'string' | 'number' | 'boolean' | 'json' -export type SubBlockType = 'short-input' | 'long-input' | 'dropdown' | 'slider' | 'table' | 'code' +export type SubBlockType = 'short-input' | 'long-input' | 'dropdown' | 'slider' | 'table' | 'code' | 'switch' export type SubBlockLayout = 'full' | 'half' export type OutputConfig = OutputType | { diff --git a/components/icons.tsx b/components/icons.tsx index a98cb6434..5ef496814 100644 --- a/components/icons.tsx +++ b/components/icons.tsx @@ -344,7 +344,7 @@ export function SectionIcon(props: SVGProps) { xmlns="http://www.w3.org/2000/svg" > ) { ) } + +export const FirecrawlIcon = (props: SVGProps) => ( + + + + +) diff --git a/components/ui/switch.tsx b/components/ui/switch.tsx new file mode 100644 index 000000000..c81058a3e --- /dev/null +++ b/components/ui/switch.tsx @@ -0,0 +1,26 @@ +import * as React from "react" +import * as SwitchPrimitives from "@radix-ui/react-switch" +import { cn } from "@/lib/utils" + +const Switch = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + + + +)) +Switch.displayName = SwitchPrimitives.Root.displayName + +export { Switch } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 23b82df9b..e55b72b4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@radix-ui/react-select": "^2.1.4", "@radix-ui/react-slider": "^1.2.2", "@radix-ui/react-slot": "^1.1.1", + "@radix-ui/react-switch": "^1.1.2", "@radix-ui/react-tabs": "^1.1.2", "@radix-ui/react-tooltip": "^1.1.6", "class-variance-authority": "^0.7.1", @@ -2568,6 +2569,35 @@ } } }, + "node_modules/@radix-ui/react-switch": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.1.2.tgz", + "integrity": "sha512-zGukiWHjEdBCRyXvKR6iXAQG6qXm2esuAD6kDOi9Cn+1X6ev3ASo4+CsYaD6Fov9r/AQFekqnD/7+V0Cs6/98g==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.1", + "@radix-ui/react-compose-refs": "1.1.1", + "@radix-ui/react-context": "1.1.1", + "@radix-ui/react-primitive": "2.0.1", + "@radix-ui/react-use-controllable-state": "1.1.0", + "@radix-ui/react-use-previous": "1.1.0", + "@radix-ui/react-use-size": "1.1.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-tabs": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.2.tgz", diff --git a/package.json b/package.json index 42f19fa73..5df96312b 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "@radix-ui/react-select": "^2.1.4", "@radix-ui/react-slider": "^1.2.2", "@radix-ui/react-slot": "^1.1.1", + "@radix-ui/react-switch": "^1.1.2", "@radix-ui/react-tabs": "^1.1.2", "@radix-ui/react-tooltip": "^1.1.6", "class-variance-authority": "^0.7.1", diff --git a/tools/firecrawl/scrape.ts b/tools/firecrawl/scrape.ts new file mode 100644 index 000000000..ceec66299 --- /dev/null +++ b/tools/firecrawl/scrape.ts @@ -0,0 +1,91 @@ +import { ToolConfig, ToolResponse } from '../types' + +interface ScrapeParams { + apiKey: string + url: string + scrapeOptions?: { + onlyMainContent?: boolean + formats?: string[] + } +} + +interface ScrapeResponse extends ToolResponse { + success: boolean + data: { + markdown: string + html?: string + metadata: { + title: string + description: string + language: string + keywords: string + robots: string + ogTitle: string + ogDescription: string + ogUrl: string + ogImage: string + ogLocaleAlternate: string[] + ogSiteName: string + sourceURL: string + statusCode: number + } + } +} + +export const scrapeTool: ToolConfig = { + id: 'firecrawl.scrape', + name: 'Firecrawl Website Scraper', + description: 'Extract clean content from any webpage in markdown format', + version: '1.0.0', + + params: { + apiKey: { + type: 'string', + required: true, + description: 'Firecrawl API key' + }, + url: { + type: 'string', + required: true, + description: 'The URL to scrape content from' + }, + scrapeOptions: { + type: 'json', + required: false, + description: 'Options for content scraping' + } + }, + + request: { + method: 'POST', + url: 'https://api.firecrawl.dev/v1/scrape', + headers: (params) => ({ + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${params.apiKey}` + }), + body: (params) => ({ + url: params.url, + formats: params.scrapeOptions?.formats || ["markdown"] + }) + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!data.success) { + throw new Error(data.error?.message || 'Unknown error occurred') + } + + return { + success: data.success, + data: data.data, + output: data.data.markdown + } + }, + + transformError: (error) => { + const message = error.error?.message || error.message + const code = error.error?.type || error.code + return `${message} (${code})` + } +} \ No newline at end of file diff --git a/tools/index.ts b/tools/index.ts index 69f77a68d..d31fd7ee5 100644 --- a/tools/index.ts +++ b/tools/index.ts @@ -1,5 +1,5 @@ -import { ToolConfig } from './types' -import { chatTool as openaiChat } from './openai/chat' +import { ToolConfig } from './types' +import { chatTool as openAIChat } from './openai/chat' import { chatTool as anthropicChat } from './anthropic/chat' import { chatTool as googleChat } from './google/chat' import { chatTool as xaiChat } from './xai/chat' @@ -8,13 +8,13 @@ import { reasonerTool as deepseekReasoner } from './deepseek/reasoner' import { requestTool as httpRequest } from './http/request' import { contactsTool as hubspotContacts } from './hubspot/contacts' import { opportunitiesTool as salesforceOpportunities } from './salesforce/opportunities' -import { functionExecuteTool as functionExecute } from './function/execute' +import { functionExecuteTool as functionExecute } from './function/execute' import { visionTool as crewAIVision } from './crewai/vision' - +import { scrapeTool } from './firecrawl/scrape' // Registry of all available tools export const tools: Record = { // AI Models - 'openai.chat': openaiChat, + 'openai.chat': openAIChat, 'anthropic.chat': anthropicChat, 'google.chat': googleChat, 'xai.chat': xaiChat, @@ -28,7 +28,9 @@ export const tools: Record = { // Function Tools 'function.execute': functionExecute, // CrewAI Tools - 'crewai.vision': crewAIVision + 'crewai.vision': crewAIVision, + // Firecrawl Tools + 'firecrawl.scrape': scrapeTool } // Get a tool by its ID @@ -65,8 +67,7 @@ export async function executeTool( throw new Error(tool.transformError(error)) } - const data = await response.json() - return tool.transformResponse(data) + return tool.transformResponse(response) } catch (error) { throw new Error(tool.transformError(error)) }