Compare commits

..

31 Commits

Author SHA1 Message Date
Waleed
dff1c9d083 v0.5.64: unsubscribe, search improvements, metrics, additional SSO configuration 2026-01-20 00:34:11 -08:00
Vikhyath Mondreti
b09f683072 v0.5.63: ui and performance improvements, more google tools 2026-01-18 15:22:42 -08:00
Vikhyath Mondreti
a8bb0db660 v0.5.62: webhook bug fixes, seeding default subblock values, block selection fixes 2026-01-16 20:27:06 -08:00
Waleed
af82820a28 v0.5.61: webhook improvements, workflow controls, react query for deployment status, chat fixes, reducto and pulse OCR, linear fixes 2026-01-16 18:06:23 -08:00
Waleed
4372841797 v0.5.60: invitation flow improvements, chat fixes, a2a improvements, additional copilot actions 2026-01-15 00:02:18 -08:00
Waleed
5e8c843241 v0.5.59: a2a support, documentation 2026-01-13 13:21:21 -08:00
Waleed
7bf3d73ee6 v0.5.58: export folders, new tools, permissions groups enhancements 2026-01-13 00:56:59 -08:00
Vikhyath Mondreti
7ffc11a738 v0.5.57: subagents, context menu improvements, bug fixes 2026-01-11 11:38:40 -08:00
Waleed
be578e2ed7 v0.5.56: batch operations, access control and permission groups, billing fixes 2026-01-10 00:31:34 -08:00
Waleed
f415e5edc4 v0.5.55: polling groups, bedrock provider, devcontainer fixes, workflow preview enhancements 2026-01-08 23:36:56 -08:00
Waleed
13a6e6c3fa v0.5.54: seo, model blacklist, helm chart updates, fireflies integration, autoconnect improvements, billing fixes 2026-01-07 16:09:45 -08:00
Waleed
f5ab7f21ae v0.5.53: hotkey improvements, added redis fallback, fixes for workflow tool 2026-01-06 23:34:52 -08:00
Waleed
bfb6fffe38 v0.5.52: new port-based router block, combobox expression and variable support 2026-01-06 16:14:10 -08:00
Waleed
4fbec0a43f v0.5.51: triggers, kb, condition block improvements, supabase and grain integration updates 2026-01-06 14:26:46 -08:00
Waleed
585f5e365b v0.5.50: import improvements, ui upgrades, kb styling and performance improvements 2026-01-05 00:35:55 -08:00
Waleed
3792bdd252 v0.5.49: hitl improvements, new email styles, imap trigger, logs context menu (#2672)
* feat(logs-context-menu): consolidated logs utils and types, added logs record context menu (#2659)

* feat(email): welcome email; improvement(emails): ui/ux (#2658)

* feat(email): welcome email; improvement(emails): ui/ux

* improvement(emails): links, accounts, preview

* refactor(emails): file structure and wrapper components

* added envvar for personal emails sent, added isHosted gate

* fixed failing tests, added env mock

* fix: removed comment

---------

Co-authored-by: waleed <walif6@gmail.com>

* fix(logging): hitl + trigger dev crash protection (#2664)

* hitl gaps

* deal with trigger worker crashes

* cleanup import strcuture

* feat(imap): added support for imap trigger (#2663)

* feat(tools): added support for imap trigger

* feat(imap): added parity, tested

* ack PR comments

* final cleanup

* feat(i18n): update translations (#2665)

Co-authored-by: waleedlatif1 <waleedlatif1@users.noreply.github.com>

* fix(grain): updated grain trigger to auto-establish trigger (#2666)

Co-authored-by: aadamgough <adam@sim.ai>

* feat(admin): routes to manage deployments (#2667)

* feat(admin): routes to manage deployments

* fix naming fo deployed by

* feat(time-picker): added timepicker emcn component, added to playground, added searchable prop for dropdown, added more timezones for schedule, updated license and notice date (#2668)

* feat(time-picker): added timepicker emcn component, added to playground, added searchable prop for dropdown, added more timezones for schedule, updated license and notice date

* removed unused params, cleaned up redundant utils

* improvement(invite): aligned styling (#2669)

* improvement(invite): aligned with rest of app

* fix(invite): error handling

* fix: addressed comments

---------

Co-authored-by: Emir Karabeg <78010029+emir-karabeg@users.noreply.github.com>
Co-authored-by: Vikhyath Mondreti <vikhyathvikku@gmail.com>
Co-authored-by: waleedlatif1 <waleedlatif1@users.noreply.github.com>
Co-authored-by: Adam Gough <77861281+aadamgough@users.noreply.github.com>
Co-authored-by: aadamgough <adam@sim.ai>
2026-01-03 13:19:18 -08:00
Waleed
eb5d1f3e5b v0.5.48: copy-paste workflow blocks, docs updates, mcp tool fixes 2025-12-31 18:00:04 -08:00
Waleed
54ab82c8dd v0.5.47: deploy workflow as mcp, kb chunks tokenizer, UI improvements, jira service management tools 2025-12-30 23:18:58 -08:00
Waleed
f895bf469b v0.5.46: build improvements, greptile, light mode improvements 2025-12-29 02:17:52 -08:00
Waleed
dd3209af06 v0.5.45: light mode fixes, realtime usage indicator, docker build improvements 2025-12-27 19:57:42 -08:00
Waleed
b6ba3b50a7 v0.5.44: keyboard shortcuts, autolayout, light mode, byok, testing improvements 2025-12-26 21:25:19 -08:00
Waleed
b304233062 v0.5.43: export logs, circleback, grain, vertex, code hygiene, schedule improvements 2025-12-23 19:19:18 -08:00
Vikhyath Mondreti
57e4b49bd6 v0.5.42: fix memory migration 2025-12-23 01:24:54 -08:00
Vikhyath Mondreti
e12dd204ed v0.5.41: memory fixes, copilot improvements, knowledgebase improvements, LLM providers standardization 2025-12-23 00:15:18 -08:00
Vikhyath Mondreti
3d9d9cbc54 v0.5.40: supabase ops to allow non-public schemas, jira uuid 2025-12-21 22:28:05 -08:00
Waleed
0f4ec962ad v0.5.39: notion, workflow variables fixes 2025-12-20 20:44:00 -08:00
Waleed
4827866f9a v0.5.38: snap to grid, copilot ux improvements, billing line items 2025-12-20 17:24:38 -08:00
Waleed
3e697d9ed9 v0.5.37: redaction utils consolidation, logs updates, autoconnect improvements, additional kb tag types 2025-12-19 22:31:55 -08:00
Martin Yankov
4431a1a484 fix(helm): add custom egress rules to realtime network policy (#2481)
The realtime service network policy was missing the custom egress rules section
that allows configuration of additional egress rules via values.yaml. This caused
the realtime pods to be unable to connect to external databases (e.g., PostgreSQL
on port 5432) when using external database configurations.

The app network policy already had this section, but the realtime network policy
was missing it, creating an inconsistency and preventing the realtime service
from accessing external databases configured via networkPolicy.egress values.

This fix adds the same custom egress rules template section to the realtime
network policy, matching the app network policy behavior and allowing users to
configure database connectivity via values.yaml.
2025-12-19 18:59:08 -08:00
Waleed
4d1a9a3f22 v0.5.36: hitl improvements, opengraph, slack fixes, one-click unsubscribe, auth checks, new db indexes 2025-12-19 01:27:49 -08:00
Vikhyath Mondreti
eb07a080fb v0.5.35: helm updates, copilot improvements, 404 for docs, salesforce fixes, subflow resize clamping 2025-12-18 16:23:19 -08:00
141 changed files with 1469 additions and 6213 deletions

View File

@@ -14,7 +14,7 @@
</p> </p>
<p align="center"> <p align="center">
<a href="https://deepwiki.com/simstudioai/sim" target="_blank" rel="noopener noreferrer"><img src="https://deepwiki.com/badge.svg" alt="Ask DeepWiki"></a> <a href="https://cursor.com/link/prompt?text=Help%20me%20set%20up%20Sim%20locally.%20Follow%20these%20steps%3A%0A%0A1.%20First%2C%20verify%20Docker%20is%20installed%20and%20running%3A%0A%20%20%20docker%20--version%0A%20%20%20docker%20info%0A%0A2.%20Clone%20the%20repository%3A%0A%20%20%20git%20clone%20https%3A%2F%2Fgithub.com%2Fsimstudioai%2Fsim.git%0A%20%20%20cd%20sim%0A%0A3.%20Start%20the%20services%20with%20Docker%20Compose%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20up%20-d%0A%0A4.%20Wait%20for%20all%20containers%20to%20be%20healthy%20(this%20may%20take%201-2%20minutes)%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20ps%0A%0A5.%20Verify%20the%20app%20is%20accessible%20at%20http%3A%2F%2Flocalhost%3A3000%0A%0AIf%20there%20are%20any%20errors%2C%20help%20me%20troubleshoot%20them.%20Common%20issues%3A%0A-%20Port%203000%2C%203002%2C%20or%205432%20already%20in%20use%0A-%20Docker%20not%20running%0A-%20Insufficient%20memory%20(needs%2012GB%2B%20RAM)%0A%0AFor%20local%20AI%20models%20with%20Ollama%2C%20use%20this%20instead%20of%20step%203%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.ollama.yml%20--profile%20setup%20up%20-d"><img src="https://img.shields.io/badge/Set%20Up%20with-Cursor-000000?logo=cursor&logoColor=white" alt="Set Up with Cursor"></a> <a href="https://deepwiki.com/simstudioai/sim" target="_blank" rel="noopener noreferrer"><img src="https://deepwiki.com/badge.svg" alt="Ask DeepWiki"></a> <a href="https://cursor.com/link/prompt?text=Help%20me%20set%20up%20Sim%20Studio%20locally.%20Follow%20these%20steps%3A%0A%0A1.%20First%2C%20verify%20Docker%20is%20installed%20and%20running%3A%0A%20%20%20docker%20--version%0A%20%20%20docker%20info%0A%0A2.%20Clone%20the%20repository%3A%0A%20%20%20git%20clone%20https%3A%2F%2Fgithub.com%2Fsimstudioai%2Fsim.git%0A%20%20%20cd%20sim%0A%0A3.%20Start%20the%20services%20with%20Docker%20Compose%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20up%20-d%0A%0A4.%20Wait%20for%20all%20containers%20to%20be%20healthy%20(this%20may%20take%201-2%20minutes)%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.prod.yml%20ps%0A%0A5.%20Verify%20the%20app%20is%20accessible%20at%20http%3A%2F%2Flocalhost%3A3000%0A%0AIf%20there%20are%20any%20errors%2C%20help%20me%20troubleshoot%20them.%20Common%20issues%3A%0A-%20Port%203000%2C%203002%2C%20or%205432%20already%20in%20use%0A-%20Docker%20not%20running%0A-%20Insufficient%20memory%20(needs%2012GB%2B%20RAM)%0A%0AFor%20local%20AI%20models%20with%20Ollama%2C%20use%20this%20instead%20of%20step%203%3A%0A%20%20%20docker%20compose%20-f%20docker-compose.ollama.yml%20--profile%20setup%20up%20-d"><img src="https://img.shields.io/badge/Set%20Up%20with-Cursor-000000?logo=cursor&logoColor=white" alt="Set Up with Cursor"></a>
</p> </p>
### Build Workflows with Ease ### Build Workflows with Ease

View File

@@ -4093,23 +4093,6 @@ export function SQSIcon(props: SVGProps<SVGSVGElement>) {
) )
} }
export function TextractIcon(props: SVGProps<SVGSVGElement>) {
return (
<svg
{...props}
viewBox='10 14 60 52'
version='1.1'
xmlns='http://www.w3.org/2000/svg'
xmlnsXlink='http://www.w3.org/1999/xlink'
>
<path
d='M22.0624102,50 C24.3763895,53.603 28.4103535,56 33.0003125,56 C40.1672485,56 45.9991964,50.168 45.9991964,43 C45.9991964,35.832 40.1672485,30 33.0003125,30 C27.6033607,30 22.9664021,33.307 21.0024196,38 L23.2143999,38 C25.0393836,34.444 28.7363506,32 33.0003125,32 C39.0652583,32 43.9992143,36.935 43.9992143,43 C43.9992143,49.065 39.0652583,54 33.0003125,54 C29.5913429,54 26.5413702,52.441 24.5213882,50 L22.0624102,50 Z M37.0002768,45 L37.0002768,43 L41.9992321,43 C41.9992321,38.038 37.9622682,34 33.0003125,34 C28.0373568,34 23.9993929,38.038 23.9993929,43 L28.9993482,43 L28.9993482,45 L24.2313908,45 C25.1443826,49.002 28.7253507,52 33.0003125,52 C35.1362934,52 37.0992759,51.249 38.6442621,50 L34.0003036,50 L34.0003036,48 L40.4782457,48 C41.0812403,47.102 41.5202364,46.087 41.7682342,45 L37.0002768,45 Z M21.0024196,48 L23.2143999,48 C22.4434068,46.498 22.0004107,44.801 22.0004107,43 C22.0004107,41.959 22.1554093,40.955 22.4264069,40 L20.3634253,40 C20.1344274,40.965 19.9994286,41.966 19.9994286,43 C19.9994286,44.771 20.3584254,46.46 21.0024196,48 L21.0024196,48 Z M19.7434309,50 L17.0004554,50 L17.0004554,48 L18.8744386,48 C18.5344417,47.04 18.2894438,46.038 18.1494451,45 L15.4144695,45 L16.707458,46.293 L15.2924706,47.707 L12.2924974,44.707 C11.9025009,44.316 11.9025009,43.684 12.2924974,43.293 L15.2924706,40.293 L16.707458,41.707 L15.4144695,43 L18.0004464,43 C18.0004464,41.973 18.1044455,40.97 18.3024437,40 L17.0004554,40 L17.0004554,38 L18.8744386,38 C20.9404202,32.184 26.4833707,28 33.0003125,28 C37.427273,28 41.4002375,29.939 44.148213,33 L59.0000804,33 L59.0000804,35 L45.6661994,35 C47.1351863,37.318 47.9991786,40.058 47.9991786,43 L59.0000804,43 L59.0000804,45 L47.8501799,45 C46.8681887,52.327 40.5912447,58 33.0003125,58 C27.2563638,58 22.2624084,54.752 19.7434309,50 L19.7434309,50 Z M37.0002768,39 C37.0002768,38.448 36.5522808,38 36.0002857,38 L29.9993482,38 C29.4473442,38 28.9993482,38.448 28.9993482,39 L28.9993482,41 L31.0003304,41 L31.0003304,40 L32.0003214,40 L32.0003214,43 L31.0003304,43 L31.0003304,45 L35.0002946,45 L35.0002946,43 L34.0003036,43 L34.0003036,40 L35.0002946,40 L35.0002946,41 L37.0002768,41 L37.0002768,39 Z M49.0001696,40 L59.0000804,40 L59.0000804,38 L49.0001696,38 L49.0001696,40 Z M49.0001696,50 L59.0000804,50 L59.0000804,48 L49.0001696,48 L49.0001696,50 Z M57.0000982,27 L60.5850662,27 L57.0000982,23.414 L57.0000982,27 Z M63.7070383,27.293 C63.8940367,27.48 64.0000357,27.735 64.0000357,28 L64.0000357,63 C64.0000357,63.552 63.5520397,64 63.0000446,64 L32.0003304,64 C31.4473264,64 31.0003304,63.552 31.0003304,63 L31.0003304,59 L33.0003125,59 L33.0003125,62 L62.0000536,62 L62.0000536,29 L56.0001071,29 C55.4471121,29 55.0001161,28.552 55.0001161,28 L55.0001161,22 L33.0003125,22 L33.0003125,27 L31.0003304,27 L31.0003304,21 C31.0003304,20.448 31.4473264,20 32.0003304,20 L56.0001071,20 C56.2651048,20 56.5191025,20.105 56.7071008,20.293 L63.7070383,27.293 Z M68,24.166 L68,61 C68,61.552 67.552004,62 67.0000089,62 L65.0000268,62 L65.0000268,60 L66.0000179,60 L66.0000179,24.612 L58.6170838,18 L36.0002857,18 L36.0002857,19 L34.0003036,19 L34.0003036,17 C34.0003036,16.448 34.4472996,16 35.0003036,16 L59.0000804,16 C59.2460782,16 59.483076,16.091 59.6660744,16.255 L67.666003,23.42 C67.8780011,23.61 68,23.881 68,24.166 L68,24.166 Z'
fill='currentColor'
/>
</svg>
)
}
export function McpIcon(props: SVGProps<SVGSVGElement>) { export function McpIcon(props: SVGProps<SVGSVGElement>) {
return ( return (
<svg <svg

View File

@@ -110,7 +110,6 @@ import {
SupabaseIcon, SupabaseIcon,
TavilyIcon, TavilyIcon,
TelegramIcon, TelegramIcon,
TextractIcon,
TinybirdIcon, TinybirdIcon,
TranslateIcon, TranslateIcon,
TrelloIcon, TrelloIcon,
@@ -144,7 +143,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
calendly: CalendlyIcon, calendly: CalendlyIcon,
circleback: CirclebackIcon, circleback: CirclebackIcon,
clay: ClayIcon, clay: ClayIcon,
confluence_v2: ConfluenceIcon, confluence: ConfluenceIcon,
cursor_v2: CursorIcon, cursor_v2: CursorIcon,
datadog: DatadogIcon, datadog: DatadogIcon,
discord: DiscordIcon, discord: DiscordIcon,
@@ -154,7 +153,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
elasticsearch: ElasticsearchIcon, elasticsearch: ElasticsearchIcon,
elevenlabs: ElevenLabsIcon, elevenlabs: ElevenLabsIcon,
exa: ExaAIIcon, exa: ExaAIIcon,
file_v2: DocumentIcon, file: DocumentIcon,
firecrawl: FirecrawlIcon, firecrawl: FirecrawlIcon,
fireflies: FirefliesIcon, fireflies: FirefliesIcon,
github_v2: GithubIcon, github_v2: GithubIcon,
@@ -196,7 +195,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
microsoft_excel_v2: MicrosoftExcelIcon, microsoft_excel_v2: MicrosoftExcelIcon,
microsoft_planner: MicrosoftPlannerIcon, microsoft_planner: MicrosoftPlannerIcon,
microsoft_teams: MicrosoftTeamsIcon, microsoft_teams: MicrosoftTeamsIcon,
mistral_parse_v2: MistralIcon, mistral_parse: MistralIcon,
mongodb: MongoDBIcon, mongodb: MongoDBIcon,
mysql: MySQLIcon, mysql: MySQLIcon,
neo4j: Neo4jIcon, neo4j: Neo4jIcon,
@@ -238,7 +237,6 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
supabase: SupabaseIcon, supabase: SupabaseIcon,
tavily: TavilyIcon, tavily: TavilyIcon,
telegram: TelegramIcon, telegram: TelegramIcon,
textract: TextractIcon,
tinybird: TinybirdIcon, tinybird: TinybirdIcon,
translate: TranslateIcon, translate: TranslateIcon,
trello: TrelloIcon, trello: TrelloIcon,
@@ -246,7 +244,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
twilio_sms: TwilioIcon, twilio_sms: TwilioIcon,
twilio_voice: TwilioIcon, twilio_voice: TwilioIcon,
typeform: TypeformIcon, typeform: TypeformIcon,
video_generator_v2: VideoIcon, video_generator: VideoIcon,
vision: EyeIcon, vision: EyeIcon,
wealthbox: WealthboxIcon, wealthbox: WealthboxIcon,
webflow: WebflowIcon, webflow: WebflowIcon,

View File

@@ -6,7 +6,7 @@ description: Interact with Confluence
import { BlockInfoCard } from "@/components/ui/block-info-card" import { BlockInfoCard } from "@/components/ui/block-info-card"
<BlockInfoCard <BlockInfoCard
type="confluence_v2" type="confluence"
color="#E0E0E0" color="#E0E0E0"
/> />

View File

@@ -6,7 +6,7 @@ description: Read and parse multiple files
import { BlockInfoCard } from "@/components/ui/block-info-card" import { BlockInfoCard } from "@/components/ui/block-info-card"
<BlockInfoCard <BlockInfoCard
type="file_v2" type="file"
color="#40916C" color="#40916C"
/> />
@@ -48,7 +48,7 @@ Parse one or more uploaded files or files from URLs (text, PDF, CSV, images, etc
| Parameter | Type | Description | | Parameter | Type | Description |
| --------- | ---- | ----------- | | --------- | ---- | ----------- |
| `files` | array | Array of parsed files with content, metadata, and file properties | | `files` | array | Array of parsed files |
| `combinedContent` | string | All file contents merged into a single text string | | `combinedContent` | string | Combined content of all parsed files |

View File

@@ -106,7 +106,6 @@
"supabase", "supabase",
"tavily", "tavily",
"telegram", "telegram",
"textract",
"tinybird", "tinybird",
"translate", "translate",
"trello", "trello",

View File

@@ -6,7 +6,7 @@ description: Extract text from PDF documents
import { BlockInfoCard } from "@/components/ui/block-info-card" import { BlockInfoCard } from "@/components/ui/block-info-card"
<BlockInfoCard <BlockInfoCard
type="mistral_parse_v2" type="mistral_parse"
color="#000000" color="#000000"
/> />
@@ -54,37 +54,18 @@ Parse PDF documents using Mistral OCR API
| Parameter | Type | Description | | Parameter | Type | Description |
| --------- | ---- | ----------- | | --------- | ---- | ----------- |
| `pages` | array | Array of page objects from Mistral OCR | | `success` | boolean | Whether the PDF was parsed successfully |
| ↳ `index` | number | Page index \(zero-based\) | | `content` | string | Extracted content in the requested format \(markdown, text, or JSON\) |
| ↳ `markdown` | string | Extracted markdown content | | `metadata` | object | Processing metadata including jobId, fileType, pageCount, and usage info |
| ↳ `images` | array | Images extracted from this page with bounding boxes | | ↳ `jobId` | string | Unique job identifier |
| ↳ `id` | string | Image identifier \(e.g., img-0.jpeg\) | | ↳ `fileType` | string | File type \(e.g., pdf\) |
| ↳ `top_left_x` | number | Top-left X coordinate in pixels | | ↳ `fileName` | string | Original file name |
| ↳ `top_left_y` | number | Top-left Y coordinate in pixels | | ↳ `source` | string | Source type \(url\) |
| ↳ `bottom_right_x` | number | Bottom-right X coordinate in pixels | | ↳ `pageCount` | number | Number of pages processed |
| ↳ `bottom_right_y` | number | Bottom-right Y coordinate in pixels | | ↳ `model` | string | Mistral model used |
| ↳ `image_base64` | string | Base64-encoded image data \(when include_image_base64=true\) | | ↳ `resultType` | string | Output format \(markdown, text, json\) |
| ↳ `id` | string | Image identifier \(e.g., img-0.jpeg\) | | ↳ `processedAt` | string | Processing timestamp |
| ↳ `top_left_x` | number | Top-left X coordinate in pixels | | ↳ `sourceUrl` | string | Source URL if applicable |
| ↳ `top_left_y` | number | Top-left Y coordinate in pixels | | ↳ `usageInfo` | object | Usage statistics from OCR processing |
| ↳ `bottom_right_x` | number | Bottom-right X coordinate in pixels |
| ↳ `bottom_right_y` | number | Bottom-right Y coordinate in pixels |
| ↳ `image_base64` | string | Base64-encoded image data \(when include_image_base64=true\) |
| ↳ `dimensions` | object | Page dimensions |
| ↳ `dpi` | number | Dots per inch |
| ↳ `height` | number | Page height in pixels |
| ↳ `width` | number | Page width in pixels |
| ↳ `dpi` | number | Dots per inch |
| ↳ `height` | number | Page height in pixels |
| ↳ `width` | number | Page width in pixels |
| ↳ `tables` | array | Extracted tables as HTML/markdown \(when table_format is set\). Referenced via placeholders like \[tbl-0.html\] |
| ↳ `hyperlinks` | array | Array of URL strings detected in the page \(e.g., \[ |
| ↳ `header` | string | Page header content \(when extract_header=true\) |
| ↳ `footer` | string | Page footer content \(when extract_footer=true\) |
| `model` | string | Mistral OCR model identifier \(e.g., mistral-ocr-latest\) |
| `usage_info` | object | Usage and processing statistics |
| ↳ `pages_processed` | number | Total number of pages processed |
| ↳ `doc_size_bytes` | number | Document file size in bytes |
| `document_annotation` | string | Structured annotation data as JSON string \(when applicable\) |

View File

@@ -58,7 +58,6 @@ Upload a file to an AWS S3 bucket
| Parameter | Type | Description | | Parameter | Type | Description |
| --------- | ---- | ----------- | | --------- | ---- | ----------- |
| `url` | string | URL of the uploaded S3 object | | `url` | string | URL of the uploaded S3 object |
| `uri` | string | S3 URI of the uploaded object \(s3://bucket/key\) |
| `metadata` | object | Upload metadata including ETag and location | | `metadata` | object | Upload metadata including ETag and location |
### `s3_get_object` ### `s3_get_object`
@@ -150,7 +149,6 @@ Copy an object within or between AWS S3 buckets
| Parameter | Type | Description | | Parameter | Type | Description |
| --------- | ---- | ----------- | | --------- | ---- | ----------- |
| `url` | string | URL of the copied S3 object | | `url` | string | URL of the copied S3 object |
| `uri` | string | S3 URI of the copied object \(s3://bucket/key\) |
| `metadata` | object | Copy operation metadata | | `metadata` | object | Copy operation metadata |

View File

@@ -1,120 +0,0 @@
---
title: AWS Textract
description: Extract text, tables, and forms from documents
---
import { BlockInfoCard } from "@/components/ui/block-info-card"
<BlockInfoCard
type="textract"
color="linear-gradient(135deg, #055F4E 0%, #56C0A7 100%)"
/>
{/* MANUAL-CONTENT-START:intro */}
[AWS Textract](https://aws.amazon.com/textract/) is a powerful AI service from Amazon Web Services designed to automatically extract printed text, handwriting, tables, forms, key-value pairs, and other structured data from scanned documents and images. Textract leverages advanced optical character recognition (OCR) and document analysis to transform documents into actionable data, enabling automation, analytics, compliance, and more.
With AWS Textract, you can:
- **Extract text from images and documents**: Recognize printed text and handwriting in formats such as PDF, JPEG, PNG, or TIFF
- **Detect and extract tables**: Automatically find tables and output their structured content
- **Parse forms and key-value pairs**: Pull structured data from forms, including fields and their corresponding values
- **Identify signatures and layout features**: Detect signatures, geometric layout, and relationships between document elements
- **Customize extraction with queries**: Extract specific fields and answers using query-based extraction (e.g., "What is the invoice number?")
In Sim, the AWS Textract integration empowers your agents to intelligently process documents as part of their workflows. This unlocks automation scenarios such as data entry from invoices, onboarding documents, contracts, receipts, and more. Your agents can extract relevant data, analyze structured forms, and generate summaries or reports directly from document uploads or URLs. By connecting Sim with AWS Textract, you can reduce manual effort, improve data accuracy, and streamline your business processes with robust document understanding.
{/* MANUAL-CONTENT-END */}
## Usage Instructions
Integrate AWS Textract into your workflow to extract text, tables, forms, and key-value pairs from documents. Single-page mode supports JPEG, PNG, and single-page PDF. Multi-page mode supports multi-page PDF and TIFF.
## Tools
### `textract_parser`
Parse documents using AWS Textract OCR and document analysis
#### Input
| Parameter | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `accessKeyId` | string | Yes | AWS Access Key ID |
| `secretAccessKey` | string | Yes | AWS Secret Access Key |
| `region` | string | Yes | AWS region for Textract service \(e.g., us-east-1\) |
| `processingMode` | string | No | Document type: single-page or multi-page. Defaults to single-page. |
| `filePath` | string | No | URL to a document to be processed \(JPEG, PNG, or single-page PDF\). |
| `s3Uri` | string | No | S3 URI for multi-page processing \(s3://bucket/key\). |
| `fileUpload` | object | No | File upload data from file-upload component |
| `featureTypes` | array | No | Feature types to detect: TABLES, FORMS, QUERIES, SIGNATURES, LAYOUT. If not specified, only text detection is performed. |
| `items` | string | No | Feature type |
| `queries` | array | No | Custom queries to extract specific information. Only used when featureTypes includes QUERIES. |
| `items` | object | No | Query configuration |
| `properties` | string | No | The query text |
| `Text` | string | No | No description |
| `Alias` | string | No | No description |
#### Output
| Parameter | Type | Description |
| --------- | ---- | ----------- |
| `blocks` | array | Array of Block objects containing detected text, tables, forms, and other elements |
| ↳ `BlockType` | string | Type of block \(PAGE, LINE, WORD, TABLE, CELL, KEY_VALUE_SET, etc.\) |
| ↳ `Id` | string | Unique identifier for the block |
| ↳ `Text` | string | Query text |
| ↳ `TextType` | string | Type of text \(PRINTED or HANDWRITING\) |
| ↳ `Confidence` | number | Confidence score \(0-100\) |
| ↳ `Page` | number | Page number |
| ↳ `Geometry` | object | Location and bounding box information |
| ↳ `BoundingBox` | object | Height as ratio of document height |
| ↳ `Height` | number | Height as ratio of document height |
| ↳ `Left` | number | Left position as ratio of document width |
| ↳ `Top` | number | Top position as ratio of document height |
| ↳ `Width` | number | Width as ratio of document width |
| ↳ `Height` | number | Height as ratio of document height |
| ↳ `Left` | number | Left position as ratio of document width |
| ↳ `Top` | number | Top position as ratio of document height |
| ↳ `Width` | number | Width as ratio of document width |
| ↳ `Polygon` | array | Polygon coordinates |
| ↳ `X` | number | X coordinate |
| ↳ `Y` | number | Y coordinate |
| ↳ `X` | number | X coordinate |
| ↳ `Y` | number | Y coordinate |
| ↳ `BoundingBox` | object | Height as ratio of document height |
| ↳ `Height` | number | Height as ratio of document height |
| ↳ `Left` | number | Left position as ratio of document width |
| ↳ `Top` | number | Top position as ratio of document height |
| ↳ `Width` | number | Width as ratio of document width |
| ↳ `Height` | number | Height as ratio of document height |
| ↳ `Left` | number | Left position as ratio of document width |
| ↳ `Top` | number | Top position as ratio of document height |
| ↳ `Width` | number | Width as ratio of document width |
| ↳ `Polygon` | array | Polygon coordinates |
| ↳ `X` | number | X coordinate |
| ↳ `Y` | number | Y coordinate |
| ↳ `X` | number | X coordinate |
| ↳ `Y` | number | Y coordinate |
| ↳ `Relationships` | array | Relationships to other blocks |
| ↳ `Type` | string | Relationship type \(CHILD, VALUE, ANSWER, etc.\) |
| ↳ `Ids` | array | IDs of related blocks |
| ↳ `Type` | string | Relationship type \(CHILD, VALUE, ANSWER, etc.\) |
| ↳ `Ids` | array | IDs of related blocks |
| ↳ `EntityTypes` | array | Entity types for KEY_VALUE_SET \(KEY or VALUE\) |
| ↳ `SelectionStatus` | string | For checkboxes: SELECTED or NOT_SELECTED |
| ↳ `RowIndex` | number | Row index for table cells |
| ↳ `ColumnIndex` | number | Column index for table cells |
| ↳ `RowSpan` | number | Row span for merged cells |
| ↳ `ColumnSpan` | number | Column span for merged cells |
| ↳ `Query` | object | Query information for QUERY blocks |
| ↳ `Text` | string | Query text |
| ↳ `Alias` | string | Query alias |
| ↳ `Pages` | array | Pages to search |
| ↳ `Alias` | string | Query alias |
| ↳ `Pages` | array | Pages to search |
| `documentMetadata` | object | Metadata about the analyzed document |
| ↳ `pages` | number | Number of pages in the document |
| `modelVersion` | string | Version of the Textract model used for processing |

View File

@@ -6,7 +6,7 @@ description: Generate videos from text using AI
import { BlockInfoCard } from "@/components/ui/block-info-card" import { BlockInfoCard } from "@/components/ui/block-info-card"
<BlockInfoCard <BlockInfoCard
type="video_generator_v2" type="video_generator"
color="#181C1E" color="#181C1E"
/> />

View File

@@ -224,7 +224,7 @@ export async function POST(req: NextRequest) {
hasApiKey: !!executionParams.apiKey, hasApiKey: !!executionParams.apiKey,
}) })
const result = await executeTool(resolvedToolName, executionParams) const result = await executeTool(resolvedToolName, executionParams, true)
logger.info(`[${tracker.requestId}] Tool execution complete`, { logger.info(`[${tracker.requestId}] Tool execution complete`, {
toolName, toolName,

View File

@@ -6,10 +6,9 @@ import { createLogger } from '@sim/logger'
import binaryExtensionsList from 'binary-extensions' import binaryExtensionsList from 'binary-extensions'
import { type NextRequest, NextResponse } from 'next/server' import { type NextRequest, NextResponse } from 'next/server'
import { checkHybridAuth } from '@/lib/auth/hybrid' import { checkHybridAuth } from '@/lib/auth/hybrid'
import { secureFetchWithPinnedIP, validateUrlWithDNS } from '@/lib/core/security/input-validation' import { createPinnedUrl, validateUrlWithDNS } from '@/lib/core/security/input-validation'
import { isSupportedFileType, parseFile } from '@/lib/file-parsers' import { isSupportedFileType, parseFile } from '@/lib/file-parsers'
import { isUsingCloudStorage, type StorageContext, StorageService } from '@/lib/uploads' import { isUsingCloudStorage, type StorageContext, StorageService } from '@/lib/uploads'
import { uploadExecutionFile } from '@/lib/uploads/contexts/execution'
import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server'
import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' import { getFileMetadataByKey } from '@/lib/uploads/server/metadata'
import { import {
@@ -22,7 +21,6 @@ import {
} from '@/lib/uploads/utils/file-utils' } from '@/lib/uploads/utils/file-utils'
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
import { verifyFileAccess } from '@/app/api/files/authorization' import { verifyFileAccess } from '@/app/api/files/authorization'
import type { UserFile } from '@/executor/types'
import '@/lib/uploads/core/setup.server' import '@/lib/uploads/core/setup.server'
export const dynamic = 'force-dynamic' export const dynamic = 'force-dynamic'
@@ -32,12 +30,6 @@ const logger = createLogger('FilesParseAPI')
const MAX_DOWNLOAD_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB const MAX_DOWNLOAD_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB
const DOWNLOAD_TIMEOUT_MS = 30000 // 30 seconds const DOWNLOAD_TIMEOUT_MS = 30000 // 30 seconds
interface ExecutionContext {
workspaceId: string
workflowId: string
executionId: string
}
interface ParseResult { interface ParseResult {
success: boolean success: boolean
content?: string content?: string
@@ -45,7 +37,6 @@ interface ParseResult {
filePath: string filePath: string
originalName?: string // Original filename from database (for workspace files) originalName?: string // Original filename from database (for workspace files)
viewerUrl?: string | null // Viewer URL for the file if available viewerUrl?: string | null // Viewer URL for the file if available
userFile?: UserFile // UserFile object for the raw file
metadata?: { metadata?: {
fileType: string fileType: string
size: number size: number
@@ -79,45 +70,27 @@ export async function POST(request: NextRequest) {
const userId = authResult.userId const userId = authResult.userId
const requestData = await request.json() const requestData = await request.json()
const { filePath, fileType, workspaceId, workflowId, executionId } = requestData const { filePath, fileType, workspaceId } = requestData
if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) { if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) {
return NextResponse.json({ success: false, error: 'No file path provided' }, { status: 400 }) return NextResponse.json({ success: false, error: 'No file path provided' }, { status: 400 })
} }
// Build execution context if all required fields are present logger.info('File parse request received:', { filePath, fileType, workspaceId, userId })
const executionContext: ExecutionContext | undefined =
workspaceId && workflowId && executionId
? { workspaceId, workflowId, executionId }
: undefined
logger.info('File parse request received:', {
filePath,
fileType,
workspaceId,
userId,
hasExecutionContext: !!executionContext,
})
if (Array.isArray(filePath)) { if (Array.isArray(filePath)) {
const results = [] const results = []
for (const singlePath of filePath) { for (const path of filePath) {
if (!singlePath || (typeof singlePath === 'string' && singlePath.trim() === '')) { if (!path || (typeof path === 'string' && path.trim() === '')) {
results.push({ results.push({
success: false, success: false,
error: 'Empty file path in array', error: 'Empty file path in array',
filePath: singlePath || '', filePath: path || '',
}) })
continue continue
} }
const result = await parseFileSingle( const result = await parseFileSingle(path, fileType, workspaceId, userId)
singlePath,
fileType,
workspaceId,
userId,
executionContext
)
if (result.metadata) { if (result.metadata) {
result.metadata.processingTime = Date.now() - startTime result.metadata.processingTime = Date.now() - startTime
} }
@@ -133,7 +106,6 @@ export async function POST(request: NextRequest) {
fileType: result.metadata?.fileType || 'application/octet-stream', fileType: result.metadata?.fileType || 'application/octet-stream',
size: result.metadata?.size || 0, size: result.metadata?.size || 0,
binary: false, binary: false,
file: result.userFile,
}, },
filePath: result.filePath, filePath: result.filePath,
viewerUrl: result.viewerUrl, viewerUrl: result.viewerUrl,
@@ -149,7 +121,7 @@ export async function POST(request: NextRequest) {
}) })
} }
const result = await parseFileSingle(filePath, fileType, workspaceId, userId, executionContext) const result = await parseFileSingle(filePath, fileType, workspaceId, userId)
if (result.metadata) { if (result.metadata) {
result.metadata.processingTime = Date.now() - startTime result.metadata.processingTime = Date.now() - startTime
@@ -165,7 +137,6 @@ export async function POST(request: NextRequest) {
fileType: result.metadata?.fileType || 'application/octet-stream', fileType: result.metadata?.fileType || 'application/octet-stream',
size: result.metadata?.size || 0, size: result.metadata?.size || 0,
binary: false, binary: false,
file: result.userFile,
}, },
filePath: result.filePath, filePath: result.filePath,
viewerUrl: result.viewerUrl, viewerUrl: result.viewerUrl,
@@ -193,8 +164,7 @@ async function parseFileSingle(
filePath: string, filePath: string,
fileType: string, fileType: string,
workspaceId: string, workspaceId: string,
userId: string, userId: string
executionContext?: ExecutionContext
): Promise<ParseResult> { ): Promise<ParseResult> {
logger.info('Parsing file:', filePath) logger.info('Parsing file:', filePath)
@@ -216,18 +186,18 @@ async function parseFileSingle(
} }
if (filePath.includes('/api/files/serve/')) { if (filePath.includes('/api/files/serve/')) {
return handleCloudFile(filePath, fileType, undefined, userId, executionContext) return handleCloudFile(filePath, fileType, undefined, userId)
} }
if (filePath.startsWith('http://') || filePath.startsWith('https://')) { if (filePath.startsWith('http://') || filePath.startsWith('https://')) {
return handleExternalUrl(filePath, fileType, workspaceId, userId, executionContext) return handleExternalUrl(filePath, fileType, workspaceId, userId)
} }
if (isUsingCloudStorage()) { if (isUsingCloudStorage()) {
return handleCloudFile(filePath, fileType, undefined, userId, executionContext) return handleCloudFile(filePath, fileType, undefined, userId)
} }
return handleLocalFile(filePath, fileType, userId, executionContext) return handleLocalFile(filePath, fileType, userId)
} }
/** /**
@@ -260,14 +230,12 @@ function validateFilePath(filePath: string): { isValid: boolean; error?: string
/** /**
* Handle external URL * Handle external URL
* If workspaceId is provided, checks if file already exists and saves to workspace if not * If workspaceId is provided, checks if file already exists and saves to workspace if not
* If executionContext is provided, also stores the file in execution storage and returns UserFile
*/ */
async function handleExternalUrl( async function handleExternalUrl(
url: string, url: string,
fileType: string, fileType: string,
workspaceId: string, workspaceId: string,
userId: string, userId: string
executionContext?: ExecutionContext
): Promise<ParseResult> { ): Promise<ParseResult> {
try { try {
logger.info('Fetching external URL:', url) logger.info('Fetching external URL:', url)
@@ -344,13 +312,17 @@ async function handleExternalUrl(
if (existingFile) { if (existingFile) {
const storageFilePath = `/api/files/serve/${existingFile.key}` const storageFilePath = `/api/files/serve/${existingFile.key}`
return handleCloudFile(storageFilePath, fileType, 'workspace', userId, executionContext) return handleCloudFile(storageFilePath, fileType, 'workspace', userId)
} }
} }
} }
const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { const pinnedUrl = createPinnedUrl(url, urlValidation.resolvedIP!)
timeout: DOWNLOAD_TIMEOUT_MS, const response = await fetch(pinnedUrl, {
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
headers: {
Host: urlValidation.originalHostname!,
},
}) })
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`) throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`)
@@ -369,19 +341,6 @@ async function handleExternalUrl(
logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`) logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`)
let userFile: UserFile | undefined
const mimeType = response.headers.get('content-type') || getMimeTypeFromExtension(extension)
if (executionContext) {
try {
userFile = await uploadExecutionFile(executionContext, buffer, filename, mimeType, userId)
logger.info(`Stored file in execution storage: ${filename}`, { key: userFile.key })
} catch (uploadError) {
logger.warn(`Failed to store file in execution storage:`, uploadError)
// Continue without userFile - parsing can still work
}
}
if (shouldCheckWorkspace) { if (shouldCheckWorkspace) {
try { try {
const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId)
@@ -394,6 +353,8 @@ async function handleExternalUrl(
}) })
} else { } else {
const { uploadWorkspaceFile } = await import('@/lib/uploads/contexts/workspace') const { uploadWorkspaceFile } = await import('@/lib/uploads/contexts/workspace')
const mimeType =
response.headers.get('content-type') || getMimeTypeFromExtension(extension)
await uploadWorkspaceFile(workspaceId, userId, buffer, filename, mimeType) await uploadWorkspaceFile(workspaceId, userId, buffer, filename, mimeType)
logger.info(`Saved URL file to workspace storage: ${filename}`) logger.info(`Saved URL file to workspace storage: ${filename}`)
} }
@@ -402,23 +363,17 @@ async function handleExternalUrl(
} }
} }
let parseResult: ParseResult
if (extension === 'pdf') { if (extension === 'pdf') {
parseResult = await handlePdfBuffer(buffer, filename, fileType, url) return await handlePdfBuffer(buffer, filename, fileType, url)
} else if (extension === 'csv') { }
parseResult = await handleCsvBuffer(buffer, filename, fileType, url) if (extension === 'csv') {
} else if (isSupportedFileType(extension)) { return await handleCsvBuffer(buffer, filename, fileType, url)
parseResult = await handleGenericTextBuffer(buffer, filename, extension, fileType, url) }
} else { if (isSupportedFileType(extension)) {
parseResult = handleGenericBuffer(buffer, filename, extension, fileType) return await handleGenericTextBuffer(buffer, filename, extension, fileType, url)
} }
// Attach userFile to the result return handleGenericBuffer(buffer, filename, extension, fileType)
if (userFile) {
parseResult.userFile = userFile
}
return parseResult
} catch (error) { } catch (error) {
logger.error(`Error handling external URL ${url}:`, error) logger.error(`Error handling external URL ${url}:`, error)
return { return {
@@ -431,15 +386,12 @@ async function handleExternalUrl(
/** /**
* Handle file stored in cloud storage * Handle file stored in cloud storage
* If executionContext is provided and file is not already from execution storage,
* copies the file to execution storage and returns UserFile
*/ */
async function handleCloudFile( async function handleCloudFile(
filePath: string, filePath: string,
fileType: string, fileType: string,
explicitContext: string | undefined, explicitContext: string | undefined,
userId: string, userId: string
executionContext?: ExecutionContext
): Promise<ParseResult> { ): Promise<ParseResult> {
try { try {
const cloudKey = extractStorageKey(filePath) const cloudKey = extractStorageKey(filePath)
@@ -486,7 +438,6 @@ async function handleCloudFile(
const filename = originalFilename || cloudKey.split('/').pop() || cloudKey const filename = originalFilename || cloudKey.split('/').pop() || cloudKey
const extension = path.extname(filename).toLowerCase().substring(1) const extension = path.extname(filename).toLowerCase().substring(1)
const mimeType = getMimeTypeFromExtension(extension)
const normalizedFilePath = `/api/files/serve/${encodeURIComponent(cloudKey)}?context=${context}` const normalizedFilePath = `/api/files/serve/${encodeURIComponent(cloudKey)}?context=${context}`
let workspaceIdFromKey: string | undefined let workspaceIdFromKey: string | undefined
@@ -502,39 +453,6 @@ async function handleCloudFile(
const viewerUrl = getViewerUrl(cloudKey, workspaceIdFromKey) const viewerUrl = getViewerUrl(cloudKey, workspaceIdFromKey)
// Store file in execution storage if executionContext is provided
let userFile: UserFile | undefined
if (executionContext) {
// If file is already from execution context, create UserFile reference without re-uploading
if (context === 'execution') {
userFile = {
id: `file_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`,
name: filename,
url: normalizedFilePath,
size: fileBuffer.length,
type: mimeType,
key: cloudKey,
context: 'execution',
}
logger.info(`Created UserFile reference for existing execution file: ${filename}`)
} else {
// Copy from workspace/other storage to execution storage
try {
userFile = await uploadExecutionFile(
executionContext,
fileBuffer,
filename,
mimeType,
userId
)
logger.info(`Copied file to execution storage: ${filename}`, { key: userFile.key })
} catch (uploadError) {
logger.warn(`Failed to copy file to execution storage:`, uploadError)
}
}
}
let parseResult: ParseResult let parseResult: ParseResult
if (extension === 'pdf') { if (extension === 'pdf') {
parseResult = await handlePdfBuffer(fileBuffer, filename, fileType, normalizedFilePath) parseResult = await handlePdfBuffer(fileBuffer, filename, fileType, normalizedFilePath)
@@ -559,11 +477,6 @@ async function handleCloudFile(
parseResult.viewerUrl = viewerUrl parseResult.viewerUrl = viewerUrl
// Attach userFile to the result
if (userFile) {
parseResult.userFile = userFile
}
return parseResult return parseResult
} catch (error) { } catch (error) {
logger.error(`Error handling cloud file ${filePath}:`, error) logger.error(`Error handling cloud file ${filePath}:`, error)
@@ -587,8 +500,7 @@ async function handleCloudFile(
async function handleLocalFile( async function handleLocalFile(
filePath: string, filePath: string,
fileType: string, fileType: string,
userId: string, userId: string
executionContext?: ExecutionContext
): Promise<ParseResult> { ): Promise<ParseResult> {
try { try {
const filename = filePath.split('/').pop() || filePath const filename = filePath.split('/').pop() || filePath
@@ -628,32 +540,13 @@ async function handleLocalFile(
const hash = createHash('md5').update(fileBuffer).digest('hex') const hash = createHash('md5').update(fileBuffer).digest('hex')
const extension = path.extname(filename).toLowerCase().substring(1) const extension = path.extname(filename).toLowerCase().substring(1)
const mimeType = fileType || getMimeTypeFromExtension(extension)
// Store file in execution storage if executionContext is provided
let userFile: UserFile | undefined
if (executionContext) {
try {
userFile = await uploadExecutionFile(
executionContext,
fileBuffer,
filename,
mimeType,
userId
)
logger.info(`Stored local file in execution storage: ${filename}`, { key: userFile.key })
} catch (uploadError) {
logger.warn(`Failed to store local file in execution storage:`, uploadError)
}
}
return { return {
success: true, success: true,
content: result.content, content: result.content,
filePath, filePath,
userFile,
metadata: { metadata: {
fileType: mimeType, fileType: fileType || getMimeTypeFromExtension(extension),
size: stats.size, size: stats.size,
hash, hash,
processingTime: 0, processingTime: 0,

View File

@@ -0,0 +1,395 @@
import { createLogger } from '@sim/logger'
import type { NextRequest } from 'next/server'
import { NextResponse } from 'next/server'
import { z } from 'zod'
import { checkHybridAuth } from '@/lib/auth/hybrid'
import { generateInternalToken } from '@/lib/auth/internal'
import { isDev } from '@/lib/core/config/feature-flags'
import { createPinnedUrl, validateUrlWithDNS } from '@/lib/core/security/input-validation'
import { generateRequestId } from '@/lib/core/utils/request'
import { getBaseUrl } from '@/lib/core/utils/urls'
import { executeTool } from '@/tools'
import { getTool, validateRequiredParametersAfterMerge } from '@/tools/utils'
const logger = createLogger('ProxyAPI')
const proxyPostSchema = z.object({
toolId: z.string().min(1, 'toolId is required'),
params: z.record(z.any()).optional().default({}),
executionContext: z
.object({
workflowId: z.string().optional(),
workspaceId: z.string().optional(),
executionId: z.string().optional(),
userId: z.string().optional(),
})
.optional(),
})
/**
* Creates a minimal set of default headers for proxy requests
* @returns Record of HTTP headers
*/
const getProxyHeaders = (): Record<string, string> => {
return {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
Accept: '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
}
}
/**
* Formats a response with CORS headers
* @param responseData Response data object
* @param status HTTP status code
* @returns NextResponse with CORS headers
*/
const formatResponse = (responseData: any, status = 200) => {
return NextResponse.json(responseData, {
status,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type, Authorization',
},
})
}
/**
* Creates an error response with consistent formatting
* @param error Error object or message
* @param status HTTP status code
* @param additionalData Additional data to include in the response
* @returns Formatted error response
*/
const createErrorResponse = (error: any, status = 500, additionalData = {}) => {
const errorMessage = error instanceof Error ? error.message : String(error)
const errorStack = error instanceof Error ? error.stack : undefined
logger.error('Creating error response', {
errorMessage,
status,
stack: isDev ? errorStack : undefined,
})
return formatResponse(
{
success: false,
error: errorMessage,
stack: isDev ? errorStack : undefined,
...additionalData,
},
status
)
}
/**
* GET handler for direct external URL proxying
* This allows for GET requests to external APIs
*/
export async function GET(request: Request) {
const url = new URL(request.url)
const targetUrl = url.searchParams.get('url')
const requestId = generateRequestId()
// Vault download proxy: /api/proxy?vaultDownload=1&bucket=...&object=...&credentialId=...
const vaultDownload = url.searchParams.get('vaultDownload')
if (vaultDownload === '1') {
try {
const bucket = url.searchParams.get('bucket')
const objectParam = url.searchParams.get('object')
const credentialId = url.searchParams.get('credentialId')
if (!bucket || !objectParam || !credentialId) {
return createErrorResponse('Missing bucket, object, or credentialId', 400)
}
// Fetch access token using existing token API
const baseUrl = new URL(getBaseUrl())
const tokenUrl = new URL('/api/auth/oauth/token', baseUrl)
// Build headers: forward session cookies if present; include internal auth for server-side
const tokenHeaders: Record<string, string> = { 'Content-Type': 'application/json' }
const incomingCookie = request.headers.get('cookie')
if (incomingCookie) tokenHeaders.Cookie = incomingCookie
try {
const internalToken = await generateInternalToken()
tokenHeaders.Authorization = `Bearer ${internalToken}`
} catch (_e) {
// best-effort internal auth
}
// Optional workflow context for collaboration auth
const workflowId = url.searchParams.get('workflowId') || undefined
const tokenRes = await fetch(tokenUrl.toString(), {
method: 'POST',
headers: tokenHeaders,
body: JSON.stringify({ credentialId, workflowId }),
})
if (!tokenRes.ok) {
const err = await tokenRes.text()
return createErrorResponse(`Failed to fetch access token: ${err}`, 401)
}
const tokenJson = await tokenRes.json()
const accessToken = tokenJson.accessToken
if (!accessToken) {
return createErrorResponse('No access token available', 401)
}
// Avoid double-encoding: incoming object may already be percent-encoded
const objectDecoded = decodeURIComponent(objectParam)
const gcsUrl = `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(
bucket
)}/o/${encodeURIComponent(objectDecoded)}?alt=media`
const fileRes = await fetch(gcsUrl, {
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!fileRes.ok) {
const errText = await fileRes.text()
return createErrorResponse(errText || 'Failed to download file', fileRes.status)
}
const headers = new Headers()
fileRes.headers.forEach((v, k) => headers.set(k, v))
return new NextResponse(fileRes.body, { status: 200, headers })
} catch (error: any) {
logger.error(`[${requestId}] Vault download proxy failed`, {
error: error instanceof Error ? error.message : String(error),
})
return createErrorResponse('Vault download failed', 500)
}
}
if (!targetUrl) {
logger.error(`[${requestId}] Missing 'url' parameter`)
return createErrorResponse("Missing 'url' parameter", 400)
}
const urlValidation = await validateUrlWithDNS(targetUrl)
if (!urlValidation.isValid) {
logger.warn(`[${requestId}] Blocked proxy request`, {
url: targetUrl.substring(0, 100),
error: urlValidation.error,
})
return createErrorResponse(urlValidation.error || 'Invalid URL', 403)
}
const method = url.searchParams.get('method') || 'GET'
const bodyParam = url.searchParams.get('body')
let body: string | undefined
if (bodyParam && ['POST', 'PUT', 'PATCH'].includes(method.toUpperCase())) {
try {
body = decodeURIComponent(bodyParam)
} catch (error) {
logger.warn(`[${requestId}] Failed to decode body parameter`, error)
}
}
const customHeaders: Record<string, string> = {}
for (const [key, value] of url.searchParams.entries()) {
if (key.startsWith('header.')) {
const headerName = key.substring(7)
customHeaders[headerName] = value
}
}
if (body && !customHeaders['Content-Type']) {
customHeaders['Content-Type'] = 'application/json'
}
logger.info(`[${requestId}] Proxying ${method} request to: ${targetUrl}`)
try {
const pinnedUrl = createPinnedUrl(targetUrl, urlValidation.resolvedIP!)
const response = await fetch(pinnedUrl, {
method: method,
headers: {
...getProxyHeaders(),
...customHeaders,
Host: urlValidation.originalHostname!,
},
body: body || undefined,
})
const contentType = response.headers.get('content-type') || ''
let data
if (contentType.includes('application/json')) {
data = await response.json()
} else {
data = await response.text()
}
const errorMessage = !response.ok
? data && typeof data === 'object' && data.error
? `${data.error.message || JSON.stringify(data.error)}`
: response.statusText || `HTTP error ${response.status}`
: undefined
if (!response.ok) {
logger.error(`[${requestId}] External API error: ${response.status} ${response.statusText}`)
}
return formatResponse({
success: response.ok,
status: response.status,
statusText: response.statusText,
headers: Object.fromEntries(response.headers.entries()),
data,
error: errorMessage,
})
} catch (error: any) {
logger.error(`[${requestId}] Proxy GET request failed`, {
url: targetUrl,
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined,
})
return createErrorResponse(error)
}
}
export async function POST(request: NextRequest) {
const requestId = generateRequestId()
const startTime = new Date()
const startTimeISO = startTime.toISOString()
try {
const authResult = await checkHybridAuth(request, { requireWorkflowId: false })
if (!authResult.success) {
logger.error(`[${requestId}] Authentication failed for proxy:`, authResult.error)
return createErrorResponse('Unauthorized', 401)
}
let requestBody
try {
requestBody = await request.json()
} catch (parseError) {
logger.error(`[${requestId}] Failed to parse request body`, {
error: parseError instanceof Error ? parseError.message : String(parseError),
})
throw new Error('Invalid JSON in request body')
}
const validationResult = proxyPostSchema.safeParse(requestBody)
if (!validationResult.success) {
logger.error(`[${requestId}] Request validation failed`, {
errors: validationResult.error.errors,
})
const errorMessages = validationResult.error.errors
.map((err) => `${err.path.join('.')}: ${err.message}`)
.join(', ')
throw new Error(`Validation failed: ${errorMessages}`)
}
const { toolId, params } = validationResult.data
logger.info(`[${requestId}] Processing tool: ${toolId}`)
const tool = getTool(toolId)
if (!tool) {
logger.error(`[${requestId}] Tool not found: ${toolId}`)
throw new Error(`Tool not found: ${toolId}`)
}
try {
validateRequiredParametersAfterMerge(toolId, tool, params)
} catch (validationError) {
logger.warn(`[${requestId}] Tool validation failed for ${toolId}`, {
error: validationError instanceof Error ? validationError.message : String(validationError),
})
const endTime = new Date()
const endTimeISO = endTime.toISOString()
const duration = endTime.getTime() - startTime.getTime()
return createErrorResponse(validationError, 400, {
startTime: startTimeISO,
endTime: endTimeISO,
duration,
})
}
const hasFileOutputs =
tool.outputs &&
Object.values(tool.outputs).some(
(output) => output.type === 'file' || output.type === 'file[]'
)
const result = await executeTool(
toolId,
params,
true, // skipProxy (we're already in the proxy)
!hasFileOutputs, // skipPostProcess (don't skip if tool has file outputs)
undefined // execution context is not available in proxy context
)
if (!result.success) {
logger.warn(`[${requestId}] Tool execution failed for ${toolId}`, {
error: result.error || 'Unknown error',
})
throw new Error(result.error || 'Tool execution failed')
}
const endTime = new Date()
const endTimeISO = endTime.toISOString()
const duration = endTime.getTime() - startTime.getTime()
const responseWithTimingData = {
...result,
startTime: startTimeISO,
endTime: endTimeISO,
duration,
timing: {
startTime: startTimeISO,
endTime: endTimeISO,
duration,
},
}
logger.info(`[${requestId}] Tool executed successfully: ${toolId} (${duration}ms)`)
return formatResponse(responseWithTimingData)
} catch (error: any) {
logger.error(`[${requestId}] Proxy request failed`, {
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined,
name: error instanceof Error ? error.name : undefined,
})
const endTime = new Date()
const endTimeISO = endTime.toISOString()
const duration = endTime.getTime() - startTime.getTime()
return createErrorResponse(error, 500, {
startTime: startTimeISO,
endTime: endTimeISO,
duration,
})
}
}
export async function OPTIONS() {
return new NextResponse(null, {
status: 204,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type, Authorization',
'Access-Control-Max-Age': '86400',
},
})
}

View File

@@ -5,11 +5,7 @@ import { checkHybridAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request' import { generateRequestId } from '@/lib/core/utils/request'
import { getBaseUrl } from '@/lib/core/utils/urls' import { getBaseUrl } from '@/lib/core/utils/urls'
import { StorageService } from '@/lib/uploads' import { StorageService } from '@/lib/uploads'
import { import { extractStorageKey, inferContextFromKey } from '@/lib/uploads/utils/file-utils'
extractStorageKey,
inferContextFromKey,
isInternalFileUrl,
} from '@/lib/uploads/utils/file-utils'
import { verifyFileAccess } from '@/app/api/files/authorization' import { verifyFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic' export const dynamic = 'force-dynamic'
@@ -51,13 +47,13 @@ export async function POST(request: NextRequest) {
logger.info(`[${requestId}] Mistral parse request`, { logger.info(`[${requestId}] Mistral parse request`, {
filePath: validatedData.filePath, filePath: validatedData.filePath,
isWorkspaceFile: isInternalFileUrl(validatedData.filePath), isWorkspaceFile: validatedData.filePath.includes('/api/files/serve/'),
userId, userId,
}) })
let fileUrl = validatedData.filePath let fileUrl = validatedData.filePath
if (isInternalFileUrl(validatedData.filePath)) { if (validatedData.filePath?.includes('/api/files/serve/')) {
try { try {
const storageKey = extractStorageKey(validatedData.filePath) const storageKey = extractStorageKey(validatedData.filePath)

View File

@@ -5,11 +5,7 @@ import { checkHybridAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request' import { generateRequestId } from '@/lib/core/utils/request'
import { getBaseUrl } from '@/lib/core/utils/urls' import { getBaseUrl } from '@/lib/core/utils/urls'
import { StorageService } from '@/lib/uploads' import { StorageService } from '@/lib/uploads'
import { import { extractStorageKey, inferContextFromKey } from '@/lib/uploads/utils/file-utils'
extractStorageKey,
inferContextFromKey,
isInternalFileUrl,
} from '@/lib/uploads/utils/file-utils'
import { verifyFileAccess } from '@/app/api/files/authorization' import { verifyFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic' export const dynamic = 'force-dynamic'
@@ -52,13 +48,13 @@ export async function POST(request: NextRequest) {
logger.info(`[${requestId}] Pulse parse request`, { logger.info(`[${requestId}] Pulse parse request`, {
filePath: validatedData.filePath, filePath: validatedData.filePath,
isWorkspaceFile: isInternalFileUrl(validatedData.filePath), isWorkspaceFile: validatedData.filePath.includes('/api/files/serve/'),
userId, userId,
}) })
let fileUrl = validatedData.filePath let fileUrl = validatedData.filePath
if (isInternalFileUrl(validatedData.filePath)) { if (validatedData.filePath?.includes('/api/files/serve/')) {
try { try {
const storageKey = extractStorageKey(validatedData.filePath) const storageKey = extractStorageKey(validatedData.filePath)
const context = inferContextFromKey(storageKey) const context = inferContextFromKey(storageKey)

View File

@@ -5,11 +5,7 @@ import { checkHybridAuth } from '@/lib/auth/hybrid'
import { generateRequestId } from '@/lib/core/utils/request' import { generateRequestId } from '@/lib/core/utils/request'
import { getBaseUrl } from '@/lib/core/utils/urls' import { getBaseUrl } from '@/lib/core/utils/urls'
import { StorageService } from '@/lib/uploads' import { StorageService } from '@/lib/uploads'
import { import { extractStorageKey, inferContextFromKey } from '@/lib/uploads/utils/file-utils'
extractStorageKey,
inferContextFromKey,
isInternalFileUrl,
} from '@/lib/uploads/utils/file-utils'
import { verifyFileAccess } from '@/app/api/files/authorization' import { verifyFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic' export const dynamic = 'force-dynamic'
@@ -48,13 +44,13 @@ export async function POST(request: NextRequest) {
logger.info(`[${requestId}] Reducto parse request`, { logger.info(`[${requestId}] Reducto parse request`, {
filePath: validatedData.filePath, filePath: validatedData.filePath,
isWorkspaceFile: isInternalFileUrl(validatedData.filePath), isWorkspaceFile: validatedData.filePath.includes('/api/files/serve/'),
userId, userId,
}) })
let fileUrl = validatedData.filePath let fileUrl = validatedData.filePath
if (isInternalFileUrl(validatedData.filePath)) { if (validatedData.filePath?.includes('/api/files/serve/')) {
try { try {
const storageKey = extractStorageKey(validatedData.filePath) const storageKey = extractStorageKey(validatedData.filePath)
const context = inferContextFromKey(storageKey) const context = inferContextFromKey(storageKey)

View File

@@ -79,13 +79,11 @@ export async function POST(request: NextRequest) {
// Generate public URL for destination (properly encode the destination key) // Generate public URL for destination (properly encode the destination key)
const encodedDestKey = validatedData.destinationKey.split('/').map(encodeURIComponent).join('/') const encodedDestKey = validatedData.destinationKey.split('/').map(encodeURIComponent).join('/')
const url = `https://${validatedData.destinationBucket}.s3.${validatedData.region}.amazonaws.com/${encodedDestKey}` const url = `https://${validatedData.destinationBucket}.s3.${validatedData.region}.amazonaws.com/${encodedDestKey}`
const uri = `s3://${validatedData.destinationBucket}/${validatedData.destinationKey}`
return NextResponse.json({ return NextResponse.json({
success: true, success: true,
output: { output: {
url, url,
uri,
copySourceVersionId: result.CopySourceVersionId, copySourceVersionId: result.CopySourceVersionId,
versionId: result.VersionId, versionId: result.VersionId,
etag: result.CopyObjectResult?.ETag, etag: result.CopyObjectResult?.ETag,

View File

@@ -117,13 +117,11 @@ export async function POST(request: NextRequest) {
const encodedKey = validatedData.objectKey.split('/').map(encodeURIComponent).join('/') const encodedKey = validatedData.objectKey.split('/').map(encodeURIComponent).join('/')
const url = `https://${validatedData.bucketName}.s3.${validatedData.region}.amazonaws.com/${encodedKey}` const url = `https://${validatedData.bucketName}.s3.${validatedData.region}.amazonaws.com/${encodedKey}`
const uri = `s3://${validatedData.bucketName}/${validatedData.objectKey}`
return NextResponse.json({ return NextResponse.json({
success: true, success: true,
output: { output: {
url, url,
uri,
etag: result.ETag, etag: result.ETag,
location: url, location: url,
key: validatedData.objectKey, key: validatedData.objectKey,

View File

@@ -1,637 +0,0 @@
import crypto from 'crypto'
import { createLogger } from '@sim/logger'
import { type NextRequest, NextResponse } from 'next/server'
import { z } from 'zod'
import { checkHybridAuth } from '@/lib/auth/hybrid'
import {
validateAwsRegion,
validateExternalUrl,
validateS3BucketName,
} from '@/lib/core/security/input-validation'
import { generateRequestId } from '@/lib/core/utils/request'
import { StorageService } from '@/lib/uploads'
import {
extractStorageKey,
inferContextFromKey,
isInternalFileUrl,
} from '@/lib/uploads/utils/file-utils'
import { verifyFileAccess } from '@/app/api/files/authorization'
export const dynamic = 'force-dynamic'
export const maxDuration = 300 // 5 minutes for large multi-page PDF processing
const logger = createLogger('TextractParseAPI')
const QuerySchema = z.object({
Text: z.string().min(1),
Alias: z.string().optional(),
Pages: z.array(z.string()).optional(),
})
const TextractParseSchema = z
.object({
accessKeyId: z.string().min(1, 'AWS Access Key ID is required'),
secretAccessKey: z.string().min(1, 'AWS Secret Access Key is required'),
region: z.string().min(1, 'AWS region is required'),
processingMode: z.enum(['sync', 'async']).optional().default('sync'),
filePath: z.string().optional(),
s3Uri: z.string().optional(),
featureTypes: z
.array(z.enum(['TABLES', 'FORMS', 'QUERIES', 'SIGNATURES', 'LAYOUT']))
.optional(),
queries: z.array(QuerySchema).optional(),
})
.superRefine((data, ctx) => {
const regionValidation = validateAwsRegion(data.region, 'AWS region')
if (!regionValidation.isValid) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: regionValidation.error,
path: ['region'],
})
}
})
function getSignatureKey(
key: string,
dateStamp: string,
regionName: string,
serviceName: string
): Buffer {
const kDate = crypto.createHmac('sha256', `AWS4${key}`).update(dateStamp).digest()
const kRegion = crypto.createHmac('sha256', kDate).update(regionName).digest()
const kService = crypto.createHmac('sha256', kRegion).update(serviceName).digest()
const kSigning = crypto.createHmac('sha256', kService).update('aws4_request').digest()
return kSigning
}
function signAwsRequest(
method: string,
host: string,
uri: string,
body: string,
accessKeyId: string,
secretAccessKey: string,
region: string,
service: string,
amzTarget: string
): Record<string, string> {
const date = new Date()
const amzDate = date.toISOString().replace(/[:-]|\.\d{3}/g, '')
const dateStamp = amzDate.slice(0, 8)
const payloadHash = crypto.createHash('sha256').update(body).digest('hex')
const canonicalHeaders =
`content-type:application/x-amz-json-1.1\n` +
`host:${host}\n` +
`x-amz-date:${amzDate}\n` +
`x-amz-target:${amzTarget}\n`
const signedHeaders = 'content-type;host;x-amz-date;x-amz-target'
const canonicalRequest = `${method}\n${uri}\n\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}`
const algorithm = 'AWS4-HMAC-SHA256'
const credentialScope = `${dateStamp}/${region}/${service}/aws4_request`
const stringToSign = `${algorithm}\n${amzDate}\n${credentialScope}\n${crypto.createHash('sha256').update(canonicalRequest).digest('hex')}`
const signingKey = getSignatureKey(secretAccessKey, dateStamp, region, service)
const signature = crypto.createHmac('sha256', signingKey).update(stringToSign).digest('hex')
const authorizationHeader = `${algorithm} Credential=${accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`
return {
'Content-Type': 'application/x-amz-json-1.1',
Host: host,
'X-Amz-Date': amzDate,
'X-Amz-Target': amzTarget,
Authorization: authorizationHeader,
}
}
async function fetchDocumentBytes(url: string): Promise<{ bytes: string; contentType: string }> {
const response = await fetch(url)
if (!response.ok) {
throw new Error(`Failed to fetch document: ${response.statusText}`)
}
const arrayBuffer = await response.arrayBuffer()
const bytes = Buffer.from(arrayBuffer).toString('base64')
const contentType = response.headers.get('content-type') || 'application/octet-stream'
return { bytes, contentType }
}
function parseS3Uri(s3Uri: string): { bucket: string; key: string } {
const match = s3Uri.match(/^s3:\/\/([^/]+)\/(.+)$/)
if (!match) {
throw new Error(
`Invalid S3 URI format: ${s3Uri}. Expected format: s3://bucket-name/path/to/object`
)
}
const bucket = match[1]
const key = match[2]
const bucketValidation = validateS3BucketName(bucket, 'S3 bucket name')
if (!bucketValidation.isValid) {
throw new Error(bucketValidation.error)
}
if (key.includes('..') || key.startsWith('/')) {
throw new Error('S3 key contains invalid path traversal sequences')
}
return { bucket, key }
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms))
}
async function callTextractAsync(
host: string,
amzTarget: string,
body: Record<string, unknown>,
accessKeyId: string,
secretAccessKey: string,
region: string
): Promise<Record<string, unknown>> {
const bodyString = JSON.stringify(body)
const headers = signAwsRequest(
'POST',
host,
'/',
bodyString,
accessKeyId,
secretAccessKey,
region,
'textract',
amzTarget
)
const response = await fetch(`https://${host}/`, {
method: 'POST',
headers,
body: bodyString,
})
if (!response.ok) {
const errorText = await response.text()
let errorMessage = `Textract API error: ${response.statusText}`
try {
const errorJson = JSON.parse(errorText)
if (errorJson.Message) {
errorMessage = errorJson.Message
} else if (errorJson.__type) {
errorMessage = `${errorJson.__type}: ${errorJson.message || errorText}`
}
} catch {
// Use default error message
}
throw new Error(errorMessage)
}
return response.json()
}
async function pollForJobCompletion(
host: string,
jobId: string,
accessKeyId: string,
secretAccessKey: string,
region: string,
useAnalyzeDocument: boolean,
requestId: string
): Promise<Record<string, unknown>> {
const pollIntervalMs = 5000 // 5 seconds between polls
const maxPollTimeMs = 180000 // 3 minutes maximum polling time
const maxAttempts = Math.ceil(maxPollTimeMs / pollIntervalMs)
const getTarget = useAnalyzeDocument
? 'Textract.GetDocumentAnalysis'
: 'Textract.GetDocumentTextDetection'
for (let attempt = 0; attempt < maxAttempts; attempt++) {
const result = await callTextractAsync(
host,
getTarget,
{ JobId: jobId },
accessKeyId,
secretAccessKey,
region
)
const jobStatus = result.JobStatus as string
if (jobStatus === 'SUCCEEDED') {
logger.info(`[${requestId}] Async job completed successfully after ${attempt + 1} polls`)
let allBlocks = (result.Blocks as unknown[]) || []
let nextToken = result.NextToken as string | undefined
while (nextToken) {
const nextResult = await callTextractAsync(
host,
getTarget,
{ JobId: jobId, NextToken: nextToken },
accessKeyId,
secretAccessKey,
region
)
allBlocks = allBlocks.concat((nextResult.Blocks as unknown[]) || [])
nextToken = nextResult.NextToken as string | undefined
}
return {
...result,
Blocks: allBlocks,
}
}
if (jobStatus === 'FAILED') {
throw new Error(`Textract job failed: ${result.StatusMessage || 'Unknown error'}`)
}
if (jobStatus === 'PARTIAL_SUCCESS') {
logger.warn(`[${requestId}] Job completed with partial success: ${result.StatusMessage}`)
let allBlocks = (result.Blocks as unknown[]) || []
let nextToken = result.NextToken as string | undefined
while (nextToken) {
const nextResult = await callTextractAsync(
host,
getTarget,
{ JobId: jobId, NextToken: nextToken },
accessKeyId,
secretAccessKey,
region
)
allBlocks = allBlocks.concat((nextResult.Blocks as unknown[]) || [])
nextToken = nextResult.NextToken as string | undefined
}
return {
...result,
Blocks: allBlocks,
}
}
logger.info(`[${requestId}] Job status: ${jobStatus}, attempt ${attempt + 1}/${maxAttempts}`)
await sleep(pollIntervalMs)
}
throw new Error(
`Timeout waiting for Textract job to complete (max ${maxPollTimeMs / 1000} seconds)`
)
}
export async function POST(request: NextRequest) {
const requestId = generateRequestId()
try {
const authResult = await checkHybridAuth(request, { requireWorkflowId: false })
if (!authResult.success || !authResult.userId) {
logger.warn(`[${requestId}] Unauthorized Textract parse attempt`, {
error: authResult.error || 'Missing userId',
})
return NextResponse.json(
{
success: false,
error: authResult.error || 'Unauthorized',
},
{ status: 401 }
)
}
const userId = authResult.userId
const body = await request.json()
const validatedData = TextractParseSchema.parse(body)
const processingMode = validatedData.processingMode || 'sync'
const featureTypes = validatedData.featureTypes ?? []
const useAnalyzeDocument = featureTypes.length > 0
const host = `textract.${validatedData.region}.amazonaws.com`
logger.info(`[${requestId}] Textract parse request`, {
processingMode,
filePath: validatedData.filePath?.substring(0, 50),
s3Uri: validatedData.s3Uri?.substring(0, 50),
featureTypes,
userId,
})
if (processingMode === 'async') {
if (!validatedData.s3Uri) {
return NextResponse.json(
{
success: false,
error: 'S3 URI is required for multi-page processing (s3://bucket/key)',
},
{ status: 400 }
)
}
const { bucket: s3Bucket, key: s3Key } = parseS3Uri(validatedData.s3Uri)
logger.info(`[${requestId}] Starting async Textract job`, { s3Bucket, s3Key })
const startTarget = useAnalyzeDocument
? 'Textract.StartDocumentAnalysis'
: 'Textract.StartDocumentTextDetection'
const startBody: Record<string, unknown> = {
DocumentLocation: {
S3Object: {
Bucket: s3Bucket,
Name: s3Key,
},
},
}
if (useAnalyzeDocument) {
startBody.FeatureTypes = featureTypes
if (
validatedData.queries &&
validatedData.queries.length > 0 &&
featureTypes.includes('QUERIES')
) {
startBody.QueriesConfig = {
Queries: validatedData.queries.map((q) => ({
Text: q.Text,
Alias: q.Alias,
Pages: q.Pages,
})),
}
}
}
const startResult = await callTextractAsync(
host,
startTarget,
startBody,
validatedData.accessKeyId,
validatedData.secretAccessKey,
validatedData.region
)
const jobId = startResult.JobId as string
if (!jobId) {
throw new Error('Failed to start Textract job: No JobId returned')
}
logger.info(`[${requestId}] Async job started`, { jobId })
const textractData = await pollForJobCompletion(
host,
jobId,
validatedData.accessKeyId,
validatedData.secretAccessKey,
validatedData.region,
useAnalyzeDocument,
requestId
)
logger.info(`[${requestId}] Textract async parse successful`, {
pageCount: (textractData.DocumentMetadata as { Pages?: number })?.Pages ?? 0,
blockCount: (textractData.Blocks as unknown[])?.length ?? 0,
})
return NextResponse.json({
success: true,
output: {
blocks: textractData.Blocks ?? [],
documentMetadata: {
pages: (textractData.DocumentMetadata as { Pages?: number })?.Pages ?? 0,
},
modelVersion: (textractData.AnalyzeDocumentModelVersion ??
textractData.DetectDocumentTextModelVersion) as string | undefined,
},
})
}
if (!validatedData.filePath) {
return NextResponse.json(
{
success: false,
error: 'File path is required for single-page processing',
},
{ status: 400 }
)
}
let fileUrl = validatedData.filePath
const isInternalFilePath = validatedData.filePath && isInternalFileUrl(validatedData.filePath)
if (isInternalFilePath) {
try {
const storageKey = extractStorageKey(validatedData.filePath)
const context = inferContextFromKey(storageKey)
const hasAccess = await verifyFileAccess(storageKey, userId, undefined, context, false)
if (!hasAccess) {
logger.warn(`[${requestId}] Unauthorized presigned URL generation attempt`, {
userId,
key: storageKey,
context,
})
return NextResponse.json(
{
success: false,
error: 'File not found',
},
{ status: 404 }
)
}
fileUrl = await StorageService.generatePresignedDownloadUrl(storageKey, context, 5 * 60)
logger.info(`[${requestId}] Generated presigned URL for ${context} file`)
} catch (error) {
logger.error(`[${requestId}] Failed to generate presigned URL:`, error)
return NextResponse.json(
{
success: false,
error: 'Failed to generate file access URL',
},
{ status: 500 }
)
}
} else if (validatedData.filePath?.startsWith('/')) {
// Reject arbitrary absolute paths that don't contain /api/files/serve/
logger.warn(`[${requestId}] Invalid internal path`, {
userId,
path: validatedData.filePath.substring(0, 50),
})
return NextResponse.json(
{
success: false,
error: 'Invalid file path. Only uploaded files are supported for internal paths.',
},
{ status: 400 }
)
} else {
const urlValidation = validateExternalUrl(fileUrl, 'Document URL')
if (!urlValidation.isValid) {
logger.warn(`[${requestId}] SSRF attempt blocked`, {
userId,
url: fileUrl.substring(0, 100),
error: urlValidation.error,
})
return NextResponse.json(
{
success: false,
error: urlValidation.error,
},
{ status: 400 }
)
}
}
const { bytes, contentType } = await fetchDocumentBytes(fileUrl)
// Track if this is a PDF for better error messaging
const isPdf = contentType.includes('pdf') || fileUrl.toLowerCase().endsWith('.pdf')
const uri = '/'
let textractBody: Record<string, unknown>
let amzTarget: string
if (useAnalyzeDocument) {
amzTarget = 'Textract.AnalyzeDocument'
textractBody = {
Document: {
Bytes: bytes,
},
FeatureTypes: featureTypes,
}
if (
validatedData.queries &&
validatedData.queries.length > 0 &&
featureTypes.includes('QUERIES')
) {
textractBody.QueriesConfig = {
Queries: validatedData.queries.map((q) => ({
Text: q.Text,
Alias: q.Alias,
Pages: q.Pages,
})),
}
}
} else {
amzTarget = 'Textract.DetectDocumentText'
textractBody = {
Document: {
Bytes: bytes,
},
}
}
const bodyString = JSON.stringify(textractBody)
const headers = signAwsRequest(
'POST',
host,
uri,
bodyString,
validatedData.accessKeyId,
validatedData.secretAccessKey,
validatedData.region,
'textract',
amzTarget
)
const textractResponse = await fetch(`https://${host}${uri}`, {
method: 'POST',
headers,
body: bodyString,
})
if (!textractResponse.ok) {
const errorText = await textractResponse.text()
logger.error(`[${requestId}] Textract API error:`, errorText)
let errorMessage = `Textract API error: ${textractResponse.statusText}`
let isUnsupportedFormat = false
try {
const errorJson = JSON.parse(errorText)
if (errorJson.Message) {
errorMessage = errorJson.Message
} else if (errorJson.__type) {
errorMessage = `${errorJson.__type}: ${errorJson.message || errorText}`
}
// Check for unsupported document format error
isUnsupportedFormat =
errorJson.__type === 'UnsupportedDocumentException' ||
errorJson.Message?.toLowerCase().includes('unsupported document') ||
errorText.toLowerCase().includes('unsupported document')
} catch {
isUnsupportedFormat = errorText.toLowerCase().includes('unsupported document')
}
// Provide helpful message for unsupported format (likely multi-page PDF)
if (isUnsupportedFormat && isPdf) {
errorMessage =
'This document format is not supported in Single Page mode. If this is a multi-page PDF, please use "Multi-Page (PDF, TIFF via S3)" mode instead, which requires uploading your document to S3 first. Single Page mode only supports JPEG, PNG, and single-page PDF files.'
}
return NextResponse.json(
{
success: false,
error: errorMessage,
},
{ status: textractResponse.status }
)
}
const textractData = await textractResponse.json()
logger.info(`[${requestId}] Textract parse successful`, {
pageCount: textractData.DocumentMetadata?.Pages ?? 0,
blockCount: textractData.Blocks?.length ?? 0,
})
return NextResponse.json({
success: true,
output: {
blocks: textractData.Blocks ?? [],
documentMetadata: {
pages: textractData.DocumentMetadata?.Pages ?? 0,
},
modelVersion:
textractData.AnalyzeDocumentModelVersion ??
textractData.DetectDocumentTextModelVersion ??
undefined,
},
})
} catch (error) {
if (error instanceof z.ZodError) {
logger.warn(`[${requestId}] Invalid request data`, { errors: error.errors })
return NextResponse.json(
{
success: false,
error: 'Invalid request data',
details: error.errors,
},
{ status: 400 }
)
}
logger.error(`[${requestId}] Error in Textract parse:`, error)
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : 'Internal server error',
},
{ status: 500 }
)
}
}

View File

@@ -12,10 +12,6 @@ import { markExecutionCancelled } from '@/lib/execution/cancellation'
import { processInputFileFields } from '@/lib/execution/files' import { processInputFileFields } from '@/lib/execution/files'
import { preprocessExecution } from '@/lib/execution/preprocessing' import { preprocessExecution } from '@/lib/execution/preprocessing'
import { LoggingSession } from '@/lib/logs/execution/logging-session' import { LoggingSession } from '@/lib/logs/execution/logging-session'
import {
cleanupExecutionBase64Cache,
hydrateUserFilesWithBase64,
} from '@/lib/uploads/utils/user-file-base64.server'
import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core' import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core'
import { type ExecutionEvent, encodeSSEEvent } from '@/lib/workflows/executor/execution-events' import { type ExecutionEvent, encodeSSEEvent } from '@/lib/workflows/executor/execution-events'
import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager' import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager'
@@ -29,7 +25,7 @@ import type { WorkflowExecutionPayload } from '@/background/workflow-execution'
import { normalizeName } from '@/executor/constants' import { normalizeName } from '@/executor/constants'
import { ExecutionSnapshot } from '@/executor/execution/snapshot' import { ExecutionSnapshot } from '@/executor/execution/snapshot'
import type { ExecutionMetadata, IterationContext } from '@/executor/execution/types' import type { ExecutionMetadata, IterationContext } from '@/executor/execution/types'
import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import type { StreamingExecution } from '@/executor/types'
import { Serializer } from '@/serializer' import { Serializer } from '@/serializer'
import { CORE_TRIGGER_TYPES, type CoreTriggerType } from '@/stores/logs/filters/types' import { CORE_TRIGGER_TYPES, type CoreTriggerType } from '@/stores/logs/filters/types'
@@ -42,8 +38,6 @@ const ExecuteWorkflowSchema = z.object({
useDraftState: z.boolean().optional(), useDraftState: z.boolean().optional(),
input: z.any().optional(), input: z.any().optional(),
isClientSession: z.boolean().optional(), isClientSession: z.boolean().optional(),
includeFileBase64: z.boolean().optional().default(true),
base64MaxBytes: z.number().int().positive().optional(),
workflowStateOverride: z workflowStateOverride: z
.object({ .object({
blocks: z.record(z.any()), blocks: z.record(z.any()),
@@ -220,8 +214,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
useDraftState, useDraftState,
input: validatedInput, input: validatedInput,
isClientSession = false, isClientSession = false,
includeFileBase64,
base64MaxBytes,
workflowStateOverride, workflowStateOverride,
} = validation.data } = validation.data
@@ -235,8 +227,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
triggerType, triggerType,
stream, stream,
useDraftState, useDraftState,
includeFileBase64,
base64MaxBytes,
workflowStateOverride, workflowStateOverride,
workflowId: _workflowId, // Also exclude workflowId used for internal JWT auth workflowId: _workflowId, // Also exclude workflowId used for internal JWT auth
...rest ...rest
@@ -437,31 +427,16 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
snapshot, snapshot,
callbacks: {}, callbacks: {},
loggingSession, loggingSession,
includeFileBase64,
base64MaxBytes,
}) })
const outputWithBase64 = includeFileBase64 const hasResponseBlock = workflowHasResponseBlock(result)
? ((await hydrateUserFilesWithBase64(result.output, {
requestId,
executionId,
maxBytes: base64MaxBytes,
})) as NormalizedBlockOutput)
: result.output
const resultWithBase64 = { ...result, output: outputWithBase64 }
// Cleanup base64 cache for this execution
await cleanupExecutionBase64Cache(executionId)
const hasResponseBlock = workflowHasResponseBlock(resultWithBase64)
if (hasResponseBlock) { if (hasResponseBlock) {
return createHttpResponseFromBlock(resultWithBase64) return createHttpResponseFromBlock(result)
} }
const filteredResult = { const filteredResult = {
success: result.success, success: result.success,
output: outputWithBase64, output: result.output,
error: result.error, error: result.error,
metadata: result.metadata metadata: result.metadata
? { ? {
@@ -523,8 +498,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
selectedOutputs: resolvedSelectedOutputs, selectedOutputs: resolvedSelectedOutputs,
isSecureMode: false, isSecureMode: false,
workflowTriggerType: triggerType === 'chat' ? 'chat' : 'api', workflowTriggerType: triggerType === 'chat' ? 'chat' : 'api',
includeFileBase64,
base64MaxBytes,
}, },
executionId, executionId,
}) })
@@ -725,8 +698,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
}, },
loggingSession, loggingSession,
abortSignal: abortController.signal, abortSignal: abortController.signal,
includeFileBase64,
base64MaxBytes,
}) })
if (result.status === 'paused') { if (result.status === 'paused') {
@@ -779,21 +750,12 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
workflowId, workflowId,
data: { data: {
success: result.success, success: result.success,
output: includeFileBase64 output: result.output,
? await hydrateUserFilesWithBase64(result.output, {
requestId,
executionId,
maxBytes: base64MaxBytes,
})
: result.output,
duration: result.metadata?.duration || 0, duration: result.metadata?.duration || 0,
startTime: result.metadata?.startTime || startTime.toISOString(), startTime: result.metadata?.startTime || startTime.toISOString(),
endTime: result.metadata?.endTime || new Date().toISOString(), endTime: result.metadata?.endTime || new Date().toISOString(),
}, },
}) })
// Cleanup base64 cache for this execution
await cleanupExecutionBase64Cache(executionId)
} catch (error: any) { } catch (error: any) {
const errorMessage = error.message || 'Unknown error' const errorMessage = error.message || 'Unknown error'
logger.error(`[${requestId}] SSE execution failed: ${errorMessage}`) logger.error(`[${requestId}] SSE execution failed: ${errorMessage}`)

View File

@@ -33,7 +33,6 @@ const BlockDataSchema = z.object({
doWhileCondition: z.string().optional(), doWhileCondition: z.string().optional(),
parallelType: z.enum(['collection', 'count']).optional(), parallelType: z.enum(['collection', 'count']).optional(),
type: z.string().optional(), type: z.string().optional(),
canonicalModes: z.record(z.enum(['basic', 'advanced'])).optional(),
}) })
const SubBlockStateSchema = z.object({ const SubBlockStateSchema = z.object({

View File

@@ -2,7 +2,7 @@
import { useRef, useState } from 'react' import { useRef, useState } from 'react'
import { createLogger } from '@sim/logger' import { createLogger } from '@sim/logger'
import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import { isUserFile } from '@/lib/core/utils/display-filters'
import type { ChatFile, ChatMessage } from '@/app/chat/components/message/message' import type { ChatFile, ChatMessage } from '@/app/chat/components/message/message'
import { CHAT_ERROR_MESSAGES } from '@/app/chat/constants' import { CHAT_ERROR_MESSAGES } from '@/app/chat/constants'
@@ -17,7 +17,7 @@ function extractFilesFromData(
return files return files
} }
if (isUserFileWithMetadata(data)) { if (isUserFile(data)) {
if (!seenIds.has(data.id)) { if (!seenIds.has(data.id)) {
seenIds.add(data.id) seenIds.add(data.id)
files.push({ files.push({
@@ -232,7 +232,7 @@ export function useChatStreaming() {
return null return null
} }
if (isUserFileWithMetadata(value)) { if (isUserFile(value)) {
return null return null
} }
@@ -285,7 +285,7 @@ export function useChatStreaming() {
const value = getOutputValue(blockOutputs, config.path) const value = getOutputValue(blockOutputs, config.path)
if (isUserFileWithMetadata(value)) { if (isUserFile(value)) {
extractedFiles.push({ extractedFiles.push({
id: value.id, id: value.id,
name: value.name, name: value.name,

View File

@@ -129,6 +129,10 @@ export function OutputSelect({
? baselineWorkflow.blocks?.[block.id]?.subBlocks?.responseFormat?.value ? baselineWorkflow.blocks?.[block.id]?.subBlocks?.responseFormat?.value
: subBlockValues?.[block.id]?.responseFormat : subBlockValues?.[block.id]?.responseFormat
const responseFormat = parseResponseFormatSafely(responseFormatValue, block.id) const responseFormat = parseResponseFormatSafely(responseFormatValue, block.id)
const operationValue =
shouldUseBaseline && baselineWorkflow
? baselineWorkflow.blocks?.[block.id]?.subBlocks?.operation?.value
: subBlockValues?.[block.id]?.operation
let outputsToProcess: Record<string, unknown> = {} let outputsToProcess: Record<string, unknown> = {}
@@ -142,20 +146,10 @@ export function OutputSelect({
outputsToProcess = blockConfig?.outputs || {} outputsToProcess = blockConfig?.outputs || {}
} }
} else { } else {
// Build subBlocks object for tool selector const toolOutputs =
const rawSubBlockValues = blockConfig && typeof operationValue === 'string'
shouldUseBaseline && baselineWorkflow ? getToolOutputs(blockConfig, operationValue)
? baselineWorkflow.blocks?.[block.id]?.subBlocks : {}
: subBlockValues?.[block.id]
const subBlocks: Record<string, { value: unknown }> = {}
if (rawSubBlockValues && typeof rawSubBlockValues === 'object') {
for (const [key, val] of Object.entries(rawSubBlockValues)) {
// Handle both { value: ... } and raw value formats
subBlocks[key] = val && typeof val === 'object' && 'value' in val ? val : { value: val }
}
}
const toolOutputs = blockConfig ? getToolOutputs(blockConfig, subBlocks) : {}
outputsToProcess = outputsToProcess =
Object.keys(toolOutputs).length > 0 ? toolOutputs : blockConfig?.outputs || {} Object.keys(toolOutputs).length > 0 ? toolOutputs : blockConfig?.outputs || {}
} }

View File

@@ -8,10 +8,9 @@ import { Button, Combobox } from '@/components/emcn/components'
import { Progress } from '@/components/ui/progress' import { Progress } from '@/components/ui/progress'
import { cn } from '@/lib/core/utils/cn' import { cn } from '@/lib/core/utils/cn'
import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace'
import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils'
import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value'
import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useWorkflowRegistry } from '@/stores/workflows/registry/store'
import { useWorkflowStore } from '@/stores/workflows/workflow/store' import { useWorkflowStore } from '@/stores/workflows/workflow/store'
import { useSubBlockValue } from '../../hooks/use-sub-block-value'
const logger = createLogger('FileUpload') const logger = createLogger('FileUpload')
@@ -86,47 +85,14 @@ export function FileUpload({
} }
} }
/**
* Checks if a file's MIME type matches the accepted types
* Supports exact matches, wildcard patterns (e.g., 'image/*'), and '*' for all types
*/
const isFileTypeAccepted = (fileType: string | undefined, accepted: string): boolean => {
if (accepted === '*') return true
if (!fileType) return false
const acceptedList = accepted.split(',').map((t) => t.trim().toLowerCase())
const normalizedFileType = fileType.toLowerCase()
return acceptedList.some((acceptedType) => {
if (acceptedType === normalizedFileType) return true
if (acceptedType.endsWith('/*')) {
const typePrefix = acceptedType.slice(0, -1) // 'image/' from 'image/*'
return normalizedFileType.startsWith(typePrefix)
}
if (acceptedType.startsWith('.')) {
const extension = acceptedType.slice(1).toLowerCase()
const fileExtension = getExtensionFromMimeType(normalizedFileType)
if (fileExtension === extension) return true
return normalizedFileType.endsWith(`/${extension}`)
}
return false
})
}
const availableWorkspaceFiles = workspaceFiles.filter((workspaceFile) => { const availableWorkspaceFiles = workspaceFiles.filter((workspaceFile) => {
const existingFiles = Array.isArray(value) ? value : value ? [value] : [] const existingFiles = Array.isArray(value) ? value : value ? [value] : []
return !existingFiles.some(
const isAlreadySelected = existingFiles.some(
(existing) => (existing) =>
existing.name === workspaceFile.name || existing.name === workspaceFile.name ||
existing.path?.includes(workspaceFile.key) || existing.path?.includes(workspaceFile.key) ||
existing.key === workspaceFile.key existing.key === workspaceFile.key
) )
return !isAlreadySelected
}) })
useEffect(() => { useEffect(() => {
@@ -455,23 +421,23 @@ export function FileUpload({
return ( return (
<div <div
key={fileKey} key={fileKey}
className='relative rounded-[4px] border border-[var(--border-1)] bg-[var(--surface-5)] px-[8px] py-[6px] hover:border-[var(--surface-7)] hover:bg-[var(--surface-5)] dark:bg-[var(--surface-5)] dark:hover:bg-[var(--border-1)]' className='flex items-center justify-between rounded-[4px] border border-[var(--border-1)] bg-[var(--surface-5)] px-[8px] py-[6px] hover:border-[var(--surface-7)] hover:bg-[var(--surface-5)] dark:bg-[var(--surface-5)] dark:hover:bg-[var(--border-1)]'
> >
<div className='truncate pr-[24px] text-sm' title={file.name}> <div className='flex-1 truncate pr-2 text-sm' title={file.name}>
<span className='text-[var(--text-primary)]'>{truncateMiddle(file.name)}</span> <span className='text-[var(--text-primary)]'>{truncateMiddle(file.name)}</span>
<span className='ml-2 text-[var(--text-muted)]'>({formatFileSize(file.size)})</span> <span className='ml-2 text-[var(--text-muted)]'>({formatFileSize(file.size)})</span>
</div> </div>
<Button <Button
type='button' type='button'
variant='ghost' variant='ghost'
className='-translate-y-1/2 absolute top-1/2 right-[4px] h-6 w-6 p-0' className='h-5 w-5 shrink-0 p-0'
onClick={(e) => handleRemoveFile(file, e)} onClick={(e) => handleRemoveFile(file, e)}
disabled={isDeleting} disabled={isDeleting}
> >
{isDeleting ? ( {isDeleting ? (
<div className='h-4 w-4 animate-spin rounded-full border-[1.5px] border-current border-t-transparent' /> <div className='h-3.5 w-3.5 animate-spin rounded-full border-[1.5px] border-current border-t-transparent' />
) : ( ) : (
<X className='h-4 w-4 opacity-50' /> <X className='h-3.5 w-3.5' />
)} )}
</Button> </Button>
</div> </div>
@@ -502,30 +468,19 @@ export function FileUpload({
const comboboxOptions = useMemo( const comboboxOptions = useMemo(
() => [ () => [
{ label: 'Upload New File', value: '__upload_new__' }, { label: 'Upload New File', value: '__upload_new__' },
...availableWorkspaceFiles.map((file) => { ...availableWorkspaceFiles.map((file) => ({
const isAccepted = label: file.name,
!acceptedTypes || acceptedTypes === '*' || isFileTypeAccepted(file.type, acceptedTypes) value: file.id,
return { })),
label: file.name,
value: file.id,
disabled: !isAccepted,
}
}),
], ],
[availableWorkspaceFiles, acceptedTypes] [availableWorkspaceFiles]
) )
const handleComboboxChange = (value: string) => { const handleComboboxChange = (value: string) => {
setInputValue(value) setInputValue(value)
const selectedFile = availableWorkspaceFiles.find((file) => file.id === value) const isValidOption =
const isAcceptedType = value === '__upload_new__' || availableWorkspaceFiles.some((file) => file.id === value)
selectedFile &&
(!acceptedTypes ||
acceptedTypes === '*' ||
isFileTypeAccepted(selectedFile.type, acceptedTypes))
const isValidOption = value === '__upload_new__' || isAcceptedType
if (!isValidOption) { if (!isValidOption) {
return return

View File

@@ -28,7 +28,6 @@ interface Field {
name: string name: string
type?: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'files' type?: 'string' | 'number' | 'boolean' | 'object' | 'array' | 'files'
value?: string value?: string
description?: string
collapsed?: boolean collapsed?: boolean
} }
@@ -42,9 +41,7 @@ interface FieldFormatProps {
placeholder?: string placeholder?: string
showType?: boolean showType?: boolean
showValue?: boolean showValue?: boolean
showDescription?: boolean
valuePlaceholder?: string valuePlaceholder?: string
descriptionPlaceholder?: string
config?: any config?: any
} }
@@ -76,7 +73,6 @@ const createDefaultField = (): Field => ({
name: '', name: '',
type: 'string', type: 'string',
value: '', value: '',
description: '',
collapsed: false, collapsed: false,
}) })
@@ -97,9 +93,7 @@ export function FieldFormat({
placeholder = 'fieldName', placeholder = 'fieldName',
showType = true, showType = true,
showValue = false, showValue = false,
showDescription = false,
valuePlaceholder = 'Enter default value', valuePlaceholder = 'Enter default value',
descriptionPlaceholder = 'Describe this field',
}: FieldFormatProps) { }: FieldFormatProps) {
const [storeValue, setStoreValue] = useSubBlockValue<Field[]>(blockId, subBlockId) const [storeValue, setStoreValue] = useSubBlockValue<Field[]>(blockId, subBlockId)
const valueInputRefs = useRef<Record<string, HTMLInputElement | HTMLTextAreaElement>>({}) const valueInputRefs = useRef<Record<string, HTMLInputElement | HTMLTextAreaElement>>({})
@@ -560,18 +554,6 @@ export function FieldFormat({
</div> </div>
)} )}
{showDescription && (
<div className='flex flex-col gap-[6px]'>
<Label className='text-[13px]'>Description</Label>
<Input
value={field.description ?? ''}
onChange={(e) => updateField(field.id, 'description', e.target.value)}
placeholder={descriptionPlaceholder}
disabled={isReadOnly}
/>
</div>
)}
{showValue && ( {showValue && (
<div className='flex flex-col gap-[6px]'> <div className='flex flex-col gap-[6px]'>
<Label className='text-[13px]'>Value</Label> <Label className='text-[13px]'>Value</Label>
@@ -586,10 +568,8 @@ export function FieldFormat({
) )
} }
export function InputFormat( export function InputFormat(props: Omit<FieldFormatProps, 'title' | 'placeholder'>) {
props: Omit<FieldFormatProps, 'title' | 'placeholder' | 'showDescription'> return <FieldFormat {...props} title='Input' placeholder='firstName' />
) {
return <FieldFormat {...props} title='Input' placeholder='firstName' showDescription={true} />
} }
export function ResponseFormat( export function ResponseFormat(

View File

@@ -241,16 +241,13 @@ const getOutputTypeForPath = (
const blockState = useWorkflowStore.getState().blocks[blockId] const blockState = useWorkflowStore.getState().blocks[blockId]
const subBlocks = mergedSubBlocksOverride ?? (blockState?.subBlocks || {}) const subBlocks = mergedSubBlocksOverride ?? (blockState?.subBlocks || {})
return getBlockOutputType(block.type, outputPath, subBlocks) return getBlockOutputType(block.type, outputPath, subBlocks)
} else if (blockConfig?.tools?.config?.tool) { } else {
const blockState = useWorkflowStore.getState().blocks[blockId] const operationValue = getSubBlockValue(blockId, 'operation')
const subBlocks = mergedSubBlocksOverride ?? (blockState?.subBlocks || {}) if (blockConfig && operationValue) {
return getToolOutputType(blockConfig, subBlocks, outputPath) return getToolOutputType(blockConfig, operationValue, outputPath)
}
} }
return 'any'
const subBlocks =
mergedSubBlocksOverride ?? useWorkflowStore.getState().blocks[blockId]?.subBlocks
const triggerMode = block?.triggerMode && blockConfig?.triggers?.enabled
return getBlockOutputType(block?.type ?? '', outputPath, subBlocks, triggerMode)
} }
/** /**
@@ -1214,7 +1211,11 @@ export const TagDropdown: React.FC<TagDropdownProps> = ({
: allTags : allTags
} }
} else { } else {
const toolOutputPaths = getToolOutputPaths(blockConfig, mergedSubBlocks) const operationValue =
mergedSubBlocks?.operation?.value ?? getSubBlockValue(activeSourceBlockId, 'operation')
const toolOutputPaths = operationValue
? getToolOutputPaths(blockConfig, operationValue, mergedSubBlocks)
: []
if (toolOutputPaths.length > 0) { if (toolOutputPaths.length > 0) {
blockTags = toolOutputPaths.map((path) => `${normalizedBlockName}.${path}`) blockTags = toolOutputPaths.map((path) => `${normalizedBlockName}.${path}`)
@@ -1534,6 +1535,7 @@ export const TagDropdown: React.FC<TagDropdownProps> = ({
if (dynamicOutputs.length > 0) { if (dynamicOutputs.length > 0) {
const allTags = dynamicOutputs.map((path) => `${normalizedBlockName}.${path}`) const allTags = dynamicOutputs.map((path) => `${normalizedBlockName}.${path}`)
// For self-reference, only show url and resumeEndpoint (not response format fields)
blockTags = isSelfReference blockTags = isSelfReference
? allTags.filter((tag) => tag.endsWith('.url') || tag.endsWith('.resumeEndpoint')) ? allTags.filter((tag) => tag.endsWith('.url') || tag.endsWith('.resumeEndpoint'))
: allTags : allTags
@@ -1541,7 +1543,11 @@ export const TagDropdown: React.FC<TagDropdownProps> = ({
blockTags = [`${normalizedBlockName}.url`, `${normalizedBlockName}.resumeEndpoint`] blockTags = [`${normalizedBlockName}.url`, `${normalizedBlockName}.resumeEndpoint`]
} }
} else { } else {
const toolOutputPaths = getToolOutputPaths(blockConfig, mergedSubBlocks) const operationValue =
mergedSubBlocks?.operation?.value ?? getSubBlockValue(accessibleBlockId, 'operation')
const toolOutputPaths = operationValue
? getToolOutputPaths(blockConfig, operationValue, mergedSubBlocks)
: []
if (toolOutputPaths.length > 0) { if (toolOutputPaths.length > 0) {
blockTags = toolOutputPaths.map((path) => `${normalizedBlockName}.${path}`) blockTags = toolOutputPaths.map((path) => `${normalizedBlockName}.${path}`)
@@ -1783,7 +1789,7 @@ export const TagDropdown: React.FC<TagDropdownProps> = ({
mergedSubBlocks mergedSubBlocks
) )
if (fieldType === 'files' || fieldType === 'file[]' || fieldType === 'array') { if (fieldType === 'files' || fieldType === 'array') {
const blockName = parts[0] const blockName = parts[0]
const remainingPath = parts.slice(2).join('.') const remainingPath = parts.slice(2).join('.')
processedTag = `${blockName}.${arrayFieldName}[0].${remainingPath}` processedTag = `${blockName}.${arrayFieldName}[0].${remainingPath}`

View File

@@ -2,15 +2,13 @@
import { useMemo } from 'react' import { useMemo } from 'react'
import { extractFieldsFromSchema } from '@/lib/core/utils/response-format' import { extractFieldsFromSchema } from '@/lib/core/utils/response-format'
import { import { getBlockOutputPaths, getBlockOutputs } from '@/lib/workflows/blocks/block-outputs'
getBlockOutputPaths,
getBlockOutputs,
getToolOutputs,
} from '@/lib/workflows/blocks/block-outputs'
import { TRIGGER_TYPES } from '@/lib/workflows/triggers/triggers' import { TRIGGER_TYPES } from '@/lib/workflows/triggers/triggers'
import type { SchemaField } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/connection-blocks/components/field-item/field-item' import type { SchemaField } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/connection-blocks/components/field-item/field-item'
import { getBlock } from '@/blocks' import { getBlock } from '@/blocks'
import type { BlockConfig } from '@/blocks/types'
import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store'
import { getTool } from '@/tools/utils'
const RESERVED_KEYS = new Set(['type', 'description']) const RESERVED_KEYS = new Set(['type', 'description'])
@@ -26,6 +24,64 @@ const getSubBlockValue = (blockId: string, property: string): any => {
return useSubBlockStore.getState().getValue(blockId, property) return useSubBlockStore.getState().getValue(blockId, property)
} }
/**
* Generates output paths for a tool-based block
*/
const generateToolOutputPaths = (blockConfig: BlockConfig, operation: string): string[] => {
if (!blockConfig?.tools?.config?.tool) return []
try {
const toolId = blockConfig.tools.config.tool({ operation })
if (!toolId) return []
const toolConfig = getTool(toolId)
if (!toolConfig?.outputs) return []
return generateOutputPaths(toolConfig.outputs)
} catch {
return []
}
}
/**
* Recursively generates all output paths from an outputs schema
*/
const generateOutputPaths = (outputs: Record<string, any>, prefix = ''): string[] => {
const paths: string[] = []
for (const [key, value] of Object.entries(outputs)) {
const currentPath = prefix ? `${prefix}.${key}` : key
if (typeof value === 'string') {
paths.push(currentPath)
} else if (typeof value === 'object' && value !== null) {
if ('type' in value && typeof value.type === 'string') {
paths.push(currentPath)
// Handle nested objects and arrays
if (value.type === 'object' && value.properties) {
paths.push(...generateOutputPaths(value.properties, currentPath))
} else if (value.type === 'array' && value.items?.properties) {
paths.push(...generateOutputPaths(value.items.properties, currentPath))
} else if (
value.type === 'array' &&
value.items &&
typeof value.items === 'object' &&
!('type' in value.items)
) {
paths.push(...generateOutputPaths(value.items, currentPath))
}
} else {
const subPaths = generateOutputPaths(value, currentPath)
paths.push(...subPaths)
}
} else {
paths.push(currentPath)
}
}
return paths
}
/** /**
* Extracts nested fields from array or object properties * Extracts nested fields from array or object properties
*/ */
@@ -99,6 +155,26 @@ const createFieldFromOutput = (
return field return field
} }
/**
* Gets tool outputs for a block's operation
*/
const getToolOutputs = (
blockConfig: BlockConfig | null,
operation?: string
): Record<string, any> => {
if (!blockConfig?.tools?.config?.tool || !operation) return {}
try {
const toolId = blockConfig.tools.config.tool({ operation })
if (!toolId) return {}
const toolConfig = getTool(toolId)
return toolConfig?.outputs || {}
} catch {
return {}
}
}
interface UseBlockOutputFieldsParams { interface UseBlockOutputFieldsParams {
blockId: string blockId: string
blockType: string blockType: string
@@ -223,11 +299,14 @@ export function useBlockOutputFields({
baseOutputs = getBlockOutputs(blockType, mergedSubBlocks) baseOutputs = getBlockOutputs(blockType, mergedSubBlocks)
} else { } else {
// For tool-based blocks, try to get tool outputs first // For tool-based blocks, try to get tool outputs first
const toolOutputs = blockConfig ? getToolOutputs(blockConfig, mergedSubBlocks) : {} const operationValue =
operation ?? mergedSubBlocks?.operation?.value ?? getSubBlockValue(blockId, 'operation')
const toolOutputs = operationValue ? getToolOutputs(blockConfig, operationValue) : {}
if (Object.keys(toolOutputs).length > 0) { if (Object.keys(toolOutputs).length > 0) {
baseOutputs = toolOutputs baseOutputs = toolOutputs
} else { } else {
// Use getBlockOutputs which handles inputFormat merging
baseOutputs = getBlockOutputs(blockType, mergedSubBlocks, triggerMode) baseOutputs = getBlockOutputs(blockType, mergedSubBlocks, triggerMode)
} }
} }

View File

@@ -208,8 +208,6 @@ async function runWorkflowExecution({
snapshot, snapshot,
callbacks: {}, callbacks: {},
loggingSession, loggingSession,
includeFileBase64: true,
base64MaxBytes: undefined,
}) })
if (executionResult.status === 'paused') { if (executionResult.status === 'paused') {

View File

@@ -240,8 +240,6 @@ async function executeWebhookJobInternal(
snapshot, snapshot,
callbacks: {}, callbacks: {},
loggingSession, loggingSession,
includeFileBase64: true, // Enable base64 hydration
base64MaxBytes: undefined, // Use default limit
}) })
if (executionResult.status === 'paused') { if (executionResult.status === 'paused') {
@@ -495,7 +493,6 @@ async function executeWebhookJobInternal(
snapshot, snapshot,
callbacks: {}, callbacks: {},
loggingSession, loggingSession,
includeFileBase64: true,
}) })
if (executionResult.status === 'paused') { if (executionResult.status === 'paused') {

View File

@@ -109,8 +109,6 @@ export async function executeWorkflowJob(payload: WorkflowExecutionPayload) {
snapshot, snapshot,
callbacks: {}, callbacks: {},
loggingSession, loggingSession,
includeFileBase64: true,
base64MaxBytes: undefined,
}) })
if (result.status === 'paused') { if (result.status === 'paused') {

View File

@@ -107,26 +107,14 @@ export const A2ABlock: BlockConfig<A2AResponse> = {
condition: { field: 'operation', value: 'a2a_send_message' }, condition: { field: 'operation', value: 'a2a_send_message' },
}, },
{ {
id: 'fileUpload', id: 'files',
title: 'Files', title: 'Files',
type: 'file-upload', type: 'file-upload',
canonicalParamId: 'files',
placeholder: 'Upload files to send', placeholder: 'Upload files to send',
description: 'Files to include with the message (FilePart)', description: 'Files to include with the message (FilePart)',
condition: { field: 'operation', value: 'a2a_send_message' }, condition: { field: 'operation', value: 'a2a_send_message' },
mode: 'basic',
multiple: true, multiple: true,
}, },
{
id: 'fileReference',
title: 'Files',
type: 'short-input',
canonicalParamId: 'files',
placeholder: 'Reference files from previous blocks',
description: 'Files to include with the message (FilePart)',
condition: { field: 'operation', value: 'a2a_send_message' },
mode: 'advanced',
},
{ {
id: 'taskId', id: 'taskId',
title: 'Task ID', title: 'Task ID',
@@ -245,14 +233,6 @@ export const A2ABlock: BlockConfig<A2AResponse> = {
type: 'array', type: 'array',
description: 'Files to include with the message', description: 'Files to include with the message',
}, },
fileUpload: {
type: 'array',
description: 'Uploaded files (basic mode)',
},
fileReference: {
type: 'json',
description: 'File reference from previous blocks (advanced mode)',
},
historyLength: { historyLength: {
type: 'number', type: 'number',
description: 'Number of history messages to include', description: 'Number of history messages to include',

View File

@@ -57,12 +57,6 @@ export const BrowserUseBlock: BlockConfig<BrowserUseResponse> = {
type: 'switch', type: 'switch',
placeholder: 'Save browser data', placeholder: 'Save browser data',
}, },
{
id: 'profile_id',
title: 'Profile ID',
type: 'short-input',
placeholder: 'Enter browser profile ID (optional)',
},
{ {
id: 'apiKey', id: 'apiKey',
title: 'API Key', title: 'API Key',
@@ -81,7 +75,6 @@ export const BrowserUseBlock: BlockConfig<BrowserUseResponse> = {
variables: { type: 'json', description: 'Task variables' }, variables: { type: 'json', description: 'Task variables' },
model: { type: 'string', description: 'AI model to use' }, model: { type: 'string', description: 'AI model to use' },
save_browser_data: { type: 'boolean', description: 'Save browser data' }, save_browser_data: { type: 'boolean', description: 'Save browser data' },
profile_id: { type: 'string', description: 'Browser profile ID for persistent sessions' },
}, },
outputs: { outputs: {
id: { type: 'string', description: 'Task execution identifier' }, id: { type: 'string', description: 'Task execution identifier' },

View File

@@ -5,9 +5,8 @@ import type { ConfluenceResponse } from '@/tools/confluence/types'
export const ConfluenceBlock: BlockConfig<ConfluenceResponse> = { export const ConfluenceBlock: BlockConfig<ConfluenceResponse> = {
type: 'confluence', type: 'confluence',
name: 'Confluence (Legacy)', name: 'Confluence',
description: 'Interact with Confluence', description: 'Interact with Confluence',
hideFromToolbar: true,
authMode: AuthMode.OAuth, authMode: AuthMode.OAuth,
longDescription: longDescription:
'Integrate Confluence into the workflow. Can read, create, update, delete pages, manage comments, attachments, labels, and search content.', 'Integrate Confluence into the workflow. Can read, create, update, delete pages, manage comments, attachments, labels, and search content.',
@@ -358,342 +357,3 @@ export const ConfluenceBlock: BlockConfig<ConfluenceResponse> = {
status: { type: 'string', description: 'Space status' }, status: { type: 'string', description: 'Space status' },
}, },
} }
export const ConfluenceV2Block: BlockConfig<ConfluenceResponse> = {
...ConfluenceBlock,
type: 'confluence_v2',
name: 'Confluence',
hideFromToolbar: false,
subBlocks: [
{
id: 'operation',
title: 'Operation',
type: 'dropdown',
options: [
{ label: 'Read Page', id: 'read' },
{ label: 'Create Page', id: 'create' },
{ label: 'Update Page', id: 'update' },
{ label: 'Delete Page', id: 'delete' },
{ label: 'Search Content', id: 'search' },
{ label: 'Create Comment', id: 'create_comment' },
{ label: 'List Comments', id: 'list_comments' },
{ label: 'Update Comment', id: 'update_comment' },
{ label: 'Delete Comment', id: 'delete_comment' },
{ label: 'Upload Attachment', id: 'upload_attachment' },
{ label: 'List Attachments', id: 'list_attachments' },
{ label: 'Delete Attachment', id: 'delete_attachment' },
{ label: 'List Labels', id: 'list_labels' },
{ label: 'Get Space', id: 'get_space' },
{ label: 'List Spaces', id: 'list_spaces' },
],
value: () => 'read',
},
{
id: 'domain',
title: 'Domain',
type: 'short-input',
placeholder: 'Enter Confluence domain (e.g., simstudio.atlassian.net)',
required: true,
},
{
id: 'credential',
title: 'Confluence Account',
type: 'oauth-input',
serviceId: 'confluence',
requiredScopes: [
'read:confluence-content.all',
'read:confluence-space.summary',
'read:space:confluence',
'read:space-details:confluence',
'write:confluence-content',
'write:confluence-space',
'write:confluence-file',
'read:content:confluence',
'read:page:confluence',
'write:page:confluence',
'read:comment:confluence',
'write:comment:confluence',
'delete:comment:confluence',
'read:attachment:confluence',
'write:attachment:confluence',
'delete:attachment:confluence',
'delete:page:confluence',
'read:label:confluence',
'write:label:confluence',
'search:confluence',
'read:me',
'offline_access',
],
placeholder: 'Select Confluence account',
required: true,
},
{
id: 'pageId',
title: 'Select Page',
type: 'file-selector',
canonicalParamId: 'pageId',
serviceId: 'confluence',
placeholder: 'Select Confluence page',
dependsOn: ['credential', 'domain'],
mode: 'basic',
},
{
id: 'manualPageId',
title: 'Page ID',
type: 'short-input',
canonicalParamId: 'pageId',
placeholder: 'Enter Confluence page ID',
mode: 'advanced',
},
{
id: 'spaceId',
title: 'Space ID',
type: 'short-input',
placeholder: 'Enter Confluence space ID',
required: true,
condition: { field: 'operation', value: ['create', 'get_space'] },
},
{
id: 'title',
title: 'Title',
type: 'short-input',
placeholder: 'Enter title for the page',
condition: { field: 'operation', value: ['create', 'update'] },
},
{
id: 'content',
title: 'Content',
type: 'long-input',
placeholder: 'Enter content for the page',
condition: { field: 'operation', value: ['create', 'update'] },
},
{
id: 'parentId',
title: 'Parent Page ID',
type: 'short-input',
placeholder: 'Enter parent page ID (optional)',
condition: { field: 'operation', value: 'create' },
},
{
id: 'query',
title: 'Search Query',
type: 'short-input',
placeholder: 'Enter search query',
required: true,
condition: { field: 'operation', value: 'search' },
},
{
id: 'comment',
title: 'Comment Text',
type: 'long-input',
placeholder: 'Enter comment text',
required: true,
condition: { field: 'operation', value: ['create_comment', 'update_comment'] },
},
{
id: 'commentId',
title: 'Comment ID',
type: 'short-input',
placeholder: 'Enter comment ID',
required: true,
condition: { field: 'operation', value: ['update_comment', 'delete_comment'] },
},
{
id: 'attachmentId',
title: 'Attachment ID',
type: 'short-input',
placeholder: 'Enter attachment ID',
required: true,
condition: { field: 'operation', value: 'delete_attachment' },
},
{
id: 'attachmentFileUpload',
title: 'File',
type: 'file-upload',
canonicalParamId: 'attachmentFile',
placeholder: 'Select file to upload',
condition: { field: 'operation', value: 'upload_attachment' },
mode: 'basic',
},
{
id: 'attachmentFileReference',
title: 'File',
type: 'short-input',
canonicalParamId: 'attachmentFile',
placeholder: 'Reference file from previous blocks',
condition: { field: 'operation', value: 'upload_attachment' },
mode: 'advanced',
},
{
id: 'attachmentFileName',
title: 'File Name',
type: 'short-input',
placeholder: 'Optional custom file name',
condition: { field: 'operation', value: 'upload_attachment' },
},
{
id: 'attachmentComment',
title: 'Comment',
type: 'short-input',
placeholder: 'Optional comment for the attachment',
condition: { field: 'operation', value: 'upload_attachment' },
},
{
id: 'labelName',
title: 'Label Name',
type: 'short-input',
placeholder: 'Enter label name',
required: true,
condition: { field: 'operation', value: ['add_label', 'remove_label'] },
},
{
id: 'limit',
title: 'Limit',
type: 'short-input',
placeholder: 'Enter maximum number of results (default: 25)',
condition: {
field: 'operation',
value: ['search', 'list_comments', 'list_attachments', 'list_spaces'],
},
},
],
tools: {
access: [
'confluence_retrieve',
'confluence_update',
'confluence_create_page',
'confluence_delete_page',
'confluence_search',
'confluence_create_comment',
'confluence_list_comments',
'confluence_update_comment',
'confluence_delete_comment',
'confluence_upload_attachment',
'confluence_list_attachments',
'confluence_delete_attachment',
'confluence_list_labels',
'confluence_get_space',
'confluence_list_spaces',
],
config: {
tool: (params) => {
switch (params.operation) {
case 'read':
return 'confluence_retrieve'
case 'create':
return 'confluence_create_page'
case 'update':
return 'confluence_update'
case 'delete':
return 'confluence_delete_page'
case 'search':
return 'confluence_search'
case 'create_comment':
return 'confluence_create_comment'
case 'list_comments':
return 'confluence_list_comments'
case 'update_comment':
return 'confluence_update_comment'
case 'delete_comment':
return 'confluence_delete_comment'
case 'upload_attachment':
return 'confluence_upload_attachment'
case 'list_attachments':
return 'confluence_list_attachments'
case 'delete_attachment':
return 'confluence_delete_attachment'
case 'list_labels':
return 'confluence_list_labels'
case 'get_space':
return 'confluence_get_space'
case 'list_spaces':
return 'confluence_list_spaces'
default:
return 'confluence_retrieve'
}
},
params: (params) => {
const {
credential,
pageId,
manualPageId,
operation,
attachmentFileUpload,
attachmentFileReference,
attachmentFile,
attachmentFileName,
attachmentComment,
...rest
} = params
const effectivePageId = (pageId || manualPageId || '').trim()
const requiresPageId = [
'read',
'update',
'delete',
'create_comment',
'list_comments',
'list_attachments',
'list_labels',
'upload_attachment',
]
const requiresSpaceId = ['create', 'get_space']
if (requiresPageId.includes(operation) && !effectivePageId) {
throw new Error('Page ID is required. Please select a page or enter a page ID manually.')
}
if (requiresSpaceId.includes(operation) && !rest.spaceId) {
throw new Error('Space ID is required for this operation.')
}
if (operation === 'upload_attachment') {
const fileInput = attachmentFileUpload || attachmentFileReference || attachmentFile
if (!fileInput) {
throw new Error('File is required for upload attachment operation.')
}
return {
credential,
pageId: effectivePageId,
operation,
file: fileInput,
fileName: attachmentFileName,
comment: attachmentComment,
...rest,
}
}
return {
credential,
pageId: effectivePageId || undefined,
operation,
...rest,
}
},
},
},
inputs: {
operation: { type: 'string', description: 'Operation to perform' },
domain: { type: 'string', description: 'Confluence domain' },
credential: { type: 'string', description: 'Confluence access token' },
pageId: { type: 'string', description: 'Page identifier' },
manualPageId: { type: 'string', description: 'Manual page identifier' },
spaceId: { type: 'string', description: 'Space identifier' },
title: { type: 'string', description: 'Page title' },
content: { type: 'string', description: 'Page content' },
parentId: { type: 'string', description: 'Parent page identifier' },
query: { type: 'string', description: 'Search query' },
comment: { type: 'string', description: 'Comment text' },
commentId: { type: 'string', description: 'Comment identifier' },
attachmentId: { type: 'string', description: 'Attachment identifier' },
attachmentFile: { type: 'json', description: 'File to upload as attachment' },
attachmentFileUpload: { type: 'json', description: 'Uploaded file (basic mode)' },
attachmentFileReference: { type: 'json', description: 'File reference (advanced mode)' },
attachmentFileName: { type: 'string', description: 'Custom file name for attachment' },
attachmentComment: { type: 'string', description: 'Comment for the attachment' },
labelName: { type: 'string', description: 'Label name' },
limit: { type: 'number', description: 'Maximum number of results' },
},
}

View File

@@ -1,14 +1,13 @@
import { createLogger } from '@sim/logger' import { createLogger } from '@sim/logger'
import { DocumentIcon } from '@/components/icons' import { DocumentIcon } from '@/components/icons'
import type { BlockConfig, SubBlockType } from '@/blocks/types' import type { BlockConfig, SubBlockType } from '@/blocks/types'
import { createVersionedToolSelector } from '@/blocks/utils'
import type { FileParserOutput } from '@/tools/file/types' import type { FileParserOutput } from '@/tools/file/types'
const logger = createLogger('FileBlock') const logger = createLogger('FileBlock')
export const FileBlock: BlockConfig<FileParserOutput> = { export const FileBlock: BlockConfig<FileParserOutput> = {
type: 'file', type: 'file',
name: 'File (Legacy)', name: 'File',
description: 'Read and parse multiple files', description: 'Read and parse multiple files',
longDescription: `Integrate File into the workflow. Can upload a file manually or insert a file url.`, longDescription: `Integrate File into the workflow. Can upload a file manually or insert a file url.`,
bestPractices: ` bestPractices: `
@@ -18,7 +17,6 @@ export const FileBlock: BlockConfig<FileParserOutput> = {
category: 'tools', category: 'tools',
bgColor: '#40916C', bgColor: '#40916C',
icon: DocumentIcon, icon: DocumentIcon,
hideFromToolbar: true,
subBlocks: [ subBlocks: [
{ {
id: 'inputMethod', id: 'inputMethod',
@@ -114,99 +112,6 @@ export const FileBlock: BlockConfig<FileParserOutput> = {
fileType: { type: 'string', description: 'File type' }, fileType: { type: 'string', description: 'File type' },
file: { type: 'json', description: 'Uploaded file data' }, file: { type: 'json', description: 'Uploaded file data' },
}, },
outputs: {
files: {
type: 'json',
description: 'Array of parsed file objects with content, metadata, and file properties',
},
combinedContent: {
type: 'string',
description: 'All file contents merged into a single text string',
},
processedFiles: {
type: 'files',
description: 'Array of UserFile objects for downstream use (attachments, uploads, etc.)',
},
},
}
export const FileV2Block: BlockConfig<FileParserOutput> = {
...FileBlock,
type: 'file_v2',
name: 'File',
description: 'Read and parse multiple files',
hideFromToolbar: false,
subBlocks: [
{
id: 'file',
title: 'Files',
type: 'file-upload' as SubBlockType,
canonicalParamId: 'fileInput',
acceptedTypes:
'.pdf,.csv,.doc,.docx,.txt,.md,.xlsx,.xls,.html,.htm,.pptx,.ppt,.json,.xml,.rtf',
placeholder: 'Upload files to process',
multiple: true,
mode: 'basic',
maxSize: 100,
},
{
id: 'filePath',
title: 'Files',
type: 'short-input' as SubBlockType,
canonicalParamId: 'fileInput',
placeholder: 'File URL',
mode: 'advanced',
},
],
tools: {
access: ['file_parser_v2'],
config: {
tool: createVersionedToolSelector({
baseToolSelector: () => 'file_parser',
suffix: '_v2',
fallbackToolId: 'file_parser_v2',
}),
params: (params) => {
const fileInput = params.file || params.filePath || params.fileInput
if (!fileInput) {
logger.error('No file input provided')
throw new Error('File is required')
}
if (typeof fileInput === 'string') {
return {
filePath: fileInput.trim(),
fileType: params.fileType || 'auto',
workspaceId: params._context?.workspaceId,
}
}
if (Array.isArray(fileInput) && fileInput.length > 0) {
const filePaths = fileInput.map((file) => file.path)
return {
filePath: filePaths.length === 1 ? filePaths[0] : filePaths,
fileType: params.fileType || 'auto',
}
}
if (fileInput?.path) {
return {
filePath: fileInput.path,
fileType: params.fileType || 'auto',
}
}
logger.error('Invalid file input format')
throw new Error('Invalid file input')
},
},
},
inputs: {
fileInput: { type: 'json', description: 'File input (upload or URL reference)' },
filePath: { type: 'string', description: 'File URL (advanced mode)' },
file: { type: 'json', description: 'Uploaded file data (basic mode)' },
fileType: { type: 'string', description: 'File type' },
},
outputs: { outputs: {
files: { files: {
type: 'json', type: 'json',

View File

@@ -1,13 +1,11 @@
import { MistralIcon } from '@/components/icons' import { MistralIcon } from '@/components/icons'
import { AuthMode, type BlockConfig, type SubBlockType } from '@/blocks/types' import { AuthMode, type BlockConfig, type SubBlockType } from '@/blocks/types'
import { createVersionedToolSelector } from '@/blocks/utils'
import type { MistralParserOutput } from '@/tools/mistral/types' import type { MistralParserOutput } from '@/tools/mistral/types'
export const MistralParseBlock: BlockConfig<MistralParserOutput> = { export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
type: 'mistral_parse', type: 'mistral_parse',
name: 'Mistral Parser (Legacy)', name: 'Mistral Parser',
description: 'Extract text from PDF documents', description: 'Extract text from PDF documents',
hideFromToolbar: true,
authMode: AuthMode.ApiKey, authMode: AuthMode.ApiKey,
longDescription: `Integrate Mistral Parse into the workflow. Can extract text from uploaded PDF documents, or from a URL.`, longDescription: `Integrate Mistral Parse into the workflow. Can extract text from uploaded PDF documents, or from a URL.`,
docsLink: 'https://docs.sim.ai/tools/mistral_parse', docsLink: 'https://docs.sim.ai/tools/mistral_parse',
@@ -15,6 +13,7 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
bgColor: '#000000', bgColor: '#000000',
icon: MistralIcon, icon: MistralIcon,
subBlocks: [ subBlocks: [
// Show input method selection
{ {
id: 'inputMethod', id: 'inputMethod',
title: 'Select Input Method', title: 'Select Input Method',
@@ -24,6 +23,8 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
{ id: 'upload', label: 'Upload PDF Document' }, { id: 'upload', label: 'Upload PDF Document' },
], ],
}, },
// URL input - conditional on inputMethod
{ {
id: 'filePath', id: 'filePath',
title: 'PDF Document URL', title: 'PDF Document URL',
@@ -34,6 +35,8 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
value: 'url', value: 'url',
}, },
}, },
// File upload option
{ {
id: 'fileUpload', id: 'fileUpload',
title: 'Upload PDF', title: 'Upload PDF',
@@ -43,8 +46,9 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
field: 'inputMethod', field: 'inputMethod',
value: 'upload', value: 'upload',
}, },
maxSize: 50, maxSize: 50, // 50MB max via direct upload
}, },
{ {
id: 'resultType', id: 'resultType',
title: 'Output Format', title: 'Output Format',
@@ -61,6 +65,28 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
type: 'short-input', type: 'short-input',
placeholder: 'e.g. 0,1,2 (leave empty for all pages)', placeholder: 'e.g. 0,1,2 (leave empty for all pages)',
}, },
/*
* Image-related parameters - temporarily disabled
* Uncomment if PDF image extraction is needed
*
{
id: 'includeImageBase64',
title: 'Include PDF Images',
type: 'switch',
},
{
id: 'imageLimit',
title: 'Max Images',
type: 'short-input',
placeholder: 'Maximum number of images to extract',
},
{
id: 'imageMinSize',
title: 'Min Image Size (px)',
type: 'short-input',
placeholder: 'Min width/height in pixels',
},
*/
{ {
id: 'apiKey', id: 'apiKey',
title: 'API Key', title: 'API Key',
@@ -75,15 +101,18 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
config: { config: {
tool: () => 'mistral_parser', tool: () => 'mistral_parser',
params: (params) => { params: (params) => {
// Basic validation
if (!params || !params.apiKey || params.apiKey.trim() === '') { if (!params || !params.apiKey || params.apiKey.trim() === '') {
throw new Error('Mistral API key is required') throw new Error('Mistral API key is required')
} }
const parameters: Record<string, unknown> = { // Build parameters object - file processing is now handled at the tool level
const parameters: any = {
apiKey: params.apiKey.trim(), apiKey: params.apiKey.trim(),
resultType: params.resultType || 'markdown', resultType: params.resultType || 'markdown',
} }
// Set filePath or fileUpload based on input method
const inputMethod = params.inputMethod || 'url' const inputMethod = params.inputMethod || 'url'
if (inputMethod === 'url') { if (inputMethod === 'url') {
if (!params.filePath || params.filePath.trim() === '') { if (!params.filePath || params.filePath.trim() === '') {
@@ -94,9 +123,11 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
if (!params.fileUpload) { if (!params.fileUpload) {
throw new Error('Please upload a PDF document') throw new Error('Please upload a PDF document')
} }
// Pass the entire fileUpload object to the tool
parameters.fileUpload = params.fileUpload parameters.fileUpload = params.fileUpload
} }
// Convert pages input from string to array of numbers if provided
let pagesArray: number[] | undefined let pagesArray: number[] | undefined
if (params.pages && params.pages.trim() !== '') { if (params.pages && params.pages.trim() !== '') {
try { try {
@@ -115,12 +146,12 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
if (pagesArray && pagesArray.length === 0) { if (pagesArray && pagesArray.length === 0) {
pagesArray = undefined pagesArray = undefined
} }
} catch (error: unknown) { } catch (error: any) {
const errorMessage = error instanceof Error ? error.message : String(error) throw new Error(`Page number format error: ${error.message}`)
throw new Error(`Page number format error: ${errorMessage}`)
} }
} }
// Add optional parameters
if (pagesArray && pagesArray.length > 0) { if (pagesArray && pagesArray.length > 0) {
parameters.pages = pagesArray parameters.pages = pagesArray
} }
@@ -142,129 +173,3 @@ export const MistralParseBlock: BlockConfig<MistralParserOutput> = {
metadata: { type: 'json', description: 'Processing metadata' }, metadata: { type: 'json', description: 'Processing metadata' },
}, },
} }
export const MistralParseV2Block: BlockConfig<MistralParserOutput> = {
...MistralParseBlock,
type: 'mistral_parse_v2',
name: 'Mistral Parser',
description: 'Extract text from PDF documents',
hideFromToolbar: false,
subBlocks: [
{
id: 'fileUpload',
title: 'PDF Document',
type: 'file-upload' as SubBlockType,
canonicalParamId: 'document',
acceptedTypes: 'application/pdf',
placeholder: 'Upload a PDF document',
mode: 'basic',
maxSize: 50,
},
{
id: 'filePath',
title: 'PDF Document',
type: 'short-input' as SubBlockType,
canonicalParamId: 'document',
placeholder: 'Document URL',
mode: 'advanced',
},
{
id: 'resultType',
title: 'Output Format',
type: 'dropdown',
options: [
{ id: 'markdown', label: 'Markdown' },
{ id: 'text', label: 'Plain Text' },
{ id: 'json', label: 'JSON' },
],
},
{
id: 'pages',
title: 'Specific Pages',
type: 'short-input',
placeholder: 'e.g. 0,1,2 (leave empty for all pages)',
},
{
id: 'apiKey',
title: 'API Key',
type: 'short-input' as SubBlockType,
placeholder: 'Enter your Mistral API key',
password: true,
required: true,
},
],
tools: {
access: ['mistral_parser_v2'],
config: {
tool: createVersionedToolSelector({
baseToolSelector: () => 'mistral_parser',
suffix: '_v2',
fallbackToolId: 'mistral_parser_v2',
}),
params: (params) => {
if (!params || !params.apiKey || params.apiKey.trim() === '') {
throw new Error('Mistral API key is required')
}
const parameters: Record<string, unknown> = {
apiKey: params.apiKey.trim(),
resultType: params.resultType || 'markdown',
}
const documentInput = params.fileUpload || params.filePath || params.document
if (!documentInput) {
throw new Error('PDF document is required')
}
if (typeof documentInput === 'object') {
parameters.fileUpload = documentInput
} else if (typeof documentInput === 'string') {
parameters.filePath = documentInput.trim()
}
let pagesArray: number[] | undefined
if (params.pages && params.pages.trim() !== '') {
try {
pagesArray = params.pages
.split(',')
.map((p: string) => p.trim())
.filter((p: string) => p.length > 0)
.map((p: string) => {
const num = Number.parseInt(p, 10)
if (Number.isNaN(num) || num < 0) {
throw new Error(`Invalid page number: ${p}`)
}
return num
})
if (pagesArray && pagesArray.length === 0) {
pagesArray = undefined
}
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error)
throw new Error(`Page number format error: ${errorMessage}`)
}
}
if (pagesArray && pagesArray.length > 0) {
parameters.pages = pagesArray
}
return parameters
},
},
},
inputs: {
document: { type: 'json', description: 'Document input (file upload or URL reference)' },
filePath: { type: 'string', description: 'PDF document URL (advanced mode)' },
fileUpload: { type: 'json', description: 'Uploaded PDF file (basic mode)' },
apiKey: { type: 'string', description: 'Mistral API key' },
resultType: { type: 'string', description: 'Output format type' },
pages: { type: 'string', description: 'Page selection' },
},
outputs: {
pages: { type: 'array', description: 'Array of page objects from Mistral OCR' },
model: { type: 'string', description: 'Mistral OCR model identifier' },
usage_info: { type: 'json', description: 'Usage statistics from the API' },
document_annotation: { type: 'string', description: 'Structured annotation data' },
},
}

View File

@@ -15,22 +15,34 @@ export const PulseBlock: BlockConfig<PulseParserOutput> = {
icon: PulseIcon, icon: PulseIcon,
subBlocks: [ subBlocks: [
{ {
id: 'fileUpload', id: 'inputMethod',
title: 'Document', title: 'Select Input Method',
type: 'file-upload' as SubBlockType, type: 'dropdown' as SubBlockType,
canonicalParamId: 'document', options: [
acceptedTypes: 'application/pdf,image/*,.docx,.pptx,.xlsx', { id: 'url', label: 'Document URL' },
placeholder: 'Upload a document', { id: 'upload', label: 'Upload Document' },
mode: 'basic', ],
maxSize: 50,
}, },
{ {
id: 'filePath', id: 'filePath',
title: 'Document', title: 'Document URL',
type: 'short-input' as SubBlockType, type: 'short-input' as SubBlockType,
canonicalParamId: 'document', placeholder: 'Enter full URL to a document (https://example.com/document.pdf)',
placeholder: 'Document URL', condition: {
mode: 'advanced', field: 'inputMethod',
value: 'url',
},
},
{
id: 'fileUpload',
title: 'Upload Document',
type: 'file-upload' as SubBlockType,
acceptedTypes: 'application/pdf,image/*,.docx,.pptx,.xlsx',
condition: {
field: 'inputMethod',
value: 'upload',
},
maxSize: 50,
}, },
{ {
id: 'pages', id: 'pages',
@@ -72,14 +84,17 @@ export const PulseBlock: BlockConfig<PulseParserOutput> = {
apiKey: params.apiKey.trim(), apiKey: params.apiKey.trim(),
} }
const documentInput = params.fileUpload || params.filePath || params.document const inputMethod = params.inputMethod || 'url'
if (!documentInput) { if (inputMethod === 'url') {
throw new Error('Document is required') if (!params.filePath || params.filePath.trim() === '') {
} throw new Error('Document URL is required')
if (typeof documentInput === 'object') { }
parameters.fileUpload = documentInput parameters.filePath = params.filePath.trim()
} else if (typeof documentInput === 'string') { } else if (inputMethod === 'upload') {
parameters.filePath = documentInput.trim() if (!params.fileUpload) {
throw new Error('Please upload a document')
}
parameters.fileUpload = params.fileUpload
} }
if (params.pages && params.pages.trim() !== '') { if (params.pages && params.pages.trim() !== '') {
@@ -102,9 +117,9 @@ export const PulseBlock: BlockConfig<PulseParserOutput> = {
}, },
}, },
inputs: { inputs: {
document: { type: 'json', description: 'Document input (file upload or URL reference)' }, inputMethod: { type: 'string', description: 'Input method selection' },
filePath: { type: 'string', description: 'Document URL (advanced mode)' }, filePath: { type: 'string', description: 'Document URL' },
fileUpload: { type: 'json', description: 'Uploaded document file (basic mode)' }, fileUpload: { type: 'json', description: 'Uploaded document file' },
apiKey: { type: 'string', description: 'Pulse API key' }, apiKey: { type: 'string', description: 'Pulse API key' },
pages: { type: 'string', description: 'Page range selection' }, pages: { type: 'string', description: 'Page range selection' },
chunking: { chunking: {

View File

@@ -14,22 +14,34 @@ export const ReductoBlock: BlockConfig<ReductoParserOutput> = {
icon: ReductoIcon, icon: ReductoIcon,
subBlocks: [ subBlocks: [
{ {
id: 'fileUpload', id: 'inputMethod',
title: 'PDF Document', title: 'Select Input Method',
type: 'file-upload' as SubBlockType, type: 'dropdown' as SubBlockType,
canonicalParamId: 'document', options: [
acceptedTypes: 'application/pdf', { id: 'url', label: 'PDF Document URL' },
placeholder: 'Upload a PDF document', { id: 'upload', label: 'Upload PDF Document' },
mode: 'basic', ],
maxSize: 50,
}, },
{ {
id: 'filePath', id: 'filePath',
title: 'PDF Document', title: 'PDF Document URL',
type: 'short-input' as SubBlockType, type: 'short-input' as SubBlockType,
canonicalParamId: 'document', placeholder: 'Enter full URL to a PDF document (https://example.com/document.pdf)',
placeholder: 'Document URL', condition: {
mode: 'advanced', field: 'inputMethod',
value: 'url',
},
},
{
id: 'fileUpload',
title: 'Upload PDF',
type: 'file-upload' as SubBlockType,
acceptedTypes: 'application/pdf',
condition: {
field: 'inputMethod',
value: 'upload',
},
maxSize: 50,
}, },
{ {
id: 'pages', id: 'pages',
@@ -68,15 +80,17 @@ export const ReductoBlock: BlockConfig<ReductoParserOutput> = {
apiKey: params.apiKey.trim(), apiKey: params.apiKey.trim(),
} }
const documentInput = params.fileUpload || params.filePath || params.document const inputMethod = params.inputMethod || 'url'
if (!documentInput) { if (inputMethod === 'url') {
throw new Error('PDF document is required') if (!params.filePath || params.filePath.trim() === '') {
} throw new Error('PDF Document URL is required')
}
if (typeof documentInput === 'object') { parameters.filePath = params.filePath.trim()
parameters.fileUpload = documentInput } else if (inputMethod === 'upload') {
} else if (typeof documentInput === 'string') { if (!params.fileUpload) {
parameters.filePath = documentInput.trim() throw new Error('Please upload a PDF document')
}
parameters.fileUpload = params.fileUpload
} }
let pagesArray: number[] | undefined let pagesArray: number[] | undefined
@@ -116,9 +130,9 @@ export const ReductoBlock: BlockConfig<ReductoParserOutput> = {
}, },
}, },
inputs: { inputs: {
document: { type: 'json', description: 'Document input (file upload or URL reference)' }, inputMethod: { type: 'string', description: 'Input method selection' },
filePath: { type: 'string', description: 'PDF document URL (advanced mode)' }, filePath: { type: 'string', description: 'PDF document URL' },
fileUpload: { type: 'json', description: 'Uploaded PDF file (basic mode)' }, fileUpload: { type: 'json', description: 'Uploaded PDF file' },
apiKey: { type: 'string', description: 'Reducto API key' }, apiKey: { type: 'string', description: 'Reducto API key' },
pages: { type: 'string', description: 'Page selection' }, pages: { type: 'string', description: 'Page selection' },
tableOutputFormat: { type: 'string', description: 'Table output format' }, tableOutputFormat: { type: 'string', description: 'Table output format' },

View File

@@ -414,10 +414,6 @@ export const S3Block: BlockConfig<S3Response> = {
}, },
outputs: { outputs: {
url: { type: 'string', description: 'URL of S3 object' }, url: { type: 'string', description: 'URL of S3 object' },
uri: {
type: 'string',
description: 'S3 URI (s3://bucket/key) for use with other AWS services',
},
objects: { type: 'json', description: 'List of objects (for list operation)' }, objects: { type: 'json', description: 'List of objects (for list operation)' },
deleted: { type: 'boolean', description: 'Deletion status' }, deleted: { type: 'boolean', description: 'Deletion status' },
metadata: { type: 'json', description: 'Operation metadata' }, metadata: { type: 'json', description: 'Operation metadata' },

View File

@@ -1,191 +0,0 @@
import { TextractIcon } from '@/components/icons'
import { AuthMode, type BlockConfig, type SubBlockType } from '@/blocks/types'
import type { TextractParserOutput } from '@/tools/textract/types'
export const TextractBlock: BlockConfig<TextractParserOutput> = {
type: 'textract',
name: 'AWS Textract',
description: 'Extract text, tables, and forms from documents',
authMode: AuthMode.ApiKey,
longDescription: `Integrate AWS Textract into your workflow to extract text, tables, forms, and key-value pairs from documents. Single-page mode supports JPEG, PNG, and single-page PDF. Multi-page mode supports multi-page PDF and TIFF.`,
docsLink: 'https://docs.sim.ai/tools/textract',
category: 'tools',
bgColor: 'linear-gradient(135deg, #055F4E 0%, #56C0A7 100%)',
icon: TextractIcon,
subBlocks: [
{
id: 'processingMode',
title: 'Processing Mode',
type: 'dropdown' as SubBlockType,
options: [
{ id: 'sync', label: 'Single Page (JPEG, PNG, 1-page PDF)' },
{ id: 'async', label: 'Multi-Page (PDF, TIFF via S3)' },
],
tooltip:
'Single Page uses synchronous API for JPEG, PNG, or single-page PDF. Multi-Page uses async API for multi-page PDF/TIFF stored in S3.',
},
{
id: 'fileUpload',
title: 'Document',
type: 'file-upload' as SubBlockType,
canonicalParamId: 'document',
acceptedTypes: 'image/jpeg,image/png,application/pdf',
placeholder: 'Upload JPEG, PNG, or single-page PDF (max 10MB)',
condition: {
field: 'processingMode',
value: 'async',
not: true,
},
mode: 'basic',
maxSize: 10,
},
{
id: 'filePath',
title: 'Document',
type: 'short-input' as SubBlockType,
canonicalParamId: 'document',
placeholder: 'URL to JPEG, PNG, or single-page PDF',
condition: {
field: 'processingMode',
value: 'async',
not: true,
},
mode: 'advanced',
},
{
id: 's3Uri',
title: 'S3 URI',
type: 'short-input' as SubBlockType,
placeholder: 's3://bucket-name/path/to/document.pdf',
condition: {
field: 'processingMode',
value: 'async',
},
},
{
id: 'region',
title: 'AWS Region',
type: 'short-input' as SubBlockType,
placeholder: 'e.g., us-east-1',
required: true,
},
{
id: 'accessKeyId',
title: 'AWS Access Key ID',
type: 'short-input' as SubBlockType,
placeholder: 'Enter your AWS Access Key ID',
password: true,
required: true,
},
{
id: 'secretAccessKey',
title: 'AWS Secret Access Key',
type: 'short-input' as SubBlockType,
placeholder: 'Enter your AWS Secret Access Key',
password: true,
required: true,
},
{
id: 'extractTables',
title: 'Extract Tables',
type: 'switch' as SubBlockType,
},
{
id: 'extractForms',
title: 'Extract Forms (Key-Value Pairs)',
type: 'switch' as SubBlockType,
},
{
id: 'detectSignatures',
title: 'Detect Signatures',
type: 'switch' as SubBlockType,
},
{
id: 'analyzeLayout',
title: 'Analyze Document Layout',
type: 'switch' as SubBlockType,
},
],
tools: {
access: ['textract_parser'],
config: {
tool: () => 'textract_parser',
params: (params) => {
if (!params.accessKeyId || params.accessKeyId.trim() === '') {
throw new Error('AWS Access Key ID is required')
}
if (!params.secretAccessKey || params.secretAccessKey.trim() === '') {
throw new Error('AWS Secret Access Key is required')
}
if (!params.region || params.region.trim() === '') {
throw new Error('AWS Region is required')
}
const processingMode = params.processingMode || 'sync'
const parameters: Record<string, unknown> = {
accessKeyId: params.accessKeyId.trim(),
secretAccessKey: params.secretAccessKey.trim(),
region: params.region.trim(),
processingMode,
}
if (processingMode === 'async') {
if (!params.s3Uri || params.s3Uri.trim() === '') {
throw new Error('S3 URI is required for multi-page processing')
}
parameters.s3Uri = params.s3Uri.trim()
} else {
const documentInput = params.fileUpload || params.filePath || params.document
if (!documentInput) {
throw new Error('Document is required')
}
if (typeof documentInput === 'object') {
parameters.fileUpload = documentInput
} else if (typeof documentInput === 'string') {
parameters.filePath = documentInput.trim()
}
}
const featureTypes: string[] = []
if (params.extractTables) featureTypes.push('TABLES')
if (params.extractForms) featureTypes.push('FORMS')
if (params.detectSignatures) featureTypes.push('SIGNATURES')
if (params.analyzeLayout) featureTypes.push('LAYOUT')
if (featureTypes.length > 0) {
parameters.featureTypes = featureTypes
}
return parameters
},
},
},
inputs: {
processingMode: { type: 'string', description: 'Document type: single-page or multi-page' },
document: { type: 'json', description: 'Document input (file upload or URL reference)' },
filePath: { type: 'string', description: 'Document URL (advanced mode)' },
fileUpload: { type: 'json', description: 'Uploaded document file (basic mode)' },
s3Uri: { type: 'string', description: 'S3 URI for multi-page processing (s3://bucket/key)' },
extractTables: { type: 'boolean', description: 'Extract tables from document' },
extractForms: { type: 'boolean', description: 'Extract form key-value pairs' },
detectSignatures: { type: 'boolean', description: 'Detect signatures' },
analyzeLayout: { type: 'boolean', description: 'Analyze document layout' },
region: { type: 'string', description: 'AWS region' },
accessKeyId: { type: 'string', description: 'AWS Access Key ID' },
secretAccessKey: { type: 'string', description: 'AWS Secret Access Key' },
},
outputs: {
blocks: {
type: 'json',
description: 'Array of detected blocks (PAGE, LINE, WORD, TABLE, CELL, KEY_VALUE_SET, etc.)',
},
documentMetadata: {
type: 'json',
description: 'Document metadata containing pages count',
},
modelVersion: {
type: 'string',
description: 'Version of the Textract model used for processing',
},
},
}

View File

@@ -4,9 +4,8 @@ import type { VideoBlockResponse } from '@/tools/video/types'
export const VideoGeneratorBlock: BlockConfig<VideoBlockResponse> = { export const VideoGeneratorBlock: BlockConfig<VideoBlockResponse> = {
type: 'video_generator', type: 'video_generator',
name: 'Video Generator (Legacy)', name: 'Video Generator',
description: 'Generate videos from text using AI', description: 'Generate videos from text using AI',
hideFromToolbar: true,
authMode: AuthMode.ApiKey, authMode: AuthMode.ApiKey,
longDescription: longDescription:
'Generate high-quality videos from text prompts using leading AI providers. Supports multiple models, aspect ratios, resolutions, and provider-specific features like world consistency, camera controls, and audio generation.', 'Generate high-quality videos from text prompts using leading AI providers. Supports multiple models, aspect ratios, resolutions, and provider-specific features like world consistency, camera controls, and audio generation.',
@@ -428,378 +427,3 @@ export const VideoGeneratorBlock: BlockConfig<VideoBlockResponse> = {
model: { type: 'string', description: 'Model used' }, model: { type: 'string', description: 'Model used' },
}, },
} }
export const VideoGeneratorV2Block: BlockConfig<VideoBlockResponse> = {
...VideoGeneratorBlock,
type: 'video_generator_v2',
name: 'Video Generator',
hideFromToolbar: false,
subBlocks: [
{
id: 'provider',
title: 'Provider',
type: 'dropdown',
options: [
{ label: 'Runway Gen-4', id: 'runway' },
{ label: 'Google Veo 3', id: 'veo' },
{ label: 'Luma Dream Machine', id: 'luma' },
{ label: 'MiniMax Hailuo', id: 'minimax' },
{ label: 'Fal.ai (Multi-Model)', id: 'falai' },
],
value: () => 'runway',
required: true,
},
{
id: 'model',
title: 'Model',
type: 'dropdown',
condition: { field: 'provider', value: 'veo' },
options: [
{ label: 'Veo 3', id: 'veo-3' },
{ label: 'Veo 3 Fast', id: 'veo-3-fast' },
{ label: 'Veo 3.1', id: 'veo-3.1' },
],
value: () => 'veo-3',
required: false,
},
{
id: 'model',
title: 'Model',
type: 'dropdown',
condition: { field: 'provider', value: 'luma' },
options: [{ label: 'Ray 2', id: 'ray-2' }],
value: () => 'ray-2',
required: false,
},
{
id: 'model',
title: 'Model',
type: 'dropdown',
condition: { field: 'provider', value: 'minimax' },
options: [{ label: 'Hailuo 2.3', id: 'hailuo-02' }],
value: () => 'hailuo-02',
required: false,
},
{
id: 'endpoint',
title: 'Quality Endpoint',
type: 'dropdown',
condition: { field: 'provider', value: 'minimax' },
options: [
{ label: 'Pro', id: 'pro' },
{ label: 'Standard', id: 'standard' },
],
value: () => 'standard',
required: false,
},
{
id: 'model',
title: 'Model',
type: 'dropdown',
condition: { field: 'provider', value: 'falai' },
options: [
{ label: 'Google Veo 3.1', id: 'veo-3.1' },
{ label: 'OpenAI Sora 2', id: 'sora-2' },
{ label: 'Kling 2.5 Turbo Pro', id: 'kling-2.5-turbo-pro' },
{ label: 'Kling 2.1 Pro', id: 'kling-2.1-pro' },
{ label: 'MiniMax Hailuo 2.3 Pro', id: 'minimax-hailuo-2.3-pro' },
{ label: 'MiniMax Hailuo 2.3 Standard', id: 'minimax-hailuo-2.3-standard' },
{ label: 'WAN 2.1', id: 'wan-2.1' },
{ label: 'LTXV 0.9.8', id: 'ltxv-0.9.8' },
],
value: () => 'veo-3.1',
required: true,
},
{
id: 'prompt',
title: 'Prompt',
type: 'long-input',
placeholder: 'Describe the video you want to generate...',
required: true,
},
{
id: 'duration',
title: 'Duration (seconds)',
type: 'dropdown',
condition: { field: 'provider', value: 'runway' },
options: [
{ label: '5', id: '5' },
{ label: '10', id: '10' },
],
value: () => '5',
required: false,
},
{
id: 'duration',
title: 'Duration (seconds)',
type: 'dropdown',
condition: { field: 'provider', value: 'veo' },
options: [
{ label: '4', id: '4' },
{ label: '6', id: '6' },
{ label: '8', id: '8' },
],
value: () => '8',
required: false,
},
{
id: 'duration',
title: 'Duration (seconds)',
type: 'dropdown',
condition: { field: 'provider', value: 'luma' },
options: [
{ label: '5', id: '5' },
{ label: '9', id: '9' },
],
value: () => '5',
required: false,
},
{
id: 'duration',
title: 'Duration (seconds)',
type: 'dropdown',
condition: { field: 'provider', value: 'minimax' },
options: [
{ label: '6', id: '6' },
{ label: '10', id: '10' },
],
value: () => '6',
required: false,
},
{
id: 'duration',
title: 'Duration (seconds)',
type: 'dropdown',
condition: {
field: 'model',
value: [
'kling-2.5-turbo-pro',
'kling-2.1-pro',
'minimax-hailuo-2.3-pro',
'minimax-hailuo-2.3-standard',
],
},
options: [
{ label: '5', id: '5' },
{ label: '8', id: '8' },
{ label: '10', id: '10' },
],
value: () => '5',
required: false,
},
{
id: 'aspectRatio',
title: 'Aspect Ratio',
type: 'dropdown',
condition: { field: 'provider', value: 'veo' },
options: [
{ label: '16:9', id: '16:9' },
{ label: '9:16', id: '9:16' },
],
value: () => '16:9',
required: false,
},
{
id: 'aspectRatio',
title: 'Aspect Ratio',
type: 'dropdown',
condition: { field: 'provider', value: 'runway' },
options: [
{ label: '16:9', id: '16:9' },
{ label: '9:16', id: '9:16' },
{ label: '1:1', id: '1:1' },
],
value: () => '16:9',
required: false,
},
{
id: 'aspectRatio',
title: 'Aspect Ratio',
type: 'dropdown',
condition: { field: 'provider', value: 'luma' },
options: [
{ label: '16:9', id: '16:9' },
{ label: '9:16', id: '9:16' },
{ label: '1:1', id: '1:1' },
],
value: () => '16:9',
required: false,
},
{
id: 'aspectRatio',
title: 'Aspect Ratio',
type: 'dropdown',
condition: {
field: 'model',
value: [
'kling-2.5-turbo-pro',
'kling-2.1-pro',
'minimax-hailuo-2.3-pro',
'minimax-hailuo-2.3-standard',
],
},
options: [
{ label: '16:9', id: '16:9' },
{ label: '9:16', id: '9:16' },
],
value: () => '16:9',
required: false,
},
{
id: 'resolution',
title: 'Resolution',
type: 'dropdown',
condition: { field: 'provider', value: 'veo' },
options: [
{ label: '720p', id: '720p' },
{ label: '1080p', id: '1080p' },
],
value: () => '1080p',
required: false,
},
{
id: 'resolution',
title: 'Resolution',
type: 'dropdown',
condition: { field: 'provider', value: 'luma' },
options: [
{ label: '540p', id: '540p' },
{ label: '720p', id: '720p' },
{ label: '1080p', id: '1080p' },
],
value: () => '1080p',
required: false,
},
{
id: 'visualReferenceUpload',
title: 'Reference Image',
type: 'file-upload',
canonicalParamId: 'visualReference',
condition: { field: 'provider', value: 'runway' },
placeholder: 'Upload reference image',
mode: 'basic',
multiple: false,
required: true,
acceptedTypes: '.jpg,.jpeg,.png,.webp',
},
{
id: 'visualReferenceInput',
title: 'Reference Image',
type: 'short-input',
canonicalParamId: 'visualReference',
condition: { field: 'provider', value: 'runway' },
placeholder: 'Reference image from previous blocks',
mode: 'advanced',
},
{
id: 'cameraControl',
title: 'Camera Controls',
type: 'long-input',
condition: { field: 'provider', value: 'luma' },
placeholder: 'JSON: [{ "key": "pan_right" }, { "key": "zoom_in" }]',
required: false,
},
{
id: 'promptOptimizer',
title: 'Prompt Optimizer',
type: 'switch',
condition: { field: 'provider', value: 'minimax' },
},
{
id: 'apiKey',
title: 'API Key',
type: 'short-input',
placeholder: 'Enter your provider API key',
password: true,
required: true,
},
],
tools: {
access: ['video_runway', 'video_veo', 'video_luma', 'video_minimax', 'video_falai'],
config: {
tool: (params) => {
switch (params.provider) {
case 'runway':
return 'video_runway'
case 'veo':
return 'video_veo'
case 'luma':
return 'video_luma'
case 'minimax':
return 'video_minimax'
case 'falai':
return 'video_falai'
default:
return 'video_runway'
}
},
params: (params) => {
const visualRef =
params.visualReferenceUpload || params.visualReferenceInput || params.visualReference
return {
provider: params.provider,
apiKey: params.apiKey,
model: params.model,
endpoint: params.endpoint,
prompt: params.prompt,
duration: params.duration ? Number(params.duration) : undefined,
aspectRatio: params.aspectRatio,
resolution: params.resolution,
visualReference: visualRef,
consistencyMode: params.consistencyMode,
stylePreset: params.stylePreset,
promptOptimizer: params.promptOptimizer,
cameraControl: params.cameraControl
? typeof params.cameraControl === 'string'
? JSON.parse(params.cameraControl)
: params.cameraControl
: undefined,
}
},
},
},
inputs: {
provider: {
type: 'string',
description: 'Video generation provider (runway, veo, luma, minimax)',
},
apiKey: { type: 'string', description: 'Provider API key' },
model: {
type: 'string',
description: 'Provider-specific model',
},
endpoint: {
type: 'string',
description: 'Quality endpoint for MiniMax (pro, standard)',
},
prompt: { type: 'string', description: 'Text prompt for video generation' },
duration: { type: 'number', description: 'Video duration in seconds' },
aspectRatio: {
type: 'string',
description: 'Aspect ratio (16:9, 9:16, 1:1) - not available for MiniMax',
},
resolution: {
type: 'string',
description: 'Video resolution - not available for MiniMax (fixed per endpoint)',
},
visualReference: { type: 'json', description: 'Reference image for Runway (UserFile)' },
visualReferenceUpload: { type: 'json', description: 'Uploaded reference image (basic mode)' },
visualReferenceInput: {
type: 'json',
description: 'Reference image from previous blocks (advanced mode)',
},
consistencyMode: {
type: 'string',
description: 'Consistency mode for Runway (character, object, style, location)',
},
stylePreset: { type: 'string', description: 'Style preset for Runway' },
promptOptimizer: {
type: 'boolean',
description: 'Enable prompt optimization for MiniMax (default: true)',
},
cameraControl: {
type: 'json',
description: 'Camera controls for Luma (pan, zoom, tilt, truck, tracking)',
},
},
}

View File

@@ -14,7 +14,7 @@ import { ChatTriggerBlock } from '@/blocks/blocks/chat_trigger'
import { CirclebackBlock } from '@/blocks/blocks/circleback' import { CirclebackBlock } from '@/blocks/blocks/circleback'
import { ClayBlock } from '@/blocks/blocks/clay' import { ClayBlock } from '@/blocks/blocks/clay'
import { ConditionBlock } from '@/blocks/blocks/condition' import { ConditionBlock } from '@/blocks/blocks/condition'
import { ConfluenceBlock, ConfluenceV2Block } from '@/blocks/blocks/confluence' import { ConfluenceBlock } from '@/blocks/blocks/confluence'
import { CursorBlock, CursorV2Block } from '@/blocks/blocks/cursor' import { CursorBlock, CursorV2Block } from '@/blocks/blocks/cursor'
import { DatadogBlock } from '@/blocks/blocks/datadog' import { DatadogBlock } from '@/blocks/blocks/datadog'
import { DiscordBlock } from '@/blocks/blocks/discord' import { DiscordBlock } from '@/blocks/blocks/discord'
@@ -25,7 +25,7 @@ import { ElasticsearchBlock } from '@/blocks/blocks/elasticsearch'
import { ElevenLabsBlock } from '@/blocks/blocks/elevenlabs' import { ElevenLabsBlock } from '@/blocks/blocks/elevenlabs'
import { EvaluatorBlock } from '@/blocks/blocks/evaluator' import { EvaluatorBlock } from '@/blocks/blocks/evaluator'
import { ExaBlock } from '@/blocks/blocks/exa' import { ExaBlock } from '@/blocks/blocks/exa'
import { FileBlock, FileV2Block } from '@/blocks/blocks/file' import { FileBlock } from '@/blocks/blocks/file'
import { FirecrawlBlock } from '@/blocks/blocks/firecrawl' import { FirecrawlBlock } from '@/blocks/blocks/firecrawl'
import { FirefliesBlock } from '@/blocks/blocks/fireflies' import { FirefliesBlock } from '@/blocks/blocks/fireflies'
import { FunctionBlock } from '@/blocks/blocks/function' import { FunctionBlock } from '@/blocks/blocks/function'
@@ -74,7 +74,7 @@ import { MemoryBlock } from '@/blocks/blocks/memory'
import { MicrosoftExcelBlock, MicrosoftExcelV2Block } from '@/blocks/blocks/microsoft_excel' import { MicrosoftExcelBlock, MicrosoftExcelV2Block } from '@/blocks/blocks/microsoft_excel'
import { MicrosoftPlannerBlock } from '@/blocks/blocks/microsoft_planner' import { MicrosoftPlannerBlock } from '@/blocks/blocks/microsoft_planner'
import { MicrosoftTeamsBlock } from '@/blocks/blocks/microsoft_teams' import { MicrosoftTeamsBlock } from '@/blocks/blocks/microsoft_teams'
import { MistralParseBlock, MistralParseV2Block } from '@/blocks/blocks/mistral_parse' import { MistralParseBlock } from '@/blocks/blocks/mistral_parse'
import { MongoDBBlock } from '@/blocks/blocks/mongodb' import { MongoDBBlock } from '@/blocks/blocks/mongodb'
import { MySQLBlock } from '@/blocks/blocks/mysql' import { MySQLBlock } from '@/blocks/blocks/mysql'
import { Neo4jBlock } from '@/blocks/blocks/neo4j' import { Neo4jBlock } from '@/blocks/blocks/neo4j'
@@ -123,7 +123,6 @@ import { SttBlock } from '@/blocks/blocks/stt'
import { SupabaseBlock } from '@/blocks/blocks/supabase' import { SupabaseBlock } from '@/blocks/blocks/supabase'
import { TavilyBlock } from '@/blocks/blocks/tavily' import { TavilyBlock } from '@/blocks/blocks/tavily'
import { TelegramBlock } from '@/blocks/blocks/telegram' import { TelegramBlock } from '@/blocks/blocks/telegram'
import { TextractBlock } from '@/blocks/blocks/textract'
import { ThinkingBlock } from '@/blocks/blocks/thinking' import { ThinkingBlock } from '@/blocks/blocks/thinking'
import { TinybirdBlock } from '@/blocks/blocks/tinybird' import { TinybirdBlock } from '@/blocks/blocks/tinybird'
import { TranslateBlock } from '@/blocks/blocks/translate' import { TranslateBlock } from '@/blocks/blocks/translate'
@@ -133,7 +132,7 @@ import { TwilioSMSBlock } from '@/blocks/blocks/twilio'
import { TwilioVoiceBlock } from '@/blocks/blocks/twilio_voice' import { TwilioVoiceBlock } from '@/blocks/blocks/twilio_voice'
import { TypeformBlock } from '@/blocks/blocks/typeform' import { TypeformBlock } from '@/blocks/blocks/typeform'
import { VariablesBlock } from '@/blocks/blocks/variables' import { VariablesBlock } from '@/blocks/blocks/variables'
import { VideoGeneratorBlock, VideoGeneratorV2Block } from '@/blocks/blocks/video_generator' import { VideoGeneratorBlock } from '@/blocks/blocks/video_generator'
import { VisionBlock } from '@/blocks/blocks/vision' import { VisionBlock } from '@/blocks/blocks/vision'
import { WaitBlock } from '@/blocks/blocks/wait' import { WaitBlock } from '@/blocks/blocks/wait'
import { WealthboxBlock } from '@/blocks/blocks/wealthbox' import { WealthboxBlock } from '@/blocks/blocks/wealthbox'
@@ -170,7 +169,6 @@ export const registry: Record<string, BlockConfig> = {
clay: ClayBlock, clay: ClayBlock,
condition: ConditionBlock, condition: ConditionBlock,
confluence: ConfluenceBlock, confluence: ConfluenceBlock,
confluence_v2: ConfluenceV2Block,
cursor: CursorBlock, cursor: CursorBlock,
cursor_v2: CursorV2Block, cursor_v2: CursorV2Block,
datadog: DatadogBlock, datadog: DatadogBlock,
@@ -183,7 +181,6 @@ export const registry: Record<string, BlockConfig> = {
evaluator: EvaluatorBlock, evaluator: EvaluatorBlock,
exa: ExaBlock, exa: ExaBlock,
file: FileBlock, file: FileBlock,
file_v2: FileV2Block,
firecrawl: FirecrawlBlock, firecrawl: FirecrawlBlock,
fireflies: FirefliesBlock, fireflies: FirefliesBlock,
function: FunctionBlock, function: FunctionBlock,
@@ -239,7 +236,6 @@ export const registry: Record<string, BlockConfig> = {
microsoft_planner: MicrosoftPlannerBlock, microsoft_planner: MicrosoftPlannerBlock,
microsoft_teams: MicrosoftTeamsBlock, microsoft_teams: MicrosoftTeamsBlock,
mistral_parse: MistralParseBlock, mistral_parse: MistralParseBlock,
mistral_parse_v2: MistralParseV2Block,
mongodb: MongoDBBlock, mongodb: MongoDBBlock,
mysql: MySQLBlock, mysql: MySQLBlock,
neo4j: Neo4jBlock, neo4j: Neo4jBlock,
@@ -290,7 +286,6 @@ export const registry: Record<string, BlockConfig> = {
supabase: SupabaseBlock, supabase: SupabaseBlock,
tavily: TavilyBlock, tavily: TavilyBlock,
telegram: TelegramBlock, telegram: TelegramBlock,
textract: TextractBlock,
thinking: ThinkingBlock, thinking: ThinkingBlock,
tinybird: TinybirdBlock, tinybird: TinybirdBlock,
translate: TranslateBlock, translate: TranslateBlock,
@@ -301,7 +296,6 @@ export const registry: Record<string, BlockConfig> = {
typeform: TypeformBlock, typeform: TypeformBlock,
variables: VariablesBlock, variables: VariablesBlock,
video_generator: VideoGeneratorBlock, video_generator: VideoGeneratorBlock,
video_generator_v2: VideoGeneratorV2Block,
vision: VisionBlock, vision: VisionBlock,
wait: WaitBlock, wait: WaitBlock,
wealthbox: WealthboxBlock, wealthbox: WealthboxBlock,

View File

@@ -4093,23 +4093,6 @@ export function SQSIcon(props: SVGProps<SVGSVGElement>) {
) )
} }
export function TextractIcon(props: SVGProps<SVGSVGElement>) {
return (
<svg
{...props}
viewBox='10 14 60 52'
version='1.1'
xmlns='http://www.w3.org/2000/svg'
xmlnsXlink='http://www.w3.org/1999/xlink'
>
<path
d='M22.0624102,50 C24.3763895,53.603 28.4103535,56 33.0003125,56 C40.1672485,56 45.9991964,50.168 45.9991964,43 C45.9991964,35.832 40.1672485,30 33.0003125,30 C27.6033607,30 22.9664021,33.307 21.0024196,38 L23.2143999,38 C25.0393836,34.444 28.7363506,32 33.0003125,32 C39.0652583,32 43.9992143,36.935 43.9992143,43 C43.9992143,49.065 39.0652583,54 33.0003125,54 C29.5913429,54 26.5413702,52.441 24.5213882,50 L22.0624102,50 Z M37.0002768,45 L37.0002768,43 L41.9992321,43 C41.9992321,38.038 37.9622682,34 33.0003125,34 C28.0373568,34 23.9993929,38.038 23.9993929,43 L28.9993482,43 L28.9993482,45 L24.2313908,45 C25.1443826,49.002 28.7253507,52 33.0003125,52 C35.1362934,52 37.0992759,51.249 38.6442621,50 L34.0003036,50 L34.0003036,48 L40.4782457,48 C41.0812403,47.102 41.5202364,46.087 41.7682342,45 L37.0002768,45 Z M21.0024196,48 L23.2143999,48 C22.4434068,46.498 22.0004107,44.801 22.0004107,43 C22.0004107,41.959 22.1554093,40.955 22.4264069,40 L20.3634253,40 C20.1344274,40.965 19.9994286,41.966 19.9994286,43 C19.9994286,44.771 20.3584254,46.46 21.0024196,48 L21.0024196,48 Z M19.7434309,50 L17.0004554,50 L17.0004554,48 L18.8744386,48 C18.5344417,47.04 18.2894438,46.038 18.1494451,45 L15.4144695,45 L16.707458,46.293 L15.2924706,47.707 L12.2924974,44.707 C11.9025009,44.316 11.9025009,43.684 12.2924974,43.293 L15.2924706,40.293 L16.707458,41.707 L15.4144695,43 L18.0004464,43 C18.0004464,41.973 18.1044455,40.97 18.3024437,40 L17.0004554,40 L17.0004554,38 L18.8744386,38 C20.9404202,32.184 26.4833707,28 33.0003125,28 C37.427273,28 41.4002375,29.939 44.148213,33 L59.0000804,33 L59.0000804,35 L45.6661994,35 C47.1351863,37.318 47.9991786,40.058 47.9991786,43 L59.0000804,43 L59.0000804,45 L47.8501799,45 C46.8681887,52.327 40.5912447,58 33.0003125,58 C27.2563638,58 22.2624084,54.752 19.7434309,50 L19.7434309,50 Z M37.0002768,39 C37.0002768,38.448 36.5522808,38 36.0002857,38 L29.9993482,38 C29.4473442,38 28.9993482,38.448 28.9993482,39 L28.9993482,41 L31.0003304,41 L31.0003304,40 L32.0003214,40 L32.0003214,43 L31.0003304,43 L31.0003304,45 L35.0002946,45 L35.0002946,43 L34.0003036,43 L34.0003036,40 L35.0002946,40 L35.0002946,41 L37.0002768,41 L37.0002768,39 Z M49.0001696,40 L59.0000804,40 L59.0000804,38 L49.0001696,38 L49.0001696,40 Z M49.0001696,50 L59.0000804,50 L59.0000804,48 L49.0001696,48 L49.0001696,50 Z M57.0000982,27 L60.5850662,27 L57.0000982,23.414 L57.0000982,27 Z M63.7070383,27.293 C63.8940367,27.48 64.0000357,27.735 64.0000357,28 L64.0000357,63 C64.0000357,63.552 63.5520397,64 63.0000446,64 L32.0003304,64 C31.4473264,64 31.0003304,63.552 31.0003304,63 L31.0003304,59 L33.0003125,59 L33.0003125,62 L62.0000536,62 L62.0000536,29 L56.0001071,29 C55.4471121,29 55.0001161,28.552 55.0001161,28 L55.0001161,22 L33.0003125,22 L33.0003125,27 L31.0003304,27 L31.0003304,21 C31.0003304,20.448 31.4473264,20 32.0003304,20 L56.0001071,20 C56.2651048,20 56.5191025,20.105 56.7071008,20.293 L63.7070383,27.293 Z M68,24.166 L68,61 C68,61.552 67.552004,62 67.0000089,62 L65.0000268,62 L65.0000268,60 L66.0000179,60 L66.0000179,24.612 L58.6170838,18 L36.0002857,18 L36.0002857,19 L34.0003036,19 L34.0003036,17 C34.0003036,16.448 34.4472996,16 35.0003036,16 L59.0000804,16 C59.2460782,16 59.483076,16.091 59.6660744,16.255 L67.666003,23.42 C67.8780011,23.61 68,23.881 68,24.166 L68,24.166 Z'
fill='currentColor'
/>
</svg>
)
}
export function McpIcon(props: SVGProps<SVGSVGElement>) { export function McpIcon(props: SVGProps<SVGSVGElement>) {
return ( return (
<svg <svg

View File

@@ -3,10 +3,6 @@ import { mcpServers } from '@sim/db/schema'
import { createLogger } from '@sim/logger' import { createLogger } from '@sim/logger'
import { and, eq, inArray, isNull } from 'drizzle-orm' import { and, eq, inArray, isNull } from 'drizzle-orm'
import { getBaseUrl } from '@/lib/core/utils/urls' import { getBaseUrl } from '@/lib/core/utils/urls'
import {
containsUserFileWithMetadata,
hydrateUserFilesWithBase64,
} from '@/lib/uploads/utils/user-file-base64.server'
import { import {
BlockType, BlockType,
buildResumeApiUrl, buildResumeApiUrl,
@@ -139,14 +135,6 @@ export class BlockExecutor {
normalizedOutput = this.normalizeOutput(output) normalizedOutput = this.normalizeOutput(output)
} }
if (ctx.includeFileBase64 && containsUserFileWithMetadata(normalizedOutput)) {
normalizedOutput = (await hydrateUserFilesWithBase64(normalizedOutput, {
requestId: ctx.metadata.requestId,
executionId: ctx.executionId,
maxBytes: ctx.base64MaxBytes,
})) as NormalizedBlockOutput
}
const duration = Date.now() - startTime const duration = Date.now() - startTime
if (blockLog) { if (blockLog) {

View File

@@ -1,599 +0,0 @@
/**
* @vitest-environment node
*/
import { loggerMock } from '@sim/testing'
import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
vi.mock('@sim/logger', () => loggerMock)
vi.mock('@/lib/execution/cancellation', () => ({
isExecutionCancelled: vi.fn(),
isRedisCancellationEnabled: vi.fn(),
}))
import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/execution/cancellation'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import type { NodeExecutionOrchestrator } from '@/executor/orchestrators/node'
import type { ExecutionContext } from '@/executor/types'
import type { SerializedBlock } from '@/serializer/types'
import { ExecutionEngine } from './engine'
function createMockBlock(id: string): SerializedBlock {
return {
id,
metadata: { id: 'test', name: 'Test Block' },
position: { x: 0, y: 0 },
config: { tool: '', params: {} },
inputs: {},
outputs: {},
enabled: true,
}
}
function createMockNode(id: string, blockType = 'test'): DAGNode {
return {
id,
block: {
...createMockBlock(id),
metadata: { id: blockType, name: `Block ${id}` },
},
outgoingEdges: new Map(),
incomingEdges: new Set(),
metadata: {},
}
}
function createMockContext(overrides: Partial<ExecutionContext> = {}): ExecutionContext {
return {
workflowId: 'test-workflow',
workspaceId: 'test-workspace',
executionId: 'test-execution',
userId: 'test-user',
blockStates: new Map(),
executedBlocks: new Set(),
blockLogs: [],
loopExecutions: new Map(),
parallelExecutions: new Map(),
completedLoops: new Set(),
activeExecutionPath: new Set(),
metadata: {
executionId: 'test-execution',
startTime: new Date().toISOString(),
pendingBlocks: [],
},
envVars: {},
...overrides,
}
}
function createMockDAG(nodes: DAGNode[]): DAG {
const nodeMap = new Map<string, DAGNode>()
nodes.forEach((node) => nodeMap.set(node.id, node))
return {
nodes: nodeMap,
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
}
interface MockEdgeManager extends EdgeManager {
processOutgoingEdges: ReturnType<typeof vi.fn>
}
function createMockEdgeManager(
processOutgoingEdgesImpl?: (node: DAGNode) => string[]
): MockEdgeManager {
const mockFn = vi.fn().mockImplementation(processOutgoingEdgesImpl || (() => []))
return {
processOutgoingEdges: mockFn,
isNodeReady: vi.fn().mockReturnValue(true),
deactivateEdgeAndDescendants: vi.fn(),
restoreIncomingEdge: vi.fn(),
clearDeactivatedEdges: vi.fn(),
clearDeactivatedEdgesForNodes: vi.fn(),
} as unknown as MockEdgeManager
}
interface MockNodeOrchestrator extends NodeExecutionOrchestrator {
executionCount: number
}
function createMockNodeOrchestrator(executeDelay = 0): MockNodeOrchestrator {
const mock = {
executionCount: 0,
executeNode: vi.fn().mockImplementation(async () => {
mock.executionCount++
if (executeDelay > 0) {
await new Promise((resolve) => setTimeout(resolve, executeDelay))
}
return { nodeId: 'test', output: {}, isFinalOutput: false }
}),
handleNodeCompletion: vi.fn(),
}
return mock as unknown as MockNodeOrchestrator
}
describe('ExecutionEngine', () => {
beforeEach(() => {
vi.clearAllMocks()
;(isExecutionCancelled as Mock).mockResolvedValue(false)
;(isRedisCancellationEnabled as Mock).mockReturnValue(false)
})
afterEach(() => {
vi.useRealTimers()
})
describe('Normal execution', () => {
it('should execute a simple linear workflow', async () => {
const startNode = createMockNode('start', 'starter')
const endNode = createMockNode('end', 'function')
startNode.outgoingEdges.set('edge1', { target: 'end' })
endNode.incomingEdges.add('start')
const dag = createMockDAG([startNode, endNode])
const context = createMockContext()
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'start') return ['end']
return []
})
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.success).toBe(true)
expect(nodeOrchestrator.executionCount).toBe(2)
})
it('should mark execution as successful when completed without cancellation', async () => {
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext()
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.success).toBe(true)
expect(result.status).toBeUndefined()
})
it('should execute all nodes in a multi-node workflow', async () => {
const nodes = [
createMockNode('start', 'starter'),
createMockNode('middle1', 'function'),
createMockNode('middle2', 'function'),
createMockNode('end', 'function'),
]
nodes[0].outgoingEdges.set('e1', { target: 'middle1' })
nodes[1].outgoingEdges.set('e2', { target: 'middle2' })
nodes[2].outgoingEdges.set('e3', { target: 'end' })
const dag = createMockDAG(nodes)
const context = createMockContext()
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'start') return ['middle1']
if (node.id === 'middle1') return ['middle2']
if (node.id === 'middle2') return ['end']
return []
})
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.success).toBe(true)
expect(nodeOrchestrator.executionCount).toBe(4)
})
})
describe('Cancellation via AbortSignal', () => {
it('should stop execution immediately when aborted before start', async () => {
const abortController = new AbortController()
abortController.abort()
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.status).toBe('cancelled')
expect(nodeOrchestrator.executionCount).toBe(0)
})
it('should stop execution when aborted mid-workflow', async () => {
const abortController = new AbortController()
const nodes = Array.from({ length: 5 }, (_, i) => createMockNode(`node${i}`, 'function'))
for (let i = 0; i < nodes.length - 1; i++) {
nodes[i].outgoingEdges.set(`e${i}`, { target: `node${i + 1}` })
}
const dag = createMockDAG(nodes)
const context = createMockContext({ abortSignal: abortController.signal })
let callCount = 0
const edgeManager = createMockEdgeManager((node) => {
callCount++
if (callCount === 2) abortController.abort()
const idx = Number.parseInt(node.id.replace('node', ''))
if (idx < 4) return [`node${idx + 1}`]
return []
})
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('node0')
expect(result.success).toBe(false)
expect(result.status).toBe('cancelled')
expect(nodeOrchestrator.executionCount).toBeLessThan(5)
})
it('should not wait for slow executions when cancelled', async () => {
const abortController = new AbortController()
const startNode = createMockNode('start', 'starter')
const slowNode = createMockNode('slow', 'function')
startNode.outgoingEdges.set('edge1', { target: 'slow' })
const dag = createMockDAG([startNode, slowNode])
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'start') return ['slow']
return []
})
const nodeOrchestrator = createMockNodeOrchestrator(500)
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const executionPromise = engine.run('start')
setTimeout(() => abortController.abort(), 50)
const startTime = Date.now()
const result = await executionPromise
const duration = Date.now() - startTime
expect(result.status).toBe('cancelled')
expect(duration).toBeLessThan(400)
})
it('should return cancelled status even if error thrown during cancellation', async () => {
const abortController = new AbortController()
abortController.abort()
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.status).toBe('cancelled')
expect(result.success).toBe(false)
})
})
describe('Cancellation via Redis', () => {
it('should check Redis for cancellation when enabled', async () => {
;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
;(isExecutionCancelled as Mock).mockResolvedValue(false)
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext()
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
await engine.run('start')
expect(isExecutionCancelled as Mock).toHaveBeenCalled()
})
it('should stop execution when Redis reports cancellation', async () => {
;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
let checkCount = 0
;(isExecutionCancelled as Mock).mockImplementation(async () => {
checkCount++
return checkCount > 1
})
const nodes = Array.from({ length: 5 }, (_, i) => createMockNode(`node${i}`, 'function'))
for (let i = 0; i < nodes.length - 1; i++) {
nodes[i].outgoingEdges.set(`e${i}`, { target: `node${i + 1}` })
}
const dag = createMockDAG(nodes)
const context = createMockContext()
const edgeManager = createMockEdgeManager((node) => {
const idx = Number.parseInt(node.id.replace('node', ''))
if (idx < 4) return [`node${idx + 1}`]
return []
})
const nodeOrchestrator = createMockNodeOrchestrator(150)
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('node0')
expect(result.success).toBe(false)
expect(result.status).toBe('cancelled')
})
it('should respect cancellation check interval', async () => {
;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
;(isExecutionCancelled as Mock).mockResolvedValue(false)
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext()
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
await engine.run('start')
expect((isExecutionCancelled as Mock).mock.calls.length).toBeGreaterThanOrEqual(1)
})
})
describe('Loop execution with cancellation', () => {
it('should break out of loop when cancelled mid-iteration', async () => {
const abortController = new AbortController()
const loopStartNode = createMockNode('loop-start', 'loop_sentinel')
loopStartNode.metadata = { isSentinel: true, sentinelType: 'start', loopId: 'loop1' }
const loopBodyNode = createMockNode('loop-body', 'function')
loopBodyNode.metadata = { isLoopNode: true, loopId: 'loop1' }
const loopEndNode = createMockNode('loop-end', 'loop_sentinel')
loopEndNode.metadata = { isSentinel: true, sentinelType: 'end', loopId: 'loop1' }
loopStartNode.outgoingEdges.set('edge1', { target: 'loop-body' })
loopBodyNode.outgoingEdges.set('edge2', { target: 'loop-end' })
loopEndNode.outgoingEdges.set('loop_continue', {
target: 'loop-start',
sourceHandle: 'loop_continue',
})
const dag = createMockDAG([loopStartNode, loopBodyNode, loopEndNode])
const context = createMockContext({ abortSignal: abortController.signal })
let iterationCount = 0
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'loop-start') return ['loop-body']
if (node.id === 'loop-body') return ['loop-end']
if (node.id === 'loop-end') {
iterationCount++
if (iterationCount === 3) abortController.abort()
return ['loop-start']
}
return []
})
const nodeOrchestrator = createMockNodeOrchestrator(5)
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('loop-start')
expect(result.status).toBe('cancelled')
expect(iterationCount).toBeLessThan(100)
})
})
describe('Parallel execution with cancellation', () => {
it('should stop queueing parallel branches when cancelled', async () => {
const abortController = new AbortController()
const startNode = createMockNode('start', 'starter')
const parallelNodes = Array.from({ length: 10 }, (_, i) =>
createMockNode(`parallel${i}`, 'function')
)
parallelNodes.forEach((_, i) => {
startNode.outgoingEdges.set(`edge${i}`, { target: `parallel${i}` })
})
const dag = createMockDAG([startNode, ...parallelNodes])
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'start') {
return parallelNodes.map((_, i) => `parallel${i}`)
}
return []
})
const nodeOrchestrator = createMockNodeOrchestrator(50)
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const executionPromise = engine.run('start')
setTimeout(() => abortController.abort(), 30)
const result = await executionPromise
expect(result.status).toBe('cancelled')
expect(nodeOrchestrator.executionCount).toBeLessThan(11)
})
it('should not wait for all parallel branches when cancelled', async () => {
const abortController = new AbortController()
const startNode = createMockNode('start', 'starter')
const slowNodes = Array.from({ length: 5 }, (_, i) => createMockNode(`slow${i}`, 'function'))
slowNodes.forEach((_, i) => {
startNode.outgoingEdges.set(`edge${i}`, { target: `slow${i}` })
})
const dag = createMockDAG([startNode, ...slowNodes])
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'start') return slowNodes.map((_, i) => `slow${i}`)
return []
})
const nodeOrchestrator = createMockNodeOrchestrator(200)
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const executionPromise = engine.run('start')
setTimeout(() => abortController.abort(), 50)
const startTime = Date.now()
const result = await executionPromise
const duration = Date.now() - startTime
expect(result.status).toBe('cancelled')
expect(duration).toBeLessThan(500)
})
})
describe('Edge cases', () => {
it('should handle empty DAG gracefully', async () => {
const dag = createMockDAG([])
const context = createMockContext()
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run()
expect(result.success).toBe(true)
expect(nodeOrchestrator.executionCount).toBe(0)
})
it('should preserve partial output when cancelled', async () => {
const abortController = new AbortController()
const startNode = createMockNode('start', 'starter')
const endNode = createMockNode('end', 'function')
endNode.outgoingEdges = new Map()
startNode.outgoingEdges.set('edge1', { target: 'end' })
const dag = createMockDAG([startNode, endNode])
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'start') return ['end']
return []
})
const nodeOrchestrator = {
executionCount: 0,
executeNode: vi.fn().mockImplementation(async (_ctx: ExecutionContext, nodeId: string) => {
if (nodeId === 'start') {
return { nodeId: 'start', output: { startData: 'value' }, isFinalOutput: false }
}
abortController.abort()
return { nodeId: 'end', output: { endData: 'value' }, isFinalOutput: true }
}),
handleNodeCompletion: vi.fn(),
} as unknown as MockNodeOrchestrator
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.status).toBe('cancelled')
expect(result.output).toBeDefined()
})
it('should populate metadata on cancellation', async () => {
const abortController = new AbortController()
abortController.abort()
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.metadata).toBeDefined()
expect(result.metadata.endTime).toBeDefined()
expect(result.metadata.duration).toBeDefined()
})
it('should return logs even when cancelled', async () => {
const abortController = new AbortController()
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext({ abortSignal: abortController.signal })
context.blockLogs.push({
blockId: 'test',
blockName: 'Test',
blockType: 'test',
startedAt: '',
endedAt: '',
durationMs: 0,
success: true,
})
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
abortController.abort()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('start')
expect(result.logs).toBeDefined()
expect(result.logs.length).toBeGreaterThan(0)
})
})
describe('Cancellation flag behavior', () => {
it('should set cancelledFlag when abort signal fires', async () => {
const abortController = new AbortController()
const nodes = Array.from({ length: 3 }, (_, i) => createMockNode(`node${i}`, 'function'))
for (let i = 0; i < nodes.length - 1; i++) {
nodes[i].outgoingEdges.set(`e${i}`, { target: `node${i + 1}` })
}
const dag = createMockDAG(nodes)
const context = createMockContext({ abortSignal: abortController.signal })
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'node0') {
abortController.abort()
return ['node1']
}
return node.id === 'node1' ? ['node2'] : []
})
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('node0')
expect(result.status).toBe('cancelled')
})
it('should cache Redis cancellation result', async () => {
;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
;(isExecutionCancelled as Mock).mockResolvedValue(true)
const nodes = Array.from({ length: 5 }, (_, i) => createMockNode(`node${i}`, 'function'))
const dag = createMockDAG(nodes)
const context = createMockContext()
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
await engine.run('node0')
expect((isExecutionCancelled as Mock).mock.calls.length).toBeLessThanOrEqual(3)
})
})
})

View File

@@ -28,8 +28,6 @@ export class ExecutionEngine {
private lastCancellationCheck = 0 private lastCancellationCheck = 0
private readonly useRedisCancellation: boolean private readonly useRedisCancellation: boolean
private readonly CANCELLATION_CHECK_INTERVAL_MS = 500 private readonly CANCELLATION_CHECK_INTERVAL_MS = 500
private abortPromise: Promise<void> | null = null
private abortResolve: (() => void) | null = null
constructor( constructor(
private context: ExecutionContext, private context: ExecutionContext,
@@ -39,34 +37,6 @@ export class ExecutionEngine {
) { ) {
this.allowResumeTriggers = this.context.metadata.resumeFromSnapshot === true this.allowResumeTriggers = this.context.metadata.resumeFromSnapshot === true
this.useRedisCancellation = isRedisCancellationEnabled() && !!this.context.executionId this.useRedisCancellation = isRedisCancellationEnabled() && !!this.context.executionId
this.initializeAbortHandler()
}
/**
* Sets up a single abort promise that can be reused throughout execution.
* This avoids creating multiple event listeners and potential memory leaks.
*/
private initializeAbortHandler(): void {
if (!this.context.abortSignal) return
if (this.context.abortSignal.aborted) {
this.cancelledFlag = true
this.abortPromise = Promise.resolve()
return
}
this.abortPromise = new Promise<void>((resolve) => {
this.abortResolve = resolve
})
this.context.abortSignal.addEventListener(
'abort',
() => {
this.cancelledFlag = true
this.abortResolve?.()
},
{ once: true }
)
} }
private async checkCancellation(): Promise<boolean> { private async checkCancellation(): Promise<boolean> {
@@ -103,15 +73,12 @@ export class ExecutionEngine {
this.initializeQueue(triggerBlockId) this.initializeQueue(triggerBlockId)
while (this.hasWork()) { while (this.hasWork()) {
if (await this.checkCancellation()) { if ((await this.checkCancellation()) && this.executing.size === 0) {
break break
} }
await this.processQueue() await this.processQueue()
} }
await this.waitForAllExecutions()
if (!this.cancelledFlag) {
await this.waitForAllExecutions()
}
if (this.pausedBlocks.size > 0) { if (this.pausedBlocks.size > 0) {
return this.buildPausedResult(startTime) return this.buildPausedResult(startTime)
@@ -197,7 +164,11 @@ export class ExecutionEngine {
private trackExecution(promise: Promise<void>): void { private trackExecution(promise: Promise<void>): void {
this.executing.add(promise) this.executing.add(promise)
promise.catch(() => {}) // Attach error handler to prevent unhandled rejection warnings
// The actual error handling happens in waitForAllExecutions/waitForAnyExecution
promise.catch(() => {
// Error will be properly handled by Promise.all/Promise.race in wait methods
})
promise.finally(() => { promise.finally(() => {
this.executing.delete(promise) this.executing.delete(promise)
}) })
@@ -205,30 +176,12 @@ export class ExecutionEngine {
private async waitForAnyExecution(): Promise<void> { private async waitForAnyExecution(): Promise<void> {
if (this.executing.size > 0) { if (this.executing.size > 0) {
const abortPromise = this.getAbortPromise() await Promise.race(this.executing)
if (abortPromise) {
await Promise.race([...this.executing, abortPromise])
} else {
await Promise.race(this.executing)
}
} }
} }
private async waitForAllExecutions(): Promise<void> { private async waitForAllExecutions(): Promise<void> {
const abortPromise = this.getAbortPromise() await Promise.all(Array.from(this.executing))
if (abortPromise) {
await Promise.race([Promise.all(this.executing), abortPromise])
} else {
await Promise.all(this.executing)
}
}
/**
* Returns the cached abort promise. This is safe to call multiple times
* as it reuses the same promise instance created during initialization.
*/
private getAbortPromise(): Promise<void> | null {
return this.abortPromise
} }
private async withQueueLock<T>(fn: () => Promise<T> | T): Promise<T> { private async withQueueLock<T>(fn: () => Promise<T> | T): Promise<T> {
@@ -324,7 +277,7 @@ export class ExecutionEngine {
this.trackExecution(promise) this.trackExecution(promise)
} }
if (this.executing.size > 0 && !this.cancelledFlag) { if (this.executing.size > 0) {
await this.waitForAnyExecution() await this.waitForAnyExecution()
} }
} }
@@ -383,6 +336,7 @@ export class ExecutionEngine {
this.addMultipleToQueue(readyNodes) this.addMultipleToQueue(readyNodes)
// Check for dynamically added nodes (e.g., from parallel expansion)
if (this.context.pendingDynamicNodes && this.context.pendingDynamicNodes.length > 0) { if (this.context.pendingDynamicNodes && this.context.pendingDynamicNodes.length > 0) {
const dynamicNodes = this.context.pendingDynamicNodes const dynamicNodes = this.context.pendingDynamicNodes
this.context.pendingDynamicNodes = [] this.context.pendingDynamicNodes = []

View File

@@ -169,8 +169,6 @@ export class DAGExecutor {
onBlockStart: this.contextExtensions.onBlockStart, onBlockStart: this.contextExtensions.onBlockStart,
onBlockComplete: this.contextExtensions.onBlockComplete, onBlockComplete: this.contextExtensions.onBlockComplete,
abortSignal: this.contextExtensions.abortSignal, abortSignal: this.contextExtensions.abortSignal,
includeFileBase64: this.contextExtensions.includeFileBase64,
base64MaxBytes: this.contextExtensions.base64MaxBytes,
} }
if (this.contextExtensions.resumeFromSnapshot) { if (this.contextExtensions.resumeFromSnapshot) {

View File

@@ -89,8 +89,6 @@ export interface ContextExtensions {
* When aborted, the execution should stop gracefully. * When aborted, the execution should stop gracefully.
*/ */
abortSignal?: AbortSignal abortSignal?: AbortSignal
includeFileBase64?: boolean
base64MaxBytes?: number
onStream?: (streamingExecution: unknown) => Promise<void> onStream?: (streamingExecution: unknown) => Promise<void>
onBlockStart?: ( onBlockStart?: (
blockId: string, blockId: string,

View File

@@ -387,6 +387,7 @@ describe('AgentBlockHandler', () => {
code: 'return { result: "auto tool executed", input }', code: 'return { result: "auto tool executed", input }',
input: 'test input', input: 'test input',
}), }),
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
expect.any(Object) // execution context expect.any(Object) // execution context
) )
@@ -399,6 +400,7 @@ describe('AgentBlockHandler', () => {
code: 'return { result: "force tool executed", input }', code: 'return { result: "force tool executed", input }',
input: 'another test', input: 'another test',
}), }),
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
expect.any(Object) // execution context expect.any(Object) // execution context
) )
@@ -1405,7 +1407,7 @@ describe('AgentBlockHandler', () => {
}) })
it('should handle MCP tools in agent execution', async () => { it('should handle MCP tools in agent execution', async () => {
mockExecuteTool.mockImplementation((toolId, params, skipPostProcess, context) => { mockExecuteTool.mockImplementation((toolId, params, skipProxy, skipPostProcess, context) => {
if (isMcpTool(toolId)) { if (isMcpTool(toolId)) {
return Promise.resolve({ return Promise.resolve({
success: true, success: true,
@@ -1680,7 +1682,7 @@ describe('AgentBlockHandler', () => {
it('should provide workspaceId context for MCP tool execution', async () => { it('should provide workspaceId context for MCP tool execution', async () => {
let capturedContext: any let capturedContext: any
mockExecuteTool.mockImplementation((toolId, params, skipPostProcess, context) => { mockExecuteTool.mockImplementation((toolId, params, skipProxy, skipPostProcess, context) => {
capturedContext = context capturedContext = context
if (isMcpTool(toolId)) { if (isMcpTool(toolId)) {
return Promise.resolve({ return Promise.resolve({

View File

@@ -325,6 +325,7 @@ export class AgentBlockHandler implements BlockHandler {
}, },
}, },
false, false,
false,
ctx ctx
) )

View File

@@ -106,6 +106,7 @@ describe('ApiBlockHandler', () => {
body: { key: 'value' }, // Expect parsed body body: { key: 'value' }, // Expect parsed body
_context: { workflowId: 'test-workflow-id' }, _context: { workflowId: 'test-workflow-id' },
}, },
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )
@@ -157,6 +158,7 @@ describe('ApiBlockHandler', () => {
expect(mockExecuteTool).toHaveBeenCalledWith( expect(mockExecuteTool).toHaveBeenCalledWith(
'http_request', 'http_request',
expect.objectContaining({ body: expectedParsedBody }), expect.objectContaining({ body: expectedParsedBody }),
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )
@@ -173,6 +175,7 @@ describe('ApiBlockHandler', () => {
expect(mockExecuteTool).toHaveBeenCalledWith( expect(mockExecuteTool).toHaveBeenCalledWith(
'http_request', 'http_request',
expect.objectContaining({ body: 'This is plain text' }), expect.objectContaining({ body: 'This is plain text' }),
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )
@@ -189,6 +192,7 @@ describe('ApiBlockHandler', () => {
expect(mockExecuteTool).toHaveBeenCalledWith( expect(mockExecuteTool).toHaveBeenCalledWith(
'http_request', 'http_request',
expect.objectContaining({ body: undefined }), expect.objectContaining({ body: undefined }),
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )

View File

@@ -82,6 +82,7 @@ export class ApiBlockHandler implements BlockHandler {
}, },
}, },
false, false,
false,
ctx ctx
) )

View File

@@ -201,6 +201,7 @@ describe('ConditionBlockHandler', () => {
}, },
}), }),
false, false,
false,
mockContext mockContext
) )
}) })

View File

@@ -44,6 +44,7 @@ export async function evaluateConditionExpression(
}, },
}, },
false, false,
false,
ctx ctx
) )

View File

@@ -84,6 +84,7 @@ describe('FunctionBlockHandler', () => {
expect(mockExecuteTool).toHaveBeenCalledWith( expect(mockExecuteTool).toHaveBeenCalledWith(
'function_execute', 'function_execute',
expectedToolParams, expectedToolParams,
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )
@@ -116,6 +117,7 @@ describe('FunctionBlockHandler', () => {
expect(mockExecuteTool).toHaveBeenCalledWith( expect(mockExecuteTool).toHaveBeenCalledWith(
'function_execute', 'function_execute',
expectedToolParams, expectedToolParams,
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )
@@ -140,6 +142,7 @@ describe('FunctionBlockHandler', () => {
expect(mockExecuteTool).toHaveBeenCalledWith( expect(mockExecuteTool).toHaveBeenCalledWith(
'function_execute', 'function_execute',
expectedToolParams, expectedToolParams,
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )

View File

@@ -42,6 +42,7 @@ export class FunctionBlockHandler implements BlockHandler {
}, },
}, },
false, false,
false,
ctx ctx
) )

View File

@@ -95,6 +95,7 @@ describe('GenericBlockHandler', () => {
expect(mockExecuteTool).toHaveBeenCalledWith( expect(mockExecuteTool).toHaveBeenCalledWith(
'some_custom_tool', 'some_custom_tool',
expectedToolParams, expectedToolParams,
false, // skipProxy
false, // skipPostProcess false, // skipPostProcess
mockContext // execution context mockContext // execution context
) )

View File

@@ -70,6 +70,7 @@ export class GenericBlockHandler implements BlockHandler {
}, },
}, },
false, false,
false,
ctx ctx
) )

View File

@@ -633,7 +633,7 @@ export class HumanInTheLoopBlockHandler implements BlockHandler {
blockNameMapping: blockNameMappingWithPause, blockNameMapping: blockNameMappingWithPause,
} }
const result = await executeTool(toolId, toolParams, false, ctx) const result = await executeTool(toolId, toolParams, false, false, ctx)
const durationMs = Date.now() - startTime const durationMs = Date.now() - startTime
if (!result.success) { if (!result.success) {

View File

@@ -11,7 +11,6 @@ export interface UserFile {
type: string type: string
key: string key: string
context?: string context?: string
base64?: string
} }
export interface ParallelPauseScope { export interface ParallelPauseScope {
@@ -237,19 +236,6 @@ export interface ExecutionContext {
// Dynamically added nodes that need to be scheduled (e.g., from parallel expansion) // Dynamically added nodes that need to be scheduled (e.g., from parallel expansion)
pendingDynamicNodes?: string[] pendingDynamicNodes?: string[]
/**
* When true, UserFile objects in block outputs will be hydrated with base64 content
* before being stored in execution state. This ensures base64 is available for
* variable resolution in downstream blocks.
*/
includeFileBase64?: boolean
/**
* Maximum file size in bytes for base64 hydration. Files larger than this limit
* will not have their base64 content fetched.
*/
base64MaxBytes?: number
} }
export interface ExecutionResult { export interface ExecutionResult {

View File

@@ -1,4 +1,4 @@
import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import { isUserFile } from '@/lib/core/utils/display-filters'
import { import {
classifyStartBlockType, classifyStartBlockType,
getLegacyStarterMode, getLegacyStarterMode,
@@ -234,7 +234,7 @@ function getFilesFromWorkflowInput(workflowInput: unknown): UserFile[] | undefin
return undefined return undefined
} }
const files = workflowInput.files const files = workflowInput.files
if (Array.isArray(files) && files.every(isUserFileWithMetadata)) { if (Array.isArray(files) && files.every(isUserFile)) {
return files return files
} }
return undefined return undefined
@@ -377,7 +377,10 @@ function buildManualTriggerOutput(
return mergeFilesIntoOutput(output, workflowInput) return mergeFilesIntoOutput(output, workflowInput)
} }
function buildIntegrationTriggerOutput(workflowInput: unknown): NormalizedBlockOutput { function buildIntegrationTriggerOutput(
_finalInput: unknown,
workflowInput: unknown
): NormalizedBlockOutput {
return isPlainObject(workflowInput) ? (workflowInput as NormalizedBlockOutput) : {} return isPlainObject(workflowInput) ? (workflowInput as NormalizedBlockOutput) : {}
} }
@@ -427,7 +430,7 @@ export function buildStartBlockOutput(options: StartBlockOutputOptions): Normali
return buildManualTriggerOutput(finalInput, workflowInput) return buildManualTriggerOutput(finalInput, workflowInput)
case StartBlockPath.EXTERNAL_TRIGGER: case StartBlockPath.EXTERNAL_TRIGGER:
return buildIntegrationTriggerOutput(workflowInput) return buildIntegrationTriggerOutput(finalInput, workflowInput)
case StartBlockPath.LEGACY_STARTER: case StartBlockPath.LEGACY_STARTER:
return buildLegacyStarterOutput( return buildLegacyStarterOutput(

View File

@@ -1,4 +1,3 @@
import { USER_FILE_ACCESSIBLE_PROPERTIES } from '@/lib/workflows/types'
import { import {
isReference, isReference,
normalizeName, normalizeName,
@@ -21,58 +20,11 @@ function isPathInOutputSchema(
return true return true
} }
const isFileArrayType = (value: any): boolean =>
value?.type === 'file[]' || value?.type === 'files'
let current: any = outputs let current: any = outputs
for (let i = 0; i < pathParts.length; i++) { for (let i = 0; i < pathParts.length; i++) {
const part = pathParts[i] const part = pathParts[i]
const arrayMatch = part.match(/^([^[]+)\[(\d+)\]$/)
if (arrayMatch) {
const [, prop] = arrayMatch
let fieldDef: any
if (prop in current) {
fieldDef = current[prop]
} else if (current.properties && prop in current.properties) {
fieldDef = current.properties[prop]
} else if (current.type === 'array' && current.items) {
if (current.items.properties && prop in current.items.properties) {
fieldDef = current.items.properties[prop]
} else if (prop in current.items) {
fieldDef = current.items[prop]
}
}
if (!fieldDef) {
return false
}
if (isFileArrayType(fieldDef)) {
if (i + 1 < pathParts.length) {
return USER_FILE_ACCESSIBLE_PROPERTIES.includes(pathParts[i + 1] as any)
}
return true
}
if (fieldDef.type === 'array' && fieldDef.items) {
current = fieldDef.items
continue
}
current = fieldDef
continue
}
if (/^\d+$/.test(part)) { if (/^\d+$/.test(part)) {
if (isFileArrayType(current)) {
if (i + 1 < pathParts.length) {
const nextPart = pathParts[i + 1]
return USER_FILE_ACCESSIBLE_PROPERTIES.includes(nextPart as any)
}
return true
}
continue continue
} }
@@ -81,15 +33,7 @@ function isPathInOutputSchema(
} }
if (part in current) { if (part in current) {
const nextCurrent = current[part] current = current[part]
if (nextCurrent?.type === 'file[]' && i + 1 < pathParts.length) {
const nextPart = pathParts[i + 1]
if (/^\d+$/.test(nextPart) && i + 2 < pathParts.length) {
const propertyPart = pathParts[i + 2]
return USER_FILE_ACCESSIBLE_PROPERTIES.includes(propertyPart as any)
}
}
current = nextCurrent
continue continue
} }
@@ -109,10 +53,6 @@ function isPathInOutputSchema(
} }
} }
if (isFileArrayType(current) && USER_FILE_ACCESSIBLE_PROPERTIES.includes(part as any)) {
return true
}
if ('type' in current && typeof current.type === 'string') { if ('type' in current && typeof current.type === 'string') {
if (!current.properties && !current.items) { if (!current.properties && !current.items) {
return false return false

View File

@@ -897,17 +897,6 @@ export function useCollaborativeWorkflow() {
// Collect all edge IDs to remove // Collect all edge IDs to remove
const edgeIdsToRemove = updates.flatMap((u) => u.affectedEdges.map((e) => e.id)) const edgeIdsToRemove = updates.flatMap((u) => u.affectedEdges.map((e) => e.id))
if (edgeIdsToRemove.length > 0) { if (edgeIdsToRemove.length > 0) {
const edgeOperationId = crypto.randomUUID()
addToQueue({
id: edgeOperationId,
operation: {
operation: EDGES_OPERATIONS.BATCH_REMOVE_EDGES,
target: OPERATION_TARGETS.EDGES,
payload: { ids: edgeIdsToRemove },
},
workflowId: activeWorkflowId || '',
userId: session?.user?.id || 'unknown',
})
useWorkflowStore.getState().batchRemoveEdges(edgeIdsToRemove) useWorkflowStore.getState().batchRemoveEdges(edgeIdsToRemove)
} }

View File

@@ -1,9 +1,9 @@
import { loggerMock } from '@sim/testing' import { loggerMock } from '@sim/testing'
import { describe, expect, it, vi } from 'vitest' import { describe, expect, it, vi } from 'vitest'
import { import {
createPinnedUrl,
validateAirtableId, validateAirtableId,
validateAlphanumericId, validateAlphanumericId,
validateAwsRegion,
validateEnum, validateEnum,
validateExternalUrl, validateExternalUrl,
validateFileExtension, validateFileExtension,
@@ -17,7 +17,6 @@ import {
validateNumericId, validateNumericId,
validatePathSegment, validatePathSegment,
validateProxyUrl, validateProxyUrl,
validateS3BucketName,
validateUrlWithDNS, validateUrlWithDNS,
} from '@/lib/core/security/input-validation' } from '@/lib/core/security/input-validation'
import { sanitizeForLogging } from '@/lib/core/security/redaction' import { sanitizeForLogging } from '@/lib/core/security/redaction'
@@ -593,6 +592,28 @@ describe('validateUrlWithDNS', () => {
}) })
}) })
describe('createPinnedUrl', () => {
it('should replace hostname with IP', () => {
const result = createPinnedUrl('https://example.com/api/data', '93.184.216.34')
expect(result).toBe('https://93.184.216.34/api/data')
})
it('should preserve port if specified', () => {
const result = createPinnedUrl('https://example.com:8443/api', '93.184.216.34')
expect(result).toBe('https://93.184.216.34:8443/api')
})
it('should preserve query string', () => {
const result = createPinnedUrl('https://example.com/api?foo=bar&baz=qux', '93.184.216.34')
expect(result).toBe('https://93.184.216.34/api?foo=bar&baz=qux')
})
it('should preserve path', () => {
const result = createPinnedUrl('https://example.com/a/b/c/d', '93.184.216.34')
expect(result).toBe('https://93.184.216.34/a/b/c/d')
})
})
describe('validateInteger', () => { describe('validateInteger', () => {
describe('valid integers', () => { describe('valid integers', () => {
it.concurrent('should accept positive integers', () => { it.concurrent('should accept positive integers', () => {
@@ -908,13 +929,13 @@ describe('validateExternalUrl', () => {
it.concurrent('should reject 127.0.0.1', () => { it.concurrent('should reject 127.0.0.1', () => {
const result = validateExternalUrl('https://127.0.0.1/api') const result = validateExternalUrl('https://127.0.0.1/api')
expect(result.isValid).toBe(false) expect(result.isValid).toBe(false)
expect(result.error).toContain('private IP') expect(result.error).toContain('localhost')
}) })
it.concurrent('should reject 0.0.0.0', () => { it.concurrent('should reject 0.0.0.0', () => {
const result = validateExternalUrl('https://0.0.0.0/api') const result = validateExternalUrl('https://0.0.0.0/api')
expect(result.isValid).toBe(false) expect(result.isValid).toBe(false)
expect(result.error).toContain('private IP') expect(result.error).toContain('localhost')
}) })
}) })
@@ -1171,282 +1192,3 @@ describe('validateAirtableId', () => {
}) })
}) })
}) })
describe('validateAwsRegion', () => {
describe('valid standard regions', () => {
it.concurrent('should accept us-east-1', () => {
const result = validateAwsRegion('us-east-1')
expect(result.isValid).toBe(true)
expect(result.sanitized).toBe('us-east-1')
})
it.concurrent('should accept us-west-2', () => {
const result = validateAwsRegion('us-west-2')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept eu-west-1', () => {
const result = validateAwsRegion('eu-west-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept eu-central-1', () => {
const result = validateAwsRegion('eu-central-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept ap-southeast-1', () => {
const result = validateAwsRegion('ap-southeast-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept ap-northeast-1', () => {
const result = validateAwsRegion('ap-northeast-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept sa-east-1', () => {
const result = validateAwsRegion('sa-east-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept me-south-1', () => {
const result = validateAwsRegion('me-south-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept af-south-1', () => {
const result = validateAwsRegion('af-south-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept ca-central-1', () => {
const result = validateAwsRegion('ca-central-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept il-central-1', () => {
const result = validateAwsRegion('il-central-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept regions with double-digit numbers', () => {
const result = validateAwsRegion('ap-northeast-12')
expect(result.isValid).toBe(true)
})
})
describe('valid GovCloud regions', () => {
it.concurrent('should accept us-gov-west-1', () => {
const result = validateAwsRegion('us-gov-west-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept us-gov-east-1', () => {
const result = validateAwsRegion('us-gov-east-1')
expect(result.isValid).toBe(true)
})
})
describe('valid China regions', () => {
it.concurrent('should accept cn-north-1', () => {
const result = validateAwsRegion('cn-north-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept cn-northwest-1', () => {
const result = validateAwsRegion('cn-northwest-1')
expect(result.isValid).toBe(true)
})
})
describe('valid ISO regions', () => {
it.concurrent('should accept us-iso-east-1', () => {
const result = validateAwsRegion('us-iso-east-1')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept us-isob-east-1', () => {
const result = validateAwsRegion('us-isob-east-1')
expect(result.isValid).toBe(true)
})
})
describe('invalid regions', () => {
it.concurrent('should reject null', () => {
const result = validateAwsRegion(null)
expect(result.isValid).toBe(false)
expect(result.error).toContain('required')
})
it.concurrent('should reject empty string', () => {
const result = validateAwsRegion('')
expect(result.isValid).toBe(false)
expect(result.error).toContain('required')
})
it.concurrent('should reject uppercase regions', () => {
const result = validateAwsRegion('US-EAST-1')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject invalid format - missing number', () => {
const result = validateAwsRegion('us-east')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject invalid format - wrong separators', () => {
const result = validateAwsRegion('us_east_1')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject invalid format - too many parts', () => {
const result = validateAwsRegion('us-east-1-extra')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject path traversal attempts', () => {
const result = validateAwsRegion('../etc/passwd')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject arbitrary strings', () => {
const result = validateAwsRegion('not-a-region')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject invalid prefix', () => {
const result = validateAwsRegion('xx-east-1')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject invalid direction', () => {
const result = validateAwsRegion('us-middle-1')
expect(result.isValid).toBe(false)
})
it.concurrent('should use custom param name in errors', () => {
const result = validateAwsRegion('', 'awsRegion')
expect(result.error).toContain('awsRegion')
})
})
})
describe('validateS3BucketName', () => {
describe('valid bucket names', () => {
it.concurrent('should accept simple bucket name', () => {
const result = validateS3BucketName('my-bucket')
expect(result.isValid).toBe(true)
expect(result.sanitized).toBe('my-bucket')
})
it.concurrent('should accept bucket name with numbers', () => {
const result = validateS3BucketName('bucket123')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept bucket name with periods', () => {
const result = validateS3BucketName('my.bucket.name')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept 3 character bucket name', () => {
const result = validateS3BucketName('abc')
expect(result.isValid).toBe(true)
})
it.concurrent('should accept 63 character bucket name', () => {
const result = validateS3BucketName('a'.repeat(63))
expect(result.isValid).toBe(true)
})
it.concurrent('should accept minimum valid bucket name (3 chars)', () => {
const result = validateS3BucketName('a1b')
expect(result.isValid).toBe(true)
})
})
describe('invalid bucket names - null/empty', () => {
it.concurrent('should reject null', () => {
const result = validateS3BucketName(null)
expect(result.isValid).toBe(false)
expect(result.error).toContain('required')
})
it.concurrent('should reject empty string', () => {
const result = validateS3BucketName('')
expect(result.isValid).toBe(false)
expect(result.error).toContain('required')
})
})
describe('invalid bucket names - length', () => {
it.concurrent('should reject 2 character bucket name', () => {
const result = validateS3BucketName('ab')
expect(result.isValid).toBe(false)
expect(result.error).toContain('between 3 and 63')
})
it.concurrent('should reject 64 character bucket name', () => {
const result = validateS3BucketName('a'.repeat(64))
expect(result.isValid).toBe(false)
expect(result.error).toContain('between 3 and 63')
})
})
describe('invalid bucket names - format', () => {
it.concurrent('should reject uppercase letters', () => {
const result = validateS3BucketName('MyBucket')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject underscores', () => {
const result = validateS3BucketName('my_bucket')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject starting with hyphen', () => {
const result = validateS3BucketName('-mybucket')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject ending with hyphen', () => {
const result = validateS3BucketName('mybucket-')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject starting with period', () => {
const result = validateS3BucketName('.mybucket')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject ending with period', () => {
const result = validateS3BucketName('mybucket.')
expect(result.isValid).toBe(false)
})
it.concurrent('should reject consecutive periods', () => {
const result = validateS3BucketName('my..bucket')
expect(result.isValid).toBe(false)
expect(result.error).toContain('consecutive periods')
})
it.concurrent('should reject IP address format', () => {
const result = validateS3BucketName('192.168.1.1')
expect(result.isValid).toBe(false)
expect(result.error).toContain('IP address')
})
it.concurrent('should reject special characters', () => {
const result = validateS3BucketName('my@bucket')
expect(result.isValid).toBe(false)
})
})
describe('error messages', () => {
it.concurrent('should use custom param name in errors', () => {
const result = validateS3BucketName('', 's3Bucket')
expect(result.error).toContain('s3Bucket')
})
})
})

View File

@@ -1,17 +1,20 @@
import dns from 'dns/promises' import dns from 'dns/promises'
import http from 'http'
import https from 'https'
import { createLogger } from '@sim/logger' import { createLogger } from '@sim/logger'
import * as ipaddr from 'ipaddr.js'
const logger = createLogger('InputValidation') const logger = createLogger('InputValidation')
/**
* Result type for validation functions
*/
export interface ValidationResult { export interface ValidationResult {
isValid: boolean isValid: boolean
error?: string error?: string
sanitized?: string sanitized?: string
} }
/**
* Options for path segment validation
*/
export interface PathSegmentOptions { export interface PathSegmentOptions {
/** Name of the parameter for error messages */ /** Name of the parameter for error messages */
paramName?: string paramName?: string
@@ -62,6 +65,7 @@ export function validatePathSegment(
customPattern, customPattern,
} = options } = options
// Check for null/undefined
if (value === null || value === undefined || value === '') { if (value === null || value === undefined || value === '') {
return { return {
isValid: false, isValid: false,
@@ -69,6 +73,7 @@ export function validatePathSegment(
} }
} }
// Check length
if (value.length > maxLength) { if (value.length > maxLength) {
logger.warn('Path segment exceeds maximum length', { logger.warn('Path segment exceeds maximum length', {
paramName, paramName,
@@ -81,6 +86,7 @@ export function validatePathSegment(
} }
} }
// Check for null bytes (potential for bypass attacks)
if (value.includes('\0') || value.includes('%00')) { if (value.includes('\0') || value.includes('%00')) {
logger.warn('Path segment contains null bytes', { paramName }) logger.warn('Path segment contains null bytes', { paramName })
return { return {
@@ -89,6 +95,7 @@ export function validatePathSegment(
} }
} }
// Check for path traversal patterns
const pathTraversalPatterns = [ const pathTraversalPatterns = [
'..', '..',
'./', './',
@@ -117,6 +124,7 @@ export function validatePathSegment(
} }
} }
// Check for directory separators
if (value.includes('/') || value.includes('\\')) { if (value.includes('/') || value.includes('\\')) {
logger.warn('Path segment contains directory separators', { paramName }) logger.warn('Path segment contains directory separators', { paramName })
return { return {
@@ -125,6 +133,7 @@ export function validatePathSegment(
} }
} }
// Use custom pattern if provided
if (customPattern) { if (customPattern) {
if (!customPattern.test(value)) { if (!customPattern.test(value)) {
logger.warn('Path segment failed custom pattern validation', { logger.warn('Path segment failed custom pattern validation', {
@@ -139,6 +148,7 @@ export function validatePathSegment(
return { isValid: true, sanitized: value } return { isValid: true, sanitized: value }
} }
// Build allowed character pattern
let pattern = '^[a-zA-Z0-9' let pattern = '^[a-zA-Z0-9'
if (allowHyphens) pattern += '\\-' if (allowHyphens) pattern += '\\-'
if (allowUnderscores) pattern += '_' if (allowUnderscores) pattern += '_'
@@ -392,20 +402,42 @@ export function validateHostname(
} }
} }
// Import the blocked IP ranges from url-validation
const BLOCKED_IP_RANGES = [
// Private IPv4 ranges (RFC 1918)
/^10\./,
/^172\.(1[6-9]|2[0-9]|3[01])\./,
/^192\.168\./,
// Loopback addresses
/^127\./,
/^localhost$/i,
// Link-local addresses (RFC 3927)
/^169\.254\./,
// Cloud metadata endpoints
/^169\.254\.169\.254$/,
// Broadcast and other reserved ranges
/^0\./,
/^224\./,
/^240\./,
/^255\./,
// IPv6 loopback and link-local
/^::1$/,
/^fe80:/i,
/^::ffff:127\./i,
/^::ffff:10\./i,
/^::ffff:172\.(1[6-9]|2[0-9]|3[01])\./i,
/^::ffff:192\.168\./i,
]
const lowerHostname = hostname.toLowerCase() const lowerHostname = hostname.toLowerCase()
// Block localhost for (const pattern of BLOCKED_IP_RANGES) {
if (lowerHostname === 'localhost') { if (pattern.test(lowerHostname)) {
logger.warn('Hostname is localhost', { paramName })
return {
isValid: false,
error: `${paramName} cannot be a private IP address or localhost`,
}
}
// Use ipaddr.js to check if hostname is an IP and if it's private/reserved
if (ipaddr.isValid(lowerHostname)) {
if (isPrivateOrReservedIP(lowerHostname)) {
logger.warn('Hostname matches blocked IP range', { logger.warn('Hostname matches blocked IP range', {
paramName, paramName,
hostname: hostname.substring(0, 100), hostname: hostname.substring(0, 100),
@@ -678,17 +710,33 @@ export function validateExternalUrl(
// Block private IP ranges and localhost // Block private IP ranges and localhost
const hostname = parsedUrl.hostname.toLowerCase() const hostname = parsedUrl.hostname.toLowerCase()
// Block localhost // Block localhost variations
if (hostname === 'localhost') { if (
hostname === 'localhost' ||
hostname === '127.0.0.1' ||
hostname === '::1' ||
hostname.startsWith('127.') ||
hostname === '0.0.0.0'
) {
return { return {
isValid: false, isValid: false,
error: `${paramName} cannot point to localhost`, error: `${paramName} cannot point to localhost`,
} }
} }
// Use ipaddr.js to check if hostname is an IP and if it's private/reserved // Block private IP ranges
if (ipaddr.isValid(hostname)) { const privateIpPatterns = [
if (isPrivateOrReservedIP(hostname)) { /^10\./,
/^172\.(1[6-9]|2[0-9]|3[0-1])\./,
/^192\.168\./,
/^169\.254\./, // Link-local
/^fe80:/i, // IPv6 link-local
/^fc00:/i, // IPv6 unique local
/^fd00:/i, // IPv6 unique local
]
for (const pattern of privateIpPatterns) {
if (pattern.test(hostname)) {
return { return {
isValid: false, isValid: false,
error: `${paramName} cannot point to private IP addresses`, error: `${paramName} cannot point to private IP addresses`,
@@ -743,25 +791,30 @@ export function validateProxyUrl(
/** /**
* Checks if an IP address is private or reserved (not routable on the public internet) * Checks if an IP address is private or reserved (not routable on the public internet)
* Uses ipaddr.js for robust handling of all IP formats including:
* - Octal notation (0177.0.0.1)
* - Hex notation (0x7f000001)
* - IPv4-mapped IPv6 (::ffff:127.0.0.1)
* - Various edge cases that regex patterns miss
*/ */
function isPrivateOrReservedIP(ip: string): boolean { function isPrivateOrReservedIP(ip: string): boolean {
try { const patterns = [
if (!ipaddr.isValid(ip)) { /^127\./, // Loopback
return true /^10\./, // Private Class A
} /^172\.(1[6-9]|2[0-9]|3[0-1])\./, // Private Class B
/^192\.168\./, // Private Class C
const addr = ipaddr.process(ip) /^169\.254\./, // Link-local
const range = addr.range() /^0\./, // Current network
/^100\.(6[4-9]|[7-9][0-9]|1[0-1][0-9]|12[0-7])\./, // Carrier-grade NAT
return range !== 'unicast' /^192\.0\.0\./, // IETF Protocol Assignments
} catch { /^192\.0\.2\./, // TEST-NET-1
return true /^198\.51\.100\./, // TEST-NET-2
} /^203\.0\.113\./, // TEST-NET-3
/^224\./, // Multicast
/^240\./, // Reserved
/^255\./, // Broadcast
/^::1$/, // IPv6 loopback
/^fe80:/i, // IPv6 link-local
/^fc00:/i, // IPv6 unique local
/^fd00:/i, // IPv6 unique local
/^::ffff:(127\.|10\.|172\.(1[6-9]|2[0-9]|3[0-1])\.|192\.168\.|169\.254\.)/i, // IPv4-mapped IPv6
]
return patterns.some((pattern) => pattern.test(ip))
} }
/** /**
@@ -829,194 +882,18 @@ export async function validateUrlWithDNS(
} }
} }
} }
export interface SecureFetchOptions {
method?: string
headers?: Record<string, string>
body?: string
timeout?: number
maxRedirects?: number
}
export class SecureFetchHeaders {
private headers: Map<string, string>
constructor(headers: Record<string, string>) {
this.headers = new Map(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v]))
}
get(name: string): string | null {
return this.headers.get(name.toLowerCase()) ?? null
}
toRecord(): Record<string, string> {
const record: Record<string, string> = {}
for (const [key, value] of this.headers) {
record[key] = value
}
return record
}
[Symbol.iterator]() {
return this.headers.entries()
}
}
export interface SecureFetchResponse {
ok: boolean
status: number
statusText: string
headers: SecureFetchHeaders
text: () => Promise<string>
json: () => Promise<unknown>
arrayBuffer: () => Promise<ArrayBuffer>
}
const DEFAULT_MAX_REDIRECTS = 5
function isRedirectStatus(status: number): boolean {
return status >= 300 && status < 400 && status !== 304
}
function resolveRedirectUrl(baseUrl: string, location: string): string {
try {
return new URL(location, baseUrl).toString()
} catch {
throw new Error(`Invalid redirect location: ${location}`)
}
}
/** /**
* Performs a fetch with IP pinning to prevent DNS rebinding attacks. * Creates a fetch URL that uses a resolved IP address to prevent DNS rebinding
* Uses the pre-resolved IP address while preserving the original hostname for TLS SNI. *
* Follows redirects securely by validating each redirect target. * @param originalUrl - The original URL
* @param resolvedIP - The resolved IP address to use
* @returns The URL with IP substituted for hostname
*/ */
export async function secureFetchWithPinnedIP( export function createPinnedUrl(originalUrl: string, resolvedIP: string): string {
url: string, const parsed = new URL(originalUrl)
resolvedIP: string, const port = parsed.port ? `:${parsed.port}` : ''
options: SecureFetchOptions = {}, return `${parsed.protocol}//${resolvedIP}${port}${parsed.pathname}${parsed.search}`
redirectCount = 0
): Promise<SecureFetchResponse> {
const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS
return new Promise((resolve, reject) => {
const parsed = new URL(url)
const isHttps = parsed.protocol === 'https:'
const defaultPort = isHttps ? 443 : 80
const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort
const isIPv6 = resolvedIP.includes(':')
const family = isIPv6 ? 6 : 4
const agentOptions = {
lookup: (
_hostname: string,
_options: unknown,
callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void
) => {
callback(null, resolvedIP, family)
},
}
const agent = isHttps
? new https.Agent(agentOptions as https.AgentOptions)
: new http.Agent(agentOptions as http.AgentOptions)
const requestOptions: http.RequestOptions = {
hostname: parsed.hostname,
port,
path: parsed.pathname + parsed.search,
method: options.method || 'GET',
headers: options.headers || {},
agent,
timeout: options.timeout || 30000,
}
const protocol = isHttps ? https : http
const req = protocol.request(requestOptions, (res) => {
const statusCode = res.statusCode || 0
const location = res.headers.location
if (isRedirectStatus(statusCode) && location && redirectCount < maxRedirects) {
res.resume()
const redirectUrl = resolveRedirectUrl(url, location)
validateUrlWithDNS(redirectUrl, 'redirectUrl')
.then((validation) => {
if (!validation.isValid) {
reject(new Error(`Redirect blocked: ${validation.error}`))
return
}
return secureFetchWithPinnedIP(
redirectUrl,
validation.resolvedIP!,
options,
redirectCount + 1
)
})
.then((response) => {
if (response) resolve(response)
})
.catch(reject)
return
}
if (isRedirectStatus(statusCode) && location && redirectCount >= maxRedirects) {
res.resume()
reject(new Error(`Too many redirects (max: ${maxRedirects})`))
return
}
const chunks: Buffer[] = []
res.on('data', (chunk: Buffer) => chunks.push(chunk))
res.on('error', (error) => {
reject(error)
})
res.on('end', () => {
const bodyBuffer = Buffer.concat(chunks)
const body = bodyBuffer.toString('utf-8')
const headersRecord: Record<string, string> = {}
for (const [key, value] of Object.entries(res.headers)) {
if (typeof value === 'string') {
headersRecord[key.toLowerCase()] = value
} else if (Array.isArray(value)) {
headersRecord[key.toLowerCase()] = value.join(', ')
}
}
resolve({
ok: statusCode >= 200 && statusCode < 300,
status: statusCode,
statusText: res.statusMessage || '',
headers: new SecureFetchHeaders(headersRecord),
text: async () => body,
json: async () => JSON.parse(body),
arrayBuffer: async () =>
bodyBuffer.buffer.slice(
bodyBuffer.byteOffset,
bodyBuffer.byteOffset + bodyBuffer.byteLength
),
})
})
})
req.on('error', (error) => {
reject(error)
})
req.on('timeout', () => {
req.destroy()
reject(new Error('Request timeout'))
})
if (options.body) {
req.write(options.body)
}
req.end()
})
} }
/** /**
@@ -1070,138 +947,6 @@ export function validateAirtableId(
return { isValid: true, sanitized: value } return { isValid: true, sanitized: value }
} }
/**
* Validates an AWS region identifier
*
* Supported region formats:
* - Standard: us-east-1, eu-west-2, ap-southeast-1, sa-east-1, af-south-1
* - GovCloud: us-gov-east-1, us-gov-west-1
* - China: cn-north-1, cn-northwest-1
* - Israel: il-central-1
* - ISO partitions: us-iso-east-1, us-isob-east-1
*
* @param value - The AWS region to validate
* @param paramName - Name of the parameter for error messages
* @returns ValidationResult
*
* @example
* ```typescript
* const result = validateAwsRegion(region, 'region')
* if (!result.isValid) {
* return NextResponse.json({ error: result.error }, { status: 400 })
* }
* ```
*/
export function validateAwsRegion(
value: string | null | undefined,
paramName = 'region'
): ValidationResult {
if (value === null || value === undefined || value === '') {
return {
isValid: false,
error: `${paramName} is required`,
}
}
// AWS region patterns:
// - Standard: af|ap|ca|eu|me|sa|us|il followed by direction and number
// - GovCloud: us-gov-east-1, us-gov-west-1
// - China: cn-north-1, cn-northwest-1
// - ISO: us-iso-east-1, us-iso-west-1, us-isob-east-1
const awsRegionPattern =
/^(af|ap|ca|cn|eu|il|me|sa|us|us-gov|us-iso|us-isob)-(central|north|northeast|northwest|south|southeast|southwest|east|west)-\d{1,2}$/
if (!awsRegionPattern.test(value)) {
logger.warn('Invalid AWS region format', {
paramName,
value: value.substring(0, 50),
})
return {
isValid: false,
error: `${paramName} must be a valid AWS region (e.g., us-east-1, eu-west-2, us-gov-west-1)`,
}
}
return { isValid: true, sanitized: value }
}
/**
* Validates an S3 bucket name according to AWS naming rules
*
* S3 bucket names must:
* - Be 3-63 characters long
* - Start and end with a letter or number
* - Contain only lowercase letters, numbers, and hyphens
* - Not contain consecutive periods
* - Not be formatted as an IP address
*
* @param value - The S3 bucket name to validate
* @param paramName - Name of the parameter for error messages
* @returns ValidationResult
*
* @example
* ```typescript
* const result = validateS3BucketName(bucket, 'bucket')
* if (!result.isValid) {
* return NextResponse.json({ error: result.error }, { status: 400 })
* }
* ```
*/
export function validateS3BucketName(
value: string | null | undefined,
paramName = 'bucket'
): ValidationResult {
if (value === null || value === undefined || value === '') {
return {
isValid: false,
error: `${paramName} is required`,
}
}
if (value.length < 3 || value.length > 63) {
logger.warn('S3 bucket name length invalid', {
paramName,
length: value.length,
})
return {
isValid: false,
error: `${paramName} must be between 3 and 63 characters`,
}
}
const bucketNamePattern = /^[a-z0-9][a-z0-9.-]*[a-z0-9]$|^[a-z0-9]$/
if (!bucketNamePattern.test(value)) {
logger.warn('Invalid S3 bucket name format', {
paramName,
value: value.substring(0, 63),
})
return {
isValid: false,
error: `${paramName} must start and end with a letter or number, and contain only lowercase letters, numbers, hyphens, and periods`,
}
}
if (value.includes('..')) {
logger.warn('S3 bucket name contains consecutive periods', { paramName })
return {
isValid: false,
error: `${paramName} cannot contain consecutive periods`,
}
}
const ipPattern = /^(\d{1,3}\.){3}\d{1,3}$/
if (ipPattern.test(value)) {
logger.warn('S3 bucket name formatted as IP address', { paramName })
return {
isValid: false,
error: `${paramName} cannot be formatted as an IP address`,
}
}
return { isValid: true, sanitized: value }
}
/** /**
* Validates a Google Calendar ID * Validates a Google Calendar ID
* *

View File

@@ -1,13 +1,11 @@
import { describe, expect, it } from 'vitest' import { describe, expect, it } from 'vitest'
import { import {
isLargeDataKey,
isSensitiveKey, isSensitiveKey,
REDACTED_MARKER, REDACTED_MARKER,
redactApiKeys, redactApiKeys,
redactSensitiveValues, redactSensitiveValues,
sanitizeEventData, sanitizeEventData,
sanitizeForLogging, sanitizeForLogging,
TRUNCATED_MARKER,
} from './redaction' } from './redaction'
/** /**
@@ -20,24 +18,6 @@ describe('REDACTED_MARKER', () => {
}) })
}) })
describe('TRUNCATED_MARKER', () => {
it.concurrent('should be the standard marker', () => {
expect(TRUNCATED_MARKER).toBe('[TRUNCATED]')
})
})
describe('isLargeDataKey', () => {
it.concurrent('should identify base64 as large data key', () => {
expect(isLargeDataKey('base64')).toBe(true)
})
it.concurrent('should not identify other keys as large data', () => {
expect(isLargeDataKey('content')).toBe(false)
expect(isLargeDataKey('data')).toBe(false)
expect(isLargeDataKey('base')).toBe(false)
})
})
describe('isSensitiveKey', () => { describe('isSensitiveKey', () => {
describe('exact matches', () => { describe('exact matches', () => {
it.concurrent('should match apiKey variations', () => { it.concurrent('should match apiKey variations', () => {
@@ -254,80 +234,6 @@ describe('redactApiKeys', () => {
expect(result.config.database.password).toBe('[REDACTED]') expect(result.config.database.password).toBe('[REDACTED]')
expect(result.config.database.host).toBe('localhost') expect(result.config.database.host).toBe('localhost')
}) })
it.concurrent('should truncate base64 fields', () => {
const obj = {
id: 'file-123',
name: 'document.pdf',
base64: 'VGhpcyBpcyBhIHZlcnkgbG9uZyBiYXNlNjQgc3RyaW5n...',
size: 12345,
}
const result = redactApiKeys(obj)
expect(result.id).toBe('file-123')
expect(result.name).toBe('document.pdf')
expect(result.base64).toBe('[TRUNCATED]')
expect(result.size).toBe(12345)
})
it.concurrent('should truncate base64 in nested UserFile objects', () => {
const obj = {
files: [
{
id: 'file-1',
name: 'doc1.pdf',
url: 'http://example.com/file1',
size: 1000,
base64: 'base64content1...',
},
{
id: 'file-2',
name: 'doc2.pdf',
url: 'http://example.com/file2',
size: 2000,
base64: 'base64content2...',
},
],
}
const result = redactApiKeys(obj)
expect(result.files[0].id).toBe('file-1')
expect(result.files[0].base64).toBe('[TRUNCATED]')
expect(result.files[1].base64).toBe('[TRUNCATED]')
})
it.concurrent('should filter UserFile objects to only expose allowed fields', () => {
const obj = {
processedFiles: [
{
id: 'file-123',
name: 'document.pdf',
url: 'http://localhost/api/files/serve/...',
size: 12345,
type: 'application/pdf',
key: 'execution/workspace/workflow/file.pdf',
context: 'execution',
base64: 'VGhpcyBpcyBhIGJhc2U2NCBzdHJpbmc=',
},
],
}
const result = redactApiKeys(obj)
// Exposed fields should be present
expect(result.processedFiles[0].id).toBe('file-123')
expect(result.processedFiles[0].name).toBe('document.pdf')
expect(result.processedFiles[0].url).toBe('http://localhost/api/files/serve/...')
expect(result.processedFiles[0].size).toBe(12345)
expect(result.processedFiles[0].type).toBe('application/pdf')
expect(result.processedFiles[0].base64).toBe('[TRUNCATED]')
// Internal fields should be filtered out
expect(result.processedFiles[0]).not.toHaveProperty('key')
expect(result.processedFiles[0]).not.toHaveProperty('context')
})
}) })
describe('primitive handling', () => { describe('primitive handling', () => {

View File

@@ -2,16 +2,10 @@
* Centralized redaction utilities for sensitive data * Centralized redaction utilities for sensitive data
*/ */
import { filterUserFileForDisplay, isUserFile } from '@/lib/core/utils/user-file'
export const REDACTED_MARKER = '[REDACTED]' export const REDACTED_MARKER = '[REDACTED]'
export const TRUNCATED_MARKER = '[TRUNCATED]'
const BYPASS_REDACTION_KEYS = new Set(['nextPageToken']) const BYPASS_REDACTION_KEYS = new Set(['nextPageToken'])
/** Keys that contain large binary/encoded data that should be truncated in logs */
const LARGE_DATA_KEYS = new Set(['base64'])
const SENSITIVE_KEY_PATTERNS: RegExp[] = [ const SENSITIVE_KEY_PATTERNS: RegExp[] = [
/^api[_-]?key$/i, /^api[_-]?key$/i,
/^access[_-]?token$/i, /^access[_-]?token$/i,
@@ -94,10 +88,6 @@ export function redactSensitiveValues(value: string): string {
return result return result
} }
export function isLargeDataKey(key: string): boolean {
return LARGE_DATA_KEYS.has(key)
}
export function redactApiKeys(obj: any): any { export function redactApiKeys(obj: any): any {
if (obj === null || obj === undefined) { if (obj === null || obj === undefined) {
return obj return obj
@@ -111,26 +101,11 @@ export function redactApiKeys(obj: any): any {
return obj.map((item) => redactApiKeys(item)) return obj.map((item) => redactApiKeys(item))
} }
if (isUserFile(obj)) {
const filtered = filterUserFileForDisplay(obj)
const result: Record<string, any> = {}
for (const [key, value] of Object.entries(filtered)) {
if (isLargeDataKey(key) && typeof value === 'string') {
result[key] = TRUNCATED_MARKER
} else {
result[key] = value
}
}
return result
}
const result: Record<string, any> = {} const result: Record<string, any> = {}
for (const [key, value] of Object.entries(obj)) { for (const [key, value] of Object.entries(obj)) {
if (isSensitiveKey(key)) { if (isSensitiveKey(key)) {
result[key] = REDACTED_MARKER result[key] = REDACTED_MARKER
} else if (isLargeDataKey(key) && typeof value === 'string') {
result[key] = TRUNCATED_MARKER
} else if (typeof value === 'object' && value !== null) { } else if (typeof value === 'object' && value !== null) {
result[key] = redactApiKeys(value) result[key] = redactApiKeys(value)
} else { } else {

View File

@@ -1,5 +1,3 @@
import { filterUserFileForDisplay, isUserFile } from '@/lib/core/utils/user-file'
const MAX_STRING_LENGTH = 15000 const MAX_STRING_LENGTH = 15000
const MAX_DEPTH = 50 const MAX_DEPTH = 50
@@ -10,9 +8,32 @@ function truncateString(value: string, maxLength = MAX_STRING_LENGTH): string {
return `${value.substring(0, maxLength)}... [truncated ${value.length - maxLength} chars]` return `${value.substring(0, maxLength)}... [truncated ${value.length - maxLength} chars]`
} }
export function isUserFile(candidate: unknown): candidate is {
id: string
name: string
url: string
key: string
size: number
type: string
context?: string
} {
if (!candidate || typeof candidate !== 'object') {
return false
}
const value = candidate as Record<string, unknown>
return (
typeof value.id === 'string' &&
typeof value.key === 'string' &&
typeof value.url === 'string' &&
typeof value.name === 'string'
)
}
function filterUserFile(data: any): any { function filterUserFile(data: any): any {
if (isUserFile(data)) { if (isUserFile(data)) {
return filterUserFileForDisplay(data) const { id, name, url, size, type } = data
return { id, name, url, size, type }
} }
return data return data
} }

View File

@@ -1,57 +0,0 @@
import type { UserFile } from '@/executor/types'
export type UserFileLike = Pick<UserFile, 'id' | 'name' | 'url' | 'key'> &
Partial<Pick<UserFile, 'size' | 'type' | 'context' | 'base64'>>
/**
* Fields exposed for UserFile objects in UI (tag dropdown) and logs.
* Internal fields like 'key' and 'context' are not exposed.
*/
export const USER_FILE_DISPLAY_FIELDS = ['id', 'name', 'url', 'size', 'type', 'base64'] as const
export type UserFileDisplayField = (typeof USER_FILE_DISPLAY_FIELDS)[number]
/**
* Checks if a value matches the minimal UserFile shape.
*/
export function isUserFile(value: unknown): value is UserFileLike {
if (!value || typeof value !== 'object') {
return false
}
const candidate = value as Record<string, unknown>
return (
typeof candidate.id === 'string' &&
typeof candidate.key === 'string' &&
typeof candidate.url === 'string' &&
typeof candidate.name === 'string'
)
}
/**
* Checks if a value matches the full UserFile metadata shape.
*/
export function isUserFileWithMetadata(value: unknown): value is UserFile {
if (!isUserFile(value)) {
return false
}
const candidate = value as Record<string, unknown>
return typeof candidate.size === 'number' && typeof candidate.type === 'string'
}
/**
* Filters a UserFile object to only include display fields.
* Used for both UI display and log sanitization.
*/
export function filterUserFileForDisplay(data: Record<string, unknown>): Record<string, unknown> {
const filtered: Record<string, unknown> = {}
for (const field of USER_FILE_DISPLAY_FIELDS) {
if (field in data) {
filtered[field] = data[field]
}
}
return filtered
}

View File

@@ -1,5 +1,5 @@
import { createLogger } from '@sim/logger' import { createLogger } from '@sim/logger'
import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import { isUserFile } from '@/lib/core/utils/display-filters'
import type { ExecutionContext } from '@/lib/uploads/contexts/execution/utils' import type { ExecutionContext } from '@/lib/uploads/contexts/execution/utils'
import { generateExecutionFileKey, generateFileId } from '@/lib/uploads/contexts/execution/utils' import { generateExecutionFileKey, generateFileId } from '@/lib/uploads/contexts/execution/utils'
import type { UserFile } from '@/executor/types' import type { UserFile } from '@/executor/types'
@@ -169,7 +169,7 @@ export async function uploadFileFromRawData(
context: ExecutionContext, context: ExecutionContext,
userId?: string userId?: string
): Promise<UserFile> { ): Promise<UserFile> {
if (isUserFileWithMetadata(rawData)) { if (isUserFile(rawData)) {
return rawData return rawData
} }

View File

@@ -455,27 +455,3 @@ export async function generatePresignedDownloadUrl(
export function hasCloudStorage(): boolean { export function hasCloudStorage(): boolean {
return USE_BLOB_STORAGE || USE_S3_STORAGE return USE_BLOB_STORAGE || USE_S3_STORAGE
} }
/**
* Get S3 bucket and key information for a storage key
* Useful for services that need direct S3 access (e.g., AWS Textract async)
*/
export function getS3InfoForKey(
key: string,
context: StorageContext
): { bucket: string; key: string } {
if (!USE_S3_STORAGE) {
throw new Error('S3 storage is not configured. Cannot retrieve S3 info for key.')
}
const config = getStorageConfig(context)
if (!config.bucket) {
throw new Error(`S3 bucket not configured for context: ${context}`)
}
return {
bucket: config.bucket,
key,
}
}

View File

@@ -1,7 +1,6 @@
'use server' 'use server'
import type { Logger } from '@sim/logger' import type { Logger } from '@sim/logger'
import { secureFetchWithPinnedIP, validateUrlWithDNS } from '@/lib/core/security/input-validation'
import type { StorageContext } from '@/lib/uploads' import type { StorageContext } from '@/lib/uploads'
import { isExecutionFile } from '@/lib/uploads/contexts/execution/utils' import { isExecutionFile } from '@/lib/uploads/contexts/execution/utils'
import { inferContextFromKey } from '@/lib/uploads/utils/file-utils' import { inferContextFromKey } from '@/lib/uploads/utils/file-utils'
@@ -10,32 +9,38 @@ import type { UserFile } from '@/executor/types'
/** /**
* Download a file from a URL (internal or external) * Download a file from a URL (internal or external)
* For internal URLs, uses direct storage access (server-side only) * For internal URLs, uses direct storage access (server-side only)
* For external URLs, validates DNS/SSRF and uses secure fetch with IP pinning * For external URLs, uses HTTP fetch
*/ */
export async function downloadFileFromUrl(fileUrl: string, timeoutMs = 180000): Promise<Buffer> { export async function downloadFileFromUrl(fileUrl: string, timeoutMs = 180000): Promise<Buffer> {
const { isInternalFileUrl } = await import('./file-utils') const { isInternalFileUrl } = await import('./file-utils')
const { parseInternalFileUrl } = await import('./file-utils') const { parseInternalFileUrl } = await import('./file-utils')
const controller = new AbortController()
const timeoutId = setTimeout(() => controller.abort(), timeoutMs)
if (isInternalFileUrl(fileUrl)) { try {
const { key, context } = parseInternalFileUrl(fileUrl) if (isInternalFileUrl(fileUrl)) {
const { downloadFile } = await import('@/lib/uploads/core/storage-service') const { key, context } = parseInternalFileUrl(fileUrl)
return downloadFile({ key, context }) const { downloadFile } = await import('@/lib/uploads/core/storage-service')
const buffer = await downloadFile({ key, context })
clearTimeout(timeoutId)
return buffer
}
const response = await fetch(fileUrl, { signal: controller.signal })
clearTimeout(timeoutId)
if (!response.ok) {
throw new Error(`Failed to download file: ${response.statusText}`)
}
return Buffer.from(await response.arrayBuffer())
} catch (error) {
clearTimeout(timeoutId)
if (error instanceof Error && error.name === 'AbortError') {
throw new Error('File download timed out')
}
throw error
} }
const urlValidation = await validateUrlWithDNS(fileUrl, 'fileUrl')
if (!urlValidation.isValid) {
throw new Error(`Invalid file URL: ${urlValidation.error}`)
}
const response = await secureFetchWithPinnedIP(fileUrl, urlValidation.resolvedIP!, {
timeout: timeoutMs,
})
if (!response.ok) {
throw new Error(`Failed to download file: ${response.statusText}`)
}
return Buffer.from(await response.arrayBuffer())
} }
/** /**

View File

@@ -1,319 +0,0 @@
import type { Logger } from '@sim/logger'
import { createLogger } from '@sim/logger'
import { getRedisClient } from '@/lib/core/config/redis'
import { isUserFileWithMetadata } from '@/lib/core/utils/user-file'
import { bufferToBase64 } from '@/lib/uploads/utils/file-utils'
import { downloadFileFromStorage, downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
import type { UserFile } from '@/executor/types'
const DEFAULT_MAX_BASE64_BYTES = 10 * 1024 * 1024
const DEFAULT_TIMEOUT_MS = 180000
const DEFAULT_CACHE_TTL_SECONDS = 300
const REDIS_KEY_PREFIX = 'user-file:base64:'
interface Base64Cache {
get(file: UserFile): Promise<string | null>
set(file: UserFile, value: string, ttlSeconds: number): Promise<void>
}
interface HydrationState {
seen: WeakSet<object>
cache: Base64Cache
cacheTtlSeconds: number
}
export interface Base64HydrationOptions {
requestId?: string
executionId?: string
logger?: Logger
maxBytes?: number
allowUnknownSize?: boolean
timeoutMs?: number
cacheTtlSeconds?: number
}
class InMemoryBase64Cache implements Base64Cache {
private entries = new Map<string, { value: string; expiresAt: number }>()
async get(file: UserFile): Promise<string | null> {
const key = getFileCacheKey(file)
const entry = this.entries.get(key)
if (!entry) {
return null
}
if (entry.expiresAt <= Date.now()) {
this.entries.delete(key)
return null
}
return entry.value
}
async set(file: UserFile, value: string, ttlSeconds: number): Promise<void> {
const key = getFileCacheKey(file)
const expiresAt = Date.now() + ttlSeconds * 1000
this.entries.set(key, { value, expiresAt })
}
}
function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Base64Cache {
const redis = getRedisClient()
const { executionId } = options
if (!redis) {
logger.warn(
`[${options.requestId}] Redis unavailable for base64 cache, using in-memory fallback`
)
return new InMemoryBase64Cache()
}
return {
async get(file: UserFile) {
try {
const key = getFullCacheKey(executionId, file)
return await redis.get(key)
} catch (error) {
logger.warn(`[${options.requestId}] Redis get failed, skipping cache`, error)
return null
}
},
async set(file: UserFile, value: string, ttlSeconds: number) {
try {
const key = getFullCacheKey(executionId, file)
await redis.set(key, value, 'EX', ttlSeconds)
} catch (error) {
logger.warn(`[${options.requestId}] Redis set failed, skipping cache`, error)
}
},
}
}
function createHydrationState(options: Base64HydrationOptions, logger: Logger): HydrationState {
return {
seen: new WeakSet<object>(),
cache: createBase64Cache(options, logger),
cacheTtlSeconds: options.cacheTtlSeconds ?? DEFAULT_CACHE_TTL_SECONDS,
}
}
function getHydrationLogger(options: Base64HydrationOptions): Logger {
return options.logger ?? createLogger('UserFileBase64')
}
function getFileCacheKey(file: UserFile): string {
if (file.key) {
return `key:${file.key}`
}
if (file.url) {
return `url:${file.url}`
}
return `id:${file.id}`
}
function getFullCacheKey(executionId: string | undefined, file: UserFile): string {
const fileKey = getFileCacheKey(file)
if (executionId) {
return `${REDIS_KEY_PREFIX}exec:${executionId}:${fileKey}`
}
return `${REDIS_KEY_PREFIX}${fileKey}`
}
async function resolveBase64(
file: UserFile,
options: Base64HydrationOptions,
logger: Logger
): Promise<string | null> {
if (file.base64) {
return file.base64
}
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES
const allowUnknownSize = options.allowUnknownSize ?? false
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS
const hasStableStorageKey = Boolean(file.key)
if (Number.isFinite(file.size) && file.size > maxBytes) {
logger.warn(
`[${options.requestId}] Skipping base64 for ${file.name} (size ${file.size} exceeds ${maxBytes})`
)
return null
}
if (
(!Number.isFinite(file.size) || file.size <= 0) &&
!allowUnknownSize &&
!hasStableStorageKey
) {
logger.warn(`[${options.requestId}] Skipping base64 for ${file.name} (unknown file size)`)
return null
}
let buffer: Buffer | null = null
const requestId = options.requestId ?? 'unknown'
if (file.key) {
try {
buffer = await downloadFileFromStorage(file, requestId, logger)
} catch (error) {
logger.warn(
`[${requestId}] Failed to download ${file.name} from storage, trying URL fallback`,
error
)
}
}
if (!buffer && file.url) {
try {
buffer = await downloadFileFromUrl(file.url, timeoutMs)
} catch (error) {
logger.warn(`[${requestId}] Failed to download ${file.name} from URL`, error)
}
}
if (!buffer) {
return null
}
if (buffer.length > maxBytes) {
logger.warn(
`[${options.requestId}] Skipping base64 for ${file.name} (downloaded ${buffer.length} exceeds ${maxBytes})`
)
return null
}
return bufferToBase64(buffer)
}
async function hydrateUserFile(
file: UserFile,
options: Base64HydrationOptions,
state: HydrationState,
logger: Logger
): Promise<UserFile> {
const cached = await state.cache.get(file)
if (cached) {
return { ...file, base64: cached }
}
const base64 = await resolveBase64(file, options, logger)
if (!base64) {
return file
}
await state.cache.set(file, base64, state.cacheTtlSeconds)
return { ...file, base64 }
}
async function hydrateValue(
value: unknown,
options: Base64HydrationOptions,
state: HydrationState,
logger: Logger
): Promise<unknown> {
if (!value || typeof value !== 'object') {
return value
}
if (isUserFileWithMetadata(value)) {
return hydrateUserFile(value, options, state, logger)
}
if (state.seen.has(value)) {
return value
}
state.seen.add(value)
if (Array.isArray(value)) {
const hydratedItems = await Promise.all(
value.map((item) => hydrateValue(item, options, state, logger))
)
return hydratedItems
}
const entries = await Promise.all(
Object.entries(value).map(async ([key, entryValue]) => {
const hydratedEntry = await hydrateValue(entryValue, options, state, logger)
return [key, hydratedEntry] as const
})
)
return Object.fromEntries(entries)
}
/**
* Hydrates UserFile objects within a value to include base64 content.
* Returns the original structure with UserFile.base64 set where available.
*/
export async function hydrateUserFilesWithBase64(
value: unknown,
options: Base64HydrationOptions
): Promise<unknown> {
const logger = getHydrationLogger(options)
const state = createHydrationState(options, logger)
return hydrateValue(value, options, state, logger)
}
function isPlainObject(value: unknown): value is Record<string, unknown> {
if (!value || typeof value !== 'object') {
return false
}
const proto = Object.getPrototypeOf(value)
return proto === Object.prototype || proto === null
}
/**
* Checks if a value contains any UserFile objects with metadata.
*/
export function containsUserFileWithMetadata(value: unknown): boolean {
if (!value || typeof value !== 'object') {
return false
}
if (isUserFileWithMetadata(value)) {
return true
}
if (Array.isArray(value)) {
return value.some((item) => containsUserFileWithMetadata(item))
}
if (!isPlainObject(value)) {
return false
}
return Object.values(value).some((entry) => containsUserFileWithMetadata(entry))
}
/**
* Cleans up base64 cache entries for a specific execution.
* Should be called at the end of workflow execution.
*/
export async function cleanupExecutionBase64Cache(executionId: string): Promise<void> {
const redis = getRedisClient()
if (!redis) {
return
}
const pattern = `${REDIS_KEY_PREFIX}exec:${executionId}:*`
const logger = createLogger('UserFileBase64')
try {
let cursor = '0'
let deletedCount = 0
do {
const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100)
cursor = nextCursor
if (keys.length > 0) {
await redis.del(...keys)
deletedCount += keys.length
}
} while (cursor !== '0')
if (deletedCount > 0) {
logger.info(`Cleaned up ${deletedCount} base64 cache entries for execution ${executionId}`)
}
} catch (error) {
logger.warn(`Failed to cleanup base64 cache for execution ${executionId}`, error)
}
}

View File

@@ -54,17 +54,6 @@ export interface SimplifiedImapEmail {
} }
export interface ImapWebhookPayload { export interface ImapWebhookPayload {
messageId: string
subject: string
from: string
to: string
cc: string
date: string | null
bodyText: string
bodyHtml: string
mailbox: string
hasAttachments: boolean
attachments: ImapAttachment[]
email: SimplifiedImapEmail email: SimplifiedImapEmail
timestamp: string timestamp: string
} }
@@ -624,17 +613,6 @@ async function processEmails(
} }
const payload: ImapWebhookPayload = { const payload: ImapWebhookPayload = {
messageId: simplifiedEmail.messageId,
subject: simplifiedEmail.subject,
from: simplifiedEmail.from,
to: simplifiedEmail.to,
cc: simplifiedEmail.cc,
date: simplifiedEmail.date,
bodyText: simplifiedEmail.bodyText,
bodyHtml: simplifiedEmail.bodyHtml,
mailbox: simplifiedEmail.mailbox,
hasAttachments: simplifiedEmail.hasAttachments,
attachments: simplifiedEmail.attachments,
email: simplifiedEmail, email: simplifiedEmail,
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
} }

View File

@@ -5,7 +5,7 @@ import { and, eq, isNull, or, sql } from 'drizzle-orm'
import { nanoid } from 'nanoid' import { nanoid } from 'nanoid'
import Parser from 'rss-parser' import Parser from 'rss-parser'
import { pollingIdempotency } from '@/lib/core/idempotency/service' import { pollingIdempotency } from '@/lib/core/idempotency/service'
import { secureFetchWithPinnedIP, validateUrlWithDNS } from '@/lib/core/security/input-validation' import { createPinnedUrl, validateUrlWithDNS } from '@/lib/core/security/input-validation'
import { getBaseUrl } from '@/lib/core/utils/urls' import { getBaseUrl } from '@/lib/core/utils/urls'
import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants' import { MAX_CONSECUTIVE_FAILURES } from '@/triggers/constants'
@@ -48,9 +48,6 @@ interface RssFeed {
} }
export interface RssWebhookPayload { export interface RssWebhookPayload {
title?: string
link?: string
pubDate?: string
item: RssItem item: RssItem
feed: { feed: {
title?: string title?: string
@@ -268,12 +265,15 @@ async function fetchNewRssItems(
throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`) throw new Error(`Invalid RSS feed URL: ${urlValidation.error}`)
} }
const response = await secureFetchWithPinnedIP(config.feedUrl, urlValidation.resolvedIP!, { const pinnedUrl = createPinnedUrl(config.feedUrl, urlValidation.resolvedIP!)
const response = await fetch(pinnedUrl, {
headers: { headers: {
Host: urlValidation.originalHostname!,
'User-Agent': 'Sim/1.0 RSS Poller', 'User-Agent': 'Sim/1.0 RSS Poller',
Accept: 'application/rss+xml, application/xml, text/xml, */*', Accept: 'application/rss+xml, application/xml, text/xml, */*',
}, },
timeout: 30000, signal: AbortSignal.timeout(30000),
}) })
if (!response.ok) { if (!response.ok) {
@@ -349,9 +349,6 @@ async function processRssItems(
`${webhookData.id}:${itemGuid}`, `${webhookData.id}:${itemGuid}`,
async () => { async () => {
const payload: RssWebhookPayload = { const payload: RssWebhookPayload = {
title: item.title,
link: item.link,
pubDate: item.pubDate,
item: { item: {
title: item.title, title: item.title,
link: item.link, link: item.link,

View File

@@ -3,11 +3,7 @@ import { account, webhook } from '@sim/db/schema'
import { createLogger } from '@sim/logger' import { createLogger } from '@sim/logger'
import { and, eq, isNull, or } from 'drizzle-orm' import { and, eq, isNull, or } from 'drizzle-orm'
import { type NextRequest, NextResponse } from 'next/server' import { type NextRequest, NextResponse } from 'next/server'
import { import { createPinnedUrl, validateUrlWithDNS } from '@/lib/core/security/input-validation'
type SecureFetchResponse,
secureFetchWithPinnedIP,
validateUrlWithDNS,
} from '@/lib/core/security/input-validation'
import type { DbOrTx } from '@/lib/db/types' import type { DbOrTx } from '@/lib/db/types'
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
@@ -102,7 +98,7 @@ async function fetchWithDNSPinning(
url: string, url: string,
accessToken: string, accessToken: string,
requestId: string requestId: string
): Promise<SecureFetchResponse | null> { ): Promise<Response | null> {
try { try {
const urlValidation = await validateUrlWithDNS(url, 'contentUrl') const urlValidation = await validateUrlWithDNS(url, 'contentUrl')
if (!urlValidation.isValid) { if (!urlValidation.isValid) {
@@ -112,14 +108,19 @@ async function fetchWithDNSPinning(
return null return null
} }
const headers: Record<string, string> = {} const pinnedUrl = createPinnedUrl(url, urlValidation.resolvedIP!)
const headers: Record<string, string> = {
Host: urlValidation.originalHostname!,
}
if (accessToken) { if (accessToken) {
headers.Authorization = `Bearer ${accessToken}` headers.Authorization = `Bearer ${accessToken}`
} }
const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { const response = await fetch(pinnedUrl, {
headers, headers,
redirect: 'follow',
}) })
return response return response
@@ -685,9 +686,6 @@ export async function formatWebhookInput(
if (foundWebhook.provider === 'rss') { if (foundWebhook.provider === 'rss') {
if (body && typeof body === 'object' && 'item' in body) { if (body && typeof body === 'object' && 'item' in body) {
return { return {
title: body.title,
link: body.link,
pubDate: body.pubDate,
item: body.item, item: body.item,
feed: body.feed, feed: body.feed,
timestamp: body.timestamp, timestamp: body.timestamp,
@@ -699,17 +697,6 @@ export async function formatWebhookInput(
if (foundWebhook.provider === 'imap') { if (foundWebhook.provider === 'imap') {
if (body && typeof body === 'object' && 'email' in body) { if (body && typeof body === 'object' && 'email' in body) {
return { return {
messageId: body.messageId,
subject: body.subject,
from: body.from,
to: body.to,
cc: body.cc,
date: body.date,
bodyText: body.bodyText,
bodyHtml: body.bodyHtml,
mailbox: body.mailbox,
hasAttachments: body.hasAttachments,
attachments: body.attachments,
email: body.email, email: body.email,
timestamp: body.timestamp, timestamp: body.timestamp,
} }

View File

@@ -351,7 +351,7 @@ function collectOutputPaths(
if (value && typeof value === 'object' && 'type' in value) { if (value && typeof value === 'object' && 'type' in value) {
const typedValue = value as { type: unknown } const typedValue = value as { type: unknown }
if (typedValue.type === 'files' || typedValue.type === 'file[]') { if (typedValue.type === 'files') {
paths.push(...expandFileTypeProperties(path)) paths.push(...expandFileTypeProperties(path))
} else { } else {
paths.push(path) paths.push(path)
@@ -393,8 +393,7 @@ function getFilePropertyType(outputs: OutputDefinition, pathParts: string[]): st
current && current &&
typeof current === 'object' && typeof current === 'object' &&
'type' in current && 'type' in current &&
((current as { type: unknown }).type === 'files' || (current as { type: unknown }).type === 'files'
(current as { type: unknown }).type === 'file[]')
) { ) {
return USER_FILE_PROPERTY_TYPES[lastPart as keyof typeof USER_FILE_PROPERTY_TYPES] return USER_FILE_PROPERTY_TYPES[lastPart as keyof typeof USER_FILE_PROPERTY_TYPES]
} }
@@ -463,11 +462,6 @@ function generateOutputPaths(outputs: Record<string, any>, prefix = ''): string[
paths.push(currentPath) paths.push(currentPath)
} else if (typeof value === 'object' && value !== null) { } else if (typeof value === 'object' && value !== null) {
if ('type' in value && typeof value.type === 'string') { if ('type' in value && typeof value.type === 'string') {
if (value.type === 'files' || value.type === 'file[]') {
paths.push(...expandFileTypeProperties(currentPath))
continue
}
const hasNestedProperties = const hasNestedProperties =
((value.type === 'object' || value.type === 'json') && value.properties) || ((value.type === 'object' || value.type === 'json') && value.properties) ||
(value.type === 'array' && value.items?.properties) || (value.type === 'array' && value.items?.properties) ||
@@ -524,17 +518,6 @@ function generateOutputPathsWithTypes(
paths.push({ path: currentPath, type: value }) paths.push({ path: currentPath, type: value })
} else if (typeof value === 'object' && value !== null) { } else if (typeof value === 'object' && value !== null) {
if ('type' in value && typeof value.type === 'string') { if ('type' in value && typeof value.type === 'string') {
if (value.type === 'files' || value.type === 'file[]') {
paths.push({ path: currentPath, type: value.type })
for (const prop of USER_FILE_ACCESSIBLE_PROPERTIES) {
paths.push({
path: `${currentPath}.${prop}`,
type: USER_FILE_PROPERTY_TYPES[prop as keyof typeof USER_FILE_PROPERTY_TYPES],
})
}
continue
}
if (value.type === 'array' && value.items?.properties) { if (value.type === 'array' && value.items?.properties) {
paths.push({ path: currentPath, type: 'array' }) paths.push({ path: currentPath, type: 'array' })
const subPaths = generateOutputPathsWithTypes(value.items.properties, currentPath) const subPaths = generateOutputPathsWithTypes(value.items.properties, currentPath)
@@ -562,26 +545,14 @@ function generateOutputPathsWithTypes(
* Gets the tool outputs for a block operation. * Gets the tool outputs for a block operation.
* *
* @param blockConfig - The block configuration containing tools config * @param blockConfig - The block configuration containing tools config
* @param subBlocks - SubBlock values to pass to the tool selector * @param operation - The selected operation for the tool
* @returns Outputs schema for the tool, or empty object on error * @returns Outputs schema for the tool, or empty object on error
*/ */
export function getToolOutputs( export function getToolOutputs(blockConfig: BlockConfig, operation: string): Record<string, any> {
blockConfig: BlockConfig,
subBlocks?: Record<string, SubBlockWithValue>
): Record<string, any> {
if (!blockConfig?.tools?.config?.tool) return {} if (!blockConfig?.tools?.config?.tool) return {}
try { try {
// Build params object from subBlock values for tool selector const toolId = blockConfig.tools.config.tool({ operation })
// This allows tool selectors to use any field (operation, provider, etc.)
const params: Record<string, any> = {}
if (subBlocks) {
for (const [key, subBlock] of Object.entries(subBlocks)) {
params[key] = subBlock.value
}
}
const toolId = blockConfig.tools.config.tool(params)
if (!toolId) return {} if (!toolId) return {}
const toolConfig = getTool(toolId) const toolConfig = getTool(toolId)
@@ -589,7 +560,7 @@ export function getToolOutputs(
return toolConfig.outputs return toolConfig.outputs
} catch (error) { } catch (error) {
logger.warn('Failed to get tool outputs', { error }) logger.warn('Failed to get tool outputs for operation', { operation, error })
return {} return {}
} }
} }
@@ -598,14 +569,16 @@ export function getToolOutputs(
* Generates output paths for a tool-based block. * Generates output paths for a tool-based block.
* *
* @param blockConfig - The block configuration containing tools config * @param blockConfig - The block configuration containing tools config
* @param subBlocks - SubBlock values for tool selection and condition evaluation * @param operation - The selected operation for the tool
* @param subBlocks - Optional subBlock values for condition evaluation
* @returns Array of output paths for the tool, or empty array on error * @returns Array of output paths for the tool, or empty array on error
*/ */
export function getToolOutputPaths( export function getToolOutputPaths(
blockConfig: BlockConfig, blockConfig: BlockConfig,
operation: string,
subBlocks?: Record<string, SubBlockWithValue> subBlocks?: Record<string, SubBlockWithValue>
): string[] { ): string[] {
const outputs = getToolOutputs(blockConfig, subBlocks) const outputs = getToolOutputs(blockConfig, operation)
if (!outputs || Object.keys(outputs).length === 0) return [] if (!outputs || Object.keys(outputs).length === 0) return []
@@ -640,16 +613,16 @@ export function getOutputPathsFromSchema(outputs: Record<string, any>): string[]
* Gets the output type for a specific path in a tool's outputs. * Gets the output type for a specific path in a tool's outputs.
* *
* @param blockConfig - The block configuration containing tools config * @param blockConfig - The block configuration containing tools config
* @param subBlocks - SubBlock values for tool selection * @param operation - The selected operation for the tool
* @param path - The dot-separated path to the output field * @param path - The dot-separated path to the output field
* @returns The type of the output field, or 'any' if not found * @returns The type of the output field, or 'any' if not found
*/ */
export function getToolOutputType( export function getToolOutputType(
blockConfig: BlockConfig, blockConfig: BlockConfig,
subBlocks: Record<string, SubBlockWithValue> | undefined, operation: string,
path: string path: string
): string { ): string {
const outputs = getToolOutputs(blockConfig, subBlocks) const outputs = getToolOutputs(blockConfig, operation)
if (!outputs || Object.keys(outputs).length === 0) return 'any' if (!outputs || Object.keys(outputs).length === 0) return 'any'
const pathsWithTypes = generateOutputPathsWithTypes(outputs) const pathsWithTypes = generateOutputPathsWithTypes(outputs)

View File

@@ -17,8 +17,6 @@ export interface ExecuteWorkflowOptions {
onStream?: (streamingExec: StreamingExecution) => Promise<void> onStream?: (streamingExec: StreamingExecution) => Promise<void>
onBlockComplete?: (blockId: string, output: unknown) => Promise<void> onBlockComplete?: (blockId: string, output: unknown) => Promise<void>
skipLoggingComplete?: boolean skipLoggingComplete?: boolean
includeFileBase64?: boolean
base64MaxBytes?: number
} }
export interface WorkflowInfo { export interface WorkflowInfo {
@@ -80,8 +78,6 @@ export async function executeWorkflow(
: undefined, : undefined,
}, },
loggingSession, loggingSession,
includeFileBase64: streamConfig?.includeFileBase64,
base64MaxBytes: streamConfig?.base64MaxBytes,
}) })
if (result.status === 'paused') { if (result.status === 'paused') {

View File

@@ -37,10 +37,12 @@ export interface ExecuteWorkflowCoreOptions {
snapshot: ExecutionSnapshot snapshot: ExecutionSnapshot
callbacks: ExecutionCallbacks callbacks: ExecutionCallbacks
loggingSession: LoggingSession loggingSession: LoggingSession
skipLogCreation?: boolean skipLogCreation?: boolean // For resume executions - reuse existing log entry
/**
* AbortSignal for cancellation support.
* When aborted (e.g., client disconnects from SSE), execution stops gracefully.
*/
abortSignal?: AbortSignal abortSignal?: AbortSignal
includeFileBase64?: boolean
base64MaxBytes?: number
} }
function parseVariableValueByType(value: unknown, type: string): unknown { function parseVariableValueByType(value: unknown, type: string): unknown {
@@ -107,15 +109,7 @@ function parseVariableValueByType(value: unknown, type: string): unknown {
export async function executeWorkflowCore( export async function executeWorkflowCore(
options: ExecuteWorkflowCoreOptions options: ExecuteWorkflowCoreOptions
): Promise<ExecutionResult> { ): Promise<ExecutionResult> {
const { const { snapshot, callbacks, loggingSession, skipLogCreation, abortSignal } = options
snapshot,
callbacks,
loggingSession,
skipLogCreation,
abortSignal,
includeFileBase64,
base64MaxBytes,
} = options
const { metadata, workflow, input, workflowVariables, selectedOutputs } = snapshot const { metadata, workflow, input, workflowVariables, selectedOutputs } = snapshot
const { requestId, workflowId, userId, triggerType, executionId, triggerBlockId, useDraftState } = const { requestId, workflowId, userId, triggerType, executionId, triggerBlockId, useDraftState } =
metadata metadata
@@ -340,8 +334,6 @@ export async function executeWorkflowCore(
snapshotState: snapshot.state, snapshotState: snapshot.state,
metadata, metadata,
abortSignal, abortSignal,
includeFileBase64,
base64MaxBytes,
} }
const executorInstance = new Executor({ const executorInstance = new Executor({

View File

@@ -751,8 +751,6 @@ export class PauseResumeManager {
callbacks: {}, callbacks: {},
loggingSession, loggingSession,
skipLogCreation: true, // Reuse existing log entry skipLogCreation: true, // Reuse existing log entry
includeFileBase64: true, // Enable base64 hydration
base64MaxBytes: undefined, // Use default limit
}) })
} }

View File

@@ -7,10 +7,6 @@ import {
import { encodeSSE } from '@/lib/core/utils/sse' import { encodeSSE } from '@/lib/core/utils/sse'
import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans'
import { processStreamingBlockLogs } from '@/lib/tokenization' import { processStreamingBlockLogs } from '@/lib/tokenization'
import {
cleanupExecutionBase64Cache,
hydrateUserFilesWithBase64,
} from '@/lib/uploads/utils/user-file-base64.server'
import { executeWorkflow } from '@/lib/workflows/executor/execute-workflow' import { executeWorkflow } from '@/lib/workflows/executor/execute-workflow'
import type { BlockLog, ExecutionResult, StreamingExecution } from '@/executor/types' import type { BlockLog, ExecutionResult, StreamingExecution } from '@/executor/types'
@@ -30,8 +26,6 @@ export interface StreamingConfig {
selectedOutputs?: string[] selectedOutputs?: string[]
isSecureMode?: boolean isSecureMode?: boolean
workflowTriggerType?: 'api' | 'chat' workflowTriggerType?: 'api' | 'chat'
includeFileBase64?: boolean
base64MaxBytes?: number
} }
export interface StreamingResponseOptions { export interface StreamingResponseOptions {
@@ -63,14 +57,12 @@ function isDangerousKey(key: string): boolean {
return DANGEROUS_KEYS.includes(key) return DANGEROUS_KEYS.includes(key)
} }
async function buildMinimalResult( function buildMinimalResult(
result: ExecutionResult, result: ExecutionResult,
selectedOutputs: string[] | undefined, selectedOutputs: string[] | undefined,
streamedContent: Map<string, string>, streamedContent: Map<string, string>,
requestId: string, requestId: string
includeFileBase64: boolean, ): { success: boolean; error?: string; output: Record<string, unknown> } {
base64MaxBytes: number | undefined
): Promise<{ success: boolean; error?: string; output: Record<string, unknown> }> {
const minimalResult = { const minimalResult = {
success: result.success, success: result.success,
error: result.error, error: result.error,
@@ -231,9 +223,6 @@ export async function createStreamingResponse(
} }
} }
const includeFileBase64 = streamConfig.includeFileBase64 ?? true
const base64MaxBytes = streamConfig.base64MaxBytes
const onBlockCompleteCallback = async (blockId: string, output: unknown) => { const onBlockCompleteCallback = async (blockId: string, output: unknown) => {
if (!streamConfig.selectedOutputs?.length) { if (!streamConfig.selectedOutputs?.length) {
return return
@@ -252,17 +241,8 @@ export async function createStreamingResponse(
const outputValue = extractOutputValue(output, path) const outputValue = extractOutputValue(output, path)
if (outputValue !== undefined) { if (outputValue !== undefined) {
const hydratedOutput = includeFileBase64
? await hydrateUserFilesWithBase64(outputValue, {
requestId,
executionId,
maxBytes: base64MaxBytes,
})
: outputValue
const formattedOutput = const formattedOutput =
typeof hydratedOutput === 'string' typeof outputValue === 'string' ? outputValue : JSON.stringify(outputValue, null, 2)
? hydratedOutput
: JSON.stringify(hydratedOutput, null, 2)
sendChunk(blockId, formattedOutput) sendChunk(blockId, formattedOutput)
} }
} }
@@ -282,8 +262,6 @@ export async function createStreamingResponse(
onStream: onStreamCallback, onStream: onStreamCallback,
onBlockComplete: onBlockCompleteCallback, onBlockComplete: onBlockCompleteCallback,
skipLoggingComplete: true, skipLoggingComplete: true,
includeFileBase64: streamConfig.includeFileBase64,
base64MaxBytes: streamConfig.base64MaxBytes,
}, },
executionId executionId
) )
@@ -295,33 +273,21 @@ export async function createStreamingResponse(
await completeLoggingSession(result) await completeLoggingSession(result)
const minimalResult = await buildMinimalResult( const minimalResult = buildMinimalResult(
result, result,
streamConfig.selectedOutputs, streamConfig.selectedOutputs,
state.streamedContent, state.streamedContent,
requestId, requestId
streamConfig.includeFileBase64 ?? true,
streamConfig.base64MaxBytes
) )
controller.enqueue(encodeSSE({ event: 'final', data: minimalResult })) controller.enqueue(encodeSSE({ event: 'final', data: minimalResult }))
controller.enqueue(encodeSSE('[DONE]')) controller.enqueue(encodeSSE('[DONE]'))
if (executionId) {
await cleanupExecutionBase64Cache(executionId)
}
controller.close() controller.close()
} catch (error: any) { } catch (error: any) {
logger.error(`[${requestId}] Stream error:`, error) logger.error(`[${requestId}] Stream error:`, error)
controller.enqueue( controller.enqueue(
encodeSSE({ event: 'error', error: error.message || 'Stream processing error' }) encodeSSE({ event: 'error', error: error.message || 'Stream processing error' })
) )
if (executionId) {
await cleanupExecutionBase64Cache(executionId)
}
controller.close() controller.close()
} }
}, },

View File

@@ -5,14 +5,7 @@ export interface InputFormatField {
value?: unknown value?: unknown
} }
export const USER_FILE_ACCESSIBLE_PROPERTIES = [ export const USER_FILE_ACCESSIBLE_PROPERTIES = ['id', 'name', 'url', 'size', 'type'] as const
'id',
'name',
'url',
'size',
'type',
'base64',
] as const
export type UserFileAccessibleProperty = (typeof USER_FILE_ACCESSIBLE_PROPERTIES)[number] export type UserFileAccessibleProperty = (typeof USER_FILE_ACCESSIBLE_PROPERTIES)[number]
@@ -22,7 +15,6 @@ export const USER_FILE_PROPERTY_TYPES: Record<UserFileAccessibleProperty, string
url: 'string', url: 'string',
size: 'number', size: 'number',
type: 'string', type: 'string',
base64: 'string',
} as const } as const
export const START_BLOCK_RESERVED_FIELDS = ['input', 'conversationId', 'files'] as const export const START_BLOCK_RESERVED_FIELDS = ['input', 'conversationId', 'files'] as const

View File

@@ -108,7 +108,6 @@
"imapflow": "1.2.4", "imapflow": "1.2.4",
"input-otp": "^1.4.2", "input-otp": "^1.4.2",
"ioredis": "^5.6.0", "ioredis": "^5.6.0",
"ipaddr.js": "2.3.0",
"isolated-vm": "6.0.2", "isolated-vm": "6.0.2",
"jose": "6.0.11", "jose": "6.0.11",
"js-tiktoken": "1.0.21", "js-tiktoken": "1.0.21",

View File

@@ -388,7 +388,7 @@ export const anthropicProvider: ProviderConfig = {
toolArgs, toolArgs,
request request
) )
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -301,7 +301,7 @@ export const azureOpenAIProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -481,7 +481,7 @@ export const bedrockProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -244,7 +244,7 @@ export const cerebrasProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -256,7 +256,7 @@ export const deepseekProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -19,7 +19,6 @@ import {
convertToGeminiFormat, convertToGeminiFormat,
convertUsageMetadata, convertUsageMetadata,
createReadableStreamFromGeminiStream, createReadableStreamFromGeminiStream,
ensureStructResponse,
extractFunctionCallPart, extractFunctionCallPart,
extractTextContent, extractTextContent,
mapToThinkingLevel, mapToThinkingLevel,
@@ -100,12 +99,12 @@ async function executeToolCall(
try { try {
const { toolParams, executionParams } = prepareToolExecution(tool, functionCall.args, request) const { toolParams, executionParams } = prepareToolExecution(tool, functionCall.args, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
const duration = toolCallEndTime - toolCallStartTime const duration = toolCallEndTime - toolCallStartTime
const resultContent: Record<string, unknown> = result.success const resultContent: Record<string, unknown> = result.success
? ensureStructResponse(result.output) ? (result.output as Record<string, unknown>)
: { error: true, message: result.error || 'Tool execution failed', tool: toolName } : { error: true, message: result.error || 'Tool execution failed', tool: toolName }
const toolCall: FunctionCallResponse = { const toolCall: FunctionCallResponse = {

View File

@@ -1,453 +0,0 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { convertToGeminiFormat, ensureStructResponse } from '@/providers/google/utils'
import type { ProviderRequest } from '@/providers/types'
describe('ensureStructResponse', () => {
describe('should return objects unchanged', () => {
it('should return plain object unchanged', () => {
const input = { key: 'value', nested: { a: 1 } }
const result = ensureStructResponse(input)
expect(result).toBe(input) // Same reference
expect(result).toEqual({ key: 'value', nested: { a: 1 } })
})
it('should return empty object unchanged', () => {
const input = {}
const result = ensureStructResponse(input)
expect(result).toBe(input)
expect(result).toEqual({})
})
})
describe('should wrap primitive values in { value: ... }', () => {
it('should wrap boolean true', () => {
const result = ensureStructResponse(true)
expect(result).toEqual({ value: true })
expect(typeof result).toBe('object')
})
it('should wrap boolean false', () => {
const result = ensureStructResponse(false)
expect(result).toEqual({ value: false })
expect(typeof result).toBe('object')
})
it('should wrap string', () => {
const result = ensureStructResponse('success')
expect(result).toEqual({ value: 'success' })
expect(typeof result).toBe('object')
})
it('should wrap empty string', () => {
const result = ensureStructResponse('')
expect(result).toEqual({ value: '' })
expect(typeof result).toBe('object')
})
it('should wrap number', () => {
const result = ensureStructResponse(42)
expect(result).toEqual({ value: 42 })
expect(typeof result).toBe('object')
})
it('should wrap zero', () => {
const result = ensureStructResponse(0)
expect(result).toEqual({ value: 0 })
expect(typeof result).toBe('object')
})
it('should wrap null', () => {
const result = ensureStructResponse(null)
expect(result).toEqual({ value: null })
expect(typeof result).toBe('object')
})
it('should wrap undefined', () => {
const result = ensureStructResponse(undefined)
expect(result).toEqual({ value: undefined })
expect(typeof result).toBe('object')
})
})
describe('should wrap arrays in { value: ... }', () => {
it('should wrap array of strings', () => {
const result = ensureStructResponse(['a', 'b', 'c'])
expect(result).toEqual({ value: ['a', 'b', 'c'] })
expect(typeof result).toBe('object')
expect(Array.isArray(result)).toBe(false)
})
it('should wrap array of objects', () => {
const result = ensureStructResponse([{ id: 1 }, { id: 2 }])
expect(result).toEqual({ value: [{ id: 1 }, { id: 2 }] })
expect(typeof result).toBe('object')
expect(Array.isArray(result)).toBe(false)
})
it('should wrap empty array', () => {
const result = ensureStructResponse([])
expect(result).toEqual({ value: [] })
expect(typeof result).toBe('object')
expect(Array.isArray(result)).toBe(false)
})
})
describe('edge cases', () => {
it('should handle nested objects correctly', () => {
const input = { a: { b: { c: 1 } }, d: [1, 2, 3] }
const result = ensureStructResponse(input)
expect(result).toBe(input) // Same reference, unchanged
})
it('should handle object with array property correctly', () => {
const input = { items: ['a', 'b'], count: 2 }
const result = ensureStructResponse(input)
expect(result).toBe(input) // Same reference, unchanged
})
})
})
describe('convertToGeminiFormat', () => {
describe('tool message handling', () => {
it('should convert tool message with object response correctly', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Hello' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_123',
type: 'function',
function: { name: 'get_weather', arguments: '{"city": "London"}' },
},
],
},
{
role: 'tool',
name: 'get_weather',
tool_call_id: 'call_123',
content: '{"temperature": 20, "condition": "sunny"}',
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
expect(toolResponseContent).toBeDefined()
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(functionResponse?.response).toEqual({ temperature: 20, condition: 'sunny' })
expect(typeof functionResponse?.response).toBe('object')
})
it('should wrap boolean true response in an object for Gemini compatibility', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Check if user exists' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_456',
type: 'function',
function: { name: 'user_exists', arguments: '{"userId": "123"}' },
},
],
},
{
role: 'tool',
name: 'user_exists',
tool_call_id: 'call_456',
content: 'true', // Boolean true as JSON string
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
expect(toolResponseContent).toBeDefined()
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
expect(functionResponse?.response).not.toBe(true)
expect(functionResponse?.response).toEqual({ value: true })
})
it('should wrap boolean false response in an object for Gemini compatibility', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Check if user exists' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_789',
type: 'function',
function: { name: 'user_exists', arguments: '{"userId": "999"}' },
},
],
},
{
role: 'tool',
name: 'user_exists',
tool_call_id: 'call_789',
content: 'false', // Boolean false as JSON string
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
expect(functionResponse?.response).toEqual({ value: false })
})
it('should wrap string response in an object for Gemini compatibility', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Get status' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_str',
type: 'function',
function: { name: 'get_status', arguments: '{}' },
},
],
},
{
role: 'tool',
name: 'get_status',
tool_call_id: 'call_str',
content: '"success"', // String as JSON
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
expect(functionResponse?.response).toEqual({ value: 'success' })
})
it('should wrap number response in an object for Gemini compatibility', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Get count' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_num',
type: 'function',
function: { name: 'get_count', arguments: '{}' },
},
],
},
{
role: 'tool',
name: 'get_count',
tool_call_id: 'call_num',
content: '42', // Number as JSON
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
expect(functionResponse?.response).toEqual({ value: 42 })
})
it('should wrap null response in an object for Gemini compatibility', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Get data' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_null',
type: 'function',
function: { name: 'get_data', arguments: '{}' },
},
],
},
{
role: 'tool',
name: 'get_data',
tool_call_id: 'call_null',
content: 'null', // null as JSON
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
expect(functionResponse?.response).toEqual({ value: null })
})
it('should keep array response as-is since arrays are valid Struct values', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Get items' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_arr',
type: 'function',
function: { name: 'get_items', arguments: '{}' },
},
],
},
{
role: 'tool',
name: 'get_items',
tool_call_id: 'call_arr',
content: '["item1", "item2"]', // Array as JSON
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
expect(functionResponse?.response).toEqual({ value: ['item1', 'item2'] })
})
it('should handle invalid JSON by wrapping in output object', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Get data' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_invalid',
type: 'function',
function: { name: 'get_data', arguments: '{}' },
},
],
},
{
role: 'tool',
name: 'get_data',
tool_call_id: 'call_invalid',
content: 'not valid json {',
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
expect(functionResponse?.response).toEqual({ output: 'not valid json {' })
})
it('should handle empty content by wrapping in output object', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{ role: 'user', content: 'Do something' },
{
role: 'assistant',
content: '',
tool_calls: [
{
id: 'call_empty',
type: 'function',
function: { name: 'do_action', arguments: '{}' },
},
],
},
{
role: 'tool',
name: 'do_action',
tool_call_id: 'call_empty',
content: '', // Empty content - falls back to default '{}'
},
],
}
const result = convertToGeminiFormat(request)
const toolResponseContent = result.contents.find(
(c) => c.parts?.[0] && 'functionResponse' in c.parts[0]
)
const functionResponse = (toolResponseContent?.parts?.[0] as { functionResponse?: unknown })
?.functionResponse as { response?: unknown }
expect(typeof functionResponse?.response).toBe('object')
// Empty string is not valid JSON, so it falls back to { output: "" }
expect(functionResponse?.response).toEqual({ output: '' })
})
})
})

View File

@@ -18,22 +18,6 @@ import { trackForcedToolUsage } from '@/providers/utils'
const logger = createLogger('GoogleUtils') const logger = createLogger('GoogleUtils')
/**
* Ensures a value is a valid object for Gemini's functionResponse.response field.
* Gemini's API requires functionResponse.response to be a google.protobuf.Struct,
* which must be an object with string keys. Primitive values (boolean, string,
* number, null) and arrays are wrapped in { value: ... }.
*
* @param value - The value to ensure is a Struct-compatible object
* @returns A Record<string, unknown> suitable for functionResponse.response
*/
export function ensureStructResponse(value: unknown): Record<string, unknown> {
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
return value as Record<string, unknown>
}
return { value }
}
/** /**
* Usage metadata for Google Gemini responses * Usage metadata for Google Gemini responses
*/ */
@@ -196,8 +180,7 @@ export function convertToGeminiFormat(request: ProviderRequest): {
} }
let responseData: Record<string, unknown> let responseData: Record<string, unknown>
try { try {
const parsed = JSON.parse(message.content ?? '{}') responseData = JSON.parse(message.content ?? '{}')
responseData = ensureStructResponse(parsed)
} catch { } catch {
responseData = { output: message.content } responseData = { output: message.content }
} }

View File

@@ -234,7 +234,7 @@ export const groqProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -299,7 +299,7 @@ export const mistralProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -307,7 +307,7 @@ export const ollamaProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -300,7 +300,7 @@ export const openaiProvider: ProviderConfig = {
} }
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

View File

@@ -286,7 +286,7 @@ export const openRouterProvider: ProviderConfig = {
if (!tool) return null if (!tool) return null
const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request)
const result = await executeTool(toolName, executionParams) const result = await executeTool(toolName, executionParams, true)
const toolCallEndTime = Date.now() const toolCallEndTime = Date.now()
return { return {

Some files were not shown because too many files have changed in this diff Show More