From 2c678f2658324aa709bc9c7a43027625476d703b Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Tue, 27 Jan 2026 21:53:09 -0600 Subject: [PATCH] refactor(backend): rename return_format options for clarity and add auto-fallback Rename store_media_file() return_format options to make intent clear: - "local_path" -> "for_local_processing" (ffmpeg, MoviePy, PIL) - "data_uri" -> "for_external_api" (Replicate, OpenAI APIs) - "workspace_ref" -> "for_block_output" (auto-adapts to context) The "for_block_output" format now gracefully handles both contexts: - CoPilot (has workspace): returns workspace:// reference - Graph execution (no workspace): falls back to data URI This prevents blocks from failing in graph execution while still providing workspace persistence in CoPilot. Also adds documentation to CLAUDE.md, new_blocks.md, and block-sdk-guide.md explaining when to use each format. Co-Authored-By: Claude Opus 4.5 --- autogpt_platform/CLAUDE.md | 44 +++++++++++++ .../backend/blocks/ai_image_customizer.py | 4 +- .../blocks/ai_image_generator_block.py | 2 +- .../blocks/ai_shortform_video_block.py | 6 +- .../backend/blocks/bannerbear/text_overlay.py | 2 +- .../backend/backend/blocks/basic.py | 13 ++-- .../backend/blocks/discord/bot_blocks.py | 2 +- .../backend/blocks/fal/ai_video_generator.py | 2 +- .../backend/backend/blocks/flux_kontext.py | 4 +- .../backend/backend/blocks/google/gmail.py | 6 +- .../backend/backend/blocks/http.py | 2 +- autogpt_platform/backend/backend/blocks/io.py | 13 ++-- .../backend/backend/blocks/media.py | 20 +++--- .../backend/backend/blocks/screenshotone.py | 2 +- .../backend/backend/blocks/spreadsheet.py | 2 +- .../backend/backend/blocks/talking_head.py | 2 +- .../backend/backend/blocks/text.py | 2 +- autogpt_platform/backend/backend/util/file.py | 59 ++++++++++------- .../backend/backend/util/file_test.py | 6 +- docs/platform/block-sdk-guide.md | 44 +++++++++++++ docs/platform/new_blocks.md | 65 +++++++++++++++++++ 21 files changed, 229 insertions(+), 73 deletions(-) diff --git a/autogpt_platform/CLAUDE.md b/autogpt_platform/CLAUDE.md index 2c76e7db80..9690178587 100644 --- a/autogpt_platform/CLAUDE.md +++ b/autogpt_platform/CLAUDE.md @@ -194,6 +194,50 @@ ex: do the inputs and outputs tie well together? If you get any pushback or hit complex block conditions check the new_blocks guide in the docs. +**Handling files in blocks with `store_media_file()`:** + +When blocks need to work with files (images, videos, documents), use `store_media_file()` from `backend.util.file`. The `return_format` parameter determines what you get back: + +| Format | Use When | Returns | +|--------|----------|---------| +| `"for_local_processing"` | Processing with local tools (ffmpeg, MoviePy, PIL) | Local file path (e.g., `"image.png"`) | +| `"for_external_api"` | Sending content to external APIs (Replicate, OpenAI) | Data URI (e.g., `"data:image/png;base64,..."`) | +| `"for_block_output"` | Returning output from your block | Smart: `workspace://` in CoPilot, data URI in graphs | + +**Examples:** +```python +# INPUT: Need to process file locally with ffmpeg +local_path = await store_media_file( + file=input_data.video, + execution_context=execution_context, + return_format="for_local_processing", +) +# local_path = "video.mp4" - use with Path/ffmpeg/etc + +# INPUT: Need to send to external API like Replicate +image_b64 = await store_media_file( + file=input_data.image, + execution_context=execution_context, + return_format="for_external_api", +) +# image_b64 = "data:image/png;base64,iVBORw0..." - send to API + +# OUTPUT: Returning result from block +result_url = await store_media_file( + file=generated_image_url, + execution_context=execution_context, + return_format="for_block_output", +) +yield "image_url", result_url +# In CoPilot: result_url = "workspace://abc123" +# In graphs: result_url = "data:image/png;base64,..." +``` + +**Key points:** +- `for_block_output` is the ONLY format that auto-adapts to execution context +- Always use `for_block_output` for block outputs unless you have a specific reason not to +- Never hardcode workspace checks - let `for_block_output` handle it + **Modifying the API:** 1. Update route in `/backend/backend/server/routers/` diff --git a/autogpt_platform/backend/backend/blocks/ai_image_customizer.py b/autogpt_platform/backend/backend/blocks/ai_image_customizer.py index aaee3bea05..39b96c3fae 100644 --- a/autogpt_platform/backend/backend/blocks/ai_image_customizer.py +++ b/autogpt_platform/backend/backend/blocks/ai_image_customizer.py @@ -143,7 +143,7 @@ class AIImageCustomizerBlock(Block): store_media_file( file=img, execution_context=execution_context, - return_format="data_uri", # Get content for external API + return_format="for_external_api", # Get content for Replicate API ) for img in input_data.images ) @@ -162,7 +162,7 @@ class AIImageCustomizerBlock(Block): stored_url = await store_media_file( file=result, execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "image_url", stored_url except Exception as e: diff --git a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py index 296c86ddbe..e40731cd97 100644 --- a/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_image_generator_block.py @@ -338,7 +338,7 @@ class AIImageGeneratorBlock(Block): stored_url = await store_media_file( file=MediaFileType(url), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "image_url", stored_url else: diff --git a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py index df906e0208..2dd937b766 100644 --- a/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py +++ b/autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py @@ -352,7 +352,7 @@ class AIShortformVideoCreatorBlock(Block): stored_url = await store_media_file( file=MediaFileType(video_url), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "video_url", stored_url @@ -556,7 +556,7 @@ class AIAdMakerVideoCreatorBlock(Block): stored_url = await store_media_file( file=MediaFileType(video_url), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "video_url", stored_url @@ -748,6 +748,6 @@ class AIScreenshotToVideoAdBlock(Block): stored_url = await store_media_file( file=MediaFileType(video_url), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "video_url", stored_url diff --git a/autogpt_platform/backend/backend/blocks/bannerbear/text_overlay.py b/autogpt_platform/backend/backend/blocks/bannerbear/text_overlay.py index d09d114706..98115d1056 100644 --- a/autogpt_platform/backend/backend/blocks/bannerbear/text_overlay.py +++ b/autogpt_platform/backend/backend/blocks/bannerbear/text_overlay.py @@ -249,7 +249,7 @@ class BannerbearTextOverlayBlock(Block): stored_url = await store_media_file( file=MediaFileType(image_url), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "image_url", stored_url else: diff --git a/autogpt_platform/backend/backend/blocks/basic.py b/autogpt_platform/backend/backend/blocks/basic.py index 3312c117db..24c8b0fe60 100644 --- a/autogpt_platform/backend/backend/blocks/basic.py +++ b/autogpt_platform/backend/backend/blocks/basic.py @@ -49,13 +49,12 @@ class FileStoreBlock(Block): execution_context: ExecutionContext, **kwargs, ) -> BlockOutput: - # Determine return format based on context and user preference - if execution_context.workspace_id and input_data.base_64: - return_format = "workspace_ref" - elif input_data.base_64: - return_format = "data_uri" - else: - return_format = "local_path" + # Determine return format based on user preference + # for_block_output: returns workspace:// if available, else data URI + # for_local_processing: returns local file path + return_format = ( + "for_block_output" if input_data.base_64 else "for_local_processing" + ) yield "file_out", await store_media_file( file=input_data.file_in, diff --git a/autogpt_platform/backend/backend/blocks/discord/bot_blocks.py b/autogpt_platform/backend/backend/blocks/discord/bot_blocks.py index 6bb5da7dd9..4438af1955 100644 --- a/autogpt_platform/backend/backend/blocks/discord/bot_blocks.py +++ b/autogpt_platform/backend/backend/blocks/discord/bot_blocks.py @@ -733,7 +733,7 @@ class SendDiscordFileBlock(Block): stored_file = await store_media_file( file=file, execution_context=execution_context, - return_format="data_uri", # Get content to send to Discord + return_format="for_external_api", # Get content to send to Discord ) # Now process as data URI header, encoded = stored_file.split(",", 1) diff --git a/autogpt_platform/backend/backend/blocks/fal/ai_video_generator.py b/autogpt_platform/backend/backend/blocks/fal/ai_video_generator.py index 7d289cec7f..ea0353cfdd 100644 --- a/autogpt_platform/backend/backend/blocks/fal/ai_video_generator.py +++ b/autogpt_platform/backend/backend/blocks/fal/ai_video_generator.py @@ -224,7 +224,7 @@ class AIVideoGeneratorBlock(Block): stored_url = await store_media_file( file=MediaFileType(video_url), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "video_url", stored_url except Exception as e: diff --git a/autogpt_platform/backend/backend/blocks/flux_kontext.py b/autogpt_platform/backend/backend/blocks/flux_kontext.py index ac63cf89cf..0f3bcead31 100644 --- a/autogpt_platform/backend/backend/blocks/flux_kontext.py +++ b/autogpt_platform/backend/backend/blocks/flux_kontext.py @@ -146,7 +146,7 @@ class AIImageEditorBlock(Block): await store_media_file( file=input_data.input_image, execution_context=execution_context, - return_format="data_uri", # Get content for external API + return_format="for_external_api", # Get content for Replicate API ) if input_data.input_image else None @@ -160,7 +160,7 @@ class AIImageEditorBlock(Block): stored_url = await store_media_file( file=result, execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "output_image", stored_url diff --git a/autogpt_platform/backend/backend/blocks/google/gmail.py b/autogpt_platform/backend/backend/blocks/google/gmail.py index bafacb4b78..8f3d9dd615 100644 --- a/autogpt_platform/backend/backend/blocks/google/gmail.py +++ b/autogpt_platform/backend/backend/blocks/google/gmail.py @@ -119,7 +119,7 @@ async def create_mime_message( local_path = await store_media_file( file=attach, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) abs_path = get_exec_file_path( execution_context.graph_exec_id or "", local_path @@ -1189,7 +1189,7 @@ async def _build_reply_message( local_path = await store_media_file( file=attach, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) abs_path = get_exec_file_path(execution_context.graph_exec_id or "", local_path) part = MIMEBase("application", "octet-stream") @@ -1719,7 +1719,7 @@ To: {original_to} local_path = await store_media_file( file=attach, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) abs_path = get_exec_file_path( execution_context.graph_exec_id or "", local_path diff --git a/autogpt_platform/backend/backend/blocks/http.py b/autogpt_platform/backend/backend/blocks/http.py index fe5f57254b..56fe592450 100644 --- a/autogpt_platform/backend/backend/blocks/http.py +++ b/autogpt_platform/backend/backend/blocks/http.py @@ -135,7 +135,7 @@ class SendWebRequestBlock(Block): rel_path = await store_media_file( file=media, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) abs_path = get_exec_file_path(graph_exec_id, rel_path) async with aiofiles.open(abs_path, "rb") as f: diff --git a/autogpt_platform/backend/backend/blocks/io.py b/autogpt_platform/backend/backend/blocks/io.py index 0617e8a321..256dd2777a 100644 --- a/autogpt_platform/backend/backend/blocks/io.py +++ b/autogpt_platform/backend/backend/blocks/io.py @@ -469,13 +469,12 @@ class AgentFileInputBlock(AgentInputBlock): if not input_data.value: return - # Determine return format based on context and user preference - if execution_context.workspace_id and input_data.base_64: - return_format = "workspace_ref" - elif input_data.base_64: - return_format = "data_uri" - else: - return_format = "local_path" + # Determine return format based on user preference + # for_block_output: returns workspace:// if available, else data URI + # for_local_processing: returns local file path + return_format = ( + "for_block_output" if input_data.base_64 else "for_local_processing" + ) yield "result", await store_media_file( file=input_data.value, diff --git a/autogpt_platform/backend/backend/blocks/media.py b/autogpt_platform/backend/backend/blocks/media.py index 3499cfb99e..f7f53b0a61 100644 --- a/autogpt_platform/backend/backend/blocks/media.py +++ b/autogpt_platform/backend/backend/blocks/media.py @@ -54,7 +54,7 @@ class MediaDurationBlock(Block): local_media_path = await store_media_file( file=input_data.media_in, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) assert execution_context.graph_exec_id is not None media_abspath = get_exec_file_path( @@ -125,7 +125,7 @@ class LoopVideoBlock(Block): local_video_path = await store_media_file( file=input_data.video_in, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) input_abspath = get_exec_file_path(graph_exec_id, local_video_path) @@ -153,13 +153,11 @@ class LoopVideoBlock(Block): looped_clip = looped_clip.with_audio(clip.audio) looped_clip.write_videofile(output_abspath, codec="libx264", audio_codec="aac") - # Return output - use workspace_ref for persistence, fallback to data_uri + # Return output - for_block_output returns workspace:// if available, else data URI video_out = await store_media_file( file=output_filename, execution_context=execution_context, - return_format="workspace_ref" if execution_context.workspace_id else ( - "data_uri" if input_data.output_return_type == "data_uri" else "local_path" - ), + return_format="for_block_output", ) yield "video_out", video_out @@ -217,12 +215,12 @@ class AddAudioToVideoBlock(Block): local_video_path = await store_media_file( file=input_data.video_in, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) local_audio_path = await store_media_file( file=input_data.audio_in, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) abs_temp_dir = os.path.join(tempfile.gettempdir(), "exec_file", graph_exec_id) @@ -246,13 +244,11 @@ class AddAudioToVideoBlock(Block): output_abspath = os.path.join(abs_temp_dir, output_filename) final_clip.write_videofile(output_abspath, codec="libx264", audio_codec="aac") - # 5) Return output - use workspace_ref for persistence, fallback to data_uri + # 5) Return output - for_block_output returns workspace:// if available, else data URI video_out = await store_media_file( file=output_filename, execution_context=execution_context, - return_format="workspace_ref" if execution_context.workspace_id else ( - "data_uri" if input_data.output_return_type == "data_uri" else "local_path" - ), + return_format="for_block_output", ) yield "video_out", video_out diff --git a/autogpt_platform/backend/backend/blocks/screenshotone.py b/autogpt_platform/backend/backend/blocks/screenshotone.py index aedc09d9dd..ee998f8da2 100644 --- a/autogpt_platform/backend/backend/blocks/screenshotone.py +++ b/autogpt_platform/backend/backend/blocks/screenshotone.py @@ -159,7 +159,7 @@ class ScreenshotWebPageBlock(Block): f"data:image/{format.value};base64,{b64encode(content).decode('utf-8')}" ), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) } diff --git a/autogpt_platform/backend/backend/blocks/spreadsheet.py b/autogpt_platform/backend/backend/blocks/spreadsheet.py index affaf7b220..6388172561 100644 --- a/autogpt_platform/backend/backend/blocks/spreadsheet.py +++ b/autogpt_platform/backend/backend/blocks/spreadsheet.py @@ -109,7 +109,7 @@ class ReadSpreadsheetBlock(Block): stored_file_path = await store_media_file( file=input_data.file_input, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) # Get full file path diff --git a/autogpt_platform/backend/backend/blocks/talking_head.py b/autogpt_platform/backend/backend/blocks/talking_head.py index 1a7fc864af..a38710f048 100644 --- a/autogpt_platform/backend/backend/blocks/talking_head.py +++ b/autogpt_platform/backend/backend/blocks/talking_head.py @@ -178,7 +178,7 @@ class CreateTalkingAvatarVideoBlock(Block): stored_url = await store_media_file( file=MediaFileType(video_url), execution_context=execution_context, - return_format="workspace_ref", + return_format="for_block_output", ) yield "video_url", stored_url return diff --git a/autogpt_platform/backend/backend/blocks/text.py b/autogpt_platform/backend/backend/blocks/text.py index 1b5770b242..fa121a07e7 100644 --- a/autogpt_platform/backend/backend/blocks/text.py +++ b/autogpt_platform/backend/backend/blocks/text.py @@ -451,7 +451,7 @@ class FileReadBlock(Block): stored_file_path = await store_media_file( file=input_data.file_input, execution_context=execution_context, - return_format="local_path", + return_format="for_local_processing", ) # Get full file path diff --git a/autogpt_platform/backend/backend/util/file.py b/autogpt_platform/backend/backend/util/file.py index d6b310fa49..fad8e8cc9a 100644 --- a/autogpt_platform/backend/backend/util/file.py +++ b/autogpt_platform/backend/backend/util/file.py @@ -9,9 +9,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Literal from urllib.parse import urlparse -# Return format options for store_media_file -MediaReturnFormat = Literal["local_path", "data_uri", "workspace_ref"] - from prisma.enums import WorkspaceFileSource from backend.util.cloud_storage import get_cloud_storage_handler @@ -23,6 +20,14 @@ from backend.util.workspace import WorkspaceManager if TYPE_CHECKING: from backend.data.execution import ExecutionContext +# Return format options for store_media_file +# - "for_local_processing": Returns local file path - use with ffmpeg, MoviePy, PIL, etc. +# - "for_external_api": Returns data URI (base64) - use when sending content to external APIs +# - "for_block_output": Returns best format for output - workspace:// in CoPilot, data URI in graphs +MediaReturnFormat = Literal[ + "for_local_processing", "for_external_api", "for_block_output" +] + TEMP_DIR = Path(tempfile.gettempdir()).resolve() # Maximum filename length (conservative limit for most filesystems) @@ -98,13 +103,13 @@ async def store_media_file( - Local path: verify it exists in exec_file directory Return format options: - - "local_path": Return relative path in exec_file dir (for local processing) - - "data_uri": Return base64 data URI (for external APIs) - - "workspace_ref": Save to workspace, return workspace://id (for CoPilot outputs) + - "for_local_processing": Returns local file path - use with ffmpeg, MoviePy, PIL, etc. + - "for_external_api": Returns data URI (base64) - use when sending to external APIs + - "for_block_output": Returns best format for output - workspace:// in CoPilot, data URI in graphs :param file: Data URI, URL, workspace://, or local (relative) path. :param execution_context: ExecutionContext with user_id, graph_exec_id, workspace_id. - :param return_format: What to return: "local_path", "data_uri", or "workspace_ref". + :param return_format: What to return: "for_local_processing", "for_external_api", or "for_block_output". :param return_content: DEPRECATED. Use return_format instead. :param save_to_workspace: DEPRECATED. Use return_format instead. :return: The requested result based on return_format. @@ -114,20 +119,22 @@ async def store_media_file( if return_content is not None or save_to_workspace is not None: warnings.warn( "return_content and save_to_workspace are deprecated. " - "Use return_format='local_path', 'data_uri', or 'workspace_ref' instead.", + "Use return_format='for_local_processing', 'for_external_api', or 'for_block_output' instead.", DeprecationWarning, stacklevel=2, ) # Map old parameters to new return_format - if return_content is False or (return_content is None and save_to_workspace is None): - # Default or explicit return_content=False -> local_path - return_format = "local_path" + if return_content is False or ( + return_content is None and save_to_workspace is None + ): + # Default or explicit return_content=False -> for_local_processing + return_format = "for_local_processing" elif save_to_workspace is False: - # return_content=True, save_to_workspace=False -> data_uri - return_format = "data_uri" + # return_content=True, save_to_workspace=False -> for_external_api + return_format = "for_external_api" else: - # return_content=True, save_to_workspace=True (or default) -> workspace_ref - return_format = "workspace_ref" + # return_content=True, save_to_workspace=True (or default) -> for_block_output + return_format = "for_block_output" # Extract values from execution_context graph_exec_id = execution_context.graph_exec_id user_id = execution_context.user_id @@ -325,21 +332,23 @@ async def store_media_file( raise ValueError(f"Local file does not exist: {target_path}") # Return based on requested format - if return_format == "local_path": - # For local file processing (MoviePy, ffmpeg, etc.) + if return_format == "for_local_processing": + # Use when processing files locally with tools like ffmpeg, MoviePy, PIL + # Returns: relative path in exec_file directory (e.g., "image.png") return MediaFileType(_strip_base_prefix(target_path, base_path)) - elif return_format == "data_uri": - # For external APIs that need base64 content + elif return_format == "for_external_api": + # Use when sending content to external APIs that need base64 + # Returns: data URI (e.g., "data:image/png;base64,iVBORw0...") return MediaFileType(_file_to_data_uri(target_path)) - elif return_format == "workspace_ref": - # For persisting outputs to workspace (CoPilot) + elif return_format == "for_block_output": + # Use when returning output from a block to user/next block + # Returns: workspace:// ref (CoPilot) or data URI (graph execution) if workspace_manager is None: - raise ValueError( - "return_format='workspace_ref' requires workspace context. " - "Ensure execution_context has workspace_id set." - ) + # No workspace available (graph execution without CoPilot) + # Fallback to data URI so the content can still be used/displayed + return MediaFileType(_file_to_data_uri(target_path)) # Don't re-save if input was already from workspace if is_from_workspace: diff --git a/autogpt_platform/backend/backend/util/file_test.py b/autogpt_platform/backend/backend/util/file_test.py index 2565734308..d75a786c76 100644 --- a/autogpt_platform/backend/backend/util/file_test.py +++ b/autogpt_platform/backend/backend/util/file_test.py @@ -84,7 +84,7 @@ class TestFileCloudIntegration: result = await store_media_file( file=MediaFileType(cloud_path), execution_context=make_test_context(graph_exec_id=graph_exec_id), - return_format="local_path", + return_format="for_local_processing", ) # Verify cloud storage operations @@ -157,7 +157,7 @@ class TestFileCloudIntegration: result = await store_media_file( file=MediaFileType(cloud_path), execution_context=make_test_context(graph_exec_id=graph_exec_id), - return_format="data_uri", + return_format="for_external_api", ) # Verify result is a data URI @@ -210,7 +210,7 @@ class TestFileCloudIntegration: await store_media_file( file=MediaFileType(data_uri), execution_context=make_test_context(graph_exec_id=graph_exec_id), - return_format="local_path", + return_format="for_local_processing", ) # Verify cloud handler was checked but not used for retrieval diff --git a/docs/platform/block-sdk-guide.md b/docs/platform/block-sdk-guide.md index 5b3eda5184..42fd883251 100644 --- a/docs/platform/block-sdk-guide.md +++ b/docs/platform/block-sdk-guide.md @@ -277,6 +277,50 @@ async def run( token = credentials.api_key.get_secret_value() ``` +### Handling Files + +When your block works with files (images, videos, documents), use `store_media_file()`: + +```python +from backend.data.execution import ExecutionContext +from backend.util.file import store_media_file +from backend.util.type import MediaFileType + +async def run( + self, + input_data: Input, + *, + execution_context: ExecutionContext, + **kwargs, +): + # PROCESSING: Need local file path for tools like ffmpeg, MoviePy, PIL + local_path = await store_media_file( + file=input_data.video, + execution_context=execution_context, + return_format="for_local_processing", + ) + + # EXTERNAL API: Need base64 content for APIs like Replicate, OpenAI + image_b64 = await store_media_file( + file=input_data.image, + execution_context=execution_context, + return_format="for_external_api", + ) + + # OUTPUT: Return to user/next block (auto-adapts to context) + result = await store_media_file( + file=generated_url, + execution_context=execution_context, + return_format="for_block_output", # workspace:// in CoPilot, data URI in graphs + ) + yield "image_url", result +``` + +**Return format options:** +- `"for_local_processing"` - Local file path for processing tools +- `"for_external_api"` - Data URI for external APIs needing base64 +- `"for_block_output"` - **Always use for outputs** - automatically picks best format + ## Testing Your Block ```bash diff --git a/docs/platform/new_blocks.md b/docs/platform/new_blocks.md index d9d329ff51..114ff8d9a4 100644 --- a/docs/platform/new_blocks.md +++ b/docs/platform/new_blocks.md @@ -111,6 +111,71 @@ Follow these steps to create and test a new block: - `graph_exec_id`: The ID of the execution of the agent. This changes every time the agent has a new "run" - `node_exec_id`: The ID of the execution of the node. This changes every time the node is executed - `node_id`: The ID of the node that is being executed. It changes every version of the graph, but not every time the node is executed. + - `execution_context`: An `ExecutionContext` object containing user_id, graph_exec_id, workspace_id, and session_id. Required for file handling. + +### Handling Files in Blocks + +When your block needs to work with files (images, videos, documents), use `store_media_file()` from `backend.util.file`. This function handles downloading, validation, virus scanning, and storage. + +**Import:** +```python +from backend.data.execution import ExecutionContext +from backend.util.file import store_media_file +from backend.util.type import MediaFileType +``` + +**The `return_format` parameter determines what you get back:** + +| Format | Use When | Returns | +|--------|----------|---------| +| `"for_local_processing"` | Processing with local tools (ffmpeg, MoviePy, PIL) | Local file path (e.g., `"image.png"`) | +| `"for_external_api"` | Sending content to external APIs (Replicate, OpenAI) | Data URI (e.g., `"data:image/png;base64,..."`) | +| `"for_block_output"` | Returning output from your block | Smart: `workspace://` in CoPilot, data URI in graphs | + +**Examples:** + +```python +async def run( + self, + input_data: Input, + *, + execution_context: ExecutionContext, + **kwargs, +) -> BlockOutput: + # PROCESSING: Need to work with file locally (ffmpeg, MoviePy, PIL) + local_path = await store_media_file( + file=input_data.video, + execution_context=execution_context, + return_format="for_local_processing", + ) + # local_path = "video.mp4" - use with Path, ffmpeg, subprocess, etc. + full_path = get_exec_file_path(execution_context.graph_exec_id, local_path) + + # EXTERNAL API: Need to send content to an API like Replicate + image_b64 = await store_media_file( + file=input_data.image, + execution_context=execution_context, + return_format="for_external_api", + ) + # image_b64 = "data:image/png;base64,iVBORw0..." - send to external API + + # OUTPUT: Returning result from block to user/next block + result_url = await store_media_file( + file=generated_image_url, + execution_context=execution_context, + return_format="for_block_output", + ) + yield "image_url", result_url + # In CoPilot: result_url = "workspace://abc123" (persistent, context-efficient) + # In graphs: result_url = "data:image/png;base64,..." (for next block/display) +``` + +**Key points:** + +- `for_block_output` is the **only** format that auto-adapts to execution context +- Always use `for_block_output` for block outputs unless you have a specific reason not to +- Never manually check for `workspace_id` - let `for_block_output` handle the logic +- The function handles URLs, data URIs, `workspace://` references, and local paths as input ### Field Types