github/AutoGPT
1
1
Fork 0
mirror of https://github.com/Significant-Gravitas/AutoGPT.git synced 2026-04-08 03:00:28 -04:00
Code Issues Packages Projects Releases Wiki Activity

merge: Merge origin/dev into swiftyos/secrt-1565-chat-based-discovery

Browse Source
This commit is contained in:
Swifty
2025-10-15 15:45:52 +02:00
parent a53545abbc 9bc9b53b99
commit bc54ccdb8e
197 changed files with 8828 additions and 2469 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
autogpt_platform/Makefile Normal file
View File

@@ -0,0 +1,47 @@
.PHONY: start-core stop-core logs-core format lint migrate run-backend run-frontend
# Run just Supabase + Redis + RabbitMQ
start-core:
docker compose up -d deps
# Stop core services
stop-core:
docker compose stop deps
# View logs for core services
logs-core:
docker compose logs -f deps
# Run formatting and linting for backend and frontend
format:
cd backend && poetry run format
cd frontend && pnpm format
cd frontend && pnpm lint
init-env:
cp -n .env.default .env || true
cd backend && cp -n .env.default .env || true
cd frontend && cp -n .env.default .env || true
# Run migrations for backend
migrate:
cd backend && poetry run prisma migrate deploy
cd backend && poetry run prisma generate
run-backend:
cd backend && poetry run app
run-frontend:
cd frontend && pnpm dev
help:
@echo "Usage: make <target>"
@echo "Targets:"
@echo " start-core - Start just the core services (Supabase, Redis, RabbitMQ) in background"
@echo " stop-core - Stop the core services"
@echo " logs-core - Tail the logs for core services"
@echo " format - Format & lint backend (Python) and frontend (TypeScript) code"
@echo " migrate - Run backend database migrations"
@echo " run-backend - Run the backend FastAPI server"
@echo " run-frontend - Run the frontend Next.js development server"

31
autogpt_platform/README.md
View File

@@ -38,6 +38,37 @@ To run the AutoGPT Platform, follow these steps:
4. After all the services are in ready state, open your browser and navigate to `http://localhost:3000` to access the AutoGPT Platform frontend.
### Running Just Core services
You can now run the following to enable just the core services.
```
# For help
make help
# Run just Supabase + Redis + RabbitMQ
make start-core
# Stop core services
make stop-core
# View logs from core services
make logs-core
# Run formatting and linting for backend and frontend
make format
# Run migrations for backend database
make migrate
# Run backend server
make run-backend
# Run frontend development server
make run-frontend
```
### Docker Compose Commands
Here are some useful Docker Compose commands for managing your AutoGPT Platform:

28
autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py
View File

@@ -4,6 +4,7 @@ import logging
import os
import socket
import sys
from logging.handlers import RotatingFileHandler
from pathlib import Path
from pydantic import Field, field_validator
@@ -139,8 +140,13 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
print(f"Log directory: {config.log_dir}")
# Activity log handler (INFO and above)
activity_log_handler = logging.FileHandler(
config.log_dir / LOG_FILE, "a", "utf-8"
# Security fix: Use RotatingFileHandler with size limits to prevent disk exhaustion
activity_log_handler = RotatingFileHandler(
config.log_dir / LOG_FILE,
mode="a",
encoding="utf-8",
maxBytes=10 * 1024 * 1024, # 10MB per file
backupCount=3, # Keep 3 backup files (40MB total)
)
activity_log_handler.setLevel(config.level)
activity_log_handler.setFormatter(
@@ -150,8 +156,13 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
if config.level == logging.DEBUG:
# Debug log handler (all levels)
debug_log_handler = logging.FileHandler(
config.log_dir / DEBUG_LOG_FILE, "a", "utf-8"
# Security fix: Use RotatingFileHandler with size limits
debug_log_handler = RotatingFileHandler(
config.log_dir / DEBUG_LOG_FILE,
mode="a",
encoding="utf-8",
maxBytes=10 * 1024 * 1024, # 10MB per file
backupCount=3, # Keep 3 backup files (40MB total)
)
debug_log_handler.setLevel(logging.DEBUG)
debug_log_handler.setFormatter(
@@ -160,8 +171,13 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
log_handlers.append(debug_log_handler)
# Error log handler (ERROR and above)
error_log_handler = logging.FileHandler(
config.log_dir / ERROR_LOG_FILE, "a", "utf-8"
# Security fix: Use RotatingFileHandler with size limits
error_log_handler = RotatingFileHandler(
config.log_dir / ERROR_LOG_FILE,
mode="a",
encoding="utf-8",
maxBytes=10 * 1024 * 1024, # 10MB per file
backupCount=3, # Keep 3 backup files (40MB total)
)
error_log_handler.setLevel(logging.ERROR)
error_log_handler.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT, no_color=True))

377
autogpt_platform/backend/backend/blocks/code_executor.py
View File

@@ -1,7 +1,8 @@
from enum import Enum
from typing import Literal, Optional
from typing import Any, Literal, Optional
from e2b_code_interpreter import AsyncSandbox
from e2b_code_interpreter import Result as E2BExecutionResult
from e2b_code_interpreter.charts import Chart as E2BExecutionResultChart
from pydantic import BaseModel, JsonValue, SecretStr
@@ -37,7 +38,7 @@ class ProgrammingLanguage(Enum):
JAVA = "java"
class CodeExecutionResult(BaseModel):
class MainCodeExecutionResult(BaseModel):
"""
*Pydantic model mirroring `e2b_code_interpreter.Result`*
@@ -47,7 +48,7 @@ class CodeExecutionResult(BaseModel):
The result can contain multiple types of data, such as text, images, plots, etc. Each type of data is represented
as a string, and the result can contain multiple types of data. The display calls don't have to have text representation,
for the actual result the representation is always present for the result, the other representations are always optional.
"""
""" # noqa
class Chart(BaseModel, E2BExecutionResultChart):
pass
@@ -68,14 +69,104 @@ class CodeExecutionResult(BaseModel):
"""Extra data that can be included. Not part of the standard types."""
class CodeExecutionBlock(Block):
class CodeExecutionResult(MainCodeExecutionResult):
__doc__ = MainCodeExecutionResult.__doc__
is_main_result: bool = False
"""Whether this data is the main result of the cell. Data can be produced by display calls of which can be multiple in a cell.""" # noqa
class BaseE2BExecutorMixin:
"""Shared implementation methods for E2B executor blocks."""
async def execute_code(
self,
api_key: str,
code: str,
language: ProgrammingLanguage,
template_id: str = "",
setup_commands: Optional[list[str]] = None,
timeout: Optional[int] = None,
sandbox_id: Optional[str] = None,
dispose_sandbox: bool = False,
):
"""
Unified code execution method that handles all three use cases:
1. Create new sandbox and execute (ExecuteCodeBlock)
2. Create new sandbox, execute, and return sandbox_id (InstantiateCodeSandboxBlock)
3. Connect to existing sandbox and execute (ExecuteCodeStepBlock)
""" # noqa
sandbox = None
try:
if sandbox_id:
# Connect to existing sandbox (ExecuteCodeStepBlock case)
sandbox = await AsyncSandbox.connect(
sandbox_id=sandbox_id, api_key=api_key
)
else:
# Create new sandbox (ExecuteCodeBlock/InstantiateCodeSandboxBlock case)
sandbox = await AsyncSandbox.create(
api_key=api_key, template=template_id, timeout=timeout
)
if setup_commands:
for cmd in setup_commands:
await sandbox.commands.run(cmd)
# Execute the code
execution = await sandbox.run_code(
code,
language=language.value,
on_error=lambda e: sandbox.kill(), # Kill the sandbox on error
)
if execution.error:
raise Exception(execution.error)
results = execution.results
text_output = execution.text
stdout_logs = "".join(execution.logs.stdout)
stderr_logs = "".join(execution.logs.stderr)
return results, text_output, stdout_logs, stderr_logs, sandbox.sandbox_id
finally:
# Dispose of sandbox if requested to reduce usage costs
if dispose_sandbox and sandbox:
await sandbox.kill()
def process_execution_results(
self, results: list[E2BExecutionResult]
) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
"""Process and filter execution results."""
# Filter out empty formats and convert to dicts
processed_results = [
{
f: value
for f in [*r.formats(), "extra", "is_main_result"]
if (value := getattr(r, f, None)) is not None
}
for r in results
]
if main_result := next(
(r for r in processed_results if r.get("is_main_result")), None
):
# Make main_result a copy we can modify & remove is_main_result
(main_result := {**main_result}).pop("is_main_result")
return main_result, processed_results
class ExecuteCodeBlock(Block, BaseE2BExecutorMixin):
# TODO : Add support to upload and download files
# Currently, You can customized the CPU and Memory, only by creating a pre customized sandbox template
# NOTE: Currently, you can only customize the CPU and Memory
# by creating a pre customized sandbox template
class Input(BlockSchema):
credentials: CredentialsMetaInput[
Literal[ProviderName.E2B], Literal["api_key"]
] = CredentialsField(
description="Enter your api key for the E2B Sandbox. You can get it in here - https://e2b.dev/docs",
description=(
"Enter your API key for the E2B platform. "
"You can get it in here - https://e2b.dev/docs"
),
)
# Todo : Option to run commond in background
@@ -108,6 +199,14 @@ class CodeExecutionBlock(Block):
description="Execution timeout in seconds", default=300
)
dispose_sandbox: bool = SchemaField(
description=(
"Whether to dispose of the sandbox immediately after execution. "
"If disabled, the sandbox will run until its timeout expires."
),
default=True,
)
template_id: str = SchemaField(
description=(
"You can use an E2B sandbox template by entering its ID here. "
@@ -119,7 +218,7 @@ class CodeExecutionBlock(Block):
)
class Output(BlockSchema):
main_result: CodeExecutionResult = SchemaField(
main_result: MainCodeExecutionResult = SchemaField(
title="Main Result", description="The main result from the code execution"
)
results: list[CodeExecutionResult] = SchemaField(
@@ -138,10 +237,10 @@ class CodeExecutionBlock(Block):
def __init__(self):
super().__init__(
id="0b02b072-abe7-11ef-8372-fb5d162dd712",
description="Executes code in an isolated sandbox environment with internet access.",
description="Executes code in a sandbox environment with internet access.",
categories={BlockCategory.DEVELOPER_TOOLS},
input_schema=CodeExecutionBlock.Input,
output_schema=CodeExecutionBlock.Output,
input_schema=ExecuteCodeBlock.Input,
output_schema=ExecuteCodeBlock.Output,
test_credentials=TEST_CREDENTIALS,
test_input={
"credentials": TEST_CREDENTIALS_INPUT,
@@ -157,102 +256,54 @@ class CodeExecutionBlock(Block):
("stdout_logs", "Hello World\n"),
],
test_mock={
"execute_code": lambda code, language, setup_commands, timeout, api_key, template_id: (
"execute_code": lambda api_key, code, language, template_id, setup_commands, timeout, dispose_sandbox: ( # noqa
[], # results
"Hello World", # text_output
"Hello World\n", # stdout_logs
"", # stderr_logs
"sandbox_id", # sandbox_id
),
},
)
async def execute_code(
self,
code: str,
language: ProgrammingLanguage,
setup_commands: list[str],
timeout: int,
api_key: str,
template_id: str,
):
try:
sandbox = None
if template_id:
sandbox = await AsyncSandbox.create(
template=template_id, api_key=api_key, timeout=timeout
)
else:
sandbox = await AsyncSandbox.create(api_key=api_key, timeout=timeout)
if not sandbox:
raise Exception("Sandbox not created")
# Running setup commands
for cmd in setup_commands:
await sandbox.commands.run(cmd)
# Executing the code
execution = await sandbox.run_code(
code,
language=language.value,
on_error=lambda e: sandbox.kill(), # Kill the sandbox if there is an error
)
if execution.error:
raise Exception(execution.error)
results = execution.results
text_output = execution.text
stdout_logs = "".join(execution.logs.stdout)
stderr_logs = "".join(execution.logs.stderr)
return results, text_output, stdout_logs, stderr_logs
except Exception as e:
raise e
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
try:
results, text_output, stdout_logs, stderr_logs = await self.execute_code(
input_data.code,
input_data.language,
input_data.setup_commands,
input_data.timeout,
credentials.api_key.get_secret_value(),
input_data.template_id,
results, text_output, stdout, stderr, _ = await self.execute_code(
api_key=credentials.api_key.get_secret_value(),
code=input_data.code,
language=input_data.language,
template_id=input_data.template_id,
setup_commands=input_data.setup_commands,
timeout=input_data.timeout,
dispose_sandbox=input_data.dispose_sandbox,
)
# Determine result object shape & filter out empty formats
results = [
{
f: r[f]
for f in [*r.formats(), "extra", "is_main_result"]
if getattr(r, f, None) is not None
}
for r in results
]
main_result, results = self.process_execution_results(results)
if main_result:
yield "main_result", main_result
yield "results", results
for r in results:
if r.pop("is_main_result", False):
yield "main_result", r
if text_output:
yield "response", text_output
if stdout_logs:
yield "stdout_logs", stdout_logs
if stderr_logs:
yield "stderr_logs", stderr_logs
if stdout:
yield "stdout_logs", stdout
if stderr:
yield "stderr_logs", stderr
except Exception as e:
yield "error", str(e)
class InstantiationBlock(Block):
class InstantiateCodeSandboxBlock(Block, BaseE2BExecutorMixin):
class Input(BlockSchema):
credentials: CredentialsMetaInput[
Literal[ProviderName.E2B], Literal["api_key"]
] = CredentialsField(
description="Enter your api key for the E2B Sandbox. You can get it in here - https://e2b.dev/docs",
description=(
"Enter your API key for the E2B platform. "
"You can get it in here - https://e2b.dev/docs"
)
)
# Todo : Option to run commond in background
@@ -310,10 +361,13 @@ class InstantiationBlock(Block):
def __init__(self):
super().__init__(
id="ff0861c9-1726-4aec-9e5b-bf53f3622112",
description="Instantiate an isolated sandbox environment with internet access where to execute code in.",
description=(
"Instantiate a sandbox environment with internet access "
"in which you can execute code with the Execute Code Step block."
),
categories={BlockCategory.DEVELOPER_TOOLS},
input_schema=InstantiationBlock.Input,
output_schema=InstantiationBlock.Output,
input_schema=InstantiateCodeSandboxBlock.Input,
output_schema=InstantiateCodeSandboxBlock.Output,
test_credentials=TEST_CREDENTIALS,
test_input={
"credentials": TEST_CREDENTIALS_INPUT,
@@ -329,11 +383,12 @@ class InstantiationBlock(Block):
("stdout_logs", "Hello World\n"),
],
test_mock={
"execute_code": lambda setup_code, language, setup_commands, timeout, api_key, template_id: (
"sandbox_id", # sandbox_id
"execute_code": lambda api_key, code, language, template_id, setup_commands, timeout: ( # noqa
[], # results
"Hello World", # text_output
"Hello World\n", # stdout_logs
"", # stderr_logs
"sandbox_id", # sandbox_id
),
},
)
@@ -342,13 +397,13 @@ class InstantiationBlock(Block):
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
try:
sandbox_id, text_output, stdout_logs, stderr_logs = await self.execute_code(
input_data.setup_code,
input_data.language,
input_data.setup_commands,
input_data.timeout,
credentials.api_key.get_secret_value(),
input_data.template_id,
_, text_output, stdout, stderr, sandbox_id = await self.execute_code(
api_key=credentials.api_key.get_secret_value(),
code=input_data.setup_code,
language=input_data.language,
template_id=input_data.template_id,
setup_commands=input_data.setup_commands,
timeout=input_data.timeout,
)
if sandbox_id:
yield "sandbox_id", sandbox_id
@@ -357,64 +412,23 @@ class InstantiationBlock(Block):
if text_output:
yield "response", text_output
if stdout_logs:
yield "stdout_logs", stdout_logs
if stderr_logs:
yield "stderr_logs", stderr_logs
if stdout:
yield "stdout_logs", stdout
if stderr:
yield "stderr_logs", stderr
except Exception as e:
yield "error", str(e)
async def execute_code(
self,
code: str,
language: ProgrammingLanguage,
setup_commands: list[str],
timeout: int,
api_key: str,
template_id: str,
):
try:
sandbox = None
if template_id:
sandbox = await AsyncSandbox.create(
template=template_id, api_key=api_key, timeout=timeout
)
else:
sandbox = await AsyncSandbox.create(api_key=api_key, timeout=timeout)
if not sandbox:
raise Exception("Sandbox not created")
# Running setup commands
for cmd in setup_commands:
await sandbox.commands.run(cmd)
# Executing the code
execution = await sandbox.run_code(
code,
language=language.value,
on_error=lambda e: sandbox.kill(), # Kill the sandbox if there is an error
)
if execution.error:
raise Exception(execution.error)
text_output = execution.text
stdout_logs = "".join(execution.logs.stdout)
stderr_logs = "".join(execution.logs.stderr)
return sandbox.sandbox_id, text_output, stdout_logs, stderr_logs
except Exception as e:
raise e
class StepExecutionBlock(Block):
class ExecuteCodeStepBlock(Block, BaseE2BExecutorMixin):
class Input(BlockSchema):
credentials: CredentialsMetaInput[
Literal[ProviderName.E2B], Literal["api_key"]
] = CredentialsField(
description="Enter your api key for the E2B Sandbox. You can get it in here - https://e2b.dev/docs",
description=(
"Enter your API key for the E2B platform. "
"You can get it in here - https://e2b.dev/docs"
),
)
sandbox_id: str = SchemaField(
@@ -435,8 +449,13 @@ class StepExecutionBlock(Block):
advanced=False,
)
dispose_sandbox: bool = SchemaField(
description="Whether to dispose of the sandbox after executing this code.",
default=False,
)
class Output(BlockSchema):
main_result: CodeExecutionResult = SchemaField(
main_result: MainCodeExecutionResult = SchemaField(
title="Main Result", description="The main result from the code execution"
)
results: list[CodeExecutionResult] = SchemaField(
@@ -455,10 +474,10 @@ class StepExecutionBlock(Block):
def __init__(self):
super().__init__(
id="82b59b8e-ea10-4d57-9161-8b169b0adba6",
description="Execute code in a previously instantiated sandbox environment.",
description="Execute code in a previously instantiated sandbox.",
categories={BlockCategory.DEVELOPER_TOOLS},
input_schema=StepExecutionBlock.Input,
output_schema=StepExecutionBlock.Output,
input_schema=ExecuteCodeStepBlock.Input,
output_schema=ExecuteCodeStepBlock.Output,
test_credentials=TEST_CREDENTIALS,
test_input={
"credentials": TEST_CREDENTIALS_INPUT,
@@ -472,74 +491,38 @@ class StepExecutionBlock(Block):
("stdout_logs", "Hello World\n"),
],
test_mock={
"execute_step_code": lambda sandbox_id, step_code, language, api_key: (
"execute_code": lambda api_key, code, language, sandbox_id, dispose_sandbox: ( # noqa
[], # results
"Hello World", # text_output
"Hello World\n", # stdout_logs
"", # stderr_logs
sandbox_id, # sandbox_id
),
},
)
async def execute_step_code(
self,
sandbox_id: str,
code: str,
language: ProgrammingLanguage,
api_key: str,
):
try:
sandbox = await AsyncSandbox.connect(sandbox_id=sandbox_id, api_key=api_key)
if not sandbox:
raise Exception("Sandbox not found")
# Executing the code
execution = await sandbox.run_code(code, language=language.value)
if execution.error:
raise Exception(execution.error)
results = execution.results
text_output = execution.text
stdout_logs = "".join(execution.logs.stdout)
stderr_logs = "".join(execution.logs.stderr)
return results, text_output, stdout_logs, stderr_logs
except Exception as e:
raise e
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
try:
results, text_output, stdout_logs, stderr_logs = (
await self.execute_step_code(
input_data.sandbox_id,
input_data.step_code,
input_data.language,
credentials.api_key.get_secret_value(),
)
results, text_output, stdout, stderr, _ = await self.execute_code(
api_key=credentials.api_key.get_secret_value(),
code=input_data.step_code,
language=input_data.language,
sandbox_id=input_data.sandbox_id,
dispose_sandbox=input_data.dispose_sandbox,
)
# Determine result object shape & filter out empty formats
results = [
{
f: r[f]
for f in [*r.formats(), "extra", "is_main_result"]
if getattr(r, f, None) is not None
}
for r in results
]
main_result, results = self.process_execution_results(results)
if main_result:
yield "main_result", main_result
yield "results", results
for r in results:
if r.pop("is_main_result", False):
yield "main_result", r
if text_output:
yield "response", text_output
if stdout_logs:
yield "stdout_logs", stdout_logs
if stderr_logs:
yield "stderr_logs", stderr_logs
if stdout:
yield "stdout_logs", stdout
if stderr:
yield "stderr_logs", stderr
except Exception as e:
yield "error", str(e)

6
autogpt_platform/backend/backend/blocks/code_extraction_block.py
View File

@@ -90,7 +90,7 @@ class CodeExtractionBlock(Block):
for aliases in language_aliases.values()
for alias in aliases
)
+ r")\s+[\s\S]*?```"
+ r")[ \t]*\n[\s\S]*?```"
)
remaining_text = re.sub(pattern, "", input_data.text).strip()
@@ -103,7 +103,9 @@ class CodeExtractionBlock(Block):
# Escape special regex characters in the language string
language = re.escape(language)
# Extract all code blocks enclosed in ```language``` blocks
pattern = re.compile(rf"```{language}\s+(.*?)```", re.DOTALL | re.IGNORECASE)
pattern = re.compile(
rf"```{language}[ \t]*\n(.*?)\n```", re.DOTALL | re.IGNORECASE
)
matches = pattern.finditer(text)
# Combine all code blocks for this language with newlines between them
code_blocks = [match.group(1).strip() for match in matches]

66
autogpt_platform/backend/backend/blocks/discord/bot_blocks.py
View File

@@ -171,11 +171,11 @@ class SendDiscordMessageBlock(Block):
description="The content of the message to send"
)
channel_name: str = SchemaField(
description="The name of the channel the message will be sent to"
description="Channel ID or channel name to send the message to"
)
server_name: str = SchemaField(
description="The name of the server where the channel is located",
advanced=True, # Optional field for server name
description="Server name (only needed if using channel name)",
advanced=True,
default="",
)
@@ -231,25 +231,49 @@ class SendDiscordMessageBlock(Block):
@client.event
async def on_ready():
print(f"Logged in as {client.user}")
for guild in client.guilds:
if server_name and guild.name != server_name:
continue
for channel in guild.text_channels:
if channel.name == channel_name:
# Split message into chunks if it exceeds 2000 characters
chunks = self.chunk_message(message_content)
last_message = None
for chunk in chunks:
last_message = await channel.send(chunk)
result["status"] = "Message sent"
result["message_id"] = (
str(last_message.id) if last_message else ""
)
result["channel_id"] = str(channel.id)
await client.close()
return
channel = None
result["status"] = "Channel not found"
# Try to parse as channel ID first
try:
channel_id = int(channel_name)
channel = client.get_channel(channel_id)
except ValueError:
# Not a valid ID, will try name lookup
pass
# If not found by ID (or not an ID), try name lookup
if not channel:
for guild in client.guilds:
if server_name and guild.name != server_name:
continue
for ch in guild.text_channels:
if ch.name == channel_name:
channel = ch
break
if channel:
break
if not channel:
result["status"] = f"Channel not found: {channel_name}"
await client.close()
return
# Type check - ensure it's a text channel that can send messages
if not hasattr(channel, "send"):
result["status"] = (
f"Channel {channel_name} cannot receive messages (not a text channel)"
)
await client.close()
return
# Split message into chunks if it exceeds 2000 characters
chunks = self.chunk_message(message_content)
last_message = None
for chunk in chunks:
last_message = await channel.send(chunk) # type: ignore
result["status"] = "Message sent"
result["message_id"] = str(last_message.id) if last_message else ""
result["channel_id"] = str(channel.id)
await client.close()
await client.start(token)

12
autogpt_platform/backend/backend/blocks/firecrawl/_api.py
View File

@@ -0,0 +1,12 @@
from enum import Enum
class ScrapeFormat(Enum):
MARKDOWN = "markdown"
HTML = "html"
RAW_HTML = "rawHtml"
LINKS = "links"
SCREENSHOT = "screenshot"
SCREENSHOT_FULL_PAGE = "screenshot@fullPage"
JSON = "json"
CHANGE_TRACKING = "changeTracking"

28
autogpt_platform/backend/backend/blocks/firecrawl/_format_utils.py Normal file
View File

@@ -0,0 +1,28 @@
"""Utility functions for converting between our ScrapeFormat enum and firecrawl FormatOption types."""
from typing import List
from firecrawl.v2.types import FormatOption, ScreenshotFormat
from backend.blocks.firecrawl._api import ScrapeFormat
def convert_to_format_options(
formats: List[ScrapeFormat],
) -> List[FormatOption]:
"""Convert our ScrapeFormat enum values to firecrawl FormatOption types.
Handles special cases like screenshot@fullPage which needs to be converted
to a ScreenshotFormat object.
"""
result: List[FormatOption] = []
for format_enum in formats:
if format_enum.value == "screenshot@fullPage":
# Special case: convert to ScreenshotFormat with full_page=True
result.append(ScreenshotFormat(type="screenshot", full_page=True))
else:
# Regular string literals
result.append(format_enum.value)
return result

33
autogpt_platform/backend/backend/blocks/firecrawl/crawl.py
View File

@@ -1,8 +1,9 @@
from enum import Enum
from typing import Any
from firecrawl import FirecrawlApp, ScrapeOptions
from firecrawl import FirecrawlApp
from firecrawl.v2.types import ScrapeOptions
from backend.blocks.firecrawl._api import ScrapeFormat
from backend.sdk import (
APIKeyCredentials,
Block,
@@ -14,21 +15,10 @@ from backend.sdk import (
)
from ._config import firecrawl
class ScrapeFormat(Enum):
MARKDOWN = "markdown"
HTML = "html"
RAW_HTML = "rawHtml"
LINKS = "links"
SCREENSHOT = "screenshot"
SCREENSHOT_FULL_PAGE = "screenshot@fullPage"
JSON = "json"
CHANGE_TRACKING = "changeTracking"
from ._format_utils import convert_to_format_options
class FirecrawlCrawlBlock(Block):
class Input(BlockSchema):
credentials: CredentialsMetaInput = firecrawl.credentials_field()
url: str = SchemaField(description="The URL to crawl")
@@ -78,18 +68,17 @@ class FirecrawlCrawlBlock(Block):
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
# Sync call
crawl_result = app.crawl_url(
crawl_result = app.crawl(
input_data.url,
limit=input_data.limit,
scrape_options=ScrapeOptions(
formats=[format.value for format in input_data.formats],
onlyMainContent=input_data.only_main_content,
maxAge=input_data.max_age,
waitFor=input_data.wait_for,
formats=convert_to_format_options(input_data.formats),
only_main_content=input_data.only_main_content,
max_age=input_data.max_age,
wait_for=input_data.wait_for,
),
)
yield "data", crawl_result.data
@@ -101,7 +90,7 @@ class FirecrawlCrawlBlock(Block):
elif f == ScrapeFormat.HTML:
yield "html", data.html
elif f == ScrapeFormat.RAW_HTML:
yield "raw_html", data.rawHtml
yield "raw_html", data.raw_html
elif f == ScrapeFormat.LINKS:
yield "links", data.links
elif f == ScrapeFormat.SCREENSHOT:
@@ -109,6 +98,6 @@ class FirecrawlCrawlBlock(Block):
elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:
yield "screenshot_full_page", data.screenshot
elif f == ScrapeFormat.CHANGE_TRACKING:
yield "change_tracking", data.changeTracking
yield "change_tracking", data.change_tracking
elif f == ScrapeFormat.JSON:
yield "json", data.json

2
autogpt_platform/backend/backend/blocks/firecrawl/extract.py
View File

@@ -20,7 +20,6 @@ from ._config import firecrawl
@cost(BlockCost(2, BlockCostType.RUN))
class FirecrawlExtractBlock(Block):
class Input(BlockSchema):
credentials: CredentialsMetaInput = firecrawl.credentials_field()
urls: list[str] = SchemaField(
@@ -53,7 +52,6 @@ class FirecrawlExtractBlock(Block):
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
extract_result = app.extract(

24
autogpt_platform/backend/backend/blocks/firecrawl/map.py
View File

@@ -1,3 +1,5 @@
from typing import Any
from firecrawl import FirecrawlApp
from backend.sdk import (
@@ -14,14 +16,16 @@ from ._config import firecrawl
class FirecrawlMapWebsiteBlock(Block):
class Input(BlockSchema):
credentials: CredentialsMetaInput = firecrawl.credentials_field()
url: str = SchemaField(description="The website url to map")
class Output(BlockSchema):
links: list[str] = SchemaField(description="The links of the website")
links: list[str] = SchemaField(description="List of URLs found on the website")
results: list[dict[str, Any]] = SchemaField(
description="List of search results with url, title, and description"
)
def __init__(self):
super().__init__(
@@ -35,12 +39,22 @@ class FirecrawlMapWebsiteBlock(Block):
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
# Sync call
map_result = app.map_url(
map_result = app.map(
url=input_data.url,
)
yield "links", map_result.links
# Convert SearchResult objects to dicts
results_data = [
{
"url": link.url,
"title": link.title,
"description": link.description,
}
for link in map_result.links
]
yield "links", [link.url for link in map_result.links]
yield "results", results_data

24
autogpt_platform/backend/backend/blocks/firecrawl/scrape.py
View File

@@ -1,8 +1,8 @@
from enum import Enum
from typing import Any
from firecrawl import FirecrawlApp
from backend.blocks.firecrawl._api import ScrapeFormat
from backend.sdk import (
APIKeyCredentials,
Block,
@@ -14,21 +14,10 @@ from backend.sdk import (
)
from ._config import firecrawl
class ScrapeFormat(Enum):
MARKDOWN = "markdown"
HTML = "html"
RAW_HTML = "rawHtml"
LINKS = "links"
SCREENSHOT = "screenshot"
SCREENSHOT_FULL_PAGE = "screenshot@fullPage"
JSON = "json"
CHANGE_TRACKING = "changeTracking"
from ._format_utils import convert_to_format_options
class FirecrawlScrapeBlock(Block):
class Input(BlockSchema):
credentials: CredentialsMetaInput = firecrawl.credentials_field()
url: str = SchemaField(description="The URL to crawl")
@@ -78,12 +67,11 @@ class FirecrawlScrapeBlock(Block):
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
scrape_result = app.scrape_url(
scrape_result = app.scrape(
input_data.url,
formats=[format.value for format in input_data.formats],
formats=convert_to_format_options(input_data.formats),
only_main_content=input_data.only_main_content,
max_age=input_data.max_age,
wait_for=input_data.wait_for,
@@ -96,7 +84,7 @@ class FirecrawlScrapeBlock(Block):
elif f == ScrapeFormat.HTML:
yield "html", scrape_result.html
elif f == ScrapeFormat.RAW_HTML:
yield "raw_html", scrape_result.rawHtml
yield "raw_html", scrape_result.raw_html
elif f == ScrapeFormat.LINKS:
yield "links", scrape_result.links
elif f == ScrapeFormat.SCREENSHOT:
@@ -104,6 +92,6 @@ class FirecrawlScrapeBlock(Block):
elif f == ScrapeFormat.SCREENSHOT_FULL_PAGE:
yield "screenshot_full_page", scrape_result.screenshot
elif f == ScrapeFormat.CHANGE_TRACKING:
yield "change_tracking", scrape_result.changeTracking
yield "change_tracking", scrape_result.change_tracking
elif f == ScrapeFormat.JSON:
yield "json", scrape_result.json

30
autogpt_platform/backend/backend/blocks/firecrawl/search.py
View File

@@ -1,8 +1,9 @@
from enum import Enum
from typing import Any
from firecrawl import FirecrawlApp, ScrapeOptions
from firecrawl import FirecrawlApp
from firecrawl.v2.types import ScrapeOptions
from backend.blocks.firecrawl._api import ScrapeFormat
from backend.sdk import (
APIKeyCredentials,
Block,
@@ -14,21 +15,10 @@ from backend.sdk import (
)
from ._config import firecrawl
class ScrapeFormat(Enum):
MARKDOWN = "markdown"
HTML = "html"
RAW_HTML = "rawHtml"
LINKS = "links"
SCREENSHOT = "screenshot"
SCREENSHOT_FULL_PAGE = "screenshot@fullPage"
JSON = "json"
CHANGE_TRACKING = "changeTracking"
from ._format_utils import convert_to_format_options
class FirecrawlSearchBlock(Block):
class Input(BlockSchema):
credentials: CredentialsMetaInput = firecrawl.credentials_field()
query: str = SchemaField(description="The query to search for")
@@ -61,7 +51,6 @@ class FirecrawlSearchBlock(Block):
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
app = FirecrawlApp(api_key=credentials.api_key.get_secret_value())
# Sync call
@@ -69,11 +58,12 @@ class FirecrawlSearchBlock(Block):
input_data.query,
limit=input_data.limit,
scrape_options=ScrapeOptions(
formats=[format.value for format in input_data.formats],
maxAge=input_data.max_age,
waitFor=input_data.wait_for,
formats=convert_to_format_options(input_data.formats) or None,
max_age=input_data.max_age,
wait_for=input_data.wait_for,
),
)
yield "data", scrape_result
for site in scrape_result.data:
yield "site", site
if hasattr(scrape_result, "web") and scrape_result.web:
for site in scrape_result.web:
yield "site", site

29
autogpt_platform/backend/backend/blocks/iteration.py
View File

@@ -54,20 +54,43 @@ class StepThroughItemsBlock(Block):
)
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
# Security fix: Add limits to prevent DoS from large iterations
MAX_ITEMS = 10000 # Maximum items to iterate
MAX_ITEM_SIZE = 1024 * 1024 # 1MB per item
for data in [input_data.items, input_data.items_object, input_data.items_str]:
if not data:
continue
# Limit string size before parsing
if isinstance(data, str):
if len(data) > MAX_ITEM_SIZE:
raise ValueError(
f"Input too large: {len(data)} bytes > {MAX_ITEM_SIZE} bytes"
)
items = json.loads(data)
else:
items = data
# Check total item count
if isinstance(items, (list, dict)):
if len(items) > MAX_ITEMS:
raise ValueError(f"Too many items: {len(items)} > {MAX_ITEMS}")
iteration_count = 0
if isinstance(items, dict):
# If items is a dictionary, iterate over its values
for item in items.values():
yield "item", item
yield "key", item
for key, value in items.items():
if iteration_count >= MAX_ITEMS:
break
yield "item", value
yield "key", key # Fixed: should yield key, not item
iteration_count += 1
else:
# If items is a list, iterate over the list
for index, item in enumerate(items):
if iteration_count >= MAX_ITEMS:
break
yield "item", item
yield "key", index
iteration_count += 1

19
autogpt_platform/backend/backend/blocks/jina/fact_checker.py
View File

@@ -1,5 +1,8 @@
from typing import List
from urllib.parse import quote
from typing_extensions import TypedDict
from backend.blocks.jina._auth import (
JinaCredentials,
JinaCredentialsField,
@@ -10,6 +13,12 @@ from backend.data.model import SchemaField
from backend.util.request import Requests
class Reference(TypedDict):
url: str
keyQuote: str
isSupportive: bool
class FactCheckerBlock(Block):
class Input(BlockSchema):
statement: str = SchemaField(
@@ -23,6 +32,10 @@ class FactCheckerBlock(Block):
)
result: bool = SchemaField(description="The result of the factuality check")
reason: str = SchemaField(description="The reason for the factuality result")
references: List[Reference] = SchemaField(
description="List of references supporting or contradicting the statement",
default=[],
)
error: str = SchemaField(description="Error message if the check fails")
def __init__(self):
@@ -53,5 +66,11 @@ class FactCheckerBlock(Block):
yield "factuality", data["factuality"]
yield "result", data["result"]
yield "reason", data["reason"]
# Yield references if present in the response
if "references" in data:
yield "references", data["references"]
else:
yield "references", []
else:
raise RuntimeError(f"Expected 'data' key not found in response: {data}")

4
autogpt_platform/backend/backend/blocks/linear/models.py
View File

@@ -37,5 +37,5 @@ class Project(BaseModel):
name: str
description: str
priority: int
progress: int
content: str
progress: float
content: str | None

18
autogpt_platform/backend/backend/blocks/llm.py
View File

@@ -1404,11 +1404,27 @@ class AITextSummarizerBlock(AIBlockBase):
@staticmethod
def _split_text(text: str, max_tokens: int, overlap: int) -> list[str]:
# Security fix: Add validation to prevent DoS attacks
# Limit text size to prevent memory exhaustion
MAX_TEXT_LENGTH = 1_000_000 # 1MB character limit
MAX_CHUNKS = 100 # Maximum number of chunks to prevent excessive memory use
if len(text) > MAX_TEXT_LENGTH:
text = text[:MAX_TEXT_LENGTH]
# Ensure chunk_size is at least 1 to prevent infinite loops
chunk_size = max(1, max_tokens - overlap)
# Ensure overlap is less than max_tokens to prevent invalid configurations
if overlap >= max_tokens:
overlap = max(0, max_tokens - 1)
words = text.split()
chunks = []
chunk_size = max_tokens - overlap
for i in range(0, len(words), chunk_size):
if len(chunks) >= MAX_CHUNKS:
break # Limit the number of chunks to prevent memory exhaustion
chunk = " ".join(words[i : i + max_tokens])
chunks.append(chunk)

226
autogpt_platform/backend/backend/blocks/perplexity.py Normal file
View File

@@ -0,0 +1,226 @@
# flake8: noqa: E501
import logging
from enum import Enum
from typing import Any, Literal
import openai
from pydantic import SecretStr
from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
from backend.data.model import (
APIKeyCredentials,
CredentialsField,
CredentialsMetaInput,
NodeExecutionStats,
SchemaField,
)
from backend.integrations.providers import ProviderName
from backend.util.logging import TruncatedLogger
logger = TruncatedLogger(logging.getLogger(__name__), "[Perplexity-Block]")
class PerplexityModel(str, Enum):
"""Perplexity sonar models available via OpenRouter"""
SONAR = "perplexity/sonar"
SONAR_PRO = "perplexity/sonar-pro"
SONAR_DEEP_RESEARCH = "perplexity/sonar-deep-research"
PerplexityCredentials = CredentialsMetaInput[
Literal[ProviderName.OPEN_ROUTER], Literal["api_key"]
]
TEST_CREDENTIALS = APIKeyCredentials(
id="test-perplexity-creds",
provider="open_router",
api_key=SecretStr("mock-openrouter-api-key"),
title="Mock OpenRouter API key",
expires_at=None,
)
TEST_CREDENTIALS_INPUT = {
"provider": TEST_CREDENTIALS.provider,
"id": TEST_CREDENTIALS.id,
"type": TEST_CREDENTIALS.type,
"title": TEST_CREDENTIALS.title,
}
def PerplexityCredentialsField() -> PerplexityCredentials:
return CredentialsField(
description="OpenRouter API key for accessing Perplexity models.",
)
class PerplexityBlock(Block):
class Input(BlockSchema):
prompt: str = SchemaField(
description="The query to send to the Perplexity model.",
placeholder="Enter your query here...",
)
model: PerplexityModel = SchemaField(
title="Perplexity Model",
default=PerplexityModel.SONAR,
description="The Perplexity sonar model to use.",
advanced=False,
)
credentials: PerplexityCredentials = PerplexityCredentialsField()
system_prompt: str = SchemaField(
title="System Prompt",
default="",
description="Optional system prompt to provide context to the model.",
advanced=True,
)
max_tokens: int | None = SchemaField(
advanced=True,
default=None,
description="The maximum number of tokens to generate.",
)
class Output(BlockSchema):
response: str = SchemaField(
description="The response from the Perplexity model."
)
annotations: list[dict[str, Any]] = SchemaField(
description="List of URL citations and annotations from the response."
)
error: str = SchemaField(description="Error message if the API call failed.")
def __init__(self):
super().__init__(
id="c8a5f2e9-8b3d-4a7e-9f6c-1d5e3c9b7a4f",
description="Query Perplexity's sonar models with real-time web search capabilities and receive annotated responses with source citations.",
categories={BlockCategory.AI, BlockCategory.SEARCH},
input_schema=PerplexityBlock.Input,
output_schema=PerplexityBlock.Output,
test_input={
"prompt": "What is the weather today?",
"model": PerplexityModel.SONAR,
"credentials": TEST_CREDENTIALS_INPUT,
},
test_credentials=TEST_CREDENTIALS,
test_output=[
("response", "The weather varies by location..."),
("annotations", list),
],
test_mock={
"call_perplexity": lambda *args, **kwargs: {
"response": "The weather varies by location...",
"annotations": [
{
"type": "url_citation",
"url_citation": {
"title": "weather.com",
"url": "https://weather.com",
},
}
],
}
},
)
self.execution_stats = NodeExecutionStats()
async def call_perplexity(
self,
credentials: APIKeyCredentials,
model: PerplexityModel,
prompt: str,
system_prompt: str = "",
max_tokens: int | None = None,
) -> dict[str, Any]:
"""Call Perplexity via OpenRouter and extract annotations."""
client = openai.AsyncOpenAI(
base_url="https://openrouter.ai/api/v1",
api_key=credentials.api_key.get_secret_value(),
)
messages = []
if system_prompt:
messages.append({"role": "system", "content": system_prompt})
messages.append({"role": "user", "content": prompt})
try:
response = await client.chat.completions.create(
extra_headers={
"HTTP-Referer": "https://agpt.co",
"X-Title": "AutoGPT",
},
model=model.value,
messages=messages,
max_tokens=max_tokens,
)
if not response.choices:
raise ValueError("No response from Perplexity via OpenRouter.")
# Extract the response content
response_content = response.choices[0].message.content or ""
# Extract annotations if present in the message
annotations = []
if hasattr(response.choices[0].message, "annotations"):
# If annotations are directly available
annotations = response.choices[0].message.annotations
else:
# Check if there's a raw response with annotations
raw = getattr(response.choices[0].message, "_raw_response", None)
if isinstance(raw, dict) and "annotations" in raw:
annotations = raw["annotations"]
if not annotations and hasattr(response, "model_extra"):
# Check model_extra for annotations
model_extra = response.model_extra
if isinstance(model_extra, dict):
# Check in choices
if "choices" in model_extra and len(model_extra["choices"]) > 0:
choice = model_extra["choices"][0]
if "message" in choice and "annotations" in choice["message"]:
annotations = choice["message"]["annotations"]
# Also check the raw response object for annotations
if not annotations:
raw = getattr(response, "_raw_response", None)
if isinstance(raw, dict):
# Check various possible locations for annotations
if "annotations" in raw:
annotations = raw["annotations"]
elif "choices" in raw and len(raw["choices"]) > 0:
choice = raw["choices"][0]
if "message" in choice and "annotations" in choice["message"]:
annotations = choice["message"]["annotations"]
# Update execution stats
if response.usage:
self.execution_stats.input_token_count = response.usage.prompt_tokens
self.execution_stats.output_token_count = (
response.usage.completion_tokens
)
return {"response": response_content, "annotations": annotations or []}
except Exception as e:
logger.error(f"Error calling Perplexity: {e}")
raise
async def run(
self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
) -> BlockOutput:
logger.debug(f"Running Perplexity block with model: {input_data.model}")
try:
result = await self.call_perplexity(
credentials=credentials,
model=input_data.model,
prompt=input_data.prompt,
system_prompt=input_data.system_prompt,
max_tokens=input_data.max_tokens,
)
yield "response", result["response"]
yield "annotations", result["annotations"]
except Exception as e:
error_msg = f"Error calling Perplexity: {str(e)}"
logger.error(error_msg)
yield "error", error_msg

36
autogpt_platform/backend/backend/blocks/rss.py
View File

@@ -1,4 +1,7 @@
import asyncio
import logging
import urllib.parse
import urllib.request
from datetime import datetime, timedelta, timezone
from typing import Any
@@ -101,7 +104,38 @@ class ReadRSSFeedBlock(Block):
@staticmethod
def parse_feed(url: str) -> dict[str, Any]:
return feedparser.parse(url) # type: ignore
# Security fix: Add protection against memory exhaustion attacks
MAX_FEED_SIZE = 10 * 1024 * 1024 # 10MB limit for RSS feeds
# Validate URL
parsed_url = urllib.parse.urlparse(url)
if parsed_url.scheme not in ("http", "https"):
raise ValueError(f"Invalid URL scheme: {parsed_url.scheme}")
# Download with size limit
try:
with urllib.request.urlopen(url, timeout=30) as response:
# Check content length if available
content_length = response.headers.get("Content-Length")
if content_length and int(content_length) > MAX_FEED_SIZE:
raise ValueError(
f"Feed too large: {content_length} bytes exceeds {MAX_FEED_SIZE} limit"
)
# Read with size limit
content = response.read(MAX_FEED_SIZE + 1)
if len(content) > MAX_FEED_SIZE:
raise ValueError(
f"Feed too large: exceeds {MAX_FEED_SIZE} byte limit"
)
# Parse with feedparser using the validated content
# feedparser has built-in protection against XML attacks
return feedparser.parse(content) # type: ignore
except Exception as e:
# Log error and return empty feed
logging.warning(f"Failed to parse RSS feed from {url}: {e}")
return {"entries": []}
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
keep_going = True

363
autogpt_platform/backend/backend/blocks/smart_decision_maker.py
View File

@@ -13,6 +13,11 @@ from backend.data.block import (
BlockSchema,
BlockType,
)
from backend.data.dynamic_fields import (
extract_base_field_name,
get_dynamic_field_description,
is_dynamic_field,
)
from backend.data.model import NodeExecutionStats, SchemaField
from backend.util import json
from backend.util.clients import get_database_manager_async_client
@@ -98,6 +103,22 @@ def _create_tool_response(call_id: str, output: Any) -> dict[str, Any]:
return {"role": "tool", "tool_call_id": call_id, "content": content}
def _convert_raw_response_to_dict(raw_response: Any) -> dict[str, Any]:
"""
Safely convert raw_response to dictionary format for conversation history.
Handles different response types from different LLM providers.
"""
if isinstance(raw_response, str):
# Ollama returns a string, convert to dict format
return {"role": "assistant", "content": raw_response}
elif isinstance(raw_response, dict):
# Already a dict (from tests or some providers)
return raw_response
else:
# OpenAI/Anthropic return objects, convert with json.to_dict
return json.to_dict(raw_response)
def get_pending_tool_calls(conversation_history: list[Any]) -> dict[str, int]:
"""
All the tool calls entry in the conversation history requires a response.
@@ -261,6 +282,7 @@ class SmartDecisionMakerBlock(Block):
@staticmethod
def cleanup(s: str):
"""Clean up block names for use as tool function names."""
return re.sub(r"[^a-zA-Z0-9_-]", "_", s).lower()
@staticmethod
@@ -288,41 +310,66 @@ class SmartDecisionMakerBlock(Block):
}
sink_block_input_schema = block.input_schema
properties = {}
field_mapping = {} # clean_name -> original_name
for link in links:
sink_name = SmartDecisionMakerBlock.cleanup(link.sink_name)
field_name = link.sink_name
is_dynamic = is_dynamic_field(field_name)
# Clean property key to ensure Anthropic API compatibility for ALL fields
clean_field_name = SmartDecisionMakerBlock.cleanup(field_name)
field_mapping[clean_field_name] = field_name
# Handle dynamic fields (e.g., values_#_*, items_$_*, etc.)
# These are fields that get merged by the executor into their base field
if (
"_#_" in link.sink_name
or "_$_" in link.sink_name
or "_@_" in link.sink_name
):
# For dynamic fields, provide a generic string schema
# The executor will handle merging these into the appropriate structure
properties[sink_name] = {
if is_dynamic:
# For dynamic fields, use cleaned name but preserve original in description
properties[clean_field_name] = {
"type": "string",
"description": f"Dynamic value for {link.sink_name}",
"description": get_dynamic_field_description(field_name),
}
else:
# For regular fields, use the block's schema
# For regular fields, use the block's schema directly
try:
properties[sink_name] = sink_block_input_schema.get_field_schema(
link.sink_name
properties[clean_field_name] = (
sink_block_input_schema.get_field_schema(field_name)
)
except (KeyError, AttributeError):
# If the field doesn't exist in the schema, provide a generic schema
properties[sink_name] = {
# If field doesn't exist in schema, provide a generic one
properties[clean_field_name] = {
"type": "string",
"description": f"Value for {link.sink_name}",
"description": f"Value for {field_name}",
}
# Build the parameters schema using a single unified path
base_schema = block.input_schema.jsonschema()
base_required = set(base_schema.get("required", []))
# Compute required fields at the leaf level:
# - If a linked field is dynamic and its base is required in the block schema, require the leaf
# - If a linked field is regular and is required in the block schema, require the leaf
required_fields: set[str] = set()
for link in links:
field_name = link.sink_name
is_dynamic = is_dynamic_field(field_name)
# Always use cleaned field name for property key (Anthropic API compliance)
clean_field_name = SmartDecisionMakerBlock.cleanup(field_name)
if is_dynamic:
base_name = extract_base_field_name(field_name)
if base_name in base_required:
required_fields.add(clean_field_name)
else:
if field_name in base_required:
required_fields.add(clean_field_name)
tool_function["parameters"] = {
**block.input_schema.jsonschema(),
"type": "object",
"properties": properties,
"additionalProperties": False,
"required": sorted(required_fields),
}
# Store field mapping for later use in output processing
tool_function["_field_mapping"] = field_mapping
return {"type": "function", "function": tool_function}
@staticmethod
@@ -366,13 +413,12 @@ class SmartDecisionMakerBlock(Block):
sink_block_properties = sink_block_input_schema.get("properties", {}).get(
link.sink_name, {}
)
sink_name = SmartDecisionMakerBlock.cleanup(link.sink_name)
description = (
sink_block_properties["description"]
if "description" in sink_block_properties
else f"The {link.sink_name} of the tool"
)
properties[sink_name] = {
properties[link.sink_name] = {
"type": "string",
"description": description,
"default": json.dumps(sink_block_properties.get("default", None)),
@@ -388,24 +434,17 @@ class SmartDecisionMakerBlock(Block):
return {"type": "function", "function": tool_function}
@staticmethod
async def _create_function_signature(node_id: str) -> list[dict[str, Any]]:
async def _create_function_signature(
node_id: str,
) -> list[dict[str, Any]]:
"""
Creates function signatures for tools linked to a specified node within a graph.
This method filters the graph links to identify those that are tools and are
connected to the given node_id. It then constructs function signatures for each
tool based on the metadata and input schema of the linked nodes.
Creates function signatures for connected tools.
Args:
node_id: The node_id for which to create function signatures.
Returns:
list[dict[str, Any]]: A list of dictionaries, each representing a function signature
for a tool, including its name, description, and parameters.
Raises:
ValueError: If no tool links are found for the specified node_id, or if a sink node
or its metadata cannot be found.
List of function signatures for tools
"""
db_client = get_database_manager_async_client()
tools = [
@@ -430,20 +469,116 @@ class SmartDecisionMakerBlock(Block):
raise ValueError(f"Sink node not found: {links[0].sink_id}")
if sink_node.block_id == AgentExecutorBlock().id:
return_tool_functions.append(
tool_func = (
await SmartDecisionMakerBlock._create_agent_function_signature(
sink_node, links
)
)
return_tool_functions.append(tool_func)
else:
return_tool_functions.append(
tool_func = (
await SmartDecisionMakerBlock._create_block_function_signature(
sink_node, links
)
)
return_tool_functions.append(tool_func)
return return_tool_functions
async def _attempt_llm_call_with_validation(
self,
credentials: llm.APIKeyCredentials,
input_data: Input,
current_prompt: list[dict],
tool_functions: list[dict[str, Any]],
):
"""
Attempt a single LLM call with tool validation.
Returns the response if successful, raises ValueError if validation fails.
"""
resp = await llm.llm_call(
credentials=credentials,
llm_model=input_data.model,
prompt=current_prompt,
max_tokens=input_data.max_tokens,
tools=tool_functions,
ollama_host=input_data.ollama_host,
parallel_tool_calls=input_data.multiple_tool_calls,
)
# Track LLM usage stats per call
self.merge_stats(
NodeExecutionStats(
input_token_count=resp.prompt_tokens,
output_token_count=resp.completion_tokens,
llm_call_count=1,
)
)
if not resp.tool_calls:
return resp
validation_errors_list: list[str] = []
for tool_call in resp.tool_calls:
tool_name = tool_call.function.name
try:
tool_args = json.loads(tool_call.function.arguments)
except Exception as e:
validation_errors_list.append(
f"Tool call '{tool_name}' has invalid JSON arguments: {e}"
)
continue
# Find the tool definition to get the expected arguments
tool_def = next(
(
tool
for tool in tool_functions
if tool["function"]["name"] == tool_name
),
None,
)
if tool_def is None and len(tool_functions) == 1:
tool_def = tool_functions[0]
# Get parameters schema from tool definition
if (
tool_def
and "function" in tool_def
and "parameters" in tool_def["function"]
):
parameters = tool_def["function"]["parameters"]
expected_args = parameters.get("properties", {})
required_params = set(parameters.get("required", []))
else:
expected_args = {arg: {} for arg in tool_args.keys()}
required_params = set()
# Validate tool call arguments
provided_args = set(tool_args.keys())
expected_args_set = set(expected_args.keys())
# Check for unexpected arguments (typos)
unexpected_args = provided_args - expected_args_set
# Only check for missing REQUIRED parameters
missing_required_args = required_params - provided_args
if unexpected_args or missing_required_args:
error_msg = f"Tool call '{tool_name}' has parameter errors:"
if unexpected_args:
error_msg += f" Unknown parameters: {sorted(unexpected_args)}."
if missing_required_args:
error_msg += f" Missing required parameters: {sorted(missing_required_args)}."
error_msg += f" Expected parameters: {sorted(expected_args_set)}."
if required_params:
error_msg += f" Required parameters: {sorted(required_params)}."
validation_errors_list.append(error_msg)
if validation_errors_list:
raise ValueError("; ".join(validation_errors_list))
return resp
async def run(
self,
input_data: Input,
@@ -466,27 +601,19 @@ class SmartDecisionMakerBlock(Block):
if pending_tool_calls and input_data.last_tool_output is None:
raise ValueError(f"Tool call requires an output for {pending_tool_calls}")
# Only assign the last tool output to the first pending tool call
tool_output = []
if pending_tool_calls and input_data.last_tool_output is not None:
# Get the first pending tool call ID
first_call_id = next(iter(pending_tool_calls.keys()))
tool_output.append(
_create_tool_response(first_call_id, input_data.last_tool_output)
)
# Add tool output to prompt right away
prompt.extend(tool_output)
# Check if there are still pending tool calls after handling the first one
remaining_pending_calls = get_pending_tool_calls(prompt)
# If there are still pending tool calls, yield the conversation and return early
if remaining_pending_calls:
yield "conversations", prompt
return
# Fallback on adding tool output in the conversation history as user prompt.
elif input_data.last_tool_output:
logger.error(
f"[SmartDecisionMakerBlock-node_exec_id={node_exec_id}] "
@@ -519,121 +646,42 @@ class SmartDecisionMakerBlock(Block):
):
prompt.append({"role": "user", "content": prefix + input_data.prompt})
# Use retry decorator for LLM calls with validation
from backend.util.retry import create_retry_decorator
current_prompt = list(prompt)
max_attempts = max(1, int(input_data.retry))
response = None
# Create retry decorator that excludes ValueError from retry (for non-LLM errors)
llm_retry = create_retry_decorator(
max_attempts=input_data.retry,
exclude_exceptions=(), # Don't exclude ValueError - we want to retry validation failures
context="SmartDecisionMaker LLM call",
)
@llm_retry
async def call_llm_with_validation():
response = await llm.llm_call(
credentials=credentials,
llm_model=input_data.model,
prompt=prompt,
max_tokens=input_data.max_tokens,
tools=tool_functions,
ollama_host=input_data.ollama_host,
parallel_tool_calls=input_data.multiple_tool_calls,
)
# Track LLM usage stats
self.merge_stats(
NodeExecutionStats(
input_token_count=response.prompt_tokens,
output_token_count=response.completion_tokens,
llm_call_count=1,
last_error = None
for attempt in range(max_attempts):
try:
response = await self._attempt_llm_call_with_validation(
credentials, input_data, current_prompt, tool_functions
)
)
break
if not response.tool_calls:
return response, None # No tool calls, return response
# Validate all tool calls before proceeding
validation_errors = []
for tool_call in response.tool_calls:
tool_name = tool_call.function.name
tool_args = json.loads(tool_call.function.arguments)
# Find the tool definition to get the expected arguments
tool_def = next(
(
tool
for tool in tool_functions
if tool["function"]["name"] == tool_name
),
None,
)
# Get parameters schema from tool definition
if (
tool_def
and "function" in tool_def
and "parameters" in tool_def["function"]
):
parameters = tool_def["function"]["parameters"]
expected_args = parameters.get("properties", {})
required_params = set(parameters.get("required", []))
else:
expected_args = {arg: {} for arg in tool_args.keys()}
required_params = set()
# Validate tool call arguments
provided_args = set(tool_args.keys())
expected_args_set = set(expected_args.keys())
# Check for unexpected arguments (typos)
unexpected_args = provided_args - expected_args_set
# Only check for missing REQUIRED parameters
missing_required_args = required_params - provided_args
if unexpected_args or missing_required_args:
error_msg = f"Tool call '{tool_name}' has parameter errors:"
if unexpected_args:
error_msg += f" Unknown parameters: {sorted(unexpected_args)}."
if missing_required_args:
error_msg += f" Missing required parameters: {sorted(missing_required_args)}."
error_msg += f" Expected parameters: {sorted(expected_args_set)}."
if required_params:
error_msg += f" Required parameters: {sorted(required_params)}."
validation_errors.append(error_msg)
# If validation failed, add feedback and raise for retry
if validation_errors:
# Add the failed response to conversation
prompt.append(response.raw_response)
# Add error feedback for retry
except ValueError as e:
last_error = e
error_feedback = (
"Your tool call had parameter errors. Please fix the following issues and try again:\n"
+ "\n".join(f"- {error}" for error in validation_errors)
+ "\n\nPlease make sure to use the exact parameter names as specified in the function schema."
+ f"- {str(e)}\n"
+ "\nPlease make sure to use the exact parameter names as specified in the function schema."
)
prompt.append({"role": "user", "content": error_feedback})
current_prompt = list(current_prompt) + [
{"role": "user", "content": error_feedback}
]
raise ValueError(
f"Tool call validation failed: {'; '.join(validation_errors)}"
)
return response, validation_errors
# Call the LLM with retry logic
response, validation_errors = await call_llm_with_validation()
if response is None:
raise last_error or ValueError(
"Failed to get valid response after all retry attempts"
)
if not response.tool_calls:
yield "finished", response.response
return
# If we get here, validation passed - yield tool outputs
for tool_call in response.tool_calls:
tool_name = tool_call.function.name
tool_args = json.loads(tool_call.function.arguments)
# Get expected arguments (already validated above)
tool_def = next(
(
tool
@@ -651,15 +699,36 @@ class SmartDecisionMakerBlock(Block):
else:
expected_args = {arg: {} for arg in tool_args.keys()}
# Yield provided arguments, use .get() for optional parameters
for arg_name in expected_args:
yield f"tools_^_{tool_name}_~_{arg_name}", tool_args.get(arg_name)
# Get field mapping from tool definition
field_mapping = (
tool_def.get("function", {}).get("_field_mapping", {})
if tool_def
else {}
)
for clean_arg_name in expected_args:
# arg_name is now always the cleaned field name (for Anthropic API compliance)
# Get the original field name from field mapping for proper emit key generation
original_field_name = field_mapping.get(clean_arg_name, clean_arg_name)
arg_value = tool_args.get(clean_arg_name)
sanitized_tool_name = self.cleanup(tool_name)
sanitized_arg_name = self.cleanup(original_field_name)
emit_key = f"tools_^_{sanitized_tool_name}_~_{sanitized_arg_name}"
logger.debug(
"[SmartDecisionMakerBlock|geid:%s|neid:%s] emit %s",
graph_exec_id,
node_exec_id,
emit_key,
)
yield emit_key, arg_value
# Add reasoning to conversation history if available
if response.reasoning:
prompt.append(
{"role": "assistant", "content": f"[Reasoning]: {response.reasoning}"}
)
prompt.append(response.raw_response)
prompt.append(_convert_raw_response_to_dict(response.raw_response))
yield "conversations", prompt

2
autogpt_platform/backend/backend/blocks/test/test_block.py
View File

@@ -19,7 +19,7 @@ async def test_block_ids_valid(block: Type[Block]):
# Skip list for blocks with known invalid UUIDs
skip_blocks = {
"GetWeatherInformationBlock",
"CodeExecutionBlock",
"ExecuteCodeBlock",
"CountdownTimerBlock",
"TwitterGetListTweetsBlock",
"TwitterRemoveListMemberBlock",

269
autogpt_platform/backend/backend/blocks/test/test_blocks_dos_vulnerability.py Normal file
View File

@@ -0,0 +1,269 @@
"""
Test security fixes for various DoS vulnerabilities.
"""
import asyncio
from unittest.mock import patch
import pytest
from backend.blocks.code_extraction_block import CodeExtractionBlock
from backend.blocks.iteration import StepThroughItemsBlock
from backend.blocks.llm import AITextSummarizerBlock
from backend.blocks.text import ExtractTextInformationBlock
from backend.blocks.xml_parser import XMLParserBlock
from backend.util.file import store_media_file
from backend.util.type import MediaFileType
class TestCodeExtractionBlockSecurity:
"""Test ReDoS fixes in CodeExtractionBlock."""
async def test_redos_protection(self):
"""Test that the regex patterns don't cause ReDoS."""
block = CodeExtractionBlock()
# Test with input that would previously cause ReDoS
malicious_input = "```python" + " " * 10000 # Large spaces
result = []
async for output_name, output_data in block.run(
CodeExtractionBlock.Input(text=malicious_input)
):
result.append((output_name, output_data))
# Should complete without hanging
assert len(result) >= 1
assert any(name == "remaining_text" for name, _ in result)
class TestAITextSummarizerBlockSecurity:
"""Test memory exhaustion fixes in AITextSummarizerBlock."""
def test_split_text_limits(self):
"""Test that _split_text has proper limits."""
# Test text size limit
large_text = "a" * 2_000_000 # 2MB text
result = AITextSummarizerBlock._split_text(large_text, 1000, 100)
# Should be truncated to 1MB
total_chars = sum(len(chunk) for chunk in result)
assert total_chars <= 1_000_000 + 1000 # Allow for chunk boundary
# Test chunk count limit
result = AITextSummarizerBlock._split_text("word " * 10000, 10, 9)
assert len(result) <= 100 # MAX_CHUNKS limit
# Test parameter validation
result = AITextSummarizerBlock._split_text(
"test", 10, 15
) # overlap > max_tokens
assert len(result) >= 1 # Should still work
class TestExtractTextInformationBlockSecurity:
"""Test ReDoS and memory exhaustion fixes in ExtractTextInformationBlock."""
async def test_text_size_limits(self):
"""Test text size limits."""
block = ExtractTextInformationBlock()
# Test with large input
large_text = "a" * 2_000_000 # 2MB
results = []
async for output_name, output_data in block.run(
ExtractTextInformationBlock.Input(
text=large_text, pattern=r"a+", find_all=True, group=0
)
):
results.append((output_name, output_data))
# Should complete and have limits applied
matched_results = [r for name, r in results if name == "matched_results"]
if matched_results:
assert len(matched_results[0]) <= 1000 # MAX_MATCHES limit
async def test_dangerous_pattern_timeout(self):
"""Test timeout protection for dangerous patterns."""
block = ExtractTextInformationBlock()
# Test with potentially dangerous lookahead pattern
test_input = "a" * 1000
# This should complete quickly due to timeout protection
start_time = asyncio.get_event_loop().time()
results = []
async for output_name, output_data in block.run(
ExtractTextInformationBlock.Input(
text=test_input, pattern=r"(?=.+)", find_all=True, group=0
)
):
results.append((output_name, output_data))
end_time = asyncio.get_event_loop().time()
# Should complete within reasonable time (much less than 5s timeout)
assert (end_time - start_time) < 10
async def test_redos_catastrophic_backtracking(self):
"""Test that ReDoS patterns with catastrophic backtracking are handled."""
block = ExtractTextInformationBlock()
# Pattern that causes catastrophic backtracking: (a+)+b
# With input "aaaaaaaaaaaaaaaaaaaaaaaaaaaa" (no 'b'), this causes exponential time
dangerous_pattern = r"(a+)+b"
test_input = "a" * 30 # 30 'a's without a 'b' at the end
# This should be handled by timeout protection or pattern detection
start_time = asyncio.get_event_loop().time()
results = []
async for output_name, output_data in block.run(
ExtractTextInformationBlock.Input(
text=test_input, pattern=dangerous_pattern, find_all=True, group=0
)
):
results.append((output_name, output_data))
end_time = asyncio.get_event_loop().time()
elapsed = end_time - start_time
# Should complete within timeout (6 seconds to be safe)
# The current threading.Timer approach doesn't work, so this will likely fail
# demonstrating the need for a fix
assert elapsed < 6, f"Regex took {elapsed}s, timeout mechanism failed"
# Should return empty results on timeout or no match
matched_results = [r for name, r in results if name == "matched_results"]
assert matched_results[0] == [] # No matches expected
class TestStepThroughItemsBlockSecurity:
"""Test iteration limits in StepThroughItemsBlock."""
async def test_item_count_limits(self):
"""Test maximum item count limits."""
block = StepThroughItemsBlock()
# Test with too many items
large_list = list(range(20000)) # Exceeds MAX_ITEMS (10000)
with pytest.raises(ValueError, match="Too many items"):
async for _ in block.run(StepThroughItemsBlock.Input(items=large_list)):
pass
async def test_string_size_limits(self):
"""Test string input size limits."""
block = StepThroughItemsBlock()
# Test with large JSON string
large_string = '["item"]' * 200000 # Large JSON string
with pytest.raises(ValueError, match="Input too large"):
async for _ in block.run(
StepThroughItemsBlock.Input(items_str=large_string)
):
pass
async def test_normal_iteration_works(self):
"""Test that normal iteration still works."""
block = StepThroughItemsBlock()
results = []
async for output_name, output_data in block.run(
StepThroughItemsBlock.Input(items=[1, 2, 3])
):
results.append((output_name, output_data))
# Should have 6 outputs (item, key for each of 3 items)
assert len(results) == 6
items = [data for name, data in results if name == "item"]
assert items == [1, 2, 3]
class TestXMLParserBlockSecurity:
"""Test XML size limits in XMLParserBlock."""
async def test_xml_size_limits(self):
"""Test XML input size limits."""
block = XMLParserBlock()
# Test with large XML - need to exceed 10MB limit
# Each "<item>data</item>" is 17 chars, need ~620K items for >10MB
large_xml = "<root>" + "<item>data</item>" * 620000 + "</root>"
with pytest.raises(ValueError, match="XML too large"):
async for _ in block.run(XMLParserBlock.Input(input_xml=large_xml)):
pass
class TestStoreMediaFileSecurity:
"""Test file storage security limits."""
@patch("backend.util.file.scan_content_safe")
@patch("backend.util.file.get_cloud_storage_handler")
async def test_file_size_limits(self, mock_cloud_storage, mock_scan):
"""Test file size limits."""
# Mock cloud storage handler - get_cloud_storage_handler is async
# but is_cloud_path and parse_cloud_path are sync methods
from unittest.mock import MagicMock
mock_handler = MagicMock()
mock_handler.is_cloud_path.return_value = False
# Make get_cloud_storage_handler an async function that returns the mock handler
async def async_get_handler():
return mock_handler
mock_cloud_storage.side_effect = async_get_handler
mock_scan.return_value = None
# Test with large base64 content
large_content = "a" * (200 * 1024 * 1024) # 200MB
large_data_uri = f"data:text/plain;base64,{large_content}"
with pytest.raises(ValueError, match="File too large"):
await store_media_file(
graph_exec_id="test",
file=MediaFileType(large_data_uri),
user_id="test_user",
)
@patch("backend.util.file.Path")
@patch("backend.util.file.scan_content_safe")
@patch("backend.util.file.get_cloud_storage_handler")
async def test_directory_size_limits(self, mock_cloud_storage, mock_scan, MockPath):
"""Test directory size limits."""
from unittest.mock import MagicMock
mock_handler = MagicMock()
mock_handler.is_cloud_path.return_value = False
async def async_get_handler():
return mock_handler
mock_cloud_storage.side_effect = async_get_handler
mock_scan.return_value = None
# Create mock path instance for the execution directory
mock_path_instance = MagicMock()
mock_path_instance.exists.return_value = True
# Mock glob to return files that total > 1GB
mock_file = MagicMock()
mock_file.is_file.return_value = True
mock_file.stat.return_value.st_size = 2 * 1024 * 1024 * 1024 # 2GB
mock_path_instance.glob.return_value = [mock_file]
# Make Path() return our mock
MockPath.return_value = mock_path_instance
# Should raise an error when directory size exceeds limit
with pytest.raises(ValueError, match="Disk usage limit exceeded"):
await store_media_file(
graph_exec_id="test",
file=MediaFileType(
"data:text/plain;base64,dGVzdA=="
), # Small test file
user_id="test_user",
)

262
autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py
View File

@@ -216,8 +216,17 @@ async def test_smart_decision_maker_tracks_llm_stats():
}
# Mock the _create_function_signature method to avoid database calls
with patch("backend.blocks.llm.llm_call", return_value=mock_response), patch.object(
SmartDecisionMakerBlock, "_create_function_signature", return_value=[]
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call",
new_callable=AsyncMock,
return_value=mock_response,
), patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=[],
):
# Create test input
@@ -301,11 +310,16 @@ async def test_smart_decision_maker_parameter_validation():
mock_response_with_typo.reasoning = None
mock_response_with_typo.raw_response = {"role": "assistant", "content": None}
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call", return_value=mock_response_with_typo
"backend.blocks.llm.llm_call",
new_callable=AsyncMock,
return_value=mock_response_with_typo,
) as mock_llm_call, patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=mock_tool_functions,
):
@@ -332,7 +346,7 @@ async def test_smart_decision_maker_parameter_validation():
# Verify error message contains details about the typo
error_msg = str(exc_info.value)
assert "Tool call validation failed" in error_msg
assert "Tool call 'search_keywords' has parameter errors" in error_msg
assert "Unknown parameters: ['maximum_keyword_difficulty']" in error_msg
# Verify that LLM was called the expected number of times (retries)
@@ -353,11 +367,16 @@ async def test_smart_decision_maker_parameter_validation():
mock_response_missing_required.reasoning = None
mock_response_missing_required.raw_response = {"role": "assistant", "content": None}
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call", return_value=mock_response_missing_required
"backend.blocks.llm.llm_call",
new_callable=AsyncMock,
return_value=mock_response_missing_required,
), patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=mock_tool_functions,
):
@@ -398,11 +417,16 @@ async def test_smart_decision_maker_parameter_validation():
mock_response_valid.reasoning = None
mock_response_valid.raw_response = {"role": "assistant", "content": None}
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call", return_value=mock_response_valid
"backend.blocks.llm.llm_call",
new_callable=AsyncMock,
return_value=mock_response_valid,
), patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=mock_tool_functions,
):
@@ -447,11 +471,16 @@ async def test_smart_decision_maker_parameter_validation():
mock_response_all_params.reasoning = None
mock_response_all_params.raw_response = {"role": "assistant", "content": None}
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call", return_value=mock_response_all_params
"backend.blocks.llm.llm_call",
new_callable=AsyncMock,
return_value=mock_response_all_params,
), patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=mock_tool_functions,
):
@@ -478,3 +507,222 @@ async def test_smart_decision_maker_parameter_validation():
assert outputs["tools_^_search_keywords_~_query"] == "test"
assert outputs["tools_^_search_keywords_~_max_keyword_difficulty"] == 50
assert outputs["tools_^_search_keywords_~_optional_param"] == "custom_value"
@pytest.mark.asyncio
async def test_smart_decision_maker_raw_response_conversion():
"""Test that SmartDecisionMaker correctly handles different raw_response types with retry mechanism."""
from unittest.mock import MagicMock, patch
import backend.blocks.llm as llm_module
from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
block = SmartDecisionMakerBlock()
# Mock tool functions
mock_tool_functions = [
{
"type": "function",
"function": {
"name": "test_tool",
"parameters": {
"type": "object",
"properties": {"param": {"type": "string"}},
"required": ["param"],
},
},
}
]
# Test case 1: Simulate ChatCompletionMessage raw_response that caused the original error
class MockChatCompletionMessage:
"""Simulate OpenAI's ChatCompletionMessage object that lacks .get() method"""
def __init__(self, role, content, tool_calls=None):
self.role = role
self.content = content
self.tool_calls = tool_calls or []
# This is what caused the error - no .get() method
# def get(self, key, default=None): # Intentionally missing
# First response: has invalid parameter name (triggers retry)
mock_tool_call_invalid = MagicMock()
mock_tool_call_invalid.function.name = "test_tool"
mock_tool_call_invalid.function.arguments = (
'{"wrong_param": "test_value"}' # Invalid parameter name
)
mock_response_retry = MagicMock()
mock_response_retry.response = None
mock_response_retry.tool_calls = [mock_tool_call_invalid]
mock_response_retry.prompt_tokens = 50
mock_response_retry.completion_tokens = 25
mock_response_retry.reasoning = None
# This would cause the original error without our fix
mock_response_retry.raw_response = MockChatCompletionMessage(
role="assistant", content=None, tool_calls=[mock_tool_call_invalid]
)
# Second response: successful (correct parameter name)
mock_tool_call_valid = MagicMock()
mock_tool_call_valid.function.name = "test_tool"
mock_tool_call_valid.function.arguments = (
'{"param": "test_value"}' # Correct parameter name
)
mock_response_success = MagicMock()
mock_response_success.response = None
mock_response_success.tool_calls = [mock_tool_call_valid]
mock_response_success.prompt_tokens = 50
mock_response_success.completion_tokens = 25
mock_response_success.reasoning = None
mock_response_success.raw_response = MockChatCompletionMessage(
role="assistant", content=None, tool_calls=[mock_tool_call_valid]
)
# Mock llm_call to return different responses on different calls
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call", new_callable=AsyncMock
) as mock_llm_call, patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=mock_tool_functions,
):
# First call returns response that will trigger retry due to validation error
# Second call returns successful response
mock_llm_call.side_effect = [mock_response_retry, mock_response_success]
input_data = SmartDecisionMakerBlock.Input(
prompt="Test prompt",
model=llm_module.LlmModel.GPT4O,
credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore
retry=2,
)
# Should succeed after retry, demonstrating our helper function works
outputs = {}
async for output_name, output_data in block.run(
input_data,
credentials=llm_module.TEST_CREDENTIALS,
graph_id="test-graph-id",
node_id="test-node-id",
graph_exec_id="test-exec-id",
node_exec_id="test-node-exec-id",
user_id="test-user-id",
):
outputs[output_name] = output_data
# Verify the tool output was generated successfully
assert "tools_^_test_tool_~_param" in outputs
assert outputs["tools_^_test_tool_~_param"] == "test_value"
# Verify conversation history was properly maintained
assert "conversations" in outputs
conversations = outputs["conversations"]
assert len(conversations) > 0
# The conversations should contain properly converted raw_response objects as dicts
# This would have failed with the original bug due to ChatCompletionMessage.get() error
for msg in conversations:
assert isinstance(msg, dict), f"Expected dict, got {type(msg)}"
if msg.get("role") == "assistant":
# Should have been converted from ChatCompletionMessage to dict
assert "role" in msg
# Verify LLM was called twice (initial + 1 retry)
assert mock_llm_call.call_count == 2
# Test case 2: Test with different raw_response types (Ollama string, dict)
# Test Ollama string response
mock_response_ollama = MagicMock()
mock_response_ollama.response = "I'll help you with that."
mock_response_ollama.tool_calls = None
mock_response_ollama.prompt_tokens = 30
mock_response_ollama.completion_tokens = 15
mock_response_ollama.reasoning = None
mock_response_ollama.raw_response = (
"I'll help you with that." # Ollama returns string
)
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call",
new_callable=AsyncMock,
return_value=mock_response_ollama,
), patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=[], # No tools for this test
):
input_data = SmartDecisionMakerBlock.Input(
prompt="Simple prompt",
model=llm_module.LlmModel.GPT4O,
credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore
)
outputs = {}
async for output_name, output_data in block.run(
input_data,
credentials=llm_module.TEST_CREDENTIALS,
graph_id="test-graph-id",
node_id="test-node-id",
graph_exec_id="test-exec-id",
node_exec_id="test-node-exec-id",
user_id="test-user-id",
):
outputs[output_name] = output_data
# Should finish since no tool calls
assert "finished" in outputs
assert outputs["finished"] == "I'll help you with that."
# Test case 3: Test with dict raw_response (some providers/tests)
mock_response_dict = MagicMock()
mock_response_dict.response = "Test response"
mock_response_dict.tool_calls = None
mock_response_dict.prompt_tokens = 25
mock_response_dict.completion_tokens = 10
mock_response_dict.reasoning = None
mock_response_dict.raw_response = {
"role": "assistant",
"content": "Test response",
} # Dict format
from unittest.mock import AsyncMock
with patch(
"backend.blocks.llm.llm_call",
new_callable=AsyncMock,
return_value=mock_response_dict,
), patch.object(
SmartDecisionMakerBlock,
"_create_function_signature",
new_callable=AsyncMock,
return_value=[],
):
input_data = SmartDecisionMakerBlock.Input(
prompt="Another test",
model=llm_module.LlmModel.GPT4O,
credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore
)
outputs = {}
async for output_name, output_data in block.run(
input_data,
credentials=llm_module.TEST_CREDENTIALS,
graph_id="test-graph-id",
node_id="test-node-id",
graph_exec_id="test-exec-id",
node_exec_id="test-node-exec-id",
user_id="test-user-id",
):
outputs[output_name] = output_data
assert "finished" in outputs
assert outputs["finished"] == "Test response"

30
autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker_dict.py
View File

@@ -48,16 +48,24 @@ async def test_smart_decision_maker_handles_dynamic_dict_fields():
assert "parameters" in signature["function"]
assert "properties" in signature["function"]["parameters"]
# Check that dynamic fields are handled
# Check that dynamic fields are handled with original names
properties = signature["function"]["parameters"]["properties"]
assert len(properties) == 3 # Should have all three fields
# Each dynamic field should have proper schema
for prop_value in properties.values():
# Check that field names are cleaned (for Anthropic API compatibility)
assert "values___name" in properties
assert "values___age" in properties
assert "values___city" in properties
# Each dynamic field should have proper schema with descriptive text
for field_name, prop_value in properties.items():
assert "type" in prop_value
assert prop_value["type"] == "string" # Dynamic fields get string type
assert "description" in prop_value
assert "Dynamic value for" in prop_value["description"]
# Check that descriptions properly explain the dynamic field
if field_name == "values___name":
assert "Dictionary field 'name'" in prop_value["description"]
assert "values['name']" in prop_value["description"]
@pytest.mark.asyncio
@@ -96,10 +104,18 @@ async def test_smart_decision_maker_handles_dynamic_list_fields():
properties = signature["function"]["parameters"]["properties"]
assert len(properties) == 2 # Should have both list items
# Each dynamic field should have proper schema
for prop_value in properties.values():
# Check that field names are cleaned (for Anthropic API compatibility)
assert "entries___0" in properties
assert "entries___1" in properties
# Each dynamic field should have proper schema with descriptive text
for field_name, prop_value in properties.items():
assert prop_value["type"] == "string"
assert "Dynamic value for" in prop_value["description"]
assert "description" in prop_value
# Check that descriptions properly explain the list field
if field_name == "entries___0":
assert "List item 0" in prop_value["description"]
assert "entries[0]" in prop_value["description"]
@pytest.mark.asyncio

553
autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker_dynamic_fields.py Normal file
View File

@@ -0,0 +1,553 @@
"""Comprehensive tests for SmartDecisionMakerBlock dynamic field handling."""
import json
from unittest.mock import AsyncMock, Mock, patch
import pytest
from backend.blocks.data_manipulation import AddToListBlock, CreateDictionaryBlock
from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock
from backend.blocks.text import MatchTextPatternBlock
from backend.data.dynamic_fields import get_dynamic_field_description
@pytest.mark.asyncio
async def test_dynamic_field_description_generation():
"""Test that dynamic field descriptions are generated correctly."""
# Test dictionary field description
desc = get_dynamic_field_description("values_#_name")
assert "Dictionary field 'name' for base field 'values'" in desc
assert "values['name']" in desc
# Test list field description
desc = get_dynamic_field_description("items_$_0")
assert "List item 0 for base field 'items'" in desc
assert "items[0]" in desc
# Test object field description
desc = get_dynamic_field_description("user_@_email")
assert "Object attribute 'email' for base field 'user'" in desc
assert "user.email" in desc
# Test regular field fallback
desc = get_dynamic_field_description("regular_field")
assert desc == "Value for regular_field"
@pytest.mark.asyncio
async def test_create_block_function_signature_with_dict_fields():
"""Test that function signatures are created correctly for dictionary dynamic fields."""
block = SmartDecisionMakerBlock()
# Create a mock node for CreateDictionaryBlock
mock_node = Mock()
mock_node.block = CreateDictionaryBlock()
mock_node.block_id = CreateDictionaryBlock().id
mock_node.input_default = {}
# Create mock links with dynamic dictionary fields (source sanitized, sink original)
mock_links = [
Mock(
source_name="tools_^_create_dict_~_values___name", # Sanitized source
sink_name="values_#_name", # Original sink
sink_id="dict_node_id",
source_id="smart_decision_node_id",
),
Mock(
source_name="tools_^_create_dict_~_values___age", # Sanitized source
sink_name="values_#_age", # Original sink
sink_id="dict_node_id",
source_id="smart_decision_node_id",
),
Mock(
source_name="tools_^_create_dict_~_values___email", # Sanitized source
sink_name="values_#_email", # Original sink
sink_id="dict_node_id",
source_id="smart_decision_node_id",
),
]
# Generate function signature
signature = await block._create_block_function_signature(mock_node, mock_links) # type: ignore
# Verify the signature structure
assert signature["type"] == "function"
assert "function" in signature
assert "parameters" in signature["function"]
assert "properties" in signature["function"]["parameters"]
# Check that dynamic fields are handled with original names
properties = signature["function"]["parameters"]["properties"]
assert len(properties) == 3
# Check cleaned field names (for Anthropic API compatibility)
assert "values___name" in properties
assert "values___age" in properties
assert "values___email" in properties
# Check descriptions mention they are dictionary fields
assert "Dictionary field" in properties["values___name"]["description"]
assert "values['name']" in properties["values___name"]["description"]
assert "Dictionary field" in properties["values___age"]["description"]
assert "values['age']" in properties["values___age"]["description"]
assert "Dictionary field" in properties["values___email"]["description"]
assert "values['email']" in properties["values___email"]["description"]
@pytest.mark.asyncio
async def test_create_block_function_signature_with_list_fields():
"""Test that function signatures are created correctly for list dynamic fields."""
block = SmartDecisionMakerBlock()
# Create a mock node for AddToListBlock
mock_node = Mock()
mock_node.block = AddToListBlock()
mock_node.block_id = AddToListBlock().id
mock_node.input_default = {}
# Create mock links with dynamic list fields
mock_links = [
Mock(
source_name="tools_^_add_list_~_0",
sink_name="entries_$_0", # Dynamic list field
sink_id="list_node_id",
source_id="smart_decision_node_id",
),
Mock(
source_name="tools_^_add_list_~_1",
sink_name="entries_$_1", # Dynamic list field
sink_id="list_node_id",
source_id="smart_decision_node_id",
),
Mock(
source_name="tools_^_add_list_~_2",
sink_name="entries_$_2", # Dynamic list field
sink_id="list_node_id",
source_id="smart_decision_node_id",
),
]
# Generate function signature
signature = await block._create_block_function_signature(mock_node, mock_links) # type: ignore
# Verify the signature structure
assert signature["type"] == "function"
properties = signature["function"]["parameters"]["properties"]
# Check cleaned field names (for Anthropic API compatibility)
assert "entries___0" in properties
assert "entries___1" in properties
assert "entries___2" in properties
# Check descriptions mention they are list items
assert "List item 0" in properties["entries___0"]["description"]
assert "entries[0]" in properties["entries___0"]["description"]
assert "List item 1" in properties["entries___1"]["description"]
assert "entries[1]" in properties["entries___1"]["description"]
@pytest.mark.asyncio
async def test_create_block_function_signature_with_object_fields():
"""Test that function signatures are created correctly for object dynamic fields."""
block = SmartDecisionMakerBlock()
# Create a mock node for MatchTextPatternBlock (simulating object fields)
mock_node = Mock()
mock_node.block = MatchTextPatternBlock()
mock_node.block_id = MatchTextPatternBlock().id
mock_node.input_default = {}
# Create mock links with dynamic object fields
mock_links = [
Mock(
source_name="tools_^_extract_~_user_name",
sink_name="data_@_user_name", # Dynamic object field
sink_id="extract_node_id",
source_id="smart_decision_node_id",
),
Mock(
source_name="tools_^_extract_~_user_email",
sink_name="data_@_user_email", # Dynamic object field
sink_id="extract_node_id",
source_id="smart_decision_node_id",
),
]
# Generate function signature
signature = await block._create_block_function_signature(mock_node, mock_links) # type: ignore
# Verify the signature structure
properties = signature["function"]["parameters"]["properties"]
# Check cleaned field names (for Anthropic API compatibility)
assert "data___user_name" in properties
assert "data___user_email" in properties
# Check descriptions mention they are object attributes
assert "Object attribute" in properties["data___user_name"]["description"]
assert "data.user_name" in properties["data___user_name"]["description"]
@pytest.mark.asyncio
async def test_create_function_signature():
"""Test that the mapping between sanitized and original field names is built correctly."""
block = SmartDecisionMakerBlock()
# Mock the database client and connected nodes
with patch(
"backend.blocks.smart_decision_maker.get_database_manager_async_client"
) as mock_db:
mock_client = AsyncMock()
mock_db.return_value = mock_client
# Create mock nodes and links
mock_dict_node = Mock()
mock_dict_node.block = CreateDictionaryBlock()
mock_dict_node.block_id = CreateDictionaryBlock().id
mock_dict_node.input_default = {}
mock_list_node = Mock()
mock_list_node.block = AddToListBlock()
mock_list_node.block_id = AddToListBlock().id
mock_list_node.input_default = {}
# Mock links with dynamic fields
dict_link1 = Mock(
source_name="tools_^_create_dictionary_~_name",
sink_name="values_#_name",
sink_id="dict_node_id",
source_id="test_node_id",
)
dict_link2 = Mock(
source_name="tools_^_create_dictionary_~_age",
sink_name="values_#_age",
sink_id="dict_node_id",
source_id="test_node_id",
)
list_link = Mock(
source_name="tools_^_add_to_list_~_0",
sink_name="entries_$_0",
sink_id="list_node_id",
source_id="test_node_id",
)
mock_client.get_connected_output_nodes.return_value = [
(dict_link1, mock_dict_node),
(dict_link2, mock_dict_node),
(list_link, mock_list_node),
]
# Call the method that builds signatures
tool_functions = await block._create_function_signature("test_node_id")
# Verify we got 2 tool functions (one for dict, one for list)
assert len(tool_functions) == 2
# Verify the tool functions contain the dynamic field names
dict_tool = next(
(
tool
for tool in tool_functions
if tool["function"]["name"] == "createdictionaryblock"
),
None,
)
assert dict_tool is not None
dict_properties = dict_tool["function"]["parameters"]["properties"]
assert "values___name" in dict_properties
assert "values___age" in dict_properties
list_tool = next(
(
tool
for tool in tool_functions
if tool["function"]["name"] == "addtolistblock"
),
None,
)
assert list_tool is not None
list_properties = list_tool["function"]["parameters"]["properties"]
assert "entries___0" in list_properties
@pytest.mark.asyncio
async def test_output_yielding_with_dynamic_fields():
"""Test that outputs are yielded correctly with dynamic field names mapped back."""
block = SmartDecisionMakerBlock()
# No more sanitized mapping needed since we removed sanitization
# Mock LLM response with tool calls
mock_response = Mock()
mock_response.tool_calls = [
Mock(
function=Mock(
arguments=json.dumps(
{
"values___name": "Alice",
"values___age": 30,
"values___email": "alice@example.com",
}
),
)
)
]
# Ensure function name is a real string, not a Mock name
mock_response.tool_calls[0].function.name = "createdictionaryblock"
mock_response.reasoning = "Creating a dictionary with user information"
mock_response.raw_response = {"role": "assistant", "content": "test"}
mock_response.prompt_tokens = 100
mock_response.completion_tokens = 50
# Mock the LLM call
with patch(
"backend.blocks.smart_decision_maker.llm.llm_call", new_callable=AsyncMock
) as mock_llm:
mock_llm.return_value = mock_response
# Mock the function signature creation
with patch.object(
block, "_create_function_signature", new_callable=AsyncMock
) as mock_sig:
mock_sig.return_value = [
{
"type": "function",
"function": {
"name": "createdictionaryblock",
"parameters": {
"type": "object",
"properties": {
"values___name": {"type": "string"},
"values___age": {"type": "number"},
"values___email": {"type": "string"},
},
},
},
}
]
# Create input data
from backend.blocks import llm
input_data = block.input_schema(
prompt="Create a user dictionary",
credentials=llm.TEST_CREDENTIALS_INPUT,
model=llm.LlmModel.GPT4O,
)
# Run the block
outputs = {}
async for output_name, output_value in block.run(
input_data,
credentials=llm.TEST_CREDENTIALS,
graph_id="test_graph",
node_id="test_node",
graph_exec_id="test_exec",
node_exec_id="test_node_exec",
user_id="test_user",
):
outputs[output_name] = output_value
# Verify the outputs use sanitized field names (matching frontend normalizeToolName)
assert "tools_^_createdictionaryblock_~_values___name" in outputs
assert outputs["tools_^_createdictionaryblock_~_values___name"] == "Alice"
assert "tools_^_createdictionaryblock_~_values___age" in outputs
assert outputs["tools_^_createdictionaryblock_~_values___age"] == 30
assert "tools_^_createdictionaryblock_~_values___email" in outputs
assert (
outputs["tools_^_createdictionaryblock_~_values___email"]
== "alice@example.com"
)
@pytest.mark.asyncio
async def test_mixed_regular_and_dynamic_fields():
"""Test handling of blocks with both regular and dynamic fields."""
block = SmartDecisionMakerBlock()
# Create a mock node
mock_node = Mock()
mock_node.block = Mock()
mock_node.block.name = "TestBlock"
mock_node.block.description = "A test block"
mock_node.block.input_schema = Mock()
# Mock the get_field_schema to return a proper schema for regular fields
def get_field_schema(field_name):
if field_name == "regular_field":
return {"type": "string", "description": "A regular field"}
elif field_name == "values":
return {"type": "object", "description": "A dictionary field"}
else:
raise KeyError(f"Field {field_name} not found")
mock_node.block.input_schema.get_field_schema = get_field_schema
mock_node.block.input_schema.jsonschema = Mock(
return_value={"properties": {}, "required": []}
)
# Create links with both regular and dynamic fields
mock_links = [
Mock(
source_name="tools_^_test_~_regular",
sink_name="regular_field", # Regular field
sink_id="test_node_id",
source_id="smart_decision_node_id",
),
Mock(
source_name="tools_^_test_~_dict_key",
sink_name="values_#_key1", # Dynamic dict field
sink_id="test_node_id",
source_id="smart_decision_node_id",
),
Mock(
source_name="tools_^_test_~_dict_key2",
sink_name="values_#_key2", # Dynamic dict field
sink_id="test_node_id",
source_id="smart_decision_node_id",
),
]
# Generate function signature
signature = await block._create_block_function_signature(mock_node, mock_links) # type: ignore
# Check properties
properties = signature["function"]["parameters"]["properties"]
assert len(properties) == 3
# Regular field should have its original schema
assert "regular_field" in properties
assert properties["regular_field"]["description"] == "A regular field"
# Dynamic fields should have generated descriptions
assert "values___key1" in properties
assert "Dictionary field" in properties["values___key1"]["description"]
assert "values___key2" in properties
assert "Dictionary field" in properties["values___key2"]["description"]
@pytest.mark.asyncio
async def test_validation_errors_dont_pollute_conversation():
"""Test that validation errors are only used during retries and don't pollute the conversation."""
block = SmartDecisionMakerBlock()
# Track conversation history changes
conversation_snapshots = []
# Mock response with invalid tool call (missing required parameter)
invalid_response = Mock()
invalid_response.tool_calls = [
Mock(
function=Mock(
arguments=json.dumps({"wrong_param": "value"}), # Wrong parameter name
)
)
]
# Ensure function name is a real string, not a Mock name
invalid_response.tool_calls[0].function.name = "test_tool"
invalid_response.reasoning = None
invalid_response.raw_response = {"role": "assistant", "content": "invalid"}
invalid_response.prompt_tokens = 100
invalid_response.completion_tokens = 50
# Mock valid response after retry
valid_response = Mock()
valid_response.tool_calls = [
Mock(function=Mock(arguments=json.dumps({"correct_param": "value"})))
]
# Ensure function name is a real string, not a Mock name
valid_response.tool_calls[0].function.name = "test_tool"
valid_response.reasoning = None
valid_response.raw_response = {"role": "assistant", "content": "valid"}
valid_response.prompt_tokens = 100
valid_response.completion_tokens = 50
call_count = 0
async def mock_llm_call(**kwargs):
nonlocal call_count
# Capture conversation state
conversation_snapshots.append(kwargs.get("prompt", []).copy())
call_count += 1
if call_count == 1:
return invalid_response
else:
return valid_response
# Mock the LLM call
with patch(
"backend.blocks.smart_decision_maker.llm.llm_call", new_callable=AsyncMock
) as mock_llm:
mock_llm.side_effect = mock_llm_call
# Mock the function signature creation
with patch.object(
block, "_create_function_signature", new_callable=AsyncMock
) as mock_sig:
mock_sig.return_value = [
{
"type": "function",
"function": {
"name": "test_tool",
"parameters": {
"type": "object",
"properties": {
"correct_param": {
"type": "string",
"description": "The correct parameter",
}
},
"required": ["correct_param"],
},
},
}
]
# Create input data
from backend.blocks import llm
input_data = block.input_schema(
prompt="Test prompt",
credentials=llm.TEST_CREDENTIALS_INPUT,
model=llm.LlmModel.GPT4O,
retry=3, # Allow retries
)
# Run the block
outputs = {}
async for output_name, output_value in block.run(
input_data,
credentials=llm.TEST_CREDENTIALS,
graph_id="test_graph",
node_id="test_node",
graph_exec_id="test_exec",
node_exec_id="test_node_exec",
user_id="test_user",
):
outputs[output_name] = output_value
# Verify we had 2 LLM calls (initial + retry)
assert call_count == 2
# Check the final conversation output
final_conversation = outputs.get("conversations", [])
# The final conversation should NOT contain the validation error message
error_messages = [
msg
for msg in final_conversation
if msg.get("role") == "user"
and "parameter errors" in msg.get("content", "")
]
assert (
len(error_messages) == 0
), "Validation error leaked into final conversation"
# The final conversation should only have the successful response
assert final_conversation[-1]["content"] == "valid"

98
autogpt_platform/backend/backend/blocks/text.py
View File

@@ -2,6 +2,8 @@ import re
from pathlib import Path
from typing import Any
import regex # Has built-in timeout support
from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
from backend.data.model import SchemaField
from backend.util import json, text
@@ -137,6 +139,11 @@ class ExtractTextInformationBlock(Block):
)
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
# Security fix: Add limits to prevent ReDoS and memory exhaustion
MAX_TEXT_LENGTH = 1_000_000 # 1MB character limit
MAX_MATCHES = 1000 # Maximum number of matches to prevent memory exhaustion
MAX_MATCH_LENGTH = 10_000 # Maximum length per match
flags = 0
if not input_data.case_sensitive:
flags = flags | re.IGNORECASE
@@ -148,20 +155,85 @@ class ExtractTextInformationBlock(Block):
else:
txt = json.dumps(input_data.text)
matches = [
match.group(input_data.group)
for match in re.finditer(input_data.pattern, txt, flags)
if input_data.group <= len(match.groups())
]
if not input_data.find_all:
matches = matches[:1]
for match in matches:
yield "positive", match
if not matches:
yield "negative", input_data.text
# Limit text size to prevent DoS
if len(txt) > MAX_TEXT_LENGTH:
txt = txt[:MAX_TEXT_LENGTH]
yield "matched_results", matches
yield "matched_count", len(matches)
# Validate regex pattern to prevent dangerous patterns
dangerous_patterns = [
r".*\+.*\+", # Nested quantifiers
r".*\*.*\*", # Nested quantifiers
r"(?=.*\+)", # Lookahead with quantifier
r"(?=.*\*)", # Lookahead with quantifier
r"\(.+\)\+", # Group with nested quantifier
r"\(.+\)\*", # Group with nested quantifier
r"\([^)]+\+\)\+", # Nested quantifiers like (a+)+
r"\([^)]+\*\)\*", # Nested quantifiers like (a*)*
]
# Check if pattern is potentially dangerous
is_dangerous = any(
re.search(dangerous, input_data.pattern) for dangerous in dangerous_patterns
)
# Use regex module with timeout for dangerous patterns
# For safe patterns, use standard re module for compatibility
try:
matches = []
match_count = 0
if is_dangerous:
# Use regex module with timeout (5 seconds) for dangerous patterns
# The regex module supports timeout parameter in finditer
try:
for match in regex.finditer(
input_data.pattern, txt, flags=flags, timeout=5.0
):
if match_count >= MAX_MATCHES:
break
if input_data.group <= len(match.groups()):
match_text = match.group(input_data.group)
# Limit match length to prevent memory exhaustion
if len(match_text) > MAX_MATCH_LENGTH:
match_text = match_text[:MAX_MATCH_LENGTH]
matches.append(match_text)
match_count += 1
except regex.error as e:
# Timeout occurred or regex error
if "timeout" in str(e).lower():
# Timeout - return empty results
pass
else:
# Other regex error
raise
else:
# Use standard re module for non-dangerous patterns
for match in re.finditer(input_data.pattern, txt, flags):
if match_count >= MAX_MATCHES:
break
if input_data.group <= len(match.groups()):
match_text = match.group(input_data.group)
# Limit match length to prevent memory exhaustion
if len(match_text) > MAX_MATCH_LENGTH:
match_text = match_text[:MAX_MATCH_LENGTH]
matches.append(match_text)
match_count += 1
if not input_data.find_all:
matches = matches[:1]
for match in matches:
yield "positive", match
if not matches:
yield "negative", input_data.text
yield "matched_results", matches
yield "matched_count", len(matches)
except Exception:
# Return empty results on any regex error
yield "negative", input_data.text
yield "matched_results", []
yield "matched_count", 0
class FillTextTemplateBlock(Block):

14
autogpt_platform/backend/backend/blocks/time_blocks.py
View File

@@ -270,13 +270,17 @@ class GetCurrentDateBlock(Block):
test_output=[
(
"date",
lambda t: abs(datetime.now() - datetime.strptime(t, "%Y-%m-%d"))
< timedelta(days=8), # 7 days difference + 1 day error margin.
lambda t: abs(
datetime.now().date() - datetime.strptime(t, "%Y-%m-%d").date()
)
<= timedelta(days=8), # 7 days difference + 1 day error margin.
),
(
"date",
lambda t: abs(datetime.now() - datetime.strptime(t, "%m/%d/%Y"))
< timedelta(days=8),
lambda t: abs(
datetime.now().date() - datetime.strptime(t, "%m/%d/%Y").date()
)
<= timedelta(days=8),
# 7 days difference + 1 day error margin.
),
(
@@ -382,7 +386,7 @@ class GetCurrentDateAndTimeBlock(Block):
lambda t: abs(
datetime.now().date() - datetime.strptime(t, "%Y/%m/%d").date()
)
< timedelta(days=1), # Date format only, no time component
<= timedelta(days=1), # Date format only, no time component
),
(
"date_time",

8
autogpt_platform/backend/backend/blocks/xml_parser.py
View File

@@ -26,6 +26,14 @@ class XMLParserBlock(Block):
)
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
# Security fix: Add size limits to prevent XML bomb attacks
MAX_XML_SIZE = 10 * 1024 * 1024 # 10MB limit for XML input
if len(input_data.input_xml) > MAX_XML_SIZE:
raise ValueError(
f"XML too large: {len(input_data.input_xml)} bytes > {MAX_XML_SIZE} bytes"
)
try:
tokens = tokenize(input_data.input_xml)
parser = Parser(tokens)

284
autogpt_platform/backend/backend/data/dynamic_fields.py Normal file
View File

@@ -0,0 +1,284 @@
"""
Utilities for handling dynamic field names with special delimiters.
Dynamic fields allow graphs to connect complex data structures using special delimiters:
- _#_ for dictionary keys (e.g., "values_#_name" → values["name"])
- _$_ for list indices (e.g., "items_$_0" → items[0])
- _@_ for object attributes (e.g., "obj_@_attr" → obj.attr)
"""
from typing import Any
from backend.util.mock import MockObject
# Dynamic field delimiters
LIST_SPLIT = "_$_"
DICT_SPLIT = "_#_"
OBJC_SPLIT = "_@_"
DYNAMIC_DELIMITERS = (LIST_SPLIT, DICT_SPLIT, OBJC_SPLIT)
def extract_base_field_name(field_name: str) -> str:
"""
Extract the base field name from a dynamic field name by removing all dynamic suffixes.
Examples:
extract_base_field_name("values_#_name") → "values"
extract_base_field_name("items_$_0") → "items"
extract_base_field_name("obj_@_attr") → "obj"
extract_base_field_name("regular_field") → "regular_field"
Args:
field_name: The field name that may contain dynamic delimiters
Returns:
The base field name without any dynamic suffixes
"""
base_name = field_name
for delimiter in DYNAMIC_DELIMITERS:
if delimiter in base_name:
base_name = base_name.split(delimiter)[0]
return base_name
def is_dynamic_field(field_name: str) -> bool:
"""
Check if a field name contains dynamic delimiters.
Args:
field_name: The field name to check
Returns:
True if the field contains any dynamic delimiters, False otherwise
"""
return any(delimiter in field_name for delimiter in DYNAMIC_DELIMITERS)
def get_dynamic_field_description(field_name: str) -> str:
"""
Generate a description for a dynamic field based on its structure.
Args:
field_name: The full dynamic field name (e.g., "values_#_name")
Returns:
A descriptive string explaining what this dynamic field represents
"""
base_name = extract_base_field_name(field_name)
if DICT_SPLIT in field_name:
# Extract the key part after _#_
parts = field_name.split(DICT_SPLIT)
if len(parts) > 1:
key = parts[1].split("_")[0] if "_" in parts[1] else parts[1]
return f"Dictionary field '{key}' for base field '{base_name}' ({base_name}['{key}'])"
elif LIST_SPLIT in field_name:
# Extract the index part after _$_
parts = field_name.split(LIST_SPLIT)
if len(parts) > 1:
index = parts[1].split("_")[0] if "_" in parts[1] else parts[1]
return (
f"List item {index} for base field '{base_name}' ({base_name}[{index}])"
)
elif OBJC_SPLIT in field_name:
# Extract the attribute part after _@_
parts = field_name.split(OBJC_SPLIT)
if len(parts) > 1:
# Get the full attribute name (everything after _@_)
attr = parts[1]
return f"Object attribute '{attr}' for base field '{base_name}' ({base_name}.{attr})"
return f"Value for {field_name}"
# --------------------------------------------------------------------------- #
# Dynamic field parsing and merging utilities
# --------------------------------------------------------------------------- #
def _next_delim(s: str) -> tuple[str | None, int]:
"""
Return the *earliest* delimiter appearing in `s` and its index.
If none present → (None, -1).
"""
first: str | None = None
pos = len(s) # sentinel: larger than any real index
for d in DYNAMIC_DELIMITERS:
i = s.find(d)
if 0 <= i < pos:
first, pos = d, i
return first, (pos if first else -1)
def _tokenise(path: str) -> list[tuple[str, str]] | None:
"""
Convert the raw path string (starting with a delimiter) into
[ (delimiter, identifier), … ] or None if the syntax is malformed.
"""
tokens: list[tuple[str, str]] = []
while path:
# 1. Which delimiter starts this chunk?
delim = next((d for d in DYNAMIC_DELIMITERS if path.startswith(d)), None)
if delim is None:
return None # invalid syntax
# 2. Slice off the delimiter, then up to the next delimiter (or EOS)
path = path[len(delim) :]
nxt_delim, pos = _next_delim(path)
token, path = (
path[: pos if pos != -1 else len(path)],
path[pos if pos != -1 else len(path) :],
)
if token == "":
return None # empty identifier is invalid
tokens.append((delim, token))
return tokens
def parse_execution_output(output: tuple[str, Any], name: str) -> Any:
"""
Retrieve a nested value out of `output` using the flattened *name*.
On any failure (wrong name, wrong type, out-of-range, bad path)
returns **None**.
Args:
output: Tuple of (base_name, data) representing a block output entry
name: The flattened field name to extract from the output data
Returns:
The value at the specified path, or None if not found/invalid
"""
base_name, data = output
# Exact match → whole object
if name == base_name:
return data
# Must start with the expected name
if not name.startswith(base_name):
return None
path = name[len(base_name) :]
if not path:
return None # nothing left to parse
tokens = _tokenise(path)
if tokens is None:
return None
cur: Any = data
for delim, ident in tokens:
if delim == LIST_SPLIT:
# list[index]
try:
idx = int(ident)
except ValueError:
return None
if not isinstance(cur, list) or idx >= len(cur):
return None
cur = cur[idx]
elif delim == DICT_SPLIT:
if not isinstance(cur, dict) or ident not in cur:
return None
cur = cur[ident]
elif delim == OBJC_SPLIT:
if not hasattr(cur, ident):
return None
cur = getattr(cur, ident)
else:
return None # unreachable
return cur
def _assign(container: Any, tokens: list[tuple[str, str]], value: Any) -> Any:
"""
Recursive helper that *returns* the (possibly new) container with
`value` assigned along the remaining `tokens` path.
"""
if not tokens:
return value # leaf reached
delim, ident = tokens[0]
rest = tokens[1:]
# ---------- list ----------
if delim == LIST_SPLIT:
try:
idx = int(ident)
except ValueError:
raise ValueError("index must be an integer")
if container is None:
container = []
elif not isinstance(container, list):
container = list(container) if hasattr(container, "__iter__") else []
while len(container) <= idx:
container.append(None)
container[idx] = _assign(container[idx], rest, value)
return container
# ---------- dict ----------
if delim == DICT_SPLIT:
if container is None:
container = {}
elif not isinstance(container, dict):
container = dict(container) if hasattr(container, "items") else {}
container[ident] = _assign(container.get(ident), rest, value)
return container
# ---------- object ----------
if delim == OBJC_SPLIT:
if container is None:
container = MockObject()
elif not hasattr(container, "__dict__"):
# If it's not an object, create a new one
container = MockObject()
setattr(
container,
ident,
_assign(getattr(container, ident, None), rest, value),
)
return container
return value # unreachable
def merge_execution_input(data: dict[str, Any]) -> dict[str, Any]:
"""
Reconstruct nested objects from a *flattened* dict of key → value.
Raises ValueError on syntactically invalid list indices.
Args:
data: Dictionary with potentially flattened dynamic field keys
Returns:
Dictionary with nested objects reconstructed from flattened keys
"""
merged: dict[str, Any] = {}
for key, value in data.items():
# Split off the base name (before the first delimiter, if any)
delim, pos = _next_delim(key)
if delim is None:
merged[key] = value
continue
base, path = key[:pos], key[pos:]
tokens = _tokenise(path)
if tokens is None:
# Invalid key; treat as scalar under the raw name
merged[key] = value
continue
merged[base] = _assign(merged.get(base), tokens, value)
data.update(merged)
return data

42
autogpt_platform/backend/backend/data/execution.py
View File

@@ -478,6 +478,48 @@ async def get_graph_executions(
return [GraphExecutionMeta.from_db(execution) for execution in executions]
async def get_graph_executions_count(
user_id: Optional[str] = None,
graph_id: Optional[str] = None,
statuses: Optional[list[ExecutionStatus]] = None,
created_time_gte: Optional[datetime] = None,
created_time_lte: Optional[datetime] = None,
) -> int:
"""
Get count of graph executions with optional filters.
Args:
user_id: Optional user ID to filter by
graph_id: Optional graph ID to filter by
statuses: Optional list of execution statuses to filter by
created_time_gte: Optional minimum creation time
created_time_lte: Optional maximum creation time
Returns:
Count of matching graph executions
"""
where_filter: AgentGraphExecutionWhereInput = {
"isDeleted": False,
}
if user_id:
where_filter["userId"] = user_id
if graph_id:
where_filter["agentGraphId"] = graph_id
if created_time_gte or created_time_lte:
where_filter["createdAt"] = {
"gte": created_time_gte or datetime.min.replace(tzinfo=timezone.utc),
"lte": created_time_lte or datetime.max.replace(tzinfo=timezone.utc),
}
if statuses:
where_filter["OR"] = [{"executionStatus": status} for status in statuses]
count = await AgentGraphExecution.prisma().count(where=where_filter)
return count
class GraphExecutionsPaginated(BaseModel):
"""Response schema for paginated graph executions."""

53
autogpt_platform/backend/backend/data/graph.py
View File

@@ -20,6 +20,7 @@ from backend.blocks.agent import AgentExecutorBlock
from backend.blocks.io import AgentInputBlock, AgentOutputBlock
from backend.blocks.llm import LlmModel
from backend.data.db import prisma as db
from backend.data.dynamic_fields import extract_base_field_name
from backend.data.includes import MAX_GRAPH_VERSIONS_FETCH
from backend.data.model import (
CredentialsField,
@@ -32,7 +33,15 @@ from backend.util import type as type_utils
from backend.util.json import SafeJson
from backend.util.models import Pagination
from .block import Block, BlockInput, BlockSchema, BlockType, get_block, get_blocks
from .block import (
Block,
BlockInput,
BlockSchema,
BlockType,
EmptySchema,
get_block,
get_blocks,
)
from .db import BaseDbModel, query_raw_with_schema, transaction
from .includes import AGENT_GRAPH_INCLUDE, AGENT_NODE_INCLUDE
@@ -73,12 +82,15 @@ class Node(BaseDbModel):
output_links: list[Link] = []
@property
def block(self) -> Block[BlockSchema, BlockSchema]:
def block(self) -> "Block[BlockSchema, BlockSchema] | _UnknownBlockBase":
"""Get the block for this node. Returns UnknownBlock if block is deleted/missing."""
block = get_block(self.block_id)
if not block:
raise ValueError(
f"Block #{self.block_id} does not exist -> Node #{self.id} is invalid"
# Log warning but don't raise exception - return a placeholder block for deleted blocks
logger.warning(
f"Block #{self.block_id} does not exist for Node #{self.id} (deleted/missing block), using UnknownBlock"
)
return _UnknownBlockBase(self.block_id)
return block
@@ -730,7 +742,7 @@ def _is_tool_pin(name: str) -> bool:
def _sanitize_pin_name(name: str) -> str:
sanitized_name = name.split("_#_")[0].split("_@_")[0].split("_$_")[0]
sanitized_name = extract_base_field_name(name)
if _is_tool_pin(sanitized_name):
return "tools"
return sanitized_name
@@ -1316,3 +1328,34 @@ async def migrate_llm_models(migrate_to: LlmModel):
id,
path,
)
# Simple placeholder class for deleted/missing blocks
class _UnknownBlockBase(Block):
"""
Placeholder for deleted/missing blocks that inherits from Block
but uses a name that doesn't end with 'Block' to avoid auto-discovery.
"""
def __init__(self, block_id: str = "00000000-0000-0000-0000-000000000000"):
# Initialize with minimal valid Block parameters
super().__init__(
id=block_id,
description=f"Unknown or deleted block (original ID: {block_id})",
disabled=True,
input_schema=EmptySchema,
output_schema=EmptySchema,
categories=set(),
contributors=[],
static_output=False,
block_type=BlockType.STANDARD,
webhook_config=None,
)
@property
def name(self):
return "UnknownBlock"
async def run(self, input_data, **kwargs):
"""Always yield an error for missing blocks."""
yield "error", f"Block {self.id} no longer exists"

76
autogpt_platform/backend/backend/data/notifications.py
View File

@@ -235,6 +235,7 @@ class BaseEventModel(BaseModel):
class NotificationEventModel(BaseEventModel, Generic[NotificationDataType_co]):
id: Optional[str] = None # None when creating, populated when reading from DB
data: NotificationDataType_co
@property
@@ -378,6 +379,7 @@ class NotificationPreference(BaseModel):
class UserNotificationEventDTO(BaseModel):
id: str # Added to track notifications for removal
type: NotificationType
data: dict
created_at: datetime
@@ -386,6 +388,7 @@ class UserNotificationEventDTO(BaseModel):
@staticmethod
def from_db(model: NotificationEvent) -> "UserNotificationEventDTO":
return UserNotificationEventDTO(
id=model.id,
type=model.type,
data=dict(model.data),
created_at=model.createdAt,
@@ -541,6 +544,79 @@ async def empty_user_notification_batch(
) from e
async def clear_all_user_notification_batches(user_id: str) -> None:
"""Clear ALL notification batches for a user across all types.
Used when user's email is bounced/inactive and we should stop
trying to send them ANY emails.
"""
try:
# Delete all notification events for this user
await NotificationEvent.prisma().delete_many(
where={"UserNotificationBatch": {"is": {"userId": user_id}}}
)
# Delete all batches for this user
await UserNotificationBatch.prisma().delete_many(where={"userId": user_id})
logger.info(f"Cleared all notification batches for user {user_id}")
except Exception as e:
raise DatabaseError(
f"Failed to clear all notification batches for user {user_id}: {e}"
) from e
async def remove_notifications_from_batch(
user_id: str, notification_type: NotificationType, notification_ids: list[str]
) -> None:
"""Remove specific notifications from a user's batch by their IDs.
This is used after successful sending to remove only the
sent notifications, preventing duplicates on retry.
"""
if not notification_ids:
return
try:
# Delete the specific notification events
deleted_count = await NotificationEvent.prisma().delete_many(
where={
"id": {"in": notification_ids},
"UserNotificationBatch": {
"is": {"userId": user_id, "type": notification_type}
},
}
)
logger.info(
f"Removed {deleted_count} notifications from batch for user {user_id}"
)
# Check if batch is now empty and delete it if so
remaining = await NotificationEvent.prisma().count(
where={
"UserNotificationBatch": {
"is": {"userId": user_id, "type": notification_type}
}
}
)
if remaining == 0:
await UserNotificationBatch.prisma().delete_many(
where=UserNotificationBatchWhereInput(
userId=user_id,
type=notification_type,
)
)
logger.info(
f"Deleted empty batch for user {user_id} and type {notification_type}"
)
except Exception as e:
raise DatabaseError(
f"Failed to remove notifications from batch for user {user_id} and type {notification_type}: {e}"
) from e
async def get_user_notification_batch(
user_id: str,
notification_type: NotificationType,

70
autogpt_platform/backend/backend/data/onboarding.py
View File

@@ -4,13 +4,13 @@ from typing import Any, Optional
import prisma
import pydantic
from autogpt_libs.utils.cache import cached
from prisma.enums import OnboardingStep
from prisma.models import UserOnboarding
from prisma.types import UserOnboardingCreateInput, UserOnboardingUpdateInput
from backend.data.block import get_blocks
from backend.data.credit import get_user_credit_model
from backend.data.graph import GraphModel
from backend.data.model import CredentialsMetaInput
from backend.server.v2.store.model import StoreAgentDetails
from backend.util.json import SafeJson
@@ -277,8 +277,14 @@ async def get_recommended_agents(user_id: str) -> list[StoreAgentDetails]:
for word in user_onboarding.integrations
]
where_clause["is_available"] = True
# Try to take only agents that are available and allowed for onboarding
storeAgents = await prisma.models.StoreAgent.prisma().find_many(
where=prisma.types.StoreAgentWhereInput(**where_clause),
where={
"is_available": True,
"useForOnboarding": True,
},
order=[
{"featured": "desc"},
{"runs": "desc"},
@@ -287,59 +293,16 @@ async def get_recommended_agents(user_id: str) -> list[StoreAgentDetails]:
take=100,
)
agentListings = await prisma.models.StoreListingVersion.prisma().find_many(
where={
"id": {"in": [agent.storeListingVersionId for agent in storeAgents]},
},
include={"AgentGraph": True},
)
for listing in agentListings:
agent = listing.AgentGraph
if agent is None:
continue
graph = GraphModel.from_db(agent)
# Remove agents with empty input schema
if not graph.input_schema:
storeAgents = [
a for a in storeAgents if a.storeListingVersionId != listing.id
]
continue
# Remove agents with empty credentials
# Get nodes from this agent that have credentials
nodes = await prisma.models.AgentNode.prisma().find_many(
where={
"agentGraphId": agent.id,
"agentBlockId": {"in": list(CREDENTIALS_FIELDS.keys())},
},
)
for node in nodes:
block_id = node.agentBlockId
field_name = CREDENTIALS_FIELDS[block_id]
# If there are no credentials or they are empty, remove the agent
# FIXME ignores default values
if (
field_name not in node.constantInput
or node.constantInput[field_name] is None
):
storeAgents = [
a for a in storeAgents if a.storeListingVersionId != listing.id
]
break
# If there are less than 2 agents, add more agents to the list
# If not enough agents found, relax the useForOnboarding filter
if len(storeAgents) < 2:
storeAgents += await prisma.models.StoreAgent.prisma().find_many(
where={
"listing_id": {"not_in": [agent.listing_id for agent in storeAgents]},
},
storeAgents = await prisma.models.StoreAgent.prisma().find_many(
where=prisma.types.StoreAgentWhereInput(**where_clause),
order=[
{"featured": "desc"},
{"runs": "desc"},
{"rating": "desc"},
],
take=2 - len(storeAgents),
take=100,
)
# Calculate points for the first X agents and choose the top 2
@@ -374,8 +337,13 @@ async def get_recommended_agents(user_id: str) -> list[StoreAgentDetails]:
]
@cached(maxsize=1, ttl_seconds=300) # Cache for 5 minutes since this rarely changes
async def onboarding_enabled() -> bool:
"""
Check if onboarding should be enabled based on store agent count.
Cached to prevent repeated slow database queries.
"""
# Use a more efficient query that stops counting after finding enough agents
count = await prisma.models.StoreAgent.prisma().count(take=MIN_AGENT_COUNT + 1)
# Onboading is enabled if there are at least 2 agents in the store
# Onboarding is enabled if there are at least 2 agents in the store
return count >= MIN_AGENT_COUNT

30
autogpt_platform/backend/backend/data/user.py
View File

@@ -354,6 +354,36 @@ async def set_user_email_verification(user_id: str, verified: bool) -> None:
) from e
async def disable_all_user_notifications(user_id: str) -> None:
"""Disable all notification preferences for a user.
Used when user's email bounces/is inactive to prevent any future notifications.
"""
try:
await PrismaUser.prisma().update(
where={"id": user_id},
data={
"notifyOnAgentRun": False,
"notifyOnZeroBalance": False,
"notifyOnLowBalance": False,
"notifyOnBlockExecutionFailed": False,
"notifyOnContinuousAgentError": False,
"notifyOnDailySummary": False,
"notifyOnWeeklySummary": False,
"notifyOnMonthlySummary": False,
"notifyOnAgentApproved": False,
"notifyOnAgentRejected": False,
},
)
# Invalidate cache for this user
get_user_by_id.cache_delete(user_id)
logger.info(f"Disabled all notification preferences for user {user_id}")
except Exception as e:
raise DatabaseError(
f"Failed to disable notifications for user {user_id}: {e}"
) from e
async def get_user_email_verification(user_id: str) -> bool:
"""Get the email verification status for a user."""
try:

59
autogpt_platform/backend/backend/executor/activity_status_generator.py
View File

@@ -4,7 +4,12 @@ Module for generating AI-based activity status for graph executions.
import json
import logging
from typing import TYPE_CHECKING, Any, NotRequired, TypedDict
from typing import TYPE_CHECKING, Any, TypedDict
try:
from typing import NotRequired
except ImportError:
from typing_extensions import NotRequired
from pydantic import SecretStr
@@ -146,17 +151,35 @@ async def generate_activity_status_for_execution(
"Focus on the ACTUAL TASK the user wanted done, not the internal workflow steps. "
"Avoid technical terms like 'workflow', 'execution', 'components', 'nodes', 'processing', etc. "
"Keep it to 3 sentences maximum. Be conversational and human-friendly.\n\n"
"UNDERSTAND THE INTENDED PURPOSE:\n"
"- FIRST: Read the graph description carefully to understand what the user wanted to accomplish\n"
"- The graph name and description tell you the main goal/intention of this automation\n"
"- Use this intended purpose as your PRIMARY criteria for success/failure evaluation\n"
"- Ask yourself: 'Did this execution actually accomplish what the graph was designed to do?'\n\n"
"CRITICAL OUTPUT ANALYSIS:\n"
"- Check if blocks that should produce user-facing results actually produced outputs\n"
"- Blocks with names containing 'Output', 'Post', 'Create', 'Send', 'Publish', 'Generate' are usually meant to produce final results\n"
"- If these critical blocks have NO outputs (empty recent_outputs), the task likely FAILED even if status shows 'completed'\n"
"- Sub-agents (AgentExecutorBlock) that produce no outputs usually indicate failed sub-tasks\n"
"- Most importantly: Does the execution result match what the graph description promised to deliver?\n\n"
"SUCCESS EVALUATION BASED ON INTENTION:\n"
"- If the graph is meant to 'create blog posts' → check if blog content was actually created\n"
"- If the graph is meant to 'send emails' → check if emails were actually sent\n"
"- If the graph is meant to 'analyze data' → check if analysis results were produced\n"
"- If the graph is meant to 'generate reports' → check if reports were generated\n"
"- Technical completion ≠ goal achievement. Focus on whether the USER'S INTENDED OUTCOME was delivered\n\n"
"IMPORTANT: Be HONEST about what actually happened:\n"
"- If the input was invalid/nonsensical, say so directly\n"
"- If the task failed, explain what went wrong in simple terms\n"
"- If errors occurred, focus on what the user needs to know\n"
"- Only claim success if the task was genuinely completed\n"
"- Don't sugar-coat failures or present them as helpful feedback\n\n"
"- Only claim success if the INTENDED PURPOSE was genuinely accomplished AND produced expected outputs\n"
"- Don't sugar-coat failures or present them as helpful feedback\n"
"- ESPECIALLY: If the graph's main purpose wasn't achieved, this is a failure regardless of 'completed' status\n\n"
"Understanding Errors:\n"
"- Node errors: Individual steps may fail but the overall task might still complete (e.g., one data source fails but others work)\n"
"- Graph error (in overall_status.graph_error): This means the entire execution failed and nothing was accomplished\n"
"- Even if execution shows 'completed', check if critical nodes failed that would prevent the desired outcome\n"
"- Focus on the end result the user wanted, not whether technical steps completed"
"- Missing outputs from critical blocks: Even if no errors, this means the task failed to produce expected results\n"
"- Focus on whether the graph's intended purpose was fulfilled, not whether technical steps completed"
),
},
{
@@ -165,15 +188,28 @@ async def generate_activity_status_for_execution(
f"A user ran '{graph_name}' to accomplish something. Based on this execution data, "
f"write what they achieved in simple, user-friendly terms:\n\n"
f"{json.dumps(execution_data, indent=2)}\n\n"
"CRITICAL: Check overall_status.graph_error FIRST - if present, the entire execution failed.\n"
"Then check individual node errors to understand partial failures.\n\n"
"ANALYSIS CHECKLIST:\n"
"1. READ graph_info.description FIRST - this tells you what the user intended to accomplish\n"
"2. Check overall_status.graph_error - if present, the entire execution failed\n"
"3. Look for nodes with 'Output', 'Post', 'Create', 'Send', 'Publish', 'Generate' in their block_name\n"
"4. Check if these critical blocks have empty recent_outputs arrays - this indicates failure\n"
"5. Look for AgentExecutorBlock (sub-agents) with no outputs - this suggests sub-task failures\n"
"6. Count how many nodes produced outputs vs total nodes - low ratio suggests problems\n"
"7. MOST IMPORTANT: Does the execution outcome match what graph_info.description promised?\n\n"
"INTENTION-BASED EVALUATION:\n"
"- If description mentions 'blog writing' → did it create blog content?\n"
"- If description mentions 'email automation' → were emails actually sent?\n"
"- If description mentions 'data analysis' → were analysis results produced?\n"
"- If description mentions 'content generation' → was content actually generated?\n"
"- If description mentions 'social media posting' → were posts actually made?\n"
"- Match the outputs to the stated intention, not just technical completion\n\n"
"Write 1-3 sentences about what the user accomplished, such as:\n"
"- 'I analyzed your resume and provided detailed feedback for the IT industry.'\n"
"- 'I couldn't analyze your resume because the input was just nonsensical text.'\n"
"- 'I failed to complete the task due to missing API access.'\n"
"- 'I couldn't complete the task because critical steps failed to produce any results.'\n"
"- 'I failed to generate the content you requested due to missing API access.'\n"
"- 'I extracted key information from your documents and organized it into a summary.'\n"
"- 'The task failed to run due to system configuration issues.'\n\n"
"Focus on what ACTUALLY happened, not what was attempted."
"- 'The task failed because the blog post creation step didn't produce any output.'\n\n"
"BE CRITICAL: If the graph's intended purpose (from description) wasn't achieved, report this as a failure even if status is 'completed'."
),
},
]
@@ -197,6 +233,7 @@ async def generate_activity_status_for_execution(
logger.debug(
f"Generated activity status for {graph_exec_id}: {activity_status}"
)
return activity_status
except Exception as e:

10
autogpt_platform/backend/backend/executor/database.py
View File

@@ -9,6 +9,7 @@ from backend.data.execution import (
get_execution_kv_data,
get_graph_execution_meta,
get_graph_executions,
get_graph_executions_count,
get_latest_node_execution,
get_node_execution,
get_node_executions,
@@ -28,11 +29,13 @@ from backend.data.graph import (
get_node,
)
from backend.data.notifications import (
clear_all_user_notification_batches,
create_or_add_to_user_notification_batch,
empty_user_notification_batch,
get_all_batches_by_type,
get_user_notification_batch,
get_user_notification_oldest_message_in_batch,
remove_notifications_from_batch,
)
from backend.data.user import (
get_active_user_ids_in_timerange,
@@ -71,7 +74,6 @@ async def _get_credits(user_id: str) -> int:
class DatabaseManager(AppService):
def run_service(self) -> None:
logger.info(f"[{self.service_name}] ⏳ Connecting to Database...")
self.run_and_wait(db.connect())
@@ -111,6 +113,7 @@ class DatabaseManager(AppService):
# Executions
get_graph_executions = _(get_graph_executions)
get_graph_executions_count = _(get_graph_executions_count)
get_graph_execution_meta = _(get_graph_execution_meta)
create_graph_execution = _(create_graph_execution)
get_node_execution = _(get_node_execution)
@@ -147,10 +150,12 @@ class DatabaseManager(AppService):
get_user_notification_preference = _(get_user_notification_preference)
# Notifications - async
clear_all_user_notification_batches = _(clear_all_user_notification_batches)
create_or_add_to_user_notification_batch = _(
create_or_add_to_user_notification_batch
)
empty_user_notification_batch = _(empty_user_notification_batch)
remove_notifications_from_batch = _(remove_notifications_from_batch)
get_all_batches_by_type = _(get_all_batches_by_type)
get_user_notification_batch = _(get_user_notification_batch)
get_user_notification_oldest_message_in_batch = _(
@@ -179,6 +184,7 @@ class DatabaseManagerClient(AppServiceClient):
# Executions
get_graph_executions = _(d.get_graph_executions)
get_graph_executions_count = _(d.get_graph_executions_count)
get_graph_execution_meta = _(d.get_graph_execution_meta)
get_node_executions = _(d.get_node_executions)
update_node_execution_status = _(d.update_node_execution_status)
@@ -241,10 +247,12 @@ class DatabaseManagerAsyncClient(AppServiceClient):
get_user_notification_preference = d.get_user_notification_preference
# Notifications
clear_all_user_notification_batches = d.clear_all_user_notification_batches
create_or_add_to_user_notification_batch = (
d.create_or_add_to_user_notification_batch
)
empty_user_notification_batch = d.empty_user_notification_batch
remove_notifications_from_batch = d.remove_notifications_from_batch
get_all_batches_by_type = d.get_all_batches_by_type
get_user_notification_batch = d.get_user_notification_batch
get_user_notification_oldest_message_in_batch = (

70
autogpt_platform/backend/backend/executor/manager.py
View File

@@ -7,6 +7,7 @@ import uuid
from collections import defaultdict
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
from pika.adapters.blocking_connection import BlockingChannel
@@ -25,6 +26,7 @@ from backend.data.block import (
get_block,
)
from backend.data.credit import UsageTransactionMetadata
from backend.data.dynamic_fields import parse_execution_output
from backend.data.execution import (
ExecutionQueue,
ExecutionStatus,
@@ -59,7 +61,6 @@ from backend.executor.utils import (
block_usage_cost,
create_execution_queue_config,
execution_usage_cost,
parse_execution_output,
validate_exec,
)
from backend.integrations.creds_manager import IntegrationCredentialsManager
@@ -84,7 +85,11 @@ from backend.util.file import clean_exec_files
from backend.util.logging import TruncatedLogger, configure_logging
from backend.util.metrics import DiscordChannel
from backend.util.process import AppProcess, set_service_name
from backend.util.retry import continuous_retry, func_retry
from backend.util.retry import (
continuous_retry,
func_retry,
send_rate_limited_discord_alert,
)
from backend.util.settings import Settings
from .cluster_lock import ClusterLock
@@ -184,6 +189,7 @@ async def execute_node(
_input_data.inputs = input_data
if nodes_input_masks:
_input_data.nodes_input_masks = nodes_input_masks
_input_data.user_id = user_id
input_data = _input_data.model_dump()
data.inputs = input_data
@@ -979,16 +985,31 @@ class ExecutionProcessor:
if isinstance(e, Exception)
else Exception(f"{e.__class__.__name__}: {e}")
)
if not execution_stats.error:
execution_stats.error = str(error)
known_errors = (InsufficientBalanceError, ModerationError)
if isinstance(error, known_errors):
execution_stats.error = str(error)
return ExecutionStatus.FAILED
execution_status = ExecutionStatus.FAILED
log_metadata.exception(
f"Failed graph execution {graph_exec.graph_exec_id}: {error}"
)
# Send rate-limited Discord alert for unknown/unexpected errors
send_rate_limited_discord_alert(
"graph_execution",
error,
"unknown_error",
f"🚨 **Unknown Graph Execution Error**\n"
f"User: {graph_exec.user_id}\n"
f"Graph ID: {graph_exec.graph_id}\n"
f"Execution ID: {graph_exec.graph_exec_id}\n"
f"Error Type: {type(error).__name__}\n"
f"Error: {str(error)[:200]}{'...' if len(str(error)) > 200 else ''}\n",
)
raise
finally:
@@ -1163,9 +1184,9 @@ class ExecutionProcessor:
f"❌ **Insufficient Funds Alert**\n"
f"User: {user_email or user_id}\n"
f"Agent: {metadata.name if metadata else 'Unknown Agent'}\n"
f"Current balance: ${e.balance/100:.2f}\n"
f"Attempted cost: ${abs(e.amount)/100:.2f}\n"
f"Shortfall: ${abs(shortfall)/100:.2f}\n"
f"Current balance: ${e.balance / 100:.2f}\n"
f"Attempted cost: ${abs(e.amount) / 100:.2f}\n"
f"Shortfall: ${abs(shortfall) / 100:.2f}\n"
f"[View User Details]({base_url}/admin/spending?search={user_email})"
)
@@ -1212,9 +1233,9 @@ class ExecutionProcessor:
alert_message = (
f"⚠️ **Low Balance Alert**\n"
f"User: {user_email or user_id}\n"
f"Balance dropped below ${LOW_BALANCE_THRESHOLD/100:.2f}\n"
f"Current balance: ${current_balance/100:.2f}\n"
f"Transaction cost: ${transaction_cost/100:.2f}\n"
f"Balance dropped below ${LOW_BALANCE_THRESHOLD / 100:.2f}\n"
f"Current balance: ${current_balance / 100:.2f}\n"
f"Transaction cost: ${transaction_cost / 100:.2f}\n"
f"[View User Details]({base_url}/admin/spending?search={user_email})"
)
get_notification_manager_client().discord_system_alert(
@@ -1445,10 +1466,39 @@ class ExecutionManager(AppProcess):
return
graph_exec_id = graph_exec_entry.graph_exec_id
user_id = graph_exec_entry.user_id
graph_id = graph_exec_entry.graph_id
logger.info(
f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}"
f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}, user_id={user_id}"
)
# Check user rate limit before processing
try:
# Only check executions from the last 24 hours for performance
current_running_count = get_db_client().get_graph_executions_count(
user_id=user_id,
graph_id=graph_id,
statuses=[ExecutionStatus.RUNNING],
created_time_gte=datetime.now(timezone.utc) - timedelta(hours=24),
)
if (
current_running_count
>= settings.config.max_concurrent_graph_executions_per_user
):
logger.warning(
f"[{self.service_name}] Rate limit exceeded for user {user_id} on graph {graph_id}: "
f"{current_running_count}/{settings.config.max_concurrent_graph_executions_per_user} running executions"
)
_ack_message(reject=True, requeue=True)
return
except Exception as e:
logger.error(
f"[{self.service_name}] Failed to check rate limit for user {user_id}: {e}, proceeding with execution"
)
# If rate limit check fails, proceed to avoid blocking executions
# Check for local duplicate execution first
if graph_exec_id in self.active_graph_runs:
logger.warning(

196
autogpt_platform/backend/backend/executor/utils.py
View File

@@ -5,7 +5,7 @@ import threading
import time
from collections import defaultdict
from concurrent.futures import Future
from typing import Any, Mapping, Optional, cast
from typing import Mapping, Optional, cast
from autogpt_libs.supabase_integration_credentials_store.types import APIKeyCredentials
from pydantic import BaseModel, JsonValue, SecretStr, ValidationError
@@ -22,6 +22,9 @@ from backend.data.block import (
)
from backend.data.block_cost_config import BLOCK_COSTS
from backend.data.db import prisma
# Import dynamic field utilities from centralized location
from backend.data.dynamic_fields import merge_execution_input
from backend.data.execution import (
ExecutionStatus,
GraphExecutionStats,
@@ -42,7 +45,6 @@ from backend.util.clients import (
)
from backend.util.exceptions import GraphValidationError, NotFoundError
from backend.util.logging import TruncatedLogger
from backend.util.mock import MockObject
from backend.util.settings import Config
from backend.util.type import convert
@@ -249,195 +251,7 @@ def _is_cost_filter_match(cost_filter: BlockInput, input_data: BlockInput) -> bo
# ============ Execution Input Helpers ============ #
# --------------------------------------------------------------------------- #
# Delimiters
# --------------------------------------------------------------------------- #
LIST_SPLIT = "_$_"
DICT_SPLIT = "_#_"
OBJC_SPLIT = "_@_"
_DELIMS = (LIST_SPLIT, DICT_SPLIT, OBJC_SPLIT)
# --------------------------------------------------------------------------- #
# Tokenisation utilities
# --------------------------------------------------------------------------- #
def _next_delim(s: str) -> tuple[str | None, int]:
"""
Return the *earliest* delimiter appearing in `s` and its index.
If none present → (None, -1).
"""
first: str | None = None
pos = len(s) # sentinel: larger than any real index
for d in _DELIMS:
i = s.find(d)
if 0 <= i < pos:
first, pos = d, i
return first, (pos if first else -1)
def _tokenise(path: str) -> list[tuple[str, str]] | None:
"""
Convert the raw path string (starting with a delimiter) into
[ (delimiter, identifier), … ] or None if the syntax is malformed.
"""
tokens: list[tuple[str, str]] = []
while path:
# 1. Which delimiter starts this chunk?
delim = next((d for d in _DELIMS if path.startswith(d)), None)
if delim is None:
return None # invalid syntax
# 2. Slice off the delimiter, then up to the next delimiter (or EOS)
path = path[len(delim) :]
nxt_delim, pos = _next_delim(path)
token, path = (
path[: pos if pos != -1 else len(path)],
path[pos if pos != -1 else len(path) :],
)
if token == "":
return None # empty identifier is invalid
tokens.append((delim, token))
return tokens
# --------------------------------------------------------------------------- #
# Public API – parsing (flattened ➜ concrete)
# --------------------------------------------------------------------------- #
def parse_execution_output(output: BlockOutputEntry, name: str) -> JsonValue | None:
"""
Retrieve a nested value out of `output` using the flattened *name*.
On any failure (wrong name, wrong type, out-of-range, bad path)
returns **None**.
"""
base_name, data = output
# Exact match → whole object
if name == base_name:
return data
# Must start with the expected name
if not name.startswith(base_name):
return None
path = name[len(base_name) :]
if not path:
return None # nothing left to parse
tokens = _tokenise(path)
if tokens is None:
return None
cur: JsonValue = data
for delim, ident in tokens:
if delim == LIST_SPLIT:
# list[index]
try:
idx = int(ident)
except ValueError:
return None
if not isinstance(cur, list) or idx >= len(cur):
return None
cur = cur[idx]
elif delim == DICT_SPLIT:
if not isinstance(cur, dict) or ident not in cur:
return None
cur = cur[ident]
elif delim == OBJC_SPLIT:
if not hasattr(cur, ident):
return None
cur = getattr(cur, ident)
else:
return None # unreachable
return cur
def _assign(container: Any, tokens: list[tuple[str, str]], value: Any) -> Any:
"""
Recursive helper that *returns* the (possibly new) container with
`value` assigned along the remaining `tokens` path.
"""
if not tokens:
return value # leaf reached
delim, ident = tokens[0]
rest = tokens[1:]
# ---------- list ----------
if delim == LIST_SPLIT:
try:
idx = int(ident)
except ValueError:
raise ValueError("index must be an integer")
if container is None:
container = []
elif not isinstance(container, list):
container = list(container) if hasattr(container, "__iter__") else []
while len(container) <= idx:
container.append(None)
container[idx] = _assign(container[idx], rest, value)
return container
# ---------- dict ----------
if delim == DICT_SPLIT:
if container is None:
container = {}
elif not isinstance(container, dict):
container = dict(container) if hasattr(container, "items") else {}
container[ident] = _assign(container.get(ident), rest, value)
return container
# ---------- object ----------
if delim == OBJC_SPLIT:
if container is None or not isinstance(container, MockObject):
container = MockObject()
setattr(
container,
ident,
_assign(getattr(container, ident, None), rest, value),
)
return container
return value # unreachable
def merge_execution_input(data: BlockInput) -> BlockInput:
"""
Reconstruct nested objects from a *flattened* dict of key → value.
Raises ValueError on syntactically invalid list indices.
"""
merged: BlockInput = {}
for key, value in data.items():
# Split off the base name (before the first delimiter, if any)
delim, pos = _next_delim(key)
if delim is None:
merged[key] = value
continue
base, path = key[:pos], key[pos:]
tokens = _tokenise(path)
if tokens is None:
# Invalid key; treat as scalar under the raw name
merged[key] = value
continue
merged[base] = _assign(merged.get(base), tokens, value)
data.update(merged)
return data
# Dynamic field utilities are now imported from backend.data.dynamic_fields
def validate_exec(

2
autogpt_platform/backend/backend/executor/utils_test.py
View File

@@ -3,7 +3,7 @@ from typing import cast
import pytest
from pytest_mock import MockerFixture
from backend.executor.utils import merge_execution_input, parse_execution_output
from backend.data.dynamic_fields import merge_execution_input, parse_execution_output
from backend.util.mock import MockObject

301
autogpt_platform/backend/backend/notifications/notifications.py
View File

@@ -25,7 +25,11 @@ from backend.data.notifications import (
get_summary_params_type,
)
from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig
from backend.data.user import generate_unsubscribe_link
from backend.data.user import (
disable_all_user_notifications,
generate_unsubscribe_link,
set_user_email_verification,
)
from backend.notifications.email import EmailSender
from backend.util.clients import get_database_manager_async_client
from backend.util.logging import TruncatedLogger
@@ -38,7 +42,7 @@ from backend.util.service import (
endpoint_to_sync,
expose,
)
from backend.util.settings import Settings
from backend.util.settings import AppEnvironment, Settings
logger = TruncatedLogger(logging.getLogger(__name__), "[NotificationManager]")
settings = Settings()
@@ -124,6 +128,12 @@ def get_routing_key(event_type: NotificationType) -> str:
def queue_notification(event: NotificationEventModel) -> NotificationResult:
"""Queue a notification - exposed method for other services to call"""
# Disable in production
if settings.config.app_env == AppEnvironment.PRODUCTION:
return NotificationResult(
success=True,
message="Queueing notifications is disabled in production",
)
try:
logger.debug(f"Received Request to queue {event=}")
@@ -151,6 +161,12 @@ def queue_notification(event: NotificationEventModel) -> NotificationResult:
async def queue_notification_async(event: NotificationEventModel) -> NotificationResult:
"""Queue a notification - exposed method for other services to call"""
# Disable in production
if settings.config.app_env == AppEnvironment.PRODUCTION:
return NotificationResult(
success=True,
message="Queueing notifications is disabled in production",
)
try:
logger.debug(f"Received Request to queue {event=}")
@@ -213,6 +229,9 @@ class NotificationManager(AppService):
@expose
async def queue_weekly_summary(self):
# disable in prod
if settings.config.app_env == AppEnvironment.PRODUCTION:
return
# Use the existing event loop instead of creating a new one with asyncio.run()
asyncio.create_task(self._queue_weekly_summary())
@@ -226,7 +245,9 @@ class NotificationManager(AppService):
logger.info(
f"Querying for active users between {start_time} and {current_time}"
)
users = await get_database_manager_async_client().get_active_user_ids_in_timerange(
users = await get_database_manager_async_client(
should_retry=False
).get_active_user_ids_in_timerange(
end_time=current_time.isoformat(),
start_time=start_time.isoformat(),
)
@@ -253,6 +274,9 @@ class NotificationManager(AppService):
async def process_existing_batches(
self, notification_types: list[NotificationType]
):
# disable in prod
if settings.config.app_env == AppEnvironment.PRODUCTION:
return
# Use the existing event loop instead of creating a new process
asyncio.create_task(self._process_existing_batches(notification_types))
@@ -266,15 +290,15 @@ class NotificationManager(AppService):
for notification_type in notification_types:
# Get all batches for this notification type
batches = (
await get_database_manager_async_client().get_all_batches_by_type(
notification_type
)
)
batches = await get_database_manager_async_client(
should_retry=False
).get_all_batches_by_type(notification_type)
for batch in batches:
# Check if batch has aged out
oldest_message = await get_database_manager_async_client().get_user_notification_oldest_message_in_batch(
oldest_message = await get_database_manager_async_client(
should_retry=False
).get_user_notification_oldest_message_in_batch(
batch.user_id, notification_type
)
@@ -289,9 +313,9 @@ class NotificationManager(AppService):
# If batch has aged out, process it
if oldest_message.created_at + max_delay < current_time:
recipient_email = await get_database_manager_async_client().get_user_email_by_id(
batch.user_id
)
recipient_email = await get_database_manager_async_client(
should_retry=False
).get_user_email_by_id(batch.user_id)
if not recipient_email:
logger.error(
@@ -308,21 +332,25 @@ class NotificationManager(AppService):
f"User {batch.user_id} does not want to receive {notification_type} notifications"
)
# Clear the batch
await get_database_manager_async_client().empty_user_notification_batch(
await get_database_manager_async_client(
should_retry=False
).empty_user_notification_batch(
batch.user_id, notification_type
)
continue
batch_data = await get_database_manager_async_client().get_user_notification_batch(
batch.user_id, notification_type
)
batch_data = await get_database_manager_async_client(
should_retry=False
).get_user_notification_batch(batch.user_id, notification_type)
if not batch_data or not batch_data.notifications:
logger.error(
f"Batch data not found for user {batch.user_id}"
)
# Clear the batch
await get_database_manager_async_client().empty_user_notification_batch(
await get_database_manager_async_client(
should_retry=False
).empty_user_notification_batch(
batch.user_id, notification_type
)
continue
@@ -358,7 +386,9 @@ class NotificationManager(AppService):
)
# Clear the batch
await get_database_manager_async_client().empty_user_notification_batch(
await get_database_manager_async_client(
should_retry=False
).empty_user_notification_batch(
batch.user_id, notification_type
)
@@ -413,15 +443,13 @@ class NotificationManager(AppService):
self, user_id: str, event_type: NotificationType
) -> bool:
"""Check if a user wants to receive a notification based on their preferences and email verification status"""
validated_email = (
await get_database_manager_async_client().get_user_email_verification(
user_id
)
)
validated_email = await get_database_manager_async_client(
should_retry=False
).get_user_email_verification(user_id)
preference = (
await get_database_manager_async_client().get_user_notification_preference(
user_id
)
await get_database_manager_async_client(
should_retry=False
).get_user_notification_preference(user_id)
).preferences.get(event_type, True)
# only if both are true, should we email this person
return validated_email and preference
@@ -437,7 +465,9 @@ class NotificationManager(AppService):
try:
# Get summary data from the database
summary_data = await get_database_manager_async_client().get_user_execution_summary_data(
summary_data = await get_database_manager_async_client(
should_retry=False
).get_user_execution_summary_data(
user_id=user_id,
start_time=params.start_date,
end_time=params.end_date,
@@ -524,13 +554,13 @@ class NotificationManager(AppService):
self, user_id: str, event_type: NotificationType, event: NotificationEventModel
) -> bool:
await get_database_manager_async_client().create_or_add_to_user_notification_batch(
user_id, event_type, event
)
await get_database_manager_async_client(
should_retry=False
).create_or_add_to_user_notification_batch(user_id, event_type, event)
oldest_message = await get_database_manager_async_client().get_user_notification_oldest_message_in_batch(
user_id, event_type
)
oldest_message = await get_database_manager_async_client(
should_retry=False
).get_user_notification_oldest_message_in_batch(user_id, event_type)
if not oldest_message:
logger.error(
f"Batch for user {user_id} and type {event_type} has no oldest message whichshould never happen!!!!!!!!!!!!!!!!"
@@ -580,11 +610,9 @@ class NotificationManager(AppService):
return False
logger.debug(f"Processing immediate notification: {event}")
recipient_email = (
await get_database_manager_async_client().get_user_email_by_id(
event.user_id
)
)
recipient_email = await get_database_manager_async_client(
should_retry=False
).get_user_email_by_id(event.user_id)
if not recipient_email:
logger.error(f"User email not found for user {event.user_id}")
return False
@@ -619,11 +647,9 @@ class NotificationManager(AppService):
return False
logger.info(f"Processing batch notification: {event}")
recipient_email = (
await get_database_manager_async_client().get_user_email_by_id(
event.user_id
)
)
recipient_email = await get_database_manager_async_client(
should_retry=False
).get_user_email_by_id(event.user_id)
if not recipient_email:
logger.error(f"User email not found for user {event.user_id}")
return False
@@ -642,11 +668,9 @@ class NotificationManager(AppService):
if not should_send:
logger.info("Batch not old enough to send")
return False
batch = (
await get_database_manager_async_client().get_user_notification_batch(
event.user_id, event.type
)
)
batch = await get_database_manager_async_client(
should_retry=False
).get_user_notification_batch(event.user_id, event.type)
if not batch or not batch.notifications:
logger.error(f"Batch not found for user {event.user_id}")
return False
@@ -657,6 +681,7 @@ class NotificationManager(AppService):
get_notif_data_type(db_event.type)
].model_validate(
{
"id": db_event.id, # Include ID from database
"user_id": event.user_id,
"type": db_event.type,
"data": db_event.data,
@@ -679,6 +704,9 @@ class NotificationManager(AppService):
chunk_sent = False
for attempt_size in [chunk_size, 50, 25, 10, 5, 1]:
chunk = batch_messages[i : i + attempt_size]
chunk_ids = [
msg.id for msg in chunk if msg.id
] # Extract IDs for removal
try:
# Try to render the email to check its size
@@ -705,6 +733,23 @@ class NotificationManager(AppService):
user_unsub_link=unsub_link,
)
# Remove successfully sent notifications immediately
if chunk_ids:
try:
await get_database_manager_async_client(
should_retry=False
).remove_notifications_from_batch(
event.user_id, event.type, chunk_ids
)
logger.info(
f"Removed {len(chunk_ids)} sent notifications from batch"
)
except Exception as e:
logger.error(
f"Failed to remove sent notifications: {e}"
)
# Continue anyway - better to risk duplicates than lose emails
# Track successful sends
successfully_sent_count += len(chunk)
@@ -722,13 +767,137 @@ class NotificationManager(AppService):
i += len(chunk)
chunk_sent = True
break
else:
# Message is too large even after size reduction
if attempt_size == 1:
logger.error(
f"Failed to send notification at index {i}: "
f"Single notification exceeds email size limit "
f"({len(test_message):,} chars > {MAX_EMAIL_SIZE:,} chars). "
f"Removing permanently from batch - will not retry."
)
# Remove the oversized notification permanently - it will NEVER fit
if chunk_ids:
try:
await get_database_manager_async_client(
should_retry=False
).remove_notifications_from_batch(
event.user_id, event.type, chunk_ids
)
logger.info(
f"Removed oversized notification {chunk_ids[0]} from batch permanently"
)
except Exception as e:
logger.error(
f"Failed to remove oversized notification: {e}"
)
failed_indices.append(i)
i += 1
chunk_sent = True
break
# Try smaller chunk size
continue
except Exception as e:
# Check if it's a Postmark API error
if attempt_size == 1:
# Even single notification is too large
logger.error(
f"Single notification too large to send: {e}. "
f"Skipping notification at index {i}"
)
# Single notification failed - determine the actual cause
error_message = str(e).lower()
error_type = type(e).__name__
# Check for HTTP 406 - Inactive recipient (common in Postmark errors)
if "406" in error_message or "inactive" in error_message:
logger.warning(
f"Failed to send notification at index {i}: "
f"Recipient marked as inactive by Postmark. "
f"Error: {e}. Disabling ALL notifications for this user."
)
# 1. Mark email as unverified
try:
await set_user_email_verification(
event.user_id, False
)
logger.info(
f"Set email verification to false for user {event.user_id}"
)
except Exception as deactivation_error:
logger.error(
f"Failed to deactivate email for user {event.user_id}: "
f"{deactivation_error}"
)
# 2. Disable all notification preferences
try:
await disable_all_user_notifications(event.user_id)
logger.info(
f"Disabled all notification preferences for user {event.user_id}"
)
except Exception as disable_error:
logger.error(
f"Failed to disable notification preferences: {disable_error}"
)
# 3. Clear ALL notification batches for this user
try:
await get_database_manager_async_client(
should_retry=False
).clear_all_user_notification_batches(event.user_id)
logger.info(
f"Cleared ALL notification batches for user {event.user_id}"
)
except Exception as remove_error:
logger.error(
f"Failed to clear batches for inactive recipient: {remove_error}"
)
# Stop processing - we've nuked everything for this user
return True
# Check for HTTP 422 - Malformed data
elif (
"422" in error_message
or "unprocessable" in error_message
):
logger.error(
f"Failed to send notification at index {i}: "
f"Malformed notification data rejected by Postmark. "
f"Error: {e}. Removing from batch permanently."
)
# Remove from batch - 422 means bad data that won't fix itself
if chunk_ids:
try:
await get_database_manager_async_client(
should_retry=False
).remove_notifications_from_batch(
event.user_id, event.type, chunk_ids
)
logger.info(
"Removed malformed notification from batch permanently"
)
except Exception as remove_error:
logger.error(
f"Failed to remove malformed notification: {remove_error}"
)
# Check if it's a ValueError for size limit
elif (
isinstance(e, ValueError)
and "too large" in error_message
):
logger.error(
f"Failed to send notification at index {i}: "
f"Notification size exceeds email limit. "
f"Error: {e}. Skipping this notification."
)
# Other API errors
else:
logger.error(
f"Failed to send notification at index {i}: "
f"Email API error ({error_type}): {e}. "
f"Skipping this notification."
)
failed_indices.append(i)
i += 1
chunk_sent = True
@@ -742,18 +911,20 @@ class NotificationManager(AppService):
failed_indices.append(i)
i += 1
# Only empty the batch if ALL notifications were sent successfully
if successfully_sent_count == len(batch_messages):
# Check what remains in the batch (notifications are removed as sent)
remaining_batch = await get_database_manager_async_client(
should_retry=False
).get_user_notification_batch(event.user_id, event.type)
if not remaining_batch or not remaining_batch.notifications:
logger.info(
f"Successfully sent all {successfully_sent_count} notifications, clearing batch"
)
await get_database_manager_async_client().empty_user_notification_batch(
event.user_id, event.type
f"All {successfully_sent_count} notifications sent and removed from batch"
)
else:
remaining_count = len(remaining_batch.notifications)
logger.warning(
f"Only sent {successfully_sent_count} of {len(batch_messages)} notifications. "
f"Failed indices: {failed_indices}. Batch will be retained for retry."
f"Sent {successfully_sent_count} notifications. "
f"{remaining_count} remain in batch for retry due to errors."
)
return True
except Exception as e:
@@ -771,11 +942,9 @@ class NotificationManager(AppService):
logger.info(f"Processing summary notification: {model}")
recipient_email = (
await get_database_manager_async_client().get_user_email_by_id(
event.user_id
)
)
recipient_email = await get_database_manager_async_client(
should_retry=False
).get_user_email_by_id(event.user_id)
if not recipient_email:
logger.error(f"User email not found for user {event.user_id}")
return False

598
autogpt_platform/backend/backend/notifications/test_notifications.py Normal file
View File

@@ -0,0 +1,598 @@
"""Tests for notification error handling in NotificationManager."""
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from prisma.enums import NotificationType
from backend.data.notifications import AgentRunData, NotificationEventModel
from backend.notifications.notifications import NotificationManager
class TestNotificationErrorHandling:
"""Test cases for notification error handling in NotificationManager."""
@pytest.fixture
def notification_manager(self):
"""Create a NotificationManager instance for testing."""
with patch("backend.notifications.notifications.AppService.__init__"):
manager = NotificationManager()
manager.email_sender = MagicMock()
# Mock the _get_template method used by _process_batch
template_mock = Mock()
template_mock.base_template = "base"
template_mock.subject_template = "subject"
template_mock.body_template = "body"
manager.email_sender._get_template = Mock(return_value=template_mock)
# Mock the formatter
manager.email_sender.formatter = Mock()
manager.email_sender.formatter.format_email = Mock(
return_value=("subject", "body content")
)
manager.email_sender.formatter.env = Mock()
manager.email_sender.formatter.env.globals = {
"base_url": "http://example.com"
}
return manager
@pytest.fixture
def sample_batch_event(self):
"""Create a sample batch event for testing."""
return NotificationEventModel(
type=NotificationType.AGENT_RUN,
user_id="user_1",
created_at=datetime.now(timezone.utc),
data=AgentRunData(
agent_name="Test Agent",
credits_used=10.0,
execution_time=5.0,
node_count=3,
graph_id="graph_1",
outputs=[],
),
)
@pytest.fixture
def sample_batch_notifications(self):
"""Create sample batch notifications for testing."""
notifications = []
for i in range(3):
notification = Mock()
notification.type = NotificationType.AGENT_RUN
notification.data = {
"agent_name": f"Test Agent {i}",
"credits_used": 10.0 * (i + 1),
"execution_time": 5.0 * (i + 1),
"node_count": 3 + i,
"graph_id": f"graph_{i}",
"outputs": [],
}
notification.created_at = datetime.now(timezone.utc)
notifications.append(notification)
return notifications
@pytest.mark.asyncio
async def test_406_stops_all_processing_for_user(
self, notification_manager, sample_batch_event
):
"""Test that 406 inactive recipient error stops ALL processing for that user."""
with patch("backend.notifications.notifications.logger"), patch(
"backend.notifications.notifications.set_user_email_verification",
new_callable=AsyncMock,
) as mock_set_verification, patch(
"backend.notifications.notifications.disable_all_user_notifications",
new_callable=AsyncMock,
) as mock_disable_all, patch(
"backend.notifications.notifications.get_database_manager_async_client"
) as mock_db_client, patch(
"backend.notifications.notifications.generate_unsubscribe_link"
) as mock_unsub_link:
# Create batch of 5 notifications
notifications = []
for i in range(5):
notification = Mock()
notification.id = f"notif_{i}"
notification.type = NotificationType.AGENT_RUN
notification.data = {
"agent_name": f"Test Agent {i}",
"credits_used": 10.0 * (i + 1),
"execution_time": 5.0 * (i + 1),
"node_count": 3 + i,
"graph_id": f"graph_{i}",
"outputs": [],
}
notification.created_at = datetime.now(timezone.utc)
notifications.append(notification)
# Setup mocks
mock_db = mock_db_client.return_value
mock_db.get_user_email_by_id = AsyncMock(return_value="test@example.com")
mock_db.get_user_notification_batch = AsyncMock(
return_value=Mock(notifications=notifications)
)
mock_db.clear_all_user_notification_batches = AsyncMock()
mock_db.remove_notifications_from_batch = AsyncMock()
mock_unsub_link.return_value = "http://example.com/unsub"
# Mock internal methods
notification_manager._should_email_user_based_on_preference = AsyncMock(
return_value=True
)
notification_manager._should_batch = AsyncMock(return_value=True)
notification_manager._parse_message = Mock(return_value=sample_batch_event)
# Track calls
call_count = [0]
def send_side_effect(*args, **kwargs):
data = kwargs.get("data", [])
if isinstance(data, list) and len(data) == 1:
current_call = call_count[0]
call_count[0] += 1
# First two succeed, third hits 406
if current_call < 2:
return None
else:
raise Exception("Recipient marked as inactive (406)")
# Force single processing
raise Exception("Force single processing")
notification_manager.email_sender.send_templated.side_effect = (
send_side_effect
)
# Act
result = await notification_manager._process_batch(
sample_batch_event.model_dump_json()
)
# Assert
assert result is True
# Only 3 calls should have been made (2 successful, 1 failed with 406)
assert call_count[0] == 3
# User should be deactivated
mock_set_verification.assert_called_once_with("user_1", False)
mock_disable_all.assert_called_once_with("user_1")
mock_db.clear_all_user_notification_batches.assert_called_once_with(
"user_1"
)
# No further processing should occur after 406
@pytest.mark.asyncio
async def test_422_permanently_removes_malformed_notification(
self, notification_manager, sample_batch_event
):
"""Test that 422 error permanently removes the malformed notification from batch and continues with others."""
with patch("backend.notifications.notifications.logger") as mock_logger, patch(
"backend.notifications.notifications.get_database_manager_async_client"
) as mock_db_client, patch(
"backend.notifications.notifications.generate_unsubscribe_link"
) as mock_unsub_link:
# Create batch of 5 notifications
notifications = []
for i in range(5):
notification = Mock()
notification.id = f"notif_{i}"
notification.type = NotificationType.AGENT_RUN
notification.data = {
"agent_name": f"Test Agent {i}",
"credits_used": 10.0 * (i + 1),
"execution_time": 5.0 * (i + 1),
"node_count": 3 + i,
"graph_id": f"graph_{i}",
"outputs": [],
}
notification.created_at = datetime.now(timezone.utc)
notifications.append(notification)
# Setup mocks
mock_db = mock_db_client.return_value
mock_db.get_user_email_by_id = AsyncMock(return_value="test@example.com")
mock_db.get_user_notification_batch = AsyncMock(
side_effect=[
Mock(notifications=notifications),
Mock(notifications=[]), # Empty after processing
]
)
mock_db.remove_notifications_from_batch = AsyncMock()
mock_unsub_link.return_value = "http://example.com/unsub"
# Mock internal methods
notification_manager._should_email_user_based_on_preference = AsyncMock(
return_value=True
)
notification_manager._should_batch = AsyncMock(return_value=True)
notification_manager._parse_message = Mock(return_value=sample_batch_event)
# Track calls
call_count = [0]
successful_indices = []
removed_notification_ids = []
# Capture what gets removed
def remove_side_effect(user_id, notif_type, notif_ids):
removed_notification_ids.extend(notif_ids)
return None
mock_db.remove_notifications_from_batch.side_effect = remove_side_effect
def send_side_effect(*args, **kwargs):
data = kwargs.get("data", [])
if isinstance(data, list) and len(data) == 1:
current_call = call_count[0]
call_count[0] += 1
# Index 2 has malformed data (422)
if current_call == 2:
raise Exception(
"Unprocessable entity (422): Malformed email data"
)
else:
successful_indices.append(current_call)
return None
# Force single processing
raise Exception("Force single processing")
notification_manager.email_sender.send_templated.side_effect = (
send_side_effect
)
# Act
result = await notification_manager._process_batch(
sample_batch_event.model_dump_json()
)
# Assert
assert result is True
assert call_count[0] == 5 # All 5 attempted
assert len(successful_indices) == 4 # 4 succeeded (all except index 2)
assert 2 not in successful_indices # Index 2 failed
# Verify 422 error was logged
error_calls = [call[0][0] for call in mock_logger.error.call_args_list]
assert any(
"422" in call or "malformed" in call.lower() for call in error_calls
)
# Verify all notifications were removed (4 successful + 1 malformed)
assert mock_db.remove_notifications_from_batch.call_count == 5
assert (
"notif_2" in removed_notification_ids
) # Malformed one was removed permanently
@pytest.mark.asyncio
async def test_oversized_notification_permanently_removed(
self, notification_manager, sample_batch_event
):
"""Test that oversized notifications are permanently removed from batch but others continue."""
with patch("backend.notifications.notifications.logger") as mock_logger, patch(
"backend.notifications.notifications.get_database_manager_async_client"
) as mock_db_client, patch(
"backend.notifications.notifications.generate_unsubscribe_link"
) as mock_unsub_link:
# Create batch of 5 notifications
notifications = []
for i in range(5):
notification = Mock()
notification.id = f"notif_{i}"
notification.type = NotificationType.AGENT_RUN
notification.data = {
"agent_name": f"Test Agent {i}",
"credits_used": 10.0 * (i + 1),
"execution_time": 5.0 * (i + 1),
"node_count": 3 + i,
"graph_id": f"graph_{i}",
"outputs": [],
}
notification.created_at = datetime.now(timezone.utc)
notifications.append(notification)
# Setup mocks
mock_db = mock_db_client.return_value
mock_db.get_user_email_by_id = AsyncMock(return_value="test@example.com")
mock_db.get_user_notification_batch = AsyncMock(
side_effect=[
Mock(notifications=notifications),
Mock(notifications=[]), # Empty after processing
]
)
mock_db.remove_notifications_from_batch = AsyncMock()
mock_unsub_link.return_value = "http://example.com/unsub"
# Mock internal methods
notification_manager._should_email_user_based_on_preference = AsyncMock(
return_value=True
)
notification_manager._should_batch = AsyncMock(return_value=True)
notification_manager._parse_message = Mock(return_value=sample_batch_event)
# Override formatter to simulate oversized on index 3
# original_format = notification_manager.email_sender.formatter.format_email
def format_side_effect(*args, **kwargs):
# Check if we're formatting index 3
data = kwargs.get("data", {}).get("notifications", [])
if data and len(data) == 1:
# Check notification content to identify index 3
if any(
"Test Agent 3" in str(n.data)
for n in data
if hasattr(n, "data")
):
# Return oversized message for index 3
return ("subject", "x" * 5_000_000) # Over 4.5MB limit
return ("subject", "normal sized content")
notification_manager.email_sender.formatter.format_email = Mock(
side_effect=format_side_effect
)
# Track calls
successful_indices = []
def send_side_effect(*args, **kwargs):
data = kwargs.get("data", [])
if isinstance(data, list) and len(data) == 1:
# Track which notification was sent based on content
for i, notif in enumerate(notifications):
if any(
f"Test Agent {i}" in str(n.data)
for n in data
if hasattr(n, "data")
):
successful_indices.append(i)
return None
return None
# Force single processing
raise Exception("Force single processing")
notification_manager.email_sender.send_templated.side_effect = (
send_side_effect
)
# Act
result = await notification_manager._process_batch(
sample_batch_event.model_dump_json()
)
# Assert
assert result is True
assert (
len(successful_indices) == 4
) # Only 4 sent (index 3 skipped due to size)
assert 3 not in successful_indices # Index 3 was not sent
# Verify oversized error was logged
error_calls = [call[0][0] for call in mock_logger.error.call_args_list]
assert any(
"exceeds email size limit" in call or "oversized" in call.lower()
for call in error_calls
)
@pytest.mark.asyncio
async def test_generic_api_error_keeps_notification_for_retry(
self, notification_manager, sample_batch_event
):
"""Test that generic API errors keep notifications in batch for retry while others continue."""
with patch("backend.notifications.notifications.logger") as mock_logger, patch(
"backend.notifications.notifications.get_database_manager_async_client"
) as mock_db_client, patch(
"backend.notifications.notifications.generate_unsubscribe_link"
) as mock_unsub_link:
# Create batch of 5 notifications
notifications = []
for i in range(5):
notification = Mock()
notification.id = f"notif_{i}"
notification.type = NotificationType.AGENT_RUN
notification.data = {
"agent_name": f"Test Agent {i}",
"credits_used": 10.0 * (i + 1),
"execution_time": 5.0 * (i + 1),
"node_count": 3 + i,
"graph_id": f"graph_{i}",
"outputs": [],
}
notification.created_at = datetime.now(timezone.utc)
notifications.append(notification)
# Notification that failed with generic error
failed_notifications = [notifications[1]] # Only index 1 remains for retry
# Setup mocks
mock_db = mock_db_client.return_value
mock_db.get_user_email_by_id = AsyncMock(return_value="test@example.com")
mock_db.get_user_notification_batch = AsyncMock(
side_effect=[
Mock(notifications=notifications),
Mock(
notifications=failed_notifications
), # Failed ones remain for retry
]
)
mock_db.remove_notifications_from_batch = AsyncMock()
mock_unsub_link.return_value = "http://example.com/unsub"
# Mock internal methods
notification_manager._should_email_user_based_on_preference = AsyncMock(
return_value=True
)
notification_manager._should_batch = AsyncMock(return_value=True)
notification_manager._parse_message = Mock(return_value=sample_batch_event)
# Track calls
successful_indices = []
failed_indices = []
removed_notification_ids = []
# Capture what gets removed
def remove_side_effect(user_id, notif_type, notif_ids):
removed_notification_ids.extend(notif_ids)
return None
mock_db.remove_notifications_from_batch.side_effect = remove_side_effect
def send_side_effect(*args, **kwargs):
data = kwargs.get("data", [])
if isinstance(data, list) and len(data) == 1:
# Track which notification based on content
for i, notif in enumerate(notifications):
if any(
f"Test Agent {i}" in str(n.data)
for n in data
if hasattr(n, "data")
):
# Index 1 has generic API error
if i == 1:
failed_indices.append(i)
raise Exception("Network timeout - temporary failure")
else:
successful_indices.append(i)
return None
return None
# Force single processing
raise Exception("Force single processing")
notification_manager.email_sender.send_templated.side_effect = (
send_side_effect
)
# Act
result = await notification_manager._process_batch(
sample_batch_event.model_dump_json()
)
# Assert
assert result is True
assert len(successful_indices) == 4 # 4 succeeded (0, 2, 3, 4)
assert len(failed_indices) == 1 # 1 failed
assert 1 in failed_indices # Index 1 failed
# Verify generic error was logged
error_calls = [call[0][0] for call in mock_logger.error.call_args_list]
assert any(
"api error" in call.lower() or "skipping" in call.lower()
for call in error_calls
)
# Only successful ones should be removed from batch (failed one stays for retry)
assert mock_db.remove_notifications_from_batch.call_count == 4
assert (
"notif_1" not in removed_notification_ids
) # Failed one NOT removed (stays for retry)
assert "notif_0" in removed_notification_ids # Successful one removed
assert "notif_2" in removed_notification_ids # Successful one removed
assert "notif_3" in removed_notification_ids # Successful one removed
assert "notif_4" in removed_notification_ids # Successful one removed
@pytest.mark.asyncio
async def test_batch_all_notifications_sent_successfully(
self, notification_manager, sample_batch_event
):
"""Test successful batch processing where all notifications are sent without errors."""
with patch("backend.notifications.notifications.logger") as mock_logger, patch(
"backend.notifications.notifications.get_database_manager_async_client"
) as mock_db_client, patch(
"backend.notifications.notifications.generate_unsubscribe_link"
) as mock_unsub_link:
# Create batch of 5 notifications
notifications = []
for i in range(5):
notification = Mock()
notification.id = f"notif_{i}"
notification.type = NotificationType.AGENT_RUN
notification.data = {
"agent_name": f"Test Agent {i}",
"credits_used": 10.0 * (i + 1),
"execution_time": 5.0 * (i + 1),
"node_count": 3 + i,
"graph_id": f"graph_{i}",
"outputs": [],
}
notification.created_at = datetime.now(timezone.utc)
notifications.append(notification)
# Setup mocks
mock_db = mock_db_client.return_value
mock_db.get_user_email_by_id = AsyncMock(return_value="test@example.com")
mock_db.get_user_notification_batch = AsyncMock(
side_effect=[
Mock(notifications=notifications),
Mock(notifications=[]), # Empty after all sent successfully
]
)
mock_db.remove_notifications_from_batch = AsyncMock()
mock_unsub_link.return_value = "http://example.com/unsub"
# Mock internal methods
notification_manager._should_email_user_based_on_preference = AsyncMock(
return_value=True
)
notification_manager._should_batch = AsyncMock(return_value=True)
notification_manager._parse_message = Mock(return_value=sample_batch_event)
# Track successful sends
successful_indices = []
removed_notification_ids = []
# Capture what gets removed
def remove_side_effect(user_id, notif_type, notif_ids):
removed_notification_ids.extend(notif_ids)
return None
mock_db.remove_notifications_from_batch.side_effect = remove_side_effect
def send_side_effect(*args, **kwargs):
data = kwargs.get("data", [])
if isinstance(data, list) and len(data) == 1:
# Track which notification was sent
for i, notif in enumerate(notifications):
if any(
f"Test Agent {i}" in str(n.data)
for n in data
if hasattr(n, "data")
):
successful_indices.append(i)
return None
return None # Success
# Force single processing
raise Exception("Force single processing")
notification_manager.email_sender.send_templated.side_effect = (
send_side_effect
)
# Act
result = await notification_manager._process_batch(
sample_batch_event.model_dump_json()
)
# Assert
assert result is True
# All 5 notifications should be sent successfully
assert len(successful_indices) == 5
assert successful_indices == [0, 1, 2, 3, 4]
# All notifications should be removed from batch
assert mock_db.remove_notifications_from_batch.call_count == 5
assert len(removed_notification_ids) == 5
for i in range(5):
assert f"notif_{i}" in removed_notification_ids
# No errors should be logged
assert mock_logger.error.call_count == 0
# Info message about successful sends should be logged
info_calls = [call[0][0] for call in mock_logger.info.call_args_list]
assert any("sent and removed" in call.lower() for call in info_calls)

12
autogpt_platform/backend/backend/server/integrations/router.py
View File

@@ -64,7 +64,7 @@ class LoginResponse(BaseModel):
state_token: str
@router.get("/{provider}/login")
@router.get("/{provider}/login", summary="Initiate OAuth flow")
async def login(
provider: Annotated[
ProviderName, Path(title="The provider to initiate an OAuth flow for")
@@ -102,7 +102,7 @@ class CredentialsMetaResponse(BaseModel):
)
@router.post("/{provider}/callback")
@router.post("/{provider}/callback", summary="Exchange OAuth code for tokens")
async def callback(
provider: Annotated[
ProviderName, Path(title="The target provider for this OAuth exchange")
@@ -180,7 +180,7 @@ async def callback(
)
@router.get("/credentials")
@router.get("/credentials", summary="List Credentials")
async def list_credentials(
user_id: Annotated[str, Security(get_user_id)],
) -> list[CredentialsMetaResponse]:
@@ -221,7 +221,9 @@ async def list_credentials_by_provider(
]
@router.get("/{provider}/credentials/{cred_id}")
@router.get(
"/{provider}/credentials/{cred_id}", summary="Get Specific Credential By ID"
)
async def get_credential(
provider: Annotated[
ProviderName, Path(title="The provider to retrieve credentials for")
@@ -242,7 +244,7 @@ async def get_credential(
return credential
@router.post("/{provider}/credentials", status_code=201)
@router.post("/{provider}/credentials", status_code=201, summary="Create Credentials")
async def create_credentials(
user_id: Annotated[str, Security(get_user_id)],
provider: Annotated[

11
autogpt_platform/backend/backend/server/v2/admin/store_admin_routes.py
View File

@@ -7,6 +7,7 @@ import fastapi
import fastapi.responses
import prisma.enums
import backend.server.v2.store.cache as store_cache
import backend.server.v2.store.db
import backend.server.v2.store.model
import backend.util.json
@@ -86,6 +87,11 @@ async def review_submission(
StoreSubmission with updated review information
"""
try:
already_approved = (
await backend.server.v2.store.db.check_submission_already_approved(
store_listing_version_id=store_listing_version_id,
)
)
submission = await backend.server.v2.store.db.review_store_submission(
store_listing_version_id=store_listing_version_id,
is_approved=request.is_approved,
@@ -93,6 +99,11 @@ async def review_submission(
internal_comments=request.internal_comments or "",
reviewer_id=user_id,
)
state_changed = already_approved != request.is_approved
# Clear caches when the request is approved as it updates what is shown on the store
if state_changed:
store_cache.clear_all_caches()
return submission
except Exception as e:
logger.exception("Error reviewing submission: %s", e)

11
autogpt_platform/backend/backend/server/v2/builder/db.py
View File

@@ -118,6 +118,17 @@ def get_blocks(
)
def get_block_by_id(block_id: str) -> BlockInfo | None:
"""
Get a specific block by its ID.
"""
for block_type in load_all_blocks().values():
block: Block[BlockSchema, BlockSchema] = block_type()
if block.id == block_id:
return block.get_info()
return None
def search_blocks(
include_blocks: bool = True,
include_integrations: bool = True,

10
autogpt_platform/backend/backend/server/v2/builder/model.py
View File

@@ -53,16 +53,6 @@ class ProviderResponse(BaseModel):
pagination: Pagination
# Search
class SearchRequest(BaseModel):
search_query: str | None = None
filter: list[FilterType] | None = None
by_creator: list[str] | None = None
search_id: str | None = None
page: int | None = None
page_size: int | None = None
class SearchBlocksResponse(BaseModel):
blocks: BlockResponse
total_block_count: int

69
autogpt_platform/backend/backend/server/v2/builder/routes.py
View File

@@ -110,6 +110,25 @@ async def get_blocks(
)
@router.get(
"/blocks/batch",
summary="Get specific blocks",
response_model=list[builder_model.BlockInfo],
)
async def get_specific_blocks(
block_ids: Annotated[list[str], fastapi.Query()],
) -> list[builder_model.BlockInfo]:
"""
Get specific blocks by their IDs.
"""
blocks = []
for block_id in block_ids:
block = builder_db.get_block_by_id(block_id)
if block:
blocks.append(block)
return blocks
@router.get(
"/providers",
summary="Get Builder integration providers",
@@ -128,30 +147,34 @@ async def get_providers(
)
@router.post(
# Not using post method because on frontend, orval doesn't support Infinite Query with POST method.
@router.get(
"/search",
summary="Builder search",
tags=["store", "private"],
response_model=builder_model.SearchResponse,
)
async def search(
options: builder_model.SearchRequest,
user_id: Annotated[str, fastapi.Security(get_user_id)],
search_query: Annotated[str | None, fastapi.Query()] = None,
filter: Annotated[list[str] | None, fastapi.Query()] = None,
search_id: Annotated[str | None, fastapi.Query()] = None,
by_creator: Annotated[list[str] | None, fastapi.Query()] = None,
page: Annotated[int, fastapi.Query()] = 1,
page_size: Annotated[int, fastapi.Query()] = 50,
) -> builder_model.SearchResponse:
"""
Search for blocks (including integrations), marketplace agents, and user library agents.
"""
# If no filters are provided, then we will return all types
if not options.filter:
options.filter = [
if not filter:
filter = [
"blocks",
"integrations",
"marketplace_agents",
"my_agents",
]
options.search_query = sanitize_query(options.search_query)
options.page = options.page or 1
options.page_size = options.page_size or 50
search_query = sanitize_query(search_query)
# Blocks&Integrations
blocks = builder_model.SearchBlocksResponse(
@@ -162,13 +185,13 @@ async def search(
total_block_count=0,
total_integration_count=0,
)
if "blocks" in options.filter or "integrations" in options.filter:
if "blocks" in filter or "integrations" in filter:
blocks = builder_db.search_blocks(
include_blocks="blocks" in options.filter,
include_integrations="integrations" in options.filter,
query=options.search_query or "",
page=options.page,
page_size=options.page_size,
include_blocks="blocks" in filter,
include_integrations="integrations" in filter,
query=search_query or "",
page=page,
page_size=page_size,
)
# Library Agents
@@ -176,12 +199,12 @@ async def search(
agents=[],
pagination=Pagination.empty(),
)
if "my_agents" in options.filter:
if "my_agents" in filter:
my_agents = await library_db.list_library_agents(
user_id=user_id,
search_term=options.search_query,
page=options.page,
page_size=options.page_size,
search_term=search_query,
page=page,
page_size=page_size,
)
# Marketplace Agents
@@ -189,12 +212,12 @@ async def search(
agents=[],
pagination=Pagination.empty(),
)
if "marketplace_agents" in options.filter:
if "marketplace_agents" in filter:
marketplace_agents = await store_db.get_store_agents(
creators=options.by_creator,
search_query=options.search_query,
page=options.page,
page_size=options.page_size,
creators=by_creator,
search_query=search_query,
page=page,
page_size=page_size,
)
more_pages = False
@@ -214,7 +237,7 @@ async def search(
"marketplace_agents": marketplace_agents.pagination.total_items,
"my_agents": my_agents.pagination.total_items,
},
page=options.page,
page=page,
more_pages=more_pages,
)

76
autogpt_platform/backend/backend/server/v2/store/cache.py Normal file
View File

@@ -0,0 +1,76 @@
from autogpt_libs.utils.cache import cached
import backend.server.v2.store.db
##############################################
############### Caches #######################
##############################################
def clear_all_caches():
"""Clear all caches."""
_get_cached_store_agents.cache_clear()
_get_cached_agent_details.cache_clear()
_get_cached_store_creators.cache_clear()
_get_cached_creator_details.cache_clear()
# Cache store agents list for 5 minutes
# Different cache entries for different query combinations
@cached(maxsize=5000, ttl_seconds=300)
async def _get_cached_store_agents(
featured: bool,
creator: str | None,
sorted_by: str | None,
search_query: str | None,
category: str | None,
page: int,
page_size: int,
):
"""Cached helper to get store agents."""
return await backend.server.v2.store.db.get_store_agents(
featured=featured,
creators=[creator] if creator else None,
sorted_by=sorted_by,
search_query=search_query,
category=category,
page=page,
page_size=page_size,
)
# Cache individual agent details for 15 minutes
@cached(maxsize=200, ttl_seconds=300)
async def _get_cached_agent_details(username: str, agent_name: str):
"""Cached helper to get agent details."""
return await backend.server.v2.store.db.get_store_agent_details(
username=username, agent_name=agent_name
)
# Cache creators list for 5 minutes
@cached(maxsize=200, ttl_seconds=300)
async def _get_cached_store_creators(
featured: bool,
search_query: str | None,
sorted_by: str | None,
page: int,
page_size: int,
):
"""Cached helper to get store creators."""
return await backend.server.v2.store.db.get_store_creators(
featured=featured,
search_query=search_query,
sorted_by=sorted_by,
page=page,
page_size=page_size,
)
# Cache individual creator details for 5 minutes
@cached(maxsize=100, ttl_seconds=300)
async def _get_cached_creator_details(username: str):
"""Cached helper to get creator details."""
return await backend.server.v2.store.db.get_store_creator_details(
username=username.lower()
)

64
autogpt_platform/backend/backend/server/v2/store/db.py
View File

@@ -197,8 +197,7 @@ async def get_store_agents(
logger.debug(
f"Getting store agents. featured={featured}, creators={creators}, sorted_by={sorted_by}, search={search_query}, category={category}, page={page}"
)
sanitized_query = sanitize_query(search_query)
search_term = sanitize_query(search_query)
where_clause: prisma.types.StoreAgentWhereInput = {"is_available": True}
if featured:
where_clause["featured"] = featured
@@ -207,10 +206,10 @@ async def get_store_agents(
if category:
where_clause["categories"] = {"has": category}
if sanitized_query:
if search_term:
where_clause["OR"] = [
{"agent_name": {"contains": sanitized_query, "mode": "insensitive"}},
{"description": {"contains": sanitized_query, "mode": "insensitive"}},
{"agent_name": {"contains": search_term, "mode": "insensitive"}},
{"description": {"contains": search_term, "mode": "insensitive"}},
]
order_by = []
@@ -271,6 +270,24 @@ async def get_store_agents(
raise backend.server.v2.store.exceptions.DatabaseError(
"Failed to fetch store agents"
) from e
# TODO: commenting this out as we concerned about potential db load issues
# finally:
# if search_term:
# await log_search_term(search_query=search_term)
async def log_search_term(search_query: str):
"""Log a search term to the database"""
# Anonymize the data by preventing correlation with other logs
date = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
try:
await prisma.models.SearchTerms.prisma().create(
data={"searchTerm": search_query, "createdDate": date}
)
except Exception as e:
# Fail silently here so that logging search terms doesn't break the app
logger.error(f"Error logging search term: {e}")
async def get_store_agent_details(
@@ -851,7 +868,21 @@ async def create_store_submission(
store_listing_version_id=store_listing_version_id,
changes_summary=changes_summary,
)
except prisma.errors.UniqueViolationError as exc:
# Attempt to check if the error was due to the slug field being unique
error_str = str(exc)
if "slug" in error_str.lower():
logger.debug(
f"Slug '{slug}' is already in use by another agent (agent_id: {agent_id}) for user {user_id}"
)
raise backend.server.v2.store.exceptions.SlugAlreadyInUseError(
f"The URL slug '{slug}' is already in use by another one of your agents. Please choose a different slug."
) from exc
else:
# Reraise as a generic database error for other unique violations
raise backend.server.v2.store.exceptions.DatabaseError(
f"Unique constraint violated (not slug): {error_str}"
) from exc
except (
backend.server.v2.store.exceptions.AgentNotFoundError,
backend.server.v2.store.exceptions.ListingExistsError,
@@ -1974,6 +2005,27 @@ async def get_admin_listings_with_versions(
)
async def check_submission_already_approved(
store_listing_version_id: str,
) -> bool:
"""Check the submission status of a store listing version."""
try:
store_listing_version = (
await prisma.models.StoreListingVersion.prisma().find_unique(
where={"id": store_listing_version_id}
)
)
if not store_listing_version:
return False
return (
store_listing_version.submissionStatus
== prisma.enums.SubmissionStatus.APPROVED
)
except Exception as e:
logger.error(f"Error checking submission status: {e}")
return False
async def get_agent_as_admin(
user_id: str | None,
store_listing_version_id: str,

5
autogpt_platform/backend/backend/server/v2/store/db_test.py
View File

@@ -42,6 +42,7 @@ async def test_get_store_agents(mocker):
versions=["1.0"],
updated_at=datetime.now(),
is_available=False,
useForOnboarding=False,
)
]
@@ -84,6 +85,7 @@ async def test_get_store_agent_details(mocker):
versions=["1.0"],
updated_at=datetime.now(),
is_available=False,
useForOnboarding=False,
)
# Mock active version agent (what we want to return for active version)
@@ -105,6 +107,7 @@ async def test_get_store_agent_details(mocker):
versions=["1.0", "2.0"],
updated_at=datetime.now(),
is_available=True,
useForOnboarding=False,
)
# Create a mock StoreListing result
@@ -248,6 +251,7 @@ async def test_create_store_submission(mocker):
isAvailable=True,
)
],
useForOnboarding=False,
)
# Mock prisma calls
@@ -275,7 +279,6 @@ async def test_create_store_submission(mocker):
# Verify mocks called correctly
mock_agent_graph.return_value.find_first.assert_called_once()
mock_store_listing.return_value.find_first.assert_called_once()
mock_store_listing.return_value.create.assert_called_once()

6
autogpt_platform/backend/backend/server/v2/store/exceptions.py
View File

@@ -106,3 +106,9 @@ class UnauthorizedError(StoreError):
"""Raised when a user is not authorized to perform an action"""
pass
class SlugAlreadyInUseError(StoreError):
"""Raised when a slug is already in use by another agent owned by the user"""
pass

216
autogpt_platform/backend/backend/server/v2/store/routes.py
View File

@@ -6,9 +6,9 @@ import urllib.parse
import autogpt_libs.auth
import fastapi
import fastapi.responses
from autogpt_libs.utils.cache import cached
import backend.data.graph
import backend.server.v2.store.cache as store_cache
import backend.server.v2.store.db
import backend.server.v2.store.exceptions
import backend.server.v2.store.image_gen
@@ -21,117 +21,6 @@ logger = logging.getLogger(__name__)
router = fastapi.APIRouter()
##############################################
############### Caches #######################
##############################################
# Cache user profiles for 1 hour per user
@cached(maxsize=1000, ttl_seconds=3600)
async def _get_cached_user_profile(user_id: str):
"""Cached helper to get user profile."""
return await backend.server.v2.store.db.get_user_profile(user_id)
# Cache store agents list for 15 minutes
# Different cache entries for different query combinations
@cached(maxsize=5000, ttl_seconds=900)
async def _get_cached_store_agents(
featured: bool,
creator: str | None,
sorted_by: str | None,
search_query: str | None,
category: str | None,
page: int,
page_size: int,
):
"""Cached helper to get store agents."""
return await backend.server.v2.store.db.get_store_agents(
featured=featured,
creators=[creator] if creator else None,
sorted_by=sorted_by,
search_query=search_query,
category=category,
page=page,
page_size=page_size,
)
# Cache individual agent details for 15 minutes
@cached(maxsize=200, ttl_seconds=900)
async def _get_cached_agent_details(username: str, agent_name: str):
"""Cached helper to get agent details."""
return await backend.server.v2.store.db.get_store_agent_details(
username=username, agent_name=agent_name
)
# Cache agent graphs for 1 hour
@cached(maxsize=200, ttl_seconds=3600)
async def _get_cached_agent_graph(store_listing_version_id: str):
"""Cached helper to get agent graph."""
return await backend.server.v2.store.db.get_available_graph(
store_listing_version_id
)
# Cache agent by version for 1 hour
@cached(maxsize=200, ttl_seconds=3600)
async def _get_cached_store_agent_by_version(store_listing_version_id: str):
"""Cached helper to get store agent by version ID."""
return await backend.server.v2.store.db.get_store_agent_by_version_id(
store_listing_version_id
)
# Cache creators list for 1 hour
@cached(maxsize=200, ttl_seconds=3600)
async def _get_cached_store_creators(
featured: bool,
search_query: str | None,
sorted_by: str | None,
page: int,
page_size: int,
):
"""Cached helper to get store creators."""
return await backend.server.v2.store.db.get_store_creators(
featured=featured,
search_query=search_query,
sorted_by=sorted_by,
page=page,
page_size=page_size,
)
# Cache individual creator details for 1 hour
@cached(maxsize=100, ttl_seconds=3600)
async def _get_cached_creator_details(username: str):
"""Cached helper to get creator details."""
return await backend.server.v2.store.db.get_store_creator_details(
username=username.lower()
)
# Cache user's own agents for 5 mins (shorter TTL as this changes more frequently)
@cached(maxsize=500, ttl_seconds=300)
async def _get_cached_my_agents(user_id: str, page: int, page_size: int):
"""Cached helper to get user's agents."""
return await backend.server.v2.store.db.get_my_agents(
user_id, page=page, page_size=page_size
)
# Cache user's submissions for 1 hour (shorter TTL as this changes frequently)
@cached(maxsize=500, ttl_seconds=3600)
async def _get_cached_submissions(user_id: str, page: int, page_size: int):
"""Cached helper to get user's submissions."""
return await backend.server.v2.store.db.get_store_submissions(
user_id=user_id,
page=page,
page_size=page_size,
)
##############################################
############### Profile Endpoints ############
##############################################
@@ -152,7 +41,7 @@ async def get_profile(
Cached for 1 hour per user.
"""
try:
profile = await _get_cached_user_profile(user_id)
profile = await backend.server.v2.store.db.get_user_profile(user_id)
if profile is None:
return fastapi.responses.JSONResponse(
status_code=404,
@@ -198,8 +87,6 @@ async def update_or_create_profile(
updated_profile = await backend.server.v2.store.db.update_profile(
user_id=user_id, profile=profile
)
# Clear the cache for this user after profile update
_get_cached_user_profile.cache_delete(user_id)
return updated_profile
except Exception as e:
logger.exception("Failed to update profile for user %s: %s", user_id, e)
@@ -234,7 +121,6 @@ async def get_agents(
):
"""
Get a paginated list of agents from the store with optional filtering and sorting.
Results are cached for 15 minutes.
Args:
featured (bool, optional): Filter to only show featured agents. Defaults to False.
@@ -270,26 +156,15 @@ async def get_agents(
)
try:
# Use vector similarity search if we have a search query, otherwise use traditional search
if search_query and search_query.strip():
agents = await backend.server.v2.store.db.search_agents(
search_query=search_query,
featured=featured,
creators=[creator] if creator else None,
category=category,
page=page,
page_size=page_size,
)
else:
agents = await backend.server.v2.store.db.get_store_agents(
featured=featured,
creators=[creator] if creator else None,
sorted_by=sorted_by,
search_query=search_query,
category=category,
page=page,
page_size=page_size,
)
agents = await store_cache._get_cached_store_agents(
featured=featured,
creator=creator,
sorted_by=sorted_by,
search_query=search_query,
category=category,
page=page,
page_size=page_size,
)
return agents
except Exception as e:
logger.exception("Failed to retrieve store agents: %s", e)
@@ -311,7 +186,6 @@ async def get_agents(
async def get_agent(username: str, agent_name: str):
"""
This is only used on the AgentDetails Page.
Results are cached for 15 minutes.
It returns the store listing agents details.
"""
@@ -319,7 +193,7 @@ async def get_agent(username: str, agent_name: str):
username = urllib.parse.unquote(username).lower()
# URL decode the agent name since it comes from the URL path
agent_name = urllib.parse.unquote(agent_name).lower()
agent = await _get_cached_agent_details(
agent = await store_cache._get_cached_agent_details(
username=username, agent_name=agent_name
)
return agent
@@ -342,10 +216,11 @@ async def get_agent(username: str, agent_name: str):
async def get_graph_meta_by_store_listing_version_id(store_listing_version_id: str):
"""
Get Agent Graph from Store Listing Version ID.
Results are cached for 1 hour.
"""
try:
graph = await _get_cached_agent_graph(store_listing_version_id)
graph = await backend.server.v2.store.db.get_available_graph(
store_listing_version_id
)
return graph
except Exception:
logger.exception("Exception occurred whilst getting agent graph")
@@ -365,10 +240,12 @@ async def get_graph_meta_by_store_listing_version_id(store_listing_version_id: s
async def get_store_agent(store_listing_version_id: str):
"""
Get Store Agent Details from Store Listing Version ID.
Results are cached for 1 hour.
"""
try:
agent = await _get_cached_store_agent_by_version(store_listing_version_id)
agent = await backend.server.v2.store.db.get_store_agent_by_version_id(
store_listing_version_id
)
return agent
except Exception:
logger.exception("Exception occurred whilst getting store agent")
@@ -446,8 +323,6 @@ async def get_creators(
- Home Page Featured Creators
- Search Results Page
Results are cached for 1 hour.
---
To support this functionality we need:
@@ -466,7 +341,7 @@ async def get_creators(
)
try:
creators = await _get_cached_store_creators(
creators = await store_cache._get_cached_store_creators(
featured=featured,
search_query=search_query,
sorted_by=sorted_by,
@@ -493,12 +368,11 @@ async def get_creator(
):
"""
Get the details of a creator.
Results are cached for 1 hour.
- Creator Details Page
"""
try:
username = urllib.parse.unquote(username).lower()
creator = await _get_cached_creator_details(username=username)
creator = await store_cache._get_cached_creator_details(username=username)
return creator
except Exception:
logger.exception("Exception occurred whilst getting creator details")
@@ -529,10 +403,11 @@ async def get_my_agents(
):
"""
Get user's own agents.
Results are cached for 5 minutes per user.
"""
try:
agents = await _get_cached_my_agents(user_id, page=page, page_size=page_size)
agents = await backend.server.v2.store.db.get_my_agents(
user_id, page=page, page_size=page_size
)
return agents
except Exception:
logger.exception("Exception occurred whilst getting my agents")
@@ -569,13 +444,6 @@ async def delete_submission(
submission_id=submission_id,
)
# Clear submissions cache for this specific user after deletion
if result:
# Clear user's own agents cache - we don't know all page/size combinations
for page in range(1, 20):
# Clear user's submissions cache for common defaults
_get_cached_submissions.cache_delete(user_id, page=page, page_size=20)
return result
except Exception:
logger.exception("Exception occurred whilst deleting store submission")
@@ -599,7 +467,6 @@ async def get_submissions(
):
"""
Get a paginated list of store submissions for the authenticated user.
Results are cached for 1 hour per user.
Args:
user_id (str): ID of the authenticated user
@@ -622,8 +489,10 @@ async def get_submissions(
status_code=422, detail="Page size must be greater than 0"
)
try:
listings = await _get_cached_submissions(
user_id, page=page, page_size=page_size
listings = await backend.server.v2.store.db.get_store_submissions(
user_id=user_id,
page=page,
page_size=page_size,
)
return listings
except Exception:
@@ -677,12 +546,13 @@ async def create_submission(
recommended_schedule_cron=submission_request.recommended_schedule_cron,
)
# Clear user's own agents cache - we don't know all page/size combinations
for page in range(1, 20):
# Clear user's submissions cache for common defaults
_get_cached_submissions.cache_delete(user_id, page=page, page_size=20)
return result
except backend.server.v2.store.exceptions.SlugAlreadyInUseError as e:
logger.warning("Slug already in use: %s", str(e))
return fastapi.responses.JSONResponse(
status_code=409,
content={"detail": str(e)},
)
except Exception:
logger.exception("Exception occurred whilst creating store submission")
return fastapi.responses.JSONResponse(
@@ -731,11 +601,6 @@ async def edit_submission(
recommended_schedule_cron=submission_request.recommended_schedule_cron,
)
# Clear user's own agents cache - we don't know all page/size combinations
for page in range(1, 20):
# Clear user's submissions cache for common defaults
_get_cached_submissions.cache_delete(user_id, page=page, page_size=20)
return result
@@ -928,15 +793,10 @@ async def get_cache_metrics():
)
# Add metrics for each cache
add_cache_metrics("user_profile", _get_cached_user_profile)
add_cache_metrics("store_agents", _get_cached_store_agents)
add_cache_metrics("agent_details", _get_cached_agent_details)
add_cache_metrics("agent_graph", _get_cached_agent_graph)
add_cache_metrics("agent_by_version", _get_cached_store_agent_by_version)
add_cache_metrics("store_creators", _get_cached_store_creators)
add_cache_metrics("creator_details", _get_cached_creator_details)
add_cache_metrics("my_agents", _get_cached_my_agents)
add_cache_metrics("submissions", _get_cached_submissions)
add_cache_metrics("store_agents", store_cache._get_cached_store_agents)
add_cache_metrics("agent_details", store_cache._get_cached_agent_details)
add_cache_metrics("store_creators", store_cache._get_cached_store_creators)
add_cache_metrics("creator_details", store_cache._get_cached_creator_details)
# Add metadata/help text at the beginning
prometheus_output = [

138
autogpt_platform/backend/backend/server/v2/store/test_cache_delete.py
View File

@@ -4,18 +4,12 @@ Test suite for verifying cache_delete functionality in store routes.
Tests that specific cache entries can be deleted while preserving others.
"""
import datetime
from unittest.mock import AsyncMock, patch
import pytest
from backend.server.v2.store import routes
from backend.server.v2.store.model import (
ProfileDetails,
StoreAgent,
StoreAgentDetails,
StoreAgentsResponse,
)
from backend.server.v2.store import cache as store_cache
from backend.server.v2.store.model import StoreAgent, StoreAgentsResponse
from backend.util.models import Pagination
@@ -54,10 +48,10 @@ class TestCacheDeletion:
return_value=mock_response,
) as mock_db:
# Clear cache first
routes._get_cached_store_agents.cache_clear()
store_cache._get_cached_store_agents.cache_clear()
# First call - should hit database
result1 = await routes._get_cached_store_agents(
result1 = await store_cache._get_cached_store_agents(
featured=False,
creator=None,
sorted_by=None,
@@ -70,7 +64,7 @@ class TestCacheDeletion:
assert result1.agents[0].agent_name == "Test Agent"
# Second call with same params - should use cache
await routes._get_cached_store_agents(
await store_cache._get_cached_store_agents(
featured=False,
creator=None,
sorted_by=None,
@@ -82,7 +76,7 @@ class TestCacheDeletion:
assert mock_db.call_count == 1 # No additional DB call
# Third call with different params - should hit database
await routes._get_cached_store_agents(
await store_cache._get_cached_store_agents(
featured=True, # Different param
creator=None,
sorted_by=None,
@@ -94,7 +88,7 @@ class TestCacheDeletion:
assert mock_db.call_count == 2 # New DB call
# Delete specific cache entry
deleted = routes._get_cached_store_agents.cache_delete(
deleted = store_cache._get_cached_store_agents.cache_delete(
featured=False,
creator=None,
sorted_by=None,
@@ -106,7 +100,7 @@ class TestCacheDeletion:
assert deleted is True # Entry was deleted
# Try to delete non-existent entry
deleted = routes._get_cached_store_agents.cache_delete(
deleted = store_cache._get_cached_store_agents.cache_delete(
featured=False,
creator="nonexistent",
sorted_by=None,
@@ -118,7 +112,7 @@ class TestCacheDeletion:
assert deleted is False # Entry didn't exist
# Call with deleted params - should hit database again
await routes._get_cached_store_agents(
await store_cache._get_cached_store_agents(
featured=False,
creator=None,
sorted_by=None,
@@ -130,7 +124,7 @@ class TestCacheDeletion:
assert mock_db.call_count == 3 # New DB call after deletion
# Call with featured=True - should still be cached
await routes._get_cached_store_agents(
await store_cache._get_cached_store_agents(
featured=True,
creator=None,
sorted_by=None,
@@ -141,105 +135,11 @@ class TestCacheDeletion:
)
assert mock_db.call_count == 3 # No additional DB call
@pytest.mark.asyncio
async def test_agent_details_cache_delete(self):
"""Test that specific agent details cache entries can be deleted."""
mock_response = StoreAgentDetails(
store_listing_version_id="version1",
slug="test-agent",
agent_name="Test Agent",
agent_video="https://example.com/video.mp4",
agent_image=["https://example.com/image.jpg"],
creator="testuser",
creator_avatar="https://example.com/avatar.jpg",
sub_heading="Test subheading",
description="Test description",
categories=["productivity"],
runs=100,
rating=4.5,
versions=[],
last_updated=datetime.datetime(2024, 1, 1),
)
with patch(
"backend.server.v2.store.db.get_store_agent_details",
new_callable=AsyncMock,
return_value=mock_response,
) as mock_db:
# Clear cache first
routes._get_cached_agent_details.cache_clear()
# First call - should hit database
await routes._get_cached_agent_details(
username="testuser", agent_name="testagent"
)
assert mock_db.call_count == 1
# Second call - should use cache
await routes._get_cached_agent_details(
username="testuser", agent_name="testagent"
)
assert mock_db.call_count == 1 # No additional DB call
# Delete specific entry
deleted = routes._get_cached_agent_details.cache_delete(
username="testuser", agent_name="testagent"
)
assert deleted is True
# Call again - should hit database
await routes._get_cached_agent_details(
username="testuser", agent_name="testagent"
)
assert mock_db.call_count == 2 # New DB call after deletion
@pytest.mark.asyncio
async def test_user_profile_cache_delete(self):
"""Test that user profile cache entries can be deleted."""
mock_response = ProfileDetails(
name="Test User",
username="testuser",
description="Test profile",
links=["https://example.com"],
)
with patch(
"backend.server.v2.store.db.get_user_profile",
new_callable=AsyncMock,
return_value=mock_response,
) as mock_db:
# Clear cache first
routes._get_cached_user_profile.cache_clear()
# First call - should hit database
await routes._get_cached_user_profile("user123")
assert mock_db.call_count == 1
# Second call - should use cache
await routes._get_cached_user_profile("user123")
assert mock_db.call_count == 1
# Different user - should hit database
await routes._get_cached_user_profile("user456")
assert mock_db.call_count == 2
# Delete specific user's cache
deleted = routes._get_cached_user_profile.cache_delete("user123")
assert deleted is True
# user123 should hit database again
await routes._get_cached_user_profile("user123")
assert mock_db.call_count == 3
# user456 should still be cached
await routes._get_cached_user_profile("user456")
assert mock_db.call_count == 3 # No additional DB call
@pytest.mark.asyncio
async def test_cache_info_after_deletions(self):
"""Test that cache_info correctly reflects deletions."""
# Clear all caches first
routes._get_cached_store_agents.cache_clear()
store_cache._get_cached_store_agents.cache_clear()
mock_response = StoreAgentsResponse(
agents=[],
@@ -258,7 +158,7 @@ class TestCacheDeletion:
):
# Add multiple entries
for i in range(5):
await routes._get_cached_store_agents(
await store_cache._get_cached_store_agents(
featured=False,
creator=f"creator{i}",
sorted_by=None,
@@ -269,12 +169,12 @@ class TestCacheDeletion:
)
# Check cache size
info = routes._get_cached_store_agents.cache_info()
info = store_cache._get_cached_store_agents.cache_info()
assert info["size"] == 5
# Delete some entries
for i in range(2):
deleted = routes._get_cached_store_agents.cache_delete(
deleted = store_cache._get_cached_store_agents.cache_delete(
featured=False,
creator=f"creator{i}",
sorted_by=None,
@@ -286,7 +186,7 @@ class TestCacheDeletion:
assert deleted is True
# Check cache size after deletion
info = routes._get_cached_store_agents.cache_info()
info = store_cache._get_cached_store_agents.cache_info()
assert info["size"] == 3
@pytest.mark.asyncio
@@ -307,10 +207,10 @@ class TestCacheDeletion:
new_callable=AsyncMock,
return_value=mock_response,
) as mock_db:
routes._get_cached_store_agents.cache_clear()
store_cache._get_cached_store_agents.cache_clear()
# Test with all parameters
await routes._get_cached_store_agents(
await store_cache._get_cached_store_agents(
featured=True,
creator="testuser",
sorted_by="rating",
@@ -322,7 +222,7 @@ class TestCacheDeletion:
assert mock_db.call_count == 1
# Delete with exact same parameters
deleted = routes._get_cached_store_agents.cache_delete(
deleted = store_cache._get_cached_store_agents.cache_delete(
featured=True,
creator="testuser",
sorted_by="rating",
@@ -334,7 +234,7 @@ class TestCacheDeletion:
assert deleted is True
# Try to delete with slightly different parameters
deleted = routes._get_cached_store_agents.cache_delete(
deleted = store_cache._get_cached_store_agents.cache_delete(
featured=True,
creator="testuser",
sorted_by="rating",

6
autogpt_platform/backend/backend/util/clients.py
View File

@@ -34,12 +34,14 @@ def get_database_manager_client() -> "DatabaseManagerClient":
@thread_cached
def get_database_manager_async_client() -> "DatabaseManagerAsyncClient":
def get_database_manager_async_client(
should_retry: bool = True,
) -> "DatabaseManagerAsyncClient":
"""Get a thread-cached DatabaseManagerAsyncClient with request retry enabled."""
from backend.executor import DatabaseManagerAsyncClient
from backend.util.service import get_service_client
return get_service_client(DatabaseManagerAsyncClient, request_retry=True)
return get_service_client(DatabaseManagerAsyncClient, request_retry=should_retry)
@thread_cached

124
autogpt_platform/backend/backend/util/dynamic_fields.py Normal file
View File

@@ -0,0 +1,124 @@
"""
Utilities for handling dynamic field names and delimiters in the AutoGPT Platform.
Dynamic fields allow graphs to connect complex data structures using special delimiters:
- _#_ for dictionary keys (e.g., "values_#_name" → values["name"])
- _$_ for list indices (e.g., "items_$_0" → items[0])
- _@_ for object attributes (e.g., "obj_@_attr" → obj.attr)
This module provides utilities for:
- Extracting base field names from dynamic field names
- Generating proper schemas for base fields
- Creating helper functions for field sanitization
"""
from backend.data.dynamic_fields import DICT_SPLIT, LIST_SPLIT, OBJC_SPLIT
# All dynamic field delimiters
DYNAMIC_DELIMITERS = (DICT_SPLIT, LIST_SPLIT, OBJC_SPLIT)
def extract_base_field_name(field_name: str) -> str:
"""
Extract the base field name from a dynamic field name.
Examples:
extract_base_field_name("values_#_name") → "values"
extract_base_field_name("items_$_0") → "items"
extract_base_field_name("obj_@_attr") → "obj"
extract_base_field_name("regular_field") → "regular_field"
Args:
field_name: The field name that may contain dynamic delimiters
Returns:
The base field name without any dynamic suffixes
"""
base_name = field_name
for delimiter in DYNAMIC_DELIMITERS:
if delimiter in base_name:
base_name = base_name.split(delimiter)[0]
return base_name
def is_dynamic_field(field_name: str) -> bool:
"""
Check if a field name contains dynamic delimiters.
Args:
field_name: The field name to check
Returns:
True if the field contains any dynamic delimiters, False otherwise
"""
return any(delimiter in field_name for delimiter in DYNAMIC_DELIMITERS)
def get_dynamic_field_description(
base_field_name: str, original_field_name: str
) -> str:
"""
Generate a description for a dynamic field based on its base field and structure.
Args:
base_field_name: The base field name (e.g., "values")
original_field_name: The full dynamic field name (e.g., "values_#_name")
Returns:
A descriptive string explaining what this dynamic field represents
"""
if DICT_SPLIT in original_field_name:
key_part = (
original_field_name.split(DICT_SPLIT, 1)[1].split(DICT_SPLIT[0])[0]
if DICT_SPLIT in original_field_name
else "key"
)
return f"Dictionary value for {base_field_name}['{key_part}']"
elif LIST_SPLIT in original_field_name:
index_part = (
original_field_name.split(LIST_SPLIT, 1)[1].split(LIST_SPLIT[0])[0]
if LIST_SPLIT in original_field_name
else "index"
)
return f"List item for {base_field_name}[{index_part}]"
elif OBJC_SPLIT in original_field_name:
attr_part = (
original_field_name.split(OBJC_SPLIT, 1)[1].split(OBJC_SPLIT[0])[0]
if OBJC_SPLIT in original_field_name
else "attr"
)
return f"Object attribute for {base_field_name}.{attr_part}"
else:
return f"Dynamic value for {base_field_name}"
def group_fields_by_base_name(field_names: list[str]) -> dict[str, list[str]]:
"""
Group a list of field names by their base field names.
Args:
field_names: List of field names that may contain dynamic delimiters
Returns:
Dictionary mapping base field names to lists of original field names
Example:
group_fields_by_base_name([
"values_#_name",
"values_#_age",
"items_$_0",
"regular_field"
])
→ {
"values": ["values_#_name", "values_#_age"],
"items": ["items_$_0"],
"regular_field": ["regular_field"]
}
"""
grouped = {}
for field_name in field_names:
base_name = extract_base_field_name(field_name)
if base_name not in grouped:
grouped[base_name] = []
grouped[base_name].append(field_name)
return grouped

175
autogpt_platform/backend/backend/util/dynamic_fields_test.py Normal file
View File

@@ -0,0 +1,175 @@
"""Tests for dynamic field utilities."""
from backend.util.dynamic_fields import (
extract_base_field_name,
get_dynamic_field_description,
group_fields_by_base_name,
is_dynamic_field,
)
class TestExtractBaseFieldName:
"""Test extracting base field names from dynamic field names."""
def test_extract_dict_field(self):
"""Test extracting base name from dictionary fields."""
assert extract_base_field_name("values_#_name") == "values"
assert extract_base_field_name("data_#_key1_#_key2") == "data"
assert extract_base_field_name("config_#_database_#_host") == "config"
def test_extract_list_field(self):
"""Test extracting base name from list fields."""
assert extract_base_field_name("items_$_0") == "items"
assert extract_base_field_name("results_$_5_$_10") == "results"
assert extract_base_field_name("nested_$_0_$_1_$_2") == "nested"
def test_extract_object_field(self):
"""Test extracting base name from object fields."""
assert extract_base_field_name("user_@_name") == "user"
assert extract_base_field_name("response_@_data_@_items") == "response"
assert extract_base_field_name("obj_@_attr1_@_attr2") == "obj"
def test_extract_mixed_fields(self):
"""Test extracting base name from mixed dynamic fields."""
assert extract_base_field_name("data_$_0_#_key") == "data"
assert extract_base_field_name("items_#_user_@_name") == "items"
assert extract_base_field_name("complex_$_0_@_attr_#_key") == "complex"
def test_extract_regular_field(self):
"""Test extracting base name from regular (non-dynamic) fields."""
assert extract_base_field_name("regular_field") == "regular_field"
assert extract_base_field_name("simple") == "simple"
assert extract_base_field_name("") == ""
def test_extract_field_with_underscores(self):
"""Test fields with regular underscores (not dynamic delimiters)."""
assert extract_base_field_name("field_name_here") == "field_name_here"
assert extract_base_field_name("my_field_#_key") == "my_field"
class TestIsDynamicField:
"""Test identifying dynamic fields."""
def test_is_dynamic_dict_field(self):
"""Test identifying dictionary dynamic fields."""
assert is_dynamic_field("values_#_name") is True
assert is_dynamic_field("data_#_key1_#_key2") is True
def test_is_dynamic_list_field(self):
"""Test identifying list dynamic fields."""
assert is_dynamic_field("items_$_0") is True
assert is_dynamic_field("results_$_5_$_10") is True
def test_is_dynamic_object_field(self):
"""Test identifying object dynamic fields."""
assert is_dynamic_field("user_@_name") is True
assert is_dynamic_field("response_@_data_@_items") is True
def test_is_dynamic_mixed_field(self):
"""Test identifying mixed dynamic fields."""
assert is_dynamic_field("data_$_0_#_key") is True
assert is_dynamic_field("items_#_user_@_name") is True
def test_is_not_dynamic_field(self):
"""Test identifying non-dynamic fields."""
assert is_dynamic_field("regular_field") is False
assert is_dynamic_field("field_name_here") is False
assert is_dynamic_field("simple") is False
assert is_dynamic_field("") is False
class TestGetDynamicFieldDescription:
"""Test generating descriptions for dynamic fields."""
def test_dict_field_description(self):
"""Test descriptions for dictionary fields."""
desc = get_dynamic_field_description("values", "values_#_name")
assert "Dictionary value for values['name']" == desc
desc = get_dynamic_field_description("config", "config_#_database")
assert "Dictionary value for config['database']" == desc
def test_list_field_description(self):
"""Test descriptions for list fields."""
desc = get_dynamic_field_description("items", "items_$_0")
assert "List item for items[0]" == desc
desc = get_dynamic_field_description("results", "results_$_5")
assert "List item for results[5]" == desc
def test_object_field_description(self):
"""Test descriptions for object fields."""
desc = get_dynamic_field_description("user", "user_@_name")
assert "Object attribute for user.name" == desc
desc = get_dynamic_field_description("response", "response_@_data")
assert "Object attribute for response.data" == desc
def test_fallback_description(self):
"""Test fallback description for non-dynamic fields."""
desc = get_dynamic_field_description("field", "field")
assert "Dynamic value for field" == desc
class TestGroupFieldsByBaseName:
"""Test grouping fields by their base names."""
def test_group_mixed_fields(self):
"""Test grouping a mix of dynamic and regular fields."""
fields = [
"values_#_name",
"values_#_age",
"items_$_0",
"items_$_1",
"user_@_email",
"regular_field",
"another_field",
]
result = group_fields_by_base_name(fields)
expected = {
"values": ["values_#_name", "values_#_age"],
"items": ["items_$_0", "items_$_1"],
"user": ["user_@_email"],
"regular_field": ["regular_field"],
"another_field": ["another_field"],
}
assert result == expected
def test_group_empty_list(self):
"""Test grouping an empty list."""
result = group_fields_by_base_name([])
assert result == {}
def test_group_single_field(self):
"""Test grouping a single field."""
result = group_fields_by_base_name(["values_#_name"])
assert result == {"values": ["values_#_name"]}
def test_group_complex_dynamic_fields(self):
"""Test grouping complex nested dynamic fields."""
fields = [
"data_$_0_#_key1",
"data_$_0_#_key2",
"data_$_1_#_key1",
"other_@_attr",
]
result = group_fields_by_base_name(fields)
expected = {
"data": ["data_$_0_#_key1", "data_$_0_#_key2", "data_$_1_#_key1"],
"other": ["other_@_attr"],
}
assert result == expected
def test_preserve_order(self):
"""Test that field order is preserved within groups."""
fields = ["values_#_c", "values_#_a", "values_#_b"]
result = group_fields_by_base_name(fields)
# Should preserve the original order
assert result["values"] == ["values_#_c", "values_#_a", "values_#_b"]

42
autogpt_platform/backend/backend/util/file.py
View File

@@ -66,6 +66,18 @@ async def store_media_file(
base_path = Path(get_exec_file_path(graph_exec_id, ""))
base_path.mkdir(parents=True, exist_ok=True)
# Security fix: Add disk space limits to prevent DoS
MAX_FILE_SIZE = 100 * 1024 * 1024 # 100MB per file
MAX_TOTAL_DISK_USAGE = 1024 * 1024 * 1024 # 1GB total per execution directory
# Check total disk usage in base_path
if base_path.exists():
current_usage = get_dir_size(base_path)
if current_usage > MAX_TOTAL_DISK_USAGE:
raise ValueError(
f"Disk usage limit exceeded: {current_usage} bytes > {MAX_TOTAL_DISK_USAGE} bytes"
)
# Helper functions
def _extension_from_mime(mime: str) -> str:
ext = mimetypes.guess_extension(mime, strict=False)
@@ -108,6 +120,12 @@ async def store_media_file(
filename = Path(path_part).name or f"{uuid.uuid4()}.bin"
target_path = _ensure_inside_base(base_path / filename, base_path)
# Check file size limit
if len(cloud_content) > MAX_FILE_SIZE:
raise ValueError(
f"File too large: {len(cloud_content)} bytes > {MAX_FILE_SIZE} bytes"
)
# Virus scan the cloud content before writing locally
await scan_content_safe(cloud_content, filename=filename)
target_path.write_bytes(cloud_content)
@@ -129,6 +147,12 @@ async def store_media_file(
target_path = _ensure_inside_base(base_path / filename, base_path)
content = base64.b64decode(b64_content)
# Check file size limit
if len(content) > MAX_FILE_SIZE:
raise ValueError(
f"File too large: {len(content)} bytes > {MAX_FILE_SIZE} bytes"
)
# Virus scan the base64 content before writing
await scan_content_safe(content, filename=filename)
target_path.write_bytes(content)
@@ -142,6 +166,12 @@ async def store_media_file(
# Download and save
resp = await Requests().get(file)
# Check file size limit
if len(resp.content) > MAX_FILE_SIZE:
raise ValueError(
f"File too large: {len(resp.content)} bytes > {MAX_FILE_SIZE} bytes"
)
# Virus scan the downloaded content before writing
await scan_content_safe(resp.content, filename=filename)
target_path.write_bytes(resp.content)
@@ -159,6 +189,18 @@ async def store_media_file(
return MediaFileType(_strip_base_prefix(target_path, base_path))
def get_dir_size(path: Path) -> int:
"""Get total size of directory."""
total = 0
try:
for entry in path.glob("**/*"):
if entry.is_file():
total += entry.stat().st_size
except Exception:
pass
return total
def get_mime_type(file: str) -> str:
"""
Get the MIME type of a file, whether it's a data URI, URL, or local path.

41
autogpt_platform/backend/backend/util/prompt.py
View File

@@ -19,9 +19,48 @@ def _msg_tokens(msg: dict, enc) -> int:
"""
OpenAI counts ≈3 wrapper tokens per chat message, plus 1 if "name"
is present, plus the tokenised content length.
For tool calls, we need to count tokens in tool_calls and content fields.
"""
WRAPPER = 3 + (1 if "name" in msg else 0)
return WRAPPER + _tok_len(msg.get("content") or "", enc)
# Count content tokens
content_tokens = _tok_len(msg.get("content") or "", enc)
# Count tool call tokens for both OpenAI and Anthropic formats
tool_call_tokens = 0
# OpenAI format: tool_calls array at message level
if "tool_calls" in msg and isinstance(msg["tool_calls"], list):
for tool_call in msg["tool_calls"]:
# Count the tool call structure tokens
tool_call_tokens += _tok_len(tool_call.get("id", ""), enc)
tool_call_tokens += _tok_len(tool_call.get("type", ""), enc)
if "function" in tool_call:
tool_call_tokens += _tok_len(tool_call["function"].get("name", ""), enc)
tool_call_tokens += _tok_len(
tool_call["function"].get("arguments", ""), enc
)
# Anthropic format: tool_use within content array
content = msg.get("content")
if isinstance(content, list):
for item in content:
if isinstance(item, dict) and item.get("type") == "tool_use":
# Count the tool use structure tokens
tool_call_tokens += _tok_len(item.get("id", ""), enc)
tool_call_tokens += _tok_len(item.get("name", ""), enc)
tool_call_tokens += _tok_len(json.dumps(item.get("input", {})), enc)
elif isinstance(item, dict) and item.get("type") == "tool_result":
# Count tool result tokens
tool_call_tokens += _tok_len(item.get("tool_use_id", ""), enc)
tool_call_tokens += _tok_len(item.get("content", ""), enc)
elif isinstance(item, dict) and "content" in item:
# Other content types with content field
tool_call_tokens += _tok_len(item.get("content", ""), enc)
# For list content, override content_tokens since we counted everything above
content_tokens = 0
return WRAPPER + content_tokens + tool_call_tokens
def _truncate_middle_tokens(text: str, enc, max_tok: int) -> str:

278
autogpt_platform/backend/backend/util/prompt_test.py Normal file
View File

@@ -0,0 +1,278 @@
"""Tests for prompt utility functions, especially tool call token counting."""
import pytest
from tiktoken import encoding_for_model
from backend.util import json
from backend.util.prompt import _msg_tokens, estimate_token_count
class TestMsgTokens:
"""Test the _msg_tokens function with various message types."""
@pytest.fixture
def enc(self):
"""Get the encoding for gpt-4o model."""
return encoding_for_model("gpt-4o")
def test_regular_message_token_counting(self, enc):
"""Test that regular messages are counted correctly (backward compatibility)."""
msg = {"role": "user", "content": "What's the weather like in San Francisco?"}
tokens = _msg_tokens(msg, enc)
# Should be wrapper (3) + content tokens
expected = 3 + len(enc.encode(msg["content"]))
assert tokens == expected
assert tokens > 3 # Has content
def test_regular_message_with_name(self, enc):
"""Test that messages with name field get extra wrapper token."""
msg = {"role": "user", "name": "test_user", "content": "Hello!"}
tokens = _msg_tokens(msg, enc)
# Should be wrapper (3 + 1 for name) + content tokens
expected = 4 + len(enc.encode(msg["content"]))
assert tokens == expected
def test_openai_tool_call_token_counting(self, enc):
"""Test OpenAI format tool call token counting."""
msg = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"location": "San Francisco", "unit": "celsius"}',
},
}
],
}
tokens = _msg_tokens(msg, enc)
# Should count wrapper + all tool call components
expected_tool_tokens = (
len(enc.encode("call_abc123"))
+ len(enc.encode("function"))
+ len(enc.encode("get_weather"))
+ len(enc.encode('{"location": "San Francisco", "unit": "celsius"}'))
)
expected = 3 + expected_tool_tokens # wrapper + tool tokens
assert tokens == expected
assert tokens > 8 # Should be significantly more than just wrapper
def test_openai_multiple_tool_calls(self, enc):
"""Test OpenAI format with multiple tool calls."""
msg = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "func1", "arguments": '{"arg": "value1"}'},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "func2", "arguments": '{"arg": "value2"}'},
},
],
}
tokens = _msg_tokens(msg, enc)
# Should count all tool calls
assert tokens > 20 # Should be more than single tool call
def test_anthropic_tool_use_token_counting(self, enc):
"""Test Anthropic format tool use token counting."""
msg = {
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": "toolu_xyz456",
"name": "get_weather",
"input": {"location": "San Francisco", "unit": "celsius"},
}
],
}
tokens = _msg_tokens(msg, enc)
# Should count wrapper + tool use components
expected_tool_tokens = (
len(enc.encode("toolu_xyz456"))
+ len(enc.encode("get_weather"))
+ len(
enc.encode(json.dumps({"location": "San Francisco", "unit": "celsius"}))
)
)
expected = 3 + expected_tool_tokens # wrapper + tool tokens
assert tokens == expected
assert tokens > 8 # Should be significantly more than just wrapper
def test_anthropic_tool_result_token_counting(self, enc):
"""Test Anthropic format tool result token counting."""
msg = {
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "toolu_xyz456",
"content": "The weather in San Francisco is 22°C and sunny.",
}
],
}
tokens = _msg_tokens(msg, enc)
# Should count wrapper + tool result components
expected_tool_tokens = len(enc.encode("toolu_xyz456")) + len(
enc.encode("The weather in San Francisco is 22°C and sunny.")
)
expected = 3 + expected_tool_tokens # wrapper + tool tokens
assert tokens == expected
assert tokens > 8 # Should be significantly more than just wrapper
def test_anthropic_mixed_content(self, enc):
"""Test Anthropic format with mixed content types."""
msg = {
"role": "assistant",
"content": [
{"type": "text", "content": "I'll check the weather for you."},
{
"type": "tool_use",
"id": "toolu_123",
"name": "get_weather",
"input": {"location": "SF"},
},
],
}
tokens = _msg_tokens(msg, enc)
# Should count all content items
assert tokens > 15 # Should count both text and tool use
def test_empty_content(self, enc):
"""Test message with empty or None content."""
msg = {"role": "assistant", "content": None}
tokens = _msg_tokens(msg, enc)
assert tokens == 3 # Just wrapper tokens
msg["content"] = ""
tokens = _msg_tokens(msg, enc)
assert tokens == 3 # Just wrapper tokens
def test_string_content_with_tool_calls(self, enc):
"""Test OpenAI format where content is string but tool_calls exist."""
msg = {
"role": "assistant",
"content": "Let me check that for you.",
"tool_calls": [
{
"id": "call_123",
"type": "function",
"function": {"name": "test_func", "arguments": "{}"},
}
],
}
tokens = _msg_tokens(msg, enc)
# Should count both content and tool calls
content_tokens = len(enc.encode("Let me check that for you."))
tool_tokens = (
len(enc.encode("call_123"))
+ len(enc.encode("function"))
+ len(enc.encode("test_func"))
+ len(enc.encode("{}"))
)
expected = 3 + content_tokens + tool_tokens
assert tokens == expected
class TestEstimateTokenCount:
"""Test the estimate_token_count function with conversations containing tool calls."""
def test_conversation_with_tool_calls(self):
"""Test token counting for a complete conversation with tool calls."""
conversation = [
{"role": "user", "content": "What's the weather like in San Francisco?"},
{
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": "toolu_123",
"name": "get_weather",
"input": {"location": "San Francisco"},
}
],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "toolu_123",
"content": "22°C and sunny",
}
],
},
{
"role": "assistant",
"content": "The weather in San Francisco is 22°C and sunny.",
},
]
total_tokens = estimate_token_count(conversation)
# Verify total equals sum of individual messages
enc = encoding_for_model("gpt-4o")
expected_total = sum(_msg_tokens(msg, enc) for msg in conversation)
assert total_tokens == expected_total
assert total_tokens > 40 # Should be substantial for this conversation
def test_openai_conversation(self):
"""Test token counting for OpenAI format conversation."""
conversation = [
{"role": "user", "content": "Calculate 2 + 2"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_calc",
"type": "function",
"function": {
"name": "calculate",
"arguments": '{"expression": "2 + 2"}',
},
}
],
},
{"role": "tool", "tool_call_id": "call_calc", "content": "4"},
{"role": "assistant", "content": "The result is 4."},
]
total_tokens = estimate_token_count(conversation)
# Verify total equals sum of individual messages
enc = encoding_for_model("gpt-4o")
expected_total = sum(_msg_tokens(msg, enc) for msg in conversation)
assert total_tokens == expected_total
assert total_tokens > 20 # Should be substantial

100
autogpt_platform/backend/backend/util/retry.py
View File

@@ -20,34 +20,73 @@ logger = logging.getLogger(__name__)
# Alert threshold for excessive retries
EXCESSIVE_RETRY_THRESHOLD = 50
# Rate limiting for alerts - track last alert time per function+error combination
_alert_rate_limiter = {}
_rate_limiter_lock = threading.Lock()
ALERT_RATE_LIMIT_SECONDS = 300 # 5 minutes between same alerts
def should_send_alert(func_name: str, exception: Exception, context: str = "") -> bool:
"""Check if we should send an alert based on rate limiting."""
# Create a unique key for this function+error+context combination
error_signature = (
f"{context}:{func_name}:{type(exception).__name__}:{str(exception)[:100]}"
)
current_time = time.time()
with _rate_limiter_lock:
last_alert_time = _alert_rate_limiter.get(error_signature, 0)
if current_time - last_alert_time >= ALERT_RATE_LIMIT_SECONDS:
_alert_rate_limiter[error_signature] = current_time
return True
return False
def send_rate_limited_discord_alert(
func_name: str, exception: Exception, context: str, alert_msg: str, channel=None
) -> bool:
"""
Send a Discord alert with rate limiting.
Returns True if alert was sent, False if rate limited.
"""
if not should_send_alert(func_name, exception, context):
return False
try:
from backend.util.clients import get_notification_manager_client
from backend.util.metrics import DiscordChannel
notification_client = get_notification_manager_client()
notification_client.discord_system_alert(
alert_msg, channel or DiscordChannel.PLATFORM
)
return True
except Exception as alert_error:
logger.error(f"Failed to send Discord alert: {alert_error}")
return False
def _send_critical_retry_alert(
func_name: str, attempt_number: int, exception: Exception, context: str = ""
):
"""Send alert when a function is approaching the retry failure threshold."""
try:
# Import here to avoid circular imports
from backend.util.clients import get_notification_manager_client
notification_client = get_notification_manager_client()
prefix = f"{context}: " if context else ""
alert_msg = (
f"🚨 CRITICAL: Operation Approaching Failure Threshold: {prefix}'{func_name}'\n\n"
f"Current attempt: {attempt_number}/{EXCESSIVE_RETRY_THRESHOLD}\n"
f"Error: {type(exception).__name__}: {exception}\n\n"
f"This operation is about to fail permanently. Investigate immediately."
)
notification_client.discord_system_alert(alert_msg)
prefix = f"{context}: " if context else ""
if send_rate_limited_discord_alert(
func_name,
exception,
context,
f"🚨 CRITICAL: Operation Approaching Failure Threshold: {prefix}'{func_name}'\n\n"
f"Current attempt: {attempt_number}/{EXCESSIVE_RETRY_THRESHOLD}\n"
f"Error: {type(exception).__name__}: {exception}\n\n"
f"This operation is about to fail permanently. Investigate immediately.",
):
logger.critical(
f"CRITICAL ALERT SENT: Operation {func_name} at attempt {attempt_number}"
)
except Exception as alert_error:
logger.error(f"Failed to send critical retry alert: {alert_error}")
# Don't let alerting failures break the main flow
def _create_retry_callback(context: str = ""):
"""Create a retry callback with optional context."""
@@ -66,7 +105,7 @@ def _create_retry_callback(context: str = ""):
f"{type(exception).__name__}: {exception}"
)
else:
# Retry attempt - send critical alert only once at threshold
# Retry attempt - send critical alert only once at threshold (rate limited)
if attempt_number == EXCESSIVE_RETRY_THRESHOLD:
_send_critical_retry_alert(
func_name, attempt_number, exception, context
@@ -131,7 +170,7 @@ def _log_prefix(resource_name: str, conn_id: str):
def conn_retry(
resource_name: str,
action_name: str,
max_retry: int = 5,
max_retry: int = 100,
max_wait: float = 30,
):
conn_id = str(uuid4())
@@ -139,10 +178,29 @@ def conn_retry(
def on_retry(retry_state):
prefix = _log_prefix(resource_name, conn_id)
exception = retry_state.outcome.exception()
attempt_number = retry_state.attempt_number
func_name = getattr(retry_state.fn, "__name__", "unknown")
if retry_state.outcome.failed and retry_state.next_action is None:
logger.error(f"{prefix} {action_name} failed after retries: {exception}")
else:
if attempt_number == EXCESSIVE_RETRY_THRESHOLD:
if send_rate_limited_discord_alert(
func_name,
exception,
f"{resource_name}_infrastructure",
f"🚨 **Critical Infrastructure Connection Issue**\n"
f"Resource: {resource_name}\n"
f"Action: {action_name}\n"
f"Function: {func_name}\n"
f"Current attempt: {attempt_number}/{max_retry + 1}\n"
f"Error: {type(exception).__name__}: {str(exception)[:200]}{'...' if len(str(exception)) > 200 else ''}\n\n"
f"Infrastructure component is approaching failure threshold. Investigate immediately.",
):
logger.critical(
f"INFRASTRUCTURE ALERT SENT: {resource_name} at {attempt_number} attempts"
)
logger.warning(
f"{prefix} {action_name} failed: {exception}. Retrying now..."
)
@@ -218,8 +276,8 @@ def continuous_retry(*, retry_delay: float = 1.0):
@wraps(func)
async def async_wrapper(*args, **kwargs):
counter = 0
while True:
counter = 0
try:
return await func(*args, **kwargs)
except Exception as exc:

204
autogpt_platform/backend/backend/util/retry_test.py
View File

@@ -1,8 +1,19 @@
import asyncio
import threading
import time
from unittest.mock import Mock, patch
import pytest
from backend.util.retry import conn_retry
from backend.util.retry import (
ALERT_RATE_LIMIT_SECONDS,
_alert_rate_limiter,
_rate_limiter_lock,
_send_critical_retry_alert,
conn_retry,
create_retry_decorator,
should_send_alert,
)
def test_conn_retry_sync_function():
@@ -47,3 +58,194 @@ async def test_conn_retry_async_function():
with pytest.raises(ValueError) as e:
await test_function()
assert str(e.value) == "Test error"
class TestRetryRateLimiting:
"""Test the rate limiting functionality for critical retry alerts."""
def setup_method(self):
"""Reset rate limiter state before each test."""
with _rate_limiter_lock:
_alert_rate_limiter.clear()
def test_should_send_alert_allows_first_occurrence(self):
"""Test that the first occurrence of an error allows alert."""
exc = ValueError("test error")
assert should_send_alert("test_func", exc, "test_context") is True
def test_should_send_alert_rate_limits_duplicate(self):
"""Test that duplicate errors are rate limited."""
exc = ValueError("test error")
# First call should be allowed
assert should_send_alert("test_func", exc, "test_context") is True
# Second call should be rate limited
assert should_send_alert("test_func", exc, "test_context") is False
def test_should_send_alert_allows_different_errors(self):
"""Test that different errors are allowed even if same function."""
exc1 = ValueError("error 1")
exc2 = ValueError("error 2")
# First error should be allowed
assert should_send_alert("test_func", exc1, "test_context") is True
# Different error should also be allowed
assert should_send_alert("test_func", exc2, "test_context") is True
def test_should_send_alert_allows_different_contexts(self):
"""Test that same error in different contexts is allowed."""
exc = ValueError("test error")
# First context should be allowed
assert should_send_alert("test_func", exc, "context1") is True
# Different context should also be allowed
assert should_send_alert("test_func", exc, "context2") is True
def test_should_send_alert_allows_different_functions(self):
"""Test that same error in different functions is allowed."""
exc = ValueError("test error")
# First function should be allowed
assert should_send_alert("func1", exc, "test_context") is True
# Different function should also be allowed
assert should_send_alert("func2", exc, "test_context") is True
def test_should_send_alert_respects_time_window(self):
"""Test that alerts are allowed again after the rate limit window."""
exc = ValueError("test error")
# First call should be allowed
assert should_send_alert("test_func", exc, "test_context") is True
# Immediately after should be rate limited
assert should_send_alert("test_func", exc, "test_context") is False
# Mock time to simulate passage of rate limit window
current_time = time.time()
with patch("backend.util.retry.time.time") as mock_time:
# Simulate time passing beyond rate limit window
mock_time.return_value = current_time + ALERT_RATE_LIMIT_SECONDS + 1
assert should_send_alert("test_func", exc, "test_context") is True
def test_should_send_alert_thread_safety(self):
"""Test that rate limiting is thread-safe."""
exc = ValueError("test error")
results = []
def check_alert():
result = should_send_alert("test_func", exc, "test_context")
results.append(result)
# Create multiple threads trying to send the same alert
threads = [threading.Thread(target=check_alert) for _ in range(10)]
# Start all threads
for thread in threads:
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Only one thread should have been allowed to send the alert
assert sum(results) == 1
assert len([r for r in results if r is True]) == 1
assert len([r for r in results if r is False]) == 9
@patch("backend.util.clients.get_notification_manager_client")
def test_send_critical_retry_alert_rate_limiting(self, mock_get_client):
"""Test that _send_critical_retry_alert respects rate limiting."""
mock_client = Mock()
mock_get_client.return_value = mock_client
exc = ValueError("spend_credits API error")
# First alert should be sent
_send_critical_retry_alert("spend_credits", 50, exc, "Service communication")
assert mock_client.discord_system_alert.call_count == 1
# Second identical alert should be rate limited (not sent)
_send_critical_retry_alert("spend_credits", 50, exc, "Service communication")
assert mock_client.discord_system_alert.call_count == 1 # Still 1, not 2
# Different error should be allowed
exc2 = ValueError("different API error")
_send_critical_retry_alert("spend_credits", 50, exc2, "Service communication")
assert mock_client.discord_system_alert.call_count == 2
@patch("backend.util.clients.get_notification_manager_client")
def test_send_critical_retry_alert_handles_notification_failure(
self, mock_get_client
):
"""Test that notification failures don't break the rate limiter."""
mock_client = Mock()
mock_client.discord_system_alert.side_effect = Exception("Notification failed")
mock_get_client.return_value = mock_client
exc = ValueError("test error")
# Should not raise exception even if notification fails
_send_critical_retry_alert("test_func", 50, exc, "test_context")
# Rate limiter should still work for subsequent calls
assert should_send_alert("test_func", exc, "test_context") is False
def test_error_signature_generation(self):
"""Test that error signatures are generated correctly for rate limiting."""
# Test with long exception message (should be truncated to 100 chars)
long_message = "x" * 200
exc = ValueError(long_message)
# Should not raise exception and should work normally
assert should_send_alert("test_func", exc, "test_context") is True
assert should_send_alert("test_func", exc, "test_context") is False
def test_real_world_scenario_spend_credits_spam(self):
"""Test the real-world scenario that was causing spam."""
# Simulate the exact error that was causing issues
exc = Exception(
"HTTP 500: Server error '500 Internal Server Error' for url 'http://autogpt-database-manager.prod-agpt.svc.cluster.local:8005/spend_credits'"
)
# First 50 attempts reach threshold - should send alert
with patch(
"backend.util.clients.get_notification_manager_client"
) as mock_get_client:
mock_client = Mock()
mock_get_client.return_value = mock_client
_send_critical_retry_alert(
"_call_method_sync", 50, exc, "Service communication"
)
assert mock_client.discord_system_alert.call_count == 1
# Next 950 failures should not send alerts (rate limited)
for _ in range(950):
_send_critical_retry_alert(
"_call_method_sync", 50, exc, "Service communication"
)
# Still only 1 alert sent total
assert mock_client.discord_system_alert.call_count == 1
@patch("backend.util.clients.get_notification_manager_client")
def test_retry_decorator_with_excessive_failures(self, mock_get_client):
"""Test retry decorator behavior when it hits the alert threshold."""
mock_client = Mock()
mock_get_client.return_value = mock_client
@create_retry_decorator(
max_attempts=60, max_wait=0.1
) # More than EXCESSIVE_RETRY_THRESHOLD, but fast
def always_failing_function():
raise ValueError("persistent failure")
with pytest.raises(ValueError):
always_failing_function()
# Should have sent exactly one alert at the threshold
assert mock_client.discord_system_alert.call_count == 1

19
autogpt_platform/backend/backend/util/service.py
View File

@@ -28,6 +28,7 @@ from fastapi import FastAPI, Request, responses
from pydantic import BaseModel, TypeAdapter, create_model
import backend.util.exceptions as exceptions
from backend.monitoring.instrumentation import instrument_fastapi
from backend.util.json import to_dict
from backend.util.metrics import sentry_init
from backend.util.process import AppProcess, get_service_name
@@ -283,6 +284,24 @@ class AppService(BaseAppService, ABC):
super().run()
self.fastapi_app = FastAPI()
# Add Prometheus instrumentation to all services
try:
instrument_fastapi(
self.fastapi_app,
service_name=self.service_name,
expose_endpoint=True,
endpoint="/metrics",
include_in_schema=False,
)
except ImportError:
logger.warning(
f"Prometheus instrumentation not available for {self.service_name}"
)
except Exception as e:
logger.error(
f"Failed to instrument {self.service_name} with Prometheus: {e}"
)
# Register the exposed API routes.
for attr_name, attr in vars(type(self)).items():
if getattr(attr, EXPOSED_FLAG, False):

6
autogpt_platform/backend/backend/util/settings.py
View File

@@ -148,6 +148,12 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
default=60 * 60,
description="Time in seconds for how far back to check for the late executions.",
)
max_concurrent_graph_executions_per_user: int = Field(
default=25,
ge=1,
le=1000,
description="Maximum number of concurrent graph executions allowed per user per graph.",
)
block_error_rate_threshold: float = Field(
default=0.5,

62
autogpt_platform/backend/migrations/20251003085016_update_store_agent_view_latest_version_only/migration.sql Normal file
View File

@@ -0,0 +1,62 @@
BEGIN;
-- Drop and recreate the StoreAgent view with isAvailable field
DROP VIEW IF EXISTS "StoreAgent";
CREATE OR REPLACE VIEW "StoreAgent" AS
WITH latest_versions AS (
SELECT
"storeListingId",
MAX(version) AS max_version
FROM "StoreListingVersion"
WHERE "submissionStatus" = 'APPROVED'
GROUP BY "storeListingId"
),
agent_versions AS (
SELECT
"storeListingId",
array_agg(DISTINCT version::text ORDER BY version::text) AS versions
FROM "StoreListingVersion"
WHERE "submissionStatus" = 'APPROVED'
GROUP BY "storeListingId"
)
SELECT
sl.id AS listing_id,
slv.id AS "storeListingVersionId",
slv."createdAt" AS updated_at,
sl.slug,
COALESCE(slv.name, '') AS agent_name,
slv."videoUrl" AS agent_video,
COALESCE(slv."imageUrls", ARRAY[]::text[]) AS agent_image,
slv."isFeatured" AS featured,
p.username AS creator_username, -- Allow NULL for malformed sub-agents
p."avatarUrl" AS creator_avatar, -- Allow NULL for malformed sub-agents
slv."subHeading" AS sub_heading,
slv.description,
slv.categories,
COALESCE(ar.run_count, 0::bigint) AS runs,
COALESCE(rs.avg_rating, 0.0)::double precision AS rating,
COALESCE(av.versions, ARRAY[slv.version::text]) AS versions,
slv."isAvailable" AS is_available -- Add isAvailable field to filter sub-agents
FROM "StoreListing" sl
JOIN latest_versions lv
ON sl.id = lv."storeListingId"
JOIN "StoreListingVersion" slv
ON slv."storeListingId" = lv."storeListingId"
AND slv.version = lv.max_version
AND slv."submissionStatus" = 'APPROVED'
JOIN "AgentGraph" a
ON slv."agentGraphId" = a.id
AND slv."agentGraphVersion" = a.version
LEFT JOIN "Profile" p
ON sl."owningUserId" = p."userId"
LEFT JOIN "mv_review_stats" rs
ON sl.id = rs."storeListingId"
LEFT JOIN "mv_agent_run_counts" ar
ON a.id = ar."agentGraphId"
LEFT JOIN agent_versions av
ON sl.id = av."storeListingId"
WHERE sl."isDeleted" = false
AND sl."hasApprovedVersion" = true;
COMMIT;

11
autogpt_platform/backend/migrations/20251007084233_log_search_terms/migration.sql Normal file
View File

@@ -0,0 +1,11 @@
-- CreateTable
CREATE TABLE "SearchTerms" (
"id" BIGSERIAL NOT NULL,
"createdDate" TIMESTAMP(3) NOT NULL,
"searchTerm" TEXT NOT NULL,
CONSTRAINT "SearchTerms_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "SearchTerms_createdDate_idx" ON "SearchTerms"("createdDate");

65
autogpt_platform/backend/migrations/20251013071709_add_use_for_onboarding/migration.sql Normal file
View File

@@ -0,0 +1,65 @@
BEGIN;
-- AlterTable
ALTER TABLE "StoreListing" ADD COLUMN "useForOnboarding" BOOLEAN NOT NULL DEFAULT false;
-- Drop and recreate the StoreAgent view with useForOnboarding field
DROP VIEW IF EXISTS "StoreAgent";
CREATE OR REPLACE VIEW "StoreAgent" AS
WITH latest_versions AS (
SELECT
"storeListingId",
MAX(version) AS max_version
FROM "StoreListingVersion"
WHERE "submissionStatus" = 'APPROVED'
GROUP BY "storeListingId"
),
agent_versions AS (
SELECT
"storeListingId",
array_agg(DISTINCT version::text ORDER BY version::text) AS versions
FROM "StoreListingVersion"
WHERE "submissionStatus" = 'APPROVED'
GROUP BY "storeListingId"
)
SELECT
sl.id AS listing_id,
slv.id AS "storeListingVersionId",
slv."createdAt" AS updated_at,
sl.slug,
COALESCE(slv.name, '') AS agent_name,
slv."videoUrl" AS agent_video,
COALESCE(slv."imageUrls", ARRAY[]::text[]) AS agent_image,
slv."isFeatured" AS featured,
p.username AS creator_username, -- Allow NULL for malformed sub-agents
p."avatarUrl" AS creator_avatar, -- Allow NULL for malformed sub-agents
slv."subHeading" AS sub_heading,
slv.description,
slv.categories,
COALESCE(ar.run_count, 0::bigint) AS runs,
COALESCE(rs.avg_rating, 0.0)::double precision AS rating,
COALESCE(av.versions, ARRAY[slv.version::text]) AS versions,
slv."isAvailable" AS is_available,
COALESCE(sl."useForOnboarding", false) AS "useForOnboarding"
FROM "StoreListing" sl
JOIN latest_versions lv
ON sl.id = lv."storeListingId"
JOIN "StoreListingVersion" slv
ON slv."storeListingId" = lv."storeListingId"
AND slv.version = lv.max_version
AND slv."submissionStatus" = 'APPROVED'
JOIN "AgentGraph" a
ON slv."agentGraphId" = a.id
AND slv."agentGraphVersion" = a.version
LEFT JOIN "Profile" p
ON sl."owningUserId" = p."userId"
LEFT JOIN "mv_review_stats" rs
ON sl.id = rs."storeListingId"
LEFT JOIN "mv_agent_run_counts" ar
ON a.id = ar."agentGraphId"
LEFT JOIN agent_versions av
ON sl.id = av."storeListingId"
WHERE sl."isDeleted" = false
AND sl."hasApprovedVersion" = true;
COMMIT;

284
autogpt_platform/backend/poetry.lock generated
View File

@@ -1307,14 +1307,14 @@ tests = ["coverage", "coveralls", "dill", "mock", "nose"]
[[package]]
name = "faker"
version = "37.6.0"
version = "37.8.0"
description = "Faker is a Python package that generates fake data for you."
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "faker-37.6.0-py3-none-any.whl", hash = "sha256:3c5209b23d7049d596a51db5d76403a0ccfea6fc294ffa2ecfef6a8843b1e6a7"},
{file = "faker-37.6.0.tar.gz", hash = "sha256:0f8cc34f30095184adf87c3c24c45b38b33ad81c35ef6eb0a3118f301143012c"},
{file = "faker-37.8.0-py3-none-any.whl", hash = "sha256:b08233118824423b5fc239f7dd51f145e7018082b4164f8da6a9994e1f1ae793"},
{file = "faker-37.8.0.tar.gz", hash = "sha256:090bb5abbec2b30949a95ce1ba6b20d1d0ed222883d63483a0d4be4a970d6fb8"},
]
[package.dependencies]
@@ -1406,20 +1406,21 @@ packaging = ">=20"
[[package]]
name = "firecrawl-py"
version = "2.16.3"
version = "4.3.6"
description = "Python SDK for Firecrawl API"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "firecrawl_py-2.16.3-py3-none-any.whl", hash = "sha256:94bb46af5e0df6c8ec414ac999a5355c0f5a46f15fd1cf5a02a3b31062db0aa8"},
{file = "firecrawl_py-2.16.3.tar.gz", hash = "sha256:5fd063ef4acc4c4be62648f1e11467336bc127780b3afc28d39078a012e6a14c"},
{file = "firecrawl_py-4.3.6-py3-none-any.whl", hash = "sha256:9b5dffdf5ed08fdbf0966f17e18c1a034d59f42a20b2bf9a6291a83190d7eb0f"},
{file = "firecrawl_py-4.3.6.tar.gz", hash = "sha256:303827a86d0f6237a8ddcaa0bcdaa4c5ee11d9a4880b0685302b8d9a0e191ee0"},
]
[package.dependencies]
aiohttp = "*"
httpx = "*"
nest-asyncio = "*"
pydantic = "*"
pydantic = ">=2.0"
python-dotenv = "*"
requests = "*"
websockets = "*"
@@ -4994,14 +4995,14 @@ files = [
[[package]]
name = "pyright"
version = "1.1.404"
version = "1.1.406"
description = "Command line wrapper for pyright"
optional = false
python-versions = ">=3.7"
groups = ["dev"]
files = [
{file = "pyright-1.1.404-py3-none-any.whl", hash = "sha256:c7b7ff1fdb7219c643079e4c3e7d4125f0dafcc19d253b47e898d130ea426419"},
{file = "pyright-1.1.404.tar.gz", hash = "sha256:455e881a558ca6be9ecca0b30ce08aa78343ecc031d37a198ffa9a7a1abeb63e"},
{file = "pyright-1.1.406-py3-none-any.whl", hash = "sha256:1d81fb43c2407bf566e97e57abb01c811973fdb21b2df8df59f870f688bdca71"},
{file = "pyright-1.1.406.tar.gz", hash = "sha256:c4872bc58c9643dac09e8a2e74d472c62036910b3bd37a32813989ef7576ea2c"},
]
[package.dependencies]
@@ -5059,14 +5060,14 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
[[package]]
name = "pytest-mock"
version = "3.14.1"
version = "3.15.1"
description = "Thin-wrapper around the mock package for easier use with pytest"
optional = false
python-versions = ">=3.8"
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0"},
{file = "pytest_mock-3.14.1.tar.gz", hash = "sha256:159e9edac4c451ce77a5cdb9fc5d1100708d2dd4ba3c3df572f14097351af80e"},
{file = "pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d"},
{file = "pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f"},
]
[package.dependencies]
@@ -5459,106 +5460,127 @@ typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""}
[[package]]
name = "regex"
version = "2024.11.6"
version = "2025.9.18"
description = "Alternative regular expression module, to replace re."
optional = false
python-versions = ">=3.8"
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"},
{file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"},
{file = "regex-2024.11.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164d8b7b3b4bcb2068b97428060b2a53be050085ef94eca7f240e7947f1b080e"},
{file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3660c82f209655a06b587d55e723f0b813d3a7db2e32e5e7dc64ac2a9e86fde"},
{file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d22326fcdef5e08c154280b71163ced384b428343ae16a5ab2b3354aed12436e"},
{file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1ac758ef6aebfc8943560194e9fd0fa18bcb34d89fd8bd2af18183afd8da3a2"},
{file = "regex-2024.11.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:997d6a487ff00807ba810e0f8332c18b4eb8d29463cfb7c820dc4b6e7562d0cf"},
{file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02a02d2bb04fec86ad61f3ea7f49c015a0681bf76abb9857f945d26159d2968c"},
{file = "regex-2024.11.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f02f93b92358ee3f78660e43b4b0091229260c5d5c408d17d60bf26b6c900e86"},
{file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:06eb1be98df10e81ebaded73fcd51989dcf534e3c753466e4b60c4697a003b67"},
{file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:040df6fe1a5504eb0f04f048e6d09cd7c7110fef851d7c567a6b6e09942feb7d"},
{file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabbfc59f2c6edba2a6622c647b716e34e8e3867e0ab975412c5c2f79b82da2"},
{file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8447d2d39b5abe381419319f942de20b7ecd60ce86f16a23b0698f22e1b70008"},
{file = "regex-2024.11.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:da8f5fc57d1933de22a9e23eec290a0d8a5927a5370d24bda9a6abe50683fe62"},
{file = "regex-2024.11.6-cp310-cp310-win32.whl", hash = "sha256:b489578720afb782f6ccf2840920f3a32e31ba28a4b162e13900c3e6bd3f930e"},
{file = "regex-2024.11.6-cp310-cp310-win_amd64.whl", hash = "sha256:5071b2093e793357c9d8b2929dfc13ac5f0a6c650559503bb81189d0a3814519"},
{file = "regex-2024.11.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5478c6962ad548b54a591778e93cd7c456a7a29f8eca9c49e4f9a806dcc5d638"},
{file = "regex-2024.11.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c89a8cc122b25ce6945f0423dc1352cb9593c68abd19223eebbd4e56612c5b7"},
{file = "regex-2024.11.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:94d87b689cdd831934fa3ce16cc15cd65748e6d689f5d2b8f4f4df2065c9fa20"},
{file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1062b39a0a2b75a9c694f7a08e7183a80c63c0d62b301418ffd9c35f55aaa114"},
{file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:167ed4852351d8a750da48712c3930b031f6efdaa0f22fa1933716bfcd6bf4a3"},
{file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d548dafee61f06ebdb584080621f3e0c23fff312f0de1afc776e2a2ba99a74f"},
{file = "regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a19f302cd1ce5dd01a9099aaa19cae6173306d1302a43b627f62e21cf18ac0"},
{file = "regex-2024.11.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bec9931dfb61ddd8ef2ebc05646293812cb6b16b60cf7c9511a832b6f1854b55"},
{file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9714398225f299aa85267fd222f7142fcb5c769e73d7733344efc46f2ef5cf89"},
{file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:202eb32e89f60fc147a41e55cb086db2a3f8cb82f9a9a88440dcfc5d37faae8d"},
{file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4181b814e56078e9b00427ca358ec44333765f5ca1b45597ec7446d3a1ef6e34"},
{file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:068376da5a7e4da51968ce4c122a7cd31afaaec4fccc7856c92f63876e57b51d"},
{file = "regex-2024.11.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f2c4184420d881a3475fb2c6f4d95d53a8d50209a2500723d831036f7c45"},
{file = "regex-2024.11.6-cp311-cp311-win32.whl", hash = "sha256:c36f9b6f5f8649bb251a5f3f66564438977b7ef8386a52460ae77e6070d309d9"},
{file = "regex-2024.11.6-cp311-cp311-win_amd64.whl", hash = "sha256:02e28184be537f0e75c1f9b2f8847dc51e08e6e171c6bde130b2687e0c33cf60"},
{file = "regex-2024.11.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:52fb28f528778f184f870b7cf8f225f5eef0a8f6e3778529bdd40c7b3920796a"},
{file = "regex-2024.11.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9"},
{file = "regex-2024.11.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:805e6b60c54bf766b251e94526ebad60b7de0c70f70a4e6210ee2891acb70bf2"},
{file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b85c2530be953a890eaffde05485238f07029600e8f098cdf1848d414a8b45e4"},
{file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb26437975da7dc36b7efad18aa9dd4ea569d2357ae6b783bf1118dabd9ea577"},
{file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abfa5080c374a76a251ba60683242bc17eeb2c9818d0d30117b4486be10c59d3"},
{file = "regex-2024.11.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b7fa6606c2881c1db9479b0eaa11ed5dfa11c8d60a474ff0e095099f39d98e"},
{file = "regex-2024.11.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c32f75920cf99fe6b6c539c399a4a128452eaf1af27f39bce8909c9a3fd8cbe"},
{file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:982e6d21414e78e1f51cf595d7f321dcd14de1f2881c5dc6a6e23bbbbd68435e"},
{file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a7c2155f790e2fb448faed6dd241386719802296ec588a8b9051c1f5c481bc29"},
{file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:149f5008d286636e48cd0b1dd65018548944e495b0265b45e1bffecce1ef7f39"},
{file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e5364a4502efca094731680e80009632ad6624084aff9a23ce8c8c6820de3e51"},
{file = "regex-2024.11.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0a86e7eeca091c09e021db8eb72d54751e527fa47b8d5787caf96d9831bd02ad"},
{file = "regex-2024.11.6-cp312-cp312-win32.whl", hash = "sha256:32f9a4c643baad4efa81d549c2aadefaeba12249b2adc5af541759237eee1c54"},
{file = "regex-2024.11.6-cp312-cp312-win_amd64.whl", hash = "sha256:a93c194e2df18f7d264092dc8539b8ffb86b45b899ab976aa15d48214138e81b"},
{file = "regex-2024.11.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a6ba92c0bcdf96cbf43a12c717eae4bc98325ca3730f6b130ffa2e3c3c723d84"},
{file = "regex-2024.11.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:525eab0b789891ac3be914d36893bdf972d483fe66551f79d3e27146191a37d4"},
{file = "regex-2024.11.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:086a27a0b4ca227941700e0b31425e7a28ef1ae8e5e05a33826e17e47fbfdba0"},
{file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde01f35767c4a7899b7eb6e823b125a64de314a8ee9791367c9a34d56af18d0"},
{file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b583904576650166b3d920d2bcce13971f6f9e9a396c673187f49811b2769dc7"},
{file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c4de13f06a0d54fa0d5ab1b7138bfa0d883220965a29616e3ea61b35d5f5fc7"},
{file = "regex-2024.11.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3cde6e9f2580eb1665965ce9bf17ff4952f34f5b126beb509fee8f4e994f143c"},
{file = "regex-2024.11.6-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d7f453dca13f40a02b79636a339c5b62b670141e63efd511d3f8f73fba162b3"},
{file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59dfe1ed21aea057a65c6b586afd2a945de04fc7db3de0a6e3ed5397ad491b07"},
{file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b97c1e0bd37c5cd7902e65f410779d39eeda155800b65fc4d04cc432efa9bc6e"},
{file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f9d1e379028e0fc2ae3654bac3cbbef81bf3fd571272a42d56c24007979bafb6"},
{file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:13291b39131e2d002a7940fb176e120bec5145f3aeb7621be6534e46251912c4"},
{file = "regex-2024.11.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f51f88c126370dcec4908576c5a627220da6c09d0bff31cfa89f2523843316d"},
{file = "regex-2024.11.6-cp313-cp313-win32.whl", hash = "sha256:63b13cfd72e9601125027202cad74995ab26921d8cd935c25f09c630436348ff"},
{file = "regex-2024.11.6-cp313-cp313-win_amd64.whl", hash = "sha256:2b3361af3198667e99927da8b84c1b010752fa4b1115ee30beaa332cabc3ef1a"},
{file = "regex-2024.11.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3a51ccc315653ba012774efca4f23d1d2a8a8f278a6072e29c7147eee7da446b"},
{file = "regex-2024.11.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ad182d02e40de7459b73155deb8996bbd8e96852267879396fb274e8700190e3"},
{file = "regex-2024.11.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ba9b72e5643641b7d41fa1f6d5abda2c9a263ae835b917348fc3c928182ad467"},
{file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40291b1b89ca6ad8d3f2b82782cc33807f1406cf68c8d440861da6304d8ffbbd"},
{file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cdf58d0e516ee426a48f7b2c03a332a4114420716d55769ff7108c37a09951bf"},
{file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a36fdf2af13c2b14738f6e973aba563623cb77d753bbbd8d414d18bfaa3105dd"},
{file = "regex-2024.11.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1cee317bfc014c2419a76bcc87f071405e3966da434e03e13beb45f8aced1a6"},
{file = "regex-2024.11.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50153825ee016b91549962f970d6a4442fa106832e14c918acd1c8e479916c4f"},
{file = "regex-2024.11.6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea1bfda2f7162605f6e8178223576856b3d791109f15ea99a9f95c16a7636fb5"},
{file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:df951c5f4a1b1910f1a99ff42c473ff60f8225baa1cdd3539fe2819d9543e9df"},
{file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:072623554418a9911446278f16ecb398fb3b540147a7828c06e2011fa531e773"},
{file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f654882311409afb1d780b940234208a252322c24a93b442ca714d119e68086c"},
{file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:89d75e7293d2b3e674db7d4d9b1bee7f8f3d1609428e293771d1a962617150cc"},
{file = "regex-2024.11.6-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:f65557897fc977a44ab205ea871b690adaef6b9da6afda4790a2484b04293a5f"},
{file = "regex-2024.11.6-cp38-cp38-win32.whl", hash = "sha256:6f44ec28b1f858c98d3036ad5d7d0bfc568bdd7a74f9c24e25f41ef1ebfd81a4"},
{file = "regex-2024.11.6-cp38-cp38-win_amd64.whl", hash = "sha256:bb8f74f2f10dbf13a0be8de623ba4f9491faf58c24064f32b65679b021ed0001"},
{file = "regex-2024.11.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5704e174f8ccab2026bd2f1ab6c510345ae8eac818b613d7d73e785f1310f839"},
{file = "regex-2024.11.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:220902c3c5cc6af55d4fe19ead504de80eb91f786dc102fbd74894b1551f095e"},
{file = "regex-2024.11.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7e351589da0850c125f1600a4c4ba3c722efefe16b297de54300f08d734fbf"},
{file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5056b185ca113c88e18223183aa1a50e66507769c9640a6ff75859619d73957b"},
{file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e34b51b650b23ed3354b5a07aab37034d9f923db2a40519139af34f485f77d0"},
{file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5670bce7b200273eee1840ef307bfa07cda90b38ae56e9a6ebcc9f50da9c469b"},
{file = "regex-2024.11.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08986dce1339bc932923e7d1232ce9881499a0e02925f7402fb7c982515419ef"},
{file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93c0b12d3d3bc25af4ebbf38f9ee780a487e8bf6954c115b9f015822d3bb8e48"},
{file = "regex-2024.11.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:764e71f22ab3b305e7f4c21f1a97e1526a25ebdd22513e251cf376760213da13"},
{file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f056bf21105c2515c32372bbc057f43eb02aae2fda61052e2f7622c801f0b4e2"},
{file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:69ab78f848845569401469da20df3e081e6b5a11cb086de3eed1d48f5ed57c95"},
{file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:86fddba590aad9208e2fa8b43b4c098bb0ec74f15718bb6a704e3c63e2cef3e9"},
{file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:684d7a212682996d21ca12ef3c17353c021fe9de6049e19ac8481ec35574a70f"},
{file = "regex-2024.11.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a03e02f48cd1abbd9f3b7e3586d97c8f7a9721c436f51a5245b3b9483044480b"},
{file = "regex-2024.11.6-cp39-cp39-win32.whl", hash = "sha256:41758407fc32d5c3c5de163888068cfee69cb4c2be844e7ac517a52770f9af57"},
{file = "regex-2024.11.6-cp39-cp39-win_amd64.whl", hash = "sha256:b2837718570f95dd41675328e111345f9b7095d821bac435aac173ac80b19983"},
{file = "regex-2024.11.6.tar.gz", hash = "sha256:7ab159b063c52a0333c884e4679f8d7a85112ee3078fe3d9004b2dd875585519"},
{file = "regex-2025.9.18-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:12296202480c201c98a84aecc4d210592b2f55e200a1d193235c4db92b9f6788"},
{file = "regex-2025.9.18-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:220381f1464a581f2ea988f2220cf2a67927adcef107d47d6897ba5a2f6d51a4"},
{file = "regex-2025.9.18-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:87f681bfca84ebd265278b5daa1dcb57f4db315da3b5d044add7c30c10442e61"},
{file = "regex-2025.9.18-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34d674cbba70c9398074c8a1fcc1a79739d65d1105de2a3c695e2b05ea728251"},
{file = "regex-2025.9.18-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:385c9b769655cb65ea40b6eea6ff763cbb6d69b3ffef0b0db8208e1833d4e746"},
{file = "regex-2025.9.18-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8900b3208e022570ae34328712bef6696de0804c122933414014bae791437ab2"},
{file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c204e93bf32cd7a77151d44b05eb36f469d0898e3fba141c026a26b79d9914a0"},
{file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3acc471d1dd7e5ff82e6cacb3b286750decd949ecd4ae258696d04f019817ef8"},
{file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6479d5555122433728760e5f29edb4c2b79655a8deb681a141beb5c8a025baea"},
{file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:431bd2a8726b000eb6f12429c9b438a24062a535d06783a93d2bcbad3698f8a8"},
{file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0cc3521060162d02bd36927e20690129200e5ac9d2c6d32b70368870b122db25"},
{file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a021217b01be2d51632ce056d7a837d3fa37c543ede36e39d14063176a26ae29"},
{file = "regex-2025.9.18-cp310-cp310-win32.whl", hash = "sha256:4a12a06c268a629cb67cc1d009b7bb0be43e289d00d5111f86a2efd3b1949444"},
{file = "regex-2025.9.18-cp310-cp310-win_amd64.whl", hash = "sha256:47acd811589301298c49db2c56bde4f9308d6396da92daf99cba781fa74aa450"},
{file = "regex-2025.9.18-cp310-cp310-win_arm64.whl", hash = "sha256:16bd2944e77522275e5ee36f867e19995bcaa533dcb516753a26726ac7285442"},
{file = "regex-2025.9.18-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:51076980cd08cd13c88eb7365427ae27f0d94e7cebe9ceb2bb9ffdae8fc4d82a"},
{file = "regex-2025.9.18-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:828446870bd7dee4e0cbeed767f07961aa07f0ea3129f38b3ccecebc9742e0b8"},
{file = "regex-2025.9.18-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28821d5637866479ec4cc23b8c990f5bc6dd24e5e4384ba4a11d38a526e1414"},
{file = "regex-2025.9.18-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:726177ade8e481db669e76bf99de0b278783be8acd11cef71165327abd1f170a"},
{file = "regex-2025.9.18-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5cca697da89b9f8ea44115ce3130f6c54c22f541943ac8e9900461edc2b8bd4"},
{file = "regex-2025.9.18-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dfbde38f38004703c35666a1e1c088b778e35d55348da2b7b278914491698d6a"},
{file = "regex-2025.9.18-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2f422214a03fab16bfa495cfec72bee4aaa5731843b771860a471282f1bf74f"},
{file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a295916890f4df0902e4286bc7223ee7f9e925daa6dcdec4192364255b70561a"},
{file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5db95ff632dbabc8c38c4e82bf545ab78d902e81160e6e455598014f0abe66b9"},
{file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fb967eb441b0f15ae610b7069bdb760b929f267efbf522e814bbbfffdf125ce2"},
{file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f04d2f20da4053d96c08f7fde6e1419b7ec9dbcee89c96e3d731fca77f411b95"},
{file = "regex-2025.9.18-cp311-cp311-win32.whl", hash = "sha256:895197241fccf18c0cea7550c80e75f185b8bd55b6924fcae269a1a92c614a07"},
{file = "regex-2025.9.18-cp311-cp311-win_amd64.whl", hash = "sha256:7e2b414deae99166e22c005e154a5513ac31493db178d8aec92b3269c9cce8c9"},
{file = "regex-2025.9.18-cp311-cp311-win_arm64.whl", hash = "sha256:fb137ec7c5c54f34a25ff9b31f6b7b0c2757be80176435bf367111e3f71d72df"},
{file = "regex-2025.9.18-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:436e1b31d7efd4dcd52091d076482031c611dde58bf9c46ca6d0a26e33053a7e"},
{file = "regex-2025.9.18-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c190af81e5576b9c5fdc708f781a52ff20f8b96386c6e2e0557a78402b029f4a"},
{file = "regex-2025.9.18-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4121f1ce2b2b5eec4b397cc1b277686e577e658d8f5870b7eb2d726bd2300ab"},
{file = "regex-2025.9.18-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:300e25dbbf8299d87205e821a201057f2ef9aa3deb29caa01cd2cac669e508d5"},
{file = "regex-2025.9.18-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b47fcf9f5316c0bdaf449e879407e1b9937a23c3b369135ca94ebc8d74b1742"},
{file = "regex-2025.9.18-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:57a161bd3acaa4b513220b49949b07e252165e6b6dc910ee7617a37ff4f5b425"},
{file = "regex-2025.9.18-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f130c3a7845ba42de42f380fff3c8aebe89a810747d91bcf56d40a069f15352"},
{file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5f96fa342b6f54dcba928dd452e8d8cb9f0d63e711d1721cd765bb9f73bb048d"},
{file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0f0d676522d68c207828dcd01fb6f214f63f238c283d9f01d85fc664c7c85b56"},
{file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:40532bff8a1a0621e7903ae57fce88feb2e8a9a9116d341701302c9302aef06e"},
{file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:039f11b618ce8d71a1c364fdee37da1012f5a3e79b1b2819a9f389cd82fd6282"},
{file = "regex-2025.9.18-cp312-cp312-win32.whl", hash = "sha256:e1dd06f981eb226edf87c55d523131ade7285137fbde837c34dc9d1bf309f459"},
{file = "regex-2025.9.18-cp312-cp312-win_amd64.whl", hash = "sha256:3d86b5247bf25fa3715e385aa9ff272c307e0636ce0c9595f64568b41f0a9c77"},
{file = "regex-2025.9.18-cp312-cp312-win_arm64.whl", hash = "sha256:032720248cbeeae6444c269b78cb15664458b7bb9ed02401d3da59fe4d68c3a5"},
{file = "regex-2025.9.18-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2a40f929cd907c7e8ac7566ac76225a77701a6221bca937bdb70d56cb61f57b2"},
{file = "regex-2025.9.18-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c90471671c2cdf914e58b6af62420ea9ecd06d1554d7474d50133ff26ae88feb"},
{file = "regex-2025.9.18-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a351aff9e07a2dabb5022ead6380cff17a4f10e4feb15f9100ee56c4d6d06af"},
{file = "regex-2025.9.18-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc4b8e9d16e20ddfe16430c23468a8707ccad3365b06d4536142e71823f3ca29"},
{file = "regex-2025.9.18-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4b8cdbddf2db1c5e80338ba2daa3cfa3dec73a46fff2a7dda087c8efbf12d62f"},
{file = "regex-2025.9.18-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a276937d9d75085b2c91fb48244349c6954f05ee97bba0963ce24a9d915b8b68"},
{file = "regex-2025.9.18-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92a8e375ccdc1256401c90e9dc02b8642894443d549ff5e25e36d7cf8a80c783"},
{file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0dc6893b1f502d73037cf807a321cdc9be29ef3d6219f7970f842475873712ac"},
{file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a61e85bfc63d232ac14b015af1261f826260c8deb19401c0597dbb87a864361e"},
{file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1ef86a9ebc53f379d921fb9a7e42b92059ad3ee800fcd9e0fe6181090e9f6c23"},
{file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d3bc882119764ba3a119fbf2bd4f1b47bc56c1da5d42df4ed54ae1e8e66fdf8f"},
{file = "regex-2025.9.18-cp313-cp313-win32.whl", hash = "sha256:3810a65675845c3bdfa58c3c7d88624356dd6ee2fc186628295e0969005f928d"},
{file = "regex-2025.9.18-cp313-cp313-win_amd64.whl", hash = "sha256:16eaf74b3c4180ede88f620f299e474913ab6924d5c4b89b3833bc2345d83b3d"},
{file = "regex-2025.9.18-cp313-cp313-win_arm64.whl", hash = "sha256:4dc98ba7dd66bd1261927a9f49bd5ee2bcb3660f7962f1ec02617280fc00f5eb"},
{file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:fe5d50572bc885a0a799410a717c42b1a6b50e2f45872e2b40f4f288f9bce8a2"},
{file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b9d9a2d6cda6621551ca8cf7a06f103adf72831153f3c0d982386110870c4d3"},
{file = "regex-2025.9.18-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:13202e4c4ac0ef9a317fff817674b293c8f7e8c68d3190377d8d8b749f566e12"},
{file = "regex-2025.9.18-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874ff523b0fecffb090f80ae53dc93538f8db954c8bb5505f05b7787ab3402a0"},
{file = "regex-2025.9.18-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d13ab0490128f2bb45d596f754148cd750411afc97e813e4b3a61cf278a23bb6"},
{file = "regex-2025.9.18-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:05440bc172bc4b4b37fb9667e796597419404dbba62e171e1f826d7d2a9ebcef"},
{file = "regex-2025.9.18-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5514b8e4031fdfaa3d27e92c75719cbe7f379e28cacd939807289bce76d0e35a"},
{file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:65d3c38c39efce73e0d9dc019697b39903ba25b1ad45ebbd730d2cf32741f40d"},
{file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ae77e447ebc144d5a26d50055c6ddba1d6ad4a865a560ec7200b8b06bc529368"},
{file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e3ef8cf53dc8df49d7e28a356cf824e3623764e9833348b655cfed4524ab8a90"},
{file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9feb29817df349c976da9a0debf775c5c33fc1c8ad7b9f025825da99374770b7"},
{file = "regex-2025.9.18-cp313-cp313t-win32.whl", hash = "sha256:168be0d2f9b9d13076940b1ed774f98595b4e3c7fc54584bba81b3cc4181742e"},
{file = "regex-2025.9.18-cp313-cp313t-win_amd64.whl", hash = "sha256:d59ecf3bb549e491c8104fea7313f3563c7b048e01287db0a90485734a70a730"},
{file = "regex-2025.9.18-cp313-cp313t-win_arm64.whl", hash = "sha256:dbef80defe9fb21310948a2595420b36c6d641d9bea4c991175829b2cc4bc06a"},
{file = "regex-2025.9.18-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c6db75b51acf277997f3adcd0ad89045d856190d13359f15ab5dda21581d9129"},
{file = "regex-2025.9.18-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8f9698b6f6895d6db810e0bda5364f9ceb9e5b11328700a90cae573574f61eea"},
{file = "regex-2025.9.18-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29cd86aa7cb13a37d0f0d7c21d8d949fe402ffa0ea697e635afedd97ab4b69f1"},
{file = "regex-2025.9.18-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c9f285a071ee55cd9583ba24dde006e53e17780bb309baa8e4289cd472bcc47"},
{file = "regex-2025.9.18-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5adf266f730431e3be9021d3e5b8d5ee65e563fec2883ea8093944d21863b379"},
{file = "regex-2025.9.18-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1137cabc0f38807de79e28d3f6e3e3f2cc8cfb26bead754d02e6d1de5f679203"},
{file = "regex-2025.9.18-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7cc9e5525cada99699ca9223cce2d52e88c52a3d2a0e842bd53de5497c604164"},
{file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bbb9246568f72dce29bcd433517c2be22c7791784b223a810225af3b50d1aafb"},
{file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6a52219a93dd3d92c675383efff6ae18c982e2d7651c792b1e6d121055808743"},
{file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:ae9b3840c5bd456780e3ddf2f737ab55a79b790f6409182012718a35c6d43282"},
{file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d488c236ac497c46a5ac2005a952c1a0e22a07be9f10c3e735bc7d1209a34773"},
{file = "regex-2025.9.18-cp314-cp314-win32.whl", hash = "sha256:0c3506682ea19beefe627a38872d8da65cc01ffa25ed3f2e422dffa1474f0788"},
{file = "regex-2025.9.18-cp314-cp314-win_amd64.whl", hash = "sha256:57929d0f92bebb2d1a83af372cd0ffba2263f13f376e19b1e4fa32aec4efddc3"},
{file = "regex-2025.9.18-cp314-cp314-win_arm64.whl", hash = "sha256:6a4b44df31d34fa51aa5c995d3aa3c999cec4d69b9bd414a8be51984d859f06d"},
{file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:b176326bcd544b5e9b17d6943f807697c0cb7351f6cfb45bf5637c95ff7e6306"},
{file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:0ffd9e230b826b15b369391bec167baed57c7ce39efc35835448618860995946"},
{file = "regex-2025.9.18-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec46332c41add73f2b57e2f5b642f991f6b15e50e9f86285e08ffe3a512ac39f"},
{file = "regex-2025.9.18-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b80fa342ed1ea095168a3f116637bd1030d39c9ff38dc04e54ef7c521e01fc95"},
{file = "regex-2025.9.18-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4d97071c0ba40f0cf2a93ed76e660654c399a0a04ab7d85472239460f3da84b"},
{file = "regex-2025.9.18-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0ac936537ad87cef9e0e66c5144484206c1354224ee811ab1519a32373e411f3"},
{file = "regex-2025.9.18-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dec57f96d4def58c422d212d414efe28218d58537b5445cf0c33afb1b4768571"},
{file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:48317233294648bf7cd068857f248e3a57222259a5304d32c7552e2284a1b2ad"},
{file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:274687e62ea3cf54846a9b25fc48a04459de50af30a7bd0b61a9e38015983494"},
{file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a78722c86a3e7e6aadf9579e3b0ad78d955f2d1f1a8ca4f67d7ca258e8719d4b"},
{file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:06104cd203cdef3ade989a1c45b6215bf42f8b9dd705ecc220c173233f7cba41"},
{file = "regex-2025.9.18-cp314-cp314t-win32.whl", hash = "sha256:2e1eddc06eeaffd249c0adb6fafc19e2118e6308c60df9db27919e96b5656096"},
{file = "regex-2025.9.18-cp314-cp314t-win_amd64.whl", hash = "sha256:8620d247fb8c0683ade51217b459cb4a1081c0405a3072235ba43a40d355c09a"},
{file = "regex-2025.9.18-cp314-cp314t-win_arm64.whl", hash = "sha256:b7531a8ef61de2c647cdf68b3229b071e46ec326b3138b2180acb4275f470b01"},
{file = "regex-2025.9.18-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3dbcfcaa18e9480669030d07371713c10b4f1a41f791ffa5cb1a99f24e777f40"},
{file = "regex-2025.9.18-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1e85f73ef7095f0380208269055ae20524bfde3f27c5384126ddccf20382a638"},
{file = "regex-2025.9.18-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9098e29b3ea4ffffeade423f6779665e2a4f8db64e699c0ed737ef0db6ba7b12"},
{file = "regex-2025.9.18-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90b6b7a2d0f45b7ecaaee1aec6b362184d6596ba2092dd583ffba1b78dd0231c"},
{file = "regex-2025.9.18-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c81b892af4a38286101502eae7aec69f7cd749a893d9987a92776954f3943408"},
{file = "regex-2025.9.18-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3b524d010973f2e1929aeb635418d468d869a5f77b52084d9f74c272189c251d"},
{file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b498437c026a3d5d0be0020023ff76d70ae4d77118e92f6f26c9d0423452446"},
{file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0716e4d6e58853d83f6563f3cf25c281ff46cf7107e5f11879e32cb0b59797d9"},
{file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:065b6956749379d41db2625f880b637d4acc14c0a4de0d25d609a62850e96d36"},
{file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d4a691494439287c08ddb9b5793da605ee80299dd31e95fa3f323fac3c33d9d4"},
{file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ef8d10cc0989565bcbe45fb4439f044594d5c2b8919d3d229ea2c4238f1d55b0"},
{file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4baeb1b16735ac969a7eeecc216f1f8b7caf60431f38a2671ae601f716a32d25"},
{file = "regex-2025.9.18-cp39-cp39-win32.whl", hash = "sha256:8e5f41ad24a1e0b5dfcf4c4e5d9f5bd54c895feb5708dd0c1d0d35693b24d478"},
{file = "regex-2025.9.18-cp39-cp39-win_amd64.whl", hash = "sha256:50e8290707f2fb8e314ab3831e594da71e062f1d623b05266f8cfe4db4949afd"},
{file = "regex-2025.9.18-cp39-cp39-win_arm64.whl", hash = "sha256:039a9d7195fd88c943d7c777d4941e8ef736731947becce773c31a1009cb3c35"},
{file = "regex-2025.9.18.tar.gz", hash = "sha256:c5ba23274c61c6fef447ba6a39333297d0c247f53059dba0bca415cac511edc4"},
]
[[package]]
@@ -5825,31 +5847,31 @@ pyasn1 = ">=0.1.3"
[[package]]
name = "ruff"
version = "0.12.11"
version = "0.13.3"
description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false
python-versions = ">=3.7"
groups = ["dev"]
files = [
{file = "ruff-0.12.11-py3-none-linux_armv6l.whl", hash = "sha256:93fce71e1cac3a8bf9200e63a38ac5c078f3b6baebffb74ba5274fb2ab276065"},
{file = "ruff-0.12.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b8e33ac7b28c772440afa80cebb972ffd823621ded90404f29e5ab6d1e2d4b93"},
{file = "ruff-0.12.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d69fb9d4937aa19adb2e9f058bc4fbfe986c2040acb1a4a9747734834eaa0bfd"},
{file = "ruff-0.12.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:411954eca8464595077a93e580e2918d0a01a19317af0a72132283e28ae21bee"},
{file = "ruff-0.12.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6a2c0a2e1a450f387bf2c6237c727dd22191ae8c00e448e0672d624b2bbd7fb0"},
{file = "ruff-0.12.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ca4c3a7f937725fd2413c0e884b5248a19369ab9bdd850b5781348ba283f644"},
{file = "ruff-0.12.11-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4d1df0098124006f6a66ecf3581a7f7e754c4df7644b2e6704cd7ca80ff95211"},
{file = "ruff-0.12.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a8dd5f230efc99a24ace3b77e3555d3fbc0343aeed3fc84c8d89e75ab2ff793"},
{file = "ruff-0.12.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4dc75533039d0ed04cd33fb8ca9ac9620b99672fe7ff1533b6402206901c34ee"},
{file = "ruff-0.12.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fc58f9266d62c6eccc75261a665f26b4ef64840887fc6cbc552ce5b29f96cc8"},
{file = "ruff-0.12.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:5a0113bd6eafd545146440225fe60b4e9489f59eb5f5f107acd715ba5f0b3d2f"},
{file = "ruff-0.12.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0d737b4059d66295c3ea5720e6efc152623bb83fde5444209b69cd33a53e2000"},
{file = "ruff-0.12.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:916fc5defee32dbc1fc1650b576a8fed68f5e8256e2180d4d9855aea43d6aab2"},
{file = "ruff-0.12.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c984f07d7adb42d3ded5be894fb4007f30f82c87559438b4879fe7aa08c62b39"},
{file = "ruff-0.12.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e07fbb89f2e9249f219d88331c833860489b49cdf4b032b8e4432e9b13e8a4b9"},
{file = "ruff-0.12.11-py3-none-win32.whl", hash = "sha256:c792e8f597c9c756e9bcd4d87cf407a00b60af77078c96f7b6366ea2ce9ba9d3"},
{file = "ruff-0.12.11-py3-none-win_amd64.whl", hash = "sha256:a3283325960307915b6deb3576b96919ee89432ebd9c48771ca12ee8afe4a0fd"},
{file = "ruff-0.12.11-py3-none-win_arm64.whl", hash = "sha256:bae4d6e6a2676f8fb0f98b74594a048bae1b944aab17e9f5d504062303c6dbea"},
{file = "ruff-0.12.11.tar.gz", hash = "sha256:c6b09ae8426a65bbee5425b9d0b82796dbb07cb1af045743c79bfb163001165d"},
{file = "ruff-0.13.3-py3-none-linux_armv6l.whl", hash = "sha256:311860a4c5e19189c89d035638f500c1e191d283d0cc2f1600c8c80d6dcd430c"},
{file = "ruff-0.13.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2bdad6512fb666b40fcadb65e33add2b040fc18a24997d2e47fee7d66f7fcae2"},
{file = "ruff-0.13.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fc6fa4637284708d6ed4e5e970d52fc3b76a557d7b4e85a53013d9d201d93286"},
{file = "ruff-0.13.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c9e6469864f94a98f412f20ea143d547e4c652f45e44f369d7b74ee78185838"},
{file = "ruff-0.13.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5bf62b705f319476c78891e0e97e965b21db468b3c999086de8ffb0d40fd2822"},
{file = "ruff-0.13.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cc1abed87ce40cb07ee0667ce99dbc766c9f519eabfd948ed87295d8737c60"},
{file = "ruff-0.13.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4fb75e7c402d504f7a9a259e0442b96403fa4a7310ffe3588d11d7e170d2b1e3"},
{file = "ruff-0.13.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:17b951f9d9afb39330b2bdd2dd144ce1c1335881c277837ac1b50bfd99985ed3"},
{file = "ruff-0.13.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6052f8088728898e0a449f0dde8fafc7ed47e4d878168b211977e3e7e854f662"},
{file = "ruff-0.13.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc742c50f4ba72ce2a3be362bd359aef7d0d302bf7637a6f942eaa763bd292af"},
{file = "ruff-0.13.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8e5640349493b378431637019366bbd73c927e515c9c1babfea3e932f5e68e1d"},
{file = "ruff-0.13.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6b139f638a80eae7073c691a5dd8d581e0ba319540be97c343d60fb12949c8d0"},
{file = "ruff-0.13.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:6b547def0a40054825de7cfa341039ebdfa51f3d4bfa6a0772940ed351d2746c"},
{file = "ruff-0.13.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9cc48a3564423915c93573f1981d57d101e617839bef38504f85f3677b3a0a3e"},
{file = "ruff-0.13.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1a993b17ec03719c502881cb2d5f91771e8742f2ca6de740034433a97c561989"},
{file = "ruff-0.13.3-py3-none-win32.whl", hash = "sha256:f14e0d1fe6460f07814d03c6e32e815bff411505178a1f539a38f6097d3e8ee3"},
{file = "ruff-0.13.3-py3-none-win_amd64.whl", hash = "sha256:621e2e5812b691d4f244638d693e640f188bacbb9bc793ddd46837cea0503dd2"},
{file = "ruff-0.13.3-py3-none-win_arm64.whl", hash = "sha256:9e9e9d699841eaf4c2c798fa783df2fabc680b72059a02ca0ed81c460bc58330"},
{file = "ruff-0.13.3.tar.gz", hash = "sha256:5b0ba0db740eefdfbcce4299f49e9eaefc643d4d007749d77d047c2bab19908e"},
]
[[package]]
@@ -7334,4 +7356,4 @@ cffi = ["cffi (>=1.11)"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.14"
content-hash = "b2363edeebb91f410039c8d4b563f683c1edb0cf4bda4f3e6c287040e93639bc"
content-hash = "ff0f6f8d90793ea95f1f7008f7c845432ff46fca0937d5068b4f7cfec0ee7674"

11
autogpt_platform/backend/pyproject.toml
View File

@@ -56,6 +56,7 @@ pytest-asyncio = "^1.1.0"
python-dotenv = "^1.1.1"
python-multipart = "^0.0.20"
redis = "^6.2.0"
regex = "^2025.9.18"
replicate = "^1.0.6"
sentry-sdk = {extras = ["anthropic", "fastapi", "launchdarkly", "openai", "sqlalchemy"], version = "^2.33.2"}
sqlalchemy = "^2.0.40"
@@ -77,7 +78,7 @@ aioclamd = "^1.0.0"
setuptools = "^80.9.0"
gcloud-aio-storage = "^9.5.0"
pandas = "^2.3.1"
firecrawl-py = "^2.16.3"
firecrawl-py = "^4.3.6"
exa-py = "^1.14.20"
croniter = "^6.0.0"
stagehand = "^0.5.1"
@@ -87,16 +88,16 @@ asyncpg = "^0.30.0"
[tool.poetry.group.dev.dependencies]
aiohappyeyeballs = "^2.6.1"
black = "^24.10.0"
faker = "^37.6.0"
faker = "^37.8.0"
httpx = "^0.28.1"
isort = "^5.13.2"
poethepoet = "^0.37.0"
pre-commit = "^4.3.0"
pyright = "^1.1.404"
pytest-mock = "^3.14.0"
pyright = "^1.1.406"
pytest-mock = "^3.15.1"
pytest-watcher = "^0.4.2"
requests = "^2.32.5"
ruff = "^0.12.11"
ruff = "^0.13.3"
# NOTE: please insert new dependencies in their alphabetical location
[build-system]

40
autogpt_platform/backend/schema.prisma
View File

@@ -119,9 +119,9 @@ model AgentGraph {
createdAt DateTime @default(now())
updatedAt DateTime? @updatedAt
name String?
description String?
instructions String?
name String?
description String?
instructions String?
recommendedScheduleCron String?
isActive Boolean @default(true)
@@ -383,9 +383,9 @@ model AgentGraphExecution {
stats Json?
// Sharing fields
isShared Boolean @default(false)
shareToken String? @unique
sharedAt DateTime?
isShared Boolean @default(false)
shareToken String? @unique
sharedAt DateTime?
@@index([agentGraphId, agentGraphVersion])
@@index([userId, isDeleted, createdAt])
@@ -546,7 +546,7 @@ model CreditTransaction {
createdAt DateTime @default(now())
userId String
User User? @relation(fields: [userId], references: [id], onDelete: NoAction)
User User? @relation(fields: [userId], references: [id], onDelete: NoAction)
amount Int
type CreditTransactionType
@@ -588,6 +588,16 @@ model CreditRefundRequest {
////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////
model SearchTerms {
// User ID not being logged as this is anonymous analytics data
// Not using uuid as we want to minimise table size
id BigInt @id @default(autoincrement())
createdDate DateTime
searchTerm String
@@index([createdDate])
}
model Profile {
id String @id @default(uuid())
createdAt DateTime @default(now())
@@ -658,6 +668,7 @@ view StoreAgent {
rating Float
versions String[]
is_available Boolean @default(true)
useForOnboarding Boolean @default(false)
// Materialized views used (refreshed every 15 minutes via pg_cron):
// - mv_agent_run_counts - Pre-aggregated agent execution counts by agentGraphId
@@ -735,6 +746,9 @@ model StoreListing {
// URL-friendly identifier for this agent (moved from StoreListingVersion)
slug String
// Allow this agent to be used during onboarding
useForOnboarding Boolean @default(false)
// The currently active version that should be shown to users
activeVersionId String? @unique
ActiveVersion StoreListingVersion? @relation("ActiveVersion", fields: [activeVersionId], references: [id])
@@ -771,13 +785,13 @@ model StoreListingVersion {
AgentGraph AgentGraph @relation(fields: [agentGraphId, agentGraphVersion], references: [id, version])
// Content fields
name String
subHeading String
videoUrl String?
imageUrls String[]
description String
name String
subHeading String
videoUrl String?
imageUrls String[]
description String
instructions String?
categories String[]
categories String[]
isFeatured Boolean @default(false)

35
autogpt_platform/docker-compose.platform.yml
View File

@@ -117,6 +117,11 @@ services:
- "8006:8006"
networks:
- app-network
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
executor:
build:
@@ -147,6 +152,11 @@ services:
- "8002:8002"
networks:
- app-network
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
websocket_server:
build:
@@ -175,6 +185,11 @@ services:
- "8001:8001"
networks:
- app-network
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
database_manager:
build:
@@ -199,6 +214,11 @@ services:
- "8005:8005"
networks:
- app-network
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
scheduler_server:
build:
@@ -242,6 +262,11 @@ services:
- "8003:8003"
networks:
- app-network
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
notification_server:
build:
@@ -270,6 +295,11 @@ services:
- "8007:8007"
networks:
- app-network
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
frontend:
build:
context: ../
@@ -286,6 +316,11 @@ services:
- "3000:3000"
networks:
- app-network
logging:
driver: json-file
options:
max-size: "10m"
max-file: "3"
# Load environment variables in order (later overrides earlier)
env_file:
- path: ./frontend/.env.default # Base defaults (always exists)

2
autogpt_platform/frontend/instrumentation-client.ts
View File

@@ -31,7 +31,7 @@ Sentry.init({
Sentry.extraErrorDataIntegration(),
Sentry.browserProfilingIntegration(),
Sentry.httpClientIntegration(),
// Sentry.launchDarklyIntegration(),
Sentry.launchDarklyIntegration(),
Sentry.replayIntegration({
unmask: [".sentry-unmask, [data-sentry-unmask]"],
}),

9
autogpt_platform/frontend/next.config.mjs
View File

@@ -2,6 +2,7 @@ import { withSentryConfig } from "@sentry/nextjs";
/** @type {import('next').NextConfig} */
const nextConfig = {
productionBrowserSourceMaps: true,
images: {
domains: [
"images.unsplash.com",
@@ -74,6 +75,14 @@ export default isDevelopmentBuild
// since the source is public anyway :)
hideSourceMaps: false,
// This helps Sentry with sourcemaps... https://docs.sentry.io/platforms/javascript/guides/nextjs/sourcemaps/
sourcemaps: {
disable: false, // Source maps are enabled by default
assets: ["**/*.js", "**/*.js.map"], // Specify which files to upload
ignore: ["**/node_modules/**"], // Files to exclude
deleteSourcemapsAfterUpload: true, // Security: delete after upload
},
// Automatically tree-shake Sentry logger statements to reduce bundle size
disableLogger: true,

6
autogpt_platform/frontend/orval.config.ts
View File

@@ -67,6 +67,12 @@ export default defineConfig({
useQuery: true,
},
},
"getV2Builder search": {
query: {
useInfinite: true,
useInfiniteQueryParam: "page",
},
},
},
},
},

1
autogpt_platform/frontend/package.json
View File

@@ -27,6 +27,7 @@
"dependencies": {
"@faker-js/faker": "10.0.0",
"@hookform/resolvers": "5.2.1",
"@marsidev/react-turnstile": "1.3.1",
"@next/third-parties": "15.4.6",
"@phosphor-icons/react": "2.1.10",
"@radix-ui/react-alert-dialog": "1.1.15",

14
autogpt_platform/frontend/pnpm-lock.yaml generated
View File

@@ -14,6 +14,9 @@ importers:
'@hookform/resolvers':
specifier: 5.2.1
version: 5.2.1(react-hook-form@7.62.0(react@18.3.1))
'@marsidev/react-turnstile':
specifier: 1.3.1
version: 1.3.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@next/third-parties':
specifier: 15.4.6
version: 15.4.6(next@15.4.7(@babel/core@7.28.4)(@opentelemetry/api@1.9.0)(@playwright/test@1.55.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
@@ -1428,6 +1431,12 @@ packages:
peerDependencies:
jsep: ^0.4.0||^1.0.0
'@marsidev/react-turnstile@1.3.1':
resolution: {integrity: sha512-h2THG/75k4Y049hgjSGPIcajxXnh+IZAiXVbryQyVmagkboN7pJtBgR16g8akjwUBSfRrg6jw6KvPDjscQflog==}
peerDependencies:
react: ^17.0.2 || ^18.0.0 || ^19.0
react-dom: ^17.0.2 || ^18.0.0 || ^19.0
'@mdx-js/react@3.1.1':
resolution: {integrity: sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==}
peerDependencies:
@@ -8668,6 +8677,11 @@ snapshots:
dependencies:
jsep: 1.4.0
'@marsidev/react-turnstile@1.3.1(react-dom@18.3.1(react@18.3.1))(react@18.3.1)':
dependencies:
react: 18.3.1
react-dom: 18.3.1(react@18.3.1)
'@mdx-js/react@3.1.1(@types/react@18.3.17)(react@18.3.1)':
dependencies:
'@types/mdx': 2.0.13

93
autogpt_platform/frontend/src/app/(no-navbar)/onboarding/5-run/helpers.ts Normal file
View File

@@ -0,0 +1,93 @@
import type { GraphMeta } from "@/lib/autogpt-server-api";
import type {
BlockIOCredentialsSubSchema,
CredentialsMetaInput,
} from "@/lib/autogpt-server-api/types";
import type { InputValues } from "./types";
export function computeInitialAgentInputs(
agent: GraphMeta | null,
existingInputs?: InputValues | null,
): InputValues {
const properties = agent?.input_schema?.properties || {};
const result: InputValues = {};
Object.entries(properties).forEach(([key, subSchema]) => {
if (
existingInputs &&
key in existingInputs &&
existingInputs[key] != null
) {
result[key] = existingInputs[key];
return;
}
// GraphIOSubSchema.default is typed as string, but server may return other primitives
const def = (subSchema as unknown as { default?: string | number }).default;
result[key] = def ?? "";
});
return result;
}
export function getAgentCredentialsInputFields(agent: GraphMeta | null) {
const hasNoInputs =
!agent?.credentials_input_schema ||
typeof agent.credentials_input_schema !== "object" ||
!("properties" in agent.credentials_input_schema) ||
!agent.credentials_input_schema.properties;
if (hasNoInputs) return {};
return agent.credentials_input_schema.properties;
}
export function areAllCredentialsSet(
fields: Record<string, BlockIOCredentialsSubSchema>,
inputs: Record<string, CredentialsMetaInput | undefined>,
) {
const required = Object.keys(fields || {});
return required.every((k) => Boolean(inputs[k]));
}
type IsRunDisabledParams = {
agent: GraphMeta | null;
isRunning: boolean;
agentInputs: InputValues | null | undefined;
credentialsRequired: boolean;
credentialsSatisfied: boolean;
};
export function isRunDisabled({
agent,
isRunning,
agentInputs,
credentialsRequired,
credentialsSatisfied,
}: IsRunDisabledParams) {
const hasEmptyInput = Object.values(agentInputs || {}).some(
(value) => String(value).trim() === "",
);
if (hasEmptyInput) return true;
if (!agent) return true;
if (isRunning) return true;
if (credentialsRequired && !credentialsSatisfied) return true;
return false;
}
export function getSchemaDefaultCredentials(
schema: BlockIOCredentialsSubSchema,
): CredentialsMetaInput | undefined {
return schema.default as CredentialsMetaInput | undefined;
}
export function sanitizeCredentials(
map: Record<string, CredentialsMetaInput | undefined>,
): Record<string, CredentialsMetaInput> {
const sanitized: Record<string, CredentialsMetaInput> = {};
for (const [key, value] of Object.entries(map)) {
if (value) sanitized[key] = value;
}
return sanitized;
}

112
autogpt_platform/frontend/src/app/(no-navbar)/onboarding/5-run/page.tsx
View File

@@ -13,13 +13,24 @@ import {
} from "@/components/__legacy__/ui/card";
import { useToast } from "@/components/molecules/Toast/use-toast";
import { GraphMeta, StoreAgentDetails } from "@/lib/autogpt-server-api";
import type { InputValues } from "./types";
import { useBackendAPI } from "@/lib/autogpt-server-api/context";
import { cn } from "@/lib/utils";
import { Play } from "lucide-react";
import { useRouter } from "next/navigation";
import { useCallback, useEffect, useState } from "react";
import { useEffect, useState } from "react";
import { RunAgentInputs } from "@/app/(platform)/library/agents/[id]/components/AgentRunsView/components/RunAgentInputs/RunAgentInputs";
import { InformationTooltip } from "@/components/molecules/InformationTooltip/InformationTooltip";
import { CredentialsInput } from "@/app/(platform)/library/agents/[id]/components/AgentRunsView/components/CredentialsInputs/CredentialsInputs";
import type { CredentialsMetaInput } from "@/lib/autogpt-server-api/types";
import {
areAllCredentialsSet,
computeInitialAgentInputs,
getAgentCredentialsInputFields,
isRunDisabled,
getSchemaDefaultCredentials,
sanitizeCredentials,
} from "./helpers";
export default function Page() {
const { state, updateState, setStep } = useOnboarding(
@@ -30,13 +41,16 @@ export default function Page() {
const [agent, setAgent] = useState<GraphMeta | null>(null);
const [storeAgent, setStoreAgent] = useState<StoreAgentDetails | null>(null);
const [runningAgent, setRunningAgent] = useState(false);
const [inputCredentials, setInputCredentials] = useState<
Record<string, CredentialsMetaInput | undefined>
>({});
const { toast } = useToast();
const router = useRouter();
const api = useBackendAPI();
useEffect(() => {
setStep(5);
}, [setStep]);
}, []);
useEffect(() => {
if (!state?.selectedStoreListingVersionId) {
@@ -49,40 +63,36 @@ export default function Page() {
});
api
.getGraphMetaByStoreListingVersionID(state.selectedStoreListingVersionId)
.then((agent) => {
setAgent(agent);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const update: { [key: string]: any } = {};
// Set default values from schema
Object.entries(agent.input_schema.properties).forEach(
([key, value]) => {
// Skip if already set
if (state.agentInput && state.agentInput[key]) {
update[key] = state.agentInput[key];
return;
}
update[key] = value.type !== "null" ? value.default || "" : "";
},
.then((meta) => {
setAgent(meta);
const update = computeInitialAgentInputs(
meta,
(state.agentInput as unknown as InputValues) || null,
);
updateState({
agentInput: update,
});
updateState({ agentInput: update });
});
}, [api, setAgent, updateState, state?.selectedStoreListingVersionId]);
const setAgentInput = useCallback(
(key: string, value: string) => {
updateState({
agentInput: {
...state?.agentInput,
[key]: value,
},
});
},
[state?.agentInput, updateState],
const agentCredentialsInputFields = getAgentCredentialsInputFields(agent);
const credentialsRequired =
Object.keys(agentCredentialsInputFields || {}).length > 0;
const allCredentialsAreSet = areAllCredentialsSet(
agentCredentialsInputFields,
inputCredentials,
);
const runAgent = useCallback(async () => {
function setAgentInput(key: string, value: string) {
updateState({
agentInput: {
...state?.agentInput,
[key]: value,
},
});
}
async function runAgent() {
if (!agent) {
return;
}
@@ -95,6 +105,7 @@ export default function Page() {
libraryAgent.graph_id,
libraryAgent.graph_version,
state?.agentInput || {},
sanitizeCredentials(inputCredentials),
);
updateState({
onboardingAgentExecutionId: runID,
@@ -111,7 +122,7 @@ export default function Page() {
});
setRunningAgent(false);
}
}, [api, agent, router, state?.agentInput, storeAgent, updateState, toast]);
}
const runYourAgent = (
<div className="ml-[104px] w-[481px] pl-5">
@@ -221,6 +232,30 @@ export default function Page() {
<span className="mt-4 text-base font-normal leading-normal text-zinc-600">
When you&apos;re done, click <b>Run Agent</b>.
</span>
{Object.entries(agentCredentialsInputFields || {}).map(
([key, inputSubSchema]) => (
<div key={key} className="mt-4">
<CredentialsInput
schema={inputSubSchema}
selectedCredentials={
inputCredentials[key] ??
getSchemaDefaultCredentials(inputSubSchema)
}
onSelectCredentials={(value) =>
setInputCredentials((prev) => ({
...prev,
[key]: value,
}))
}
siblingInputs={
(state?.agentInput || undefined) as
| Record<string, any>
| undefined
}
/>
</div>
),
)}
<Card className="agpt-box mt-4">
<CardHeader>
<CardTitle className="font-poppins text-lg">Input</CardTitle>
@@ -250,13 +285,14 @@ export default function Page() {
variant="violet"
className="mt-8 w-[136px]"
loading={runningAgent}
disabled={
Object.values(state?.agentInput || {}).some(
(value) => String(value).trim() === "",
) ||
!agent ||
runningAgent
}
disabled={isRunDisabled({
agent,
isRunning: runningAgent,
agentInputs:
(state?.agentInput as unknown as InputValues) || null,
credentialsRequired,
credentialsSatisfied: allCredentialsAreSet,
})}
onClick={runAgent}
icon={<Play className="mr-2" size={18} />}
>

2
autogpt_platform/frontend/src/app/(no-navbar)/onboarding/5-run/types.ts Normal file
View File

@@ -0,0 +1,2 @@
export type InputPrimitive = string | number;
export type InputValues = Record<string, InputPrimitive>;

137
autogpt_platform/frontend/src/components/__legacy__/Wallet.tsx → autogpt_platform/frontend/src/app/(no-navbar)/onboarding/components/Wallet/Wallet.tsx
View File

@@ -9,14 +9,16 @@ import {
import { X } from "lucide-react";
import { Text } from "@/components/atoms/Text/Text";
import { PopoverClose } from "@radix-ui/react-popover";
import { TaskGroups } from "@/app/(no-navbar)/onboarding/components/WalletTaskGroups";
import { ScrollArea } from "./ui/scroll-area";
import { TaskGroups } from "@/app/(no-navbar)/onboarding/components/Wallet/components/WalletTaskGroups";
import { ScrollArea } from "../../../../../components/__legacy__/ui/scroll-area";
import { useOnboarding } from "@/providers/onboarding/onboarding-provider";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { cn } from "@/lib/utils";
import * as party from "party-js";
import WalletRefill from "./WalletRefill";
import WalletRefill from "./components/WalletRefill";
import { OnboardingStep } from "@/lib/autogpt-server-api";
import { storage, Key as StorageKey } from "@/services/storage/local-storage";
import { WalletIcon } from "@phosphor-icons/react";
export interface Task {
id: OnboardingStep;
@@ -38,6 +40,7 @@ export interface TaskGroup {
export default function Wallet() {
const { state, updateState } = useOnboarding();
const groups = useMemo<TaskGroup[]>(() => {
return [
{
@@ -164,27 +167,67 @@ export default function Wallet() {
const [prevCredits, setPrevCredits] = useState<number | null>(credits);
const [flash, setFlash] = useState(false);
const [walletOpen, setWalletOpen] = useState(state?.walletShown || false);
const [walletOpen, setWalletOpen] = useState(false);
const [lastSeenCredits, setLastSeenCredits] = useState<number | null>(null);
const totalCount = useMemo(() => {
return groups.reduce((acc, group) => acc + group.tasks.length, 0);
}, [groups]);
// Get total completed count for all groups
const completedCount = useMemo(() => {
return groups.reduce(
const [completedCount, setCompletedCount] = useState<number | null>(null);
// Needed to show confetti when a new step is completed
const [prevCompletedCount, setPrevCompletedCount] = useState<number | null>(
null,
);
const walletRef = useRef<HTMLButtonElement | null>(null);
useEffect(() => {
if (!state) {
return;
}
const completed = groups.reduce(
(acc, group) =>
acc +
group.tasks.filter((task) => state?.completedSteps?.includes(task.id))
.length,
0,
);
setCompletedCount(completed);
}, [groups, state?.completedSteps]);
// Needed to show confetti when a new step is completed
const [stepsLength, setStepsLength] = useState(completedCount);
// Load last seen credits from localStorage once on mount
useEffect(() => {
const stored = storage.get(StorageKey.WALLET_LAST_SEEN_CREDITS);
if (stored !== undefined && stored !== null) {
const parsed = parseFloat(stored);
if (!Number.isNaN(parsed)) setLastSeenCredits(parsed);
else setLastSeenCredits(0);
} else {
setLastSeenCredits(0);
}
}, []);
const walletRef = useRef<HTMLButtonElement | null>(null);
// Auto-open once if never shown, otherwise open only when credits increase beyond last seen
useEffect(() => {
if (typeof credits !== "number") return;
// Open once for first-time users
if (state && state.walletShown === false) {
setWalletOpen(true);
// Mark as shown so it won't reopen on every reload
updateState({ walletShown: true });
return;
}
// Open if user gained more credits than last acknowledged
if (
lastSeenCredits !== null &&
credits > lastSeenCredits &&
walletOpen === false
) {
setWalletOpen(true);
}
}, [credits, lastSeenCredits, state?.walletShown, updateState, walletOpen]);
const onWalletOpen = useCallback(async () => {
if (!state?.walletShown) {
@@ -206,19 +249,25 @@ export default function Wallet() {
// Confetti effect on the wallet button
useEffect(() => {
if (!state?.completedSteps) {
return;
}
// It's enough to check completed count,
// because the order of completed steps is not important
// If the count is the same, we don't need to do anything
if (completedCount === stepsLength) {
if (completedCount === null || completedCount === prevCompletedCount) {
return;
}
// Otherwise, we need to set the new prevCompletedCount
setPrevCompletedCount(completedCount);
// If there was no previous count, we don't show confetti
if (prevCompletedCount === null) {
return;
}
// Otherwise, we need to set the new length
setStepsLength(completedCount);
// And emit confetti
if (walletRef.current) {
// Fix confetti appearing in the top left corner
const rect = walletRef.current.getBoundingClientRect();
if (rect.width === 0 || rect.height === 0) {
return;
}
setTimeout(() => {
fetchCredits();
party.confetti(walletRef.current!, {
@@ -236,7 +285,8 @@ export default function Wallet() {
state?.notified,
fadeOut,
fetchCredits,
stepsLength,
completedCount,
prevCompletedCount,
walletRef,
]);
@@ -255,28 +305,46 @@ export default function Wallet() {
}, 300);
}, [credits, prevCredits]);
// Do not render until we have both credits and onboarding data
if (credits === null || !state) return null;
return (
<Popover open={walletOpen} onOpenChange={setWalletOpen}>
<Popover
open={walletOpen}
onOpenChange={(open) => {
setWalletOpen(open);
if (!open) {
// Persist the latest acknowledged credits so we only auto-open on future gains
if (typeof credits === "number") {
storage.set(StorageKey.WALLET_LAST_SEEN_CREDITS, String(credits));
setLastSeenCredits(credits);
}
}
}}
>
<PopoverTrigger asChild>
<div className="relative inline-block">
<button
ref={walletRef}
className={cn(
"relative flex items-center gap-1 rounded-md bg-zinc-50 px-3 py-2 text-sm",
"relative flex flex-nowrap items-center gap-2 rounded-md bg-zinc-50 px-3 py-2 text-sm",
)}
onClick={onWalletOpen}
>
Earn credits{" "}
<span className="text-sm font-semibold">
{formatCredits(credits)}
</span>
{completedCount < totalCount && (
<span className="absolute right-1 top-1 h-2 w-2 rounded-full bg-violet-600"></span>
)}
<div className="absolute bottom-[-2.5rem] left-1/2 z-50 hidden -translate-x-1/2 transform whitespace-nowrap rounded-small bg-white px-4 py-2 shadow-md group-hover:block">
<Text variant="body-medium">
{completedCount} of {totalCount} rewards claimed
</Text>
<WalletIcon size={20} className="inline-block md:hidden" />
<div>
<span className="mr-1 hidden md:inline-block">Earn credits </span>
<span className="text-sm font-semibold">
{formatCredits(credits)}
</span>
{completedCount && completedCount < totalCount && (
<span className="absolute right-1 top-1 h-2 w-2 rounded-full bg-violet-600"></span>
)}
<div className="absolute bottom-[-2.5rem] left-1/2 z-50 hidden -translate-x-1/2 transform whitespace-nowrap rounded-small bg-white px-4 py-2 shadow-md group-hover:block">
<Text variant="body-medium">
{completedCount} of {totalCount} rewards claimed
</Text>
</div>
</div>
</button>
<div
@@ -288,10 +356,11 @@ export default function Wallet() {
</div>
</PopoverTrigger>
<PopoverContent
className={cn(
"absolute -right-[7.9rem] -top-[3.2rem] z-50 w-[28.5rem] px-[0.625rem] py-2",
"rounded-xl border-zinc-100 bg-white shadow-[0_3px_3px] shadow-zinc-200",
)}
side="bottom"
align="end"
sideOffset={12}
collisionPadding={16}
className={cn("z-50 w-[28.5rem] px-[0.625rem] py-2")}
>
{/* Header */}
<div className="mx-1 flex items-center justify-between border-b border-zinc-200 pb-3">
@@ -303,7 +372,7 @@ export default function Wallet() {
Earn credits{" "}
<span className="font-semibold">{formatCredits(credits)}</span>
</div>
<PopoverClose>
<PopoverClose aria-label="Close wallet">
<X className="ml-2 h-5 w-5 text-zinc-800 hover:text-foreground" />
</PopoverClose>
</div>

7
autogpt_platform/frontend/src/components/__legacy__/WalletRefill.tsx → autogpt_platform/frontend/src/app/(no-navbar)/onboarding/components/Wallet/components/WalletRefill.tsx
View File

@@ -16,9 +16,12 @@ import {
FormLabel,
FormMessage,
} from "@/components/__legacy__/ui/form";
import { Input } from "./ui/input";
import { Input } from "../../../../../../components/__legacy__/ui/input";
import Link from "next/link";
import { useToast, useToastOnFail } from "../molecules/Toast/use-toast";
import {
useToast,
useToastOnFail,
} from "../../../../../../components/molecules/Toast/use-toast";
import useCredits from "@/hooks/useCredits";
import { useCallback, useEffect, useState } from "react";

5
autogpt_platform/frontend/src/app/(no-navbar)/onboarding/components/WalletTaskGroups.tsx → autogpt_platform/frontend/src/app/(no-navbar)/onboarding/components/Wallet/components/WalletTaskGroups.tsx
View File

@@ -3,7 +3,10 @@ import { ChevronDown, Check, BadgeQuestionMark } from "lucide-react";
import { cn } from "@/lib/utils";
import * as party from "party-js";
import { useOnboarding } from "@/providers/onboarding/onboarding-provider";
import { Task, TaskGroup } from "@/components/__legacy__/Wallet";
import {
Task,
TaskGroup,
} from "@/app/(no-navbar)/onboarding/components/Wallet/Wallet";
interface Props {
groups: TaskGroup[];

6
autogpt_platform/frontend/src/app/(no-navbar)/onboarding/page.tsx
View File

@@ -1,14 +1,16 @@
import BackendAPI from "@/lib/autogpt-server-api";
import { redirect } from "next/navigation";
import { finishOnboarding } from "./6-congrats/actions";
import { shouldShowOnboarding } from "@/app/api/helpers";
// Force dynamic rendering to avoid static generation issues with cookies
export const dynamic = "force-dynamic";
export default async function OnboardingPage() {
const api = new BackendAPI();
const isOnboardingEnabled = await shouldShowOnboarding();
if (!api.isOnboardingEnabled()) {
if (!isOnboardingEnabled) {
redirect("/marketplace");
}
@@ -23,7 +25,7 @@ export default async function OnboardingPage() {
else if (onboarding.completedSteps.includes("AGENT_NEW_RUN"))
redirect("/onboarding/5-run");
else if (onboarding.completedSteps.includes("AGENT_CHOICE"))
redirect("/onboarding/5-agent");
redirect("/onboarding/5-run");
else if (onboarding.completedSteps.includes("INTEGRATIONS"))
redirect("/onboarding/4-agent");
else if (onboarding.completedSteps.includes("USAGE_REASON"))

2
autogpt_platform/frontend/src/app/(platform)/admin/layout.tsx
View File

@@ -31,7 +31,7 @@ export default function AdminLayout({
children: React.ReactNode;
}) {
return (
<div className="flex min-h-screen w-screen flex-col lg:flex-row">
<div className="flex min-h-screen w-full flex-col lg:flex-row">
<Sidebar linkGroups={sidebarLinkGroups} />
<div className="flex-1 pl-4">{children}</div>
</div>

66
autogpt_platform/frontend/src/app/(platform)/auth/callback/route.ts
View File

@@ -2,34 +2,14 @@ import { getServerSupabase } from "@/lib/supabase/server/getServerSupabase";
import BackendAPI from "@/lib/autogpt-server-api";
import { NextResponse } from "next/server";
import { revalidatePath } from "next/cache";
async function shouldShowOnboarding() {
const api = new BackendAPI();
return (
(await api.isOnboardingEnabled()) &&
!(await api.getUserOnboarding()).completedSteps.includes("CONGRATS")
);
}
// Validate redirect URL to prevent open redirect attacks
function validateRedirectUrl(url: string): string {
// Only allow relative URLs that start with /
if (url.startsWith("/") && !url.startsWith("//")) {
return url;
}
// Default to home page for any invalid URLs
return "/";
}
import { shouldShowOnboarding } from "@/app/api/helpers";
// Handle the callback to complete the user session login
export async function GET(request: Request) {
const { searchParams, origin } = new URL(request.url);
const code = searchParams.get("code");
// if "next" is in param, use it as the redirect URL
const nextParam = searchParams.get("next") ?? "/";
// Validate redirect URL to prevent open redirect attacks
let next = validateRedirectUrl(nextParam);
let next = "/marketplace";
if (code) {
const supabase = await getServerSupabase();
@@ -39,7 +19,7 @@ export async function GET(request: Request) {
}
const { error } = await supabase.auth.exchangeCodeForSession(code);
// data.session?.refresh_token is available if you need to store it for later use
if (!error) {
try {
const api = new BackendAPI();
@@ -54,7 +34,45 @@ export async function GET(request: Request) {
}
} catch (createUserError) {
console.error("Error creating user:", createUserError);
// Continue with redirect even if createUser fails
// Handle ApiError from the backend API client
if (
createUserError &&
typeof createUserError === "object" &&
"status" in createUserError
) {
const apiError = createUserError as any;
if (apiError.status === 401) {
// Authentication issues - token missing/invalid
return NextResponse.redirect(
`${origin}/error?message=auth-token-invalid`,
);
} else if (apiError.status >= 500) {
// Server/database errors
return NextResponse.redirect(
`${origin}/error?message=server-error`,
);
} else if (apiError.status === 429) {
// Rate limiting
return NextResponse.redirect(
`${origin}/error?message=rate-limited`,
);
}
}
// Handle network/fetch errors
if (
createUserError instanceof TypeError &&
createUserError.message.includes("fetch")
) {
return NextResponse.redirect(`${origin}/error?message=network-error`);
}
// Generic user creation failure
return NextResponse.redirect(
`${origin}/error?message=user-creation-failed`,
);
}
const forwardedHost = request.headers.get("x-forwarded-host"); // original origin before load balancer

23
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/Flow.tsx → autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/Flow/Flow.tsx
View File

@@ -1,16 +1,14 @@
import { ReactFlow, Background, Controls } from "@xyflow/react";
import { useNodeStore } from "../../stores/nodeStore";
import NewControlPanel from "../NewBlockMenu/NewControlPanel/NewControlPanel";
import NewControlPanel from "../../NewBlockMenu/NewControlPanel/NewControlPanel";
import CustomEdge from "../edges/CustomEdge";
import { useFlow } from "./useFlow";
import { useShallow } from "zustand/react/shallow";
import { useNodeStore } from "../../../stores/nodeStore";
import { useMemo } from "react";
import { CustomNode } from "./nodes/CustomNode";
import { useCustomEdge } from "./edges/useCustomEdge";
import CustomEdge from "./edges/CustomEdge";
import { RightSidebar } from "../RIghtSidebar";
import { CustomNode } from "../nodes/CustomNode";
import { useCustomEdge } from "../edges/useCustomEdge";
export const Flow = () => {
// All these 3 are working perfectly
const nodes = useNodeStore(useShallow((state) => state.nodes));
const onNodesChange = useNodeStore(
useShallow((state) => state.onNodesChange),
@@ -18,9 +16,11 @@ export const Flow = () => {
const nodeTypes = useMemo(() => ({ custom: CustomNode }), []);
const { edges, onConnect, onEdgesChange } = useCustomEdge();
// We use this hook to load the graph and convert them into custom nodes and edges.
useFlow();
return (
<div className="flex h-full w-full dark:bg-slate-900">
{/* Builder area - flexible width */}
<div className="relative flex-1">
<ReactFlow
nodes={nodes}
@@ -30,15 +30,14 @@ export const Flow = () => {
onConnect={onConnect}
onEdgesChange={onEdgesChange}
edgeTypes={{ custom: CustomEdge }}
maxZoom={2}
minZoom={0.1}
>
<Background />
<Controls />
<NewControlPanel />
</ReactFlow>
</div>
<div className="w-[30%]">
<RightSidebar />
</div>
</div>
);
};

69
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/Flow/useFlow.ts Normal file
View File

@@ -0,0 +1,69 @@
import { useGetV2GetSpecificBlocks } from "@/app/api/__generated__/endpoints/default/default";
import { useGetV1GetSpecificGraph } from "@/app/api/__generated__/endpoints/graphs/graphs";
import { BlockInfo } from "@/app/api/__generated__/models/blockInfo";
import { GraphModel } from "@/app/api/__generated__/models/graphModel";
import { parseAsInteger, parseAsString, useQueryStates } from "nuqs";
import { useNodeStore } from "../../../stores/nodeStore";
import { useShallow } from "zustand/react/shallow";
import { useEffect, useMemo } from "react";
import { convertNodesPlusBlockInfoIntoCustomNodes } from "../../helper";
import { useEdgeStore } from "../../../stores/edgeStore";
export const useFlow = () => {
const addNodes = useNodeStore(useShallow((state) => state.addNodes));
const addLinks = useEdgeStore(useShallow((state) => state.addLinks));
const [{ flowID, flowVersion }] = useQueryStates({
flowID: parseAsString,
flowVersion: parseAsInteger,
});
const { data: graph } = useGetV1GetSpecificGraph(
flowID ?? "",
flowVersion !== null ? { version: flowVersion } : {},
{
query: {
select: (res) => res.data as GraphModel,
enabled: !!flowID,
},
},
);
const nodes = graph?.nodes;
const blockIds = nodes?.map((node) => node.block_id);
const { data: blocks } = useGetV2GetSpecificBlocks(
{ block_ids: blockIds ?? [] },
{
query: {
select: (res) => res.data as BlockInfo[],
enabled: !!flowID && !!blockIds,
},
},
);
const customNodes = useMemo(() => {
if (!nodes || !blocks) return [];
return nodes.map((node) => {
const customNode = convertNodesPlusBlockInfoIntoCustomNodes(
node,
blocks?.find((block) => block.id === node.block_id) ??
({} as BlockInfo),
);
return customNode;
});
}, [nodes, blocks]);
useEffect(() => {
if (customNodes.length > 0) {
addNodes(customNodes);
}
if (graph?.links) {
addLinks(graph.links);
}
}, [customNodes, addNodes, graph?.links]);
return {};
};

11
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/components/ArrayEditor/ArrayEditor.tsx
View File

@@ -22,7 +22,7 @@ export const ArrayEditor = ({
onAddClick,
disabled,
readonly,
id,
id: fieldId,
}: ArrayEditorProps) => {
const { isInputConnected } = useEdgeStore();
@@ -31,12 +31,12 @@ export const ArrayEditor = ({
<div className="flex items-center gap-2">
<div className="flex-1">
{items?.map((element) => {
const fieldKey = generateHandleId(
id,
const arrayFieldHandleId = generateHandleId(
fieldId,
[element.index.toString()],
HandleIdType.ARRAY,
);
const isConnected = isInputConnected(nodeId, fieldKey);
const isConnected = isInputConnected(nodeId, fieldId);
return (
<div
key={element.key}
@@ -45,7 +45,7 @@ export const ArrayEditor = ({
<ArrayEditorContext.Provider
value={{
isArrayItem: true,
fieldKey,
arrayFieldHandleId: arrayFieldHandleId,
isConnected,
}}
>
@@ -75,6 +75,7 @@ export const ArrayEditor = ({
{canAdd && !readonly && !disabled && (
<Button
type="button"
variant="secondary"
size="small"
onClick={onAddClick}
className="w-full"

4
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/components/ArrayEditor/ArrayEditorContext.tsx
View File

@@ -2,10 +2,10 @@ import { createContext } from "react";
export const ArrayEditorContext = createContext<{
isArrayItem: boolean;
fieldKey: string;
arrayFieldHandleId: string;
isConnected: boolean;
}>({
isArrayItem: false,
fieldKey: "",
arrayFieldHandleId: "",
isConnected: false,
});

49
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/components/ObjectEditor/ObjectEditor.tsx
View File

@@ -7,7 +7,11 @@ import { Button } from "@/components/atoms/Button/Button";
import { Input } from "@/components/atoms/Input/Input";
import NodeHandle from "../../handlers/NodeHandle";
import { useEdgeStore } from "@/app/(platform)/build/stores/edgeStore";
import { generateHandleId, HandleIdType } from "../../handlers/helpers";
import {
generateHandleId,
HandleIdType,
parseKeyValueHandleId,
} from "../../handlers/helpers";
export interface ObjectEditorProps {
id: string;
@@ -23,7 +27,7 @@ export interface ObjectEditorProps {
export const ObjectEditor = React.forwardRef<HTMLDivElement, ObjectEditorProps>(
(
{
id,
id: parentFieldId,
value = {},
onChange,
placeholder = "Enter value",
@@ -34,6 +38,9 @@ export const ObjectEditor = React.forwardRef<HTMLDivElement, ObjectEditorProps>(
},
ref,
) => {
const getAllHandleIdsOfANode = useEdgeStore(
(state) => state.getAllHandleIdsOfANode,
);
const setProperty = (key: string, propertyValue: any) => {
if (!onChange) return;
@@ -72,44 +79,47 @@ export const ObjectEditor = React.forwardRef<HTMLDivElement, ObjectEditorProps>(
const { isInputConnected } = useEdgeStore();
const allHandleIdsOfANode = getAllHandleIdsOfANode(nodeId);
const allKeyValueHandleIdsOfANode = allHandleIdsOfANode.filter((handleId) =>
handleId.includes("_#_"),
);
allKeyValueHandleIdsOfANode.forEach((handleId) => {
const key = parseKeyValueHandleId(handleId, HandleIdType.KEY_VALUE);
if (!value[key]) {
value[key] = null;
}
});
return (
<div
ref={ref}
className={`flex flex-col gap-2 ${className || ""}`}
id={id}
id={parentFieldId}
>
{Object.entries(value).map(([key, propertyValue], idx) => {
const dynamicHandleId = generateHandleId(
fieldKey,
const isDynamicPropertyConnected = isInputConnected(nodeId, fieldKey);
const handleId = generateHandleId(
parentFieldId,
[key],
HandleIdType.KEY_VALUE,
);
const isDynamicPropertyConnected = isInputConnected(
nodeId,
dynamicHandleId,
);
console.log("dynamicHandleId", dynamicHandleId);
console.log("key", key);
console.log("fieldKey", fieldKey);
return (
<div key={idx} className="flex flex-col gap-2">
<div className="-ml-2 flex items-center gap-1">
<NodeHandle
id={dynamicHandleId}
isConnected={isDynamicPropertyConnected}
handleId={handleId}
side="left"
/>
<Text variant="small" className="text-gray-600">
<Text variant="small" className="!text-gray-500">
#{key.trim() === "" ? "" : key}
</Text>
<Text variant="small" className="!text-green-500">
(string)
</Text>
</div>
{!isDynamicPropertyConnected && (
{!isDynamicPropertyConnected && propertyValue !== null && (
<div className="flex items-center gap-2">
<Input
hideLabel={true}
@@ -150,12 +160,13 @@ export const ObjectEditor = React.forwardRef<HTMLDivElement, ObjectEditorProps>(
<Button
type="button"
variant="secondary"
size="small"
onClick={addProperty}
className="w-full"
className="w-fit"
disabled={hasEmptyKeys || disabled}
>
<Plus className="mr-2 h-4 w-4" />
<Plus className="h-4 w-4" />
Add Property
</Button>
</div>

2
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/edges/CustomEdge.tsx
View File

@@ -42,7 +42,7 @@ const CustomEdge = ({
<EdgeLabelRenderer>
<Button
onClick={() => removeConnection(id)}
className={`absolute z-10 min-w-0 p-1`}
className={`absolute h-fit min-w-0 p-1`}
variant="secondary"
style={{
transform: `translate(-50%, -50%) translate(${labelX}px, ${labelY}px)`,

7
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/handlers/NodeHandle.tsx
View File

@@ -2,20 +2,19 @@ import { CircleIcon } from "@phosphor-icons/react";
import { Handle, Position } from "@xyflow/react";
const NodeHandle = ({
id,
handleId,
isConnected,
side,
}: {
id: string;
handleId: string;
isConnected: boolean;
side: "left" | "right";
}) => {
console.log("id", id);
return (
<Handle
type={side === "left" ? "target" : "source"}
position={side === "left" ? Position.Left : Position.Right}
id={id}
id={handleId}
className={side === "left" ? "-ml-4 mr-2" : "-mr-2 ml-2"}
>
<div className="pointer-events-none">

71
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/handlers/helpers.ts
View File

@@ -39,17 +39,17 @@ const sanitizeForHandleId = (str: string): string => {
};
export const generateHandleId = (
mainKey: string,
fieldKey: string,
nestedValues: string[] = [],
type: HandleIdType = HandleIdType.SIMPLE,
): string => {
if (!mainKey) return "";
if (!fieldKey) return "";
mainKey = fromRjsfId(mainKey);
mainKey = sanitizeForHandleId(mainKey);
fieldKey = fromRjsfId(fieldKey);
fieldKey = sanitizeForHandleId(fieldKey);
if (type === HandleIdType.SIMPLE || nestedValues.length === 0) {
return mainKey;
return fieldKey;
}
const sanitizedNestedValues = nestedValues.map((value) =>
@@ -58,60 +58,31 @@ export const generateHandleId = (
switch (type) {
case HandleIdType.NESTED:
return [mainKey, ...sanitizedNestedValues].join(".");
return [fieldKey, ...sanitizedNestedValues].join(".");
case HandleIdType.ARRAY:
return [mainKey, ...sanitizedNestedValues].join("_$_");
return [fieldKey, ...sanitizedNestedValues].join("_$_");
case HandleIdType.KEY_VALUE:
return [mainKey, ...sanitizedNestedValues].join("_#_");
return [fieldKey, ...sanitizedNestedValues].join("_#_");
default:
return mainKey;
return fieldKey;
}
};
export const parseHandleId = (
export const parseKeyValueHandleId = (
handleId: string,
): {
mainKey: string;
nestedValues: string[];
type: HandleIdType;
} => {
if (!handleId) {
return { mainKey: "", nestedValues: [], type: HandleIdType.SIMPLE };
type: HandleIdType,
): string => {
if (type === HandleIdType.KEY_VALUE) {
return handleId.split("_#_")[1];
} else if (type === HandleIdType.ARRAY) {
return handleId.split("_$_")[1];
} else if (type === HandleIdType.NESTED) {
return handleId.split(".")[1];
} else if (type === HandleIdType.SIMPLE) {
return handleId.split("_")[1];
}
if (handleId.includes("_#_")) {
const parts = handleId.split("_#_");
return {
mainKey: parts[0],
nestedValues: parts.slice(1),
type: HandleIdType.KEY_VALUE,
};
}
if (handleId.includes("_$_")) {
const parts = handleId.split("_$_");
return {
mainKey: parts[0],
nestedValues: parts.slice(1),
type: HandleIdType.ARRAY,
};
}
if (handleId.includes(".")) {
const parts = handleId.split(".");
return {
mainKey: parts[0],
nestedValues: parts.slice(1),
type: HandleIdType.NESTED,
};
}
return {
mainKey: handleId,
nestedValues: [],
type: HandleIdType.SIMPLE,
};
return "";
};

28
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/CustomNode.tsx
View File

@@ -8,6 +8,9 @@ import { Switch } from "@/components/atoms/Switch/Switch";
import { preprocessInputSchema } from "../processors/input-schema-pre-processor";
import { OutputHandler } from "./OutputHandler";
import { useNodeStore } from "../../../stores/nodeStore";
import { cn } from "@/lib/utils";
import { BlockUIType } from "../../types";
import { StickyNoteBlock } from "./StickyNoteBlock";
export type CustomNodeData = {
hardcodedValues: {
@@ -17,26 +20,36 @@ export type CustomNodeData = {
description: string;
inputSchema: RJSFSchema;
outputSchema: RJSFSchema;
uiType: BlockUIType;
};
export type CustomNode = XYNode<CustomNodeData, "custom">;
export const CustomNode: React.FC<NodeProps<CustomNode>> = React.memo(
({ data, id }) => {
({ data, id: nodeId, selected }) => {
const showAdvanced = useNodeStore(
(state) => state.nodeAdvancedStates[id] || false,
(state) => state.nodeAdvancedStates[nodeId] || false,
);
const setShowAdvanced = useNodeStore((state) => state.setShowAdvanced);
if (data.uiType === BlockUIType.NOTE) {
return <StickyNoteBlock selected={selected} data={data} id={nodeId} />;
}
return (
<div className="rounded-xl border border-slate-200/60 bg-gradient-to-br from-white to-slate-50/30 shadow-lg shadow-slate-900/5 backdrop-blur-sm">
<div
className={cn(
"z-12 rounded-xl bg-gradient-to-br from-white to-slate-50/30 shadow-lg shadow-slate-900/5 ring-1 ring-slate-200/60 backdrop-blur-sm",
selected && "shadow-2xl ring-2 ring-slate-200",
)}
>
{/* Header */}
<div className="flex h-14 items-center justify-center rounded-xl border-b border-slate-200/50 bg-gradient-to-r from-slate-50/80 to-white/90">
<Text
variant="large-semibold"
className="tracking-tight text-slate-800"
>
{data.title} #{id}
{data.title}
</Text>
</div>
@@ -44,7 +57,8 @@ export const CustomNode: React.FC<NodeProps<CustomNode>> = React.memo(
<div className="bg-white/40 pb-6 pr-6">
<FormCreator
jsonSchema={preprocessInputSchema(data.inputSchema)}
nodeId={id}
nodeId={nodeId}
uiType={data.uiType}
/>
</div>
@@ -54,13 +68,13 @@ export const CustomNode: React.FC<NodeProps<CustomNode>> = React.memo(
Advanced
</Text>
<Switch
onCheckedChange={(checked) => setShowAdvanced(id, checked)}
onCheckedChange={(checked) => setShowAdvanced(nodeId, checked)}
checked={showAdvanced}
/>
</div>
{/* Output Handles */}
<OutputHandler outputSchema={data.outputSchema} nodeId={id} />
<OutputHandler outputSchema={data.outputSchema} nodeId={nodeId} />
</div>
);
},

19
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/FormCreator.tsx
View File

@@ -7,14 +7,28 @@ import { fields } from "./fields";
import { templates } from "./templates";
import { uiSchema } from "./uiSchema";
import { useNodeStore } from "../../../stores/nodeStore";
import { BlockUIType } from "../../types";
export const FormCreator = React.memo(
({ jsonSchema, nodeId }: { jsonSchema: RJSFSchema; nodeId: string }) => {
({
jsonSchema,
nodeId,
uiType,
}: {
jsonSchema: RJSFSchema;
nodeId: string;
uiType: BlockUIType;
}) => {
const updateNodeData = useNodeStore((state) => state.updateNodeData);
const getHardCodedValues = useNodeStore(
(state) => state.getHardCodedValues,
);
const handleChange = ({ formData }: any) => {
updateNodeData(nodeId, { hardcodedValues: formData });
};
const initialValues = getHardCodedValues(nodeId);
return (
<Form
schema={jsonSchema}
@@ -22,9 +36,10 @@ export const FormCreator = React.memo(
fields={fields}
templates={templates}
widgets={widgets}
formContext={{ nodeId: nodeId }}
formContext={{ nodeId: nodeId, uiType: uiType }}
onChange={handleChange}
uiSchema={uiSchema}
formData={initialValues}
/>
);
},

53
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/OutputHandler.tsx
View File

@@ -13,6 +13,7 @@ import {
} from "@/components/atoms/Tooltip/BaseTooltip";
import { useEdgeStore } from "@/app/(platform)/build/stores/edgeStore";
import { getTypeDisplayInfo } from "./helpers";
import { generateHandleId } from "../handlers/helpers";
export const OutputHandler = ({
outputSchema,
@@ -29,7 +30,7 @@ export const OutputHandler = ({
<div className="flex flex-col items-end justify-between gap-2 rounded-b-xl border-t border-slate-200/50 bg-white py-3.5">
<Button
variant="ghost"
className="mr-4 p-0"
className="mr-4 h-fit min-w-0 p-0 hover:border-transparent hover:bg-transparent"
onClick={() => setIsOutputVisible(!isOutputVisible)}
>
<Text
@@ -54,32 +55,34 @@ export const OutputHandler = ({
return shouldShow ? (
<div key={key} className="relative flex items-center gap-2">
<Text
variant="body"
className="flex items-center gap-2 font-medium text-slate-700"
>
{property?.description && (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<span
style={{ marginLeft: 6, cursor: "pointer" }}
aria-label="info"
tabIndex={0}
>
<InfoIcon />
</span>
</TooltipTrigger>
<TooltipContent>{property?.description}</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
{property?.description && (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<span
style={{ marginLeft: 6, cursor: "pointer" }}
aria-label="info"
tabIndex={0}
>
<InfoIcon />
</span>
</TooltipTrigger>
<TooltipContent>{property?.description}</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
<Text variant="body" className="text-slate-700">
{property?.title || key}{" "}
<Text variant="small" as="span" className={colorClass}>
({displayType})
</Text>
</Text>
<NodeHandle id={key} isConnected={isConnected} side="right" />
<Text variant="small" as="span" className={colorClass}>
({displayType})
</Text>
<NodeHandle
handleId={generateHandleId(key)}
isConnected={isConnected}
side="right"
/>
</div>
) : null;
})}

54
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/StickyNoteBlock.tsx Normal file
View File

@@ -0,0 +1,54 @@
import { useMemo } from "react";
import { FormCreator } from "./FormCreator";
import { preprocessInputSchema } from "../processors/input-schema-pre-processor";
import { CustomNodeData } from "./CustomNode";
import { Text } from "@/components/atoms/Text/Text";
import { cn } from "@/lib/utils";
type StickyNoteBlockType = {
selected: boolean;
data: CustomNodeData;
id: string;
};
export const StickyNoteBlock = ({ data, id }: StickyNoteBlockType) => {
const { angle, color } = useMemo(() => {
const hash = id.split("").reduce((acc, char) => {
return char.charCodeAt(0) + ((acc << 5) - acc);
}, 0);
const colors = [
"bg-orange-200",
"bg-red-200",
"bg-yellow-200",
"bg-green-200",
"bg-blue-200",
"bg-purple-200",
"bg-pink-200",
];
return {
angle: (hash % 7) - 3,
color: colors[Math.abs(hash) % colors.length],
};
}, [id]);
return (
<div
className={cn(
"relative h-76 w-76 p-4 text-black shadow-[rgba(0,0,0,0.3)_-2px_5px_5px_0px]",
color,
)}
style={{ transform: `rotate(${angle}deg)` }}
>
<Text variant="h3" className="tracking-tight text-slate-800">
Notes #{id}
</Text>
<FormCreator
jsonSchema={preprocessInputSchema(data.inputSchema)}
nodeId={id}
uiType={data.uiType}
/>
</div>
);
};

12
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/fields/AnyOfField/AnyOfField.tsx
View File

@@ -43,12 +43,12 @@ export const AnyOfField = ({
onBlur,
onFocus,
}: FieldProps) => {
const fieldKey = generateHandleId(idSchema.$id ?? "");
const handleId = generateHandleId(idSchema.$id ?? "");
const updatedFormContexrt = { ...formContext, fromAnyOf: true };
const { nodeId } = updatedFormContexrt;
const { isInputConnected } = useEdgeStore();
const isConnected = isInputConnected(nodeId, fieldKey);
const isConnected = isInputConnected(nodeId, handleId);
const {
isNullableType,
nonNull,
@@ -127,7 +127,11 @@ export const AnyOfField = ({
<div className="flex flex-col">
<div className="flex items-center justify-between gap-2">
<div className="-ml-2 flex items-center gap-1">
<NodeHandle id={fieldKey} isConnected={isConnected} side="left" />
<NodeHandle
handleId={handleId}
isConnected={isConnected}
side="left"
/>
<Text variant="body">
{name.charAt(0).toUpperCase() + name.slice(1)}
</Text>
@@ -151,7 +155,7 @@ export const AnyOfField = ({
return (
<div className="flex flex-col">
<div className="-mb-3 -ml-2 flex items-center gap-1">
<NodeHandle id={fieldKey} isConnected={isConnected} side="left" />
<NodeHandle handleId={handleId} isConnected={isConnected} side="left" />
<Text variant="body">
{name.charAt(0).toUpperCase() + name.slice(1)}
</Text>

34
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/fields/CredentialField.tsx
View File

@@ -1,34 +0,0 @@
import React from "react";
import { FieldProps } from "@rjsf/utils";
import { Input } from "@/components/atoms/Input/Input";
// We need to add all the logic for the credential fields here
export const CredentialsField = (props: FieldProps) => {
const { formData = {}, onChange, required: _required, schema } = props;
const _credentialProvider = schema.credentials_provider;
const _credentialType = schema.credentials_types;
const _description = schema.description;
const _title = schema.title;
// Helper to update one property
const setField = (key: string, value: any) =>
onChange({ ...formData, [key]: value });
return (
<div className="flex flex-col gap-2">
<Input
hideLabel={true}
label={""}
id="credentials-id"
type="text"
value={formData.id || ""}
onChange={(e) => setField("id", e.target.value)}
placeholder="Enter your API Key"
required
size="small"
wrapperClassName="mb-0"
/>
</div>
);
};

77
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/fields/CredentialField/CredentialField.tsx Normal file
View File

@@ -0,0 +1,77 @@
import React from "react";
import { FieldProps } from "@rjsf/utils";
import { useCredentialField } from "./useCredentialField";
import { SelectCredential } from "./SelectCredential";
import { Skeleton } from "@/components/__legacy__/ui/skeleton";
import { BlockIOCredentialsSubSchema } from "@/lib/autogpt-server-api";
import { APIKeyCredentialsModal } from "./models/APIKeyCredentialModal/APIKeyCredentialModal";
import { OAuthCredentialModal } from "./models/OAuthCredentialModal/OAuthCredentialModal";
import { PasswordCredentialsModal } from "./models/PasswordCredentialModal/PasswordCredentialModal";
export const CredentialsField = (props: FieldProps) => {
const {
formData = {},
onChange,
required: _required,
schema,
formContext,
} = props;
const {
credentials,
isCredentialListLoading,
supportsApiKey,
supportsOAuth2,
supportsUserPassword,
credentialsExists,
credentialProvider,
setCredential,
} = useCredentialField({
credentialSchema: schema as BlockIOCredentialsSubSchema,
formData,
nodeId: formContext.nodeId,
onChange,
});
if (isCredentialListLoading) {
return (
<div className="flex flex-col gap-2">
<Skeleton className="h-8 w-full rounded-xlarge" />
<Skeleton className="h-8 w-[30%] rounded-xlarge" />
</div>
);
}
if (!credentialProvider) {
return null;
}
return (
<div className="flex flex-col gap-2">
{credentialsExists && (
<SelectCredential
credentials={credentials}
value={formData.id || ""}
onChange={setCredential}
disabled={false}
label="Credential"
placeholder="Select credential"
/>
)}
<div>
{supportsApiKey && (
<APIKeyCredentialsModal
schema={schema as BlockIOCredentialsSubSchema}
provider={credentialProvider}
/>
)}
{supportsOAuth2 && (
<OAuthCredentialModal provider={credentialProvider} />
)}
{supportsUserPassword && (
<PasswordCredentialsModal provider={credentialProvider} />
)}
</div>
</div>
);
};

91
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/fields/CredentialField/SelectCredential.tsx Normal file
View File

@@ -0,0 +1,91 @@
import React from "react";
import { Select } from "@/components/atoms/Select/Select";
import { CredentialsMetaResponse } from "@/app/api/__generated__/models/credentialsMetaResponse";
import {
ArrowSquareOutIcon,
KeyholeIcon,
KeyIcon,
} from "@phosphor-icons/react";
import { Button } from "@/components/atoms/Button/Button";
import Link from "next/link";
import { providerIcons } from "./helpers";
type SelectCredentialProps = {
credentials: CredentialsMetaResponse[];
value?: string;
defaultValue?: string;
onChange: (credentialId: string) => void;
disabled?: boolean;
label?: string;
placeholder?: string;
};
export const SelectCredential: React.FC<SelectCredentialProps> = ({
credentials,
value,
onChange,
disabled = false,
label = "Credential",
placeholder = "Select credential",
}) => {
const options = credentials.map((cred) => {
const details: string[] = [];
if (cred.title && cred.title !== cred.provider) {
details.push(cred.title);
}
if (cred.username) {
details.push(cred.username);
}
if (cred.host) {
details.push(cred.host);
}
const label =
details.length > 0
? `${cred.provider} (${details.join(" - ")})`
: cred.provider;
const Icon = providerIcons[cred.provider];
const icon =
cred.type === "oauth2" ? (
Icon ? (
<Icon />
) : (
<KeyholeIcon />
)
) : (
<KeyIcon className="h-4 w-4" />
);
return {
value: cred.id,
label,
icon,
};
});
return (
<div className="flex w-full items-center gap-2">
<Select
label={label}
id="select-credential"
wrapperClassName="!mb-0 flex-1 !max-w-[90%]"
value={value}
onValueChange={onChange}
options={options}
disabled={disabled}
placeholder={placeholder}
size="small"
hideLabel
/>
<Link href={`/profile/integrations`}>
<Button
variant="outline"
size="icon"
className="h-8 w-8 border-zinc-300 p-0"
>
<ArrowSquareOutIcon className="h-4 w-4 text-zinc-600" />
</Button>
</Link>
</div>
);
};

134
autogpt_platform/frontend/src/app/(platform)/build/components/FlowEditor/nodes/fields/CredentialField/helpers.ts Normal file
View File

@@ -0,0 +1,134 @@
import { CredentialsMetaResponse } from "@/app/api/__generated__/models/credentialsMetaResponse";
import { BlockIOCredentialsSubSchema } from "@/lib/autogpt-server-api";
import {
GoogleLogoIcon,
KeyholeIcon,
NotionLogoIcon,
DiscordLogoIcon,
MediumLogoIcon,
GithubLogoIcon,
TwitterLogoIcon,
Icon,
} from "@phosphor-icons/react";
export const filterCredentialsByProvider = (
credentials: CredentialsMetaResponse[] | undefined,
provider: string,
) => {
const filtered =
credentials?.filter((credential) => provider === credential.provider) ?? [];
return {
credentials: filtered,
exists: filtered.length > 0,
};
};
export function toDisplayName(provider: string): string {
// Special cases that need manual handling
const specialCases: Record<string, string> = {
aiml_api: "AI/ML",
d_id: "D-ID",
e2b: "E2B",
llama_api: "Llama API",
open_router: "Open Router",
smtp: "SMTP",
revid: "Rev.ID",
};
if (specialCases[provider]) {
return specialCases[provider];
}
// General case: convert snake_case to Title Case
return provider
.split(/[_-]/)
.map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
.join(" ");
}
export function isCredentialFieldSchema(schema: any): boolean {
return (
typeof schema === "object" &&
schema !== null &&
"credentials_provider" in schema
);
}
export const providerIcons: Partial<Record<string, Icon>> = {
aiml_api: KeyholeIcon,
anthropic: KeyholeIcon,
apollo: KeyholeIcon,
e2b: KeyholeIcon,
github: GithubLogoIcon,
google: GoogleLogoIcon,
groq: KeyholeIcon,
http: KeyholeIcon,
notion: NotionLogoIcon,
nvidia: KeyholeIcon,
discord: DiscordLogoIcon,
d_id: KeyholeIcon,
google_maps: GoogleLogoIcon,
jina: KeyholeIcon,
ideogram: KeyholeIcon,
linear: KeyholeIcon,
medium: MediumLogoIcon,
mem0: KeyholeIcon,
ollama: KeyholeIcon,
openai: KeyholeIcon,
openweathermap: KeyholeIcon,
open_router: KeyholeIcon,
llama_api: KeyholeIcon,
pinecone: KeyholeIcon,
enrichlayer: KeyholeIcon,
slant3d: KeyholeIcon,
screenshotone: KeyholeIcon,
smtp: KeyholeIcon,
replicate: KeyholeIcon,
reddit: KeyholeIcon,
fal: KeyholeIcon,
revid: KeyholeIcon,
twitter: TwitterLogoIcon,
unreal_speech: KeyholeIcon,
exa: KeyholeIcon,
hubspot: KeyholeIcon,
smartlead: KeyholeIcon,
todoist: KeyholeIcon,
zerobounce: KeyholeIcon,
};
export const getCredentialProviderFromSchema = (
formData: Record<string, any>,
schema: BlockIOCredentialsSubSchema,
) => {
const discriminator = schema.discriminator;
const discriminatorMapping = schema.discriminator_mapping;
const discriminatorValues = schema.discriminator_values;
const providers = schema.credentials_provider;
const discriminatorValue = [
discriminator ? formData[discriminator] : null,
...(discriminatorValues || []),
].find(Boolean);
const discriminatedProvider = discriminatorMapping
? discriminatorMapping[discriminatorValue]
: null;
if (providers.length > 1) {
if (!discriminator) {
throw new Error(
"Multi-provider credential input requires discriminator!",
);
}
if (!discriminatedProvider) {
console.warn(
`Missing discriminator value from '${discriminator}': ` +
"hiding credentials input until it is set.",
);
return null;
}
return discriminatedProvider;
} else {
return providers[0];
}
};

Some files were not shown because too many files have changed in this diff Show More