mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-02-16 01:36:36 -05:00
Compare commits
2 Commits
otto/secrt
...
dev
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
647c8ed8d4 | ||
|
|
27d94e395c |
@@ -11,45 +11,15 @@ import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any, cast
|
||||
|
||||
from backend.api.features.chat.sdk.tool_adapter import MCP_TOOL_PREFIX
|
||||
from backend.api.features.chat.sdk.tool_adapter import (
|
||||
BLOCKED_TOOLS,
|
||||
DANGEROUS_PATTERNS,
|
||||
MCP_TOOL_PREFIX,
|
||||
WORKSPACE_SCOPED_TOOLS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Tools that are blocked entirely (CLI/system access).
|
||||
# "Bash" (capital) is the SDK built-in — it's NOT in allowed_tools but blocked
|
||||
# here as defence-in-depth. The agent uses mcp__copilot__bash_exec instead,
|
||||
# which has kernel-level network isolation (unshare --net).
|
||||
BLOCKED_TOOLS = {
|
||||
"Bash",
|
||||
"bash",
|
||||
"shell",
|
||||
"exec",
|
||||
"terminal",
|
||||
"command",
|
||||
}
|
||||
|
||||
# Tools allowed only when their path argument stays within the SDK workspace.
|
||||
# The SDK uses these to handle oversized tool results (writes to tool-results/
|
||||
# files, then reads them back) and for workspace file operations.
|
||||
WORKSPACE_SCOPED_TOOLS = {"Read", "Write", "Edit", "Glob", "Grep"}
|
||||
|
||||
# Dangerous patterns in tool inputs
|
||||
DANGEROUS_PATTERNS = [
|
||||
r"sudo",
|
||||
r"rm\s+-rf",
|
||||
r"dd\s+if=",
|
||||
r"/etc/passwd",
|
||||
r"/etc/shadow",
|
||||
r"chmod\s+777",
|
||||
r"curl\s+.*\|.*sh",
|
||||
r"wget\s+.*\|.*sh",
|
||||
r"eval\s*\(",
|
||||
r"exec\s*\(",
|
||||
r"__import__",
|
||||
r"os\.system",
|
||||
r"subprocess",
|
||||
]
|
||||
|
||||
|
||||
def _deny(reason: str) -> dict[str, Any]:
|
||||
"""Return a hook denial response."""
|
||||
|
||||
@@ -41,6 +41,7 @@ from .response_adapter import SDKResponseAdapter
|
||||
from .security_hooks import create_security_hooks
|
||||
from .tool_adapter import (
|
||||
COPILOT_TOOL_NAMES,
|
||||
SDK_DISALLOWED_TOOLS,
|
||||
LongRunningCallback,
|
||||
create_copilot_mcp_server,
|
||||
set_execution_context,
|
||||
@@ -543,7 +544,7 @@ async def stream_chat_completion_sdk(
|
||||
"system_prompt": system_prompt,
|
||||
"mcp_servers": {"copilot": mcp_server},
|
||||
"allowed_tools": COPILOT_TOOL_NAMES,
|
||||
"disallowed_tools": ["Bash"],
|
||||
"disallowed_tools": SDK_DISALLOWED_TOOLS,
|
||||
"hooks": security_hooks,
|
||||
"cwd": sdk_cwd,
|
||||
"max_buffer_size": config.claude_agent_max_buffer_size,
|
||||
|
||||
@@ -310,7 +310,48 @@ def create_copilot_mcp_server():
|
||||
# Bash is NOT included — use the sandboxed MCP bash_exec tool instead,
|
||||
# which provides kernel-level network isolation via unshare --net.
|
||||
# Task allows spawning sub-agents (rate-limited by security hooks).
|
||||
_SDK_BUILTIN_TOOLS = ["Read", "Write", "Edit", "Glob", "Grep", "Task"]
|
||||
# WebSearch uses Brave Search via Anthropic's API — safe, no SSRF risk.
|
||||
_SDK_BUILTIN_TOOLS = ["Read", "Write", "Edit", "Glob", "Grep", "Task", "WebSearch"]
|
||||
|
||||
# SDK built-in tools that must be explicitly blocked.
|
||||
# Bash: dangerous — agent uses mcp__copilot__bash_exec with kernel-level
|
||||
# network isolation (unshare --net) instead.
|
||||
# WebFetch: SSRF risk — can reach internal network (localhost, 10.x, etc.).
|
||||
# Agent uses the SSRF-protected mcp__copilot__web_fetch tool instead.
|
||||
SDK_DISALLOWED_TOOLS = ["Bash", "WebFetch"]
|
||||
|
||||
# Tools that are blocked entirely in security hooks (defence-in-depth).
|
||||
# Includes SDK_DISALLOWED_TOOLS plus common aliases/synonyms.
|
||||
BLOCKED_TOOLS = {
|
||||
*SDK_DISALLOWED_TOOLS,
|
||||
"bash",
|
||||
"shell",
|
||||
"exec",
|
||||
"terminal",
|
||||
"command",
|
||||
}
|
||||
|
||||
# Tools allowed only when their path argument stays within the SDK workspace.
|
||||
# The SDK uses these to handle oversized tool results (writes to tool-results/
|
||||
# files, then reads them back) and for workspace file operations.
|
||||
WORKSPACE_SCOPED_TOOLS = {"Read", "Write", "Edit", "Glob", "Grep"}
|
||||
|
||||
# Dangerous patterns in tool inputs
|
||||
DANGEROUS_PATTERNS = [
|
||||
r"sudo",
|
||||
r"rm\s+-rf",
|
||||
r"dd\s+if=",
|
||||
r"/etc/passwd",
|
||||
r"/etc/shadow",
|
||||
r"chmod\s+777",
|
||||
r"curl\s+.*\|.*sh",
|
||||
r"wget\s+.*\|.*sh",
|
||||
r"eval\s*\(",
|
||||
r"exec\s*\(",
|
||||
r"__import__",
|
||||
r"os\.system",
|
||||
r"subprocess",
|
||||
]
|
||||
|
||||
# List of tool names for allowed_tools configuration
|
||||
# Include MCP tools, the MCP Read tool for oversized results,
|
||||
|
||||
@@ -682,17 +682,219 @@ class ListIsEmptyBlock(Block):
|
||||
yield "is_empty", len(input_data.list) == 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# List Concatenation Helpers
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _validate_list_input(item: Any, index: int) -> str | None:
|
||||
"""Validate that an item is a list. Returns error message or None."""
|
||||
if item is None:
|
||||
return None # None is acceptable, will be skipped
|
||||
if not isinstance(item, list):
|
||||
return (
|
||||
f"Invalid input at index {index}: expected a list, "
|
||||
f"got {type(item).__name__}. "
|
||||
f"All items in 'lists' must be lists (e.g., [[1, 2], [3, 4]])."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _validate_all_lists(lists: List[Any]) -> str | None:
|
||||
"""Validate that all items in a sequence are lists. Returns first error or None."""
|
||||
for idx, item in enumerate(lists):
|
||||
error = _validate_list_input(item, idx)
|
||||
if error is not None and item is not None:
|
||||
return error
|
||||
return None
|
||||
|
||||
|
||||
def _concatenate_lists_simple(lists: List[List[Any]]) -> List[Any]:
|
||||
"""Concatenate a sequence of lists into a single list, skipping None values."""
|
||||
result: List[Any] = []
|
||||
for lst in lists:
|
||||
if lst is None:
|
||||
continue
|
||||
result.extend(lst)
|
||||
return result
|
||||
|
||||
|
||||
def _flatten_nested_list(nested: List[Any], max_depth: int = -1) -> List[Any]:
|
||||
"""
|
||||
Recursively flatten a nested list structure.
|
||||
|
||||
Args:
|
||||
nested: The list to flatten.
|
||||
max_depth: Maximum recursion depth. -1 means unlimited.
|
||||
|
||||
Returns:
|
||||
A flat list with all nested elements extracted.
|
||||
"""
|
||||
result: List[Any] = []
|
||||
_flatten_recursive(nested, result, current_depth=0, max_depth=max_depth)
|
||||
return result
|
||||
|
||||
|
||||
_MAX_FLATTEN_DEPTH = 1000
|
||||
|
||||
|
||||
def _flatten_recursive(
|
||||
items: List[Any],
|
||||
result: List[Any],
|
||||
current_depth: int,
|
||||
max_depth: int,
|
||||
) -> None:
|
||||
"""Internal recursive helper for flattening nested lists."""
|
||||
if current_depth > _MAX_FLATTEN_DEPTH:
|
||||
raise RecursionError(
|
||||
f"Flattening exceeded maximum depth of {_MAX_FLATTEN_DEPTH} levels. "
|
||||
"Input may be too deeply nested."
|
||||
)
|
||||
for item in items:
|
||||
if isinstance(item, list) and (max_depth == -1 or current_depth < max_depth):
|
||||
_flatten_recursive(item, result, current_depth + 1, max_depth)
|
||||
else:
|
||||
result.append(item)
|
||||
|
||||
|
||||
def _deduplicate_list(items: List[Any]) -> List[Any]:
|
||||
"""
|
||||
Remove duplicate elements from a list, preserving order of first occurrences.
|
||||
|
||||
Args:
|
||||
items: The list to deduplicate.
|
||||
|
||||
Returns:
|
||||
A list with duplicates removed, maintaining original order.
|
||||
"""
|
||||
seen: set = set()
|
||||
result: List[Any] = []
|
||||
for item in items:
|
||||
item_id = _make_hashable(item)
|
||||
if item_id not in seen:
|
||||
seen.add(item_id)
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def _make_hashable(item: Any):
|
||||
"""
|
||||
Create a hashable representation of any item for deduplication.
|
||||
Converts unhashable types (dicts, lists) into deterministic tuple structures.
|
||||
"""
|
||||
if isinstance(item, dict):
|
||||
return tuple(
|
||||
sorted(
|
||||
((_make_hashable(k), _make_hashable(v)) for k, v in item.items()),
|
||||
key=lambda x: (str(type(x[0])), str(x[0])),
|
||||
)
|
||||
)
|
||||
if isinstance(item, (list, tuple)):
|
||||
return tuple(_make_hashable(i) for i in item)
|
||||
if isinstance(item, set):
|
||||
return frozenset(_make_hashable(i) for i in item)
|
||||
return item
|
||||
|
||||
|
||||
def _filter_none_values(items: List[Any]) -> List[Any]:
|
||||
"""Remove None values from a list."""
|
||||
return [item for item in items if item is not None]
|
||||
|
||||
|
||||
def _compute_nesting_depth(
|
||||
items: Any, current: int = 0, max_depth: int = _MAX_FLATTEN_DEPTH
|
||||
) -> int:
|
||||
"""
|
||||
Compute the maximum nesting depth of a list structure using iteration to avoid RecursionError.
|
||||
|
||||
Uses a stack-based approach to handle deeply nested structures without hitting Python's
|
||||
recursion limit (~1000 levels).
|
||||
"""
|
||||
if not isinstance(items, list):
|
||||
return current
|
||||
|
||||
# Stack contains tuples of (item, depth)
|
||||
stack = [(items, current)]
|
||||
max_observed_depth = current
|
||||
|
||||
while stack:
|
||||
item, depth = stack.pop()
|
||||
|
||||
if depth > max_depth:
|
||||
return depth
|
||||
|
||||
if not isinstance(item, list):
|
||||
max_observed_depth = max(max_observed_depth, depth)
|
||||
continue
|
||||
|
||||
if len(item) == 0:
|
||||
max_observed_depth = max(max_observed_depth, depth + 1)
|
||||
continue
|
||||
|
||||
# Add all children to stack with incremented depth
|
||||
for child in item:
|
||||
stack.append((child, depth + 1))
|
||||
|
||||
return max_observed_depth
|
||||
|
||||
|
||||
def _interleave_lists(lists: List[List[Any]]) -> List[Any]:
|
||||
"""
|
||||
Interleave elements from multiple lists in round-robin fashion.
|
||||
Example: [[1,2,3], [a,b], [x,y,z]] -> [1, a, x, 2, b, y, 3, z]
|
||||
"""
|
||||
if not lists:
|
||||
return []
|
||||
filtered = [lst for lst in lists if lst is not None]
|
||||
if not filtered:
|
||||
return []
|
||||
result: List[Any] = []
|
||||
max_len = max(len(lst) for lst in filtered)
|
||||
for i in range(max_len):
|
||||
for lst in filtered:
|
||||
if i < len(lst):
|
||||
result.append(lst[i])
|
||||
return result
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# List Concatenation Blocks
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class ConcatenateListsBlock(Block):
|
||||
"""
|
||||
Concatenates two or more lists into a single list.
|
||||
|
||||
This block accepts a list of lists and combines all their elements
|
||||
in order into one flat output list. It supports options for
|
||||
deduplication and None-filtering to provide flexible list merging
|
||||
capabilities for workflow pipelines.
|
||||
"""
|
||||
|
||||
class Input(BlockSchemaInput):
|
||||
lists: List[List[Any]] = SchemaField(
|
||||
description="A list of lists to concatenate together. All lists will be combined in order into a single list.",
|
||||
placeholder="e.g., [[1, 2], [3, 4], [5, 6]]",
|
||||
)
|
||||
deduplicate: bool = SchemaField(
|
||||
description="If True, remove duplicate elements from the concatenated result while preserving order.",
|
||||
default=False,
|
||||
advanced=True,
|
||||
)
|
||||
remove_none: bool = SchemaField(
|
||||
description="If True, remove None values from the concatenated result.",
|
||||
default=False,
|
||||
advanced=True,
|
||||
)
|
||||
|
||||
class Output(BlockSchemaOutput):
|
||||
concatenated_list: List[Any] = SchemaField(
|
||||
description="The concatenated list containing all elements from all input lists in order."
|
||||
)
|
||||
length: int = SchemaField(
|
||||
description="The total number of elements in the concatenated list."
|
||||
)
|
||||
error: str = SchemaField(
|
||||
description="Error message if concatenation failed due to invalid input types."
|
||||
)
|
||||
@@ -700,7 +902,7 @@ class ConcatenateListsBlock(Block):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="3cf9298b-5817-4141-9d80-7c2cc5199c8e",
|
||||
description="Concatenates multiple lists into a single list. All elements from all input lists are combined in order.",
|
||||
description="Concatenates multiple lists into a single list. All elements from all input lists are combined in order. Supports optional deduplication and None removal.",
|
||||
categories={BlockCategory.BASIC},
|
||||
input_schema=ConcatenateListsBlock.Input,
|
||||
output_schema=ConcatenateListsBlock.Output,
|
||||
@@ -709,29 +911,497 @@ class ConcatenateListsBlock(Block):
|
||||
{"lists": [["a", "b"], ["c"], ["d", "e", "f"]]},
|
||||
{"lists": [[1, 2], []]},
|
||||
{"lists": []},
|
||||
{"lists": [[1, 2, 2, 3], [3, 4]], "deduplicate": True},
|
||||
{"lists": [[1, None, 2], [None, 3]], "remove_none": True},
|
||||
],
|
||||
test_output=[
|
||||
("concatenated_list", [1, 2, 3, 4, 5, 6]),
|
||||
("length", 6),
|
||||
("concatenated_list", ["a", "b", "c", "d", "e", "f"]),
|
||||
("length", 6),
|
||||
("concatenated_list", [1, 2]),
|
||||
("length", 2),
|
||||
("concatenated_list", []),
|
||||
("length", 0),
|
||||
("concatenated_list", [1, 2, 3, 4]),
|
||||
("length", 4),
|
||||
("concatenated_list", [1, 2, 3]),
|
||||
("length", 3),
|
||||
],
|
||||
)
|
||||
|
||||
def _validate_inputs(self, lists: List[Any]) -> str | None:
|
||||
return _validate_all_lists(lists)
|
||||
|
||||
def _perform_concatenation(self, lists: List[List[Any]]) -> List[Any]:
|
||||
return _concatenate_lists_simple(lists)
|
||||
|
||||
def _apply_deduplication(self, items: List[Any]) -> List[Any]:
|
||||
return _deduplicate_list(items)
|
||||
|
||||
def _apply_none_removal(self, items: List[Any]) -> List[Any]:
|
||||
return _filter_none_values(items)
|
||||
|
||||
def _post_process(
|
||||
self, items: List[Any], deduplicate: bool, remove_none: bool
|
||||
) -> List[Any]:
|
||||
"""Apply all post-processing steps to the concatenated result."""
|
||||
result = items
|
||||
if remove_none:
|
||||
result = self._apply_none_removal(result)
|
||||
if deduplicate:
|
||||
result = self._apply_deduplication(result)
|
||||
return result
|
||||
|
||||
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
|
||||
concatenated = []
|
||||
for idx, lst in enumerate(input_data.lists):
|
||||
if lst is None:
|
||||
# Skip None values to avoid errors
|
||||
continue
|
||||
if not isinstance(lst, list):
|
||||
# Type validation: each item must be a list
|
||||
# Strings are iterable and would cause extend() to iterate character-by-character
|
||||
# Non-iterable types would raise TypeError
|
||||
yield "error", (
|
||||
f"Invalid input at index {idx}: expected a list, got {type(lst).__name__}. "
|
||||
f"All items in 'lists' must be lists (e.g., [[1, 2], [3, 4]])."
|
||||
)
|
||||
return
|
||||
concatenated.extend(lst)
|
||||
yield "concatenated_list", concatenated
|
||||
# Validate all inputs are lists
|
||||
validation_error = self._validate_inputs(input_data.lists)
|
||||
if validation_error is not None:
|
||||
yield "error", validation_error
|
||||
return
|
||||
|
||||
# Perform concatenation
|
||||
concatenated = self._perform_concatenation(input_data.lists)
|
||||
|
||||
# Apply post-processing
|
||||
result = self._post_process(
|
||||
concatenated, input_data.deduplicate, input_data.remove_none
|
||||
)
|
||||
|
||||
yield "concatenated_list", result
|
||||
yield "length", len(result)
|
||||
|
||||
|
||||
class FlattenListBlock(Block):
|
||||
"""
|
||||
Flattens a nested list structure into a single flat list.
|
||||
|
||||
This block takes a list that may contain nested lists at any depth
|
||||
and produces a single-level list with all leaf elements. Useful
|
||||
for normalizing data structures from multiple sources that may
|
||||
have varying levels of nesting.
|
||||
"""
|
||||
|
||||
class Input(BlockSchemaInput):
|
||||
nested_list: List[Any] = SchemaField(
|
||||
description="A potentially nested list to flatten into a single-level list.",
|
||||
placeholder="e.g., [[1, [2, 3]], [4, [5, [6]]]]",
|
||||
)
|
||||
max_depth: int = SchemaField(
|
||||
description="Maximum depth to flatten. -1 means flatten completely. 1 means flatten only one level.",
|
||||
default=-1,
|
||||
advanced=True,
|
||||
)
|
||||
|
||||
class Output(BlockSchemaOutput):
|
||||
flattened_list: List[Any] = SchemaField(
|
||||
description="The flattened list with all nested elements extracted."
|
||||
)
|
||||
length: int = SchemaField(
|
||||
description="The number of elements in the flattened list."
|
||||
)
|
||||
original_depth: int = SchemaField(
|
||||
description="The maximum nesting depth of the original input list."
|
||||
)
|
||||
error: str = SchemaField(description="Error message if flattening failed.")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="cc45bb0f-d035-4756-96a7-fe3e36254b4d",
|
||||
description="Flattens a nested list structure into a single flat list. Supports configurable maximum flattening depth.",
|
||||
categories={BlockCategory.BASIC},
|
||||
input_schema=FlattenListBlock.Input,
|
||||
output_schema=FlattenListBlock.Output,
|
||||
test_input=[
|
||||
{"nested_list": [[1, 2], [3, [4, 5]]]},
|
||||
{"nested_list": [1, [2, [3, [4]]]]},
|
||||
{"nested_list": [1, [2, [3, [4]]], 5], "max_depth": 1},
|
||||
{"nested_list": []},
|
||||
{"nested_list": [1, 2, 3]},
|
||||
],
|
||||
test_output=[
|
||||
("flattened_list", [1, 2, 3, 4, 5]),
|
||||
("length", 5),
|
||||
("original_depth", 3),
|
||||
("flattened_list", [1, 2, 3, 4]),
|
||||
("length", 4),
|
||||
("original_depth", 4),
|
||||
("flattened_list", [1, 2, [3, [4]], 5]),
|
||||
("length", 4),
|
||||
("original_depth", 4),
|
||||
("flattened_list", []),
|
||||
("length", 0),
|
||||
("original_depth", 1),
|
||||
("flattened_list", [1, 2, 3]),
|
||||
("length", 3),
|
||||
("original_depth", 1),
|
||||
],
|
||||
)
|
||||
|
||||
def _compute_depth(self, items: List[Any]) -> int:
|
||||
"""Compute the nesting depth of the input list."""
|
||||
return _compute_nesting_depth(items)
|
||||
|
||||
def _flatten(self, items: List[Any], max_depth: int) -> List[Any]:
|
||||
"""Flatten the list to the specified depth."""
|
||||
return _flatten_nested_list(items, max_depth=max_depth)
|
||||
|
||||
def _validate_max_depth(self, max_depth: int) -> str | None:
|
||||
"""Validate the max_depth parameter."""
|
||||
if max_depth < -1:
|
||||
return f"max_depth must be -1 (unlimited) or a non-negative integer, got {max_depth}"
|
||||
return None
|
||||
|
||||
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
|
||||
# Validate max_depth
|
||||
depth_error = self._validate_max_depth(input_data.max_depth)
|
||||
if depth_error is not None:
|
||||
yield "error", depth_error
|
||||
return
|
||||
|
||||
original_depth = self._compute_depth(input_data.nested_list)
|
||||
flattened = self._flatten(input_data.nested_list, input_data.max_depth)
|
||||
|
||||
yield "flattened_list", flattened
|
||||
yield "length", len(flattened)
|
||||
yield "original_depth", original_depth
|
||||
|
||||
|
||||
class InterleaveListsBlock(Block):
|
||||
"""
|
||||
Interleaves elements from multiple lists in round-robin fashion.
|
||||
|
||||
Given multiple input lists, this block takes one element from each
|
||||
list in turn, producing an output where elements alternate between
|
||||
sources. Lists of different lengths are handled gracefully - shorter
|
||||
lists simply stop contributing once exhausted.
|
||||
"""
|
||||
|
||||
class Input(BlockSchemaInput):
|
||||
lists: List[List[Any]] = SchemaField(
|
||||
description="A list of lists to interleave. Elements will be taken in round-robin order.",
|
||||
placeholder="e.g., [[1, 2, 3], ['a', 'b', 'c']]",
|
||||
)
|
||||
|
||||
class Output(BlockSchemaOutput):
|
||||
interleaved_list: List[Any] = SchemaField(
|
||||
description="The interleaved list with elements alternating from each input list."
|
||||
)
|
||||
length: int = SchemaField(
|
||||
description="The total number of elements in the interleaved list."
|
||||
)
|
||||
error: str = SchemaField(description="Error message if interleaving failed.")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="9f616084-1d9f-4f8e-bc00-5b9d2a75cd75",
|
||||
description="Interleaves elements from multiple lists in round-robin fashion, alternating between sources.",
|
||||
categories={BlockCategory.BASIC},
|
||||
input_schema=InterleaveListsBlock.Input,
|
||||
output_schema=InterleaveListsBlock.Output,
|
||||
test_input=[
|
||||
{"lists": [[1, 2, 3], ["a", "b", "c"]]},
|
||||
{"lists": [[1, 2, 3], ["a", "b"], ["x", "y", "z"]]},
|
||||
{"lists": [[1], [2], [3]]},
|
||||
{"lists": []},
|
||||
],
|
||||
test_output=[
|
||||
("interleaved_list", [1, "a", 2, "b", 3, "c"]),
|
||||
("length", 6),
|
||||
("interleaved_list", [1, "a", "x", 2, "b", "y", 3, "z"]),
|
||||
("length", 8),
|
||||
("interleaved_list", [1, 2, 3]),
|
||||
("length", 3),
|
||||
("interleaved_list", []),
|
||||
("length", 0),
|
||||
],
|
||||
)
|
||||
|
||||
def _validate_inputs(self, lists: List[Any]) -> str | None:
|
||||
return _validate_all_lists(lists)
|
||||
|
||||
def _interleave(self, lists: List[List[Any]]) -> List[Any]:
|
||||
return _interleave_lists(lists)
|
||||
|
||||
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
|
||||
validation_error = self._validate_inputs(input_data.lists)
|
||||
if validation_error is not None:
|
||||
yield "error", validation_error
|
||||
return
|
||||
|
||||
result = self._interleave(input_data.lists)
|
||||
yield "interleaved_list", result
|
||||
yield "length", len(result)
|
||||
|
||||
|
||||
class ZipListsBlock(Block):
|
||||
"""
|
||||
Zips multiple lists together into a list of grouped tuples/lists.
|
||||
|
||||
Takes two or more input lists and combines corresponding elements
|
||||
into sub-lists. For example, zipping [1,2,3] and ['a','b','c']
|
||||
produces [[1,'a'], [2,'b'], [3,'c']]. Supports both truncating
|
||||
to shortest list and padding to longest list with a fill value.
|
||||
"""
|
||||
|
||||
class Input(BlockSchemaInput):
|
||||
lists: List[List[Any]] = SchemaField(
|
||||
description="A list of lists to zip together. Corresponding elements will be grouped.",
|
||||
placeholder="e.g., [[1, 2, 3], ['a', 'b', 'c']]",
|
||||
)
|
||||
pad_to_longest: bool = SchemaField(
|
||||
description="If True, pad shorter lists with fill_value to match the longest list. If False, truncate to shortest.",
|
||||
default=False,
|
||||
advanced=True,
|
||||
)
|
||||
fill_value: Any = SchemaField(
|
||||
description="Value to use for padding when pad_to_longest is True.",
|
||||
default=None,
|
||||
advanced=True,
|
||||
)
|
||||
|
||||
class Output(BlockSchemaOutput):
|
||||
zipped_list: List[List[Any]] = SchemaField(
|
||||
description="The zipped list of grouped elements."
|
||||
)
|
||||
length: int = SchemaField(
|
||||
description="The number of groups in the zipped result."
|
||||
)
|
||||
error: str = SchemaField(description="Error message if zipping failed.")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="0d0e684f-5cb9-4c4b-b8d1-47a0860e0c07",
|
||||
description="Zips multiple lists together into a list of grouped elements. Supports padding to longest or truncating to shortest.",
|
||||
categories={BlockCategory.BASIC},
|
||||
input_schema=ZipListsBlock.Input,
|
||||
output_schema=ZipListsBlock.Output,
|
||||
test_input=[
|
||||
{"lists": [[1, 2, 3], ["a", "b", "c"]]},
|
||||
{"lists": [[1, 2, 3], ["a", "b"]]},
|
||||
{
|
||||
"lists": [[1, 2], ["a", "b", "c"]],
|
||||
"pad_to_longest": True,
|
||||
"fill_value": 0,
|
||||
},
|
||||
{"lists": []},
|
||||
],
|
||||
test_output=[
|
||||
("zipped_list", [[1, "a"], [2, "b"], [3, "c"]]),
|
||||
("length", 3),
|
||||
("zipped_list", [[1, "a"], [2, "b"]]),
|
||||
("length", 2),
|
||||
("zipped_list", [[1, "a"], [2, "b"], [0, "c"]]),
|
||||
("length", 3),
|
||||
("zipped_list", []),
|
||||
("length", 0),
|
||||
],
|
||||
)
|
||||
|
||||
def _validate_inputs(self, lists: List[Any]) -> str | None:
|
||||
return _validate_all_lists(lists)
|
||||
|
||||
def _zip_truncate(self, lists: List[List[Any]]) -> List[List[Any]]:
|
||||
"""Zip lists, truncating to shortest."""
|
||||
filtered = [lst for lst in lists if lst is not None]
|
||||
if not filtered:
|
||||
return []
|
||||
return [list(group) for group in zip(*filtered)]
|
||||
|
||||
def _zip_pad(self, lists: List[List[Any]], fill_value: Any) -> List[List[Any]]:
|
||||
"""Zip lists, padding shorter ones with fill_value."""
|
||||
if not lists:
|
||||
return []
|
||||
lists = [lst for lst in lists if lst is not None]
|
||||
if not lists:
|
||||
return []
|
||||
max_len = max(len(lst) for lst in lists)
|
||||
result: List[List[Any]] = []
|
||||
for i in range(max_len):
|
||||
group: List[Any] = []
|
||||
for lst in lists:
|
||||
if i < len(lst):
|
||||
group.append(lst[i])
|
||||
else:
|
||||
group.append(fill_value)
|
||||
result.append(group)
|
||||
return result
|
||||
|
||||
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
|
||||
validation_error = self._validate_inputs(input_data.lists)
|
||||
if validation_error is not None:
|
||||
yield "error", validation_error
|
||||
return
|
||||
|
||||
if not input_data.lists:
|
||||
yield "zipped_list", []
|
||||
yield "length", 0
|
||||
return
|
||||
|
||||
if input_data.pad_to_longest:
|
||||
result = self._zip_pad(input_data.lists, input_data.fill_value)
|
||||
else:
|
||||
result = self._zip_truncate(input_data.lists)
|
||||
|
||||
yield "zipped_list", result
|
||||
yield "length", len(result)
|
||||
|
||||
|
||||
class ListDifferenceBlock(Block):
|
||||
"""
|
||||
Computes the difference between two lists (elements in the first
|
||||
list that are not in the second list).
|
||||
|
||||
This is useful for finding items that exist in one dataset but
|
||||
not in another, such as finding new items, missing items, or
|
||||
items that need to be processed.
|
||||
"""
|
||||
|
||||
class Input(BlockSchemaInput):
|
||||
list_a: List[Any] = SchemaField(
|
||||
description="The primary list to check elements from.",
|
||||
placeholder="e.g., [1, 2, 3, 4, 5]",
|
||||
)
|
||||
list_b: List[Any] = SchemaField(
|
||||
description="The list to subtract. Elements found here will be removed from list_a.",
|
||||
placeholder="e.g., [3, 4, 5, 6]",
|
||||
)
|
||||
symmetric: bool = SchemaField(
|
||||
description="If True, compute symmetric difference (elements in either list but not both).",
|
||||
default=False,
|
||||
advanced=True,
|
||||
)
|
||||
|
||||
class Output(BlockSchemaOutput):
|
||||
difference: List[Any] = SchemaField(
|
||||
description="Elements from list_a not found in list_b (or symmetric difference if enabled)."
|
||||
)
|
||||
length: int = SchemaField(
|
||||
description="The number of elements in the difference result."
|
||||
)
|
||||
error: str = SchemaField(description="Error message if the operation failed.")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="05309873-9d61-447e-96b5-b804e2511829",
|
||||
description="Computes the difference between two lists. Returns elements in the first list not found in the second, or symmetric difference.",
|
||||
categories={BlockCategory.BASIC},
|
||||
input_schema=ListDifferenceBlock.Input,
|
||||
output_schema=ListDifferenceBlock.Output,
|
||||
test_input=[
|
||||
{"list_a": [1, 2, 3, 4, 5], "list_b": [3, 4, 5, 6, 7]},
|
||||
{
|
||||
"list_a": [1, 2, 3, 4, 5],
|
||||
"list_b": [3, 4, 5, 6, 7],
|
||||
"symmetric": True,
|
||||
},
|
||||
{"list_a": ["a", "b", "c"], "list_b": ["b"]},
|
||||
{"list_a": [], "list_b": [1, 2, 3]},
|
||||
],
|
||||
test_output=[
|
||||
("difference", [1, 2]),
|
||||
("length", 2),
|
||||
("difference", [1, 2, 6, 7]),
|
||||
("length", 4),
|
||||
("difference", ["a", "c"]),
|
||||
("length", 2),
|
||||
("difference", []),
|
||||
("length", 0),
|
||||
],
|
||||
)
|
||||
|
||||
def _compute_difference(self, list_a: List[Any], list_b: List[Any]) -> List[Any]:
|
||||
"""Compute elements in list_a not in list_b."""
|
||||
b_hashes = {_make_hashable(item) for item in list_b}
|
||||
return [item for item in list_a if _make_hashable(item) not in b_hashes]
|
||||
|
||||
def _compute_symmetric_difference(
|
||||
self, list_a: List[Any], list_b: List[Any]
|
||||
) -> List[Any]:
|
||||
"""Compute elements in either list but not both."""
|
||||
a_hashes = {_make_hashable(item) for item in list_a}
|
||||
b_hashes = {_make_hashable(item) for item in list_b}
|
||||
only_in_a = [item for item in list_a if _make_hashable(item) not in b_hashes]
|
||||
only_in_b = [item for item in list_b if _make_hashable(item) not in a_hashes]
|
||||
return only_in_a + only_in_b
|
||||
|
||||
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
|
||||
if input_data.symmetric:
|
||||
result = self._compute_symmetric_difference(
|
||||
input_data.list_a, input_data.list_b
|
||||
)
|
||||
else:
|
||||
result = self._compute_difference(input_data.list_a, input_data.list_b)
|
||||
|
||||
yield "difference", result
|
||||
yield "length", len(result)
|
||||
|
||||
|
||||
class ListIntersectionBlock(Block):
|
||||
"""
|
||||
Computes the intersection of two lists (elements present in both lists).
|
||||
|
||||
This is useful for finding common items between two datasets,
|
||||
such as shared tags, mutual connections, or overlapping categories.
|
||||
"""
|
||||
|
||||
class Input(BlockSchemaInput):
|
||||
list_a: List[Any] = SchemaField(
|
||||
description="The first list to intersect.",
|
||||
placeholder="e.g., [1, 2, 3, 4, 5]",
|
||||
)
|
||||
list_b: List[Any] = SchemaField(
|
||||
description="The second list to intersect.",
|
||||
placeholder="e.g., [3, 4, 5, 6, 7]",
|
||||
)
|
||||
|
||||
class Output(BlockSchemaOutput):
|
||||
intersection: List[Any] = SchemaField(
|
||||
description="Elements present in both list_a and list_b."
|
||||
)
|
||||
length: int = SchemaField(
|
||||
description="The number of elements in the intersection."
|
||||
)
|
||||
error: str = SchemaField(description="Error message if the operation failed.")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="b6eb08b6-dbe3-411b-b9b4-2508cb311a1f",
|
||||
description="Computes the intersection of two lists, returning only elements present in both.",
|
||||
categories={BlockCategory.BASIC},
|
||||
input_schema=ListIntersectionBlock.Input,
|
||||
output_schema=ListIntersectionBlock.Output,
|
||||
test_input=[
|
||||
{"list_a": [1, 2, 3, 4, 5], "list_b": [3, 4, 5, 6, 7]},
|
||||
{"list_a": ["a", "b", "c"], "list_b": ["c", "d", "e"]},
|
||||
{"list_a": [1, 2], "list_b": [3, 4]},
|
||||
{"list_a": [], "list_b": [1, 2, 3]},
|
||||
],
|
||||
test_output=[
|
||||
("intersection", [3, 4, 5]),
|
||||
("length", 3),
|
||||
("intersection", ["c"]),
|
||||
("length", 1),
|
||||
("intersection", []),
|
||||
("length", 0),
|
||||
("intersection", []),
|
||||
("length", 0),
|
||||
],
|
||||
)
|
||||
|
||||
def _compute_intersection(self, list_a: List[Any], list_b: List[Any]) -> List[Any]:
|
||||
"""Compute elements present in both lists, preserving order from list_a."""
|
||||
b_hashes = {_make_hashable(item) for item in list_b}
|
||||
seen: set = set()
|
||||
result: List[Any] = []
|
||||
for item in list_a:
|
||||
h = _make_hashable(item)
|
||||
if h in b_hashes and h not in seen:
|
||||
result.append(item)
|
||||
seen.add(h)
|
||||
return result
|
||||
|
||||
async def run(self, input_data: Input, **kwargs) -> BlockOutput:
|
||||
result = self._compute_intersection(input_data.list_a, input_data.list_b)
|
||||
yield "intersection", result
|
||||
yield "length", len(result)
|
||||
|
||||
@@ -867,67 +867,9 @@ class GraphModel(Graph, GraphMeta):
|
||||
|
||||
return node_errors
|
||||
|
||||
@staticmethod
|
||||
def prune_invalid_links(graph: BaseGraph) -> int:
|
||||
"""
|
||||
Remove invalid/orphan links from the graph.
|
||||
|
||||
This removes links that:
|
||||
- Reference non-existent source or sink nodes
|
||||
- Reference invalid block IDs
|
||||
|
||||
Note: Pin name validation is handled separately in _validate_graph_structure.
|
||||
|
||||
Returns the number of links pruned.
|
||||
"""
|
||||
node_map = {v.id: v for v in graph.nodes}
|
||||
original_count = len(graph.links)
|
||||
valid_links = []
|
||||
|
||||
for link in graph.links:
|
||||
source_node = node_map.get(link.source_id)
|
||||
sink_node = node_map.get(link.sink_id)
|
||||
|
||||
# Skip if either node doesn't exist
|
||||
if not source_node or not sink_node:
|
||||
logger.warning(
|
||||
f"Pruning orphan link: source={link.source_id}, sink={link.sink_id} "
|
||||
f"- node(s) not found"
|
||||
)
|
||||
continue
|
||||
|
||||
# Skip if source block doesn't exist
|
||||
source_block = get_block(source_node.block_id)
|
||||
if not source_block:
|
||||
logger.warning(
|
||||
f"Pruning link with invalid source block: {source_node.block_id}"
|
||||
)
|
||||
continue
|
||||
|
||||
# Skip if sink block doesn't exist
|
||||
sink_block = get_block(sink_node.block_id)
|
||||
if not sink_block:
|
||||
logger.warning(
|
||||
f"Pruning link with invalid sink block: {sink_node.block_id}"
|
||||
)
|
||||
continue
|
||||
|
||||
valid_links.append(link)
|
||||
|
||||
graph.links = valid_links
|
||||
pruned_count = original_count - len(valid_links)
|
||||
|
||||
if pruned_count > 0:
|
||||
logger.info(f"Pruned {pruned_count} invalid link(s) from graph {graph.id}")
|
||||
|
||||
return pruned_count
|
||||
|
||||
@staticmethod
|
||||
def _validate_graph_structure(graph: BaseGraph):
|
||||
"""Validate graph structure (links, connections, etc.)"""
|
||||
# First, prune invalid links to clean up orphan edges
|
||||
GraphModel.prune_invalid_links(graph)
|
||||
|
||||
node_map = {v.id: v for v in graph.nodes}
|
||||
|
||||
def is_static_output_block(nid: str) -> bool:
|
||||
|
||||
1276
autogpt_platform/backend/test/blocks/test_list_concatenation.py
Normal file
1276
autogpt_platform/backend/test/blocks/test_list_concatenation.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -133,23 +133,22 @@ export const useFlow = () => {
|
||||
}
|
||||
}, [availableGraphs, setAvailableSubGraphs]);
|
||||
|
||||
// adding nodes and links together to avoid race condition
|
||||
// Links depend on nodes existing, so we must add nodes first
|
||||
// adding nodes
|
||||
useEffect(() => {
|
||||
if (customNodes.length > 0) {
|
||||
// Clear both stores to prevent stale data from previous graphs
|
||||
useNodeStore.getState().setNodes([]);
|
||||
useNodeStore.getState().clearResolutionState();
|
||||
useEdgeStore.getState().setEdges([]);
|
||||
|
||||
addNodes(customNodes);
|
||||
|
||||
// Only add links after nodes are in the store
|
||||
if (graph?.links) {
|
||||
addLinks(graph.links);
|
||||
}
|
||||
}
|
||||
}, [customNodes, graph?.links, addNodes, addLinks]);
|
||||
}, [customNodes, addNodes]);
|
||||
|
||||
// adding links
|
||||
useEffect(() => {
|
||||
if (graph?.links) {
|
||||
useEdgeStore.getState().setEdges([]);
|
||||
addLinks(graph.links);
|
||||
}
|
||||
}, [graph?.links, addLinks]);
|
||||
|
||||
useEffect(() => {
|
||||
if (customNodes.length > 0 && graph?.links) {
|
||||
|
||||
@@ -13,7 +13,6 @@ import { Graph } from "@/app/api/__generated__/models/graph";
|
||||
import { useNodeStore } from "../stores/nodeStore";
|
||||
import { useEdgeStore } from "../stores/edgeStore";
|
||||
import { graphsEquivalent } from "../components/NewControlPanel/NewSaveControl/helpers";
|
||||
import { linkToCustomEdge } from "../components/helper";
|
||||
import { useGraphStore } from "../stores/graphStore";
|
||||
import { useShallow } from "zustand/react/shallow";
|
||||
import {
|
||||
@@ -22,18 +21,6 @@ import {
|
||||
getTempFlowId,
|
||||
} from "@/services/builder-draft/draft-service";
|
||||
|
||||
/**
|
||||
* Sync the edge store with the authoritative backend state.
|
||||
* This ensures the frontend matches what the backend accepted after save.
|
||||
*/
|
||||
function syncEdgesWithBackend(links: GraphModel["links"]) {
|
||||
if (links !== undefined) {
|
||||
// Replace all edges with the authoritative backend state
|
||||
const newEdges = links.map(linkToCustomEdge);
|
||||
useEdgeStore.getState().setEdges(newEdges);
|
||||
}
|
||||
}
|
||||
|
||||
export type SaveGraphOptions = {
|
||||
showToast?: boolean;
|
||||
onSuccess?: (graph: GraphModel) => void;
|
||||
@@ -77,9 +64,6 @@ export const useSaveGraph = ({
|
||||
flowVersion: data.version,
|
||||
});
|
||||
|
||||
// Sync edge store with authoritative backend state
|
||||
syncEdgesWithBackend(data.links);
|
||||
|
||||
const tempFlowId = getTempFlowId();
|
||||
if (tempFlowId) {
|
||||
await draftService.deleteDraft(tempFlowId);
|
||||
@@ -117,9 +101,6 @@ export const useSaveGraph = ({
|
||||
flowVersion: data.version,
|
||||
});
|
||||
|
||||
// Sync edge store with authoritative backend state
|
||||
syncEdgesWithBackend(data.links);
|
||||
|
||||
// Clear the draft for this flow after successful save
|
||||
if (data.id) {
|
||||
await draftService.deleteDraft(data.id);
|
||||
|
||||
@@ -120,64 +120,12 @@ export const useEdgeStore = create<EdgeStore>((set, get) => ({
|
||||
isOutputConnected: (nodeId, handle) =>
|
||||
get().edges.some((e) => e.source === nodeId && e.sourceHandle === handle),
|
||||
|
||||
getBackendLinks: () => {
|
||||
// Filter out edges referencing non-existent nodes before converting to links
|
||||
const nodeIds = new Set(useNodeStore.getState().nodes.map((n) => n.id));
|
||||
const validEdges = get().edges.filter((edge) => {
|
||||
const isValid = nodeIds.has(edge.source) && nodeIds.has(edge.target);
|
||||
if (!isValid) {
|
||||
console.warn(
|
||||
`[EdgeStore] Filtering out invalid edge during save: source=${edge.source}, target=${edge.target}`,
|
||||
);
|
||||
}
|
||||
return isValid;
|
||||
});
|
||||
return validEdges.map(customEdgeToLink);
|
||||
},
|
||||
getBackendLinks: () => get().edges.map(customEdgeToLink),
|
||||
|
||||
addLinks: (links) => {
|
||||
// Get current node IDs to validate links
|
||||
const nodeIds = new Set(useNodeStore.getState().nodes.map((n) => n.id));
|
||||
|
||||
// Convert and filter links in one pass, avoiding individual addEdge calls
|
||||
// which would push to history for each edge (causing history pollution)
|
||||
const newEdges: CustomEdge[] = [];
|
||||
const existingEdges = get().edges;
|
||||
|
||||
for (const link of links) {
|
||||
// Skip invalid links (orphan edges referencing non-existent nodes)
|
||||
if (!nodeIds.has(link.source_id) || !nodeIds.has(link.sink_id)) {
|
||||
console.warn(
|
||||
`[EdgeStore] Skipping invalid link: source=${link.source_id}, sink=${link.sink_id} - node(s) not found`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const edge = linkToCustomEdge(link);
|
||||
|
||||
// Skip if edge already exists
|
||||
const exists = existingEdges.some(
|
||||
(e) =>
|
||||
e.source === edge.source &&
|
||||
e.target === edge.target &&
|
||||
e.sourceHandle === edge.sourceHandle &&
|
||||
e.targetHandle === edge.targetHandle,
|
||||
);
|
||||
if (!exists) {
|
||||
newEdges.push(edge);
|
||||
}
|
||||
}
|
||||
|
||||
if (newEdges.length > 0) {
|
||||
// Bulk add all edges at once, pushing to history only once
|
||||
const prevState = {
|
||||
nodes: useNodeStore.getState().nodes,
|
||||
edges: existingEdges,
|
||||
};
|
||||
|
||||
set((state) => ({ edges: [...state.edges, ...newEdges] }));
|
||||
useHistoryStore.getState().pushState(prevState);
|
||||
}
|
||||
links.forEach((link) => {
|
||||
get().addEdge(linkToCustomEdge(link));
|
||||
});
|
||||
},
|
||||
|
||||
getAllHandleIdsOfANode: (nodeId) =>
|
||||
|
||||
@@ -56,12 +56,16 @@ Below is a comprehensive list of all available blocks, categorized by their prim
|
||||
| [File Store](block-integrations/basic.md#file-store) | Downloads and stores a file from a URL, data URI, or local path |
|
||||
| [Find In Dictionary](block-integrations/basic.md#find-in-dictionary) | A block that looks up a value in a dictionary, list, or object by key or index and returns the corresponding value |
|
||||
| [Find In List](block-integrations/basic.md#find-in-list) | Finds the index of the value in the list |
|
||||
| [Flatten List](block-integrations/basic.md#flatten-list) | Flattens a nested list structure into a single flat list |
|
||||
| [Get All Memories](block-integrations/basic.md#get-all-memories) | Retrieve all memories from Mem0 with optional conversation filtering |
|
||||
| [Get Latest Memory](block-integrations/basic.md#get-latest-memory) | Retrieve the latest memory from Mem0 with optional key filtering |
|
||||
| [Get List Item](block-integrations/basic.md#get-list-item) | Returns the element at the given index |
|
||||
| [Get Store Agent Details](block-integrations/system/store_operations.md#get-store-agent-details) | Get detailed information about an agent from the store |
|
||||
| [Get Weather Information](block-integrations/basic.md#get-weather-information) | Retrieves weather information for a specified location using OpenWeatherMap API |
|
||||
| [Human In The Loop](block-integrations/basic.md#human-in-the-loop) | Pause execution for human review |
|
||||
| [Interleave Lists](block-integrations/basic.md#interleave-lists) | Interleaves elements from multiple lists in round-robin fashion, alternating between sources |
|
||||
| [List Difference](block-integrations/basic.md#list-difference) | Computes the difference between two lists |
|
||||
| [List Intersection](block-integrations/basic.md#list-intersection) | Computes the intersection of two lists, returning only elements present in both |
|
||||
| [List Is Empty](block-integrations/basic.md#list-is-empty) | Checks if a list is empty |
|
||||
| [List Library Agents](block-integrations/system/library_operations.md#list-library-agents) | List all agents in your personal library |
|
||||
| [Note](block-integrations/basic.md#note) | A visual annotation block that displays a sticky note in the workflow editor for documentation and organization purposes |
|
||||
@@ -84,6 +88,7 @@ Below is a comprehensive list of all available blocks, categorized by their prim
|
||||
| [Store Value](block-integrations/basic.md#store-value) | A basic block that stores and forwards a value throughout workflows, allowing it to be reused without changes across multiple blocks |
|
||||
| [Universal Type Converter](block-integrations/basic.md#universal-type-converter) | This block is used to convert a value to a universal type |
|
||||
| [XML Parser](block-integrations/basic.md#xml-parser) | Parses XML using gravitasml to tokenize and coverts it to dict |
|
||||
| [Zip Lists](block-integrations/basic.md#zip-lists) | Zips multiple lists together into a list of grouped elements |
|
||||
|
||||
## Data Processing
|
||||
|
||||
|
||||
@@ -637,7 +637,7 @@ This enables extensibility by allowing custom blocks to be added without modifyi
|
||||
## Concatenate Lists
|
||||
|
||||
### What it is
|
||||
Concatenates multiple lists into a single list. All elements from all input lists are combined in order.
|
||||
Concatenates multiple lists into a single list. All elements from all input lists are combined in order. Supports optional deduplication and None removal.
|
||||
|
||||
### How it works
|
||||
<!-- MANUAL: how_it_works -->
|
||||
@@ -651,6 +651,8 @@ The block includes validation to ensure each item is actually a list. If a non-l
|
||||
| Input | Description | Type | Required |
|
||||
|-------|-------------|------|----------|
|
||||
| lists | A list of lists to concatenate together. All lists will be combined in order into a single list. | List[List[Any]] | Yes |
|
||||
| deduplicate | If True, remove duplicate elements from the concatenated result while preserving order. | bool | No |
|
||||
| remove_none | If True, remove None values from the concatenated result. | bool | No |
|
||||
|
||||
### Outputs
|
||||
|
||||
@@ -658,6 +660,7 @@ The block includes validation to ensure each item is actually a list. If a non-l
|
||||
|--------|-------------|------|
|
||||
| error | Error message if concatenation failed due to invalid input types. | str |
|
||||
| concatenated_list | The concatenated list containing all elements from all input lists in order. | List[Any] |
|
||||
| length | The total number of elements in the concatenated list. | int |
|
||||
|
||||
### Possible use case
|
||||
<!-- MANUAL: use_case -->
|
||||
@@ -820,6 +823,45 @@ This enables conditional logic based on list membership and helps locate items f
|
||||
|
||||
---
|
||||
|
||||
## Flatten List
|
||||
|
||||
### What it is
|
||||
Flattens a nested list structure into a single flat list. Supports configurable maximum flattening depth.
|
||||
|
||||
### How it works
|
||||
<!-- MANUAL: how_it_works -->
|
||||
This block recursively traverses a nested list and extracts all leaf elements into a single flat list. You can control how deep the flattening goes with the max_depth parameter: set it to -1 to flatten completely, or to a positive integer to flatten only that many levels.
|
||||
|
||||
The block also reports the original nesting depth of the input, which is useful for understanding the structure of data coming from sources with varying levels of nesting.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
### Inputs
|
||||
|
||||
| Input | Description | Type | Required |
|
||||
|-------|-------------|------|----------|
|
||||
| nested_list | A potentially nested list to flatten into a single-level list. | List[Any] | Yes |
|
||||
| max_depth | Maximum depth to flatten. -1 means flatten completely. 1 means flatten only one level. | int | No |
|
||||
|
||||
### Outputs
|
||||
|
||||
| Output | Description | Type |
|
||||
|--------|-------------|------|
|
||||
| error | Error message if flattening failed. | str |
|
||||
| flattened_list | The flattened list with all nested elements extracted. | List[Any] |
|
||||
| length | The number of elements in the flattened list. | int |
|
||||
| original_depth | The maximum nesting depth of the original input list. | int |
|
||||
|
||||
### Possible use case
|
||||
<!-- MANUAL: use_case -->
|
||||
**Normalizing API Responses**: Flatten nested JSON arrays from different API endpoints into a uniform single-level list for consistent processing.
|
||||
|
||||
**Aggregating Nested Results**: Combine results from recursive file searches or nested category trees into a flat list of items for display or export.
|
||||
|
||||
**Data Pipeline Cleanup**: Simplify deeply nested data structures from multiple transformation steps into a clean flat list before final output.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
---
|
||||
|
||||
## Get All Memories
|
||||
|
||||
### What it is
|
||||
@@ -1012,6 +1054,120 @@ This enables human oversight at critical points in automated workflows, ensuring
|
||||
|
||||
---
|
||||
|
||||
## Interleave Lists
|
||||
|
||||
### What it is
|
||||
Interleaves elements from multiple lists in round-robin fashion, alternating between sources.
|
||||
|
||||
### How it works
|
||||
<!-- MANUAL: how_it_works -->
|
||||
This block takes elements from each input list in round-robin order, picking one element from each list in turn. For example, given `[[1, 2, 3], ['a', 'b', 'c']]`, it produces `[1, 'a', 2, 'b', 3, 'c']`.
|
||||
|
||||
When lists have different lengths, shorter lists stop contributing once exhausted, and remaining elements from longer lists continue to be added in order.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
### Inputs
|
||||
|
||||
| Input | Description | Type | Required |
|
||||
|-------|-------------|------|----------|
|
||||
| lists | A list of lists to interleave. Elements will be taken in round-robin order. | List[List[Any]] | Yes |
|
||||
|
||||
### Outputs
|
||||
|
||||
| Output | Description | Type |
|
||||
|--------|-------------|------|
|
||||
| error | Error message if interleaving failed. | str |
|
||||
| interleaved_list | The interleaved list with elements alternating from each input list. | List[Any] |
|
||||
| length | The total number of elements in the interleaved list. | int |
|
||||
|
||||
### Possible use case
|
||||
<!-- MANUAL: use_case -->
|
||||
**Balanced Content Mixing**: Alternate between content from different sources (e.g., mixing promotional and organic posts) for a balanced feed.
|
||||
|
||||
**Round-Robin Scheduling**: Distribute tasks evenly across workers or queues by interleaving items from separate task lists.
|
||||
|
||||
**Multi-Language Output**: Weave together translated text segments with their original counterparts for side-by-side comparison.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
---
|
||||
|
||||
## List Difference
|
||||
|
||||
### What it is
|
||||
Computes the difference between two lists. Returns elements in the first list not found in the second, or symmetric difference.
|
||||
|
||||
### How it works
|
||||
<!-- MANUAL: how_it_works -->
|
||||
This block compares two lists and returns elements from list_a that do not appear in list_b. It uses hash-based lookup for efficient comparison. When symmetric mode is enabled, it returns elements that are in either list but not in both.
|
||||
|
||||
The order of elements from list_a is preserved in the output, and elements from list_b are appended when using symmetric difference.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
### Inputs
|
||||
|
||||
| Input | Description | Type | Required |
|
||||
|-------|-------------|------|----------|
|
||||
| list_a | The primary list to check elements from. | List[Any] | Yes |
|
||||
| list_b | The list to subtract. Elements found here will be removed from list_a. | List[Any] | Yes |
|
||||
| symmetric | If True, compute symmetric difference (elements in either list but not both). | bool | No |
|
||||
|
||||
### Outputs
|
||||
|
||||
| Output | Description | Type |
|
||||
|--------|-------------|------|
|
||||
| error | Error message if the operation failed. | str |
|
||||
| difference | Elements from list_a not found in list_b (or symmetric difference if enabled). | List[Any] |
|
||||
| length | The number of elements in the difference result. | int |
|
||||
|
||||
### Possible use case
|
||||
<!-- MANUAL: use_case -->
|
||||
**Change Detection**: Compare a current list of records against a previous snapshot to find newly added or removed items.
|
||||
|
||||
**Exclusion Filtering**: Remove items from a list that appear in a blocklist or already-processed list to avoid duplicates.
|
||||
|
||||
**Data Sync**: Identify which items exist in one system but not another to determine what needs to be synced.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
---
|
||||
|
||||
## List Intersection
|
||||
|
||||
### What it is
|
||||
Computes the intersection of two lists, returning only elements present in both.
|
||||
|
||||
### How it works
|
||||
<!-- MANUAL: how_it_works -->
|
||||
This block finds elements that appear in both input lists by hashing elements from list_b for efficient lookup, then checking each element of list_a against that set. The output preserves the order from list_a and removes duplicates.
|
||||
|
||||
This is useful for finding common items between two datasets without needing to manually iterate or compare.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
### Inputs
|
||||
|
||||
| Input | Description | Type | Required |
|
||||
|-------|-------------|------|----------|
|
||||
| list_a | The first list to intersect. | List[Any] | Yes |
|
||||
| list_b | The second list to intersect. | List[Any] | Yes |
|
||||
|
||||
### Outputs
|
||||
|
||||
| Output | Description | Type |
|
||||
|--------|-------------|------|
|
||||
| error | Error message if the operation failed. | str |
|
||||
| intersection | Elements present in both list_a and list_b. | List[Any] |
|
||||
| length | The number of elements in the intersection. | int |
|
||||
|
||||
### Possible use case
|
||||
<!-- MANUAL: use_case -->
|
||||
**Finding Common Tags**: Identify shared tags or categories between two items for recommendation or grouping purposes.
|
||||
|
||||
**Mutual Connections**: Find users or contacts that appear in both of two different lists, such as shared friends or overlapping team members.
|
||||
|
||||
**Feature Comparison**: Determine which features or capabilities are supported by both of two systems or products.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
---
|
||||
|
||||
## List Is Empty
|
||||
|
||||
### What it is
|
||||
@@ -1452,3 +1608,42 @@ This makes XML data accessible using standard dictionary operations, allowing yo
|
||||
<!-- END MANUAL -->
|
||||
|
||||
---
|
||||
|
||||
## Zip Lists
|
||||
|
||||
### What it is
|
||||
Zips multiple lists together into a list of grouped elements. Supports padding to longest or truncating to shortest.
|
||||
|
||||
### How it works
|
||||
<!-- MANUAL: how_it_works -->
|
||||
This block pairs up corresponding elements from multiple input lists into sub-lists. For example, zipping `[[1, 2, 3], ['a', 'b', 'c']]` produces `[[1, 'a'], [2, 'b'], [3, 'c']]`.
|
||||
|
||||
By default, the result is truncated to the length of the shortest input list. Enable pad_to_longest to instead pad shorter lists with a fill_value so no elements from longer lists are lost.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
### Inputs
|
||||
|
||||
| Input | Description | Type | Required |
|
||||
|-------|-------------|------|----------|
|
||||
| lists | A list of lists to zip together. Corresponding elements will be grouped. | List[List[Any]] | Yes |
|
||||
| pad_to_longest | If True, pad shorter lists with fill_value to match the longest list. If False, truncate to shortest. | bool | No |
|
||||
| fill_value | Value to use for padding when pad_to_longest is True. | Fill Value | No |
|
||||
|
||||
### Outputs
|
||||
|
||||
| Output | Description | Type |
|
||||
|--------|-------------|------|
|
||||
| error | Error message if zipping failed. | str |
|
||||
| zipped_list | The zipped list of grouped elements. | List[List[Any]] |
|
||||
| length | The number of groups in the zipped result. | int |
|
||||
|
||||
### Possible use case
|
||||
<!-- MANUAL: use_case -->
|
||||
**Creating Key-Value Pairs**: Combine a list of field names with a list of values to build structured records or dictionaries.
|
||||
|
||||
**Parallel Data Alignment**: Pair up corresponding items from separate data sources (e.g., names and email addresses) for processing together.
|
||||
|
||||
**Table Row Construction**: Group column data into rows by zipping each column's values together for CSV export or display.
|
||||
<!-- END MANUAL -->
|
||||
|
||||
---
|
||||
|
||||
Reference in New Issue
Block a user