feat(rnd): Add dynamic input pin for input object construction (#7871)

### Background Currently, there is no way to construct the output of nodes into a composite data structure (list/dict/object) using the builder UI. The backend already supports this feature by connecting the output pin to the input pin using these format: * <pin_name>_$_<list_index> for constructing list * <pin_name>_#_<dict_key> for constructing dict * <pin_name>_@_<field_name> for constructing object The scope of this PR is implementing the UX for this in the builder UI. ### Changes 🏗️ <img width="765" alt="image" src="https://github.com/user-attachments/assets/8fc319a4-1350-410f-98cf-24f2aa2bc34b"> This allows you to add more pins in a key value & list input: `_$_` list constructor & `_#_` dict constructor.
2026-04-08 03:00:28 -04:00 · 2024-08-23 20:21:38 +02:00
parent e59e138352
commit f9b8b0a41a
10 changed files with 193 additions and 150 deletions
--- a/rnd/autogpt_builder/src/components/CustomNode.tsx
+++ b/rnd/autogpt_builder/src/components/CustomNode.tsx
@@ -21,13 +21,20 @@ import { Switch } from "@/components/ui/switch";
 import { Copy, Trash2 } from "lucide-react";
 import { history } from "./history";
 import NodeHandle from "./NodeHandle";
-import { CustomEdgeData } from "./CustomEdge";
 import { NodeGenericInputField } from "./node-input-components";
 import SchemaTooltip from "./SchemaTooltip";
 import { getPrimaryCategoryColor } from "@/lib/utils";

 type ParsedKey = { key: string; index?: number };

+export type ConnectionData = Array<{
+  edge_id: string;
+  source: string;
+  sourceHandle: string;
+  target: string;
+  targetHandle: string;
+}>;
+
 export type CustomNodeData = {
  blockType: string;
  title: string;
@@ -37,13 +44,7 @@ export type CustomNodeData = {
  outputSchema: BlockIORootSchema;
  hardcodedValues: { [key: string]: any };
  setHardcodedValues: (values: { [key: string]: any }) => void;
-  connections: Array<{
-    edge_id: string;
-    source: string;
-    sourceHandle: string;
-    target: string;
-    targetHandle: string;
-  }>;
+  connections: ConnectionData;
  isOutputOpen: boolean;
  status?: NodeExecutionResult["status"];
  output_data?: NodeExecutionResult["output_data"];
@@ -161,26 +162,26 @@ const CustomNode: FC<NodeProps<CustomNodeData>> = ({ data, id }) => {

  // Helper function to parse keys with array indices
  const parseKeys = (key: string): ParsedKey[] => {
-    const regex = /(\w+)|\[(\d+)\]/g;
+    const splits = key.split(/_@_|_#_|_\$_|\./);
    const keys: ParsedKey[] = [];
-    let match;
    let currentKey: string | null = null;

-    while ((match = regex.exec(key)) !== null) {
-      if (match[1]) {
+    splits.forEach((split) => {
+      const isInteger = /^\d+$/.test(split);
+      if (!isInteger) {
        if (currentKey !== null) {
          keys.push({ key: currentKey });
        }
-        currentKey = match[1];
-      } else if (match[2]) {
+        currentKey = split;
+      } else {
        if (currentKey !== null) {
-          keys.push({ key: currentKey, index: parseInt(match[2], 10) });
+          keys.push({ key: currentKey, index: parseInt(split, 10) });
          currentKey = null;
        } else {
          throw new Error("Invalid key format: array index without a key");
        }
      }
-    }
+    });

    if (currentKey !== null) {
      keys.push({ key: currentKey });
@@ -343,6 +344,7 @@ const CustomNode: FC<NodeProps<CustomNodeData>> = ({ data, id }) => {
                          propKey={propKey}
                          propSchema={propSchema}
                          currentValue={getValue(propKey)}
+                          connections={data.connections}
                          handleInputChange={handleInputChange}
                          handleInputClick={handleInputClick}
                          errors={data.errors ?? {}}
--- a/rnd/autogpt_builder/src/components/NodeHandle.tsx
+++ b/rnd/autogpt_builder/src/components/NodeHandle.tsx
@@ -23,7 +23,7 @@ const NodeHandle: FC<HandleProps> = ({
    string: "text",
    number: "number",
    boolean: "true/false",
-    object: "complex",
+    object: "object",
    array: "list",
    null: "null",
  };
--- a/rnd/autogpt_builder/src/components/customnode.css
+++ b/rnd/autogpt_builder/src/components/customnode.css
@@ -26,7 +26,6 @@
  margin-bottom: 0px;
  padding: 5px;
  min-height: 44px;
-  width: 100%;
  height: 100%;
 }

--- a/rnd/autogpt_builder/src/components/node-input-components.tsx
+++ b/rnd/autogpt_builder/src/components/node-input-components.tsx
@@ -21,11 +21,14 @@ import {
  SelectValue,
 } from "./ui/select";
 import { Input } from "./ui/input";
+import NodeHandle from "./NodeHandle";
+import { ConnectionData } from "./CustomNode";

 type NodeObjectInputTreeProps = {
  selfKey?: string;
  schema: BlockIORootSchema | BlockIOObjectSubSchema;
  object?: { [key: string]: any };
+  connections: ConnectionData;
  handleInputClick: (key: string) => void;
  handleInputChange: (key: string, value: any) => void;
  errors: { [key: string]: string | undefined };
@@ -37,6 +40,7 @@ const NodeObjectInputTree: FC<NodeObjectInputTreeProps> = ({
  selfKey = "",
  schema,
  object,
+  connections,
  handleInputClick,
  handleInputChange,
  errors,
@@ -64,6 +68,7 @@ const NodeObjectInputTree: FC<NodeObjectInputTreeProps> = ({
              propSchema={propSchema}
              currentValue={object ? object[propKey] : undefined}
              errors={errors}
+              connections={connections}
              handleInputChange={handleInputChange}
              handleInputClick={handleInputClick}
              displayName={propSchema.title || beautifyString(propKey)}
@@ -82,6 +87,7 @@ export const NodeGenericInputField: FC<{
  propSchema: BlockIOSubSchema;
  currentValue?: any;
  errors: NodeObjectInputTreeProps["errors"];
+  connections: NodeObjectInputTreeProps["connections"];
  handleInputChange: NodeObjectInputTreeProps["handleInputChange"];
  handleInputClick: NodeObjectInputTreeProps["handleInputClick"];
  className?: string;
@@ -91,6 +97,7 @@ export const NodeGenericInputField: FC<{
  propSchema,
  currentValue,
  errors,
+  connections,
  handleInputChange,
  handleInputClick,
  className,
@@ -116,6 +123,7 @@ export const NodeGenericInputField: FC<{
        errors={errors}
        className={cn("border-l border-gray-500 pl-2", className)} // visual indent
        displayName={displayName}
+        connections={connections}
        handleInputClick={handleInputClick}
        handleInputChange={handleInputChange}
      />
@@ -131,6 +139,7 @@ export const NodeGenericInputField: FC<{
        errors={errors}
        className={className}
        displayName={displayName}
+        connections={connections}
        handleInputChange={handleInputChange}
      />
    );
@@ -230,10 +239,24 @@ export const NodeGenericInputField: FC<{
          errors={errors}
          className={className}
          displayName={displayName}
+          connections={connections}
          handleInputChange={handleInputChange}
          handleInputClick={handleInputClick}
        />
      );
+    case "object":
+      return (
+        <NodeKeyValueInput
+          selfKey={propKey}
+          schema={propSchema}
+          entries={currentValue}
+          errors={errors}
+          className={className}
+          displayName={displayName}
+          connections={connections}
+          handleInputChange={handleInputChange}
+        />
+      );
    default:
      console.warn(
        `Schema for '${propKey}' specifies unknown type:`,
@@ -259,6 +282,7 @@ const NodeKeyValueInput: FC<{
  schema: BlockIOKVSubSchema;
  entries?: { [key: string]: string } | { [key: string]: number };
  errors: { [key: string]: string | undefined };
+  connections: NodeObjectInputTreeProps["connections"];
  handleInputChange: NodeObjectInputTreeProps["handleInputChange"];
  className?: string;
  displayName?: string;
@@ -266,22 +290,29 @@ const NodeKeyValueInput: FC<{
  selfKey,
  entries,
  schema,
+  connections,
  handleInputChange,
  errors,
  className,
  displayName,
 }) => {
+  let defaultEntries = new Map<string, any>();
+  connections
+    .filter((c) => c.targetHandle.startsWith(`${selfKey}_`))
+    .forEach((c) => {
+      const key = c.targetHandle.slice(`${selfKey}_#_`.length);
+      defaultEntries.set(key, "");
+    });
+  Object.entries(entries ?? schema.default ?? {}).forEach(([key, value]) => {
+    defaultEntries.set(key, value);
+  });
+
  const [keyValuePairs, setKeyValuePairs] = useState<
    {
      key: string;
      value: string | number | null;
    }[]
-  >(
-    Object.entries(entries ?? schema.default ?? {}).map(([key, value]) => ({
-      key,
-      value: value,
-    })),
-  );
+  >(Array.from(defaultEntries, ([key, value]) => ({ key, value })));

  function updateKeyValuePairs(newPairs: typeof keyValuePairs) {
    setKeyValuePairs(newPairs);
@@ -292,54 +323,76 @@ const NodeKeyValueInput: FC<{
  }

  function convertValueType(value: string): string | number | null {
-    if (schema.additionalProperties.type == "string") return value;
+    if (
+      !schema.additionalProperties ||
+      schema.additionalProperties.type == "string"
+    )
+      return value;
    if (!value) return null;
    return Number(value);
  }

+  function getEntryKey(key: string): string {
+    return `${selfKey}_#_${key}`;
+  }
+  function isConnected(key: string): boolean {
+    return connections.some((c) => c.targetHandle === getEntryKey(key));
+  }
+
  return (
    <div className={cn(className, "flex flex-col")}>
      {displayName && <strong>{displayName}</strong>}
      <div>
        {keyValuePairs.map(({ key, value }, index) => (
          <div key={index}>
-            <div className="nodrag mb-2 flex items-center space-x-2">
-              <Input
-                type="text"
-                placeholder="Key"
-                value={key}
-                onChange={(e) =>
-                  updateKeyValuePairs(
-                    keyValuePairs.toSpliced(index, 1, {
-                      key: e.target.value,
-                      value: value,
-                    }),
-                  )
-                }
+            {key && (
+              <NodeHandle
+                keyName={getEntryKey(key)}
+                schema={{ type: "string" }}
+                isConnected={isConnected(key)}
+                isRequired={false}
+                side="left"
              />
-              <Input
-                type="text"
-                placeholder="Value"
-                value={value ?? ""}
-                onChange={(e) =>
-                  updateKeyValuePairs(
-                    keyValuePairs.toSpliced(index, 1, {
-                      key: key,
-                      value: convertValueType(e.target.value),
-                    }),
-                  )
-                }
-              />
-              <Button
-                variant="ghost"
-                className="px-2"
-                onClick={() =>
-                  updateKeyValuePairs(keyValuePairs.toSpliced(index, 1))
-                }
-              >
-                <Cross2Icon />
-              </Button>
-            </div>
+            )}
+            {!isConnected(key) && (
+              <div className="nodrag mb-2 flex items-center space-x-2">
+                <Input
+                  type="text"
+                  placeholder="Key"
+                  value={key}
+                  onChange={(e) =>
+                    updateKeyValuePairs(
+                      keyValuePairs.toSpliced(index, 1, {
+                        key: e.target.value,
+                        value: value,
+                      }),
+                    )
+                  }
+                />
+                <Input
+                  type="text"
+                  placeholder="Value"
+                  value={value ?? ""}
+                  onChange={(e) =>
+                    updateKeyValuePairs(
+                      keyValuePairs.toSpliced(index, 1, {
+                        key: key,
+                        value: convertValueType(e.target.value),
+                      }),
+                    )
+                  }
+                />
+                <Button
+                  variant="ghost"
+                  className="px-2"
+                  onClick={() =>
+                    updateKeyValuePairs(keyValuePairs.toSpliced(index, 1))
+                  }
+                >
+                  <Cross2Icon />
+                </Button>
+              </div>
+            )}
            {errors[`${selfKey}.${key}`] && (
              <span className="error-message">
                {errors[`${selfKey}.${key}`]}
@@ -368,6 +421,7 @@ const NodeArrayInput: FC<{
  schema: BlockIOArraySubSchema;
  entries?: string[];
  errors: { [key: string]: string | undefined };
+  connections: NodeObjectInputTreeProps["connections"];
  handleInputChange: NodeObjectInputTreeProps["handleInputChange"];
  handleInputClick: NodeObjectInputTreeProps["handleInputClick"];
  className?: string;
@@ -377,6 +431,7 @@ const NodeArrayInput: FC<{
  schema,
  entries,
  errors,
+  connections,
  handleInputChange,
  handleInputClick,
  className,
@@ -390,39 +445,52 @@ const NodeArrayInput: FC<{
    <div className={cn(className, "flex flex-col")}>
      {displayName && <strong>{displayName}</strong>}
      {entries.map((entry: any, index: number) => {
-        const entryKey = `${selfKey}[${index}]`;
+        const entryKey = `${selfKey}_$_${index}`;
+        const isConnected =
+          connections && connections.some((c) => c.targetHandle === entryKey);
        return (
-          <div key={entryKey}>
-            <div className="mb-2 flex items-center space-x-2">
-              {schema.items ? (
-                <NodeGenericInputField
-                  propKey={entryKey}
-                  propSchema={schema.items}
-                  currentValue={entry}
-                  errors={errors}
-                  handleInputChange={handleInputChange}
-                  handleInputClick={handleInputClick}
-                />
-              ) : (
-                <NodeFallbackInput
-                  selfKey={entryKey}
-                  schema={schema.items}
-                  value={entry}
-                  error={errors[entryKey]}
-                  displayName={displayName || beautifyString(selfKey)}
-                  handleInputChange={handleInputChange}
-                  handleInputClick={handleInputClick}
-                />
+          <div key={entryKey} className="self-start">
+            <div className="mb-2 flex space-x-2">
+              <NodeHandle
+                keyName={entryKey}
+                schema={schema.items!}
+                isConnected={isConnected}
+                isRequired={false}
+                side="left"
+              />
+              {!isConnected &&
+                (schema.items ? (
+                  <NodeGenericInputField
+                    propKey={entryKey}
+                    propSchema={schema.items}
+                    currentValue={entry}
+                    errors={errors}
+                    connections={connections}
+                    handleInputChange={handleInputChange}
+                    handleInputClick={handleInputClick}
+                  />
+                ) : (
+                  <NodeFallbackInput
+                    selfKey={entryKey}
+                    schema={schema.items}
+                    value={entry}
+                    error={errors[entryKey]}
+                    displayName={displayName || beautifyString(selfKey)}
+                    handleInputChange={handleInputChange}
+                    handleInputClick={handleInputClick}
+                  />
+                ))}
+              {!isConnected && (
+                <Button
+                  variant="ghost"
+                  size="icon"
+                  onClick={() =>
+                    handleInputChange(selfKey, entries.toSpliced(index, 1))
+                  }
+                >
+                  <Cross2Icon />
+                </Button>
              )}
-              <Button
-                variant="ghost"
-                size="icon"
-                onClick={() =>
-                  handleInputChange(selfKey, entries.toSpliced(index, 1))
-                }
-              >
-                <Cross2Icon />
-              </Button>
            </div>
            {errors[entryKey] && typeof errors[entryKey] === "string" && (
              <span className="error-message">{errors[entryKey]}</span>
--- a/rnd/autogpt_server/autogpt_server/blocks/text.py
+++ b/rnd/autogpt_server/autogpt_server/blocks/text.py
@@ -115,13 +115,8 @@ class TextParserBlock(Block):

 class TextFormatterBlock(Block):
    class Input(BlockSchema):
-        texts: list[Any] = Field(description="Texts (list) to format", default=[])
-        named_texts: dict[str, Any] = Field(
-            description="Texts (dict) to format", default={}
-        )
-        format: str = Field(
-            description="Template to format the text using `texts` and `named_texts`",
-        )
+        values: dict[str, Any] = Field(description="Values (dict) to be used in format")
+        format: str = Field(description="Template to format the text using `values`")

    class Output(BlockSchema):
        output: str
@@ -134,39 +129,27 @@ class TextFormatterBlock(Block):
            input_schema=TextFormatterBlock.Input,
            output_schema=TextFormatterBlock.Output,
            test_input=[
-                {"texts": ["Hello"], "format": "{texts[0]}"},
                {
-                    "texts": ["Hello", "World!"],
-                    "named_texts": {"name": "Alice"},
-                    "format": "{texts[0]} {texts[1]} {name}",
+                    "values": {"name": "Alice", "hello": "Hello", "world": "World!"},
+                    "format": "{hello}, {world} {name}",
                },
-                {"format": "Hello, World!"},
            ],
            test_output=[
-                ("output", "Hello"),
-                ("output", "Hello World! Alice"),
-                ("output", "Hello, World!"),
+                ("output", "Hello, World! Alice"),
            ],
        )

    def run(self, input_data: Input) -> BlockOutput:
-        texts = [
-            text if isinstance(text, str) else json.dumps(text)
-            for text in input_data.texts
-        ]
-        named_texts = {
+        values = {
            key: value if isinstance(value, str) else json.dumps(value)
-            for key, value in input_data.named_texts.items()
+            for key, value in input_data.values.items()
        }
-        yield "output", input_data.format.format(texts=texts, **named_texts)
+        yield "output", input_data.format.format(**values)


 class TextCombinerBlock(Block):
    class Input(BlockSchema):
-        input1: str = Field(description="First text input", default="")
-        input2: str = Field(description="Second text input", default="")
-        input3: str = Field(description="Second text input", default="")
-        input4: str = Field(description="Second text input", default="")
+        input: list[str] = Field(description="text input to combine")
        delimiter: str = Field(description="Delimiter to combine texts", default="")

    class Output(BlockSchema):
@@ -180,24 +163,15 @@ class TextCombinerBlock(Block):
            input_schema=TextCombinerBlock.Input,
            output_schema=TextCombinerBlock.Output,
            test_input=[
-                {"input1": "Hello world I like ", "input2": "cake and to go for walks"},
-                {"input1": "This is a test. ", "input2": "Let's see how it works."},
+                {"input": ["Hello world I like ", "cake and to go for walks"]},
+                {"input": ["This is a test", "Hi!"], "delimiter": "! "},
            ],
            test_output=[
                ("output", "Hello world I like cake and to go for walks"),
-                ("output", "This is a test. Let's see how it works."),
+                ("output", "This is a test! Hi!"),
            ],
        )

    def run(self, input_data: Input) -> BlockOutput:
-        combined_text = input_data.delimiter.join(
-            text
-            for text in [
-                input_data.input1,
-                input_data.input2,
-                input_data.input3,
-                input_data.input4,
-            ]
-            if text
-        )
+        combined_text = input_data.delimiter.join(input_data.input)
        yield "output", combined_text
--- a/rnd/autogpt_server/autogpt_server/data/execution.py
+++ b/rnd/autogpt_server/autogpt_server/data/execution.py
@@ -13,7 +13,7 @@ from prisma.types import AgentGraphExecutionWhereInput
 from pydantic import BaseModel

 from autogpt_server.data.block import BlockData, BlockInput, CompletedBlockOutput
-from autogpt_server.util import json
+from autogpt_server.util import json, mock


 class GraphExecution(BaseModel):
@@ -363,8 +363,8 @@ def merge_execution_input(data: BlockInput) -> BlockInput:
        if OBJC_SPLIT not in key:
            continue
        name, index = key.split(OBJC_SPLIT)
-        if not isinstance(data[name], object):
-            data[name] = type("Object", (object,), data[name])()
+        if name not in data or not isinstance(data[name], object):
+            data[name] = mock.MockObject()
        setattr(data[name], index, value)

    return data
--- a/rnd/autogpt_server/autogpt_server/usecases/block_autogen.py
+++ b/rnd/autogpt_server/autogpt_server/usecases/block_autogen.py
@@ -97,7 +97,7 @@ Here is the information I get to write a Python code for that:
 Here is your previous attempt:
 {previous_attempt}
 """,
-            "named_texts_#_previous_attempt": "No previous attempt found.",
+            "values_#_previous_attempt": "No previous attempt found.",
        },
    )
    code_gen_llm_call = Node(
@@ -162,7 +162,7 @@ Here are a couple of sample of the Block class implementation:
            source_id=input_data.id,
            sink_id=input_text_formatter.id,
            source_name="output",
-            sink_name="named_texts_#_query",
+            sink_name="values_#_query",
        ),
        Link(
            source_id=input_query_constant.id,
@@ -192,13 +192,13 @@ Here are a couple of sample of the Block class implementation:
            source_id=search_result_constant.id,
            sink_id=prompt_text_formatter.id,
            source_name="output",
-            sink_name="named_texts_#_search_result",
+            sink_name="values_#_search_result",
        ),
        Link(
            source_id=input_query_constant.id,
            sink_id=prompt_text_formatter.id,
            source_name="output",
-            sink_name="named_texts_#_query",
+            sink_name="values_#_query",
        ),
        Link(
            source_id=prompt_text_formatter.id,
@@ -222,7 +222,7 @@ Here are a couple of sample of the Block class implementation:
            source_id=block_installation.id,
            sink_id=prompt_text_formatter.id,
            source_name="error",
-            sink_name="named_texts_#_previous_attempt",
+            sink_name="values_#_previous_attempt",
        ),
        Link(  # Re-trigger search result.
            source_id=block_installation.id,
--- a/rnd/autogpt_server/autogpt_server/usecases/reddit_marketing.py
+++ b/rnd/autogpt_server/autogpt_server/usecases/reddit_marketing.py
@@ -95,7 +95,7 @@ Make sure to only comment on a relevant post.
            source_id=reddit_get_post_node.id,
            sink_id=text_formatter_node.id,
            source_name="post",
-            sink_name="named_texts",
+            sink_name="values",
        ),
        Link(
            source_id=text_formatter_node.id,
--- a/rnd/autogpt_server/autogpt_server/usecases/sample.py
+++ b/rnd/autogpt_server/autogpt_server/usecases/sample.py
@@ -11,7 +11,7 @@ from autogpt_server.util.test import SpinTestServer, wait_execution
 async def create_test_user() -> User:
    test_user_data = {
        "sub": "ef3b97d7-1161-4eb4-92b2-10c24fb154c1",
-        "email": "testuser@example.com",
+        "email": "testuser#example.com",
        "name": "Test User",
    }
    user = await get_or_create_user(test_user_data)
@@ -38,8 +38,8 @@ def create_test_graph() -> graph.Graph:
        graph.Node(
            block_id=TextFormatterBlock().id,
            input_default={
-                "format": "{texts[0]}, {texts[1]}{texts[2]}",
-                "texts_$_3": "!!!",
+                "format": "{a}, {b}{c}",
+                "values_#_c": "!!!",
            },
        ),
        graph.Node(block_id=PrintingBlock().id),
@@ -49,13 +49,13 @@ def create_test_graph() -> graph.Graph:
            source_id=nodes[0].id,
            sink_id=nodes[2].id,
            source_name="output",
-            sink_name="texts_$_1",
+            sink_name="values_#_a",
        ),
        graph.Link(
            source_id=nodes[1].id,
            sink_id=nodes[2].id,
            source_name="output",
-            sink_name="texts_$_2",
+            sink_name="values_#_b",
        ),
        graph.Link(
            source_id=nodes[2].id,
--- a/rnd/autogpt_server/test/executor/test_manager.py
+++ b/rnd/autogpt_server/test/executor/test_manager.py
@@ -62,11 +62,11 @@ async def assert_sample_graph_executions(
    assert exec.graph_exec_id == graph_exec_id
    assert exec.output_data == {"output": ["Hello, World!!!"]}
    assert exec.input_data == {
-        "format": "{texts[0]}, {texts[1]}{texts[2]}",
-        "texts": ["Hello", "World", "!!!"],
-        "texts_$_1": "Hello",
-        "texts_$_2": "World",
-        "texts_$_3": "!!!",
+        "format": "{a}, {b}{c}",
+        "values": {"a": "Hello", "b": "World", "c": "!!!"},
+        "values_#_a": "Hello",
+        "values_#_b": "World",
+        "values_#_c": "!!!",
    }
    assert exec.node_id == test_graph.nodes[2].id