feat: Add VideoRendererBlock for rendering videos

This commit adds a new `VideoRendererBlock` component to the `autogpt_builder` module. The `VideoRendererBlock` is responsible for rendering videos based on the provided video URL. It supports both YouTube videos and direct video URLs. The `CustomNode` component in the `CustomNode.tsx` file has been updated to handle the new `VideoRendererBlock` component. When the `block_id` matches the ID of the `VideoRendererBlock`, the video is rendered using an iframe for YouTube videos or a video tag for direct video URLs. This change improves the rendering capabilities of the application by allowing the display of videos within the UI.
Ensures the full artifact is always visible but isolated.
2026-01-14 01:28:11 -05:00 · 2024-08-14 11:29:44 +01:00 · 2024-08-11 13:26:32 +01:00 · 2024-08-10 19:42:25 +01:00
5 changed files with 280 additions and 25 deletions
--- a/rnd/autogpt_builder/src/components/ArtifactRenderer.tsx
+++ b/rnd/autogpt_builder/src/components/ArtifactRenderer.tsx
@@ -0,0 +1,106 @@
+import React, { useEffect, useRef, useState } from 'react';
+import ReactMarkdown from 'react-markdown';
+import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { tomorrow } from 'react-syntax-highlighter/dist/esm/styles/prism';
+
+const ArtifactRenderer = ({ artifactData }) => {
+  const iframeRef = useRef(null);
+  const [iframeHeight, setIframeHeight] = useState('300px');
+
+  useEffect(() => {
+    const data = Array.isArray(artifactData) && artifactData.length > 0 ? artifactData[0] : artifactData;
+    if (data && (data.type === 'image/svg+xml' || data.type === 'text/html')) {
+      const resizeIframe = () => {
+        if (iframeRef.current && iframeRef.current.contentWindow) {
+          const height = iframeRef.current.contentWindow.document.body.scrollHeight;
+          setIframeHeight(`${height + 20}px`); // Add a small buffer
+        }
+      };
+
+      // Resize on load and after a short delay (for any dynamic content)
+      if (iframeRef.current) {
+        iframeRef.current.onload = resizeIframe;
+        setTimeout(resizeIframe, 100);
+      }
+    }
+  }, [artifactData]);
+
+  if (!artifactData) {
+    console.error("No artifact data received");
+    return <div>No artifact data received</div>;
+  }
+
+  const data = Array.isArray(artifactData) && artifactData.length > 0 ? artifactData[0] : artifactData;
+
+  if (!data || typeof data !== 'object') {
+    console.error("Invalid artifact data structure:", artifactData);
+    return <div>Error: Invalid artifact data structure</div>;
+  }
+
+  const { type, title, content, language } = data;
+
+  if (!type) {
+    console.error("Artifact type is missing:", data);
+    return <div>Error: Artifact type is missing</div>;
+  }
+
+  const renderContent = () => {
+    switch (type) {
+      case 'image/png':
+      case 'image/jpeg':
+      case 'image/gif':
+        return <img src={content} alt={title} style={{ maxWidth: '100%', height: 'auto' }} />;
+      case 'application/vnd.agpt.code':
+        return (
+          <SyntaxHighlighter language={language || 'text'} style={tomorrow}>
+            {content}
+          </SyntaxHighlighter>
+        );
+      case 'text/markdown':
+        return <ReactMarkdown>{content}</ReactMarkdown>;
+      case 'text/html':
+      case 'image/svg+xml':
+        return (
+          <iframe
+            ref={iframeRef}
+            srcDoc={`
+              <html>
+                <head>
+                  <base target="_blank">
+                  <style>
+                    body {
+                      margin: 0;
+                      padding: 0;
+                      overflow: hidden;
+                    }
+                    svg, img {
+                      max-width: 100%;
+                      height: auto;
+                    }
+                  </style>
+                </head>
+                <body>${content}</body>
+              </html>
+            `}
+            style={{
+              width: '100%',
+              height: iframeHeight,
+              border: 'none',
+              overflow: 'hidden'
+            }}
+          />
+        );
+      default:
+        return <p>Unsupported artifact type: {type}</p>;
+    }
+  };
+
+  return (
+    <div className="artifact-renderer" style={{ width: '100%', overflow: 'hidden' }}>
+      <h3>{title || 'Untitled Artifact'}</h3>
+      {renderContent()}
+    </div>
+  );
+};
+
+export default ArtifactRenderer;
--- a/rnd/autogpt_builder/src/components/CustomNode.tsx
+++ b/rnd/autogpt_builder/src/components/CustomNode.tsx
@@ -23,6 +23,8 @@ import { history } from "./history";
 import NodeHandle from "./NodeHandle";
 import { CustomEdgeData } from "./CustomEdge";
 import { NodeGenericInputField } from "./node-input-components";
+import ArtifactRenderer from './ArtifactRenderer';
+import VideoRendererBlock from './VideoRendererBlock';

 type ParsedKey = { key: string; index?: number };

@@ -275,6 +277,44 @@ const CustomNode: FC<NodeProps<CustomNodeData>> = ({ data, id }) => {
    console.log("Copy node:", id);
  }, [id]);

+  const renderOutput = () => {
+    console.log("CustomNode renderOutput, full data:", data);
+    console.log("CustomNode renderOutput, output_data:", data.output_data);
+    
+    switch (data.block_id) {
+      case "7a8b9c0d-1e2f-3g4h-5i6j-7k8l9m0n1o2p":
+        return <ArtifactRenderer artifactData={data.output_data.artifact_data} />;
+      case "a92a0017-2390-425f-b5a8-fb3c50c81400":
+        return <VideoRendererBlock data={data} />;
+      default:
+        return (
+          <div className="node-output" onClick={handleOutputClick}>
+            <p>
+              <strong>Status:</strong>{" "}
+              {typeof data.status === "object"
+                ? JSON.stringify(data.status)
+                : data.status || "N/A"}
+            </p>
+            <p>
+              <strong>Output Data:</strong>{" "}
+              {(() => {
+                const outputText =
+                  typeof data.output_data === "object"
+                    ? JSON.stringify(data.output_data)
+                    : data.output_data;
+
+                if (!outputText) return "No output data";
+
+                return outputText.length > 100
+                  ? `${outputText.slice(0, 100)}... Press To Read More`
+                  : outputText;
+              })()}
+            </p>
+          </div>
+        );
+      }
+  };
+
  return (
    <div
      className={`custom-node dark-theme border rounded-xl shandow-md bg-white/[.8] ${data.status?.toLowerCase() ?? ""}`}
@@ -349,31 +389,7 @@ const CustomNode: FC<NodeProps<CustomNodeData>> = ({ data, id }) => {
          {data.outputSchema && generateOutputHandles(data.outputSchema)}
        </div>
      </div>
-      {isOutputOpen && (
-        <div className="node-output" onClick={handleOutputClick}>
-          <p>
-            <strong>Status:</strong>{" "}
-            {typeof data.status === "object"
-              ? JSON.stringify(data.status)
-              : data.status || "N/A"}
-          </p>
-          <p>
-            <strong>Output Data:</strong>{" "}
-            {(() => {
-              const outputText =
-                typeof data.output_data === "object"
-                  ? JSON.stringify(data.output_data)
-                  : data.output_data;
-
-              if (!outputText) return "No output data";
-
-              return outputText.length > 100
-                ? `${outputText.slice(0, 100)}... Press To Read More`
-                : outputText;
-            })()}
-          </p>
-        </div>
-      )}
+      {isOutputOpen && renderOutput()}
      <div className="flex items-center pl-4 pb-4 mt-2.5">
        <Switch onCheckedChange={toggleOutput} />
        <span className="m-1 mr-4">Output</span>
--- a/rnd/autogpt_builder/src/components/VideoRendererBlock.tsx
+++ b/rnd/autogpt_builder/src/components/VideoRendererBlock.tsx
@@ -0,0 +1,45 @@
+import React from 'react';
+
+interface VideoRendererProps {
+  data: any;
+}
+
+const VideoRendererBlock: React.FC<VideoRendererProps> = ({ data }) => {
+  // Extract video URL from the correct location in the data structure
+  const videoUrl = data.hardcodedValues?.video_url || 
+                   (Array.isArray(data.output_data?.video_url) && data.output_data.video_url[0]);
+
+  if (!videoUrl || typeof videoUrl !== 'string') {
+    return <div>Invalid or missing video URL</div>;
+  }
+
+  const getYouTubeVideoId = (url: string) => {
+    const regExp = /^.*((youtu.be\/)|(v\/)|(\/u\/\w\/)|(embed\/)|(watch\?))\??v?=?([^#&?]*).*/;
+    const match = url.match(regExp);
+    return (match && match[7].length === 11) ? match[7] : null;
+  };
+
+  const videoId = getYouTubeVideoId(videoUrl);
+
+  return (
+    <div style={{ width: '100%', padding: '10px' }}>
+      {videoId ? (
+        <iframe
+          width="100%"
+          height="315"
+          src={`https://www.youtube.com/embed/${videoId}`}
+          frameBorder="0"
+          allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+          allowFullScreen
+        ></iframe>
+      ) : (
+        <video controls width="100%" height="315">
+          <source src={videoUrl} type="video/mp4" />
+          Your browser does not support the video tag.
+        </video>
+      )}
+    </div>
+  );
+};
+
+export default VideoRendererBlock;
--- a/rnd/autogpt_server/autogpt_server/blocks/artifact_renderer_block.py
+++ b/rnd/autogpt_server/autogpt_server/blocks/artifact_renderer_block.py
@@ -0,0 +1,68 @@
+from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from autogpt_server.data.model import SchemaField
+import re
+import base64
+import markdown
+
+class ArtifactRendererBlock(Block):
+    class Input(BlockSchema):
+        artifact_string: str = SchemaField(description="The input string containing an AGPT artifact to be rendered.")
+
+    class Output(BlockSchema):
+        artifact_data: dict = SchemaField(description="Processed artifact data for frontend rendering.")
+
+    def __init__(self):
+        super().__init__(
+            id="7a8b9c0d-1e2f-3g4h-5i6j-7k8l9m0n1o2p",
+            description="Processes an AGPT artifact for visual rendering within the block.",
+            categories={BlockCategory.TEXT, BlockCategory.BASIC},
+            input_schema=ArtifactRendererBlock.Input,
+            output_schema=ArtifactRendererBlock.Output,
+        )
+        
+    def parse_artifact(self, artifact_string):
+        pattern = r'<agptArtifact\s+(.*?)>(.*?)</agptArtifact>'
+        match = re.search(pattern, artifact_string, re.DOTALL)
+        if match:
+            attributes = dict(re.findall(r'(\w+)="([^"]*)"', match.group(1)))
+            content = match.group(2).strip()
+            return attributes, content
+        return None, None
+
+    def process_artifact(self, attributes, content):
+        artifact_type = attributes.get('type', '')
+        title = attributes.get('title', 'Untitled Artifact')
+        identifier = attributes.get('identifier', '')
+        language = attributes.get('language', '')
+
+        processed_data = {
+            'type': artifact_type,
+            'title': title,
+            'identifier': identifier,
+            'language': language,
+            'content': content
+        }
+
+        if artifact_type.startswith('image/'):
+            processed_data['content'] = f"data:{artifact_type};base64,{content}"
+        elif artifact_type == 'text/markdown':
+            # Send markdown as plain text, don't convert to HTML
+            processed_data['content'] = content
+        elif artifact_type == 'application/vnd.agpt.code':
+            # Keep the content as is for code snippets
+            pass
+        elif artifact_type == 'text/html' or artifact_type == 'image/svg+xml':
+            # Keep HTML and SVG content as is
+            pass
+        else:
+            processed_data['content'] = content
+
+        return processed_data
+
+    def run(self, input_data: Input) -> BlockOutput:
+        attributes, content = self.parse_artifact(input_data.artifact_string)
+        if attributes and content:
+            processed_data = self.process_artifact(attributes, content)
+            yield "artifact_data", processed_data
+        else:
+            yield "artifact_data", {"error": "Invalid artifact format"}
--- a/rnd/autogpt_server/autogpt_server/blocks/videorendererblock.py
+++ b/rnd/autogpt_server/autogpt_server/blocks/videorendererblock.py
@@ -0,0 +1,20 @@
+from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from autogpt_server.data.model import SchemaField
+
+class VideoRendererBlock(Block):
+    class Input(BlockSchema):
+        video_url: str = SchemaField(description="The URL of the video to be rendered.")
+
+    class Output(BlockSchema):
+        video_url: str = SchemaField(description="The URL of the video to be rendered.")
+
+    def __init__(self):
+        super().__init__(
+            id="a92a0017-2390-425f-b5a8-fb3c50c81400",
+            description="Renders a video from a given URL within the block.",
+            input_schema=VideoRendererBlock.Input,
+            output_schema=VideoRendererBlock.Output
+        )
+
+    def run(self, input_data: Input) -> BlockOutput:
+        yield "video_url", input_data.video_url