Merge branch 'dev' into redesigning-block-menu

Request on block execution can be throttled, and requests between
services can sometimes break. The scope of this PR is to add an
appropriate retry on those.

### Changes 🏗️

* Block request retry: Retry on throttled status code only (504, 429,
etc).
* RPC request retry: Retry connection issues (ConnectError, Timeout,
etc).
* Truncate logging on executor/utils.

### Checklist 📋

#### For code changes:
- [x] I have clearly listed my changes in the PR description
- [x] I have made a test plan
- [x] I have tested my changes according to the test plan:
  <!-- Put your test plan here: -->
  - [x] Manual graph execution

.deepsource.toml

@@ -0,0 +1,18 @@
+version = 1
+
+test_patterns = ["**/*.spec.ts","**/*_test.py","**/*_tests.py","**/test_*.py"]
+
+exclude_patterns = ["classic/**"]
+
+[[analyzers]]
+name = "javascript"
+
+[analyzers.meta]
+plugins = ["react"]
+environment = ["nodejs"]
+
+[[analyzers]]
+name = "python"
+
+[analyzers.meta]
+runtime_version = "3.x.x"

.dockerignore

@@ -1,40 +1,61 @@
 # Ignore everything by default, selectively add things to context
-classic/run
+*
 
-# AutoGPT
+# Platform - Libs
+!autogpt_platform/autogpt_libs/autogpt_libs/
+!autogpt_platform/autogpt_libs/pyproject.toml
+!autogpt_platform/autogpt_libs/poetry.lock
+!autogpt_platform/autogpt_libs/README.md
+
+# Platform - Backend
+!autogpt_platform/backend/backend/
+!autogpt_platform/backend/migrations/
+!autogpt_platform/backend/schema.prisma
+!autogpt_platform/backend/pyproject.toml
+!autogpt_platform/backend/poetry.lock
+!autogpt_platform/backend/README.md
+
+# Platform - Market
+!autogpt_platform/market/market/
+!autogpt_platform/market/scripts.py
+!autogpt_platform/market/schema.prisma
+!autogpt_platform/market/pyproject.toml
+!autogpt_platform/market/poetry.lock
+!autogpt_platform/market/README.md
+
+# Platform - Frontend
+!autogpt_platform/frontend/src/
+!autogpt_platform/frontend/public/
+!autogpt_platform/frontend/package.json
+!autogpt_platform/frontend/pnpm-lock.yaml
+!autogpt_platform/frontend/tsconfig.json
+!autogpt_platform/frontend/README.md
+## config
+!autogpt_platform/frontend/*.config.*
+!autogpt_platform/frontend/.env.*
+
+# Classic - AutoGPT
 !classic/original_autogpt/autogpt/
 !classic/original_autogpt/pyproject.toml
 !classic/original_autogpt/poetry.lock
 !classic/original_autogpt/README.md
 !classic/original_autogpt/tests/
 
-# Benchmark
+# Classic - Benchmark
 !classic/benchmark/agbenchmark/
 !classic/benchmark/pyproject.toml
 !classic/benchmark/poetry.lock
 !classic/benchmark/README.md
 
-# Forge
+# Classic - Forge
 !classic/forge/
 !classic/forge/pyproject.toml
 !classic/forge/poetry.lock
 !classic/forge/README.md
 
-# Frontend
+# Classic - Frontend
 !classic/frontend/build/web/
 
-# Platform
-!autogpt_platform/
-
 # Explicitly re-ignore some folders
 .*
 **/__pycache__
-
-autogpt_platform/frontend/.next/
-autogpt_platform/frontend/node_modules
-autogpt_platform/frontend/.env.example
-autogpt_platform/frontend/.env.local
-autogpt_platform/backend/.env
-autogpt_platform/backend/.venv/
-
-autogpt_platform/market/.env

.github/dependabot.yml

@@ -7,17 +7,22 @@ updates:
       interval: "weekly"
     open-pull-requests-limit: 10
     target-branch: "dev"
+    commit-message:
+      prefix: "chore(libs/deps)"
+      prefix-development: "chore(libs/deps-dev)"
+    ignore:
+      - dependency-name: "poetry"
     groups:
       production-dependencies:
         dependency-type: "production"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
       development-dependencies:
         dependency-type: "development"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
 
   # backend (Poetry project)
   - package-ecosystem: "pip"
@@ -26,18 +31,22 @@ updates:
       interval: "weekly"
     open-pull-requests-limit: 10
     target-branch: "dev"
+    commit-message:
+      prefix: "chore(backend/deps)"
+      prefix-development: "chore(backend/deps-dev)"
+    ignore:
+      - dependency-name: "poetry"
     groups:
       production-dependencies:
         dependency-type: "production"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
       development-dependencies:
         dependency-type: "development"
         update-types:
-        - "minor"
-        - "patch"
-
+          - "minor"
+          - "patch"
 
   # frontend (Next.js project)
   - package-ecosystem: "npm"
@@ -46,18 +55,20 @@ updates:
       interval: "weekly"
     open-pull-requests-limit: 10
     target-branch: "dev"
+    commit-message:
+      prefix: "chore(frontend/deps)"
+      prefix-development: "chore(frontend/deps-dev)"
     groups:
       production-dependencies:
         dependency-type: "production"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
       development-dependencies:
         dependency-type: "development"
         update-types:
-        - "minor"
-        - "patch"
-
+          - "minor"
+          - "patch"
 
   # infra (Terraform)
   - package-ecosystem: "terraform"
@@ -66,38 +77,21 @@ updates:
       interval: "weekly"
     open-pull-requests-limit: 5
     target-branch: "dev"
+    commit-message:
+      prefix: "chore(infra/deps)"
+      prefix-development: "chore(infra/deps-dev)"
+
     groups:
       production-dependencies:
         dependency-type: "production"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
       development-dependencies:
         dependency-type: "development"
         update-types:
-        - "minor"
-        - "patch"
-
-
-  # market (Poetry project)
-  - package-ecosystem: "pip"
-    directory: "autogpt_platform/market"
-    schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 10
-    target-branch: "dev"
-    groups:
-      production-dependencies:
-        dependency-type: "production"
-        update-types:
-        - "minor"
-        - "patch"
-      development-dependencies:
-        dependency-type: "development"
-        update-types:
-        - "minor"
-        - "patch"
-
+          - "minor"
+          - "patch"
 
   # GitHub Actions
   - package-ecosystem: "github-actions"
@@ -110,14 +104,13 @@ updates:
       production-dependencies:
         dependency-type: "production"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
       development-dependencies:
         dependency-type: "development"
         update-types:
-        - "minor"
-        - "patch"
-
+          - "minor"
+          - "patch"
 
   # Docker
   - package-ecosystem: "docker"
@@ -130,50 +123,31 @@ updates:
       production-dependencies:
         dependency-type: "production"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
       development-dependencies:
         dependency-type: "development"
         update-types:
-        - "minor"
-        - "patch"
-
-
-  # Submodules
-  - package-ecosystem: "gitsubmodule"
-    directory: "autogpt_platform/supabase"
-    schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 1
-    target-branch: "dev"
-    groups:
-      production-dependencies:
-        dependency-type: "production"
-        update-types:
-        - "minor"
-        - "patch"
-      development-dependencies:
-        dependency-type: "development"
-        update-types:
-        - "minor"
-        - "patch"
-
+          - "minor"
+          - "patch"
 
   # Docs
-  - package-ecosystem: 'pip'
+  - package-ecosystem: "pip"
     directory: "docs/"
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 1
     target-branch: "dev"
+    commit-message:
+      prefix: "chore(docs/deps)"
     groups:
       production-dependencies:
         dependency-type: "production"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"
       development-dependencies:
         dependency-type: "development"
         update-types:
-        - "minor"
-        - "patch"
+          - "minor"
+          - "patch"

.github/labeler.yml

@@ -24,8 +24,9 @@ platform/frontend:
 
 platform/backend:
 - changed-files:
-  - any-glob-to-any-file: autogpt_platform/backend/**
-  - all-globs-to-all-files: '!autogpt_platform/backend/backend/blocks/**'
+  - all-globs-to-any-file:
+    - autogpt_platform/backend/**
+    - '!autogpt_platform/backend/backend/blocks/**'
 
 platform/blocks:
 - changed-files:

.github/workflows/classic-autogpt-ci.yml

@@ -115,6 +115,7 @@ jobs:
           poetry run pytest -vv \
             --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \
             --numprocesses=logical --durations=10 \
+            --junitxml=junit.xml -o junit_family=legacy \
             tests/unit tests/integration
         env:
           CI: true
@@ -124,8 +125,14 @@ jobs:
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}  # Run even if tests fail
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: autogpt-agent,${{ runner.os }}

.github/workflows/classic-benchmark-ci.yml

@@ -87,13 +87,20 @@ jobs:
           poetry run pytest -vv \
             --cov=agbenchmark --cov-branch --cov-report term-missing --cov-report xml \
             --durations=10 \
+            --junitxml=junit.xml -o junit_family=legacy \
             tests
         env:
           CI: true
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}  # Run even if tests fail
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: agbenchmark,${{ runner.os }}

.github/workflows/classic-forge-ci.yml

@@ -139,6 +139,7 @@ jobs:
           poetry run pytest -vv \
             --cov=forge --cov-branch --cov-report term-missing --cov-report xml \
             --durations=10 \
+            --junitxml=junit.xml -o junit_family=legacy \
             forge
         env:
           CI: true
@@ -148,8 +149,14 @@ jobs:
           AWS_ACCESS_KEY_ID: minioadmin
           AWS_SECRET_ACCESS_KEY: minioadmin
 
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}  # Run even if tests fail
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: forge,${{ runner.os }}

.github/workflows/claude.yml

@@ -0,0 +1,47 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (
+        (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+        (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+        (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+        (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+      ) && (
+        github.event.comment.author_association == 'OWNER' ||
+        github.event.comment.author_association == 'MEMBER' ||
+        github.event.comment.author_association == 'COLLABORATOR' ||
+        github.event.review.author_association == 'OWNER' ||
+        github.event.review.author_association == 'MEMBER' ||
+        github.event.review.author_association == 'COLLABORATOR' ||
+        github.event.issue.author_association == 'OWNER' ||
+        github.event.issue.author_association == 'MEMBER' ||
+        github.event.issue.author_association == 'COLLABORATOR'
+      )
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@beta
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

.github/workflows/platform-autgpt-deploy-prod.yml

@@ -34,13 +34,8 @@ jobs:
           python -m prisma migrate deploy
         env:
           DATABASE_URL: ${{ secrets.BACKEND_DATABASE_URL }}
+          DIRECT_URL: ${{ secrets.BACKEND_DATABASE_URL }}
 
-      - name: Run Market Migrations
-        working-directory: ./autogpt_platform/market
-        run: |
-          python -m prisma migrate deploy
-        env:
-          DATABASE_URL: ${{ secrets.MARKET_DATABASE_URL }}
           
   trigger:
     needs: migrate

.github/workflows/platform-autogpt-deploy-dev.yaml

@@ -36,13 +36,7 @@ jobs:
           python -m prisma migrate deploy
         env:
           DATABASE_URL: ${{ secrets.BACKEND_DATABASE_URL }}
-
-      - name: Run Market Migrations
-        working-directory: ./autogpt_platform/market
-        run: |
-          python -m prisma migrate deploy
-        env:
-          DATABASE_URL: ${{ secrets.MARKET_DATABASE_URL }}
+          DIRECT_URL: ${{ secrets.BACKEND_DATABASE_URL }}
 
   trigger:
     needs: migrate

.github/workflows/platform-backend-ci.yml

@@ -6,11 +6,13 @@ on:
     paths:
       - ".github/workflows/platform-backend-ci.yml"
       - "autogpt_platform/backend/**"
+      - "autogpt_platform/autogpt_libs/**"
   pull_request:
     branches: [master, dev, release-*]
     paths:
       - ".github/workflows/platform-backend-ci.yml"
       - "autogpt_platform/backend/**"
+      - "autogpt_platform/autogpt_libs/**"
   merge_group:
 
 concurrency:
@@ -30,7 +32,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
     runs-on: ubuntu-latest
 
     services:
@@ -40,6 +42,14 @@ jobs:
           REDIS_PASSWORD: testpassword
         ports:
           - 6379:6379
+      rabbitmq:
+        image: rabbitmq:3.12-management
+        ports:
+          - 5672:5672
+          - 15672:15672
+        env:
+          RABBITMQ_DEFAULT_USER: ${{ env.RABBITMQ_DEFAULT_USER }}
+          RABBITMQ_DEFAULT_PASS: ${{ env.RABBITMQ_DEFAULT_PASS }}
 
     steps:
       - name: Checkout repository
@@ -56,7 +66,7 @@ jobs:
       - name: Setup Supabase
         uses: supabase/setup-cli@v1
         with:
-          version: latest
+          version: 1.178.1
 
       - id: get_date
         name: Get date
@@ -70,12 +80,40 @@ jobs:
 
       - name: Install Poetry (Unix)
         run: |
-          curl -sSL https://install.python-poetry.org | python3 -
+          # Extract Poetry version from backend/poetry.lock
+          HEAD_POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry)
+          echo "Found Poetry version ${HEAD_POETRY_VERSION} in backend/poetry.lock"
+
+          if [ -n "$BASE_REF" ]; then
+            BASE_BRANCH=${BASE_REF/refs\/heads\//}
+            BASE_POETRY_VERSION=$((git show "origin/$BASE_BRANCH":./poetry.lock; true) | python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry -)
+            echo "Found Poetry version ${BASE_POETRY_VERSION} in backend/poetry.lock on ${BASE_REF}"
+            POETRY_VERSION=$(printf '%s\n' "$HEAD_POETRY_VERSION" "$BASE_POETRY_VERSION" | sort -V | tail -n1)
+          else
+            POETRY_VERSION=$HEAD_POETRY_VERSION
+          fi
+          echo "Using Poetry version ${POETRY_VERSION}"
+
+          # Install Poetry
+          curl -sSL https://install.python-poetry.org | POETRY_VERSION=$POETRY_VERSION python3 -
 
           if [ "${{ runner.os }}" = "macOS" ]; then
             PATH="$HOME/.local/bin:$PATH"
             echo "$HOME/.local/bin" >> $GITHUB_PATH
           fi
+        env:
+          BASE_REF: ${{ github.base_ref || github.event.merge_group.base_ref }}
+
+      - name: Check poetry.lock
+        run: |
+          poetry lock
+
+          if ! git diff --quiet --ignore-matching-lines="^# " poetry.lock; then
+            echo "Error: poetry.lock not up to date."
+            echo
+            git diff poetry.lock
+            exit 1
+          fi
 
       - name: Install Python dependencies
         run: poetry install
@@ -97,6 +135,7 @@ jobs:
         run: poetry run prisma migrate dev --name updates
         env:
           DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
+          DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}
 
       - id: lint
         name: Run Linter
@@ -113,12 +152,13 @@ jobs:
         env:
           LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
           DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
+          DIRECT_URL: ${{ steps.supabase.outputs.DB_URL }}
           SUPABASE_URL: ${{ steps.supabase.outputs.API_URL }}
           SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.SERVICE_ROLE_KEY }}
           SUPABASE_JWT_SECRET: ${{ steps.supabase.outputs.JWT_SECRET }}
-          REDIS_HOST: 'localhost'
-          REDIS_PORT: '6379'
-          REDIS_PASSWORD: 'testpassword'
+          REDIS_HOST: "localhost"
+          REDIS_PORT: "6379"
+          REDIS_PASSWORD: "testpassword"
 
     env:
       CI: true
@@ -126,6 +166,13 @@ jobs:
       RUN_ENV: local
       PORT: 8080
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      # We know these are here, don't report this as a security vulnerability
+      # This is used as the default credential for the entire system's RabbitMQ instance
+      # If you want to replace this, you can do so by making our entire system generate
+      # new credentials for each local user and update the environment variables in
+      # the backend service, docker composes, and examples
+      RABBITMQ_DEFAULT_USER: "rabbitmq_user_default"
+      RABBITMQ_DEFAULT_PASS: "k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7"
 
       # - name: Upload coverage reports to Codecov
       #   uses: codecov/codecov-action@v4

.github/workflows/platform-dev-deploy-event-dispatcher.yml

@@ -0,0 +1,198 @@
+name: AutoGPT Platform - Dev Deploy PR Event Dispatcher
+
+on:
+  pull_request:
+    types: [closed]
+  issue_comment:
+    types: [created]
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check comment permissions and deployment status
+        id: check_status
+        if: github.event_name == 'issue_comment' && github.event.issue.pull_request
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const commentBody = context.payload.comment.body.trim();
+            const commentUser = context.payload.comment.user.login;
+            const prAuthor = context.payload.issue.user.login;
+            const authorAssociation = context.payload.comment.author_association;
+            
+            // Check permissions
+            const hasPermission = (
+              authorAssociation === 'OWNER' ||
+              authorAssociation === 'MEMBER' ||
+              authorAssociation === 'COLLABORATOR'
+            );
+            
+            core.setOutput('comment_body', commentBody);
+            core.setOutput('has_permission', hasPermission);
+            
+            if (!hasPermission && (commentBody === '!deploy' || commentBody === '!undeploy')) {
+              core.setOutput('permission_denied', 'true');
+              return;
+            }
+            
+            if (commentBody !== '!deploy' && commentBody !== '!undeploy') {
+              return;
+            }
+            
+            // Process deploy command
+            if (commentBody === '!deploy') {
+              core.setOutput('should_deploy', 'true');
+            }
+            // Process undeploy command
+            else if (commentBody === '!undeploy') {
+              core.setOutput('should_undeploy', 'true');
+            }
+
+      - name: Post permission denied comment
+        if: steps.check_status.outputs.permission_denied == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: `❌ **Permission denied**: Only the repository owners, members, or collaborators can use deployment commands.`
+            });
+
+      - name: Get PR details for deployment
+        id: pr_details
+        if: steps.check_status.outputs.should_deploy == 'true' || steps.check_status.outputs.should_undeploy == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+            core.setOutput('pr_number', pr.data.number);
+            core.setOutput('pr_title', pr.data.title);
+            core.setOutput('pr_state', pr.data.state);
+          
+      - name: Dispatch Deploy Event
+        if: steps.check_status.outputs.should_deploy == 'true'
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.DISPATCH_TOKEN }}
+          repository: Significant-Gravitas/AutoGPT_cloud_infrastructure
+          event-type: pr-event
+          client-payload: |
+            {
+              "action": "deploy",
+              "pr_number": "${{ steps.pr_details.outputs.pr_number }}",
+              "pr_title": "${{ steps.pr_details.outputs.pr_title }}",
+              "pr_state": "${{ steps.pr_details.outputs.pr_state }}",
+              "repo": "${{ github.repository }}"
+            }
+
+      - name: Post deploy success comment
+        if: steps.check_status.outputs.should_deploy == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: `🚀 **Deploying PR #${{ steps.pr_details.outputs.pr_number }}** to development environment...`
+            });
+
+      - name: Dispatch Undeploy Event (from comment)
+        if: steps.check_status.outputs.should_undeploy == 'true'
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.DISPATCH_TOKEN }}
+          repository: Significant-Gravitas/AutoGPT_cloud_infrastructure
+          event-type: pr-event
+          client-payload: |
+            {
+              "action": "undeploy",
+              "pr_number": "${{ steps.pr_details.outputs.pr_number }}",
+              "pr_title": "${{ steps.pr_details.outputs.pr_title }}",
+              "pr_state": "${{ steps.pr_details.outputs.pr_state }}",
+              "repo": "${{ github.repository }}"
+            }
+
+      - name: Post undeploy success comment
+        if: steps.check_status.outputs.should_undeploy == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: `🗑️ **Undeploying PR #${{ steps.pr_details.outputs.pr_number }}** from development environment...`
+            });
+
+      - name: Check deployment status on PR close
+        id: check_pr_close
+        if: github.event_name == 'pull_request' && github.event.action == 'closed'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const comments = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number
+            });
+            
+            let lastDeployIndex = -1;
+            let lastUndeployIndex = -1;
+            
+            comments.data.forEach((comment, index) => {
+              if (comment.body.trim() === '!deploy') {
+                lastDeployIndex = index;
+              } else if (comment.body.trim() === '!undeploy') {
+                lastUndeployIndex = index;
+              }
+            });
+            
+            // Should undeploy if there's a !deploy without a subsequent !undeploy
+            const shouldUndeploy = lastDeployIndex !== -1 && lastDeployIndex > lastUndeployIndex;
+            core.setOutput('should_undeploy', shouldUndeploy);
+            
+      - name: Dispatch Undeploy Event (PR closed with active deployment)
+        if: >-
+          github.event_name == 'pull_request' &&
+          github.event.action == 'closed' &&
+          steps.check_pr_close.outputs.should_undeploy == 'true'
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.DISPATCH_TOKEN }}
+          repository: Significant-Gravitas/AutoGPT_cloud_infrastructure
+          event-type: pr-event
+          client-payload: |
+            {
+              "action": "undeploy",
+              "pr_number": "${{ github.event.pull_request.number }}",
+              "pr_title": "${{ github.event.pull_request.title }}",
+              "pr_state": "${{ github.event.pull_request.state }}",
+              "repo": "${{ github.repository }}"
+            }
+
+      - name: Post PR close undeploy comment
+        if: >-
+          github.event_name == 'pull_request' &&
+          github.event.action == 'closed' &&
+          steps.check_pr_close.outputs.should_undeploy == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: `🧹 **Auto-undeploying**: PR closed with active deployment. Cleaning up development environment for PR #${{ github.event.pull_request.number }}.`
+            });

.github/workflows/platform-frontend-ci.yml

@@ -23,39 +23,49 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
           node-version: "21"
 
+      - name: Enable corepack
+        run: corepack enable
+
       - name: Install dependencies
-        run: |
-          yarn install --frozen-lockfile
+        run: pnpm install --frozen-lockfile
 
       - name: Run lint
-        run: |
-          yarn lint
+        run: pnpm lint
 
-  test:
+  type-check:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
         with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
+          node-version: "21"
 
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: false
-          dotnet: false
-          haskell: false
-          large-packages: true
-          docker-images: true
-          swap-storage: true
+      - name: Enable corepack
+        run: corepack enable
 
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Run tsc check
+        run: pnpm type-check
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium, webkit]
+
+    steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
@@ -66,9 +76,18 @@ jobs:
         with:
           node-version: "21"
 
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          large-packages: false # slow
+          docker-images: false # limited benefit
+
       - name: Copy default supabase .env
         run: |
-          cp ../supabase/docker/.env.example ../.env
+          cp ../.env.example ../.env
 
       - name: Copy backend .env
         run: |
@@ -79,23 +98,28 @@ jobs:
           docker compose -f ../docker-compose.yml up -d
 
       - name: Install dependencies
-        run: |
-          yarn install --frozen-lockfile
+        run: pnpm install --frozen-lockfile
 
-      - name: Setup Builder .env
-        run: |
-          cp .env.example .env
+      - name: Setup .env
+        run: cp .env.example .env
 
-      - name: Install Playwright Browsers
-        run: yarn playwright install --with-deps
+      - name: Build frontend
+        run: pnpm build --turbo
+        # uses Turbopack, much faster and safe enough for a test pipeline
 
-      - name: Run tests
-        run: |
-          yarn test
+      - name: Install Browser '${{ matrix.browser }}'
+        run: pnpm playwright install --with-deps ${{ matrix.browser }}
+
+      - name: Run Playwright tests
+        run: pnpm test:no-build --project=${{ matrix.browser }}
+
+      - name: Print Final Docker Compose logs
+        if: always()
+        run: docker compose -f ../docker-compose.yml logs
 
       - uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
         with:
-          name: playwright-report
+          name: playwright-report-${{ matrix.browser }}
           path: playwright-report/
           retention-days: 30

.github/workflows/platform-market-ci.yml

@@ -1,126 +0,0 @@
-name: AutoGPT Platform - Backend CI
-
-on:
-  push:
-    branches: [master, dev, ci-test*]
-    paths:
-      - ".github/workflows/platform-market-ci.yml"
-      - "autogpt_platform/market/**"
-  pull_request:
-    branches: [master, dev, release-*]
-    paths:
-      - ".github/workflows/platform-market-ci.yml"
-      - "autogpt_platform/market/**"
-  merge_group:
-
-concurrency:
-  group: ${{ format('backend-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
-  cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}
-
-defaults:
-  run:
-    shell: bash
-    working-directory: autogpt_platform/market
-
-jobs:
-  test:
-    permissions:
-      contents: read
-    timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10"]
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Setup Supabase
-        uses: supabase/setup-cli@v1
-        with:
-          version: latest
-
-      - id: get_date
-        name: Get date
-        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
-
-      - name: Set up Python dependency cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pypoetry
-          key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/market/poetry.lock') }}
-
-      - name: Install Poetry (Unix)
-        run: |
-          curl -sSL https://install.python-poetry.org | python3 -
-
-          if [ "${{ runner.os }}" = "macOS" ]; then
-            PATH="$HOME/.local/bin:$PATH"
-            echo "$HOME/.local/bin" >> $GITHUB_PATH
-          fi
-
-      - name: Install Python dependencies
-        run: poetry install
-
-      - name: Generate Prisma Client
-        run: poetry run prisma generate
-
-      - id: supabase
-        name: Start Supabase
-        working-directory: .
-        run: |
-          supabase init
-          supabase start --exclude postgres-meta,realtime,storage-api,imgproxy,inbucket,studio,edge-runtime,logflare,vector,supavisor
-          supabase status -o env | sed 's/="/=/; s/"$//' >> $GITHUB_OUTPUT
-        # outputs:
-        # DB_URL, API_URL, GRAPHQL_URL, ANON_KEY, SERVICE_ROLE_KEY, JWT_SECRET
-
-      - name: Run Database Migrations
-        run: poetry run prisma migrate dev --name updates
-        env:
-          DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
-
-      - id: lint
-        name: Run Linter
-        run: poetry run lint
-
-      # Tests comment out because they do not work with prisma mock, nor have they been updated since they were created
-      # - name: Run pytest with coverage
-      #   run: |
-      #     if [[ "${{ runner.debug }}" == "1" ]]; then
-      #       poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG test
-      #     else
-      #       poetry run pytest -s -vv test
-      #     fi
-      #   if: success() || (failure() && steps.lint.outcome == 'failure')
-      #   env:
-      #     LOG_LEVEL: ${{ runner.debug && 'DEBUG' || 'INFO' }}
-      #     DATABASE_URL: ${{ steps.supabase.outputs.DB_URL }}
-      #     SUPABASE_URL: ${{ steps.supabase.outputs.API_URL }}
-      #     SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.SERVICE_ROLE_KEY }}
-      #     SUPABASE_JWT_SECRET: ${{ steps.supabase.outputs.JWT_SECRET }}
-      #     REDIS_HOST: 'localhost'
-      #     REDIS_PORT: '6379'
-          # REDIS_PASSWORD: 'testpassword'
-
-    env:
-      CI: true
-      PLAIN_OUTPUT: True
-      RUN_ENV: local
-      PORT: 8080
-
-      # - name: Upload coverage reports to Codecov
-      #   uses: codecov/codecov-action@v4
-      #   with:
-      #     token: ${{ secrets.CODECOV_TOKEN }}
-      #     flags: backend,${{ runner.os }}

.github/workflows/repo-close-stale-issues.yml

@@ -16,7 +16,7 @@ jobs:
           # operations-per-run: 5000
           stale-issue-message: >
             This issue has automatically been marked as _stale_ because it has not had
-            any activity in the last 50 days. You can _unstale_ it by commenting or
+            any activity in the last 170 days. You can _unstale_ it by commenting or
             removing the label. Otherwise, this issue will be closed in 10 days.
           stale-pr-message: >
             This pull request has automatically been marked as _stale_ because it has
@@ -25,7 +25,7 @@ jobs:
           close-issue-message: >
             This issue was closed automatically because it has been stale for 10 days
             with no activity.
-          days-before-stale: 50
+          days-before-stale: 170
           days-before-close: 10
           # Do not touch meta issues:
           exempt-issue-labels: meta,fridge,project management

.github/workflows/scripts/get_package_version_from_lockfile.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import sys
+
+if sys.version_info < (3, 11):
+    print("Python version 3.11 or higher required")
+    sys.exit(1)
+
+import tomllib
+
+
+def get_package_version(package_name: str, lockfile_path: str) -> str | None:
+    """Extract package version from poetry.lock file."""
+    try:
+        if lockfile_path == "-":
+            data = tomllib.load(sys.stdin.buffer)
+        else:
+            with open(lockfile_path, "rb") as f:
+                data = tomllib.load(f)
+    except FileNotFoundError:
+        print(f"Error: File '{lockfile_path}' not found", file=sys.stderr)
+        sys.exit(1)
+    except tomllib.TOMLDecodeError as e:
+        print(f"Error parsing TOML file: {e}", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error reading file: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    # Look for the package in the packages list
+    packages = data.get("package", [])
+    for package in packages:
+        if package.get("name", "").lower() == package_name.lower():
+            return package.get("version")
+
+    return None
+
+
+def main():
+    if len(sys.argv) not in (2, 3):
+        print(
+            "Usages: python get_package_version_from_lockfile.py <package name> [poetry.lock path]\n"
+            "        cat poetry.lock | python get_package_version_from_lockfile.py <package name> -",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    package_name = sys.argv[1]
+    lockfile_path = sys.argv[2] if len(sys.argv) == 3 else "poetry.lock"
+
+    version = get_package_version(package_name, lockfile_path)
+
+    if version:
+        print(version)
+    else:
+        print(f"Package '{package_name}' not found in {lockfile_path}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

.gitignore

@@ -173,3 +173,7 @@ LICENSE.rtf
 autogpt_platform/backend/settings.py
 /.auth
 /autogpt_platform/frontend/.auth
+
+*.ign.*
+.test-contents
+.claude/settings.local.json

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "classic/forge/tests/vcr_cassettes"]
 	path = classic/forge/tests/vcr_cassettes
 	url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes
-[submodule "autogpt_platform/supabase"]
-	path = autogpt_platform/supabase
-	url = https://github.com/supabase/supabase.git

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
         name: Detect secrets
         description: Detects high entropy strings that are likely to be passwords.
         files: ^autogpt_platform/
-        stages: [push]
+        stages: [pre-push]
 
   - repo: local
     # For proper type checking, all dependencies need to be up-to-date.
@@ -98,6 +98,11 @@ repos:
         files: ^autogpt_platform/autogpt_libs/
         args: [--fix]
 
+      - id: ruff-format
+        name: Format (Ruff) - AutoGPT Platform - Libs
+        alias: ruff-lint-platform-libs
+        files: ^autogpt_platform/autogpt_libs/
+
   - repo: local
     # isort needs the context of which packages are installed to function, so we
     # can't use a vendored isort pre-commit hook (which runs in its own isolated venv).
@@ -105,7 +110,7 @@ repos:
       - id: isort
         name: Lint (isort) - AutoGPT Platform - Backend
         alias: isort-platform-backend
-        entry: poetry -C autogpt_platform/backend run isort -p backend
+        entry: poetry -P autogpt_platform/backend run isort -p backend
         files: ^autogpt_platform/backend/
         types: [file, python]
         language: system
@@ -113,7 +118,7 @@ repos:
       - id: isort
         name: Lint (isort) - Classic - AutoGPT
         alias: isort-classic-autogpt
-        entry: poetry -C classic/original_autogpt run isort -p autogpt
+        entry: poetry -P classic/original_autogpt run isort -p autogpt
         files: ^classic/original_autogpt/
         types: [file, python]
         language: system
@@ -121,7 +126,7 @@ repos:
       - id: isort
         name: Lint (isort) - Classic - Forge
         alias: isort-classic-forge
-        entry: poetry -C classic/forge run isort -p forge
+        entry: poetry -P classic/forge run isort -p forge
         files: ^classic/forge/
         types: [file, python]
         language: system
@@ -129,18 +134,18 @@ repos:
       - id: isort
         name: Lint (isort) - Classic - Benchmark
         alias: isort-classic-benchmark
-        entry: poetry -C classic/benchmark run isort -p agbenchmark
+        entry: poetry -P classic/benchmark run isort -p agbenchmark
         files: ^classic/benchmark/
         types: [file, python]
         language: system
 
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.10.0
     # Black has sensible defaults, doesn't need package context, and ignores
     # everything in .gitignore, so it works fine without any config or arguments.
     hooks:
       - id: black
-        name: Lint (Black)
+        name: Format (Black)
 
   - repo: https://github.com/PyCQA/flake8
     rev: 7.0.0
@@ -165,6 +170,16 @@ repos:
         files: ^classic/benchmark/(agbenchmark|tests)/((?!reports).)*[/.]
         args: [--config=classic/benchmark/.flake8]
 
+  - repo: local
+    hooks:
+      - id: prettier
+        name: Format (Prettier) - AutoGPT Platform - Frontend
+        alias: format-platform-frontend
+        entry: bash -c 'cd autogpt_platform/frontend && npx prettier --write $(echo "$@" | sed "s|autogpt_platform/frontend/||g")' --
+        files: ^autogpt_platform/frontend/
+        types: [file]
+        language: system
+
   - repo: local
     # To have watertight type checking, we check *all* the files in an affected
     # project. To trigger on poetry.lock we also reset the file `types` filter.
@@ -173,7 +188,6 @@ repos:
         name: Typecheck - AutoGPT Platform - Backend
         alias: pyright-platform-backend
         entry: poetry -C autogpt_platform/backend run pyright
-        args: [-p, autogpt_platform/backend, autogpt_platform/backend]
         # include forge source (since it's a path dependency) but exclude *_test.py files:
         files: ^autogpt_platform/(backend/((backend|test)/|(\w+\.py|poetry\.lock)$)|autogpt_libs/(autogpt_libs/.*(?<!_test)\.py|poetry\.lock)$)
         types: [file]
@@ -184,7 +198,6 @@ repos:
         name: Typecheck - AutoGPT Platform - Libs
         alias: pyright-platform-libs
         entry: poetry -C autogpt_platform/autogpt_libs run pyright
-        args: [-p, autogpt_platform/autogpt_libs, autogpt_platform/autogpt_libs]
         files: ^autogpt_platform/autogpt_libs/(autogpt_libs/|poetry\.lock$)
         types: [file]
         language: system
@@ -194,7 +207,6 @@ repos:
         name: Typecheck - Classic - AutoGPT
         alias: pyright-classic-autogpt
         entry: poetry -C classic/original_autogpt run pyright
-        args: [-p, classic/original_autogpt, classic/original_autogpt]
         # include forge source (since it's a path dependency) but exclude *_test.py files:
         files: ^(classic/original_autogpt/((autogpt|scripts|tests)/|poetry\.lock$)|classic/forge/(forge/.*(?<!_test)\.py|poetry\.lock)$)
         types: [file]
@@ -205,7 +217,6 @@ repos:
         name: Typecheck - Classic - Forge
         alias: pyright-classic-forge
         entry: poetry -C classic/forge run pyright
-        args: [-p, classic/forge, classic/forge]
         files: ^classic/forge/(forge/|poetry\.lock$)
         types: [file]
         language: system
@@ -215,7 +226,6 @@ repos:
         name: Typecheck - Classic - Benchmark
         alias: pyright-classic-benchmark
         entry: poetry -C classic/benchmark run pyright
-        args: [-p, classic/benchmark, classic/benchmark]
         files: ^classic/benchmark/(agbenchmark/|tests/|poetry\.lock$)
         types: [file]
         language: system
@@ -223,36 +233,46 @@ repos:
 
   - repo: local
     hooks:
-      - id: pytest
-        name: Run tests - AutoGPT Platform - Backend
-        alias: pytest-platform-backend
-        entry: bash -c 'cd autogpt_platform/backend && poetry run pytest'
-        # include autogpt_libs source (since it's a path dependency) but exclude *_test.py files:
-        files: ^autogpt_platform/(backend/((backend|test)/|poetry\.lock$)|autogpt_libs/(autogpt_libs/.*(?<!_test)\.py|poetry\.lock)$)
+      - id: tsc
+        name: Typecheck - AutoGPT Platform - Frontend
+        entry: bash -c 'cd autogpt_platform/frontend && pnpm type-check'
+        files: ^autogpt_platform/frontend/
+        types: [file]
         language: system
         pass_filenames: false
 
-      - id: pytest
-        name: Run tests - Classic - AutoGPT (excl. slow tests)
-        alias: pytest-classic-autogpt
-        entry: bash -c 'cd classic/original_autogpt && poetry run pytest --cov=autogpt -m "not slow" tests/unit tests/integration'
-        # include forge source (since it's a path dependency) but exclude *_test.py files:
-        files: ^(classic/original_autogpt/((autogpt|tests)/|poetry\.lock$)|classic/forge/(forge/.*(?<!_test)\.py|poetry\.lock)$)
-        language: system
-        pass_filenames: false
+  # - repo: local
+  #   hooks:
+  #     - id: pytest
+  #       name: Run tests - AutoGPT Platform - Backend
+  #       alias: pytest-platform-backend
+  #       entry: bash -c 'cd autogpt_platform/backend && poetry run pytest'
+  #       # include autogpt_libs source (since it's a path dependency) but exclude *_test.py files:
+  #       files: ^autogpt_platform/(backend/((backend|test)/|poetry\.lock$)|autogpt_libs/(autogpt_libs/.*(?<!_test)\.py|poetry\.lock)$)
+  #       language: system
+  #       pass_filenames: false
 
-      - id: pytest
-        name: Run tests - Classic - Forge (excl. slow tests)
-        alias: pytest-classic-forge
-        entry: bash -c 'cd classic/forge && poetry run pytest --cov=forge -m "not slow"'
-        files: ^classic/forge/(forge/|tests/|poetry\.lock$)
-        language: system
-        pass_filenames: false
+  #     - id: pytest
+  #       name: Run tests - Classic - AutoGPT (excl. slow tests)
+  #       alias: pytest-classic-autogpt
+  #       entry: bash -c 'cd classic/original_autogpt && poetry run pytest --cov=autogpt -m "not slow" tests/unit tests/integration'
+  #       # include forge source (since it's a path dependency) but exclude *_test.py files:
+  #       files: ^(classic/original_autogpt/((autogpt|tests)/|poetry\.lock$)|classic/forge/(forge/.*(?<!_test)\.py|poetry\.lock)$)
+  #       language: system
+  #       pass_filenames: false
 
-      - id: pytest
-        name: Run tests - Classic - Benchmark
-        alias: pytest-classic-benchmark
-        entry: bash -c 'cd classic/benchmark && poetry run pytest --cov=benchmark'
-        files: ^classic/benchmark/(agbenchmark/|tests/|poetry\.lock$)
-        language: system
-        pass_filenames: false
+  #     - id: pytest
+  #       name: Run tests - Classic - Forge (excl. slow tests)
+  #       alias: pytest-classic-forge
+  #       entry: bash -c 'cd classic/forge && poetry run pytest --cov=forge -m "not slow"'
+  #       files: ^classic/forge/(forge/|tests/|poetry\.lock$)
+  #       language: system
+  #       pass_filenames: false
+
+  #     - id: pytest
+  #       name: Run tests - Classic - Benchmark
+  #       alias: pytest-classic-benchmark
+  #       entry: bash -c 'cd classic/benchmark && poetry run pytest --cov=benchmark'
+  #       files: ^classic/benchmark/(agbenchmark/|tests/|poetry\.lock$)
+  #       language: system
+  #       pass_filenames: false

.vscode/launch.json

@@ -32,9 +32,9 @@
       "type": "debugpy",
       "request": "launch",
       "module": "backend.app",
-      // "env": {
-      //   "ENV": "dev"
-      // },
+      "env": {
+        "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES"
+      },
       "envFile": "${workspaceFolder}/backend/.env",
       "justMyCode": false,
       "cwd": "${workspaceFolder}/autogpt_platform/backend"

AGENTS.md

@@ -0,0 +1,53 @@
+# AutoGPT Platform Contribution Guide
+
+This guide provides context for Codex when updating the **autogpt_platform** folder.
+
+## Directory overview
+
+- `autogpt_platform/backend` – FastAPI based backend service.
+- `autogpt_platform/autogpt_libs` – Shared Python libraries.
+- `autogpt_platform/frontend` – Next.js + Typescript frontend.
+- `autogpt_platform/docker-compose.yml` – development stack.
+
+See `docs/content/platform/getting-started.md` for setup instructions.
+
+## Code style
+
+- Format Python code with `poetry run format`.
+- Format frontend code using `pnpm format`.
+
+## Testing
+
+- Backend: `poetry run test` (runs pytest with a docker based postgres + prisma).
+- Frontend: `pnpm test` or `pnpm test-ui` for Playwright tests. See `docs/content/platform/contributing/tests.md` for tips.
+
+Always run the relevant linters and tests before committing.
+Use conventional commit messages for all commits (e.g. `feat(backend): add API`).
+  Types:
+    - feat
+    - fix
+    - refactor
+    - ci
+    - dx (developer experience)
+  Scopes:
+    - platform
+      - platform/library
+      - platform/marketplace
+      - backend
+        - backend/executor
+      - frontend
+        - frontend/library
+        - frontend/marketplace
+      - blocks
+
+## Pull requests
+
+- Use the template in `.github/PULL_REQUEST_TEMPLATE.md`.
+- Rely on the pre-commit checks for linting and formatting
+- Fill out the **Changes** section and the checklist.
+- Use conventional commit titles with a scope (e.g. `feat(frontend): add feature`).
+- Keep out-of-scope changes under 20% of the PR.
+- Ensure PR descriptions are complete.
+- For changes touching `data/*.py`, validate user ID checks or explain why not needed.
+- If adding protected frontend routes, update `frontend/lib/supabase/middleware.ts`.
+- Use the linear ticket branch structure if given codex/open-1668-resume-dropped-runs

CONTRIBUTING.md

@@ -2,9 +2,6 @@
 If you are reading this, you are probably looking for the full **[contribution guide]**,
 which is part of our [wiki].
 
-Also check out our [🚀 Roadmap][roadmap] for information about our priorities and associated tasks.
-<!-- You can find our immediate priorities and their progress on our public [kanban board]. -->
-
 [contribution guide]: https://github.com/Significant-Gravitas/AutoGPT/wiki/Contributing
 [wiki]: https://github.com/Significant-Gravitas/AutoGPT/wiki
 [roadmap]: https://github.com/Significant-Gravitas/AutoGPT/discussions/6971

README.md

@@ -15,7 +15,38 @@
 > Setting up and hosting the AutoGPT Platform yourself is a technical process. 
 > If you'd rather something that just works, we recommend [joining the waitlist](https://bit.ly/3ZDijAI) for the cloud-hosted beta.
 
-https://github.com/user-attachments/assets/d04273a5-b36a-4a37-818e-f631ce72d603
+### System Requirements
+
+Before proceeding with the installation, ensure your system meets the following requirements:
+
+#### Hardware Requirements
+- CPU: 4+ cores recommended
+- RAM: Minimum 8GB, 16GB recommended
+- Storage: At least 10GB of free space
+
+#### Software Requirements
+- Operating Systems:
+  - Linux (Ubuntu 20.04 or newer recommended)
+  - macOS (10.15 or newer)
+  - Windows 10/11 with WSL2
+- Required Software (with minimum versions):
+  - Docker Engine (20.10.0 or newer)
+  - Docker Compose (2.0.0 or newer)
+  - Git (2.30 or newer)
+  - Node.js (16.x or newer)
+  - npm (8.x or newer)
+  - VSCode (1.60 or newer) or any modern code editor
+
+#### Network Requirements
+- Stable internet connection
+- Access to required ports (will be configured in Docker)
+- Ability to make outbound HTTPS connections
+
+### Updated Setup Instructions:
+We've moved to a fully maintained and regularly updated documentation site.
+
+👉 [Follow the official self-hosting guide here](https://docs.agpt.co/platform/getting-started/)
+
 
 This tutorial assumes you have Docker, VSCode, git and npm installed.
 
@@ -148,7 +179,7 @@ Just clone the repo, install dependencies with `./run setup`, and you should be
 
 [![Join us on Discord](https://invidget.switchblade.xyz/autogpt)](https://discord.gg/autogpt)
 
-To report a bug or request a feature, create a [GitHub Issue](https://github.com/Significant-Gravitas/AutoGPT/issues/new/choose). Please ensure someone else hasn’t created an issue for the same topic.
+To report a bug or request a feature, create a [GitHub Issue](https://github.com/Significant-Gravitas/AutoGPT/issues/new/choose). Please ensure someone else hasn't created an issue for the same topic.
 
 ## 🤝 Sister projects
 

SECURITY.md

@@ -8,7 +8,7 @@ We take the security of our project seriously. If you believe you have found a s
 
 Instead, please report them via:
 - [GitHub Security Advisory](https://github.com/Significant-Gravitas/AutoGPT/security/advisories/new)
-- [Huntr.dev](https://huntr.com/repos/significant-gravitas/autogpt) - where you may be eligible for a bounty
+<!--- [Huntr.dev](https://huntr.com/repos/significant-gravitas/autogpt) - where you may be eligible for a bounty-->
 
 ### Reporting Process
 1. **Submit Report**: Use one of the above channels to submit your report
@@ -20,6 +20,7 @@ Instead, please report them via:
 - Please provide detailed reports with reproducible steps
 - Include the version/commit hash where you discovered the vulnerability
 - Allow us a 90-day security fix window before any public disclosure
+- After patch is released, allow 30 days for users to update before public disclosure (for a total of 120 days max between update time and fix time)
 - Share any potential mitigations or workarounds if known
 
 ## Supported Versions

autogpt_platform/.env.example

@@ -0,0 +1,123 @@
+############
+# Secrets
+# YOU MUST CHANGE THESE BEFORE GOING INTO PRODUCTION
+############
+
+POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password
+JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long
+ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE
+SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q
+DASHBOARD_USERNAME=supabase
+DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated
+SECRET_KEY_BASE=UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq
+VAULT_ENC_KEY=your-encryption-key-32-chars-min
+
+
+############
+# Database - You can change these to any PostgreSQL database that has logical replication enabled.
+############
+
+POSTGRES_HOST=db
+POSTGRES_DB=postgres
+POSTGRES_PORT=5432
+# default user is postgres
+
+
+############
+# Supavisor -- Database pooler
+############
+POOLER_PROXY_PORT_TRANSACTION=6543
+POOLER_DEFAULT_POOL_SIZE=20
+POOLER_MAX_CLIENT_CONN=100
+POOLER_TENANT_ID=your-tenant-id
+
+
+############
+# API Proxy - Configuration for the Kong Reverse proxy.
+############
+
+KONG_HTTP_PORT=8000
+KONG_HTTPS_PORT=8443
+
+
+############
+# API - Configuration for PostgREST.
+############
+
+PGRST_DB_SCHEMAS=public,storage,graphql_public
+
+
+############
+# Auth - Configuration for the GoTrue authentication server.
+############
+
+## General
+SITE_URL=http://localhost:3000
+ADDITIONAL_REDIRECT_URLS=
+JWT_EXPIRY=3600
+DISABLE_SIGNUP=false
+API_EXTERNAL_URL=http://localhost:8000
+
+## Mailer Config
+MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify"
+MAILER_URLPATHS_INVITE="/auth/v1/verify"
+MAILER_URLPATHS_RECOVERY="/auth/v1/verify"
+MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify"
+
+## Email auth
+ENABLE_EMAIL_SIGNUP=true
+ENABLE_EMAIL_AUTOCONFIRM=false
+SMTP_ADMIN_EMAIL=admin@example.com
+SMTP_HOST=supabase-mail
+SMTP_PORT=2500
+SMTP_USER=fake_mail_user
+SMTP_PASS=fake_mail_password
+SMTP_SENDER_NAME=fake_sender
+ENABLE_ANONYMOUS_USERS=false
+
+## Phone auth
+ENABLE_PHONE_SIGNUP=true
+ENABLE_PHONE_AUTOCONFIRM=true
+
+
+############
+# Studio - Configuration for the Dashboard
+############
+
+STUDIO_DEFAULT_ORGANIZATION=Default Organization
+STUDIO_DEFAULT_PROJECT=Default Project
+
+STUDIO_PORT=3000
+# replace if you intend to use Studio outside of localhost
+SUPABASE_PUBLIC_URL=http://localhost:8000
+
+# Enable webp support
+IMGPROXY_ENABLE_WEBP_DETECTION=true
+
+# Add your OpenAI API key to enable SQL Editor Assistant
+OPENAI_API_KEY=
+
+
+############
+# Functions - Configuration for Functions
+############
+# NOTE: VERIFY_JWT applies to all functions. Per-function VERIFY_JWT is not supported yet.
+FUNCTIONS_VERIFY_JWT=false
+
+
+############
+# Logs - Configuration for Logflare
+# Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction
+############
+
+LOGFLARE_LOGGER_BACKEND_API_KEY=your-super-secret-and-long-logflare-key
+
+# Change vector.toml sinks to reflect this change
+LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key
+
+# Docker socket location - this value will differ depending on your OS
+DOCKER_SOCKET_LOCATION=/var/run/docker.sock
+
+# Google Cloud Project details
+GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID
+GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER

autogpt_platform/CLAUDE.md

@@ -0,0 +1,132 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Repository Overview
+
+AutoGPT Platform is a monorepo containing:
+- **Backend** (`/backend`): Python FastAPI server with async support
+- **Frontend** (`/frontend`): Next.js React application
+- **Shared Libraries** (`/autogpt_libs`): Common Python utilities
+
+## Essential Commands
+
+### Backend Development
+```bash
+# Install dependencies
+cd backend && poetry install
+
+# Run database migrations
+poetry run prisma migrate dev
+
+# Start all services (database, redis, rabbitmq)
+docker compose up -d
+
+# Run the backend server
+poetry run serve
+
+# Run tests
+poetry run test
+
+# Run specific test
+poetry run pytest path/to/test_file.py::test_function_name
+
+# Lint and format
+poetry run format  # Black + isort
+poetry run lint    # ruff
+```
+More details can be found in TESTING.md
+
+#### Creating/Updating Snapshots
+
+When you first write a test or when the expected output changes:
+
+```bash
+poetry run pytest path/to/test.py --snapshot-update
+```
+
+⚠️ **Important**: Always review snapshot changes before committing! Use `git diff` to verify the changes are expected.
+
+
+### Frontend Development
+```bash
+# Install dependencies
+cd frontend && npm install
+
+# Start development server
+npm run dev
+
+# Run E2E tests
+npm run test
+
+# Run Storybook for component development
+npm run storybook
+
+# Build production
+npm run build
+
+# Type checking
+npm run type-check
+```
+
+## Architecture Overview
+
+### Backend Architecture
+- **API Layer**: FastAPI with REST and WebSocket endpoints
+- **Database**: PostgreSQL with Prisma ORM, includes pgvector for embeddings
+- **Queue System**: RabbitMQ for async task processing
+- **Execution Engine**: Separate executor service processes agent workflows
+- **Authentication**: JWT-based with Supabase integration
+
+### Frontend Architecture
+- **Framework**: Next.js App Router with React Server Components
+- **State Management**: React hooks + Supabase client for real-time updates
+- **Workflow Builder**: Visual graph editor using @xyflow/react
+- **UI Components**: Radix UI primitives with Tailwind CSS styling
+- **Feature Flags**: LaunchDarkly integration
+
+### Key Concepts
+1. **Agent Graphs**: Workflow definitions stored as JSON, executed by the backend
+2. **Blocks**: Reusable components in `/backend/blocks/` that perform specific tasks
+3. **Integrations**: OAuth and API connections stored per user
+4. **Store**: Marketplace for sharing agent templates
+
+### Testing Approach
+- Backend uses pytest with snapshot testing for API responses
+- Test files are colocated with source files (`*_test.py`)
+- Frontend uses Playwright for E2E tests
+- Component testing via Storybook
+
+### Database Schema
+Key models (defined in `/backend/schema.prisma`):
+- `User`: Authentication and profile data
+- `AgentGraph`: Workflow definitions with version control
+- `AgentGraphExecution`: Execution history and results
+- `AgentNode`: Individual nodes in a workflow
+- `StoreListing`: Marketplace listings for sharing agents
+
+### Environment Configuration
+- Backend: `.env` file in `/backend`
+- Frontend: `.env.local` file in `/frontend`
+- Both require Supabase credentials and API keys for various services
+
+### Common Development Tasks
+
+**Adding a new block:**
+1. Create new file in `/backend/backend/blocks/`
+2. Inherit from `Block` base class
+3. Define input/output schemas
+4. Implement `run` method
+5. Register in block registry
+
+**Modifying the API:**
+1. Update route in `/backend/backend/server/routers/`
+2. Add/update Pydantic models in same directory
+3. Write tests alongside the route file
+4. Run `poetry run test` to verify
+
+**Frontend feature development:**
+1. Components go in `/frontend/src/components/`
+2. Use existing UI components from `/frontend/src/components/ui/`
+3. Add Storybook stories for new components
+4. Test with Playwright if user-facing

autogpt_platform/README.md

@@ -15,53 +15,60 @@ Welcome to the AutoGPT Platform - a powerful system for creating and running AI
 To run the AutoGPT Platform, follow these steps:
 
 1. Clone this repository to your local machine and navigate to the `autogpt_platform` directory within the repository:
+
    ```
    git clone <https://github.com/Significant-Gravitas/AutoGPT.git | git@github.com:Significant-Gravitas/AutoGPT.git>
    cd AutoGPT/autogpt_platform
    ```
 
 2. Run the following command:
+
    ```
-   git submodule update --init --recursive
+   cp .env.example .env
    ```
-   This command will initialize and update the submodules in the repository. The `supabase` folder will be cloned to the root directory.
+
+   This command will copy the `.env.example` file to `.env`. You can modify the `.env` file to add your own environment variables.
 
 3. Run the following command:
-   ```
-   cp supabase/docker/.env.example .env
-   ```
-   This command will copy the `.env.example` file to `.env` in the `supabase/docker` directory. You can modify the `.env` file to add your own environment variables.
 
-4. Run the following command:
    ```
    docker compose up -d
    ```
+
    This command will start all the necessary backend services defined in the `docker-compose.yml` file in detached mode.
 
-5. Navigate to `frontend` within the `autogpt_platform` directory:
+4. Navigate to `frontend` within the `autogpt_platform` directory:
+
    ```
    cd frontend
    ```
+
    You will need to run your frontend application separately on your local machine.
 
-6. Run the following command: 
+5. Run the following command:
+
    ```
    cp .env.example .env.local
    ```
+
    This command will copy the `.env.example` file to `.env.local` in the `frontend` directory. You can modify the `.env.local` within this folder to add your own environment variables for the frontend application.
 
-7. Run the following command:
+6. Run the following command:
+
+   Enable corepack and install dependencies by running:
+
    ```
-   npm install
-   npm run dev
-   ```
-   This command will install the necessary dependencies and start the frontend application in development mode.
-   If you are using Yarn, you can run the following commands instead:
-   ```
-   yarn install && yarn dev
+   corepack enable
+   pnpm i
    ```
 
-8. Open your browser and navigate to `http://localhost:3000` to access the AutoGPT Platform frontend.
+   Then start the frontend application in development mode:
+
+   ```
+   pnpm dev
+   ```
+
+7. Open your browser and navigate to `http://localhost:3000` to access the AutoGPT Platform frontend.
 
 ### Docker Compose Commands
 
@@ -74,43 +81,52 @@ Here are some useful Docker Compose commands for managing your AutoGPT Platform:
 - `docker compose down`: Stop and remove containers, networks, and volumes.
 - `docker compose watch`: Watch for changes in your services and automatically update them.
 
-
 ### Sample Scenarios
 
 Here are some common scenarios where you might use multiple Docker Compose commands:
 
 1. Updating and restarting a specific service:
+
    ```
    docker compose build api_srv
    docker compose up -d --no-deps api_srv
    ```
+
    This rebuilds the `api_srv` service and restarts it without affecting other services.
 
 2. Viewing logs for troubleshooting:
+
    ```
    docker compose logs -f api_srv ws_srv
    ```
+
    This shows and follows the logs for both `api_srv` and `ws_srv` services.
 
 3. Scaling a service for increased load:
+
    ```
    docker compose up -d --scale executor=3
    ```
+
    This scales the `executor` service to 3 instances to handle increased load.
 
 4. Stopping the entire system for maintenance:
+
    ```
    docker compose stop
    docker compose rm -f
    docker compose pull
    docker compose up -d
    ```
+
    This stops all services, removes containers, pulls the latest images, and restarts the system.
 
 5. Developing with live updates:
+
    ```
    docker compose watch
    ```
+
    This watches for changes in your code and automatically updates the relevant services.
 
 6. Checking the status of services:
@@ -121,7 +137,6 @@ Here are some common scenarios where you might use multiple Docker Compose comma
 
 These scenarios demonstrate how to use Docker Compose commands in combination to manage your AutoGPT Platform effectively.
 
-
 ### Persisting Data
 
 To persist data for PostgreSQL and Redis, you can modify the `docker-compose.yml` file to add volumes. Here's how:

autogpt_platform/autogpt_libs/README.md

@@ -1,3 +1,3 @@
 # AutoGPT Libs
 
-This is a new project to store shared functionality across different services in NextGen AutoGPT (e.g. authentication)
+This is a new project to store shared functionality across different services in the AutoGPT Platform (e.g. authentication)

autogpt_platform/autogpt_libs/autogpt_libs/api_key/key_manager.py

@@ -31,4 +31,5 @@ class APIKeyManager:
         """Verify if a provided API key matches the stored hash."""
         if not provided_key.startswith(self.PREFIX):
             return False
-        return hashlib.sha256(provided_key.encode()).hexdigest() == stored_hash
+        provided_hash = hashlib.sha256(provided_key.encode()).hexdigest()
+        return secrets.compare_digest(provided_hash, stored_hash)

autogpt_platform/autogpt_libs/autogpt_libs/auth/__init__.py

@@ -1,14 +1,13 @@
-from .config import Settings
 from .depends import requires_admin_user, requires_user
 from .jwt_utils import parse_jwt_token
-from .middleware import auth_middleware
+from .middleware import APIKeyValidator, auth_middleware
 from .models import User
 
 __all__ = [
-    "Settings",
     "parse_jwt_token",
     "requires_user",
     "requires_admin_user",
+    "APIKeyValidator",
     "auth_middleware",
     "User",
 ]

autogpt_platform/autogpt_libs/autogpt_libs/auth/config.py

@@ -1,14 +1,11 @@
 import os
 
-from dotenv import load_dotenv
-
-load_dotenv()
-
 
 class Settings:
-    JWT_SECRET_KEY: str = os.getenv("SUPABASE_JWT_SECRET", "")
-    ENABLE_AUTH: bool = os.getenv("ENABLE_AUTH", "false").lower() == "true"
-    JWT_ALGORITHM: str = "HS256"
+    def __init__(self):
+        self.JWT_SECRET_KEY: str = os.getenv("SUPABASE_JWT_SECRET", "")
+        self.ENABLE_AUTH: bool = os.getenv("ENABLE_AUTH", "false").lower() == "true"
+        self.JWT_ALGORITHM: str = "HS256"
 
     @property
     def is_configured(self) -> bool:

autogpt_platform/autogpt_libs/autogpt_libs/auth/depends.py

@@ -1,6 +1,6 @@
 import fastapi
 
-from .config import Settings
+from .config import settings
 from .middleware import auth_middleware
 from .models import DEFAULT_USER_ID, User
 
@@ -17,7 +17,7 @@ def requires_admin_user(
 
 def verify_user(payload: dict | None, admin_only: bool) -> User:
     if not payload:
-        if Settings.ENABLE_AUTH:
+        if settings.ENABLE_AUTH:
             raise fastapi.HTTPException(
                 status_code=401, detail="Authorization header is missing"
             )
@@ -35,3 +35,12 @@ def verify_user(payload: dict | None, admin_only: bool) -> User:
         raise fastapi.HTTPException(status_code=403, detail="Admin access required")
 
     return User.from_payload(payload)
+
+
+def get_user_id(payload: dict = fastapi.Depends(auth_middleware)) -> str:
+    user_id = payload.get("sub")
+    if not user_id:
+        raise fastapi.HTTPException(
+            status_code=401, detail="User ID not found in token"
+        )
+    return user_id

autogpt_platform/autogpt_libs/autogpt_libs/auth/middleware.py

@@ -1,7 +1,11 @@
+import inspect
 import logging
+import secrets
+from typing import Any, Callable, Optional
 
-from fastapi import HTTPException, Request
-from fastapi.security import HTTPBearer
+from fastapi import HTTPException, Request, Security
+from fastapi.security import APIKeyHeader, HTTPBearer
+from starlette.status import HTTP_401_UNAUTHORIZED
 
 from .config import settings
 from .jwt_utils import parse_jwt_token
@@ -13,7 +17,7 @@ logger = logging.getLogger(__name__)
 async def auth_middleware(request: Request):
     if not settings.ENABLE_AUTH:
         # If authentication is disabled, allow the request to proceed
-        logger.warn("Auth disabled")
+        logger.warning("Auth disabled")
         return {}
 
     security = HTTPBearer()
@@ -29,3 +33,108 @@ async def auth_middleware(request: Request):
     except ValueError as e:
         raise HTTPException(status_code=401, detail=str(e))
     return payload
+
+
+class APIKeyValidator:
+    """
+    Configurable API key validator that supports custom validation functions
+    for FastAPI applications.
+
+    This class provides a flexible way to implement API key authentication with optional
+    custom validation logic. It can be used for simple token matching
+    or more complex validation scenarios like database lookups.
+
+    Examples:
+        Simple token validation:
+        ```python
+        validator = APIKeyValidator(
+            header_name="X-API-Key",
+            expected_token="your-secret-token"
+        )
+
+        @app.get("/protected", dependencies=[Depends(validator.get_dependency())])
+        def protected_endpoint():
+            return {"message": "Access granted"}
+        ```
+
+        Custom validation with database lookup:
+        ```python
+        async def validate_with_db(api_key: str):
+            api_key_obj = await db.get_api_key(api_key)
+            return api_key_obj if api_key_obj and api_key_obj.is_active else None
+
+        validator = APIKeyValidator(
+            header_name="X-API-Key",
+            validate_fn=validate_with_db
+        )
+        ```
+
+    Args:
+        header_name (str): The name of the header containing the API key
+        expected_token (Optional[str]): The expected API key value for simple token matching
+        validate_fn (Optional[Callable]): Custom validation function that takes an API key
+            string and returns a boolean or object. Can be async.
+        error_status (int): HTTP status code to use for validation errors
+        error_message (str): Error message to return when validation fails
+    """
+
+    def __init__(
+        self,
+        header_name: str,
+        expected_token: Optional[str] = None,
+        validate_fn: Optional[Callable[[str], bool]] = None,
+        error_status: int = HTTP_401_UNAUTHORIZED,
+        error_message: str = "Invalid API key",
+    ):
+        # Create the APIKeyHeader as a class property
+        self.security_scheme = APIKeyHeader(name=header_name)
+        self.expected_token = expected_token
+        self.custom_validate_fn = validate_fn
+        self.error_status = error_status
+        self.error_message = error_message
+
+    async def default_validator(self, api_key: str) -> bool:
+        if not self.expected_token:
+            raise ValueError(
+                "Expected Token Required to be set when uisng API Key Validator default validation"
+            )
+        return secrets.compare_digest(api_key, self.expected_token)
+
+    async def __call__(
+        self, request: Request, api_key: str = Security(APIKeyHeader)
+    ) -> Any:
+        if api_key is None:
+            raise HTTPException(status_code=self.error_status, detail="Missing API key")
+
+        # Use custom validation if provided, otherwise use default equality check
+        validator = self.custom_validate_fn or self.default_validator
+        result = (
+            await validator(api_key)
+            if inspect.iscoroutinefunction(validator)
+            else validator(api_key)
+        )
+
+        if not result:
+            raise HTTPException(
+                status_code=self.error_status, detail=self.error_message
+            )
+
+        # Store validation result in request state if it's not just a boolean
+        if result is not True:
+            request.state.api_key = result
+
+        return result
+
+    def get_dependency(self):
+        """
+        Returns a callable dependency that FastAPI will recognize as a security scheme
+        """
+
+        async def validate_api_key(
+            request: Request, api_key: str = Security(self.security_scheme)
+        ) -> Any:
+            return await self(request, api_key)
+
+        # This helps FastAPI recognize it as a security dependency
+        validate_api_key.__name__ = f"validate_{self.security_scheme.model.name}"
+        return validate_api_key

autogpt_platform/autogpt_libs/autogpt_libs/feature_flag/client.py

@@ -13,7 +13,6 @@ from typing_extensions import ParamSpec
 from .config import SETTINGS
 
 logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.DEBUG)
 
 P = ParamSpec("P")
 T = TypeVar("T")
@@ -72,7 +71,7 @@ def feature_flag(
     """
 
     def decorator(
-        func: Callable[P, Union[T, Awaitable[T]]]
+        func: Callable[P, Union[T, Awaitable[T]]],
     ) -> Callable[P, Union[T, Awaitable[T]]]:
         @wraps(func)
         async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:

autogpt_platform/autogpt_libs/autogpt_libs/feature_flag/client_test.py

@@ -1,7 +1,8 @@
 import pytest
-from autogpt_libs.feature_flag.client import feature_flag, mock_flag_variation
 from ldclient import LDClient
 
+from autogpt_libs.feature_flag.client import feature_flag, mock_flag_variation
+
 
 @pytest.fixture
 def ld_client(mocker):

autogpt_platform/autogpt_libs/autogpt_libs/feature_flag/config.py

@@ -6,7 +6,7 @@ class Settings(BaseSettings):
     launch_darkly_sdk_key: str = Field(
         default="",
         description="The Launch Darkly SDK key",
-        validation_alias="LAUNCH_DARKLY_SDK_KEY"
+        validation_alias="LAUNCH_DARKLY_SDK_KEY",
     )
 
     model_config = SettingsConfigDict(case_sensitive=True, extra="ignore")

autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py

@@ -8,7 +8,7 @@ from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from .filters import BelowLevelFilter
-from .formatters import AGPTFormatter, StructuredLoggingFormatter
+from .formatters import AGPTFormatter
 
 LOG_DIR = Path(__file__).parent.parent.parent.parent / "logs"
 LOG_FILE = "activity.log"
@@ -18,12 +18,11 @@ ERROR_LOG_FILE = "error.log"
 SIMPLE_LOG_FORMAT = "%(asctime)s %(levelname)s  %(title)s%(message)s"
 
 DEBUG_LOG_FORMAT = (
-    "%(asctime)s %(levelname)s %(filename)s:%(lineno)d" "  %(title)s%(message)s"
+    "%(asctime)s %(levelname)s %(filename)s:%(lineno)d  %(title)s%(message)s"
 )
 
 
 class LoggingConfig(BaseSettings):
-
     level: str = Field(
         default="INFO",
         description="Logging level",
@@ -82,9 +81,26 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
     """
 
     config = LoggingConfig()
-
     log_handlers: list[logging.Handler] = []
 
+    # Console output handlers
+    stdout = logging.StreamHandler(stream=sys.stdout)
+    stdout.setLevel(config.level)
+    stdout.addFilter(BelowLevelFilter(logging.WARNING))
+    if config.level == logging.DEBUG:
+        stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
+    else:
+        stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
+
+    stderr = logging.StreamHandler()
+    stderr.setLevel(logging.WARNING)
+    if config.level == logging.DEBUG:
+        stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
+    else:
+        stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
+
+    log_handlers += [stdout, stderr]
+
     # Cloud logging setup
     if config.enable_cloud_logging or force_cloud_logging:
         import google.cloud.logging
@@ -98,28 +114,7 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
             transport=SyncTransport,
         )
         cloud_handler.setLevel(config.level)
-        cloud_handler.setFormatter(StructuredLoggingFormatter())
         log_handlers.append(cloud_handler)
-        print("Cloud logging enabled")
-    else:
-        # Console output handlers
-        stdout = logging.StreamHandler(stream=sys.stdout)
-        stdout.setLevel(config.level)
-        stdout.addFilter(BelowLevelFilter(logging.WARNING))
-        if config.level == logging.DEBUG:
-            stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
-        else:
-            stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
-
-        stderr = logging.StreamHandler()
-        stderr.setLevel(logging.WARNING)
-        if config.level == logging.DEBUG:
-            stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
-        else:
-            stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
-
-        log_handlers += [stdout, stderr]
-        print("Console logging enabled")
 
     # File logging setup
     if config.enable_file_logging:
@@ -157,7 +152,6 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
         error_log_handler.setLevel(logging.ERROR)
         error_log_handler.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT, no_color=True))
         log_handlers.append(error_log_handler)
-        print("File logging enabled")
 
     # Configure the root logger
     logging.basicConfig(

autogpt_platform/autogpt_libs/autogpt_libs/logging/formatters.py

@@ -1,7 +1,6 @@
 import logging
 
 from colorama import Fore, Style
-from google.cloud.logging_v2.handlers import CloudLoggingFilter, StructuredLogHandler
 
 from .utils import remove_color_codes
 
@@ -80,16 +79,3 @@ class AGPTFormatter(FancyConsoleFormatter):
             return remove_color_codes(super().format(record))
         else:
             return super().format(record)
-
-
-class StructuredLoggingFormatter(StructuredLogHandler, logging.Formatter):
-    def __init__(self):
-        # Set up CloudLoggingFilter to add diagnostic info to the log records
-        self.cloud_logging_filter = CloudLoggingFilter()
-
-        # Init StructuredLogHandler
-        super().__init__()
-
-    def format(self, record: logging.LogRecord) -> str:
-        self.cloud_logging_filter.filter(record)
-        return super().format(record)

autogpt_platform/autogpt_libs/autogpt_libs/logging/test_utils.py

@@ -24,10 +24,10 @@ from .utils import remove_color_codes
         ),
         ("", ""),
         ("hello", "hello"),
-        ("hello\x1B[31m world", "hello world"),
-        ("\x1B[36mHello,\x1B[32m World!", "Hello, World!"),
+        ("hello\x1b[31m world", "hello world"),
+        ("\x1b[36mHello,\x1b[32m World!", "Hello, World!"),
         (
-            "\x1B[1m\x1B[31mError:\x1B[0m\x1B[31m file not found",
+            "\x1b[1m\x1b[31mError:\x1b[0m\x1b[31m file not found",
             "Error: file not found",
         ),
     ],

autogpt_platform/autogpt_libs/autogpt_libs/logging/utils.py

@@ -2,6 +2,7 @@ import logging
 import re
 from typing import Any
 
+import uvicorn.config
 from colorama import Fore
 
 
@@ -25,3 +26,14 @@ def print_attribute(
             "color": value_color,
         },
     )
+
+
+def generate_uvicorn_config():
+    """
+    Generates a uvicorn logging config that silences uvicorn's default logging and tells it to use the native logging module.
+    """
+    log_config = dict(uvicorn.config.LOGGING_CONFIG)
+    log_config["loggers"]["uvicorn"] = {"handlers": []}
+    log_config["loggers"]["uvicorn.error"] = {"handlers": []}
+    log_config["loggers"]["uvicorn.access"] = {"handlers": []}
+    return log_config

autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/config.py

@@ -0,0 +1,31 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class RateLimitSettings(BaseSettings):
+    redis_host: str = Field(
+        default="redis://localhost:6379",
+        description="Redis host",
+        validation_alias="REDIS_HOST",
+    )
+
+    redis_port: str = Field(
+        default="6379", description="Redis port", validation_alias="REDIS_PORT"
+    )
+
+    redis_password: str = Field(
+        default="password",
+        description="Redis password",
+        validation_alias="REDIS_PASSWORD",
+    )
+
+    requests_per_minute: int = Field(
+        default=60,
+        description="Maximum number of requests allowed per minute per API key",
+        validation_alias="RATE_LIMIT_REQUESTS_PER_MINUTE",
+    )
+
+    model_config = SettingsConfigDict(case_sensitive=True, extra="ignore")
+
+
+RATE_LIMIT_SETTINGS = RateLimitSettings()

autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/limiter.py

@@ -0,0 +1,51 @@
+import time
+from typing import Tuple
+
+from redis import Redis
+
+from .config import RATE_LIMIT_SETTINGS
+
+
+class RateLimiter:
+    def __init__(
+        self,
+        redis_host: str = RATE_LIMIT_SETTINGS.redis_host,
+        redis_port: str = RATE_LIMIT_SETTINGS.redis_port,
+        redis_password: str = RATE_LIMIT_SETTINGS.redis_password,
+        requests_per_minute: int = RATE_LIMIT_SETTINGS.requests_per_minute,
+    ):
+        self.redis = Redis(
+            host=redis_host,
+            port=int(redis_port),
+            password=redis_password,
+            decode_responses=True,
+        )
+        self.window = 60
+        self.max_requests = requests_per_minute
+
+    async def check_rate_limit(self, api_key_id: str) -> Tuple[bool, int, int]:
+        """
+        Check if request is within rate limits.
+
+        Args:
+            api_key_id: The API key identifier to check
+
+        Returns:
+            Tuple of (is_allowed, remaining_requests, reset_time)
+        """
+        now = time.time()
+        window_start = now - self.window
+        key = f"ratelimit:{api_key_id}:1min"
+
+        pipe = self.redis.pipeline()
+        pipe.zremrangebyscore(key, 0, window_start)
+        pipe.zadd(key, {str(now): now})
+        pipe.zcount(key, window_start, now)
+        pipe.expire(key, self.window)
+
+        _, _, request_count, _ = pipe.execute()
+
+        remaining = max(0, self.max_requests - request_count)
+        reset_time = int(now + self.window)
+
+        return request_count <= self.max_requests, remaining, reset_time

autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/middleware.py

@@ -0,0 +1,32 @@
+from fastapi import HTTPException, Request
+from starlette.middleware.base import RequestResponseEndpoint
+
+from .limiter import RateLimiter
+
+
+async def rate_limit_middleware(request: Request, call_next: RequestResponseEndpoint):
+    """FastAPI middleware for rate limiting API requests."""
+    limiter = RateLimiter()
+
+    if not request.url.path.startswith("/api"):
+        return await call_next(request)
+
+    api_key = request.headers.get("Authorization")
+    if not api_key:
+        return await call_next(request)
+
+    api_key = api_key.replace("Bearer ", "")
+
+    is_allowed, remaining, reset_time = await limiter.check_rate_limit(api_key)
+
+    if not is_allowed:
+        raise HTTPException(
+            status_code=429, detail="Rate limit exceeded. Please try again later."
+        )
+
+    response = await call_next(request)
+    response.headers["X-RateLimit-Limit"] = str(limiter.max_requests)
+    response.headers["X-RateLimit-Remaining"] = str(remaining)
+    response.headers["X-RateLimit-Reset"] = str(reset_time)
+
+    return response

autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py

@@ -0,0 +1,76 @@
+from typing import Annotated, Any, Literal, Optional, TypedDict
+from uuid import uuid4
+
+from pydantic import BaseModel, Field, SecretStr, field_serializer
+
+
+class _BaseCredentials(BaseModel):
+    id: str = Field(default_factory=lambda: str(uuid4()))
+    provider: str
+    title: Optional[str]
+
+    @field_serializer("*")
+    def dump_secret_strings(value: Any, _info):
+        if isinstance(value, SecretStr):
+            return value.get_secret_value()
+        return value
+
+
+class OAuth2Credentials(_BaseCredentials):
+    type: Literal["oauth2"] = "oauth2"
+    username: Optional[str]
+    """Username of the third-party service user that these credentials belong to"""
+    access_token: SecretStr
+    access_token_expires_at: Optional[int]
+    """Unix timestamp (seconds) indicating when the access token expires (if at all)"""
+    refresh_token: Optional[SecretStr]
+    refresh_token_expires_at: Optional[int]
+    """Unix timestamp (seconds) indicating when the refresh token expires (if at all)"""
+    scopes: list[str]
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    def bearer(self) -> str:
+        return f"Bearer {self.access_token.get_secret_value()}"
+
+
+class APIKeyCredentials(_BaseCredentials):
+    type: Literal["api_key"] = "api_key"
+    api_key: SecretStr
+    expires_at: Optional[int]
+    """Unix timestamp (seconds) indicating when the API key expires (if at all)"""
+
+    def bearer(self) -> str:
+        return f"Bearer {self.api_key.get_secret_value()}"
+
+
+Credentials = Annotated[
+    OAuth2Credentials | APIKeyCredentials,
+    Field(discriminator="type"),
+]
+
+
+CredentialsType = Literal["api_key", "oauth2"]
+
+
+class OAuthState(BaseModel):
+    token: str
+    provider: str
+    expires_at: int
+    code_verifier: Optional[str] = None
+    scopes: list[str]
+    """Unix timestamp (seconds) indicating when this OAuth state expires"""
+
+
+class UserMetadata(BaseModel):
+    integration_credentials: list[Credentials] = Field(default_factory=list)
+    integration_oauth_states: list[OAuthState] = Field(default_factory=list)
+
+
+class UserMetadataRaw(TypedDict, total=False):
+    integration_credentials: list[dict]
+    integration_oauth_states: list[dict]
+
+
+class UserIntegrations(BaseModel):
+    credentials: list[Credentials] = Field(default_factory=list)
+    oauth_states: list[OAuthState] = Field(default_factory=list)

autogpt_platform/autogpt_libs/autogpt_libs/utils/cache.py

@@ -1,20 +1,59 @@
+import inspect
 import threading
-from typing import Callable, ParamSpec, TypeVar
+from typing import Awaitable, Callable, ParamSpec, TypeVar, cast, overload
 
 P = ParamSpec("P")
 R = TypeVar("R")
 
 
-def thread_cached(func: Callable[P, R]) -> Callable[P, R]:
+@overload
+def thread_cached(func: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[R]]: ...
+
+
+@overload
+def thread_cached(func: Callable[P, R]) -> Callable[P, R]: ...
+
+
+def thread_cached(
+    func: Callable[P, R] | Callable[P, Awaitable[R]],
+) -> Callable[P, R] | Callable[P, Awaitable[R]]:
     thread_local = threading.local()
 
-    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
-        cache = getattr(thread_local, "cache", None)
-        if cache is None:
-            cache = thread_local.cache = {}
-        key = (args, tuple(sorted(kwargs.items())))
-        if key not in cache:
-            cache[key] = func(*args, **kwargs)
-        return cache[key]
+    def _clear():
+        if hasattr(thread_local, "cache"):
+            del thread_local.cache
 
-    return wrapper
+    if inspect.iscoroutinefunction(func):
+
+        async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+            cache = getattr(thread_local, "cache", None)
+            if cache is None:
+                cache = thread_local.cache = {}
+            key = (args, tuple(sorted(kwargs.items())))
+            if key not in cache:
+                cache[key] = await cast(Callable[P, Awaitable[R]], func)(
+                    *args, **kwargs
+                )
+            return cache[key]
+
+        setattr(async_wrapper, "clear_cache", _clear)
+        return async_wrapper
+
+    else:
+
+        def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+            cache = getattr(thread_local, "cache", None)
+            if cache is None:
+                cache = thread_local.cache = {}
+            key = (args, tuple(sorted(kwargs.items())))
+            if key not in cache:
+                cache[key] = func(*args, **kwargs)
+            return cache[key]
+
+        setattr(sync_wrapper, "clear_cache", _clear)
+        return sync_wrapper
+
+
+def clear_thread_cache(func: Callable) -> None:
+    if clear := getattr(func, "clear_cache", None):
+        clear()

autogpt_platform/autogpt_libs/autogpt_libs/utils/synchronize.py

@@ -1,15 +1,15 @@
-from contextlib import contextmanager
-from threading import Lock
+import asyncio
+from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Any
 
 from expiringdict import ExpiringDict
 
 if TYPE_CHECKING:
-    from redis import Redis
-    from redis.lock import Lock as RedisLock
+    from redis.asyncio import Redis as AsyncRedis
+    from redis.asyncio.lock import Lock as AsyncRedisLock
 
 
-class RedisKeyedMutex:
+class AsyncRedisKeyedMutex:
     """
     This class provides a mutex that can be locked and unlocked by a specific key,
     using Redis as a distributed locking provider.
@@ -17,40 +17,45 @@ class RedisKeyedMutex:
     in case the key is not unlocked for a specified duration, to prevent memory leaks.
     """
 
-    def __init__(self, redis: "Redis", timeout: int | None = 60):
+    def __init__(self, redis: "AsyncRedis", timeout: int | None = 60):
         self.redis = redis
         self.timeout = timeout
-        self.locks: dict[Any, "RedisLock"] = ExpiringDict(
+        self.locks: dict[Any, "AsyncRedisLock"] = ExpiringDict(
             max_len=6000, max_age_seconds=self.timeout
         )
-        self.locks_lock = Lock()
+        self.locks_lock = asyncio.Lock()
 
-    @contextmanager
-    def locked(self, key: Any):
-        lock = self.acquire(key)
+    @asynccontextmanager
+    async def locked(self, key: Any):
+        lock = await self.acquire(key)
         try:
             yield
         finally:
-            lock.release()
+            if (await lock.locked()) and (await lock.owned()):
+                await lock.release()
 
-    def acquire(self, key: Any) -> "RedisLock":
+    async def acquire(self, key: Any) -> "AsyncRedisLock":
         """Acquires and returns a lock with the given key"""
-        with self.locks_lock:
+        async with self.locks_lock:
             if key not in self.locks:
                 self.locks[key] = self.redis.lock(
                     str(key), self.timeout, thread_local=False
                 )
             lock = self.locks[key]
-        lock.acquire()
+        await lock.acquire()
         return lock
 
-    def release(self, key: Any):
-        if lock := self.locks.get(key):
-            lock.release()
+    async def release(self, key: Any):
+        if (
+            (lock := self.locks.get(key))
+            and (await lock.locked())
+            and (await lock.owned())
+        ):
+            await lock.release()
 
-    def release_all_locks(self):
+    async def release_all_locks(self):
         """Call this on process termination to ensure all locks are released"""
-        self.locks_lock.acquire(blocking=False)
-        for lock in self.locks.values():
-            if lock.locked() and lock.owned():
-                lock.release()
+        async with self.locks_lock:
+            for lock in self.locks.values():
+                if (await lock.locked()) and (await lock.owned()):
+                    await lock.release()

autogpt_platform/autogpt_libs/pyproject.toml

@@ -7,21 +7,23 @@ readme = "README.md"
 packages = [{ include = "autogpt_libs" }]
 
 [tool.poetry.dependencies]
+python = ">=3.10,<4.0"
 colorama = "^0.4.6"
 expiringdict = "^1.2.2"
-google-cloud-logging = "^3.11.3"
-pydantic = "^2.10.2"
-pydantic-settings = "^2.6.1"
-pyjwt = "^2.10.0"
-pytest-asyncio = "^0.24.0"
+google-cloud-logging = "^3.12.1"
+pydantic = "^2.11.4"
+pydantic-settings = "^2.9.1"
+pyjwt = "^2.10.1"
+pytest-asyncio = "^0.26.0"
 pytest-mock = "^3.14.0"
-python = ">=3.10,<4.0"
-python-dotenv = "^1.0.1"
-supabase = "^2.10.0"
+supabase = "^2.15.1"
+launchdarkly-server-sdk = "^9.11.1"
+fastapi = "^0.115.12"
+uvicorn = "^0.34.3"
 
 [tool.poetry.group.dev.dependencies]
-redis = "^5.2.0"
-ruff = "^0.8.0"
+redis = "^5.2.1"
+ruff = "^0.11.10"
 
 [build-system]
 requires = ["poetry-core"]

autogpt_platform/backend/.env.example

@@ -2,19 +2,32 @@ DB_USER=postgres
 DB_PASS=your-super-secret-and-long-postgres-password
 DB_NAME=postgres
 DB_PORT=5432
-DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@localhost:${DB_PORT}/${DB_NAME}?connect_timeout=60&schema=platform"
+DB_HOST=localhost
+DB_CONNECTION_LIMIT=12
+DB_CONNECT_TIMEOUT=60
+DB_POOL_TIMEOUT=300
+DB_SCHEMA=platform
+DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}?schema=${DB_SCHEMA}&connect_timeout=${DB_CONNECT_TIMEOUT}"
+DIRECT_URL="postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}?schema=${DB_SCHEMA}&connect_timeout=${DB_CONNECT_TIMEOUT}"
 PRISMA_SCHEMA="postgres/schema.prisma"
 
+# EXECUTOR
+NUM_GRAPH_WORKERS=10
+
 BACKEND_CORS_ALLOW_ORIGINS=["http://localhost:3000"]
 
 # generate using `from cryptography.fernet import Fernet;Fernet.generate_key().decode()`
 ENCRYPTION_KEY='dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw='
+UNSUBSCRIBE_SECRET_KEY = 'HlP8ivStJjmbf6NKi78m_3FnOogut0t5ckzjsIqeaio='
 
 REDIS_HOST=localhost
 REDIS_PORT=6379
 REDIS_PASSWORD=password
 
 ENABLE_CREDIT=false
+STRIPE_API_KEY=
+STRIPE_WEBHOOK_SECRET=
+
 # What environment things should be logged under: local dev or prod
 APP_ENV=local
 # What environment to behave as: "local" or "cloud"
@@ -22,12 +35,26 @@ BEHAVE_AS=local
 PYRO_HOST=localhost
 SENTRY_DSN=
 
+# Email For Postmark so we can send emails
+POSTMARK_SERVER_API_TOKEN=
+POSTMARK_SENDER_EMAIL=invalid@invalid.com
+POSTMARK_WEBHOOK_TOKEN=
+
 ## User auth with Supabase is required for any of the 3rd party integrations with auth to work.
 ENABLE_AUTH=true
 SUPABASE_URL=http://localhost:8000
 SUPABASE_SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q
 SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long
 
+# RabbitMQ credentials -- Used for communication between services
+RABBITMQ_HOST=localhost
+RABBITMQ_PORT=5672
+RABBITMQ_DEFAULT_USER=rabbitmq_user_default
+RABBITMQ_DEFAULT_PASS=k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7
+
+## GCS bucket is required for marketplace and library functionality
+MEDIA_GCS_BUCKET_NAME=
+
 ## For local development, you may need to set FRONTEND_BASE_URL for the OAuth flow
 ## for integrations to work. Defaults to the value of PLATFORM_BASE_URL if not set.
 # FRONTEND_BASE_URL=http://localhost:3000
@@ -36,7 +63,14 @@ SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long
 ## to use the platform's webhook-related functionality.
 ## If you are developing locally, you can use something like ngrok to get a publc URL
 ## and tunnel it to your locally running backend.
-PLATFORM_BASE_URL=https://your-public-url-here
+PLATFORM_BASE_URL=http://localhost:3000
+
+## Cloudflare Turnstile (CAPTCHA) Configuration
+## Get these from the Cloudflare Turnstile dashboard: https://dash.cloudflare.com/?to=/:account/turnstile
+## This is the backend secret key
+TURNSTILE_SECRET_KEY=
+## This is the verify URL
+TURNSTILE_VERIFY_URL=https://challenges.cloudflare.com/turnstile/v0/siteverify
 
 ## == INTEGRATION CREDENTIALS == ##
 # Each set of server side credentials is required for the corresponding 3rd party
@@ -58,19 +92,52 @@ GITHUB_CLIENT_SECRET=
 GOOGLE_CLIENT_ID=
 GOOGLE_CLIENT_SECRET=
 
+# Twitter (X) OAuth 2.0 with PKCE Configuration
+# 1. Create a Twitter Developer Account:
+#    - Visit https://developer.x.com/en and sign up
+# 2. Set up your application:
+#    - Navigate to Developer Portal > Projects > Create Project
+#    - Add a new app to your project
+# 3. Configure app settings:
+#    - App Permissions: Read + Write + Direct Messages
+#    - App Type: Web App, Automated App or Bot
+#    - OAuth 2.0 Callback URL: http://localhost:3000/auth/integrations/oauth_callback
+#    - Save your Client ID and Client Secret below
+TWITTER_CLIENT_ID=
+TWITTER_CLIENT_SECRET=
+
+# Linear App
+# Make a new workspace for your OAuth APP -- trust me
+# https://linear.app/settings/api/applications/new
+# Callback URL: http://localhost:3000/auth/integrations/oauth_callback
+LINEAR_CLIENT_ID=
+LINEAR_CLIENT_SECRET=
+
+# To obtain Todoist API credentials:
+# 1. Create a Todoist account at todoist.com
+# 2. Visit the Developer Console: https://developer.todoist.com/appconsole.html
+# 3. Click "Create new app"
+# 4. Once created, copy your Client ID and Client Secret below
+TODOIST_CLIENT_ID=
+TODOIST_CLIENT_SECRET=
+
 ## ===== OPTIONAL API KEYS ===== ##
 
 # LLM
 OPENAI_API_KEY=
 ANTHROPIC_API_KEY=
+AIML_API_KEY=
 GROQ_API_KEY=
 OPEN_ROUTER_API_KEY=
+LLAMA_API_KEY=
 
 # Reddit
+# Go to https://www.reddit.com/prefs/apps and create a new app
+# Choose "script" for the type
+# Fill in the redirect uri as <your_frontend_url>/auth/integrations/oauth_callback, e.g. http://localhost:3000/auth/integrations/oauth_callback
 REDDIT_CLIENT_ID=
 REDDIT_CLIENT_SECRET=
-REDDIT_USERNAME=
-REDDIT_PASSWORD=
+REDDIT_USER_AGENT="AutoGPT:1.0 (by /u/autogpt)"
 
 # Discord
 DISCORD_BOT_TOKEN=
@@ -106,6 +173,32 @@ REPLICATE_API_KEY=
 # Ideogram
 IDEOGRAM_API_KEY=
 
+# Fal
+FAL_API_KEY=
+
+# Exa
+EXA_API_KEY=
+
+# E2B
+E2B_API_KEY=
+
+# Mem0
+MEM0_API_KEY=
+
+# Nvidia
+NVIDIA_API_KEY=
+
+# Apollo
+APOLLO_API_KEY=
+
+# SmartLead
+SMARTLEAD_API_KEY=
+
+# ZeroBounce
+ZEROBOUNCE_API_KEY=
+
+## ===== OPTIONAL API KEYS END ===== ##
+
 # Logging Configuration
 LOG_LEVEL=INFO
 ENABLE_CLOUD_LOGGING=false

autogpt_platform/backend/Dockerfile

@@ -6,17 +6,21 @@ ENV PYTHONUNBUFFERED 1
 
 WORKDIR /app
 
-# Install build dependencies
-RUN apt-get update \
-    && apt-get install -y build-essential curl ffmpeg wget libcurl4-gnutls-dev libexpat1-dev libpq5 gettext libz-dev libssl-dev postgresql-client git \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+RUN echo 'Acquire::http::Pipeline-Depth 0;\nAcquire::http::No-Cache true;\nAcquire::BrokenProxy true;\n' > /etc/apt/apt.conf.d/99fixbadproxy
 
-ENV POETRY_VERSION=1.8.3 \
-    POETRY_HOME="/opt/poetry" \
-    POETRY_NO_INTERACTION=1 \
-    POETRY_VIRTUALENVS_CREATE=false \
-    PATH="$POETRY_HOME/bin:$PATH"
+RUN apt-get update --allow-releaseinfo-change --fix-missing
+
+# Install build dependencies
+RUN apt-get install -y build-essential 
+RUN apt-get install -y libpq5
+RUN apt-get install -y libz-dev
+RUN apt-get install -y libssl-dev
+RUN apt-get install -y postgresql-client
+
+ENV POETRY_HOME=/opt/poetry
+ENV POETRY_NO_INTERACTION=1
+ENV POETRY_VIRTUALENVS_CREATE=false
+ENV PATH=/opt/poetry/bin:$PATH
 
 # Upgrade pip and setuptools to fix security vulnerabilities
 RUN pip3 install --upgrade pip setuptools
@@ -27,24 +31,20 @@ RUN pip3 install poetry
 COPY autogpt_platform/autogpt_libs /app/autogpt_platform/autogpt_libs
 COPY autogpt_platform/backend/poetry.lock autogpt_platform/backend/pyproject.toml /app/autogpt_platform/backend/
 WORKDIR /app/autogpt_platform/backend
-RUN poetry config virtualenvs.create false \
-    && poetry install --no-interaction --no-ansi
+RUN poetry install --no-ansi --no-root
 
 # Generate Prisma client
 COPY autogpt_platform/backend/schema.prisma ./
-RUN poetry config virtualenvs.create false \
-    && poetry run prisma generate
+RUN poetry run prisma generate
 
 FROM python:3.11.10-slim-bookworm AS server_dependencies
 
 WORKDIR /app
 
-ENV POETRY_VERSION=1.8.3 \
-    POETRY_HOME="/opt/poetry" \
+ENV POETRY_HOME=/opt/poetry \
     POETRY_NO_INTERACTION=1 \
-    POETRY_VIRTUALENVS_CREATE=false \
-    PATH="$POETRY_HOME/bin:$PATH"
-
+    POETRY_VIRTUALENVS_CREATE=false
+ENV PATH=/opt/poetry/bin:$PATH
 
 # Upgrade pip and setuptools to fix security vulnerabilities
 RUN pip3 install --upgrade pip setuptools
@@ -71,8 +71,8 @@ WORKDIR /app/autogpt_platform/backend
 FROM server_dependencies AS server
 
 COPY autogpt_platform/backend /app/autogpt_platform/backend
+RUN poetry install --no-ansi --only-root
 
-ENV DATABASE_URL=""
 ENV PORT=8000
 
 CMD ["poetry", "run", "rest"]

autogpt_platform/backend/README.advanced.md

@@ -1,75 +1 @@
-# AutoGPT Agent Server Advanced set up
-
-This guide walks you through a dockerized set up, with an external DB (postgres)
-
-## Setup
-
-We use the Poetry to manage the dependencies. To set up the project, follow these steps inside this directory:
-
-0. Install Poetry
-    ```sh
-    pip install poetry
-    ```
-    
-1. Configure Poetry to use .venv in your project directory
-    ```sh
-    poetry config virtualenvs.in-project true
-    ```
-
-2. Enter the poetry shell
-
-   ```sh
-   poetry shell
-   ```
-   
-3. Install dependencies
-
-   ```sh
-   poetry install
-   ```
-   
-4. Copy .env.example to .env
-
-   ```sh
-   cp .env.example .env
-   ```
-
-5. Generate the Prisma client
-
-   ```sh
-   poetry run prisma generate
-   ```
-   
-
-   > In case Prisma generates the client for the global Python installation instead of the virtual environment, the current mitigation is to just uninstall the global Prisma package:
-   >
-   > ```sh
-   > pip uninstall prisma
-   > ```
-   >
-   > Then run the generation again. The path *should* look something like this:  
-   > `<some path>/pypoetry/virtualenvs/backend-TQIRSwR6-py3.12/bin/prisma`
-
-6. Run the postgres database from the /rnd folder
-
-   ```sh
-   cd autogpt_platform/
-   docker compose up -d
-   ```
-
-7. Run the migrations (from the backend folder)
-
-   ```sh
-   cd ../backend
-   prisma migrate deploy
-   ```
-
-## Running The Server
-
-### Starting the server directly
-
-Run the following command:
-
-```sh
-poetry run app
-```
+[Advanced Setup (Dev Branch)](https://dev-docs.agpt.co/platform/advanced_setup/#autogpt_agent_server_advanced_set_up)

autogpt_platform/backend/README.md

@@ -1,203 +1 @@
-# AutoGPT Agent Server 
-
-This is an initial project for creating the next generation of agent execution, which is an AutoGPT agent server.
-The agent server will enable the creation of composite multi-agent systems that utilize AutoGPT agents and other non-agent components as its primitives.
-
-## Docs
-
-You can access the docs for the [AutoGPT Agent Server here](https://docs.agpt.co/server/setup).
-
-## Setup
-
-We use the Poetry to manage the dependencies. To set up the project, follow these steps inside this directory:
-
-0. Install Poetry
-    ```sh
-    pip install poetry
-    ```
-    
-1. Configure Poetry to use .venv in your project directory
-    ```sh
-    poetry config virtualenvs.in-project true
-    ```
-
-2. Enter the poetry shell
-
-   ```sh
-   poetry shell
-   ```
-   
-3. Install dependencies
-
-   ```sh
-   poetry install
-   ```
-
-4. Copy .env.example to .env
-
-   ```sh
-   cp .env.example .env
-   ```
-   
-5. Generate the Prisma client
-
-   ```sh
-   poetry run prisma generate
-   ```
-   
-
-   > In case Prisma generates the client for the global Python installation instead of the virtual environment, the current mitigation is to just uninstall the global Prisma package:
-   >
-   > ```sh
-   > pip uninstall prisma
-   > ```
-   >
-   > Then run the generation again. The path *should* look something like this:  
-   > `<some path>/pypoetry/virtualenvs/backend-TQIRSwR6-py3.12/bin/prisma`
-
-6. Migrate the database. Be careful because this deletes current data in the database.
-
-   ```sh
-   docker compose up db -d
-   poetry run prisma migrate deploy
-   ```
-
-## Running The Server
-
-### Starting the server without Docker
-
-Run the following command to run database in docker but the application locally:
-
-```sh
-docker compose --profile local up deps --build --detach
-poetry run app
-```
-
-### Starting the server with Docker
-
-Run the following command to build the dockerfiles:
-
-```sh
-docker compose build
-```
-
-Run the following command to run the app:
-
-```sh
-docker compose up
-```
-
-Run the following to automatically rebuild when code changes, in another terminal:
-
-```sh
-docker compose watch
-```
-
-Run the following command to shut down:
-
-```sh
-docker compose down
-```
-
-If you run into issues with dangling orphans, try:
-
-```sh
-docker compose down --volumes --remove-orphans && docker-compose up --force-recreate --renew-anon-volumes --remove-orphans  
-```
-
-## Testing
-
-To run the tests:
-
-```sh
-poetry run test
-```
-
-## Development
-
-### Formatting & Linting
-Auto formatter and linter are set up in the project. To run them:
-
-Install:
-```sh
-poetry install --with dev
-```
-
-Format the code:
-```sh
-poetry run format
-```
-
-Lint the code:
-```sh
-poetry run lint
-```
-
-## Project Outline
-
-The current project has the following main modules:
-
-### **blocks**
-
-This module stores all the Agent Blocks, which are reusable components to build a graph that represents the agent's behavior.
-
-### **data**
-
-This module stores the logical model that is persisted in the database.
-It abstracts the database operations into functions that can be called by the service layer.
-Any code that interacts with Prisma objects or the database should reside in this module.
-The main models are:
-* `block`: anything related to the block used in the graph
-* `execution`: anything related to the execution graph execution
-* `graph`: anything related to the graph, node, and its relations
-
-### **execution**
-
-This module stores the business logic of executing the graph.
-It currently has the following main modules:
-* `manager`: A service that consumes the queue of the graph execution and executes the graph. It contains both pieces of logic.
-* `scheduler`: A service that triggers scheduled graph execution based on a cron expression. It pushes an execution request to the manager.
-
-### **server**
-
-This module stores the logic for the server API.
-It contains all the logic used for the API that allows the client to create, execute, and monitor the graph and its execution.
-This API service interacts with other services like those defined in `manager` and `scheduler`.
-
-### **utils**
-
-This module stores utility functions that are used across the project.
-Currently, it has two main modules:
-* `process`: A module that contains the logic to spawn a new process.
-* `service`: A module that serves as a parent class for all the services in the project.
-
-## Service Communication
-
-Currently, there are only 3 active services:
-
-- AgentServer (the API, defined in `server.py`)
-- ExecutionManager (the executor, defined in `manager.py`)
-- ExecutionScheduler (the scheduler, defined in `scheduler.py`)
-
-The services run in independent Python processes and communicate through an IPC.
-A communication layer (`service.py`) is created to decouple the communication library from the implementation.
-
-Currently, the IPC is done using Pyro5 and abstracted in a way that allows a function decorated with `@expose` to be called from a different process.
-
-
-By default the daemons run on the following ports: 
-
-Execution Manager Daemon: 8002
-Execution Scheduler Daemon: 8003
-Rest Server Daemon: 8004
-
-## Adding a New Agent Block
-
-To add a new agent block, you need to create a new class that inherits from `Block` and provides the following information:
-* All the block code should live in the `blocks` (`backend.blocks`) module.
-* `input_schema`: the schema of the input data, represented by a Pydantic object.
-* `output_schema`: the schema of the output data, represented by a Pydantic object.
-* `run` method: the main logic of the block.
-* `test_input` & `test_output`: the sample input and output data for the block, which will be used to auto-test the block.
-* You can mock the functions declared in the block using the `test_mock` field for your unit tests.
-* Once you finish creating the block, you can test it by running `pytest -s test/block/test_block.py`.
+[Getting Started (Released)](https://docs.agpt.co/platform/getting-started/#autogpt_agent_server)

autogpt_platform/backend/TESTING.md

@@ -0,0 +1,237 @@
+# Backend Testing Guide
+
+This guide covers testing practices for the AutoGPT Platform backend, with a focus on snapshot testing for API endpoints.
+
+## Table of Contents
+- [Overview](#overview)
+- [Running Tests](#running-tests)
+- [Snapshot Testing](#snapshot-testing)
+- [Writing Tests for API Routes](#writing-tests-for-api-routes)
+- [Best Practices](#best-practices)
+
+## Overview
+
+The backend uses pytest for testing with the following key libraries:
+- `pytest` - Test framework
+- `pytest-asyncio` - Async test support
+- `pytest-mock` - Mocking support
+- `pytest-snapshot` - Snapshot testing for API responses
+
+## Running Tests
+
+### Run all tests
+```bash
+poetry run test
+```
+
+### Run specific test file
+```bash
+poetry run pytest path/to/test_file.py
+```
+
+### Run with verbose output
+```bash
+poetry run pytest -v
+```
+
+### Run with coverage
+```bash
+poetry run pytest --cov=backend
+```
+
+## Snapshot Testing
+
+Snapshot testing captures the output of your code and compares it against previously saved snapshots. This is particularly useful for testing API responses.
+
+### How Snapshot Testing Works
+
+1. First run: Creates snapshot files in `snapshots/` directories
+2. Subsequent runs: Compares output against saved snapshots
+3. Changes detected: Test fails if output differs from snapshot
+
+### Creating/Updating Snapshots
+
+When you first write a test or when the expected output changes:
+
+```bash
+poetry run pytest path/to/test.py --snapshot-update
+```
+
+⚠️ **Important**: Always review snapshot changes before committing! Use `git diff` to verify the changes are expected.
+
+### Snapshot Test Example
+
+```python
+import json
+from pytest_snapshot.plugin import Snapshot
+
+def test_api_endpoint(snapshot: Snapshot):
+    response = client.get("/api/endpoint")
+    
+    # Snapshot the response
+    snapshot.snapshot_dir = "snapshots"
+snapshot.assert_match(
+        json.dumps(response.json(), indent=2, sort_keys=True),
+        "endpoint_response"
+    )
+```
+
+### Best Practices for Snapshots
+
+1. **Use descriptive names**: `"user_list_response"` not `"response1"`
+2. **Sort JSON keys**: Ensures consistent snapshots
+3. **Format JSON**: Use `indent=2` for readable diffs
+4. **Exclude dynamic data**: Remove timestamps, IDs, etc. that change between runs
+
+Example of excluding dynamic data:
+```python
+response_data = response.json()
+# Remove dynamic fields for snapshot
+response_data.pop("created_at", None)
+response_data.pop("id", None)
+
+snapshot.snapshot_dir = "snapshots"
+snapshot.assert_match(
+    json.dumps(response_data, indent=2, sort_keys=True),
+    "static_response_data"
+)
+```
+
+## Writing Tests for API Routes
+
+### Basic Structure
+
+```python
+import json
+import fastapi
+import fastapi.testclient
+import pytest
+from pytest_snapshot.plugin import Snapshot
+
+from backend.server.v2.myroute import router
+
+app = fastapi.FastAPI()
+app.include_router(router)
+client = fastapi.testclient.TestClient(app)
+
+def test_endpoint_success(snapshot: Snapshot):
+    response = client.get("/endpoint")
+    assert response.status_code == 200
+    
+    # Test specific fields
+    data = response.json()
+    assert data["status"] == "success"
+    
+    # Snapshot the full response
+    snapshot.snapshot_dir = "snapshots"
+    snapshot.assert_match(
+        json.dumps(data, indent=2, sort_keys=True),
+        "endpoint_success_response"
+    )
+```
+
+### Testing with Authentication
+
+```python
+def override_auth_middleware():
+    return {"sub": "test-user-id"}
+
+def override_get_user_id():
+    return "test-user-id"
+
+app.dependency_overrides[auth_middleware] = override_auth_middleware
+app.dependency_overrides[get_user_id] = override_get_user_id
+```
+
+### Mocking External Services
+
+```python
+def test_external_api_call(mocker, snapshot):
+    # Mock external service
+    mock_response = {"external": "data"}
+    mocker.patch(
+        "backend.services.external_api.call",
+        return_value=mock_response
+    )
+    
+    response = client.post("/api/process")
+    assert response.status_code == 200
+    
+    snapshot.snapshot_dir = "snapshots"
+    snapshot.assert_match(
+        json.dumps(response.json(), indent=2, sort_keys=True),
+        "process_with_external_response"
+    )
+```
+
+## Best Practices
+
+### 1. Test Organization
+- Place tests next to the code: `routes.py` → `routes_test.py`
+- Use descriptive test names: `test_create_user_with_invalid_email`
+- Group related tests in classes when appropriate
+
+### 2. Test Coverage
+- Test happy path and error cases
+- Test edge cases (empty data, invalid formats)
+- Test authentication and authorization
+
+### 3. Snapshot Testing Guidelines
+- Review all snapshot changes carefully
+- Don't snapshot sensitive data
+- Keep snapshots focused and minimal
+- Update snapshots intentionally, not accidentally
+
+### 4. Async Testing
+- Use regular `def` for FastAPI TestClient tests
+- Use `async def` with `@pytest.mark.asyncio` for testing async functions directly
+
+### 5. Fixtures
+Create reusable fixtures for common test data:
+
+```python
+@pytest.fixture
+def sample_user():
+    return {
+        "email": "test@example.com",
+        "name": "Test User"
+    }
+
+def test_create_user(sample_user, snapshot):
+    response = client.post("/users", json=sample_user)
+    # ... test implementation
+```
+
+## CI/CD Integration
+
+The GitHub Actions workflow automatically runs tests on:
+- Pull requests
+- Pushes to main branch
+
+Snapshot tests work in CI by:
+1. Committing snapshot files to the repository
+2. CI compares against committed snapshots
+3. Fails if snapshots don't match
+
+## Troubleshooting
+
+### Snapshot Mismatches
+- Review the diff carefully
+- If changes are expected: `poetry run pytest --snapshot-update`
+- If changes are unexpected: Fix the code causing the difference
+
+### Async Test Issues
+- Ensure async functions use `@pytest.mark.asyncio`
+- Use `AsyncMock` for mocking async functions
+- FastAPI TestClient handles async automatically
+
+### Import Errors
+- Check that all dependencies are in `pyproject.toml`
+- Run `poetry install` to ensure dependencies are installed
+- Verify import paths are correct
+
+## Summary
+
+Snapshot testing provides a powerful way to ensure API responses remain consistent. Combined with traditional assertions, it creates a robust test suite that catches regressions while remaining maintainable.
+
+Remember: Good tests are as important as good code!

autogpt_platform/backend/backend/app.py

@@ -1,22 +1,30 @@
+import logging
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from backend.util.process import AppProcess
 
+logger = logging.getLogger(__name__)
+
 
 def run_processes(*processes: "AppProcess", **kwargs):
     """
     Execute all processes in the app. The last process is run in the foreground.
+    Includes enhanced error handling and process lifecycle management.
     """
     try:
+        # Run all processes except the last one in the background.
         for process in processes[:-1]:
             process.start(background=True, **kwargs)
 
-        # Run the last process in the foreground
+        # Run the last process in the foreground.
         processes[-1].start(background=False, **kwargs)
     finally:
         for process in processes:
-            process.stop()
+            try:
+                process.stop()
+            except Exception as e:
+                logger.exception(f"[{process.service_name}] unable to stop: {e}")
 
 
 def main(**kwargs):
@@ -24,14 +32,16 @@ def main(**kwargs):
     Run all the processes required for the AutoGPT-server (REST and WebSocket APIs).
     """
 
-    from backend.executor import DatabaseManager, ExecutionManager, ExecutionScheduler
+    from backend.executor import DatabaseManager, ExecutionManager, Scheduler
+    from backend.notifications import NotificationManager
     from backend.server.rest_api import AgentServer
     from backend.server.ws_api import WebsocketServer
 
     run_processes(
         DatabaseManager(),
         ExecutionManager(),
-        ExecutionScheduler(),
+        Scheduler(),
+        NotificationManager(),
         WebsocketServer(),
         AgentServer(),
         **kwargs,

autogpt_platform/backend/backend/blocks/__init__.py

@@ -1,89 +1,99 @@
+import functools
 import importlib
 import os
 import re
 from pathlib import Path
-from typing import Type, TypeVar
-
-from backend.data.block import Block
-
-# Dynamically load all modules under backend.blocks
-AVAILABLE_MODULES = []
-current_dir = Path(__file__).parent
-modules = [
-    str(f.relative_to(current_dir))[:-3].replace(os.path.sep, ".")
-    for f in current_dir.rglob("*.py")
-    if f.is_file() and f.name != "__init__.py"
-]
-for module in modules:
-    if not re.match("^[a-z_.]+$", module):
-        raise ValueError(
-            f"Block module {module} error: module name must be lowercase, "
-            "separated by underscores, and contain only alphabet characters"
-        )
-
-    importlib.import_module(f".{module}", package=__name__)
-    AVAILABLE_MODULES.append(module)
-
-# Load all Block instances from the available modules
-AVAILABLE_BLOCKS: dict[str, Type[Block]] = {}
+from typing import TYPE_CHECKING, TypeVar
 
+if TYPE_CHECKING:
+    from backend.data.block import Block
 
 T = TypeVar("T")
 
 
-def all_subclasses(cls: Type[T]) -> list[Type[T]]:
+@functools.cache
+def load_all_blocks() -> dict[str, type["Block"]]:
+    from backend.data.block import Block
+
+    # Dynamically load all modules under backend.blocks
+    current_dir = Path(__file__).parent
+    modules = [
+        str(f.relative_to(current_dir))[:-3].replace(os.path.sep, ".")
+        for f in current_dir.rglob("*.py")
+        if f.is_file() and f.name != "__init__.py"
+    ]
+    for module in modules:
+        if not re.match("^[a-z0-9_.]+$", module):
+            raise ValueError(
+                f"Block module {module} error: module name must be lowercase, "
+                "and contain only alphanumeric characters and underscores."
+            )
+
+        importlib.import_module(f".{module}", package=__name__)
+
+    # Load all Block instances from the available modules
+    available_blocks: dict[str, type["Block"]] = {}
+    for block_cls in all_subclasses(Block):
+        class_name = block_cls.__name__
+
+        if class_name.endswith("Base"):
+            continue
+
+        if not class_name.endswith("Block"):
+            raise ValueError(
+                f"Block class {class_name} does not end with 'Block'. "
+                "If you are creating an abstract class, "
+                "please name the class with 'Base' at the end"
+            )
+
+        block = block_cls.create()
+
+        if not isinstance(block.id, str) or len(block.id) != 36:
+            raise ValueError(
+                f"Block ID {block.name} error: {block.id} is not a valid UUID"
+            )
+
+        if block.id in available_blocks:
+            raise ValueError(
+                f"Block ID {block.name} error: {block.id} is already in use"
+            )
+
+        input_schema = block.input_schema.model_fields
+        output_schema = block.output_schema.model_fields
+
+        # Make sure `error` field is a string in the output schema
+        if "error" in output_schema and output_schema["error"].annotation is not str:
+            raise ValueError(
+                f"{block.name} `error` field in output_schema must be a string"
+            )
+
+        # Ensure all fields in input_schema and output_schema are annotated SchemaFields
+        for field_name, field in [*input_schema.items(), *output_schema.items()]:
+            if field.annotation is None:
+                raise ValueError(
+                    f"{block.name} has a field {field_name} that is not annotated"
+                )
+            if field.json_schema_extra is None:
+                raise ValueError(
+                    f"{block.name} has a field {field_name} not defined as SchemaField"
+                )
+
+        for field in block.input_schema.model_fields.values():
+            if field.annotation is bool and field.default not in (True, False):
+                raise ValueError(
+                    f"{block.name} has a boolean field with no default value"
+                )
+
+        available_blocks[block.id] = block_cls
+
+    return available_blocks
+
+
+__all__ = ["load_all_blocks"]
+
+
+def all_subclasses(cls: type[T]) -> list[type[T]]:
     subclasses = cls.__subclasses__()
     for subclass in subclasses:
         subclasses += all_subclasses(subclass)
     return subclasses
-
-
-for block_cls in all_subclasses(Block):
-    name = block_cls.__name__
-
-    if block_cls.__name__.endswith("Base"):
-        continue
-
-    if not block_cls.__name__.endswith("Block"):
-        raise ValueError(
-            f"Block class {block_cls.__name__} does not end with 'Block', If you are creating an abstract class, please name the class with 'Base' at the end"
-        )
-
-    block = block_cls.create()
-
-    if not isinstance(block.id, str) or len(block.id) != 36:
-        raise ValueError(f"Block ID {block.name} error: {block.id} is not a valid UUID")
-
-    if block.id in AVAILABLE_BLOCKS:
-        raise ValueError(f"Block ID {block.name} error: {block.id} is already in use")
-
-    input_schema = block.input_schema.model_fields
-    output_schema = block.output_schema.model_fields
-
-    # Make sure `error` field is a string in the output schema
-    if "error" in output_schema and output_schema["error"].annotation is not str:
-        raise ValueError(
-            f"{block.name} `error` field in output_schema must be a string"
-        )
-
-    # Make sure all fields in input_schema and output_schema are annotated and has a value
-    for field_name, field in [*input_schema.items(), *output_schema.items()]:
-        if field.annotation is None:
-            raise ValueError(
-                f"{block.name} has a field {field_name} that is not annotated"
-            )
-        if field.json_schema_extra is None:
-            raise ValueError(
-                f"{block.name} has a field {field_name} not defined as SchemaField"
-            )
-
-    for field in block.input_schema.model_fields.values():
-        if field.annotation is bool and field.default not in (True, False):
-            raise ValueError(f"{block.name} has a boolean field with no default value")
-
-    if block.disabled:
-        continue
-
-    AVAILABLE_BLOCKS[block.id] = block_cls
-
-__all__ = ["AVAILABLE_MODULES", "AVAILABLE_BLOCKS"]

autogpt_platform/backend/backend/blocks/agent.py

@@ -1,6 +1,6 @@
+import asyncio
 import logging
-
-from autogpt_libs.utils.cache import thread_cached
+from typing import Any, Optional
 
 from backend.data.block import (
     Block,
@@ -12,36 +12,46 @@ from backend.data.block import (
     get_block,
 )
 from backend.data.execution import ExecutionStatus
-from backend.data.model import SchemaField
+from backend.data.model import CredentialsMetaInput, SchemaField
+from backend.util import json
 
 logger = logging.getLogger(__name__)
 
 
-@thread_cached
-def get_executor_manager_client():
-    from backend.executor import ExecutionManager
-    from backend.util.service import get_service_client
-
-    return get_service_client(ExecutionManager)
-
-
-@thread_cached
-def get_event_bus():
-    from backend.data.execution import RedisExecutionEventBus
-
-    return RedisExecutionEventBus()
-
-
 class AgentExecutorBlock(Block):
     class Input(BlockSchema):
         user_id: str = SchemaField(description="User ID")
         graph_id: str = SchemaField(description="Graph ID")
         graph_version: int = SchemaField(description="Graph Version")
+        agent_name: Optional[str] = SchemaField(
+            default=None, description="Name to display in the Builder UI"
+        )
 
-        data: BlockInput = SchemaField(description="Input data for the graph")
+        inputs: BlockInput = SchemaField(description="Input data for the graph")
         input_schema: dict = SchemaField(description="Input schema for the graph")
         output_schema: dict = SchemaField(description="Output schema for the graph")
 
+        node_credentials_input_map: Optional[
+            dict[str, dict[str, CredentialsMetaInput]]
+        ] = SchemaField(default=None, hidden=True)
+
+        @classmethod
+        def get_input_schema(cls, data: BlockInput) -> dict[str, Any]:
+            return data.get("input_schema", {})
+
+        @classmethod
+        def get_input_defaults(cls, data: BlockInput) -> BlockInput:
+            return data.get("inputs", {})
+
+        @classmethod
+        def get_missing_input(cls, data: BlockInput) -> set[str]:
+            required_fields = cls.get_input_schema(data).get("required", [])
+            return set(required_fields) - set(data)
+
+        @classmethod
+        def get_mismatch_error(cls, data: BlockInput) -> str | None:
+            return json.validate_with_jsonschema(cls.get_input_schema(data), data)
+
     class Output(BlockSchema):
         pass
 
@@ -55,33 +65,83 @@ class AgentExecutorBlock(Block):
             categories={BlockCategory.AGENT},
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
-        executor_manager = get_executor_manager_client()
-        event_bus = get_event_bus()
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
 
-        graph_exec = executor_manager.add_execution(
+        from backend.executor import utils as execution_utils
+
+        graph_exec = await execution_utils.add_graph_execution(
             graph_id=input_data.graph_id,
             graph_version=input_data.graph_version,
             user_id=input_data.user_id,
-            data=input_data.data,
+            inputs=input_data.inputs,
+            node_credentials_input_map=input_data.node_credentials_input_map,
+            use_db_query=False,
         )
-        log_id = f"Graph #{input_data.graph_id}-V{input_data.graph_version}, exec-id: {graph_exec.graph_exec_id}"
+
+        try:
+            async for name, data in self._run(
+                graph_id=input_data.graph_id,
+                graph_version=input_data.graph_version,
+                graph_exec_id=graph_exec.id,
+                user_id=input_data.user_id,
+            ):
+                yield name, data
+        except asyncio.CancelledError:
+            logger.warning(
+                f"Execution of graph {input_data.graph_id} version {input_data.graph_version} was cancelled."
+            )
+            await execution_utils.stop_graph_execution(
+                graph_exec.id, use_db_query=False
+            )
+        except Exception as e:
+            logger.error(
+                f"Execution of graph {input_data.graph_id} version {input_data.graph_version} failed: {e}, stopping execution."
+            )
+            await execution_utils.stop_graph_execution(
+                graph_exec.id, use_db_query=False
+            )
+            raise
+
+    async def _run(
+        self,
+        graph_id: str,
+        graph_version: int,
+        graph_exec_id: str,
+        user_id: str,
+    ) -> BlockOutput:
+
+        from backend.data.execution import ExecutionEventType
+        from backend.executor import utils as execution_utils
+
+        event_bus = execution_utils.get_async_execution_event_bus()
+
+        log_id = f"Graph #{graph_id}-V{graph_version}, exec-id: {graph_exec_id}"
         logger.info(f"Starting execution of {log_id}")
 
-        for event in event_bus.listen(
-            graph_id=graph_exec.graph_id, graph_exec_id=graph_exec.graph_exec_id
+        async for event in event_bus.listen(
+            user_id=user_id,
+            graph_id=graph_id,
+            graph_exec_id=graph_exec_id,
         ):
-            logger.info(
+            if event.status not in [
+                ExecutionStatus.COMPLETED,
+                ExecutionStatus.TERMINATED,
+                ExecutionStatus.FAILED,
+            ]:
+                logger.debug(
+                    f"Execution {log_id} received event {event.event_type} with status {event.status}"
+                )
+                continue
+
+            if event.event_type == ExecutionEventType.GRAPH_EXEC_UPDATE:
+                # If the graph execution is COMPLETED, TERMINATED, or FAILED,
+                # we can stop listening for further events.
+                break
+
+            logger.debug(
                 f"Execution {log_id} produced input {event.input_data} output {event.output_data}"
             )
 
-            if not event.node_id:
-                if event.status in [ExecutionStatus.COMPLETED, ExecutionStatus.FAILED]:
-                    logger.info(f"Execution {log_id} ended with status {event.status}")
-                    break
-                else:
-                    continue
-
             if not event.block_id:
                 logger.warning(f"{log_id} received event without block_id {event}")
                 continue
@@ -96,5 +156,7 @@ class AgentExecutorBlock(Block):
                 continue
 
             for output_data in event.output_data.get("output", []):
-                logger.info(f"Execution {log_id} produced {output_name}: {output_data}")
+                logger.debug(
+                    f"Execution {log_id} produced {output_name}: {output_data}"
+                )
                 yield output_name, output_data

autogpt_platform/backend/backend/blocks/ai_image_generator_block.py

@@ -1,8 +1,8 @@
 from enum import Enum
 from typing import Literal
 
-import replicate
 from pydantic import SecretStr
+from replicate.client import Client as ReplicateClient
 from replicate.helpers import FileOutput
 
 from backend.data.block import Block, BlockCategory, BlockSchema
@@ -12,6 +12,7 @@ from backend.data.model import (
     CredentialsMetaInput,
     SchemaField,
 )
+from backend.integrations.providers import ProviderName
 
 
 class ImageSize(str, Enum):
@@ -101,12 +102,10 @@ class ImageGenModel(str, Enum):
 
 class AIImageGeneratorBlock(Block):
     class Input(BlockSchema):
-        credentials: CredentialsMetaInput[Literal["replicate"], Literal["api_key"]] = (
-            CredentialsField(
-                provider="replicate",
-                supported_credential_types={"api_key"},
-                description="Enter your Replicate API key to access the image generation API. You can obtain an API key from https://replicate.com/account/api-tokens.",
-            )
+        credentials: CredentialsMetaInput[
+            Literal[ProviderName.REPLICATE], Literal["api_key"]
+        ] = CredentialsField(
+            description="Enter your Replicate API key to access the image generation API. You can obtain an API key from https://replicate.com/account/api-tokens.",
         )
         prompt: str = SchemaField(
             description="Text prompt for image generation",
@@ -166,15 +165,15 @@ class AIImageGeneratorBlock(Block):
             },
         )
 
-    def _run_client(
+    async def _run_client(
         self, credentials: APIKeyCredentials, model_name: str, input_params: dict
     ):
         try:
             # Initialize Replicate client
-            client = replicate.Client(api_token=credentials.api_key.get_secret_value())
+            client = ReplicateClient(api_token=credentials.api_key.get_secret_value())
 
             # Run the model with input parameters
-            output = client.run(model_name, input=input_params, wait=False)
+            output = await client.async_run(model_name, input=input_params, wait=False)
 
             # Process output
             if isinstance(output, list) and len(output) > 0:
@@ -196,7 +195,7 @@ class AIImageGeneratorBlock(Block):
         except Exception as e:
             raise RuntimeError(f"Unexpected error during model execution: {e}")
 
-    def generate_image(self, input_data: Input, credentials: APIKeyCredentials):
+    async def generate_image(self, input_data: Input, credentials: APIKeyCredentials):
         try:
             # Handle style-based prompt modification for models without native style support
             modified_prompt = input_data.prompt
@@ -214,7 +213,7 @@ class AIImageGeneratorBlock(Block):
                     "steps": 40,
                     "cfg_scale": 7.0,
                 }
-                output = self._run_client(
+                output = await self._run_client(
                     credentials,
                     "stability-ai/stable-diffusion-3.5-medium",
                     input_params,
@@ -232,7 +231,7 @@ class AIImageGeneratorBlock(Block):
                     "output_format": "jpg",  # Set to jpg for Flux models
                     "output_quality": 90,
                 }
-                output = self._run_client(
+                output = await self._run_client(
                     credentials, "black-forest-labs/flux-1.1-pro", input_params
                 )
                 return output
@@ -247,7 +246,7 @@ class AIImageGeneratorBlock(Block):
                     "output_format": "jpg",
                     "output_quality": 90,
                 }
-                output = self._run_client(
+                output = await self._run_client(
                     credentials, "black-forest-labs/flux-1.1-pro-ultra", input_params
                 )
                 return output
@@ -258,7 +257,7 @@ class AIImageGeneratorBlock(Block):
                     "size": SIZE_TO_RECRAFT_DIMENSIONS[input_data.size],
                     "style": input_data.style.value,
                 }
-                output = self._run_client(
+                output = await self._run_client(
                     credentials, "recraft-ai/recraft-v3", input_params
                 )
                 return output
@@ -297,9 +296,9 @@ class AIImageGeneratorBlock(Block):
         style_text = style_map.get(style, "")
         return f"{style_text} of" if style_text else ""
 
-    def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs):
+    async def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs):
         try:
-            url = self.generate_image(input_data, credentials)
+            url = await self.generate_image(input_data, credentials)
             if url:
                 yield "image_url", url
             else:

autogpt_platform/backend/backend/blocks/ai_music_generator.py

@@ -1,10 +1,10 @@
+import asyncio
 import logging
-import time
 from enum import Enum
 from typing import Literal
 
-import replicate
 from pydantic import SecretStr
+from replicate.client import Client as ReplicateClient
 
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 from backend.data.model import (
@@ -13,6 +13,7 @@ from backend.data.model import (
     CredentialsMetaInput,
     SchemaField,
 )
+from backend.integrations.providers import ProviderName
 
 logger = logging.getLogger(__name__)
 
@@ -54,13 +55,11 @@ class NormalizationStrategy(str, Enum):
 
 class AIMusicGeneratorBlock(Block):
     class Input(BlockSchema):
-        credentials: CredentialsMetaInput[Literal["replicate"], Literal["api_key"]] = (
-            CredentialsField(
-                provider="replicate",
-                supported_credential_types={"api_key"},
-                description="The Replicate integration can be used with "
-                "any API key with sufficient permissions for the blocks it is used on.",
-            )
+        credentials: CredentialsMetaInput[
+            Literal[ProviderName.REPLICATE], Literal["api_key"]
+        ] = CredentialsField(
+            description="The Replicate integration can be used with "
+            "any API key with sufficient permissions for the blocks it is used on.",
         )
         prompt: str = SchemaField(
             description="A description of the music you want to generate",
@@ -143,7 +142,7 @@ class AIMusicGeneratorBlock(Block):
             test_credentials=TEST_CREDENTIALS,
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
         max_retries = 3
@@ -155,7 +154,7 @@ class AIMusicGeneratorBlock(Block):
                 logger.debug(
                     f"[AIMusicGeneratorBlock] - Running model (attempt {attempt + 1})"
                 )
-                result = self.run_model(
+                result = await self.run_model(
                     api_key=credentials.api_key,
                     music_gen_model_version=input_data.music_gen_model_version,
                     prompt=input_data.prompt,
@@ -177,13 +176,13 @@ class AIMusicGeneratorBlock(Block):
                 last_error = f"Unexpected error: {str(e)}"
                 logger.error(f"[AIMusicGeneratorBlock] - Error: {last_error}")
                 if attempt < max_retries - 1:
-                    time.sleep(retry_delay)
+                    await asyncio.sleep(retry_delay)
                     continue
 
         # If we've exhausted all retries, yield the error
         yield "error", f"Failed after {max_retries} attempts. Last error: {last_error}"
 
-    def run_model(
+    async def run_model(
         self,
         api_key: SecretStr,
         music_gen_model_version: MusicGenModelVersion,
@@ -197,10 +196,10 @@ class AIMusicGeneratorBlock(Block):
         normalization_strategy: NormalizationStrategy,
     ):
         # Initialize Replicate client with the API key
-        client = replicate.Client(api_token=api_key.get_secret_value())
+        client = ReplicateClient(api_token=api_key.get_secret_value())
 
         # Run the model with parameters
-        output = client.run(
+        output = await client.async_run(
             "meta/musicgen:671ac645ce5e552cc63a54a2bbff63fcf798043055d2dac5fc9e36a837eedcfb",
             input={
                 "prompt": prompt,

autogpt_platform/backend/backend/blocks/ai_shortform_video_block.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import time
 from enum import Enum
@@ -12,7 +13,8 @@ from backend.data.model import (
     CredentialsMetaInput,
     SchemaField,
 )
-from backend.util.request import requests
+from backend.integrations.providers import ProviderName
+from backend.util.request import Requests
 
 TEST_CREDENTIALS = APIKeyCredentials(
     id="01234567-89ab-cdef-0123-456789abcdef",
@@ -140,13 +142,11 @@ logger = logging.getLogger(__name__)
 
 class AIShortformVideoCreatorBlock(Block):
     class Input(BlockSchema):
-        credentials: CredentialsMetaInput[Literal["revid"], Literal["api_key"]] = (
-            CredentialsField(
-                provider="revid",
-                supported_credential_types={"api_key"},
-                description="The revid.ai integration can be used with "
-                "any API key with sufficient permissions for the blocks it is used on.",
-            )
+        credentials: CredentialsMetaInput[
+            Literal[ProviderName.REVID], Literal["api_key"]
+        ] = CredentialsField(
+            description="The revid.ai integration can be used with "
+            "any API key with sufficient permissions for the blocks it is used on.",
         )
         script: str = SchemaField(
             description="""1. Use short and punctuated sentences\n\n2. Use linebreaks to create a new clip\n\n3. Text outside of brackets is spoken by the AI, and [text between brackets] will be used to guide the visual generation. For example, [close-up of a cat] will show a close-up of a cat.""",
@@ -217,29 +217,29 @@ class AIShortformVideoCreatorBlock(Block):
             test_credentials=TEST_CREDENTIALS,
         )
 
-    def create_webhook(self):
+    async def create_webhook(self):
         url = "https://webhook.site/token"
         headers = {"Accept": "application/json", "Content-Type": "application/json"}
-        response = requests.post(url, headers=headers)
+        response = await Requests().post(url, headers=headers)
         webhook_data = response.json()
         return webhook_data["uuid"], f"https://webhook.site/{webhook_data['uuid']}"
 
-    def create_video(self, api_key: SecretStr, payload: dict) -> dict:
+    async def create_video(self, api_key: SecretStr, payload: dict) -> dict:
         url = "https://www.revid.ai/api/public/v2/render"
         headers = {"key": api_key.get_secret_value()}
-        response = requests.post(url, json=payload, headers=headers)
+        response = await Requests().post(url, json=payload, headers=headers)
         logger.debug(
-            f"API Response Status Code: {response.status_code}, Content: {response.text}"
+            f"API Response Status Code: {response.status}, Content: {response.text}"
         )
         return response.json()
 
-    def check_video_status(self, api_key: SecretStr, pid: str) -> dict:
+    async def check_video_status(self, api_key: SecretStr, pid: str) -> dict:
         url = f"https://www.revid.ai/api/public/v2/status?pid={pid}"
         headers = {"key": api_key.get_secret_value()}
-        response = requests.get(url, headers=headers)
+        response = await Requests().get(url, headers=headers)
         return response.json()
 
-    def wait_for_video(
+    async def wait_for_video(
         self,
         api_key: SecretStr,
         pid: str,
@@ -248,7 +248,7 @@ class AIShortformVideoCreatorBlock(Block):
     ) -> str:
         start_time = time.time()
         while time.time() - start_time < max_wait_time:
-            status = self.check_video_status(api_key, pid)
+            status = await self.check_video_status(api_key, pid)
             logger.debug(f"Video status: {status}")
 
             if status.get("status") == "ready" and "videoUrl" in status:
@@ -261,16 +261,16 @@ class AIShortformVideoCreatorBlock(Block):
                 logger.error(f"Video creation failed: {status.get('message')}")
                 raise ValueError(f"Video creation failed: {status.get('message')}")
 
-            time.sleep(10)
+            await asyncio.sleep(10)
 
         logger.error("Video creation timed out")
         raise TimeoutError("Video creation timed out")
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
         # Create a new Webhook.site URL
-        webhook_token, webhook_url = self.create_webhook()
+        webhook_token, webhook_url = await self.create_webhook()
         logger.debug(f"Webhook URL: {webhook_url}")
 
         audio_url = input_data.background_music.audio_url
@@ -307,7 +307,7 @@ class AIShortformVideoCreatorBlock(Block):
         }
 
         logger.debug("Creating video...")
-        response = self.create_video(credentials.api_key, payload)
+        response = await self.create_video(credentials.api_key, payload)
         pid = response.get("pid")
 
         if not pid:
@@ -319,6 +319,8 @@ class AIShortformVideoCreatorBlock(Block):
             logger.debug(
                 f"Video created with project ID: {pid}. Waiting for completion..."
             )
-            video_url = self.wait_for_video(credentials.api_key, pid, webhook_token)
+            video_url = await self.wait_for_video(
+                credentials.api_key, pid, webhook_token
+            )
             logger.debug(f"Video ready: {video_url}")
             yield "video_url", video_url

autogpt_platform/backend/backend/blocks/apollo/_api.py

@@ -0,0 +1,112 @@
+import logging
+from typing import List
+
+from backend.blocks.apollo._auth import ApolloCredentials
+from backend.blocks.apollo.models import (
+    Contact,
+    Organization,
+    SearchOrganizationsRequest,
+    SearchOrganizationsResponse,
+    SearchPeopleRequest,
+    SearchPeopleResponse,
+)
+from backend.util.request import Requests
+
+logger = logging.getLogger(name=__name__)
+
+
+class ApolloClient:
+    """Client for the Apollo API"""
+
+    API_URL = "https://api.apollo.io/api/v1"
+
+    def __init__(self, credentials: ApolloCredentials):
+        self.credentials = credentials
+        self.requests = Requests()
+
+    def _get_headers(self) -> dict[str, str]:
+        return {"x-api-key": self.credentials.api_key.get_secret_value()}
+
+    async def search_people(self, query: SearchPeopleRequest) -> List[Contact]:
+        """Search for people in Apollo"""
+        response = await self.requests.get(
+            f"{self.API_URL}/mixed_people/search",
+            headers=self._get_headers(),
+            params=query.model_dump(exclude={"credentials", "max_results"}),
+        )
+        data = response.json()
+        parsed_response = SearchPeopleResponse(**data)
+        if parsed_response.pagination.total_entries == 0:
+            return []
+
+        people = parsed_response.people
+
+        # handle pagination
+        if (
+            query.max_results is not None
+            and query.max_results < parsed_response.pagination.total_entries
+            and len(people) < query.max_results
+        ):
+            while (
+                len(people) < query.max_results
+                and query.page < parsed_response.pagination.total_pages
+                and len(parsed_response.people) > 0
+            ):
+                query.page += 1
+                response = await self.requests.get(
+                    f"{self.API_URL}/mixed_people/search",
+                    headers=self._get_headers(),
+                    params=query.model_dump(exclude={"credentials", "max_results"}),
+                )
+                data = response.json()
+                parsed_response = SearchPeopleResponse(**data)
+                people.extend(parsed_response.people[: query.max_results - len(people)])
+
+        logger.info(f"Found {len(people)} people")
+        return people[: query.max_results] if query.max_results else people
+
+    async def search_organizations(
+        self, query: SearchOrganizationsRequest
+    ) -> List[Organization]:
+        """Search for organizations in Apollo"""
+        response = await self.requests.get(
+            f"{self.API_URL}/mixed_companies/search",
+            headers=self._get_headers(),
+            params=query.model_dump(exclude={"credentials", "max_results"}),
+        )
+        data = response.json()
+        parsed_response = SearchOrganizationsResponse(**data)
+        if parsed_response.pagination.total_entries == 0:
+            return []
+
+        organizations = parsed_response.organizations
+
+        # handle pagination
+        if (
+            query.max_results is not None
+            and query.max_results < parsed_response.pagination.total_entries
+            and len(organizations) < query.max_results
+        ):
+            while (
+                len(organizations) < query.max_results
+                and query.page < parsed_response.pagination.total_pages
+                and len(parsed_response.organizations) > 0
+            ):
+                query.page += 1
+                response = await self.requests.get(
+                    f"{self.API_URL}/mixed_companies/search",
+                    headers=self._get_headers(),
+                    params=query.model_dump(exclude={"credentials", "max_results"}),
+                )
+                data = response.json()
+                parsed_response = SearchOrganizationsResponse(**data)
+                organizations.extend(
+                    parsed_response.organizations[
+                        : query.max_results - len(organizations)
+                    ]
+                )
+
+        logger.info(f"Found {len(organizations)} organizations")
+        return (
+            organizations[: query.max_results] if query.max_results else organizations
+        )

autogpt_platform/backend/backend/blocks/apollo/_auth.py

@@ -0,0 +1,35 @@
+from typing import Literal
+
+from pydantic import SecretStr
+
+from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput
+from backend.integrations.providers import ProviderName
+
+ApolloCredentials = APIKeyCredentials
+ApolloCredentialsInput = CredentialsMetaInput[
+    Literal[ProviderName.APOLLO],
+    Literal["api_key"],
+]
+
+TEST_CREDENTIALS = APIKeyCredentials(
+    id="01234567-89ab-cdef-0123-456789abcdef",
+    provider="apollo",
+    api_key=SecretStr("mock-apollo-api-key"),
+    title="Mock Apollo API key",
+    expires_at=None,
+)
+TEST_CREDENTIALS_INPUT = {
+    "provider": TEST_CREDENTIALS.provider,
+    "id": TEST_CREDENTIALS.id,
+    "type": TEST_CREDENTIALS.type,
+    "title": TEST_CREDENTIALS.title,
+}
+
+
+def ApolloCredentialsField() -> ApolloCredentialsInput:
+    """
+    Creates a Apollo credentials input on a block.
+    """
+    return CredentialsField(
+        description="The Apollo integration can be used with an API Key.",
+    )

autogpt_platform/backend/backend/blocks/apollo/models.py

@@ -0,0 +1,548 @@
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from backend.data.model import SchemaField
+
+
+class PrimaryPhone(BaseModel):
+    """A primary phone in Apollo"""
+
+    number: str
+    source: str
+    sanitized_number: str
+
+
+class SenorityLevels(str, Enum):
+    """Seniority levels in Apollo"""
+
+    OWNER = "owner"
+    FOUNDER = "founder"
+    C_SUITE = "c_suite"
+    PARTNER = "partner"
+    VP = "vp"
+    HEAD = "head"
+    DIRECTOR = "director"
+    MANAGER = "manager"
+    SENIOR = "senior"
+    ENTRY = "entry"
+    INTERN = "intern"
+
+
+class ContactEmailStatuses(str, Enum):
+    """Contact email statuses in Apollo"""
+
+    VERIFIED = "verified"
+    UNVERIFIED = "unverified"
+    LIKELY_TO_ENGAGE = "likely_to_engage"
+    UNAVAILABLE = "unavailable"
+
+
+class RuleConfigStatus(BaseModel):
+    """A rule config status in Apollo"""
+
+    _id: str
+    created_at: str
+    rule_action_config_id: str
+    rule_config_id: str
+    status_cd: str
+    updated_at: str
+    id: str
+    key: str
+
+
+class ContactCampaignStatus(BaseModel):
+    """A contact campaign status in Apollo"""
+
+    id: str
+    emailer_campaign_id: str
+    send_email_from_user_id: str
+    inactive_reason: str
+    status: str
+    added_at: str
+    added_by_user_id: str
+    finished_at: str
+    paused_at: str
+    auto_unpause_at: str
+    send_email_from_email_address: str
+    send_email_from_email_account_id: str
+    manually_set_unpause: str
+    failure_reason: str
+    current_step_id: str
+    in_response_to_emailer_message_id: str
+    cc_emails: str
+    bcc_emails: str
+    to_emails: str
+
+
+class Account(BaseModel):
+    """An account in Apollo"""
+
+    id: str
+    name: str
+    website_url: str
+    blog_url: str
+    angellist_url: str
+    linkedin_url: str
+    twitter_url: str
+    facebook_url: str
+    primary_phone: PrimaryPhone
+    languages: list[str]
+    alexa_ranking: int
+    phone: str
+    linkedin_uid: str
+    founded_year: int
+    publicly_traded_symbol: str
+    publicly_traded_exchange: str
+    logo_url: str
+    chrunchbase_url: str
+    primary_domain: str
+    domain: str
+    team_id: str
+    organization_id: str
+    account_stage_id: str
+    source: str
+    original_source: str
+    creator_id: str
+    owner_id: str
+    created_at: str
+    phone_status: str
+    hubspot_id: str
+    salesforce_id: str
+    crm_owner_id: str
+    parent_account_id: str
+    sanitized_phone: str
+    # no listed type on the API docs
+    account_playbook_statues: list[Any]
+    account_rule_config_statuses: list[RuleConfigStatus]
+    existence_level: str
+    label_ids: list[str]
+    typed_custom_fields: Any
+    custom_field_errors: Any
+    modality: str
+    source_display_name: str
+    salesforce_record_id: str
+    crm_record_url: str
+
+
+class ContactEmail(BaseModel):
+    """A contact email in Apollo"""
+
+    email: str = ""
+    email_md5: str = ""
+    email_sha256: str = ""
+    email_status: str = ""
+    email_source: str = ""
+    extrapolated_email_confidence: str = ""
+    position: int = 0
+    email_from_customer: str = ""
+    free_domain: bool = True
+
+
+class EmploymentHistory(BaseModel):
+    """An employment history in Apollo"""
+
+    model_config = ConfigDict(
+        extra="allow",
+        arbitrary_types_allowed=True,
+        from_attributes=True,
+        populate_by_name=True,
+    )
+
+    _id: Optional[str] = None
+    created_at: Optional[str] = None
+    current: Optional[bool] = None
+    degree: Optional[str] = None
+    description: Optional[str] = None
+    emails: Optional[str] = None
+    end_date: Optional[str] = None
+    grade_level: Optional[str] = None
+    kind: Optional[str] = None
+    major: Optional[str] = None
+    organization_id: Optional[str] = None
+    organization_name: Optional[str] = None
+    raw_address: Optional[str] = None
+    start_date: Optional[str] = None
+    title: Optional[str] = None
+    updated_at: Optional[str] = None
+    id: Optional[str] = None
+    key: Optional[str] = None
+
+
+class Breadcrumb(BaseModel):
+    """A breadcrumb in Apollo"""
+
+    label: Optional[str] = "N/A"
+    signal_field_name: Optional[str] = "N/A"
+    value: str | list | None = "N/A"
+    display_name: Optional[str] = "N/A"
+
+
+class TypedCustomField(BaseModel):
+    """A typed custom field in Apollo"""
+
+    id: Optional[str] = "N/A"
+    value: Optional[str] = "N/A"
+
+
+class Pagination(BaseModel):
+    """Pagination in Apollo"""
+
+    model_config = ConfigDict(
+        extra="allow",
+        arbitrary_types_allowed=True,
+        from_attributes=True,
+        populate_by_name=True,
+    )
+
+    page: int = 0
+    per_page: int = 0
+    total_entries: int = 0
+    total_pages: int = 0
+
+
+class DialerFlags(BaseModel):
+    """A dialer flags in Apollo"""
+
+    country_name: str
+    country_enabled: bool
+    high_risk_calling_enabled: bool
+    potential_high_risk_number: bool
+
+
+class PhoneNumber(BaseModel):
+    """A phone number in Apollo"""
+
+    raw_number: str = ""
+    sanitized_number: str = ""
+    type: str = ""
+    position: int = 0
+    status: str = ""
+    dnc_status: str = ""
+    dnc_other_info: str = ""
+    dailer_flags: DialerFlags = DialerFlags(
+        country_name="",
+        country_enabled=True,
+        high_risk_calling_enabled=True,
+        potential_high_risk_number=True,
+    )
+
+
+class Organization(BaseModel):
+    """An organization in Apollo"""
+
+    model_config = ConfigDict(
+        extra="allow",
+        arbitrary_types_allowed=True,
+        from_attributes=True,
+        populate_by_name=True,
+    )
+
+    id: Optional[str] = "N/A"
+    name: Optional[str] = "N/A"
+    website_url: Optional[str] = "N/A"
+    blog_url: Optional[str] = "N/A"
+    angellist_url: Optional[str] = "N/A"
+    linkedin_url: Optional[str] = "N/A"
+    twitter_url: Optional[str] = "N/A"
+    facebook_url: Optional[str] = "N/A"
+    primary_phone: Optional[PrimaryPhone] = PrimaryPhone(
+        number="N/A", source="N/A", sanitized_number="N/A"
+    )
+    languages: list[str] = []
+    alexa_ranking: Optional[int] = 0
+    phone: Optional[str] = "N/A"
+    linkedin_uid: Optional[str] = "N/A"
+    founded_year: Optional[int] = 0
+    publicly_traded_symbol: Optional[str] = "N/A"
+    publicly_traded_exchange: Optional[str] = "N/A"
+    logo_url: Optional[str] = "N/A"
+    chrunchbase_url: Optional[str] = "N/A"
+    primary_domain: Optional[str] = "N/A"
+    sanitized_phone: Optional[str] = "N/A"
+    owned_by_organization_id: Optional[str] = "N/A"
+    intent_strength: Optional[str] = "N/A"
+    show_intent: bool = True
+    has_intent_signal_account: Optional[bool] = True
+    intent_signal_account: Optional[str] = "N/A"
+
+
+class Contact(BaseModel):
+    """A contact in Apollo"""
+
+    model_config = ConfigDict(
+        extra="allow",
+        arbitrary_types_allowed=True,
+        from_attributes=True,
+        populate_by_name=True,
+    )
+
+    contact_roles: list[Any] = []
+    id: Optional[str] = None
+    first_name: Optional[str] = None
+    last_name: Optional[str] = None
+    name: Optional[str] = None
+    linkedin_url: Optional[str] = None
+    title: Optional[str] = None
+    contact_stage_id: Optional[str] = None
+    owner_id: Optional[str] = None
+    creator_id: Optional[str] = None
+    person_id: Optional[str] = None
+    email_needs_tickling: bool = True
+    organization_name: Optional[str] = None
+    source: Optional[str] = None
+    original_source: Optional[str] = None
+    organization_id: Optional[str] = None
+    headline: Optional[str] = None
+    photo_url: Optional[str] = None
+    present_raw_address: Optional[str] = None
+    linkededin_uid: Optional[str] = None
+    extrapolated_email_confidence: Optional[float] = None
+    salesforce_id: Optional[str] = None
+    salesforce_lead_id: Optional[str] = None
+    salesforce_contact_id: Optional[str] = None
+    saleforce_account_id: Optional[str] = None
+    crm_owner_id: Optional[str] = None
+    created_at: Optional[str] = None
+    emailer_campaign_ids: list[str] = []
+    direct_dial_status: Optional[str] = None
+    direct_dial_enrichment_failed_at: Optional[str] = None
+    email_status: Optional[str] = None
+    email_source: Optional[str] = None
+    account_id: Optional[str] = None
+    last_activity_date: Optional[str] = None
+    hubspot_vid: Optional[str] = None
+    hubspot_company_id: Optional[str] = None
+    crm_id: Optional[str] = None
+    sanitized_phone: Optional[str] = None
+    merged_crm_ids: Optional[str] = None
+    updated_at: Optional[str] = None
+    queued_for_crm_push: bool = True
+    suggested_from_rule_engine_config_id: Optional[str] = None
+    email_unsubscribed: Optional[str] = None
+    label_ids: list[Any] = []
+    has_pending_email_arcgate_request: bool = True
+    has_email_arcgate_request: bool = True
+    existence_level: Optional[str] = None
+    email: Optional[str] = None
+    email_from_customer: Optional[str] = None
+    typed_custom_fields: list[TypedCustomField] = []
+    custom_field_errors: Any = None
+    salesforce_record_id: Optional[str] = None
+    crm_record_url: Optional[str] = None
+    email_status_unavailable_reason: Optional[str] = None
+    email_true_status: Optional[str] = None
+    updated_email_true_status: bool = True
+    contact_rule_config_statuses: list[RuleConfigStatus] = []
+    source_display_name: Optional[str] = None
+    twitter_url: Optional[str] = None
+    contact_campaign_statuses: list[ContactCampaignStatus] = []
+    state: Optional[str] = None
+    city: Optional[str] = None
+    country: Optional[str] = None
+    account: Optional[Account] = None
+    contact_emails: list[ContactEmail] = []
+    organization: Optional[Organization] = None
+    employment_history: list[EmploymentHistory] = []
+    time_zone: Optional[str] = None
+    intent_strength: Optional[str] = None
+    show_intent: bool = True
+    phone_numbers: list[PhoneNumber] = []
+    account_phone_note: Optional[str] = None
+    free_domain: bool = True
+    is_likely_to_engage: bool = True
+    email_domain_catchall: bool = True
+    contact_job_change_event: Optional[str] = None
+
+
+class SearchOrganizationsRequest(BaseModel):
+    """Request for Apollo's search organizations API"""
+
+    organization_num_empoloyees_range: list[int] = SchemaField(
+        description="""The number range of employees working for the company. This enables you to find companies based on headcount. You can add multiple ranges to expand your search results.
+
+Each range you add needs to be a string, with the upper and lower numbers of the range separated only by a comma.""",
+        default=[0, 1000000],
+    )
+
+    organization_locations: list[str] = SchemaField(
+        description="""The location of the company headquarters. You can search across cities, US states, and countries.
+
+If a company has several office locations, results are still based on the headquarters location. For example, if you search chicago but a company's HQ location is in boston, any Boston-based companies will not appearch in your search results, even if they match other parameters.
+
+To exclude companies based on location, use the organization_not_locations parameter.
+""",
+        default_factory=list,
+    )
+    organizations_not_locations: list[str] = SchemaField(
+        description="""Exclude companies from search results based on the location of the company headquarters. You can use cities, US states, and countries as locations to exclude.
+
+This parameter is useful for ensuring you do not prospect in an undesirable territory. For example, if you use ireland as a value, no Ireland-based companies will appear in your search results.
+""",
+        default_factory=list,
+    )
+    q_organization_keyword_tags: list[str] = SchemaField(
+        description="""Filter search results based on keywords associated with companies. For example, you can enter mining as a value to return only companies that have an association with the mining industry."""
+    )
+    q_organization_name: str = SchemaField(
+        description="""Filter search results to include a specific company name.
+
+If the value you enter for this parameter does not match with a company's name, the company will not appear in search results, even if it matches other parameters. Partial matches are accepted. For example, if you filter by the value marketing, a company called NY Marketing Unlimited would still be eligible as a search result, but NY Market Analysis would not be eligible."""
+    )
+    organization_ids: list[str] = SchemaField(
+        description="""The Apollo IDs for the companies you want to include in your search results. Each company in the Apollo database is assigned a unique ID.
+
+To find IDs, identify the values for organization_id when you call this endpoint.""",
+        default_factory=list,
+    )
+    max_results: int = SchemaField(
+        description="""The maximum number of results to return. If you don't specify this parameter, the default is 100.""",
+        default=100,
+        ge=1,
+        le=50000,
+        advanced=True,
+    )
+
+    page: int = SchemaField(
+        description="""The page number of the Apollo data that you want to retrieve.
+
+Use this parameter in combination with the per_page parameter to make search results for navigable and improve the performance of the endpoint.""",
+        default=1,
+    )
+    per_page: int = SchemaField(
+        description="""The number of search results that should be returned for each page. Limited the number of results per page improves the endpoint's performance.
+
+Use the page parameter to search the different pages of data.""",
+        default=100,
+    )
+
+
+class SearchOrganizationsResponse(BaseModel):
+    """Response from Apollo's search organizations API"""
+
+    breadcrumbs: list[Breadcrumb] = []
+    partial_results_only: bool = True
+    has_join: bool = True
+    disable_eu_prospecting: bool = True
+    partial_results_limit: int = 0
+    pagination: Pagination = Pagination(
+        page=0, per_page=0, total_entries=0, total_pages=0
+    )
+    # no listed type on the API docs
+    accounts: list[Any] = []
+    organizations: list[Organization] = []
+    models_ids: list[str] = []
+    num_fetch_result: Optional[str] = "N/A"
+    derived_params: Optional[str] = "N/A"
+
+
+class SearchPeopleRequest(BaseModel):
+    """Request for Apollo's search people API"""
+
+    person_titles: list[str] = SchemaField(
+        description="""Job titles held by the people you want to find. For a person to be included in search results, they only need to match 1 of the job titles you add. Adding more job titles expands your search results.
+
+Results also include job titles with the same terms, even if they are not exact matches. For example, searching for marketing manager might return people with the job title content marketing manager.
+
+Use this parameter in combination with the person_seniorities[] parameter to find people based on specific job functions and seniority levels.
+""",
+        default_factory=list,
+        placeholder="marketing manager",
+    )
+    person_locations: list[str] = SchemaField(
+        description="""The location where people live. You can search across cities, US states, and countries.
+
+To find people based on the headquarters locations of their current employer, use the organization_locations parameter.""",
+        default_factory=list,
+    )
+    person_seniorities: list[SenorityLevels] = SchemaField(
+        description="""The job seniority that people hold within their current employer. This enables you to find people that currently hold positions at certain reporting levels, such as Director level or senior IC level.
+
+For a person to be included in search results, they only need to match 1 of the seniorities you add. Adding more seniorities expands your search results.
+
+Searches only return results based on their current job title, so searching for Director-level employees only returns people that currently hold a Director-level title. If someone was previously a Director, but is currently a VP, they would not be included in your search results.
+
+Use this parameter in combination with the person_titles[] parameter to find people based on specific job functions and seniority levels.""",
+        default_factory=list,
+    )
+    organization_locations: list[str] = SchemaField(
+        description="""The location of the company headquarters for a person's current employer. You can search across cities, US states, and countries.
+
+If a company has several office locations, results are still based on the headquarters location. For example, if you search chicago but a company's HQ location is in boston, people that work for the Boston-based company will not appear in your results, even if they match other parameters.
+
+To find people based on their personal location, use the person_locations parameter.""",
+        default_factory=list,
+    )
+    q_organization_domains: list[str] = SchemaField(
+        description="""The domain name for the person's employer. This can be the current employer or a previous employer. Do not include www., the @ symbol, or similar.
+
+You can add multiple domains to search across companies.
+
+  Examples: apollo.io and microsoft.com""",
+        default_factory=list,
+    )
+    contact_email_statuses: list[ContactEmailStatuses] = SchemaField(
+        description="""The email statuses for the people you want to find. You can add multiple statuses to expand your search.""",
+        default_factory=list,
+    )
+    organization_ids: list[str] = SchemaField(
+        description="""The Apollo IDs for the companies (employers) you want to include in your search results. Each company in the Apollo database is assigned a unique ID.
+
+To find IDs, call the Organization Search endpoint and identify the values for organization_id.""",
+        default_factory=list,
+    )
+    organization_num_empoloyees_range: list[int] = SchemaField(
+        description="""The number range of employees working for the company. This enables you to find companies based on headcount. You can add multiple ranges to expand your search results.
+
+Each range you add needs to be a string, with the upper and lower numbers of the range separated only by a comma.""",
+        default_factory=list,
+    )
+    q_keywords: str = SchemaField(
+        description="""A string of words over which we want to filter the results""",
+        default="",
+    )
+    page: int = SchemaField(
+        description="""The page number of the Apollo data that you want to retrieve.
+
+Use this parameter in combination with the per_page parameter to make search results for navigable and improve the performance of the endpoint.""",
+        default=1,
+    )
+    per_page: int = SchemaField(
+        description="""The number of search results that should be returned for each page. Limited the number of results per page improves the endpoint's performance.
+
+Use the page parameter to search the different pages of data.""",
+        default=100,
+    )
+    max_results: int = SchemaField(
+        description="""The maximum number of results to return. If you don't specify this parameter, the default is 100.""",
+        default=100,
+        ge=1,
+        le=50000,
+        advanced=True,
+    )
+
+
+class SearchPeopleResponse(BaseModel):
+    """Response from Apollo's search people API"""
+
+    model_config = ConfigDict(
+        extra="allow",
+        arbitrary_types_allowed=True,
+        from_attributes=True,
+        populate_by_name=True,
+    )
+
+    breadcrumbs: list[Breadcrumb] = []
+    partial_results_only: bool = True
+    has_join: bool = True
+    disable_eu_prospecting: bool = True
+    partial_results_limit: int = 0
+    pagination: Pagination = Pagination(
+        page=0, per_page=0, total_entries=0, total_pages=0
+    )
+    contacts: list[Contact] = []
+    people: list[Contact] = []
+    model_ids: list[str] = []
+    num_fetch_result: Optional[str] = "N/A"
+    derived_params: Optional[str] = "N/A"

autogpt_platform/backend/backend/blocks/apollo/organization.py

@@ -0,0 +1,219 @@
+from backend.blocks.apollo._api import ApolloClient
+from backend.blocks.apollo._auth import (
+    TEST_CREDENTIALS,
+    TEST_CREDENTIALS_INPUT,
+    ApolloCredentials,
+    ApolloCredentialsInput,
+)
+from backend.blocks.apollo.models import (
+    Organization,
+    PrimaryPhone,
+    SearchOrganizationsRequest,
+)
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+
+
+class SearchOrganizationsBlock(Block):
+    """Search for organizations in Apollo"""
+
+    class Input(BlockSchema):
+        organization_num_empoloyees_range: list[int] = SchemaField(
+            description="""The number range of employees working for the company. This enables you to find companies based on headcount. You can add multiple ranges to expand your search results.
+
+Each range you add needs to be a string, with the upper and lower numbers of the range separated only by a comma.""",
+            default=[0, 1000000],
+        )
+
+        organization_locations: list[str] = SchemaField(
+            description="""The location of the company headquarters. You can search across cities, US states, and countries.
+
+If a company has several office locations, results are still based on the headquarters location. For example, if you search chicago but a company's HQ location is in boston, any Boston-based companies will not appearch in your search results, even if they match other parameters.
+
+To exclude companies based on location, use the organization_not_locations parameter.
+""",
+            default_factory=list,
+        )
+        organizations_not_locations: list[str] = SchemaField(
+            description="""Exclude companies from search results based on the location of the company headquarters. You can use cities, US states, and countries as locations to exclude.
+
+This parameter is useful for ensuring you do not prospect in an undesirable territory. For example, if you use ireland as a value, no Ireland-based companies will appear in your search results.
+""",
+            default_factory=list,
+        )
+        q_organization_keyword_tags: list[str] = SchemaField(
+            description="""Filter search results based on keywords associated with companies. For example, you can enter mining as a value to return only companies that have an association with the mining industry.""",
+            default_factory=list,
+        )
+        q_organization_name: str = SchemaField(
+            description="""Filter search results to include a specific company name.
+
+If the value you enter for this parameter does not match with a company's name, the company will not appear in search results, even if it matches other parameters. Partial matches are accepted. For example, if you filter by the value marketing, a company called NY Marketing Unlimited would still be eligible as a search result, but NY Market Analysis would not be eligible.""",
+            default="",
+            advanced=False,
+        )
+        organization_ids: list[str] = SchemaField(
+            description="""The Apollo IDs for the companies you want to include in your search results. Each company in the Apollo database is assigned a unique ID.
+
+To find IDs, identify the values for organization_id when you call this endpoint.""",
+            default_factory=list,
+        )
+        max_results: int = SchemaField(
+            description="""The maximum number of results to return. If you don't specify this parameter, the default is 100.""",
+            default=100,
+            ge=1,
+            le=50000,
+            advanced=True,
+        )
+        credentials: ApolloCredentialsInput = SchemaField(
+            description="Apollo credentials",
+        )
+
+    class Output(BlockSchema):
+        organizations: list[Organization] = SchemaField(
+            description="List of organizations found",
+            default_factory=list,
+        )
+        organization: Organization = SchemaField(
+            description="Each found organization, one at a time",
+        )
+        error: str = SchemaField(
+            description="Error message if the search failed",
+            default="",
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="3d71270d-599e-4148-9b95-71b35d2f44f0",
+            description="Search for organizations in Apollo",
+            categories={BlockCategory.SEARCH},
+            input_schema=SearchOrganizationsBlock.Input,
+            output_schema=SearchOrganizationsBlock.Output,
+            test_credentials=TEST_CREDENTIALS,
+            test_input={"query": "Google", "credentials": TEST_CREDENTIALS_INPUT},
+            test_output=[
+                (
+                    "organization",
+                    Organization(
+                        id="1",
+                        name="Google",
+                        website_url="https://google.com",
+                        blog_url="https://google.com/blog",
+                        angellist_url="https://angel.co/google",
+                        linkedin_url="https://linkedin.com/company/google",
+                        twitter_url="https://twitter.com/google",
+                        facebook_url="https://facebook.com/google",
+                        primary_phone=PrimaryPhone(
+                            source="google",
+                            number="1234567890",
+                            sanitized_number="1234567890",
+                        ),
+                        languages=["en"],
+                        alexa_ranking=1000,
+                        phone="1234567890",
+                        linkedin_uid="1234567890",
+                        founded_year=2000,
+                        publicly_traded_symbol="GOOGL",
+                        publicly_traded_exchange="NASDAQ",
+                        logo_url="https://google.com/logo.png",
+                        chrunchbase_url="https://chrunchbase.com/google",
+                        primary_domain="google.com",
+                        sanitized_phone="1234567890",
+                        owned_by_organization_id="1",
+                        intent_strength="strong",
+                        show_intent=True,
+                        has_intent_signal_account=True,
+                        intent_signal_account="1",
+                    ),
+                ),
+                (
+                    "organizations",
+                    [
+                        Organization(
+                            id="1",
+                            name="Google",
+                            website_url="https://google.com",
+                            blog_url="https://google.com/blog",
+                            angellist_url="https://angel.co/google",
+                            linkedin_url="https://linkedin.com/company/google",
+                            twitter_url="https://twitter.com/google",
+                            facebook_url="https://facebook.com/google",
+                            primary_phone=PrimaryPhone(
+                                source="google",
+                                number="1234567890",
+                                sanitized_number="1234567890",
+                            ),
+                            languages=["en"],
+                            alexa_ranking=1000,
+                            phone="1234567890",
+                            linkedin_uid="1234567890",
+                            founded_year=2000,
+                            publicly_traded_symbol="GOOGL",
+                            publicly_traded_exchange="NASDAQ",
+                            logo_url="https://google.com/logo.png",
+                            chrunchbase_url="https://chrunchbase.com/google",
+                            primary_domain="google.com",
+                            sanitized_phone="1234567890",
+                            owned_by_organization_id="1",
+                            intent_strength="strong",
+                            show_intent=True,
+                            has_intent_signal_account=True,
+                            intent_signal_account="1",
+                        ),
+                    ],
+                ),
+            ],
+            test_mock={
+                "search_organizations": lambda *args, **kwargs: [
+                    Organization(
+                        id="1",
+                        name="Google",
+                        website_url="https://google.com",
+                        blog_url="https://google.com/blog",
+                        angellist_url="https://angel.co/google",
+                        linkedin_url="https://linkedin.com/company/google",
+                        twitter_url="https://twitter.com/google",
+                        facebook_url="https://facebook.com/google",
+                        primary_phone=PrimaryPhone(
+                            source="google",
+                            number="1234567890",
+                            sanitized_number="1234567890",
+                        ),
+                        languages=["en"],
+                        alexa_ranking=1000,
+                        phone="1234567890",
+                        linkedin_uid="1234567890",
+                        founded_year=2000,
+                        publicly_traded_symbol="GOOGL",
+                        publicly_traded_exchange="NASDAQ",
+                        logo_url="https://google.com/logo.png",
+                        chrunchbase_url="https://chrunchbase.com/google",
+                        primary_domain="google.com",
+                        sanitized_phone="1234567890",
+                        owned_by_organization_id="1",
+                        intent_strength="strong",
+                        show_intent=True,
+                        has_intent_signal_account=True,
+                        intent_signal_account="1",
+                    )
+                ]
+            },
+        )
+
+    @staticmethod
+    async def search_organizations(
+        query: SearchOrganizationsRequest, credentials: ApolloCredentials
+    ) -> list[Organization]:
+        client = ApolloClient(credentials)
+        return await client.search_organizations(query)
+
+    async def run(
+        self, input_data: Input, *, credentials: ApolloCredentials, **kwargs
+    ) -> BlockOutput:
+        query = SearchOrganizationsRequest(
+            **input_data.model_dump(exclude={"credentials"})
+        )
+        organizations = await self.search_organizations(query, credentials)
+        for organization in organizations:
+            yield "organization", organization
+        yield "organizations", organizations

autogpt_platform/backend/backend/blocks/apollo/people.py

@@ -0,0 +1,394 @@
+from backend.blocks.apollo._api import ApolloClient
+from backend.blocks.apollo._auth import (
+    TEST_CREDENTIALS,
+    TEST_CREDENTIALS_INPUT,
+    ApolloCredentials,
+    ApolloCredentialsInput,
+)
+from backend.blocks.apollo.models import (
+    Contact,
+    ContactEmailStatuses,
+    SearchPeopleRequest,
+    SenorityLevels,
+)
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+
+
+class SearchPeopleBlock(Block):
+    """Search for people in Apollo"""
+
+    class Input(BlockSchema):
+        person_titles: list[str] = SchemaField(
+            description="""Job titles held by the people you want to find. For a person to be included in search results, they only need to match 1 of the job titles you add. Adding more job titles expands your search results.
+
+        Results also include job titles with the same terms, even if they are not exact matches. For example, searching for marketing manager might return people with the job title content marketing manager.
+
+        Use this parameter in combination with the person_seniorities[] parameter to find people based on specific job functions and seniority levels.
+        """,
+            default_factory=list,
+            advanced=False,
+        )
+        person_locations: list[str] = SchemaField(
+            description="""The location where people live. You can search across cities, US states, and countries.
+
+        To find people based on the headquarters locations of their current employer, use the organization_locations parameter.""",
+            default_factory=list,
+            advanced=False,
+        )
+        person_seniorities: list[SenorityLevels] = SchemaField(
+            description="""The job seniority that people hold within their current employer. This enables you to find people that currently hold positions at certain reporting levels, such as Director level or senior IC level.
+
+        For a person to be included in search results, they only need to match 1 of the seniorities you add. Adding more seniorities expands your search results.
+
+        Searches only return results based on their current job title, so searching for Director-level employees only returns people that currently hold a Director-level title. If someone was previously a Director, but is currently a VP, they would not be included in your search results.
+
+        Use this parameter in combination with the person_titles[] parameter to find people based on specific job functions and seniority levels.""",
+            default_factory=list,
+            advanced=False,
+        )
+        organization_locations: list[str] = SchemaField(
+            description="""The location of the company headquarters for a person's current employer. You can search across cities, US states, and countries.
+
+        If a company has several office locations, results are still based on the headquarters location. For example, if you search chicago but a company's HQ location is in boston, people that work for the Boston-based company will not appear in your results, even if they match other parameters.
+
+        To find people based on their personal location, use the person_locations parameter.""",
+            default_factory=list,
+            advanced=False,
+        )
+        q_organization_domains: list[str] = SchemaField(
+            description="""The domain name for the person's employer. This can be the current employer or a previous employer. Do not include www., the @ symbol, or similar.
+
+        You can add multiple domains to search across companies.
+
+          Examples: apollo.io and microsoft.com""",
+            default_factory=list,
+            advanced=False,
+        )
+        contact_email_statuses: list[ContactEmailStatuses] = SchemaField(
+            description="""The email statuses for the people you want to find. You can add multiple statuses to expand your search.""",
+            default_factory=list,
+            advanced=False,
+        )
+        organization_ids: list[str] = SchemaField(
+            description="""The Apollo IDs for the companies (employers) you want to include in your search results. Each company in the Apollo database is assigned a unique ID.
+
+        To find IDs, call the Organization Search endpoint and identify the values for organization_id.""",
+            default_factory=list,
+            advanced=False,
+        )
+        organization_num_empoloyees_range: list[int] = SchemaField(
+            description="""The number range of employees working for the company. This enables you to find companies based on headcount. You can add multiple ranges to expand your search results.
+
+        Each range you add needs to be a string, with the upper and lower numbers of the range separated only by a comma.""",
+            default_factory=list,
+            advanced=False,
+        )
+        q_keywords: str = SchemaField(
+            description="""A string of words over which we want to filter the results""",
+            default="",
+            advanced=False,
+        )
+        max_results: int = SchemaField(
+            description="""The maximum number of results to return. If you don't specify this parameter, the default is 100.""",
+            default=100,
+            ge=1,
+            le=50000,
+            advanced=True,
+        )
+
+        credentials: ApolloCredentialsInput = SchemaField(
+            description="Apollo credentials",
+        )
+
+    class Output(BlockSchema):
+        people: list[Contact] = SchemaField(
+            description="List of people found",
+            default_factory=list,
+        )
+        person: Contact = SchemaField(
+            description="Each found person, one at a time",
+        )
+        error: str = SchemaField(
+            description="Error message if the search failed",
+            default="",
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="c2adb3aa-5aae-488d-8a6e-4eb8c23e2ed6",
+            description="Search for people in Apollo",
+            categories={BlockCategory.SEARCH},
+            input_schema=SearchPeopleBlock.Input,
+            output_schema=SearchPeopleBlock.Output,
+            test_credentials=TEST_CREDENTIALS,
+            test_input={"credentials": TEST_CREDENTIALS_INPUT},
+            test_output=[
+                (
+                    "person",
+                    Contact(
+                        contact_roles=[],
+                        id="1",
+                        name="John Doe",
+                        first_name="John",
+                        last_name="Doe",
+                        linkedin_url="https://www.linkedin.com/in/johndoe",
+                        title="Software Engineer",
+                        organization_name="Google",
+                        organization_id="123456",
+                        contact_stage_id="1",
+                        owner_id="1",
+                        creator_id="1",
+                        person_id="1",
+                        email_needs_tickling=True,
+                        source="apollo",
+                        original_source="apollo",
+                        headline="Software Engineer",
+                        photo_url="https://www.linkedin.com/in/johndoe",
+                        present_raw_address="123 Main St, Anytown, USA",
+                        linkededin_uid="123456",
+                        extrapolated_email_confidence=0.8,
+                        salesforce_id="123456",
+                        salesforce_lead_id="123456",
+                        salesforce_contact_id="123456",
+                        saleforce_account_id="123456",
+                        crm_owner_id="123456",
+                        created_at="2021-01-01",
+                        emailer_campaign_ids=[],
+                        direct_dial_status="active",
+                        direct_dial_enrichment_failed_at="2021-01-01",
+                        email_status="active",
+                        email_source="apollo",
+                        account_id="123456",
+                        last_activity_date="2021-01-01",
+                        hubspot_vid="123456",
+                        hubspot_company_id="123456",
+                        crm_id="123456",
+                        sanitized_phone="123456",
+                        merged_crm_ids="123456",
+                        updated_at="2021-01-01",
+                        queued_for_crm_push=True,
+                        suggested_from_rule_engine_config_id="123456",
+                        email_unsubscribed=None,
+                        label_ids=[],
+                        has_pending_email_arcgate_request=True,
+                        has_email_arcgate_request=True,
+                        existence_level=None,
+                        email=None,
+                        email_from_customer=None,
+                        typed_custom_fields=[],
+                        custom_field_errors=None,
+                        salesforce_record_id=None,
+                        crm_record_url=None,
+                        email_status_unavailable_reason=None,
+                        email_true_status=None,
+                        updated_email_true_status=True,
+                        contact_rule_config_statuses=[],
+                        source_display_name=None,
+                        twitter_url=None,
+                        contact_campaign_statuses=[],
+                        state=None,
+                        city=None,
+                        country=None,
+                        account=None,
+                        contact_emails=[],
+                        organization=None,
+                        employment_history=[],
+                        time_zone=None,
+                        intent_strength=None,
+                        show_intent=True,
+                        phone_numbers=[],
+                        account_phone_note=None,
+                        free_domain=True,
+                        is_likely_to_engage=True,
+                        email_domain_catchall=True,
+                        contact_job_change_event=None,
+                    ),
+                ),
+                (
+                    "people",
+                    [
+                        Contact(
+                            contact_roles=[],
+                            id="1",
+                            name="John Doe",
+                            first_name="John",
+                            last_name="Doe",
+                            linkedin_url="https://www.linkedin.com/in/johndoe",
+                            title="Software Engineer",
+                            organization_name="Google",
+                            organization_id="123456",
+                            contact_stage_id="1",
+                            owner_id="1",
+                            creator_id="1",
+                            person_id="1",
+                            email_needs_tickling=True,
+                            source="apollo",
+                            original_source="apollo",
+                            headline="Software Engineer",
+                            photo_url="https://www.linkedin.com/in/johndoe",
+                            present_raw_address="123 Main St, Anytown, USA",
+                            linkededin_uid="123456",
+                            extrapolated_email_confidence=0.8,
+                            salesforce_id="123456",
+                            salesforce_lead_id="123456",
+                            salesforce_contact_id="123456",
+                            saleforce_account_id="123456",
+                            crm_owner_id="123456",
+                            created_at="2021-01-01",
+                            emailer_campaign_ids=[],
+                            direct_dial_status="active",
+                            direct_dial_enrichment_failed_at="2021-01-01",
+                            email_status="active",
+                            email_source="apollo",
+                            account_id="123456",
+                            last_activity_date="2021-01-01",
+                            hubspot_vid="123456",
+                            hubspot_company_id="123456",
+                            crm_id="123456",
+                            sanitized_phone="123456",
+                            merged_crm_ids="123456",
+                            updated_at="2021-01-01",
+                            queued_for_crm_push=True,
+                            suggested_from_rule_engine_config_id="123456",
+                            email_unsubscribed=None,
+                            label_ids=[],
+                            has_pending_email_arcgate_request=True,
+                            has_email_arcgate_request=True,
+                            existence_level=None,
+                            email=None,
+                            email_from_customer=None,
+                            typed_custom_fields=[],
+                            custom_field_errors=None,
+                            salesforce_record_id=None,
+                            crm_record_url=None,
+                            email_status_unavailable_reason=None,
+                            email_true_status=None,
+                            updated_email_true_status=True,
+                            contact_rule_config_statuses=[],
+                            source_display_name=None,
+                            twitter_url=None,
+                            contact_campaign_statuses=[],
+                            state=None,
+                            city=None,
+                            country=None,
+                            account=None,
+                            contact_emails=[],
+                            organization=None,
+                            employment_history=[],
+                            time_zone=None,
+                            intent_strength=None,
+                            show_intent=True,
+                            phone_numbers=[],
+                            account_phone_note=None,
+                            free_domain=True,
+                            is_likely_to_engage=True,
+                            email_domain_catchall=True,
+                            contact_job_change_event=None,
+                        ),
+                    ],
+                ),
+            ],
+            test_mock={
+                "search_people": lambda query, credentials: [
+                    Contact(
+                        id="1",
+                        name="John Doe",
+                        first_name="John",
+                        last_name="Doe",
+                        linkedin_url="https://www.linkedin.com/in/johndoe",
+                        title="Software Engineer",
+                        organization_name="Google",
+                        organization_id="123456",
+                        contact_stage_id="1",
+                        owner_id="1",
+                        creator_id="1",
+                        person_id="1",
+                        email_needs_tickling=True,
+                        source="apollo",
+                        original_source="apollo",
+                        headline="Software Engineer",
+                        photo_url="https://www.linkedin.com/in/johndoe",
+                        present_raw_address="123 Main St, Anytown, USA",
+                        linkededin_uid="123456",
+                        extrapolated_email_confidence=0.8,
+                        salesforce_id="123456",
+                        salesforce_lead_id="123456",
+                        salesforce_contact_id="123456",
+                        saleforce_account_id="123456",
+                        crm_owner_id="123456",
+                        created_at="2021-01-01",
+                        emailer_campaign_ids=[],
+                        direct_dial_status="active",
+                        direct_dial_enrichment_failed_at="2021-01-01",
+                        email_status="active",
+                        email_source="apollo",
+                        account_id="123456",
+                        last_activity_date="2021-01-01",
+                        hubspot_vid="123456",
+                        hubspot_company_id="123456",
+                        crm_id="123456",
+                        sanitized_phone="123456",
+                        merged_crm_ids="123456",
+                        updated_at="2021-01-01",
+                        queued_for_crm_push=True,
+                        suggested_from_rule_engine_config_id="123456",
+                        email_unsubscribed=None,
+                        label_ids=[],
+                        has_pending_email_arcgate_request=True,
+                        has_email_arcgate_request=True,
+                        existence_level=None,
+                        email=None,
+                        email_from_customer=None,
+                        typed_custom_fields=[],
+                        custom_field_errors=None,
+                        salesforce_record_id=None,
+                        crm_record_url=None,
+                        email_status_unavailable_reason=None,
+                        email_true_status=None,
+                        updated_email_true_status=True,
+                        contact_rule_config_statuses=[],
+                        source_display_name=None,
+                        twitter_url=None,
+                        contact_campaign_statuses=[],
+                        state=None,
+                        city=None,
+                        country=None,
+                        account=None,
+                        contact_emails=[],
+                        organization=None,
+                        employment_history=[],
+                        time_zone=None,
+                        intent_strength=None,
+                        show_intent=True,
+                        phone_numbers=[],
+                        account_phone_note=None,
+                        free_domain=True,
+                        is_likely_to_engage=True,
+                        email_domain_catchall=True,
+                        contact_job_change_event=None,
+                    ),
+                ]
+            },
+        )
+
+    @staticmethod
+    async def search_people(
+        query: SearchPeopleRequest, credentials: ApolloCredentials
+    ) -> list[Contact]:
+        client = ApolloClient(credentials)
+        return await client.search_people(query)
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: ApolloCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+
+        query = SearchPeopleRequest(**input_data.model_dump(exclude={"credentials"}))
+        people = await self.search_people(query, credentials)
+        for person in people:
+            yield "person", person
+        yield "people", people

autogpt_platform/backend/backend/blocks/basic.py

@@ -1,13 +1,48 @@
-import re
+import enum
 from typing import Any, List
 
-from jinja2 import BaseLoader, Environment
-
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema, BlockType
 from backend.data.model import SchemaField
+from backend.util import json
+from backend.util.file import store_media_file
 from backend.util.mock import MockObject
+from backend.util.type import MediaFileType, convert
 
-jinja = Environment(loader=BaseLoader())
+
+class FileStoreBlock(Block):
+    class Input(BlockSchema):
+        file_in: MediaFileType = SchemaField(
+            description="The file to store in the temporary directory, it can be a URL, data URI, or local path."
+        )
+
+    class Output(BlockSchema):
+        file_out: MediaFileType = SchemaField(
+            description="The relative path to the stored file in the temporary directory."
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="cbb50872-625b-42f0-8203-a2ae78242d8a",
+            description="Stores the input file in the temporary directory.",
+            categories={BlockCategory.BASIC, BlockCategory.MULTIMEDIA},
+            input_schema=FileStoreBlock.Input,
+            output_schema=FileStoreBlock.Output,
+            static_output=True,
+        )
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        graph_exec_id: str,
+        **kwargs,
+    ) -> BlockOutput:
+        file_path = await store_media_file(
+            graph_exec_id=graph_exec_id,
+            file=input_data.file_in,
+            return_content=False,
+        )
+        yield "file_out", file_path
 
 
 class StoreValueBlock(Block):
@@ -49,15 +84,16 @@ class StoreValueBlock(Block):
             static_output=True,
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         yield "output", input_data.data or input_data.input
 
 
 class PrintToConsoleBlock(Block):
     class Input(BlockSchema):
-        text: str = SchemaField(description="The text to print to the console.")
+        text: Any = SchemaField(description="The data to print to the console.")
 
     class Output(BlockSchema):
+        output: Any = SchemaField(description="The data printed to the console.")
         status: str = SchemaField(description="The status of the print operation.")
 
     def __init__(self):
@@ -68,11 +104,14 @@ class PrintToConsoleBlock(Block):
             input_schema=PrintToConsoleBlock.Input,
             output_schema=PrintToConsoleBlock.Output,
             test_input={"text": "Hello, World!"},
-            test_output=("status", "printed"),
+            test_output=[
+                ("output", "Hello, World!"),
+                ("status", "printed"),
+            ],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
-        print(">>>>> Print: ", input_data.text)
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        yield "output", input_data.text
         yield "status", "printed"
 
 
@@ -112,10 +151,13 @@ class FindInDictionaryBlock(Block):
             categories={BlockCategory.BASIC},
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         obj = input_data.input
         key = input_data.key
 
+        if isinstance(obj, str):
+            obj = json.loads(obj)
+
         if isinstance(obj, dict) and key in obj:
             yield "output", obj[key]
         elif isinstance(obj, list) and isinstance(key, int) and 0 <= key < len(obj):
@@ -133,190 +175,10 @@ class FindInDictionaryBlock(Block):
             yield "missing", input_data.input
 
 
-class AgentInputBlock(Block):
-    """
-    This block is used to provide input to the graph.
-
-    It takes in a value, name, description, default values list and bool to limit selection to default values.
-
-    It Outputs the value passed as input.
-    """
-
-    class Input(BlockSchema):
-        name: str = SchemaField(description="The name of the input.")
-        value: Any = SchemaField(
-            description="The value to be passed as input.",
-            default=None,
-        )
-        title: str | None = SchemaField(
-            description="The title of the input.", default=None, advanced=True
-        )
-        description: str | None = SchemaField(
-            description="The description of the input.",
-            default=None,
-            advanced=True,
-        )
-        placeholder_values: List[Any] = SchemaField(
-            description="The placeholder values to be passed as input.",
-            default=[],
-            advanced=True,
-        )
-        limit_to_placeholder_values: bool = SchemaField(
-            description="Whether to limit the selection to placeholder values.",
-            default=False,
-            advanced=True,
-        )
-        advanced: bool = SchemaField(
-            description="Whether to show the input in the advanced section, if the field is not required.",
-            default=False,
-            advanced=True,
-        )
-        secret: bool = SchemaField(
-            description="Whether the input should be treated as a secret.",
-            default=False,
-            advanced=True,
-        )
-
-    class Output(BlockSchema):
-        result: Any = SchemaField(description="The value passed as input.")
-
-    def __init__(self):
-        super().__init__(
-            id="c0a8e994-ebf1-4a9c-a4d8-89d09c86741b",
-            description="This block is used to provide input to the graph.",
-            input_schema=AgentInputBlock.Input,
-            output_schema=AgentInputBlock.Output,
-            test_input=[
-                {
-                    "value": "Hello, World!",
-                    "name": "input_1",
-                    "description": "This is a test input.",
-                    "placeholder_values": [],
-                    "limit_to_placeholder_values": False,
-                },
-                {
-                    "value": "Hello, World!",
-                    "name": "input_2",
-                    "description": "This is a test input.",
-                    "placeholder_values": ["Hello, World!"],
-                    "limit_to_placeholder_values": True,
-                },
-            ],
-            test_output=[
-                ("result", "Hello, World!"),
-                ("result", "Hello, World!"),
-            ],
-            categories={BlockCategory.INPUT, BlockCategory.BASIC},
-            block_type=BlockType.INPUT,
-            static_output=True,
-        )
-
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
-        yield "result", input_data.value
-
-
-class AgentOutputBlock(Block):
-    """
-    Records the output of the graph for users to see.
-
-    Behavior:
-        If `format` is provided and the `value` is of a type that can be formatted,
-        the block attempts to format the recorded_value using the `format`.
-        If formatting fails or no `format` is provided, the raw `value` is output.
-    """
-
-    class Input(BlockSchema):
-        value: Any = SchemaField(
-            description="The value to be recorded as output.",
-            default=None,
-            advanced=False,
-        )
-        name: str = SchemaField(description="The name of the output.")
-        title: str | None = SchemaField(
-            description="The title of the output.",
-            default=None,
-            advanced=True,
-        )
-        description: str | None = SchemaField(
-            description="The description of the output.",
-            default=None,
-            advanced=True,
-        )
-        format: str = SchemaField(
-            description="The format string to be used to format the recorded_value.",
-            default="",
-            advanced=True,
-        )
-        advanced: bool = SchemaField(
-            description="Whether to treat the output as advanced.",
-            default=False,
-            advanced=True,
-        )
-        secret: bool = SchemaField(
-            description="Whether the output should be treated as a secret.",
-            default=False,
-            advanced=True,
-        )
-
-    class Output(BlockSchema):
-        output: Any = SchemaField(description="The value recorded as output.")
-
-    def __init__(self):
-        super().__init__(
-            id="363ae599-353e-4804-937e-b2ee3cef3da4",
-            description="Stores the output of the graph for users to see.",
-            input_schema=AgentOutputBlock.Input,
-            output_schema=AgentOutputBlock.Output,
-            test_input=[
-                {
-                    "value": "Hello, World!",
-                    "name": "output_1",
-                    "description": "This is a test output.",
-                    "format": "{{ output_1 }}!!",
-                },
-                {
-                    "value": "42",
-                    "name": "output_2",
-                    "description": "This is another test output.",
-                    "format": "{{ output_2 }}",
-                },
-                {
-                    "value": MockObject(value="!!", key="key"),
-                    "name": "output_3",
-                    "description": "This is a test output with a mock object.",
-                    "format": "{{ output_3 }}",
-                },
-            ],
-            test_output=[
-                ("output", "Hello, World!!!"),
-                ("output", "42"),
-                ("output", MockObject(value="!!", key="key")),
-            ],
-            categories={BlockCategory.OUTPUT, BlockCategory.BASIC},
-            block_type=BlockType.OUTPUT,
-            static_output=True,
-        )
-
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
-        """
-        Attempts to format the recorded_value using the fmt_string if provided.
-        If formatting fails or no fmt_string is given, returns the original recorded_value.
-        """
-        if input_data.format:
-            try:
-                fmt = re.sub(r"(?<!{){[ a-zA-Z0-9_]+}", r"{\g<0>}", input_data.format)
-                template = jinja.from_string(fmt)
-                yield "output", template.render({input_data.name: input_data.value})
-            except Exception as e:
-                yield "output", f"Error: {e}, {input_data.value}"
-        else:
-            yield "output", input_data.value
-
-
 class AddToDictionaryBlock(Block):
     class Input(BlockSchema):
         dictionary: dict[Any, Any] = SchemaField(
-            default={},
+            default_factory=dict,
             description="The dictionary to add the entry to. If not provided, a new dictionary will be created.",
         )
         key: str = SchemaField(
@@ -332,7 +194,7 @@ class AddToDictionaryBlock(Block):
             advanced=False,
         )
         entries: dict[Any, Any] = SchemaField(
-            default={},
+            default_factory=dict,
             description="The entries to add to the dictionary. This is the batch version of the `key` and `value` fields.",
             advanced=True,
         )
@@ -379,7 +241,7 @@ class AddToDictionaryBlock(Block):
             ],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         updated_dict = input_data.dictionary.copy()
 
         if input_data.value is not None and input_data.key:
@@ -394,7 +256,7 @@ class AddToDictionaryBlock(Block):
 class AddToListBlock(Block):
     class Input(BlockSchema):
         list: List[Any] = SchemaField(
-            default=[],
+            default_factory=list,
             advanced=False,
             description="The list to add the entry to. If not provided, a new list will be created.",
         )
@@ -404,7 +266,7 @@ class AddToListBlock(Block):
             default=None,
         )
         entries: List[Any] = SchemaField(
-            default=[],
+            default_factory=lambda: list(),
             description="The entries to add to the list. This is the batch version of the `entry` field.",
             advanced=True,
         )
@@ -457,7 +319,7 @@ class AddToListBlock(Block):
             ],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         entries_added = input_data.entries.copy()
         if input_data.entry:
             entries_added.append(input_data.entry)
@@ -471,6 +333,48 @@ class AddToListBlock(Block):
         yield "updated_list", updated_list
 
 
+class FindInListBlock(Block):
+    class Input(BlockSchema):
+        list: List[Any] = SchemaField(description="The list to search in.")
+        value: Any = SchemaField(description="The value to search for.")
+
+    class Output(BlockSchema):
+        index: int = SchemaField(description="The index of the value in the list.")
+        found: bool = SchemaField(
+            description="Whether the value was found in the list."
+        )
+        not_found_value: Any = SchemaField(
+            description="The value that was not found in the list."
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="5e2c6d0a-1e37-489f-b1d0-8e1812b23333",
+            description="Finds the index of the value in the list.",
+            categories={BlockCategory.BASIC},
+            input_schema=FindInListBlock.Input,
+            output_schema=FindInListBlock.Output,
+            test_input=[
+                {"list": [1, 2, 3, 4, 5], "value": 3},
+                {"list": [1, 2, 3, 4, 5], "value": 6},
+            ],
+            test_output=[
+                ("index", 2),
+                ("found", True),
+                ("found", False),
+                ("not_found_value", 6),
+            ],
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        try:
+            yield "index", input_data.list.index(input_data.value)
+            yield "found", True
+        except ValueError:
+            yield "found", False
+            yield "not_found_value", input_data.value
+
+
 class NoteBlock(Block):
     class Input(BlockSchema):
         text: str = SchemaField(description="The text to display in the sticky note.")
@@ -492,5 +396,147 @@ class NoteBlock(Block):
             block_type=BlockType.NOTE,
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         yield "output", input_data.text
+
+
+class CreateDictionaryBlock(Block):
+    class Input(BlockSchema):
+        values: dict[str, Any] = SchemaField(
+            description="Key-value pairs to create the dictionary with",
+            placeholder="e.g., {'name': 'Alice', 'age': 25}",
+        )
+
+    class Output(BlockSchema):
+        dictionary: dict[str, Any] = SchemaField(
+            description="The created dictionary containing the specified key-value pairs"
+        )
+        error: str = SchemaField(
+            description="Error message if dictionary creation failed"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="b924ddf4-de4f-4b56-9a85-358930dcbc91",
+            description="Creates a dictionary with the specified key-value pairs. Use this when you know all the values you want to add upfront.",
+            categories={BlockCategory.DATA},
+            input_schema=CreateDictionaryBlock.Input,
+            output_schema=CreateDictionaryBlock.Output,
+            test_input=[
+                {
+                    "values": {"name": "Alice", "age": 25, "city": "New York"},
+                },
+                {
+                    "values": {"numbers": [1, 2, 3], "active": True, "score": 95.5},
+                },
+            ],
+            test_output=[
+                (
+                    "dictionary",
+                    {"name": "Alice", "age": 25, "city": "New York"},
+                ),
+                (
+                    "dictionary",
+                    {"numbers": [1, 2, 3], "active": True, "score": 95.5},
+                ),
+            ],
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        try:
+            # The values are already validated by Pydantic schema
+            yield "dictionary", input_data.values
+        except Exception as e:
+            yield "error", f"Failed to create dictionary: {str(e)}"
+
+
+class CreateListBlock(Block):
+    class Input(BlockSchema):
+        values: List[Any] = SchemaField(
+            description="A list of values to be combined into a new list.",
+            placeholder="e.g., ['Alice', 25, True]",
+        )
+
+    class Output(BlockSchema):
+        list: List[Any] = SchemaField(
+            description="The created list containing the specified values."
+        )
+        error: str = SchemaField(description="Error message if list creation failed.")
+
+    def __init__(self):
+        super().__init__(
+            id="a912d5c7-6e00-4542-b2a9-8034136930e4",
+            description="Creates a list with the specified values. Use this when you know all the values you want to add upfront.",
+            categories={BlockCategory.DATA},
+            input_schema=CreateListBlock.Input,
+            output_schema=CreateListBlock.Output,
+            test_input=[
+                {
+                    "values": ["Alice", 25, True],
+                },
+                {
+                    "values": [1, 2, 3, "four", {"key": "value"}],
+                },
+            ],
+            test_output=[
+                (
+                    "list",
+                    ["Alice", 25, True],
+                ),
+                (
+                    "list",
+                    [1, 2, 3, "four", {"key": "value"}],
+                ),
+            ],
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        try:
+            # The values are already validated by Pydantic schema
+            yield "list", input_data.values
+        except Exception as e:
+            yield "error", f"Failed to create list: {str(e)}"
+
+
+class TypeOptions(enum.Enum):
+    STRING = "string"
+    NUMBER = "number"
+    BOOLEAN = "boolean"
+    LIST = "list"
+    DICTIONARY = "dictionary"
+
+
+class UniversalTypeConverterBlock(Block):
+    class Input(BlockSchema):
+        value: Any = SchemaField(
+            description="The value to convert to a universal type."
+        )
+        type: TypeOptions = SchemaField(description="The type to convert the value to.")
+
+    class Output(BlockSchema):
+        value: Any = SchemaField(description="The converted value.")
+
+    def __init__(self):
+        super().__init__(
+            id="95d1b990-ce13-4d88-9737-ba5c2070c97b",
+            description="This block is used to convert a value to a universal type.",
+            categories={BlockCategory.BASIC},
+            input_schema=UniversalTypeConverterBlock.Input,
+            output_schema=UniversalTypeConverterBlock.Output,
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        try:
+            converted_value = convert(
+                input_data.value,
+                {
+                    TypeOptions.STRING: str,
+                    TypeOptions.NUMBER: float,
+                    TypeOptions.BOOLEAN: bool,
+                    TypeOptions.LIST: list,
+                    TypeOptions.DICTIONARY: dict,
+                }[input_data.type],
+            )
+            yield "value", converted_value
+        except Exception as e:
+            yield "error", f"Failed to convert value: {str(e)}"

autogpt_platform/backend/backend/blocks/block.py

@@ -38,7 +38,7 @@ class BlockInstallationBlock(Block):
             disabled=True,
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         code = input_data.code
 
         if search := re.search(r"class (\w+)\(Block\):", code):
@@ -64,7 +64,7 @@ class BlockInstallationBlock(Block):
 
             from backend.util.test import execute_block_test
 
-            execute_block_test(block)
+            await execute_block_test(block)
             yield "success", "Block installed successfully."
         except Exception as e:
             os.remove(file_path)

autogpt_platform/backend/backend/blocks/branching.py

@@ -70,7 +70,7 @@ class ConditionBlock(Block):
             ],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         operator = input_data.operator
 
         value1 = input_data.value1
@@ -107,3 +107,83 @@ class ConditionBlock(Block):
             yield "yes_output", yes_value
         else:
             yield "no_output", no_value
+
+
+class IfInputMatchesBlock(Block):
+    class Input(BlockSchema):
+        input: Any = SchemaField(
+            description="The input to match against",
+            placeholder="For example: 10 or 'hello' or True",
+        )
+        value: Any = SchemaField(
+            description="The value to output if the input matches",
+            placeholder="For example: 'Greater' or 20 or False",
+        )
+        yes_value: Any = SchemaField(
+            description="The value to output if the input matches",
+            placeholder="For example: 'Greater' or 20 or False",
+            default=None,
+        )
+        no_value: Any = SchemaField(
+            description="The value to output if the input does not match",
+            placeholder="For example: 'Greater' or 20 or False",
+            default=None,
+        )
+
+    class Output(BlockSchema):
+        result: bool = SchemaField(
+            description="The result of the condition evaluation (True or False)"
+        )
+        yes_output: Any = SchemaField(
+            description="The output value if the condition is true"
+        )
+        no_output: Any = SchemaField(
+            description="The output value if the condition is false"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="6dbbc4b3-ca6c-42b6-b508-da52d23e13f2",
+            input_schema=IfInputMatchesBlock.Input,
+            output_schema=IfInputMatchesBlock.Output,
+            description="Handles conditional logic based on comparison operators",
+            categories={BlockCategory.LOGIC},
+            test_input=[
+                {
+                    "input": 10,
+                    "value": 10,
+                    "yes_value": "Greater",
+                    "no_value": "Not greater",
+                },
+                {
+                    "input": 10,
+                    "value": 20,
+                    "yes_value": "Greater",
+                    "no_value": "Not greater",
+                },
+                {
+                    "input": 10,
+                    "value": None,
+                    "yes_value": "Yes",
+                    "no_value": "No",
+                },
+            ],
+            test_output=[
+                ("result", True),
+                ("yes_output", "Greater"),
+                ("result", False),
+                ("no_output", "Not greater"),
+                ("result", False),
+                ("no_output", "No"),
+                # ("result", True),
+                # ("yes_output", "Yes"),
+            ],
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        if input_data.input == input_data.value or input_data.input is input_data.value:
+            yield "result", True
+            yield "yes_output", input_data.yes_value
+        else:
+            yield "result", False
+            yield "no_output", input_data.no_value

autogpt_platform/backend/backend/blocks/code_executor.py

@@ -0,0 +1,457 @@
+from enum import Enum
+from typing import Literal
+
+from e2b_code_interpreter import AsyncSandbox
+from pydantic import SecretStr
+
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import (
+    APIKeyCredentials,
+    CredentialsField,
+    CredentialsMetaInput,
+    SchemaField,
+)
+from backend.integrations.providers import ProviderName
+
+TEST_CREDENTIALS = APIKeyCredentials(
+    id="01234567-89ab-cdef-0123-456789abcdef",
+    provider="e2b",
+    api_key=SecretStr("mock-e2b-api-key"),
+    title="Mock E2B API key",
+    expires_at=None,
+)
+TEST_CREDENTIALS_INPUT = {
+    "provider": TEST_CREDENTIALS.provider,
+    "id": TEST_CREDENTIALS.id,
+    "type": TEST_CREDENTIALS.type,
+    "title": TEST_CREDENTIALS.type,
+}
+
+
+class ProgrammingLanguage(Enum):
+    PYTHON = "python"
+    JAVASCRIPT = "js"
+    BASH = "bash"
+    R = "r"
+    JAVA = "java"
+
+
+class CodeExecutionBlock(Block):
+    # TODO : Add support to upload and download files
+    # Currently, You can customized the CPU and Memory, only by creating a pre customized sandbox template
+    class Input(BlockSchema):
+        credentials: CredentialsMetaInput[
+            Literal[ProviderName.E2B], Literal["api_key"]
+        ] = CredentialsField(
+            description="Enter your api key for the E2B Sandbox. You can get it in here - https://e2b.dev/docs",
+        )
+
+        # Todo : Option to run commond in background
+        setup_commands: list[str] = SchemaField(
+            description=(
+                "Shell commands to set up the sandbox before running the code. "
+                "You can use `curl` or `git` to install your desired Debian based "
+                "package manager. `pip` and `npm` are pre-installed.\n\n"
+                "These commands are executed with `sh`, in the foreground."
+            ),
+            placeholder="pip install cowsay",
+            default_factory=list,
+            advanced=False,
+        )
+
+        code: str = SchemaField(
+            description="Code to execute in the sandbox",
+            placeholder="print('Hello, World!')",
+            default="",
+            advanced=False,
+        )
+
+        language: ProgrammingLanguage = SchemaField(
+            description="Programming language to execute",
+            default=ProgrammingLanguage.PYTHON,
+            advanced=False,
+        )
+
+        timeout: int = SchemaField(
+            description="Execution timeout in seconds", default=300
+        )
+
+        template_id: str = SchemaField(
+            description=(
+                "You can use an E2B sandbox template by entering its ID here. "
+                "Check out the E2B docs for more details: "
+                "[E2B - Sandbox template](https://e2b.dev/docs/sandbox-template)"
+            ),
+            default="",
+            advanced=True,
+        )
+
+    class Output(BlockSchema):
+        response: str = SchemaField(description="Response from code execution")
+        stdout_logs: str = SchemaField(
+            description="Standard output logs from execution"
+        )
+        stderr_logs: str = SchemaField(description="Standard error logs from execution")
+        error: str = SchemaField(description="Error message if execution failed")
+
+    def __init__(self):
+        super().__init__(
+            id="0b02b072-abe7-11ef-8372-fb5d162dd712",
+            description="Executes code in an isolated sandbox environment with internet access.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=CodeExecutionBlock.Input,
+            output_schema=CodeExecutionBlock.Output,
+            test_credentials=TEST_CREDENTIALS,
+            test_input={
+                "credentials": TEST_CREDENTIALS_INPUT,
+                "code": "print('Hello World')",
+                "language": ProgrammingLanguage.PYTHON.value,
+                "setup_commands": [],
+                "timeout": 300,
+                "template_id": "",
+            },
+            test_output=[
+                ("response", "Hello World"),
+                ("stdout_logs", "Hello World\n"),
+            ],
+            test_mock={
+                "execute_code": lambda code, language, setup_commands, timeout, api_key, template_id: (
+                    "Hello World",
+                    "Hello World\n",
+                    "",
+                ),
+            },
+        )
+
+    async def execute_code(
+        self,
+        code: str,
+        language: ProgrammingLanguage,
+        setup_commands: list[str],
+        timeout: int,
+        api_key: str,
+        template_id: str,
+    ):
+        try:
+            sandbox = None
+            if template_id:
+                sandbox = await AsyncSandbox.create(
+                    template=template_id, api_key=api_key, timeout=timeout
+                )
+            else:
+                sandbox = await AsyncSandbox.create(api_key=api_key, timeout=timeout)
+
+            if not sandbox:
+                raise Exception("Sandbox not created")
+
+            # Running setup commands
+            for cmd in setup_commands:
+                await sandbox.commands.run(cmd)
+
+            # Executing the code
+            execution = await sandbox.run_code(
+                code,
+                language=language.value,
+                on_error=lambda e: sandbox.kill(),  # Kill the sandbox if there is an error
+            )
+
+            if execution.error:
+                raise Exception(execution.error)
+
+            response = execution.text
+            stdout_logs = "".join(execution.logs.stdout)
+            stderr_logs = "".join(execution.logs.stderr)
+
+            return response, stdout_logs, stderr_logs
+
+        except Exception as e:
+            raise e
+
+    async def run(
+        self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
+    ) -> BlockOutput:
+        try:
+            response, stdout_logs, stderr_logs = await self.execute_code(
+                input_data.code,
+                input_data.language,
+                input_data.setup_commands,
+                input_data.timeout,
+                credentials.api_key.get_secret_value(),
+                input_data.template_id,
+            )
+
+            if response:
+                yield "response", response
+            if stdout_logs:
+                yield "stdout_logs", stdout_logs
+            if stderr_logs:
+                yield "stderr_logs", stderr_logs
+        except Exception as e:
+            yield "error", str(e)
+
+
+class InstantiationBlock(Block):
+    class Input(BlockSchema):
+        credentials: CredentialsMetaInput[
+            Literal[ProviderName.E2B], Literal["api_key"]
+        ] = CredentialsField(
+            description="Enter your api key for the E2B Sandbox. You can get it in here - https://e2b.dev/docs",
+        )
+
+        # Todo : Option to run commond in background
+        setup_commands: list[str] = SchemaField(
+            description=(
+                "Shell commands to set up the sandbox before running the code. "
+                "You can use `curl` or `git` to install your desired Debian based "
+                "package manager. `pip` and `npm` are pre-installed.\n\n"
+                "These commands are executed with `sh`, in the foreground."
+            ),
+            placeholder="pip install cowsay",
+            default_factory=list,
+            advanced=False,
+        )
+
+        setup_code: str = SchemaField(
+            description="Code to execute in the sandbox",
+            placeholder="print('Hello, World!')",
+            default="",
+            advanced=False,
+        )
+
+        language: ProgrammingLanguage = SchemaField(
+            description="Programming language to execute",
+            default=ProgrammingLanguage.PYTHON,
+            advanced=False,
+        )
+
+        timeout: int = SchemaField(
+            description="Execution timeout in seconds", default=300
+        )
+
+        template_id: str = SchemaField(
+            description=(
+                "You can use an E2B sandbox template by entering its ID here. "
+                "Check out the E2B docs for more details: "
+                "[E2B - Sandbox template](https://e2b.dev/docs/sandbox-template)"
+            ),
+            default="",
+            advanced=True,
+        )
+
+    class Output(BlockSchema):
+        sandbox_id: str = SchemaField(description="ID of the sandbox instance")
+        response: str = SchemaField(description="Response from code execution")
+        stdout_logs: str = SchemaField(
+            description="Standard output logs from execution"
+        )
+        stderr_logs: str = SchemaField(description="Standard error logs from execution")
+        error: str = SchemaField(description="Error message if execution failed")
+
+    def __init__(self):
+        super().__init__(
+            id="ff0861c9-1726-4aec-9e5b-bf53f3622112",
+            description="Instantiate an isolated sandbox environment with internet access where to execute code in.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=InstantiationBlock.Input,
+            output_schema=InstantiationBlock.Output,
+            test_credentials=TEST_CREDENTIALS,
+            test_input={
+                "credentials": TEST_CREDENTIALS_INPUT,
+                "setup_code": "print('Hello World')",
+                "language": ProgrammingLanguage.PYTHON.value,
+                "setup_commands": [],
+                "timeout": 300,
+                "template_id": "",
+            },
+            test_output=[
+                ("sandbox_id", str),
+                ("response", "Hello World"),
+                ("stdout_logs", "Hello World\n"),
+            ],
+            test_mock={
+                "execute_code": lambda setup_code, language, setup_commands, timeout, api_key, template_id: (
+                    "sandbox_id",
+                    "Hello World",
+                    "Hello World\n",
+                    "",
+                ),
+            },
+        )
+
+    async def run(
+        self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
+    ) -> BlockOutput:
+        try:
+            sandbox_id, response, stdout_logs, stderr_logs = await self.execute_code(
+                input_data.setup_code,
+                input_data.language,
+                input_data.setup_commands,
+                input_data.timeout,
+                credentials.api_key.get_secret_value(),
+                input_data.template_id,
+            )
+            if sandbox_id:
+                yield "sandbox_id", sandbox_id
+            else:
+                yield "error", "Sandbox ID not found"
+            if response:
+                yield "response", response
+            if stdout_logs:
+                yield "stdout_logs", stdout_logs
+            if stderr_logs:
+                yield "stderr_logs", stderr_logs
+        except Exception as e:
+            yield "error", str(e)
+
+    async def execute_code(
+        self,
+        code: str,
+        language: ProgrammingLanguage,
+        setup_commands: list[str],
+        timeout: int,
+        api_key: str,
+        template_id: str,
+    ):
+        try:
+            sandbox = None
+            if template_id:
+                sandbox = await AsyncSandbox.create(
+                    template=template_id, api_key=api_key, timeout=timeout
+                )
+            else:
+                sandbox = await AsyncSandbox.create(api_key=api_key, timeout=timeout)
+
+            if not sandbox:
+                raise Exception("Sandbox not created")
+
+            # Running setup commands
+            for cmd in setup_commands:
+                await sandbox.commands.run(cmd)
+
+            # Executing the code
+            execution = await sandbox.run_code(
+                code,
+                language=language.value,
+                on_error=lambda e: sandbox.kill(),  # Kill the sandbox if there is an error
+            )
+
+            if execution.error:
+                raise Exception(execution.error)
+
+            response = execution.text
+            stdout_logs = "".join(execution.logs.stdout)
+            stderr_logs = "".join(execution.logs.stderr)
+
+            return sandbox.sandbox_id, response, stdout_logs, stderr_logs
+
+        except Exception as e:
+            raise e
+
+
+class StepExecutionBlock(Block):
+    class Input(BlockSchema):
+        credentials: CredentialsMetaInput[
+            Literal[ProviderName.E2B], Literal["api_key"]
+        ] = CredentialsField(
+            description="Enter your api key for the E2B Sandbox. You can get it in here - https://e2b.dev/docs",
+        )
+
+        sandbox_id: str = SchemaField(
+            description="ID of the sandbox instance to execute the code in",
+            advanced=False,
+        )
+
+        step_code: str = SchemaField(
+            description="Code to execute in the sandbox",
+            placeholder="print('Hello, World!')",
+            default="",
+            advanced=False,
+        )
+
+        language: ProgrammingLanguage = SchemaField(
+            description="Programming language to execute",
+            default=ProgrammingLanguage.PYTHON,
+            advanced=False,
+        )
+
+    class Output(BlockSchema):
+        response: str = SchemaField(description="Response from code execution")
+        stdout_logs: str = SchemaField(
+            description="Standard output logs from execution"
+        )
+        stderr_logs: str = SchemaField(description="Standard error logs from execution")
+        error: str = SchemaField(description="Error message if execution failed")
+
+    def __init__(self):
+        super().__init__(
+            id="82b59b8e-ea10-4d57-9161-8b169b0adba6",
+            description="Execute code in a previously instantiated sandbox environment.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=StepExecutionBlock.Input,
+            output_schema=StepExecutionBlock.Output,
+            test_credentials=TEST_CREDENTIALS,
+            test_input={
+                "credentials": TEST_CREDENTIALS_INPUT,
+                "sandbox_id": "sandbox_id",
+                "step_code": "print('Hello World')",
+                "language": ProgrammingLanguage.PYTHON.value,
+            },
+            test_output=[
+                ("response", "Hello World"),
+                ("stdout_logs", "Hello World\n"),
+            ],
+            test_mock={
+                "execute_step_code": lambda sandbox_id, step_code, language, api_key: (
+                    "Hello World",
+                    "Hello World\n",
+                    "",
+                ),
+            },
+        )
+
+    async def execute_step_code(
+        self,
+        sandbox_id: str,
+        code: str,
+        language: ProgrammingLanguage,
+        api_key: str,
+    ):
+        try:
+            sandbox = await AsyncSandbox.connect(sandbox_id=sandbox_id, api_key=api_key)
+            if not sandbox:
+                raise Exception("Sandbox not found")
+
+            # Executing the code
+            execution = await sandbox.run_code(code, language=language.value)
+
+            if execution.error:
+                raise Exception(execution.error)
+
+            response = execution.text
+            stdout_logs = "".join(execution.logs.stdout)
+            stderr_logs = "".join(execution.logs.stderr)
+
+            return response, stdout_logs, stderr_logs
+
+        except Exception as e:
+            raise e
+
+    async def run(
+        self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
+    ) -> BlockOutput:
+        try:
+            response, stdout_logs, stderr_logs = await self.execute_step_code(
+                input_data.sandbox_id,
+                input_data.step_code,
+                input_data.language,
+                credentials.api_key.get_secret_value(),
+            )
+
+            if response:
+                yield "response", response
+            if stdout_logs:
+                yield "stdout_logs", stdout_logs
+            if stderr_logs:
+                yield "stderr_logs", stderr_logs
+        except Exception as e:
+            yield "error", str(e)

autogpt_platform/backend/backend/blocks/code_extraction_block.py

@@ -0,0 +1,110 @@
+import re
+
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+
+
+class CodeExtractionBlock(Block):
+    class Input(BlockSchema):
+        text: str = SchemaField(
+            description="Text containing code blocks to extract (e.g., AI response)",
+            placeholder="Enter text containing code blocks",
+        )
+
+    class Output(BlockSchema):
+        html: str = SchemaField(description="Extracted HTML code")
+        css: str = SchemaField(description="Extracted CSS code")
+        javascript: str = SchemaField(description="Extracted JavaScript code")
+        python: str = SchemaField(description="Extracted Python code")
+        sql: str = SchemaField(description="Extracted SQL code")
+        java: str = SchemaField(description="Extracted Java code")
+        cpp: str = SchemaField(description="Extracted C++ code")
+        csharp: str = SchemaField(description="Extracted C# code")
+        json_code: str = SchemaField(description="Extracted JSON code")
+        bash: str = SchemaField(description="Extracted Bash code")
+        php: str = SchemaField(description="Extracted PHP code")
+        ruby: str = SchemaField(description="Extracted Ruby code")
+        yaml: str = SchemaField(description="Extracted YAML code")
+        markdown: str = SchemaField(description="Extracted Markdown code")
+        typescript: str = SchemaField(description="Extracted TypeScript code")
+        xml: str = SchemaField(description="Extracted XML code")
+        remaining_text: str = SchemaField(
+            description="Remaining text after code extraction"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="d3a7d896-3b78-4f44-8b4b-48fbf4f0bcd8",
+            description="Extracts code blocks from text and identifies their programming languages",
+            categories={BlockCategory.TEXT},
+            input_schema=CodeExtractionBlock.Input,
+            output_schema=CodeExtractionBlock.Output,
+            test_input={
+                "text": "Here's a Python example:\n```python\nprint('Hello World')\n```\nAnd some HTML:\n```html\n<h1>Title</h1>\n```"
+            },
+            test_output=[
+                ("html", "<h1>Title</h1>"),
+                ("python", "print('Hello World')"),
+                ("remaining_text", "Here's a Python example:\nAnd some HTML:"),
+            ],
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        # List of supported programming languages with mapped aliases
+        language_aliases = {
+            "html": ["html", "htm"],
+            "css": ["css"],
+            "javascript": ["javascript", "js"],
+            "python": ["python", "py"],
+            "sql": ["sql"],
+            "java": ["java"],
+            "cpp": ["cpp", "c++"],
+            "csharp": ["csharp", "c#", "cs"],
+            "json_code": ["json"],
+            "bash": ["bash", "shell", "sh"],
+            "php": ["php"],
+            "ruby": ["ruby", "rb"],
+            "yaml": ["yaml", "yml"],
+            "markdown": ["markdown", "md"],
+            "typescript": ["typescript", "ts"],
+            "xml": ["xml"],
+        }
+
+        # Extract code for each language
+        for canonical_name, aliases in language_aliases.items():
+            code = ""
+            # Try each alias for the language
+            for alias in aliases:
+                code_for_alias = self.extract_code(input_data.text, alias)
+                if code_for_alias:
+                    code = code + "\n\n" + code_for_alias if code else code_for_alias
+
+            if code:  # Only yield if there's actual code content
+                yield canonical_name, code
+
+        # Remove all code blocks from the text to get remaining text
+        pattern = (
+            r"```(?:"
+            + "|".join(
+                re.escape(alias)
+                for aliases in language_aliases.values()
+                for alias in aliases
+            )
+            + r")\s+[\s\S]*?```"
+        )
+
+        remaining_text = re.sub(pattern, "", input_data.text).strip()
+        remaining_text = re.sub(r"\n\s*\n", "\n", remaining_text)
+
+        if remaining_text:  # Only yield if there's remaining text
+            yield "remaining_text", remaining_text
+
+    def extract_code(self, text: str, language: str) -> str:
+        # Escape special regex characters in the language string
+        language = re.escape(language)
+        # Extract all code blocks enclosed in ```language``` blocks
+        pattern = re.compile(rf"```{language}\s+(.*?)```", re.DOTALL | re.IGNORECASE)
+        matches = pattern.finditer(text)
+        # Combine all code blocks for this language with newlines between them
+        code_blocks = [match.group(1).strip() for match in matches]
+        return "\n\n".join(code_blocks) if code_blocks else ""

autogpt_platform/backend/backend/blocks/compass/triggers.py

@@ -0,0 +1,60 @@
+from pydantic import BaseModel
+
+from backend.data.block import (
+    Block,
+    BlockCategory,
+    BlockManualWebhookConfig,
+    BlockOutput,
+    BlockSchema,
+)
+from backend.data.model import SchemaField
+from backend.integrations.providers import ProviderName
+from backend.integrations.webhooks.compass import CompassWebhookType
+
+
+class Transcription(BaseModel):
+    text: str
+    speaker: str
+    end: float
+    start: float
+    duration: float
+
+
+class TranscriptionDataModel(BaseModel):
+    date: str
+    transcription: str
+    transcriptions: list[Transcription]
+
+
+class CompassAITriggerBlock(Block):
+    class Input(BlockSchema):
+        payload: TranscriptionDataModel = SchemaField(hidden=True)
+
+    class Output(BlockSchema):
+        transcription: str = SchemaField(
+            description="The contents of the compass transcription."
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="9464a020-ed1d-49e1-990f-7f2ac924a2b7",
+            description="This block will output the contents of the compass transcription.",
+            categories={BlockCategory.HARDWARE},
+            input_schema=CompassAITriggerBlock.Input,
+            output_schema=CompassAITriggerBlock.Output,
+            webhook_config=BlockManualWebhookConfig(
+                provider=ProviderName.COMPASS,
+                webhook_type=CompassWebhookType.TRANSCRIPTION,
+            ),
+            test_input=[
+                {"input": "Hello, World!"},
+                {"input": "Hello, World!", "data": "Existing Data"},
+            ],
+            # test_output=[
+            #     ("output", "Hello, World!"),  # No data provided, so trigger is returned
+            #     ("output", "Existing Data"),  # Data is provided, so data is returned.
+            # ],
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        yield "transcription", input_data.payload.transcription

autogpt_platform/backend/backend/blocks/count_words_and_char_block.py

@@ -30,7 +30,7 @@ class WordCharacterCountBlock(Block):
             test_output=[("word_count", 4), ("character_count", 19)],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         try:
             text = input_data.text
             word_count = len(text.split())

autogpt_platform/backend/backend/blocks/csv.py

@@ -34,7 +34,7 @@ class ReadCsvBlock(Block):
         )
         skip_columns: list[str] = SchemaField(
             description="The columns to skip from the start of the row",
-            default=[],
+            default_factory=list,
         )
 
     class Output(BlockSchema):
@@ -69,7 +69,7 @@ class ReadCsvBlock(Block):
             ],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         import csv
         from io import StringIO
 

autogpt_platform/backend/backend/blocks/decoder_block.py

@@ -34,6 +34,6 @@ This is a "quoted" string.""",
             ],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         decoded_text = codecs.decode(input_data.text, "unicode_escape")
         yield "decoded_text", decoded_text

autogpt_platform/backend/backend/blocks/discord.py

@@ -1,4 +1,3 @@
-import asyncio
 from typing import Literal
 
 import aiohttp
@@ -12,16 +11,15 @@ from backend.data.model import (
     CredentialsMetaInput,
     SchemaField,
 )
+from backend.integrations.providers import ProviderName
 
-DiscordCredentials = CredentialsMetaInput[Literal["discord"], Literal["api_key"]]
+DiscordCredentials = CredentialsMetaInput[
+    Literal[ProviderName.DISCORD], Literal["api_key"]
+]
 
 
 def DiscordCredentialsField() -> DiscordCredentials:
-    return CredentialsField(
-        description="Discord bot token",
-        provider="discord",
-        supported_credential_types={"api_key"},
-    )
+    return CredentialsField(description="Discord bot token")
 
 
 TEST_CREDENTIALS = APIKeyCredentials(
@@ -75,7 +73,11 @@ class ReadDiscordMessagesBlock(Block):
                 ("username", "test_user"),
             ],
             test_mock={
-                "run_bot": lambda token: asyncio.Future()  # Create a Future object for mocking
+                "run_bot": lambda token: {
+                    "output_data": "Hello!\n\nFile from user: example.txt\nContent: This is the content of the file.",
+                    "channel_name": "general",
+                    "username": "test_user",
+                }
             },
         )
 
@@ -107,37 +109,24 @@ class ReadDiscordMessagesBlock(Block):
                 if attachment.filename.endswith((".txt", ".py")):
                     async with aiohttp.ClientSession() as session:
                         async with session.get(attachment.url) as response:
-                            file_content = await response.text()
+                            file_content = response.text()
                             self.output_data += f"\n\nFile from user: {attachment.filename}\nContent: {file_content}"
 
             await client.close()
 
         await client.start(token.get_secret_value())
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
-        while True:
-            for output_name, output_value in self.__run(input_data, credentials):
-                yield output_name, output_value
-            break
+        async for output_name, output_value in self.__run(input_data, credentials):
+            yield output_name, output_value
 
-    def __run(self, input_data: Input, credentials: APIKeyCredentials) -> BlockOutput:
+    async def __run(
+        self, input_data: Input, credentials: APIKeyCredentials
+    ) -> BlockOutput:
         try:
-            loop = asyncio.get_event_loop()
-            future = self.run_bot(credentials.api_key)
-
-            # If it's a Future (mock), set the result
-            if isinstance(future, asyncio.Future):
-                future.set_result(
-                    {
-                        "output_data": "Hello!\n\nFile from user: example.txt\nContent: This is the content of the file.",
-                        "channel_name": "general",
-                        "username": "test_user",
-                    }
-                )
-
-            result = loop.run_until_complete(future)
+            result = await self.run_bot(credentials.api_key)
 
             # For testing purposes, use the mocked result
             if isinstance(result, dict):
@@ -191,7 +180,7 @@ class SendDiscordMessageBlock(Block):
             },
             test_output=[("status", "Message sent")],
             test_mock={
-                "send_message": lambda token, channel_name, message_content: asyncio.Future()
+                "send_message": lambda token, channel_name, message_content: "Message sent"
             },
             test_credentials=TEST_CREDENTIALS,
         )
@@ -223,23 +212,16 @@ class SendDiscordMessageBlock(Block):
         """Splits a message into chunks not exceeding the Discord limit."""
         return [message[i : i + limit] for i in range(0, len(message), limit)]
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
         try:
-            loop = asyncio.get_event_loop()
-            future = self.send_message(
+            result = await self.send_message(
                 credentials.api_key.get_secret_value(),
                 input_data.channel_name,
                 input_data.message_content,
             )
 
-            # If it's a Future (mock), set the result
-            if isinstance(future, asyncio.Future):
-                future.set_result("Message sent")
-
-            result = loop.run_until_complete(future)
-
             # For testing purposes, use the mocked result
             if isinstance(result, str):
                 self.output_data = result

autogpt_platform/backend/backend/blocks/email_block.py

@@ -1,22 +1,53 @@
 import smtplib
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
+from typing import Literal
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, SecretStr
 
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from backend.data.model import BlockSecret, SchemaField, SecretField
+from backend.data.model import (
+    CredentialsField,
+    CredentialsMetaInput,
+    SchemaField,
+    UserPasswordCredentials,
+)
+from backend.integrations.providers import ProviderName
+
+TEST_CREDENTIALS = UserPasswordCredentials(
+    id="01234567-89ab-cdef-0123-456789abcdef",
+    provider="smtp",
+    username=SecretStr("mock-smtp-username"),
+    password=SecretStr("mock-smtp-password"),
+    title="Mock SMTP credentials",
+)
+
+TEST_CREDENTIALS_INPUT = {
+    "provider": TEST_CREDENTIALS.provider,
+    "id": TEST_CREDENTIALS.id,
+    "type": TEST_CREDENTIALS.type,
+    "title": TEST_CREDENTIALS.title,
+}
+SMTPCredentials = UserPasswordCredentials
+SMTPCredentialsInput = CredentialsMetaInput[
+    Literal[ProviderName.SMTP],
+    Literal["user_password"],
+]
 
 
-class EmailCredentials(BaseModel):
+def SMTPCredentialsField() -> SMTPCredentialsInput:
+    return CredentialsField(
+        description="The SMTP integration requires a username and password.",
+    )
+
+
+class SMTPConfig(BaseModel):
     smtp_server: str = SchemaField(
-        default="smtp.gmail.com", description="SMTP server address"
+        default="smtp.example.com", description="SMTP server address"
     )
     smtp_port: int = SchemaField(default=25, description="SMTP port number")
-    smtp_username: BlockSecret = SecretField(key="smtp_username")
-    smtp_password: BlockSecret = SecretField(key="smtp_password")
 
-    model_config = ConfigDict(title="Email Credentials")
+    model_config = ConfigDict(title="SMTP Config")
 
 
 class SendEmailBlock(Block):
@@ -30,10 +61,11 @@ class SendEmailBlock(Block):
         body: str = SchemaField(
             description="Body of the email", placeholder="Enter the email body"
         )
-        creds: EmailCredentials = SchemaField(
-            description="SMTP credentials",
-            default=EmailCredentials(),
+        config: SMTPConfig = SchemaField(
+            description="SMTP Config",
+            default=SMTPConfig(),
         )
+        credentials: SMTPCredentialsInput = SMTPCredentialsField()
 
     class Output(BlockSchema):
         status: str = SchemaField(description="Status of the email sending operation")
@@ -43,7 +75,6 @@ class SendEmailBlock(Block):
 
     def __init__(self):
         super().__init__(
-            disabled=True,
             id="4335878a-394e-4e67-adf2-919877ff49ae",
             description="This block sends an email using the provided SMTP credentials.",
             categories={BlockCategory.OUTPUT},
@@ -53,25 +84,29 @@ class SendEmailBlock(Block):
                 "to_email": "recipient@example.com",
                 "subject": "Test Email",
                 "body": "This is a test email.",
-                "creds": {
+                "config": {
                     "smtp_server": "smtp.gmail.com",
                     "smtp_port": 25,
-                    "smtp_username": "your-email@gmail.com",
-                    "smtp_password": "your-gmail-password",
                 },
+                "credentials": TEST_CREDENTIALS_INPUT,
             },
+            test_credentials=TEST_CREDENTIALS,
             test_output=[("status", "Email sent successfully")],
             test_mock={"send_email": lambda *args, **kwargs: "Email sent successfully"},
         )
 
     @staticmethod
     def send_email(
-        creds: EmailCredentials, to_email: str, subject: str, body: str
+        config: SMTPConfig,
+        to_email: str,
+        subject: str,
+        body: str,
+        credentials: SMTPCredentials,
     ) -> str:
-        smtp_server = creds.smtp_server
-        smtp_port = creds.smtp_port
-        smtp_username = creds.smtp_username.get_secret_value()
-        smtp_password = creds.smtp_password.get_secret_value()
+        smtp_server = config.smtp_server
+        smtp_port = config.smtp_port
+        smtp_username = credentials.username.get_secret_value()
+        smtp_password = credentials.password.get_secret_value()
 
         msg = MIMEMultipart()
         msg["From"] = smtp_username
@@ -86,10 +121,13 @@ class SendEmailBlock(Block):
 
         return "Email sent successfully"
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(
+        self, input_data: Input, *, credentials: SMTPCredentials, **kwargs
+    ) -> BlockOutput:
         yield "status", self.send_email(
-            input_data.creds,
-            input_data.to_email,
-            input_data.subject,
-            input_data.body,
+            config=input_data.config,
+            to_email=input_data.to_email,
+            subject=input_data.subject,
+            body=input_data.body,
+            credentials=credentials,
         )

autogpt_platform/backend/backend/blocks/exa/_auth.py

@@ -0,0 +1,32 @@
+from typing import Literal
+
+from pydantic import SecretStr
+
+from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput
+from backend.integrations.providers import ProviderName
+
+ExaCredentials = APIKeyCredentials
+ExaCredentialsInput = CredentialsMetaInput[
+    Literal[ProviderName.EXA],
+    Literal["api_key"],
+]
+
+TEST_CREDENTIALS = APIKeyCredentials(
+    id="01234567-89ab-cdef-0123-456789abcdef",
+    provider="exa",
+    api_key=SecretStr("mock-exa-api-key"),
+    title="Mock Exa API key",
+    expires_at=None,
+)
+
+TEST_CREDENTIALS_INPUT = {
+    "provider": TEST_CREDENTIALS.provider,
+    "id": TEST_CREDENTIALS.id,
+    "type": TEST_CREDENTIALS.type,
+    "title": TEST_CREDENTIALS.title,
+}
+
+
+def ExaCredentialsField() -> ExaCredentialsInput:
+    """Creates an Exa credentials input on a block."""
+    return CredentialsField(description="The Exa integration requires an API Key.")

autogpt_platform/backend/backend/blocks/exa/contents.py

@@ -0,0 +1,86 @@
+from typing import List
+
+from pydantic import BaseModel
+
+from backend.blocks.exa._auth import (
+    ExaCredentials,
+    ExaCredentialsField,
+    ExaCredentialsInput,
+)
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+from backend.util.request import Requests
+
+
+class ContentRetrievalSettings(BaseModel):
+    text: dict = SchemaField(
+        description="Text content settings",
+        default={"maxCharacters": 1000, "includeHtmlTags": False},
+        advanced=True,
+    )
+    highlights: dict = SchemaField(
+        description="Highlight settings",
+        default={
+            "numSentences": 3,
+            "highlightsPerUrl": 3,
+            "query": "",
+        },
+        advanced=True,
+    )
+    summary: dict = SchemaField(
+        description="Summary settings",
+        default={"query": ""},
+        advanced=True,
+    )
+
+
+class ExaContentsBlock(Block):
+    class Input(BlockSchema):
+        credentials: ExaCredentialsInput = ExaCredentialsField()
+        ids: List[str] = SchemaField(
+            description="Array of document IDs obtained from searches",
+        )
+        contents: ContentRetrievalSettings = SchemaField(
+            description="Content retrieval settings",
+            default=ContentRetrievalSettings(),
+            advanced=True,
+        )
+
+    class Output(BlockSchema):
+        results: list = SchemaField(
+            description="List of document contents",
+            default_factory=list,
+        )
+        error: str = SchemaField(description="Error message if the request failed")
+
+    def __init__(self):
+        super().__init__(
+            id="c52be83f-f8cd-4180-b243-af35f986b461",
+            description="Retrieves document contents using Exa's contents API",
+            categories={BlockCategory.SEARCH},
+            input_schema=ExaContentsBlock.Input,
+            output_schema=ExaContentsBlock.Output,
+        )
+
+    async def run(
+        self, input_data: Input, *, credentials: ExaCredentials, **kwargs
+    ) -> BlockOutput:
+        url = "https://api.exa.ai/contents"
+        headers = {
+            "Content-Type": "application/json",
+            "x-api-key": credentials.api_key.get_secret_value(),
+        }
+
+        payload = {
+            "ids": input_data.ids,
+            "text": input_data.contents.text,
+            "highlights": input_data.contents.highlights,
+            "summary": input_data.contents.summary,
+        }
+
+        try:
+            response = await Requests().post(url, headers=headers, json=payload)
+            data = response.json()
+            yield "results", data.get("results", [])
+        except Exception as e:
+            yield "error", str(e)

autogpt_platform/backend/backend/blocks/exa/helpers.py

@@ -0,0 +1,54 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from backend.data.model import SchemaField
+
+
+class TextSettings(BaseModel):
+    max_characters: int = SchemaField(
+        default=1000,
+        description="Maximum number of characters to return",
+        placeholder="1000",
+    )
+    include_html_tags: bool = SchemaField(
+        default=False,
+        description="Whether to include HTML tags in the text",
+        placeholder="False",
+    )
+
+
+class HighlightSettings(BaseModel):
+    num_sentences: int = SchemaField(
+        default=3,
+        description="Number of sentences per highlight",
+        placeholder="3",
+    )
+    highlights_per_url: int = SchemaField(
+        default=3,
+        description="Number of highlights per URL",
+        placeholder="3",
+    )
+
+
+class SummarySettings(BaseModel):
+    query: Optional[str] = SchemaField(
+        default="",
+        description="Query string for summarization",
+        placeholder="Enter query",
+    )
+
+
+class ContentSettings(BaseModel):
+    text: TextSettings = SchemaField(
+        default=TextSettings(),
+        description="Text content settings",
+    )
+    highlights: HighlightSettings = SchemaField(
+        default=HighlightSettings(),
+        description="Highlight settings",
+    )
+    summary: SummarySettings = SchemaField(
+        default=SummarySettings(),
+        description="Summary settings",
+    )

autogpt_platform/backend/backend/blocks/exa/search.py

@@ -0,0 +1,144 @@
+from datetime import datetime
+from typing import List
+
+from backend.blocks.exa._auth import (
+    ExaCredentials,
+    ExaCredentialsField,
+    ExaCredentialsInput,
+)
+from backend.blocks.exa.helpers import ContentSettings
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+from backend.util.request import Requests
+
+
+class ExaSearchBlock(Block):
+    class Input(BlockSchema):
+        credentials: ExaCredentialsInput = ExaCredentialsField()
+        query: str = SchemaField(description="The search query")
+        use_auto_prompt: bool = SchemaField(
+            description="Whether to use autoprompt",
+            default=True,
+            advanced=True,
+        )
+        type: str = SchemaField(
+            description="Type of search",
+            default="",
+            advanced=True,
+        )
+        category: str = SchemaField(
+            description="Category to search within",
+            default="",
+            advanced=True,
+        )
+        number_of_results: int = SchemaField(
+            description="Number of results to return",
+            default=10,
+            advanced=True,
+        )
+        include_domains: List[str] = SchemaField(
+            description="Domains to include in search",
+            default_factory=list,
+        )
+        exclude_domains: List[str] = SchemaField(
+            description="Domains to exclude from search",
+            default_factory=list,
+            advanced=True,
+        )
+        start_crawl_date: datetime = SchemaField(
+            description="Start date for crawled content",
+        )
+        end_crawl_date: datetime = SchemaField(
+            description="End date for crawled content",
+        )
+        start_published_date: datetime = SchemaField(
+            description="Start date for published content",
+        )
+        end_published_date: datetime = SchemaField(
+            description="End date for published content",
+        )
+        include_text: List[str] = SchemaField(
+            description="Text patterns to include",
+            default_factory=list,
+            advanced=True,
+        )
+        exclude_text: List[str] = SchemaField(
+            description="Text patterns to exclude",
+            default_factory=list,
+            advanced=True,
+        )
+        contents: ContentSettings = SchemaField(
+            description="Content retrieval settings",
+            default=ContentSettings(),
+            advanced=True,
+        )
+
+    class Output(BlockSchema):
+        results: list = SchemaField(
+            description="List of search results",
+            default_factory=list,
+        )
+        error: str = SchemaField(
+            description="Error message if the request failed",
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="996cec64-ac40-4dde-982f-b0dc60a5824d",
+            description="Searches the web using Exa's advanced search API",
+            categories={BlockCategory.SEARCH},
+            input_schema=ExaSearchBlock.Input,
+            output_schema=ExaSearchBlock.Output,
+        )
+
+    async def run(
+        self, input_data: Input, *, credentials: ExaCredentials, **kwargs
+    ) -> BlockOutput:
+        url = "https://api.exa.ai/search"
+        headers = {
+            "Content-Type": "application/json",
+            "x-api-key": credentials.api_key.get_secret_value(),
+        }
+
+        payload = {
+            "query": input_data.query,
+            "useAutoprompt": input_data.use_auto_prompt,
+            "numResults": input_data.number_of_results,
+            "contents": input_data.contents.dict(),
+        }
+
+        date_field_mapping = {
+            "start_crawl_date": "startCrawlDate",
+            "end_crawl_date": "endCrawlDate",
+            "start_published_date": "startPublishedDate",
+            "end_published_date": "endPublishedDate",
+        }
+
+        # Add dates if they exist
+        for input_field, api_field in date_field_mapping.items():
+            value = getattr(input_data, input_field, None)
+            if value:
+                payload[api_field] = value.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+        optional_field_mapping = {
+            "type": "type",
+            "category": "category",
+            "include_domains": "includeDomains",
+            "exclude_domains": "excludeDomains",
+            "include_text": "includeText",
+            "exclude_text": "excludeText",
+        }
+
+        # Add other fields
+        for input_field, api_field in optional_field_mapping.items():
+            value = getattr(input_data, input_field)
+            if value:  # Only add non-empty values
+                payload[api_field] = value
+
+        try:
+            response = await Requests().post(url, headers=headers, json=payload)
+            data = response.json()
+            # Extract just the results array from the response
+            yield "results", data.get("results", [])
+        except Exception as e:
+            yield "error", str(e)

autogpt_platform/backend/backend/blocks/exa/similar.py

@@ -0,0 +1,127 @@
+from datetime import datetime
+from typing import Any, List
+
+from backend.blocks.exa._auth import (
+    ExaCredentials,
+    ExaCredentialsField,
+    ExaCredentialsInput,
+)
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+from backend.util.request import Requests
+
+from .helpers import ContentSettings
+
+
+class ExaFindSimilarBlock(Block):
+    class Input(BlockSchema):
+        credentials: ExaCredentialsInput = ExaCredentialsField()
+        url: str = SchemaField(
+            description="The url for which you would like to find similar links"
+        )
+        number_of_results: int = SchemaField(
+            description="Number of results to return",
+            default=10,
+            advanced=True,
+        )
+        include_domains: List[str] = SchemaField(
+            description="Domains to include in search",
+            default_factory=list,
+            advanced=True,
+        )
+        exclude_domains: List[str] = SchemaField(
+            description="Domains to exclude from search",
+            default_factory=list,
+            advanced=True,
+        )
+        start_crawl_date: datetime = SchemaField(
+            description="Start date for crawled content",
+        )
+        end_crawl_date: datetime = SchemaField(
+            description="End date for crawled content",
+        )
+        start_published_date: datetime = SchemaField(
+            description="Start date for published content",
+        )
+        end_published_date: datetime = SchemaField(
+            description="End date for published content",
+        )
+        include_text: List[str] = SchemaField(
+            description="Text patterns to include (max 1 string, up to 5 words)",
+            default_factory=list,
+            advanced=True,
+        )
+        exclude_text: List[str] = SchemaField(
+            description="Text patterns to exclude (max 1 string, up to 5 words)",
+            default_factory=list,
+            advanced=True,
+        )
+        contents: ContentSettings = SchemaField(
+            description="Content retrieval settings",
+            default=ContentSettings(),
+            advanced=True,
+        )
+
+    class Output(BlockSchema):
+        results: List[Any] = SchemaField(
+            description="List of similar documents with title, URL, published date, author, and score",
+            default_factory=list,
+        )
+        error: str = SchemaField(description="Error message if the request failed")
+
+    def __init__(self):
+        super().__init__(
+            id="5e7315d1-af61-4a0c-9350-7c868fa7438a",
+            description="Finds similar links using Exa's findSimilar API",
+            categories={BlockCategory.SEARCH},
+            input_schema=ExaFindSimilarBlock.Input,
+            output_schema=ExaFindSimilarBlock.Output,
+        )
+
+    async def run(
+        self, input_data: Input, *, credentials: ExaCredentials, **kwargs
+    ) -> BlockOutput:
+        url = "https://api.exa.ai/findSimilar"
+        headers = {
+            "Content-Type": "application/json",
+            "x-api-key": credentials.api_key.get_secret_value(),
+        }
+
+        payload = {
+            "url": input_data.url,
+            "numResults": input_data.number_of_results,
+            "contents": input_data.contents.dict(),
+        }
+
+        optional_field_mapping = {
+            "include_domains": "includeDomains",
+            "exclude_domains": "excludeDomains",
+            "include_text": "includeText",
+            "exclude_text": "excludeText",
+        }
+
+        # Add optional fields if they have values
+        for input_field, api_field in optional_field_mapping.items():
+            value = getattr(input_data, input_field)
+            if value:  # Only add non-empty values
+                payload[api_field] = value
+
+        date_field_mapping = {
+            "start_crawl_date": "startCrawlDate",
+            "end_crawl_date": "endCrawlDate",
+            "start_published_date": "startPublishedDate",
+            "end_published_date": "endPublishedDate",
+        }
+
+        # Add dates if they exist
+        for input_field, api_field in date_field_mapping.items():
+            value = getattr(input_data, input_field, None)
+            if value:
+                payload[api_field] = value.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+        try:
+            response = await Requests().post(url, headers=headers, json=payload)
+            data = response.json()
+            yield "results", data.get("results", [])
+        except Exception as e:
+            yield "error", str(e)

autogpt_platform/backend/backend/blocks/fal/_auth.py

@@ -3,10 +3,11 @@ from typing import Literal
 from pydantic import SecretStr
 
 from backend.data.model import APIKeyCredentials, CredentialsField, CredentialsMetaInput
+from backend.integrations.providers import ProviderName
 
 FalCredentials = APIKeyCredentials
 FalCredentialsInput = CredentialsMetaInput[
-    Literal["fal"],
+    Literal[ProviderName.FAL],
     Literal["api_key"],
 ]
 
@@ -30,7 +31,5 @@ def FalCredentialsField() -> FalCredentialsInput:
     Creates a FAL credentials input on a block.
     """
     return CredentialsField(
-        provider="fal",
-        supported_credential_types={"api_key"},
         description="The FAL integration can be used with an API Key.",
     )

autogpt_platform/backend/backend/blocks/fal/ai_video_generator.py

@@ -1,9 +1,7 @@
+import asyncio
 import logging
-import time
 from enum import Enum
-from typing import Any, Dict
-
-import httpx
+from typing import Any
 
 from backend.blocks.fal._auth import (
     TEST_CREDENTIALS,
@@ -14,6 +12,7 @@ from backend.blocks.fal._auth import (
 )
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 from backend.data.model import SchemaField
+from backend.util.request import ClientResponseError, Requests
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +20,7 @@ logger = logging.getLogger(__name__)
 class FalModel(str, Enum):
     MOCHI = "fal-ai/mochi-v1"
     LUMA = "fal-ai/luma-dream-machine"
+    VEO3 = "fal-ai/veo3"
 
 
 class AIVideoGeneratorBlock(Block):
@@ -42,7 +42,7 @@ class AIVideoGeneratorBlock(Block):
             description="Error message if video generation failed."
         )
         logs: list[str] = SchemaField(
-            description="Generation progress logs.", optional=True
+            description="Generation progress logs.",
         )
 
     def __init__(self):
@@ -64,36 +64,38 @@ class AIVideoGeneratorBlock(Block):
             },
         )
 
-    def _get_headers(self, api_key: str) -> Dict[str, str]:
-        """Get headers for FAL API requests."""
+    def _get_headers(self, api_key: str) -> dict[str, str]:
+        """Get headers for FAL API Requests."""
         return {
             "Authorization": f"Key {api_key}",
             "Content-Type": "application/json",
         }
 
-    def _submit_request(
-        self, url: str, headers: Dict[str, str], data: Dict[str, Any]
-    ) -> Dict[str, Any]:
+    async def _submit_request(
+        self, url: str, headers: dict[str, str], data: dict[str, Any]
+    ) -> dict[str, Any]:
         """Submit a request to the FAL API."""
         try:
-            response = httpx.post(url, headers=headers, json=data)
-            response.raise_for_status()
+            response = await Requests().post(url, headers=headers, json=data)
             return response.json()
-        except httpx.HTTPError as e:
+        except ClientResponseError as e:
             logger.error(f"FAL API request failed: {str(e)}")
             raise RuntimeError(f"Failed to submit request: {str(e)}")
 
-    def _poll_status(self, status_url: str, headers: Dict[str, str]) -> Dict[str, Any]:
+    async def _poll_status(
+        self, status_url: str, headers: dict[str, str]
+    ) -> dict[str, Any]:
         """Poll the status endpoint until completion or failure."""
         try:
-            response = httpx.get(status_url, headers=headers)
-            response.raise_for_status()
+            response = await Requests().get(status_url, headers=headers)
             return response.json()
-        except httpx.HTTPError as e:
+        except ClientResponseError as e:
             logger.error(f"Failed to get status: {str(e)}")
             raise RuntimeError(f"Failed to get status: {str(e)}")
 
-    def generate_video(self, input_data: Input, credentials: FalCredentials) -> str:
+    async def generate_video(
+        self, input_data: Input, credentials: FalCredentials
+    ) -> str:
         """Generate video using the specified FAL model."""
         base_url = "https://queue.fal.run"
         api_key = credentials.api_key.get_secret_value()
@@ -102,13 +104,16 @@ class AIVideoGeneratorBlock(Block):
         # Submit generation request
         submit_url = f"{base_url}/{input_data.model.value}"
         submit_data = {"prompt": input_data.prompt}
+        if input_data.model == FalModel.VEO3:
+            submit_data["generate_audio"] = True  # type: ignore
 
         seen_logs = set()
 
         try:
             # Submit request to queue
-            submit_response = httpx.post(submit_url, headers=headers, json=submit_data)
-            submit_response.raise_for_status()
+            submit_response = await Requests().post(
+                submit_url, headers=headers, json=submit_data
+            )
             request_data = submit_response.json()
 
             # Get request_id and urls from initial response
@@ -119,14 +124,23 @@ class AIVideoGeneratorBlock(Block):
             if not all([request_id, status_url, result_url]):
                 raise ValueError("Missing required data in submission response")
 
+            # Ensure status_url is a string
+            if not isinstance(status_url, str):
+                raise ValueError("Invalid status URL format")
+
+            # Ensure result_url is a string
+            if not isinstance(result_url, str):
+                raise ValueError("Invalid result URL format")
+
             # Poll for status with exponential backoff
             max_attempts = 30
             attempt = 0
             base_wait_time = 5
 
             while attempt < max_attempts:
-                status_response = httpx.get(f"{status_url}?logs=1", headers=headers)
-                status_response.raise_for_status()
+                status_response = await Requests().get(
+                    f"{status_url}?logs=1", headers=headers
+                )
                 status_data = status_response.json()
 
                 # Process new logs only
@@ -149,8 +163,7 @@ class AIVideoGeneratorBlock(Block):
                 status = status_data.get("status")
                 if status == "COMPLETED":
                     # Get the final result
-                    result_response = httpx.get(result_url, headers=headers)
-                    result_response.raise_for_status()
+                    result_response = await Requests().get(result_url, headers=headers)
                     result_data = result_response.json()
 
                     if "video" not in result_data or not isinstance(
@@ -159,8 +172,8 @@ class AIVideoGeneratorBlock(Block):
                         raise ValueError("Invalid response format - missing video data")
 
                     video_url = result_data["video"].get("url")
-                    if not video_url:
-                        raise ValueError("No video URL in response")
+                    if not video_url or not isinstance(video_url, str):
+                        raise ValueError("No valid video URL in response")
 
                     return video_url
 
@@ -180,19 +193,19 @@ class AIVideoGeneratorBlock(Block):
                     logger.info(f"[FAL Generation] Status: Unknown status: {status}")
 
                 wait_time = min(base_wait_time * (2**attempt), 60)  # Cap at 60 seconds
-                time.sleep(wait_time)
+                await asyncio.sleep(wait_time)
                 attempt += 1
 
             raise RuntimeError("Maximum polling attempts reached")
 
-        except httpx.HTTPError as e:
+        except ClientResponseError as e:
             raise RuntimeError(f"API request failed: {str(e)}")
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: FalCredentials, **kwargs
     ) -> BlockOutput:
         try:
-            video_url = self.generate_video(input_data, credentials)
+            video_url = await self.generate_video(input_data, credentials)
             yield "video_url", video_url
         except Exception as e:
             error_message = str(e)

autogpt_platform/backend/backend/blocks/flux_kontext.py

@@ -0,0 +1,174 @@
+from enum import Enum
+from typing import Literal, Optional
+
+from pydantic import SecretStr
+from replicate.client import Client as ReplicateClient
+from replicate.helpers import FileOutput
+
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import (
+    APIKeyCredentials,
+    CredentialsField,
+    CredentialsMetaInput,
+    SchemaField,
+)
+from backend.integrations.providers import ProviderName
+from backend.util.file import MediaFileType
+
+TEST_CREDENTIALS = APIKeyCredentials(
+    id="01234567-89ab-cdef-0123-456789abcdef",
+    provider="replicate",
+    api_key=SecretStr("mock-replicate-api-key"),
+    title="Mock Replicate API key",
+    expires_at=None,
+)
+TEST_CREDENTIALS_INPUT = {
+    "provider": TEST_CREDENTIALS.provider,
+    "id": TEST_CREDENTIALS.id,
+    "type": TEST_CREDENTIALS.type,
+    "title": TEST_CREDENTIALS.type,
+}
+
+
+class FluxKontextModelName(str, Enum):
+    PRO = "Flux Kontext Pro"
+    MAX = "Flux Kontext Max"
+
+    @property
+    def api_name(self) -> str:
+        return f"black-forest-labs/flux-kontext-{self.name.lower()}"
+
+
+class AspectRatio(str, Enum):
+    MATCH_INPUT_IMAGE = "match_input_image"
+    ASPECT_1_1 = "1:1"
+    ASPECT_16_9 = "16:9"
+    ASPECT_9_16 = "9:16"
+    ASPECT_4_3 = "4:3"
+    ASPECT_3_4 = "3:4"
+    ASPECT_3_2 = "3:2"
+    ASPECT_2_3 = "2:3"
+    ASPECT_4_5 = "4:5"
+    ASPECT_5_4 = "5:4"
+    ASPECT_21_9 = "21:9"
+    ASPECT_9_21 = "9:21"
+    ASPECT_2_1 = "2:1"
+    ASPECT_1_2 = "1:2"
+
+
+class AIImageEditorBlock(Block):
+    class Input(BlockSchema):
+        credentials: CredentialsMetaInput[
+            Literal[ProviderName.REPLICATE], Literal["api_key"]
+        ] = CredentialsField(
+            description="Replicate API key with permissions for Flux Kontext models",
+        )
+        prompt: str = SchemaField(
+            description="Text instruction describing the desired edit",
+            title="Prompt",
+        )
+        input_image: Optional[MediaFileType] = SchemaField(
+            description="Reference image URI (jpeg, png, gif, webp)",
+            default=None,
+            title="Input Image",
+        )
+        aspect_ratio: AspectRatio = SchemaField(
+            description="Aspect ratio of the generated image",
+            default=AspectRatio.MATCH_INPUT_IMAGE,
+            title="Aspect Ratio",
+            advanced=False,
+        )
+        seed: Optional[int] = SchemaField(
+            description="Random seed. Set for reproducible generation",
+            default=None,
+            title="Seed",
+            advanced=True,
+        )
+        model: FluxKontextModelName = SchemaField(
+            description="Model variant to use",
+            default=FluxKontextModelName.PRO,
+            title="Model",
+        )
+
+    class Output(BlockSchema):
+        output_image: MediaFileType = SchemaField(
+            description="URL of the transformed image"
+        )
+        error: str = SchemaField(description="Error message if generation failed")
+
+    def __init__(self):
+        super().__init__(
+            id="3fd9c73d-4370-4925-a1ff-1b86b99fabfa",
+            description=(
+                "Edit images using BlackForest Labs' Flux Kontext models. Provide a prompt "
+                "and optional reference image to generate a modified image."
+            ),
+            categories={BlockCategory.AI, BlockCategory.MULTIMEDIA},
+            input_schema=AIImageEditorBlock.Input,
+            output_schema=AIImageEditorBlock.Output,
+            test_input={
+                "prompt": "Add a hat to the cat",
+                "input_image": "https://example.com/cat.png",
+                "aspect_ratio": AspectRatio.MATCH_INPUT_IMAGE,
+                "seed": None,
+                "model": FluxKontextModelName.PRO,
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_output=[
+                ("output_image", "https://replicate.com/output/edited-image.png"),
+            ],
+            test_mock={
+                "run_model": lambda *args, **kwargs: "https://replicate.com/output/edited-image.png",
+            },
+            test_credentials=TEST_CREDENTIALS,
+        )
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: APIKeyCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        result = await self.run_model(
+            api_key=credentials.api_key,
+            model_name=input_data.model.api_name,
+            prompt=input_data.prompt,
+            input_image=input_data.input_image,
+            aspect_ratio=input_data.aspect_ratio.value,
+            seed=input_data.seed,
+        )
+        yield "output_image", result
+
+    async def run_model(
+        self,
+        api_key: SecretStr,
+        model_name: str,
+        prompt: str,
+        input_image: Optional[MediaFileType],
+        aspect_ratio: str,
+        seed: Optional[int],
+    ) -> MediaFileType:
+        client = ReplicateClient(api_token=api_key.get_secret_value())
+        input_params = {
+            "prompt": prompt,
+            "input_image": input_image,
+            "aspect_ratio": aspect_ratio,
+            **({"seed": seed} if seed is not None else {}),
+        }
+
+        output: FileOutput | list[FileOutput] = await client.async_run(  # type: ignore
+            model_name,
+            input=input_params,
+            wait=False,
+        )
+
+        if isinstance(output, list) and output:
+            output = output[0]
+
+        if isinstance(output, FileOutput):
+            return MediaFileType(output.url)
+        if isinstance(output, str):
+            return MediaFileType(output)
+
+        raise ValueError("No output received")

autogpt_platform/backend/backend/blocks/generic_webhook/triggers.py

@@ -0,0 +1,51 @@
+from backend.data.block import (
+    Block,
+    BlockCategory,
+    BlockManualWebhookConfig,
+    BlockOutput,
+    BlockSchema,
+)
+from backend.data.model import SchemaField
+from backend.integrations.providers import ProviderName
+from backend.integrations.webhooks.generic import GenericWebhookType
+
+
+class GenericWebhookTriggerBlock(Block):
+    class Input(BlockSchema):
+        payload: dict = SchemaField(hidden=True, default_factory=dict)
+        constants: dict = SchemaField(
+            description="The constants to be set when the block is put on the graph",
+            default_factory=dict,
+        )
+
+    class Output(BlockSchema):
+        payload: dict = SchemaField(
+            description="The complete webhook payload that was received from the generic webhook."
+        )
+        constants: dict = SchemaField(
+            description="The constants to be set when the block is put on the graph"
+        )
+
+    example_payload = {"message": "Hello, World!"}
+
+    def __init__(self):
+        super().__init__(
+            id="8fa8c167-2002-47ce-aba8-97572fc5d387",
+            description="This block will output the contents of the generic input for the webhook.",
+            categories={BlockCategory.INPUT},
+            input_schema=GenericWebhookTriggerBlock.Input,
+            output_schema=GenericWebhookTriggerBlock.Output,
+            webhook_config=BlockManualWebhookConfig(
+                provider=ProviderName.GENERIC_WEBHOOK,
+                webhook_type=GenericWebhookType.PLAIN,
+            ),
+            test_input={"constants": {"key": "value"}, "payload": self.example_payload},
+            test_output=[
+                ("constants", {"key": "value"}),
+                ("payload", self.example_payload),
+            ],
+        )
+
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+        yield "constants", input_data.constants
+        yield "payload", input_data.payload

autogpt_platform/backend/backend/blocks/github/_api.py

@@ -1,16 +1,30 @@
+from typing import overload
 from urllib.parse import urlparse
 
-from backend.blocks.github._auth import GithubCredentials
-from backend.util.request import Requests
+from backend.blocks.github._auth import (
+    GithubCredentials,
+    GithubFineGrainedAPICredentials,
+)
+from backend.util.request import URL, Requests
 
 
-def _convert_to_api_url(url: str) -> str:
+@overload
+def _convert_to_api_url(url: str) -> str: ...
+
+
+@overload
+def _convert_to_api_url(url: URL) -> URL: ...
+
+
+def _convert_to_api_url(url: str | URL) -> str | URL:
     """
     Converts a standard GitHub URL to the corresponding GitHub API URL.
     Handles repository URLs, issue URLs, pull request URLs, and more.
     """
-    parsed_url = urlparse(url)
-    path_parts = parsed_url.path.strip("/").split("/")
+    if url_as_str := isinstance(url, str):
+        url = urlparse(url)
+
+    path_parts = url.path.strip("/").split("/")
 
     if len(path_parts) >= 2:
         owner, repo = path_parts[0], path_parts[1]
@@ -25,17 +39,73 @@ def _convert_to_api_url(url: str) -> str:
     else:
         raise ValueError("Invalid GitHub URL format.")
 
-    return api_url
+    return api_url if url_as_str else urlparse(api_url)
 
 
 def _get_headers(credentials: GithubCredentials) -> dict[str, str]:
     return {
-        "Authorization": credentials.bearer(),
+        "Authorization": credentials.auth_header(),
         "Accept": "application/vnd.github.v3+json",
     }
 
 
-def get_api(credentials: GithubCredentials, convert_urls: bool = True) -> Requests:
+def convert_comment_url_to_api_endpoint(comment_url: str) -> str:
+    """
+    Converts a GitHub comment URL (web interface) to the appropriate API endpoint URL.
+
+    Handles:
+    1. Issue/PR comments: #issuecomment-{id}
+    2. PR review comments: #discussion_r{id}
+
+    Returns the appropriate API endpoint path for the comment.
+    """
+    # First, check if this is already an API URL
+    parsed_url = urlparse(comment_url)
+    if parsed_url.hostname == "api.github.com":
+        return comment_url
+
+    # Replace pull with issues for comment endpoints
+    if "/pull/" in comment_url:
+        comment_url = comment_url.replace("/pull/", "/issues/")
+
+    # Handle issue/PR comments (#issuecomment-xxx)
+    if "#issuecomment-" in comment_url:
+        base_url, comment_part = comment_url.split("#issuecomment-")
+        comment_id = comment_part
+
+        # Extract repo information from base URL
+        parsed_url = urlparse(base_url)
+        path_parts = parsed_url.path.strip("/").split("/")
+        owner, repo = path_parts[0], path_parts[1]
+
+        # Construct API URL for issue comments
+        return (
+            f"https://api.github.com/repos/{owner}/{repo}/issues/comments/{comment_id}"
+        )
+
+    # Handle PR review comments (#discussion_r)
+    elif "#discussion_r" in comment_url:
+        base_url, comment_part = comment_url.split("#discussion_r")
+        comment_id = comment_part
+
+        # Extract repo information from base URL
+        parsed_url = urlparse(base_url)
+        path_parts = parsed_url.path.strip("/").split("/")
+        owner, repo = path_parts[0], path_parts[1]
+
+        # Construct API URL for PR review comments
+        return (
+            f"https://api.github.com/repos/{owner}/{repo}/pulls/comments/{comment_id}"
+        )
+
+    # If no specific comment identifiers are found, use the general URL conversion
+    return _convert_to_api_url(comment_url)
+
+
+def get_api(
+    credentials: GithubCredentials | GithubFineGrainedAPICredentials,
+    convert_urls: bool = True,
+) -> Requests:
     return Requests(
         trusted_origins=["https://api.github.com", "https://github.com"],
         extra_url_validator=_convert_to_api_url if convert_urls else None,

autogpt_platform/backend/backend/blocks/github/_auth.py

@@ -8,6 +8,7 @@ from backend.data.model import (
     CredentialsMetaInput,
     OAuth2Credentials,
 )
+from backend.integrations.providers import ProviderName
 from backend.util.settings import Secrets
 
 secrets = Secrets()
@@ -17,10 +18,15 @@ GITHUB_OAUTH_IS_CONFIGURED = bool(
 
 GithubCredentials = APIKeyCredentials | OAuth2Credentials
 GithubCredentialsInput = CredentialsMetaInput[
-    Literal["github"],
+    Literal[ProviderName.GITHUB],
     Literal["api_key", "oauth2"] if GITHUB_OAUTH_IS_CONFIGURED else Literal["api_key"],
 ]
 
+GithubFineGrainedAPICredentials = APIKeyCredentials
+GithubFineGrainedAPICredentialsInput = CredentialsMetaInput[
+    Literal[ProviderName.GITHUB], Literal["api_key"]
+]
+
 
 def GithubCredentialsField(scope: str) -> GithubCredentialsInput:
     """
@@ -30,10 +36,16 @@ def GithubCredentialsField(scope: str) -> GithubCredentialsInput:
         scope: The authorization scope needed for the block to work. ([list of available scopes](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes))
     """  # noqa
     return CredentialsField(
-        provider="github",
-        supported_credential_types=(
-            {"api_key", "oauth2"} if GITHUB_OAUTH_IS_CONFIGURED else {"api_key"}
-        ),
+        required_scopes={scope},
+        description="The GitHub integration can be used with OAuth, "
+        "or any API key with sufficient permissions for the blocks it is used on.",
+    )
+
+
+def GithubFineGrainedAPICredentialsField(
+    scope: str,
+) -> GithubFineGrainedAPICredentialsInput:
+    return CredentialsField(
         required_scopes={scope},
         description="The GitHub integration can be used with OAuth, "
         "or any API key with sufficient permissions for the blocks it is used on.",
@@ -53,3 +65,18 @@ TEST_CREDENTIALS_INPUT = {
     "type": TEST_CREDENTIALS.type,
     "title": TEST_CREDENTIALS.type,
 }
+
+TEST_FINE_GRAINED_CREDENTIALS = APIKeyCredentials(
+    id="01234567-89ab-cdef-0123-456789abcdef",
+    provider="github",
+    api_key=SecretStr("mock-github-api-key"),
+    title="Mock GitHub API key",
+    expires_at=None,
+)
+
+TEST_FINE_GRAINED_CREDENTIALS_INPUT = {
+    "provider": TEST_FINE_GRAINED_CREDENTIALS.provider,
+    "id": TEST_FINE_GRAINED_CREDENTIALS.id,
+    "type": TEST_FINE_GRAINED_CREDENTIALS.type,
+    "title": TEST_FINE_GRAINED_CREDENTIALS.type,
+}

autogpt_platform/backend/backend/blocks/github/checks.py

@@ -0,0 +1,360 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+
+from ._api import get_api
+from ._auth import (
+    TEST_CREDENTIALS,
+    TEST_CREDENTIALS_INPUT,
+    GithubCredentials,
+    GithubCredentialsField,
+    GithubCredentialsInput,
+)
+
+
+# queued, in_progress, completed, waiting, requested, pending
+class ChecksStatus(Enum):
+    QUEUED = "queued"
+    IN_PROGRESS = "in_progress"
+    COMPLETED = "completed"
+    WAITING = "waiting"
+    REQUESTED = "requested"
+    PENDING = "pending"
+
+
+class ChecksConclusion(Enum):
+    SUCCESS = "success"
+    FAILURE = "failure"
+    NEUTRAL = "neutral"
+    CANCELLED = "cancelled"
+    TIMED_OUT = "timed_out"
+    ACTION_REQUIRED = "action_required"
+    SKIPPED = "skipped"
+
+
+class GithubCreateCheckRunBlock(Block):
+    """Block for creating a new check run on a GitHub repository."""
+
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo:status")
+        repo_url: str = SchemaField(
+            description="URL of the GitHub repository",
+            placeholder="https://github.com/owner/repo",
+        )
+        name: str = SchemaField(
+            description="The name of the check run (e.g., 'code-coverage')",
+        )
+        head_sha: str = SchemaField(
+            description="The SHA of the commit to check",
+        )
+        status: ChecksStatus = SchemaField(
+            description="Current status of the check run",
+            default=ChecksStatus.QUEUED,
+        )
+        conclusion: Optional[ChecksConclusion] = SchemaField(
+            description="The final conclusion of the check (required if status is completed)",
+            default=None,
+        )
+        details_url: str = SchemaField(
+            description="The URL for the full details of the check",
+            default="",
+        )
+        output_title: str = SchemaField(
+            description="Title of the check run output",
+            default="",
+        )
+        output_summary: str = SchemaField(
+            description="Summary of the check run output",
+            default="",
+        )
+        output_text: str = SchemaField(
+            description="Detailed text of the check run output",
+            default="",
+        )
+
+    class Output(BlockSchema):
+        class CheckRunResult(BaseModel):
+            id: int
+            html_url: str
+            status: str
+
+        check_run: CheckRunResult = SchemaField(
+            description="Details of the created check run"
+        )
+        error: str = SchemaField(
+            description="Error message if check run creation failed"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="2f45e89a-3b7d-4f22-b89e-6c4f5c7e1234",
+            description="Creates a new check run for a specific commit in a GitHub repository",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubCreateCheckRunBlock.Input,
+            output_schema=GithubCreateCheckRunBlock.Output,
+            test_input={
+                "repo_url": "https://github.com/owner/repo",
+                "name": "test-check",
+                "head_sha": "ce587453ced02b1526dfb4cb910479d431683101",
+                "status": ChecksStatus.COMPLETED.value,
+                "conclusion": ChecksConclusion.SUCCESS.value,
+                "output_title": "Test Results",
+                "output_summary": "All tests passed",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            # requires a github app not available to oauth in our current system
+            disabled=True,
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                (
+                    "check_run",
+                    {
+                        "id": 4,
+                        "html_url": "https://github.com/owner/repo/runs/4",
+                        "status": "completed",
+                    },
+                ),
+            ],
+            test_mock={
+                "create_check_run": lambda *args, **kwargs: {
+                    "id": 4,
+                    "html_url": "https://github.com/owner/repo/runs/4",
+                    "status": "completed",
+                }
+            },
+        )
+
+    @staticmethod
+    async def create_check_run(
+        credentials: GithubCredentials,
+        repo_url: str,
+        name: str,
+        head_sha: str,
+        status: ChecksStatus,
+        conclusion: Optional[ChecksConclusion] = None,
+        details_url: Optional[str] = None,
+        output_title: Optional[str] = None,
+        output_summary: Optional[str] = None,
+        output_text: Optional[str] = None,
+    ) -> dict:
+        api = get_api(credentials)
+
+        class CheckRunData(BaseModel):
+            name: str
+            head_sha: str
+            status: str
+            conclusion: Optional[str] = None
+            details_url: Optional[str] = None
+            output: Optional[dict[str, str]] = None
+
+        data = CheckRunData(
+            name=name,
+            head_sha=head_sha,
+            status=status.value,
+        )
+
+        if conclusion:
+            data.conclusion = conclusion.value
+
+        if details_url:
+            data.details_url = details_url
+
+        if output_title or output_summary or output_text:
+            output_data = {
+                "title": output_title or "",
+                "summary": output_summary or "",
+                "text": output_text or "",
+            }
+            data.output = output_data
+
+        check_runs_url = f"{repo_url}/check-runs"
+        response = await api.post(
+            check_runs_url, data=data.model_dump_json(exclude_none=True)
+        )
+        result = response.json()
+
+        return {
+            "id": result["id"],
+            "html_url": result["html_url"],
+            "status": result["status"],
+        }
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        try:
+            result = await self.create_check_run(
+                credentials=credentials,
+                repo_url=input_data.repo_url,
+                name=input_data.name,
+                head_sha=input_data.head_sha,
+                status=input_data.status,
+                conclusion=input_data.conclusion,
+                details_url=input_data.details_url,
+                output_title=input_data.output_title,
+                output_summary=input_data.output_summary,
+                output_text=input_data.output_text,
+            )
+            yield "check_run", result
+        except Exception as e:
+            yield "error", str(e)
+
+
+class GithubUpdateCheckRunBlock(Block):
+    """Block for updating an existing check run on a GitHub repository."""
+
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo:status")
+        repo_url: str = SchemaField(
+            description="URL of the GitHub repository",
+            placeholder="https://github.com/owner/repo",
+        )
+        check_run_id: int = SchemaField(
+            description="The ID of the check run to update",
+        )
+        status: ChecksStatus = SchemaField(
+            description="New status of the check run",
+        )
+        conclusion: ChecksConclusion = SchemaField(
+            description="The final conclusion of the check (required if status is completed)",
+        )
+        output_title: Optional[str] = SchemaField(
+            description="New title of the check run output",
+            default=None,
+        )
+        output_summary: Optional[str] = SchemaField(
+            description="New summary of the check run output",
+            default=None,
+        )
+        output_text: Optional[str] = SchemaField(
+            description="New detailed text of the check run output",
+            default=None,
+        )
+
+    class Output(BlockSchema):
+        class CheckRunResult(BaseModel):
+            id: int
+            html_url: str
+            status: str
+            conclusion: Optional[str]
+
+        check_run: CheckRunResult = SchemaField(
+            description="Details of the updated check run"
+        )
+        error: str = SchemaField(description="Error message if check run update failed")
+
+    def __init__(self):
+        super().__init__(
+            id="8a23c567-9d01-4e56-b789-0c12d3e45678",  # Generated UUID
+            description="Updates an existing check run in a GitHub repository",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubUpdateCheckRunBlock.Input,
+            output_schema=GithubUpdateCheckRunBlock.Output,
+            # requires a github app not available to oauth in our current system
+            disabled=True,
+            test_input={
+                "repo_url": "https://github.com/owner/repo",
+                "check_run_id": 4,
+                "status": ChecksStatus.COMPLETED.value,
+                "conclusion": ChecksConclusion.SUCCESS.value,
+                "output_title": "Updated Results",
+                "output_summary": "All tests passed after retry",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                (
+                    "check_run",
+                    {
+                        "id": 4,
+                        "html_url": "https://github.com/owner/repo/runs/4",
+                        "status": "completed",
+                        "conclusion": "success",
+                    },
+                ),
+            ],
+            test_mock={
+                "update_check_run": lambda *args, **kwargs: {
+                    "id": 4,
+                    "html_url": "https://github.com/owner/repo/runs/4",
+                    "status": "completed",
+                    "conclusion": "success",
+                }
+            },
+        )
+
+    @staticmethod
+    async def update_check_run(
+        credentials: GithubCredentials,
+        repo_url: str,
+        check_run_id: int,
+        status: ChecksStatus,
+        conclusion: Optional[ChecksConclusion] = None,
+        output_title: Optional[str] = None,
+        output_summary: Optional[str] = None,
+        output_text: Optional[str] = None,
+    ) -> dict:
+        api = get_api(credentials)
+
+        class UpdateCheckRunData(BaseModel):
+            status: str
+            conclusion: Optional[str] = None
+            output: Optional[dict[str, str]] = None
+
+        data = UpdateCheckRunData(
+            status=status.value,
+        )
+
+        if conclusion:
+            data.conclusion = conclusion.value
+
+        if output_title or output_summary or output_text:
+            output_data = {
+                "title": output_title or "",
+                "summary": output_summary or "",
+                "text": output_text or "",
+            }
+            data.output = output_data
+
+        check_run_url = f"{repo_url}/check-runs/{check_run_id}"
+        response = await api.patch(
+            check_run_url, data=data.model_dump_json(exclude_none=True)
+        )
+        result = response.json()
+
+        return {
+            "id": result["id"],
+            "html_url": result["html_url"],
+            "status": result["status"],
+            "conclusion": result.get("conclusion"),
+        }
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        try:
+            result = await self.update_check_run(
+                credentials=credentials,
+                repo_url=input_data.repo_url,
+                check_run_id=input_data.check_run_id,
+                status=input_data.status,
+                conclusion=input_data.conclusion,
+                output_title=input_data.output_title,
+                output_summary=input_data.output_summary,
+                output_text=input_data.output_text,
+            )
+            yield "check_run", result
+        except Exception as e:
+            yield "error", str(e)

autogpt_platform/backend/backend/blocks/github/issues.py

@@ -1,3 +1,4 @@
+import logging
 from urllib.parse import urlparse
 
 from typing_extensions import TypedDict
@@ -5,7 +6,7 @@ from typing_extensions import TypedDict
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 from backend.data.model import SchemaField
 
-from ._api import get_api
+from ._api import convert_comment_url_to_api_endpoint, get_api
 from ._auth import (
     TEST_CREDENTIALS,
     TEST_CREDENTIALS_INPUT,
@@ -14,6 +15,8 @@ from ._auth import (
     GithubCredentialsInput,
 )
 
+logger = logging.getLogger(__name__)
+
 
 def is_github_url(url: str) -> bool:
     return urlparse(url).netloc == "github.com"
@@ -77,7 +80,7 @@ class GithubCommentBlock(Block):
         )
 
     @staticmethod
-    def post_comment(
+    async def post_comment(
         credentials: GithubCredentials, issue_url: str, body_text: str
     ) -> tuple[int, str]:
         api = get_api(credentials)
@@ -85,18 +88,18 @@ class GithubCommentBlock(Block):
         if "pull" in issue_url:
             issue_url = issue_url.replace("pull", "issues")
         comments_url = issue_url + "/comments"
-        response = api.post(comments_url, json=data)
+        response = await api.post(comments_url, json=data)
         comment = response.json()
         return comment["id"], comment["html_url"]
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        id, url = self.post_comment(
+        id, url = await self.post_comment(
             credentials,
             input_data.issue_url,
             input_data.comment,
@@ -108,6 +111,229 @@ class GithubCommentBlock(Block):
 # --8<-- [end:GithubCommentBlockExample]
 
 
+class GithubUpdateCommentBlock(Block):
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo")
+        comment_url: str = SchemaField(
+            description="URL of the GitHub comment",
+            placeholder="https://github.com/owner/repo/issues/1#issuecomment-123456789",
+            default="",
+            advanced=False,
+        )
+        issue_url: str = SchemaField(
+            description="URL of the GitHub issue or pull request",
+            placeholder="https://github.com/owner/repo/issues/1",
+            default="",
+        )
+        comment_id: str = SchemaField(
+            description="ID of the GitHub comment",
+            placeholder="123456789",
+            default="",
+        )
+        comment: str = SchemaField(
+            description="Comment to update",
+            placeholder="Enter your comment",
+        )
+
+    class Output(BlockSchema):
+        id: int = SchemaField(description="ID of the updated comment")
+        url: str = SchemaField(description="URL to the comment on GitHub")
+        error: str = SchemaField(
+            description="Error message if the comment update failed"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="b3f4d747-10e3-4e69-8c51-f2be1d99c9a7",
+            description="This block updates a comment on a specified GitHub issue or pull request.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubUpdateCommentBlock.Input,
+            output_schema=GithubUpdateCommentBlock.Output,
+            test_input={
+                "comment_url": "https://github.com/owner/repo/issues/1#issuecomment-123456789",
+                "comment": "This is an updated comment.",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                ("id", 123456789),
+                (
+                    "url",
+                    "https://github.com/owner/repo/issues/1#issuecomment-123456789",
+                ),
+            ],
+            test_mock={
+                "update_comment": lambda *args, **kwargs: (
+                    123456789,
+                    "https://github.com/owner/repo/issues/1#issuecomment-123456789",
+                )
+            },
+        )
+
+    @staticmethod
+    async def update_comment(
+        credentials: GithubCredentials, comment_url: str, body_text: str
+    ) -> tuple[int, str]:
+        api = get_api(credentials, convert_urls=False)
+        data = {"body": body_text}
+        url = convert_comment_url_to_api_endpoint(comment_url)
+
+        logger.info(url)
+        response = await api.patch(url, json=data)
+        comment = response.json()
+        return comment["id"], comment["html_url"]
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        if (
+            not input_data.comment_url
+            and input_data.comment_id
+            and input_data.issue_url
+        ):
+            parsed_url = urlparse(input_data.issue_url)
+            path_parts = parsed_url.path.strip("/").split("/")
+            owner, repo = path_parts[0], path_parts[1]
+
+            input_data.comment_url = f"https://api.github.com/repos/{owner}/{repo}/issues/comments/{input_data.comment_id}"
+
+        elif (
+            not input_data.comment_url
+            and not input_data.comment_id
+            and input_data.issue_url
+        ):
+            raise ValueError(
+                "Must provide either comment_url or comment_id and issue_url"
+            )
+        id, url = await self.update_comment(
+            credentials,
+            input_data.comment_url,
+            input_data.comment,
+        )
+        yield "id", id
+        yield "url", url
+
+
+class GithubListCommentsBlock(Block):
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo")
+        issue_url: str = SchemaField(
+            description="URL of the GitHub issue or pull request",
+            placeholder="https://github.com/owner/repo/issues/1",
+        )
+
+    class Output(BlockSchema):
+        class CommentItem(TypedDict):
+            id: int
+            body: str
+            user: str
+            url: str
+
+        comment: CommentItem = SchemaField(
+            title="Comment", description="Comments with their ID, body, user, and URL"
+        )
+        comments: list[CommentItem] = SchemaField(
+            description="List of comments with their ID, body, user, and URL"
+        )
+        error: str = SchemaField(description="Error message if listing comments failed")
+
+    def __init__(self):
+        super().__init__(
+            id="c4b5fb63-0005-4a11-b35a-0c2467bd6b59",
+            description="This block lists all comments for a specified GitHub issue or pull request.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubListCommentsBlock.Input,
+            output_schema=GithubListCommentsBlock.Output,
+            test_input={
+                "issue_url": "https://github.com/owner/repo/issues/1",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                (
+                    "comment",
+                    {
+                        "id": 123456789,
+                        "body": "This is a test comment.",
+                        "user": "test_user",
+                        "url": "https://github.com/owner/repo/issues/1#issuecomment-123456789",
+                    },
+                ),
+                (
+                    "comments",
+                    [
+                        {
+                            "id": 123456789,
+                            "body": "This is a test comment.",
+                            "user": "test_user",
+                            "url": "https://github.com/owner/repo/issues/1#issuecomment-123456789",
+                        }
+                    ],
+                ),
+            ],
+            test_mock={
+                "list_comments": lambda *args, **kwargs: [
+                    {
+                        "id": 123456789,
+                        "body": "This is a test comment.",
+                        "user": "test_user",
+                        "url": "https://github.com/owner/repo/issues/1#issuecomment-123456789",
+                    }
+                ]
+            },
+        )
+
+    @staticmethod
+    async def list_comments(
+        credentials: GithubCredentials, issue_url: str
+    ) -> list[Output.CommentItem]:
+        parsed_url = urlparse(issue_url)
+        path_parts = parsed_url.path.strip("/").split("/")
+
+        owner = path_parts[0]
+        repo = path_parts[1]
+
+        # GitHub API uses 'issues' for both issues and pull requests when it comes to comments
+        issue_number = path_parts[3]  # Whether 'issues/123' or 'pull/123'
+
+        # Construct the proper API URL directly
+        api_url = f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_number}/comments"
+
+        # Set convert_urls=False since we're already providing an API URL
+        api = get_api(credentials, convert_urls=False)
+        response = await api.get(api_url)
+        comments = response.json()
+        parsed_comments: list[GithubListCommentsBlock.Output.CommentItem] = [
+            {
+                "id": comment["id"],
+                "body": comment["body"],
+                "user": comment["user"]["login"],
+                "url": comment["html_url"],
+            }
+            for comment in comments
+        ]
+        return parsed_comments
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        comments = await self.list_comments(
+            credentials,
+            input_data.issue_url,
+        )
+        for comment in comments:
+            yield "comment", comment
+        yield "comments", comments
+
+
 class GithubMakeIssueBlock(Block):
     class Input(BlockSchema):
         credentials: GithubCredentialsInput = GithubCredentialsField("repo")
@@ -156,24 +382,24 @@ class GithubMakeIssueBlock(Block):
         )
 
     @staticmethod
-    def create_issue(
+    async def create_issue(
         credentials: GithubCredentials, repo_url: str, title: str, body: str
     ) -> tuple[int, str]:
         api = get_api(credentials)
         data = {"title": title, "body": body}
         issues_url = repo_url + "/issues"
-        response = api.post(issues_url, json=data)
+        response = await api.post(issues_url, json=data)
         issue = response.json()
         return issue["number"], issue["html_url"]
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        number, url = self.create_issue(
+        number, url = await self.create_issue(
             credentials,
             input_data.repo_url,
             input_data.title,
@@ -226,25 +452,25 @@ class GithubReadIssueBlock(Block):
         )
 
     @staticmethod
-    def read_issue(
+    async def read_issue(
         credentials: GithubCredentials, issue_url: str
     ) -> tuple[str, str, str]:
         api = get_api(credentials)
-        response = api.get(issue_url)
+        response = await api.get(issue_url)
         data = response.json()
         title = data.get("title", "No title found")
         body = data.get("body", "No body content found")
         user = data.get("user", {}).get("login", "No user found")
         return title, body, user
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        title, body, user = self.read_issue(
+        title, body, user = await self.read_issue(
             credentials,
             input_data.issue_url,
         )
@@ -306,30 +532,30 @@ class GithubListIssuesBlock(Block):
         )
 
     @staticmethod
-    def list_issues(
+    async def list_issues(
         credentials: GithubCredentials, repo_url: str
     ) -> list[Output.IssueItem]:
         api = get_api(credentials)
         issues_url = repo_url + "/issues"
-        response = api.get(issues_url)
+        response = await api.get(issues_url)
         data = response.json()
         issues: list[GithubListIssuesBlock.Output.IssueItem] = [
             {"title": issue["title"], "url": issue["html_url"]} for issue in data
         ]
         return issues
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        issues = self.list_issues(
+        for issue in await self.list_issues(
             credentials,
             input_data.repo_url,
-        )
-        yield from (("issue", issue) for issue in issues)
+        ):
+            yield "issue", issue
 
 
 class GithubAddLabelBlock(Block):
@@ -368,21 +594,23 @@ class GithubAddLabelBlock(Block):
         )
 
     @staticmethod
-    def add_label(credentials: GithubCredentials, issue_url: str, label: str) -> str:
+    async def add_label(
+        credentials: GithubCredentials, issue_url: str, label: str
+    ) -> str:
         api = get_api(credentials)
         data = {"labels": [label]}
         labels_url = issue_url + "/labels"
-        api.post(labels_url, json=data)
+        await api.post(labels_url, json=data)
         return "Label added successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        status = self.add_label(
+        status = await self.add_label(
             credentials,
             input_data.issue_url,
             input_data.label,
@@ -428,20 +656,22 @@ class GithubRemoveLabelBlock(Block):
         )
 
     @staticmethod
-    def remove_label(credentials: GithubCredentials, issue_url: str, label: str) -> str:
+    async def remove_label(
+        credentials: GithubCredentials, issue_url: str, label: str
+    ) -> str:
         api = get_api(credentials)
         label_url = issue_url + f"/labels/{label}"
-        api.delete(label_url)
+        await api.delete(label_url)
         return "Label removed successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        status = self.remove_label(
+        status = await self.remove_label(
             credentials,
             input_data.issue_url,
             input_data.label,
@@ -489,7 +719,7 @@ class GithubAssignIssueBlock(Block):
         )
 
     @staticmethod
-    def assign_issue(
+    async def assign_issue(
         credentials: GithubCredentials,
         issue_url: str,
         assignee: str,
@@ -497,17 +727,17 @@ class GithubAssignIssueBlock(Block):
         api = get_api(credentials)
         assignees_url = issue_url + "/assignees"
         data = {"assignees": [assignee]}
-        api.post(assignees_url, json=data)
+        await api.post(assignees_url, json=data)
         return "Issue assigned successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        status = self.assign_issue(
+        status = await self.assign_issue(
             credentials,
             input_data.issue_url,
             input_data.assignee,
@@ -555,7 +785,7 @@ class GithubUnassignIssueBlock(Block):
         )
 
     @staticmethod
-    def unassign_issue(
+    async def unassign_issue(
         credentials: GithubCredentials,
         issue_url: str,
         assignee: str,
@@ -563,17 +793,17 @@ class GithubUnassignIssueBlock(Block):
         api = get_api(credentials)
         assignees_url = issue_url + "/assignees"
         data = {"assignees": [assignee]}
-        api.delete(assignees_url, json=data)
+        await api.delete(assignees_url, json=data)
         return "Issue unassigned successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        status = self.unassign_issue(
+        status = await self.unassign_issue(
             credentials,
             input_data.issue_url,
             input_data.assignee,

autogpt_platform/backend/backend/blocks/github/pull_requests.py

@@ -1,3 +1,5 @@
+import re
+
 from typing_extensions import TypedDict
 
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
@@ -63,28 +65,31 @@ class GithubListPullRequestsBlock(Block):
         )
 
     @staticmethod
-    def list_prs(credentials: GithubCredentials, repo_url: str) -> list[Output.PRItem]:
+    async def list_prs(
+        credentials: GithubCredentials, repo_url: str
+    ) -> list[Output.PRItem]:
         api = get_api(credentials)
         pulls_url = repo_url + "/pulls"
-        response = api.get(pulls_url)
+        response = await api.get(pulls_url)
         data = response.json()
         pull_requests: list[GithubListPullRequestsBlock.Output.PRItem] = [
             {"title": pr["title"], "url": pr["html_url"]} for pr in data
         ]
         return pull_requests
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        pull_requests = self.list_prs(
+        pull_requests = await self.list_prs(
             credentials,
             input_data.repo_url,
         )
-        yield from (("pull_request", pr) for pr in pull_requests)
+        for pr in pull_requests:
+            yield "pull_request", pr
 
 
 class GithubMakePullRequestBlock(Block):
@@ -151,7 +156,7 @@ class GithubMakePullRequestBlock(Block):
         )
 
     @staticmethod
-    def create_pr(
+    async def create_pr(
         credentials: GithubCredentials,
         repo_url: str,
         title: str,
@@ -162,11 +167,11 @@ class GithubMakePullRequestBlock(Block):
         api = get_api(credentials)
         pulls_url = repo_url + "/pulls"
         data = {"title": title, "body": body, "head": head, "base": base}
-        response = api.post(pulls_url, json=data)
+        response = await api.post(pulls_url, json=data)
         pr_data = response.json()
         return pr_data["number"], pr_data["html_url"]
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
@@ -174,7 +179,7 @@ class GithubMakePullRequestBlock(Block):
         **kwargs,
     ) -> BlockOutput:
         try:
-            number, url = self.create_pr(
+            number, url = await self.create_pr(
                 credentials,
                 input_data.repo_url,
                 input_data.title,
@@ -198,6 +203,7 @@ class GithubReadPullRequestBlock(Block):
         include_pr_changes: bool = SchemaField(
             description="Whether to include the changes made in the pull request",
             default=False,
+            advanced=False,
         )
 
     class Output(BlockSchema):
@@ -239,39 +245,39 @@ class GithubReadPullRequestBlock(Block):
         )
 
     @staticmethod
-    def read_pr(credentials: GithubCredentials, pr_url: str) -> tuple[str, str, str]:
+    async def read_pr(
+        credentials: GithubCredentials, pr_url: str
+    ) -> tuple[str, str, str]:
         api = get_api(credentials)
-        # Adjust the URL to access the issue endpoint for PR metadata
         issue_url = pr_url.replace("/pull/", "/issues/")
-        response = api.get(issue_url)
+        response = await api.get(issue_url)
         data = response.json()
         title = data.get("title", "No title found")
         body = data.get("body", "No body content found")
-        author = data.get("user", {}).get("login", "No user found")
+        author = data.get("user", {}).get("login", "Unknown author")
         return title, body, author
 
     @staticmethod
-    def read_pr_changes(credentials: GithubCredentials, pr_url: str) -> str:
+    async def read_pr_changes(credentials: GithubCredentials, pr_url: str) -> str:
         api = get_api(credentials)
-        files_url = pr_url + "/files"
-        response = api.get(files_url)
+        files_url = prepare_pr_api_url(pr_url=pr_url, path="files")
+        response = await api.get(files_url)
         files = response.json()
         changes = []
         for file in files:
-            filename = file.get("filename")
-            patch = file.get("patch")
-            if filename and patch:
-                changes.append(f"File: {filename}\n{patch}")
-        return "\n\n".join(changes)
+            filename = file.get("filename", "")
+            status = file.get("status", "")
+            changes.append(f"{filename}: {status}")
+        return "\n".join(changes)
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        title, body, author = self.read_pr(
+        title, body, author = await self.read_pr(
             credentials,
             input_data.pr_url,
         )
@@ -280,7 +286,7 @@ class GithubReadPullRequestBlock(Block):
         yield "author", author
 
         if input_data.include_pr_changes:
-            changes = self.read_pr_changes(
+            changes = await self.read_pr_changes(
                 credentials,
                 input_data.pr_url,
             )
@@ -327,16 +333,16 @@ class GithubAssignPRReviewerBlock(Block):
         )
 
     @staticmethod
-    def assign_reviewer(
+    async def assign_reviewer(
         credentials: GithubCredentials, pr_url: str, reviewer: str
     ) -> str:
         api = get_api(credentials)
-        reviewers_url = pr_url + "/requested_reviewers"
+        reviewers_url = prepare_pr_api_url(pr_url=pr_url, path="requested_reviewers")
         data = {"reviewers": [reviewer]}
-        api.post(reviewers_url, json=data)
+        await api.post(reviewers_url, json=data)
         return "Reviewer assigned successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
@@ -344,7 +350,7 @@ class GithubAssignPRReviewerBlock(Block):
         **kwargs,
     ) -> BlockOutput:
         try:
-            status = self.assign_reviewer(
+            status = await self.assign_reviewer(
                 credentials,
                 input_data.pr_url,
                 input_data.reviewer,
@@ -394,16 +400,16 @@ class GithubUnassignPRReviewerBlock(Block):
         )
 
     @staticmethod
-    def unassign_reviewer(
+    async def unassign_reviewer(
         credentials: GithubCredentials, pr_url: str, reviewer: str
     ) -> str:
         api = get_api(credentials)
-        reviewers_url = pr_url + "/requested_reviewers"
+        reviewers_url = prepare_pr_api_url(pr_url=pr_url, path="requested_reviewers")
         data = {"reviewers": [reviewer]}
-        api.delete(reviewers_url, json=data)
+        await api.delete(reviewers_url, json=data)
         return "Reviewer unassigned successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
@@ -411,7 +417,7 @@ class GithubUnassignPRReviewerBlock(Block):
         **kwargs,
     ) -> BlockOutput:
         try:
-            status = self.unassign_reviewer(
+            status = await self.unassign_reviewer(
                 credentials,
                 input_data.pr_url,
                 input_data.reviewer,
@@ -474,12 +480,12 @@ class GithubListPRReviewersBlock(Block):
         )
 
     @staticmethod
-    def list_reviewers(
+    async def list_reviewers(
         credentials: GithubCredentials, pr_url: str
     ) -> list[Output.ReviewerItem]:
         api = get_api(credentials)
-        reviewers_url = pr_url + "/requested_reviewers"
-        response = api.get(reviewers_url)
+        reviewers_url = prepare_pr_api_url(pr_url=pr_url, path="requested_reviewers")
+        response = await api.get(reviewers_url)
         data = response.json()
         reviewers: list[GithubListPRReviewersBlock.Output.ReviewerItem] = [
             {"username": reviewer["login"], "url": reviewer["html_url"]}
@@ -487,15 +493,26 @@ class GithubListPRReviewersBlock(Block):
         ]
         return reviewers
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        reviewers = self.list_reviewers(
+        for reviewer in await self.list_reviewers(
             credentials,
             input_data.pr_url,
-        )
-        yield from (("reviewer", reviewer) for reviewer in reviewers)
+        ):
+            yield "reviewer", reviewer
+
+
+def prepare_pr_api_url(pr_url: str, path: str) -> str:
+    # Pattern to capture the base repository URL and the pull request number
+    pattern = r"^(?:https?://)?([^/]+/[^/]+/[^/]+)/pull/(\d+)"
+    match = re.match(pattern, pr_url)
+    if not match:
+        return pr_url
+
+    base_url, pr_number = match.groups()
+    return f"{base_url}/pulls/{pr_number}/{path}"

autogpt_platform/backend/backend/blocks/github/repo.py

@@ -65,12 +65,12 @@ class GithubListTagsBlock(Block):
         )
 
     @staticmethod
-    def list_tags(
+    async def list_tags(
         credentials: GithubCredentials, repo_url: str
     ) -> list[Output.TagItem]:
         api = get_api(credentials)
         tags_url = repo_url + "/tags"
-        response = api.get(tags_url)
+        response = await api.get(tags_url)
         data = response.json()
         repo_path = repo_url.replace("https://github.com/", "")
         tags: list[GithubListTagsBlock.Output.TagItem] = [
@@ -82,18 +82,19 @@ class GithubListTagsBlock(Block):
         ]
         return tags
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        tags = self.list_tags(
+        tags = await self.list_tags(
             credentials,
             input_data.repo_url,
         )
-        yield from (("tag", tag) for tag in tags)
+        for tag in tags:
+            yield "tag", tag
 
 
 class GithubListBranchesBlock(Block):
@@ -147,12 +148,12 @@ class GithubListBranchesBlock(Block):
         )
 
     @staticmethod
-    def list_branches(
+    async def list_branches(
         credentials: GithubCredentials, repo_url: str
     ) -> list[Output.BranchItem]:
         api = get_api(credentials)
         branches_url = repo_url + "/branches"
-        response = api.get(branches_url)
+        response = await api.get(branches_url)
         data = response.json()
         repo_path = repo_url.replace("https://github.com/", "")
         branches: list[GithubListBranchesBlock.Output.BranchItem] = [
@@ -164,18 +165,19 @@ class GithubListBranchesBlock(Block):
         ]
         return branches
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        branches = self.list_branches(
+        branches = await self.list_branches(
             credentials,
             input_data.repo_url,
         )
-        yield from (("branch", branch) for branch in branches)
+        for branch in branches:
+            yield "branch", branch
 
 
 class GithubListDiscussionsBlock(Block):
@@ -234,7 +236,7 @@ class GithubListDiscussionsBlock(Block):
         )
 
     @staticmethod
-    def list_discussions(
+    async def list_discussions(
         credentials: GithubCredentials, repo_url: str, num_discussions: int
     ) -> list[Output.DiscussionItem]:
         api = get_api(credentials)
@@ -254,7 +256,7 @@ class GithubListDiscussionsBlock(Block):
         }
         """
         variables = {"owner": owner, "repo": repo, "num": num_discussions}
-        response = api.post(
+        response = await api.post(
             "https://api.github.com/graphql",
             json={"query": query, "variables": variables},
         )
@@ -265,17 +267,20 @@ class GithubListDiscussionsBlock(Block):
         ]
         return discussions
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        discussions = self.list_discussions(
-            credentials, input_data.repo_url, input_data.num_discussions
+        discussions = await self.list_discussions(
+            credentials,
+            input_data.repo_url,
+            input_data.num_discussions,
         )
-        yield from (("discussion", discussion) for discussion in discussions)
+        for discussion in discussions:
+            yield "discussion", discussion
 
 
 class GithubListReleasesBlock(Block):
@@ -329,30 +334,31 @@ class GithubListReleasesBlock(Block):
         )
 
     @staticmethod
-    def list_releases(
+    async def list_releases(
         credentials: GithubCredentials, repo_url: str
     ) -> list[Output.ReleaseItem]:
         api = get_api(credentials)
         releases_url = repo_url + "/releases"
-        response = api.get(releases_url)
+        response = await api.get(releases_url)
         data = response.json()
         releases: list[GithubListReleasesBlock.Output.ReleaseItem] = [
             {"name": release["name"], "url": release["html_url"]} for release in data
         ]
         return releases
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        releases = self.list_releases(
+        releases = await self.list_releases(
             credentials,
             input_data.repo_url,
         )
-        yield from (("release", release) for release in releases)
+        for release in releases:
+            yield "release", release
 
 
 class GithubReadFileBlock(Block):
@@ -405,40 +411,40 @@ class GithubReadFileBlock(Block):
         )
 
     @staticmethod
-    def read_file(
+    async def read_file(
         credentials: GithubCredentials, repo_url: str, file_path: str, branch: str
     ) -> tuple[str, int]:
         api = get_api(credentials)
         content_url = repo_url + f"/contents/{file_path}?ref={branch}"
-        response = api.get(content_url)
-        content = response.json()
+        response = await api.get(content_url)
+        data = response.json()
 
-        if isinstance(content, list):
+        if isinstance(data, list):
             # Multiple entries of different types exist at this path
-            if not (file := next((f for f in content if f["type"] == "file"), None)):
+            if not (file := next((f for f in data if f["type"] == "file"), None)):
                 raise TypeError("Not a file")
-            content = file
+            data = file
 
-        if content["type"] != "file":
+        if data["type"] != "file":
             raise TypeError("Not a file")
 
-        return content["content"], content["size"]
+        return data["content"], data["size"]
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        raw_content, size = self.read_file(
+        content, size = await self.read_file(
             credentials,
             input_data.repo_url,
-            input_data.file_path.lstrip("/"),
+            input_data.file_path,
             input_data.branch,
         )
-        yield "raw_content", raw_content
-        yield "text_content", base64.b64decode(raw_content).decode("utf-8")
+        yield "raw_content", content
+        yield "text_content", base64.b64decode(content).decode("utf-8")
         yield "size", size
 
 
@@ -515,52 +521,55 @@ class GithubReadFolderBlock(Block):
         )
 
     @staticmethod
-    def read_folder(
+    async def read_folder(
         credentials: GithubCredentials, repo_url: str, folder_path: str, branch: str
     ) -> tuple[list[Output.FileEntry], list[Output.DirEntry]]:
         api = get_api(credentials)
         contents_url = repo_url + f"/contents/{folder_path}?ref={branch}"
-        response = api.get(contents_url)
-        content = response.json()
+        response = await api.get(contents_url)
+        data = response.json()
 
-        if not isinstance(content, list):
+        if not isinstance(data, list):
             raise TypeError("Not a folder")
 
-        files = [
+        files: list[GithubReadFolderBlock.Output.FileEntry] = [
             GithubReadFolderBlock.Output.FileEntry(
                 name=entry["name"],
                 path=entry["path"],
                 size=entry["size"],
             )
-            for entry in content
+            for entry in data
             if entry["type"] == "file"
         ]
-        dirs = [
+
+        dirs: list[GithubReadFolderBlock.Output.DirEntry] = [
             GithubReadFolderBlock.Output.DirEntry(
                 name=entry["name"],
                 path=entry["path"],
             )
-            for entry in content
+            for entry in data
             if entry["type"] == "dir"
         ]
 
         return files, dirs
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        files, dirs = self.read_folder(
+        files, dirs = await self.read_folder(
             credentials,
             input_data.repo_url,
             input_data.folder_path.lstrip("/"),
             input_data.branch,
         )
-        yield from (("file", file) for file in files)
-        yield from (("dir", dir) for dir in dirs)
+        for file in files:
+            yield "file", file
+        for dir in dirs:
+            yield "dir", dir
 
 
 class GithubMakeBranchBlock(Block):
@@ -606,32 +615,35 @@ class GithubMakeBranchBlock(Block):
         )
 
     @staticmethod
-    def create_branch(
+    async def create_branch(
         credentials: GithubCredentials,
         repo_url: str,
         new_branch: str,
         source_branch: str,
     ) -> str:
         api = get_api(credentials)
-        # Get the SHA of the source branch
         ref_url = repo_url + f"/git/refs/heads/{source_branch}"
-        response = api.get(ref_url)
-        sha = response.json()["object"]["sha"]
+        response = await api.get(ref_url)
+        data = response.json()
+        sha = data["object"]["sha"]
 
         # Create the new branch
-        create_ref_url = repo_url + "/git/refs"
-        data = {"ref": f"refs/heads/{new_branch}", "sha": sha}
-        response = api.post(create_ref_url, json=data)
+        new_ref_url = repo_url + "/git/refs"
+        data = {
+            "ref": f"refs/heads/{new_branch}",
+            "sha": sha,
+        }
+        response = await api.post(new_ref_url, json=data)
         return "Branch created successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        status = self.create_branch(
+        status = await self.create_branch(
             credentials,
             input_data.repo_url,
             input_data.new_branch,
@@ -678,24 +690,419 @@ class GithubDeleteBranchBlock(Block):
         )
 
     @staticmethod
-    def delete_branch(
+    async def delete_branch(
         credentials: GithubCredentials, repo_url: str, branch: str
     ) -> str:
         api = get_api(credentials)
         ref_url = repo_url + f"/git/refs/heads/{branch}"
-        api.delete(ref_url)
+        await api.delete(ref_url)
         return "Branch deleted successfully"
 
-    def run(
+    async def run(
         self,
         input_data: Input,
         *,
         credentials: GithubCredentials,
         **kwargs,
     ) -> BlockOutput:
-        status = self.delete_branch(
+        status = await self.delete_branch(
             credentials,
             input_data.repo_url,
             input_data.branch,
         )
         yield "status", status
+
+
+class GithubCreateFileBlock(Block):
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo")
+        repo_url: str = SchemaField(
+            description="URL of the GitHub repository",
+            placeholder="https://github.com/owner/repo",
+        )
+        file_path: str = SchemaField(
+            description="Path where the file should be created",
+            placeholder="path/to/file.txt",
+        )
+        content: str = SchemaField(
+            description="Content to write to the file",
+            placeholder="File content here",
+        )
+        branch: str = SchemaField(
+            description="Branch where the file should be created",
+            default="main",
+        )
+        commit_message: str = SchemaField(
+            description="Message for the commit",
+            default="Create new file",
+        )
+
+    class Output(BlockSchema):
+        url: str = SchemaField(description="URL of the created file")
+        sha: str = SchemaField(description="SHA of the commit")
+        error: str = SchemaField(
+            description="Error message if the file creation failed"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="8fd132ac-b917-428a-8159-d62893e8a3fe",
+            description="This block creates a new file in a GitHub repository.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubCreateFileBlock.Input,
+            output_schema=GithubCreateFileBlock.Output,
+            test_input={
+                "repo_url": "https://github.com/owner/repo",
+                "file_path": "test/file.txt",
+                "content": "Test content",
+                "branch": "main",
+                "commit_message": "Create test file",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                ("url", "https://github.com/owner/repo/blob/main/test/file.txt"),
+                ("sha", "abc123"),
+            ],
+            test_mock={
+                "create_file": lambda *args, **kwargs: (
+                    "https://github.com/owner/repo/blob/main/test/file.txt",
+                    "abc123",
+                )
+            },
+        )
+
+    @staticmethod
+    async def create_file(
+        credentials: GithubCredentials,
+        repo_url: str,
+        file_path: str,
+        content: str,
+        branch: str,
+        commit_message: str,
+    ) -> tuple[str, str]:
+        api = get_api(credentials)
+        contents_url = repo_url + f"/contents/{file_path}"
+        content_base64 = base64.b64encode(content.encode()).decode()
+        data = {
+            "message": commit_message,
+            "content": content_base64,
+            "branch": branch,
+        }
+        response = await api.put(contents_url, json=data)
+        data = response.json()
+        return data["content"]["html_url"], data["commit"]["sha"]
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        try:
+            url, sha = await self.create_file(
+                credentials,
+                input_data.repo_url,
+                input_data.file_path,
+                input_data.content,
+                input_data.branch,
+                input_data.commit_message,
+            )
+            yield "url", url
+            yield "sha", sha
+        except Exception as e:
+            yield "error", str(e)
+
+
+class GithubUpdateFileBlock(Block):
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo")
+        repo_url: str = SchemaField(
+            description="URL of the GitHub repository",
+            placeholder="https://github.com/owner/repo",
+        )
+        file_path: str = SchemaField(
+            description="Path to the file to update",
+            placeholder="path/to/file.txt",
+        )
+        content: str = SchemaField(
+            description="New content for the file",
+            placeholder="Updated content here",
+        )
+        branch: str = SchemaField(
+            description="Branch containing the file",
+            default="main",
+        )
+        commit_message: str = SchemaField(
+            description="Message for the commit",
+            default="Update file",
+        )
+
+    class Output(BlockSchema):
+        url: str = SchemaField(description="URL of the updated file")
+        sha: str = SchemaField(description="SHA of the commit")
+        error: str = SchemaField(description="Error message if the file update failed")
+
+    def __init__(self):
+        super().__init__(
+            id="30be12a4-57cb-4aa4-baf5-fcc68d136076",
+            description="This block updates an existing file in a GitHub repository.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubUpdateFileBlock.Input,
+            output_schema=GithubUpdateFileBlock.Output,
+            test_input={
+                "repo_url": "https://github.com/owner/repo",
+                "file_path": "test/file.txt",
+                "content": "Updated content",
+                "branch": "main",
+                "commit_message": "Update test file",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                ("url", "https://github.com/owner/repo/blob/main/test/file.txt"),
+                ("sha", "def456"),
+            ],
+            test_mock={
+                "update_file": lambda *args, **kwargs: (
+                    "https://github.com/owner/repo/blob/main/test/file.txt",
+                    "def456",
+                )
+            },
+        )
+
+    @staticmethod
+    async def update_file(
+        credentials: GithubCredentials,
+        repo_url: str,
+        file_path: str,
+        content: str,
+        branch: str,
+        commit_message: str,
+    ) -> tuple[str, str]:
+        api = get_api(credentials)
+        contents_url = repo_url + f"/contents/{file_path}"
+        params = {"ref": branch}
+        response = await api.get(contents_url, params=params)
+        data = response.json()
+
+        # Convert new content to base64
+        content_base64 = base64.b64encode(content.encode()).decode()
+        data = {
+            "message": commit_message,
+            "content": content_base64,
+            "sha": data["sha"],
+            "branch": branch,
+        }
+        response = await api.put(contents_url, json=data)
+        data = response.json()
+        return data["content"]["html_url"], data["commit"]["sha"]
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        try:
+            url, sha = await self.update_file(
+                credentials,
+                input_data.repo_url,
+                input_data.file_path,
+                input_data.content,
+                input_data.branch,
+                input_data.commit_message,
+            )
+            yield "url", url
+            yield "sha", sha
+        except Exception as e:
+            yield "error", str(e)
+
+
+class GithubCreateRepositoryBlock(Block):
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo")
+        name: str = SchemaField(
+            description="Name of the repository to create",
+            placeholder="my-new-repo",
+        )
+        description: str = SchemaField(
+            description="Description of the repository",
+            placeholder="A description of the repository",
+            default="",
+        )
+        private: bool = SchemaField(
+            description="Whether the repository should be private",
+            default=False,
+        )
+        auto_init: bool = SchemaField(
+            description="Whether to initialize the repository with a README",
+            default=True,
+        )
+        gitignore_template: str = SchemaField(
+            description="Git ignore template to use (e.g., Python, Node, Java)",
+            default="",
+        )
+
+    class Output(BlockSchema):
+        url: str = SchemaField(description="URL of the created repository")
+        clone_url: str = SchemaField(description="Git clone URL of the repository")
+        error: str = SchemaField(
+            description="Error message if the repository creation failed"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="029ec3b8-1cfd-46d3-b6aa-28e4a706efd1",
+            description="This block creates a new GitHub repository.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubCreateRepositoryBlock.Input,
+            output_schema=GithubCreateRepositoryBlock.Output,
+            test_input={
+                "name": "test-repo",
+                "description": "A test repository",
+                "private": False,
+                "auto_init": True,
+                "gitignore_template": "Python",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                ("url", "https://github.com/owner/test-repo"),
+                ("clone_url", "https://github.com/owner/test-repo.git"),
+            ],
+            test_mock={
+                "create_repository": lambda *args, **kwargs: (
+                    "https://github.com/owner/test-repo",
+                    "https://github.com/owner/test-repo.git",
+                )
+            },
+        )
+
+    @staticmethod
+    async def create_repository(
+        credentials: GithubCredentials,
+        name: str,
+        description: str,
+        private: bool,
+        auto_init: bool,
+        gitignore_template: str,
+    ) -> tuple[str, str]:
+        api = get_api(credentials)
+        data = {
+            "name": name,
+            "description": description,
+            "private": private,
+            "auto_init": auto_init,
+            "gitignore_template": gitignore_template,
+        }
+        response = await api.post("https://api.github.com/user/repos", json=data)
+        data = response.json()
+        return data["html_url"], data["clone_url"]
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        try:
+            url, clone_url = await self.create_repository(
+                credentials,
+                input_data.name,
+                input_data.description,
+                input_data.private,
+                input_data.auto_init,
+                input_data.gitignore_template,
+            )
+            yield "url", url
+            yield "clone_url", clone_url
+        except Exception as e:
+            yield "error", str(e)
+
+
+class GithubListStargazersBlock(Block):
+    class Input(BlockSchema):
+        credentials: GithubCredentialsInput = GithubCredentialsField("repo")
+        repo_url: str = SchemaField(
+            description="URL of the GitHub repository",
+            placeholder="https://github.com/owner/repo",
+        )
+
+    class Output(BlockSchema):
+        class StargazerItem(TypedDict):
+            username: str
+            url: str
+
+        stargazer: StargazerItem = SchemaField(
+            title="Stargazer",
+            description="Stargazers with their username and profile URL",
+        )
+        error: str = SchemaField(
+            description="Error message if listing stargazers failed"
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="a4b9c2d1-e5f6-4g7h-8i9j-0k1l2m3n4o5p",  # Generated unique UUID
+            description="This block lists all users who have starred a specified GitHub repository.",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubListStargazersBlock.Input,
+            output_schema=GithubListStargazersBlock.Output,
+            test_input={
+                "repo_url": "https://github.com/owner/repo",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                (
+                    "stargazer",
+                    {
+                        "username": "octocat",
+                        "url": "https://github.com/octocat",
+                    },
+                )
+            ],
+            test_mock={
+                "list_stargazers": lambda *args, **kwargs: [
+                    {
+                        "username": "octocat",
+                        "url": "https://github.com/octocat",
+                    }
+                ]
+            },
+        )
+
+    @staticmethod
+    async def list_stargazers(
+        credentials: GithubCredentials, repo_url: str
+    ) -> list[Output.StargazerItem]:
+        api = get_api(credentials)
+        stargazers_url = repo_url + "/stargazers"
+        response = await api.get(stargazers_url)
+        data = response.json()
+        stargazers: list[GithubListStargazersBlock.Output.StargazerItem] = [
+            {
+                "username": stargazer["login"],
+                "url": stargazer["html_url"],
+            }
+            for stargazer in data
+        ]
+        return stargazers
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubCredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        stargazers = await self.list_stargazers(
+            credentials,
+            input_data.repo_url,
+        )
+        for stargazer in stargazers:
+            yield "stargazer", stargazer

autogpt_platform/backend/backend/blocks/github/statuses.py

@@ -0,0 +1,182 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+
+from ._api import get_api
+from ._auth import (
+    TEST_CREDENTIALS,
+    TEST_CREDENTIALS_INPUT,
+    GithubFineGrainedAPICredentials,
+    GithubFineGrainedAPICredentialsField,
+    GithubFineGrainedAPICredentialsInput,
+)
+
+
+class StatusState(Enum):
+    ERROR = "error"
+    FAILURE = "failure"
+    PENDING = "pending"
+    SUCCESS = "success"
+
+
+class GithubCreateStatusBlock(Block):
+    """Block for creating a commit status on a GitHub repository."""
+
+    class Input(BlockSchema):
+        credentials: GithubFineGrainedAPICredentialsInput = (
+            GithubFineGrainedAPICredentialsField("repo:status")
+        )
+        repo_url: str = SchemaField(
+            description="URL of the GitHub repository",
+            placeholder="https://github.com/owner/repo",
+        )
+        sha: str = SchemaField(
+            description="The SHA of the commit to set status for",
+        )
+        state: StatusState = SchemaField(
+            description="The state of the status (error, failure, pending, success)",
+        )
+        target_url: Optional[str] = SchemaField(
+            description="URL with additional details about this status",
+            default=None,
+        )
+        description: Optional[str] = SchemaField(
+            description="Short description of the status",
+            default=None,
+        )
+        check_name: Optional[str] = SchemaField(
+            description="Label to differentiate this status from others",
+            default="AutoGPT Platform Checks",
+            advanced=False,
+        )
+
+    class Output(BlockSchema):
+        class StatusResult(BaseModel):
+            id: int
+            url: str
+            state: str
+            context: str
+            description: Optional[str]
+            target_url: Optional[str]
+            created_at: str
+            updated_at: str
+
+        status: StatusResult = SchemaField(description="Details of the created status")
+        error: str = SchemaField(description="Error message if status creation failed")
+
+    def __init__(self):
+        super().__init__(
+            id="3d67f123-a4b5-4c89-9d01-2e34f5c67890",  # Generated UUID
+            description="Creates a new commit status in a GitHub repository",
+            categories={BlockCategory.DEVELOPER_TOOLS},
+            input_schema=GithubCreateStatusBlock.Input,
+            output_schema=GithubCreateStatusBlock.Output,
+            test_input={
+                "repo_url": "https://github.com/owner/repo",
+                "sha": "ce587453ced02b1526dfb4cb910479d431683101",
+                "state": StatusState.SUCCESS.value,
+                "target_url": "https://example.com/build/status",
+                "description": "The build succeeded!",
+                "check_name": "continuous-integration/jenkins",
+                "credentials": TEST_CREDENTIALS_INPUT,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                (
+                    "status",
+                    {
+                        "id": 1234567890,
+                        "url": "https://api.github.com/repos/owner/repo/statuses/ce587453ced02b1526dfb4cb910479d431683101",
+                        "state": "success",
+                        "context": "continuous-integration/jenkins",
+                        "description": "The build succeeded!",
+                        "target_url": "https://example.com/build/status",
+                        "created_at": "2024-01-21T10:00:00Z",
+                        "updated_at": "2024-01-21T10:00:00Z",
+                    },
+                ),
+            ],
+            test_mock={
+                "create_status": lambda *args, **kwargs: {
+                    "id": 1234567890,
+                    "url": "https://api.github.com/repos/owner/repo/statuses/ce587453ced02b1526dfb4cb910479d431683101",
+                    "state": "success",
+                    "context": "continuous-integration/jenkins",
+                    "description": "The build succeeded!",
+                    "target_url": "https://example.com/build/status",
+                    "created_at": "2024-01-21T10:00:00Z",
+                    "updated_at": "2024-01-21T10:00:00Z",
+                }
+            },
+        )
+
+    @staticmethod
+    async def create_status(
+        credentials: GithubFineGrainedAPICredentials,
+        repo_url: str,
+        sha: str,
+        state: StatusState,
+        target_url: Optional[str] = None,
+        description: Optional[str] = None,
+        context: str = "default",
+    ) -> dict:
+        api = get_api(credentials)
+
+        class StatusData(BaseModel):
+            state: str
+            target_url: Optional[str] = None
+            description: Optional[str] = None
+            context: str
+
+        data = StatusData(
+            state=state.value,
+            context=context,
+        )
+
+        if target_url:
+            data.target_url = target_url
+
+        if description:
+            data.description = description
+
+        status_url = f"{repo_url}/statuses/{sha}"
+        response = await api.post(
+            status_url, data=data.model_dump_json(exclude_none=True)
+        )
+        result = response.json()
+
+        return {
+            "id": result["id"],
+            "url": result["url"],
+            "state": result["state"],
+            "context": result["context"],
+            "description": result.get("description"),
+            "target_url": result.get("target_url"),
+            "created_at": result["created_at"],
+            "updated_at": result["updated_at"],
+        }
+
+    async def run(
+        self,
+        input_data: Input,
+        *,
+        credentials: GithubFineGrainedAPICredentials,
+        **kwargs,
+    ) -> BlockOutput:
+        try:
+            result = await self.create_status(
+                credentials=credentials,
+                repo_url=input_data.repo_url,
+                sha=input_data.sha,
+                state=input_data.state,
+                target_url=input_data.target_url,
+                description=input_data.description,
+                context=input_data.check_name or "AutoGPT Platform Checks",
+            )
+            yield "status", result
+        except Exception as e:
+            yield "error", str(e)

autogpt_platform/backend/backend/blocks/github/triggers.py

@@ -12,6 +12,7 @@ from backend.data.block import (
     BlockWebhookConfig,
 )
 from backend.data.model import SchemaField
+from backend.integrations.providers import ProviderName
 
 from ._auth import (
     TEST_CREDENTIALS,
@@ -36,7 +37,7 @@ class GitHubTriggerBase:
             placeholder="{owner}/{repo}",
         )
         # --8<-- [start:example-payload-field]
-        payload: dict = SchemaField(hidden=True, default={})
+        payload: dict = SchemaField(hidden=True, default_factory=dict)
         # --8<-- [end:example-payload-field]
 
     class Output(BlockSchema):
@@ -52,7 +53,7 @@ class GitHubTriggerBase:
             description="Error message if the payload could not be processed"
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:
         yield "payload", input_data.payload
         yield "triggered_by_user", input_data.payload["sender"]
 
@@ -111,7 +112,9 @@ class GithubPullRequestTriggerBlock(GitHubTriggerBase, Block):
     def __init__(self):
         from backend.integrations.webhooks.github import GithubWebhookType
 
-        example_payload = json.loads(self.EXAMPLE_PAYLOAD_FILE.read_text())
+        example_payload = json.loads(
+            self.EXAMPLE_PAYLOAD_FILE.read_text(encoding="utf-8")
+        )
 
         super().__init__(
             id="6c60ec01-8128-419e-988f-96a063ee2fea",
@@ -121,7 +124,7 @@ class GithubPullRequestTriggerBlock(GitHubTriggerBase, Block):
             output_schema=GithubPullRequestTriggerBlock.Output,
             # --8<-- [start:example-webhook_config]
             webhook_config=BlockWebhookConfig(
-                provider="github",
+                provider=ProviderName.GITHUB,
                 webhook_type=GithubWebhookType.REPO,
                 resource_format="{repo}",
                 event_filter_input="events",
@@ -145,8 +148,9 @@ class GithubPullRequestTriggerBlock(GitHubTriggerBase, Block):
             ],
         )
 
-    def run(self, input_data: Input, **kwargs) -> BlockOutput:  # type: ignore
-        yield from super().run(input_data, **kwargs)
+    async def run(self, input_data: Input, **kwargs) -> BlockOutput:  # type: ignore
+        async for name, value in super().run(input_data, **kwargs):
+            yield name, value
         yield "event", input_data.payload["action"]
         yield "number", input_data.payload["number"]
         yield "pull_request", input_data.payload["pull_request"]

autogpt_platform/backend/backend/blocks/google/_auth.py

@@ -3,6 +3,7 @@ from typing import Literal
 from pydantic import SecretStr
 
 from backend.data.model import CredentialsField, CredentialsMetaInput, OAuth2Credentials
+from backend.integrations.providers import ProviderName
 from backend.util.settings import Secrets
 
 # --8<-- [start:GoogleOAuthIsConfigured]
@@ -12,7 +13,9 @@ GOOGLE_OAUTH_IS_CONFIGURED = bool(
 )
 # --8<-- [end:GoogleOAuthIsConfigured]
 GoogleCredentials = OAuth2Credentials
-GoogleCredentialsInput = CredentialsMetaInput[Literal["google"], Literal["oauth2"]]
+GoogleCredentialsInput = CredentialsMetaInput[
+    Literal[ProviderName.GOOGLE], Literal["oauth2"]
+]
 
 
 def GoogleCredentialsField(scopes: list[str]) -> GoogleCredentialsInput:
@@ -23,8 +26,6 @@ def GoogleCredentialsField(scopes: list[str]) -> GoogleCredentialsInput:
         scopes: The authorization scopes needed for the block to work.
     """
     return CredentialsField(
-        provider="google",
-        supported_credential_types={"oauth2"},
         required_scopes=set(scopes),
         description="The Google integration requires OAuth2 authentication.",
     )

autogpt_platform/backend/backend/blocks/google/calendar.py

@@ -0,0 +1,603 @@
+import asyncio
+import enum
+import uuid
+from datetime import datetime, timedelta, timezone
+from typing import Literal
+
+from google.oauth2.credentials import Credentials
+from googleapiclient.discovery import build
+from pydantic import BaseModel
+
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+from backend.util.settings import AppEnvironment, Settings
+
+from ._auth import (
+    GOOGLE_OAUTH_IS_CONFIGURED,
+    TEST_CREDENTIALS,
+    TEST_CREDENTIALS_INPUT,
+    GoogleCredentials,
+    GoogleCredentialsField,
+    GoogleCredentialsInput,
+)
+
+
+class CalendarEvent(BaseModel):
+    """Structured representation of a Google Calendar event."""
+
+    id: str
+    title: str
+    start_time: str
+    end_time: str
+    is_all_day: bool
+    location: str | None
+    description: str | None
+    organizer: str | None
+    attendees: list[str]
+    has_video_call: bool
+    video_link: str | None
+    calendar_link: str
+    is_recurring: bool
+
+
+class GoogleCalendarReadEventsBlock(Block):
+    class Input(BlockSchema):
+        credentials: GoogleCredentialsInput = GoogleCredentialsField(
+            ["https://www.googleapis.com/auth/calendar.readonly"]
+        )
+        calendar_id: str = SchemaField(
+            description="Calendar ID (use 'primary' for your main calendar)",
+            default="primary",
+        )
+        max_events: int = SchemaField(
+            description="Maximum number of events to retrieve", default=10
+        )
+        start_time: datetime = SchemaField(
+            description="Retrieve events starting from this time",
+            default_factory=lambda: datetime.now(tz=timezone.utc),
+        )
+        time_range_days: int = SchemaField(
+            description="Number of days to look ahead for events", default=30
+        )
+        search_term: str | None = SchemaField(
+            description="Optional search term to filter events by", default=None
+        )
+
+        page_token: str | None = SchemaField(
+            description="Page token from previous request to get the next batch of events. You can use this if you have lots of events you want to process in a loop",
+            default=None,
+        )
+        include_declined_events: bool = SchemaField(
+            description="Include events you've declined", default=False
+        )
+
+    class Output(BlockSchema):
+        events: list[CalendarEvent] = SchemaField(
+            description="List of calendar events in the requested time range",
+            default_factory=list,
+        )
+        event: CalendarEvent = SchemaField(
+            description="One of the calendar events in the requested time range"
+        )
+        next_page_token: str | None = SchemaField(
+            description="Token for retrieving the next page of events if more exist",
+            default=None,
+        )
+        error: str = SchemaField(
+            description="Error message if the request failed",
+        )
+
+    def __init__(self):
+        settings = Settings()
+
+        # Create realistic test data for events
+        test_now = datetime.now(tz=timezone.utc)
+        test_tomorrow = test_now + timedelta(days=1)
+
+        test_event_dict = {
+            "id": "event1id",
+            "title": "Team Meeting",
+            "start_time": test_tomorrow.strftime("%Y-%m-%d %H:%M"),
+            "end_time": (test_tomorrow + timedelta(hours=1)).strftime("%Y-%m-%d %H:%M"),
+            "is_all_day": False,
+            "location": "Conference Room A",
+            "description": "Weekly team sync",
+            "organizer": "manager@example.com",
+            "attendees": ["colleague1@example.com", "colleague2@example.com"],
+            "has_video_call": True,
+            "video_link": "https://meet.google.com/abc-defg-hij",
+            "calendar_link": "https://calendar.google.com/calendar/event?eid=event1id",
+            "is_recurring": True,
+        }
+
+        super().__init__(
+            id="80bc3ed1-e9a4-449e-8163-a8fc86f74f6a",
+            description="Retrieves upcoming events from a Google Calendar with filtering options",
+            categories={BlockCategory.PRODUCTIVITY, BlockCategory.DATA},
+            input_schema=GoogleCalendarReadEventsBlock.Input,
+            output_schema=GoogleCalendarReadEventsBlock.Output,
+            disabled=not GOOGLE_OAUTH_IS_CONFIGURED
+            or settings.config.app_env == AppEnvironment.PRODUCTION,
+            test_input={
+                "credentials": TEST_CREDENTIALS_INPUT,
+                "calendar_id": "primary",
+                "max_events": 5,
+                "start_time": test_now.isoformat(),
+                "time_range_days": 7,
+                "search_term": None,
+                "include_declined_events": False,
+                "page_token": None,
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                ("event", test_event_dict),
+                ("events", [test_event_dict]),
+            ],
+            test_mock={
+                "_read_calendar": lambda *args, **kwargs: {
+                    "items": [
+                        {
+                            "id": "event1id",
+                            "summary": "Team Meeting",
+                            "start": {
+                                "dateTime": test_tomorrow.isoformat(),
+                                "timeZone": "UTC",
+                            },
+                            "end": {
+                                "dateTime": (
+                                    test_tomorrow + timedelta(hours=1)
+                                ).isoformat(),
+                                "timeZone": "UTC",
+                            },
+                            "location": "Conference Room A",
+                            "description": "Weekly team sync",
+                            "organizer": {"email": "manager@example.com"},
+                            "attendees": [
+                                {"email": "colleague1@example.com"},
+                                {"email": "colleague2@example.com"},
+                            ],
+                            "conferenceData": {
+                                "conferenceUrl": "https://meet.google.com/abc-defg-hij"
+                            },
+                            "htmlLink": "https://calendar.google.com/calendar/event?eid=event1id",
+                            "recurrence": ["RRULE:FREQ=WEEKLY;COUNT=10"],
+                        }
+                    ],
+                    "nextPageToken": None,
+                },
+                "_format_events": lambda *args, **kwargs: [test_event_dict],
+            },
+        )
+
+    async def run(
+        self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
+    ) -> BlockOutput:
+        try:
+            service = self._build_service(credentials, **kwargs)
+
+            # Calculate end time based on start time and time range
+            end_time = input_data.start_time + timedelta(
+                days=input_data.time_range_days
+            )
+
+            # Call Google Calendar API
+            result = await asyncio.to_thread(
+                self._read_calendar,
+                service=service,
+                calendarId=input_data.calendar_id,
+                time_min=input_data.start_time.isoformat(),
+                time_max=end_time.isoformat(),
+                max_results=input_data.max_events,
+                single_events=True,
+                search_term=input_data.search_term,
+                show_deleted=False,
+                show_hidden=input_data.include_declined_events,
+                page_token=input_data.page_token,
+            )
+
+            # Format events into a user-friendly structure
+            formatted_events = self._format_events(result.get("items", []))
+
+            # Include next page token if available
+            if next_page_token := result.get("nextPageToken"):
+                yield "next_page_token", next_page_token
+
+            for event in formatted_events:
+                yield "event", event
+
+            yield "events", formatted_events
+
+        except Exception as e:
+            yield "error", str(e)
+
+    @staticmethod
+    def _build_service(credentials: GoogleCredentials, **kwargs):
+        creds = Credentials(
+            token=(
+                credentials.access_token.get_secret_value()
+                if credentials.access_token
+                else None
+            ),
+            refresh_token=(
+                credentials.refresh_token.get_secret_value()
+                if credentials.refresh_token
+                else None
+            ),
+            token_uri="https://oauth2.googleapis.com/token",
+            client_id=Settings().secrets.google_client_id,
+            client_secret=Settings().secrets.google_client_secret,
+            scopes=credentials.scopes,
+        )
+        return build("calendar", "v3", credentials=creds)
+
+    def _read_calendar(
+        self,
+        service,
+        calendarId: str,
+        time_min: str,
+        time_max: str,
+        max_results: int,
+        single_events: bool,
+        search_term: str | None = None,
+        show_deleted: bool = False,
+        show_hidden: bool = False,
+        page_token: str | None = None,
+    ) -> dict:
+        """Read calendar events with optional filtering."""
+        calendar = service.events()
+
+        # Build query parameters
+        params = {
+            "calendarId": calendarId,
+            "timeMin": time_min,
+            "timeMax": time_max,
+            "maxResults": max_results,
+            "singleEvents": single_events,
+            "orderBy": "startTime",
+            "showDeleted": show_deleted,
+            "showHiddenInvitations": show_hidden,
+            **({"pageToken": page_token} if page_token else {}),
+        }
+
+        # Add search term if provided
+        if search_term:
+            params["q"] = search_term
+
+        result = calendar.list(**params).execute()
+        return result
+
+    def _format_events(self, events: list[dict]) -> list[CalendarEvent]:
+        """Format Google Calendar API events into user-friendly structure."""
+        formatted_events = []
+
+        for event in events:
+            # Determine if all-day event
+            is_all_day = "date" in event.get("start", {})
+
+            # Format start and end times
+            if is_all_day:
+                start_time = event.get("start", {}).get("date", "")
+                end_time = event.get("end", {}).get("date", "")
+            else:
+                # Convert ISO format to more readable format
+                start_datetime = datetime.fromisoformat(
+                    event.get("start", {}).get("dateTime", "").replace("Z", "+00:00")
+                )
+                end_datetime = datetime.fromisoformat(
+                    event.get("end", {}).get("dateTime", "").replace("Z", "+00:00")
+                )
+                start_time = start_datetime.strftime("%Y-%m-%d %H:%M")
+                end_time = end_datetime.strftime("%Y-%m-%d %H:%M")
+
+            # Extract attendees
+            attendees = []
+            for attendee in event.get("attendees", []):
+                if email := attendee.get("email"):
+                    attendees.append(email)
+
+            # Check for video call link
+            has_video_call = False
+            video_link = None
+            if conf_data := event.get("conferenceData"):
+                if conf_url := conf_data.get("conferenceUrl"):
+                    has_video_call = True
+                    video_link = conf_url
+                elif entry_points := conf_data.get("entryPoints", []):
+                    for entry in entry_points:
+                        if entry.get("entryPointType") == "video":
+                            has_video_call = True
+                            video_link = entry.get("uri")
+                            break
+
+            # Create formatted event
+            formatted_event = CalendarEvent(
+                id=event.get("id", ""),
+                title=event.get("summary", "Untitled Event"),
+                start_time=start_time,
+                end_time=end_time,
+                is_all_day=is_all_day,
+                location=event.get("location"),
+                description=event.get("description"),
+                organizer=event.get("organizer", {}).get("email"),
+                attendees=attendees,
+                has_video_call=has_video_call,
+                video_link=video_link,
+                calendar_link=event.get("htmlLink", ""),
+                is_recurring=bool(event.get("recurrence")),
+            )
+
+            formatted_events.append(formatted_event)
+
+        return formatted_events
+
+
+class ReminderPreset(enum.Enum):
+    """Common reminder times before an event."""
+
+    TEN_MINUTES = 10
+    THIRTY_MINUTES = 30
+    ONE_HOUR = 60
+    ONE_DAY = 1440  # 24 hours in minutes
+
+
+class RecurrenceFrequency(enum.Enum):
+    """Frequency options for recurring events."""
+
+    DAILY = "DAILY"
+    WEEKLY = "WEEKLY"
+    MONTHLY = "MONTHLY"
+    YEARLY = "YEARLY"
+
+
+class ExactTiming(BaseModel):
+    """Model for specifying start and end times."""
+
+    discriminator: Literal["exact_timing"]
+    start_datetime: datetime
+    end_datetime: datetime
+
+
+class DurationTiming(BaseModel):
+    """Model for specifying start time and duration."""
+
+    discriminator: Literal["duration_timing"]
+    start_datetime: datetime
+    duration_minutes: int
+
+
+class OneTimeEvent(BaseModel):
+    """Model for a one-time event."""
+
+    discriminator: Literal["one_time"]
+
+
+class RecurringEvent(BaseModel):
+    """Model for a recurring event."""
+
+    discriminator: Literal["recurring"]
+    frequency: RecurrenceFrequency
+    count: int
+
+
+class GoogleCalendarCreateEventBlock(Block):
+    class Input(BlockSchema):
+        credentials: GoogleCredentialsInput = GoogleCredentialsField(
+            ["https://www.googleapis.com/auth/calendar"]
+        )
+        # Event Details
+        event_title: str = SchemaField(description="Title of the event")
+        location: str | None = SchemaField(
+            description="Location of the event", default=None
+        )
+        description: str | None = SchemaField(
+            description="Description of the event", default=None
+        )
+
+        # Timing
+        timing: ExactTiming | DurationTiming = SchemaField(
+            discriminator="discriminator",
+            advanced=False,
+            description="Specify when the event starts and ends",
+            default_factory=lambda: DurationTiming(
+                discriminator="duration_timing",
+                start_datetime=datetime.now().replace(microsecond=0, second=0, minute=0)
+                + timedelta(hours=1),
+                duration_minutes=60,
+            ),
+        )
+
+        # Calendar selection
+        calendar_id: str = SchemaField(
+            description="Calendar ID (use 'primary' for your main calendar)",
+            default="primary",
+        )
+
+        # Guests
+        guest_emails: list[str] = SchemaField(
+            description="Email addresses of guests to invite", default_factory=list
+        )
+        send_notifications: bool = SchemaField(
+            description="Send email notifications to guests", default=True
+        )
+
+        # Extras
+        add_google_meet: bool = SchemaField(
+            description="Include a Google Meet video conference link", default=False
+        )
+        recurrence: OneTimeEvent | RecurringEvent = SchemaField(
+            discriminator="discriminator",
+            description="Whether the event repeats",
+            default_factory=lambda: OneTimeEvent(discriminator="one_time"),
+        )
+        reminder_minutes: list[ReminderPreset] = SchemaField(
+            description="When to send reminders before the event",
+            default_factory=lambda: [ReminderPreset.TEN_MINUTES],
+        )
+
+    class Output(BlockSchema):
+        event_id: str = SchemaField(description="ID of the created event")
+        event_link: str = SchemaField(
+            description="Link to view the event in Google Calendar"
+        )
+        error: str = SchemaField(description="Error message if event creation failed")
+
+    def __init__(self):
+        settings = Settings()
+
+        super().__init__(
+            id="ed2ec950-fbff-4204-94c0-023fb1d625e0",
+            description="This block creates a new event in Google Calendar with customizable parameters.",
+            categories={BlockCategory.PRODUCTIVITY},
+            input_schema=GoogleCalendarCreateEventBlock.Input,
+            output_schema=GoogleCalendarCreateEventBlock.Output,
+            disabled=not GOOGLE_OAUTH_IS_CONFIGURED
+            or settings.config.app_env == AppEnvironment.PRODUCTION,
+            test_input={
+                "credentials": TEST_CREDENTIALS_INPUT,
+                "event_title": "Team Meeting",
+                "location": "Conference Room A",
+                "description": "Weekly team sync-up",
+                "calendar_id": "primary",
+                "guest_emails": ["colleague1@example.com", "colleague2@example.com"],
+                "add_google_meet": True,
+                "send_notifications": True,
+                "reminder_minutes": [
+                    ReminderPreset.TEN_MINUTES.value,
+                    ReminderPreset.ONE_HOUR.value,
+                ],
+            },
+            test_credentials=TEST_CREDENTIALS,
+            test_output=[
+                ("event_id", "abc123event_id"),
+                ("event_link", "https://calendar.google.com/calendar/event?eid=abc123"),
+            ],
+            test_mock={
+                "_create_event": lambda *args, **kwargs: {
+                    "id": "abc123event_id",
+                    "htmlLink": "https://calendar.google.com/calendar/event?eid=abc123",
+                }
+            },
+        )
+
+    async def run(
+        self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
+    ) -> BlockOutput:
+        try:
+            service = self._build_service(credentials, **kwargs)
+
+            # Create event body
+            # Get start and end times based on the timing option
+            if input_data.timing.discriminator == "exact_timing":
+                start_datetime = input_data.timing.start_datetime
+                end_datetime = input_data.timing.end_datetime
+            else:  # duration_timing
+                start_datetime = input_data.timing.start_datetime
+                end_datetime = start_datetime + timedelta(
+                    minutes=input_data.timing.duration_minutes
+                )
+
+            # Format datetimes for Google Calendar API
+            start_time_str = start_datetime.isoformat()
+            end_time_str = end_datetime.isoformat()
+
+            # Build the event body
+            event_body = {
+                "summary": input_data.event_title,
+                "start": {"dateTime": start_time_str},
+                "end": {"dateTime": end_time_str},
+            }
+
+            # Add optional fields
+            if input_data.location:
+                event_body["location"] = input_data.location
+
+            if input_data.description:
+                event_body["description"] = input_data.description
+
+            # Add guests
+            if input_data.guest_emails:
+                event_body["attendees"] = [
+                    {"email": email} for email in input_data.guest_emails
+                ]
+
+            # Add reminders
+            if input_data.reminder_minutes:
+                event_body["reminders"] = {
+                    "useDefault": False,
+                    "overrides": [
+                        {"method": "popup", "minutes": reminder.value}
+                        for reminder in input_data.reminder_minutes
+                    ],
+                }
+
+            # Add Google Meet
+            if input_data.add_google_meet:
+                event_body["conferenceData"] = {
+                    "createRequest": {
+                        "requestId": f"meet-{uuid.uuid4()}",
+                        "conferenceSolutionKey": {"type": "hangoutsMeet"},
+                    }
+                }
+
+            # Add recurrence
+            if input_data.recurrence.discriminator == "recurring":
+                rule = f"RRULE:FREQ={input_data.recurrence.frequency.value}"
+                rule += f";COUNT={input_data.recurrence.count}"
+                event_body["recurrence"] = [rule]
+
+            # Create the event
+            result = await asyncio.to_thread(
+                self._create_event,
+                service=service,
+                calendar_id=input_data.calendar_id,
+                event_body=event_body,
+                send_notifications=input_data.send_notifications,
+                conference_data_version=1 if input_data.add_google_meet else 0,
+            )
+
+            yield "event_id", result["id"]
+            yield "event_link", result["htmlLink"]
+
+        except Exception as e:
+            yield "error", str(e)
+
+    @staticmethod
+    def _build_service(credentials: GoogleCredentials, **kwargs):
+        creds = Credentials(
+            token=(
+                credentials.access_token.get_secret_value()
+                if credentials.access_token
+                else None
+            ),
+            refresh_token=(
+                credentials.refresh_token.get_secret_value()
+                if credentials.refresh_token
+                else None
+            ),
+            token_uri="https://oauth2.googleapis.com/token",
+            client_id=Settings().secrets.google_client_id,
+            client_secret=Settings().secrets.google_client_secret,
+            scopes=credentials.scopes,
+        )
+        return build("calendar", "v3", credentials=creds)
+
+    def _create_event(
+        self,
+        service,
+        calendar_id: str,
+        event_body: dict,
+        send_notifications: bool = False,
+        conference_data_version: int = 0,
+    ) -> dict:
+        """Create a new event in Google Calendar."""
+        calendar = service.events()
+
+        # Make the API call
+        result = calendar.insert(
+            calendarId=calendar_id,
+            body=event_body,
+            sendNotifications=send_notifications,
+            conferenceDataVersion=conference_data_version,
+        ).execute()
+
+        return result

autogpt_platform/backend/backend/blocks/google/gmail.py

@@ -1,3 +1,4 @@
+import asyncio
 import base64
 from email.utils import parseaddr
 from typing import List
@@ -8,6 +9,7 @@ from pydantic import BaseModel
 
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 from backend.data.model import SchemaField
+from backend.util.settings import Settings
 
 from ._auth import (
     GOOGLE_OAUTH_IS_CONFIGURED,
@@ -127,11 +129,13 @@ class GmailReadBlock(Block):
             },
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
     ) -> BlockOutput:
-        service = self._build_service(credentials, **kwargs)
-        messages = self._read_emails(service, input_data.query, input_data.max_results)
+        service = GmailReadBlock._build_service(credentials, **kwargs)
+        messages = await asyncio.to_thread(
+            self._read_emails, service, input_data.query, input_data.max_results
+        )
         for email in messages:
             yield "email", email
         yield "emails", messages
@@ -150,8 +154,8 @@ class GmailReadBlock(Block):
                 else None
             ),
             token_uri="https://oauth2.googleapis.com/token",
-            client_id=kwargs.get("client_id"),
-            client_secret=kwargs.get("client_secret"),
+            client_id=Settings().secrets.google_client_id,
+            client_secret=Settings().secrets.google_client_secret,
             scopes=credentials.scopes,
         )
         return build("gmail", "v1", credentials=creds)
@@ -285,14 +289,18 @@ class GmailSendBlock(Block):
             },
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
     ) -> BlockOutput:
         service = GmailReadBlock._build_service(credentials, **kwargs)
-        send_result = self._send_email(
-            service, input_data.to, input_data.subject, input_data.body
+        result = await asyncio.to_thread(
+            self._send_email,
+            service,
+            input_data.to,
+            input_data.subject,
+            input_data.body,
         )
-        yield "result", send_result
+        yield "result", result
 
     def _send_email(self, service, to: str, subject: str, body: str) -> dict:
         if not to or not subject or not body:
@@ -357,12 +365,12 @@ class GmailListLabelsBlock(Block):
             },
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
     ) -> BlockOutput:
         service = GmailReadBlock._build_service(credentials, **kwargs)
-        labels = self._list_labels(service)
-        yield "result", labels
+        result = await asyncio.to_thread(self._list_labels, service)
+        yield "result", result
 
     def _list_labels(self, service) -> list[dict]:
         results = service.users().labels().list(userId="me").execute()
@@ -418,11 +426,13 @@ class GmailAddLabelBlock(Block):
             },
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
     ) -> BlockOutput:
         service = GmailReadBlock._build_service(credentials, **kwargs)
-        result = self._add_label(service, input_data.message_id, input_data.label_name)
+        result = await asyncio.to_thread(
+            self._add_label, service, input_data.message_id, input_data.label_name
+        )
         yield "result", result
 
     def _add_label(self, service, message_id: str, label_name: str) -> dict:
@@ -501,12 +511,12 @@ class GmailRemoveLabelBlock(Block):
             },
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
     ) -> BlockOutput:
         service = GmailReadBlock._build_service(credentials, **kwargs)
-        result = self._remove_label(
-            service, input_data.message_id, input_data.label_name
+        result = await asyncio.to_thread(
+            self._remove_label, service, input_data.message_id, input_data.label_name
         )
         yield "result", result
 

autogpt_platform/backend/backend/blocks/google/sheets.py

@@ -1,8 +1,11 @@
+import asyncio
+
 from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import build
 
 from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 from backend.data.model import SchemaField
+from backend.util.settings import AppEnvironment, Settings
 
 from ._auth import (
     GOOGLE_OAUTH_IS_CONFIGURED,
@@ -35,13 +38,15 @@ class GoogleSheetsReadBlock(Block):
         )
 
     def __init__(self):
+        settings = Settings()
         super().__init__(
             id="5724e902-3635-47e9-a108-aaa0263a4988",
             description="This block reads data from a Google Sheets spreadsheet.",
             categories={BlockCategory.DATA},
             input_schema=GoogleSheetsReadBlock.Input,
             output_schema=GoogleSheetsReadBlock.Output,
-            disabled=not GOOGLE_OAUTH_IS_CONFIGURED,
+            disabled=not GOOGLE_OAUTH_IS_CONFIGURED
+            or settings.config.app_env == AppEnvironment.PRODUCTION,
             test_input={
                 "spreadsheet_id": "1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms",
                 "range": "Sheet1!A1:B2",
@@ -65,11 +70,13 @@ class GoogleSheetsReadBlock(Block):
             },
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
     ) -> BlockOutput:
         service = self._build_service(credentials, **kwargs)
-        data = self._read_sheet(service, input_data.spreadsheet_id, input_data.range)
+        data = await asyncio.to_thread(
+            self._read_sheet, service, input_data.spreadsheet_id, input_data.range
+        )
         yield "result", data
 
     @staticmethod
@@ -86,8 +93,8 @@ class GoogleSheetsReadBlock(Block):
                 else None
             ),
             token_uri="https://oauth2.googleapis.com/token",
-            client_id=kwargs.get("client_id"),
-            client_secret=kwargs.get("client_secret"),
+            client_id=Settings().secrets.google_client_id,
+            client_secret=Settings().secrets.google_client_secret,
             scopes=credentials.scopes,
         )
         return build("sheets", "v4", credentials=creds)
@@ -154,11 +161,12 @@ class GoogleSheetsWriteBlock(Block):
             },
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: GoogleCredentials, **kwargs
     ) -> BlockOutput:
         service = GoogleSheetsReadBlock._build_service(credentials, **kwargs)
-        result = self._write_sheet(
+        result = await asyncio.to_thread(
+            self._write_sheet,
             service,
             input_data.spreadsheet_id,
             input_data.range,

autogpt_platform/backend/backend/blocks/google_maps.py

@@ -10,6 +10,7 @@ from backend.data.model import (
     CredentialsMetaInput,
     SchemaField,
 )
+from backend.integrations.providers import ProviderName
 
 TEST_CREDENTIALS = APIKeyCredentials(
     id="01234567-89ab-cdef-0123-456789abcdef",
@@ -38,12 +39,8 @@ class Place(BaseModel):
 class GoogleMapsSearchBlock(Block):
     class Input(BlockSchema):
         credentials: CredentialsMetaInput[
-            Literal["google_maps"], Literal["api_key"]
-        ] = CredentialsField(
-            provider="google_maps",
-            supported_credential_types={"api_key"},
-            description="Google Maps API Key",
-        )
+            Literal[ProviderName.GOOGLE_MAPS], Literal["api_key"]
+        ] = CredentialsField(description="Google Maps API Key")
         query: str = SchemaField(
             description="Search query for local businesses",
             placeholder="e.g., 'restaurants in New York'",
@@ -106,7 +103,7 @@ class GoogleMapsSearchBlock(Block):
             test_credentials=TEST_CREDENTIALS,
         )
 
-    def run(
+    async def run(
         self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs
     ) -> BlockOutput:
         places = self.search_places(