remove work dir

Restructuring the Repo to make it clear the difference between classic autogpt and the autogpt platform:
* Move the "classic" projects `autogpt`, `forge`, `frontend`, and `benchmark` into a `classic` folder
  * Also rename `autogpt` to `original_autogpt` for absolute clarity
* Rename `rnd/` to `autogpt_platform/`
  * `rnd/autogpt_builder` -> `autogpt_platform/frontend`
  * `rnd/autogpt_server` -> `autogpt_platform/backend`
* Adjust any paths accordingly

.gitattributes

@@ -1,10 +1,10 @@
-frontend/build/** linguist-generated
+classic/frontend/build/** linguist-generated
 
 **/poetry.lock linguist-generated
 
 docs/_javascript/** linguist-vendored
 
 # Exclude VCR cassettes from stats
-forge/tests/vcr_cassettes/**/**.y*ml linguist-generated
+classic/forge/tests/vcr_cassettes/**/**.y*ml linguist-generated
 
 * text=auto

.github/CODEOWNERS

@@ -1,7 +1,7 @@
 * @Significant-Gravitas/maintainers
 .github/workflows/ @Significant-Gravitas/devops
-forge/ @Significant-Gravitas/forge-maintainers
-benchmark/ @Significant-Gravitas/benchmark-maintainers
-frontend/ @Significant-Gravitas/frontend-maintainers
-rnd/infra @Significant-Gravitas/devops
+classic/forge/ @Significant-Gravitas/forge-maintainers
+classic/benchmark/ @Significant-Gravitas/benchmark-maintainers
+classic/frontend/ @Significant-Gravitas/frontend-maintainers
+autogpt_platform/infra @Significant-Gravitas/devops
 .github/CODEOWNERS @Significant-Gravitas/admins

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,26 +6,18 @@
 
 <!-- Concisely describe all of the changes made in this pull request: -->
 
-### PR Quality Scorecard ✨
+
+### Testing 🔍
+> [!NOTE] 
+Only for the new autogpt platform, currently in autogpt_platform/
 
 <!--
-Check out our contribution guide:
-https://github.com/Significant-Gravitas/AutoGPT/wiki/Contributing
-
-1. Avoid duplicate work, issues, PRs etc.
-2. Also consider contributing something other than code; see the [contribution guide]
-   for options.
-3. Clearly explain your changes.
-4. Avoid making unnecessary changes, especially if they're purely based on personal
-   preferences. Doing so is the maintainers' job. ;-)
+Please make sure your changes have been tested and are in good working condition. 
+Here is a list of our critical paths, if you need some inspiration on what and how to test:
 -->
 
-- [x] Have you used the PR description template? &ensp; `+2 pts`
-- [ ] Is your pull request atomic, focusing on a single change? &ensp; `+5 pts`
-- [ ] Have you linked the GitHub issue(s) that this PR addresses? &ensp; `+5 pts`
-- [ ] Have you documented your changes clearly and comprehensively? &ensp; `+5 pts`
-- [ ] Have you changed or added a feature? &ensp; `-4 pts`
-  - [ ] Have you added/updated corresponding documentation? &ensp; `+4 pts`
-  - [ ] Have you added/updated corresponding integration tests? &ensp; `+5 pts`
-- [ ] Have you changed the behavior of AutoGPT? &ensp; `-5 pts`
-  - [ ] Have you also run `agbenchmark` to verify that these changes do not regress performance? &ensp; `+10 pts`
+- Create from scratch and execute an agent with at least 3 blocks
+- Import an agent from file upload, and confirm it executes correctly
+- Upload agent to marketplace
+- Import an agent from marketplace and confirm it executes correctly
+- Edit an agent from monitor, and confirm it executes correctly

.github/labeler.yml

@@ -1,27 +1,27 @@
-AutoGPT Agent:
+Classic AutoGPT Agent:
 - changed-files:
-  - any-glob-to-any-file: autogpt/**
+  - any-glob-to-any-file: classic/original_autogpt/**
+
+Classic Benchmark:
+- changed-files:
+  - any-glob-to-any-file: classic/benchmark/**
+
+Classic Frontend:
+- changed-files:
+  - any-glob-to-any-file: classic/frontend/**
 
 Forge:
 - changed-files:
-  - any-glob-to-any-file: forge/**
-
-Benchmark:
-- changed-files:
-  - any-glob-to-any-file: benchmark/**
-
-Frontend:
-- changed-files:
-  - any-glob-to-any-file: frontend/**
+  - any-glob-to-any-file: classic/forge/**
 
 documentation:
 - changed-files:
   - any-glob-to-any-file: docs/**
 
-Builder:
+platform/frontend:
 - changed-files:
-  - any-glob-to-any-file: rnd/autogpt_builder/**
+  - any-glob-to-any-file: autogpt_platform/frontend/**
 
-Server:
+platform/backend:
 - changed-files:
-  - any-glob-to-any-file: rnd/autogpt_server/**
+  - any-glob-to-any-file: autogpt_platform/backend/**

.github/workflows/autogpts-benchmark.yml

@@ -1,97 +0,0 @@
-name: AutoGPTs Nightly Benchmark
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 2 * * *'
-
-jobs:
-  benchmark:
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        agent-name: [ autogpt ]
-      fail-fast: false
-    timeout-minutes: 120
-    env:
-      min-python-version: '3.10'
-      REPORTS_BRANCH: data/benchmark-reports
-      REPORTS_FOLDER: ${{ format('benchmark/reports/{0}', matrix.agent-name) }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-
-      - name: Set up Python ${{ env.min-python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.min-python-version }}
-
-      - name: Install Poetry
-        run: curl -sSL https://install.python-poetry.org | python -
-
-      - name: Prepare reports folder
-        run: mkdir -p ${{ env.REPORTS_FOLDER }}
-
-      - run: poetry -C benchmark install
-
-      - name: Benchmark ${{ matrix.agent-name }}
-        run: |
-          ./run agent start ${{ matrix.agent-name }}
-          cd ${{ matrix.agent-name }}
-
-          set +e  # Do not quit on non-zero exit codes
-          poetry run agbenchmark run -N 3 \
-            --test=ReadFile \
-            --test=BasicRetrieval --test=RevenueRetrieval2 \
-            --test=CombineCsv --test=LabelCsv --test=AnswerQuestionCombineCsv \
-            --test=UrlShortener --test=TicTacToe --test=Battleship \
-            --test=WebArenaTask_0 --test=WebArenaTask_21 --test=WebArenaTask_124 \
-            --test=WebArenaTask_134 --test=WebArenaTask_163
-
-            # Convert exit code 1 (some challenges failed) to exit code 0
-            if [ $? -eq 0 ] || [ $? -eq 1 ]; then
-              exit 0
-            else
-              exit $?
-            fi
-        env:
-          AGENT_NAME: ${{ matrix.agent-name }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
-          REPORTS_FOLDER: ${{ format('../../{0}', env.REPORTS_FOLDER) }}  # account for changed workdir
-
-          TELEMETRY_ENVIRONMENT: autogpt-benchmark-ci
-          TELEMETRY_OPT_IN: ${{ github.ref_name == 'master' }}
-
-      - name: Push reports to data branch
-        run: |
-          # BODGE: Remove success_rate.json and regression_tests.json to avoid conflicts on checkout
-          rm ${{ env.REPORTS_FOLDER }}/*.json
-
-          # Find folder with newest (untracked) report in it
-          report_subfolder=$(find ${{ env.REPORTS_FOLDER }} -type f -name 'report.json' \
-            | xargs -I {} dirname {} \
-            | xargs -I {} git ls-files --others --exclude-standard {} \
-            | xargs -I {} dirname {} \
-            | sort -u)
-          json_report_file="$report_subfolder/report.json"
-
-          # Convert JSON report to Markdown
-          markdown_report_file="$report_subfolder/report.md"
-          poetry -C benchmark run benchmark/reports/format.py "$json_report_file" > "$markdown_report_file"
-          cat "$markdown_report_file" >> $GITHUB_STEP_SUMMARY
-
-          git config --global user.name 'GitHub Actions'
-          git config --global user.email 'github-actions@agpt.co'
-          git fetch origin ${{ env.REPORTS_BRANCH }}:${{ env.REPORTS_BRANCH }} \
-            && git checkout ${{ env.REPORTS_BRANCH }} \
-            || git checkout --orphan ${{ env.REPORTS_BRANCH }}
-          git reset --hard
-          git add ${{ env.REPORTS_FOLDER }}
-          git commit -m "Benchmark report for ${{ matrix.agent-name }} @ $(date +'%Y-%m-%d')" \
-            && git push origin ${{ env.REPORTS_BRANCH }}

.github/workflows/classic-autogpt-ci.yml

@@ -1,25 +1,25 @@
-name: AutoGPT CI
+name: Classic - AutoGPT CI
 
 on:
   push:
     branches: [ master, development, ci-test* ]
     paths:
-      - '.github/workflows/autogpt-ci.yml'
-      - 'autogpt/**'
+      - '.github/workflows/classic-autogpt-ci.yml'
+      - 'classic/original_autogpt/**'
   pull_request:
     branches: [ master, development, release-* ]
     paths:
-      - '.github/workflows/autogpt-ci.yml'
-      - 'autogpt/**'
+      - '.github/workflows/classic-autogpt-ci.yml'
+      - 'classic/original_autogpt/**'
 
 concurrency:
-  group: ${{ format('autogpt-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
+  group: ${{ format('classic-autogpt-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
   cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}
 
 defaults:
   run:
     shell: bash
-    working-directory: autogpt
+    working-directory: classic/original_autogpt
 
 jobs:
   test:
@@ -86,7 +86,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }}
-          key: poetry-${{ runner.os }}-${{ hashFiles('autogpt/poetry.lock') }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('classic/original_autogpt/poetry.lock') }}
 
       - name: Install Poetry (Unix)
         if: runner.os != 'Windows'
@@ -135,4 +135,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: test-logs
-          path: autogpt/logs/
+          path: classic/original_autogpt/logs/

.github/workflows/classic-autogpt-docker-cache-clean.yml

@@ -1,4 +1,4 @@
-name: Purge Auto-GPT Docker CI cache
+name: Classic - Purge Auto-GPT Docker CI cache
 
 on:
   schedule:
@@ -25,7 +25,8 @@ jobs:
       name: Build image
       uses: docker/build-push-action@v5
       with:
-        file: Dockerfile.autogpt
+        context: classic/
+        file: classic/Dockerfile.autogpt
         build-args: BUILD_TYPE=${{ matrix.build-type }}
         load: true    # save to docker images
         # use GHA cache as read-only

.github/workflows/classic-autogpt-docker-ci.yml

@@ -1,24 +1,26 @@
-name: AutoGPT Docker CI
+name: Classic - AutoGPT Docker CI
 
 on:
   push:
     branches: [ master, development ]
     paths:
-      - '.github/workflows/autogpt-docker-ci.yml'
-      - 'autogpt/**'
+      - '.github/workflows/classic-autogpt-docker-ci.yml'
+      - 'classic/original_autogpt/**'
+      - 'classic/forge/**'
   pull_request:
     branches: [ master, development, release-* ]
     paths:
-      - '.github/workflows/autogpt-docker-ci.yml'
-      - 'autogpt/**'
+      - '.github/workflows/classic-autogpt-docker-ci.yml'
+      - 'classic/original_autogpt/**'
+      - 'classic/forge/**'
 
 concurrency:
-  group: ${{ format('autogpt-docker-ci-{0}', github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }}
+  group: ${{ format('classic-autogpt-docker-ci-{0}', github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 defaults:
   run:
-    working-directory: autogpt
+    working-directory: classic/original_autogpt
 
 env:
   IMAGE_NAME: auto-gpt
@@ -47,7 +49,8 @@ jobs:
       name: Build image
       uses: docker/build-push-action@v5
       with:
-        file: Dockerfile.autogpt
+        context: classic/
+        file: classic/Dockerfile.autogpt
         build-args: BUILD_TYPE=${{ matrix.build-type }}
         tags: ${{ env.IMAGE_NAME }}
         labels: GIT_REVISION=${{ github.sha }}
@@ -116,7 +119,8 @@ jobs:
         name: Build image
         uses: docker/build-push-action@v5
         with:
-          file: Dockerfile.autogpt
+          context: classic/
+          file: classic/Dockerfile.autogpt
           build-args: BUILD_TYPE=dev  # include pytest
           tags: >
             ${{ env.IMAGE_NAME }},

.github/workflows/classic-autogpt-docker-release.yml

@@ -1,4 +1,4 @@
-name: AutoGPT Docker Release
+name: Classic - AutoGPT Docker Release
 
 on:
   release:
@@ -44,6 +44,7 @@ jobs:
       name: Build image
       uses: docker/build-push-action@v5
       with:
+        context: classic/
         file: Dockerfile.autogpt
         build-args: BUILD_TYPE=release
         load: true    # save to docker images

.github/workflows/classic-autogpts-ci.yml

@@ -1,4 +1,4 @@
-name: Agent smoke tests
+name: Classic - Agent smoke tests
 
 on:
   workflow_dispatch:
@@ -7,32 +7,37 @@ on:
   push:
     branches: [ master, development, ci-test* ]
     paths:
-      - '.github/workflows/autogpts-ci.yml'
-      - 'autogpt/**'
-      - 'forge/**'
-      - 'benchmark/**'
-      - 'run'
-      - 'cli.py'
-      - 'setup.py'
+      - '.github/workflows/classic-autogpts-ci.yml'
+      - 'classic/original_autogpt/**'
+      - 'classic/forge/**'
+      - 'classic/benchmark/**'
+      - 'classic/run'
+      - 'classic/cli.py'
+      - 'classic/setup.py'
       - '!**/*.md'
   pull_request:
     branches: [ master, development, release-* ]
     paths:
-      - '.github/workflows/autogpts-ci.yml'
-      - 'autogpt/**'
-      - 'forge/**'
-      - 'benchmark/**'
-      - 'run'
-      - 'cli.py'
-      - 'setup.py'
+      - '.github/workflows/classic-autogpts-ci.yml'
+      - 'classic/original_autogpt/**'
+      - 'classic/forge/**'
+      - 'classic/benchmark/**'
+      - 'classic/run'
+      - 'classic/cli.py'
+      - 'classic/setup.py'
       - '!**/*.md'
 
+defaults:
+  run:
+    shell: bash
+    working-directory: classic
+
 jobs:
   serve-agent-protocol:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        agent-name: [ autogpt ]
+        agent-name: [ original_autogpt ]
       fail-fast: false
     timeout-minutes: 20
     env:
@@ -50,7 +55,7 @@ jobs:
           python-version: ${{ env.min-python-version }}
 
       - name: Install Poetry
-        working-directory: ./${{ matrix.agent-name }}/
+        working-directory: ./classic/${{ matrix.agent-name }}/
         run: |
           curl -sSL https://install.python-poetry.org | python -
 

.github/workflows/classic-benchmark-ci.yml

@@ -1,18 +1,18 @@
-name: AGBenchmark CI
+name: Classic - AGBenchmark CI
 
 on:
   push:
     branches: [ master, development, ci-test* ]
     paths:
-      - 'benchmark/**'
-      - .github/workflows/benchmark-ci.yml
-      - '!benchmark/reports/**'
+      - 'classic/benchmark/**'
+      - '!classic/benchmark/reports/**'
+      - .github/workflows/classic-benchmark-ci.yml
   pull_request:
     branches: [ master, development, release-* ]
     paths:
-      - 'benchmark/**'
-      - '!benchmark/reports/**'
-      - .github/workflows/benchmark-ci.yml
+      - 'classic/benchmark/**'
+      - '!classic/benchmark/reports/**'
+      - .github/workflows/classic-benchmark-ci.yml
 
 concurrency:
   group: ${{ format('benchmark-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
@@ -39,7 +39,7 @@ jobs:
     defaults:
       run:
         shell: bash
-        working-directory: benchmark
+        working-directory: classic/benchmark
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -58,7 +58,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }}
-          key: poetry-${{ runner.os }}-${{ hashFiles('benchmark/poetry.lock') }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('classic/benchmark/poetry.lock') }}
 
       - name: Install Poetry (Unix)
         if: runner.os != 'Windows'
@@ -122,7 +122,7 @@ jobs:
           curl -sSL https://install.python-poetry.org | python -
 
       - name: Run regression tests
-        working-directory: .
+        working-directory: classic
         run: |
           ./run agent start ${{ matrix.agent-name }}
           cd ${{ matrix.agent-name }}
@@ -155,7 +155,7 @@ jobs:
 
           poetry run agbenchmark --mock
 
-          CHANGED=$(git diff --name-only | grep -E '(agbenchmark/challenges)|(../frontend/assets)') || echo "No diffs"
+          CHANGED=$(git diff --name-only | grep -E '(agbenchmark/challenges)|(../classic/frontend/assets)') || echo "No diffs"
           if [ ! -z "$CHANGED" ]; then
             echo "There are unstaged changes please run agbenchmark and commit those changes since they are needed."
             echo "$CHANGED"

.github/workflows/classic-benchmark_publish_package.yml

@@ -1,4 +1,4 @@
-name: Publish to PyPI
+name: Classic - Publish to PyPI
 
 on:
   workflow_dispatch:
@@ -21,21 +21,21 @@ jobs:
         python-version: 3.8
 
     - name: Install Poetry
-      working-directory: ./benchmark/
+      working-directory: ./classic/benchmark/
       run: |
         curl -sSL https://install.python-poetry.org | python3 -
         echo "$HOME/.poetry/bin" >> $GITHUB_PATH
 
     - name: Build project for distribution
-      working-directory: ./benchmark/
+      working-directory: ./classic/benchmark/
       run: poetry build
 
     - name: Install dependencies
-      working-directory: ./benchmark/
+      working-directory: ./classic/benchmark/
       run: poetry install
 
     - name: Check Version
-      working-directory: ./benchmark/
+      working-directory: ./classic/benchmark/
       id: check-version
       run: |
         echo version=$(poetry version --short) >> $GITHUB_OUTPUT
@@ -43,7 +43,7 @@ jobs:
     - name: Create Release
       uses: ncipollo/release-action@v1
       with:
-        artifacts: "benchmark/dist/*"
+        artifacts: "classic/benchmark/dist/*"
         token: ${{ secrets.GITHUB_TOKEN }}
         draft: false
         generateReleaseNotes: false
@@ -51,5 +51,5 @@ jobs:
         commit: master
 
     - name: Build and publish
-      working-directory: ./benchmark/
+      working-directory: ./classic/benchmark/
       run: poetry publish -u __token__ -p ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/classic-forge-ci.yml

@@ -1,18 +1,18 @@
-name: Forge CI
+name: Classic - Forge CI
 
 on:
   push:
     branches: [ master, development, ci-test* ]
     paths:
-      - '.github/workflows/forge-ci.yml'
-      - 'forge/**'
-      - '!forge/tests/vcr_cassettes'
+      - '.github/workflows/classic-forge-ci.yml'
+      - 'classic/forge/**'
+      - '!classic/forge/tests/vcr_cassettes'
   pull_request:
     branches: [ master, development, release-* ]
     paths:
-      - '.github/workflows/forge-ci.yml'
-      - 'forge/**'
-      - '!forge/tests/vcr_cassettes'
+      - '.github/workflows/classic-forge-ci.yml'
+      - 'classic/forge/**'
+      - '!classic/forge/tests/vcr_cassettes'
 
 concurrency:
   group: ${{ format('forge-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
@@ -21,7 +21,7 @@ concurrency:
 defaults:
   run:
     shell: bash
-    working-directory: forge
+    working-directory: classic/forge
 
 jobs:
   test:
@@ -110,7 +110,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }}
-          key: poetry-${{ runner.os }}-${{ hashFiles('forge/poetry.lock') }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('classic/forge/poetry.lock') }}
 
       - name: Install Poetry (Unix)
         if: runner.os != 'Windows'
@@ -233,4 +233,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: test-logs
-          path: forge/logs/
+          path: classic/forge/logs/

.github/workflows/classic-frontend-ci.yml

@@ -1,4 +1,4 @@
-name: Frontend CI/CD
+name: Classic - Frontend CI/CD
 
 on:
   push:
@@ -7,12 +7,12 @@ on:
       - development
       - 'ci-test*' # This will match any branch that starts with "ci-test"
     paths:
-      - 'frontend/**'
-      - '.github/workflows/frontend-ci.yml'
+      - 'classic/frontend/**'
+      - '.github/workflows/classic-frontend-ci.yml'
   pull_request:
     paths:
-      - 'frontend/**'
-      - '.github/workflows/frontend-ci.yml'
+      - 'classic/frontend/**'
+      - '.github/workflows/classic-frontend-ci.yml'
 
 jobs:
   build:
@@ -21,7 +21,7 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     env:
-      BUILD_BRANCH: ${{ format('frontend-build/{0}', github.ref_name) }}
+      BUILD_BRANCH: ${{ format('classic-frontend-build/{0}', github.ref_name) }}
 
     steps:
     - name: Checkout Repo
@@ -34,7 +34,7 @@ jobs:
 
     - name: Build Flutter to Web
       run: |
-        cd frontend
+        cd classic/frontend
         flutter build web --base-href /app/
 
     # - name: Commit and Push to ${{ env.BUILD_BRANCH }}
@@ -42,7 +42,7 @@ jobs:
     #   run: |
     #     git config --local user.email "action@github.com"
     #     git config --local user.name "GitHub Action"
-    #     git add frontend/build/web
+    #     git add classic/frontend/build/web
     #     git checkout -B ${{ env.BUILD_BRANCH }}
     #     git commit -m "Update frontend build to ${GITHUB_SHA:0:7}" -a
     #     git push -f origin ${{ env.BUILD_BRANCH }}
@@ -51,7 +51,7 @@ jobs:
       if: github.event_name == 'push'
       uses: peter-evans/create-pull-request@v6
       with:
-        add-paths: frontend/build/web
+        add-paths: classic/frontend/build/web
         base: ${{ github.ref_name }}
         branch: ${{ env.BUILD_BRANCH }}
         delete-branch: true

.github/workflows/classic-python-checks.yml

@@ -1,27 +1,27 @@
-name: Python checks
+name: Classic - Python checks
 
 on:
   push:
     branches: [ master, development, ci-test* ]
     paths:
-      - '.github/workflows/lint-ci.yml'
-      - 'autogpt/**'
-      - 'forge/**'
-      - 'benchmark/**'
+      - '.github/workflows/classic-python-checks-ci.yml'
+      - 'classic/original_autogpt/**'
+      - 'classic/forge/**'
+      - 'classic/benchmark/**'
       - '**.py'
-      - '!forge/tests/vcr_cassettes'
+      - '!classic/forge/tests/vcr_cassettes'
   pull_request:
     branches: [ master, development, release-* ]
     paths:
-      - '.github/workflows/lint-ci.yml'
-      - 'autogpt/**'
-      - 'forge/**'
-      - 'benchmark/**'
+      - '.github/workflows/classic-python-checks-ci.yml'
+      - 'classic/original_autogpt/**'
+      - 'classic/forge/**'
+      - 'classic/benchmark/**'
       - '**.py'
-      - '!forge/tests/vcr_cassettes'
+      - '!classic/forge/tests/vcr_cassettes'
 
 concurrency:
-  group: ${{ format('lint-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
+  group: ${{ format('classic-python-checks-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
   cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}
 
 defaults:
@@ -40,18 +40,18 @@ jobs:
         uses: dorny/paths-filter@v3
         with:
           filters: |
-            autogpt:
-              - autogpt/autogpt/**
-              - autogpt/tests/**
-              - autogpt/poetry.lock
+            original_autogpt:
+              - classic/original_autogpt/autogpt/**
+              - classic/original_autogpt/tests/**
+              - classic/original_autogpt/poetry.lock
             forge:
-              - forge/forge/**
-              - forge/tests/**
-              - forge/poetry.lock
+              - classic/forge/forge/**
+              - classic/forge/tests/**
+              - classic/forge/poetry.lock
             benchmark:
-              - benchmark/agbenchmark/**
-              - benchmark/tests/**
-              - benchmark/poetry.lock
+              - classic/benchmark/agbenchmark/**
+              - classic/benchmark/tests/**
+              - classic/benchmark/poetry.lock
     outputs:
       changed-parts: ${{ steps.changes-in.outputs.changes }}
 
@@ -89,23 +89,23 @@ jobs:
       # Install dependencies
 
       - name: Install Python dependencies
-        run: poetry -C ${{ matrix.sub-package }} install
+        run: poetry -C classic/${{ matrix.sub-package }} install
 
       # Lint
 
       - name: Lint (isort)
         run: poetry run isort --check .
-        working-directory: ${{ matrix.sub-package }}
+        working-directory: classic/${{ matrix.sub-package }}
 
       - name: Lint (Black)
         if: success() || failure()
         run: poetry run black --check .
-        working-directory: ${{ matrix.sub-package }}
+        working-directory: classic/${{ matrix.sub-package }}
 
       - name: Lint (Flake8)
         if: success() || failure()
         run: poetry run flake8 .
-        working-directory: ${{ matrix.sub-package }}
+        working-directory: classic/${{ matrix.sub-package }}
 
   types:
     needs: get-changed-parts
@@ -141,11 +141,11 @@ jobs:
       # Install dependencies
 
       - name: Install Python dependencies
-        run: poetry -C ${{ matrix.sub-package }} install
+        run: poetry -C classic/${{ matrix.sub-package }} install
 
       # Typecheck
 
       - name: Typecheck
         if: success() || failure()
         run: poetry run pyright
-        working-directory: ${{ matrix.sub-package }}
+        working-directory: classic/${{ matrix.sub-package }}

.github/workflows/hackathon.yml

@@ -1,133 +0,0 @@
-name: Hackathon
-
-on:
-  workflow_dispatch:
-    inputs:
-      agents:
-        description: "Agents to run (comma-separated)"
-        required: false
-        default: "autogpt" # Default agents if none are specified
-
-jobs:
-  matrix-setup:
-    runs-on: ubuntu-latest
-    # Service containers to run with `matrix-setup`
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-      env-name: ${{ steps.set-matrix.outputs.env-name }}
-    steps:
-      - id: set-matrix
-        run: |
-          if [ "${{ github.event_name }}" == "schedule" ]; then
-            echo "::set-output name=env-name::production"
-            echo "::set-output name=matrix::[ 'irrelevant']"
-          elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            IFS=',' read -ra matrix_array <<< "${{ github.event.inputs.agents }}"
-            matrix_string="[ \"$(echo "${matrix_array[@]}" | sed 's/ /", "/g')\" ]"
-            echo "::set-output name=env-name::production"
-            echo "::set-output name=matrix::$matrix_string"
-          else
-            echo "::set-output name=env-name::testing"
-            echo "::set-output name=matrix::[ 'irrelevant' ]"
-          fi
-
-  tests:
-    environment:
-      name: "${{ needs.matrix-setup.outputs.env-name }}"
-    needs: matrix-setup
-    env:
-      min-python-version: "3.10"
-    name: "${{ matrix.agent-name }}"
-    runs-on: ubuntu-latest
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-    timeout-minutes: 50
-    strategy:
-      fail-fast: false
-      matrix:
-        agent-name: ${{fromJson(needs.matrix-setup.outputs.matrix)}}
-    steps:
-      - name: Print Environment Name
-        run: |
-          echo "Matrix Setup Environment Name: ${{ needs.matrix-setup.outputs.env-name }}"
-
-      - name: Check Docker Container
-        id: check
-        run: docker ps
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-
-      - name: Set up Python ${{ env.min-python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.min-python-version }}
-
-      - id: get_date
-        name: Get date
-        run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
-
-      - name: Install Poetry
-        run: |
-          curl -sSL https://install.python-poetry.org | python -
-
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: v18.15
-
-      - name: Run benchmark
-        run: |
-          link=$(jq -r '.["github_repo_url"]' arena/$AGENT_NAME.json)
-          branch=$(jq -r '.["branch_to_benchmark"]' arena/$AGENT_NAME.json)
-          git clone "$link" -b "$branch" "$AGENT_NAME"
-          cd $AGENT_NAME
-          cp ./$AGENT_NAME/.env.example ./$AGENT_NAME/.env || echo "file not found"
-          ./run agent start $AGENT_NAME
-          cd ../benchmark
-          poetry install
-          poetry run agbenchmark --no-dep
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          SERP_API_KEY: ${{ secrets.SERP_API_KEY }}
-          SERPAPI_API_KEY: ${{ secrets.SERP_API_KEY }}
-          WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }}
-          WEAVIATE_URL: ${{ secrets.WEAVIATE_URL }}
-          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
-          GOOGLE_CUSTOM_SEARCH_ENGINE_ID: ${{ secrets.GOOGLE_CUSTOM_SEARCH_ENGINE_ID }}
-          AGENT_NAME: ${{ matrix.agent-name }}

.github/workflows/platform-autogpt-docker-ci.yml

@@ -0,0 +1,40 @@
+name: AutoGPT Server Docker Build & Push
+
+on:
+  push:
+    branches: [ update-docker-ci ]
+    paths:
+      - '**'
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  PROJECT_ID: agpt-dev
+  IMAGE_NAME: agpt-server-dev
+  REGION: us-central1
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0.2.1
+        with:
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Configure Docker
+        run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev
+
+      - name: Build Docker image
+        run: docker build -t ${{ env.REGION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.IMAGE_NAME }}:${{ github.sha }} -f autogpt_platform/backend/Dockerfile .
+
+      - name: Push Docker image
+        run: docker push ${{ env.REGION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.IMAGE_NAME }}:${{ github.sha }}

.github/workflows/platform-autogpt-infra-ci.yml

@@ -1,20 +1,20 @@
-name: AutoGPT Builder Infra
+name: AutoGPT Platform - Infra
 
 on:
   push:
     branches: [ master ]
     paths:
-      - '.github/workflows/autogpt-infra-ci.yml'
-      - 'rnd/infra/**'
+      - '.github/workflows/platform-autogpt-infra-ci.yml'
+      - 'autogpt_platform/infra/**'
   pull_request:
     paths:
-      - '.github/workflows/autogpt-infra-ci.yml'
-      - 'rnd/infra/**'
+      - '.github/workflows/platform-autogpt-infra-ci.yml'
+      - 'autogpt_platform/infra/**'
 
 defaults:
   run:
     shell: bash
-    working-directory: rnd/infra
+    working-directory: autogpt_platform/infra
 
 jobs:
   lint:
@@ -53,4 +53,4 @@ jobs:
 
       - name: Run chart-testing (lint)
         if: steps.list-changed.outputs.changed == 'true'
-        run: ct lint --target-branch ${{ github.event.repository.default_branch }}
+        run: ct lint --target-branch ${{ github.event.repository.default_branch }}

.github/workflows/platform-backend-ci.yml

@@ -1,25 +1,25 @@
-name: AutoGPT Server CI
+name: AutoGPT Platform - Backend CI
 
 on:
   push:
     branches: [master, development, ci-test*]
     paths:
-      - ".github/workflows/autogpt-server-ci.yml"
-      - "rnd/autogpt_server/**"
+      - ".github/workflows/platform-backend-ci.yml"
+      - "autogpt_platform/backend/**"
   pull_request:
     branches: [master, development, release-*]
     paths:
-      - ".github/workflows/autogpt-server-ci.yml"
-      - "rnd/autogpt_server/**"
+      - ".github/workflows/platform-backend-ci.yml"
+      - "autogpt_platform/backend/**"
 
 concurrency:
-  group: ${{ format('autogpt-server-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
+  group: ${{ format('backend-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }}
   cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }}
 
 defaults:
   run:
     shell: bash
-    working-directory: rnd/autogpt_server
+    working-directory: autogpt_platform/backend
 
 jobs:
   test:
@@ -90,7 +90,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }}
-          key: poetry-${{ runner.os }}-${{ hashFiles('rnd/autogpt_server/poetry.lock') }}
+          key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }}
 
       - name: Install Poetry (Unix)
         if: runner.os != 'Windows'
@@ -152,4 +152,4 @@ jobs:
       #   uses: codecov/codecov-action@v4
       #   with:
       #     token: ${{ secrets.CODECOV_TOKEN }}
-      #     flags: autogpt-server,${{ runner.os }}
+      #     flags: backend,${{ runner.os }}

.github/workflows/platform-frontend-ci.yml

@@ -1,20 +1,20 @@
-name: AutoGPT Builder CI
+name: AutoGPT Platform - Frontend CI
 
 on:
   push:
     branches: [ master ]
     paths:
-      - '.github/workflows/autogpt-builder-ci.yml'
-      - 'rnd/autogpt_builder/**'
+      - '.github/workflows/platform-frontend-ci.yml'
+      - 'autogpt_platform/frontend/**'
   pull_request:
     paths:
-      - '.github/workflows/autogpt-builder-ci.yml'
-      - 'rnd/autogpt_builder/**'
+      - '.github/workflows/platform-frontend-ci.yml'
+      - 'autogpt_platform/frontend/**'
 
 defaults:
   run:
     shell: bash
-    working-directory: rnd/autogpt_builder
+    working-directory: autogpt_platform/frontend
 
 jobs:
 

.github/workflows/repo-close-stale-issues.yml

@@ -1,4 +1,4 @@
-name: 'Close stale issues'
+name: Repo - Close stale issues
 on:
   schedule:
     - cron: '30 1 * * *'

.github/workflows/repo-pr-label.yml

@@ -1,12 +1,12 @@
-name: "Pull Request auto-label"
+name: Repo - Pull Request auto-label
 
 on:
   # So that PRs touching the same files as the push are updated
   push:
     branches: [ master, development, release-* ]
     paths-ignore:
-      - 'forge/tests/vcr_cassettes'
-      - 'benchmark/reports/**'
+      - 'classic/forge/tests/vcr_cassettes'
+      - 'classic/benchmark/reports/**'
   # So that the `dirtyLabel` is removed if conflicts are resolve
   # We recommend `pull_request_target` so that github secrets are available.
   # In `pull_request` we wouldn't be able to change labels of fork PRs

.github/workflows/repo-stats.yml

@@ -1,4 +1,4 @@
-name: github-repo-stats
+name: Repo - Github Stats
 
 on:
   schedule:

.github/workflows/repo-workflow-checker.yml

@@ -1,4 +1,4 @@
-name: PR Status Checker
+name: Repo - PR Status Checker
 on:
   pull_request:
     types: [opened, synchronize, reopened]
@@ -26,6 +26,6 @@ jobs:
           echo "Current directory before running Python script:"
           pwd
           echo "Attempting to run Python script:"
-          python check_actions_status.py
+          python .github/workflows/scripts/check_actions_status.py
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,7 +1,7 @@
 ## Original ignores
 .github_access_token
-autogpt/keys.py
-autogpt/*.json
+classic/original_autogpt/keys.py
+classic/original_autogpt/*.json
 auto_gpt_workspace/*
 *.mpeg
 .env
@@ -157,7 +157,7 @@ openai/
 CURRENT_BULLETIN.md
 
 # AgBenchmark
-agbenchmark/reports/
+classic/benchmark/agbenchmark/reports/
 
 # Nodejs
 package-lock.json
@@ -170,4 +170,4 @@ pri*
 ig*
 .github_access_token
 LICENSE.rtf
-rnd/autogpt_server/settings.py
+autogpt_platform/backend/settings.py

.gitmodules

@@ -1,3 +1,6 @@
-[submodule "forge/tests/vcr_cassettes"]
-	path = forge/tests/vcr_cassettes
+[submodule "classic/forge/tests/vcr_cassettes"]
+	path = classic/forge/tests/vcr_cassettes
 	url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes
+[submodule "autogpt_platform/supabase"]
+	path = autogpt_platform/supabase
+	url = https://github.com/supabase/supabase.git

.pre-commit-config.yaml

@@ -16,22 +16,22 @@ repos:
     hooks:
       - id: isort-autogpt
         name: Lint (isort) - AutoGPT
-        entry: poetry -C autogpt run isort
-        files: ^autogpt/
+        entry: poetry -C classic/original_autogpt run isort
+        files: ^classic/original_autogpt/
         types: [file, python]
         language: system
 
       - id: isort-forge
         name: Lint (isort) - Forge
-        entry: poetry -C forge run isort
-        files: ^forge/
+        entry: poetry -C classic/forge run isort
+        files: ^classic/forge/
         types: [file, python]
         language: system
 
       - id: isort-benchmark
         name: Lint (isort) - Benchmark
-        entry: poetry -C benchmark run isort
-        files: ^benchmark/
+        entry: poetry -C classic/benchmark run isort
+        files: ^classic/benchmark/
         types: [file, python]
         language: system
 
@@ -52,20 +52,20 @@ repos:
       - id: flake8
         name: Lint (Flake8) - AutoGPT
         alias: flake8-autogpt
-        files: ^autogpt/(autogpt|scripts|tests)/
-        args: [--config=autogpt/.flake8]
+        files: ^classic/original_autogpt/(autogpt|scripts|tests)/
+        args: [--config=classic/original_autogpt/.flake8]
 
       - id: flake8
         name: Lint (Flake8) - Forge
         alias: flake8-forge
-        files: ^forge/(forge|tests)/
-        args: [--config=forge/.flake8]
+        files: ^classic/forge/(forge|tests)/
+        args: [--config=classic/forge/.flake8]
 
       - id: flake8
         name: Lint (Flake8) - Benchmark
         alias: flake8-benchmark
-        files: ^benchmark/(agbenchmark|tests)/((?!reports).)*[/.]
-        args: [--config=benchmark/.flake8]
+        files: ^classic/benchmark/(agbenchmark|tests)/((?!reports).)*[/.]
+        args: [--config=classic/benchmark/.flake8]
 
   - repo: local
     # To have watertight type checking, we check *all* the files in an affected
@@ -74,10 +74,10 @@ repos:
       - id: pyright
         name: Typecheck - AutoGPT
         alias: pyright-autogpt
-        entry: poetry -C autogpt run pyright
+        entry: poetry -C classic/original_autogpt run pyright
         args: [-p, autogpt, autogpt]
         # include forge source (since it's a path dependency) but exclude *_test.py files:
-        files: ^(autogpt/((autogpt|scripts|tests)/|poetry\.lock$)|forge/(forge/.*(?<!_test)\.py|poetry\.lock)$)
+        files: ^(classic/original_autogpt/((autogpt|scripts|tests)/|poetry\.lock$)|classic/forge/(classic/forge/.*(?<!_test)\.py|poetry\.lock)$)
         types: [file]
         language: system
         pass_filenames: false
@@ -85,9 +85,9 @@ repos:
       - id: pyright
         name: Typecheck - Forge
         alias: pyright-forge
-        entry: poetry -C forge run pyright
+        entry: poetry -C classic/forge run pyright
         args: [-p, forge, forge]
-        files: ^forge/(forge/|poetry\.lock$)
+        files: ^classic/forge/(classic/forge/|poetry\.lock$)
         types: [file]
         language: system
         pass_filenames: false
@@ -95,9 +95,9 @@ repos:
       - id: pyright
         name: Typecheck - Benchmark
         alias: pyright-benchmark
-        entry: poetry -C benchmark run pyright
+        entry: poetry -C classic/benchmark run pyright
         args: [-p, benchmark, benchmark]
-        files: ^benchmark/(agbenchmark/|tests/|poetry\.lock$)
+        files: ^classic/benchmark/(agbenchmark/|tests/|poetry\.lock$)
         types: [file]
         language: system
         pass_filenames: false
@@ -106,22 +106,22 @@ repos:
     hooks:
       - id: pytest-autogpt
         name: Run tests - AutoGPT (excl. slow tests)
-        entry: bash -c 'cd autogpt && poetry run pytest --cov=autogpt -m "not slow" tests/unit tests/integration'
+        entry: bash -c 'cd classic/original_autogpt && poetry run pytest --cov=autogpt -m "not slow" tests/unit tests/integration'
         # include forge source (since it's a path dependency) but exclude *_test.py files:
-        files: ^(autogpt/((autogpt|tests)/|poetry\.lock$)|forge/(forge/.*(?<!_test)\.py|poetry\.lock)$)
+        files: ^(classic/original_autogpt/((autogpt|tests)/|poetry\.lock$)|classic/forge/(classic/forge/.*(?<!_test)\.py|poetry\.lock)$)
         language: system
         pass_filenames: false
 
       - id: pytest-forge
         name: Run tests - Forge (excl. slow tests)
-        entry: bash -c 'cd forge && poetry run pytest --cov=forge -m "not slow"'
-        files: ^forge/(forge/|tests/|poetry\.lock$)
+        entry: bash -c 'cd classic/forge && poetry run pytest --cov=forge -m "not slow"'
+        files: ^classic/forge/(classic/forge/|tests/|poetry\.lock$)
         language: system
         pass_filenames: false
 
       - id: pytest-benchmark
         name: Run tests - Benchmark
-        entry: bash -c 'cd benchmark && poetry run pytest --cov=benchmark'
-        files: ^benchmark/(agbenchmark/|tests/|poetry\.lock$)
+        entry: bash -c 'cd classic/benchmark && poetry run pytest --cov=benchmark'
+        files: ^classic/benchmark/(agbenchmark/|tests/|poetry\.lock$)
         language: system
         pass_filenames: false

.vscode/all-projects.code-workspace

@@ -1,49 +1,49 @@
 {
   "folders": [
     {
-      "name": "autogpt",
-      "path": "../autogpt"
+      "name": "autogpt_server",
+      "path": "../autogpt_platform/autogpt_server"
     },
     {
-      "name": "benchmark",
-      "path": "../benchmark"
+      "name": "autogpt_builder",
+      "path": "../autogpt_platform/autogpt_builder"
+    },
+    {
+      "name": "market",
+      "path": "../autogpt_platform/market"
+    },
+    {
+      "name": "lib",
+      "path": "../autogpt_platform/autogpt_libs"
+    },
+    {
+      "name": "infra",
+      "path": "../autogpt_platform/infra"
     },
     {
       "name": "docs",
       "path": "../docs"
     },
-    {
-      "name": "forge",
-      "path": "../forge"
-    },
-    {
-      "name": "frontend",
-      "path": "../frontend"
-    },
-    {
-      "name": "autogpt_server",
-      "path": "../rnd/autogpt_server"
-    },
-    {
-      "name": "autogpt_builder",
-      "path": "../rnd/autogpt_builder"
-    },
-    {
-      "name": "market",
-      "path": "../rnd/market"
-    },
-    {
-      "name": "lib",
-      "path": "../rnd/autogpt_libs"
-    },
-    {
-      "name": "infra",
-      "path": "../rnd/infra"
-    },
     {
       "name": "[root]",
       "path": ".."
-    }
+    },
+    {
+      "name": "classic - autogpt",
+      "path": "../classic/original_autogpt"
+    },
+    {
+      "name": "classic - benchmark",
+      "path": "../classic/benchmark"
+    },
+    {
+      "name": "classic - forge",
+      "path": "../classic/forge"
+    },
+    {
+      "name": "classic - frontend",
+      "path": "../classic/frontend"
+    },
   ],
   "settings": {
     "python.analysis.typeCheckingMode": "basic"

README.md

@@ -55,15 +55,16 @@ Be part of the revolution! **AutoGPT** is here to stay, at the forefront of AI i
 ## 🤖 AutoGPT Classic
 > Below is information about the classic version of AutoGPT.
 
-**🛠️ [Build your own Agent - Quickstart](FORGE-QUICKSTART.md)**
+**🛠️ [Build your own Agent - Quickstart](classic/FORGE-QUICKSTART.md)**
+
 ### 🏗️ Forge
 
-**Forge your own agent!** – Forge is a ready-to-go template for your agent application. All the boilerplate code is already handled, letting you channel all your creativity into the things that set *your* agent apart. All tutorials are located [here](https://medium.com/@aiedge/autogpt-forge-e3de53cc58ec). Components from the [`forge.sdk`](/forge/forge/sdk) can also be used individually to speed up development and reduce boilerplate in your agent project.
+**Forge your own agent!** – Forge is a ready-to-go toolkit to build your own agent application. It handles most of the boilerplate code, letting you channel all your creativity into the things that set *your* agent apart. All tutorials are located [here](https://medium.com/@aiedge/autogpt-forge-e3de53cc58ec). Components from [`forge`](/classic/forge/) can also be used individually to speed up development and reduce boilerplate in your agent project.
 
-🚀 [**Getting Started with Forge**](https://github.com/Significant-Gravitas/AutoGPT/blob/master/forge/tutorials/001_getting_started.md) –
+🚀 [**Getting Started with Forge**](https://github.com/Significant-Gravitas/AutoGPT/blob/master/classic/forge/tutorials/001_getting_started.md) –
 This guide will walk you through the process of creating your own agent and using the benchmark and user interface.
 
-📘 [Learn More](https://github.com/Significant-Gravitas/AutoGPT/tree/master/forge) about Forge
+📘 [Learn More](https://github.com/Significant-Gravitas/AutoGPT/tree/master/classic/forge) about Forge
 
 ### 🎯 Benchmark
 
@@ -83,7 +84,7 @@ This guide will walk you through the process of creating your own agent and usin
 
 The frontend works out-of-the-box with all agents in the repo. Just use the [CLI] to run your agent of choice!
 
-📘 [Learn More](https://github.com/Significant-Gravitas/AutoGPT/tree/master/frontend) about the Frontend
+📘 [Learn More](https://github.com/Significant-Gravitas/AutoGPT/tree/master/classic/frontend) about the Frontend
 
 ### ⌨️ CLI
 

autogpt/.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-  "python.analysis.typeCheckingMode": "basic",
-}

autogpt_platform/README.md

@@ -14,21 +14,40 @@ Welcome to the AutoGPT Platform - a powerful system for creating and running AI
 To run the AutoGPT Platform, follow these steps:
 
 1. Clone this repository to your local machine.
-2. Navigate to the project directory.
+2. Navigate to autogpt_platform/supabase
 3. Run the following command:
+   ```
+      git submodule update --init --recursive
+   ```
+4. Navigate back to autogpt_platform (cd ..)
+5. Run the following command:
+   ```
+      cp supabase/docker/.env.example .env
+   ```
+6. Run the following command:
 
    ```
-   docker compose up -d
+   docker compose -f docker-compose.combined.yml up -d
+
    ```
 
-   This command will start all the necessary services defined in the `docker-compose.yml` file in detached mode.
+   This command will start all the necessary backend services defined in the `docker-compose.combined.yml` file in detached mode.
+7. Navigate to autogpt_platform/frontend.
+8. Run the following command: 
+   ```
+      cp .env.example .env.local
+   ```
+9. Run the following command: 
+   ```
+      yarn dev
+   ```
 
 ### Docker Compose Commands
 
 Here are some useful Docker Compose commands for managing your AutoGPT Platform:
 
-- `docker compose up -d`: Start the services in detached mode.
-- `docker compose stop`: Stop the running services without removing them.
+- `docker compose -f docker-compose.combined.yml up -d`: Start the services in detached mode.
+- `docker compose -f docker-compose.combined.yml stop`: Stop the running services without removing them.
 - `docker compose rm`: Remove stopped service containers.
 - `docker compose build`: Build or rebuild services.
 - `docker compose down`: Stop and remove containers, networks, and volumes.

autogpt_platform/autogpt_libs/autogpt_libs/supabase_integration_credentials_store/types.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field, SecretStr, field_serializer
 class _BaseCredentials(BaseModel):
     id: str = Field(default_factory=lambda: str(uuid4()))
     provider: str
-    title: str
+    title: Optional[str]
 
     @field_serializer("*")
     def dump_secret_strings(value: Any, _info):
@@ -18,6 +18,8 @@ class _BaseCredentials(BaseModel):
 
 class OAuth2Credentials(_BaseCredentials):
     type: Literal["oauth2"] = "oauth2"
+    username: Optional[str]
+    """Username of the third-party service user that these credentials belong to"""
     access_token: SecretStr
     access_token_expires_at: Optional[int]
     """Unix timestamp (seconds) indicating when the access token expires (if at all)"""

autogpt_platform/backend/.env.example

@@ -1,7 +1,7 @@
 DB_USER=agpt_user
 DB_PASS=pass123
 DB_NAME=agpt_local
-DB_PORT=5432
+DB_PORT=5433
 DATABASE_URL="postgresql://${DB_USER}:${DB_PASS}@localhost:${DB_PORT}/${DB_NAME}"
 PRISMA_SCHEMA="postgres/schema.prisma"
 
@@ -9,10 +9,13 @@ REDIS_HOST=localhost
 REDIS_PORT=6379
 REDIS_PASSWORD=password
 
-AUTH_ENABLED=false
+ENABLE_AUTH=false
+ENABLE_CREDIT=false
 APP_ENV="local"
 PYRO_HOST=localhost
 SENTRY_DSN=
+# This is needed when ENABLE_AUTH is true
+SUPABASE_JWT_SECRET=
 
 ## ===== OPTIONAL API KEYS ===== ##
 

autogpt_platform/backend/Dockerfile

@@ -17,17 +17,21 @@ ENV POETRY_VERSION=1.8.3 \
     POETRY_NO_INTERACTION=1 \
     POETRY_VIRTUALENVS_CREATE=false \
     PATH="$POETRY_HOME/bin:$PATH"
+
+# Upgrade pip and setuptools to fix security vulnerabilities
+RUN pip3 install --upgrade pip setuptools
+
 RUN pip3 install poetry
 
 # Copy and install dependencies
-COPY rnd/autogpt_libs /app/rnd/autogpt_libs
-COPY rnd/autogpt_server/poetry.lock rnd/autogpt_server/pyproject.toml /app/rnd/autogpt_server/
-WORKDIR /app/rnd/autogpt_server
+COPY autogpt_platform/autogpt_libs /app/autogpt_platform/autogpt_libs
+COPY autogpt_platform/backend/poetry.lock autogpt_platform/backend/pyproject.toml /app/autogpt_platform/backend/
+WORKDIR /app/autogpt_platform/backend
 RUN poetry config virtualenvs.create false \
     && poetry install --no-interaction --no-ansi
 
 # Generate Prisma client
-COPY rnd/autogpt_server/schema.prisma ./
+COPY autogpt_platform/backend/schema.prisma ./
 RUN poetry config virtualenvs.create false \
     && poetry run prisma generate
 
@@ -41,6 +45,10 @@ ENV POETRY_VERSION=1.8.3 \
     POETRY_VIRTUALENVS_CREATE=false \
     PATH="$POETRY_HOME/bin:$PATH"
 
+
+# Upgrade pip and setuptools to fix security vulnerabilities
+RUN pip3 install --upgrade pip setuptools
+
 # Copy only necessary files from builder
 COPY --from=builder /app /app
 COPY --from=builder /usr/local/lib/python3.11 /usr/local/lib/python3.11
@@ -51,21 +59,20 @@ COPY --from=builder /root/.cache/prisma-python/binaries /root/.cache/prisma-pyth
 
 ENV PATH="/app/.venv/bin:$PATH"
 
-RUN mkdir -p /app/rnd/autogpt_libs
-RUN mkdir -p /app/rnd/autogpt_server
+RUN mkdir -p /app/autogpt_platform/autogpt_libs
+RUN mkdir -p /app/autogpt_platform/backend
 
-COPY rnd/autogpt_libs /app/rnd/autogpt_libs
+COPY autogpt_platform/autogpt_libs /app/autogpt_platform/autogpt_libs
 
-COPY rnd/autogpt_server/poetry.lock rnd/autogpt_server/pyproject.toml /app/rnd/autogpt_server/
+COPY autogpt_platform/backend/poetry.lock autogpt_platform/backend/pyproject.toml /app/autogpt_platform/backend/
 
-WORKDIR /app/rnd/autogpt_server
+WORKDIR /app/autogpt_platform/backend
 
 FROM server_dependencies AS server
 
-COPY rnd/autogpt_server /app/rnd/autogpt_server
+COPY autogpt_platform/backend /app/autogpt_platform/backend
 
 ENV DATABASE_URL=""
 ENV PORT=8000
 
 CMD ["poetry", "run", "rest"]
-

autogpt_platform/backend/README.advanced.md

@@ -48,19 +48,19 @@ We use the Poetry to manage the dependencies. To set up the project, follow thes
    > ```
    >
    > Then run the generation again. The path *should* look something like this:  
-   > `<some path>/pypoetry/virtualenvs/autogpt-server-TQIRSwR6-py3.12/bin/prisma`
+   > `<some path>/pypoetry/virtualenvs/backend-TQIRSwR6-py3.12/bin/prisma`
 
 6. Run the postgres database from the /rnd folder
 
    ```sh
-   cd rnd/
+   cd autogpt_platform/
    docker compose up -d
    ```
 
-7. Run the migrations (from the autogpt_server folder)
+7. Run the migrations (from the backend folder)
 
    ```sh
-   cd ../autogpt_server
+   cd ../backend
    prisma migrate dev --schema postgres/schema.prisma
    ```
 

autogpt_platform/backend/README.md

@@ -53,7 +53,7 @@ We use the Poetry to manage the dependencies. To set up the project, follow thes
    > ```
    >
    > Then run the generation again. The path *should* look something like this:  
-   > `<some path>/pypoetry/virtualenvs/autogpt-server-TQIRSwR6-py3.12/bin/prisma`
+   > `<some path>/pypoetry/virtualenvs/backend-TQIRSwR6-py3.12/bin/prisma`
 
 6. Migrate the database. Be careful because this deletes current data in the database.
 
@@ -193,7 +193,7 @@ Rest Server Daemon: 8004
 ## Adding a New Agent Block
 
 To add a new agent block, you need to create a new class that inherits from `Block` and provides the following information:
-* All the block code should live in the `blocks` (`autogpt_server.blocks`) module.
+* All the block code should live in the `blocks` (`backend.blocks`) module.
 * `input_schema`: the schema of the input data, represented by a Pydantic object.
 * `output_schema`: the schema of the output data, represented by a Pydantic object.
 * `run` method: the main logic of the block.

autogpt_platform/backend/backend/app.py

@@ -1,7 +1,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from autogpt_server.util.process import AppProcess
+    from backend.util.process import AppProcess
 
 
 def run_processes(*processes: "AppProcess", **kwargs):
@@ -24,8 +24,8 @@ def main(**kwargs):
     Run all the processes required for the AutoGPT-server (REST and WebSocket APIs).
     """
 
-    from autogpt_server.executor import ExecutionManager, ExecutionScheduler
-    from autogpt_server.server import AgentServer, WebsocketServer
+    from backend.executor import ExecutionManager, ExecutionScheduler
+    from backend.server import AgentServer, WebsocketServer
 
     run_processes(
         ExecutionManager(),

autogpt_platform/backend/backend/blocks/__init__.py

@@ -4,9 +4,9 @@ import os
 import re
 from pathlib import Path
 
-from autogpt_server.data.block import Block
+from backend.data.block import Block
 
-# Dynamically load all modules under autogpt_server.blocks
+# Dynamically load all modules under backend.blocks
 AVAILABLE_MODULES = []
 current_dir = os.path.dirname(__file__)
 modules = glob.glob(os.path.join(current_dir, "*.py"))

autogpt_platform/backend/backend/blocks/basic.py

@@ -4,15 +4,15 @@ from typing import Any, List
 from jinja2 import BaseLoader, Environment
 from pydantic import Field
 
-from autogpt_server.data.block import (
+from backend.data.block import (
     Block,
     BlockCategory,
     BlockOutput,
     BlockSchema,
     BlockUIType,
 )
-from autogpt_server.data.model import SchemaField
-from autogpt_server.util.mock import MockObject
+from backend.data.model import SchemaField
+from backend.util.mock import MockObject
 
 jinja = Environment(loader=BaseLoader())
 
@@ -85,7 +85,6 @@ class PrintToConsoleBlock(Block):
 
 
 class FindInDictionaryBlock(Block):
-
     class Input(BlockSchema):
         input: Any = Field(description="Dictionary to lookup from")
         key: str | int = Field(description="Key to lookup in the dictionary")

autogpt_platform/backend/backend/blocks/block.py

@@ -2,7 +2,7 @@ import os
 import re
 from typing import Type
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 
 
 class BlockInstallationBlock(Block):
@@ -48,7 +48,7 @@ class BlockInstallationBlock(Block):
 
         block_dir = os.path.dirname(__file__)
         file_path = f"{block_dir}/{file_name}.py"
-        module_name = f"autogpt_server.blocks.{file_name}"
+        module_name = f"backend.blocks.{file_name}"
         with open(file_path, "w") as f:
             f.write(code)
 
@@ -57,7 +57,7 @@ class BlockInstallationBlock(Block):
             block_class: Type[Block] = getattr(module, class_name)
             block = block_class()
 
-            from autogpt_server.util.test import execute_block_test
+            from backend.util.test import execute_block_test
 
             execute_block_test(block)
             yield "success", "Block installed successfully."

autogpt_platform/backend/backend/blocks/branching.py

@@ -1,8 +1,8 @@
 from enum import Enum
 from typing import Any
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import SchemaField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
 
 
 class ComparisonOperator(Enum):

autogpt_platform/backend/backend/blocks/csv.py

@@ -1,5 +1,5 @@
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import ContributorDetails
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import ContributorDetails
 
 
 class ReadCsvBlock(Block):
@@ -14,7 +14,8 @@ class ReadCsvBlock(Block):
         skip_columns: list[str] = []
 
     class Output(BlockSchema):
-        data: dict[str, str]
+        row: dict[str, str]
+        all_data: list[dict[str, str]]
 
     def __init__(self):
         super().__init__(
@@ -27,8 +28,15 @@ class ReadCsvBlock(Block):
                 "contents": "a, b, c\n1,2,3\n4,5,6",
             },
             test_output=[
-                ("data", {"a": "1", "b": "2", "c": "3"}),
-                ("data", {"a": "4", "b": "5", "c": "6"}),
+                ("row", {"a": "1", "b": "2", "c": "3"}),
+                ("row", {"a": "4", "b": "5", "c": "6"}),
+                (
+                    "all_data",
+                    [
+                        {"a": "1", "b": "2", "c": "3"},
+                        {"a": "4", "b": "5", "c": "6"},
+                    ],
+                ),
             ],
         )
 
@@ -53,8 +61,7 @@ class ReadCsvBlock(Block):
         for _ in range(input_data.skip_rows):
             next(reader)
 
-        # join the data with the header
-        for row in reader:
+        def process_row(row):
             data = {}
             for i, value in enumerate(row):
                 if i not in input_data.skip_columns:
@@ -62,4 +69,12 @@ class ReadCsvBlock(Block):
                         data[header[i]] = value.strip() if input_data.strip else value
                     else:
                         data[str(i)] = value.strip() if input_data.strip else value
-            yield "data", data
+            return data
+
+        all_data = []
+        for row in reader:
+            processed_row = process_row(row)
+            all_data.append(processed_row)
+            yield "row", processed_row
+
+        yield "all_data", all_data

autogpt_platform/backend/backend/blocks/discord.py

@@ -4,8 +4,8 @@ import aiohttp
 import discord
 from pydantic import Field
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import BlockSecret, SecretField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import BlockSecret, SecretField
 
 
 class ReadDiscordMessagesBlock(Block):

autogpt_platform/backend/backend/blocks/email_block.py

@@ -4,8 +4,8 @@ from email.mime.text import MIMEText
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import BlockSecret, SchemaField, SecretField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import BlockSecret, SchemaField, SecretField
 
 
 class EmailCredentials(BaseModel):

autogpt_platform/backend/backend/blocks/http.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import requests
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 
 
 class HttpMethod(Enum):

autogpt_platform/backend/backend/blocks/iteration.py

@@ -1,7 +1,7 @@
 from typing import Any, List, Tuple
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import SchemaField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
 
 
 class ListIteratorBlock(Block):

autogpt_platform/backend/backend/blocks/llm.py

@@ -1,15 +1,16 @@
 import logging
 from enum import Enum
-from typing import List, NamedTuple
+from json import JSONDecodeError
+from typing import Any, List, NamedTuple
 
 import anthropic
 import ollama
 import openai
 from groq import Groq
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import BlockSecret, SchemaField, SecretField
-from autogpt_server.util import json
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import BlockSecret, SchemaField, SecretField
+from backend.util import json
 
 logger = logging.getLogger(__name__)
 
@@ -24,6 +25,7 @@ LlmApiKeys = {
 class ModelMetadata(NamedTuple):
     provider: str
     context_window: int
+    cost_factor: int
 
 
 class LlmModel(str, Enum):
@@ -55,26 +57,29 @@ class LlmModel(str, Enum):
 
 
 MODEL_METADATA = {
-    LlmModel.GPT4O_MINI: ModelMetadata("openai", 128000),
-    LlmModel.GPT4O: ModelMetadata("openai", 128000),
-    LlmModel.GPT4_TURBO: ModelMetadata("openai", 128000),
-    LlmModel.GPT3_5_TURBO: ModelMetadata("openai", 16385),
-    LlmModel.CLAUDE_3_5_SONNET: ModelMetadata("anthropic", 200000),
-    LlmModel.CLAUDE_3_HAIKU: ModelMetadata("anthropic", 200000),
-    LlmModel.LLAMA3_8B: ModelMetadata("groq", 8192),
-    LlmModel.LLAMA3_70B: ModelMetadata("groq", 8192),
-    LlmModel.MIXTRAL_8X7B: ModelMetadata("groq", 32768),
-    LlmModel.GEMMA_7B: ModelMetadata("groq", 8192),
-    LlmModel.GEMMA2_9B: ModelMetadata("groq", 8192),
-    LlmModel.LLAMA3_1_405B: ModelMetadata(
-        "groq", 8192
-    ),  # Limited to 16k during preview
-    LlmModel.LLAMA3_1_70B: ModelMetadata("groq", 131072),
-    LlmModel.LLAMA3_1_8B: ModelMetadata("groq", 131072),
-    LlmModel.OLLAMA_LLAMA3_8B: ModelMetadata("ollama", 8192),
-    LlmModel.OLLAMA_LLAMA3_405B: ModelMetadata("ollama", 8192),
+    LlmModel.GPT4O_MINI: ModelMetadata("openai", 128000, cost_factor=10),
+    LlmModel.GPT4O: ModelMetadata("openai", 128000, cost_factor=12),
+    LlmModel.GPT4_TURBO: ModelMetadata("openai", 128000, cost_factor=11),
+    LlmModel.GPT3_5_TURBO: ModelMetadata("openai", 16385, cost_factor=8),
+    LlmModel.CLAUDE_3_5_SONNET: ModelMetadata("anthropic", 200000, cost_factor=14),
+    LlmModel.CLAUDE_3_HAIKU: ModelMetadata("anthropic", 200000, cost_factor=13),
+    LlmModel.LLAMA3_8B: ModelMetadata("groq", 8192, cost_factor=6),
+    LlmModel.LLAMA3_70B: ModelMetadata("groq", 8192, cost_factor=9),
+    LlmModel.MIXTRAL_8X7B: ModelMetadata("groq", 32768, cost_factor=7),
+    LlmModel.GEMMA_7B: ModelMetadata("groq", 8192, cost_factor=6),
+    LlmModel.GEMMA2_9B: ModelMetadata("groq", 8192, cost_factor=7),
+    LlmModel.LLAMA3_1_405B: ModelMetadata("groq", 8192, cost_factor=10),
+    # Limited to 16k during preview
+    LlmModel.LLAMA3_1_70B: ModelMetadata("groq", 131072, cost_factor=15),
+    LlmModel.LLAMA3_1_8B: ModelMetadata("groq", 131072, cost_factor=13),
+    LlmModel.OLLAMA_LLAMA3_8B: ModelMetadata("ollama", 8192, cost_factor=7),
+    LlmModel.OLLAMA_LLAMA3_405B: ModelMetadata("ollama", 8192, cost_factor=11),
 }
 
+for model in LlmModel:
+    if model not in MODEL_METADATA:
+        raise ValueError(f"Missing MODEL_METADATA metadata for model: {model}")
+
 
 class AIStructuredResponseGeneratorBlock(Block):
     class Input(BlockSchema):
@@ -89,7 +94,7 @@ class AIStructuredResponseGeneratorBlock(Block):
         )
 
     class Output(BlockSchema):
-        response: dict[str, str]
+        response: dict[str, Any]
         error: str
 
     def __init__(self):
@@ -135,16 +140,33 @@ class AIStructuredResponseGeneratorBlock(Block):
             )
             return response.choices[0].message.content or ""
         elif provider == "anthropic":
-            sysprompt = "".join([p["content"] for p in prompt if p["role"] == "system"])
-            usrprompt = [p for p in prompt if p["role"] == "user"]
+            system_messages = [p["content"] for p in prompt if p["role"] == "system"]
+            sysprompt = " ".join(system_messages)
+
+            messages = []
+            last_role = None
+            for p in prompt:
+                if p["role"] in ["user", "assistant"]:
+                    if p["role"] != last_role:
+                        messages.append({"role": p["role"], "content": p["content"]})
+                        last_role = p["role"]
+                    else:
+                        # If the role is the same as the last one, combine the content
+                        messages[-1]["content"] += "\n" + p["content"]
+
             client = anthropic.Anthropic(api_key=api_key)
-            response = client.messages.create(
-                model=model.value,
-                max_tokens=4096,
-                system=sysprompt,
-                messages=usrprompt,  # type: ignore
-            )
-            return response.content[0].text if response.content else ""
+            try:
+                response = client.messages.create(
+                    model=model.value,
+                    max_tokens=4096,
+                    system=sysprompt,
+                    messages=messages,
+                )
+                return response.content[0].text if response.content else ""
+            except anthropic.APIError as e:
+                error_message = f"Anthropic API error: {str(e)}"
+                logger.error(error_message)
+                raise ValueError(error_message)
         elif provider == "groq":
             client = Groq(api_key=api_key)
             response_format = {"type": "json_object"} if json_format else None
@@ -195,14 +217,16 @@ class AIStructuredResponseGeneratorBlock(Block):
 
         prompt.append({"role": "user", "content": input_data.prompt})
 
-        def parse_response(resp: str) -> tuple[dict[str, str], str | None]:
+        def parse_response(resp: str) -> tuple[dict[str, Any], str | None]:
             try:
                 parsed = json.loads(resp)
+                if not isinstance(parsed, dict):
+                    return {}, f"Expected a dictionary, but got {type(parsed)}"
                 miss_keys = set(input_data.expected_format.keys()) - set(parsed.keys())
                 if miss_keys:
                     return parsed, f"Missing keys: {miss_keys}"
                 return parsed, None
-            except Exception as e:
+            except JSONDecodeError as e:
                 return {}, f"JSON decode error: {e}"
 
         logger.info(f"LLM request: {prompt}")
@@ -226,7 +250,16 @@ class AIStructuredResponseGeneratorBlock(Block):
                 if input_data.expected_format:
                     parsed_dict, parsed_error = parse_response(response_text)
                     if not parsed_error:
-                        yield "response", {k: str(v) for k, v in parsed_dict.items()}
+                        yield "response", {
+                            k: (
+                                json.loads(v)
+                                if isinstance(v, str)
+                                and v.startswith("[")
+                                and v.endswith("]")
+                                else (", ".join(v) if isinstance(v, list) else v)
+                            )
+                            for k, v in parsed_dict.items()
+                        }
                         return
                 else:
                     yield "response", {"response": response_text}
@@ -287,7 +320,7 @@ class AITextGeneratorBlock(Block):
             if output_name == "response":
                 return output_data["response"]
             else:
-                raise output_data
+                raise RuntimeError(output_data)
         raise ValueError("Failed to get a response from the LLM.")
 
     def run(self, input_data: Input) -> BlockOutput:
@@ -301,7 +334,7 @@ class AITextGeneratorBlock(Block):
             yield "error", str(e)
 
 
-class TextSummarizerBlock(Block):
+class AITextSummarizerBlock(Block):
     class Input(BlockSchema):
         text: str
         model: LlmModel = LlmModel.GPT4_TURBO
@@ -319,8 +352,8 @@ class TextSummarizerBlock(Block):
             id="c3d4e5f6-7g8h-9i0j-1k2l-m3n4o5p6q7r8",
             description="Utilize a Large Language Model (LLM) to summarize a long text.",
             categories={BlockCategory.AI, BlockCategory.TEXT},
-            input_schema=TextSummarizerBlock.Input,
-            output_schema=TextSummarizerBlock.Output,
+            input_schema=AITextSummarizerBlock.Input,
+            output_schema=AITextSummarizerBlock.Output,
             test_input={"text": "Lorem ipsum..." * 100},
             test_output=("summary", "Final summary of a long text"),
             test_mock={
@@ -412,7 +445,7 @@ class TextSummarizerBlock(Block):
         else:
             # If combined summaries are still too long, recursively summarize
             return self._run(
-                TextSummarizerBlock.Input(
+                AITextSummarizerBlock.Input(
                     text=combined_text,
                     api_key=input_data.api_key,
                     model=input_data.model,

autogpt_platform/backend/backend/blocks/maths.py

@@ -2,8 +2,8 @@ import operator
 from enum import Enum
 from typing import Any
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import SchemaField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
 
 
 class Operation(Enum):

autogpt_platform/backend/backend/blocks/medium.py

@@ -2,8 +2,8 @@ from typing import List
 
 import requests
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import BlockSecret, SchemaField, SecretField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import BlockSecret, SchemaField, SecretField
 
 
 class PublishToMediumBlock(Block):

autogpt_platform/backend/backend/blocks/reddit.py

@@ -4,9 +4,9 @@ from typing import Iterator
 import praw
 from pydantic import BaseModel, ConfigDict, Field
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import BlockSecret, SecretField
-from autogpt_server.util.mock import MockObject
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import BlockSecret, SecretField
+from backend.util.mock import MockObject
 
 
 class RedditCredentials(BaseModel):

autogpt_platform/backend/backend/blocks/rss.py

@@ -5,8 +5,8 @@ from typing import Any
 import feedparser
 import pydantic
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import SchemaField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
 
 
 class RSSEntry(pydantic.BaseModel):

autogpt_platform/backend/backend/blocks/sampling.py

@@ -0,0 +1,264 @@
+import random
+from collections import defaultdict
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
+
+
+class SamplingMethod(str, Enum):
+    RANDOM = "random"
+    SYSTEMATIC = "systematic"
+    TOP = "top"
+    BOTTOM = "bottom"
+    STRATIFIED = "stratified"
+    WEIGHTED = "weighted"
+    RESERVOIR = "reservoir"
+    CLUSTER = "cluster"
+
+
+class DataSamplingBlock(Block):
+    class Input(BlockSchema):
+        data: Union[Dict[str, Any], List[Union[dict, List[Any]]]] = SchemaField(
+            description="The dataset to sample from. Can be a single dictionary, a list of dictionaries, or a list of lists.",
+            placeholder="{'id': 1, 'value': 'a'} or [{'id': 1, 'value': 'a'}, {'id': 2, 'value': 'b'}, ...]",
+        )
+        sample_size: int = SchemaField(
+            description="The number of samples to take from the dataset.",
+            placeholder="10",
+            default=10,
+        )
+        sampling_method: SamplingMethod = SchemaField(
+            description="The method to use for sampling.",
+            default=SamplingMethod.RANDOM,
+        )
+        accumulate: bool = SchemaField(
+            description="Whether to accumulate data before sampling.",
+            default=False,
+        )
+        random_seed: Optional[int] = SchemaField(
+            description="Seed for random number generator (optional).",
+            default=None,
+        )
+        stratify_key: Optional[str] = SchemaField(
+            description="Key to use for stratified sampling (required for stratified sampling).",
+            default=None,
+        )
+        weight_key: Optional[str] = SchemaField(
+            description="Key to use for weighted sampling (required for weighted sampling).",
+            default=None,
+        )
+        cluster_key: Optional[str] = SchemaField(
+            description="Key to use for cluster sampling (required for cluster sampling).",
+            default=None,
+        )
+
+    class Output(BlockSchema):
+        sampled_data: List[Union[dict, List[Any]]] = SchemaField(
+            description="The sampled subset of the input data."
+        )
+        sample_indices: List[int] = SchemaField(
+            description="The indices of the sampled data in the original dataset."
+        )
+
+    def __init__(self):
+        super().__init__(
+            id="4a448883-71fa-49cf-91cf-70d793bd7d87",
+            description="This block samples data from a given dataset using various sampling methods.",
+            categories={BlockCategory.LOGIC},
+            input_schema=DataSamplingBlock.Input,
+            output_schema=DataSamplingBlock.Output,
+            test_input={
+                "data": [
+                    {"id": i, "value": chr(97 + i), "group": i % 3} for i in range(10)
+                ],
+                "sample_size": 3,
+                "sampling_method": SamplingMethod.STRATIFIED,
+                "accumulate": False,
+                "random_seed": 42,
+                "stratify_key": "group",
+            },
+            test_output=[
+                (
+                    "sampled_data",
+                    [
+                        {"id": 0, "value": "a", "group": 0},
+                        {"id": 1, "value": "b", "group": 1},
+                        {"id": 8, "value": "i", "group": 2},
+                    ],
+                ),
+                ("sample_indices", [0, 1, 8]),
+            ],
+        )
+        self.accumulated_data = []
+
+    def run(self, input_data: Input) -> BlockOutput:
+        if input_data.accumulate:
+            if isinstance(input_data.data, dict):
+                self.accumulated_data.append(input_data.data)
+            elif isinstance(input_data.data, list):
+                self.accumulated_data.extend(input_data.data)
+            else:
+                raise ValueError(f"Unsupported data type: {type(input_data.data)}")
+
+            # If we don't have enough data yet, return without sampling
+            if len(self.accumulated_data) < input_data.sample_size:
+                return
+
+            data_to_sample = self.accumulated_data
+        else:
+            # If not accumulating, use the input data directly
+            data_to_sample = (
+                input_data.data
+                if isinstance(input_data.data, list)
+                else [input_data.data]
+            )
+
+        if input_data.random_seed is not None:
+            random.seed(input_data.random_seed)
+
+        data_size = len(data_to_sample)
+
+        if input_data.sample_size > data_size:
+            raise ValueError(
+                f"Sample size ({input_data.sample_size}) cannot be larger than the dataset size ({data_size})."
+            )
+
+        indices = []
+
+        if input_data.sampling_method == SamplingMethod.RANDOM:
+            indices = random.sample(range(data_size), input_data.sample_size)
+        elif input_data.sampling_method == SamplingMethod.SYSTEMATIC:
+            step = data_size // input_data.sample_size
+            start = random.randint(0, step - 1)
+            indices = list(range(start, data_size, step))[: input_data.sample_size]
+        elif input_data.sampling_method == SamplingMethod.TOP:
+            indices = list(range(input_data.sample_size))
+        elif input_data.sampling_method == SamplingMethod.BOTTOM:
+            indices = list(range(data_size - input_data.sample_size, data_size))
+        elif input_data.sampling_method == SamplingMethod.STRATIFIED:
+            if not input_data.stratify_key:
+                raise ValueError(
+                    "Stratify key must be provided for stratified sampling."
+                )
+            strata = defaultdict(list)
+            for i, item in enumerate(data_to_sample):
+                if isinstance(item, dict):
+                    strata_value = item.get(input_data.stratify_key)
+                elif hasattr(item, input_data.stratify_key):
+                    strata_value = getattr(item, input_data.stratify_key)
+                else:
+                    raise ValueError(
+                        f"Stratify key '{input_data.stratify_key}' not found in item {item}"
+                    )
+
+                if strata_value is None:
+                    raise ValueError(
+                        f"Stratify value for key '{input_data.stratify_key}' is None"
+                    )
+
+                strata[str(strata_value)].append(i)
+
+            # Calculate the number of samples to take from each stratum
+            stratum_sizes = {
+                k: max(1, int(len(v) / data_size * input_data.sample_size))
+                for k, v in strata.items()
+            }
+
+            # Adjust sizes to ensure we get exactly sample_size samples
+            while sum(stratum_sizes.values()) != input_data.sample_size:
+                if sum(stratum_sizes.values()) < input_data.sample_size:
+                    stratum_sizes[
+                        max(stratum_sizes, key=lambda k: stratum_sizes[k])
+                    ] += 1
+                else:
+                    stratum_sizes[
+                        max(stratum_sizes, key=lambda k: stratum_sizes[k])
+                    ] -= 1
+
+            for stratum, size in stratum_sizes.items():
+                indices.extend(random.sample(strata[stratum], size))
+        elif input_data.sampling_method == SamplingMethod.WEIGHTED:
+            if not input_data.weight_key:
+                raise ValueError("Weight key must be provided for weighted sampling.")
+            weights = []
+            for item in data_to_sample:
+                if isinstance(item, dict):
+                    weight = item.get(input_data.weight_key)
+                elif hasattr(item, input_data.weight_key):
+                    weight = getattr(item, input_data.weight_key)
+                else:
+                    raise ValueError(
+                        f"Weight key '{input_data.weight_key}' not found in item {item}"
+                    )
+
+                if weight is None:
+                    raise ValueError(
+                        f"Weight value for key '{input_data.weight_key}' is None"
+                    )
+                try:
+                    weights.append(float(weight))
+                except ValueError:
+                    raise ValueError(
+                        f"Weight value '{weight}' cannot be converted to a number"
+                    )
+
+            if not weights:
+                raise ValueError(
+                    f"No valid weights found using key '{input_data.weight_key}'"
+                )
+
+            indices = random.choices(
+                range(data_size), weights=weights, k=input_data.sample_size
+            )
+        elif input_data.sampling_method == SamplingMethod.RESERVOIR:
+            indices = list(range(input_data.sample_size))
+            for i in range(input_data.sample_size, data_size):
+                j = random.randint(0, i)
+                if j < input_data.sample_size:
+                    indices[j] = i
+        elif input_data.sampling_method == SamplingMethod.CLUSTER:
+            if not input_data.cluster_key:
+                raise ValueError("Cluster key must be provided for cluster sampling.")
+            clusters = defaultdict(list)
+            for i, item in enumerate(data_to_sample):
+                if isinstance(item, dict):
+                    cluster_value = item.get(input_data.cluster_key)
+                elif hasattr(item, input_data.cluster_key):
+                    cluster_value = getattr(item, input_data.cluster_key)
+                else:
+                    raise TypeError(
+                        f"Item {item} does not have the cluster key '{input_data.cluster_key}'"
+                    )
+
+                clusters[str(cluster_value)].append(i)
+
+            # Randomly select clusters until we have enough samples
+            selected_clusters = []
+            while (
+                sum(len(clusters[c]) for c in selected_clusters)
+                < input_data.sample_size
+            ):
+                available_clusters = [c for c in clusters if c not in selected_clusters]
+                if not available_clusters:
+                    break
+                selected_clusters.append(random.choice(available_clusters))
+
+            for cluster in selected_clusters:
+                indices.extend(clusters[cluster])
+
+            # If we have more samples than needed, randomly remove some
+            if len(indices) > input_data.sample_size:
+                indices = random.sample(indices, input_data.sample_size)
+        else:
+            raise ValueError(f"Unknown sampling method: {input_data.sampling_method}")
+
+        sampled_data = [data_to_sample[i] for i in indices]
+
+        # Clear accumulated data after sampling if accumulation is enabled
+        if input_data.accumulate:
+            self.accumulated_data = []
+
+        yield "sampled_data", sampled_data
+        yield "sample_indices", indices

autogpt_platform/backend/backend/blocks/search.py

@@ -3,8 +3,8 @@ from urllib.parse import quote
 
 import requests
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import BlockSecret, SecretField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import BlockSecret, SecretField
 
 
 class GetRequest:

autogpt_platform/backend/backend/blocks/talking_head.py

@@ -3,8 +3,8 @@ from typing import Literal
 
 import requests
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import BlockSecret, SchemaField, SecretField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import BlockSecret, SchemaField, SecretField
 
 
 class CreateTalkingAvatarVideoBlock(Block):

autogpt_platform/backend/backend/blocks/text.py

@@ -4,8 +4,8 @@ from typing import Any
 from jinja2 import BaseLoader, Environment
 from pydantic import Field
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.util import json
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.util import json
 
 jinja = Environment(loader=BaseLoader())
 

autogpt_platform/backend/backend/blocks/time_blocks.py

@@ -2,7 +2,7 @@ import time
 from datetime import datetime, timedelta
 from typing import Any, Union
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
 
 
 class GetCurrentTimeBlock(Block):
@@ -23,7 +23,7 @@ class GetCurrentTimeBlock(Block):
                 {"trigger": "Hello", "format": "{time}"},
             ],
             test_output=[
-                ("time", time.strftime("%H:%M:%S")),
+                ("time", lambda _: time.strftime("%H:%M:%S")),
             ],
         )
 
@@ -130,7 +130,6 @@ class CountdownTimerBlock(Block):
         )
 
     def run(self, input_data: Input) -> BlockOutput:
-
         seconds = int(input_data.seconds)
         minutes = int(input_data.minutes)
         hours = int(input_data.hours)

autogpt_platform/backend/backend/blocks/youtube.py

@@ -3,8 +3,8 @@ from urllib.parse import parse_qs, urlparse
 from youtube_transcript_api import YouTubeTranscriptApi
 from youtube_transcript_api.formatters import TextFormatter
 
-from autogpt_server.data.block import Block, BlockCategory, BlockOutput, BlockSchema
-from autogpt_server.data.model import SchemaField
+from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
+from backend.data.model import SchemaField
 
 
 class TranscribeYouTubeVideoBlock(Block):

autogpt_platform/backend/backend/cli.py

@@ -8,8 +8,8 @@ import pathlib
 import click
 import psutil
 
-from autogpt_server import app
-from autogpt_server.util.process import AppProcess
+from backend import app
+from backend.util.process import AppProcess
 
 
 def get_pid_path() -> pathlib.Path:
@@ -109,7 +109,7 @@ def reddit(server_address: str):
     """
     import requests
 
-    from autogpt_server.usecases.reddit_marketing import create_test_graph
+    from backend.usecases.reddit_marketing import create_test_graph
 
     test_graph = create_test_graph()
     url = f"{server_address}/graphs"
@@ -130,7 +130,7 @@ def populate_db(server_address: str):
     """
     import requests
 
-    from autogpt_server.usecases.sample import create_test_graph
+    from backend.usecases.sample import create_test_graph
 
     test_graph = create_test_graph()
     url = f"{server_address}/graphs"
@@ -166,7 +166,7 @@ def graph(server_address: str):
     """
     import requests
 
-    from autogpt_server.usecases.sample import create_test_graph
+    from backend.usecases.sample import create_test_graph
 
     url = f"{server_address}/graphs"
     headers = {"Content-Type": "application/json"}
@@ -219,7 +219,7 @@ def websocket(server_address: str, graph_id: str):
 
     import websockets
 
-    from autogpt_server.server.ws_api import ExecutionSubscription, Methods, WsMessage
+    from backend.server.ws_api import ExecutionSubscription, Methods, WsMessage
 
     async def send_message(server_address: str):
         uri = f"ws://{server_address}"

autogpt_platform/backend/backend/data/analytics.py

@@ -0,0 +1,43 @@
+import logging
+
+import prisma.types
+
+logger = logging.getLogger(__name__)
+
+
+async def log_raw_analytics(
+    user_id: str,
+    type: str,
+    data: dict,
+    data_index: str,
+):
+    details = await prisma.models.AnalyticsDetails.prisma().create(
+        data={
+            "userId": user_id,
+            "type": type,
+            "data": prisma.Json(data),
+            "dataIndex": data_index,
+        }
+    )
+    return details
+
+
+async def log_raw_metric(
+    user_id: str,
+    metric_name: str,
+    metric_value: float,
+    data_string: str,
+):
+    if metric_value < 0:
+        raise ValueError("metric_value must be non-negative")
+
+    result = await prisma.models.AnalyticsMetrics.prisma().create(
+        data={
+            "value": metric_value,
+            "analyticMetric": metric_name,
+            "userId": user_id,
+            "dataString": data_string,
+        },
+    )
+
+    return result

autogpt_platform/backend/backend/data/block.py

@@ -7,8 +7,8 @@ import jsonschema
 from prisma.models import AgentBlock
 from pydantic import BaseModel
 
-from autogpt_server.data.model import ContributorDetails
-from autogpt_server.util import json
+from backend.data.model import ContributorDetails
+from backend.util import json
 
 BlockData = tuple[str, Any]  # Input & Output data should be a tuple of (name, data).
 BlockInput = dict[str, Any]  # Input: 1 input pin consumes 1 data.
@@ -225,7 +225,7 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
 
 
 def get_blocks() -> dict[str, Block]:
-    from autogpt_server.blocks import AVAILABLE_BLOCKS  # noqa: E402
+    from backend.blocks import AVAILABLE_BLOCKS  # noqa: E402
 
     return AVAILABLE_BLOCKS
 

autogpt_platform/backend/backend/data/credit.py

@@ -0,0 +1,274 @@
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Optional, Type
+
+import prisma.errors
+from prisma import Json
+from prisma.enums import UserBlockCreditType
+from prisma.models import UserBlockCredit
+from pydantic import BaseModel
+
+from backend.blocks.llm import (
+    MODEL_METADATA,
+    AIConversationBlock,
+    AIStructuredResponseGeneratorBlock,
+    AITextGeneratorBlock,
+    AITextSummarizerBlock,
+    LlmModel,
+)
+from backend.blocks.talking_head import CreateTalkingAvatarVideoBlock
+from backend.data.block import Block, BlockInput
+from backend.util.settings import Config
+
+
+class BlockCostType(str, Enum):
+    RUN = "run"  # cost X credits per run
+    BYTE = "byte"  # cost X credits per byte
+    SECOND = "second"  # cost X credits per second
+
+
+class BlockCost(BaseModel):
+    cost_amount: int
+    cost_filter: BlockInput
+    cost_type: BlockCostType
+
+    def __init__(
+        self,
+        cost_amount: int,
+        cost_type: BlockCostType = BlockCostType.RUN,
+        cost_filter: Optional[BlockInput] = None,
+        **data: Any,
+    ) -> None:
+        super().__init__(
+            cost_amount=cost_amount,
+            cost_filter=cost_filter or {},
+            cost_type=cost_type,
+            **data,
+        )
+
+
+llm_cost = [
+    BlockCost(
+        cost_type=BlockCostType.RUN,
+        cost_filter={
+            "model": model,
+            "api_key": None,  # Running LLM with user own API key is free.
+        },
+        cost_amount=metadata.cost_factor,
+    )
+    for model, metadata in MODEL_METADATA.items()
+] + [
+    BlockCost(
+        # Default cost is running LlmModel.GPT4O.
+        cost_amount=MODEL_METADATA[LlmModel.GPT4O].cost_factor,
+        cost_filter={"api_key": None},
+    ),
+]
+
+BLOCK_COSTS: dict[Type[Block], list[BlockCost]] = {
+    AIConversationBlock: llm_cost,
+    AITextGeneratorBlock: llm_cost,
+    AIStructuredResponseGeneratorBlock: llm_cost,
+    AITextSummarizerBlock: llm_cost,
+    CreateTalkingAvatarVideoBlock: [
+        BlockCost(cost_amount=15, cost_filter={"api_key": None})
+    ],
+}
+
+
+class UserCreditBase(ABC):
+    def __init__(self, num_user_credits_refill: int):
+        self.num_user_credits_refill = num_user_credits_refill
+
+    @abstractmethod
+    async def get_or_refill_credit(self, user_id: str) -> int:
+        """
+        Get the current credit for the user and refill if no transaction has been made in the current cycle.
+
+        Returns:
+            int: The current credit for the user.
+        """
+        pass
+
+    @abstractmethod
+    async def spend_credits(
+        self,
+        user_id: str,
+        user_credit: int,
+        block: Block,
+        input_data: BlockInput,
+        data_size: float,
+        run_time: float,
+    ) -> int:
+        """
+        Spend the credits for the user based on the block usage.
+
+        Args:
+            user_id (str): The user ID.
+            user_credit (int): The current credit for the user.
+            block (Block): The block that is being used.
+            input_data (BlockInput): The input data for the block.
+            data_size (float): The size of the data being processed.
+            run_time (float): The time taken to run the block.
+
+        Returns:
+            int: amount of credit spent
+        """
+        pass
+
+    @abstractmethod
+    async def top_up_credits(self, user_id: str, amount: int):
+        """
+        Top up the credits for the user.
+
+        Args:
+            user_id (str): The user ID.
+            amount (int): The amount to top up.
+        """
+        pass
+
+
+class UserCredit(UserCreditBase):
+    async def get_or_refill_credit(self, user_id: str) -> int:
+        cur_time = self.time_now()
+        cur_month = cur_time.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+        nxt_month = cur_month.replace(month=cur_month.month + 1)
+
+        user_credit = await UserBlockCredit.prisma().group_by(
+            by=["userId"],
+            sum={"amount": True},
+            where={
+                "userId": user_id,
+                "createdAt": {"gte": cur_month, "lt": nxt_month},
+                "isActive": True,
+            },
+        )
+
+        if user_credit:
+            credit_sum = user_credit[0].get("_sum") or {}
+            return credit_sum.get("amount", 0)
+
+        key = f"MONTHLY-CREDIT-TOP-UP-{cur_month}"
+
+        try:
+            await UserBlockCredit.prisma().create(
+                data={
+                    "amount": self.num_user_credits_refill,
+                    "type": UserBlockCreditType.TOP_UP,
+                    "userId": user_id,
+                    "transactionKey": key,
+                    "createdAt": self.time_now(),
+                }
+            )
+        except prisma.errors.UniqueViolationError:
+            pass  # Already refilled this month
+
+        return self.num_user_credits_refill
+
+    @staticmethod
+    def time_now():
+        return datetime.now(timezone.utc)
+
+    @staticmethod
+    def _block_usage_cost(
+        block: Block,
+        input_data: BlockInput,
+        data_size: float,
+        run_time: float,
+    ) -> tuple[int, BlockInput]:
+        block_costs = BLOCK_COSTS.get(type(block))
+        if not block_costs:
+            return 0, {}
+
+        for block_cost in block_costs:
+            if all(
+                # None, [], {}, "", are considered the same value.
+                input_data.get(k) == b or (not input_data.get(k) and not b)
+                for k, b in block_cost.cost_filter.items()
+            ):
+                if block_cost.cost_type == BlockCostType.RUN:
+                    return block_cost.cost_amount, block_cost.cost_filter
+
+                if block_cost.cost_type == BlockCostType.SECOND:
+                    return (
+                        int(run_time * block_cost.cost_amount),
+                        block_cost.cost_filter,
+                    )
+
+                if block_cost.cost_type == BlockCostType.BYTE:
+                    return (
+                        int(data_size * block_cost.cost_amount),
+                        block_cost.cost_filter,
+                    )
+
+        return 0, {}
+
+    async def spend_credits(
+        self,
+        user_id: str,
+        user_credit: int,
+        block: Block,
+        input_data: BlockInput,
+        data_size: float,
+        run_time: float,
+        validate_balance: bool = True,
+    ) -> int:
+        cost, matching_filter = self._block_usage_cost(
+            block=block, input_data=input_data, data_size=data_size, run_time=run_time
+        )
+        if cost <= 0:
+            return 0
+
+        if validate_balance and user_credit < cost:
+            raise ValueError(f"Insufficient credit: {user_credit} < {cost}")
+
+        await UserBlockCredit.prisma().create(
+            data={
+                "userId": user_id,
+                "amount": -cost,
+                "type": UserBlockCreditType.USAGE,
+                "blockId": block.id,
+                "metadata": Json(
+                    {
+                        "block": block.name,
+                        "input": matching_filter,
+                    }
+                ),
+                "createdAt": self.time_now(),
+            }
+        )
+        return cost
+
+    async def top_up_credits(self, user_id: str, amount: int):
+        await UserBlockCredit.prisma().create(
+            data={
+                "userId": user_id,
+                "amount": amount,
+                "type": UserBlockCreditType.TOP_UP,
+                "createdAt": self.time_now(),
+            }
+        )
+
+
+class DisabledUserCredit(UserCreditBase):
+    async def get_or_refill_credit(self, *args, **kwargs) -> int:
+        return 0
+
+    async def spend_credits(self, *args, **kwargs) -> int:
+        return 0
+
+    async def top_up_credits(self, *args, **kwargs):
+        pass
+
+
+def get_user_credit_model() -> UserCreditBase:
+    config = Config()
+    if config.enable_credit.lower() == "true":
+        return UserCredit(config.num_user_credits_refill)
+    else:
+        return DisabledUserCredit(0)
+
+
+def get_block_costs() -> dict[str, list[BlockCost]]:
+    return {block().id: costs for block, costs in BLOCK_COSTS.items()}

autogpt_platform/backend/backend/data/execution.py

@@ -1,9 +1,9 @@
 from collections import defaultdict
 from datetime import datetime, timezone
-from enum import Enum
 from multiprocessing import Manager
 from typing import Any, Generic, TypeVar
 
+from prisma.enums import AgentExecutionStatus
 from prisma.models import (
     AgentGraphExecution,
     AgentNodeExecution,
@@ -16,17 +16,19 @@ from prisma.types import (
 )
 from pydantic import BaseModel
 
-from autogpt_server.data.block import BlockData, BlockInput, CompletedBlockOutput
-from autogpt_server.util import json, mock
+from backend.data.block import BlockData, BlockInput, CompletedBlockOutput
+from backend.util import json, mock
 
 
 class GraphExecution(BaseModel):
+    user_id: str
     graph_exec_id: str
-    start_node_execs: list["NodeExecution"]
     graph_id: str
+    start_node_execs: list["NodeExecution"]
 
 
 class NodeExecution(BaseModel):
+    user_id: str
     graph_exec_id: str
     graph_id: str
     node_exec_id: str
@@ -34,13 +36,7 @@ class NodeExecution(BaseModel):
     data: BlockInput
 
 
-class ExecutionStatus(str, Enum):
-    INCOMPLETE = "INCOMPLETE"
-    QUEUED = "QUEUED"
-    RUNNING = "RUNNING"
-    COMPLETED = "COMPLETED"
-    FAILED = "FAILED"
-
+ExecutionStatus = AgentExecutionStatus
 
 T = TypeVar("T")
 
@@ -148,6 +144,7 @@ async def create_graph_execution(
         data={
             "agentGraphId": graph_id,
             "agentGraphVersion": graph_version,
+            "executionStatus": ExecutionStatus.QUEUED,
             "AgentNodeExecutions": {
                 "create": [  # type: ignore
                     {
@@ -259,10 +256,20 @@ async def upsert_execution_output(
     )
 
 
+async def update_graph_execution_start_time(graph_exec_id: str):
+    await AgentGraphExecution.prisma().update(
+        where={"id": graph_exec_id},
+        data={
+            "executionStatus": ExecutionStatus.RUNNING,
+            "startedAt": datetime.now(tz=timezone.utc),
+        },
+    )
+
+
 async def update_graph_execution_stats(graph_exec_id: str, stats: dict[str, Any]):
     await AgentGraphExecution.prisma().update(
         where={"id": graph_exec_id},
-        data={"stats": json.dumps(stats)},
+        data={"executionStatus": ExecutionStatus.COMPLETED, "stats": json.dumps(stats)},
     )
 
 
@@ -389,19 +396,19 @@ def merge_execution_input(data: BlockInput) -> BlockInput:
 
     # Merge all input with <input_name>_$_<index> into a single list.
     items = list(data.items())
-    list_input: list[Any] = []
+
     for key, value in items:
         if LIST_SPLIT not in key:
             continue
         name, index = key.split(LIST_SPLIT)
         if not index.isdigit():
-            list_input.append((name, value, 0))
-        else:
-            list_input.append((name, value, int(index)))
+            raise ValueError(f"Invalid key: {key}, #{index} index must be an integer.")
 
-    for name, value, _ in sorted(list_input, key=lambda x: x[2]):
         data[name] = data.get(name, [])
-        data[name].append(value)
+        if int(index) >= len(data[name]):
+            # Pad list with empty string on missing indices.
+            data[name].extend([""] * (int(index) - len(data[name]) + 1))
+        data[name][int(index)] = value
 
     # Merge all input with <input_name>_#_<index> into a single dict.
     for key, value in items:

autogpt_platform/backend/backend/data/graph.py

@@ -9,11 +9,11 @@ from prisma.models import AgentGraph, AgentNode, AgentNodeLink
 from pydantic import BaseModel, PrivateAttr
 from pydantic_core import PydanticUndefinedType
 
-from autogpt_server.blocks.basic import AgentInputBlock, AgentOutputBlock
-from autogpt_server.data.block import BlockInput, get_block, get_blocks
-from autogpt_server.data.db import BaseDbModel, transaction
-from autogpt_server.data.user import DEFAULT_USER_ID
-from autogpt_server.util import json
+from backend.blocks.basic import AgentInputBlock, AgentOutputBlock
+from backend.data.block import BlockInput, get_block, get_blocks
+from backend.data.db import BaseDbModel, transaction
+from backend.data.user import DEFAULT_USER_ID
+from backend.util import json
 
 logger = logging.getLogger(__name__)
 
@@ -274,7 +274,6 @@ class Graph(GraphMeta):
                         PydanticUndefinedType,
                     )
                 ):
-
                     input_schema.append(
                         InputSchemaItem(
                             node_id=node.id,

autogpt_platform/backend/backend/data/model.py

@@ -11,7 +11,7 @@ from pydantic_core import (
     core_schema,
 )
 
-from autogpt_server.util.settings import Secrets
+from backend.util.settings import Secrets
 
 T = TypeVar("T")
 logger = logging.getLogger(__name__)

autogpt_platform/backend/backend/data/queue.py

@@ -6,7 +6,7 @@ from datetime import datetime
 
 from redis.asyncio import Redis
 
-from autogpt_server.data.execution import ExecutionResult
+from backend.data.execution import ExecutionResult
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +37,6 @@ class AsyncEventQueue(ABC):
 
 
 class AsyncRedisEventQueue(AsyncEventQueue):
-
     def __init__(self):
         self.host = os.getenv("REDIS_HOST", "localhost")
         self.port = int(os.getenv("REDIS_PORT", "6379"))

autogpt_platform/backend/backend/data/schedule.py

@@ -3,9 +3,9 @@ from typing import Optional
 
 from prisma.models import AgentGraphExecutionSchedule
 
-from autogpt_server.data.block import BlockInput
-from autogpt_server.data.db import BaseDbModel
-from autogpt_server.util import json
+from backend.data.block import BlockInput
+from backend.data.db import BaseDbModel
+from backend.util import json
 
 
 class ExecutionSchedule(BaseDbModel):

autogpt_platform/backend/backend/data/user.py

@@ -3,14 +3,13 @@ from typing import Optional
 from fastapi import HTTPException
 from prisma.models import User
 
-from autogpt_server.data.db import prisma
+from backend.data.db import prisma
 
 DEFAULT_USER_ID = "3e53486c-cf57-477e-ba2a-cb02dc828e1a"
 DEFAULT_EMAIL = "default@example.com"
 
 
 async def get_or_create_user(user_data: dict) -> User:
-
     user_id = user_data.get("sub")
     if not user_id:
         raise HTTPException(status_code=401, detail="User ID not found in token")

autogpt_platform/backend/backend/exec.py

@@ -1,5 +1,5 @@
-from autogpt_server.app import run_processes
-from autogpt_server.executor import ExecutionManager
+from backend.app import run_processes
+from backend.executor import ExecutionManager
 
 
 def main():

autogpt_platform/backend/backend/executor/manager.py

@@ -12,13 +12,15 @@ from multiprocessing.pool import AsyncResult, Pool
 from typing import TYPE_CHECKING, Any, Coroutine, Generator, TypeVar
 
 if TYPE_CHECKING:
-    from autogpt_server.server.rest_api import AgentServer
+    from backend.server.rest_api import AgentServer
 
-from autogpt_server.blocks.basic import AgentInputBlock
-from autogpt_server.data import db
-from autogpt_server.data.block import Block, BlockData, BlockInput, get_block
-from autogpt_server.data.execution import (
+from backend.blocks.basic import AgentInputBlock
+from backend.data import db
+from backend.data.block import Block, BlockData, BlockInput, get_block
+from backend.data.credit import get_user_credit_model
+from backend.data.execution import (
     ExecutionQueue,
+    ExecutionResult,
     ExecutionStatus,
     GraphExecution,
     NodeExecution,
@@ -34,36 +36,60 @@ from autogpt_server.data.execution import (
     upsert_execution_input,
     upsert_execution_output,
 )
-from autogpt_server.data.graph import Graph, Link, Node, get_graph, get_node
-from autogpt_server.util import json
-from autogpt_server.util.decorator import error_logged, time_measured
-from autogpt_server.util.logging import configure_logging
-from autogpt_server.util.service import AppService, expose, get_service_client
-from autogpt_server.util.settings import Config
-from autogpt_server.util.type import convert
+from backend.data.graph import Graph, Link, Node, get_graph, get_node
+from backend.util import json
+from backend.util.decorator import error_logged, time_measured
+from backend.util.logging import configure_logging
+from backend.util.service import AppService, expose, get_service_client
+from backend.util.settings import Config
+from backend.util.type import convert
 
 logger = logging.getLogger(__name__)
 
 
-def get_log_metadata(
-    graph_eid: str,
-    graph_id: str,
-    node_eid: str,
-    node_id: str,
-    block_name: str,
-) -> dict:
-    return {
-        "component": "ExecutionManager",
-        "graph_eid": graph_eid,
-        "graph_id": graph_id,
-        "node_eid": node_eid,
-        "node_id": node_id,
-        "block_name": block_name,
-    }
+class LogMetadata:
+    def __init__(
+        self,
+        user_id: str,
+        graph_eid: str,
+        graph_id: str,
+        node_eid: str,
+        node_id: str,
+        block_name: str,
+    ):
+        self.metadata = {
+            "component": "ExecutionManager",
+            "user_id": user_id,
+            "graph_eid": graph_eid,
+            "graph_id": graph_id,
+            "node_eid": node_eid,
+            "node_id": node_id,
+            "block_name": block_name,
+        }
+        self.prefix = f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|nid:{node_eid}|{block_name}]"
 
+    def info(self, msg: str, **extra):
+        msg = self._wrap(msg, **extra)
+        logger.info(msg, extra={"json_fields": {**self.metadata, **extra}})
 
-def get_log_prefix(graph_eid: str, node_eid: str, block_name: str = "-"):
-    return f"[ExecutionManager][graph-eid-{graph_eid}|node-eid-{node_eid}|{block_name}]"
+    def warning(self, msg: str, **extra):
+        msg = self._wrap(msg, **extra)
+        logger.warning(msg, extra={"json_fields": {**self.metadata, **extra}})
+
+    def error(self, msg: str, **extra):
+        msg = self._wrap(msg, **extra)
+        logger.error(msg, extra={"json_fields": {**self.metadata, **extra}})
+
+    def debug(self, msg: str, **extra):
+        msg = self._wrap(msg, **extra)
+        logger.debug(msg, extra={"json_fields": {**self.metadata, **extra}})
+
+    def exception(self, msg: str, **extra):
+        msg = self._wrap(msg, **extra)
+        logger.exception(msg, extra={"json_fields": {**self.metadata, **extra}})
+
+    def _wrap(self, msg: str, **extra):
+        return f"{self.prefix} {msg} {extra}"
 
 
 T = TypeVar("T")
@@ -89,6 +115,7 @@ def execute_node(
     Returns:
         The subsequent node to be enqueued, or None if there is no subsequent node.
     """
+    user_id = data.user_id
     graph_exec_id = data.graph_exec_id
     graph_id = data.graph_id
     node_exec_id = data.node_exec_id
@@ -99,9 +126,10 @@ def execute_node(
     def wait(f: Coroutine[Any, Any, T]) -> T:
         return loop.run_until_complete(f)
 
-    def update_execution(status: ExecutionStatus):
+    def update_execution(status: ExecutionStatus) -> ExecutionResult:
         exec_update = wait(update_execution_status(node_exec_id, status))
         api_client.send_execution_update(exec_update.model_dump())
+        return exec_update
 
     node = wait(get_node(node_id))
 
@@ -111,43 +139,35 @@ def execute_node(
         return
 
     # Sanity check: validate the execution input.
-    log_metadata = get_log_metadata(
+    log_metadata = LogMetadata(
+        user_id=user_id,
         graph_eid=graph_exec_id,
         graph_id=graph_id,
         node_eid=node_exec_id,
         node_id=node_id,
         block_name=node_block.name,
     )
-    prefix = get_log_prefix(
-        graph_eid=graph_exec_id,
-        node_eid=node_exec_id,
-        block_name=node_block.name,
-    )
     input_data, error = validate_exec(node, data.data, resolve_input=False)
     if input_data is None:
-        logger.error(
-            "{prefix} Skip execution, input validation error",
-            extra={"json_fields": {**log_metadata, "error": error}},
-        )
+        log_metadata.error(f"Skip execution, input validation error: {error}")
         return
 
     # Execute the node
     input_data_str = json.dumps(input_data)
     input_size = len(input_data_str)
-    logger.info(
-        f"{prefix} Executed node with input",
-        extra={"json_fields": {**log_metadata, "input": input_data_str}},
-    )
+    log_metadata.info("Executed node with input", input=input_data_str)
     update_execution(ExecutionStatus.RUNNING)
+    user_credit = get_user_credit_model()
 
     output_size = 0
     try:
+        credit = wait(user_credit.get_or_refill_credit(user_id))
+        if credit < 0:
+            raise ValueError(f"Insufficient credit: {credit}")
+
         for output_name, output_data in node_block.execute(input_data):
             output_size += len(json.dumps(output_data))
-            logger.info(
-                f"{prefix} Node produced output",
-                extra={"json_fields": {**log_metadata, output_name: output_data}},
-            )
+            log_metadata.info("Node produced output", output_name=output_data)
             wait(upsert_execution_output(node_exec_id, output_name, output_data))
 
             for execution in _enqueue_next_nodes(
@@ -155,20 +175,25 @@ def execute_node(
                 loop=loop,
                 node=node,
                 output=(output_name, output_data),
+                user_id=user_id,
                 graph_exec_id=graph_exec_id,
                 graph_id=graph_id,
                 log_metadata=log_metadata,
             ):
                 yield execution
 
-        update_execution(ExecutionStatus.COMPLETED)
+        r = update_execution(ExecutionStatus.COMPLETED)
+        s = input_size + output_size
+        t = (
+            (r.end_time - r.start_time).total_seconds()
+            if r.end_time and r.start_time
+            else 0
+        )
+        wait(user_credit.spend_credits(user_id, credit, node_block, input_data, s, t))
 
     except Exception as e:
-        error_msg = f"{e.__class__.__name__}: {e}"
-        logger.exception(
-            f"{prefix} Node execution failed with error",
-            extra={"json_fields": {**log_metadata, error: error_msg}},
-        )
+        error_msg = str(e)
+        log_metadata.exception(f"Node execution failed with error {error_msg}")
         wait(upsert_execution_output(node_exec_id, "error", error_msg))
         update_execution(ExecutionStatus.FAILED)
 
@@ -194,9 +219,10 @@ def _enqueue_next_nodes(
     loop: asyncio.AbstractEventLoop,
     node: Node,
     output: BlockData,
+    user_id: str,
     graph_exec_id: str,
     graph_id: str,
-    log_metadata: dict,
+    log_metadata: LogMetadata,
 ) -> list[NodeExecution]:
     def wait(f: Coroutine[Any, Any, T]) -> T:
         return loop.run_until_complete(f)
@@ -209,6 +235,7 @@ def _enqueue_next_nodes(
         )
         api_client.send_execution_update(exec_update.model_dump())
         return NodeExecution(
+            user_id=user_id,
             graph_exec_id=graph_exec_id,
             graph_id=graph_id,
             node_exec_id=node_exec_id,
@@ -262,17 +289,11 @@ def _enqueue_next_nodes(
 
             # Incomplete input data, skip queueing the execution.
             if not next_node_input:
-                logger.warning(
-                    f"Skipped queueing {suffix}",
-                    extra={"json_fields": {**log_metadata}},
-                )
+                log_metadata.warning(f"Skipped queueing {suffix}")
                 return enqueued_executions
 
             # Input is complete, enqueue the execution.
-            logger.info(
-                f"Enqueued {suffix}",
-                extra={"json_fields": {**log_metadata}},
-            )
+            log_metadata.info(f"Enqueued {suffix}")
             enqueued_executions.append(
                 add_enqueued_execution(next_node_exec_id, next_node_id, next_node_input)
             )
@@ -298,11 +319,9 @@ def _enqueue_next_nodes(
                 idata, msg = validate_exec(next_node, idata)
                 suffix = f"{next_output_name}>{next_input_name}~{ineid}:{msg}"
                 if not idata:
-                    logger.info(
-                        f"{log_metadata} Enqueueing static-link skipped: {suffix}"
-                    )
+                    log_metadata.info(f"Enqueueing static-link skipped: {suffix}")
                     continue
-                logger.info(f"{log_metadata} Enqueueing static-link execution {suffix}")
+                log_metadata.info(f"Enqueueing static-link execution {suffix}")
                 enqueued_executions.append(
                     add_enqueued_execution(iexec.node_exec_id, next_node_id, idata)
                 )
@@ -371,7 +390,7 @@ def validate_exec(
 
 
 def get_agent_server_client() -> "AgentServer":
-    from autogpt_server.server.rest_api import AgentServer
+    from backend.server.rest_api import AgentServer
 
     return get_service_client(AgentServer, Config().agent_server_port)
 
@@ -443,22 +462,18 @@ class Executor:
     def on_node_execution(
         cls, q: ExecutionQueue[NodeExecution], node_exec: NodeExecution
     ):
-        log_metadata = get_log_metadata(
+        log_metadata = LogMetadata(
+            user_id=node_exec.user_id,
             graph_eid=node_exec.graph_exec_id,
             graph_id=node_exec.graph_id,
             node_eid=node_exec.node_exec_id,
             node_id=node_exec.node_id,
             block_name="-",
         )
-        prefix = get_log_prefix(
-            graph_eid=node_exec.graph_exec_id,
-            node_eid=node_exec.node_exec_id,
-            block_name="-",
-        )
 
         execution_stats = {}
         timing_info, _ = cls._on_node_execution(
-            q, node_exec, log_metadata, prefix, execution_stats
+            q, node_exec, log_metadata, execution_stats
         )
         execution_stats["walltime"] = timing_info.wall_time
         execution_stats["cputime"] = timing_info.cpu_time
@@ -473,29 +488,19 @@ class Executor:
         cls,
         q: ExecutionQueue[NodeExecution],
         node_exec: NodeExecution,
-        log_metadata: dict,
-        prefix: str,
+        log_metadata: LogMetadata,
         stats: dict[str, Any] | None = None,
     ):
         try:
-            logger.info(
-                f"{prefix} Start node execution {node_exec.node_exec_id}",
-                extra={"json_fields": {**log_metadata}},
-            )
+            log_metadata.info(f"Start node execution {node_exec.node_exec_id}")
             for execution in execute_node(
                 cls.loop, cls.agent_server_client, node_exec, stats
             ):
                 q.add(execution)
-            logger.info(
-                f"{prefix} Finished node execution {node_exec.node_exec_id}",
-                extra={"json_fields": {**log_metadata}},
-            )
+            log_metadata.info(f"Finished node execution {node_exec.node_exec_id}")
         except Exception as e:
-            logger.exception(
-                f"Failed node execution {node_exec.node_exec_id}: {e}",
-                extra={
-                    **log_metadata,
-                },
+            log_metadata.exception(
+                f"Failed node execution {node_exec.node_exec_id}: {e}"
             )
 
     @classmethod
@@ -517,10 +522,12 @@ class Executor:
 
     @classmethod
     def on_graph_executor_stop(cls):
-        logger.info(
-            f"[on_graph_executor_stop {cls.pid}] ⏳ Terminating node executor pool..."
-        )
+        prefix = f"[on_graph_executor_stop {cls.pid}]"
+        logger.info(f"{prefix} ⏳ Disconnecting DB...")
+        cls.loop.run_until_complete(db.disconnect())
+        logger.info(f"{prefix} ⏳ Terminating node executor pool...")
         cls.executor.terminate()
+        logger.info(f"{prefix} ✅ Finished cleanup")
 
     @classmethod
     def _init_node_executor_pool(cls):
@@ -532,20 +539,16 @@ class Executor:
     @classmethod
     @error_logged
     def on_graph_execution(cls, graph_exec: GraphExecution, cancel: threading.Event):
-        log_metadata = get_log_metadata(
+        log_metadata = LogMetadata(
+            user_id=graph_exec.user_id,
             graph_eid=graph_exec.graph_exec_id,
             graph_id=graph_exec.graph_id,
             node_id="*",
             node_eid="*",
             block_name="-",
         )
-        prefix = get_log_prefix(
-            graph_eid=graph_exec.graph_exec_id,
-            node_eid="*",
-            block_name="-",
-        )
         timing_info, node_count = cls._on_graph_execution(
-            graph_exec, cancel, log_metadata, prefix
+            graph_exec, cancel, log_metadata
         )
 
         cls.loop.run_until_complete(
@@ -565,13 +568,9 @@ class Executor:
         cls,
         graph_exec: GraphExecution,
         cancel: threading.Event,
-        log_metadata: dict,
-        prefix: str,
+        log_metadata: LogMetadata,
     ) -> int:
-        logger.info(
-            f"{prefix} Start graph execution {graph_exec.graph_exec_id}",
-            extra={"json_fields": {**log_metadata}},
-        )
+        log_metadata.info(f"Start graph execution {graph_exec.graph_exec_id}")
         n_node_executions = 0
         finished = False
 
@@ -581,10 +580,7 @@ class Executor:
             if finished:
                 return
             cls.executor.terminate()
-            logger.info(
-                f"{prefix} Terminated graph execution {graph_exec.graph_exec_id}",
-                extra={"json_fields": {**log_metadata}},
-            )
+            log_metadata.info(f"Terminated graph execution {graph_exec.graph_exec_id}")
             cls._init_node_executor_pool()
 
         cancel_thread = threading.Thread(target=cancel_handler)
@@ -622,10 +618,9 @@ class Executor:
                     #   Re-enqueueing the data back to the queue will disrupt the order.
                     execution.wait()
 
-                logger.debug(
-                    f"{prefix} Dispatching node execution {exec_data.node_exec_id} "
+                log_metadata.debug(
+                    f"Dispatching node execution {exec_data.node_exec_id} "
                     f"for node {exec_data.node_id}",
-                    extra={**log_metadata},
                 )
                 running_executions[exec_data.node_id] = cls.executor.apply_async(
                     cls.on_node_execution,
@@ -635,10 +630,8 @@ class Executor:
 
                 # Avoid terminating graph execution when some nodes are still running.
                 while queue.empty() and running_executions:
-                    logger.debug(
-                        "Queue empty; running nodes: "
-                        f"{list(running_executions.keys())}",
-                        extra={"json_fields": {**log_metadata}},
+                    log_metadata.debug(
+                        f"Queue empty; running nodes: {list(running_executions.keys())}"
                     )
                     for node_id, execution in list(running_executions.items()):
                         if cancel.is_set():
@@ -647,20 +640,13 @@ class Executor:
                         if not queue.empty():
                             break  # yield to parent loop to execute new queue items
 
-                        logger.debug(
-                            f"Waiting on execution of node {node_id}",
-                            extra={"json_fields": {**log_metadata}},
-                        )
+                        log_metadata.debug(f"Waiting on execution of node {node_id}")
                         execution.wait(3)
 
-            logger.info(
-                f"{prefix} Finished graph execution {graph_exec.graph_exec_id}",
-                extra={"json_fields": {**log_metadata}},
-            )
+            log_metadata.info(f"Finished graph execution {graph_exec.graph_exec_id}")
         except Exception as e:
-            logger.exception(
-                f"{prefix} Failed graph execution {graph_exec.graph_exec_id}: {e}",
-                extra={"json_fields": {**log_metadata}},
+            log_metadata.exception(
+                f"Failed graph execution {graph_exec.graph_exec_id}: {e}"
             )
         finally:
             if not cancel.is_set():
@@ -747,6 +733,7 @@ class ExecutionManager(AppService):
         for node_exec in node_execs:
             starting_node_execs.append(
                 NodeExecution(
+                    user_id=user_id,
                     graph_exec_id=node_exec.graph_exec_id,
                     graph_id=node_exec.graph_id,
                     node_exec_id=node_exec.node_exec_id,
@@ -762,6 +749,7 @@ class ExecutionManager(AppService):
             self.agent_server_client.send_execution_update(exec_update.model_dump())
 
         graph_exec = GraphExecution(
+            user_id=user_id,
             graph_id=graph_id,
             graph_exec_id=graph_exec_id,
             start_node_execs=starting_node_execs,

autogpt_platform/backend/backend/executor/scheduler.py

@@ -5,11 +5,11 @@ from datetime import datetime
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.triggers.cron import CronTrigger
 
-from autogpt_server.data import schedule as model
-from autogpt_server.data.block import BlockInput
-from autogpt_server.executor.manager import ExecutionManager
-from autogpt_server.util.service import AppService, expose, get_service_client
-from autogpt_server.util.settings import Config
+from backend.data import schedule as model
+from backend.data.block import BlockInput
+from backend.executor.manager import ExecutionManager
+from backend.util.service import AppService, expose, get_service_client
+from backend.util.settings import Config
 
 logger = logging.getLogger(__name__)
 

autogpt_platform/backend/backend/integrations/oauth/github.py

@@ -23,6 +23,7 @@ class GitHubOAuthHandler(BaseOAuthHandler):
     """  # noqa
 
     PROVIDER_NAME = "github"
+    EMAIL_ENDPOINT = "https://api.github.com/user/emails"
 
     def __init__(self, client_id: str, client_secret: str, redirect_uri: str):
         self.client_id = client_id
@@ -69,10 +70,13 @@ class GitHubOAuthHandler(BaseOAuthHandler):
         response.raise_for_status()
         token_data: dict = response.json()
 
+        username = self._request_username(token_data["access_token"])
+
         now = int(time.time())
         new_credentials = OAuth2Credentials(
             provider=self.PROVIDER_NAME,
-            title=current_credentials.title if current_credentials else "GitHub",
+            title=current_credentials.title if current_credentials else None,
+            username=username,
             access_token=token_data["access_token"],
             # Token refresh responses have an empty `scope` property (see docs),
             # so we have to get the scope from the existing credentials object.
@@ -97,3 +101,19 @@ class GitHubOAuthHandler(BaseOAuthHandler):
         if current_credentials:
             new_credentials.id = current_credentials.id
         return new_credentials
+
+    def _request_username(self, access_token: str) -> str | None:
+        url = "https://api.github.com/user"
+        headers = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {access_token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+
+        response = requests.get(url, headers=headers)
+
+        if not response.ok:
+            return None
+
+        # Get the login (username)
+        return response.json().get("login")