From 505320fcd39901dc057e61fcd7c1f30a8a39ea21 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Mon, 5 May 2025 13:59:28 -0500 Subject: [PATCH] feat(backend): Move Scheduler (#9904) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want the scheduler shouldn't scale with the rest API lol ### Changes 🏗️ pulls out the scheduler into its own service ### Checklist 📋 #### For code changes: - [x] I have clearly listed my changes in the PR description - [x] I have made a test plan - [x] I have tested my changes according to the test plan: - [x] test it --------- Co-authored-by: Zamil Majdy --- autogpt_platform/backend/backend/rest.py | 3 +- autogpt_platform/backend/backend/scheduler.py | 13 ++++ autogpt_platform/backend/pyproject.toml | 1 + autogpt_platform/docker-compose.platform.yml | 60 ++++++++++++++++++- autogpt_platform/docker-compose.yml | 16 +++-- docs/content/platform/getting-started.md | 2 +- 6 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 autogpt_platform/backend/backend/scheduler.py diff --git a/autogpt_platform/backend/backend/rest.py b/autogpt_platform/backend/backend/rest.py index a20df1091d..0fb1eed875 100644 --- a/autogpt_platform/backend/backend/rest.py +++ b/autogpt_platform/backend/backend/rest.py @@ -1,5 +1,5 @@ from backend.app import run_processes -from backend.executor import DatabaseManager, Scheduler +from backend.executor import DatabaseManager from backend.notifications.notifications import NotificationManager from backend.server.rest_api import AgentServer @@ -11,7 +11,6 @@ def main(): run_processes( NotificationManager(), DatabaseManager(), - Scheduler(), AgentServer(), ) diff --git a/autogpt_platform/backend/backend/scheduler.py b/autogpt_platform/backend/backend/scheduler.py new file mode 100644 index 0000000000..22be4bf7fd --- /dev/null +++ b/autogpt_platform/backend/backend/scheduler.py @@ -0,0 +1,13 @@ +from backend.app import run_processes +from backend.executor.scheduler import Scheduler + + +def main(): + """ + Run all the processes required for the AutoGPT-server Scheduling System. + """ + run_processes(Scheduler()) + + +if __name__ == "__main__": + main() diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.toml index bd47502269..3460c31f80 100644 --- a/autogpt_platform/backend/pyproject.toml +++ b/autogpt_platform/backend/pyproject.toml @@ -88,6 +88,7 @@ build-backend = "poetry.core.masonry.api" app = "backend.app:main" rest = "backend.rest:main" ws = "backend.ws:main" +scheduler = "backend.scheduler:main" executor = "backend.exec:main" cli = "backend.cli:main" format = "linter:format" diff --git a/autogpt_platform/docker-compose.platform.yml b/autogpt_platform/docker-compose.platform.yml index 21064a900a..cc5e8c3d6a 100644 --- a/autogpt_platform/docker-compose.platform.yml +++ b/autogpt_platform/docker-compose.platform.yml @@ -73,6 +73,8 @@ services: condition: service_completed_successfully rabbitmq: condition: service_healthy + # scheduler_server: + # condition: service_healthy environment: - SUPABASE_URL=http://kong:8000 - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long @@ -88,7 +90,7 @@ services: - REDIS_PASSWORD=password - ENABLE_AUTH=true - PYRO_HOST=0.0.0.0 - - EXECUTIONSCHEDULER_HOST=rest_server + - SCHEDULER_HOST=scheduler_server - EXECUTIONMANAGER_HOST=executor - NOTIFICATIONMANAGER_HOST=rest_server - FRONTEND_BASE_URL=http://localhost:3000 @@ -98,7 +100,6 @@ services: ports: - "8006:8006" - "8007:8007" - - "8003:8003" # execution scheduler networks: - app-network @@ -187,6 +188,61 @@ services: networks: - app-network + scheduler_server: + build: + context: ../ + dockerfile: autogpt_platform/backend/Dockerfile + target: server + command: ["python", "-m", "backend.scheduler"] + develop: + watch: + - path: ./ + target: autogpt_platform/backend/ + action: rebuild + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + rabbitmq: + condition: service_healthy + migrate: + condition: service_completed_successfully + # healthcheck: + # test: + # [ + # "CMD", + # "curl", + # "-f", + # "-X", + # "POST", + # "http://localhost:8003/health_check", + # ] + # interval: 10s + # timeout: 10s + # retries: 5 + environment: + - DATABASEMANAGER_HOST=rest_server + - NOTIFICATIONMANAGER_HOST=rest_server + - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long + - DATABASE_URL=postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres?connect_timeout=60&schema=platform + - DIRECT_URL=postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres?connect_timeout=60&schema=platform + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=password + - RABBITMQ_HOST=rabbitmq + - RABBITMQ_PORT=5672 + - RABBITMQ_DEFAULT_USER=rabbitmq_user_default + - RABBITMQ_DEFAULT_PASS=k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7 + - ENABLE_AUTH=true + - PYRO_HOST=0.0.0.0 + - BACKEND_CORS_ALLOW_ORIGINS=["http://localhost:3000"] + + ports: + - "8003:8003" + networks: + - app-network + # frontend: # build: # context: ../ diff --git a/autogpt_platform/docker-compose.yml b/autogpt_platform/docker-compose.yml index 5863357fc7..1fcce8e24c 100644 --- a/autogpt_platform/docker-compose.yml +++ b/autogpt_platform/docker-compose.yml @@ -57,11 +57,17 @@ services: file: ./docker-compose.platform.yml service: websocket_server -# frontend: -# <<: *agpt-services -# extends: -# file: ./docker-compose.platform.yml -# service: frontend + scheduler_server: + <<: *agpt-services + extends: + file: ./docker-compose.platform.yml + service: scheduler_server + + # frontend: + # <<: *agpt-services + # extends: + # file: ./docker-compose.platform.yml + # service: frontend # Supabase services studio: diff --git a/docs/content/platform/getting-started.md b/docs/content/platform/getting-started.md index c66d922f0f..1cfaac7aed 100644 --- a/docs/content/platform/getting-started.md +++ b/docs/content/platform/getting-started.md @@ -382,7 +382,7 @@ Currently, there are only 3 active services: - AgentServer (the API, defined in `server.py`) - ExecutionManager (the executor, defined in `manager.py`) -- ExecutionScheduler (the scheduler, defined in `scheduler.py`) +- Scheduler (the scheduler, defined in `scheduler.py`) The services run in independent Python processes and communicate through an IPC. A communication layer (`service.py`) is created to decouple the communication library from the implementation.