mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-02-10 23:05:17 -05:00
## Summary Implement comprehensive k6 load testing infrastructure for the AutoGPT Platform with clean file organization, unified test runner, and cloud integration. ## Key Features ### 🗂️ Clean File Organization - **tests/basic/**: Simple validation tests (connectivity, single endpoints) - **tests/api/**: Core functionality tests (API endpoints, graph execution) - **tests/marketplace/**: User-facing feature tests (public/library access) - **tests/comprehensive/**: End-to-end scenario tests (complete user journeys) - **orchestrator/**: Advanced test orchestration for full suites ### 🚀 Unified Test Runner - **Single entry point**: `run-tests.js` for both local and cloud execution - **7 available tests**: From basic connectivity to comprehensive platform journeys - **Flexible execution**: Run individual tests, comma-separated lists, or all tests - **Auto-configuration**: Different VU/duration settings for local vs cloud execution ### 🔐 Advanced Authentication - **Pre-authenticated tokens**: 24-hour JWT tokens eliminate Supabase rate limiting - **Configurable generation**: Default 10 tokens, scalable to 150+ for high concurrency - **Graceful handling**: Proper auth failure detection and recovery - **ES module compatible**: Modern JavaScript with full import/export support ### ☁️ k6 Cloud Integration - **Cloud execution**: Tests run on k6 cloud infrastructure for consistent results - **Real-time monitoring**: Live dashboards with performance metrics - **URL tracking**: Automatic test result URL capture and storage - **Sequential orchestration**: Proper delays between tests for resource management ## Test Coverage ### Performance Validated - **Core API**: 100 VUs successfully testing `/api/credits`, `/api/graphs`, `/api/blocks`, `/api/executions` - **Graph Execution**: 80 VUs for complete workflow pipeline testing - **Marketplace**: 150 VUs for public browsing, 100 VUs for authenticated library operations - **Authentication**: 150+ concurrent users with pre-authenticated token scaling ### User Journey Simulation - **Dashboard workflows**: Credits checking, graph management, execution monitoring - **Marketplace browsing**: Public search, agent discovery, category filtering - **Library operations**: Agent adding, favoriting, forking, detailed views - **Complete workflows**: End-to-end platform usage with realistic user behavior ## Technical Implementation ### ES Module Compatibility - Full ES module support with modern JavaScript imports/exports - Proper module execution patterns for Node.js compatibility - Clean separation between CommonJS legacy and modern ES modules ### Error Handling & Monitoring - **Separate metrics**: HTTP status, authentication, JSON validation, overall success - **Graceful degradation**: Auth failures don't crash VUs, proper error tracking - **Performance thresholds**: Configurable P95/P99 latency and error rate limits - **Custom counters**: Track operation types, success rates, user journey completion ### Infrastructure Benefits - **Rate limit protection**: Pre-auth tokens prevent Supabase auth bottlenecks - **Scalable testing**: Support for 150+ concurrent users with proper token management - **Cloud consistency**: Tests run on dedicated k6 cloud servers for reliable results - **Development workflow**: Local execution for debugging, cloud for performance validation ## Usage ### Quick Start ```bash # Setup and verification export SUPABASE_SERVICE_KEY="your-service-key" node generate-tokens.js node run-tests.js verify # Local testing (development) node run-tests.js run core-api-test DEV # Cloud testing (performance) node run-tests.js cloud all DEV ``` ### NPM Scripts ```bash npm run verify # Quick setup check npm test # All tests locally npm run cloud # All tests in k6 cloud ``` ## Validation Results ✅ **Authentication**: 100% success rate with fresh 24-hour tokens ✅ **File Structure**: All imports and references verified correct ✅ **Test Execution**: All 7 tests execute successfully with proper metrics ✅ **Cloud Integration**: k6 cloud execution working with proper credentials ✅ **Documentation**: Complete README with usage examples and troubleshooting ## Files Changed ### Core Infrastructure - `run-tests.js`: Unified test runner supporting local/cloud execution - `generate-tokens.js`: ES module compatible token generation with 24-hour expiry - `README.md`: Comprehensive documentation with updated file references ### Organized Test Structure - `tests/basic/connectivity-test.js`: Basic connectivity validation - `tests/basic/single-endpoint-test.js`: Individual API endpoint testing - `tests/api/core-api-test.js`: Core authenticated API endpoints - `tests/api/graph-execution-test.js`: Graph workflow pipeline testing - `tests/marketplace/public-access-test.js`: Public marketplace browsing - `tests/marketplace/library-access-test.js`: Authenticated marketplace/library operations - `tests/comprehensive/platform-journey-test.js`: Complete user journey simulation ### Configuration - `configs/environment.js`: Environment URLs and performance settings - `package.json`: NPM scripts and dependencies for unified workflow This infrastructure provides a solid foundation for continuous performance monitoring and load testing of the AutoGPT Platform. 🤖 Generated with [Claude Code](https://claude.ai/code) --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nicholas Tindle <nicholas.tindle@agpt.co> Co-authored-by: Reinier van der Leer <pwuts@agpt.co>
142 lines
5.2 KiB
JavaScript
142 lines
5.2 KiB
JavaScript
// Environment configuration for AutoGPT Platform load tests
|
|
export const ENV_CONFIG = {
|
|
DEV: {
|
|
API_BASE_URL: "https://dev-server.agpt.co",
|
|
BUILDER_BASE_URL: "https://dev-builder.agpt.co",
|
|
WS_BASE_URL: "wss://dev-ws-server.agpt.co",
|
|
SUPABASE_URL: "https://adfjtextkuilwuhzdjpf.supabase.co",
|
|
SUPABASE_ANON_KEY:
|
|
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFkZmp0ZXh0a3VpbHd1aHpkanBmIiwicm9sZSI6ImFub24iLCJpYXQiOjE3MzAyNTE3MDIsImV4cCI6MjA0NTgyNzcwMn0.IuQNXsHEKJNxtS9nyFeqO0BGMYN8sPiObQhuJLSK9xk",
|
|
},
|
|
LOCAL: {
|
|
API_BASE_URL: "http://localhost:8006",
|
|
BUILDER_BASE_URL: "http://localhost:3000",
|
|
WS_BASE_URL: "ws://localhost:8001",
|
|
SUPABASE_URL: "http://localhost:8000",
|
|
SUPABASE_ANON_KEY:
|
|
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE",
|
|
},
|
|
PROD: {
|
|
API_BASE_URL: "https://api.agpt.co",
|
|
BUILDER_BASE_URL: "https://builder.agpt.co",
|
|
WS_BASE_URL: "wss://ws-server.agpt.co",
|
|
SUPABASE_URL: "https://supabase.agpt.co",
|
|
SUPABASE_ANON_KEY:
|
|
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImJnd3B3ZHN4YmxyeWloaW51dGJ4Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3MzAyODYzMDUsImV4cCI6MjA0NTg2MjMwNX0.ISa2IofTdQIJmmX5JwKGGNajqjsD8bjaGBzK90SubE0",
|
|
},
|
|
};
|
|
|
|
// Get environment config based on K6_ENVIRONMENT variable (default: DEV)
|
|
export function getEnvironmentConfig() {
|
|
const env = __ENV.K6_ENVIRONMENT || "DEV";
|
|
return ENV_CONFIG[env];
|
|
}
|
|
|
|
// Authentication configuration
|
|
export const AUTH_CONFIG = {
|
|
// Test user credentials - REPLACE WITH ACTUAL TEST ACCOUNTS
|
|
TEST_USERS: [
|
|
{
|
|
email: "loadtest1@example.com",
|
|
password: "LoadTest123!",
|
|
user_id: "test-user-1",
|
|
},
|
|
{
|
|
email: "loadtest2@example.com",
|
|
password: "LoadTest123!",
|
|
user_id: "test-user-2",
|
|
},
|
|
{
|
|
email: "loadtest3@example.com",
|
|
password: "LoadTest123!",
|
|
user_id: "test-user-3",
|
|
},
|
|
],
|
|
|
|
// JWT token for API access (will be set during test execution)
|
|
JWT_TOKEN: null,
|
|
};
|
|
|
|
// Performance test configurations - Environment variable overrides supported
|
|
export const PERFORMANCE_CONFIG = {
|
|
// Default load test parameters (override with env vars: VUS, DURATION, RAMP_UP, RAMP_DOWN)
|
|
DEFAULT_VUS: parseInt(__ENV.VUS) || 10,
|
|
DEFAULT_DURATION: __ENV.DURATION || "2m",
|
|
DEFAULT_RAMP_UP: __ENV.RAMP_UP || "30s",
|
|
DEFAULT_RAMP_DOWN: __ENV.RAMP_DOWN || "30s",
|
|
|
|
// Stress test parameters (override with env vars: STRESS_VUS, STRESS_DURATION, etc.)
|
|
STRESS_VUS: parseInt(__ENV.STRESS_VUS) || 50,
|
|
STRESS_DURATION: __ENV.STRESS_DURATION || "5m",
|
|
STRESS_RAMP_UP: __ENV.STRESS_RAMP_UP || "1m",
|
|
STRESS_RAMP_DOWN: __ENV.STRESS_RAMP_DOWN || "1m",
|
|
|
|
// Spike test parameters (override with env vars: SPIKE_VUS, SPIKE_DURATION, etc.)
|
|
SPIKE_VUS: parseInt(__ENV.SPIKE_VUS) || 100,
|
|
SPIKE_DURATION: __ENV.SPIKE_DURATION || "30s",
|
|
SPIKE_RAMP_UP: __ENV.SPIKE_RAMP_UP || "10s",
|
|
SPIKE_RAMP_DOWN: __ENV.SPIKE_RAMP_DOWN || "10s",
|
|
|
|
// Volume test parameters (override with env vars: VOLUME_VUS, VOLUME_DURATION, etc.)
|
|
VOLUME_VUS: parseInt(__ENV.VOLUME_VUS) || 20,
|
|
VOLUME_DURATION: __ENV.VOLUME_DURATION || "10m",
|
|
VOLUME_RAMP_UP: __ENV.VOLUME_RAMP_UP || "2m",
|
|
VOLUME_RAMP_DOWN: __ENV.VOLUME_RAMP_DOWN || "2m",
|
|
|
|
// SLA thresholds (adjustable via env vars: THRESHOLD_P95, THRESHOLD_P99, etc.)
|
|
THRESHOLDS: {
|
|
http_req_duration: [
|
|
`p(95)<${__ENV.THRESHOLD_P95 || "2000"}`,
|
|
`p(99)<${__ENV.THRESHOLD_P99 || "5000"}`,
|
|
],
|
|
http_req_failed: [`rate<${__ENV.THRESHOLD_ERROR_RATE || "0.05"}`],
|
|
http_reqs: [`rate>${__ENV.THRESHOLD_RPS || "10"}`],
|
|
checks: [`rate>${__ENV.THRESHOLD_CHECK_RATE || "0.95"}`],
|
|
},
|
|
};
|
|
|
|
// Helper function to get load test configuration based on test type
|
|
export function getLoadTestConfig(testType = "default") {
|
|
const configs = {
|
|
default: {
|
|
vus: PERFORMANCE_CONFIG.DEFAULT_VUS,
|
|
duration: PERFORMANCE_CONFIG.DEFAULT_DURATION,
|
|
rampUp: PERFORMANCE_CONFIG.DEFAULT_RAMP_UP,
|
|
rampDown: PERFORMANCE_CONFIG.DEFAULT_RAMP_DOWN,
|
|
},
|
|
stress: {
|
|
vus: PERFORMANCE_CONFIG.STRESS_VUS,
|
|
duration: PERFORMANCE_CONFIG.STRESS_DURATION,
|
|
rampUp: PERFORMANCE_CONFIG.STRESS_RAMP_UP,
|
|
rampDown: PERFORMANCE_CONFIG.STRESS_RAMP_DOWN,
|
|
},
|
|
spike: {
|
|
vus: PERFORMANCE_CONFIG.SPIKE_VUS,
|
|
duration: PERFORMANCE_CONFIG.SPIKE_DURATION,
|
|
rampUp: PERFORMANCE_CONFIG.SPIKE_RAMP_UP,
|
|
rampDown: PERFORMANCE_CONFIG.SPIKE_RAMP_DOWN,
|
|
},
|
|
volume: {
|
|
vus: PERFORMANCE_CONFIG.VOLUME_VUS,
|
|
duration: PERFORMANCE_CONFIG.VOLUME_DURATION,
|
|
rampUp: PERFORMANCE_CONFIG.VOLUME_RAMP_UP,
|
|
rampDown: PERFORMANCE_CONFIG.VOLUME_RAMP_DOWN,
|
|
},
|
|
};
|
|
|
|
return configs[testType] || configs.default;
|
|
}
|
|
|
|
// Grafana Cloud K6 configuration
|
|
export const GRAFANA_CONFIG = {
|
|
PROJECT_ID: __ENV.K6_CLOUD_PROJECT_ID || "",
|
|
TOKEN: __ENV.K6_CLOUD_TOKEN || "",
|
|
// Tags for organizing test results
|
|
TEST_TAGS: {
|
|
team: "platform",
|
|
service: "autogpt-platform",
|
|
environment: __ENV.K6_ENVIRONMENT || "dev",
|
|
version: __ENV.GIT_COMMIT || "unknown",
|
|
},
|
|
};
|