mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-04-08 03:00:28 -04:00
Updated claude removed extra docs
This commit is contained in:
SwiftyOS
@@ -2,6 +2,238 @@
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Block Development with SDK
|
||||
|
||||
The AutoGPT Platform now includes a comprehensive SDK that dramatically simplifies block creation. Blocks can be fully self-contained with zero external configuration required.
|
||||
|
||||
### Quick Start - Creating a New Block
|
||||
|
||||
```python
|
||||
from backend.sdk import *
|
||||
|
||||
@provider("my-service") # Auto-registers new provider
|
||||
@cost_config(
|
||||
BlockCost(cost_amount=5, cost_type=BlockCostType.RUN),
|
||||
BlockCost(cost_amount=1, cost_type=BlockCostType.BYTE)
|
||||
)
|
||||
@default_credentials(
|
||||
APIKeyCredentials(
|
||||
id="my-service-default",
|
||||
provider="my-service",
|
||||
api_key=SecretStr("default-api-key"),
|
||||
title="My Service Default API Key"
|
||||
)
|
||||
)
|
||||
class MyServiceBlock(Block):
|
||||
class Input(BlockSchema):
|
||||
credentials: CredentialsMetaInput = CredentialsField(
|
||||
provider="my-service",
|
||||
supported_credential_types={"api_key"}
|
||||
)
|
||||
text: String = SchemaField(description="Input text")
|
||||
|
||||
class Output(BlockSchema):
|
||||
result: String = SchemaField(description="Output result")
|
||||
error: String = SchemaField(description="Error message", default="")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="my-service-block-12345678-1234-1234-1234-123456789012",
|
||||
description="Process text using My Service",
|
||||
categories={BlockCategory.TEXT},
|
||||
input_schema=MyServiceBlock.Input,
|
||||
output_schema=MyServiceBlock.Output,
|
||||
)
|
||||
|
||||
def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:
|
||||
try:
|
||||
api_key = credentials.api_key.get_secret_value()
|
||||
# Process with API
|
||||
yield "result", f"Processed: {input_data.text}"
|
||||
except Exception as e:
|
||||
yield "error", str(e)
|
||||
```
|
||||
|
||||
### Key Features
|
||||
|
||||
1. **Single Import**: `from backend.sdk import *` provides everything needed
|
||||
2. **Auto-Registration**: No manual configuration files to update
|
||||
3. **Dynamic Providers**: Any string works as a provider name
|
||||
4. **Self-Contained**: All configuration via decorators
|
||||
|
||||
### Available Decorators
|
||||
|
||||
- `@provider("name")` - Register new provider
|
||||
- `@cost_config(...)` - Set block execution costs
|
||||
- `@default_credentials(...)` - Provide default API credentials
|
||||
- `@webhook_config("provider", ManagerClass)` - Register webhook manager
|
||||
- `@oauth_config("provider", HandlerClass)` - Register OAuth handler
|
||||
|
||||
### Creating Blocks with Webhooks
|
||||
|
||||
```python
|
||||
from backend.sdk import *
|
||||
|
||||
# First, create webhook manager
|
||||
class MyWebhookManager(BaseWebhooksManager):
|
||||
PROVIDER_NAME = "my-service"
|
||||
|
||||
class WebhookType(str, Enum):
|
||||
DATA_UPDATE = "data_update"
|
||||
|
||||
async def validate_payload(self, webhook, request) -> tuple[dict, str]:
|
||||
payload = await request.json()
|
||||
event_type = request.headers.get("X-MyService-Event", "unknown")
|
||||
return payload, event_type
|
||||
|
||||
async def _register_webhook(self, webhook, credentials) -> tuple[str, dict]:
|
||||
# Register with external service
|
||||
return "webhook-id", {"status": "registered"}
|
||||
|
||||
async def _deregister_webhook(self, webhook, credentials) -> None:
|
||||
# Deregister from external service
|
||||
pass
|
||||
|
||||
# Then create webhook block
|
||||
@provider("my-service")
|
||||
@webhook_config("my-service", MyWebhookManager)
|
||||
class MyWebhookBlock(Block):
|
||||
class Input(BlockSchema):
|
||||
events: BaseModel = SchemaField(
|
||||
description="Events to listen for",
|
||||
default={"data_update": True}
|
||||
)
|
||||
payload: Dict = SchemaField(
|
||||
description="Webhook payload",
|
||||
default={},
|
||||
hidden=True
|
||||
)
|
||||
|
||||
class Output(BlockSchema):
|
||||
event_type: String = SchemaField(description="Event type")
|
||||
event_data: Dict = SchemaField(description="Event data")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="my-webhook-block-12345678-1234-1234-1234-123456789012",
|
||||
description="Listen for My Service webhooks",
|
||||
categories={BlockCategory.INPUT},
|
||||
input_schema=MyWebhookBlock.Input,
|
||||
output_schema=MyWebhookBlock.Output,
|
||||
block_type=BlockType.WEBHOOK,
|
||||
webhook_config=BlockWebhookConfig(
|
||||
provider="my-service",
|
||||
webhook_type="data_update",
|
||||
event_filter_input="events",
|
||||
),
|
||||
)
|
||||
|
||||
def run(self, input_data: Input, **kwargs) -> BlockOutput:
|
||||
payload = input_data.payload
|
||||
yield "event_type", payload.get("type", "unknown")
|
||||
yield "event_data", payload
|
||||
```
|
||||
|
||||
### Creating Blocks with OAuth
|
||||
|
||||
```python
|
||||
from backend.sdk import *
|
||||
|
||||
# First, create OAuth handler
|
||||
class MyServiceOAuthHandler(BaseOAuthHandler):
|
||||
PROVIDER_NAME = "my-service"
|
||||
DEFAULT_SCOPES = ["read", "write"]
|
||||
|
||||
def get_login_url(self, scopes: list[str], state: str, code_challenge: Optional[str]) -> str:
|
||||
# Build OAuth authorization URL
|
||||
return f"https://my-service.com/oauth/authorize?..."
|
||||
|
||||
def exchange_code_for_tokens(self, code: str, scopes: list[str], code_verifier: Optional[str]) -> OAuth2Credentials:
|
||||
# Exchange authorization code for tokens
|
||||
return OAuth2Credentials(
|
||||
provider="my-service",
|
||||
access_token=SecretStr("access-token"),
|
||||
refresh_token=SecretStr("refresh-token"),
|
||||
scopes=scopes,
|
||||
access_token_expires_at=int(time.time() + 3600)
|
||||
)
|
||||
|
||||
# Then create OAuth-enabled block
|
||||
@provider("my-service")
|
||||
@oauth_config("my-service", MyServiceOAuthHandler)
|
||||
class MyOAuthBlock(Block):
|
||||
class Input(BlockSchema):
|
||||
credentials: CredentialsMetaInput = CredentialsField(
|
||||
provider="my-service",
|
||||
supported_credential_types={"oauth2"},
|
||||
required_scopes={"read", "write"}
|
||||
)
|
||||
action: String = SchemaField(description="Action to perform")
|
||||
|
||||
class Output(BlockSchema):
|
||||
result: Dict = SchemaField(description="API response")
|
||||
error: String = SchemaField(description="Error message", default="")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="my-oauth-block-12345678-1234-1234-1234-123456789012",
|
||||
description="Interact with My Service using OAuth",
|
||||
categories={BlockCategory.DEVELOPER_TOOLS},
|
||||
input_schema=MyOAuthBlock.Input,
|
||||
output_schema=MyOAuthBlock.Output,
|
||||
)
|
||||
|
||||
def run(self, input_data: Input, *, credentials: OAuth2Credentials, **kwargs) -> BlockOutput:
|
||||
try:
|
||||
headers = {"Authorization": f"Bearer {credentials.access_token.get_secret_value()}"}
|
||||
# Make API call with OAuth token
|
||||
yield "result", {"status": "success", "action": input_data.action}
|
||||
except Exception as e:
|
||||
yield "error", str(e)
|
||||
```
|
||||
|
||||
### SDK Components Available
|
||||
|
||||
The SDK provides 68+ components via `from backend.sdk import *`:
|
||||
|
||||
**Core Block Components:**
|
||||
- `Block`, `BlockSchema`, `BlockOutput`, `BlockCategory`, `BlockType`
|
||||
- `SchemaField`, `CredentialsField`, `CredentialsMetaInput`
|
||||
|
||||
**Credential Types:**
|
||||
- `APIKeyCredentials`, `OAuth2Credentials`, `UserPasswordCredentials`
|
||||
|
||||
**Cost System:**
|
||||
- `BlockCost`, `BlockCostType`, `NodeExecutionStats`
|
||||
|
||||
**Type Aliases:**
|
||||
- `String`, `Integer`, `Float`, `Boolean` (cleaner than str, int, etc.)
|
||||
|
||||
**Common Types:**
|
||||
- `List`, `Dict`, `Optional`, `Any`, `Union`, `BaseModel`, `SecretStr`
|
||||
|
||||
**Utilities:**
|
||||
- `json`, `logging`, `store_media_file`, `MediaFileType`
|
||||
|
||||
### Best Practices
|
||||
|
||||
1. **Use UUID for Block ID**: Generate a unique UUID for each block
|
||||
2. **Handle Errors**: Always include error handling in the run method
|
||||
3. **Yield All Outputs**: Ensure all output schema fields are yielded
|
||||
4. **Test Your Block**: Include test_input and test_output in __init__
|
||||
5. **Document Well**: Provide clear descriptions for the block and all fields
|
||||
|
||||
### No Manual Configuration Needed
|
||||
|
||||
With the SDK, you never need to manually update these files:
|
||||
- ❌ `backend/data/block_cost_config.py`
|
||||
- ❌ `backend/integrations/credentials_store.py`
|
||||
- ❌ `backend/integrations/providers.py`
|
||||
- ❌ `backend/integrations/oauth/__init__.py`
|
||||
- ❌ `backend/integrations/webhooks/__init__.py`
|
||||
|
||||
Everything is handled automatically by the decorators!
|
||||
|
||||
## Project Architecture
|
||||
|
||||
The AutoGPT Platform is a microservice-based system for creating and running AI-powered agent workflows. It consists of three main components:
|
||||
|
||||
@@ -1,582 +0,0 @@
|
||||
# AutoGPT Platform Block SDK - Complete Simplification Plan
|
||||
|
||||
## Executive Summary
|
||||
|
||||
This plan provides a **complete solution** to simplify block development by:
|
||||
|
||||
1. **Single Import**: `from backend.sdk import *` provides everything needed for block development
|
||||
2. **Zero External Changes**: Adding new blocks requires **no code changes outside the blocks folder**
|
||||
3. **Auto-Registration**: Credentials, costs, OAuth, and webhooks are automatically discovered and registered
|
||||
|
||||
## Current Problem Analysis
|
||||
|
||||
### Manual Registration Locations (Must Be Eliminated)
|
||||
|
||||
Currently, adding a new block requires manual updates in **5+ files outside the blocks folder**:
|
||||
|
||||
1. **`backend/data/block_cost_config.py`**: Block cost configurations (lines 184-300)
|
||||
2. **`backend/integrations/credentials_store.py`**: Default credentials (lines 188-210)
|
||||
3. **`backend/integrations/oauth/__init__.py`**: OAuth handler registration (lines 16-26)
|
||||
4. **`backend/integrations/webhooks/__init__.py`**: Webhook manager registration (lines 20-30)
|
||||
5. **`backend/integrations/providers.py`**: Provider name enum (lines 6-43)
|
||||
|
||||
### Import Complexity Problem
|
||||
|
||||
Current blocks require **8-15 import statements** from various backend modules:
|
||||
```python
|
||||
from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
|
||||
from backend.data.model import SchemaField, CredentialsField, CredentialsMetaInput
|
||||
from backend.integrations.providers import ProviderName
|
||||
from backend.blocks.github._auth import GithubCredentials, GithubCredentialsField
|
||||
from backend.data.cost import BlockCost, BlockCostType
|
||||
# ... and more
|
||||
```
|
||||
|
||||
## Proposed Solution: Complete SDK with Auto-Registration
|
||||
|
||||
### 1. SDK Module Structure
|
||||
|
||||
```
|
||||
backend/
|
||||
├── sdk/
|
||||
│ ├── __init__.py # Complete re-export of all block dependencies
|
||||
│ ├── auto_registry.py # Auto-registration system for costs/credentials/webhooks
|
||||
│ └── decorators.py # Registration decorators for blocks
|
||||
```
|
||||
|
||||
### 2. Complete Re-Export System (`backend/sdk/__init__.py`)
|
||||
|
||||
```python
|
||||
"""
|
||||
AutoGPT Platform Block Development SDK
|
||||
|
||||
Complete re-export of all dependencies needed for block development.
|
||||
Usage: from backend.sdk import *
|
||||
|
||||
This module provides:
|
||||
- All block base classes and types
|
||||
- All credential and authentication components
|
||||
- All cost tracking components
|
||||
- All webhook components
|
||||
- All utility functions
|
||||
- Auto-registration decorators
|
||||
"""
|
||||
|
||||
# === CORE BLOCK SYSTEM ===
|
||||
from backend.data.block import (
|
||||
Block, BlockCategory, BlockOutput, BlockSchema, BlockType,
|
||||
BlockWebhookConfig, BlockManualWebhookConfig
|
||||
)
|
||||
from backend.data.model import (
|
||||
SchemaField, CredentialsField, CredentialsMetaInput,
|
||||
APIKeyCredentials, OAuth2Credentials, UserPasswordCredentials,
|
||||
NodeExecutionStats
|
||||
)
|
||||
|
||||
# === INTEGRATIONS ===
|
||||
from backend.integrations.providers import ProviderName
|
||||
from backend.integrations.webhooks._base import BaseWebhooksManager, ManualWebhookManagerBase
|
||||
|
||||
# === COST SYSTEM ===
|
||||
from backend.data.cost import BlockCost, BlockCostType
|
||||
from backend.data.credit import UsageTransactionMetadata
|
||||
from backend.executor.utils import block_usage_cost
|
||||
|
||||
# === UTILITIES ===
|
||||
from backend.util import json
|
||||
from backend.util.file import store_media_file
|
||||
from backend.util.type import MediaFileType, convert
|
||||
from backend.util.text import TextFormatter
|
||||
from backend.util.logging import TruncatedLogger
|
||||
|
||||
# === COMMON TYPES ===
|
||||
from typing import Any, Dict, List, Literal, Optional, Union, TypeVar, Type
|
||||
from pydantic import BaseModel, SecretStr, Field
|
||||
from enum import Enum
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
# === TYPE ALIASES ===
|
||||
String = str
|
||||
Integer = int
|
||||
Float = float
|
||||
Boolean = bool
|
||||
|
||||
# === AUTO-REGISTRATION DECORATORS ===
|
||||
from .decorators import (
|
||||
register_credentials, register_cost, register_oauth, register_webhook_manager,
|
||||
provider, cost_config, webhook_config, default_credentials
|
||||
)
|
||||
|
||||
# === RE-EXPORT PROVIDER-SPECIFIC COMPONENTS ===
|
||||
# Dynamically import and re-export provider-specific components
|
||||
try:
|
||||
from backend.blocks.github._auth import (
|
||||
GithubCredentials, GithubCredentialsInput, GithubCredentialsField
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from backend.blocks.google._auth import (
|
||||
GoogleCredentials, GoogleCredentialsInput, GoogleCredentialsField
|
||||
)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from backend.integrations.oauth.github import GitHubOAuthHandler
|
||||
from backend.integrations.oauth.google import GoogleOAuthHandler
|
||||
from backend.integrations.oauth.base import BaseOAuthHandler
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from backend.integrations.webhooks.github import GitHubWebhooksManager
|
||||
from backend.integrations.webhooks.generic import GenericWebhookManager
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# === COMPREHENSIVE __all__ EXPORT ===
|
||||
__all__ = [
|
||||
# Core Block System
|
||||
"Block", "BlockCategory", "BlockOutput", "BlockSchema", "BlockType",
|
||||
"BlockWebhookConfig", "BlockManualWebhookConfig",
|
||||
|
||||
# Schema and Model Components
|
||||
"SchemaField", "CredentialsField", "CredentialsMetaInput",
|
||||
"APIKeyCredentials", "OAuth2Credentials", "UserPasswordCredentials",
|
||||
"NodeExecutionStats",
|
||||
|
||||
# Cost System
|
||||
"BlockCost", "BlockCostType", "UsageTransactionMetadata", "block_usage_cost",
|
||||
|
||||
# Integrations
|
||||
"ProviderName", "BaseWebhooksManager", "ManualWebhookManagerBase",
|
||||
|
||||
# Provider-Specific (when available)
|
||||
"GithubCredentials", "GithubCredentialsInput", "GithubCredentialsField",
|
||||
"GoogleCredentials", "GoogleCredentialsInput", "GoogleCredentialsField",
|
||||
"BaseOAuthHandler", "GitHubOAuthHandler", "GoogleOAuthHandler",
|
||||
"GitHubWebhooksManager", "GenericWebhookManager",
|
||||
|
||||
# Utilities
|
||||
"json", "store_media_file", "MediaFileType", "convert", "TextFormatter",
|
||||
"TruncatedLogger", "logging", "asyncio",
|
||||
|
||||
# Types
|
||||
"String", "Integer", "Float", "Boolean", "List", "Dict", "Optional",
|
||||
"Any", "Literal", "Union", "TypeVar", "Type", "BaseModel", "SecretStr",
|
||||
"Field", "Enum",
|
||||
|
||||
# Auto-Registration Decorators
|
||||
"register_credentials", "register_cost", "register_oauth", "register_webhook_manager",
|
||||
"provider", "cost_config", "webhook_config", "default_credentials",
|
||||
]
|
||||
```
|
||||
|
||||
### 3. Auto-Registration System (`backend/sdk/auto_registry.py`)
|
||||
|
||||
```python
|
||||
"""
|
||||
Auto-Registration System for AutoGPT Platform
|
||||
|
||||
Automatically discovers and registers:
|
||||
- Block costs
|
||||
- Default credentials
|
||||
- OAuth handlers
|
||||
- Webhook managers
|
||||
- Provider names
|
||||
|
||||
This eliminates the need to manually update configuration files
|
||||
outside the blocks folder when adding new blocks.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Set, Type, Any
|
||||
import inspect
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# === GLOBAL REGISTRIES ===
|
||||
class AutoRegistry:
|
||||
"""Central registry for auto-discovered block configurations."""
|
||||
|
||||
def __init__(self):
|
||||
self.block_costs: Dict[Type, List] = {}
|
||||
self.default_credentials: List[Any] = []
|
||||
self.oauth_handlers: Dict[str, Type] = {}
|
||||
self.webhook_managers: Dict[str, Type] = {}
|
||||
self.providers: Set[str] = set()
|
||||
|
||||
def register_block_cost(self, block_class: Type, cost_config: List):
|
||||
"""Register cost configuration for a block."""
|
||||
self.block_costs[block_class] = cost_config
|
||||
|
||||
def register_default_credential(self, credential):
|
||||
"""Register a default platform credential."""
|
||||
self.default_credentials.append(credential)
|
||||
|
||||
def register_oauth_handler(self, provider_name: str, handler_class: Type):
|
||||
"""Register an OAuth handler for a provider."""
|
||||
self.oauth_handlers[provider_name] = handler_class
|
||||
|
||||
def register_webhook_manager(self, provider_name: str, manager_class: Type):
|
||||
"""Register a webhook manager for a provider."""
|
||||
self.webhook_managers[provider_name] = manager_class
|
||||
|
||||
def register_provider(self, provider_name: str):
|
||||
"""Register a new provider name."""
|
||||
self.providers.add(provider_name)
|
||||
|
||||
def get_block_costs_dict(self) -> Dict[Type, List]:
|
||||
"""Get block costs in format expected by current system."""
|
||||
return self.block_costs.copy()
|
||||
|
||||
def get_default_credentials_list(self) -> List[Any]:
|
||||
"""Get default credentials in format expected by current system."""
|
||||
return self.default_credentials.copy()
|
||||
|
||||
def get_oauth_handlers_dict(self) -> Dict[str, Type]:
|
||||
"""Get OAuth handlers in format expected by current system."""
|
||||
return self.oauth_handlers.copy()
|
||||
|
||||
def get_webhook_managers_dict(self) -> Dict[str, Type]:
|
||||
"""Get webhook managers in format expected by current system."""
|
||||
return self.webhook_managers.copy()
|
||||
|
||||
# Global registry instance
|
||||
_registry = AutoRegistry()
|
||||
|
||||
def get_registry() -> AutoRegistry:
|
||||
"""Get the global auto-registry instance."""
|
||||
return _registry
|
||||
|
||||
# === DISCOVERY FUNCTIONS ===
|
||||
def discover_block_configurations():
|
||||
"""
|
||||
Discover all block configurations by scanning loaded blocks.
|
||||
Called during application startup after blocks are loaded.
|
||||
"""
|
||||
from backend.blocks import load_all_blocks
|
||||
|
||||
# Load all blocks (this also imports all block modules)
|
||||
load_all_blocks()
|
||||
|
||||
# Registry is populated by decorators during import
|
||||
return _registry
|
||||
|
||||
def patch_existing_systems():
|
||||
"""
|
||||
Patch existing configuration systems to use auto-discovered data.
|
||||
This maintains backward compatibility while enabling auto-registration.
|
||||
"""
|
||||
|
||||
# Patch block cost configuration
|
||||
import backend.data.block_cost_config as cost_config
|
||||
original_block_costs = getattr(cost_config, 'BLOCK_COSTS', {})
|
||||
cost_config.BLOCK_COSTS = {**original_block_costs, **_registry.get_block_costs_dict()}
|
||||
|
||||
# Patch credentials store
|
||||
import backend.integrations.credentials_store as cred_store
|
||||
if hasattr(cred_store, 'DEFAULT_CREDENTIALS'):
|
||||
cred_store.DEFAULT_CREDENTIALS.extend(_registry.get_default_credentials_list())
|
||||
|
||||
# Patch OAuth handlers
|
||||
import backend.integrations.oauth as oauth_module
|
||||
if hasattr(oauth_module, 'HANDLERS_BY_NAME'):
|
||||
oauth_module.HANDLERS_BY_NAME.update(_registry.get_oauth_handlers_dict())
|
||||
|
||||
# Patch webhook managers
|
||||
import backend.integrations.webhooks as webhook_module
|
||||
if hasattr(webhook_module, '_WEBHOOK_MANAGERS'):
|
||||
webhook_module._WEBHOOK_MANAGERS.update(_registry.get_webhook_managers_dict())
|
||||
```
|
||||
|
||||
### 4. Registration Decorators (`backend/sdk/decorators.py`)
|
||||
|
||||
```python
|
||||
"""
|
||||
Registration Decorators for AutoGPT Platform Blocks
|
||||
|
||||
These decorators allow blocks to self-register their configurations:
|
||||
- @cost_config: Register block cost configuration
|
||||
- @default_credentials: Register default platform credentials
|
||||
- @provider: Register new provider name
|
||||
- @webhook_config: Register webhook manager
|
||||
- @oauth_config: Register OAuth handler
|
||||
"""
|
||||
|
||||
from typing import List, Type, Any, Optional
|
||||
from functools import wraps
|
||||
from .auto_registry import get_registry
|
||||
|
||||
def cost_config(*cost_configurations):
|
||||
"""
|
||||
Decorator to register cost configuration for a block.
|
||||
|
||||
Usage:
|
||||
@cost_config(
|
||||
BlockCost(cost_amount=5, cost_type=BlockCostType.RUN),
|
||||
BlockCost(cost_amount=1, cost_type=BlockCostType.BYTE)
|
||||
)
|
||||
class MyBlock(Block):
|
||||
pass
|
||||
"""
|
||||
def decorator(block_class: Type):
|
||||
registry = get_registry()
|
||||
registry.register_block_cost(block_class, list(cost_configurations))
|
||||
return block_class
|
||||
return decorator
|
||||
|
||||
def default_credentials(*credentials):
|
||||
"""
|
||||
Decorator to register default platform credentials.
|
||||
|
||||
Usage:
|
||||
@default_credentials(
|
||||
APIKeyCredentials(provider="myservice", api_key="default-key")
|
||||
)
|
||||
class MyBlock(Block):
|
||||
pass
|
||||
"""
|
||||
def decorator(block_class: Type):
|
||||
registry = get_registry()
|
||||
for credential in credentials:
|
||||
registry.register_default_credential(credential)
|
||||
return block_class
|
||||
return decorator
|
||||
|
||||
def provider(provider_name: str):
|
||||
"""
|
||||
Decorator to register a new provider name.
|
||||
|
||||
Usage:
|
||||
@provider("myservice")
|
||||
class MyBlock(Block):
|
||||
pass
|
||||
"""
|
||||
def decorator(block_class: Type):
|
||||
registry = get_registry()
|
||||
registry.register_provider(provider_name)
|
||||
return block_class
|
||||
return decorator
|
||||
|
||||
def webhook_config(provider_name: str, manager_class: Type):
|
||||
"""
|
||||
Decorator to register a webhook manager.
|
||||
|
||||
Usage:
|
||||
@webhook_config("github", GitHubWebhooksManager)
|
||||
class GitHubWebhookBlock(Block):
|
||||
pass
|
||||
"""
|
||||
def decorator(block_class: Type):
|
||||
registry = get_registry()
|
||||
registry.register_webhook_manager(provider_name, manager_class)
|
||||
return block_class
|
||||
return decorator
|
||||
|
||||
def oauth_config(provider_name: str, handler_class: Type):
|
||||
"""
|
||||
Decorator to register an OAuth handler.
|
||||
|
||||
Usage:
|
||||
@oauth_config("github", GitHubOAuthHandler)
|
||||
class GitHubBlock(Block):
|
||||
pass
|
||||
"""
|
||||
def decorator(block_class: Type):
|
||||
registry = get_registry()
|
||||
registry.register_oauth_handler(provider_name, handler_class)
|
||||
return block_class
|
||||
return decorator
|
||||
|
||||
# === CONVENIENCE DECORATORS ===
|
||||
def register_credentials(*credentials):
|
||||
"""Alias for default_credentials decorator."""
|
||||
return default_credentials(*credentials)
|
||||
|
||||
def register_cost(*cost_configurations):
|
||||
"""Alias for cost_config decorator."""
|
||||
return cost_config(*cost_configurations)
|
||||
|
||||
def register_oauth(provider_name: str, handler_class: Type):
|
||||
"""Alias for oauth_config decorator."""
|
||||
return oauth_config(provider_name, handler_class)
|
||||
|
||||
def register_webhook_manager(provider_name: str, manager_class: Type):
|
||||
"""Alias for webhook_config decorator."""
|
||||
return webhook_config(provider_name, manager_class)
|
||||
```
|
||||
|
||||
### 5. Integration with Existing Systems
|
||||
|
||||
To maintain backward compatibility, we need to patch the existing configuration loading:
|
||||
|
||||
#### A. Patch Application Startup (`backend/app.py` or `backend/server/rest_api.py`)
|
||||
|
||||
```python
|
||||
# Add this after block loading
|
||||
from backend.sdk.auto_registry import discover_block_configurations, patch_existing_systems
|
||||
|
||||
# During application startup (after initialize_blocks())
|
||||
def setup_auto_registration():
|
||||
"""Set up auto-registration system."""
|
||||
# Discover all block configurations
|
||||
registry = discover_block_configurations()
|
||||
|
||||
# Patch existing systems to use discovered configurations
|
||||
patch_existing_systems()
|
||||
|
||||
print(f"Auto-registered {len(registry.block_costs)} block costs")
|
||||
print(f"Auto-registered {len(registry.default_credentials)} default credentials")
|
||||
print(f"Auto-registered {len(registry.oauth_handlers)} OAuth handlers")
|
||||
print(f"Auto-registered {len(registry.webhook_managers)} webhook managers")
|
||||
print(f"Auto-registered {len(registry.providers)} providers")
|
||||
|
||||
# Call during lifespan startup
|
||||
setup_auto_registration()
|
||||
```
|
||||
|
||||
#### B. Extend Provider Enum Dynamically
|
||||
|
||||
```python
|
||||
# backend/integrations/providers.py - Add dynamic extension
|
||||
def extend_provider_enum():
|
||||
"""Dynamically extend ProviderName enum with auto-discovered providers."""
|
||||
from backend.sdk.auto_registry import get_registry
|
||||
|
||||
registry = get_registry()
|
||||
for provider_name in registry.providers:
|
||||
if not hasattr(ProviderName, provider_name.upper()):
|
||||
setattr(ProviderName, provider_name.upper(), provider_name)
|
||||
```
|
||||
|
||||
### 6. Example Block with Auto-Registration
|
||||
|
||||
```python
|
||||
# Example: backend/blocks/my_service.py
|
||||
from backend.sdk import *
|
||||
|
||||
@provider("myservice")
|
||||
@cost_config(
|
||||
BlockCost(cost_amount=5, cost_type=BlockCostType.RUN),
|
||||
BlockCost(cost_amount=1, cost_type=BlockCostType.BYTE)
|
||||
)
|
||||
@default_credentials(
|
||||
APIKeyCredentials(
|
||||
id="myservice-default",
|
||||
provider="myservice",
|
||||
api_key=SecretStr("default-key-from-env"),
|
||||
title="MyService Default API Key"
|
||||
)
|
||||
)
|
||||
class MyServiceBlock(Block):
|
||||
class Input(BlockSchema):
|
||||
credentials: CredentialsMetaInput = CredentialsField(
|
||||
provider="myservice",
|
||||
supported_credential_types={"api_key"}
|
||||
)
|
||||
text: String = SchemaField(description="Text to process")
|
||||
|
||||
class Output(BlockSchema):
|
||||
result: String = SchemaField(description="Processing result")
|
||||
error: String = SchemaField(description="Error message if failed")
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="myservice-block-uuid",
|
||||
description="Process text using MyService API",
|
||||
categories={BlockCategory.TEXT},
|
||||
input_schema=MyServiceBlock.Input,
|
||||
output_schema=MyServiceBlock.Output,
|
||||
)
|
||||
|
||||
def run(self, input_data: Input, *, credentials: APIKeyCredentials, **kwargs) -> BlockOutput:
|
||||
try:
|
||||
# Use MyService API with credentials
|
||||
api_key = credentials.api_key.get_secret_value()
|
||||
# ... implementation
|
||||
yield "result", f"Processed: {input_data.text}"
|
||||
except Exception as e:
|
||||
yield "error", str(e)
|
||||
```
|
||||
|
||||
**Key Benefits of This Example:**
|
||||
1. **Single Import**: `from backend.sdk import *` provides everything needed
|
||||
2. **Self-Contained**: All configuration is in the block file via decorators
|
||||
3. **No External Changes**: Adding this block requires zero changes outside the blocks folder
|
||||
4. **Auto-Discovery**: Provider, costs, and credentials are automatically registered
|
||||
|
||||
### 7. Migration Strategy
|
||||
|
||||
#### Phase 1: Implement SDK and Auto-Registration (Week 1)
|
||||
1. Create `backend/sdk/__init__.py` with complete re-exports
|
||||
2. Create `backend/sdk/auto_registry.py` with registration system
|
||||
3. Create `backend/sdk/decorators.py` with registration decorators
|
||||
4. Patch application startup to use auto-registration
|
||||
5. Test with existing blocks (should work unchanged)
|
||||
|
||||
#### Phase 2: Migrate Configuration (Week 2)
|
||||
1. Move cost configurations from `block_cost_config.py` to block decorators
|
||||
2. Move default credentials from `credentials_store.py` to block decorators
|
||||
3. Move OAuth handlers from `oauth/__init__.py` to block decorators
|
||||
4. Move webhook managers from `webhooks/__init__.py` to block decorators
|
||||
5. Update 5-10 example blocks to demonstrate new patterns
|
||||
|
||||
#### Phase 3: Documentation and Adoption (Week 3)
|
||||
1. Update developer documentation with new patterns
|
||||
2. Create migration guide for existing blocks
|
||||
3. Add VS Code snippets for new decorators
|
||||
4. Create video tutorial showing complete workflow
|
||||
|
||||
#### Phase 4: Cleanup (Week 4)
|
||||
1. Remove old configuration files (optional, for backward compatibility)
|
||||
2. Migrate remaining blocks to new pattern
|
||||
3. Simplify application startup code
|
||||
4. Performance optimizations
|
||||
|
||||
### 8. Implementation Checklist
|
||||
|
||||
#### Core SDK Implementation ✅
|
||||
- [x] Create `backend/sdk/__init__.py` with complete re-exports (~200 lines) ✅ DONE - 180 lines
|
||||
- [x] Create `backend/sdk/auto_registry.py` with registry system (~150 lines) ✅ DONE - 167 lines
|
||||
- [x] Create `backend/sdk/decorators.py` with decorators (~100 lines) ✅ DONE - 132 lines
|
||||
- [x] Test that `from backend.sdk import *` provides all needed imports ✅ DONE - test created
|
||||
- [x] Test that existing blocks work unchanged ✅ DONE - backward compatible
|
||||
|
||||
#### Auto-Registration Implementation
|
||||
- [x] Patch application startup to call auto-registration ✅ DONE - rest_api.py modified
|
||||
- [x] Patch `block_cost_config.py` to use auto-discovered costs ✅ DONE via auto_registry.py
|
||||
- [x] Patch `credentials_store.py` to use auto-discovered credentials ✅ DONE via auto_registry.py
|
||||
- [x] Patch `oauth/__init__.py` to use auto-discovered handlers ✅ DONE via auto_registry.py
|
||||
- [x] Patch `webhooks/__init__.py` to use auto-discovered managers ✅ DONE via auto_registry.py
|
||||
- [x] Extend `ProviderName` enum dynamically ✅ DONE - added `_missing_` method for dynamic providers
|
||||
|
||||
#### Testing and Migration
|
||||
- [x] Create test blocks using new decorators ✅ DONE - 3 example blocks created
|
||||
- [ ] Migrate 5 existing blocks to demonstrate patterns ❌ NOT DONE - left for future PR
|
||||
- [x] Add comprehensive tests for auto-registration ✅ DONE - test_sdk_imports.py
|
||||
- [ ] Performance test auto-discovery system ❌ NOT DONE - left for optimization phase
|
||||
- [x] Create migration guide and documentation ✅ DONE - SDK_DEMONSTRATION.md
|
||||
|
||||
### 9. Benefits Summary
|
||||
|
||||
#### For Block Developers
|
||||
- **Single Import**: `from backend.sdk import *` provides everything needed
|
||||
- **Zero External Changes**: Adding blocks requires no modifications outside blocks folder
|
||||
- **Self-Documenting**: All configuration is visible in the block file
|
||||
- **Type Safety**: Full IDE support and type checking
|
||||
|
||||
#### For Platform Maintainers
|
||||
- **Eliminates Manual Updates**: No more updating 5+ configuration files
|
||||
- **Reduces Errors**: No risk of forgetting to update configuration files
|
||||
- **Easier Code Reviews**: All configuration changes are in the block PR
|
||||
- **Better Modularity**: Blocks are truly self-contained
|
||||
|
||||
#### For the Platform
|
||||
- **Faster Development**: New blocks can be added without cross-cutting changes
|
||||
- **Better Scalability**: System handles 100s of blocks without complexity
|
||||
- **Improved Documentation**: Configuration is co-located with implementation
|
||||
- **Easier Testing**: Blocks can be tested in isolation
|
||||
|
||||
This comprehensive solution achieves both goals: **complete import simplification** via `from backend.sdk import *` and **zero external configuration** via auto-registration decorators.
|
||||
@@ -1,221 +0,0 @@
|
||||
# Provider Enum Elimination Plan
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The `ProviderName` enum in `backend/integrations/providers.py` currently requires manual updates when adding new providers, which conflicts with our SDK's goal of zero external configuration. This plan proposes a **hybrid solution** that maintains backward compatibility while enabling dynamic provider registration.
|
||||
|
||||
## Problem Analysis
|
||||
|
||||
### Current Issues
|
||||
1. **Static Enum**: Python enums are immutable after class definition
|
||||
2. **Manual Updates**: Each new provider requires updating the enum
|
||||
3. **Type Safety**: FastAPI and Pydantic rely on the enum for validation
|
||||
4. **Wide Usage**: Used in 50+ locations for type hints, validation, and database storage
|
||||
|
||||
### Why Dynamic Extension Fails
|
||||
- Python enums cannot be modified at runtime
|
||||
- Type checkers analyze enums statically
|
||||
- FastAPI generates OpenAPI schemas at startup
|
||||
- Circular imports prevent late modification
|
||||
|
||||
## Proposed Solution: Provider Registry Pattern
|
||||
|
||||
### Overview
|
||||
Replace the static enum with a **provider registry** that:
|
||||
1. Maintains core providers as constants
|
||||
2. Allows runtime registration of new providers
|
||||
3. Provides backward-compatible validation
|
||||
4. Works seamlessly with FastAPI/Pydantic
|
||||
|
||||
### Architecture
|
||||
|
||||
```python
|
||||
# backend/integrations/provider_registry.py
|
||||
class ProviderRegistry:
|
||||
"""Central registry for all providers"""
|
||||
|
||||
# Core providers (backward compatibility)
|
||||
ANTHROPIC = "anthropic"
|
||||
GITHUB = "github"
|
||||
GOOGLE = "google"
|
||||
# ... all existing providers
|
||||
|
||||
_registry: set[str] = {
|
||||
"anthropic", "github", "google", ...
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def register(cls, provider: str) -> None:
|
||||
"""Register a new provider dynamically"""
|
||||
cls._registry.add(provider.lower())
|
||||
|
||||
@classmethod
|
||||
def is_valid(cls, provider: str) -> bool:
|
||||
"""Check if provider is registered"""
|
||||
return provider.lower() in cls._registry
|
||||
|
||||
@classmethod
|
||||
def validate(cls, provider: str) -> str:
|
||||
"""Validate and normalize provider name"""
|
||||
normalized = provider.lower()
|
||||
if not cls.is_valid(normalized):
|
||||
raise ValueError(f"Unknown provider: {provider}")
|
||||
return normalized
|
||||
```
|
||||
|
||||
### Custom Pydantic Type
|
||||
|
||||
```python
|
||||
# backend/integrations/provider_type.py
|
||||
from typing import Any
|
||||
from pydantic import GetCoreSchemaHandler
|
||||
from pydantic_core import core_schema
|
||||
|
||||
class Provider(str):
|
||||
"""Custom Pydantic type for provider validation"""
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(
|
||||
cls, source_type: Any, handler: GetCoreSchemaHandler
|
||||
) -> core_schema.CoreSchema:
|
||||
def validate(value: Any) -> str:
|
||||
if isinstance(value, ProviderName): # Accept old enum
|
||||
return value.value
|
||||
if isinstance(value, str):
|
||||
return ProviderRegistry.validate(value)
|
||||
raise ValueError(f"Invalid provider type: {type(value)}")
|
||||
|
||||
return core_schema.no_info_plain_validator_function(validate)
|
||||
```
|
||||
|
||||
### Migration Strategy
|
||||
|
||||
1. **Phase 1**: Add registry alongside enum
|
||||
2. **Phase 2**: Update type hints to use `Provider` type
|
||||
3. **Phase 3**: Deprecate enum (keep for compatibility)
|
||||
4. **Phase 4**: Remove enum in major version
|
||||
|
||||
## Implementation Checklist
|
||||
|
||||
### Phase 1: Core Implementation (Week 1)
|
||||
- [ ] Create `backend/integrations/provider_registry.py` with ProviderRegistry class
|
||||
- [ ] Create `backend/integrations/provider_type.py` with custom Pydantic type
|
||||
- [ ] Add all existing enum values to registry initialization
|
||||
- [ ] Create migration utilities to copy enum values to registry
|
||||
- [ ] Add tests for provider registry functionality
|
||||
|
||||
### Phase 2: SDK Integration (Week 1)
|
||||
- [ ] Update SDK decorators to use `ProviderRegistry.register()`
|
||||
- [ ] Modify auto-registration to register providers via registry
|
||||
- [ ] Update SDK imports to include Provider type
|
||||
- [ ] Test that @provider decorator works with registry
|
||||
- [ ] Verify backward compatibility with existing blocks
|
||||
|
||||
### Phase 3: Type Migration (Week 2)
|
||||
- [ ] Create compatibility type alias: `ProviderNameType = Union[ProviderName, Provider]`
|
||||
- [ ] Update critical paths to accept both enum and registry
|
||||
- [ ] Start with webhook models: `provider: ProviderNameType`
|
||||
- [ ] Update OAuth handlers to use provider type
|
||||
- [ ] Update credential models to use provider type
|
||||
|
||||
### Phase 4: FastAPI Integration (Week 2)
|
||||
- [ ] Create custom FastAPI query parameter validator
|
||||
- [ ] Update route handlers to use Provider type
|
||||
- [ ] Test OpenAPI schema generation
|
||||
- [ ] Ensure proper error messages for invalid providers
|
||||
- [ ] Update API documentation
|
||||
|
||||
### Phase 5: Database Layer (Week 3)
|
||||
- [ ] Ensure database continues storing providers as strings
|
||||
- [ ] Update Prisma models if needed
|
||||
- [ ] Test database queries with new providers
|
||||
- [ ] Verify migration compatibility
|
||||
- [ ] Add database constraints if necessary
|
||||
|
||||
### Phase 6: Testing & Documentation (Week 3)
|
||||
- [ ] Create comprehensive test suite for provider registry
|
||||
- [ ] Test migration from enum to registry
|
||||
- [ ] Performance test registry lookups
|
||||
- [ ] Update developer documentation
|
||||
- [ ] Create migration guide for existing code
|
||||
|
||||
### Phase 7: Gradual Rollout (Week 4)
|
||||
- [ ] Deploy with feature flag for registry usage
|
||||
- [ ] Monitor for any validation errors
|
||||
- [ ] Gradually migrate internal code to use registry
|
||||
- [ ] Collect feedback from block developers
|
||||
- [ ] Plan enum deprecation timeline
|
||||
|
||||
## Code Examples
|
||||
|
||||
### Before (Current)
|
||||
```python
|
||||
# Must manually add to enum
|
||||
class ProviderName(str, Enum):
|
||||
MYSERVICE = "myservice" # Manual addition required
|
||||
|
||||
# Block usage
|
||||
@provider("myservice") # Fails - not in enum
|
||||
class MyServiceBlock(Block):
|
||||
pass
|
||||
```
|
||||
|
||||
### After (With Registry)
|
||||
```python
|
||||
# Automatic registration via decorator
|
||||
@provider("myservice") # Auto-registers in registry
|
||||
class MyServiceBlock(Block):
|
||||
pass
|
||||
|
||||
# Or explicit registration
|
||||
ProviderRegistry.register("myservice")
|
||||
```
|
||||
|
||||
### Backward Compatible Usage
|
||||
```python
|
||||
# Old code continues to work
|
||||
def process(provider: ProviderName): # Still accepts enum
|
||||
pass
|
||||
|
||||
# New code uses registry
|
||||
def process_new(provider: Provider): # Accepts any registered provider
|
||||
pass
|
||||
|
||||
# Transition code
|
||||
def process_both(provider: Union[ProviderName, Provider]): # Accepts both
|
||||
pass
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **Zero Configuration**: New providers auto-register via decorators
|
||||
2. **Backward Compatible**: Existing code continues to work
|
||||
3. **Type Safe**: Maintains validation and type checking
|
||||
4. **Scalable**: No manual enum updates needed
|
||||
5. **Flexible**: Supports dynamic provider discovery
|
||||
|
||||
## Risks and Mitigations
|
||||
|
||||
### Risk 1: Type Safety Loss
|
||||
**Mitigation**: Custom Pydantic type maintains runtime validation
|
||||
|
||||
### Risk 2: Breaking Changes
|
||||
**Mitigation**: Phased migration with compatibility layer
|
||||
|
||||
### Risk 3: Performance Impact
|
||||
**Mitigation**: Set lookups are O(1), negligible performance impact
|
||||
|
||||
### Risk 4: OpenAPI Schema
|
||||
**Mitigation**: Custom schema generation for dynamic providers
|
||||
|
||||
## Success Metrics
|
||||
|
||||
1. **Zero Manual Updates**: New providers require no code changes outside blocks
|
||||
2. **100% Backward Compatibility**: All existing code continues to work
|
||||
3. **Type Safety Maintained**: No increase in runtime errors
|
||||
4. **Developer Satisfaction**: Easier provider addition process
|
||||
5. **Performance**: No measurable performance degradation
|
||||
|
||||
## Conclusion
|
||||
|
||||
This plan provides a path to eliminate the static ProviderName enum requirement while maintaining all benefits of type safety and validation. The provider registry pattern aligns perfectly with the SDK's goal of zero external configuration and will enable true plug-and-play block development.
|
||||
@@ -1,126 +0,0 @@
|
||||
# Simple Provider Enum Solution
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Instead of complex registries or major refactoring, we can solve the dynamic provider problem with a **single method** added to the existing `ProviderName` enum using Python's built-in `_missing_` feature. This PR implements this simple solution.
|
||||
|
||||
## The Solution
|
||||
|
||||
Add this method to the `ProviderName` enum:
|
||||
|
||||
```python
|
||||
class ProviderName(str, Enum):
|
||||
# ... existing providers ...
|
||||
|
||||
@classmethod
|
||||
def _missing_(cls, value):
|
||||
"""
|
||||
Allow any string to be a valid provider name.
|
||||
This enables custom providers without modifying the enum.
|
||||
"""
|
||||
# Create a new enum member dynamically
|
||||
new_member = object.__new__(cls)
|
||||
new_member._name_ = value.upper()
|
||||
new_member._value_ = value
|
||||
return new_member
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
```python
|
||||
# Existing providers work as before
|
||||
provider1 = ProviderName.GITHUB # Standard enum member
|
||||
|
||||
# New providers work automatically
|
||||
provider2 = ProviderName("myservice") # Creates dynamic enum member
|
||||
|
||||
# Both work identically
|
||||
assert provider1.value == "github"
|
||||
assert provider2.value == "myservice"
|
||||
assert isinstance(provider2, ProviderName) # True!
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **Zero Breaking Changes**: All existing code continues to work
|
||||
2. **Minimal Code Change**: Only ~10 lines added to one file
|
||||
3. **Type Safety Maintained**: Still validates as ProviderName type
|
||||
4. **No External Dependencies**: Uses Python's built-in enum features
|
||||
5. **Simple to Understand**: No complex patterns or abstractions
|
||||
|
||||
## Implementation Status
|
||||
|
||||
### ✅ Implemented in This PR
|
||||
The `ProviderName` enum in `backend/integrations/providers.py` now includes the `_missing_` method that enables dynamic provider support. This means:
|
||||
|
||||
- ✅ Any string can be used as a provider name
|
||||
- ✅ The SDK's `@provider` decorator now works with custom providers
|
||||
- ✅ Only 15 lines added to providers.py
|
||||
- ✅ Full backward compatibility maintained
|
||||
- ✅ Type safety preserved
|
||||
|
||||
### Remaining Tasks (Optional Improvements)
|
||||
|
||||
#### Documentation Updates (30 minutes)
|
||||
- [ ] Update SDK documentation to clarify any string works as provider
|
||||
- [ ] Add example showing custom provider usage
|
||||
- [ ] Document this feature in developer guides
|
||||
|
||||
#### Testing (30 minutes)
|
||||
- [ ] Add specific test for SDK with custom provider name
|
||||
- [ ] Verify the example blocks work with custom providers
|
||||
|
||||
## Example Usage in SDK
|
||||
|
||||
```python
|
||||
from backend.sdk import *
|
||||
|
||||
# Works with any provider name now!
|
||||
@provider("my-custom-llm-service")
|
||||
@cost_config(BlockCost(cost_amount=5, cost_type=BlockCostType.RUN))
|
||||
class MyCustomLLMBlock(Block):
|
||||
class Input(BlockSchema):
|
||||
credentials: CredentialsMetaInput = CredentialsField(
|
||||
provider="my-custom-llm-service", # Works automatically!
|
||||
supported_credential_types={"api_key"}
|
||||
)
|
||||
```
|
||||
|
||||
## Comparison with Complex Solution
|
||||
|
||||
### Complex Registry Pattern (Original Plan)
|
||||
- 200+ lines of new code
|
||||
- 7 phases, 30+ tasks
|
||||
- New abstractions to learn
|
||||
- Weeks of implementation
|
||||
|
||||
### Simple Enum Enhancement (This Plan)
|
||||
- 10 lines of code
|
||||
- 1 method addition
|
||||
- No new concepts
|
||||
- Hours of implementation
|
||||
|
||||
## Potential Concerns
|
||||
|
||||
### Q: Is this a hack?
|
||||
A: No, `_missing_` is an official Python enum feature designed exactly for this use case.
|
||||
|
||||
### Q: Will this work with type checkers?
|
||||
A: Yes, the type is still `ProviderName`, so type checkers are happy.
|
||||
|
||||
### Q: What about performance?
|
||||
A: Dynamic member creation is cached, so it only happens once per unique provider.
|
||||
|
||||
### Q: Any limitations?
|
||||
A: Provider names should be valid Python identifiers when uppercase (no spaces, special chars).
|
||||
|
||||
## Conclusion
|
||||
|
||||
This simple solution achieves all our goals:
|
||||
- ✅ Zero configuration for new providers
|
||||
- ✅ Full backward compatibility
|
||||
- ✅ Type safety maintained
|
||||
- ✅ Minimal code changes
|
||||
- ✅ Easy to understand and maintain
|
||||
|
||||
Sometimes the simplest solution is the best solution.
|
||||
@@ -1,148 +0,0 @@
|
||||
# AutoGPT SDK - Complete Implementation Demonstration
|
||||
|
||||
## ✅ Implementation Complete
|
||||
|
||||
The SDK has been successfully implemented with the following features:
|
||||
|
||||
### 1. **Single Import Statement**
|
||||
```python
|
||||
from backend.sdk import *
|
||||
```
|
||||
|
||||
This single import provides access to **68+ components** including:
|
||||
- All block base classes (`Block`, `BlockSchema`, `BlockOutput`, etc.)
|
||||
- All credential types (`APIKeyCredentials`, `OAuth2Credentials`, etc.)
|
||||
- All decorators (`@provider`, `@cost_config`, `@default_credentials`, etc.)
|
||||
- Type aliases (`String`, `Integer`, `Float`, `Boolean`)
|
||||
- Utilities (`json`, `logging`, `store_media_file`, etc.)
|
||||
|
||||
### 2. **Auto-Registration System**
|
||||
|
||||
No more manual updates to configuration files! The SDK provides decorators that automatically register:
|
||||
|
||||
- **Providers**: `@provider("myservice")`
|
||||
- **Block Costs**: `@cost_config(BlockCost(...))`
|
||||
- **Default Credentials**: `@default_credentials(APIKeyCredentials(...))`
|
||||
- **OAuth Handlers**: `@oauth_config("myservice", MyOAuthHandler)`
|
||||
- **Webhook Managers**: `@webhook_config("myservice", MyWebhookManager)`
|
||||
|
||||
### 3. **Example: Before vs After**
|
||||
|
||||
#### Before SDK (Old Way):
|
||||
```python
|
||||
# Multiple imports from various modules
|
||||
from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
|
||||
from backend.data.model import SchemaField, CredentialsField, CredentialsMetaInput
|
||||
from backend.integrations.providers import ProviderName
|
||||
from backend.data.cost import BlockCost, BlockCostType
|
||||
from backend.data.model import APIKeyCredentials
|
||||
from typing import List, Optional, Dict
|
||||
from pydantic import SecretStr
|
||||
|
||||
# PLUS: Manual updates required in 5+ configuration files:
|
||||
# - backend/data/block_cost_config.py (add to BLOCK_COSTS dict)
|
||||
# - backend/integrations/credentials_store.py (add to DEFAULT_CREDENTIALS)
|
||||
# - backend/integrations/providers.py (add to ProviderName enum)
|
||||
# - backend/integrations/oauth/__init__.py (if OAuth needed)
|
||||
# - backend/integrations/webhooks/__init__.py (if webhooks needed)
|
||||
|
||||
class MyServiceBlock(Block):
|
||||
# ... block implementation
|
||||
```
|
||||
|
||||
#### After SDK (New Way):
|
||||
```python
|
||||
# Single import provides everything
|
||||
from backend.sdk import *
|
||||
|
||||
# All configuration via decorators - no external files to modify!
|
||||
@provider("myservice")
|
||||
@cost_config(
|
||||
BlockCost(cost_amount=5, cost_type=BlockCostType.RUN)
|
||||
)
|
||||
@default_credentials(
|
||||
APIKeyCredentials(
|
||||
id="myservice-default",
|
||||
provider="myservice",
|
||||
api_key=SecretStr("default-key"),
|
||||
title="MyService Default API Key"
|
||||
)
|
||||
)
|
||||
class MyServiceBlock(Block):
|
||||
# ... block implementation
|
||||
```
|
||||
|
||||
### 4. **Implementation Files**
|
||||
|
||||
The SDK consists of just 3 files:
|
||||
|
||||
1. **`backend/sdk/__init__.py`** (~200 lines)
|
||||
- Complete re-export of all block development dependencies
|
||||
- Smart handling of optional components
|
||||
- Comprehensive `__all__` list for `import *`
|
||||
|
||||
2. **`backend/sdk/auto_registry.py`** (~170 lines)
|
||||
- Global registry for auto-discovered configurations
|
||||
- Patches existing systems for backward compatibility
|
||||
- Called during application startup
|
||||
|
||||
3. **`backend/sdk/decorators.py`** (~130 lines)
|
||||
- Registration decorators for all configuration types
|
||||
- Simple, intuitive API
|
||||
- Supports individual or combined decorators
|
||||
|
||||
### 5. **Working Examples**
|
||||
|
||||
Three example blocks have been created to demonstrate the SDK:
|
||||
|
||||
1. **`backend/blocks/example_sdk_block.py`**
|
||||
- Shows basic SDK usage with API credentials
|
||||
- Auto-registers provider, costs, and default credentials
|
||||
|
||||
2. **`backend/blocks/example_webhook_sdk_block.py`**
|
||||
- Demonstrates webhook manager registration
|
||||
- No manual webhook configuration needed
|
||||
|
||||
3. **`backend/blocks/simple_example_block.py`**
|
||||
- Minimal example showing the import simplification
|
||||
|
||||
### 6. **Key Benefits Achieved**
|
||||
|
||||
✅ **Single Import**: `from backend.sdk import *` provides everything needed
|
||||
|
||||
✅ **Zero External Changes**: Adding blocks requires NO modifications outside the blocks folder
|
||||
|
||||
✅ **Auto-Registration**: All configurations are discovered and registered automatically
|
||||
|
||||
✅ **Backward Compatible**: Existing blocks continue to work unchanged
|
||||
|
||||
✅ **Type Safety**: Full IDE support with autocomplete and type checking
|
||||
|
||||
✅ **Minimal Footprint**: Only ~500 lines of code across 3 files
|
||||
|
||||
### 7. **How It Works**
|
||||
|
||||
1. **During Development**: Developers use `from backend.sdk import *` and decorators
|
||||
2. **During Startup**: The application calls `setup_auto_registration()`
|
||||
3. **Auto-Discovery**: The system scans all blocks and collects decorator configurations
|
||||
4. **Patching**: Existing configuration systems are patched with discovered data
|
||||
5. **Runtime**: Everything works as before, but with zero manual configuration
|
||||
|
||||
### 8. **Testing**
|
||||
|
||||
A comprehensive test suite has been created in `test/sdk/test_sdk_imports.py` that verifies:
|
||||
- All expected imports are available
|
||||
- Auto-registration system works correctly
|
||||
- Decorators properly register configurations
|
||||
- Example blocks can be imported and used
|
||||
|
||||
### 9. **Next Steps**
|
||||
|
||||
To fully adopt the SDK:
|
||||
|
||||
1. **Migrate existing blocks** to use SDK imports (optional, for consistency)
|
||||
2. **Update documentation** to show SDK patterns
|
||||
3. **Create VS Code snippets** for common SDK patterns
|
||||
4. **Remove old configuration entries** as blocks are migrated (optional)
|
||||
|
||||
The SDK is now ready for use and will significantly improve the developer experience for creating AutoGPT blocks!
|
||||
@@ -1,120 +0,0 @@
|
||||
# SDK Implementation Summary
|
||||
|
||||
## ✅ Implementation Complete
|
||||
|
||||
The AutoGPT Platform Block Development SDK has been fully implemented and tested. Here's what was accomplished:
|
||||
|
||||
### 1. Core SDK Implementation (100% Complete)
|
||||
- ✅ Created `backend/sdk/__init__.py` with comprehensive re-exports
|
||||
- ✅ Created `backend/sdk/auto_registry.py` with auto-registration system
|
||||
- ✅ Created `backend/sdk/decorators.py` with all registration decorators
|
||||
- ✅ Implemented dynamic provider support via `_missing_` method in ProviderName enum
|
||||
- ✅ Patched application startup to use auto-registration
|
||||
|
||||
### 2. Features Implemented
|
||||
|
||||
#### Single Import Statement
|
||||
```python
|
||||
from backend.sdk import *
|
||||
```
|
||||
This provides access to:
|
||||
- 68+ components needed for block development
|
||||
- All core block classes and types
|
||||
- All credential and authentication types
|
||||
- All decorators for auto-registration
|
||||
- Type aliases for better readability
|
||||
- Common utilities and types
|
||||
|
||||
#### Auto-Registration Decorators
|
||||
- `@provider("service-name")` - Registers new provider
|
||||
- `@cost_config(...)` - Registers block costs
|
||||
- `@default_credentials(...)` - Registers default credentials
|
||||
- `@webhook_config(...)` - Registers webhook managers
|
||||
- `@oauth_config(...)` - Registers OAuth handlers
|
||||
|
||||
#### Dynamic Provider Support
|
||||
The ProviderName enum now accepts any string as a valid provider through the `_missing_` method, eliminating the need for manual enum updates.
|
||||
|
||||
### 3. Test Coverage
|
||||
|
||||
#### Comprehensive Test Suite (`test_sdk_comprehensive.py`)
|
||||
- ✅ SDK imports verification (68+ components)
|
||||
- ✅ Auto-registry system functionality
|
||||
- ✅ All decorators working correctly
|
||||
- ✅ Dynamic provider enum support
|
||||
- ✅ Complete block example with all features
|
||||
- ✅ Backward compatibility verification
|
||||
- ✅ Import * syntax support
|
||||
|
||||
**Result: 8/8 tests passed**
|
||||
|
||||
#### Integration Demo (`demo_sdk_block.py`)
|
||||
Created a complete working example showing:
|
||||
- ✅ Custom provider registration ("ultra-translate-ai")
|
||||
- ✅ Automatic cost configuration
|
||||
- ✅ Default credentials setup
|
||||
- ✅ Block execution with test data
|
||||
- ✅ Zero external configuration needed
|
||||
|
||||
### 4. Key Benefits Achieved
|
||||
|
||||
#### For Developers
|
||||
- **90% reduction in imports**: From 8-15 imports to just 1
|
||||
- **Zero configuration**: No manual updates to external files
|
||||
- **Self-contained blocks**: All configuration via decorators
|
||||
- **Type safety**: Full IDE support maintained
|
||||
|
||||
#### For the Platform
|
||||
- **Scalability**: Can handle hundreds of blocks without complexity
|
||||
- **Maintainability**: Only 3 SDK files (~500 lines total)
|
||||
- **Backward compatibility**: All existing blocks continue to work
|
||||
- **Easy onboarding**: New developers productive in minutes
|
||||
|
||||
### 5. Example Usage
|
||||
|
||||
```python
|
||||
from backend.sdk import *
|
||||
|
||||
@provider("my-ai-service")
|
||||
@cost_config(BlockCost(cost_amount=5, cost_type=BlockCostType.RUN))
|
||||
@default_credentials(
|
||||
APIKeyCredentials(
|
||||
id="my-ai-default",
|
||||
provider="my-ai-service",
|
||||
api_key=SecretStr("default-key"),
|
||||
title="My AI Service Default Key"
|
||||
)
|
||||
)
|
||||
class MyAIBlock(Block):
|
||||
# Block implementation
|
||||
pass
|
||||
```
|
||||
|
||||
### 6. Files Created/Modified
|
||||
|
||||
#### New Files
|
||||
- `backend/sdk/__init__.py` (180 lines)
|
||||
- `backend/sdk/auto_registry.py` (167 lines)
|
||||
- `backend/sdk/decorators.py` (132 lines)
|
||||
- `backend/integrations/providers.py` (added `_missing_` method)
|
||||
- 3 example blocks demonstrating SDK usage
|
||||
- Comprehensive test suite
|
||||
|
||||
#### Modified Files
|
||||
- `backend/server/rest_api.py` (added auto-registration setup)
|
||||
|
||||
### 7. Next Steps
|
||||
|
||||
The SDK is production-ready. Future enhancements could include:
|
||||
- Migration of existing blocks to use SDK imports
|
||||
- VS Code snippets for common patterns
|
||||
- Extended documentation and tutorials
|
||||
- Performance optimizations if needed
|
||||
|
||||
## Conclusion
|
||||
|
||||
The SDK successfully achieves both primary goals:
|
||||
1. **Single import statement** (`from backend.sdk import *`) for all block development needs
|
||||
2. **Zero external configuration** through auto-registration decorators
|
||||
|
||||
The implementation is minimal (~500 lines), maintainable, and provides massive developer productivity improvements while maintaining full backward compatibility.
|
||||
Reference in New Issue
Block a user