mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-04-08 03:00:28 -04:00
Compare commits
29 Commits
feat/impro
...
hotfix/wai
| Author | SHA1 | Date | |
|---|---|---|---|
|
d89b84ba2b | ||
|
|
4619b07945 | ||
|
|
d43535e491 | ||
|
|
a35914889a | ||
|
|
7c248f2d6e | ||
|
|
d4a7ce3846 | ||
|
|
605a198c09 | ||
|
|
a3389485a7 | ||
|
|
cd439e912a | ||
|
|
7b32290582 | ||
|
|
e3137382c3 | ||
|
|
097a19141d | ||
|
|
65f2c04ef1 | ||
|
|
865abdb9e0 | ||
|
|
b59b200bd6 | ||
|
|
e7fb4cce5a | ||
|
|
85e2aef6ad | ||
|
|
85a8fb598e | ||
|
|
ae20da8aaa | ||
|
|
0bb2b87c32 | ||
|
|
73c0b6899a | ||
|
|
4c853a54d7 | ||
|
|
dfdd632161 | ||
|
|
e4bc728d40 | ||
|
|
2c6d85d15e | ||
|
|
8258338caf | ||
|
|
374f35874c | ||
|
|
e62a56e8ba | ||
|
|
f3f9a60157 |
@@ -1,339 +0,0 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from functools import wraps
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
ParamSpec,
|
||||
Protocol,
|
||||
TypeVar,
|
||||
cast,
|
||||
runtime_checkable,
|
||||
)
|
||||
|
||||
P = ParamSpec("P")
|
||||
R = TypeVar("R")
|
||||
R_co = TypeVar("R_co", covariant=True)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _make_hashable_key(
|
||||
args: tuple[Any, ...], kwargs: dict[str, Any]
|
||||
) -> tuple[Any, ...]:
|
||||
"""
|
||||
Convert args and kwargs into a hashable cache key.
|
||||
|
||||
Handles unhashable types like dict, list, set by converting them to
|
||||
their sorted string representations.
|
||||
"""
|
||||
|
||||
def make_hashable(obj: Any) -> Any:
|
||||
"""Recursively convert an object to a hashable representation."""
|
||||
if isinstance(obj, dict):
|
||||
# Sort dict items to ensure consistent ordering
|
||||
return (
|
||||
"__dict__",
|
||||
tuple(sorted((k, make_hashable(v)) for k, v in obj.items())),
|
||||
)
|
||||
elif isinstance(obj, (list, tuple)):
|
||||
return ("__list__", tuple(make_hashable(item) for item in obj))
|
||||
elif isinstance(obj, set):
|
||||
return ("__set__", tuple(sorted(make_hashable(item) for item in obj)))
|
||||
elif hasattr(obj, "__dict__"):
|
||||
# Handle objects with __dict__ attribute
|
||||
return ("__obj__", obj.__class__.__name__, make_hashable(obj.__dict__))
|
||||
else:
|
||||
# For basic hashable types (str, int, bool, None, etc.)
|
||||
try:
|
||||
hash(obj)
|
||||
return obj
|
||||
except TypeError:
|
||||
# Fallback: convert to string representation
|
||||
return ("__str__", str(obj))
|
||||
|
||||
hashable_args = tuple(make_hashable(arg) for arg in args)
|
||||
hashable_kwargs = tuple(sorted((k, make_hashable(v)) for k, v in kwargs.items()))
|
||||
return (hashable_args, hashable_kwargs)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class CachedFunction(Protocol[P, R_co]):
|
||||
"""Protocol for cached functions with cache management methods."""
|
||||
|
||||
def cache_clear(self) -> None:
|
||||
"""Clear all cached entries."""
|
||||
return None
|
||||
|
||||
def cache_info(self) -> dict[str, int | None]:
|
||||
"""Get cache statistics."""
|
||||
return {}
|
||||
|
||||
def cache_delete(self, *args: P.args, **kwargs: P.kwargs) -> bool:
|
||||
"""Delete a specific cache entry by its arguments. Returns True if entry existed."""
|
||||
return False
|
||||
|
||||
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R_co:
|
||||
"""Call the cached function."""
|
||||
return None # type: ignore
|
||||
|
||||
|
||||
def cached(
|
||||
*,
|
||||
maxsize: int = 128,
|
||||
ttl_seconds: int | None = None,
|
||||
) -> Callable[[Callable], CachedFunction]:
|
||||
"""
|
||||
Thundering herd safe cache decorator for both sync and async functions.
|
||||
|
||||
Uses double-checked locking to prevent multiple threads/coroutines from
|
||||
executing the expensive operation simultaneously during cache misses.
|
||||
|
||||
Args:
|
||||
func: The function to cache (when used without parentheses)
|
||||
maxsize: Maximum number of cached entries
|
||||
ttl_seconds: Time to live in seconds. If None, entries never expire
|
||||
|
||||
Returns:
|
||||
Decorated function or decorator
|
||||
|
||||
Example:
|
||||
@cache() # Default: maxsize=128, no TTL
|
||||
def expensive_sync_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
|
||||
@cache() # Works with async too
|
||||
async def expensive_async_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
|
||||
@cache(maxsize=1000, ttl_seconds=300) # Custom maxsize and TTL
|
||||
def another_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
"""
|
||||
|
||||
def decorator(target_func):
|
||||
# Cache storage and per-event-loop locks
|
||||
cache_storage = {}
|
||||
_event_loop_locks = {} # Maps event loop to its asyncio.Lock
|
||||
|
||||
if inspect.iscoroutinefunction(target_func):
|
||||
|
||||
def _get_cache_lock():
|
||||
"""Get or create an asyncio.Lock for the current event loop."""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
# No event loop, use None as default key
|
||||
loop = None
|
||||
|
||||
if loop not in _event_loop_locks:
|
||||
return _event_loop_locks.setdefault(loop, asyncio.Lock())
|
||||
return _event_loop_locks[loop]
|
||||
|
||||
@wraps(target_func)
|
||||
async def async_wrapper(*args: P.args, **kwargs: P.kwargs):
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
current_time = time.time()
|
||||
|
||||
# Fast path: check cache without lock
|
||||
if key in cache_storage:
|
||||
if ttl_seconds is None:
|
||||
logger.debug(f"Cache hit for {target_func.__name__}")
|
||||
return cache_storage[key]
|
||||
else:
|
||||
cached_data = cache_storage[key]
|
||||
if isinstance(cached_data, tuple):
|
||||
result, timestamp = cached_data
|
||||
if current_time - timestamp < ttl_seconds:
|
||||
logger.debug(f"Cache hit for {target_func.__name__}")
|
||||
return result
|
||||
|
||||
# Slow path: acquire lock for cache miss/expiry
|
||||
async with _get_cache_lock():
|
||||
# Double-check: another coroutine might have populated cache
|
||||
if key in cache_storage:
|
||||
if ttl_seconds is None:
|
||||
return cache_storage[key]
|
||||
else:
|
||||
cached_data = cache_storage[key]
|
||||
if isinstance(cached_data, tuple):
|
||||
result, timestamp = cached_data
|
||||
if current_time - timestamp < ttl_seconds:
|
||||
return result
|
||||
|
||||
# Cache miss - execute function
|
||||
logger.debug(f"Cache miss for {target_func.__name__}")
|
||||
result = await target_func(*args, **kwargs)
|
||||
|
||||
# Store result
|
||||
if ttl_seconds is None:
|
||||
cache_storage[key] = result
|
||||
else:
|
||||
cache_storage[key] = (result, current_time)
|
||||
|
||||
# Cleanup if needed
|
||||
if len(cache_storage) > maxsize:
|
||||
cutoff = maxsize // 2
|
||||
oldest_keys = (
|
||||
list(cache_storage.keys())[:-cutoff] if cutoff > 0 else []
|
||||
)
|
||||
for old_key in oldest_keys:
|
||||
cache_storage.pop(old_key, None)
|
||||
|
||||
return result
|
||||
|
||||
wrapper = async_wrapper
|
||||
|
||||
else:
|
||||
# Sync function with threading.Lock
|
||||
cache_lock = threading.Lock()
|
||||
|
||||
@wraps(target_func)
|
||||
def sync_wrapper(*args: P.args, **kwargs: P.kwargs):
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
current_time = time.time()
|
||||
|
||||
# Fast path: check cache without lock
|
||||
if key in cache_storage:
|
||||
if ttl_seconds is None:
|
||||
logger.debug(f"Cache hit for {target_func.__name__}")
|
||||
return cache_storage[key]
|
||||
else:
|
||||
cached_data = cache_storage[key]
|
||||
if isinstance(cached_data, tuple):
|
||||
result, timestamp = cached_data
|
||||
if current_time - timestamp < ttl_seconds:
|
||||
logger.debug(f"Cache hit for {target_func.__name__}")
|
||||
return result
|
||||
|
||||
# Slow path: acquire lock for cache miss/expiry
|
||||
with cache_lock:
|
||||
# Double-check: another thread might have populated cache
|
||||
if key in cache_storage:
|
||||
if ttl_seconds is None:
|
||||
return cache_storage[key]
|
||||
else:
|
||||
cached_data = cache_storage[key]
|
||||
if isinstance(cached_data, tuple):
|
||||
result, timestamp = cached_data
|
||||
if current_time - timestamp < ttl_seconds:
|
||||
return result
|
||||
|
||||
# Cache miss - execute function
|
||||
logger.debug(f"Cache miss for {target_func.__name__}")
|
||||
result = target_func(*args, **kwargs)
|
||||
|
||||
# Store result
|
||||
if ttl_seconds is None:
|
||||
cache_storage[key] = result
|
||||
else:
|
||||
cache_storage[key] = (result, current_time)
|
||||
|
||||
# Cleanup if needed
|
||||
if len(cache_storage) > maxsize:
|
||||
cutoff = maxsize // 2
|
||||
oldest_keys = (
|
||||
list(cache_storage.keys())[:-cutoff] if cutoff > 0 else []
|
||||
)
|
||||
for old_key in oldest_keys:
|
||||
cache_storage.pop(old_key, None)
|
||||
|
||||
return result
|
||||
|
||||
wrapper = sync_wrapper
|
||||
|
||||
# Add cache management methods
|
||||
def cache_clear() -> None:
|
||||
cache_storage.clear()
|
||||
|
||||
def cache_info() -> dict[str, int | None]:
|
||||
return {
|
||||
"size": len(cache_storage),
|
||||
"maxsize": maxsize,
|
||||
"ttl_seconds": ttl_seconds,
|
||||
}
|
||||
|
||||
def cache_delete(*args, **kwargs) -> bool:
|
||||
"""Delete a specific cache entry. Returns True if entry existed."""
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
if key in cache_storage:
|
||||
del cache_storage[key]
|
||||
return True
|
||||
return False
|
||||
|
||||
setattr(wrapper, "cache_clear", cache_clear)
|
||||
setattr(wrapper, "cache_info", cache_info)
|
||||
setattr(wrapper, "cache_delete", cache_delete)
|
||||
|
||||
return cast(CachedFunction, wrapper)
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def thread_cached(func):
|
||||
"""
|
||||
Thread-local cache decorator for both sync and async functions.
|
||||
|
||||
Each thread gets its own cache, which is useful for request-scoped caching
|
||||
in web applications where you want to cache within a single request but
|
||||
not across requests.
|
||||
|
||||
Args:
|
||||
func: The function to cache
|
||||
|
||||
Returns:
|
||||
Decorated function with thread-local caching
|
||||
|
||||
Example:
|
||||
@thread_cached
|
||||
def expensive_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
|
||||
@thread_cached # Works with async too
|
||||
async def expensive_async_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
"""
|
||||
thread_local = threading.local()
|
||||
|
||||
def _clear():
|
||||
if hasattr(thread_local, "cache"):
|
||||
del thread_local.cache
|
||||
|
||||
if inspect.iscoroutinefunction(func):
|
||||
|
||||
@wraps(func)
|
||||
async def async_wrapper(*args, **kwargs):
|
||||
cache = getattr(thread_local, "cache", None)
|
||||
if cache is None:
|
||||
cache = thread_local.cache = {}
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
if key not in cache:
|
||||
cache[key] = await func(*args, **kwargs)
|
||||
return cache[key]
|
||||
|
||||
setattr(async_wrapper, "clear_cache", _clear)
|
||||
return async_wrapper
|
||||
|
||||
else:
|
||||
|
||||
@wraps(func)
|
||||
def sync_wrapper(*args, **kwargs):
|
||||
cache = getattr(thread_local, "cache", None)
|
||||
if cache is None:
|
||||
cache = thread_local.cache = {}
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
if key not in cache:
|
||||
cache[key] = func(*args, **kwargs)
|
||||
return cache[key]
|
||||
|
||||
setattr(sync_wrapper, "clear_cache", _clear)
|
||||
return sync_wrapper
|
||||
|
||||
|
||||
def clear_thread_cache(func: Callable) -> None:
|
||||
"""Clear thread-local cache for a function."""
|
||||
if clear := getattr(func, "clear_cache", None):
|
||||
clear()
|
||||
@@ -5,7 +5,7 @@ import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from autogpt_libs.utils.cache import cached
|
||||
from backend.util.cache import cached
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
def load_all_blocks() -> dict[str, type["Block"]]:
|
||||
from backend.data.block import Block
|
||||
from backend.util.settings import Config
|
||||
|
||||
@@ -2,7 +2,7 @@ from typing import Any
|
||||
|
||||
from backend.data.block import Block, BlockCategory, BlockOutput, BlockSchema
|
||||
from backend.data.model import SchemaField
|
||||
from backend.util.json import json
|
||||
from backend.util.json import loads
|
||||
|
||||
|
||||
class StepThroughItemsBlock(Block):
|
||||
@@ -68,7 +68,7 @@ class StepThroughItemsBlock(Block):
|
||||
raise ValueError(
|
||||
f"Input too large: {len(data)} bytes > {MAX_ITEM_SIZE} bytes"
|
||||
)
|
||||
items = json.loads(data)
|
||||
items = loads(data)
|
||||
else:
|
||||
items = data
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ from typing import (
|
||||
|
||||
import jsonref
|
||||
import jsonschema
|
||||
from autogpt_libs.utils.cache import cached
|
||||
from prisma.models import AgentBlock
|
||||
from prisma.types import AgentBlockCreateInput
|
||||
from pydantic import BaseModel
|
||||
@@ -28,6 +27,7 @@ from pydantic import BaseModel
|
||||
from backend.data.model import NodeExecutionStats
|
||||
from backend.integrations.providers import ProviderName
|
||||
from backend.util import json
|
||||
from backend.util.cache import cached
|
||||
from backend.util.settings import Config
|
||||
|
||||
from .model import (
|
||||
@@ -722,7 +722,7 @@ def get_block(block_id: str) -> Block[BlockSchema, BlockSchema] | None:
|
||||
return cls() if cls else None
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
def get_webhook_block_ids() -> Sequence[str]:
|
||||
return [
|
||||
id
|
||||
@@ -731,7 +731,7 @@ def get_webhook_block_ids() -> Sequence[str]:
|
||||
]
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
def get_io_block_ids() -> Sequence[str]:
|
||||
return [
|
||||
id
|
||||
|
||||
@@ -5,7 +5,6 @@ from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import stripe
|
||||
from prisma import Json
|
||||
from prisma.enums import (
|
||||
CreditRefundRequestStatus,
|
||||
CreditTransactionType,
|
||||
@@ -13,16 +12,12 @@ from prisma.enums import (
|
||||
OnboardingStep,
|
||||
)
|
||||
from prisma.errors import UniqueViolationError
|
||||
from prisma.models import CreditRefundRequest, CreditTransaction, User
|
||||
from prisma.types import (
|
||||
CreditRefundRequestCreateInput,
|
||||
CreditTransactionCreateInput,
|
||||
CreditTransactionWhereInput,
|
||||
)
|
||||
from prisma.models import CreditRefundRequest, CreditTransaction, User, UserBalance
|
||||
from prisma.types import CreditRefundRequestCreateInput, CreditTransactionWhereInput
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.data import db
|
||||
from backend.data.block_cost_config import BLOCK_COSTS
|
||||
from backend.data.db import query_raw_with_schema
|
||||
from backend.data.includes import MAX_CREDIT_REFUND_REQUESTS_FETCH
|
||||
from backend.data.model import (
|
||||
AutoTopUpConfig,
|
||||
@@ -36,7 +31,8 @@ from backend.data.user import get_user_by_id, get_user_email_by_id
|
||||
from backend.notifications.notifications import queue_notification_async
|
||||
from backend.server.v2.admin.model import UserHistoryResponse
|
||||
from backend.util.exceptions import InsufficientBalanceError
|
||||
from backend.util.json import SafeJson
|
||||
from backend.util.feature_flag import Flag, is_feature_enabled
|
||||
from backend.util.json import SafeJson, dumps
|
||||
from backend.util.models import Pagination
|
||||
from backend.util.retry import func_retry
|
||||
from backend.util.settings import Settings
|
||||
@@ -49,6 +45,10 @@ stripe.api_key = settings.secrets.stripe_api_key
|
||||
logger = logging.getLogger(__name__)
|
||||
base_url = settings.config.frontend_base_url or settings.config.platform_base_url
|
||||
|
||||
# Constants for test compatibility
|
||||
POSTGRES_INT_MAX = 2147483647
|
||||
POSTGRES_INT_MIN = -2147483648
|
||||
|
||||
|
||||
class UsageTransactionMetadata(BaseModel):
|
||||
graph_exec_id: str | None = None
|
||||
@@ -139,14 +139,20 @@ class UserCreditBase(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def onboarding_reward(self, user_id: str, credits: int, step: OnboardingStep):
|
||||
async def onboarding_reward(
|
||||
self, user_id: str, credits: int, step: OnboardingStep
|
||||
) -> bool:
|
||||
"""
|
||||
Reward the user with credits for completing an onboarding step.
|
||||
Won't reward if the user has already received credits for the step.
|
||||
|
||||
Args:
|
||||
user_id (str): The user ID.
|
||||
credits (int): The amount to reward.
|
||||
step (OnboardingStep): The onboarding step.
|
||||
|
||||
Returns:
|
||||
bool: True if rewarded, False if already rewarded.
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -236,6 +242,12 @@ class UserCreditBase(ABC):
|
||||
"""
|
||||
Returns the current balance of the user & the latest balance snapshot time.
|
||||
"""
|
||||
# Check UserBalance first for efficiency and consistency
|
||||
user_balance = await UserBalance.prisma().find_unique(where={"userId": user_id})
|
||||
if user_balance:
|
||||
return user_balance.balance, user_balance.updatedAt
|
||||
|
||||
# Fallback to transaction history computation if UserBalance doesn't exist
|
||||
top_time = self.time_now()
|
||||
snapshot = await CreditTransaction.prisma().find_first(
|
||||
where={
|
||||
@@ -250,72 +262,86 @@ class UserCreditBase(ABC):
|
||||
snapshot_balance = snapshot.runningBalance or 0 if snapshot else 0
|
||||
snapshot_time = snapshot.createdAt if snapshot else datetime_min
|
||||
|
||||
# Get transactions after the snapshot, this should not exist, but just in case.
|
||||
transactions = await CreditTransaction.prisma().group_by(
|
||||
by=["userId"],
|
||||
sum={"amount": True},
|
||||
max={"createdAt": True},
|
||||
where={
|
||||
"userId": user_id,
|
||||
"createdAt": {
|
||||
"gt": snapshot_time,
|
||||
"lte": top_time,
|
||||
},
|
||||
"isActive": True,
|
||||
},
|
||||
)
|
||||
transaction_balance = (
|
||||
int(transactions[0].get("_sum", {}).get("amount", 0) + snapshot_balance)
|
||||
if transactions
|
||||
else snapshot_balance
|
||||
)
|
||||
transaction_time = (
|
||||
datetime.fromisoformat(
|
||||
str(transactions[0].get("_max", {}).get("createdAt", datetime_min))
|
||||
)
|
||||
if transactions
|
||||
else snapshot_time
|
||||
)
|
||||
return transaction_balance, transaction_time
|
||||
return snapshot_balance, snapshot_time
|
||||
|
||||
@func_retry
|
||||
async def _enable_transaction(
|
||||
self,
|
||||
transaction_key: str,
|
||||
user_id: str,
|
||||
metadata: Json,
|
||||
metadata: SafeJson,
|
||||
new_transaction_key: str | None = None,
|
||||
):
|
||||
transaction = await CreditTransaction.prisma().find_first_or_raise(
|
||||
where={"transactionKey": transaction_key, "userId": user_id}
|
||||
# First check if transaction exists and is inactive (safety check)
|
||||
transaction = await CreditTransaction.prisma().find_first(
|
||||
where={
|
||||
"transactionKey": transaction_key,
|
||||
"userId": user_id,
|
||||
"isActive": False,
|
||||
}
|
||||
)
|
||||
if transaction.isActive:
|
||||
return
|
||||
if not transaction:
|
||||
# Transaction doesn't exist or is already active, return early
|
||||
return None
|
||||
|
||||
async with db.locked_transaction(f"usr_trx_{user_id}"):
|
||||
|
||||
transaction = await CreditTransaction.prisma().find_first_or_raise(
|
||||
where={"transactionKey": transaction_key, "userId": user_id}
|
||||
# Atomic operation to enable transaction and update user balance using UserBalance
|
||||
result = await query_raw_with_schema(
|
||||
"""
|
||||
WITH user_balance_lock AS (
|
||||
SELECT
|
||||
$2::text as userId,
|
||||
COALESCE(
|
||||
(SELECT balance FROM {schema_prefix}"UserBalance" WHERE "userId" = $2 FOR UPDATE),
|
||||
-- Fallback: compute balance from transaction history if UserBalance doesn't exist
|
||||
(SELECT COALESCE(ct."runningBalance", 0)
|
||||
FROM {schema_prefix}"CreditTransaction" ct
|
||||
WHERE ct."userId" = $2
|
||||
AND ct."isActive" = true
|
||||
AND ct."runningBalance" IS NOT NULL
|
||||
ORDER BY ct."createdAt" DESC
|
||||
LIMIT 1),
|
||||
0
|
||||
) as balance
|
||||
),
|
||||
transaction_check AS (
|
||||
SELECT * FROM {schema_prefix}"CreditTransaction"
|
||||
WHERE "transactionKey" = $1 AND "userId" = $2 AND "isActive" = false
|
||||
),
|
||||
balance_update AS (
|
||||
INSERT INTO {schema_prefix}"UserBalance" ("userId", "balance", "updatedAt")
|
||||
SELECT
|
||||
$2::text,
|
||||
user_balance_lock.balance + transaction_check.amount,
|
||||
CURRENT_TIMESTAMP
|
||||
FROM user_balance_lock, transaction_check
|
||||
ON CONFLICT ("userId") DO UPDATE SET
|
||||
"balance" = EXCLUDED."balance",
|
||||
"updatedAt" = EXCLUDED."updatedAt"
|
||||
RETURNING "balance", "updatedAt"
|
||||
),
|
||||
transaction_update AS (
|
||||
UPDATE {schema_prefix}"CreditTransaction"
|
||||
SET "transactionKey" = COALESCE($4, $1),
|
||||
"isActive" = true,
|
||||
"runningBalance" = balance_update.balance,
|
||||
"createdAt" = balance_update."updatedAt",
|
||||
"metadata" = $3::jsonb
|
||||
FROM balance_update, transaction_check
|
||||
WHERE {schema_prefix}"CreditTransaction"."transactionKey" = transaction_check."transactionKey"
|
||||
AND {schema_prefix}"CreditTransaction"."userId" = transaction_check."userId"
|
||||
RETURNING {schema_prefix}"CreditTransaction"."runningBalance"
|
||||
)
|
||||
if transaction.isActive:
|
||||
return
|
||||
SELECT "runningBalance" as balance FROM transaction_update;
|
||||
""",
|
||||
transaction_key, # $1
|
||||
user_id, # $2
|
||||
dumps(metadata.data), # $3 - use pre-serialized JSON string for JSONB
|
||||
new_transaction_key, # $4
|
||||
)
|
||||
|
||||
user_balance, _ = await self._get_credits(user_id)
|
||||
await CreditTransaction.prisma().update(
|
||||
where={
|
||||
"creditTransactionIdentifier": {
|
||||
"transactionKey": transaction_key,
|
||||
"userId": user_id,
|
||||
}
|
||||
},
|
||||
data={
|
||||
"transactionKey": new_transaction_key or transaction_key,
|
||||
"isActive": True,
|
||||
"runningBalance": user_balance + transaction.amount,
|
||||
"createdAt": self.time_now(),
|
||||
"metadata": metadata,
|
||||
},
|
||||
)
|
||||
if result:
|
||||
# UserBalance is already updated by the CTE
|
||||
return result[0]["balance"]
|
||||
|
||||
async def _add_transaction(
|
||||
self,
|
||||
@@ -326,12 +352,54 @@ class UserCreditBase(ABC):
|
||||
transaction_key: str | None = None,
|
||||
ceiling_balance: int | None = None,
|
||||
fail_insufficient_credits: bool = True,
|
||||
metadata: Json = SafeJson({}),
|
||||
metadata: SafeJson = SafeJson({}),
|
||||
) -> tuple[int, str]:
|
||||
"""
|
||||
Add a new transaction for the user.
|
||||
This is the only method that should be used to add a new transaction.
|
||||
|
||||
ATOMIC OPERATION DESIGN DECISION:
|
||||
================================
|
||||
This method uses PostgreSQL row-level locking (FOR UPDATE) for atomic credit operations.
|
||||
After extensive analysis of concurrency patterns and correctness requirements, we determined
|
||||
that the FOR UPDATE approach is necessary despite the latency overhead.
|
||||
|
||||
WHY FOR UPDATE LOCKING IS REQUIRED:
|
||||
----------------------------------
|
||||
1. **Data Consistency**: Credit operations must be ACID-compliant. The balance check,
|
||||
calculation, and update must be atomic to prevent race conditions where:
|
||||
- Multiple spend operations could exceed available balance
|
||||
- Lost update problems could occur with concurrent top-ups
|
||||
- Refunds could create negative balances incorrectly
|
||||
|
||||
2. **Serializability**: FOR UPDATE ensures operations are serialized at the database level,
|
||||
guaranteeing that each transaction sees a consistent view of the balance before applying changes.
|
||||
|
||||
3. **Correctness Over Performance**: Financial operations require absolute correctness.
|
||||
The ~10-50ms latency increase from row locking is acceptable for the guarantee that
|
||||
no user will ever have an incorrect balance due to race conditions.
|
||||
|
||||
4. **PostgreSQL Optimization**: Modern PostgreSQL versions optimize row locks efficiently.
|
||||
The performance cost is minimal compared to the complexity and risk of lock-free approaches.
|
||||
|
||||
ALTERNATIVES CONSIDERED AND REJECTED:
|
||||
------------------------------------
|
||||
- **Optimistic Concurrency**: Using version numbers or timestamps would require complex
|
||||
retry logic and could still fail under high contention scenarios.
|
||||
- **Application-Level Locking**: Redis locks or similar would add network overhead and
|
||||
single points of failure while being less reliable than database locks.
|
||||
- **Event Sourcing**: Would require complete architectural changes and eventual consistency
|
||||
models that don't fit our real-time balance requirements.
|
||||
|
||||
PERFORMANCE CHARACTERISTICS:
|
||||
---------------------------
|
||||
- Single user operations: 10-50ms latency (acceptable for financial operations)
|
||||
- Concurrent operations on same user: Serialized (prevents data corruption)
|
||||
- Concurrent operations on different users: Fully parallel (no blocking)
|
||||
|
||||
This design prioritizes correctness and data integrity over raw performance,
|
||||
which is the appropriate choice for a credit/payment system.
|
||||
|
||||
Args:
|
||||
user_id (str): The user ID.
|
||||
amount (int): The amount of credits to add.
|
||||
@@ -345,40 +413,142 @@ class UserCreditBase(ABC):
|
||||
Returns:
|
||||
tuple[int, str]: The new balance & the transaction key.
|
||||
"""
|
||||
async with db.locked_transaction(f"usr_trx_{user_id}"):
|
||||
# Get latest balance snapshot
|
||||
user_balance, _ = await self._get_credits(user_id)
|
||||
|
||||
if ceiling_balance and amount > 0 and user_balance >= ceiling_balance:
|
||||
# Quick validation for ceiling balance to avoid unnecessary database operations
|
||||
if ceiling_balance and amount > 0:
|
||||
current_balance, _ = await self._get_credits(user_id)
|
||||
if current_balance >= ceiling_balance:
|
||||
raise ValueError(
|
||||
f"You already have enough balance of ${user_balance/100}, top-up is not required when you already have at least ${ceiling_balance/100}"
|
||||
f"You already have enough balance of ${current_balance/100}, top-up is not required when you already have at least ${ceiling_balance/100}"
|
||||
)
|
||||
|
||||
if amount < 0 and user_balance + amount < 0:
|
||||
if fail_insufficient_credits:
|
||||
raise InsufficientBalanceError(
|
||||
message=f"Insufficient balance of ${user_balance/100}, where this will cost ${abs(amount)/100}",
|
||||
user_id=user_id,
|
||||
balance=user_balance,
|
||||
amount=amount,
|
||||
# Single unified atomic operation for all transaction types using UserBalance
|
||||
try:
|
||||
result = await query_raw_with_schema(
|
||||
"""
|
||||
WITH user_balance_lock AS (
|
||||
SELECT
|
||||
$1::text as userId,
|
||||
-- CRITICAL: FOR UPDATE lock prevents concurrent modifications to the same user's balance
|
||||
-- This ensures atomic read-modify-write operations and prevents race conditions
|
||||
COALESCE(
|
||||
(SELECT balance FROM {schema_prefix}"UserBalance" WHERE "userId" = $1 FOR UPDATE),
|
||||
-- Fallback: compute balance from transaction history if UserBalance doesn't exist
|
||||
(SELECT COALESCE(ct."runningBalance", 0)
|
||||
FROM {schema_prefix}"CreditTransaction" ct
|
||||
WHERE ct."userId" = $1
|
||||
AND ct."isActive" = true
|
||||
AND ct."runningBalance" IS NOT NULL
|
||||
ORDER BY ct."createdAt" DESC
|
||||
LIMIT 1),
|
||||
0
|
||||
) as balance
|
||||
),
|
||||
balance_update AS (
|
||||
INSERT INTO {schema_prefix}"UserBalance" ("userId", "balance", "updatedAt")
|
||||
SELECT
|
||||
$1::text,
|
||||
CASE
|
||||
-- For inactive transactions: Don't update balance
|
||||
WHEN $5::boolean = false THEN user_balance_lock.balance
|
||||
-- For ceiling balance (amount > 0): Apply ceiling
|
||||
WHEN $2 > 0 AND $7::int IS NOT NULL AND user_balance_lock.balance > $7::int - $2 THEN $7::int
|
||||
-- For regular operations: Apply with overflow/underflow protection
|
||||
WHEN user_balance_lock.balance + $2 > $6::int THEN $6::int
|
||||
WHEN user_balance_lock.balance + $2 < $10::int THEN $10::int
|
||||
ELSE user_balance_lock.balance + $2
|
||||
END,
|
||||
CURRENT_TIMESTAMP
|
||||
FROM user_balance_lock
|
||||
WHERE (
|
||||
$5::boolean = false OR -- Allow inactive transactions
|
||||
$2 >= 0 OR -- Allow positive amounts (top-ups, grants)
|
||||
$8::boolean = false OR -- Allow when insufficient balance check is disabled
|
||||
user_balance_lock.balance + $2 >= 0 -- Allow spending only when sufficient balance
|
||||
)
|
||||
ON CONFLICT ("userId") DO UPDATE SET
|
||||
"balance" = EXCLUDED."balance",
|
||||
"updatedAt" = EXCLUDED."updatedAt"
|
||||
RETURNING "balance", "updatedAt"
|
||||
),
|
||||
transaction_insert AS (
|
||||
INSERT INTO {schema_prefix}"CreditTransaction" (
|
||||
"userId", "amount", "type", "runningBalance",
|
||||
"metadata", "isActive", "createdAt", "transactionKey"
|
||||
)
|
||||
SELECT
|
||||
$1::text,
|
||||
$2::int,
|
||||
$3::text::{schema_prefix}"CreditTransactionType",
|
||||
CASE
|
||||
-- For inactive transactions: Set runningBalance to original balance (don't apply the change yet)
|
||||
WHEN $5::boolean = false THEN user_balance_lock.balance
|
||||
ELSE COALESCE(balance_update.balance, user_balance_lock.balance)
|
||||
END,
|
||||
$4::jsonb,
|
||||
$5::boolean,
|
||||
COALESCE(balance_update."updatedAt", CURRENT_TIMESTAMP),
|
||||
COALESCE($9, gen_random_uuid()::text)
|
||||
FROM user_balance_lock
|
||||
LEFT JOIN balance_update ON true
|
||||
WHERE (
|
||||
$5::boolean = false OR -- Allow inactive transactions
|
||||
$2 >= 0 OR -- Allow positive amounts (top-ups, grants)
|
||||
$8::boolean = false OR -- Allow when insufficient balance check is disabled
|
||||
user_balance_lock.balance + $2 >= 0 -- Allow spending only when sufficient balance
|
||||
)
|
||||
RETURNING "runningBalance", "transactionKey"
|
||||
)
|
||||
SELECT "runningBalance" as balance, "transactionKey" FROM transaction_insert;
|
||||
""",
|
||||
user_id, # $1
|
||||
amount, # $2
|
||||
transaction_type.value, # $3
|
||||
dumps(metadata.data), # $4 - use pre-serialized JSON string for JSONB
|
||||
is_active, # $5
|
||||
POSTGRES_INT_MAX, # $6 - overflow protection
|
||||
ceiling_balance, # $7 - ceiling balance (nullable)
|
||||
fail_insufficient_credits, # $8 - check balance for spending
|
||||
transaction_key, # $9 - transaction key (nullable)
|
||||
POSTGRES_INT_MIN, # $10 - underflow protection
|
||||
)
|
||||
except Exception as e:
|
||||
# Convert raw SQL unique constraint violations to UniqueViolationError
|
||||
# for consistent exception handling throughout the codebase
|
||||
error_str = str(e).lower()
|
||||
if (
|
||||
"already exists" in error_str
|
||||
or "duplicate key" in error_str
|
||||
or "unique constraint" in error_str
|
||||
):
|
||||
# Extract table and constraint info for better error messages
|
||||
# Re-raise as a UniqueViolationError but with proper format
|
||||
# Create a minimal data structure that the error constructor expects
|
||||
raise UniqueViolationError({"error": str(e), "user_facing_error": {}})
|
||||
# For any other error, re-raise as-is
|
||||
raise
|
||||
|
||||
amount = min(-user_balance, 0)
|
||||
if result:
|
||||
new_balance, tx_key = result[0]["balance"], result[0]["transactionKey"]
|
||||
# UserBalance is already updated by the CTE
|
||||
return new_balance, tx_key
|
||||
|
||||
# Create the transaction
|
||||
transaction_data: CreditTransactionCreateInput = {
|
||||
"userId": user_id,
|
||||
"amount": amount,
|
||||
"runningBalance": user_balance + amount,
|
||||
"type": transaction_type,
|
||||
"metadata": metadata,
|
||||
"isActive": is_active,
|
||||
"createdAt": self.time_now(),
|
||||
}
|
||||
if transaction_key:
|
||||
transaction_data["transactionKey"] = transaction_key
|
||||
tx = await CreditTransaction.prisma().create(data=transaction_data)
|
||||
return user_balance + amount, tx.transactionKey
|
||||
# If no result, either user doesn't exist or insufficient balance
|
||||
user = await User.prisma().find_unique(where={"id": user_id})
|
||||
if not user:
|
||||
raise ValueError(f"User {user_id} not found")
|
||||
|
||||
# Must be insufficient balance for spending operation
|
||||
if amount < 0 and fail_insufficient_credits:
|
||||
current_balance, _ = await self._get_credits(user_id)
|
||||
raise InsufficientBalanceError(
|
||||
message=f"Insufficient balance of ${current_balance/100}, where this will cost ${abs(amount)/100}",
|
||||
user_id=user_id,
|
||||
balance=current_balance,
|
||||
amount=amount,
|
||||
)
|
||||
|
||||
# Unexpected case
|
||||
raise ValueError(f"Transaction failed for user {user_id}, amount {amount}")
|
||||
|
||||
|
||||
class UserCredit(UserCreditBase):
|
||||
@@ -453,9 +623,10 @@ class UserCredit(UserCreditBase):
|
||||
{"reason": f"Reward for completing {step.value} onboarding step."}
|
||||
),
|
||||
)
|
||||
return True
|
||||
except UniqueViolationError:
|
||||
# Already rewarded for this step
|
||||
pass
|
||||
# User already received this reward
|
||||
return False
|
||||
|
||||
async def top_up_refund(
|
||||
self, user_id: str, transaction_key: str, metadata: dict[str, str]
|
||||
@@ -644,7 +815,7 @@ class UserCredit(UserCreditBase):
|
||||
):
|
||||
# init metadata, without sharing it with the world
|
||||
metadata = metadata or {}
|
||||
if not metadata["reason"]:
|
||||
if not metadata.get("reason"):
|
||||
match top_up_type:
|
||||
case TopUpType.MANUAL:
|
||||
metadata["reason"] = {"reason": f"Top up credits for {user_id}"}
|
||||
@@ -974,8 +1145,8 @@ class DisabledUserCredit(UserCreditBase):
|
||||
async def top_up_credits(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
async def onboarding_reward(self, *args, **kwargs):
|
||||
pass
|
||||
async def onboarding_reward(self, *args, **kwargs) -> bool:
|
||||
return True
|
||||
|
||||
async def top_up_intent(self, *args, **kwargs) -> str:
|
||||
return ""
|
||||
@@ -993,14 +1164,31 @@ class DisabledUserCredit(UserCreditBase):
|
||||
pass
|
||||
|
||||
|
||||
def get_user_credit_model() -> UserCreditBase:
|
||||
async def get_user_credit_model(user_id: str) -> UserCreditBase:
|
||||
"""
|
||||
Get the credit model for a user, considering LaunchDarkly flags.
|
||||
|
||||
Args:
|
||||
user_id (str): The user ID to check flags for.
|
||||
|
||||
Returns:
|
||||
UserCreditBase: The appropriate credit model for the user
|
||||
"""
|
||||
if not settings.config.enable_credit:
|
||||
return DisabledUserCredit()
|
||||
|
||||
if settings.config.enable_beta_monthly_credit:
|
||||
return BetaUserCredit(settings.config.num_user_credits_refill)
|
||||
# Check LaunchDarkly flag for payment pilot users
|
||||
# Default to False (beta monthly credit behavior) to maintain current behavior
|
||||
is_payment_enabled = await is_feature_enabled(
|
||||
Flag.ENABLE_PLATFORM_PAYMENT, user_id, default=False
|
||||
)
|
||||
|
||||
return UserCredit()
|
||||
if is_payment_enabled:
|
||||
# Payment enabled users get UserCredit (no monthly refills, enable payments)
|
||||
return UserCredit()
|
||||
else:
|
||||
# Default behavior: users get beta monthly credits
|
||||
return BetaUserCredit(settings.config.num_user_credits_refill)
|
||||
|
||||
|
||||
def get_block_costs() -> dict[str, list["BlockCost"]]:
|
||||
@@ -1090,7 +1278,8 @@ async def admin_get_user_history(
|
||||
)
|
||||
reason = metadata.get("reason", "No reason provided")
|
||||
|
||||
balance, last_update = await get_user_credit_model()._get_credits(tx.userId)
|
||||
user_credit_model = await get_user_credit_model(tx.userId)
|
||||
balance, _ = await user_credit_model._get_credits(tx.userId)
|
||||
|
||||
history.append(
|
||||
UserTransaction(
|
||||
|
||||
172
autogpt_platform/backend/backend/data/credit_ceiling_test.py
Normal file
172
autogpt_platform/backend/backend/data/credit_ceiling_test.py
Normal file
@@ -0,0 +1,172 @@
|
||||
"""
|
||||
Test ceiling balance functionality to ensure auto top-up limits work correctly.
|
||||
|
||||
This test was added to cover a previously untested code path that could lead to
|
||||
incorrect balance capping behavior.
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.errors import UniqueViolationError
|
||||
from prisma.models import CreditTransaction, User, UserBalance
|
||||
|
||||
from backend.data.credit import UserCredit
|
||||
from backend.util.json import SafeJson
|
||||
from backend.util.test import SpinTestServer
|
||||
|
||||
|
||||
async def create_test_user(user_id: str) -> None:
|
||||
"""Create a test user for ceiling tests."""
|
||||
try:
|
||||
await User.prisma().create(
|
||||
data={
|
||||
"id": user_id,
|
||||
"email": f"test-{user_id}@example.com",
|
||||
"name": f"Test User {user_id[:8]}",
|
||||
}
|
||||
)
|
||||
except UniqueViolationError:
|
||||
# User already exists, continue
|
||||
pass
|
||||
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={"create": {"userId": user_id, "balance": 0}, "update": {"balance": 0}},
|
||||
)
|
||||
|
||||
|
||||
async def cleanup_test_user(user_id: str) -> None:
|
||||
"""Clean up test user and their transactions."""
|
||||
try:
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": user_id})
|
||||
await User.prisma().delete_many(where={"id": user_id})
|
||||
except Exception as e:
|
||||
# Log cleanup failures but don't fail the test
|
||||
print(f"Warning: Failed to cleanup test user {user_id}: {e}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_ceiling_balance_rejects_when_above_threshold(server: SpinTestServer):
|
||||
"""Test that ceiling balance correctly rejects top-ups when balance is above threshold."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"ceiling-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Give user balance of 1000 ($10) using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=1000,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "initial_balance"}),
|
||||
)
|
||||
current_balance = await credit_system.get_credits(user_id)
|
||||
assert current_balance == 1000
|
||||
|
||||
# Try to add 200 more with ceiling of 800 (should reject since 1000 > 800)
|
||||
with pytest.raises(ValueError, match="You already have enough balance"):
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=200,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
ceiling_balance=800, # Ceiling lower than current balance
|
||||
)
|
||||
|
||||
# Balance should remain unchanged
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 1000, f"Balance should remain 1000, got {final_balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_ceiling_balance_clamps_when_would_exceed(server: SpinTestServer):
|
||||
"""Test that ceiling balance correctly clamps amounts that would exceed the ceiling."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"ceiling-clamp-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Give user balance of 500 ($5) using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=500,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "initial_balance"}),
|
||||
)
|
||||
|
||||
# Add 800 more with ceiling of 1000 (should clamp to 1000, not reach 1300)
|
||||
final_balance, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=800,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
ceiling_balance=1000, # Ceiling should clamp 500 + 800 = 1300 to 1000
|
||||
)
|
||||
|
||||
# Balance should be clamped to ceiling
|
||||
assert (
|
||||
final_balance == 1000
|
||||
), f"Balance should be clamped to 1000, got {final_balance}"
|
||||
|
||||
# Verify with get_credits too
|
||||
stored_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
stored_balance == 1000
|
||||
), f"Stored balance should be 1000, got {stored_balance}"
|
||||
|
||||
# Verify transaction shows the clamped amount
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={"userId": user_id, "type": CreditTransactionType.TOP_UP},
|
||||
order={"createdAt": "desc"},
|
||||
)
|
||||
|
||||
# Should have 2 transactions: 500 + (500 to reach ceiling of 1000)
|
||||
assert len(transactions) == 2
|
||||
|
||||
# The second transaction should show it only added 500, not 800
|
||||
second_tx = transactions[0] # Most recent
|
||||
assert second_tx.runningBalance == 1000
|
||||
# The actual amount recorded could be 800 (what was requested) but balance was clamped
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_ceiling_balance_allows_when_under_threshold(server: SpinTestServer):
|
||||
"""Test that ceiling balance allows top-ups when balance is under threshold."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"ceiling-under-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Give user balance of 300 ($3) using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=300,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "initial_balance"}),
|
||||
)
|
||||
|
||||
# Add 200 more with ceiling of 1000 (should succeed: 300 + 200 = 500 < 1000)
|
||||
final_balance, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=200,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
ceiling_balance=1000,
|
||||
)
|
||||
|
||||
# Balance should be exactly 500
|
||||
assert final_balance == 500, f"Balance should be 500, got {final_balance}"
|
||||
|
||||
# Verify with get_credits too
|
||||
stored_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
stored_balance == 500
|
||||
), f"Stored balance should be 500, got {stored_balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
737
autogpt_platform/backend/backend/data/credit_concurrency_test.py
Normal file
737
autogpt_platform/backend/backend/data/credit_concurrency_test.py
Normal file
@@ -0,0 +1,737 @@
|
||||
"""
|
||||
Concurrency and atomicity tests for the credit system.
|
||||
|
||||
These tests ensure the credit system handles high-concurrency scenarios correctly
|
||||
without race conditions, deadlocks, or inconsistent state.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
from uuid import uuid4
|
||||
|
||||
import prisma.enums
|
||||
import pytest
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.errors import UniqueViolationError
|
||||
from prisma.models import CreditTransaction, User, UserBalance
|
||||
|
||||
from backend.data.credit import POSTGRES_INT_MAX, UsageTransactionMetadata, UserCredit
|
||||
from backend.util.exceptions import InsufficientBalanceError
|
||||
from backend.util.json import SafeJson
|
||||
from backend.util.test import SpinTestServer
|
||||
|
||||
# Test with both UserCredit and BetaUserCredit if needed
|
||||
credit_system = UserCredit()
|
||||
|
||||
|
||||
async def create_test_user(user_id: str) -> None:
|
||||
"""Create a test user with initial balance."""
|
||||
try:
|
||||
await User.prisma().create(
|
||||
data={
|
||||
"id": user_id,
|
||||
"email": f"test-{user_id}@example.com",
|
||||
"name": f"Test User {user_id[:8]}",
|
||||
}
|
||||
)
|
||||
except UniqueViolationError:
|
||||
# User already exists, continue
|
||||
pass
|
||||
|
||||
# Ensure UserBalance record exists
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={"create": {"userId": user_id, "balance": 0}, "update": {"balance": 0}},
|
||||
)
|
||||
|
||||
|
||||
async def cleanup_test_user(user_id: str) -> None:
|
||||
"""Clean up test user and their transactions."""
|
||||
try:
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": user_id})
|
||||
await UserBalance.prisma().delete_many(where={"userId": user_id})
|
||||
await User.prisma().delete_many(where={"id": user_id})
|
||||
except Exception as e:
|
||||
# Log cleanup failures but don't fail the test
|
||||
print(f"Warning: Failed to cleanup test user {user_id}: {e}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_concurrent_spends_same_user(server: SpinTestServer):
|
||||
"""Test multiple concurrent spends from the same user don't cause race conditions."""
|
||||
user_id = f"concurrent-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Give user initial balance using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=1000,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "initial_balance"}),
|
||||
)
|
||||
|
||||
# Try to spend 10 x $1 concurrently
|
||||
async def spend_one_dollar(idx: int):
|
||||
try:
|
||||
return await credit_system.spend_credits(
|
||||
user_id,
|
||||
100, # $1
|
||||
UsageTransactionMetadata(
|
||||
graph_exec_id=f"concurrent-{idx}",
|
||||
reason=f"Concurrent spend {idx}",
|
||||
),
|
||||
)
|
||||
except InsufficientBalanceError:
|
||||
return None
|
||||
|
||||
# Run 10 concurrent spends
|
||||
results = await asyncio.gather(
|
||||
*[spend_one_dollar(i) for i in range(10)], return_exceptions=True
|
||||
)
|
||||
|
||||
# Count successful spends
|
||||
successful = [
|
||||
r for r in results if r is not None and not isinstance(r, Exception)
|
||||
]
|
||||
failed = [r for r in results if isinstance(r, InsufficientBalanceError)]
|
||||
|
||||
# All 10 should succeed since we have exactly $10
|
||||
assert len(successful) == 10, f"Expected 10 successful, got {len(successful)}"
|
||||
assert len(failed) == 0, f"Expected 0 failures, got {len(failed)}"
|
||||
|
||||
# Final balance should be exactly 0
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 0, f"Expected balance 0, got {final_balance}"
|
||||
|
||||
# Verify transaction history is consistent
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={"userId": user_id, "type": prisma.enums.CreditTransactionType.USAGE}
|
||||
)
|
||||
assert (
|
||||
len(transactions) == 10
|
||||
), f"Expected 10 transactions, got {len(transactions)}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_concurrent_spends_insufficient_balance(server: SpinTestServer):
|
||||
"""Test that concurrent spends correctly enforce balance limits."""
|
||||
user_id = f"insufficient-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Give user limited balance using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=500,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "limited_balance"}),
|
||||
)
|
||||
|
||||
# Try to spend 10 x $1 concurrently (but only have $5)
|
||||
async def spend_one_dollar(idx: int):
|
||||
try:
|
||||
return await credit_system.spend_credits(
|
||||
user_id,
|
||||
100, # $1
|
||||
UsageTransactionMetadata(
|
||||
graph_exec_id=f"insufficient-{idx}",
|
||||
reason=f"Insufficient spend {idx}",
|
||||
),
|
||||
)
|
||||
except InsufficientBalanceError:
|
||||
return "FAILED"
|
||||
|
||||
# Run 10 concurrent spends
|
||||
results = await asyncio.gather(
|
||||
*[spend_one_dollar(i) for i in range(10)], return_exceptions=True
|
||||
)
|
||||
|
||||
# Count successful vs failed
|
||||
successful = [
|
||||
r
|
||||
for r in results
|
||||
if r not in ["FAILED", None] and not isinstance(r, Exception)
|
||||
]
|
||||
failed = [r for r in results if r == "FAILED"]
|
||||
|
||||
# Exactly 5 should succeed, 5 should fail
|
||||
assert len(successful) == 5, f"Expected 5 successful, got {len(successful)}"
|
||||
assert len(failed) == 5, f"Expected 5 failures, got {len(failed)}"
|
||||
|
||||
# Final balance should be exactly 0
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 0, f"Expected balance 0, got {final_balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_concurrent_mixed_operations(server: SpinTestServer):
|
||||
"""Test concurrent mix of spends, top-ups, and balance checks."""
|
||||
user_id = f"mixed-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Initial balance using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=1000,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "initial_balance"}),
|
||||
)
|
||||
|
||||
# Mix of operations
|
||||
async def mixed_operations():
|
||||
operations = []
|
||||
|
||||
# 5 spends of $1 each
|
||||
for i in range(5):
|
||||
operations.append(
|
||||
credit_system.spend_credits(
|
||||
user_id,
|
||||
100,
|
||||
UsageTransactionMetadata(reason=f"Mixed spend {i}"),
|
||||
)
|
||||
)
|
||||
|
||||
# 3 top-ups of $2 each using internal method
|
||||
for i in range(3):
|
||||
operations.append(
|
||||
credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=200,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": f"concurrent_topup_{i}"}),
|
||||
)
|
||||
)
|
||||
|
||||
# 10 balance checks
|
||||
for i in range(10):
|
||||
operations.append(credit_system.get_credits(user_id))
|
||||
|
||||
return await asyncio.gather(*operations, return_exceptions=True)
|
||||
|
||||
results = await mixed_operations()
|
||||
|
||||
# Check no exceptions occurred
|
||||
exceptions = [
|
||||
r
|
||||
for r in results
|
||||
if isinstance(r, Exception) and not isinstance(r, InsufficientBalanceError)
|
||||
]
|
||||
assert len(exceptions) == 0, f"Unexpected exceptions: {exceptions}"
|
||||
|
||||
# Final balance should be: 1000 - 500 + 600 = 1100
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 1100, f"Expected balance 1100, got {final_balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_race_condition_exact_balance(server: SpinTestServer):
|
||||
"""Test spending exact balance amount concurrently doesn't go negative."""
|
||||
user_id = f"exact-balance-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Give exact amount using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=100,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "exact_amount"}),
|
||||
)
|
||||
|
||||
# Try to spend $1 twice concurrently
|
||||
async def spend_exact():
|
||||
try:
|
||||
return await credit_system.spend_credits(
|
||||
user_id, 100, UsageTransactionMetadata(reason="Exact spend")
|
||||
)
|
||||
except InsufficientBalanceError:
|
||||
return "FAILED"
|
||||
|
||||
# Both try to spend the full balance
|
||||
result1, result2 = await asyncio.gather(spend_exact(), spend_exact())
|
||||
|
||||
# Exactly one should succeed
|
||||
results = [result1, result2]
|
||||
successful = [
|
||||
r for r in results if r != "FAILED" and not isinstance(r, Exception)
|
||||
]
|
||||
failed = [r for r in results if r == "FAILED"]
|
||||
|
||||
assert len(successful) == 1, f"Expected 1 success, got {len(successful)}"
|
||||
assert len(failed) == 1, f"Expected 1 failure, got {len(failed)}"
|
||||
|
||||
# Balance should be exactly 0, never negative
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 0, f"Expected balance 0, got {final_balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_onboarding_reward_idempotency(server: SpinTestServer):
|
||||
"""Test that onboarding rewards are idempotent (can't be claimed twice)."""
|
||||
user_id = f"onboarding-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Use WELCOME step which is defined in the OnboardingStep enum
|
||||
# Try to claim same reward multiple times concurrently
|
||||
async def claim_reward():
|
||||
try:
|
||||
result = await credit_system.onboarding_reward(
|
||||
user_id, 500, prisma.enums.OnboardingStep.WELCOME
|
||||
)
|
||||
return "SUCCESS" if result else "DUPLICATE"
|
||||
except Exception as e:
|
||||
print(f"Claim reward failed: {e}")
|
||||
return "FAILED"
|
||||
|
||||
# Try 5 concurrent claims of the same reward
|
||||
results = await asyncio.gather(*[claim_reward() for _ in range(5)])
|
||||
|
||||
# Count results
|
||||
success_count = results.count("SUCCESS")
|
||||
failed_count = results.count("FAILED")
|
||||
|
||||
# At least one should succeed, others should be duplicates
|
||||
assert success_count >= 1, f"At least one claim should succeed, got {results}"
|
||||
assert failed_count == 0, f"No claims should fail, got {results}"
|
||||
|
||||
# Check balance - should only have 500, not 2500
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 500, f"Expected balance 500, got {final_balance}"
|
||||
|
||||
# Check only one transaction exists
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={
|
||||
"userId": user_id,
|
||||
"type": prisma.enums.CreditTransactionType.GRANT,
|
||||
"transactionKey": f"REWARD-{user_id}-WELCOME",
|
||||
}
|
||||
)
|
||||
assert (
|
||||
len(transactions) == 1
|
||||
), f"Expected 1 reward transaction, got {len(transactions)}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_integer_overflow_protection(server: SpinTestServer):
|
||||
"""Test that integer overflow is prevented by clamping to POSTGRES_INT_MAX."""
|
||||
user_id = f"overflow-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Try to add amount that would overflow
|
||||
max_int = POSTGRES_INT_MAX
|
||||
|
||||
# First, set balance near max
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={
|
||||
"create": {"userId": user_id, "balance": max_int - 100},
|
||||
"update": {"balance": max_int - 100},
|
||||
},
|
||||
)
|
||||
|
||||
# Try to add more than possible - should clamp to POSTGRES_INT_MAX
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=200,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "overflow_protection"}),
|
||||
)
|
||||
|
||||
# Balance should be clamped to max_int, not overflowed
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
final_balance == max_int
|
||||
), f"Balance should be clamped to {max_int}, got {final_balance}"
|
||||
|
||||
# Verify transaction was created with clamped amount
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={
|
||||
"userId": user_id,
|
||||
"type": prisma.enums.CreditTransactionType.TOP_UP,
|
||||
},
|
||||
order={"createdAt": "desc"},
|
||||
)
|
||||
assert len(transactions) > 0, "Transaction should be created"
|
||||
assert (
|
||||
transactions[0].runningBalance == max_int
|
||||
), "Transaction should show clamped balance"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_high_concurrency_stress(server: SpinTestServer):
|
||||
"""Stress test with many concurrent operations."""
|
||||
user_id = f"stress-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Initial balance using internal method (bypasses Stripe)
|
||||
initial_balance = 10000 # $100
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=initial_balance,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "stress_test_balance"}),
|
||||
)
|
||||
|
||||
# Run many concurrent operations
|
||||
async def random_operation(idx: int):
|
||||
operation = random.choice(["spend", "check"])
|
||||
|
||||
if operation == "spend":
|
||||
amount = random.randint(1, 50) # $0.01 to $0.50
|
||||
try:
|
||||
return (
|
||||
"spend",
|
||||
amount,
|
||||
await credit_system.spend_credits(
|
||||
user_id,
|
||||
amount,
|
||||
UsageTransactionMetadata(reason=f"Stress {idx}"),
|
||||
),
|
||||
)
|
||||
except InsufficientBalanceError:
|
||||
return ("spend_failed", amount, None)
|
||||
else:
|
||||
balance = await credit_system.get_credits(user_id)
|
||||
return ("check", 0, balance)
|
||||
|
||||
# Run 100 concurrent operations
|
||||
results = await asyncio.gather(
|
||||
*[random_operation(i) for i in range(100)], return_exceptions=True
|
||||
)
|
||||
|
||||
# Calculate expected final balance
|
||||
total_spent = sum(
|
||||
r[1]
|
||||
for r in results
|
||||
if not isinstance(r, Exception) and isinstance(r, tuple) and r[0] == "spend"
|
||||
)
|
||||
expected_balance = initial_balance - total_spent
|
||||
|
||||
# Verify final balance
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
final_balance == expected_balance
|
||||
), f"Expected {expected_balance}, got {final_balance}"
|
||||
assert final_balance >= 0, "Balance went negative!"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_concurrent_multiple_spends_sufficient_balance(server: SpinTestServer):
|
||||
"""Test multiple concurrent spends when there's sufficient balance for all."""
|
||||
user_id = f"multi-spend-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Give user 150 balance ($1.50) using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=150,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "sufficient_balance"}),
|
||||
)
|
||||
|
||||
# Track individual timing to see serialization
|
||||
timings = {}
|
||||
|
||||
async def spend_with_detailed_timing(amount: int, label: str):
|
||||
start = asyncio.get_event_loop().time()
|
||||
try:
|
||||
await credit_system.spend_credits(
|
||||
user_id,
|
||||
amount,
|
||||
UsageTransactionMetadata(
|
||||
graph_exec_id=f"concurrent-{label}",
|
||||
reason=f"Concurrent spend {label}",
|
||||
),
|
||||
)
|
||||
end = asyncio.get_event_loop().time()
|
||||
timings[label] = {"start": start, "end": end, "duration": end - start}
|
||||
return f"{label}-SUCCESS"
|
||||
except Exception as e:
|
||||
end = asyncio.get_event_loop().time()
|
||||
timings[label] = {
|
||||
"start": start,
|
||||
"end": end,
|
||||
"duration": end - start,
|
||||
"error": str(e),
|
||||
}
|
||||
return f"{label}-FAILED: {e}"
|
||||
|
||||
# Run concurrent spends: 10, 20, 30 (total 60, well under 150)
|
||||
overall_start = asyncio.get_event_loop().time()
|
||||
results = await asyncio.gather(
|
||||
spend_with_detailed_timing(10, "spend-10"),
|
||||
spend_with_detailed_timing(20, "spend-20"),
|
||||
spend_with_detailed_timing(30, "spend-30"),
|
||||
return_exceptions=True,
|
||||
)
|
||||
overall_end = asyncio.get_event_loop().time()
|
||||
|
||||
print(f"Results: {results}")
|
||||
print(f"Overall duration: {overall_end - overall_start:.4f}s")
|
||||
|
||||
# Analyze timing to detect serialization vs true concurrency
|
||||
print("\nTiming analysis:")
|
||||
for label, timing in timings.items():
|
||||
print(
|
||||
f" {label}: started at {timing['start']:.4f}, ended at {timing['end']:.4f}, duration {timing['duration']:.4f}s"
|
||||
)
|
||||
|
||||
# Check if operations overlapped (true concurrency) or were serialized
|
||||
sorted_timings = sorted(timings.items(), key=lambda x: x[1]["start"])
|
||||
print("\nExecution order by start time:")
|
||||
for i, (label, timing) in enumerate(sorted_timings):
|
||||
print(f" {i+1}. {label}: {timing['start']:.4f} -> {timing['end']:.4f}")
|
||||
|
||||
# Check for overlap (true concurrency) vs serialization
|
||||
overlaps = []
|
||||
for i in range(len(sorted_timings) - 1):
|
||||
current = sorted_timings[i]
|
||||
next_op = sorted_timings[i + 1]
|
||||
if current[1]["end"] > next_op[1]["start"]:
|
||||
overlaps.append(f"{current[0]} overlaps with {next_op[0]}")
|
||||
|
||||
if overlaps:
|
||||
print(f"✅ TRUE CONCURRENCY detected: {overlaps}")
|
||||
else:
|
||||
print("🔒 SERIALIZATION detected: No overlapping execution times")
|
||||
|
||||
# Check final balance
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
print(f"Final balance: {final_balance}")
|
||||
|
||||
# Count successes/failures
|
||||
successful = [r for r in results if "SUCCESS" in str(r)]
|
||||
failed = [r for r in results if "FAILED" in str(r)]
|
||||
|
||||
print(f"Successful: {len(successful)}, Failed: {len(failed)}")
|
||||
|
||||
# All should succeed since 150 - (10 + 20 + 30) = 90 > 0
|
||||
assert (
|
||||
len(successful) == 3
|
||||
), f"Expected all 3 to succeed, got {len(successful)} successes: {results}"
|
||||
assert final_balance == 90, f"Expected balance 90, got {final_balance}"
|
||||
|
||||
# Check transaction timestamps to confirm database-level serialization
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={"userId": user_id, "type": prisma.enums.CreditTransactionType.USAGE},
|
||||
order={"createdAt": "asc"},
|
||||
)
|
||||
print("\nDatabase transaction order (by createdAt):")
|
||||
for i, tx in enumerate(transactions):
|
||||
print(
|
||||
f" {i+1}. Amount {tx.amount}, Running balance: {tx.runningBalance}, Created: {tx.createdAt}"
|
||||
)
|
||||
|
||||
# Verify running balances are chronologically consistent (ordered by createdAt)
|
||||
actual_balances = [
|
||||
tx.runningBalance for tx in transactions if tx.runningBalance is not None
|
||||
]
|
||||
print(f"Running balances: {actual_balances}")
|
||||
|
||||
# The balances should be valid intermediate states regardless of execution order
|
||||
# Starting balance: 150, spending 10+20+30=60, so final should be 90
|
||||
# The intermediate balances depend on execution order but should all be valid
|
||||
expected_possible_balances = {
|
||||
# If order is 10, 20, 30: [140, 120, 90]
|
||||
# If order is 10, 30, 20: [140, 110, 90]
|
||||
# If order is 20, 10, 30: [130, 120, 90]
|
||||
# If order is 20, 30, 10: [130, 100, 90]
|
||||
# If order is 30, 10, 20: [120, 110, 90]
|
||||
# If order is 30, 20, 10: [120, 100, 90]
|
||||
90,
|
||||
100,
|
||||
110,
|
||||
120,
|
||||
130,
|
||||
140, # All possible intermediate balances
|
||||
}
|
||||
|
||||
# Verify all balances are valid intermediate states
|
||||
for balance in actual_balances:
|
||||
assert (
|
||||
balance in expected_possible_balances
|
||||
), f"Invalid balance {balance}, expected one of {expected_possible_balances}"
|
||||
|
||||
# Final balance should always be 90 (150 - 60)
|
||||
assert (
|
||||
min(actual_balances) == 90
|
||||
), f"Final balance should be 90, got {min(actual_balances)}"
|
||||
|
||||
# The final transaction should always have balance 90
|
||||
# The other transactions should have valid intermediate balances
|
||||
assert (
|
||||
90 in actual_balances
|
||||
), f"Final balance 90 should be in actual_balances: {actual_balances}"
|
||||
|
||||
# All balances should be >= 90 (the final state)
|
||||
assert all(
|
||||
balance >= 90 for balance in actual_balances
|
||||
), f"All balances should be >= 90, got {actual_balances}"
|
||||
|
||||
# CRITICAL: Transactions are atomic but can complete in any order
|
||||
# What matters is that all running balances are valid intermediate states
|
||||
# Each balance should be between 90 (final) and 140 (after first transaction)
|
||||
for balance in actual_balances:
|
||||
assert (
|
||||
90 <= balance <= 140
|
||||
), f"Balance {balance} is outside valid range [90, 140]"
|
||||
|
||||
# Final balance (minimum) should always be 90
|
||||
assert (
|
||||
min(actual_balances) == 90
|
||||
), f"Final balance should be 90, got {min(actual_balances)}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_prove_database_locking_behavior(server: SpinTestServer):
|
||||
"""Definitively prove whether database locking causes waiting vs failures."""
|
||||
user_id = f"locking-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Set balance to exact amount that can handle all spends using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=60, # Exactly 10+20+30
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "exact_amount_test"}),
|
||||
)
|
||||
|
||||
async def spend_with_precise_timing(amount: int, label: str):
|
||||
request_start = asyncio.get_event_loop().time()
|
||||
db_operation_start = asyncio.get_event_loop().time()
|
||||
try:
|
||||
# Add a small delay to increase chance of true concurrency
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
db_operation_start = asyncio.get_event_loop().time()
|
||||
await credit_system.spend_credits(
|
||||
user_id,
|
||||
amount,
|
||||
UsageTransactionMetadata(
|
||||
graph_exec_id=f"locking-{label}",
|
||||
reason=f"Locking test {label}",
|
||||
),
|
||||
)
|
||||
db_operation_end = asyncio.get_event_loop().time()
|
||||
|
||||
return {
|
||||
"label": label,
|
||||
"status": "SUCCESS",
|
||||
"request_start": request_start,
|
||||
"db_start": db_operation_start,
|
||||
"db_end": db_operation_end,
|
||||
"db_duration": db_operation_end - db_operation_start,
|
||||
}
|
||||
except Exception as e:
|
||||
db_operation_end = asyncio.get_event_loop().time()
|
||||
return {
|
||||
"label": label,
|
||||
"status": "FAILED",
|
||||
"error": str(e),
|
||||
"request_start": request_start,
|
||||
"db_start": db_operation_start,
|
||||
"db_end": db_operation_end,
|
||||
"db_duration": db_operation_end - db_operation_start,
|
||||
}
|
||||
|
||||
# Launch all requests simultaneously
|
||||
results = await asyncio.gather(
|
||||
spend_with_precise_timing(10, "A"),
|
||||
spend_with_precise_timing(20, "B"),
|
||||
spend_with_precise_timing(30, "C"),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
print("\n🔍 LOCKING BEHAVIOR ANALYSIS:")
|
||||
print("=" * 50)
|
||||
|
||||
successful = [
|
||||
r for r in results if isinstance(r, dict) and r.get("status") == "SUCCESS"
|
||||
]
|
||||
failed = [
|
||||
r for r in results if isinstance(r, dict) and r.get("status") == "FAILED"
|
||||
]
|
||||
|
||||
print(f"✅ Successful operations: {len(successful)}")
|
||||
print(f"❌ Failed operations: {len(failed)}")
|
||||
|
||||
if len(failed) > 0:
|
||||
print(
|
||||
"\n🚫 CONCURRENT FAILURES - Some requests failed due to insufficient balance:"
|
||||
)
|
||||
for result in failed:
|
||||
if isinstance(result, dict):
|
||||
print(
|
||||
f" {result['label']}: {result.get('error', 'Unknown error')}"
|
||||
)
|
||||
|
||||
if len(successful) == 3:
|
||||
print(
|
||||
"\n🔒 SERIALIZATION CONFIRMED - All requests succeeded, indicating they were queued:"
|
||||
)
|
||||
|
||||
# Sort by actual execution time to see order
|
||||
dict_results = [r for r in results if isinstance(r, dict)]
|
||||
sorted_results = sorted(dict_results, key=lambda x: x["db_start"])
|
||||
|
||||
for i, result in enumerate(sorted_results):
|
||||
print(
|
||||
f" {i+1}. {result['label']}: DB operation took {result['db_duration']:.4f}s"
|
||||
)
|
||||
|
||||
# Check if any operations overlapped at the database level
|
||||
print("\n⏱️ Database operation timeline:")
|
||||
for result in sorted_results:
|
||||
print(
|
||||
f" {result['label']}: {result['db_start']:.4f} -> {result['db_end']:.4f}"
|
||||
)
|
||||
|
||||
# Verify final state
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
print(f"\n💰 Final balance: {final_balance}")
|
||||
|
||||
if len(successful) == 3:
|
||||
assert (
|
||||
final_balance == 0
|
||||
), f"If all succeeded, balance should be 0, got {final_balance}"
|
||||
print(
|
||||
"✅ CONCLUSION: Database row locking causes requests to WAIT and execute serially"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
"❌ CONCLUSION: Some requests failed, indicating different concurrency behavior"
|
||||
)
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
277
autogpt_platform/backend/backend/data/credit_integration_test.py
Normal file
277
autogpt_platform/backend/backend/data/credit_integration_test.py
Normal file
@@ -0,0 +1,277 @@
|
||||
"""
|
||||
Integration tests for credit system to catch SQL enum casting issues.
|
||||
|
||||
These tests run actual database operations to ensure SQL queries work correctly,
|
||||
which would have caught the CreditTransactionType enum casting bug.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.models import CreditTransaction, User, UserBalance
|
||||
|
||||
from backend.data.credit import (
|
||||
AutoTopUpConfig,
|
||||
BetaUserCredit,
|
||||
UsageTransactionMetadata,
|
||||
get_auto_top_up,
|
||||
set_auto_top_up,
|
||||
)
|
||||
from backend.util.json import SafeJson
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def cleanup_test_user():
|
||||
"""Clean up test user data before and after tests."""
|
||||
import uuid
|
||||
|
||||
user_id = str(uuid.uuid4()) # Use unique user ID for each test
|
||||
|
||||
# Create the user first
|
||||
try:
|
||||
await User.prisma().create(
|
||||
data={
|
||||
"id": user_id,
|
||||
"email": f"test-{user_id}@example.com",
|
||||
"topUpConfig": SafeJson({}),
|
||||
"timezone": "UTC",
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
# User might already exist, that's fine
|
||||
pass
|
||||
|
||||
yield user_id
|
||||
|
||||
# Cleanup after test
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": user_id})
|
||||
await UserBalance.prisma().delete_many(where={"userId": user_id})
|
||||
# Clear auto-top-up config before deleting user
|
||||
await User.prisma().update(
|
||||
where={"id": user_id}, data={"topUpConfig": SafeJson({})}
|
||||
)
|
||||
await User.prisma().delete(where={"id": user_id})
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_credit_transaction_enum_casting_integration(cleanup_test_user):
|
||||
"""
|
||||
Integration test to verify CreditTransactionType enum casting works in SQL queries.
|
||||
|
||||
This test would have caught the enum casting bug where PostgreSQL expected
|
||||
platform."CreditTransactionType" but got "CreditTransactionType".
|
||||
"""
|
||||
user_id = cleanup_test_user
|
||||
credit_system = BetaUserCredit(1000)
|
||||
|
||||
# Test each transaction type to ensure enum casting works
|
||||
test_cases = [
|
||||
(CreditTransactionType.TOP_UP, 100, "Test top-up"),
|
||||
(CreditTransactionType.USAGE, -50, "Test usage"),
|
||||
(CreditTransactionType.GRANT, 200, "Test grant"),
|
||||
(CreditTransactionType.REFUND, -25, "Test refund"),
|
||||
(CreditTransactionType.CARD_CHECK, 0, "Test card check"),
|
||||
]
|
||||
|
||||
for transaction_type, amount, reason in test_cases:
|
||||
metadata = SafeJson({"reason": reason, "test": "enum_casting"})
|
||||
|
||||
# This call would fail with enum casting error before the fix
|
||||
balance, tx_key = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=amount,
|
||||
transaction_type=transaction_type,
|
||||
metadata=metadata,
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
# Verify transaction was created with correct type
|
||||
transaction = await CreditTransaction.prisma().find_first(
|
||||
where={"userId": user_id, "transactionKey": tx_key}
|
||||
)
|
||||
|
||||
assert transaction is not None
|
||||
assert transaction.type == transaction_type
|
||||
assert transaction.amount == amount
|
||||
assert transaction.metadata is not None
|
||||
|
||||
# Verify metadata content
|
||||
assert transaction.metadata["reason"] == reason
|
||||
assert transaction.metadata["test"] == "enum_casting"
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_auto_top_up_integration(cleanup_test_user, monkeypatch):
|
||||
"""
|
||||
Integration test for auto-top-up functionality that triggers enum casting.
|
||||
|
||||
This tests the complete auto-top-up flow which involves SQL queries with
|
||||
CreditTransactionType enums, ensuring enum casting works end-to-end.
|
||||
"""
|
||||
# Enable credits for this test
|
||||
from backend.data.credit import settings
|
||||
|
||||
monkeypatch.setattr(settings.config, "enable_credit", True)
|
||||
monkeypatch.setattr(settings.config, "enable_beta_monthly_credit", True)
|
||||
monkeypatch.setattr(settings.config, "num_user_credits_refill", 1000)
|
||||
|
||||
user_id = cleanup_test_user
|
||||
credit_system = BetaUserCredit(1000)
|
||||
|
||||
# First add some initial credits so we can test the configuration and subsequent behavior
|
||||
balance, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=50, # Below threshold that we'll set
|
||||
transaction_type=CreditTransactionType.GRANT,
|
||||
metadata=SafeJson({"reason": "Initial credits before auto top-up config"}),
|
||||
)
|
||||
assert balance == 50
|
||||
|
||||
# Configure auto top-up with threshold above current balance
|
||||
config = AutoTopUpConfig(threshold=100, amount=500)
|
||||
await set_auto_top_up(user_id, config)
|
||||
|
||||
# Verify configuration was saved but no immediate top-up occurred
|
||||
current_balance = await credit_system.get_credits(user_id)
|
||||
assert current_balance == 50 # Balance should be unchanged
|
||||
|
||||
# Simulate spending credits that would trigger auto top-up
|
||||
# This involves multiple SQL operations with enum casting
|
||||
try:
|
||||
metadata = UsageTransactionMetadata(reason="Test spend to trigger auto top-up")
|
||||
await credit_system.spend_credits(user_id=user_id, cost=10, metadata=metadata)
|
||||
|
||||
# The auto top-up mechanism should have been triggered
|
||||
# Verify the transaction types were handled correctly
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={"userId": user_id}, order={"createdAt": "desc"}
|
||||
)
|
||||
|
||||
# Should have at least: GRANT (initial), USAGE (spend), and TOP_UP (auto top-up)
|
||||
assert len(transactions) >= 3
|
||||
|
||||
# Verify different transaction types exist and enum casting worked
|
||||
transaction_types = {t.type for t in transactions}
|
||||
assert CreditTransactionType.GRANT in transaction_types
|
||||
assert CreditTransactionType.USAGE in transaction_types
|
||||
assert (
|
||||
CreditTransactionType.TOP_UP in transaction_types
|
||||
) # Auto top-up should have triggered
|
||||
|
||||
except Exception as e:
|
||||
# If this fails with enum casting error, the test successfully caught the bug
|
||||
if "CreditTransactionType" in str(e) and (
|
||||
"cast" in str(e).lower() or "type" in str(e).lower()
|
||||
):
|
||||
pytest.fail(f"Enum casting error detected: {e}")
|
||||
else:
|
||||
# Re-raise other unexpected errors
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_enable_transaction_enum_casting_integration(cleanup_test_user):
|
||||
"""
|
||||
Integration test for _enable_transaction with enum casting.
|
||||
|
||||
Tests the scenario where inactive transactions are enabled, which also
|
||||
involves SQL queries with CreditTransactionType enum casting.
|
||||
"""
|
||||
user_id = cleanup_test_user
|
||||
credit_system = BetaUserCredit(1000)
|
||||
|
||||
# Create an inactive transaction
|
||||
balance, tx_key = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=100,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"reason": "Inactive transaction test"}),
|
||||
is_active=False, # Create as inactive
|
||||
)
|
||||
|
||||
# Balance should be 0 since transaction is inactive
|
||||
assert balance == 0
|
||||
|
||||
# Enable the transaction with new metadata
|
||||
enable_metadata = SafeJson(
|
||||
{
|
||||
"payment_method": "test_payment",
|
||||
"activation_reason": "Integration test activation",
|
||||
}
|
||||
)
|
||||
|
||||
# This would fail with enum casting error before the fix
|
||||
final_balance = await credit_system._enable_transaction(
|
||||
transaction_key=tx_key,
|
||||
user_id=user_id,
|
||||
metadata=enable_metadata,
|
||||
)
|
||||
|
||||
# Now balance should reflect the activated transaction
|
||||
assert final_balance == 100
|
||||
|
||||
# Verify transaction was properly enabled with correct enum type
|
||||
transaction = await CreditTransaction.prisma().find_first(
|
||||
where={"userId": user_id, "transactionKey": tx_key}
|
||||
)
|
||||
|
||||
assert transaction is not None
|
||||
assert transaction.isActive is True
|
||||
assert transaction.type == CreditTransactionType.TOP_UP
|
||||
assert transaction.runningBalance == 100
|
||||
|
||||
# Verify metadata was updated
|
||||
assert transaction.metadata is not None
|
||||
assert transaction.metadata["payment_method"] == "test_payment"
|
||||
assert transaction.metadata["activation_reason"] == "Integration test activation"
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_auto_top_up_configuration_storage(cleanup_test_user, monkeypatch):
|
||||
"""
|
||||
Test that auto-top-up configuration is properly stored and retrieved.
|
||||
|
||||
The immediate top-up logic is handled by the API routes, not the core
|
||||
set_auto_top_up function. This test verifies the configuration is correctly
|
||||
saved and can be retrieved.
|
||||
"""
|
||||
# Enable credits for this test
|
||||
from backend.data.credit import settings
|
||||
|
||||
monkeypatch.setattr(settings.config, "enable_credit", True)
|
||||
monkeypatch.setattr(settings.config, "enable_beta_monthly_credit", True)
|
||||
monkeypatch.setattr(settings.config, "num_user_credits_refill", 1000)
|
||||
|
||||
user_id = cleanup_test_user
|
||||
credit_system = BetaUserCredit(1000)
|
||||
|
||||
# Set initial balance
|
||||
balance, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=50,
|
||||
transaction_type=CreditTransactionType.GRANT,
|
||||
metadata=SafeJson({"reason": "Initial balance for config test"}),
|
||||
)
|
||||
|
||||
assert balance == 50
|
||||
|
||||
# Configure auto top-up
|
||||
config = AutoTopUpConfig(threshold=100, amount=200)
|
||||
await set_auto_top_up(user_id, config)
|
||||
|
||||
# Verify the configuration was saved
|
||||
retrieved_config = await get_auto_top_up(user_id)
|
||||
assert retrieved_config.threshold == config.threshold
|
||||
assert retrieved_config.amount == config.amount
|
||||
|
||||
# Verify balance is unchanged (no immediate top-up from set_auto_top_up)
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 50 # Should be unchanged
|
||||
|
||||
# Verify no immediate auto-top-up transaction was created by set_auto_top_up
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={"userId": user_id}, order={"createdAt": "desc"}
|
||||
)
|
||||
|
||||
# Should only have the initial GRANT transaction
|
||||
assert len(transactions) == 1
|
||||
assert transactions[0].type == CreditTransactionType.GRANT
|
||||
141
autogpt_platform/backend/backend/data/credit_metadata_test.py
Normal file
141
autogpt_platform/backend/backend/data/credit_metadata_test.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""
|
||||
Tests for credit system metadata handling to ensure JSON casting works correctly.
|
||||
|
||||
This test verifies that metadata parameters are properly serialized when passed
|
||||
to raw SQL queries with JSONB columns.
|
||||
"""
|
||||
|
||||
# type: ignore
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.models import CreditTransaction, UserBalance
|
||||
|
||||
from backend.data.credit import BetaUserCredit
|
||||
from backend.data.user import DEFAULT_USER_ID
|
||||
from backend.util.json import SafeJson
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def setup_test_user():
|
||||
"""Setup test user and cleanup after test."""
|
||||
user_id = DEFAULT_USER_ID
|
||||
|
||||
# Cleanup before test
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": user_id})
|
||||
await UserBalance.prisma().delete_many(where={"userId": user_id})
|
||||
|
||||
yield user_id
|
||||
|
||||
# Cleanup after test
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": user_id})
|
||||
await UserBalance.prisma().delete_many(where={"userId": user_id})
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_metadata_json_serialization(setup_test_user):
|
||||
"""Test that metadata is properly serialized for JSONB column in raw SQL."""
|
||||
user_id = setup_test_user
|
||||
credit_system = BetaUserCredit(1000)
|
||||
|
||||
# Test with complex metadata that would fail if not properly serialized
|
||||
complex_metadata = SafeJson(
|
||||
{
|
||||
"graph_exec_id": "test-12345",
|
||||
"reason": "Testing metadata serialization",
|
||||
"nested_data": {
|
||||
"key1": "value1",
|
||||
"key2": ["array", "of", "values"],
|
||||
"key3": {"deeply": {"nested": "object"}},
|
||||
},
|
||||
"special_chars": "Testing 'quotes' and \"double quotes\" and unicode: 🚀",
|
||||
}
|
||||
)
|
||||
|
||||
# This should work without throwing a JSONB casting error
|
||||
balance, tx_key = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=500, # $5 top-up
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=complex_metadata,
|
||||
is_active=True,
|
||||
)
|
||||
|
||||
# Verify the transaction was created successfully
|
||||
assert balance == 500
|
||||
|
||||
# Verify the metadata was stored correctly in the database
|
||||
transaction = await CreditTransaction.prisma().find_first(
|
||||
where={"userId": user_id, "transactionKey": tx_key}
|
||||
)
|
||||
|
||||
assert transaction is not None
|
||||
assert transaction.metadata is not None
|
||||
|
||||
# Verify the metadata contains our complex data
|
||||
metadata_dict: dict[str, Any] = dict(transaction.metadata) # type: ignore
|
||||
assert metadata_dict["graph_exec_id"] == "test-12345"
|
||||
assert metadata_dict["reason"] == "Testing metadata serialization"
|
||||
assert metadata_dict["nested_data"]["key1"] == "value1"
|
||||
assert metadata_dict["nested_data"]["key3"]["deeply"]["nested"] == "object"
|
||||
assert (
|
||||
metadata_dict["special_chars"]
|
||||
== "Testing 'quotes' and \"double quotes\" and unicode: 🚀"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_enable_transaction_metadata_serialization(setup_test_user):
|
||||
"""Test that _enable_transaction also handles metadata JSON serialization correctly."""
|
||||
user_id = setup_test_user
|
||||
credit_system = BetaUserCredit(1000)
|
||||
|
||||
# First create an inactive transaction
|
||||
balance, tx_key = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=300,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"initial": "inactive_transaction"}),
|
||||
is_active=False, # Create as inactive
|
||||
)
|
||||
|
||||
# Initial balance should be 0 because transaction is inactive
|
||||
assert balance == 0
|
||||
|
||||
# Now enable the transaction with new metadata
|
||||
enable_metadata = SafeJson(
|
||||
{
|
||||
"payment_method": "stripe",
|
||||
"payment_intent": "pi_test_12345",
|
||||
"activation_reason": "Payment confirmed",
|
||||
"complex_data": {"array": [1, 2, 3], "boolean": True, "null_value": None},
|
||||
}
|
||||
)
|
||||
|
||||
# This should work without JSONB casting errors
|
||||
final_balance = await credit_system._enable_transaction(
|
||||
transaction_key=tx_key,
|
||||
user_id=user_id,
|
||||
metadata=enable_metadata,
|
||||
)
|
||||
|
||||
# Now balance should reflect the activated transaction
|
||||
assert final_balance == 300
|
||||
|
||||
# Verify the metadata was updated correctly
|
||||
transaction = await CreditTransaction.prisma().find_first(
|
||||
where={"userId": user_id, "transactionKey": tx_key}
|
||||
)
|
||||
|
||||
assert transaction is not None
|
||||
assert transaction.isActive is True
|
||||
|
||||
# Verify the metadata was updated with enable_metadata
|
||||
metadata_dict: dict[str, Any] = dict(transaction.metadata) # type: ignore
|
||||
assert metadata_dict["payment_method"] == "stripe"
|
||||
assert metadata_dict["payment_intent"] == "pi_test_12345"
|
||||
assert metadata_dict["complex_data"]["array"] == [1, 2, 3]
|
||||
assert metadata_dict["complex_data"]["boolean"] is True
|
||||
assert metadata_dict["complex_data"]["null_value"] is None
|
||||
372
autogpt_platform/backend/backend/data/credit_refund_test.py
Normal file
372
autogpt_platform/backend/backend/data/credit_refund_test.py
Normal file
@@ -0,0 +1,372 @@
|
||||
"""
|
||||
Tests for credit system refund and dispute operations.
|
||||
|
||||
These tests ensure that refund operations (deduct_credits, handle_dispute)
|
||||
are atomic and maintain data consistency.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import stripe
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.models import CreditRefundRequest, CreditTransaction, User, UserBalance
|
||||
|
||||
from backend.data.credit import UserCredit
|
||||
from backend.util.json import SafeJson
|
||||
from backend.util.test import SpinTestServer
|
||||
|
||||
credit_system = UserCredit()
|
||||
|
||||
# Test user ID for refund tests
|
||||
REFUND_TEST_USER_ID = "refund-test-user"
|
||||
|
||||
|
||||
async def setup_test_user_with_topup():
|
||||
"""Create a test user with initial balance and a top-up transaction."""
|
||||
# Clean up any existing data
|
||||
await CreditRefundRequest.prisma().delete_many(
|
||||
where={"userId": REFUND_TEST_USER_ID}
|
||||
)
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": REFUND_TEST_USER_ID})
|
||||
await UserBalance.prisma().delete_many(where={"userId": REFUND_TEST_USER_ID})
|
||||
await User.prisma().delete_many(where={"id": REFUND_TEST_USER_ID})
|
||||
|
||||
# Create user
|
||||
await User.prisma().create(
|
||||
data={
|
||||
"id": REFUND_TEST_USER_ID,
|
||||
"email": f"{REFUND_TEST_USER_ID}@example.com",
|
||||
"name": "Refund Test User",
|
||||
}
|
||||
)
|
||||
|
||||
# Create user balance
|
||||
await UserBalance.prisma().create(
|
||||
data={
|
||||
"userId": REFUND_TEST_USER_ID,
|
||||
"balance": 1000, # $10
|
||||
}
|
||||
)
|
||||
|
||||
# Create a top-up transaction that can be refunded
|
||||
topup_tx = await CreditTransaction.prisma().create(
|
||||
data={
|
||||
"userId": REFUND_TEST_USER_ID,
|
||||
"amount": 1000,
|
||||
"type": CreditTransactionType.TOP_UP,
|
||||
"transactionKey": "pi_test_12345",
|
||||
"runningBalance": 1000,
|
||||
"isActive": True,
|
||||
"metadata": SafeJson({"stripe_payment_intent": "pi_test_12345"}),
|
||||
}
|
||||
)
|
||||
|
||||
return topup_tx
|
||||
|
||||
|
||||
async def cleanup_test_user():
|
||||
"""Clean up test data."""
|
||||
await CreditRefundRequest.prisma().delete_many(
|
||||
where={"userId": REFUND_TEST_USER_ID}
|
||||
)
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": REFUND_TEST_USER_ID})
|
||||
await UserBalance.prisma().delete_many(where={"userId": REFUND_TEST_USER_ID})
|
||||
await User.prisma().delete_many(where={"id": REFUND_TEST_USER_ID})
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_deduct_credits_atomic(server: SpinTestServer):
|
||||
"""Test that deduct_credits is atomic and creates transaction correctly."""
|
||||
topup_tx = await setup_test_user_with_topup()
|
||||
|
||||
try:
|
||||
# Create a mock refund object
|
||||
refund = MagicMock(spec=stripe.Refund)
|
||||
refund.id = "re_test_refund_123"
|
||||
refund.payment_intent = topup_tx.transactionKey
|
||||
refund.amount = 500 # Refund $5 of the $10 top-up
|
||||
refund.status = "succeeded"
|
||||
refund.reason = "requested_by_customer"
|
||||
refund.created = int(datetime.now(timezone.utc).timestamp())
|
||||
|
||||
# Create refund request record (simulating webhook flow)
|
||||
await CreditRefundRequest.prisma().create(
|
||||
data={
|
||||
"userId": REFUND_TEST_USER_ID,
|
||||
"amount": 500,
|
||||
"transactionKey": topup_tx.transactionKey, # Should match the original transaction
|
||||
"reason": "Test refund",
|
||||
}
|
||||
)
|
||||
|
||||
# Call deduct_credits
|
||||
await credit_system.deduct_credits(refund)
|
||||
|
||||
# Verify the user's balance was deducted
|
||||
user_balance = await UserBalance.prisma().find_unique(
|
||||
where={"userId": REFUND_TEST_USER_ID}
|
||||
)
|
||||
assert user_balance is not None
|
||||
assert (
|
||||
user_balance.balance == 500
|
||||
), f"Expected balance 500, got {user_balance.balance}"
|
||||
|
||||
# Verify refund transaction was created
|
||||
refund_tx = await CreditTransaction.prisma().find_first(
|
||||
where={
|
||||
"userId": REFUND_TEST_USER_ID,
|
||||
"type": CreditTransactionType.REFUND,
|
||||
"transactionKey": refund.id,
|
||||
}
|
||||
)
|
||||
assert refund_tx is not None
|
||||
assert refund_tx.amount == -500
|
||||
assert refund_tx.runningBalance == 500
|
||||
assert refund_tx.isActive
|
||||
|
||||
# Verify refund request was updated
|
||||
refund_request = await CreditRefundRequest.prisma().find_first(
|
||||
where={
|
||||
"userId": REFUND_TEST_USER_ID,
|
||||
"transactionKey": topup_tx.transactionKey,
|
||||
}
|
||||
)
|
||||
assert refund_request is not None
|
||||
assert (
|
||||
refund_request.result
|
||||
== "The refund request has been approved, the amount will be credited back to your account."
|
||||
)
|
||||
|
||||
finally:
|
||||
await cleanup_test_user()
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_deduct_credits_user_not_found(server: SpinTestServer):
|
||||
"""Test that deduct_credits raises error if transaction not found (which means user doesn't exist)."""
|
||||
# Create a mock refund object that references a non-existent payment intent
|
||||
refund = MagicMock(spec=stripe.Refund)
|
||||
refund.id = "re_test_refund_nonexistent"
|
||||
refund.payment_intent = "pi_test_nonexistent" # This payment intent doesn't exist
|
||||
refund.amount = 500
|
||||
refund.status = "succeeded"
|
||||
refund.reason = "requested_by_customer"
|
||||
refund.created = int(datetime.now(timezone.utc).timestamp())
|
||||
|
||||
# Should raise error for missing transaction
|
||||
with pytest.raises(Exception): # Should raise NotFoundError for missing transaction
|
||||
await credit_system.deduct_credits(refund)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
@patch("backend.data.credit.settings")
|
||||
@patch("stripe.Dispute.modify")
|
||||
@patch("backend.data.credit.get_user_by_id")
|
||||
async def test_handle_dispute_with_sufficient_balance(
|
||||
mock_get_user, mock_stripe_modify, mock_settings, server: SpinTestServer
|
||||
):
|
||||
"""Test handling dispute when user has sufficient balance (dispute gets closed)."""
|
||||
topup_tx = await setup_test_user_with_topup()
|
||||
|
||||
try:
|
||||
# Mock settings to have a low tolerance threshold
|
||||
mock_settings.config.refund_credit_tolerance_threshold = 0
|
||||
|
||||
# Mock the user lookup
|
||||
mock_user = MagicMock()
|
||||
mock_user.email = f"{REFUND_TEST_USER_ID}@example.com"
|
||||
mock_get_user.return_value = mock_user
|
||||
|
||||
# Create a mock dispute object for small amount (user has 1000, disputing 100)
|
||||
dispute = MagicMock(spec=stripe.Dispute)
|
||||
dispute.id = "dp_test_dispute_123"
|
||||
dispute.payment_intent = topup_tx.transactionKey
|
||||
dispute.amount = 100 # Small dispute amount
|
||||
dispute.status = "pending"
|
||||
dispute.reason = "fraudulent"
|
||||
dispute.created = int(datetime.now(timezone.utc).timestamp())
|
||||
|
||||
# Mock the close method to prevent real API calls
|
||||
dispute.close = MagicMock()
|
||||
|
||||
# Handle the dispute
|
||||
await credit_system.handle_dispute(dispute)
|
||||
|
||||
# Verify dispute.close() was called (since user has sufficient balance)
|
||||
dispute.close.assert_called_once()
|
||||
|
||||
# Verify no stripe evidence was added since dispute was closed
|
||||
mock_stripe_modify.assert_not_called()
|
||||
|
||||
# Verify the user's balance was NOT deducted (dispute was closed)
|
||||
user_balance = await UserBalance.prisma().find_unique(
|
||||
where={"userId": REFUND_TEST_USER_ID}
|
||||
)
|
||||
assert user_balance is not None
|
||||
assert (
|
||||
user_balance.balance == 1000
|
||||
), f"Balance should remain 1000, got {user_balance.balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user()
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
@patch("backend.data.credit.settings")
|
||||
@patch("stripe.Dispute.modify")
|
||||
@patch("backend.data.credit.get_user_by_id")
|
||||
async def test_handle_dispute_with_insufficient_balance(
|
||||
mock_get_user, mock_stripe_modify, mock_settings, server: SpinTestServer
|
||||
):
|
||||
"""Test handling dispute when user has insufficient balance (evidence gets added)."""
|
||||
topup_tx = await setup_test_user_with_topup()
|
||||
|
||||
# Save original method for restoration before any try blocks
|
||||
original_get_history = credit_system.get_transaction_history
|
||||
|
||||
try:
|
||||
# Mock settings to have a high tolerance threshold so dispute isn't closed
|
||||
mock_settings.config.refund_credit_tolerance_threshold = 2000
|
||||
|
||||
# Mock the user lookup
|
||||
mock_user = MagicMock()
|
||||
mock_user.email = f"{REFUND_TEST_USER_ID}@example.com"
|
||||
mock_get_user.return_value = mock_user
|
||||
|
||||
# Mock the transaction history method to return an async result
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
mock_history = MagicMock()
|
||||
mock_history.transactions = []
|
||||
credit_system.get_transaction_history = AsyncMock(return_value=mock_history)
|
||||
|
||||
# Create a mock dispute object for full amount (user has 1000, disputing 1000)
|
||||
dispute = MagicMock(spec=stripe.Dispute)
|
||||
dispute.id = "dp_test_dispute_pending"
|
||||
dispute.payment_intent = topup_tx.transactionKey
|
||||
dispute.amount = 1000
|
||||
dispute.status = "warning_needs_response"
|
||||
dispute.created = int(datetime.now(timezone.utc).timestamp())
|
||||
|
||||
# Mock the close method to prevent real API calls
|
||||
dispute.close = MagicMock()
|
||||
|
||||
# Handle the dispute (evidence should be added)
|
||||
await credit_system.handle_dispute(dispute)
|
||||
|
||||
# Verify dispute.close() was NOT called (insufficient balance after tolerance)
|
||||
dispute.close.assert_not_called()
|
||||
|
||||
# Verify stripe evidence was added since dispute wasn't closed
|
||||
mock_stripe_modify.assert_called_once()
|
||||
|
||||
# Verify the user's balance was NOT deducted (handle_dispute doesn't deduct credits)
|
||||
user_balance = await UserBalance.prisma().find_unique(
|
||||
where={"userId": REFUND_TEST_USER_ID}
|
||||
)
|
||||
assert user_balance is not None
|
||||
assert user_balance.balance == 1000, "Balance should remain unchanged"
|
||||
|
||||
finally:
|
||||
credit_system.get_transaction_history = original_get_history
|
||||
await cleanup_test_user()
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_concurrent_refunds(server: SpinTestServer):
|
||||
"""Test that concurrent refunds are handled atomically."""
|
||||
import asyncio
|
||||
|
||||
topup_tx = await setup_test_user_with_topup()
|
||||
|
||||
try:
|
||||
# Create multiple refund requests
|
||||
refund_requests = []
|
||||
for i in range(5):
|
||||
req = await CreditRefundRequest.prisma().create(
|
||||
data={
|
||||
"userId": REFUND_TEST_USER_ID,
|
||||
"amount": 100, # $1 each
|
||||
"transactionKey": topup_tx.transactionKey,
|
||||
"reason": f"Test refund {i}",
|
||||
}
|
||||
)
|
||||
refund_requests.append(req)
|
||||
|
||||
# Create refund tasks to run concurrently
|
||||
async def process_refund(index: int):
|
||||
refund = MagicMock(spec=stripe.Refund)
|
||||
refund.id = f"re_test_concurrent_{index}"
|
||||
refund.payment_intent = topup_tx.transactionKey
|
||||
refund.amount = 100 # $1 refund
|
||||
refund.status = "succeeded"
|
||||
refund.reason = "requested_by_customer"
|
||||
refund.created = int(datetime.now(timezone.utc).timestamp())
|
||||
|
||||
try:
|
||||
await credit_system.deduct_credits(refund)
|
||||
return "success"
|
||||
except Exception as e:
|
||||
return f"error: {e}"
|
||||
|
||||
# Run refunds concurrently
|
||||
results = await asyncio.gather(
|
||||
*[process_refund(i) for i in range(5)], return_exceptions=True
|
||||
)
|
||||
|
||||
# All should succeed
|
||||
assert all(r == "success" for r in results), f"Some refunds failed: {results}"
|
||||
|
||||
# Verify final balance - with non-atomic implementation, this will demonstrate race condition
|
||||
# EXPECTED BEHAVIOR: Due to race conditions, not all refunds will be properly processed
|
||||
# The balance will be incorrect (higher than expected) showing lost updates
|
||||
user_balance = await UserBalance.prisma().find_unique(
|
||||
where={"userId": REFUND_TEST_USER_ID}
|
||||
)
|
||||
assert user_balance is not None
|
||||
|
||||
# With atomic implementation, this should be 500 (1000 - 5*100)
|
||||
# With current non-atomic implementation, this will likely be wrong due to race conditions
|
||||
print(f"DEBUG: Final balance = {user_balance.balance}, expected = 500")
|
||||
|
||||
# With atomic implementation, all 5 refunds should process correctly
|
||||
assert (
|
||||
user_balance.balance == 500
|
||||
), f"Expected balance 500 after 5 refunds of 100 each, got {user_balance.balance}"
|
||||
|
||||
# Verify all refund transactions exist
|
||||
refund_txs = await CreditTransaction.prisma().find_many(
|
||||
where={
|
||||
"userId": REFUND_TEST_USER_ID,
|
||||
"type": CreditTransactionType.REFUND,
|
||||
}
|
||||
)
|
||||
assert (
|
||||
len(refund_txs) == 5
|
||||
), f"Expected 5 refund transactions, got {len(refund_txs)}"
|
||||
|
||||
running_balances: set[int] = {
|
||||
tx.runningBalance for tx in refund_txs if tx.runningBalance is not None
|
||||
}
|
||||
|
||||
# Verify all balances are valid intermediate states
|
||||
for balance in running_balances:
|
||||
assert (
|
||||
500 <= balance <= 1000
|
||||
), f"Invalid balance {balance}, should be between 500 and 1000"
|
||||
|
||||
# Final balance should be present
|
||||
assert (
|
||||
500 in running_balances
|
||||
), f"Final balance 500 should be in {running_balances}"
|
||||
|
||||
# All balances should be unique and form a valid sequence
|
||||
sorted_balances = sorted(running_balances, reverse=True)
|
||||
assert (
|
||||
len(sorted_balances) == 5
|
||||
), f"Expected 5 unique balances, got {len(sorted_balances)}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user()
|
||||
@@ -1,8 +1,8 @@
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.models import CreditTransaction
|
||||
from prisma.models import CreditTransaction, UserBalance
|
||||
|
||||
from backend.blocks.llm import AITextGeneratorBlock
|
||||
from backend.data.block import get_block
|
||||
@@ -19,14 +19,24 @@ user_credit = BetaUserCredit(REFILL_VALUE)
|
||||
|
||||
async def disable_test_user_transactions():
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": DEFAULT_USER_ID})
|
||||
# Also reset the balance to 0 and set updatedAt to old date to trigger monthly refill
|
||||
old_date = datetime.now(timezone.utc) - timedelta(days=35) # More than a month ago
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": DEFAULT_USER_ID},
|
||||
data={
|
||||
"create": {"userId": DEFAULT_USER_ID, "balance": 0},
|
||||
"update": {"balance": 0, "updatedAt": old_date},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def top_up(amount: int):
|
||||
await user_credit._add_transaction(
|
||||
balance, _ = await user_credit._add_transaction(
|
||||
DEFAULT_USER_ID,
|
||||
amount,
|
||||
CreditTransactionType.TOP_UP,
|
||||
)
|
||||
return balance
|
||||
|
||||
|
||||
async def spend_credits(entry: NodeExecutionEntry) -> int:
|
||||
@@ -111,29 +121,90 @@ async def test_block_credit_top_up(server: SpinTestServer):
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_block_credit_reset(server: SpinTestServer):
|
||||
"""Test that BetaUserCredit provides monthly refills correctly."""
|
||||
await disable_test_user_transactions()
|
||||
month1 = 1
|
||||
month2 = 2
|
||||
|
||||
# set the calendar to month 2 but use current time from now
|
||||
user_credit.time_now = lambda: datetime.now(timezone.utc).replace(
|
||||
month=month2, day=1
|
||||
)
|
||||
month2credit = await user_credit.get_credits(DEFAULT_USER_ID)
|
||||
# Save original time_now function for restoration
|
||||
original_time_now = user_credit.time_now
|
||||
|
||||
# Month 1 result should only affect month 1
|
||||
user_credit.time_now = lambda: datetime.now(timezone.utc).replace(
|
||||
month=month1, day=1
|
||||
)
|
||||
month1credit = await user_credit.get_credits(DEFAULT_USER_ID)
|
||||
await top_up(100)
|
||||
assert await user_credit.get_credits(DEFAULT_USER_ID) == month1credit + 100
|
||||
try:
|
||||
# Test month 1 behavior
|
||||
month1 = datetime.now(timezone.utc).replace(month=1, day=1)
|
||||
user_credit.time_now = lambda: month1
|
||||
|
||||
# Month 2 balance is unaffected
|
||||
user_credit.time_now = lambda: datetime.now(timezone.utc).replace(
|
||||
month=month2, day=1
|
||||
)
|
||||
assert await user_credit.get_credits(DEFAULT_USER_ID) == month2credit
|
||||
# First call in month 1 should trigger refill
|
||||
balance = await user_credit.get_credits(DEFAULT_USER_ID)
|
||||
assert balance == REFILL_VALUE # Should get 1000 credits
|
||||
|
||||
# Manually create a transaction with month 1 timestamp to establish history
|
||||
await CreditTransaction.prisma().create(
|
||||
data={
|
||||
"userId": DEFAULT_USER_ID,
|
||||
"amount": 100,
|
||||
"type": CreditTransactionType.TOP_UP,
|
||||
"runningBalance": 1100,
|
||||
"isActive": True,
|
||||
"createdAt": month1, # Set specific timestamp
|
||||
}
|
||||
)
|
||||
|
||||
# Update user balance to match
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": DEFAULT_USER_ID},
|
||||
data={
|
||||
"create": {"userId": DEFAULT_USER_ID, "balance": 1100},
|
||||
"update": {"balance": 1100},
|
||||
},
|
||||
)
|
||||
|
||||
# Now test month 2 behavior
|
||||
month2 = datetime.now(timezone.utc).replace(month=2, day=1)
|
||||
user_credit.time_now = lambda: month2
|
||||
|
||||
# In month 2, since balance (1100) > refill (1000), no refill should happen
|
||||
month2_balance = await user_credit.get_credits(DEFAULT_USER_ID)
|
||||
assert month2_balance == 1100 # Balance persists, no reset
|
||||
|
||||
# Now test the refill behavior when balance is low
|
||||
# Set balance below refill threshold
|
||||
await UserBalance.prisma().update(
|
||||
where={"userId": DEFAULT_USER_ID}, data={"balance": 400}
|
||||
)
|
||||
|
||||
# Create a month 2 transaction to update the last transaction time
|
||||
await CreditTransaction.prisma().create(
|
||||
data={
|
||||
"userId": DEFAULT_USER_ID,
|
||||
"amount": -700, # Spent 700 to get to 400
|
||||
"type": CreditTransactionType.USAGE,
|
||||
"runningBalance": 400,
|
||||
"isActive": True,
|
||||
"createdAt": month2,
|
||||
}
|
||||
)
|
||||
|
||||
# Move to month 3
|
||||
month3 = datetime.now(timezone.utc).replace(month=3, day=1)
|
||||
user_credit.time_now = lambda: month3
|
||||
|
||||
# Should get refilled since balance (400) < refill value (1000)
|
||||
month3_balance = await user_credit.get_credits(DEFAULT_USER_ID)
|
||||
assert month3_balance == REFILL_VALUE # Should be refilled to 1000
|
||||
|
||||
# Verify the refill transaction was created
|
||||
refill_tx = await CreditTransaction.prisma().find_first(
|
||||
where={
|
||||
"userId": DEFAULT_USER_ID,
|
||||
"type": CreditTransactionType.GRANT,
|
||||
"transactionKey": {"contains": "MONTHLY-CREDIT-TOP-UP"},
|
||||
},
|
||||
order={"createdAt": "desc"},
|
||||
)
|
||||
assert refill_tx is not None, "Monthly refill transaction should be created"
|
||||
assert refill_tx.amount == 600, "Refill should be 600 (1000 - 400)"
|
||||
finally:
|
||||
# Restore original time_now function
|
||||
user_credit.time_now = original_time_now
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
361
autogpt_platform/backend/backend/data/credit_underflow_test.py
Normal file
361
autogpt_platform/backend/backend/data/credit_underflow_test.py
Normal file
@@ -0,0 +1,361 @@
|
||||
"""
|
||||
Test underflow protection for cumulative refunds and negative transactions.
|
||||
|
||||
This test ensures that when multiple large refunds are processed, the user balance
|
||||
doesn't underflow below POSTGRES_INT_MIN, which could cause integer wraparound issues.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.errors import UniqueViolationError
|
||||
from prisma.models import CreditTransaction, User, UserBalance
|
||||
|
||||
from backend.data.credit import POSTGRES_INT_MIN, UserCredit
|
||||
from backend.util.test import SpinTestServer
|
||||
|
||||
|
||||
async def create_test_user(user_id: str) -> None:
|
||||
"""Create a test user for underflow tests."""
|
||||
try:
|
||||
await User.prisma().create(
|
||||
data={
|
||||
"id": user_id,
|
||||
"email": f"test-{user_id}@example.com",
|
||||
"name": f"Test User {user_id[:8]}",
|
||||
}
|
||||
)
|
||||
except UniqueViolationError:
|
||||
# User already exists, continue
|
||||
pass
|
||||
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={"create": {"userId": user_id, "balance": 0}, "update": {"balance": 0}},
|
||||
)
|
||||
|
||||
|
||||
async def cleanup_test_user(user_id: str) -> None:
|
||||
"""Clean up test user and their transactions."""
|
||||
try:
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": user_id})
|
||||
await UserBalance.prisma().delete_many(where={"userId": user_id})
|
||||
await User.prisma().delete_many(where={"id": user_id})
|
||||
except Exception as e:
|
||||
# Log cleanup failures but don't fail the test
|
||||
print(f"Warning: Failed to cleanup test user {user_id}: {e}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_debug_underflow_step_by_step(server: SpinTestServer):
|
||||
"""Debug underflow behavior step by step."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"debug-underflow-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
print(f"POSTGRES_INT_MIN: {POSTGRES_INT_MIN}")
|
||||
|
||||
# Test 1: Set up balance close to underflow threshold
|
||||
print("\n=== Test 1: Setting up balance close to underflow threshold ===")
|
||||
# First, manually set balance to a value very close to POSTGRES_INT_MIN
|
||||
# We'll set it to POSTGRES_INT_MIN + 100, then try to subtract 200
|
||||
# This should trigger underflow protection: (POSTGRES_INT_MIN + 100) + (-200) = POSTGRES_INT_MIN - 100
|
||||
initial_balance_target = POSTGRES_INT_MIN + 100
|
||||
|
||||
# Use direct database update to set the balance close to underflow
|
||||
from prisma.models import UserBalance
|
||||
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={
|
||||
"create": {"userId": user_id, "balance": initial_balance_target},
|
||||
"update": {"balance": initial_balance_target},
|
||||
},
|
||||
)
|
||||
|
||||
current_balance = await credit_system.get_credits(user_id)
|
||||
print(f"Set balance to: {current_balance}")
|
||||
assert current_balance == initial_balance_target
|
||||
|
||||
# Test 2: Apply amount that should cause underflow
|
||||
print("\n=== Test 2: Testing underflow protection ===")
|
||||
test_amount = (
|
||||
-200
|
||||
) # This should cause underflow: (POSTGRES_INT_MIN + 100) + (-200) = POSTGRES_INT_MIN - 100
|
||||
expected_without_protection = current_balance + test_amount
|
||||
print(f"Current balance: {current_balance}")
|
||||
print(f"Test amount: {test_amount}")
|
||||
print(f"Without protection would be: {expected_without_protection}")
|
||||
print(f"Should be clamped to POSTGRES_INT_MIN: {POSTGRES_INT_MIN}")
|
||||
|
||||
# Apply the amount that should trigger underflow protection
|
||||
balance_result, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=test_amount,
|
||||
transaction_type=CreditTransactionType.REFUND,
|
||||
fail_insufficient_credits=False,
|
||||
)
|
||||
print(f"Actual result: {balance_result}")
|
||||
|
||||
# Check if underflow protection worked
|
||||
assert (
|
||||
balance_result == POSTGRES_INT_MIN
|
||||
), f"Expected underflow protection to clamp balance to {POSTGRES_INT_MIN}, got {balance_result}"
|
||||
|
||||
# Test 3: Edge case - exactly at POSTGRES_INT_MIN
|
||||
print("\n=== Test 3: Testing exact POSTGRES_INT_MIN boundary ===")
|
||||
# Set balance to exactly POSTGRES_INT_MIN
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={
|
||||
"create": {"userId": user_id, "balance": POSTGRES_INT_MIN},
|
||||
"update": {"balance": POSTGRES_INT_MIN},
|
||||
},
|
||||
)
|
||||
|
||||
edge_balance = await credit_system.get_credits(user_id)
|
||||
print(f"Balance set to exactly POSTGRES_INT_MIN: {edge_balance}")
|
||||
|
||||
# Try to subtract 1 - should stay at POSTGRES_INT_MIN
|
||||
edge_result, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=-1,
|
||||
transaction_type=CreditTransactionType.REFUND,
|
||||
fail_insufficient_credits=False,
|
||||
)
|
||||
print(f"After subtracting 1: {edge_result}")
|
||||
|
||||
assert (
|
||||
edge_result == POSTGRES_INT_MIN
|
||||
), f"Expected balance to remain clamped at {POSTGRES_INT_MIN}, got {edge_result}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_underflow_protection_large_refunds(server: SpinTestServer):
|
||||
"""Test that large cumulative refunds don't cause integer underflow."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"underflow-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Set up balance close to underflow threshold to test the protection
|
||||
# Set balance to POSTGRES_INT_MIN + 1000, then try to subtract 2000
|
||||
# This should trigger underflow protection
|
||||
from prisma.models import UserBalance
|
||||
|
||||
test_balance = POSTGRES_INT_MIN + 1000
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={
|
||||
"create": {"userId": user_id, "balance": test_balance},
|
||||
"update": {"balance": test_balance},
|
||||
},
|
||||
)
|
||||
|
||||
current_balance = await credit_system.get_credits(user_id)
|
||||
assert current_balance == test_balance
|
||||
|
||||
# Try to deduct amount that would cause underflow: test_balance + (-2000) = POSTGRES_INT_MIN - 1000
|
||||
underflow_amount = -2000
|
||||
expected_without_protection = (
|
||||
current_balance + underflow_amount
|
||||
) # Should be POSTGRES_INT_MIN - 1000
|
||||
|
||||
# Use _add_transaction directly with amount that would cause underflow
|
||||
final_balance, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=underflow_amount,
|
||||
transaction_type=CreditTransactionType.REFUND,
|
||||
fail_insufficient_credits=False, # Allow going negative for refunds
|
||||
)
|
||||
|
||||
# Balance should be clamped to POSTGRES_INT_MIN, not the calculated underflow value
|
||||
assert (
|
||||
final_balance == POSTGRES_INT_MIN
|
||||
), f"Balance should be clamped to {POSTGRES_INT_MIN}, got {final_balance}"
|
||||
assert (
|
||||
final_balance > expected_without_protection
|
||||
), f"Balance should be greater than underflow result {expected_without_protection}, got {final_balance}"
|
||||
|
||||
# Verify with get_credits too
|
||||
stored_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
stored_balance == POSTGRES_INT_MIN
|
||||
), f"Stored balance should be {POSTGRES_INT_MIN}, got {stored_balance}"
|
||||
|
||||
# Verify transaction was created with the underflow-protected balance
|
||||
transactions = await CreditTransaction.prisma().find_many(
|
||||
where={"userId": user_id, "type": CreditTransactionType.REFUND},
|
||||
order={"createdAt": "desc"},
|
||||
)
|
||||
assert len(transactions) > 0, "Refund transaction should be created"
|
||||
assert (
|
||||
transactions[0].runningBalance == POSTGRES_INT_MIN
|
||||
), f"Transaction should show clamped balance {POSTGRES_INT_MIN}, got {transactions[0].runningBalance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_multiple_large_refunds_cumulative_underflow(server: SpinTestServer):
|
||||
"""Test that multiple large refunds applied sequentially don't cause underflow."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"cumulative-underflow-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Set up balance close to underflow threshold
|
||||
from prisma.models import UserBalance
|
||||
|
||||
initial_balance = POSTGRES_INT_MIN + 500 # Close to minimum but with some room
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={
|
||||
"create": {"userId": user_id, "balance": initial_balance},
|
||||
"update": {"balance": initial_balance},
|
||||
},
|
||||
)
|
||||
|
||||
# Apply multiple refunds that would cumulatively underflow
|
||||
refund_amount = -300 # Each refund that would cause underflow when cumulative
|
||||
|
||||
# First refund: (POSTGRES_INT_MIN + 500) + (-300) = POSTGRES_INT_MIN + 200 (still above minimum)
|
||||
balance_1, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=refund_amount,
|
||||
transaction_type=CreditTransactionType.REFUND,
|
||||
fail_insufficient_credits=False,
|
||||
)
|
||||
|
||||
# Should be above minimum for first refund
|
||||
expected_balance_1 = (
|
||||
initial_balance + refund_amount
|
||||
) # Should be POSTGRES_INT_MIN + 200
|
||||
assert (
|
||||
balance_1 == expected_balance_1
|
||||
), f"First refund should result in {expected_balance_1}, got {balance_1}"
|
||||
assert (
|
||||
balance_1 >= POSTGRES_INT_MIN
|
||||
), f"First refund should not go below {POSTGRES_INT_MIN}, got {balance_1}"
|
||||
|
||||
# Second refund: (POSTGRES_INT_MIN + 200) + (-300) = POSTGRES_INT_MIN - 100 (would underflow)
|
||||
balance_2, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=refund_amount,
|
||||
transaction_type=CreditTransactionType.REFUND,
|
||||
fail_insufficient_credits=False,
|
||||
)
|
||||
|
||||
# Should be clamped to minimum due to underflow protection
|
||||
assert (
|
||||
balance_2 == POSTGRES_INT_MIN
|
||||
), f"Second refund should be clamped to {POSTGRES_INT_MIN}, got {balance_2}"
|
||||
|
||||
# Third refund: Should stay at minimum
|
||||
balance_3, _ = await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=refund_amount,
|
||||
transaction_type=CreditTransactionType.REFUND,
|
||||
fail_insufficient_credits=False,
|
||||
)
|
||||
|
||||
# Should still be at minimum
|
||||
assert (
|
||||
balance_3 == POSTGRES_INT_MIN
|
||||
), f"Third refund should stay at {POSTGRES_INT_MIN}, got {balance_3}"
|
||||
|
||||
# Final balance check
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
final_balance == POSTGRES_INT_MIN
|
||||
), f"Final balance should be {POSTGRES_INT_MIN}, got {final_balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_concurrent_large_refunds_no_underflow(server: SpinTestServer):
|
||||
"""Test that concurrent large refunds don't cause race condition underflow."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"concurrent-underflow-test-{uuid4()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Set up balance close to underflow threshold
|
||||
from prisma.models import UserBalance
|
||||
|
||||
initial_balance = POSTGRES_INT_MIN + 1000 # Close to minimum
|
||||
await UserBalance.prisma().upsert(
|
||||
where={"userId": user_id},
|
||||
data={
|
||||
"create": {"userId": user_id, "balance": initial_balance},
|
||||
"update": {"balance": initial_balance},
|
||||
},
|
||||
)
|
||||
|
||||
async def large_refund(amount: int, label: str):
|
||||
try:
|
||||
return await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=-amount,
|
||||
transaction_type=CreditTransactionType.REFUND,
|
||||
fail_insufficient_credits=False,
|
||||
)
|
||||
except Exception as e:
|
||||
return f"FAILED-{label}: {e}"
|
||||
|
||||
# Run concurrent refunds that would cause underflow if not protected
|
||||
# Each refund of 500 would cause underflow: initial_balance + (-500) could go below POSTGRES_INT_MIN
|
||||
refund_amount = 500
|
||||
results = await asyncio.gather(
|
||||
large_refund(refund_amount, "A"),
|
||||
large_refund(refund_amount, "B"),
|
||||
large_refund(refund_amount, "C"),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
# Check all results are valid and no underflow occurred
|
||||
valid_results = []
|
||||
for i, result in enumerate(results):
|
||||
if isinstance(result, tuple):
|
||||
balance, _ = result
|
||||
assert (
|
||||
balance >= POSTGRES_INT_MIN
|
||||
), f"Result {i} balance {balance} underflowed below {POSTGRES_INT_MIN}"
|
||||
valid_results.append(balance)
|
||||
elif isinstance(result, str) and "FAILED" in result:
|
||||
# Some operations might fail due to validation, that's okay
|
||||
pass
|
||||
else:
|
||||
# Unexpected exception
|
||||
assert not isinstance(
|
||||
result, Exception
|
||||
), f"Unexpected exception in result {i}: {result}"
|
||||
|
||||
# At least one operation should succeed
|
||||
assert (
|
||||
len(valid_results) > 0
|
||||
), f"At least one refund should succeed, got results: {results}"
|
||||
|
||||
# All successful results should be >= POSTGRES_INT_MIN
|
||||
for balance in valid_results:
|
||||
assert (
|
||||
balance >= POSTGRES_INT_MIN
|
||||
), f"Balance {balance} should not be below {POSTGRES_INT_MIN}"
|
||||
|
||||
# Final balance should be valid and at or above POSTGRES_INT_MIN
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
final_balance >= POSTGRES_INT_MIN
|
||||
), f"Final balance {final_balance} should not underflow below {POSTGRES_INT_MIN}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
Integration test to verify complete migration from User.balance to UserBalance table.
|
||||
|
||||
This test ensures that:
|
||||
1. No User.balance queries exist in the system
|
||||
2. All balance operations go through UserBalance table
|
||||
3. User and UserBalance stay synchronized properly
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from prisma.enums import CreditTransactionType
|
||||
from prisma.errors import UniqueViolationError
|
||||
from prisma.models import CreditTransaction, User, UserBalance
|
||||
|
||||
from backend.data.credit import UsageTransactionMetadata, UserCredit
|
||||
from backend.util.json import SafeJson
|
||||
from backend.util.test import SpinTestServer
|
||||
|
||||
|
||||
async def create_test_user(user_id: str) -> None:
|
||||
"""Create a test user for migration tests."""
|
||||
try:
|
||||
await User.prisma().create(
|
||||
data={
|
||||
"id": user_id,
|
||||
"email": f"test-{user_id}@example.com",
|
||||
"name": f"Test User {user_id[:8]}",
|
||||
}
|
||||
)
|
||||
except UniqueViolationError:
|
||||
# User already exists, continue
|
||||
pass
|
||||
|
||||
|
||||
async def cleanup_test_user(user_id: str) -> None:
|
||||
"""Clean up test user and their data."""
|
||||
try:
|
||||
await CreditTransaction.prisma().delete_many(where={"userId": user_id})
|
||||
await UserBalance.prisma().delete_many(where={"userId": user_id})
|
||||
await User.prisma().delete_many(where={"id": user_id})
|
||||
except Exception as e:
|
||||
# Log cleanup failures but don't fail the test
|
||||
print(f"Warning: Failed to cleanup test user {user_id}: {e}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_user_balance_migration_complete(server: SpinTestServer):
|
||||
"""Test that User table balance is never used and UserBalance is source of truth."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"migration-test-{datetime.now().timestamp()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# 1. Verify User table does NOT have balance set initially
|
||||
user = await User.prisma().find_unique(where={"id": user_id})
|
||||
assert user is not None
|
||||
# User.balance should not exist or should be None/0 if it exists
|
||||
user_balance_attr = getattr(user, "balance", None)
|
||||
if user_balance_attr is not None:
|
||||
assert (
|
||||
user_balance_attr == 0 or user_balance_attr is None
|
||||
), f"User.balance should be 0 or None, got {user_balance_attr}"
|
||||
|
||||
# 2. Perform various credit operations using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=1000,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "migration_test"}),
|
||||
)
|
||||
balance1 = await credit_system.get_credits(user_id)
|
||||
assert balance1 == 1000
|
||||
|
||||
await credit_system.spend_credits(
|
||||
user_id,
|
||||
300,
|
||||
UsageTransactionMetadata(
|
||||
graph_exec_id="test", reason="Migration test spend"
|
||||
),
|
||||
)
|
||||
balance2 = await credit_system.get_credits(user_id)
|
||||
assert balance2 == 700
|
||||
|
||||
# 3. Verify UserBalance table has correct values
|
||||
user_balance = await UserBalance.prisma().find_unique(where={"userId": user_id})
|
||||
assert user_balance is not None
|
||||
assert (
|
||||
user_balance.balance == 700
|
||||
), f"UserBalance should be 700, got {user_balance.balance}"
|
||||
|
||||
# 4. CRITICAL: Verify User.balance is NEVER updated during operations
|
||||
user_after = await User.prisma().find_unique(where={"id": user_id})
|
||||
assert user_after is not None
|
||||
user_balance_after = getattr(user_after, "balance", None)
|
||||
if user_balance_after is not None:
|
||||
# If User.balance exists, it should still be 0 (never updated)
|
||||
assert (
|
||||
user_balance_after == 0 or user_balance_after is None
|
||||
), f"User.balance should remain 0/None after operations, got {user_balance_after}. This indicates User.balance is still being used!"
|
||||
|
||||
# 5. Verify get_credits always returns UserBalance value, not User.balance
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
final_balance == user_balance.balance
|
||||
), f"get_credits should return UserBalance value {user_balance.balance}, got {final_balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_detect_stale_user_balance_queries(server: SpinTestServer):
|
||||
"""Test to detect if any operations are still using User.balance instead of UserBalance."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"stale-query-test-{datetime.now().timestamp()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Create UserBalance with specific value
|
||||
await UserBalance.prisma().create(
|
||||
data={"userId": user_id, "balance": 5000} # $50
|
||||
)
|
||||
|
||||
# Verify that get_credits returns UserBalance value (5000), not any stale User.balance value
|
||||
balance = await credit_system.get_credits(user_id)
|
||||
assert (
|
||||
balance == 5000
|
||||
), f"Expected get_credits to return 5000 from UserBalance, got {balance}"
|
||||
|
||||
# Verify all operations use UserBalance using internal method (bypasses Stripe)
|
||||
await credit_system._add_transaction(
|
||||
user_id=user_id,
|
||||
amount=1000,
|
||||
transaction_type=CreditTransactionType.TOP_UP,
|
||||
metadata=SafeJson({"test": "final_verification"}),
|
||||
)
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 6000, f"Expected 6000, got {final_balance}"
|
||||
|
||||
# Verify UserBalance table has the correct value
|
||||
user_balance = await UserBalance.prisma().find_unique(where={"userId": user_id})
|
||||
assert user_balance is not None
|
||||
assert (
|
||||
user_balance.balance == 6000
|
||||
), f"UserBalance should be 6000, got {user_balance.balance}"
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_concurrent_operations_use_userbalance_only(server: SpinTestServer):
|
||||
"""Test that concurrent operations all use UserBalance locking, not User.balance."""
|
||||
credit_system = UserCredit()
|
||||
user_id = f"concurrent-userbalance-test-{datetime.now().timestamp()}"
|
||||
await create_test_user(user_id)
|
||||
|
||||
try:
|
||||
# Set initial balance in UserBalance
|
||||
await UserBalance.prisma().create(data={"userId": user_id, "balance": 1000})
|
||||
|
||||
# Run concurrent operations to ensure they all use UserBalance atomic operations
|
||||
async def concurrent_spend(amount: int, label: str):
|
||||
try:
|
||||
await credit_system.spend_credits(
|
||||
user_id,
|
||||
amount,
|
||||
UsageTransactionMetadata(
|
||||
graph_exec_id=f"concurrent-{label}",
|
||||
reason=f"Concurrent test {label}",
|
||||
),
|
||||
)
|
||||
return f"{label}-SUCCESS"
|
||||
except Exception as e:
|
||||
return f"{label}-FAILED: {e}"
|
||||
|
||||
# Run concurrent operations
|
||||
results = await asyncio.gather(
|
||||
concurrent_spend(100, "A"),
|
||||
concurrent_spend(200, "B"),
|
||||
concurrent_spend(300, "C"),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
# All should succeed (1000 >= 100+200+300)
|
||||
successful = [r for r in results if "SUCCESS" in str(r)]
|
||||
assert len(successful) == 3, f"All operations should succeed, got {results}"
|
||||
|
||||
# Final balance should be 1000 - 600 = 400
|
||||
final_balance = await credit_system.get_credits(user_id)
|
||||
assert final_balance == 400, f"Expected final balance 400, got {final_balance}"
|
||||
|
||||
# Verify UserBalance has correct value
|
||||
user_balance = await UserBalance.prisma().find_unique(where={"userId": user_id})
|
||||
assert user_balance is not None
|
||||
assert (
|
||||
user_balance.balance == 400
|
||||
), f"UserBalance should be 400, got {user_balance.balance}"
|
||||
|
||||
# Critical: If User.balance exists and was used, it might have wrong value
|
||||
try:
|
||||
user = await User.prisma().find_unique(where={"id": user_id})
|
||||
user_balance_attr = getattr(user, "balance", None)
|
||||
if user_balance_attr is not None:
|
||||
# If User.balance exists, it should NOT be used for operations
|
||||
# The fact that our final balance is correct from UserBalance proves the system is working
|
||||
print(
|
||||
f"✅ User.balance exists ({user_balance_attr}) but UserBalance ({user_balance.balance}) is being used correctly"
|
||||
)
|
||||
except Exception:
|
||||
print("✅ User.balance column doesn't exist - migration is complete")
|
||||
|
||||
finally:
|
||||
await cleanup_test_user(user_id)
|
||||
@@ -98,42 +98,6 @@ async def transaction(timeout: int = TRANSACTION_TIMEOUT):
|
||||
yield tx
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def locked_transaction(key: str, timeout: int = TRANSACTION_TIMEOUT):
|
||||
"""
|
||||
Create a transaction and take a per-key advisory *transaction* lock.
|
||||
|
||||
- Uses a 64-bit lock id via hashtextextended(key, 0) to avoid 32-bit collisions.
|
||||
- Bound by lock_timeout and statement_timeout so it won't block indefinitely.
|
||||
- Lock is held for the duration of the transaction and auto-released on commit/rollback.
|
||||
|
||||
Args:
|
||||
key: String lock key (e.g., "usr_trx_<uuid>").
|
||||
timeout: Transaction/lock/statement timeout in milliseconds.
|
||||
"""
|
||||
async with transaction(timeout=timeout) as tx:
|
||||
# Ensure we don't wait longer than desired
|
||||
# Note: SET LOCAL doesn't support parameterized queries, must use string interpolation
|
||||
await tx.execute_raw(f"SET LOCAL statement_timeout = '{int(timeout)}ms'") # type: ignore[arg-type]
|
||||
await tx.execute_raw(f"SET LOCAL lock_timeout = '{int(timeout)}ms'") # type: ignore[arg-type]
|
||||
|
||||
# Block until acquired or lock_timeout hits
|
||||
try:
|
||||
await tx.execute_raw(
|
||||
"SELECT pg_advisory_xact_lock(hashtextextended($1, 0))",
|
||||
key,
|
||||
)
|
||||
except Exception as e:
|
||||
# Normalize PG's lock timeout error to TimeoutError for callers
|
||||
if "lock timeout" in str(e).lower():
|
||||
raise TimeoutError(
|
||||
f"Could not acquire lock for key={key!r} within {timeout}ms"
|
||||
) from e
|
||||
raise
|
||||
|
||||
yield tx
|
||||
|
||||
|
||||
def get_database_schema() -> str:
|
||||
"""Extract database schema from DATABASE_URL."""
|
||||
parsed_url = urlparse(DATABASE_URL)
|
||||
|
||||
@@ -4,7 +4,6 @@ from typing import Any, Optional
|
||||
|
||||
import prisma
|
||||
import pydantic
|
||||
from autogpt_libs.utils.cache import cached
|
||||
from prisma.enums import OnboardingStep
|
||||
from prisma.models import UserOnboarding
|
||||
from prisma.types import UserOnboardingCreateInput, UserOnboardingUpdateInput
|
||||
@@ -13,6 +12,7 @@ from backend.data.block import get_blocks
|
||||
from backend.data.credit import get_user_credit_model
|
||||
from backend.data.model import CredentialsMetaInput
|
||||
from backend.server.v2.store.model import StoreAgentDetails
|
||||
from backend.util.cache import cached
|
||||
from backend.util.json import SafeJson
|
||||
|
||||
# Mapping from user reason id to categories to search for when choosing agent to show
|
||||
@@ -26,8 +26,6 @@ REASON_MAPPING: dict[str, list[str]] = {
|
||||
POINTS_AGENT_COUNT = 50 # Number of agents to calculate points for
|
||||
MIN_AGENT_COUNT = 2 # Minimum number of marketplace agents to enable onboarding
|
||||
|
||||
user_credit = get_user_credit_model()
|
||||
|
||||
|
||||
class UserOnboardingUpdate(pydantic.BaseModel):
|
||||
completedSteps: Optional[list[OnboardingStep]] = None
|
||||
@@ -147,7 +145,8 @@ async def reward_user(user_id: str, step: OnboardingStep):
|
||||
return
|
||||
|
||||
onboarding.rewardedFor.append(step)
|
||||
await user_credit.onboarding_reward(user_id, reward, step)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
await user_credit_model.onboarding_reward(user_id, reward, step)
|
||||
await UserOnboarding.prisma().update(
|
||||
where={"userId": user_id},
|
||||
data={
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from autogpt_libs.utils.cache import cached, thread_cached
|
||||
from dotenv import load_dotenv
|
||||
from redis import Redis
|
||||
from redis.asyncio import Redis as AsyncRedis
|
||||
|
||||
from backend.util.cache import cached, thread_cached
|
||||
from backend.util.retry import conn_retry
|
||||
|
||||
load_dotenv()
|
||||
@@ -34,7 +34,7 @@ def disconnect():
|
||||
get_redis().close()
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
def get_redis() -> Redis:
|
||||
return connect()
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ from typing import Optional, cast
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from autogpt_libs.auth.models import DEFAULT_USER_ID
|
||||
from autogpt_libs.utils.cache import cached
|
||||
from fastapi import HTTPException
|
||||
from prisma.enums import NotificationType
|
||||
from prisma.models import User as PrismaUser
|
||||
@@ -16,6 +15,7 @@ from prisma.types import JsonFilter, UserCreateInput, UserUpdateInput
|
||||
from backend.data.db import prisma
|
||||
from backend.data.model import User, UserIntegrations, UserMetadata
|
||||
from backend.data.notifications import NotificationPreference, NotificationPreferenceDTO
|
||||
from backend.util.cache import cached
|
||||
from backend.util.encryption import JSONCryptor
|
||||
from backend.util.exceptions import DatabaseError
|
||||
from backend.util.json import SafeJson
|
||||
|
||||
@@ -57,7 +57,6 @@ from backend.util.service import (
|
||||
from backend.util.settings import Config
|
||||
|
||||
config = Config()
|
||||
_user_credit_model = get_user_credit_model()
|
||||
logger = logging.getLogger(__name__)
|
||||
P = ParamSpec("P")
|
||||
R = TypeVar("R")
|
||||
@@ -66,11 +65,13 @@ R = TypeVar("R")
|
||||
async def _spend_credits(
|
||||
user_id: str, cost: int, metadata: UsageTransactionMetadata
|
||||
) -> int:
|
||||
return await _user_credit_model.spend_credits(user_id, cost, metadata)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
return await user_credit_model.spend_credits(user_id, cost, metadata)
|
||||
|
||||
|
||||
async def _get_credits(user_id: str) -> int:
|
||||
return await _user_credit_model.get_credits(user_id)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
return await user_credit_model.get_credits(user_id)
|
||||
|
||||
|
||||
class DatabaseManager(AppService):
|
||||
|
||||
@@ -246,7 +246,7 @@ async def execute_node(
|
||||
async for output_name, output_data in node_block.execute(
|
||||
input_data, **extra_exec_kwargs
|
||||
):
|
||||
output_data = json.convert_pydantic_to_json(output_data)
|
||||
output_data = json.to_dict(output_data)
|
||||
output_size += len(json.dumps(output_data))
|
||||
log_metadata.debug("Node produced output", **{output_name: output_data})
|
||||
yield output_name, output_data
|
||||
@@ -1548,11 +1548,12 @@ class ExecutionManager(AppProcess):
|
||||
logger.warning(
|
||||
f"[{self.service_name}] Graph {graph_exec_id} already running on pod {current_owner}"
|
||||
)
|
||||
_ack_message(reject=True, requeue=False)
|
||||
else:
|
||||
logger.warning(
|
||||
f"[{self.service_name}] Could not acquire lock for {graph_exec_id} - Redis unavailable"
|
||||
)
|
||||
_ack_message(reject=True, requeue=True)
|
||||
_ack_message(reject=True, requeue=True)
|
||||
return
|
||||
self._execution_locks[graph_exec_id] = cluster_lock
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from autogpt_libs.utils.cache import cached
|
||||
from backend.util.cache import cached
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..providers import ProviderName
|
||||
@@ -8,7 +8,7 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
# --8<-- [start:load_webhook_managers]
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
def load_webhook_managers() -> dict["ProviderName", type["BaseWebhooksManager"]]:
|
||||
webhook_managers = {}
|
||||
|
||||
|
||||
@@ -14,19 +14,49 @@ def configured_snapshot(snapshot: Snapshot) -> Snapshot:
|
||||
@pytest.fixture
|
||||
def test_user_id() -> str:
|
||||
"""Test user ID fixture."""
|
||||
return "test-user-id"
|
||||
return "3e53486c-cf57-477e-ba2a-cb02dc828e1a"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_user_id() -> str:
|
||||
"""Admin user ID fixture."""
|
||||
return "admin-user-id"
|
||||
return "4e53486c-cf57-477e-ba2a-cb02dc828e1b"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target_user_id() -> str:
|
||||
"""Target user ID fixture."""
|
||||
return "target-user-id"
|
||||
return "5e53486c-cf57-477e-ba2a-cb02dc828e1c"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def setup_test_user(test_user_id):
|
||||
"""Create test user in database before tests."""
|
||||
from backend.data.user import get_or_create_user
|
||||
|
||||
# Create the test user in the database using JWT token format
|
||||
user_data = {
|
||||
"sub": test_user_id,
|
||||
"email": "test@example.com",
|
||||
"user_metadata": {"name": "Test User"},
|
||||
}
|
||||
await get_or_create_user(user_data)
|
||||
return test_user_id
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def setup_admin_user(admin_user_id):
|
||||
"""Create admin user in database before tests."""
|
||||
from backend.data.user import get_or_create_user
|
||||
|
||||
# Create the admin user in the database using JWT token format
|
||||
user_data = {
|
||||
"sub": admin_user_id,
|
||||
"email": "test-admin@example.com",
|
||||
"user_metadata": {"name": "Test Admin"},
|
||||
}
|
||||
await get_or_create_user(user_data)
|
||||
return admin_user_id
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -11,7 +11,6 @@ import pydantic
|
||||
import stripe
|
||||
from autogpt_libs.auth import get_user_id, requires_user
|
||||
from autogpt_libs.auth.jwt_utils import get_jwt_payload
|
||||
from autogpt_libs.utils.cache import cached
|
||||
from fastapi import (
|
||||
APIRouter,
|
||||
Body,
|
||||
@@ -40,6 +39,7 @@ from backend.data.credit import (
|
||||
AutoTopUpConfig,
|
||||
RefundRequest,
|
||||
TransactionHistory,
|
||||
UserCredit,
|
||||
get_auto_top_up,
|
||||
get_user_credit_model,
|
||||
set_auto_top_up,
|
||||
@@ -84,6 +84,7 @@ from backend.server.model import (
|
||||
UpdateTimezoneRequest,
|
||||
UploadFileResponse,
|
||||
)
|
||||
from backend.util.cache import cached
|
||||
from backend.util.clients import get_scheduler_client
|
||||
from backend.util.cloud_storage import get_cloud_storage_handler
|
||||
from backend.util.exceptions import GraphValidationError, NotFoundError
|
||||
@@ -107,9 +108,6 @@ def _create_file_size_error(size_bytes: int, max_size_mb: int) -> HTTPException:
|
||||
settings = Settings()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_user_credit_model = get_user_credit_model()
|
||||
|
||||
# Define the API routes
|
||||
v1_router = APIRouter()
|
||||
|
||||
@@ -291,7 +289,7 @@ def _compute_blocks_sync() -> str:
|
||||
return dumps(result)
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
async def _get_cached_blocks() -> str:
|
||||
"""
|
||||
Async cached function with thundering herd protection.
|
||||
@@ -478,7 +476,8 @@ async def upload_file(
|
||||
async def get_user_credits(
|
||||
user_id: Annotated[str, Security(get_user_id)],
|
||||
) -> dict[str, int]:
|
||||
return {"credits": await _user_credit_model.get_credits(user_id)}
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
return {"credits": await user_credit_model.get_credits(user_id)}
|
||||
|
||||
|
||||
@v1_router.post(
|
||||
@@ -490,9 +489,8 @@ async def get_user_credits(
|
||||
async def request_top_up(
|
||||
request: RequestTopUp, user_id: Annotated[str, Security(get_user_id)]
|
||||
):
|
||||
checkout_url = await _user_credit_model.top_up_intent(
|
||||
user_id, request.credit_amount
|
||||
)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
checkout_url = await user_credit_model.top_up_intent(user_id, request.credit_amount)
|
||||
return {"checkout_url": checkout_url}
|
||||
|
||||
|
||||
@@ -507,7 +505,8 @@ async def refund_top_up(
|
||||
transaction_key: str,
|
||||
metadata: dict[str, str],
|
||||
) -> int:
|
||||
return await _user_credit_model.top_up_refund(user_id, transaction_key, metadata)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
return await user_credit_model.top_up_refund(user_id, transaction_key, metadata)
|
||||
|
||||
|
||||
@v1_router.patch(
|
||||
@@ -517,7 +516,8 @@ async def refund_top_up(
|
||||
dependencies=[Security(requires_user)],
|
||||
)
|
||||
async def fulfill_checkout(user_id: Annotated[str, Security(get_user_id)]):
|
||||
await _user_credit_model.fulfill_checkout(user_id=user_id)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
await user_credit_model.fulfill_checkout(user_id=user_id)
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
@@ -531,18 +531,23 @@ async def configure_user_auto_top_up(
|
||||
request: AutoTopUpConfig, user_id: Annotated[str, Security(get_user_id)]
|
||||
) -> str:
|
||||
if request.threshold < 0:
|
||||
raise ValueError("Threshold must be greater than 0")
|
||||
raise HTTPException(status_code=422, detail="Threshold must be greater than 0")
|
||||
if request.amount < 500 and request.amount != 0:
|
||||
raise ValueError("Amount must be greater than or equal to 500")
|
||||
if request.amount < request.threshold:
|
||||
raise ValueError("Amount must be greater than or equal to threshold")
|
||||
raise HTTPException(
|
||||
status_code=422, detail="Amount must be greater than or equal to 500"
|
||||
)
|
||||
if request.amount != 0 and request.amount < request.threshold:
|
||||
raise HTTPException(
|
||||
status_code=422, detail="Amount must be greater than or equal to threshold"
|
||||
)
|
||||
|
||||
current_balance = await _user_credit_model.get_credits(user_id)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
current_balance = await user_credit_model.get_credits(user_id)
|
||||
|
||||
if current_balance < request.threshold:
|
||||
await _user_credit_model.top_up_credits(user_id, request.amount)
|
||||
await user_credit_model.top_up_credits(user_id, request.amount)
|
||||
else:
|
||||
await _user_credit_model.top_up_credits(user_id, 0)
|
||||
await user_credit_model.top_up_credits(user_id, 0)
|
||||
|
||||
await set_auto_top_up(
|
||||
user_id, AutoTopUpConfig(threshold=request.threshold, amount=request.amount)
|
||||
@@ -590,15 +595,13 @@ async def stripe_webhook(request: Request):
|
||||
event["type"] == "checkout.session.completed"
|
||||
or event["type"] == "checkout.session.async_payment_succeeded"
|
||||
):
|
||||
await _user_credit_model.fulfill_checkout(
|
||||
session_id=event["data"]["object"]["id"]
|
||||
)
|
||||
await UserCredit().fulfill_checkout(session_id=event["data"]["object"]["id"])
|
||||
|
||||
if event["type"] == "charge.dispute.created":
|
||||
await _user_credit_model.handle_dispute(event["data"]["object"])
|
||||
await UserCredit().handle_dispute(event["data"]["object"])
|
||||
|
||||
if event["type"] == "refund.created" or event["type"] == "charge.dispute.closed":
|
||||
await _user_credit_model.deduct_credits(event["data"]["object"])
|
||||
await UserCredit().deduct_credits(event["data"]["object"])
|
||||
|
||||
return Response(status_code=200)
|
||||
|
||||
@@ -612,7 +615,8 @@ async def stripe_webhook(request: Request):
|
||||
async def manage_payment_method(
|
||||
user_id: Annotated[str, Security(get_user_id)],
|
||||
) -> dict[str, str]:
|
||||
return {"url": await _user_credit_model.create_billing_portal_session(user_id)}
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
return {"url": await user_credit_model.create_billing_portal_session(user_id)}
|
||||
|
||||
|
||||
@v1_router.get(
|
||||
@@ -630,7 +634,8 @@ async def get_credit_history(
|
||||
if transaction_count_limit < 1 or transaction_count_limit > 1000:
|
||||
raise ValueError("Transaction count limit must be between 1 and 1000")
|
||||
|
||||
return await _user_credit_model.get_transaction_history(
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
return await user_credit_model.get_transaction_history(
|
||||
user_id=user_id,
|
||||
transaction_time_ceiling=transaction_time,
|
||||
transaction_count_limit=transaction_count_limit,
|
||||
@@ -647,7 +652,8 @@ async def get_credit_history(
|
||||
async def get_refund_requests(
|
||||
user_id: Annotated[str, Security(get_user_id)],
|
||||
) -> list[RefundRequest]:
|
||||
return await _user_credit_model.get_refund_requests(user_id)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
return await user_credit_model.get_refund_requests(user_id)
|
||||
|
||||
|
||||
########################################################
|
||||
@@ -869,7 +875,8 @@ async def execute_graph(
|
||||
graph_version: Optional[int] = None,
|
||||
preset_id: Optional[str] = None,
|
||||
) -> execution_db.GraphExecutionMeta:
|
||||
current_balance = await _user_credit_model.get_credits(user_id)
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
current_balance = await user_credit_model.get_credits(user_id)
|
||||
if current_balance <= 0:
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
|
||||
@@ -23,10 +23,13 @@ client = fastapi.testclient.TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_app_auth(mock_jwt_user):
|
||||
def setup_app_auth(mock_jwt_user, setup_test_user):
|
||||
"""Setup auth overrides for all tests in this module"""
|
||||
from autogpt_libs.auth.jwt_utils import get_jwt_payload
|
||||
|
||||
# setup_test_user fixture already executed and user is created in database
|
||||
# It returns the user_id which we don't need to await
|
||||
|
||||
app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]
|
||||
yield
|
||||
app.dependency_overrides.clear()
|
||||
@@ -194,8 +197,12 @@ def test_get_user_credits(
|
||||
snapshot: Snapshot,
|
||||
) -> None:
|
||||
"""Test get user credits endpoint"""
|
||||
mock_credit_model = mocker.patch("backend.server.routers.v1._user_credit_model")
|
||||
mock_credit_model = Mock()
|
||||
mock_credit_model.get_credits = AsyncMock(return_value=1000)
|
||||
mocker.patch(
|
||||
"backend.server.routers.v1.get_user_credit_model",
|
||||
return_value=mock_credit_model,
|
||||
)
|
||||
|
||||
response = client.get("/credits")
|
||||
|
||||
@@ -215,10 +222,14 @@ def test_request_top_up(
|
||||
snapshot: Snapshot,
|
||||
) -> None:
|
||||
"""Test request top up endpoint"""
|
||||
mock_credit_model = mocker.patch("backend.server.routers.v1._user_credit_model")
|
||||
mock_credit_model = Mock()
|
||||
mock_credit_model.top_up_intent = AsyncMock(
|
||||
return_value="https://checkout.example.com/session123"
|
||||
)
|
||||
mocker.patch(
|
||||
"backend.server.routers.v1.get_user_credit_model",
|
||||
return_value=mock_credit_model,
|
||||
)
|
||||
|
||||
request_data = {"credit_amount": 500}
|
||||
|
||||
@@ -261,6 +272,74 @@ def test_get_auto_top_up(
|
||||
)
|
||||
|
||||
|
||||
def test_configure_auto_top_up(
|
||||
mocker: pytest_mock.MockFixture,
|
||||
snapshot: Snapshot,
|
||||
) -> None:
|
||||
"""Test configure auto top-up endpoint - this test would have caught the enum casting bug"""
|
||||
# Mock the set_auto_top_up function to avoid database operations
|
||||
mocker.patch(
|
||||
"backend.server.routers.v1.set_auto_top_up",
|
||||
return_value=None,
|
||||
)
|
||||
|
||||
# Mock credit model to avoid Stripe API calls
|
||||
mock_credit_model = mocker.AsyncMock()
|
||||
mock_credit_model.get_credits.return_value = 50 # Current balance below threshold
|
||||
mock_credit_model.top_up_credits.return_value = None
|
||||
|
||||
mocker.patch(
|
||||
"backend.server.routers.v1.get_user_credit_model",
|
||||
return_value=mock_credit_model,
|
||||
)
|
||||
|
||||
# Test data
|
||||
request_data = {
|
||||
"threshold": 100,
|
||||
"amount": 500,
|
||||
}
|
||||
|
||||
response = client.post("/credits/auto-top-up", json=request_data)
|
||||
|
||||
# This should succeed with our fix, but would have failed before with the enum casting error
|
||||
assert response.status_code == 200
|
||||
assert response.json() == "Auto top-up settings updated"
|
||||
|
||||
|
||||
def test_configure_auto_top_up_validation_errors(
|
||||
mocker: pytest_mock.MockFixture,
|
||||
) -> None:
|
||||
"""Test configure auto top-up endpoint validation"""
|
||||
# Mock set_auto_top_up to avoid database operations for successful case
|
||||
mocker.patch("backend.server.routers.v1.set_auto_top_up")
|
||||
|
||||
# Mock credit model to avoid Stripe API calls for the successful case
|
||||
mock_credit_model = mocker.AsyncMock()
|
||||
mock_credit_model.get_credits.return_value = 50
|
||||
mock_credit_model.top_up_credits.return_value = None
|
||||
|
||||
mocker.patch(
|
||||
"backend.server.routers.v1.get_user_credit_model",
|
||||
return_value=mock_credit_model,
|
||||
)
|
||||
|
||||
# Test negative threshold
|
||||
response = client.post(
|
||||
"/credits/auto-top-up", json={"threshold": -1, "amount": 500}
|
||||
)
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
# Test amount too small (but not 0)
|
||||
response = client.post(
|
||||
"/credits/auto-top-up", json={"threshold": 100, "amount": 100}
|
||||
)
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
# Test amount = 0 (should be allowed)
|
||||
response = client.post("/credits/auto-top-up", json={"threshold": 100, "amount": 0})
|
||||
assert response.status_code == 200 # Should succeed
|
||||
|
||||
|
||||
# Graphs endpoints tests
|
||||
def test_get_graphs(
|
||||
mocker: pytest_mock.MockFixture,
|
||||
|
||||
@@ -11,8 +11,6 @@ from backend.util.json import SafeJson
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_user_credit_model = get_user_credit_model()
|
||||
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/admin",
|
||||
@@ -33,7 +31,8 @@ async def add_user_credits(
|
||||
logger.info(
|
||||
f"Admin user {admin_user_id} is adding {amount} credits to user {user_id}"
|
||||
)
|
||||
new_balance, transaction_key = await _user_credit_model._add_transaction(
|
||||
user_credit_model = await get_user_credit_model(user_id)
|
||||
new_balance, transaction_key = await user_credit_model._add_transaction(
|
||||
user_id,
|
||||
amount,
|
||||
transaction_type=CreditTransactionType.GRANT,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import json
|
||||
from unittest.mock import AsyncMock
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import fastapi
|
||||
import fastapi.testclient
|
||||
@@ -7,12 +7,12 @@ import prisma.enums
|
||||
import pytest
|
||||
import pytest_mock
|
||||
from autogpt_libs.auth.jwt_utils import get_jwt_payload
|
||||
from prisma import Json
|
||||
from pytest_snapshot.plugin import Snapshot
|
||||
|
||||
import backend.server.v2.admin.credit_admin_routes as credit_admin_routes
|
||||
import backend.server.v2.admin.model as admin_model
|
||||
from backend.data.model import UserTransaction
|
||||
from backend.util.json import SafeJson
|
||||
from backend.util.models import Pagination
|
||||
|
||||
app = fastapi.FastAPI()
|
||||
@@ -37,12 +37,14 @@ def test_add_user_credits_success(
|
||||
) -> None:
|
||||
"""Test successful credit addition by admin"""
|
||||
# Mock the credit model
|
||||
mock_credit_model = mocker.patch(
|
||||
"backend.server.v2.admin.credit_admin_routes._user_credit_model"
|
||||
)
|
||||
mock_credit_model = Mock()
|
||||
mock_credit_model._add_transaction = AsyncMock(
|
||||
return_value=(1500, "transaction-123-uuid")
|
||||
)
|
||||
mocker.patch(
|
||||
"backend.server.v2.admin.credit_admin_routes.get_user_credit_model",
|
||||
return_value=mock_credit_model,
|
||||
)
|
||||
|
||||
request_data = {
|
||||
"user_id": target_user_id,
|
||||
@@ -62,11 +64,17 @@ def test_add_user_credits_success(
|
||||
call_args = mock_credit_model._add_transaction.call_args
|
||||
assert call_args[0] == (target_user_id, 500)
|
||||
assert call_args[1]["transaction_type"] == prisma.enums.CreditTransactionType.GRANT
|
||||
# Check that metadata is a Json object with the expected content
|
||||
assert isinstance(call_args[1]["metadata"], Json)
|
||||
assert call_args[1]["metadata"] == Json(
|
||||
{"admin_id": admin_user_id, "reason": "Test credit grant for debugging"}
|
||||
)
|
||||
# Check that metadata is a SafeJson object with the expected content
|
||||
assert isinstance(call_args[1]["metadata"], SafeJson)
|
||||
actual_metadata = call_args[1]["metadata"]
|
||||
expected_data = {
|
||||
"admin_id": admin_user_id,
|
||||
"reason": "Test credit grant for debugging",
|
||||
}
|
||||
|
||||
# SafeJson inherits from Json which stores parsed data in the .data attribute
|
||||
assert actual_metadata.data["admin_id"] == expected_data["admin_id"]
|
||||
assert actual_metadata.data["reason"] == expected_data["reason"]
|
||||
|
||||
# Snapshot test the response
|
||||
configured_snapshot.assert_match(
|
||||
@@ -81,12 +89,14 @@ def test_add_user_credits_negative_amount(
|
||||
) -> None:
|
||||
"""Test credit deduction by admin (negative amount)"""
|
||||
# Mock the credit model
|
||||
mock_credit_model = mocker.patch(
|
||||
"backend.server.v2.admin.credit_admin_routes._user_credit_model"
|
||||
)
|
||||
mock_credit_model = Mock()
|
||||
mock_credit_model._add_transaction = AsyncMock(
|
||||
return_value=(200, "transaction-456-uuid")
|
||||
)
|
||||
mocker.patch(
|
||||
"backend.server.v2.admin.credit_admin_routes.get_user_credit_model",
|
||||
return_value=mock_credit_model,
|
||||
)
|
||||
|
||||
request_data = {
|
||||
"user_id": "target-user-id",
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import prisma
|
||||
from autogpt_libs.utils.cache import cached
|
||||
|
||||
import backend.data.block
|
||||
from backend.blocks import load_all_blocks
|
||||
@@ -18,6 +17,7 @@ from backend.server.v2.builder.model import (
|
||||
ProviderResponse,
|
||||
SearchBlocksResponse,
|
||||
)
|
||||
from backend.util.cache import cached
|
||||
from backend.util.models import Pagination
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -307,7 +307,7 @@ def _matches_llm_model(schema_cls: type[BlockSchema], query: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
def _get_all_providers() -> dict[ProviderName, Provider]:
|
||||
providers: dict[ProviderName, Provider] = {}
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from autogpt_libs.utils.cache import cached
|
||||
|
||||
import backend.server.v2.store.db
|
||||
from backend.util.cache import cached
|
||||
|
||||
##############################################
|
||||
############### Caches #######################
|
||||
@@ -17,7 +16,7 @@ def clear_all_caches():
|
||||
|
||||
# Cache store agents list for 5 minutes
|
||||
# Different cache entries for different query combinations
|
||||
@cached(maxsize=5000, ttl_seconds=300)
|
||||
@cached(maxsize=5000, ttl_seconds=300, shared_cache=True)
|
||||
async def _get_cached_store_agents(
|
||||
featured: bool,
|
||||
creator: str | None,
|
||||
@@ -40,7 +39,7 @@ async def _get_cached_store_agents(
|
||||
|
||||
|
||||
# Cache individual agent details for 15 minutes
|
||||
@cached(maxsize=200, ttl_seconds=300)
|
||||
@cached(maxsize=200, ttl_seconds=300, shared_cache=True)
|
||||
async def _get_cached_agent_details(username: str, agent_name: str):
|
||||
"""Cached helper to get agent details."""
|
||||
return await backend.server.v2.store.db.get_store_agent_details(
|
||||
@@ -49,7 +48,7 @@ async def _get_cached_agent_details(username: str, agent_name: str):
|
||||
|
||||
|
||||
# Cache creators list for 5 minutes
|
||||
@cached(maxsize=200, ttl_seconds=300)
|
||||
@cached(maxsize=200, ttl_seconds=300, shared_cache=True)
|
||||
async def _get_cached_store_creators(
|
||||
featured: bool,
|
||||
search_query: str | None,
|
||||
@@ -68,7 +67,7 @@ async def _get_cached_store_creators(
|
||||
|
||||
|
||||
# Cache individual creator details for 5 minutes
|
||||
@cached(maxsize=100, ttl_seconds=300)
|
||||
@cached(maxsize=100, ttl_seconds=300, shared_cache=True)
|
||||
async def _get_cached_creator_details(username: str):
|
||||
"""Cached helper to get creator details."""
|
||||
return await backend.server.v2.store.db.get_store_creator_details(
|
||||
|
||||
457
autogpt_platform/backend/backend/util/cache.py
Normal file
457
autogpt_platform/backend/backend/util/cache.py
Normal file
@@ -0,0 +1,457 @@
|
||||
"""
|
||||
Caching utilities for the AutoGPT platform.
|
||||
|
||||
Provides decorators for caching function results with support for:
|
||||
- In-memory caching with TTL
|
||||
- Shared Redis-backed caching across processes
|
||||
- Thread-local caching for request-scoped data
|
||||
- Thundering herd protection
|
||||
- LRU eviction with optional TTL refresh
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
import pickle
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from functools import wraps
|
||||
from typing import Any, Callable, ParamSpec, Protocol, TypeVar, cast, runtime_checkable
|
||||
|
||||
from redis import ConnectionPool, Redis
|
||||
|
||||
from backend.util.retry import conn_retry
|
||||
from backend.util.settings import Settings
|
||||
|
||||
P = ParamSpec("P")
|
||||
R = TypeVar("R")
|
||||
R_co = TypeVar("R_co", covariant=True)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
settings = Settings()
|
||||
|
||||
# RECOMMENDED REDIS CONFIGURATION FOR PRODUCTION:
|
||||
# Configure Redis with the following settings for optimal caching performance:
|
||||
# maxmemory-policy allkeys-lru # Evict least recently used keys when memory limit reached
|
||||
# maxmemory 2gb # Set memory limit (adjust based on your needs)
|
||||
# save "" # Disable persistence if using Redis purely for caching
|
||||
|
||||
# Create a dedicated Redis connection pool for caching (binary mode for pickle)
|
||||
_cache_pool: ConnectionPool | None = None
|
||||
|
||||
|
||||
@conn_retry("Redis", "Acquiring cache connection pool")
|
||||
def _get_cache_pool() -> ConnectionPool:
|
||||
"""Get or create a connection pool for cache operations."""
|
||||
global _cache_pool
|
||||
if _cache_pool is None:
|
||||
_cache_pool = ConnectionPool(
|
||||
host=settings.config.redis_host,
|
||||
port=settings.config.redis_port,
|
||||
password=settings.config.redis_password or None,
|
||||
decode_responses=False, # Binary mode for pickle
|
||||
max_connections=50,
|
||||
socket_keepalive=True,
|
||||
socket_connect_timeout=5,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
return _cache_pool
|
||||
|
||||
|
||||
redis = Redis(connection_pool=_get_cache_pool())
|
||||
|
||||
|
||||
@dataclass
|
||||
class CachedValue:
|
||||
"""Wrapper for cached values with timestamp to avoid tuple ambiguity."""
|
||||
|
||||
result: Any
|
||||
timestamp: float
|
||||
|
||||
|
||||
def _make_hashable_key(
|
||||
args: tuple[Any, ...], kwargs: dict[str, Any]
|
||||
) -> tuple[Any, ...]:
|
||||
"""
|
||||
Convert args and kwargs into a hashable cache key.
|
||||
|
||||
Handles unhashable types like dict, list, set by converting them to
|
||||
their sorted string representations.
|
||||
"""
|
||||
|
||||
def make_hashable(obj: Any) -> Any:
|
||||
"""Recursively convert an object to a hashable representation."""
|
||||
if isinstance(obj, dict):
|
||||
# Sort dict items to ensure consistent ordering
|
||||
return (
|
||||
"__dict__",
|
||||
tuple(sorted((k, make_hashable(v)) for k, v in obj.items())),
|
||||
)
|
||||
elif isinstance(obj, (list, tuple)):
|
||||
return ("__list__", tuple(make_hashable(item) for item in obj))
|
||||
elif isinstance(obj, set):
|
||||
return ("__set__", tuple(sorted(make_hashable(item) for item in obj)))
|
||||
elif hasattr(obj, "__dict__"):
|
||||
# Handle objects with __dict__ attribute
|
||||
return ("__obj__", obj.__class__.__name__, make_hashable(obj.__dict__))
|
||||
else:
|
||||
# For basic hashable types (str, int, bool, None, etc.)
|
||||
try:
|
||||
hash(obj)
|
||||
return obj
|
||||
except TypeError:
|
||||
# Fallback: convert to string representation
|
||||
return ("__str__", str(obj))
|
||||
|
||||
hashable_args = tuple(make_hashable(arg) for arg in args)
|
||||
hashable_kwargs = tuple(sorted((k, make_hashable(v)) for k, v in kwargs.items()))
|
||||
return (hashable_args, hashable_kwargs)
|
||||
|
||||
|
||||
def _make_redis_key(key: tuple[Any, ...], func_name: str) -> str:
|
||||
"""Convert a hashable key tuple to a Redis key string."""
|
||||
# Ensure key is already hashable
|
||||
hashable_key = key if isinstance(key, tuple) else (key,)
|
||||
return f"cache:{func_name}:{hash(hashable_key)}"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class CachedFunction(Protocol[P, R_co]):
|
||||
"""Protocol for cached functions with cache management methods."""
|
||||
|
||||
def cache_clear(self, pattern: str | None = None) -> None:
|
||||
"""Clear cached entries. If pattern provided, clear matching entries only."""
|
||||
return None
|
||||
|
||||
def cache_info(self) -> dict[str, int | None]:
|
||||
"""Get cache statistics."""
|
||||
return {}
|
||||
|
||||
def cache_delete(self, *args: P.args, **kwargs: P.kwargs) -> bool:
|
||||
"""Delete a specific cache entry by its arguments. Returns True if entry existed."""
|
||||
return False
|
||||
|
||||
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R_co:
|
||||
"""Call the cached function."""
|
||||
return None # type: ignore
|
||||
|
||||
|
||||
def cached(
|
||||
*,
|
||||
maxsize: int = 128,
|
||||
ttl_seconds: int,
|
||||
shared_cache: bool = False,
|
||||
refresh_ttl_on_get: bool = False,
|
||||
) -> Callable[[Callable], CachedFunction]:
|
||||
"""
|
||||
Thundering herd safe cache decorator for both sync and async functions.
|
||||
|
||||
Uses double-checked locking to prevent multiple threads/coroutines from
|
||||
executing the expensive operation simultaneously during cache misses.
|
||||
|
||||
Args:
|
||||
maxsize: Maximum number of cached entries (only for in-memory cache)
|
||||
ttl_seconds: Time to live in seconds. Required - entries must expire.
|
||||
shared_cache: If True, use Redis for cross-process caching
|
||||
refresh_ttl_on_get: If True, refresh TTL when cache entry is accessed (LRU behavior)
|
||||
|
||||
Returns:
|
||||
Decorated function with caching capabilities
|
||||
|
||||
Example:
|
||||
@cached(ttl_seconds=300) # 5 minute TTL
|
||||
def expensive_sync_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
|
||||
@cached(ttl_seconds=600, shared_cache=True, refresh_ttl_on_get=True)
|
||||
async def expensive_async_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
"""
|
||||
|
||||
def decorator(target_func):
|
||||
cache_storage: dict[tuple, CachedValue] = {}
|
||||
_event_loop_locks: dict[Any, asyncio.Lock] = {}
|
||||
|
||||
def _get_from_redis(redis_key: str) -> Any | None:
|
||||
"""Get value from Redis, optionally refreshing TTL."""
|
||||
try:
|
||||
if refresh_ttl_on_get:
|
||||
# Use GETEX to get value and refresh expiry atomically
|
||||
cached_bytes = redis.getex(redis_key, ex=ttl_seconds)
|
||||
else:
|
||||
cached_bytes = redis.get(redis_key)
|
||||
|
||||
if cached_bytes and isinstance(cached_bytes, bytes):
|
||||
return pickle.loads(cached_bytes)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Redis error during cache check for {target_func.__name__}: {e}"
|
||||
)
|
||||
return None
|
||||
|
||||
def _set_to_redis(redis_key: str, value: Any) -> None:
|
||||
"""Set value in Redis with TTL."""
|
||||
try:
|
||||
pickled_value = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
redis.setex(redis_key, ttl_seconds, pickled_value)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Redis error storing cache for {target_func.__name__}: {e}"
|
||||
)
|
||||
|
||||
def _get_from_memory(key: tuple) -> Any | None:
|
||||
"""Get value from in-memory cache, checking TTL."""
|
||||
if key in cache_storage:
|
||||
cached_data = cache_storage[key]
|
||||
if time.time() - cached_data.timestamp < ttl_seconds:
|
||||
logger.debug(
|
||||
f"Cache hit for {target_func.__name__} args: {key[0]} kwargs: {key[1]}"
|
||||
)
|
||||
return cached_data.result
|
||||
return None
|
||||
|
||||
def _set_to_memory(key: tuple, value: Any) -> None:
|
||||
"""Set value in in-memory cache with timestamp."""
|
||||
cache_storage[key] = CachedValue(result=value, timestamp=time.time())
|
||||
|
||||
# Cleanup if needed
|
||||
if len(cache_storage) > maxsize:
|
||||
cutoff = maxsize // 2
|
||||
oldest_keys = list(cache_storage.keys())[:-cutoff] if cutoff > 0 else []
|
||||
for old_key in oldest_keys:
|
||||
cache_storage.pop(old_key, None)
|
||||
|
||||
if inspect.iscoroutinefunction(target_func):
|
||||
|
||||
def _get_cache_lock():
|
||||
"""Get or create an asyncio.Lock for the current event loop."""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop not in _event_loop_locks:
|
||||
return _event_loop_locks.setdefault(loop, asyncio.Lock())
|
||||
return _event_loop_locks[loop]
|
||||
|
||||
@wraps(target_func)
|
||||
async def async_wrapper(*args: P.args, **kwargs: P.kwargs):
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
redis_key = (
|
||||
_make_redis_key(key, target_func.__name__) if shared_cache else ""
|
||||
)
|
||||
|
||||
# Fast path: check cache without lock
|
||||
if shared_cache:
|
||||
result = _get_from_redis(redis_key)
|
||||
if result is not None:
|
||||
return result
|
||||
else:
|
||||
result = _get_from_memory(key)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Slow path: acquire lock for cache miss/expiry
|
||||
async with _get_cache_lock():
|
||||
# Double-check: another coroutine might have populated cache
|
||||
if shared_cache:
|
||||
result = _get_from_redis(redis_key)
|
||||
if result is not None:
|
||||
return result
|
||||
else:
|
||||
result = _get_from_memory(key)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Cache miss - execute function
|
||||
logger.debug(f"Cache miss for {target_func.__name__}")
|
||||
result = await target_func(*args, **kwargs)
|
||||
|
||||
# Store result
|
||||
if shared_cache:
|
||||
_set_to_redis(redis_key, result)
|
||||
else:
|
||||
_set_to_memory(key, result)
|
||||
|
||||
return result
|
||||
|
||||
wrapper = async_wrapper
|
||||
|
||||
else:
|
||||
# Sync function with threading.Lock
|
||||
cache_lock = threading.Lock()
|
||||
|
||||
@wraps(target_func)
|
||||
def sync_wrapper(*args: P.args, **kwargs: P.kwargs):
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
redis_key = (
|
||||
_make_redis_key(key, target_func.__name__) if shared_cache else ""
|
||||
)
|
||||
|
||||
# Fast path: check cache without lock
|
||||
if shared_cache:
|
||||
result = _get_from_redis(redis_key)
|
||||
if result is not None:
|
||||
return result
|
||||
else:
|
||||
result = _get_from_memory(key)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Slow path: acquire lock for cache miss/expiry
|
||||
with cache_lock:
|
||||
# Double-check: another thread might have populated cache
|
||||
if shared_cache:
|
||||
result = _get_from_redis(redis_key)
|
||||
if result is not None:
|
||||
return result
|
||||
else:
|
||||
result = _get_from_memory(key)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Cache miss - execute function
|
||||
logger.debug(f"Cache miss for {target_func.__name__}")
|
||||
result = target_func(*args, **kwargs)
|
||||
|
||||
# Store result
|
||||
if shared_cache:
|
||||
_set_to_redis(redis_key, result)
|
||||
else:
|
||||
_set_to_memory(key, result)
|
||||
|
||||
return result
|
||||
|
||||
wrapper = sync_wrapper
|
||||
|
||||
# Add cache management methods
|
||||
def cache_clear(pattern: str | None = None) -> None:
|
||||
"""Clear cache entries. If pattern provided, clear matching entries."""
|
||||
if shared_cache:
|
||||
if pattern:
|
||||
# Clear entries matching pattern
|
||||
keys = list(
|
||||
redis.scan_iter(f"cache:{target_func.__name__}:{pattern}")
|
||||
)
|
||||
else:
|
||||
# Clear all cache keys
|
||||
keys = list(redis.scan_iter(f"cache:{target_func.__name__}:*"))
|
||||
|
||||
if keys:
|
||||
pipeline = redis.pipeline()
|
||||
for key in keys:
|
||||
pipeline.delete(key)
|
||||
pipeline.execute()
|
||||
else:
|
||||
if pattern:
|
||||
# For in-memory cache, pattern matching not supported
|
||||
logger.warning(
|
||||
"Pattern-based clearing not supported for in-memory cache"
|
||||
)
|
||||
else:
|
||||
cache_storage.clear()
|
||||
|
||||
def cache_info() -> dict[str, int | None]:
|
||||
if shared_cache:
|
||||
cache_keys = list(redis.scan_iter(f"cache:{target_func.__name__}:*"))
|
||||
return {
|
||||
"size": len(cache_keys),
|
||||
"maxsize": None, # Redis manages its own size
|
||||
"ttl_seconds": ttl_seconds,
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"size": len(cache_storage),
|
||||
"maxsize": maxsize,
|
||||
"ttl_seconds": ttl_seconds,
|
||||
}
|
||||
|
||||
def cache_delete(*args, **kwargs) -> bool:
|
||||
"""Delete a specific cache entry. Returns True if entry existed."""
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
if shared_cache:
|
||||
redis_key = _make_redis_key(key, target_func.__name__)
|
||||
if redis.exists(redis_key):
|
||||
redis.delete(redis_key)
|
||||
return True
|
||||
return False
|
||||
else:
|
||||
if key in cache_storage:
|
||||
del cache_storage[key]
|
||||
return True
|
||||
return False
|
||||
|
||||
setattr(wrapper, "cache_clear", cache_clear)
|
||||
setattr(wrapper, "cache_info", cache_info)
|
||||
setattr(wrapper, "cache_delete", cache_delete)
|
||||
|
||||
return cast(CachedFunction, wrapper)
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def thread_cached(func):
|
||||
"""
|
||||
Thread-local cache decorator for both sync and async functions.
|
||||
|
||||
Each thread gets its own cache, which is useful for request-scoped caching
|
||||
in web applications where you want to cache within a single request but
|
||||
not across requests.
|
||||
|
||||
Args:
|
||||
func: The function to cache
|
||||
|
||||
Returns:
|
||||
Decorated function with thread-local caching
|
||||
|
||||
Example:
|
||||
@thread_cached
|
||||
def expensive_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
|
||||
@thread_cached # Works with async too
|
||||
async def expensive_async_operation(param: str) -> dict:
|
||||
return {"result": param}
|
||||
"""
|
||||
thread_local = threading.local()
|
||||
|
||||
def _clear():
|
||||
if hasattr(thread_local, "cache"):
|
||||
del thread_local.cache
|
||||
|
||||
if inspect.iscoroutinefunction(func):
|
||||
|
||||
@wraps(func)
|
||||
async def async_wrapper(*args, **kwargs):
|
||||
cache = getattr(thread_local, "cache", None)
|
||||
if cache is None:
|
||||
cache = thread_local.cache = {}
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
if key not in cache:
|
||||
cache[key] = await func(*args, **kwargs)
|
||||
return cache[key]
|
||||
|
||||
setattr(async_wrapper, "clear_cache", _clear)
|
||||
return async_wrapper
|
||||
|
||||
else:
|
||||
|
||||
@wraps(func)
|
||||
def sync_wrapper(*args, **kwargs):
|
||||
cache = getattr(thread_local, "cache", None)
|
||||
if cache is None:
|
||||
cache = thread_local.cache = {}
|
||||
key = _make_hashable_key(args, kwargs)
|
||||
if key not in cache:
|
||||
cache[key] = func(*args, **kwargs)
|
||||
return cache[key]
|
||||
|
||||
setattr(sync_wrapper, "clear_cache", _clear)
|
||||
return sync_wrapper
|
||||
|
||||
|
||||
def clear_thread_cache(func: Callable) -> None:
|
||||
"""Clear thread-local cache for a function."""
|
||||
if clear := getattr(func, "clear_cache", None):
|
||||
clear()
|
||||
@@ -16,7 +16,7 @@ from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from autogpt_libs.utils.cache import cached, clear_thread_cache, thread_cached
|
||||
from backend.util.cache import cached, clear_thread_cache, thread_cached
|
||||
|
||||
|
||||
class TestThreadCached:
|
||||
@@ -332,7 +332,7 @@ class TestCache:
|
||||
"""Test basic sync caching functionality."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
def expensive_sync_function(x: int, y: int = 0) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -358,7 +358,7 @@ class TestCache:
|
||||
"""Test basic async caching functionality."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
async def expensive_async_function(x: int, y: int = 0) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -385,7 +385,7 @@ class TestCache:
|
||||
call_count = 0
|
||||
results = []
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
def slow_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -412,7 +412,7 @@ class TestCache:
|
||||
"""Test that concurrent async calls don't cause thundering herd."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
async def slow_async_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -508,7 +508,7 @@ class TestCache:
|
||||
"""Test cache clearing functionality."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
def clearable_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -537,7 +537,7 @@ class TestCache:
|
||||
"""Test cache clearing functionality with async function."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
async def async_clearable_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -567,7 +567,7 @@ class TestCache:
|
||||
"""Test that cached async functions return actual results, not coroutines."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
async def async_result_function(x: int) -> str:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -593,7 +593,7 @@ class TestCache:
|
||||
"""Test selective cache deletion functionality."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
def deletable_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -636,7 +636,7 @@ class TestCache:
|
||||
"""Test selective cache deletion functionality with async function."""
|
||||
call_count = 0
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=300)
|
||||
async def async_deletable_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
@@ -674,3 +674,450 @@ class TestCache:
|
||||
# Try to delete non-existent entry
|
||||
was_deleted = async_deletable_function.cache_delete(99)
|
||||
assert was_deleted is False
|
||||
|
||||
|
||||
class TestSharedCache:
|
||||
"""Tests for shared_cache (Redis-backed) functionality."""
|
||||
|
||||
def test_sync_shared_cache_basic(self):
|
||||
"""Test basic shared cache functionality with sync function."""
|
||||
call_count = 0
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
def shared_sync_function(x: int, y: int = 0) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return x + y
|
||||
|
||||
# Clear any existing cache
|
||||
shared_sync_function.cache_clear()
|
||||
|
||||
# First call
|
||||
result1 = shared_sync_function(10, 20)
|
||||
assert result1 == 30
|
||||
assert call_count == 1
|
||||
|
||||
# Second call - should use Redis cache
|
||||
result2 = shared_sync_function(10, 20)
|
||||
assert result2 == 30
|
||||
assert call_count == 1
|
||||
|
||||
# Different args - should call function again
|
||||
result3 = shared_sync_function(15, 25)
|
||||
assert result3 == 40
|
||||
assert call_count == 2
|
||||
|
||||
# Cleanup
|
||||
shared_sync_function.cache_clear()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_shared_cache_basic(self):
|
||||
"""Test basic shared cache functionality with async function."""
|
||||
call_count = 0
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
async def shared_async_function(x: int, y: int = 0) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
await asyncio.sleep(0.01)
|
||||
return x + y
|
||||
|
||||
# Clear any existing cache
|
||||
shared_async_function.cache_clear()
|
||||
|
||||
# First call
|
||||
result1 = await shared_async_function(10, 20)
|
||||
assert result1 == 30
|
||||
assert call_count == 1
|
||||
|
||||
# Second call - should use Redis cache
|
||||
result2 = await shared_async_function(10, 20)
|
||||
assert result2 == 30
|
||||
assert call_count == 1
|
||||
|
||||
# Different args - should call function again
|
||||
result3 = await shared_async_function(15, 25)
|
||||
assert result3 == 40
|
||||
assert call_count == 2
|
||||
|
||||
# Cleanup
|
||||
shared_async_function.cache_clear()
|
||||
|
||||
def test_shared_cache_ttl_refresh(self):
|
||||
"""Test TTL refresh functionality with shared cache."""
|
||||
call_count = 0
|
||||
|
||||
@cached(ttl_seconds=2, shared_cache=True, refresh_ttl_on_get=True)
|
||||
def ttl_refresh_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return x * 10
|
||||
|
||||
# Clear any existing cache
|
||||
ttl_refresh_function.cache_clear()
|
||||
|
||||
# First call
|
||||
result1 = ttl_refresh_function(3)
|
||||
assert result1 == 30
|
||||
assert call_count == 1
|
||||
|
||||
# Wait 1 second
|
||||
time.sleep(1)
|
||||
|
||||
# Second call - should refresh TTL and use cache
|
||||
result2 = ttl_refresh_function(3)
|
||||
assert result2 == 30
|
||||
assert call_count == 1
|
||||
|
||||
# Wait another 1.5 seconds (total 2.5s from first call, 1.5s from second)
|
||||
time.sleep(1.5)
|
||||
|
||||
# Third call - TTL should have been refreshed, so still cached
|
||||
result3 = ttl_refresh_function(3)
|
||||
assert result3 == 30
|
||||
assert call_count == 1
|
||||
|
||||
# Wait 2.1 seconds - now it should expire
|
||||
time.sleep(2.1)
|
||||
|
||||
# Fourth call - should call function again
|
||||
result4 = ttl_refresh_function(3)
|
||||
assert result4 == 30
|
||||
assert call_count == 2
|
||||
|
||||
# Cleanup
|
||||
ttl_refresh_function.cache_clear()
|
||||
|
||||
def test_shared_cache_without_ttl_refresh(self):
|
||||
"""Test that TTL doesn't refresh when refresh_ttl_on_get=False."""
|
||||
call_count = 0
|
||||
|
||||
@cached(ttl_seconds=2, shared_cache=True, refresh_ttl_on_get=False)
|
||||
def no_ttl_refresh_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return x * 10
|
||||
|
||||
# Clear any existing cache
|
||||
no_ttl_refresh_function.cache_clear()
|
||||
|
||||
# First call
|
||||
result1 = no_ttl_refresh_function(4)
|
||||
assert result1 == 40
|
||||
assert call_count == 1
|
||||
|
||||
# Wait 1 second
|
||||
time.sleep(1)
|
||||
|
||||
# Second call - should use cache but NOT refresh TTL
|
||||
result2 = no_ttl_refresh_function(4)
|
||||
assert result2 == 40
|
||||
assert call_count == 1
|
||||
|
||||
# Wait another 1.1 seconds (total 2.1s from first call)
|
||||
time.sleep(1.1)
|
||||
|
||||
# Third call - should have expired
|
||||
result3 = no_ttl_refresh_function(4)
|
||||
assert result3 == 40
|
||||
assert call_count == 2
|
||||
|
||||
# Cleanup
|
||||
no_ttl_refresh_function.cache_clear()
|
||||
|
||||
def test_shared_cache_complex_objects(self):
|
||||
"""Test caching complex objects with shared cache (pickle serialization)."""
|
||||
call_count = 0
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
def complex_object_function(x: int) -> dict:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return {
|
||||
"number": x,
|
||||
"squared": x**2,
|
||||
"nested": {"list": [1, 2, x], "tuple": (x, x * 2)},
|
||||
"string": f"value_{x}",
|
||||
}
|
||||
|
||||
# Clear any existing cache
|
||||
complex_object_function.cache_clear()
|
||||
|
||||
# First call
|
||||
result1 = complex_object_function(5)
|
||||
assert result1["number"] == 5
|
||||
assert result1["squared"] == 25
|
||||
assert result1["nested"]["list"] == [1, 2, 5]
|
||||
assert call_count == 1
|
||||
|
||||
# Second call - should use cache
|
||||
result2 = complex_object_function(5)
|
||||
assert result2 == result1
|
||||
assert call_count == 1
|
||||
|
||||
# Cleanup
|
||||
complex_object_function.cache_clear()
|
||||
|
||||
def test_shared_cache_info(self):
|
||||
"""Test cache_info for shared cache."""
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
def info_shared_function(x: int) -> int:
|
||||
return x * 2
|
||||
|
||||
# Clear any existing cache
|
||||
info_shared_function.cache_clear()
|
||||
|
||||
# Check initial info
|
||||
info = info_shared_function.cache_info()
|
||||
assert info["size"] == 0
|
||||
assert info["maxsize"] is None # Redis manages size
|
||||
assert info["ttl_seconds"] == 30
|
||||
|
||||
# Add some entries
|
||||
info_shared_function(1)
|
||||
info_shared_function(2)
|
||||
info_shared_function(3)
|
||||
|
||||
info = info_shared_function.cache_info()
|
||||
assert info["size"] == 3
|
||||
|
||||
# Cleanup
|
||||
info_shared_function.cache_clear()
|
||||
|
||||
def test_shared_cache_delete(self):
|
||||
"""Test selective deletion with shared cache."""
|
||||
call_count = 0
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
def delete_shared_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return x * 3
|
||||
|
||||
# Clear any existing cache
|
||||
delete_shared_function.cache_clear()
|
||||
|
||||
# Add entries
|
||||
delete_shared_function(1)
|
||||
delete_shared_function(2)
|
||||
delete_shared_function(3)
|
||||
assert call_count == 3
|
||||
|
||||
# Verify cached
|
||||
delete_shared_function(1)
|
||||
delete_shared_function(2)
|
||||
assert call_count == 3
|
||||
|
||||
# Delete specific entry
|
||||
was_deleted = delete_shared_function.cache_delete(2)
|
||||
assert was_deleted is True
|
||||
|
||||
# Entry for x=2 should be gone
|
||||
delete_shared_function(2)
|
||||
assert call_count == 4
|
||||
|
||||
# Others should still be cached
|
||||
delete_shared_function(1)
|
||||
delete_shared_function(3)
|
||||
assert call_count == 4
|
||||
|
||||
# Try to delete non-existent
|
||||
was_deleted = delete_shared_function.cache_delete(99)
|
||||
assert was_deleted is False
|
||||
|
||||
# Cleanup
|
||||
delete_shared_function.cache_clear()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_shared_cache_thundering_herd(self):
|
||||
"""Test that shared cache prevents thundering herd for async functions."""
|
||||
call_count = 0
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
async def shared_slow_function(x: int) -> int:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
await asyncio.sleep(0.1)
|
||||
return x * x
|
||||
|
||||
# Clear any existing cache
|
||||
shared_slow_function.cache_clear()
|
||||
|
||||
# Launch multiple concurrent tasks
|
||||
tasks = [shared_slow_function(8) for _ in range(10)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
# All should return same result
|
||||
assert all(r == 64 for r in results)
|
||||
# Only one should have executed
|
||||
assert call_count == 1
|
||||
|
||||
# Cleanup
|
||||
shared_slow_function.cache_clear()
|
||||
|
||||
def test_shared_cache_clear_pattern(self):
|
||||
"""Test pattern-based cache clearing (Redis feature)."""
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
def pattern_function(category: str, item: int) -> str:
|
||||
return f"{category}_{item}"
|
||||
|
||||
# Clear any existing cache
|
||||
pattern_function.cache_clear()
|
||||
|
||||
# Add various entries
|
||||
pattern_function("fruit", 1)
|
||||
pattern_function("fruit", 2)
|
||||
pattern_function("vegetable", 1)
|
||||
pattern_function("vegetable", 2)
|
||||
|
||||
info = pattern_function.cache_info()
|
||||
assert info["size"] == 4
|
||||
|
||||
# Note: Pattern clearing with wildcards requires specific Redis scan
|
||||
# implementation. The current code clears by pattern but needs
|
||||
# adjustment for partial matching. For now, test full clear.
|
||||
pattern_function.cache_clear()
|
||||
info = pattern_function.cache_info()
|
||||
assert info["size"] == 0
|
||||
|
||||
def test_shared_vs_local_cache_isolation(self):
|
||||
"""Test that shared and local caches are isolated."""
|
||||
shared_count = 0
|
||||
local_count = 0
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
def shared_function(x: int) -> int:
|
||||
nonlocal shared_count
|
||||
shared_count += 1
|
||||
return x * 2
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=False)
|
||||
def local_function(x: int) -> int:
|
||||
nonlocal local_count
|
||||
local_count += 1
|
||||
return x * 2
|
||||
|
||||
# Clear caches
|
||||
shared_function.cache_clear()
|
||||
local_function.cache_clear()
|
||||
|
||||
# Call both with same args
|
||||
shared_result = shared_function(5)
|
||||
local_result = local_function(5)
|
||||
|
||||
assert shared_result == local_result == 10
|
||||
assert shared_count == 1
|
||||
assert local_count == 1
|
||||
|
||||
# Call again - both should use their respective caches
|
||||
shared_function(5)
|
||||
local_function(5)
|
||||
assert shared_count == 1
|
||||
assert local_count == 1
|
||||
|
||||
# Clear only shared cache
|
||||
shared_function.cache_clear()
|
||||
|
||||
# Shared should recompute, local should still use cache
|
||||
shared_function(5)
|
||||
local_function(5)
|
||||
assert shared_count == 2
|
||||
assert local_count == 1
|
||||
|
||||
# Cleanup
|
||||
shared_function.cache_clear()
|
||||
local_function.cache_clear()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shared_cache_concurrent_different_keys(self):
|
||||
"""Test that concurrent calls with different keys work correctly."""
|
||||
call_counts = {}
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
async def multi_key_function(key: str) -> str:
|
||||
if key not in call_counts:
|
||||
call_counts[key] = 0
|
||||
call_counts[key] += 1
|
||||
await asyncio.sleep(0.05)
|
||||
return f"result_{key}"
|
||||
|
||||
# Clear cache
|
||||
multi_key_function.cache_clear()
|
||||
|
||||
# Launch concurrent tasks with different keys
|
||||
keys = ["a", "b", "c", "d", "e"]
|
||||
tasks = []
|
||||
for key in keys:
|
||||
# Multiple calls per key
|
||||
tasks.extend([multi_key_function(key) for _ in range(3)])
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
# Verify results
|
||||
for i, key in enumerate(keys):
|
||||
expected = f"result_{key}"
|
||||
# Each key appears 3 times in results
|
||||
key_results = results[i * 3 : (i + 1) * 3]
|
||||
assert all(r == expected for r in key_results)
|
||||
|
||||
# Each key should only be computed once
|
||||
for key in keys:
|
||||
assert call_counts[key] == 1
|
||||
|
||||
# Cleanup
|
||||
multi_key_function.cache_clear()
|
||||
|
||||
def test_shared_cache_performance_comparison(self):
|
||||
"""Compare performance of shared vs local cache."""
|
||||
import statistics
|
||||
|
||||
shared_times = []
|
||||
local_times = []
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=True)
|
||||
def shared_perf_function(x: int) -> int:
|
||||
time.sleep(0.01) # Simulate work
|
||||
return x * 2
|
||||
|
||||
@cached(ttl_seconds=30, shared_cache=False)
|
||||
def local_perf_function(x: int) -> int:
|
||||
time.sleep(0.01) # Simulate work
|
||||
return x * 2
|
||||
|
||||
# Clear caches
|
||||
shared_perf_function.cache_clear()
|
||||
local_perf_function.cache_clear()
|
||||
|
||||
# Warm up both caches
|
||||
for i in range(5):
|
||||
shared_perf_function(i)
|
||||
local_perf_function(i)
|
||||
|
||||
# Measure cache hit times
|
||||
for i in range(5):
|
||||
# Shared cache hit
|
||||
start = time.time()
|
||||
shared_perf_function(i)
|
||||
shared_times.append(time.time() - start)
|
||||
|
||||
# Local cache hit
|
||||
start = time.time()
|
||||
local_perf_function(i)
|
||||
local_times.append(time.time() - start)
|
||||
|
||||
# Local cache should be faster (no Redis round-trip)
|
||||
avg_shared = statistics.mean(shared_times)
|
||||
avg_local = statistics.mean(local_times)
|
||||
|
||||
print(f"Avg shared cache hit time: {avg_shared:.6f}s")
|
||||
print(f"Avg local cache hit time: {avg_local:.6f}s")
|
||||
|
||||
# Local should be significantly faster for cache hits
|
||||
# Redis adds network latency even for cache hits
|
||||
assert avg_local < avg_shared
|
||||
|
||||
# Cleanup
|
||||
shared_perf_function.cache_clear()
|
||||
local_perf_function.cache_clear()
|
||||
@@ -4,8 +4,7 @@ Centralized service client helpers with thread caching.
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from autogpt_libs.utils.cache import cached, thread_cached
|
||||
|
||||
from backend.util.cache import cached, thread_cached
|
||||
from backend.util.settings import Settings
|
||||
|
||||
settings = Settings()
|
||||
@@ -120,7 +119,7 @@ def get_integration_credentials_store() -> "IntegrationCredentialsStore":
|
||||
# ============ Supabase Clients ============ #
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
def get_supabase() -> "Client":
|
||||
"""Get a process-cached synchronous Supabase client instance."""
|
||||
from supabase import create_client
|
||||
@@ -130,7 +129,7 @@ def get_supabase() -> "Client":
|
||||
)
|
||||
|
||||
|
||||
@cached()
|
||||
@cached(ttl_seconds=3600)
|
||||
async def get_async_supabase() -> "AClient":
|
||||
"""Get a process-cached asynchronous Supabase client instance."""
|
||||
from supabase import create_async_client
|
||||
|
||||
@@ -5,12 +5,12 @@ from functools import wraps
|
||||
from typing import Any, Awaitable, Callable, TypeVar
|
||||
|
||||
import ldclient
|
||||
from autogpt_libs.utils.cache import cached
|
||||
from fastapi import HTTPException
|
||||
from ldclient import Context, LDClient
|
||||
from ldclient.config import Config
|
||||
from typing_extensions import ParamSpec
|
||||
|
||||
from backend.util.cache import cached
|
||||
from backend.util.settings import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -35,6 +35,7 @@ class Flag(str, Enum):
|
||||
AI_ACTIVITY_STATUS = "ai-agent-execution-summary"
|
||||
BETA_BLOCKS = "beta-blocks"
|
||||
AGENT_ACTIVITY = "agent-activity"
|
||||
ENABLE_PLATFORM_PAYMENT = "enable-platform-payment"
|
||||
|
||||
|
||||
def is_configured() -> bool:
|
||||
|
||||
@@ -1,35 +1,22 @@
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Type, TypeGuard, TypeVar, overload
|
||||
from typing import Any, Type, TypeVar, overload
|
||||
|
||||
import jsonschema
|
||||
import orjson
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.encoders import jsonable_encoder as to_dict
|
||||
from prisma import Json
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .truncate import truncate
|
||||
from .type import type_match
|
||||
|
||||
__all__ = [
|
||||
"json",
|
||||
"dumps",
|
||||
"loads",
|
||||
"validate_with_jsonschema",
|
||||
"SafeJson",
|
||||
"convert_pydantic_to_json",
|
||||
]
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Precompiled regex to remove PostgreSQL-incompatible control characters
|
||||
# Removes \u0000-\u0008, \u000B-\u000C, \u000E-\u001F, \u007F (keeps tab \u0009, newline \u000A, carriage return \u000D)
|
||||
POSTGRES_CONTROL_CHARS = re.compile(r"[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]")
|
||||
|
||||
|
||||
def to_dict(data) -> dict:
|
||||
if isinstance(data, BaseModel):
|
||||
data = data.model_dump()
|
||||
return jsonable_encoder(data)
|
||||
|
||||
|
||||
def dumps(
|
||||
data: Any, *args: Any, indent: int | None = None, option: int = 0, **kwargs: Any
|
||||
) -> str:
|
||||
@@ -118,64 +105,46 @@ def validate_with_jsonschema(
|
||||
return str(e)
|
||||
|
||||
|
||||
def is_list_of_basemodels(value: object) -> TypeGuard[list[BaseModel]]:
|
||||
return isinstance(value, list) and all(
|
||||
isinstance(item, BaseModel) for item in value
|
||||
)
|
||||
def _sanitize_string(value: str) -> str:
|
||||
"""Remove PostgreSQL-incompatible control characters from string."""
|
||||
return POSTGRES_CONTROL_CHARS.sub("", value)
|
||||
|
||||
|
||||
def convert_pydantic_to_json(output_data: Any) -> Any:
|
||||
if isinstance(output_data, BaseModel):
|
||||
return output_data.model_dump()
|
||||
if is_list_of_basemodels(output_data):
|
||||
return [item.model_dump() for item in output_data]
|
||||
return output_data
|
||||
def sanitize_json(data: Any) -> Any:
|
||||
try:
|
||||
# Use two-pass approach for consistent string sanitization:
|
||||
# 1. First convert to basic JSON-serializable types (handles Pydantic models)
|
||||
# 2. Then sanitize strings in the result
|
||||
basic_result = to_dict(data)
|
||||
return to_dict(basic_result, custom_encoder={str: _sanitize_string})
|
||||
except Exception as e:
|
||||
# Log the failure and fall back to string representation
|
||||
logger.error(
|
||||
"SafeJson fallback to string representation due to serialization error: %s (%s). "
|
||||
"Data type: %s, Data preview: %s",
|
||||
type(e).__name__,
|
||||
truncate(str(e), 200),
|
||||
type(data).__name__,
|
||||
truncate(str(data), 100),
|
||||
)
|
||||
|
||||
# Ultimate fallback: convert to string representation and sanitize
|
||||
return _sanitize_string(str(data))
|
||||
|
||||
|
||||
def _sanitize_value(value: Any) -> Any:
|
||||
"""
|
||||
Recursively sanitize values by removing PostgreSQL-incompatible control characters.
|
||||
|
||||
This function walks through data structures and removes control characters from strings.
|
||||
It handles:
|
||||
- Strings: Remove control chars directly from the string
|
||||
- Lists: Recursively sanitize each element
|
||||
- Dicts: Recursively sanitize keys and values
|
||||
- Other types: Return as-is
|
||||
|
||||
Args:
|
||||
value: The value to sanitize
|
||||
|
||||
Returns:
|
||||
Sanitized version of the value with control characters removed
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
# Remove control characters directly from the string
|
||||
return POSTGRES_CONTROL_CHARS.sub("", value)
|
||||
elif isinstance(value, dict):
|
||||
# Recursively sanitize dictionary keys and values
|
||||
return {_sanitize_value(k): _sanitize_value(v) for k, v in value.items()}
|
||||
elif isinstance(value, list):
|
||||
# Recursively sanitize list elements
|
||||
return [_sanitize_value(item) for item in value]
|
||||
elif isinstance(value, tuple):
|
||||
# Recursively sanitize tuple elements
|
||||
return tuple(_sanitize_value(item) for item in value)
|
||||
else:
|
||||
# For other types (int, float, bool, None, etc.), return as-is
|
||||
return value
|
||||
|
||||
|
||||
def SafeJson(data: Any) -> Json:
|
||||
class SafeJson(Json):
|
||||
"""
|
||||
Safely serialize data and return Prisma's Json type.
|
||||
Sanitizes control characters to prevent PostgreSQL 22P05 errors.
|
||||
|
||||
This function:
|
||||
1. Converts Pydantic models to dicts
|
||||
1. Converts Pydantic models to dicts (recursively using to_dict)
|
||||
2. Recursively removes PostgreSQL-incompatible control characters from strings
|
||||
3. Returns a Prisma Json object safe for database storage
|
||||
|
||||
Uses to_dict (jsonable_encoder) with a custom encoder to handle both Pydantic
|
||||
conversion and control character sanitization in a two-pass approach.
|
||||
|
||||
Args:
|
||||
data: Input data to sanitize and convert to Json
|
||||
|
||||
@@ -187,12 +156,6 @@ def SafeJson(data: Any) -> Json:
|
||||
>>> SafeJson({"path": "C:\\\\temp"}) # backslashes preserved
|
||||
>>> SafeJson({"data": "Text\\\\u0000here"}) # literal backslash-u preserved
|
||||
"""
|
||||
# Convert Pydantic models to dict first
|
||||
if isinstance(data, BaseModel):
|
||||
data = data.model_dump(exclude_none=True)
|
||||
|
||||
# Sanitize the data structure by removing control characters
|
||||
sanitized_data = _sanitize_value(data)
|
||||
|
||||
# Return as Prisma Json type
|
||||
return Json(sanitized_data)
|
||||
def __init__(self, data: Any):
|
||||
super().__init__(sanitize_json(data))
|
||||
|
||||
@@ -8,18 +8,9 @@ from typing import Optional
|
||||
|
||||
from backend.util.logging import configure_logging
|
||||
from backend.util.metrics import sentry_init
|
||||
from backend.util.settings import set_service_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_SERVICE_NAME = "MainProcess"
|
||||
|
||||
|
||||
def get_service_name():
|
||||
return _SERVICE_NAME
|
||||
|
||||
|
||||
def set_service_name(name: str):
|
||||
global _SERVICE_NAME
|
||||
_SERVICE_NAME = name
|
||||
|
||||
|
||||
class AppProcess(ABC):
|
||||
|
||||
@@ -13,7 +13,7 @@ import idna
|
||||
from aiohttp import FormData, abc
|
||||
from tenacity import retry, retry_if_result, wait_exponential_jitter
|
||||
|
||||
from backend.util.json import json
|
||||
from backend.util.json import loads
|
||||
|
||||
# Retry status codes for which we will automatically retry the request
|
||||
THROTTLE_RETRY_STATUS_CODES: set[int] = {429, 500, 502, 503, 504, 408}
|
||||
@@ -259,7 +259,7 @@ class Response:
|
||||
"""
|
||||
Parse the body as JSON and return the resulting Python object.
|
||||
"""
|
||||
return json.loads(
|
||||
return loads(
|
||||
self.content.decode(encoding or "utf-8", errors="replace"), **kwargs
|
||||
)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from tenacity import (
|
||||
wait_exponential_jitter,
|
||||
)
|
||||
|
||||
from backend.util.process import get_service_name
|
||||
from backend.util.settings import get_service_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -31,9 +31,9 @@ import backend.util.exceptions as exceptions
|
||||
from backend.monitoring.instrumentation import instrument_fastapi
|
||||
from backend.util.json import to_dict
|
||||
from backend.util.metrics import sentry_init
|
||||
from backend.util.process import AppProcess, get_service_name
|
||||
from backend.util.process import AppProcess
|
||||
from backend.util.retry import conn_retry, create_retry_decorator
|
||||
from backend.util.settings import Config
|
||||
from backend.util.settings import Config, get_service_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
T = TypeVar("T")
|
||||
|
||||
@@ -15,6 +15,17 @@ from backend.util.data import get_data_path
|
||||
|
||||
T = TypeVar("T", bound=BaseSettings)
|
||||
|
||||
_SERVICE_NAME = "MainProcess"
|
||||
|
||||
|
||||
def get_service_name():
|
||||
return _SERVICE_NAME
|
||||
|
||||
|
||||
def set_service_name(name: str):
|
||||
global _SERVICE_NAME
|
||||
_SERVICE_NAME = name
|
||||
|
||||
|
||||
class AppEnvironment(str, Enum):
|
||||
LOCAL = "local"
|
||||
@@ -254,6 +265,7 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
|
||||
default="localhost",
|
||||
description="The host for the RabbitMQ server",
|
||||
)
|
||||
|
||||
rabbitmq_port: int = Field(
|
||||
default=5672,
|
||||
description="The port for the RabbitMQ server",
|
||||
@@ -264,6 +276,21 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
|
||||
description="The vhost for the RabbitMQ server",
|
||||
)
|
||||
|
||||
redis_host: str = Field(
|
||||
default="localhost",
|
||||
description="The host for the Redis server",
|
||||
)
|
||||
|
||||
redis_port: int = Field(
|
||||
default=6379,
|
||||
description="The port for the Redis server",
|
||||
)
|
||||
|
||||
redis_password: str = Field(
|
||||
default="",
|
||||
description="The password for the Redis server (empty string if no password)",
|
||||
)
|
||||
|
||||
postmark_sender_email: str = Field(
|
||||
default="invalid@invalid.com",
|
||||
description="The email address to use for sending emails",
|
||||
|
||||
@@ -411,3 +411,346 @@ class TestSafeJson:
|
||||
assert "C:\\temp\\file" in str(file_path_with_null)
|
||||
assert ".txt" in str(file_path_with_null)
|
||||
assert "\x00" not in str(file_path_with_null) # Null removed from path
|
||||
|
||||
def test_invalid_escape_error_prevention(self):
|
||||
"""Test that SafeJson prevents 'Invalid \\escape' errors that occurred in upsert_execution_output."""
|
||||
# This reproduces the exact scenario that was causing the error:
|
||||
# POST /upsert_execution_output failed: Invalid \escape: line 1 column 36404 (char 36403)
|
||||
|
||||
# Create data with various problematic escape sequences that could cause JSON parsing errors
|
||||
problematic_output_data = {
|
||||
"web_content": "Article text\x00with null\x01and control\x08chars\x0C\x1F\x7F",
|
||||
"file_path": "C:\\Users\\test\\file\x00.txt",
|
||||
"json_like_string": '{"text": "data\x00\x08\x1F"}',
|
||||
"escaped_sequences": "Text with \\u0000 and \\u0008 sequences",
|
||||
"mixed_content": "Normal text\tproperly\nformatted\rwith\x00invalid\x08chars\x1Fmixed",
|
||||
"large_text": "A" * 35000
|
||||
+ "\x00\x08\x1F"
|
||||
+ "B" * 5000, # Large text like in the error
|
||||
}
|
||||
|
||||
# This should not raise any JSON parsing errors
|
||||
result = SafeJson(problematic_output_data)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
# Verify the result is a valid Json object that can be safely stored in PostgreSQL
|
||||
result_data = cast(dict[str, Any], result.data)
|
||||
assert isinstance(result_data, dict)
|
||||
|
||||
# Verify problematic characters are removed but safe content preserved
|
||||
web_content = result_data.get("web_content", "")
|
||||
file_path = result_data.get("file_path", "")
|
||||
large_text = result_data.get("large_text", "")
|
||||
|
||||
# Check that control characters are removed
|
||||
assert "\x00" not in str(web_content)
|
||||
assert "\x01" not in str(web_content)
|
||||
assert "\x08" not in str(web_content)
|
||||
assert "\x0C" not in str(web_content)
|
||||
assert "\x1F" not in str(web_content)
|
||||
assert "\x7F" not in str(web_content)
|
||||
|
||||
# Check that legitimate content is preserved
|
||||
assert "Article text" in str(web_content)
|
||||
assert "with null" in str(web_content)
|
||||
assert "and control" in str(web_content)
|
||||
assert "chars" in str(web_content)
|
||||
|
||||
# Check file path handling
|
||||
assert "C:\\Users\\test\\file" in str(file_path)
|
||||
assert ".txt" in str(file_path)
|
||||
assert "\x00" not in str(file_path)
|
||||
|
||||
# Check large text handling (the scenario from the error at char 36403)
|
||||
assert len(str(large_text)) > 35000 # Content preserved
|
||||
assert "A" * 1000 in str(large_text) # A's preserved
|
||||
assert "B" * 1000 in str(large_text) # B's preserved
|
||||
assert "\x00" not in str(large_text) # Control chars removed
|
||||
assert "\x08" not in str(large_text)
|
||||
assert "\x1F" not in str(large_text)
|
||||
|
||||
# Most importantly: ensure the result can be JSON-serialized without errors
|
||||
# This would have failed with the old approach
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data) # Should not raise "Invalid \escape"
|
||||
assert len(json_string) > 0
|
||||
|
||||
# And can be parsed back
|
||||
parsed_back = json.loads(json_string)
|
||||
assert isinstance(parsed_back, dict)
|
||||
|
||||
def test_dict_containing_pydantic_models(self):
|
||||
"""Test that dicts containing Pydantic models are properly serialized."""
|
||||
# This reproduces the bug from PR #11187 where credential_inputs failed
|
||||
model1 = SamplePydanticModel(name="Alice", age=30)
|
||||
model2 = SamplePydanticModel(name="Bob", age=25)
|
||||
|
||||
data = {
|
||||
"user1": model1,
|
||||
"user2": model2,
|
||||
"regular_data": "test",
|
||||
}
|
||||
|
||||
result = SafeJson(data)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
# Verify it can be JSON serialized (this was the bug)
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data)
|
||||
assert "Alice" in json_string
|
||||
assert "Bob" in json_string
|
||||
|
||||
def test_nested_pydantic_in_dict(self):
|
||||
"""Test deeply nested Pydantic models in dicts."""
|
||||
inner_model = SamplePydanticModel(name="Inner", age=20)
|
||||
middle_model = SamplePydanticModel(
|
||||
name="Middle", age=30, metadata={"inner": inner_model}
|
||||
)
|
||||
|
||||
data = {
|
||||
"level1": {
|
||||
"level2": {
|
||||
"model": middle_model,
|
||||
"other": "data",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = SafeJson(data)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data)
|
||||
assert "Middle" in json_string
|
||||
assert "Inner" in json_string
|
||||
|
||||
def test_list_containing_pydantic_models_in_dict(self):
|
||||
"""Test list of Pydantic models inside a dict."""
|
||||
models = [SamplePydanticModel(name=f"User{i}", age=20 + i) for i in range(5)]
|
||||
|
||||
data = {
|
||||
"users": models,
|
||||
"count": len(models),
|
||||
}
|
||||
|
||||
result = SafeJson(data)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data)
|
||||
assert "User0" in json_string
|
||||
assert "User4" in json_string
|
||||
|
||||
def test_credentials_meta_input_scenario(self):
|
||||
"""Test the exact scenario from create_graph_execution that was failing."""
|
||||
|
||||
# Simulate CredentialsMetaInput structure
|
||||
class MockCredentialsMetaInput(BaseModel):
|
||||
id: str
|
||||
title: Optional[str] = None
|
||||
provider: str
|
||||
type: str
|
||||
|
||||
cred_input = MockCredentialsMetaInput(
|
||||
id="test-123", title="Test Credentials", provider="github", type="oauth2"
|
||||
)
|
||||
|
||||
# This is how credential_inputs is structured in create_graph_execution
|
||||
credential_inputs = {"github_creds": cred_input}
|
||||
|
||||
# This should work without TypeError
|
||||
result = SafeJson(credential_inputs)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
# Verify it can be JSON serialized
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data)
|
||||
assert "test-123" in json_string
|
||||
assert "github" in json_string
|
||||
assert "oauth2" in json_string
|
||||
|
||||
def test_mixed_pydantic_and_primitives(self):
|
||||
"""Test complex mix of Pydantic models and primitive types."""
|
||||
model = SamplePydanticModel(name="Test", age=25)
|
||||
|
||||
data = {
|
||||
"models": [model, {"plain": "dict"}, "string", 123],
|
||||
"nested": {
|
||||
"model": model,
|
||||
"list": [1, 2, model, 4],
|
||||
"plain": "text",
|
||||
},
|
||||
"plain_list": [1, 2, 3],
|
||||
}
|
||||
|
||||
result = SafeJson(data)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data)
|
||||
assert "Test" in json_string
|
||||
assert "plain" in json_string
|
||||
|
||||
def test_pydantic_model_with_control_chars_in_dict(self):
|
||||
"""Test Pydantic model with control chars when nested in dict."""
|
||||
model = SamplePydanticModel(
|
||||
name="Test\x00User", # Has null byte
|
||||
age=30,
|
||||
metadata={"info": "data\x08with\x0Ccontrols"},
|
||||
)
|
||||
|
||||
data = {"credential": model}
|
||||
|
||||
result = SafeJson(data)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
# Verify control characters are removed
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data)
|
||||
assert "\x00" not in json_string
|
||||
assert "\x08" not in json_string
|
||||
assert "\x0C" not in json_string
|
||||
assert "TestUser" in json_string # Name preserved minus null byte
|
||||
|
||||
def test_deeply_nested_pydantic_models_control_char_sanitization(self):
|
||||
"""Test that control characters are sanitized in deeply nested Pydantic models."""
|
||||
|
||||
# Create nested Pydantic models with control characters at different levels
|
||||
class InnerModel(BaseModel):
|
||||
deep_string: str
|
||||
value: int = 42
|
||||
metadata: dict = {}
|
||||
|
||||
class MiddleModel(BaseModel):
|
||||
middle_string: str
|
||||
inner: InnerModel
|
||||
data: str
|
||||
|
||||
class OuterModel(BaseModel):
|
||||
outer_string: str
|
||||
middle: MiddleModel
|
||||
|
||||
# Create test data with control characters at every nesting level
|
||||
inner = InnerModel(
|
||||
deep_string="Deepest\x00Level\x08Control\x0CChars", # Multiple control chars at deepest level
|
||||
metadata={
|
||||
"nested_key": "Nested\x1FValue\x7FDelete"
|
||||
}, # Control chars in nested dict
|
||||
)
|
||||
|
||||
middle = MiddleModel(
|
||||
middle_string="Middle\x01StartOfHeading\x1FUnitSeparator",
|
||||
inner=inner,
|
||||
data="Some\x0BVerticalTab\x0EShiftOut",
|
||||
)
|
||||
|
||||
outer = OuterModel(outer_string="Outer\x00Null\x07Bell", middle=middle)
|
||||
|
||||
# Wrap in a dict with additional control characters
|
||||
data = {
|
||||
"top_level": "Top\x00Level\x08Backspace",
|
||||
"nested_model": outer,
|
||||
"list_with_strings": [
|
||||
"List\x00Item1",
|
||||
"List\x0CItem2\x1F",
|
||||
{"dict_in_list": "Dict\x08Value"},
|
||||
],
|
||||
}
|
||||
|
||||
# Process with SafeJson
|
||||
result = SafeJson(data)
|
||||
assert isinstance(result, Json)
|
||||
|
||||
# Verify all control characters are removed at every level
|
||||
import json
|
||||
|
||||
json_string = json.dumps(result.data)
|
||||
|
||||
# Check that NO control characters remain anywhere
|
||||
control_chars = [
|
||||
"\x00",
|
||||
"\x01",
|
||||
"\x02",
|
||||
"\x03",
|
||||
"\x04",
|
||||
"\x05",
|
||||
"\x06",
|
||||
"\x07",
|
||||
"\x08",
|
||||
"\x0B",
|
||||
"\x0C",
|
||||
"\x0E",
|
||||
"\x0F",
|
||||
"\x10",
|
||||
"\x11",
|
||||
"\x12",
|
||||
"\x13",
|
||||
"\x14",
|
||||
"\x15",
|
||||
"\x16",
|
||||
"\x17",
|
||||
"\x18",
|
||||
"\x19",
|
||||
"\x1A",
|
||||
"\x1B",
|
||||
"\x1C",
|
||||
"\x1D",
|
||||
"\x1E",
|
||||
"\x1F",
|
||||
"\x7F",
|
||||
]
|
||||
|
||||
for char in control_chars:
|
||||
assert (
|
||||
char not in json_string
|
||||
), f"Control character {repr(char)} found in result"
|
||||
|
||||
# Verify specific sanitized content is present (control chars removed but text preserved)
|
||||
result_data = cast(dict[str, Any], result.data)
|
||||
|
||||
# Top level
|
||||
assert "TopLevelBackspace" in json_string
|
||||
|
||||
# Outer model level
|
||||
assert "OuterNullBell" in json_string
|
||||
|
||||
# Middle model level
|
||||
assert "MiddleStartOfHeadingUnitSeparator" in json_string
|
||||
assert "SomeVerticalTabShiftOut" in json_string
|
||||
|
||||
# Inner model level (deepest nesting)
|
||||
assert "DeepestLevelControlChars" in json_string
|
||||
|
||||
# Nested dict in model
|
||||
assert "NestedValueDelete" in json_string
|
||||
|
||||
# List items
|
||||
assert "ListItem1" in json_string
|
||||
assert "ListItem2" in json_string
|
||||
assert "DictValue" in json_string
|
||||
|
||||
# Verify structure is preserved (not just converted to string)
|
||||
assert isinstance(result_data, dict)
|
||||
assert isinstance(result_data["nested_model"], dict)
|
||||
assert isinstance(result_data["nested_model"]["middle"], dict)
|
||||
assert isinstance(result_data["nested_model"]["middle"]["inner"], dict)
|
||||
assert isinstance(result_data["list_with_strings"], list)
|
||||
|
||||
# Verify specific deep values are accessible and sanitized
|
||||
nested_model = cast(dict[str, Any], result_data["nested_model"])
|
||||
middle = cast(dict[str, Any], nested_model["middle"])
|
||||
inner = cast(dict[str, Any], middle["inner"])
|
||||
|
||||
deep_string = inner["deep_string"]
|
||||
assert deep_string == "DeepestLevelControlChars"
|
||||
|
||||
metadata = cast(dict[str, Any], inner["metadata"])
|
||||
nested_metadata = metadata["nested_key"]
|
||||
assert nested_metadata == "NestedValueDelete"
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Create UserBalance table for atomic credit operations
|
||||
-- This replaces the need for User.balance column and provides better separation of concerns
|
||||
-- UserBalance records are automatically created by the application when users interact with the credit system
|
||||
|
||||
-- CreateTable (only if it doesn't exist)
|
||||
CREATE TABLE IF NOT EXISTS "UserBalance" (
|
||||
"userId" TEXT NOT NULL,
|
||||
"balance" INTEGER NOT NULL DEFAULT 0,
|
||||
"updatedAt" TIMESTAMP(3) NOT NULL,
|
||||
|
||||
CONSTRAINT "UserBalance_pkey" PRIMARY KEY ("userId"),
|
||||
CONSTRAINT "UserBalance_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
-- CreateIndex (only if it doesn't exist)
|
||||
CREATE INDEX IF NOT EXISTS "UserBalance_userId_idx" ON "UserBalance"("userId");
|
||||
@@ -45,6 +45,7 @@ model User {
|
||||
AnalyticsDetails AnalyticsDetails[]
|
||||
AnalyticsMetrics AnalyticsMetrics[]
|
||||
CreditTransactions CreditTransaction[]
|
||||
UserBalance UserBalance?
|
||||
|
||||
AgentPresets AgentPreset[]
|
||||
LibraryAgents LibraryAgent[]
|
||||
@@ -887,6 +888,16 @@ model APIKey {
|
||||
@@index([userId, status])
|
||||
}
|
||||
|
||||
model UserBalance {
|
||||
userId String @id
|
||||
balance Int @default(0)
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([userId])
|
||||
}
|
||||
|
||||
enum APIKeyStatus {
|
||||
ACTIVE
|
||||
REVOKED
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"email": "test@example.com",
|
||||
"id": "test-user-id",
|
||||
"id": "3e53486c-cf57-477e-ba2a-cb02dc828e1a",
|
||||
"name": "Test User"
|
||||
}
|
||||
@@ -28,6 +28,6 @@
|
||||
"recommended_schedule_cron": null,
|
||||
"sub_graphs": [],
|
||||
"trigger_setup_info": null,
|
||||
"user_id": "test-user-id",
|
||||
"user_id": "3e53486c-cf57-477e-ba2a-cb02dc828e1a",
|
||||
"version": 1
|
||||
}
|
||||
@@ -26,7 +26,7 @@
|
||||
"recommended_schedule_cron": null,
|
||||
"sub_graphs": [],
|
||||
"trigger_setup_info": null,
|
||||
"user_id": "test-user-id",
|
||||
"user_id": "3e53486c-cf57-477e-ba2a-cb02dc828e1a",
|
||||
"version": 1
|
||||
}
|
||||
]
|
||||
@@ -749,10 +749,11 @@ class TestDataCreator:
|
||||
"""Add credits to users."""
|
||||
print("Adding credits to users...")
|
||||
|
||||
credit_model = get_user_credit_model()
|
||||
|
||||
for user in self.users:
|
||||
try:
|
||||
# Get user-specific credit model
|
||||
credit_model = await get_user_credit_model(user["id"])
|
||||
|
||||
# Skip credits for disabled credit model to avoid errors
|
||||
if (
|
||||
hasattr(credit_model, "__class__")
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
NEXT_PUBLIC_LAUNCHDARKLY_ENABLED=false
|
||||
NEXT_PUBLIC_LAUNCHDARKLY_CLIENT_ID=687ab1372f497809b131e06e
|
||||
|
||||
NEXT_PUBLIC_SHOW_BILLING_PAGE=false
|
||||
NEXT_PUBLIC_TURNSTILE=disabled
|
||||
NEXT_PUBLIC_REACT_QUERY_DEVTOOL=true
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import WalletRefill from "./components/WalletRefill";
|
||||
import { OnboardingStep } from "@/lib/autogpt-server-api";
|
||||
import { storage, Key as StorageKey } from "@/services/storage/local-storage";
|
||||
import { WalletIcon } from "@phosphor-icons/react";
|
||||
import { useGetFlag, Flag } from "@/services/feature-flags/use-get-flag";
|
||||
|
||||
export interface Task {
|
||||
id: OnboardingStep;
|
||||
@@ -40,6 +41,7 @@ export interface TaskGroup {
|
||||
|
||||
export default function Wallet() {
|
||||
const { state, updateState } = useOnboarding();
|
||||
const isPaymentEnabled = useGetFlag(Flag.ENABLE_PLATFORM_PAYMENT);
|
||||
|
||||
const groups = useMemo<TaskGroup[]>(() => {
|
||||
return [
|
||||
@@ -379,9 +381,7 @@ export default function Wallet() {
|
||||
</div>
|
||||
<ScrollArea className="max-h-[85vh] overflow-y-auto">
|
||||
{/* Top ups */}
|
||||
{process.env.NEXT_PUBLIC_SHOW_BILLING_PAGE === "true" && (
|
||||
<WalletRefill />
|
||||
)}
|
||||
{isPaymentEnabled && <WalletRefill />}
|
||||
{/* Tasks */}
|
||||
<p className="mx-1 my-3 font-sans text-xs font-normal text-zinc-400">
|
||||
Complete the following tasks to earn more credits!
|
||||
|
||||
@@ -2,11 +2,18 @@
|
||||
|
||||
import { isServerSide } from "@/lib/utils/is-server-side";
|
||||
import { useEffect, useState } from "react";
|
||||
import { Button } from "@/components/atoms/Button/Button";
|
||||
import { Text } from "@/components/atoms/Text/Text";
|
||||
import { Card } from "@/components/atoms/Card/Card";
|
||||
import { WaitlistErrorContent } from "@/components/auth/WaitlistErrorContent";
|
||||
import { isWaitlistError } from "@/app/api/auth/utils";
|
||||
import { useRouter } from "next/navigation";
|
||||
|
||||
export default function AuthErrorPage() {
|
||||
const [errorType, setErrorType] = useState<string | null>(null);
|
||||
const [errorCode, setErrorCode] = useState<string | null>(null);
|
||||
const [errorDescription, setErrorDescription] = useState<string | null>(null);
|
||||
const router = useRouter();
|
||||
|
||||
useEffect(() => {
|
||||
// This code only runs on the client side
|
||||
@@ -23,15 +30,57 @@ export default function AuthErrorPage() {
|
||||
}, []);
|
||||
|
||||
if (!errorType && !errorCode && !errorDescription) {
|
||||
return <div>Loading...</div>;
|
||||
return (
|
||||
<div className="flex h-screen items-center justify-center">
|
||||
<Text variant="body">Loading...</Text>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Check if this is a waitlist/not allowed error
|
||||
const isWaitlistErr = isWaitlistError(errorCode, errorDescription);
|
||||
|
||||
if (isWaitlistErr) {
|
||||
return (
|
||||
<div className="flex h-screen items-center justify-center">
|
||||
<Card className="w-full max-w-md p-8">
|
||||
<WaitlistErrorContent
|
||||
onClose={() => router.push("/login")}
|
||||
closeButtonText="Back to Login"
|
||||
/>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Default error display for other types of errors
|
||||
return (
|
||||
<div>
|
||||
<h1>Authentication Error</h1>
|
||||
{errorType && <p>Error Type: {errorType}</p>}
|
||||
{errorCode && <p>Error Code: {errorCode}</p>}
|
||||
{errorDescription && <p>Error Description: {errorDescription}</p>}
|
||||
<div className="flex h-screen items-center justify-center">
|
||||
<Card className="w-full max-w-md p-8">
|
||||
<div className="flex flex-col items-center gap-6">
|
||||
<Text variant="h3">Authentication Error</Text>
|
||||
<div className="flex flex-col gap-2 text-center">
|
||||
{errorType && (
|
||||
<Text variant="body">
|
||||
<strong>Error Type:</strong> {errorType}
|
||||
</Text>
|
||||
)}
|
||||
{errorCode && (
|
||||
<Text variant="body">
|
||||
<strong>Error Code:</strong> {errorCode}
|
||||
</Text>
|
||||
)}
|
||||
{errorDescription && (
|
||||
<Text variant="body">
|
||||
<strong>Description:</strong> {errorDescription}
|
||||
</Text>
|
||||
)}
|
||||
</div>
|
||||
<Button variant="primary" onClick={() => router.push("/login")}>
|
||||
Back to Login
|
||||
</Button>
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -20,6 +20,11 @@ export async function GET(request: Request) {
|
||||
|
||||
const { error } = await supabase.auth.exchangeCodeForSession(code);
|
||||
|
||||
// Keep minimal error logging for OAuth debugging if needed
|
||||
if (error) {
|
||||
console.error("OAuth code exchange failed:", error.message);
|
||||
}
|
||||
|
||||
if (!error) {
|
||||
try {
|
||||
const api = new BackendAPI();
|
||||
|
||||
@@ -5,7 +5,7 @@ import { useFlow } from "./useFlow";
|
||||
import { useShallow } from "zustand/react/shallow";
|
||||
import { useNodeStore } from "../../../stores/nodeStore";
|
||||
import { useMemo } from "react";
|
||||
import { CustomNode } from "../nodes/CustomNode";
|
||||
import { CustomNode } from "../nodes/CustomNode/CustomNode";
|
||||
import { useCustomEdge } from "../edges/useCustomEdge";
|
||||
import { GraphLoadingBox } from "./GraphLoadingBox";
|
||||
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
import React from "react";
|
||||
import { Node as XYNode, NodeProps } from "@xyflow/react";
|
||||
import { FormCreator } from "./FormCreator";
|
||||
import { RJSFSchema } from "@rjsf/utils";
|
||||
import { Text } from "@/components/atoms/Text/Text";
|
||||
|
||||
import { Switch } from "@/components/atoms/Switch/Switch";
|
||||
import { preprocessInputSchema } from "../processors/input-schema-pre-processor";
|
||||
import { OutputHandler } from "./OutputHandler";
|
||||
import { useNodeStore } from "../../../stores/nodeStore";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { BlockUIType } from "../../types";
|
||||
import { StickyNoteBlock } from "./StickyNoteBlock";
|
||||
|
||||
export type CustomNodeData = {
|
||||
hardcodedValues: {
|
||||
[key: string]: any;
|
||||
};
|
||||
title: string;
|
||||
block_id: string;
|
||||
description: string;
|
||||
inputSchema: RJSFSchema;
|
||||
outputSchema: RJSFSchema;
|
||||
uiType: BlockUIType;
|
||||
};
|
||||
|
||||
export type CustomNode = XYNode<CustomNodeData, "custom">;
|
||||
|
||||
export const CustomNode: React.FC<NodeProps<CustomNode>> = React.memo(
|
||||
({ data, id: nodeId, selected }) => {
|
||||
const showAdvanced = useNodeStore(
|
||||
(state) => state.nodeAdvancedStates[nodeId] || false,
|
||||
);
|
||||
const setShowAdvanced = useNodeStore((state) => state.setShowAdvanced);
|
||||
|
||||
if (data.uiType === BlockUIType.NOTE) {
|
||||
return <StickyNoteBlock selected={selected} data={data} id={nodeId} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"z-12 rounded-xl bg-gradient-to-br from-white to-slate-50/30 shadow-lg shadow-slate-900/5 ring-1 ring-slate-200/60 backdrop-blur-sm",
|
||||
selected && "shadow-2xl ring-2 ring-slate-200",
|
||||
)}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex h-14 items-center justify-center rounded-xl border-b border-slate-200/50 bg-gradient-to-r from-slate-50/80 to-white/90">
|
||||
<Text
|
||||
variant="large-semibold"
|
||||
className="tracking-tight text-slate-800"
|
||||
>
|
||||
{data.title}
|
||||
</Text>
|
||||
</div>
|
||||
|
||||
{/* Input Handles */}
|
||||
<div className="bg-white/40 pb-6 pr-6">
|
||||
<FormCreator
|
||||
jsonSchema={preprocessInputSchema(data.inputSchema)}
|
||||
nodeId={nodeId}
|
||||
uiType={data.uiType}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Advanced Button */}
|
||||
<div className="flex items-center justify-between gap-2 rounded-b-xl border-t border-slate-200/50 bg-gradient-to-r from-slate-50/60 to-white/80 px-5 py-3.5">
|
||||
<Text variant="body" className="font-medium text-slate-700">
|
||||
Advanced
|
||||
</Text>
|
||||
<Switch
|
||||
onCheckedChange={(checked) => setShowAdvanced(nodeId, checked)}
|
||||
checked={showAdvanced}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Output Handles */}
|
||||
<OutputHandler outputSchema={data.outputSchema} nodeId={nodeId} />
|
||||
</div>
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
CustomNode.displayName = "CustomNode";
|
||||
@@ -0,0 +1,45 @@
|
||||
import React from "react";
|
||||
import { Node as XYNode, NodeProps } from "@xyflow/react";
|
||||
import { RJSFSchema } from "@rjsf/utils";
|
||||
import { BlockUIType } from "../../../types";
|
||||
import { StickyNoteBlock } from "./StickyNoteBlock";
|
||||
import { BlockInfoCategoriesItem } from "@/app/api/__generated__/models/blockInfoCategoriesItem";
|
||||
import { StandardNodeBlock } from "./StandardNodeBlock";
|
||||
import { BlockCost } from "@/app/api/__generated__/models/blockCost";
|
||||
|
||||
export type CustomNodeData = {
|
||||
hardcodedValues: {
|
||||
[key: string]: any;
|
||||
};
|
||||
title: string;
|
||||
description: string;
|
||||
inputSchema: RJSFSchema;
|
||||
outputSchema: RJSFSchema;
|
||||
uiType: BlockUIType;
|
||||
block_id: string;
|
||||
// TODO : We need better type safety for the following backend fields.
|
||||
costs: BlockCost[];
|
||||
categories: BlockInfoCategoriesItem[];
|
||||
};
|
||||
|
||||
export type CustomNode = XYNode<CustomNodeData, "custom">;
|
||||
|
||||
export const CustomNode: React.FC<NodeProps<CustomNode>> = React.memo(
|
||||
({ data, id: nodeId, selected }) => {
|
||||
if (data.uiType === BlockUIType.NOTE) {
|
||||
return <StickyNoteBlock selected={selected} data={data} id={nodeId} />;
|
||||
}
|
||||
|
||||
if (data.uiType === BlockUIType.STANDARD) {
|
||||
return (
|
||||
<StandardNodeBlock data={data} selected={selected} nodeId={nodeId} />
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<StandardNodeBlock data={data} selected={selected} nodeId={nodeId} />
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
CustomNode.displayName = "CustomNode";
|
||||
@@ -0,0 +1,79 @@
|
||||
import { beautifyString, cn } from "@/lib/utils";
|
||||
import { CustomNodeData } from "./CustomNode";
|
||||
import { Text } from "@/components/atoms/Text/Text";
|
||||
import { FormCreator } from "../FormCreator";
|
||||
import { preprocessInputSchema } from "../../processors/input-schema-pre-processor";
|
||||
import { Switch } from "@/components/atoms/Switch/Switch";
|
||||
import { useNodeStore } from "@/app/(platform)/build/stores/nodeStore";
|
||||
import { OutputHandler } from "../OutputHandler";
|
||||
import { NodeCost } from "./components/NodeCost";
|
||||
import { NodeBadges } from "./components/NodeBadges";
|
||||
|
||||
type StandardNodeBlockType = {
|
||||
data: CustomNodeData;
|
||||
selected: boolean;
|
||||
nodeId: string;
|
||||
};
|
||||
export const StandardNodeBlock = ({
|
||||
data,
|
||||
selected,
|
||||
nodeId,
|
||||
}: StandardNodeBlockType) => {
|
||||
const showAdvanced = useNodeStore(
|
||||
(state) => state.nodeAdvancedStates[nodeId] || false,
|
||||
);
|
||||
const setShowAdvanced = useNodeStore((state) => state.setShowAdvanced);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"z-12 rounded-xl bg-gradient-to-br from-white to-slate-50/30 shadow-lg shadow-slate-900/5 ring-1 ring-slate-200/60 backdrop-blur-sm",
|
||||
selected && "shadow-2xl ring-2 ring-slate-200",
|
||||
)}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex h-auto flex-col gap-2 rounded-xl border-b border-slate-200/50 bg-gradient-to-r from-slate-50/80 to-white/90 px-4 py-4">
|
||||
{/* Upper section */}
|
||||
<div className="flex items-center gap-2">
|
||||
<Text
|
||||
variant="large-semibold"
|
||||
className="tracking-tight text-slate-800"
|
||||
>
|
||||
{beautifyString(data.title)}
|
||||
</Text>
|
||||
<Text variant="small" className="!font-medium !text-slate-500">
|
||||
#{nodeId.split("-")[0]}
|
||||
</Text>
|
||||
</div>
|
||||
{/* Lower section */}
|
||||
<div className="flex space-x-2">
|
||||
<NodeCost blockCosts={data.costs} nodeId={nodeId} />
|
||||
<NodeBadges categories={data.categories} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Input Handles */}
|
||||
<div className="bg-white/40 pb-6 pr-6">
|
||||
<FormCreator
|
||||
jsonSchema={preprocessInputSchema(data.inputSchema)}
|
||||
nodeId={nodeId}
|
||||
uiType={data.uiType}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Advanced Button */}
|
||||
<div className="flex items-center justify-between gap-2 rounded-b-xl border-t border-slate-200/50 bg-gradient-to-r from-slate-50/60 to-white/80 px-5 py-3.5">
|
||||
<Text variant="body" className="font-medium text-slate-700">
|
||||
Advanced
|
||||
</Text>
|
||||
<Switch
|
||||
onCheckedChange={(checked) => setShowAdvanced(nodeId, checked)}
|
||||
checked={showAdvanced}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Output Handles */}
|
||||
<OutputHandler outputSchema={data.outputSchema} nodeId={nodeId} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useMemo } from "react";
|
||||
import { FormCreator } from "./FormCreator";
|
||||
import { preprocessInputSchema } from "../processors/input-schema-pre-processor";
|
||||
import { FormCreator } from "../FormCreator";
|
||||
import { preprocessInputSchema } from "../../processors/input-schema-pre-processor";
|
||||
import { CustomNodeData } from "./CustomNode";
|
||||
import { Text } from "@/components/atoms/Text/Text";
|
||||
import { cn } from "@/lib/utils";
|
||||
@@ -0,0 +1,20 @@
|
||||
import { BlockInfoCategoriesItem } from "@/app/api/__generated__/models/blockInfoCategoriesItem";
|
||||
import { Badge } from "@/components/__legacy__/ui/badge";
|
||||
import { beautifyString, cn } from "@/lib/utils";
|
||||
|
||||
export const NodeBadges = ({
|
||||
categories,
|
||||
}: {
|
||||
categories: BlockInfoCategoriesItem[];
|
||||
}) => {
|
||||
return categories.map((category) => (
|
||||
<Badge
|
||||
key={category.category}
|
||||
className={cn(
|
||||
"rounded-full border border-slate-500 bg-slate-100 text-black shadow-none",
|
||||
)}
|
||||
>
|
||||
{beautifyString(category.category.toLowerCase())}
|
||||
</Badge>
|
||||
));
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
import { BlockCost } from "@/app/api/__generated__/models/blockCost";
|
||||
import { Text } from "@/components/atoms/Text/Text";
|
||||
import useCredits from "@/hooks/useCredits";
|
||||
import { CoinIcon } from "@phosphor-icons/react";
|
||||
import { isCostFilterMatch } from "../../../../helper";
|
||||
import { useNodeStore } from "@/app/(platform)/build/stores/nodeStore";
|
||||
|
||||
export const NodeCost = ({
|
||||
blockCosts,
|
||||
nodeId,
|
||||
}: {
|
||||
blockCosts: BlockCost[];
|
||||
nodeId: string;
|
||||
}) => {
|
||||
const { formatCredits } = useCredits();
|
||||
const hardcodedValues = useNodeStore((state) =>
|
||||
state.getHardCodedValues(nodeId),
|
||||
);
|
||||
const blockCost =
|
||||
blockCosts &&
|
||||
blockCosts.find((cost) =>
|
||||
isCostFilterMatch(cost.cost_filter, hardcodedValues),
|
||||
);
|
||||
|
||||
if (!blockCost) return null;
|
||||
|
||||
return (
|
||||
<div className="mr-3 flex items-center gap-1 text-base font-light">
|
||||
<CoinIcon className="h-3 w-3" />
|
||||
<Text variant="small" className="!font-medium">
|
||||
{formatCredits(blockCost.cost_amount)}
|
||||
</Text>
|
||||
<Text variant="small">
|
||||
{" \/"}
|
||||
{blockCost.cost_type}
|
||||
</Text>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -8,11 +8,11 @@ import { GraphExecutionID } from "@/lib/autogpt-server-api";
|
||||
// import { ControlPanelButton } from "../ControlPanelButton";
|
||||
import { ArrowUUpLeftIcon, ArrowUUpRightIcon } from "@phosphor-icons/react";
|
||||
// import { GraphSearchMenu } from "../GraphMenu/GraphMenu";
|
||||
import { CustomNode } from "../FlowEditor/nodes/CustomNode";
|
||||
import { history } from "@/app/(platform)/build/components/legacy-builder/history";
|
||||
import { Flag, useGetFlag } from "@/services/feature-flags/use-get-flag";
|
||||
import { Separator } from "@/components/__legacy__/ui/separator";
|
||||
import { NewSaveControl } from "./NewSaveControl/NewSaveControl";
|
||||
import { CustomNode } from "../FlowEditor/nodes/CustomNode/CustomNode";
|
||||
|
||||
export type Control = {
|
||||
icon: React.ReactNode;
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { BlockInfo } from "@/app/api/__generated__/models/blockInfo";
|
||||
import { CustomNode, CustomNodeData } from "./FlowEditor/nodes/CustomNode";
|
||||
import {
|
||||
CustomNode,
|
||||
CustomNodeData,
|
||||
} from "./FlowEditor/nodes/CustomNode/CustomNode";
|
||||
import { BlockUIType } from "./types";
|
||||
import { NodeModel } from "@/app/api/__generated__/models/nodeModel";
|
||||
import { NodeModelMetadata } from "@/app/api/__generated__/models/nodeModelMetadata";
|
||||
@@ -14,8 +17,10 @@ export const convertBlockInfoIntoCustomNodeData = (
|
||||
description: block.description,
|
||||
inputSchema: block.inputSchema,
|
||||
outputSchema: block.outputSchema,
|
||||
categories: block.categories,
|
||||
uiType: block.uiType as BlockUIType,
|
||||
block_id: block.id,
|
||||
costs: block.costs,
|
||||
};
|
||||
return customNodeData;
|
||||
};
|
||||
@@ -51,3 +56,38 @@ export const convertNodesPlusBlockInfoIntoCustomNodes = (
|
||||
};
|
||||
return customNode;
|
||||
};
|
||||
|
||||
export enum BlockCategory {
|
||||
AI = "AI",
|
||||
SOCIAL = "SOCIAL",
|
||||
TEXT = "TEXT",
|
||||
SEARCH = "SEARCH",
|
||||
BASIC = "BASIC",
|
||||
INPUT = "INPUT",
|
||||
OUTPUT = "OUTPUT",
|
||||
LOGIC = "LOGIC",
|
||||
COMMUNICATION = "COMMUNICATION",
|
||||
DEVELOPER_TOOLS = "DEVELOPER_TOOLS",
|
||||
DATA = "DATA",
|
||||
HARDWARE = "HARDWARE",
|
||||
AGENT = "AGENT",
|
||||
CRM = "CRM",
|
||||
SAFETY = "SAFETY",
|
||||
PRODUCTIVITY = "PRODUCTIVITY",
|
||||
ISSUE_TRACKING = "ISSUE_TRACKING",
|
||||
MULTIMEDIA = "MULTIMEDIA",
|
||||
MARKETING = "MARKETING",
|
||||
}
|
||||
|
||||
// Cost related helpers
|
||||
export const isCostFilterMatch = (
|
||||
costFilter: any,
|
||||
inputValues: any,
|
||||
): boolean => {
|
||||
return typeof costFilter === "object" && typeof inputValues === "object"
|
||||
? Object.entries(costFilter).every(
|
||||
([k, v]) =>
|
||||
(!v && !inputValues[k]) || isCostFilterMatch(v, inputValues[k]),
|
||||
)
|
||||
: costFilter === inputValues;
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { create } from "zustand";
|
||||
import { NodeChange, applyNodeChanges } from "@xyflow/react";
|
||||
import { CustomNode } from "../components/FlowEditor/nodes/CustomNode";
|
||||
import { CustomNode } from "../components/FlowEditor/nodes/CustomNode/CustomNode";
|
||||
import { BlockInfo } from "@/app/api/__generated__/models/blockInfo";
|
||||
import { convertBlockInfoIntoCustomNodeData } from "../components/helper";
|
||||
import { Node } from "@/app/api/__generated__/models/node";
|
||||
|
||||
@@ -67,7 +67,7 @@ export function useLoginPage() {
|
||||
|
||||
if (!response.ok) {
|
||||
const { error } = await response.json();
|
||||
if (typeof error === "string" && error.includes("not_allowed")) {
|
||||
if (error === "not_allowed") {
|
||||
setShowNotAllowedModal(true);
|
||||
} else {
|
||||
setFeedback(error || "Failed to start OAuth flow");
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
"use client";
|
||||
|
||||
import * as React from "react";
|
||||
import { Sidebar } from "@/components/__legacy__/Sidebar";
|
||||
import {
|
||||
@@ -8,8 +10,11 @@ import {
|
||||
IconCoin,
|
||||
} from "@/components/__legacy__/ui/icons";
|
||||
import { KeyIcon } from "lucide-react";
|
||||
import { useGetFlag, Flag } from "@/services/feature-flags/use-get-flag";
|
||||
|
||||
export default function Layout({ children }: { children: React.ReactNode }) {
|
||||
const isPaymentEnabled = useGetFlag(Flag.ENABLE_PLATFORM_PAYMENT);
|
||||
|
||||
const sidebarLinkGroups = [
|
||||
{
|
||||
links: [
|
||||
@@ -18,7 +23,7 @@ export default function Layout({ children }: { children: React.ReactNode }) {
|
||||
href: "/profile/dashboard",
|
||||
icon: <IconDashboardLayout className="h-6 w-6" />,
|
||||
},
|
||||
...(process.env.NEXT_PUBLIC_SHOW_BILLING_PAGE === "true"
|
||||
...(isPaymentEnabled
|
||||
? [
|
||||
{
|
||||
text: "Billing",
|
||||
|
||||
@@ -59,6 +59,7 @@ export function useSignupPage() {
|
||||
resetCaptcha();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/auth/provider", {
|
||||
method: "POST",
|
||||
@@ -70,6 +71,13 @@ export function useSignupPage() {
|
||||
const { error } = await response.json();
|
||||
setIsGoogleLoading(false);
|
||||
resetCaptcha();
|
||||
|
||||
// Check for waitlist error
|
||||
if (error === "not_allowed") {
|
||||
setShowNotAllowedModal(true);
|
||||
return;
|
||||
}
|
||||
|
||||
toast({
|
||||
title: error || "Failed to start OAuth flow",
|
||||
variant: "destructive",
|
||||
@@ -142,6 +150,7 @@ export function useSignupPage() {
|
||||
setShowNotAllowedModal(true);
|
||||
return;
|
||||
}
|
||||
|
||||
toast({
|
||||
title: result?.error || "Signup failed",
|
||||
variant: "destructive",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { getServerSupabase } from "@/lib/supabase/server/getServerSupabase";
|
||||
import { NextResponse } from "next/server";
|
||||
import { LoginProvider } from "@/types/auth";
|
||||
import { isWaitlistError, logWaitlistError } from "../utils";
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
@@ -31,8 +32,9 @@ export async function POST(request: Request) {
|
||||
});
|
||||
|
||||
if (error) {
|
||||
// FIXME: supabase doesn't return the correct error message for this case
|
||||
if (error.message.includes("P0001")) {
|
||||
// Check for waitlist/allowlist error
|
||||
if (isWaitlistError(error?.code, error?.message)) {
|
||||
logWaitlistError("OAuth Provider", error.message);
|
||||
return NextResponse.json({ error: "not_allowed" }, { status: 403 });
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import { getServerSupabase } from "@/lib/supabase/server/getServerSupabase";
|
||||
import { verifyTurnstileToken } from "@/lib/turnstile";
|
||||
import { signupFormSchema } from "@/types/auth";
|
||||
import { shouldShowOnboarding } from "../../helpers";
|
||||
import { isWaitlistError, logWaitlistError } from "../utils";
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
@@ -29,6 +30,7 @@ export async function POST(request: Request) {
|
||||
turnstileToken ?? "",
|
||||
"signup",
|
||||
);
|
||||
|
||||
if (!captchaOk) {
|
||||
return NextResponse.json(
|
||||
{ error: "CAPTCHA verification failed. Please try again." },
|
||||
@@ -47,16 +49,19 @@ export async function POST(request: Request) {
|
||||
const { data, error } = await supabase.auth.signUp(parsed.data);
|
||||
|
||||
if (error) {
|
||||
// FIXME: supabase doesn't return the correct error message for this case
|
||||
if (error.message.includes("P0001")) {
|
||||
// Check for waitlist/allowlist error
|
||||
if (isWaitlistError(error?.code, error?.message)) {
|
||||
logWaitlistError("Signup", error.message);
|
||||
return NextResponse.json({ error: "not_allowed" }, { status: 403 });
|
||||
}
|
||||
|
||||
if ((error as any).code === "user_already_exists") {
|
||||
return NextResponse.json(
|
||||
{ error: "user_already_exists" },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json({ error: error.message }, { status: 400 });
|
||||
}
|
||||
|
||||
|
||||
60
autogpt_platform/frontend/src/app/api/auth/utils.ts
Normal file
60
autogpt_platform/frontend/src/app/api/auth/utils.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Checks if an error is related to the waitlist/allowlist
|
||||
*
|
||||
* Can be used with either:
|
||||
* - Error objects from Supabase auth operations: `isWaitlistError(error?.code, error?.message)`
|
||||
* - URL parameters from OAuth callbacks: `isWaitlistError(errorCode, errorDescription)`
|
||||
*
|
||||
* The PostgreSQL trigger raises P0001 with message format:
|
||||
* "The email address "email" is not allowed to register. Please contact support for assistance."
|
||||
*
|
||||
* @param code - Error code (e.g., "P0001", "unexpected_failure") or null
|
||||
* @param message - Error message/description or null
|
||||
* @returns true if this appears to be a waitlist/allowlist error
|
||||
*/
|
||||
export function isWaitlistError(
|
||||
code?: string | null,
|
||||
message?: string | null,
|
||||
): boolean {
|
||||
// Check for explicit PostgreSQL trigger error code
|
||||
if (code === "P0001") return true;
|
||||
|
||||
if (!message) return false;
|
||||
|
||||
const lowerMessage = message.toLowerCase();
|
||||
|
||||
// Check for the generic database error that occurs during waitlist check
|
||||
// This happens when Supabase wraps the PostgreSQL trigger error
|
||||
if (
|
||||
code === "unexpected_failure" &&
|
||||
message === "Database error saving new user"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for various waitlist-related patterns in the message
|
||||
return (
|
||||
lowerMessage.includes("p0001") || // PostgreSQL error code in message
|
||||
lowerMessage.includes("not allowed") || // Common waitlist message
|
||||
lowerMessage.includes("waitlist") || // Explicit waitlist mention
|
||||
lowerMessage.includes("allowlist") || // Explicit allowlist mention
|
||||
lowerMessage.includes("allowed_users") || // Database table reference
|
||||
lowerMessage.includes("not allowed to register") // Full trigger message
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs a waitlist error for debugging purposes
|
||||
* Does not expose user email in logs for privacy
|
||||
*
|
||||
* @param context - Where the error occurred (e.g., "Signup", "OAuth Provider")
|
||||
* @param errorMessage - The full error message
|
||||
*/
|
||||
export function logWaitlistError(context: string, errorMessage: string): void {
|
||||
// Only log the error code and general message, not the email
|
||||
const sanitizedMessage = errorMessage.replace(
|
||||
/"[^"]+@[^"]+"/g, // Matches email addresses in quotes
|
||||
'"[email]"',
|
||||
);
|
||||
console.log(`[${context}] Waitlist check failed:`, sanitizedMessage);
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { Button } from "../atoms/Button/Button";
|
||||
import { Text } from "../atoms/Text/Text";
|
||||
import { Dialog } from "../molecules/Dialog/Dialog";
|
||||
import { WaitlistErrorContent } from "./WaitlistErrorContent";
|
||||
|
||||
interface Props {
|
||||
isOpen: boolean;
|
||||
@@ -14,18 +13,8 @@ export function EmailNotAllowedModal({ isOpen, onClose }: Props) {
|
||||
styling={{ maxWidth: "35rem" }}
|
||||
>
|
||||
<Dialog.Content>
|
||||
<div className="flex flex-col items-center gap-8 py-4">
|
||||
<Text variant="h3">Access Restricted</Text>
|
||||
<Text variant="large-medium" className="text-center">
|
||||
We're currently in a limited access phase. Your email address
|
||||
isn't on our current allowlist for early access. If you believe
|
||||
this is an error or would like to request access, please contact us.
|
||||
</Text>
|
||||
<div className="flex justify-end pt-4">
|
||||
<Button variant="primary" onClick={onClose}>
|
||||
I understand
|
||||
</Button>
|
||||
</div>
|
||||
<div className="py-4">
|
||||
<WaitlistErrorContent onClose={onClose} />
|
||||
</div>
|
||||
</Dialog.Content>
|
||||
</Dialog>
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { Button } from "../atoms/Button/Button";
|
||||
import { Text } from "../atoms/Text/Text";
|
||||
|
||||
interface WaitlistErrorContentProps {
|
||||
onClose: () => void;
|
||||
closeButtonText?: string;
|
||||
closeButtonVariant?: "primary" | "secondary";
|
||||
}
|
||||
|
||||
export function WaitlistErrorContent({
|
||||
onClose,
|
||||
closeButtonText = "Close",
|
||||
closeButtonVariant = "primary",
|
||||
}: WaitlistErrorContentProps) {
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-6">
|
||||
<Text variant="h3">Join the Waitlist</Text>
|
||||
<div className="flex flex-col gap-4 text-center">
|
||||
<Text variant="large-medium" className="text-center">
|
||||
The AutoGPT Platform is currently in closed beta. Your email address
|
||||
isn't on our current allowlist for early access.
|
||||
</Text>
|
||||
<Text variant="body" className="text-center">
|
||||
Join our waitlist to get notified when we open up access!
|
||||
</Text>
|
||||
</div>
|
||||
<div className="flex gap-3">
|
||||
<Button
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
window.open("https://agpt.co/waitlist", "_blank");
|
||||
}}
|
||||
>
|
||||
Join Waitlist
|
||||
</Button>
|
||||
<Button variant={closeButtonVariant} onClick={onClose}>
|
||||
{closeButtonText}
|
||||
</Button>
|
||||
</div>
|
||||
<div className="flex flex-col gap-2">
|
||||
<Text variant="small" className="text-center text-muted-foreground">
|
||||
Already signed up for the waitlist? Make sure you're using the
|
||||
exact same email address you used when signing up.
|
||||
</Text>
|
||||
<Text variant="small" className="text-center text-muted-foreground">
|
||||
If you're not sure which email you used or need help,{" "}
|
||||
<a
|
||||
href="https://discord.gg/autogpt"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="underline hover:text-foreground"
|
||||
>
|
||||
reach out on Discord
|
||||
</a>
|
||||
</Text>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -15,6 +15,7 @@ export enum Flag {
|
||||
SHARE_EXECUTION_RESULTS = "share-execution-results",
|
||||
AGENT_FAVORITING = "agent-favoriting",
|
||||
MARKETPLACE_SEARCH_TERMS = "marketplace-search-terms",
|
||||
ENABLE_PLATFORM_PAYMENT = "enable-platform-payment",
|
||||
}
|
||||
|
||||
export type FlagValues = {
|
||||
@@ -28,6 +29,7 @@ export type FlagValues = {
|
||||
[Flag.SHARE_EXECUTION_RESULTS]: boolean;
|
||||
[Flag.AGENT_FAVORITING]: boolean;
|
||||
[Flag.MARKETPLACE_SEARCH_TERMS]: string[];
|
||||
[Flag.ENABLE_PLATFORM_PAYMENT]: boolean;
|
||||
};
|
||||
|
||||
const isPwMockEnabled = process.env.NEXT_PUBLIC_PW_TEST === "true";
|
||||
@@ -43,6 +45,7 @@ const mockFlags = {
|
||||
[Flag.SHARE_EXECUTION_RESULTS]: false,
|
||||
[Flag.AGENT_FAVORITING]: false,
|
||||
[Flag.MARKETPLACE_SEARCH_TERMS]: DEFAULT_SEARCH_TERMS,
|
||||
[Flag.ENABLE_PLATFORM_PAYMENT]: false,
|
||||
};
|
||||
|
||||
export function useGetFlag<T extends Flag>(flag: T): FlagValues[T] | null {
|
||||
@@ -50,7 +53,9 @@ export function useGetFlag<T extends Flag>(flag: T): FlagValues[T] | null {
|
||||
const flagValue = currentFlags[flag];
|
||||
const isCloud = getBehaveAs() === BehaveAs.CLOUD;
|
||||
|
||||
if (isPwMockEnabled && !isCloud) return mockFlags[flag];
|
||||
if ((isPwMockEnabled && !isCloud) || flagValue === undefined) {
|
||||
return mockFlags[flag];
|
||||
}
|
||||
|
||||
return flagValue;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user