From ef7e50403e25a904bc43a256a540f544e35a11aa Mon Sep 17 00:00:00 2001 From: Zamil Majdy Date: Tue, 12 Nov 2024 21:09:59 +0700 Subject: [PATCH] refactor(backend): Centralize Block Cost into a Single File (#8623) --- .../backend/backend/blocks/llm.py | 41 ++-- .../backend/backend/data/block_cost_config.py | 194 +++++++++++++++++ autogpt_platform/backend/backend/data/cost.py | 32 +++ .../backend/backend/data/credit.py | 195 +----------------- 4 files changed, 248 insertions(+), 214 deletions(-) create mode 100644 autogpt_platform/backend/backend/data/block_cost_config.py create mode 100644 autogpt_platform/backend/backend/data/cost.py diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py index 129879298b..7663178702 100644 --- a/autogpt_platform/backend/backend/blocks/llm.py +++ b/autogpt_platform/backend/backend/blocks/llm.py @@ -58,7 +58,6 @@ def AICredentialsField() -> AICredentials: class ModelMetadata(NamedTuple): provider: str context_window: int - cost_factor: int class LlmModelMeta(EnumMeta): @@ -117,31 +116,27 @@ class LlmModel(str, Enum, metaclass=LlmModelMeta): def context_window(self) -> int: return self.metadata.context_window - @property - def cost_factor(self) -> int: - return self.metadata.cost_factor - MODEL_METADATA = { - LlmModel.O1_PREVIEW: ModelMetadata("openai", 32000, cost_factor=16), - LlmModel.O1_MINI: ModelMetadata("openai", 62000, cost_factor=4), - LlmModel.GPT4O_MINI: ModelMetadata("openai", 128000, cost_factor=1), - LlmModel.GPT4O: ModelMetadata("openai", 128000, cost_factor=3), - LlmModel.GPT4_TURBO: ModelMetadata("openai", 128000, cost_factor=10), - LlmModel.GPT3_5_TURBO: ModelMetadata("openai", 16385, cost_factor=1), - LlmModel.CLAUDE_3_5_SONNET: ModelMetadata("anthropic", 200000, cost_factor=4), - LlmModel.CLAUDE_3_HAIKU: ModelMetadata("anthropic", 200000, cost_factor=1), - LlmModel.LLAMA3_8B: ModelMetadata("groq", 8192, cost_factor=1), - LlmModel.LLAMA3_70B: ModelMetadata("groq", 8192, cost_factor=1), - LlmModel.MIXTRAL_8X7B: ModelMetadata("groq", 32768, cost_factor=1), - LlmModel.GEMMA_7B: ModelMetadata("groq", 8192, cost_factor=1), - LlmModel.GEMMA2_9B: ModelMetadata("groq", 8192, cost_factor=1), - LlmModel.LLAMA3_1_405B: ModelMetadata("groq", 8192, cost_factor=1), + LlmModel.O1_PREVIEW: ModelMetadata("openai", 32000), + LlmModel.O1_MINI: ModelMetadata("openai", 62000), + LlmModel.GPT4O_MINI: ModelMetadata("openai", 128000), + LlmModel.GPT4O: ModelMetadata("openai", 128000), + LlmModel.GPT4_TURBO: ModelMetadata("openai", 128000), + LlmModel.GPT3_5_TURBO: ModelMetadata("openai", 16385), + LlmModel.CLAUDE_3_5_SONNET: ModelMetadata("anthropic", 200000), + LlmModel.CLAUDE_3_HAIKU: ModelMetadata("anthropic", 200000), + LlmModel.LLAMA3_8B: ModelMetadata("groq", 8192), + LlmModel.LLAMA3_70B: ModelMetadata("groq", 8192), + LlmModel.MIXTRAL_8X7B: ModelMetadata("groq", 32768), + LlmModel.GEMMA_7B: ModelMetadata("groq", 8192), + LlmModel.GEMMA2_9B: ModelMetadata("groq", 8192), + LlmModel.LLAMA3_1_405B: ModelMetadata("groq", 8192), # Limited to 16k during preview - LlmModel.LLAMA3_1_70B: ModelMetadata("groq", 131072, cost_factor=1), - LlmModel.LLAMA3_1_8B: ModelMetadata("groq", 131072, cost_factor=1), - LlmModel.OLLAMA_LLAMA3_8B: ModelMetadata("ollama", 8192, cost_factor=1), - LlmModel.OLLAMA_LLAMA3_405B: ModelMetadata("ollama", 8192, cost_factor=1), + LlmModel.LLAMA3_1_70B: ModelMetadata("groq", 131072), + LlmModel.LLAMA3_1_8B: ModelMetadata("groq", 131072), + LlmModel.OLLAMA_LLAMA3_8B: ModelMetadata("ollama", 8192), + LlmModel.OLLAMA_LLAMA3_405B: ModelMetadata("ollama", 8192), } for model in LlmModel: diff --git a/autogpt_platform/backend/backend/data/block_cost_config.py b/autogpt_platform/backend/backend/data/block_cost_config.py new file mode 100644 index 0000000000..4a5d6419b9 --- /dev/null +++ b/autogpt_platform/backend/backend/data/block_cost_config.py @@ -0,0 +1,194 @@ +from typing import Type + +from autogpt_libs.supabase_integration_credentials_store.store import ( + anthropic_credentials, + did_credentials, + groq_credentials, + ideogram_credentials, + jina_credentials, + openai_credentials, + replicate_credentials, + revid_credentials, +) + +from backend.blocks.ai_shortform_video_block import AIShortformVideoCreatorBlock +from backend.blocks.ideogram import IdeogramModelBlock +from backend.blocks.jina.search import SearchTheWebBlock +from backend.blocks.llm import ( + MODEL_METADATA, + AIConversationBlock, + AIStructuredResponseGeneratorBlock, + AITextGeneratorBlock, + AITextSummarizerBlock, + LlmModel, +) +from backend.blocks.replicate_flux_advanced import ReplicateFluxAdvancedModelBlock +from backend.blocks.search import ExtractWebsiteContentBlock +from backend.blocks.talking_head import CreateTalkingAvatarVideoBlock +from backend.data.block import Block +from backend.data.cost import BlockCost, BlockCostType + +# =============== Configure the cost for each LLM Model call =============== # + +MODEL_COST: dict[LlmModel, int] = { + LlmModel.O1_PREVIEW: 16, + LlmModel.O1_MINI: 4, + LlmModel.GPT4O_MINI: 1, + LlmModel.GPT4O: 3, + LlmModel.GPT4_TURBO: 10, + LlmModel.GPT3_5_TURBO: 1, + LlmModel.CLAUDE_3_5_SONNET: 4, + LlmModel.CLAUDE_3_HAIKU: 1, + LlmModel.LLAMA3_8B: 1, + LlmModel.LLAMA3_70B: 1, + LlmModel.MIXTRAL_8X7B: 1, + LlmModel.GEMMA_7B: 1, + LlmModel.GEMMA2_9B: 1, + LlmModel.LLAMA3_1_405B: 1, + LlmModel.LLAMA3_1_70B: 1, + LlmModel.LLAMA3_1_8B: 1, + LlmModel.OLLAMA_LLAMA3_8B: 1, + LlmModel.OLLAMA_LLAMA3_405B: 1, +} + +for model in LlmModel: + if model not in MODEL_COST: + raise ValueError(f"Missing MODEL_COST for model: {model}") + + +LLM_COST = ( + [ + BlockCost( + cost_type=BlockCostType.RUN, + cost_filter={ + "model": model, + "api_key": None, # Running LLM with user own API key is free. + }, + cost_amount=cost, + ) + for model, cost in MODEL_COST.items() + ] + + [ + BlockCost( + cost_type=BlockCostType.RUN, + cost_filter={ + "model": model, + "credentials": { + "id": anthropic_credentials.id, + "provider": anthropic_credentials.provider, + "type": anthropic_credentials.type, + }, + }, + cost_amount=cost, + ) + for model, cost in MODEL_COST.items() + if MODEL_METADATA[model].provider == "anthropic" + ] + + [ + BlockCost( + cost_type=BlockCostType.RUN, + cost_filter={ + "model": model, + "credentials": { + "id": openai_credentials.id, + "provider": openai_credentials.provider, + "type": openai_credentials.type, + }, + }, + cost_amount=cost, + ) + for model, cost in MODEL_COST.items() + if MODEL_METADATA[model].provider == "openai" + ] + + [ + BlockCost( + cost_type=BlockCostType.RUN, + cost_filter={ + "model": model, + "credentials": {"id": groq_credentials.id}, + }, + cost_amount=cost, + ) + for model, cost in MODEL_COST.items() + if MODEL_METADATA[model].provider == "groq" + ] + + [ + BlockCost( + # Default cost is running LlmModel.GPT4O. + cost_amount=MODEL_COST[LlmModel.GPT4O], + cost_filter={"api_key": None}, + ), + ] +) + +# =============== This is the exhaustive list of cost for each Block =============== # + +BLOCK_COSTS: dict[Type[Block], list[BlockCost]] = { + AIConversationBlock: LLM_COST, + AITextGeneratorBlock: LLM_COST, + AIStructuredResponseGeneratorBlock: LLM_COST, + AITextSummarizerBlock: LLM_COST, + CreateTalkingAvatarVideoBlock: [ + BlockCost( + cost_amount=15, + cost_filter={ + "credentials": { + "id": did_credentials.id, + "provider": did_credentials.provider, + "type": did_credentials.type, + } + }, + ) + ], + SearchTheWebBlock: [ + BlockCost( + cost_amount=1, + cost_filter={ + "credentials": { + "id": jina_credentials.id, + "provider": jina_credentials.provider, + "type": jina_credentials.type, + } + }, + ) + ], + ExtractWebsiteContentBlock: [ + BlockCost(cost_amount=1, cost_filter={"raw_content": False}) + ], + IdeogramModelBlock: [ + BlockCost( + cost_amount=1, + cost_filter={ + "credentials": { + "id": ideogram_credentials.id, + "provider": ideogram_credentials.provider, + "type": ideogram_credentials.type, + } + }, + ) + ], + AIShortformVideoCreatorBlock: [ + BlockCost( + cost_amount=10, + cost_filter={ + "credentials": { + "id": revid_credentials.id, + "provider": revid_credentials.provider, + "type": revid_credentials.type, + } + }, + ) + ], + ReplicateFluxAdvancedModelBlock: [ + BlockCost( + cost_amount=10, + cost_filter={ + "credentials": { + "id": replicate_credentials.id, + "provider": replicate_credentials.provider, + "type": replicate_credentials.type, + } + }, + ) + ], +} diff --git a/autogpt_platform/backend/backend/data/cost.py b/autogpt_platform/backend/backend/data/cost.py new file mode 100644 index 0000000000..2318d72d7a --- /dev/null +++ b/autogpt_platform/backend/backend/data/cost.py @@ -0,0 +1,32 @@ +from enum import Enum +from typing import Any, Optional + +from pydantic import BaseModel + +from backend.data.block import BlockInput + + +class BlockCostType(str, Enum): + RUN = "run" # cost X credits per run + BYTE = "byte" # cost X credits per byte + SECOND = "second" # cost X credits per second + + +class BlockCost(BaseModel): + cost_amount: int + cost_filter: BlockInput + cost_type: BlockCostType + + def __init__( + self, + cost_amount: int, + cost_type: BlockCostType = BlockCostType.RUN, + cost_filter: Optional[BlockInput] = None, + **data: Any, + ) -> None: + super().__init__( + cost_amount=cost_amount, + cost_filter=cost_filter or {}, + cost_type=cost_type, + **data, + ) diff --git a/autogpt_platform/backend/backend/data/credit.py b/autogpt_platform/backend/backend/data/credit.py index d3618e658b..fbb79a54a4 100644 --- a/autogpt_platform/backend/backend/data/credit.py +++ b/autogpt_platform/backend/backend/data/credit.py @@ -1,204 +1,17 @@ from abc import ABC, abstractmethod from datetime import datetime, timezone -from enum import Enum -from typing import Any, Optional, Type -import prisma.errors -from autogpt_libs.supabase_integration_credentials_store.store import ( - anthropic_credentials, - did_credentials, - groq_credentials, - ideogram_credentials, - jina_credentials, - openai_credentials, - replicate_credentials, - revid_credentials, -) from prisma import Json from prisma.enums import UserBlockCreditType +from prisma.errors import UniqueViolationError from prisma.models import UserBlockCredit -from pydantic import BaseModel -from backend.blocks.ai_shortform_video_block import AIShortformVideoCreatorBlock -from backend.blocks.ideogram import IdeogramModelBlock -from backend.blocks.jina.search import SearchTheWebBlock -from backend.blocks.llm import ( - MODEL_METADATA, - AIConversationBlock, - AIStructuredResponseGeneratorBlock, - AITextGeneratorBlock, - AITextSummarizerBlock, - LlmModel, -) -from backend.blocks.replicate_flux_advanced import ReplicateFluxAdvancedModelBlock -from backend.blocks.search import ExtractWebsiteContentBlock -from backend.blocks.talking_head import CreateTalkingAvatarVideoBlock from backend.data.block import Block, BlockInput, get_block +from backend.data.block_cost_config import BLOCK_COSTS +from backend.data.cost import BlockCost, BlockCostType from backend.util.settings import Config -class BlockCostType(str, Enum): - RUN = "run" # cost X credits per run - BYTE = "byte" # cost X credits per byte - SECOND = "second" # cost X credits per second - - -class BlockCost(BaseModel): - cost_amount: int - cost_filter: BlockInput - cost_type: BlockCostType - - def __init__( - self, - cost_amount: int, - cost_type: BlockCostType = BlockCostType.RUN, - cost_filter: Optional[BlockInput] = None, - **data: Any, - ) -> None: - super().__init__( - cost_amount=cost_amount, - cost_filter=cost_filter or {}, - cost_type=cost_type, - **data, - ) - - -llm_cost = ( - [ - BlockCost( - cost_type=BlockCostType.RUN, - cost_filter={ - "model": model, - "api_key": None, # Running LLM with user own API key is free. - }, - cost_amount=metadata.cost_factor, - ) - for model, metadata in MODEL_METADATA.items() - ] - + [ - BlockCost( - cost_type=BlockCostType.RUN, - cost_filter={ - "model": model, - "credentials": { - "id": anthropic_credentials.id, - "provider": anthropic_credentials.provider, - "type": anthropic_credentials.type, - }, - }, - cost_amount=metadata.cost_factor, - ) - for model, metadata in MODEL_METADATA.items() - if metadata.provider == "anthropic" - ] - + [ - BlockCost( - cost_type=BlockCostType.RUN, - cost_filter={ - "model": model, - "credentials": { - "id": openai_credentials.id, - "provider": openai_credentials.provider, - "type": openai_credentials.type, - }, - }, - cost_amount=metadata.cost_factor, - ) - for model, metadata in MODEL_METADATA.items() - if metadata.provider == "openai" - ] - + [ - BlockCost( - cost_type=BlockCostType.RUN, - cost_filter={ - "model": model, - "credentials": {"id": groq_credentials.id}, - }, - cost_amount=metadata.cost_factor, - ) - for model, metadata in MODEL_METADATA.items() - if metadata.provider == "groq" - ] - + [ - BlockCost( - # Default cost is running LlmModel.GPT4O. - cost_amount=MODEL_METADATA[LlmModel.GPT4O].cost_factor, - cost_filter={"api_key": None}, - ), - ] -) - -BLOCK_COSTS: dict[Type[Block], list[BlockCost]] = { - AIConversationBlock: llm_cost, - AITextGeneratorBlock: llm_cost, - AIStructuredResponseGeneratorBlock: llm_cost, - AITextSummarizerBlock: llm_cost, - CreateTalkingAvatarVideoBlock: [ - BlockCost( - cost_amount=15, - cost_filter={ - "credentials": { - "id": did_credentials.id, - "provider": did_credentials.provider, - "type": did_credentials.type, - } - }, - ) - ], - SearchTheWebBlock: [ - BlockCost( - cost_amount=1, - cost_filter={ - "credentials": { - "id": jina_credentials.id, - "provider": jina_credentials.provider, - "type": jina_credentials.type, - } - }, - ) - ], - ExtractWebsiteContentBlock: [ - BlockCost(cost_amount=1, cost_filter={"raw_content": False}) - ], - IdeogramModelBlock: [ - BlockCost( - cost_amount=1, - cost_filter={ - "credentials": { - "id": ideogram_credentials.id, - "provider": ideogram_credentials.provider, - "type": ideogram_credentials.type, - } - }, - ) - ], - AIShortformVideoCreatorBlock: [ - BlockCost( - cost_amount=10, - cost_filter={ - "credentials": { - "id": revid_credentials.id, - "provider": revid_credentials.provider, - "type": revid_credentials.type, - } - }, - ) - ], - ReplicateFluxAdvancedModelBlock: [ - BlockCost( - cost_amount=10, - cost_filter={ - "credentials": { - "id": replicate_credentials.id, - "provider": replicate_credentials.provider, - "type": replicate_credentials.type, - } - }, - ) - ], -} - - class UserCreditBase(ABC): def __init__(self, num_user_credits_refill: int): self.num_user_credits_refill = num_user_credits_refill @@ -283,7 +96,7 @@ class UserCredit(UserCreditBase): "createdAt": self.time_now(), } ) - except prisma.errors.UniqueViolationError: + except UniqueViolationError: pass # Already refilled this month return self.num_user_credits_refill