From a259eac9ffb190eb0bf01535b31815bb1a791d95 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Jun 2025 19:46:44 +0200 Subject: [PATCH] feat(blocks): Add AI/ML API support to LLM blocks (#9996) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hi! Taking over this feature from the previous author in [#9163](https://github.com/Significant-Gravitas/AutoGPT/pull/9163). This PR continues the work to integrate AI/ML API support into LLM blocks, addressing pending review comments and ensuring compatibility with the current codebase. I’ve reviewed and fixed the outstanding issues from the previous PR. Kindly recheck the previous concerns — let me know if anything still needs improvement. Previous description: > Changes 🏗️ > > - Added basic functionality to enable users to send requests to our models. > - Added instructions for users on how to use the AI/ML API in AutoGPT. > > Checklist 📋 > For code changes: > > - [x] I have clearly listed my changes in the PR description > - [x] I have made a test plan > - [x] I have tested my changes according to the test plan: > - [x] The API key has been successfully added and saved to the user's profile. > - [x] Sending requests to each model provided by us, enabling users to test them in a project with various max_tokens parameter values and other configurations. --------- Signed-off-by: dependabot[bot] Co-authored-by: Ivan Co-authored-by: waterstark <84220220+waterstark@users.noreply.github.com> Co-authored-by: Aarushi <50577581+aarushik93@users.noreply.github.com> Co-authored-by: Reinier van der Leer Co-authored-by: Nicholas Tindle Co-authored-by: snyk-bot Co-authored-by: Krzysztof Czerwinski <34861343+kcze@users.noreply.github.com> Co-authored-by: Zamil Majdy Co-authored-by: Krzysztof Czerwinski Co-authored-by: Reinier van der Leer Co-authored-by: Bently Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ayush Mittal <130590402+Ayush-Mittal10@users.noreply.github.com> Co-authored-by: Azraf Nahian <69325302+turboslapper@users.noreply.github.com> Co-authored-by: Nicholas Tindle Co-authored-by: Swifty Co-authored-by: Mario Sacaj Co-authored-by: Toran Bruce Richards Co-authored-by: Abhimanyu Yadav <122007096+Abhi1992002@users.noreply.github.com> Co-authored-by: Ritik Dutta Co-authored-by: Pratim Sadhu --- autogpt_platform/backend/.env.example | 1 + .../backend/backend/blocks/llm.py | 36 +++++++++++++++++++ .../backend/backend/data/block_cost_config.py | 23 ++++++++++++ .../backend/integrations/credentials_store.py | 10 ++++++ .../backend/backend/integrations/providers.py | 1 + .../backend/backend/util/settings.py | 1 + .../profile/(user)/integrations/page.tsx | 1 + .../integrations/credentials-input.tsx | 1 + .../integrations/credentials-provider.tsx | 1 + .../src/lib/autogpt-server-api/types.ts | 1 + docs/content/index.md | 1 + 11 files changed, 77 insertions(+) diff --git a/autogpt_platform/backend/.env.example b/autogpt_platform/backend/.env.example index ebd9d7c6ad..4c5830d704 100644 --- a/autogpt_platform/backend/.env.example +++ b/autogpt_platform/backend/.env.example @@ -127,6 +127,7 @@ TODOIST_CLIENT_SECRET= # LLM OPENAI_API_KEY= ANTHROPIC_API_KEY= +AIML_API_KEY= GROQ_API_KEY= OPEN_ROUTER_API_KEY= LLAMA_API_KEY= diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py index f4c16e48ad..a1fedeec52 100644 --- a/autogpt_platform/backend/backend/blocks/llm.py +++ b/autogpt_platform/backend/backend/blocks/llm.py @@ -31,6 +31,7 @@ logger = TruncatedLogger(logging.getLogger(__name__), "[LLM-Block]") fmt = TextFormatter() LLMProviderName = Literal[ + ProviderName.AIML_API, ProviderName.ANTHROPIC, ProviderName.GROQ, ProviderName.OLLAMA, @@ -107,6 +108,12 @@ class LlmModel(str, Enum, metaclass=LlmModelMeta): CLAUDE_3_5_SONNET = "claude-3-5-sonnet-latest" CLAUDE_3_5_HAIKU = "claude-3-5-haiku-latest" CLAUDE_3_HAIKU = "claude-3-haiku-20240307" + # AI/ML API models + AIML_API_QWEN2_5_72B = "Qwen/Qwen2.5-72B-Instruct-Turbo" + AIML_API_LLAMA3_1_70B = "nvidia/llama-3.1-nemotron-70b-instruct" + AIML_API_LLAMA3_3_70B = "meta-llama/Llama-3.3-70B-Instruct-Turbo" + AIML_API_META_LLAMA_3_1_70B = "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo" + AIML_API_LLAMA_3_2_3B = "meta-llama/Llama-3.2-3B-Instruct-Turbo" # Groq models GEMMA2_9B = "gemma2-9b-it" LLAMA3_3_70B = "llama-3.3-70b-versatile" @@ -204,6 +211,12 @@ MODEL_METADATA = { LlmModel.CLAUDE_3_HAIKU: ModelMetadata( "anthropic", 200000, 4096 ), # claude-3-haiku-20240307 + # https://docs.aimlapi.com/api-overview/model-database/text-models + LlmModel.AIML_API_QWEN2_5_72B: ModelMetadata("aiml_api", 32000, 8000), + LlmModel.AIML_API_LLAMA3_1_70B: ModelMetadata("aiml_api", 128000, 40000), + LlmModel.AIML_API_LLAMA3_3_70B: ModelMetadata("aiml_api", 128000, None), + LlmModel.AIML_API_META_LLAMA_3_1_70B: ModelMetadata("aiml_api", 131000, 2000), + LlmModel.AIML_API_LLAMA_3_2_3B: ModelMetadata("aiml_api", 128000, None), # https://console.groq.com/docs/models LlmModel.GEMMA2_9B: ModelMetadata("groq", 8192, None), LlmModel.LLAMA3_3_70B: ModelMetadata("groq", 128000, 32768), @@ -616,6 +629,29 @@ def llm_call( prompt_tokens=response.usage.prompt_tokens if response.usage else 0, completion_tokens=response.usage.completion_tokens if response.usage else 0, ) + elif provider == "aiml_api": + client = openai.OpenAI( + base_url="https://api.aimlapi.com/v2", + api_key=credentials.api_key.get_secret_value(), + default_headers={"X-Project": "AutoGPT"}, + ) + + completion = client.chat.completions.create( + model=llm_model.value, + messages=prompt, # type: ignore + max_tokens=max_tokens, + ) + + return LLMResponse( + raw_response=completion.choices[0].message, + prompt=prompt, + response=completion.choices[0].message.content or "", + tool_calls=None, + prompt_tokens=completion.usage.prompt_tokens if completion.usage else 0, + completion_tokens=( + completion.usage.completion_tokens if completion.usage else 0 + ), + ) else: raise ValueError(f"Unsupported LLM provider: {provider}") diff --git a/autogpt_platform/backend/backend/data/block_cost_config.py b/autogpt_platform/backend/backend/data/block_cost_config.py index e099ebfea0..864ed0e989 100644 --- a/autogpt_platform/backend/backend/data/block_cost_config.py +++ b/autogpt_platform/backend/backend/data/block_cost_config.py @@ -22,6 +22,7 @@ from backend.blocks.text_to_speech_block import UnrealTextToSpeechBlock from backend.data.block import Block from backend.data.cost import BlockCost, BlockCostType from backend.integrations.credentials_store import ( + aiml_api_credentials, anthropic_credentials, did_credentials, groq_credentials, @@ -54,6 +55,11 @@ MODEL_COST: dict[LlmModel, int] = { LlmModel.CLAUDE_3_5_SONNET: 4, LlmModel.CLAUDE_3_5_HAIKU: 1, # $0.80 / $4.00 LlmModel.CLAUDE_3_HAIKU: 1, + LlmModel.AIML_API_QWEN2_5_72B: 1, + LlmModel.AIML_API_LLAMA3_1_70B: 1, + LlmModel.AIML_API_LLAMA3_3_70B: 1, + LlmModel.AIML_API_META_LLAMA_3_1_70B: 1, + LlmModel.AIML_API_LLAMA_3_2_3B: 1, LlmModel.LLAMA3_8B: 1, LlmModel.LLAMA3_70B: 1, LlmModel.MIXTRAL_8X7B: 1, @@ -178,6 +184,23 @@ LLM_COST = ( for model, cost in MODEL_COST.items() if MODEL_METADATA[model].provider == "llama_api" ] + # AI/ML Api Models + + [ + BlockCost( + cost_type=BlockCostType.RUN, + cost_filter={ + "model": model, + "credentials": { + "id": aiml_api_credentials.id, + "provider": aiml_api_credentials.provider, + "type": aiml_api_credentials.type, + }, + }, + cost_amount=cost, + ) + for model, cost in MODEL_COST.items() + if MODEL_METADATA[model].provider == "aiml_api" + ] ) # =============== This is the exhaustive list of cost for each Block =============== # diff --git a/autogpt_platform/backend/backend/integrations/credentials_store.py b/autogpt_platform/backend/backend/integrations/credentials_store.py index 09849536c6..847b20fa6c 100644 --- a/autogpt_platform/backend/backend/integrations/credentials_store.py +++ b/autogpt_platform/backend/backend/integrations/credentials_store.py @@ -60,6 +60,13 @@ openai_credentials = APIKeyCredentials( title="Use Credits for OpenAI", expires_at=None, ) +aiml_api_credentials = APIKeyCredentials( + id="aad82a89-9794-4ebb-977f-d736aa5260a3", + provider="aiml_api", + api_key=SecretStr(settings.secrets.aiml_api_key), + title="Use Credits for AI/ML API", + expires_at=None, +) anthropic_credentials = APIKeyCredentials( id="24e5d942-d9e3-4798-8151-90143ee55629", provider="anthropic", @@ -191,6 +198,7 @@ DEFAULT_CREDENTIALS = [ ideogram_credentials, replicate_credentials, openai_credentials, + aiml_api_credentials, anthropic_credentials, groq_credentials, did_credentials, @@ -252,6 +260,8 @@ class IntegrationCredentialsStore: all_credentials.append(replicate_credentials) if settings.secrets.openai_api_key: all_credentials.append(openai_credentials) + if settings.secrets.aiml_api_key: + all_credentials.append(aiml_api_credentials) if settings.secrets.anthropic_api_key: all_credentials.append(anthropic_credentials) if settings.secrets.did_api_key: diff --git a/autogpt_platform/backend/backend/integrations/providers.py b/autogpt_platform/backend/backend/integrations/providers.py index bd054641b8..30444c1ea3 100644 --- a/autogpt_platform/backend/backend/integrations/providers.py +++ b/autogpt_platform/backend/backend/integrations/providers.py @@ -3,6 +3,7 @@ from enum import Enum # --8<-- [start:ProviderName] class ProviderName(str, Enum): + AIML_API = "aiml_api" ANTHROPIC = "anthropic" APOLLO = "apollo" COMPASS = "compass" diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py index 3ae35d6794..c349f0c804 100644 --- a/autogpt_platform/backend/backend/util/settings.py +++ b/autogpt_platform/backend/backend/util/settings.py @@ -385,6 +385,7 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings): ) openai_api_key: str = Field(default="", description="OpenAI API key") + aiml_api_key: str = Field(default="", description="'AI/ML API' key") anthropic_api_key: str = Field(default="", description="Anthropic API key") groq_api_key: str = Field(default="", description="Groq API key") open_router_api_key: str = Field(default="", description="Open Router API Key") diff --git a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx index 80e34a134c..3cceb3c6b3 100644 --- a/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx +++ b/autogpt_platform/frontend/src/app/(platform)/profile/(user)/integrations/page.tsx @@ -103,6 +103,7 @@ export default function UserIntegrationsPage() { "6b9fc200-4726-4973-86c9-cd526f5ce5db", // Replicate "53c25cb8-e3ee-465c-a4d1-e75a4c899c2a", // OpenAI "24e5d942-d9e3-4798-8151-90143ee55629", // Anthropic + "aad82a89-9794-4ebb-977f-d736aa5260a3", // AI/ML "4ec22295-8f97-4dd1-b42b-2c6957a02545", // Groq "7f7b0654-c36b-4565-8fa7-9a52575dfae2", // D-ID "7f26de70-ba0d-494e-ba76-238e65e7b45f", // Jina diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx index 83e337efe9..690f58de33 100644 --- a/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx +++ b/autogpt_platform/frontend/src/components/integrations/credentials-input.tsx @@ -56,6 +56,7 @@ export const providerIcons: Record< CredentialsProviderName, React.FC<{ className?: string }> > = { + aiml_api: fallbackIcon, anthropic: fallbackIcon, apollo: fallbackIcon, e2b: fallbackIcon, diff --git a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx index 91bdc40f46..f74fc3c7c5 100644 --- a/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx +++ b/autogpt_platform/frontend/src/components/integrations/credentials-provider.tsx @@ -18,6 +18,7 @@ const CREDENTIALS_PROVIDER_NAMES = Object.values( // --8<-- [start:CredentialsProviderNames] const providerDisplayNames: Record = { + aiml_api: "AI/ML", anthropic: "Anthropic", apollo: "Apollo", discord: "Discord", diff --git a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts index 635fa44e2d..873a6c735f 100644 --- a/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts +++ b/autogpt_platform/frontend/src/lib/autogpt-server-api/types.ts @@ -149,6 +149,7 @@ export type Credentials = // --8<-- [start:BlockIOCredentialsSubSchema] export const PROVIDER_NAMES = { + AIML_API: "aiml_api", ANTHROPIC: "anthropic", APOLLO: "apollo", D_ID: "d_id", diff --git a/docs/content/index.md b/docs/content/index.md index 243b56ea41..cd69844364 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -66,6 +66,7 @@ The platform comes pre-integrated with cutting-edge LLM providers: - OpenAI - Anthropic +- AI/ML API - Groq - Llama