mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-04-08 03:00:28 -04:00
refactor(platform): address autogpt-reviewer feedback (batch 1)
- cost_tracking.py: replace `Any` types with NodeExecutionEntry + Block
- Extract usd_to_microdollars utility in platform_cost.py, used by
cost_tracking.py and copilot/token_tracking.py.
- llm.py: extract x-total-cost header parsing to extract_openrouter_cost()
helper + 8 unit tests covering present/absent/empty/non-numeric/zero
cases. Previously untested blocker.
- token_tracking.py: extract COPILOT_BLOCK_ID, COPILOT_CREDENTIAL_ID
constants + _copilot_block_name() helper (clearer than inline
f"copilot:{log_prefix.strip(' []')}".rstrip(":")).
- platform_cost.py: cap by_provider query at LIMIT 500 (defensive bound).
- TrackingBadge.tsx: drop dark: classes per frontend convention, add
"items" badge color.
- PlatformCostContent.tsx: drop dark: classes from error banner,
add role="tablist"/role="tab"/aria-selected to tabs, add htmlFor
to filter input labels.
- admin/layout.tsx: Receipt icon moved from lucide-react to phosphor.
- ProviderTable.tsx: add "(unsaved)" label to Rate column header to
signal per-session only.
This commit is contained in:
@@ -722,6 +722,26 @@ def convert_openai_tool_fmt_to_anthropic(
|
||||
return anthropic_tools
|
||||
|
||||
|
||||
def extract_openrouter_cost(response) -> float | None:
|
||||
"""Extract OpenRouter's `x-total-cost` header from an OpenAI SDK response.
|
||||
|
||||
OpenRouter returns the per-request USD cost in a response header. The
|
||||
OpenAI SDK exposes the raw httpx response via an undocumented `_response`
|
||||
attribute. If the SDK ever drops or renames that attribute, we silently
|
||||
degrade to no cost tracking rather than raising.
|
||||
"""
|
||||
try:
|
||||
raw_resp = getattr(response, "_response", None)
|
||||
if raw_resp is None or not hasattr(raw_resp, "headers"):
|
||||
return None
|
||||
cost_header = raw_resp.headers.get("x-total-cost")
|
||||
if not cost_header:
|
||||
return None
|
||||
return float(cost_header)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
def extract_openai_reasoning(response) -> str | None:
|
||||
"""Extract reasoning from OpenAI-compatible response if available."""
|
||||
"""Note: This will likely not working since the reasoning is not present in another Response API"""
|
||||
@@ -1046,16 +1066,6 @@ async def llm_call(
|
||||
tool_calls = extract_openai_tool_calls(response)
|
||||
reasoning = extract_openai_reasoning(response)
|
||||
|
||||
cost = None
|
||||
try:
|
||||
raw_resp = getattr(response, "_response", None)
|
||||
if raw_resp and hasattr(raw_resp, "headers"):
|
||||
cost_header = raw_resp.headers.get("x-total-cost")
|
||||
if cost_header:
|
||||
cost = float(cost_header)
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
return LLMResponse(
|
||||
raw_response=response.choices[0].message,
|
||||
prompt=prompt,
|
||||
@@ -1064,7 +1074,7 @@ async def llm_call(
|
||||
prompt_tokens=response.usage.prompt_tokens if response.usage else 0,
|
||||
completion_tokens=response.usage.completion_tokens if response.usage else 0,
|
||||
reasoning=reasoning,
|
||||
provider_cost=cost,
|
||||
provider_cost=extract_openrouter_cost(response),
|
||||
)
|
||||
elif provider == "llama_api":
|
||||
tools_param = tools if tools else openai.NOT_GIVEN
|
||||
|
||||
@@ -987,3 +987,51 @@ class TestLlmModelMissing:
|
||||
assert (
|
||||
llm.LlmModel("extra/google/gemini-2.5-pro") == llm.LlmModel.GEMINI_2_5_PRO
|
||||
)
|
||||
|
||||
|
||||
class TestExtractOpenRouterCost:
|
||||
"""Tests for extract_openrouter_cost — the x-total-cost header parser."""
|
||||
|
||||
def _mk_response(self, headers: dict | None):
|
||||
response = MagicMock()
|
||||
if headers is None:
|
||||
response._response = None
|
||||
else:
|
||||
raw = MagicMock()
|
||||
raw.headers = headers
|
||||
response._response = raw
|
||||
return response
|
||||
|
||||
def test_extracts_numeric_cost(self):
|
||||
response = self._mk_response({"x-total-cost": "0.0042"})
|
||||
assert llm.extract_openrouter_cost(response) == 0.0042
|
||||
|
||||
def test_returns_none_when_header_missing(self):
|
||||
response = self._mk_response({})
|
||||
assert llm.extract_openrouter_cost(response) is None
|
||||
|
||||
def test_returns_none_when_header_empty_string(self):
|
||||
response = self._mk_response({"x-total-cost": ""})
|
||||
assert llm.extract_openrouter_cost(response) is None
|
||||
|
||||
def test_returns_none_when_header_non_numeric(self):
|
||||
response = self._mk_response({"x-total-cost": "not-a-number"})
|
||||
assert llm.extract_openrouter_cost(response) is None
|
||||
|
||||
def test_returns_none_when_no_response_attr(self):
|
||||
response = MagicMock(spec=[]) # no _response attr
|
||||
assert llm.extract_openrouter_cost(response) is None
|
||||
|
||||
def test_returns_none_when_raw_is_none(self):
|
||||
response = self._mk_response(None)
|
||||
assert llm.extract_openrouter_cost(response) is None
|
||||
|
||||
def test_returns_none_when_raw_has_no_headers(self):
|
||||
response = MagicMock()
|
||||
response._response = MagicMock(spec=[]) # no headers attr
|
||||
assert llm.extract_openrouter_cost(response) is None
|
||||
|
||||
def test_returns_zero_for_zero_cost(self):
|
||||
"""Zero-cost is a valid value (free tier) and must not become None."""
|
||||
response = self._mk_response({"x-total-cost": "0"})
|
||||
assert llm.extract_openrouter_cost(response) == 0.0
|
||||
|
||||
@@ -13,9 +13,9 @@ import asyncio
|
||||
import logging
|
||||
|
||||
from backend.data.platform_cost import (
|
||||
MICRODOLLARS_PER_USD,
|
||||
PlatformCostEntry,
|
||||
log_platform_cost_safe,
|
||||
usd_to_microdollars,
|
||||
)
|
||||
|
||||
from .model import ChatSession, Usage
|
||||
@@ -23,6 +23,11 @@ from .rate_limit import record_token_usage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Identifiers used by PlatformCostLog for copilot turns (not tied to a real
|
||||
# block/credential in the block_cost_config or credentials_store tables).
|
||||
COPILOT_BLOCK_ID = "copilot"
|
||||
COPILOT_CREDENTIAL_ID = "copilot_system"
|
||||
|
||||
# Hold strong references to in-flight log tasks to prevent GC.
|
||||
_pending_log_tasks: set[asyncio.Task] = set()
|
||||
|
||||
@@ -33,6 +38,13 @@ def _schedule_log(entry: PlatformCostEntry) -> None:
|
||||
task.add_done_callback(_pending_log_tasks.discard)
|
||||
|
||||
|
||||
def _copilot_block_name(log_prefix: str) -> str:
|
||||
"""Turn a log prefix like ``"[SDK]"`` into a stable block_name
|
||||
``"copilot:SDK"``. Empty prefix becomes just ``"copilot"``."""
|
||||
tag = log_prefix.strip(" []")
|
||||
return f"{COPILOT_BLOCK_ID}:{tag}" if tag else COPILOT_BLOCK_ID
|
||||
|
||||
|
||||
async def persist_and_record_usage(
|
||||
*,
|
||||
session: ChatSession | None,
|
||||
@@ -124,9 +136,7 @@ async def persist_and_record_usage(
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
cost_microdollars = (
|
||||
round(cost_float * MICRODOLLARS_PER_USD) if cost_float is not None else None
|
||||
)
|
||||
cost_microdollars = usd_to_microdollars(cost_float)
|
||||
session_id = session.session_id if session else None
|
||||
|
||||
if cost_float is not None:
|
||||
@@ -140,10 +150,10 @@ async def persist_and_record_usage(
|
||||
PlatformCostEntry(
|
||||
user_id=user_id,
|
||||
graph_exec_id=session_id,
|
||||
block_id="copilot",
|
||||
block_name=f"copilot:{log_prefix.strip(' []')}".rstrip(":"),
|
||||
block_id=COPILOT_BLOCK_ID,
|
||||
block_name=_copilot_block_name(log_prefix),
|
||||
provider=provider,
|
||||
credential_id="copilot_system",
|
||||
credential_id=COPILOT_CREDENTIAL_ID,
|
||||
cost_microdollars=cost_microdollars,
|
||||
input_tokens=prompt_tokens,
|
||||
output_tokens=completion_tokens,
|
||||
|
||||
@@ -13,6 +13,13 @@ logger = logging.getLogger(__name__)
|
||||
MICRODOLLARS_PER_USD = 1_000_000
|
||||
|
||||
|
||||
def usd_to_microdollars(cost_usd: float | None) -> int | None:
|
||||
"""Convert a USD amount (float) to microdollars (int). None-safe."""
|
||||
if cost_usd is None:
|
||||
return None
|
||||
return round(cost_usd * MICRODOLLARS_PER_USD)
|
||||
|
||||
|
||||
class PlatformCostEntry(BaseModel):
|
||||
user_id: str
|
||||
graph_exec_id: str | None = None
|
||||
@@ -199,6 +206,7 @@ async def get_platform_cost_dashboard(
|
||||
GROUP BY p."provider",
|
||||
COALESCE(p."trackingType", p."metadata"->>'tracking_type')
|
||||
ORDER BY total_cost DESC
|
||||
LIMIT 500
|
||||
""",
|
||||
*params_p,
|
||||
),
|
||||
|
||||
@@ -4,12 +4,13 @@ import asyncio
|
||||
import logging
|
||||
from typing import Any, cast
|
||||
|
||||
from backend.blocks._base import BlockSchema
|
||||
from backend.blocks._base import Block, BlockSchema
|
||||
from backend.data.execution import NodeExecutionEntry
|
||||
from backend.data.model import NodeExecutionStats
|
||||
from backend.data.platform_cost import (
|
||||
MICRODOLLARS_PER_USD,
|
||||
PlatformCostEntry,
|
||||
log_platform_cost_safe,
|
||||
usd_to_microdollars,
|
||||
)
|
||||
from backend.executor.utils import block_usage_cost
|
||||
from backend.integrations.credentials_store import is_system_credential
|
||||
@@ -84,8 +85,8 @@ def resolve_tracking(
|
||||
|
||||
|
||||
async def log_system_credential_cost(
|
||||
node_exec: Any,
|
||||
block: Any,
|
||||
node_exec: NodeExecutionEntry,
|
||||
block: Block,
|
||||
stats: NodeExecutionStats,
|
||||
) -> None:
|
||||
"""Check if a system credential was used and log the platform cost.
|
||||
@@ -132,8 +133,8 @@ async def log_system_credential_cost(
|
||||
# For other types (items, characters, per_run, ...) the
|
||||
# provider_cost field holds the raw amount, not a dollar value.
|
||||
cost_microdollars = None
|
||||
if tracking_type == "cost_usd" and stats.provider_cost is not None:
|
||||
cost_microdollars = round(stats.provider_cost * MICRODOLLARS_PER_USD)
|
||||
if tracking_type == "cost_usd":
|
||||
cost_microdollars = usd_to_microdollars(stats.provider_cost)
|
||||
|
||||
meta: dict[str, Any] = {
|
||||
"tracking_type": tracking_type,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Sidebar } from "@/components/__legacy__/Sidebar";
|
||||
import { Users, DollarSign, UserSearch, FileText, Receipt } from "lucide-react";
|
||||
import { Gauge } from "@phosphor-icons/react/dist/ssr";
|
||||
import { Users, DollarSign, UserSearch, FileText } from "lucide-react";
|
||||
import { Gauge, Receipt } from "@phosphor-icons/react/dist/ssr";
|
||||
|
||||
import { IconSliders } from "@/components/__legacy__/ui/icons";
|
||||
|
||||
|
||||
@@ -129,8 +129,11 @@ function PlatformCostContent({ searchParams }: Props) {
|
||||
<div className="flex flex-col gap-6">
|
||||
<div className="flex flex-wrap items-end gap-3 rounded-lg border p-4">
|
||||
<div className="flex flex-col gap-1">
|
||||
<label className="text-sm text-muted-foreground">Start Date</label>
|
||||
<label htmlFor="start-date" className="text-sm text-muted-foreground">
|
||||
Start Date
|
||||
</label>
|
||||
<input
|
||||
id="start-date"
|
||||
type="datetime-local"
|
||||
className="rounded border px-3 py-1.5 text-sm"
|
||||
value={startInput}
|
||||
@@ -138,8 +141,11 @@ function PlatformCostContent({ searchParams }: Props) {
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1">
|
||||
<label className="text-sm text-muted-foreground">End Date</label>
|
||||
<label htmlFor="end-date" className="text-sm text-muted-foreground">
|
||||
End Date
|
||||
</label>
|
||||
<input
|
||||
id="end-date"
|
||||
type="datetime-local"
|
||||
className="rounded border px-3 py-1.5 text-sm"
|
||||
value={endInput}
|
||||
@@ -147,8 +153,14 @@ function PlatformCostContent({ searchParams }: Props) {
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1">
|
||||
<label className="text-sm text-muted-foreground">Provider</label>
|
||||
<label
|
||||
htmlFor="provider-filter"
|
||||
className="text-sm text-muted-foreground"
|
||||
>
|
||||
Provider
|
||||
</label>
|
||||
<input
|
||||
id="provider-filter"
|
||||
type="text"
|
||||
placeholder="e.g. openai"
|
||||
className="rounded border px-3 py-1.5 text-sm"
|
||||
@@ -157,8 +169,14 @@ function PlatformCostContent({ searchParams }: Props) {
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1">
|
||||
<label className="text-sm text-muted-foreground">User ID</label>
|
||||
<label
|
||||
htmlFor="user-id-filter"
|
||||
className="text-sm text-muted-foreground"
|
||||
>
|
||||
User ID
|
||||
</label>
|
||||
<input
|
||||
id="user-id-filter"
|
||||
type="text"
|
||||
placeholder="Filter by user"
|
||||
className="rounded border px-3 py-1.5 text-sm"
|
||||
@@ -175,7 +193,7 @@ function PlatformCostContent({ searchParams }: Props) {
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="rounded-lg border border-red-300 bg-red-50 p-4 text-sm text-red-700 dark:border-red-700 dark:bg-red-900/20 dark:text-red-400">
|
||||
<div className="rounded-lg border border-red-300 bg-red-50 p-4 text-sm text-red-700">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
@@ -209,10 +227,12 @@ function PlatformCostContent({ searchParams }: Props) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex gap-2 border-b">
|
||||
<div role="tablist" className="flex gap-2 border-b">
|
||||
{["overview", "by-user", "logs"].map((t) => (
|
||||
<button
|
||||
key={t}
|
||||
role="tab"
|
||||
aria-selected={tab === t}
|
||||
onClick={() => updateUrl({ tab: t, page: "1" })}
|
||||
className={`px-4 py-2 text-sm font-medium ${tab === t ? "border-b-2 border-blue-600 text-blue-600" : "text-muted-foreground hover:text-foreground"}`}
|
||||
>
|
||||
|
||||
@@ -27,7 +27,9 @@ function ProviderTable({ data, rateOverrides, onRateOverride }: Props) {
|
||||
<th className="px-4 py-3 text-right">Requests</th>
|
||||
<th className="px-4 py-3 text-right">Known Cost</th>
|
||||
<th className="px-4 py-3 text-right">Est. Cost</th>
|
||||
<th className="px-4 py-3 text-right">Rate</th>
|
||||
<th className="px-4 py-3 text-right" title="Per-session only">
|
||||
Rate <span className="text-[10px] font-normal">(unsaved)</span>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
||||
@@ -1,17 +1,13 @@
|
||||
function trackingBadge(trackingType: string | null | undefined) {
|
||||
const colors: Record<string, string> = {
|
||||
cost_usd:
|
||||
"bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-400",
|
||||
tokens: "bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-400",
|
||||
duration_seconds:
|
||||
"bg-orange-100 text-orange-800 dark:bg-orange-900/30 dark:text-orange-400",
|
||||
characters:
|
||||
"bg-purple-100 text-purple-800 dark:bg-purple-900/30 dark:text-purple-400",
|
||||
sandbox_seconds:
|
||||
"bg-orange-100 text-orange-800 dark:bg-orange-900/30 dark:text-orange-400",
|
||||
walltime_seconds:
|
||||
"bg-orange-100 text-orange-800 dark:bg-orange-900/30 dark:text-orange-400",
|
||||
per_run: "bg-gray-100 text-gray-800 dark:bg-gray-900/30 dark:text-gray-400",
|
||||
cost_usd: "bg-green-100 text-green-800",
|
||||
tokens: "bg-blue-100 text-blue-800",
|
||||
duration_seconds: "bg-orange-100 text-orange-800",
|
||||
characters: "bg-purple-100 text-purple-800",
|
||||
sandbox_seconds: "bg-orange-100 text-orange-800",
|
||||
walltime_seconds: "bg-orange-100 text-orange-800",
|
||||
items: "bg-pink-100 text-pink-800",
|
||||
per_run: "bg-gray-100 text-gray-800",
|
||||
};
|
||||
const label = trackingType || "per_run";
|
||||
return (
|
||||
|
||||
Reference in New Issue
Block a user