mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-04-30 03:00:41 -04:00
feat(platform): Add execution analytics admin endpoint with feature flag bypass (#11327)
This PR adds a comprehensive execution analytics admin endpoint that generates AI-powered activity summaries and correctness scores for graph executions, with proper feature flag bypass for admin use. ### Changes 🏗️ **Backend Changes:** - Added admin endpoint: `/api/executions/admin/execution_analytics` - Implemented feature flag bypass with `skip_feature_flag=True` parameter for admin operations - Fixed async database client usage (`get_db_async_client`) to resolve async/await errors - Added batch processing with configurable size limits to handle large datasets - Comprehensive error handling and logging for troubleshooting - Renamed entire feature from "Activity Backfill" to "Execution Analytics" for clarity **Frontend Changes:** - Created clean admin UI for execution analytics generation at `/admin/execution-analytics` - Built form with graph ID input, model selection dropdown, and optional filters - Implemented results table with status badges and detailed execution information - Added CSV export functionality for analytics results - Integrated with generated TypeScript API client for proper authentication - Added proper error handling with toast notifications and loading states **Database & API:** - Fixed critical async/await issue by switching from sync to async database client - Updated router configuration and endpoint naming for consistency - Generated proper TypeScript types and API client integration - Applied feature flag filtering at API level while bypassing for admin operations ### Checklist 📋 #### For code changes: - [x] I have clearly listed my changes in the PR description - [x] I have made a test plan - [x] I have tested my changes according to the test plan: **Test Plan:** - [x] Admin can access execution analytics page at `/admin/execution-analytics` - [x] Form validation works correctly (requires graph ID, validates inputs) - [x] API endpoint `/api/executions/admin/execution_analytics` responds correctly - [x] Authentication works properly through generated API client - [x] Analytics generation works with different LLM models (gpt-4o-mini, gpt-4o, etc.) - [x] Results display correctly with appropriate status badges (success/failed/skipped) - [x] CSV export functionality downloads correct data - [x] Error handling displays appropriate toast messages - [x] Feature flag bypass works for admin users (generates analytics regardless of user flags) - [x] Batch processing handles multiple executions correctly - [x] Loading states show proper feedback during processing #### For configuration changes: - [x] `.env.default` is updated or already compatible with my changes - [x] `docker-compose.yml` is updated or already compatible with my changes - [x] No configuration changes required for this feature **Related to:** PR #11325 (base correctness score functionality) 🤖 Generated with [Claude Code](https://claude.ai/code) --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Zamil Majdy <majdyz@users.noreply.github.com>
This commit is contained in:
Zamil Majdy
@@ -0,0 +1,319 @@
|
||||
"use client";
|
||||
|
||||
import React, { useState } from "react";
|
||||
import { Button } from "@/components/atoms/Button/Button";
|
||||
import { Text } from "@/components/atoms/Text/Text";
|
||||
import { Badge } from "@/components/atoms/Badge/Badge";
|
||||
import { DownloadIcon, EyeIcon, CopyIcon } from "@phosphor-icons/react";
|
||||
import { useToast } from "@/components/molecules/Toast/use-toast";
|
||||
import type { ExecutionAnalyticsResponse } from "@/app/api/__generated__/models/executionAnalyticsResponse";
|
||||
|
||||
interface Props {
|
||||
results: ExecutionAnalyticsResponse;
|
||||
}
|
||||
|
||||
export function AnalyticsResultsTable({ results }: Props) {
|
||||
const [expandedRows, setExpandedRows] = useState<Set<string>>(new Set());
|
||||
const { toast } = useToast();
|
||||
|
||||
const createCopyableId = (value: string, label: string) => (
|
||||
<div
|
||||
className="group flex cursor-pointer items-center gap-1 font-mono text-xs text-gray-500 hover:text-gray-700"
|
||||
onClick={() => {
|
||||
navigator.clipboard.writeText(value);
|
||||
toast({
|
||||
title: "Copied",
|
||||
description: `${label} copied to clipboard`,
|
||||
});
|
||||
}}
|
||||
title={`Click to copy ${label.toLowerCase()}`}
|
||||
>
|
||||
{value.substring(0, 8)}...
|
||||
<CopyIcon className="h-3 w-3 opacity-0 transition-opacity group-hover:opacity-100" />
|
||||
</div>
|
||||
);
|
||||
|
||||
const toggleRowExpansion = (execId: string) => {
|
||||
const newExpanded = new Set(expandedRows);
|
||||
if (newExpanded.has(execId)) {
|
||||
newExpanded.delete(execId);
|
||||
} else {
|
||||
newExpanded.add(execId);
|
||||
}
|
||||
setExpandedRows(newExpanded);
|
||||
};
|
||||
|
||||
const exportToCSV = () => {
|
||||
const headers = [
|
||||
"Agent ID",
|
||||
"Version",
|
||||
"User ID",
|
||||
"Execution ID",
|
||||
"Status",
|
||||
"Score",
|
||||
"Summary Text",
|
||||
"Error Message",
|
||||
];
|
||||
|
||||
const csvData = results.results.map((result) => [
|
||||
result.agent_id,
|
||||
result.version_id.toString(),
|
||||
result.user_id,
|
||||
result.exec_id,
|
||||
result.status,
|
||||
result.score?.toString() || "",
|
||||
`"${(result.summary_text || "").replace(/"/g, '""')}"`, // Escape quotes in summary
|
||||
`"${(result.error_message || "").replace(/"/g, '""')}"`, // Escape quotes in error
|
||||
]);
|
||||
|
||||
const csvContent = [
|
||||
headers.join(","),
|
||||
...csvData.map((row) => row.join(",")),
|
||||
].join("\n");
|
||||
|
||||
const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" });
|
||||
const link = document.createElement("a");
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
link.setAttribute("href", url);
|
||||
link.setAttribute(
|
||||
"download",
|
||||
`execution-analytics-results-${new Date().toISOString().split("T")[0]}.csv`,
|
||||
);
|
||||
link.style.visibility = "hidden";
|
||||
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
};
|
||||
|
||||
const getStatusBadge = (status: string) => {
|
||||
switch (status) {
|
||||
case "success":
|
||||
return <Badge variant="success">Success</Badge>;
|
||||
case "failed":
|
||||
return <Badge variant="error">Failed</Badge>;
|
||||
case "skipped":
|
||||
return <Badge variant="info">Skipped</Badge>;
|
||||
default:
|
||||
return <Badge variant="info">{status}</Badge>;
|
||||
}
|
||||
};
|
||||
|
||||
const getScoreDisplay = (score?: number) => {
|
||||
if (score === undefined || score === null) return "—";
|
||||
|
||||
const percentage = Math.round(score * 100);
|
||||
let colorClass = "";
|
||||
|
||||
if (score >= 0.8) colorClass = "text-green-600";
|
||||
else if (score >= 0.6) colorClass = "text-yellow-600";
|
||||
else if (score >= 0.4) colorClass = "text-orange-600";
|
||||
else colorClass = "text-red-600";
|
||||
|
||||
return <span className={colorClass}>{percentage}%</span>;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{/* Summary Stats */}
|
||||
<div className="rounded-lg bg-gray-50 p-4">
|
||||
<Text variant="h3" className="mb-3">
|
||||
Analytics Summary
|
||||
</Text>
|
||||
<div className="grid grid-cols-2 gap-4 text-sm md:grid-cols-5">
|
||||
<div>
|
||||
<Text variant="body" className="text-gray-600">
|
||||
Total Executions:
|
||||
</Text>
|
||||
<Text variant="h4" className="font-semibold">
|
||||
{results.total_executions}
|
||||
</Text>
|
||||
</div>
|
||||
<div>
|
||||
<Text variant="body" className="text-gray-600">
|
||||
Processed:
|
||||
</Text>
|
||||
<Text variant="h4" className="font-semibold">
|
||||
{results.processed_executions}
|
||||
</Text>
|
||||
</div>
|
||||
<div>
|
||||
<Text variant="body" className="text-gray-600">
|
||||
Successful:
|
||||
</Text>
|
||||
<Text variant="h4" className="font-semibold text-green-600">
|
||||
{results.successful_analytics}
|
||||
</Text>
|
||||
</div>
|
||||
<div>
|
||||
<Text variant="body" className="text-gray-600">
|
||||
Failed:
|
||||
</Text>
|
||||
<Text variant="h4" className="font-semibold text-red-600">
|
||||
{results.failed_analytics}
|
||||
</Text>
|
||||
</div>
|
||||
<div>
|
||||
<Text variant="body" className="text-gray-600">
|
||||
Skipped:
|
||||
</Text>
|
||||
<Text variant="h4" className="font-semibold text-gray-600">
|
||||
{results.skipped_executions}
|
||||
</Text>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Export Button */}
|
||||
<div className="flex justify-end">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={exportToCSV}
|
||||
disabled={results.results.length === 0}
|
||||
>
|
||||
<DownloadIcon size={16} className="mr-2" />
|
||||
Export CSV
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* Results Table */}
|
||||
{results.results.length > 0 ? (
|
||||
<div className="overflow-hidden rounded-lg border">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th className="px-4 py-3 text-left">
|
||||
<Text variant="body" className="font-medium text-gray-600">
|
||||
Agent ID
|
||||
</Text>
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left">
|
||||
<Text variant="body" className="font-medium text-gray-600">
|
||||
Version
|
||||
</Text>
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left">
|
||||
<Text variant="body" className="font-medium text-gray-600">
|
||||
User ID
|
||||
</Text>
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left">
|
||||
<Text variant="body" className="font-medium text-gray-600">
|
||||
Execution ID
|
||||
</Text>
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left">
|
||||
<Text variant="body" className="font-medium text-gray-600">
|
||||
Status
|
||||
</Text>
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left">
|
||||
<Text variant="body" className="font-medium text-gray-600">
|
||||
Score
|
||||
</Text>
|
||||
</th>
|
||||
<th className="px-4 py-3 text-left">
|
||||
<Text variant="body" className="font-medium text-gray-600">
|
||||
Actions
|
||||
</Text>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{results.results.map((result) => (
|
||||
<React.Fragment key={result.exec_id}>
|
||||
<tr className="hover:bg-gray-50">
|
||||
<td className="px-4 py-3">
|
||||
{createCopyableId(result.agent_id, "Agent ID")}
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<Text variant="body">{result.version_id}</Text>
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
{createCopyableId(result.user_id, "User ID")}
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
{createCopyableId(result.exec_id, "Execution ID")}
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
{getStatusBadge(result.status)}
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
{getScoreDisplay(
|
||||
typeof result.score === "number"
|
||||
? result.score
|
||||
: undefined,
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
{(result.summary_text || result.error_message) && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="small"
|
||||
onClick={() => toggleRowExpansion(result.exec_id)}
|
||||
>
|
||||
<EyeIcon size={16} />
|
||||
</Button>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
{expandedRows.has(result.exec_id) && (
|
||||
<tr>
|
||||
<td colSpan={7} className="bg-gray-50 px-4 py-3">
|
||||
<div className="space-y-3">
|
||||
{result.summary_text && (
|
||||
<div>
|
||||
<Text
|
||||
variant="body"
|
||||
className="mb-1 font-medium text-gray-700"
|
||||
>
|
||||
Summary:
|
||||
</Text>
|
||||
<Text
|
||||
variant="body"
|
||||
className="leading-relaxed text-gray-600"
|
||||
>
|
||||
{result.summary_text}
|
||||
</Text>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{result.error_message && (
|
||||
<div>
|
||||
<Text
|
||||
variant="body"
|
||||
className="mb-1 font-medium text-red-700"
|
||||
>
|
||||
Error:
|
||||
</Text>
|
||||
<Text
|
||||
variant="body"
|
||||
className="leading-relaxed text-red-600"
|
||||
>
|
||||
{result.error_message}
|
||||
</Text>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
)}
|
||||
</React.Fragment>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="py-8 text-center">
|
||||
<Text variant="body" className="text-gray-500">
|
||||
No executions were processed.
|
||||
</Text>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { Button } from "@/components/atoms/Button/Button";
|
||||
import { Input } from "@/components/__legacy__/ui/input";
|
||||
import { Label } from "@/components/__legacy__/ui/label";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/__legacy__/ui/select";
|
||||
import { useToast } from "@/components/molecules/Toast/use-toast";
|
||||
import { usePostV2GenerateExecutionAnalytics } from "@/app/api/__generated__/endpoints/admin/admin";
|
||||
import type { ExecutionAnalyticsRequest } from "@/app/api/__generated__/models/executionAnalyticsRequest";
|
||||
import type { ExecutionAnalyticsResponse } from "@/app/api/__generated__/models/executionAnalyticsResponse";
|
||||
|
||||
// Local interface for form state to simplify handling
|
||||
interface FormData {
|
||||
graph_id: string;
|
||||
graph_version?: number;
|
||||
user_id?: string;
|
||||
created_after?: string;
|
||||
model_name: string;
|
||||
batch_size: number;
|
||||
}
|
||||
import { AnalyticsResultsTable } from "./AnalyticsResultsTable";
|
||||
|
||||
const MODEL_OPTIONS = [
|
||||
{ value: "gpt-4o-mini", label: "GPT-4o Mini (Recommended)" },
|
||||
{ value: "gpt-4o", label: "GPT-4o" },
|
||||
{ value: "gpt-4-turbo", label: "GPT-4 Turbo" },
|
||||
{ value: "gpt-4.1", label: "GPT-4.1" },
|
||||
{ value: "gpt-4.1-mini", label: "GPT-4.1 Mini" },
|
||||
];
|
||||
|
||||
export function ExecutionAnalyticsForm() {
|
||||
const [results, setResults] = useState<ExecutionAnalyticsResponse | null>(
|
||||
null,
|
||||
);
|
||||
const { toast } = useToast();
|
||||
|
||||
const generateAnalytics = usePostV2GenerateExecutionAnalytics({
|
||||
mutation: {
|
||||
onSuccess: (res) => {
|
||||
if (res.status !== 200) {
|
||||
throw new Error("Something went wrong!");
|
||||
}
|
||||
const result = res.data;
|
||||
setResults(result);
|
||||
toast({
|
||||
title: "Analytics Generated",
|
||||
description: `Processed ${result.processed_executions} executions. ${result.successful_analytics} successful, ${result.failed_analytics} failed, ${result.skipped_executions} skipped.`,
|
||||
variant: "default",
|
||||
});
|
||||
},
|
||||
onError: (error: any) => {
|
||||
console.error("Analytics generation error:", error);
|
||||
toast({
|
||||
title: "Analytics Generation Failed",
|
||||
description:
|
||||
error?.message || error?.detail || "An unexpected error occurred",
|
||||
variant: "destructive",
|
||||
});
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const [formData, setFormData] = useState<FormData>({
|
||||
graph_id: "",
|
||||
model_name: "gpt-4o-mini",
|
||||
batch_size: 10, // Fixed internal value
|
||||
});
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
|
||||
if (!formData.graph_id.trim()) {
|
||||
toast({
|
||||
title: "Validation Error",
|
||||
description: "Graph ID is required",
|
||||
variant: "destructive",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
setResults(null);
|
||||
|
||||
// Prepare the request payload
|
||||
const payload: ExecutionAnalyticsRequest = {
|
||||
graph_id: formData.graph_id.trim(),
|
||||
model_name: formData.model_name,
|
||||
batch_size: formData.batch_size,
|
||||
};
|
||||
|
||||
if (formData.graph_version) {
|
||||
payload.graph_version = formData.graph_version;
|
||||
}
|
||||
|
||||
if (formData.user_id?.trim()) {
|
||||
payload.user_id = formData.user_id.trim();
|
||||
}
|
||||
|
||||
if (
|
||||
formData.created_after &&
|
||||
typeof formData.created_after === "string" &&
|
||||
formData.created_after.trim()
|
||||
) {
|
||||
payload.created_after = new Date(formData.created_after.trim());
|
||||
}
|
||||
|
||||
generateAnalytics.mutate({ data: payload });
|
||||
};
|
||||
|
||||
const handleInputChange = (field: keyof FormData, value: any) => {
|
||||
setFormData((prev: FormData) => ({ ...prev, [field]: value }));
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<form onSubmit={handleSubmit} className="space-y-4">
|
||||
<div className="grid grid-cols-1 gap-4 md:grid-cols-2">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="graph_id">
|
||||
Graph ID <span className="text-red-500">*</span>
|
||||
</Label>
|
||||
<Input
|
||||
id="graph_id"
|
||||
value={formData.graph_id}
|
||||
onChange={(e) => handleInputChange("graph_id", e.target.value)}
|
||||
placeholder="Enter graph/agent ID"
|
||||
required
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="graph_version">Graph Version</Label>
|
||||
<Input
|
||||
id="graph_version"
|
||||
type="number"
|
||||
value={formData.graph_version || ""}
|
||||
onChange={(e) =>
|
||||
handleInputChange(
|
||||
"graph_version",
|
||||
e.target.value ? parseInt(e.target.value) : undefined,
|
||||
)
|
||||
}
|
||||
placeholder="Optional - leave empty for all versions"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="user_id">User ID</Label>
|
||||
<Input
|
||||
id="user_id"
|
||||
value={formData.user_id || ""}
|
||||
onChange={(e) => handleInputChange("user_id", e.target.value)}
|
||||
placeholder="Optional - leave empty for all users"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="created_after">Created After</Label>
|
||||
<Input
|
||||
id="created_after"
|
||||
type="datetime-local"
|
||||
value={formData.created_after || ""}
|
||||
onChange={(e) =>
|
||||
handleInputChange("created_after", e.target.value)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="model_name">AI Model</Label>
|
||||
<Select
|
||||
value={formData.model_name}
|
||||
onValueChange={(value) => handleInputChange("model_name", value)}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Select AI model" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{MODEL_OPTIONS.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex justify-end">
|
||||
<Button
|
||||
variant="primary"
|
||||
size="large"
|
||||
type="submit"
|
||||
disabled={generateAnalytics.isPending}
|
||||
>
|
||||
{generateAnalytics.isPending
|
||||
? "Processing..."
|
||||
: "Generate Analytics"}
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{results && <AnalyticsResultsTable results={results} />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { withRoleAccess } from "@/lib/withRoleAccess";
|
||||
import { Suspense } from "react";
|
||||
import { ExecutionAnalyticsForm } from "./components/ExecutionAnalyticsForm";
|
||||
|
||||
function ExecutionAnalyticsDashboard() {
|
||||
return (
|
||||
<div className="mx-auto p-6">
|
||||
<div className="flex flex-col gap-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-3xl font-bold">Execution Analytics</h1>
|
||||
<p className="text-gray-500">
|
||||
Generate missing activity summaries and success scores for agent
|
||||
executions
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border bg-white p-6 shadow-sm">
|
||||
<h2 className="mb-4 text-xl font-semibold">Analytics Generation</h2>
|
||||
<p className="mb-6 text-gray-600">
|
||||
This tool will identify completed executions missing activity
|
||||
summaries or success scores and generate them using AI. Only
|
||||
executions that meet the criteria and are missing these fields will
|
||||
be processed.
|
||||
</p>
|
||||
|
||||
<Suspense
|
||||
fallback={<div className="py-10 text-center">Loading...</div>}
|
||||
>
|
||||
<ExecutionAnalyticsForm />
|
||||
</Suspense>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default async function ExecutionAnalyticsPage() {
|
||||
"use server";
|
||||
const withAdminAccess = await withRoleAccess(["admin"]);
|
||||
const ProtectedExecutionAnalyticsDashboard = await withAdminAccess(
|
||||
ExecutionAnalyticsDashboard,
|
||||
);
|
||||
return <ProtectedExecutionAnalyticsDashboard />;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Sidebar } from "@/components/__legacy__/Sidebar";
|
||||
import { Users, DollarSign, UserSearch } from "lucide-react";
|
||||
import { Users, DollarSign, UserSearch, FileText } from "lucide-react";
|
||||
|
||||
import { IconSliders } from "@/components/__legacy__/ui/icons";
|
||||
|
||||
@@ -21,6 +21,11 @@ const sidebarLinkGroups = [
|
||||
href: "/admin/impersonation",
|
||||
icon: <UserSearch className="h-6 w-6" />,
|
||||
},
|
||||
{
|
||||
text: "Execution Analytics",
|
||||
href: "/admin/execution-analytics",
|
||||
icon: <FileText className="h-6 w-6" />,
|
||||
},
|
||||
{
|
||||
text: "Admin User Management",
|
||||
href: "/admin/settings",
|
||||
|
||||
@@ -305,10 +305,62 @@ export function AgentRunDetailsView({
|
||||
</TooltipProvider>
|
||||
</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<CardContent className="space-y-4">
|
||||
<p className="text-sm leading-relaxed text-neutral-700">
|
||||
{run.stats.activity_status}
|
||||
</p>
|
||||
|
||||
{/* Correctness Score */}
|
||||
{typeof run.stats.correctness_score === "number" && (
|
||||
<div className="flex items-center gap-3 rounded-lg bg-neutral-50 p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm font-medium text-neutral-600">
|
||||
Success Estimate:
|
||||
</span>
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="relative h-2 w-16 overflow-hidden rounded-full bg-neutral-200">
|
||||
<div
|
||||
className={`h-full transition-all ${
|
||||
run.stats.correctness_score >= 0.8
|
||||
? "bg-green-500"
|
||||
: run.stats.correctness_score >= 0.6
|
||||
? "bg-yellow-500"
|
||||
: run.stats.correctness_score >= 0.4
|
||||
? "bg-orange-500"
|
||||
: "bg-red-500"
|
||||
}`}
|
||||
style={{
|
||||
width: `${Math.round(run.stats.correctness_score * 100)}%`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<span className="text-sm font-medium">
|
||||
{Math.round(run.stats.correctness_score * 100)}%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<TooltipProvider>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<IconCircleAlert className="size-4 cursor-help text-neutral-400 hover:text-neutral-600" />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p className="max-w-xs">
|
||||
AI-generated estimate of how well this execution
|
||||
achieved its intended purpose. This score indicates
|
||||
{run.stats.correctness_score >= 0.8
|
||||
? " the agent was highly successful."
|
||||
: run.stats.correctness_score >= 0.6
|
||||
? " the agent was mostly successful with minor issues."
|
||||
: run.stats.correctness_score >= 0.4
|
||||
? " the agent was partially successful with some gaps."
|
||||
: " the agent had limited success with significant issues."}
|
||||
</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
@@ -3886,6 +3886,48 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/executions/admin/execution_analytics": {
|
||||
"post": {
|
||||
"tags": ["v2", "admin", "admin", "execution_analytics"],
|
||||
"summary": "Generate Execution Analytics",
|
||||
"description": "Generate activity summaries and correctness scores for graph executions.\n\nThis endpoint:\n1. Fetches all completed executions matching the criteria\n2. Identifies executions missing activity_status or correctness_score\n3. Generates missing data using AI in batches\n4. Updates the database with new stats\n5. Returns a detailed report of the analytics operation",
|
||||
"operationId": "postV2Generate execution analytics",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ExecutionAnalyticsRequest"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Successful Response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ExecutionAnalyticsResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"422": {
|
||||
"description": "Validation Error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": { "$ref": "#/components/schemas/HTTPValidationError" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/HTTP401NotAuthenticatedError"
|
||||
}
|
||||
},
|
||||
"security": [{ "HTTPBearerJWT": [] }]
|
||||
}
|
||||
},
|
||||
"/api/library/presets": {
|
||||
"get": {
|
||||
"tags": ["v2", "presets"],
|
||||
@@ -5755,6 +5797,123 @@
|
||||
"required": ["url", "relevance_score"],
|
||||
"title": "Document"
|
||||
},
|
||||
"ExecutionAnalyticsRequest": {
|
||||
"properties": {
|
||||
"graph_id": {
|
||||
"type": "string",
|
||||
"title": "Graph Id",
|
||||
"description": "Graph ID to analyze"
|
||||
},
|
||||
"graph_version": {
|
||||
"anyOf": [{ "type": "integer" }, { "type": "null" }],
|
||||
"title": "Graph Version",
|
||||
"description": "Optional graph version"
|
||||
},
|
||||
"user_id": {
|
||||
"anyOf": [{ "type": "string" }, { "type": "null" }],
|
||||
"title": "User Id",
|
||||
"description": "Optional user ID filter"
|
||||
},
|
||||
"created_after": {
|
||||
"anyOf": [
|
||||
{ "type": "string", "format": "date-time" },
|
||||
{ "type": "null" }
|
||||
],
|
||||
"title": "Created After",
|
||||
"description": "Optional created date lower bound"
|
||||
},
|
||||
"model_name": {
|
||||
"anyOf": [{ "type": "string" }, { "type": "null" }],
|
||||
"title": "Model Name",
|
||||
"description": "Model to use for generation",
|
||||
"default": "gpt-4o-mini"
|
||||
},
|
||||
"batch_size": {
|
||||
"type": "integer",
|
||||
"maximum": 25.0,
|
||||
"minimum": 1.0,
|
||||
"title": "Batch Size",
|
||||
"description": "Batch size for concurrent processing",
|
||||
"default": 10
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": ["graph_id"],
|
||||
"title": "ExecutionAnalyticsRequest"
|
||||
},
|
||||
"ExecutionAnalyticsResponse": {
|
||||
"properties": {
|
||||
"total_executions": {
|
||||
"type": "integer",
|
||||
"title": "Total Executions"
|
||||
},
|
||||
"processed_executions": {
|
||||
"type": "integer",
|
||||
"title": "Processed Executions"
|
||||
},
|
||||
"successful_analytics": {
|
||||
"type": "integer",
|
||||
"title": "Successful Analytics"
|
||||
},
|
||||
"failed_analytics": {
|
||||
"type": "integer",
|
||||
"title": "Failed Analytics"
|
||||
},
|
||||
"skipped_executions": {
|
||||
"type": "integer",
|
||||
"title": "Skipped Executions"
|
||||
},
|
||||
"results": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ExecutionAnalyticsResult"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "Results"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"total_executions",
|
||||
"processed_executions",
|
||||
"successful_analytics",
|
||||
"failed_analytics",
|
||||
"skipped_executions",
|
||||
"results"
|
||||
],
|
||||
"title": "ExecutionAnalyticsResponse"
|
||||
},
|
||||
"ExecutionAnalyticsResult": {
|
||||
"properties": {
|
||||
"agent_id": { "type": "string", "title": "Agent Id" },
|
||||
"version_id": { "type": "integer", "title": "Version Id" },
|
||||
"user_id": { "type": "string", "title": "User Id" },
|
||||
"exec_id": { "type": "string", "title": "Exec Id" },
|
||||
"summary_text": {
|
||||
"anyOf": [{ "type": "string" }, { "type": "null" }],
|
||||
"title": "Summary Text"
|
||||
},
|
||||
"score": {
|
||||
"anyOf": [{ "type": "number" }, { "type": "null" }],
|
||||
"title": "Score"
|
||||
},
|
||||
"status": { "type": "string", "title": "Status" },
|
||||
"error_message": {
|
||||
"anyOf": [{ "type": "string" }, { "type": "null" }],
|
||||
"title": "Error Message"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"agent_id",
|
||||
"version_id",
|
||||
"user_id",
|
||||
"exec_id",
|
||||
"summary_text",
|
||||
"score",
|
||||
"status"
|
||||
],
|
||||
"title": "ExecutionAnalyticsResult"
|
||||
},
|
||||
"Graph": {
|
||||
"properties": {
|
||||
"id": { "type": "string", "title": "Id" },
|
||||
|
||||
Reference in New Issue
Block a user