github/sim
1
1
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-04-06 03:00:16 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(copilot): stats tracking (#1227)

Browse Source 
* Add copilot stats table schema

* Move db to agent

* Lint

* Fix tests
This commit is contained in:
Siddharth Ganesan
2025-09-02 18:17:50 -07:00
committed by GitHub
parent 1a5d5ddffa
commit c2d668c3eb
12 changed files with 283 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/sim/app/api/copilot/chat/route.test.ts
View File

@@ -224,6 +224,7 @@ describe('Copilot Chat API Route', () => {
stream: true,
streamToolCalls: true,
mode: 'agent',
messageId: 'mock-uuid-1234-5678',
depth: 0,
}),
})
@@ -286,6 +287,7 @@ describe('Copilot Chat API Route', () => {
stream: true,
streamToolCalls: true,
mode: 'agent',
messageId: 'mock-uuid-1234-5678',
depth: 0,
}),
})
@@ -337,6 +339,7 @@ describe('Copilot Chat API Route', () => {
stream: true,
streamToolCalls: true,
mode: 'agent',
messageId: 'mock-uuid-1234-5678',
depth: 0,
}),
})
@@ -425,6 +428,7 @@ describe('Copilot Chat API Route', () => {
stream: true,
streamToolCalls: true,
mode: 'ask',
messageId: 'mock-uuid-1234-5678',
depth: 0,
}),
})

7
apps/sim/app/api/copilot/chat/route.ts
View File

@@ -108,6 +108,8 @@ export async function POST(req: NextRequest) {
conversationId,
contexts,
} = ChatMessageSchema.parse(body)
// Ensure we have a consistent user message ID for this request
const userMessageIdToUse = userMessageId || crypto.randomUUID()
try {
logger.info(`[${tracker.requestId}] Received chat POST`, {
hasContexts: Array.isArray(contexts),
@@ -369,6 +371,7 @@ export async function POST(req: NextRequest) {
stream: stream,
streamToolCalls: true,
mode: mode,
messageId: userMessageIdToUse,
...(providerConfig ? { provider: providerConfig } : {}),
...(effectiveConversationId ? { conversationId: effectiveConversationId } : {}),
...(typeof effectiveDepth === 'number' ? { depth: effectiveDepth } : {}),
@@ -414,7 +417,7 @@ export async function POST(req: NextRequest) {
if (stream && simAgentResponse.body) {
// Create user message to save
const userMessage = {
id: userMessageId || crypto.randomUUID(), // Use frontend ID if provided
id: userMessageIdToUse, // Consistent ID used for request and persistence
role: 'user',
content: message,
timestamp: new Date().toISOString(),
@@ -810,7 +813,7 @@ export async function POST(req: NextRequest) {
// Save messages if we have a chat
if (currentChat && responseData.content) {
const userMessage = {
id: userMessageId || crypto.randomUUID(), // Use frontend ID if provided
id: userMessageIdToUse, // Consistent ID used for request and persistence
role: 'user',
content: message,
timestamp: new Date().toISOString(),

80
apps/sim/app/api/copilot/stats/route.ts Normal file
View File

@@ -0,0 +1,80 @@
import { type NextRequest, NextResponse } from 'next/server'
import { z } from 'zod'
import {
authenticateCopilotRequestSessionOnly,
createBadRequestResponse,
createInternalServerErrorResponse,
createRequestTracker,
createUnauthorizedResponse,
} from '@/lib/copilot/auth'
import { env } from '@/lib/env'
import { SIM_AGENT_API_URL_DEFAULT } from '@/lib/sim-agent'
const SIM_AGENT_API_URL = env.SIM_AGENT_API_URL || SIM_AGENT_API_URL_DEFAULT
const BodySchema = z
.object({
// Do NOT send id; messageId is the unique correlator
userId: z.string().optional(),
chatId: z.string().uuid().optional(),
messageId: z.string().optional(),
depth: z.number().int().nullable().optional(),
maxEnabled: z.boolean().nullable().optional(),
createdAt: z.union([z.string().datetime(), z.date()]).optional(),
diffCreated: z.boolean().nullable().optional(),
diffAccepted: z.boolean().nullable().optional(),
duration: z.number().int().nullable().optional(),
inputTokens: z.number().int().nullable().optional(),
outputTokens: z.number().int().nullable().optional(),
aborted: z.boolean().nullable().optional(),
})
.passthrough()
export async function POST(req: NextRequest) {
const tracker = createRequestTracker()
try {
const { userId, isAuthenticated } = await authenticateCopilotRequestSessionOnly()
if (!isAuthenticated || !userId) {
return createUnauthorizedResponse()
}
const json = await req.json().catch(() => ({}))
const parsed = BodySchema.safeParse(json)
if (!parsed.success) {
return createBadRequestResponse('Invalid request body for copilot stats')
}
const body = parsed.data as any
// Build outgoing payload for Sim Agent; do not include id
const payload: Record<string, any> = {
...body,
userId: body.userId || userId,
createdAt: body.createdAt || new Date().toISOString(),
}
payload.id = undefined
const agentRes = await fetch(`${SIM_AGENT_API_URL}/api/stats`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
...(env.COPILOT_API_KEY ? { 'x-api-key': env.COPILOT_API_KEY } : {}),
},
body: JSON.stringify(payload),
})
// Prefer not to block clients; still relay status
let agentJson: any = null
try {
agentJson = await agentRes.json()
} catch {}
if (!agentRes.ok) {
const message = (agentJson && (agentJson.error || agentJson.message)) || 'Upstream error'
return NextResponse.json({ success: false, error: message }, { status: 400 })
}
return NextResponse.json({ success: true })
} catch (error) {
return createInternalServerErrorResponse('Failed to forward copilot stats')
}
}

15
apps/sim/db/migrations/meta/0080_snapshot.json
View File

@@ -5747,21 +5747,6 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"workspace_environment_workspace_id_idx": {
"name": "workspace_environment_workspace_id_idx",
"columns": [
{
"expression": "workspace_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {

15
apps/sim/db/migrations/meta/0081_snapshot.json
View File

@@ -5747,21 +5747,6 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"workspace_environment_workspace_id_idx": {
"name": "workspace_environment_workspace_id_idx",
"columns": [
{
"expression": "workspace_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {

15
apps/sim/db/migrations/meta/0082_snapshot.json
View File

@@ -5872,21 +5872,6 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"workspace_environment_workspace_id_idx": {
"name": "workspace_environment_workspace_id_idx",
"columns": [
{
"expression": "workspace_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {

15
apps/sim/db/migrations/meta/0083_snapshot.json
View File

@@ -5872,21 +5872,6 @@
"concurrently": false,
"method": "btree",
"with": {}
},
"workspace_environment_workspace_id_idx": {
"name": "workspace_environment_workspace_id_idx",
"columns": [
{
"expression": "workspace_id",
"isExpression": false,
"asc": true,
"nulls": "last"
}
],
"isUnique": false,
"concurrently": false,
"method": "btree",
"with": {}
}
},
"foreignKeys": {

19
apps/sim/lib/copilot/tools/client/workflow/build-workflow.ts
View File

@@ -95,6 +95,25 @@ export class BuildWorkflowClientTool extends BaseClientTool {
// Populate diff preview immediately (without marking complete yet)
try {
const diffStore = useWorkflowDiffStore.getState()
// Send early stats upsert with the triggering user message id if available
try {
const { useCopilotStore } = await import('@/stores/copilot/store')
const { currentChat, currentUserMessageId, agentDepth, agentPrefetch } =
useCopilotStore.getState() as any
if (currentChat?.id && currentUserMessageId) {
fetch('/api/copilot/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
chatId: currentChat.id,
messageId: currentUserMessageId,
depth: agentDepth,
maxEnabled: agentDepth >= 2 && !agentPrefetch,
diffCreated: true,
}),
}).catch(() => {})
}
} catch {}
await diffStore.setProposedChanges(result.yamlContent)
logger.info('diff proposed changes set')
} catch (e) {

19
apps/sim/lib/copilot/tools/client/workflow/edit-workflow.ts
View File

@@ -151,6 +151,25 @@ export class EditWorkflowClientTool extends BaseClientTool {
try {
if (!this.hasAppliedDiff) {
const diffStore = useWorkflowDiffStore.getState()
// Send early stats upsert with the triggering user message id if available
try {
const { useCopilotStore } = await import('@/stores/copilot/store')
const { currentChat, currentUserMessageId, agentDepth, agentPrefetch } =
useCopilotStore.getState() as any
if (currentChat?.id && currentUserMessageId) {
fetch('/api/copilot/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
chatId: currentChat.id,
messageId: currentUserMessageId,
depth: agentDepth,
maxEnabled: agentDepth >= 2 && !agentPrefetch,
diffCreated: true,
}),
}).catch(() => {})
}
} catch {}
await diffStore.setProposedChanges(result.yamlContent)
logger.info('diff proposed changes set for edit_workflow')
this.hasAppliedDiff = true

98
apps/sim/stores/copilot/store.ts
View File

@@ -1539,7 +1539,17 @@ export const useCopilotStore = create<CopilotStore>()(
}
const isFirstMessage = get().messages.length === 0 && !currentChat?.title
set({ messages: newMessages })
// Capture send-time meta for reliable stats
const sendDepth = get().agentDepth
const sendMaxEnabled = sendDepth >= 2 && !get().agentPrefetch
set((state) => ({
messages: newMessages,
currentUserMessageId: userMessage.id,
messageMetaById: {
...(state.messageMetaById || {}),
[userMessage.id]: { depth: sendDepth, maxEnabled: sendMaxEnabled },
},
}))
if (isFirstMessage) {
const optimisticTitle = message.length > 50 ? `${message.substring(0, 47)}...` : message
@@ -1583,7 +1593,12 @@ export const useCopilotStore = create<CopilotStore>()(
})
if (result.success && result.stream) {
await get().handleStreamingResponse(result.stream, streamingMessage.id)
await get().handleStreamingResponse(
result.stream,
streamingMessage.id,
false,
userMessage.id
)
set({ chatsLastLoadedAt: null, chatsLoadedForWorkflow: null })
} else {
if (result.error === 'Request was aborted') {
@@ -1670,6 +1685,27 @@ export const useCopilotStore = create<CopilotStore>()(
}).catch(() => {})
} catch {}
}
// Optimistic stats: mark aborted for the in-flight user message
try {
const { currentChat: cc, currentUserMessageId, messageMetaById } = get() as any
if (cc?.id && currentUserMessageId) {
const meta = messageMetaById?.[currentUserMessageId] || null
const agentDepth = meta?.depth
const maxEnabled = meta?.maxEnabled
fetch('/api/copilot/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
chatId: cc.id,
messageId: currentUserMessageId,
...(typeof agentDepth === 'number' ? { depth: agentDepth } : {}),
...(typeof maxEnabled === 'boolean' ? { maxEnabled } : {}),
aborted: true,
}),
}).catch(() => {})
}
} catch {}
} catch {
set({ isSendingMessage: false, isAborting: false, abortController: null })
}
@@ -1981,14 +2017,16 @@ export const useCopilotStore = create<CopilotStore>()(
// Handle streaming response
handleStreamingResponse: async (
stream: ReadableStream,
messageId: string,
isContinuation = false
assistantMessageId: string,
isContinuation = false,
triggerUserMessageId?: string
) => {
const reader = stream.getReader()
const decoder = new TextDecoder()
const startTimeMs = Date.now()
const context: StreamingContext = {
messageId,
messageId: assistantMessageId,
accumulatedContent: new StringBuilder(),
contentBlocks: [],
currentTextBlock: null,
@@ -2000,7 +2038,7 @@ export const useCopilotStore = create<CopilotStore>()(
if (isContinuation) {
const { messages } = get()
const existingMessage = messages.find((m) => m.id === messageId)
const existingMessage = messages.find((m) => m.id === assistantMessageId)
if (existingMessage) {
if (existingMessage.content) context.accumulatedContent.append(existingMessage.content)
context.contentBlocks = existingMessage.contentBlocks
@@ -2042,7 +2080,7 @@ export const useCopilotStore = create<CopilotStore>()(
const finalContent = context.accumulatedContent.toString()
set((state) => ({
messages: state.messages.map((msg) =>
msg.id === messageId
msg.id === assistantMessageId
? {
...msg,
content: finalContent,
@@ -2052,6 +2090,7 @@ export const useCopilotStore = create<CopilotStore>()(
),
isSendingMessage: false,
abortController: null,
currentUserMessageId: null,
}))
if (context.newChatId && !get().currentChat) {
@@ -2071,6 +2110,51 @@ export const useCopilotStore = create<CopilotStore>()(
})
} catch {}
}
// Post copilot_stats record (input/output tokens can be null for now)
try {
const { messageMetaById } = get() as any
const meta =
(messageMetaById && (messageMetaById as any)[triggerUserMessageId || '']) || null
const agentDepth = meta?.depth ?? get().agentDepth
const maxEnabled = meta?.maxEnabled ?? (agentDepth >= 2 && !get().agentPrefetch)
const { useWorkflowDiffStore } = await import('@/stores/workflow-diff/store')
const diffState = useWorkflowDiffStore.getState() as any
const diffCreated = !!diffState?.isShowingDiff
const diffAccepted = false // acceptance may arrive earlier or later via diff store
const endMs = Date.now()
const duration = Math.max(0, endMs - startTimeMs)
const chatIdToUse = get().currentChat?.id || context.newChatId
// Prefer provided trigger user message id; fallback to last user message
let userMessageIdToUse = triggerUserMessageId
if (!userMessageIdToUse) {
const msgs = get().messages
for (let i = msgs.length - 1; i >= 0; i--) {
const m = msgs[i]
if (m.role === 'user') {
userMessageIdToUse = m.id
break
}
}
}
if (chatIdToUse) {
fetch('/api/copilot/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
chatId: chatIdToUse,
messageId: userMessageIdToUse || assistantMessageId,
depth: agentDepth,
maxEnabled,
diffCreated,
diffAccepted,
duration: duration ?? null,
inputTokens: null,
outputTokens: null,
}),
}).catch(() => {})
}
} catch {}
} finally {
clearTimeout(timeoutId)
}

9
apps/sim/stores/copilot/types.ts
View File

@@ -107,6 +107,12 @@ export interface CopilotState {
// Transient flag to prevent auto-selecting a chat during new-chat UX
suppressAutoSelect?: boolean
// Explicitly track the current user message id for this in-flight query (for stats/diff correlation)
currentUserMessageId?: string | null
// Per-message metadata captured at send-time for reliable stats
messageMetaById?: Record<string, { depth: 0 | 1 | 2 | 3; maxEnabled: boolean }>
}
export interface CopilotActions {
@@ -171,7 +177,8 @@ export interface CopilotActions {
handleStreamingResponse: (
stream: ReadableStream,
messageId: string,
isContinuation?: boolean
isContinuation?: boolean,
triggerUserMessageId?: string
) => Promise<void>
handleNewChatCreation: (newChatId: string) => Promise<void>
updateDiffStore: (yamlContent: string, toolName?: string) => Promise<void>

58
apps/sim/stores/workflow-diff/store.ts
View File

@@ -54,6 +54,8 @@ interface WorkflowDiffState {
// PERFORMANCE OPTIMIZATION: Cache frequently accessed computed values
_cachedDisplayState?: WorkflowState
_lastDisplayStateHash?: string
// Track the user message id that triggered the current diff (for stats correlation)
_triggerMessageId?: string | null
}
interface WorkflowDiffActions {
@@ -112,6 +114,7 @@ export const useWorkflowDiffStore = create<WorkflowDiffState & WorkflowDiffActio
diffError: null,
_cachedDisplayState: undefined,
_lastDisplayStateHash: undefined,
_triggerMessageId: null,
_batchedStateUpdate: batchedUpdate,
@@ -147,6 +150,22 @@ export const useWorkflowDiffStore = create<WorkflowDiffState & WorkflowDiffActio
return
}
// Attempt to capture the triggering user message id from copilot store
let triggerMessageId: string | null = null
try {
const { useCopilotStore } = await import('@/stores/copilot/store')
const { messages } = useCopilotStore.getState() as any
if (Array.isArray(messages) && messages.length > 0) {
for (let i = messages.length - 1; i >= 0; i--) {
const m = messages[i]
if (m?.role === 'user' && m?.id) {
triggerMessageId = m.id
break
}
}
}
} catch {}
// PERFORMANCE OPTIMIZATION: Log diff analysis efficiently
if (result.diff.diffAnalysis) {
const analysis = result.diff.diffAnalysis
@@ -168,6 +187,7 @@ export const useWorkflowDiffStore = create<WorkflowDiffState & WorkflowDiffActio
diffError: null,
_cachedDisplayState: undefined, // Clear cache
_lastDisplayStateHash: undefined,
_triggerMessageId: triggerMessageId,
})
logger.info('Diff created successfully')
@@ -273,6 +293,25 @@ export const useWorkflowDiffStore = create<WorkflowDiffState & WorkflowDiffActio
return
}
// Immediately flag diffAccepted on stats if we can (early upsert with minimal fields)
try {
const { useCopilotStore } = await import('@/stores/copilot/store')
const { currentChat } = useCopilotStore.getState() as any
const triggerMessageId = get()._triggerMessageId
if (currentChat?.id && triggerMessageId) {
fetch('/api/copilot/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
chatId: currentChat.id,
messageId: triggerMessageId,
diffCreated: true,
diffAccepted: true,
}),
}).catch(() => {})
}
} catch {}
// Update the main workflow store state
useWorkflowStore.setState({
blocks: cleanState.blocks,
@@ -392,7 +431,24 @@ export const useWorkflowDiffStore = create<WorkflowDiffState & WorkflowDiffActio
// Update copilot tool call state to 'rejected'
try {
const { useCopilotStore } = await import('@/stores/copilot/store')
const { messages, toolCallsById } = useCopilotStore.getState()
const { currentChat, messages, toolCallsById } = useCopilotStore.getState() as any
// Post early diffAccepted=false if we have trigger + chat
try {
const triggerMessageId = get()._triggerMessageId
if (currentChat?.id && triggerMessageId) {
fetch('/api/copilot/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
chatId: currentChat.id,
messageId: triggerMessageId,
diffCreated: true,
diffAccepted: false,
}),
}).catch(() => {})
}
} catch {}
// Prefer the latest assistant message's build/edit tool_call from contentBlocks
let toolCallId: string | undefined