feat(langsmith): add langsmith tools for logging, output selector use tool-aware listing (#2821)

* feat(langsmith): add langsmith tools for logging, output selector use tool-aware listing

* fix

* fix docs

* fix positioning of outputs

* fix docs script

apps/sim/tools/langsmith/create_run.ts

@@ -0,0 +1,188 @@
+import type { LangsmithCreateRunParams, LangsmithCreateRunResponse } from '@/tools/langsmith/types'
+import { normalizeLangsmithRunPayload } from '@/tools/langsmith/utils'
+import type { ToolConfig } from '@/tools/types'
+
+export const langsmithCreateRunTool: ToolConfig<
+  LangsmithCreateRunParams,
+  LangsmithCreateRunResponse
+> = {
+  id: 'langsmith_create_run',
+  name: 'LangSmith Create Run',
+  description: 'Forward a single run to LangSmith for ingestion.',
+  version: '1.0.0',
+  params: {
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'LangSmith API key',
+    },
+    id: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Unique run identifier',
+    },
+    name: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'Run name',
+    },
+    run_type: {
+      type: 'string',
+      required: true,
+      visibility: 'user-or-llm',
+      description: 'Run type (tool, chain, llm, retriever, embedding, prompt, parser)',
+    },
+    start_time: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Run start time in ISO-8601 format',
+    },
+    end_time: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Run end time in ISO-8601 format',
+    },
+    inputs: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Inputs payload',
+    },
+    run_outputs: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Outputs payload',
+    },
+    extra: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Additional metadata (extra)',
+    },
+    tags: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Array of tag strings',
+    },
+    parent_run_id: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Parent run ID',
+    },
+    trace_id: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Trace ID',
+    },
+    session_id: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Session ID',
+    },
+    session_name: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Session name',
+    },
+    status: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Run status',
+    },
+    error: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Error details',
+    },
+    dotted_order: {
+      type: 'string',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Dotted order string',
+    },
+    events: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Structured events array',
+    },
+  },
+  request: {
+    url: () => 'https://api.smith.langchain.com/runs',
+    method: 'POST',
+    headers: (params) => ({
+      'X-Api-Key': params.apiKey,
+      'Content-Type': 'application/json',
+    }),
+    body: (params) => {
+      const { payload } = normalizeLangsmithRunPayload(params)
+      const normalizedPayload: Record<string, unknown> = {
+        ...payload,
+        name: payload.name?.trim(),
+        inputs: params.inputs,
+        outputs: params.run_outputs,
+        extra: params.extra,
+        tags: params.tags,
+        status: params.status,
+        error: params.error,
+        events: params.events,
+      }
+
+      return Object.fromEntries(
+        Object.entries(normalizedPayload).filter(([, value]) => value !== undefined)
+      )
+    },
+  },
+  transformResponse: async (response, params) => {
+    const runId = params ? normalizeLangsmithRunPayload(params).runId : null
+    const data = (await response.json()) as Record<string, unknown>
+    const directMessage =
+      typeof (data as { message?: unknown }).message === 'string'
+        ? (data as { message: string }).message
+        : null
+    const nestedPayload =
+      runId && typeof data[runId] === 'object' && data[runId] !== null
+        ? (data[runId] as Record<string, unknown>)
+        : null
+    const nestedMessage =
+      nestedPayload && typeof nestedPayload.message === 'string' ? nestedPayload.message : null
+
+    return {
+      success: true,
+      output: {
+        accepted: true,
+        runId: runId ?? null,
+        message: directMessage ?? nestedMessage ?? null,
+      },
+    }
+  },
+  outputs: {
+    accepted: {
+      type: 'boolean',
+      description: 'Whether the run was accepted for ingestion',
+    },
+    runId: {
+      type: 'string',
+      description: 'Run identifier provided in the request',
+      optional: true,
+    },
+    message: {
+      type: 'string',
+      description: 'Response message from LangSmith',
+      optional: true,
+    },
+  },
+}

apps/sim/tools/langsmith/create_runs_batch.ts

@@ -0,0 +1,112 @@
+import type {
+  LangsmithCreateRunsBatchParams,
+  LangsmithCreateRunsBatchResponse,
+  LangsmithRunPayload,
+} from '@/tools/langsmith/types'
+import { normalizeLangsmithRunPayload } from '@/tools/langsmith/utils'
+import type { ToolConfig } from '@/tools/types'
+
+export const langsmithCreateRunsBatchTool: ToolConfig<
+  LangsmithCreateRunsBatchParams,
+  LangsmithCreateRunsBatchResponse
+> = {
+  id: 'langsmith_create_runs_batch',
+  name: 'LangSmith Create Runs Batch',
+  description: 'Forward multiple runs to LangSmith in a single batch.',
+  version: '1.0.0',
+  params: {
+    apiKey: {
+      type: 'string',
+      required: true,
+      visibility: 'user-only',
+      description: 'LangSmith API key',
+    },
+    post: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Array of new runs to ingest',
+    },
+    patch: {
+      type: 'json',
+      required: false,
+      visibility: 'user-or-llm',
+      description: 'Array of runs to update/patch',
+    },
+  },
+  request: {
+    url: () => 'https://api.smith.langchain.com/runs/batch',
+    method: 'POST',
+    headers: (params) => ({
+      'X-Api-Key': params.apiKey,
+      'Content-Type': 'application/json',
+    }),
+    body: (params) => {
+      const payload: Record<string, unknown> = {
+        post: params.post
+          ? params.post.map((run) => normalizeLangsmithRunPayload(run).payload)
+          : undefined,
+        patch: params.patch
+          ? params.patch.map((run) => normalizeLangsmithRunPayload(run).payload)
+          : undefined,
+      }
+
+      return Object.fromEntries(Object.entries(payload).filter(([, value]) => value !== undefined))
+    },
+  },
+  transformResponse: async (response, params) => {
+    const data = (await response.json()) as Record<string, unknown>
+    const directMessage =
+      typeof (data as { message?: unknown }).message === 'string'
+        ? (data as { message: string }).message
+        : null
+    const messages = Object.values(data)
+      .map((value) => {
+        if (typeof value !== 'object' || value === null) {
+          return null
+        }
+        const messageValue = (value as Record<string, unknown>).message
+        return typeof messageValue === 'string' ? messageValue : null
+      })
+      .filter((value): value is string => Boolean(value))
+
+    const collectRunIds = (runs?: LangsmithRunPayload[]) =>
+      runs?.map((run) => normalizeLangsmithRunPayload(run).runId) ?? []
+
+    return {
+      success: true,
+      output: {
+        accepted: true,
+        runIds: [...collectRunIds(params?.post), ...collectRunIds(params?.patch)],
+        message: directMessage ?? null,
+        messages: messages.length ? messages : undefined,
+      },
+    }
+  },
+  outputs: {
+    accepted: {
+      type: 'boolean',
+      description: 'Whether the batch was accepted for ingestion',
+    },
+    runIds: {
+      type: 'array',
+      description: 'Run identifiers provided in the request',
+      items: {
+        type: 'string',
+      },
+    },
+    message: {
+      type: 'string',
+      description: 'Response message from LangSmith',
+      optional: true,
+    },
+    messages: {
+      type: 'array',
+      description: 'Per-run response messages, when provided',
+      optional: true,
+      items: {
+        type: 'string',
+      },
+    },
+  },
+}

apps/sim/tools/langsmith/index.ts

@@ -0,0 +1,2 @@
+export { langsmithCreateRunTool } from '@/tools/langsmith/create_run'
+export { langsmithCreateRunsBatchTool } from '@/tools/langsmith/create_runs_batch'

apps/sim/tools/langsmith/types.ts

@@ -0,0 +1,60 @@
+import type { ToolResponse } from '@/tools/types'
+
+export type LangsmithRunType =
+  | 'tool'
+  | 'chain'
+  | 'llm'
+  | 'retriever'
+  | 'embedding'
+  | 'prompt'
+  | 'parser'
+
+export interface LangsmithRunPayload {
+  id?: string
+  name: string
+  run_type: LangsmithRunType
+  start_time?: string
+  end_time?: string
+  inputs?: Record<string, unknown>
+  outputs?: Record<string, unknown>
+  extra?: Record<string, unknown>
+  tags?: string[]
+  parent_run_id?: string
+  trace_id?: string
+  session_id?: string
+  session_name?: string
+  status?: string
+  error?: string
+  dotted_order?: string
+  events?: Record<string, unknown>[]
+}
+
+export interface LangsmithCreateRunParams extends Omit<LangsmithRunPayload, 'outputs'> {
+  apiKey: string
+  run_outputs?: Record<string, unknown>
+}
+
+export interface LangsmithCreateRunsBatchParams {
+  apiKey: string
+  post?: LangsmithRunPayload[]
+  patch?: LangsmithRunPayload[]
+}
+
+export interface LangsmithCreateRunResponse extends ToolResponse {
+  output: {
+    accepted: boolean
+    runId: string | null
+    message: string | null
+  }
+}
+
+export interface LangsmithCreateRunsBatchResponse extends ToolResponse {
+  output: {
+    accepted: boolean
+    runIds: string[]
+    message: string | null
+    messages?: string[]
+  }
+}
+
+export type LangsmithResponse = LangsmithCreateRunResponse | LangsmithCreateRunsBatchResponse

apps/sim/tools/langsmith/utils.ts

@@ -0,0 +1,38 @@
+import type { LangsmithRunPayload } from '@/tools/langsmith/types'
+
+interface NormalizedRunPayload {
+  payload: LangsmithRunPayload
+  runId: string
+}
+
+const toCompactTimestamp = (startTime?: string): string => {
+  const parsed = startTime ? new Date(startTime) : new Date()
+  const date = Number.isNaN(parsed.getTime()) ? new Date() : parsed
+  const pad = (value: number, length: number) => value.toString().padStart(length, '0')
+  const year = date.getUTCFullYear()
+  const month = pad(date.getUTCMonth() + 1, 2)
+  const day = pad(date.getUTCDate(), 2)
+  const hours = pad(date.getUTCHours(), 2)
+  const minutes = pad(date.getUTCMinutes(), 2)
+  const seconds = pad(date.getUTCSeconds(), 2)
+  const micros = pad(date.getUTCMilliseconds() * 1000, 6)
+  return `${year}${month}${day}T${hours}${minutes}${seconds}${micros}`
+}
+
+export const normalizeLangsmithRunPayload = (run: LangsmithRunPayload): NormalizedRunPayload => {
+  const runId = run.id ?? crypto.randomUUID()
+  const traceId = run.trace_id ?? runId
+  const startTime = run.start_time ?? new Date().toISOString()
+  const dottedOrder = run.dotted_order ?? `${toCompactTimestamp(startTime)}Z${runId}`
+
+  return {
+    runId,
+    payload: {
+      ...run,
+      id: runId,
+      trace_id: traceId,
+      start_time: startTime,
+      dotted_order: dottedOrder,
+    },
+  }
+}

apps/sim/tools/registry.ts

@@ -653,6 +653,7 @@ import {
   knowledgeSearchTool,
   knowledgeUploadChunkTool,
 } from '@/tools/knowledge'
+import { langsmithCreateRunsBatchTool, langsmithCreateRunTool } from '@/tools/langsmith'
 import { lemlistGetActivitiesTool, lemlistGetLeadTool, lemlistSendEmailTool } from '@/tools/lemlist'
 import {
   linearAddLabelToIssueTool,
@@ -2442,6 +2443,8 @@ export const tools: Record<string, ToolConfig> = {
   linear_update_project_status: linearUpdateProjectStatusTool,
   linear_delete_project_status: linearDeleteProjectStatusTool,
   linear_list_project_statuses: linearListProjectStatusesTool,
+  langsmith_create_run: langsmithCreateRunTool,
+  langsmith_create_runs_batch: langsmithCreateRunsBatchTool,
   lemlist_get_activities: lemlistGetActivitiesTool,
   lemlist_get_lead: lemlistGetLeadTool,
   lemlist_send_email: lemlistSendEmailTool,