From c05d58f0741838b070053fad80ce035ac51a0ccc Mon Sep 17 00:00:00 2001 From: vignesh07 Date: Tue, 17 Feb 2026 17:17:32 -0800 Subject: [PATCH] feat(memory): add mcporter option for warm qmd searches --- src/config/schema.help.ts | 7 ++ src/config/types.memory.ts | 15 +++ src/config/zod-schema.ts | 9 ++ src/memory/backend-config.ts | 36 ++++++ src/memory/embeddings.ts | 4 +- src/memory/qmd-manager.ts | 221 ++++++++++++++++++++++++++++++++++- 6 files changed, 287 insertions(+), 5 deletions(-) diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 81c194016f..09241fbc04 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -221,6 +221,13 @@ export const FIELD_HELP: Record = { "memory.backend": 'Memory backend ("builtin" for OpenClaw embeddings, "qmd" for QMD sidecar).', "memory.citations": 'Default citation behavior ("auto", "on", or "off").', "memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).", + "memory.qmd.mcporter": + "Optional: route QMD searches through mcporter (MCP runtime) instead of spawning `qmd` per query. Intended to avoid per-search cold starts when QMD models are large.", + "memory.qmd.mcporter.enabled": "Enable mcporter-backed QMD searches (default: false).", + "memory.qmd.mcporter.serverName": + "mcporter server name to call (default: qmd). Server should run `qmd mcp` with lifecycle keep-alive.", + "memory.qmd.mcporter.startDaemon": + "Start `mcporter daemon start` automatically when enabled (default: true).", "memory.qmd.includeDefaultMemory": "Whether to automatically index MEMORY.md + memory/**/*.md (default: true).", "memory.qmd.paths": diff --git a/src/config/types.memory.ts b/src/config/types.memory.ts index 74479baaaa..54581f65fa 100644 --- a/src/config/types.memory.ts +++ b/src/config/types.memory.ts @@ -12,6 +12,7 @@ export type MemoryConfig = { export type MemoryQmdConfig = { command?: string; + mcporter?: MemoryQmdMcporterConfig; searchMode?: MemoryQmdSearchMode; includeDefaultMemory?: boolean; paths?: MemoryQmdIndexPath[]; @@ -21,6 +22,20 @@ export type MemoryQmdConfig = { scope?: SessionSendPolicyConfig; }; +export type MemoryQmdMcporterConfig = { + /** + * Route QMD searches through mcporter (MCP runtime) instead of spawning `qmd` per query. + * Requires: + * - `mcporter` installed and on PATH + * - A configured mcporter server that runs `qmd mcp` with `lifecycle: keep-alive` + */ + enabled?: boolean; + /** mcporter server name (defaults to "qmd") */ + serverName?: string; + /** Start the mcporter daemon automatically (defaults to true when enabled). */ + startDaemon?: boolean; +}; + export type MemoryQmdIndexPath = { path: string; name?: string; diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index ca1a781fa3..26f1b13b4a 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -72,9 +72,18 @@ const MemoryQmdLimitsSchema = z }) .strict(); +const MemoryQmdMcporterSchema = z + .object({ + enabled: z.boolean().optional(), + serverName: z.string().optional(), + startDaemon: z.boolean().optional(), + }) + .strict(); + const MemoryQmdSchema = z .object({ command: z.string().optional(), + mcporter: MemoryQmdMcporterSchema.optional(), searchMode: z.union([z.literal("query"), z.literal("search"), z.literal("vsearch")]).optional(), includeDefaultMemory: z.boolean().optional(), paths: z.array(MemoryQmdPathSchema).optional(), diff --git a/src/memory/backend-config.ts b/src/memory/backend-config.ts index 02573f3a54..da1c13819a 100644 --- a/src/memory/backend-config.ts +++ b/src/memory/backend-config.ts @@ -8,6 +8,7 @@ import type { MemoryCitationsMode, MemoryQmdConfig, MemoryQmdIndexPath, + MemoryQmdMcporterConfig, MemoryQmdSearchMode, } from "../config/types.memory.js"; import { resolveUserPath } from "../utils.js"; @@ -50,8 +51,15 @@ export type ResolvedQmdSessionConfig = { retentionDays?: number; }; +export type ResolvedQmdMcporterConfig = { + enabled: boolean; + serverName: string; + startDaemon: boolean; +}; + export type ResolvedQmdConfig = { command: string; + mcporter: ResolvedQmdMcporterConfig; searchMode: MemoryQmdSearchMode; collections: ResolvedQmdCollection[]; sessions: ResolvedQmdSessionConfig; @@ -79,6 +87,12 @@ const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = { maxInjectedChars: 4_000, timeoutMs: DEFAULT_QMD_TIMEOUT_MS, }; +const DEFAULT_QMD_MCPORTER: ResolvedQmdMcporterConfig = { + enabled: false, + serverName: "qmd", + startDaemon: true, +}; + const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = { default: "deny", rules: [ @@ -237,6 +251,27 @@ function resolveCustomPaths( return collections; } +function resolveMcporterConfig(raw?: MemoryQmdMcporterConfig): ResolvedQmdMcporterConfig { + const parsed: ResolvedQmdMcporterConfig = { ...DEFAULT_QMD_MCPORTER }; + if (!raw) { + return parsed; + } + if (raw.enabled !== undefined) { + parsed.enabled = raw.enabled; + } + if (typeof raw.serverName === "string" && raw.serverName.trim()) { + parsed.serverName = raw.serverName.trim(); + } + if (raw.startDaemon !== undefined) { + parsed.startDaemon = raw.startDaemon; + } + // When enabled, default startDaemon to true. + if (parsed.enabled && raw.startDaemon === undefined) { + parsed.startDaemon = true; + } + return parsed; +} + function resolveDefaultCollections( include: boolean, workspaceDir: string, @@ -283,6 +318,7 @@ export function resolveMemoryBackendConfig(params: { const command = parsedCommand?.[0] || rawCommand.split(/\s+/)[0] || "qmd"; const resolved: ResolvedQmdConfig = { command, + mcporter: resolveMcporterConfig(qmdCfg?.mcporter), searchMode: resolveSearchMode(qmdCfg?.searchMode), collections, includeDefaultMemory, diff --git a/src/memory/embeddings.ts b/src/memory/embeddings.ts index fc60218931..ef7d7b2168 100644 --- a/src/memory/embeddings.ts +++ b/src/memory/embeddings.ts @@ -228,7 +228,9 @@ export async function createEmbeddingProvider( }; } // Non-auth errors are still fatal - throw new Error(combinedReason, { cause: fallbackErr }); + const wrapped = new Error(combinedReason) as Error & { cause?: unknown }; + wrapped.cause = fallbackErr; + throw wrapped; } } // No fallback configured - check if we should degrade to FTS-only diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 380f4175c9..54f4c7cc40 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -24,7 +24,11 @@ import type { } from "./types.js"; type SqliteDatabase = import("node:sqlite").DatabaseSync; -import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js"; +import type { + ResolvedMemoryBackendConfig, + ResolvedQmdConfig, + ResolvedQmdMcporterConfig, +} from "./backend-config.js"; import { parseQmdQueryJson, type QmdQueryResult } from "./qmd-query-parser.js"; const log = createSubsystemLogger("memory"); @@ -404,9 +408,37 @@ export class QmdMemoryManager implements MemorySearchManager { return []; } const qmdSearchCommand = this.qmd.searchMode; + const mcporterEnabled = this.qmd.mcporter.enabled; let parsed: QmdQueryResult[]; try { - if (qmdSearchCommand === "query" && collectionNames.length > 1) { + if (mcporterEnabled) { + const tool: "search" | "vector_search" | "deep_search" = + qmdSearchCommand === "search" + ? "search" + : qmdSearchCommand === "vsearch" + ? "vector_search" + : "deep_search"; + const minScore = opts?.minScore ?? 0; + if (collectionNames.length > 1) { + parsed = await this.runMcporterAcrossCollections({ + tool, + query: trimmed, + limit, + minScore, + collectionNames, + }); + } else { + parsed = await this.runQmdSearchViaMcporter({ + mcporter: this.qmd.mcporter, + tool, + query: trimmed, + limit, + minScore, + collection: collectionNames[0], + timeoutMs: this.qmd.limits.timeoutMs, + }); + } + } else if (qmdSearchCommand === "query" && collectionNames.length > 1) { parsed = await this.runQueryAcrossCollections(trimmed, limit, collectionNames); } else { const args = this.buildSearchArgs(qmdSearchCommand, trimmed, limit); @@ -417,7 +449,7 @@ export class QmdMemoryManager implements MemorySearchManager { parsed = parseQmdQueryJson(result.stdout, result.stderr); } } catch (err) { - if (qmdSearchCommand !== "query" && this.isUnsupportedQmdOptionError(err)) { + if (!mcporterEnabled && qmdSearchCommand !== "query" && this.isUnsupportedQmdOptionError(err)) { log.warn( `qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`, ); @@ -437,7 +469,8 @@ export class QmdMemoryManager implements MemorySearchManager { throw fallbackErr instanceof Error ? fallbackErr : new Error(String(fallbackErr)); } } else { - log.warn(`qmd ${qmdSearchCommand} failed: ${String(err)}`); + const label = mcporterEnabled ? "mcporter/qmd" : `qmd ${qmdSearchCommand}`; + log.warn(`${label} failed: ${String(err)}`); throw err instanceof Error ? err : new Error(String(err)); } } @@ -764,6 +797,153 @@ export class QmdMemoryManager implements MemorySearchManager { }); } + private async ensureMcporterDaemonStarted(mcporter: ResolvedQmdMcporterConfig): Promise { + if (!mcporter.enabled || !mcporter.startDaemon) { + return; + } + type McporterGlobal = typeof globalThis & { + __openclawMcporterDaemonStart?: Promise; + }; + const g: McporterGlobal = globalThis; + if (!g.__openclawMcporterDaemonStart) { + g.__openclawMcporterDaemonStart = (async () => { + try { + await this.runMcporter(["daemon", "start"], { timeoutMs: 10_000 }); + } catch (err) { + log.warn(`mcporter daemon start failed: ${String(err)}`); + } + })(); + } + await g.__openclawMcporterDaemonStart; + } + + private async runMcporter( + args: string[], + opts?: { timeoutMs?: number }, + ): Promise<{ stdout: string; stderr: string }> { + return await new Promise((resolve, reject) => { + const child = spawn("mcporter", args, { + env: process.env, + cwd: this.workspaceDir, + }); + let stdout = ""; + let stderr = ""; + let stdoutTruncated = false; + let stderrTruncated = false; + const timer = opts?.timeoutMs + ? setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`mcporter ${args.join(" ")} timed out after ${opts.timeoutMs}ms`)); + }, opts.timeoutMs) + : null; + child.stdout.on("data", (data) => { + const next = appendOutputWithCap(stdout, data.toString("utf8"), this.maxQmdOutputChars); + stdout = next.text; + stdoutTruncated = stdoutTruncated || next.truncated; + }); + child.stderr.on("data", (data) => { + const next = appendOutputWithCap(stderr, data.toString("utf8"), this.maxQmdOutputChars); + stderr = next.text; + stderrTruncated = stderrTruncated || next.truncated; + }); + child.on("error", (err) => { + if (timer) { + clearTimeout(timer); + } + reject(err); + }); + child.on("close", (code) => { + if (timer) { + clearTimeout(timer); + } + if (stdoutTruncated || stderrTruncated) { + reject( + new Error( + `mcporter ${args.join(" ")} produced too much output (limit ${this.maxQmdOutputChars} chars)`, + ), + ); + return; + } + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject( + new Error(`mcporter ${args.join(" ")} failed (code ${code}): ${stderr || stdout}`), + ); + } + }); + }); + } + + private async runQmdSearchViaMcporter(params: { + mcporter: ResolvedQmdMcporterConfig; + tool: "search" | "vector_search" | "deep_search"; + query: string; + limit: number; + minScore: number; + collection?: string; + timeoutMs: number; + }): Promise { + await this.ensureMcporterDaemonStarted(params.mcporter); + + const selector = `${params.mcporter.serverName}.${params.tool}`; + const callArgs: Record = { + query: params.query, + limit: params.limit, + minScore: params.minScore, + }; + if (params.collection) { + callArgs.collection = params.collection; + } + + const result = await this.runMcporter( + [ + "call", + selector, + "--args", + JSON.stringify(callArgs), + "--output", + "json", + "--timeout", + String(Math.max(0, params.timeoutMs)), + ], + { timeoutMs: Math.max(params.timeoutMs + 2_000, 5_000) }, + ); + + const parsedUnknown: unknown = JSON.parse(result.stdout); + const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + + const structured = + isRecord(parsedUnknown) && isRecord(parsedUnknown.structuredContent) + ? parsedUnknown.structuredContent + : parsedUnknown; + + const results: unknown[] = + isRecord(structured) && Array.isArray(structured.results) + ? (structured.results as unknown[]) + : Array.isArray(structured) + ? structured + : []; + + const out: QmdQueryResult[] = []; + for (const item of results) { + if (!isRecord(item)) { + continue; + } + const docidRaw = item.docid; + const docid = typeof docidRaw === "string" ? docidRaw.replace(/^#/, "").trim() : ""; + if (!docid) { + continue; + } + const scoreRaw = item.score; + const score = typeof scoreRaw === "number" ? scoreRaw : Number(scoreRaw); + const snippet = typeof item.snippet === "string" ? item.snippet : ""; + out.push({ docid, score: Number.isFinite(score) ? score : 0, snippet }); + } + return out; + } + private async readPartialText(absPath: string, from?: number, lines?: number): Promise { const start = Math.max(1, from ?? 1); const count = Math.max(1, lines ?? Number.POSITIVE_INFINITY); @@ -1191,6 +1371,39 @@ export class QmdMemoryManager implements MemorySearchManager { return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0)); } + private async runMcporterAcrossCollections(params: { + tool: "search" | "vector_search" | "deep_search"; + query: string; + limit: number; + minScore: number; + collectionNames: string[]; + }): Promise { + const bestByDocId = new Map(); + for (const collectionName of params.collectionNames) { + const parsed = await this.runQmdSearchViaMcporter({ + mcporter: this.qmd.mcporter, + tool: params.tool, + query: params.query, + limit: params.limit, + minScore: params.minScore, + collection: collectionName, + timeoutMs: this.qmd.limits.timeoutMs, + }); + for (const entry of parsed) { + if (typeof entry.docid !== "string" || !entry.docid.trim()) { + continue; + } + const prev = bestByDocId.get(entry.docid); + const prevScore = typeof prev?.score === "number" ? prev.score : Number.NEGATIVE_INFINITY; + const nextScore = typeof entry.score === "number" ? entry.score : Number.NEGATIVE_INFINITY; + if (!prev || nextScore > prevScore) { + bestByDocId.set(entry.docid, entry); + } + } + } + return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0)); + } + private listManagedCollectionNames(): string[] { const seen = new Set(); const names: string[] = [];