diff --git a/extensions/memory-neo4j/cli.ts b/extensions/memory-neo4j/cli.ts index 168cbc9af8..eca188f760 100644 --- a/extensions/memory-neo4j/cli.ts +++ b/extensions/memory-neo4j/cli.ts @@ -290,6 +290,7 @@ export function registerCli(api: OpenClawPluginApi, deps: CliDeps): void { "Skip LLM-based semantic dedup (Phase 1b) and conflict detection (Phase 1c)", ) .option("--workspace ", "Workspace directory for TASKS.md cleanup") + .option("--report", "Show quality metrics after sleep cycle completes") .action( async (opts: { agent?: string; @@ -302,6 +303,7 @@ export function registerCli(api: OpenClawPluginApi, deps: CliDeps): void { concurrency?: string; skipSemantic?: boolean; workspace?: string; + report?: boolean; }) => { console.log("\nšŸŒ™ Memory Sleep Cycle"); console.log("═════════════════════════════════════════════════════════════"); @@ -440,6 +442,54 @@ export function registerCli(api: OpenClawPluginApi, deps: CliDeps): void { if (result.aborted) { console.log("\nāš ļø Sleep cycle was aborted before completion."); } + + // Quality report (optional) + if (opts.report) { + console.log("\n═════════════════════════════════════════════════════════════"); + console.log("šŸ“Š Quality Report"); + console.log("─────────────────────────────────────────────────────────────"); + + try { + // Extraction coverage + const statusCounts = await db.countByExtractionStatus(opts.agent); + const totalMems = + statusCounts.pending + + statusCounts.complete + + statusCounts.failed + + statusCounts.skipped; + const coveragePct = + totalMems > 0 ? ((statusCounts.complete / totalMems) * 100).toFixed(1) : "0.0"; + console.log( + `\n Extraction Coverage: ${coveragePct}% (${statusCounts.complete}/${totalMems})`, + ); + console.log( + ` pending=${statusCounts.pending} complete=${statusCounts.complete} failed=${statusCounts.failed} skipped=${statusCounts.skipped}`, + ); + + // Entity graph stats + const graphStats = await db.getEntityGraphStats(opts.agent); + console.log(`\n Entity Graph:`); + console.log( + ` Entities: ${graphStats.entityCount} Mentions: ${graphStats.mentionCount} Density: ${graphStats.density.toFixed(2)}`, + ); + + // Decay distribution + const decayDist = await db.getDecayDistribution(opts.agent); + if (decayDist.length > 0) { + const maxCount = Math.max(...decayDist.map((d) => d.count)); + const BAR_W = 20; + console.log(`\n Decay Distribution:`); + for (const { bucket, count } of decayDist) { + const filled = maxCount > 0 ? Math.round((count / maxCount) * BAR_W) : 0; + const bar = "ā–ˆ".repeat(filled) + "ā–‘".repeat(BAR_W - filled); + console.log(` ${bucket.padEnd(13)} ${bar} ${count}`); + } + } + } catch (reportErr) { + console.log(`\n āš ļø Could not generate quality report: ${String(reportErr)}`); + } + } + console.log(""); } catch (err) { console.error( @@ -567,6 +617,192 @@ export function registerCli(api: OpenClawPluginApi, deps: CliDeps): void { process.exitCode = 1; } }); + memory + .command("health") + .description("Memory system health dashboard") + .option("--agent ", "Scope to a specific agent") + .option("--json", "Output all sections as JSON") + .action(async (opts: { agent?: string; json?: boolean }) => { + try { + await db.ensureInitialized(); + + const agentId = opts.agent; + + // Gather all data in parallel + const [ + memoryStats, + totalCount, + statusCounts, + graphStats, + decayDist, + orphanEntities, + orphanTags, + singleUseTags, + ] = await Promise.all([ + db.getMemoryStats(), + db.countMemories(agentId), + db.countByExtractionStatus(agentId), + db.getEntityGraphStats(agentId), + db.getDecayDistribution(agentId), + db.findOrphanEntities(500), + db.findOrphanTags(500), + db.findSingleUseTags(14, 500), + ]); + + // Filter stats by agent if specified + const filteredStats = agentId + ? memoryStats.filter((s) => s.agentId === agentId) + : memoryStats; + + if (opts.json) { + const totalExtraction = + statusCounts.pending + + statusCounts.complete + + statusCounts.failed + + statusCounts.skipped; + console.log( + JSON.stringify( + { + memoryOverview: { + total: totalCount, + byAgentCategory: filteredStats, + }, + extractionHealth: { + ...statusCounts, + total: totalExtraction, + coveragePercent: + totalExtraction > 0 + ? Number(((statusCounts.complete / totalExtraction) * 100).toFixed(1)) + : 0, + }, + entityGraph: { + ...graphStats, + orphanCount: orphanEntities.length, + }, + tagHealth: { + orphanCount: orphanTags.length, + singleUseCount: singleUseTags.length, + }, + decayDistribution: decayDist, + }, + null, + 2, + ), + ); + return; + } + + const BAR_W = 20; + const bar = (ratio: number) => { + const filled = Math.round(Math.min(1, Math.max(0, ratio)) * BAR_W); + return "ā–ˆ".repeat(filled) + "ā–‘".repeat(BAR_W - filled); + }; + + console.log("\n╔═══════════════════════════════════════════════════════════╗"); + console.log("ā•‘ Memory (Neo4j) Health Dashboard ā•‘"); + if (agentId) { + console.log(`ā•‘ Agent: ${agentId.padEnd(49)}ā•‘`); + } + console.log("ā•šā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•ā•"); + + // Section 1: Memory Overview + console.log("\nā”Œā”€ Memory Overview"); + console.log("│"); + console.log(`│ Total: ${totalCount} memories`); + + if (filteredStats.length > 0) { + // Group by agent + const byAgent = new Map< + string, + Array<{ category: string; count: number; avgImportance: number }> + >(); + for (const row of filteredStats) { + const list = byAgent.get(row.agentId) || []; + list.push({ + category: row.category, + count: row.count, + avgImportance: row.avgImportance, + }); + byAgent.set(row.agentId, list); + } + + for (const [agent, categories] of byAgent) { + const agentTotal = categories.reduce((s, c) => s + c.count, 0); + const maxCat = Math.max(...categories.map((c) => c.count)); + console.log(`│`); + console.log(`│ ${agent} (${agentTotal}):`); + for (const { category, count } of categories) { + const ratio = maxCat > 0 ? count / maxCat : 0; + console.log(`│ ${category.padEnd(12)} ${bar(ratio)} ${count}`); + } + } + } + console.log("ā””"); + + // Section 2: Extraction Health + const totalExtraction = + statusCounts.pending + + statusCounts.complete + + statusCounts.failed + + statusCounts.skipped; + const coveragePct = + totalExtraction > 0 + ? ((statusCounts.complete / totalExtraction) * 100).toFixed(1) + : "0.0"; + + console.log("\nā”Œā”€ Extraction Health"); + console.log("│"); + console.log( + `│ Coverage: ${coveragePct}% (${statusCounts.complete}/${totalExtraction})`, + ); + console.log(`│`); + const statusEntries: Array<[string, number]> = [ + ["pending", statusCounts.pending], + ["complete", statusCounts.complete], + ["failed", statusCounts.failed], + ["skipped", statusCounts.skipped], + ]; + const maxStatus = Math.max(...statusEntries.map(([, c]) => c)); + for (const [label, count] of statusEntries) { + const ratio = maxStatus > 0 ? count / maxStatus : 0; + console.log(`│ ${label.padEnd(10)} ${bar(ratio)} ${count}`); + } + console.log("ā””"); + + // Section 3: Entity Graph + console.log("\nā”Œā”€ Entity Graph"); + console.log("│"); + console.log(`│ Entities: ${graphStats.entityCount}`); + console.log(`│ Mentions: ${graphStats.mentionCount}`); + console.log(`│ Density: ${graphStats.density.toFixed(2)} mentions/entity`); + console.log(`│ Orphans: ${orphanEntities.length}`); + console.log("ā””"); + + // Section 4: Tag Health + console.log("\nā”Œā”€ Tag Health"); + console.log("│"); + console.log(`│ Orphan tags: ${orphanTags.length}`); + console.log(`│ Single-use tags: ${singleUseTags.length}`); + console.log("ā””"); + + // Section 5: Decay Distribution + console.log("\nā”Œā”€ Decay Distribution"); + console.log("│"); + if (decayDist.length > 0) { + const maxDecay = Math.max(...decayDist.map((d) => d.count)); + for (const { bucket, count } of decayDist) { + const ratio = maxDecay > 0 ? count / maxDecay : 0; + console.log(`│ ${bucket.padEnd(13)} ${bar(ratio)} ${count}`); + } + } else { + console.log("│ No non-core memories found."); + } + console.log("ā””\n"); + } catch (err) { + console.error(`Error: ${err instanceof Error ? err.message : String(err)}`); + process.exitCode = 1; + } + }); }, { commands: [] }, // Adds subcommands to existing "memory" command, no conflict ); diff --git a/extensions/memory-neo4j/index.ts b/extensions/memory-neo4j/index.ts index efa9e531ca..84a2be9141 100644 --- a/extensions/memory-neo4j/index.ts +++ b/extensions/memory-neo4j/index.ts @@ -134,7 +134,15 @@ const memoryNeo4jPlugin = { } const text = results - .map((r, i) => `${i + 1}. [${r.category}] ${r.text} (${(r.score * 100).toFixed(0)}%)`) + .map((r, i) => { + const base = `${i + 1}. [${r.category}] ${r.text} (${(r.score * 100).toFixed(0)}%)`; + if (!r.signals) return base; + const parts: string[] = []; + if (r.signals.vector.rank > 0) parts.push(`vec:#${r.signals.vector.rank}`); + if (r.signals.bm25.rank > 0) parts.push(`bm25:#${r.signals.bm25.rank}`); + if (r.signals.graph.rank > 0) parts.push(`graph:#${r.signals.graph.rank}`); + return parts.length > 0 ? `${base} [${parts.join(" ")}]` : base; + }) .join("\n"); const sanitizedResults = results.map((r) => ({ @@ -565,7 +573,7 @@ const memoryNeo4jPlugin = { const memoryContext = results.map((r) => `- [${r.category}] ${r.text}`).join("\n"); api.logger.debug?.( - `memory-neo4j: auto-recall memories: ${JSON.stringify(results.map((r) => ({ id: r.id, text: r.text.slice(0, 80), category: r.category, score: r.score })))}`, + `memory-neo4j: auto-recall memories: ${JSON.stringify(results.map((r) => ({ id: r.id, text: r.text.slice(0, 80), score: r.score, vec: r.signals?.vector.rank || "-", bm25: r.signals?.bm25.rank || "-", graph: r.signals?.graph.rank || "-" })))}`, ); return { diff --git a/extensions/memory-neo4j/neo4j-client.ts b/extensions/memory-neo4j/neo4j-client.ts index d75b7b5865..ea52fe45d8 100644 --- a/extensions/memory-neo4j/neo4j-client.ts +++ b/extensions/memory-neo4j/neo4j-client.ts @@ -1771,6 +1771,90 @@ export class Neo4jMemoryClient { throw lastError; } + // -------------------------------------------------------------------------- + // Health & Stats Queries + // -------------------------------------------------------------------------- + + /** + * Get entity graph statistics: entity count, mention count, and density. + * Density = mentionCount / max(entityCount, 1). + */ + async getEntityGraphStats( + agentId?: string, + ): Promise<{ entityCount: number; mentionCount: number; density: number }> { + await this.ensureInitialized(); + const session = this.driver!.session(); + try { + // When agentId is provided, only count entities connected to that agent's memories + const query = agentId + ? `OPTIONAL MATCH (m:Memory {agentId: $agentId})-[r:MENTIONS]->(e:Entity) + WITH collect(DISTINCT e) AS entities, count(r) AS mentionCount + RETURN size(entities) AS entityCount, mentionCount` + : `OPTIONAL MATCH (e:Entity) + WITH count(DISTINCT e) AS entityCount + OPTIONAL MATCH ()-[r:MENTIONS]->() + RETURN entityCount, count(r) AS mentionCount`; + + const result = await session.run(query, agentId ? { agentId } : {}); + const entityCount = (result.records[0]?.get("entityCount") as number) ?? 0; + const mentionCount = (result.records[0]?.get("mentionCount") as number) ?? 0; + return { + entityCount, + mentionCount, + density: mentionCount / Math.max(entityCount, 1), + }; + } finally { + await session.close(); + } + } + + /** + * Get decay score distribution bucketed into health categories. + * Computes decay scores server-side and buckets them. + */ + async getDecayDistribution(agentId?: string): Promise> { + await this.ensureInitialized(); + const session = this.driver!.session(); + try { + const agentFilter = agentId ? "AND m.agentId = $agentId" : ""; + const result = await session.run( + `MATCH (m:Memory) + WHERE m.createdAt IS NOT NULL AND m.category <> 'core' ${agentFilter} + WITH m, + m.importance AS importance, + CASE + WHEN m.lastRetrievedAt IS NOT NULL + THEN duration.between(datetime(m.lastRetrievedAt), datetime()).days + ELSE duration.between(datetime(m.createdAt), datetime()).days + END AS effectiveAgeDays, + coalesce(m.retrievalCount, 0) AS retrievalCount + WITH m, importance, + 30.0 * (1.0 + importance * 2.0) * (1.0 + log(1.0 + retrievalCount) * 0.2) AS halfLife, + effectiveAgeDays + WITH CASE + WHEN importance * exp(-1.0 * effectiveAgeDays / halfLife) >= 0.8 THEN 'healthy' + WHEN importance * exp(-1.0 * effectiveAgeDays / halfLife) >= 0.5 THEN 'moderate' + WHEN importance * exp(-1.0 * effectiveAgeDays / halfLife) >= 0.2 THEN 'fading' + ELSE 'near-pruning' + END AS bucket + RETURN bucket, count(*) AS cnt + ORDER BY CASE bucket + WHEN 'healthy' THEN 1 + WHEN 'moderate' THEN 2 + WHEN 'fading' THEN 3 + WHEN 'near-pruning' THEN 4 + END`, + agentId ? { agentId } : {}, + ); + return result.records.map((r) => ({ + bucket: r.get("bucket") as string, + count: (r.get("cnt") as number) ?? 0, + })); + } finally { + await session.close(); + } + } + // -------------------------------------------------------------------------- // Sleep Cycle: Entity Deduplication // -------------------------------------------------------------------------- diff --git a/extensions/memory-neo4j/schema.ts b/extensions/memory-neo4j/schema.ts index 4a1ff46488..f60fbade61 100644 --- a/extensions/memory-neo4j/schema.ts +++ b/extensions/memory-neo4j/schema.ts @@ -105,6 +105,11 @@ export type SearchSignalResult = { score: number; }; +export type SignalAttribution = { + rank: number; // 1-indexed, 0 = absent from this signal + score: number; // raw signal score, 0 = absent +}; + export type HybridSearchResult = { id: string; text: string; @@ -112,6 +117,11 @@ export type HybridSearchResult = { importance: number; createdAt: string; score: number; + signals?: { + vector: SignalAttribution; + bm25: SignalAttribution; + graph: SignalAttribution; + }; }; // ============================================================================ diff --git a/extensions/memory-neo4j/search.ts b/extensions/memory-neo4j/search.ts index 5e05b363a5..7aa6c145d7 100644 --- a/extensions/memory-neo4j/search.ts +++ b/extensions/memory-neo4j/search.ts @@ -14,7 +14,12 @@ import type { Embeddings } from "./embeddings.js"; import type { Neo4jMemoryClient } from "./neo4j-client.js"; -import type { HybridSearchResult, Logger, SearchSignalResult } from "./schema.js"; +import type { + HybridSearchResult, + Logger, + SearchSignalResult, + SignalAttribution, +} from "./schema.js"; // ============================================================================ // Query Classification @@ -107,6 +112,11 @@ type FusedCandidate = { importance: number; createdAt: string; rrfScore: number; + signals: { + vector: SignalAttribution; + bm25: SignalAttribution; + graph: SignalAttribution; + }; }; /** @@ -159,6 +169,7 @@ export function fuseWithConfidenceRRF( // Calculate confidence-weighted RRF score for each candidate const results: FusedCandidate[] = []; + const NO_SIGNAL: SignalAttribution = { rank: 0, score: 0 }; for (const [id, meta] of candidateMetadata) { let rrfScore = 0; @@ -171,6 +182,13 @@ export function fuseWithConfidenceRRF( } } + // Build per-signal attribution from the existing signal maps + const signals = { + vector: signalMaps[0]?.get(id) ?? NO_SIGNAL, + bm25: signalMaps[1]?.get(id) ?? NO_SIGNAL, + graph: signalMaps[2]?.get(id) ?? NO_SIGNAL, + }; + results.push({ id, text: meta.text, @@ -178,6 +196,7 @@ export function fuseWithConfidenceRRF( importance: meta.importance, createdAt: meta.createdAt, rrfScore, + signals, }); } @@ -269,6 +288,7 @@ export async function hybridSearch( importance: r.importance, createdAt: r.createdAt, score: Math.min(1, r.rrfScore * normalizer), // Normalize to 0-1 + signals: r.signals, })); // 6. Record retrieval events (fire-and-forget for latency)