mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-03 03:03:24 -04:00
fix(memory): index FTS in provider-missing mode
This commit is contained in:
@@ -56,6 +56,7 @@ Docs: https://docs.openclaw.ai
|
||||
- Memory/QMD: scope managed collection names per agent and precreate glob-backed collection directories before registration, preventing cross-agent collection clobbering and startup ENOENT failures in fresh workspaces. (#17194) Thanks @jonathanadams96.
|
||||
- Auto-reply/WhatsApp/TUI/Web: when a final assistant message is `NO_REPLY` and a messaging tool send succeeded, mirror the delivered messaging-tool text into session-visible assistant output so TUI/Web no longer show `NO_REPLY` placeholders. (#7010) Thanks @Morrowind-Xie.
|
||||
- Cron: infer `payload.kind="agentTurn"` for model-only `cron.update` payload patches, so partial agent-turn updates do not fail validation when `kind` is omitted. (#15664) Thanks @rodrigouroz.
|
||||
- Memory/FTS: in embedding-provider fallback mode, ensure file/session indexing still writes and refreshes FTS rows so first-run memory search works and stale removed entries are removed from FTS too. Thanks @irchelper.
|
||||
- TUI: make searchable-select filtering and highlight rendering ANSI-aware so queries ignore hidden escape codes and no longer corrupt ANSI styling sequences during match highlighting. (#4519) Thanks @bee4come.
|
||||
- TUI/Windows: coalesce rapid single-line submit bursts in Git Bash into one multiline message as a fallback when bracketed paste is unavailable, preventing pasted multiline text from being split into multiple sends. (#4986) Thanks @adamkane.
|
||||
- TUI: suppress false `(no output)` placeholders for non-local empty final events during concurrent runs, preventing external-channel replies from showing empty assistant bubbles while a local run is still streaming. (#5782) Thanks @LagWizard and @vignesh07.
|
||||
|
||||
@@ -694,30 +694,31 @@ class MemoryManagerEmbeddingOps {
|
||||
entry: MemoryFileEntry | SessionFileEntry,
|
||||
options: { source: MemorySource; content?: string },
|
||||
) {
|
||||
// FTS-only mode: skip indexing if no provider
|
||||
if (!this.provider) {
|
||||
log.debug("Skipping embedding indexing in FTS-only mode", {
|
||||
path: entry.path,
|
||||
source: options.source,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const content = options.content ?? (await fs.readFile(entry.absPath, "utf-8"));
|
||||
const chunks = enforceEmbeddingMaxInputTokens(
|
||||
this.provider,
|
||||
chunkMarkdown(content, this.settings.chunking).filter(
|
||||
(chunk) => chunk.text.trim().length > 0,
|
||||
),
|
||||
const parsedChunks = chunkMarkdown(content, this.settings.chunking).filter(
|
||||
(chunk) => chunk.text.trim().length > 0,
|
||||
);
|
||||
const chunks =
|
||||
this.provider === null
|
||||
? parsedChunks
|
||||
: enforceEmbeddingMaxInputTokens(this.provider, parsedChunks);
|
||||
if (options.source === "sessions" && "lineMap" in entry) {
|
||||
remapChunkLines(chunks, entry.lineMap);
|
||||
}
|
||||
const embeddings = this.batch.enabled
|
||||
? await this.embedChunksWithBatch(chunks, entry, options.source)
|
||||
: await this.embedChunksInBatches(chunks);
|
||||
|
||||
const embeddings = this.provider
|
||||
? this.batch.enabled
|
||||
? await this.embedChunksWithBatch(chunks, entry, options.source)
|
||||
: await this.embedChunksInBatches(chunks)
|
||||
: [];
|
||||
|
||||
const sample = embeddings.find((embedding) => embedding.length > 0);
|
||||
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
|
||||
let vectorReady = false;
|
||||
if (this.provider && sample) {
|
||||
vectorReady = await this.ensureVectorReady(sample.length);
|
||||
}
|
||||
const model = this.provider?.model ?? "fts-only";
|
||||
|
||||
const now = Date.now();
|
||||
if (vectorReady) {
|
||||
try {
|
||||
@@ -729,10 +730,16 @@ class MemoryManagerEmbeddingOps {
|
||||
} catch {}
|
||||
}
|
||||
if (this.fts.enabled && this.fts.available) {
|
||||
const deleteFtsSql =
|
||||
this.provider === null
|
||||
? `DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`
|
||||
: `DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`;
|
||||
const deleteFtsParams =
|
||||
this.provider === null ? [entry.path, options.source] : [entry.path, options.source, model];
|
||||
try {
|
||||
this.db
|
||||
.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`)
|
||||
.run(entry.path, options.source, this.provider.model);
|
||||
.prepare(deleteFtsSql)
|
||||
.run(...deleteFtsParams);
|
||||
} catch {}
|
||||
}
|
||||
this.db
|
||||
@@ -742,7 +749,7 @@ class MemoryManagerEmbeddingOps {
|
||||
const chunk = chunks[i];
|
||||
const embedding = embeddings[i] ?? [];
|
||||
const id = hashText(
|
||||
`${options.source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${this.provider.model}`,
|
||||
`${options.source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${model}`,
|
||||
);
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -762,7 +769,7 @@ class MemoryManagerEmbeddingOps {
|
||||
chunk.startLine,
|
||||
chunk.endLine,
|
||||
chunk.hash,
|
||||
this.provider.model,
|
||||
model,
|
||||
chunk.text,
|
||||
JSON.stringify(embedding),
|
||||
now,
|
||||
@@ -786,7 +793,7 @@ class MemoryManagerEmbeddingOps {
|
||||
id,
|
||||
entry.path,
|
||||
options.source,
|
||||
this.provider.model,
|
||||
model,
|
||||
chunk.startLine,
|
||||
chunk.endLine,
|
||||
);
|
||||
|
||||
@@ -544,12 +544,6 @@ class MemoryManagerSyncOps {
|
||||
needsFullReindex: boolean;
|
||||
progress?: MemorySyncProgressState;
|
||||
}) {
|
||||
// FTS-only mode: skip embedding sync (no provider)
|
||||
if (!this.provider) {
|
||||
log.debug("Skipping memory file sync in FTS-only mode (no embedding provider)");
|
||||
return;
|
||||
}
|
||||
|
||||
const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
|
||||
const fileEntries = await Promise.all(
|
||||
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
|
||||
@@ -612,10 +606,16 @@ class MemoryManagerSyncOps {
|
||||
} catch {}
|
||||
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
||||
if (this.fts.enabled && this.fts.available) {
|
||||
const deleteFtsSql =
|
||||
this.provider === null
|
||||
? `DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`
|
||||
: `DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`;
|
||||
try {
|
||||
const deleteFtsParams =
|
||||
this.provider === null ? [stale.path, "memory"] : [stale.path, "memory", this.provider.model];
|
||||
this.db
|
||||
.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`)
|
||||
.run(stale.path, "memory", this.provider.model);
|
||||
.prepare(deleteFtsSql)
|
||||
.run(...deleteFtsParams);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
@@ -625,12 +625,6 @@ class MemoryManagerSyncOps {
|
||||
needsFullReindex: boolean;
|
||||
progress?: MemorySyncProgressState;
|
||||
}) {
|
||||
// FTS-only mode: skip embedding sync (no provider)
|
||||
if (!this.provider) {
|
||||
log.debug("Skipping session file sync in FTS-only mode (no embedding provider)");
|
||||
return;
|
||||
}
|
||||
|
||||
const files = await listSessionFilesForAgent(this.agentId);
|
||||
const activePaths = new Set(files.map((file) => sessionPathForFile(file)));
|
||||
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
|
||||
@@ -719,10 +713,18 @@ class MemoryManagerSyncOps {
|
||||
.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`)
|
||||
.run(stale.path, "sessions");
|
||||
if (this.fts.enabled && this.fts.available) {
|
||||
const deleteFtsSql =
|
||||
this.provider === null
|
||||
? `DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`
|
||||
: `DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`;
|
||||
try {
|
||||
const deleteFtsParams =
|
||||
this.provider === null
|
||||
? [stale.path, "sessions"]
|
||||
: [stale.path, "sessions", this.provider.model];
|
||||
this.db
|
||||
.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`)
|
||||
.run(stale.path, "sessions", this.provider.model);
|
||||
.prepare(deleteFtsSql)
|
||||
.run(...deleteFtsParams);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user