mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-03 03:03:24 -04:00
fix: enable FTS fallback when no embedding provider available (#17725)
When no embedding provider is available (e.g., OAuth mode without API keys), memory_search now falls back to FTS-only mode instead of returning disabled: true. Changes: - embeddings.ts: return null provider with reason instead of throwing - manager.ts: handle null provider, use FTS-only search mode - manager-search.ts: allow searching all models when provider is undefined - memory-tool.ts: expose search mode in results The search results now include a 'mode' field indicating 'hybrid' or 'fts-only'.
This commit is contained in:
@@ -81,12 +81,14 @@ export function createMemorySearchTool(options: {
|
||||
status.backend === "qmd"
|
||||
? clampResultsByInjectedChars(decorated, resolved.qmd?.limits.maxInjectedChars)
|
||||
: decorated;
|
||||
const searchMode = (status.custom as { searchMode?: string } | undefined)?.searchMode;
|
||||
return jsonResult({
|
||||
results,
|
||||
provider: status.provider,
|
||||
model: status.model,
|
||||
fallback: status.fallback,
|
||||
citations: citationsMode,
|
||||
mode: searchMode,
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
|
||||
@@ -432,3 +432,63 @@ describe("local embedding normalization", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("FTS-only fallback when no provider available", () => {
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("returns null provider with reason when auto mode finds no providers", async () => {
|
||||
vi.mocked(authModule.resolveApiKeyForProvider).mockRejectedValue(
|
||||
new Error('No API key found for provider "openai"'),
|
||||
);
|
||||
|
||||
const result = await createEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "auto",
|
||||
model: "",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
expect(result.provider).toBeNull();
|
||||
expect(result.requestedProvider).toBe("auto");
|
||||
expect(result.providerUnavailableReason).toBeDefined();
|
||||
expect(result.providerUnavailableReason).toContain("No API key");
|
||||
});
|
||||
|
||||
it("returns null provider when explicit provider fails with missing API key", async () => {
|
||||
vi.mocked(authModule.resolveApiKeyForProvider).mockRejectedValue(
|
||||
new Error('No API key found for provider "openai"'),
|
||||
);
|
||||
|
||||
const result = await createEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
expect(result.provider).toBeNull();
|
||||
expect(result.requestedProvider).toBe("openai");
|
||||
expect(result.providerUnavailableReason).toBeDefined();
|
||||
});
|
||||
|
||||
it("returns null provider when both primary and fallback fail with missing API keys", async () => {
|
||||
vi.mocked(authModule.resolveApiKeyForProvider).mockRejectedValue(
|
||||
new Error("No API key found for provider"),
|
||||
);
|
||||
|
||||
const result = await createEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
fallback: "gemini",
|
||||
});
|
||||
|
||||
expect(result.provider).toBeNull();
|
||||
expect(result.requestedProvider).toBe("openai");
|
||||
expect(result.fallbackFrom).toBe("openai");
|
||||
expect(result.providerUnavailableReason).toContain("Fallback to gemini failed");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -36,10 +36,11 @@ export type EmbeddingProviderFallback = EmbeddingProviderId | "none";
|
||||
const REMOTE_EMBEDDING_PROVIDER_IDS = ["openai", "gemini", "voyage"] as const;
|
||||
|
||||
export type EmbeddingProviderResult = {
|
||||
provider: EmbeddingProvider;
|
||||
provider: EmbeddingProvider | null;
|
||||
requestedProvider: EmbeddingProviderRequest;
|
||||
fallbackFrom?: EmbeddingProviderId;
|
||||
fallbackReason?: string;
|
||||
providerUnavailableReason?: string;
|
||||
openAi?: OpenAiEmbeddingClient;
|
||||
gemini?: GeminiEmbeddingClient;
|
||||
voyage?: VoyageEmbeddingClient;
|
||||
@@ -183,15 +184,19 @@ export async function createEmbeddingProvider(
|
||||
missingKeyErrors.push(message);
|
||||
continue;
|
||||
}
|
||||
// Non-auth errors (e.g., network) are still fatal
|
||||
throw new Error(message, { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
// All providers failed due to missing API keys - return null provider for FTS-only mode
|
||||
const details = [...missingKeyErrors, localError].filter(Boolean) as string[];
|
||||
if (details.length > 0) {
|
||||
throw new Error(details.join("\n\n"));
|
||||
}
|
||||
throw new Error("No embeddings provider available.");
|
||||
const reason = details.length > 0 ? details.join("\n\n") : "No embeddings provider available.";
|
||||
return {
|
||||
provider: null,
|
||||
requestedProvider,
|
||||
providerUnavailableReason: reason,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -209,13 +214,31 @@ export async function createEmbeddingProvider(
|
||||
fallbackReason: reason,
|
||||
};
|
||||
} catch (fallbackErr) {
|
||||
// oxlint-disable-next-line preserve-caught-error
|
||||
throw new Error(
|
||||
`${reason}\n\nFallback to ${fallback} failed: ${formatErrorMessage(fallbackErr)}`,
|
||||
{ cause: fallbackErr },
|
||||
);
|
||||
// Both primary and fallback failed - check if it's auth-related
|
||||
const fallbackReason = formatErrorMessage(fallbackErr);
|
||||
const combinedReason = `${reason}\n\nFallback to ${fallback} failed: ${fallbackReason}`;
|
||||
if (isMissingApiKeyError(primaryErr) && isMissingApiKeyError(fallbackErr)) {
|
||||
// Both failed due to missing API keys - return null for FTS-only mode
|
||||
return {
|
||||
provider: null,
|
||||
requestedProvider,
|
||||
fallbackFrom: requestedProvider,
|
||||
fallbackReason: reason,
|
||||
providerUnavailableReason: combinedReason,
|
||||
};
|
||||
}
|
||||
// Non-auth errors are still fatal
|
||||
throw new Error(combinedReason, { cause: fallbackErr });
|
||||
}
|
||||
}
|
||||
// No fallback configured - check if we should degrade to FTS-only
|
||||
if (isMissingApiKeyError(primaryErr)) {
|
||||
return {
|
||||
provider: null,
|
||||
requestedProvider,
|
||||
providerUnavailableReason: reason,
|
||||
};
|
||||
}
|
||||
throw new Error(reason, { cause: primaryErr });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +202,10 @@ class MemoryManagerEmbeddingOps {
|
||||
}
|
||||
|
||||
private computeProviderKey(): string {
|
||||
// FTS-only mode: no provider, use a constant key
|
||||
if (!this.provider) {
|
||||
return hashText(JSON.stringify({ provider: "none", model: "fts-only" }));
|
||||
}
|
||||
if (this.provider.id === "openai" && this.openAi) {
|
||||
const entries = Object.entries(this.openAi.headers)
|
||||
.filter(([key]) => key.toLowerCase() !== "authorization")
|
||||
|
||||
@@ -136,7 +136,7 @@ export function listChunks(params: {
|
||||
export async function searchKeyword(params: {
|
||||
db: DatabaseSync;
|
||||
ftsTable: string;
|
||||
providerModel: string;
|
||||
providerModel: string | undefined;
|
||||
query: string;
|
||||
limit: number;
|
||||
snippetMaxChars: number;
|
||||
@@ -152,16 +152,20 @@ export async function searchKeyword(params: {
|
||||
return [];
|
||||
}
|
||||
|
||||
// When providerModel is undefined (FTS-only mode), search all models
|
||||
const modelClause = params.providerModel ? " AND model = ?" : "";
|
||||
const modelParams = params.providerModel ? [params.providerModel] : [];
|
||||
|
||||
const rows = params.db
|
||||
.prepare(
|
||||
`SELECT id, path, source, start_line, end_line, text,\n` +
|
||||
` bm25(${params.ftsTable}) AS rank\n` +
|
||||
` FROM ${params.ftsTable}\n` +
|
||||
` WHERE ${params.ftsTable} MATCH ? AND model = ?${params.sourceFilter.sql}\n` +
|
||||
` WHERE ${params.ftsTable} MATCH ?${modelClause}${params.sourceFilter.sql}\n` +
|
||||
` ORDER BY rank ASC\n` +
|
||||
` LIMIT ?`,
|
||||
)
|
||||
.all(ftsQuery, params.providerModel, ...params.sourceFilter.params, params.limit) as Array<{
|
||||
.all(ftsQuery, ...modelParams, ...params.sourceFilter.params, params.limit) as Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
source: SearchSource;
|
||||
|
||||
@@ -46,10 +46,11 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
private readonly agentId: string;
|
||||
private readonly workspaceDir: string;
|
||||
private readonly settings: ResolvedMemorySearchConfig;
|
||||
private provider: EmbeddingProvider;
|
||||
private provider: EmbeddingProvider | null;
|
||||
private readonly requestedProvider: "openai" | "local" | "gemini" | "voyage" | "auto";
|
||||
private fallbackFrom?: "openai" | "local" | "gemini" | "voyage";
|
||||
private fallbackReason?: string;
|
||||
private readonly providerUnavailableReason?: string;
|
||||
private openAi?: OpenAiEmbeddingClient;
|
||||
private gemini?: GeminiEmbeddingClient;
|
||||
private voyage?: VoyageEmbeddingClient;
|
||||
@@ -154,6 +155,7 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
this.requestedProvider = params.providerResult.requestedProvider;
|
||||
this.fallbackFrom = params.providerResult.fallbackFrom;
|
||||
this.fallbackReason = params.providerResult.fallbackReason;
|
||||
this.providerUnavailableReason = params.providerResult.providerUnavailableReason;
|
||||
this.openAi = params.providerResult.openAi;
|
||||
this.gemini = params.providerResult.gemini;
|
||||
this.voyage = params.providerResult.voyage;
|
||||
@@ -225,6 +227,16 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)),
|
||||
);
|
||||
|
||||
// FTS-only mode: no embedding provider available
|
||||
if (!this.provider) {
|
||||
if (!this.fts.enabled || !this.fts.available) {
|
||||
log.warn("memory search: no provider and FTS unavailable");
|
||||
return [];
|
||||
}
|
||||
const ftsResults = await this.searchKeyword(cleaned, candidates).catch(() => []);
|
||||
return ftsResults.filter((entry) => entry.score >= minScore).slice(0, maxResults);
|
||||
}
|
||||
|
||||
const keywordResults = hybrid.enabled
|
||||
? await this.searchKeyword(cleaned, candidates).catch(() => [])
|
||||
: [];
|
||||
@@ -253,6 +265,10 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
queryVec: number[],
|
||||
limit: number,
|
||||
): Promise<Array<MemorySearchResult & { id: string }>> {
|
||||
// This method should never be called without a provider
|
||||
if (!this.provider) {
|
||||
return [];
|
||||
}
|
||||
const results = await searchVector({
|
||||
db: this.db,
|
||||
vectorTable: VECTOR_TABLE,
|
||||
@@ -279,10 +295,12 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
return [];
|
||||
}
|
||||
const sourceFilter = this.buildSourceFilter();
|
||||
// In FTS-only mode (no provider), search all models; otherwise filter by current provider's model
|
||||
const providerModel = this.provider?.model;
|
||||
const results = await searchKeyword({
|
||||
db: this.db,
|
||||
ftsTable: FTS_TABLE,
|
||||
providerModel: this.provider.model,
|
||||
providerModel,
|
||||
query,
|
||||
limit,
|
||||
snippetMaxChars: SNIPPET_MAX_CHARS,
|
||||
@@ -446,6 +464,13 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
return sources.map((source) => Object.assign({ source }, bySource.get(source)!));
|
||||
})();
|
||||
|
||||
// Determine search mode: "fts-only" if no provider, "hybrid" otherwise
|
||||
const searchMode = this.provider ? "hybrid" : "fts-only";
|
||||
const providerInfo = this.provider
|
||||
? { provider: this.provider.id, model: this.provider.model }
|
||||
: { provider: "none", model: undefined };
|
||||
|
||||
return {
|
||||
backend: "builtin",
|
||||
files: files?.c ?? 0,
|
||||
@@ -453,8 +478,8 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
dirty: this.dirty || this.sessionsDirty,
|
||||
workspaceDir: this.workspaceDir,
|
||||
dbPath: this.settings.store.path,
|
||||
provider: this.provider.id,
|
||||
model: this.provider.model,
|
||||
provider: providerInfo.provider,
|
||||
model: providerInfo.model,
|
||||
requestedProvider: this.requestedProvider,
|
||||
sources: Array.from(this.sources),
|
||||
extraPaths: this.settings.extraPaths,
|
||||
@@ -497,10 +522,18 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
lastError: this.batchFailureLastError,
|
||||
lastProvider: this.batchFailureLastProvider,
|
||||
},
|
||||
custom: {
|
||||
searchMode,
|
||||
providerUnavailableReason: this.providerUnavailableReason,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async probeVectorAvailability(): Promise<boolean> {
|
||||
// FTS-only mode: vector search not available
|
||||
if (!this.provider) {
|
||||
return false;
|
||||
}
|
||||
if (!this.vector.enabled) {
|
||||
return false;
|
||||
}
|
||||
@@ -508,6 +541,13 @@ export class MemoryIndexManager implements MemorySearchManager {
|
||||
}
|
||||
|
||||
async probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult> {
|
||||
// FTS-only mode: embeddings not available but search still works
|
||||
if (!this.provider) {
|
||||
return {
|
||||
ok: false,
|
||||
error: this.providerUnavailableReason ?? "No embedding provider available (FTS-only mode)",
|
||||
};
|
||||
}
|
||||
try {
|
||||
await this.embedBatchWithRetry(["ping"]);
|
||||
return { ok: true };
|
||||
|
||||
Reference in New Issue
Block a user