perf(test): speed up line, models list, and memory batch

This commit is contained in:
Peter Steinberger
2026-02-14 15:20:35 +00:00
parent 9fb48f4dff
commit 684c18458a
6 changed files with 126 additions and 97 deletions

View File

@@ -1,4 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
let modelsListCommand: typeof import("./models/list.list-command.js").modelsListCommand;
const loadConfig = vi.fn();
const ensureOpenClawModelsJson = vi.fn().mockResolvedValue(undefined);
@@ -49,15 +51,13 @@ vi.mock("../agents/model-auth.js", () => ({
getCustomProviderApiKey,
}));
vi.mock("@mariozechner/pi-coding-agent", async () => {
class MockAuthStorage {}
vi.mock("../agents/pi-model-discovery.js", () => {
class MockModelRegistry {
find(provider: string, id: string) {
const found =
return (
modelRegistryState.models.find((model) => model.provider === provider && model.id === id) ??
null;
return found;
null
);
}
getAll() {
@@ -76,11 +76,17 @@ vi.mock("@mariozechner/pi-coding-agent", async () => {
}
return {
AuthStorage: MockAuthStorage,
ModelRegistry: MockModelRegistry,
discoverAuthStorage: () => ({}) as unknown,
discoverModels: () => new MockModelRegistry() as unknown,
};
});
vi.mock("../agents/pi-embedded-runner/model.js", () => ({
resolveModel: () => {
throw new Error("resolveModel should not be called from models.list tests");
},
}));
function makeRuntime() {
return {
log: vi.fn(),
@@ -101,6 +107,10 @@ afterEach(() => {
});
describe("models list/status", () => {
beforeAll(async () => {
({ modelsListCommand } = await import("./models/list.list-command.js"));
});
it("models list outputs canonical zai key for configured z.ai model", async () => {
loadConfig.mockReturnValue({
agents: { defaults: { model: "z.ai/glm-4.7" } },
@@ -118,8 +128,6 @@ describe("models list/status", () => {
modelRegistryState.models = [model];
modelRegistryState.available = [model];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -144,8 +152,6 @@ describe("models list/status", () => {
modelRegistryState.models = [model];
modelRegistryState.available = [model];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ plain: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -179,8 +185,6 @@ describe("models list/status", () => {
modelRegistryState.models = models;
modelRegistryState.available = models;
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ all: true, provider: "z.ai", json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -216,8 +220,6 @@ describe("models list/status", () => {
modelRegistryState.models = models;
modelRegistryState.available = models;
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ all: true, provider: "Z.AI", json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -253,8 +255,6 @@ describe("models list/status", () => {
modelRegistryState.models = models;
modelRegistryState.available = models;
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ all: true, provider: "z-ai", json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -280,8 +280,6 @@ describe("models list/status", () => {
modelRegistryState.models = [model];
modelRegistryState.available = [];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ all: true, json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -317,8 +315,6 @@ describe("models list/status", () => {
},
];
modelRegistryState.available = [];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -357,8 +353,6 @@ describe("models list/status", () => {
},
];
modelRegistryState.available = [];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -396,8 +390,6 @@ describe("models list/status", () => {
};
modelRegistryState.models = [template];
modelRegistryState.available = [template];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -434,8 +426,6 @@ describe("models list/status", () => {
};
modelRegistryState.models = [template];
modelRegistryState.available = [template];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -477,8 +467,6 @@ describe("models list/status", () => {
};
modelRegistryState.models = [template];
modelRegistryState.available = [];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.log).toHaveBeenCalledTimes(1);
@@ -526,8 +514,6 @@ describe("models list/status", () => {
},
];
modelRegistryState.available = [];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.error).toHaveBeenCalledTimes(1);
@@ -573,8 +559,6 @@ describe("models list/status", () => {
cost: { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 },
},
];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.error).toHaveBeenCalledTimes(1);
@@ -623,8 +607,6 @@ describe("models list/status", () => {
},
];
modelRegistryState.available = [];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.error).toHaveBeenCalledTimes(1);
@@ -654,8 +636,6 @@ describe("models list/status", () => {
code: "MODEL_AVAILABILITY_UNAVAILABLE",
});
const runtime = makeRuntime();
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.error).toHaveBeenCalledTimes(1);
@@ -689,8 +669,6 @@ describe("models list/status", () => {
modelRegistryState.models = [];
modelRegistryState.available = [];
const { modelsListCommand } = await import("./models/list.list-command.js");
await modelsListCommand({ json: true }, runtime);
expect(runtime.error).toHaveBeenCalledTimes(1);

View File

@@ -1,6 +1,36 @@
import type { MessageEvent } from "@line/bot-sdk";
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
// Avoid pulling in globals/pairing/media dependencies; this suite only asserts
// allowlist/groupPolicy gating and message-context wiring.
vi.mock("../globals.js", () => ({
danger: (text: string) => text,
logVerbose: () => {},
}));
vi.mock("../pairing/pairing-labels.js", () => ({
resolvePairingIdLabel: () => "lineUserId",
}));
vi.mock("../pairing/pairing-messages.js", () => ({
buildPairingReply: () => "pairing-reply",
}));
vi.mock("./download.js", () => ({
downloadLineMedia: async () => {
throw new Error("downloadLineMedia should not be called from bot-handlers tests");
},
}));
vi.mock("./send.js", () => ({
pushMessageLine: async () => {
throw new Error("pushMessageLine should not be called from bot-handlers tests");
},
replyMessageLine: async () => {
throw new Error("replyMessageLine should not be called from bot-handlers tests");
},
}));
const { buildLineMessageContextMock, buildLinePostbackContextMock } = vi.hoisted(() => ({
buildLineMessageContextMock: vi.fn(async () => ({
ctxPayload: { From: "line:group:group-1" },

View File

@@ -6,6 +6,10 @@ import { getMemorySearchManager, type MemoryIndexManager } from "./index.js";
let embedBatchCalls = 0;
vi.mock("./sqlite-vec.js", () => ({
loadSqliteVecExtension: async () => ({ ok: false, error: "sqlite-vec disabled in tests" }),
}));
vi.mock("./embeddings.js", () => {
const embedText = (text: string) => {
const lower = text.toLowerCase();

View File

@@ -7,6 +7,10 @@ import { getMemorySearchManager, type MemoryIndexManager } from "./index.js";
const embedBatch = vi.fn(async () => []);
const embedQuery = vi.fn(async () => [0.5, 0.5, 0.5]);
vi.mock("./sqlite-vec.js", () => ({
loadSqliteVecExtension: async () => ({ ok: false, error: "sqlite-vec disabled in tests" }),
}));
vi.mock("./embeddings.js", () => ({
createEmbeddingProvider: async () => ({
requestedProvider: "openai",
@@ -289,6 +293,7 @@ describe("memory indexing with OpenAI batches", () => {
});
it("tracks batch failures, resets on success, and disables after repeated failures", async () => {
const restoreTimeouts = useFastShortTimeouts();
const content = ["flaky", "batch"].join("\n\n");
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-09.md"), content);
@@ -376,62 +381,66 @@ describe("memory indexing with OpenAI batches", () => {
},
};
const result = await getMemorySearchManager({ cfg, agentId: "main" });
expect(result.manager).not.toBeNull();
if (!result.manager) {
throw new Error("manager missing");
try {
const result = await getMemorySearchManager({ cfg, agentId: "main" });
expect(result.manager).not.toBeNull();
if (!result.manager) {
throw new Error("manager missing");
}
manager = result.manager;
// First failure: fallback to regular embeddings and increment failure count.
await manager.sync({ force: true });
expect(embedBatch).toHaveBeenCalled();
let status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(1);
// Success should reset failure count.
embedBatch.mockClear();
mode = "ok";
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "recovery"].join("\n\n"),
);
await manager.sync({ force: true });
status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(0);
expect(embedBatch).not.toHaveBeenCalled();
// Two more failures after reset should disable remote batching.
mode = "fail";
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fail-a"].join("\n\n"),
);
await manager.sync({ force: true });
status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(1);
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fail-b"].join("\n\n"),
);
await manager.sync({ force: true });
status = manager.status();
expect(status.batch?.enabled).toBe(false);
expect(status.batch?.failures).toBeGreaterThanOrEqual(2);
// Once disabled, batch endpoints are skipped and fallback embeddings run directly.
const fetchCalls = fetchMock.mock.calls.length;
embedBatch.mockClear();
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fallback"].join("\n\n"),
);
await manager.sync({ force: true });
expect(fetchMock.mock.calls.length).toBe(fetchCalls);
expect(embedBatch).toHaveBeenCalled();
} finally {
restoreTimeouts();
}
manager = result.manager;
// First failure: fallback to regular embeddings and increment failure count.
await manager.sync({ force: true });
expect(embedBatch).toHaveBeenCalled();
let status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(1);
// Success should reset failure count.
embedBatch.mockClear();
mode = "ok";
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "recovery"].join("\n\n"),
);
await manager.sync({ force: true });
status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(0);
expect(embedBatch).not.toHaveBeenCalled();
// Two more failures after reset should disable remote batching.
mode = "fail";
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fail-a"].join("\n\n"),
);
await manager.sync({ force: true });
status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(1);
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fail-b"].join("\n\n"),
);
await manager.sync({ force: true });
status = manager.status();
expect(status.batch?.enabled).toBe(false);
expect(status.batch?.failures).toBeGreaterThanOrEqual(2);
// Once disabled, batch endpoints are skipped and fallback embeddings run directly.
const fetchCalls = fetchMock.mock.calls.length;
embedBatch.mockClear();
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fallback"].join("\n\n"),
);
await manager.sync({ force: true });
expect(fetchMock.mock.calls.length).toBe(fetchCalls);
expect(embedBatch).toHaveBeenCalled();
});
});

View File

@@ -7,6 +7,10 @@ import { getMemorySearchManager, type MemoryIndexManager } from "./index.js";
const embedBatch = vi.fn(async (texts: string[]) => texts.map(() => [0, 1, 0]));
const embedQuery = vi.fn(async () => [0, 1, 0]);
vi.mock("./sqlite-vec.js", () => ({
loadSqliteVecExtension: async () => ({ ok: false, error: "sqlite-vec disabled in tests" }),
}));
vi.mock("./embeddings.js", () => ({
createEmbeddingProvider: async () => ({
requestedProvider: "openai",

View File

@@ -7,6 +7,10 @@ import { getMemorySearchManager, type MemoryIndexManager } from "./index.js";
const embedBatch = vi.fn(async (texts: string[]) => texts.map(() => [0, 1, 0]));
const embedQuery = vi.fn(async () => [0, 1, 0]);
vi.mock("./sqlite-vec.js", () => ({
loadSqliteVecExtension: async () => ({ ok: false, error: "sqlite-vec disabled in tests" }),
}));
vi.mock("./embeddings.js", () => ({
createEmbeddingProvider: async () => ({
requestedProvider: "openai",