diff --git a/CHANGELOG.md b/CHANGELOG.md index 39e4f66387..237103ac5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,7 +68,7 @@ Docs: https://docs.openclaw.ai - Gateway/Config: prevent `config.patch` object-array merges from falling back to full-array replacement when some patch entries lack `id`, so partial `agents.list` updates no longer drop unrelated agents. (#17989) Thanks @stakeswky. - Config/Discord: require string IDs in Discord allowlists, keep onboarding inputs string-only, and add doctor repair for numeric entries. (#18220) Thanks @thewilloftheshadow. - Security/Sessions: create new session transcript JSONL files with user-only (`0o600`) permissions and extend `openclaw security audit --fix` to remediate existing transcript file permissions. -- Sessions/Maintenance: archive transcripts when pruning stale sessions, clean expired media in subdirectories, and purge old deleted transcript archives to prevent disk leaks. (#18538) +- Sessions/Maintenance: archive transcripts when pruning stale sessions, clean expired media in subdirectories, and purge `.deleted` transcript archives after the prune window to prevent disk leaks. (#18538) - Infra/Fetch: ensure foreign abort-signal listener cleanup never masks original fetch successes/failures, while still preventing detached-finally unhandled rejection noise in `wrapFetchWithAbortSignal`. Thanks @Jackten. - Heartbeat: allow suppressing tool error warning payloads during heartbeat runs via a new heartbeat config flag. (#18497) Thanks @thewilloftheshadow. - Heartbeat: include sender metadata (From/To/Provider) in heartbeat prompts so model context matches the delivery target. (#18532) diff --git a/src/config/sessions/store.pruning.e2e.test.ts b/src/config/sessions/store.pruning.e2e.test.ts index 95ae5ee03e..a382e050a5 100644 --- a/src/config/sessions/store.pruning.e2e.test.ts +++ b/src/config/sessions/store.pruning.e2e.test.ts @@ -3,8 +3,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { clearSessionStoreCacheForTest, loadSessionStore, saveSessionStore } from "./store.js"; import type { SessionEntry } from "./types.js"; +import { clearSessionStoreCacheForTest, loadSessionStore, saveSessionStore } from "./store.js"; // Keep integration tests deterministic: never read a real openclaw.json. vi.mock("../config.js", () => ({ @@ -13,6 +13,8 @@ vi.mock("../config.js", () => ({ const DAY_MS = 24 * 60 * 60 * 1000; +const archiveTimestamp = (ms: number) => new Date(ms).toISOString().replaceAll(":", "-"); + let fixtureRoot = ""; let fixtureCount = 0; @@ -124,6 +126,51 @@ describe("Integration: saveSessionStore with pruning", () => { expect(archived).toHaveLength(1); }); + it("cleans up archived transcripts older than the prune window", async () => { + mockLoadConfig.mockReturnValue({ + session: { + maintenance: { + mode: "enforce", + pruneAfter: "7d", + maxEntries: 500, + rotateBytes: 10_485_760, + }, + }, + }); + + const now = Date.now(); + const staleSessionId = "stale-session"; + const store: Record = { + stale: { sessionId: staleSessionId, updatedAt: now - 30 * DAY_MS }, + fresh: { sessionId: "fresh-session", updatedAt: now }, + }; + + const staleTranscript = path.join(testDir, `${staleSessionId}.jsonl`); + await fs.writeFile(staleTranscript, '{"type":"session"}\n', "utf-8"); + + const oldArchived = path.join( + testDir, + `old-session.jsonl.deleted.${archiveTimestamp(now - 9 * DAY_MS)}`, + ); + const recentArchived = path.join( + testDir, + `recent-session.jsonl.deleted.${archiveTimestamp(now - 2 * DAY_MS)}`, + ); + const bakArchived = path.join( + testDir, + `bak-session.jsonl.bak.${archiveTimestamp(now - 20 * DAY_MS)}`, + ); + await fs.writeFile(oldArchived, "old", "utf-8"); + await fs.writeFile(recentArchived, "recent", "utf-8"); + await fs.writeFile(bakArchived, "bak", "utf-8"); + + await saveSessionStore(storePath, store); + + await expect(fs.stat(oldArchived)).rejects.toThrow(); + await expect(fs.stat(recentArchived)).resolves.toBeDefined(); + await expect(fs.stat(bakArchived)).resolves.toBeDefined(); + }); + it("saveSessionStore skips enforcement when maintenance mode is warn", async () => { mockLoadConfig.mockReturnValue({ session: { diff --git a/src/config/sessions/store.ts b/src/config/sessions/store.ts index 60e741dcb3..e0ff6d36ff 100644 --- a/src/config/sessions/store.ts +++ b/src/config/sessions/store.ts @@ -1,11 +1,15 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { acquireSessionWriteLock } from "../../agents/session-write-lock.js"; import type { MsgContext } from "../../auto-reply/templating.js"; +import type { SessionMaintenanceConfig, SessionMaintenanceMode } from "../types.base.js"; +import { acquireSessionWriteLock } from "../../agents/session-write-lock.js"; import { parseByteSize } from "../../cli/parse-bytes.js"; import { parseDurationMs } from "../../cli/parse-duration.js"; -import { archiveSessionTranscripts } from "../../gateway/session-utils.fs.js"; +import { + archiveSessionTranscripts, + cleanupArchivedSessionTranscripts, +} from "../../gateway/session-utils.fs.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { deliveryContextFromSession, @@ -16,7 +20,6 @@ import { } from "../../utils/delivery-context.js"; import { getFileMtimeMs, isCacheEnabled, resolveCacheTtlMs } from "../cache-utils.js"; import { loadConfig } from "../config.js"; -import type { SessionMaintenanceConfig, SessionMaintenanceMode } from "../types.base.js"; import { deriveSessionMetaPatch } from "./metadata.js"; import { mergeSessionEntry, type SessionEntry } from "./types.js"; @@ -538,13 +541,24 @@ async function saveSessionStoreUnlocked( }, }); capEntryCount(store, maintenance.maxEntries); + const archivedDirs = new Set(); for (const [sessionId, sessionFile] of prunedSessionFiles) { - archiveSessionTranscripts({ + const archived = archiveSessionTranscripts({ sessionId, storePath, sessionFile, reason: "deleted", }); + for (const archivedPath of archived) { + archivedDirs.add(path.dirname(archivedPath)); + } + } + if (archivedDirs.size > 0) { + await cleanupArchivedSessionTranscripts({ + directories: [...archivedDirs], + olderThanMs: maintenance.pruneAfterMs, + reason: "deleted", + }); } // Rotate the on-disk file if it exceeds the size threshold. diff --git a/src/gateway/session-utils.fs.ts b/src/gateway/session-utils.fs.ts index 38d478ee32..c3edd8cf0a 100644 --- a/src/gateway/session-utils.fs.ts +++ b/src/gateway/session-utils.fs.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { SessionPreviewItem } from "./session-utils.types.js"; import { resolveSessionFilePath, resolveSessionTranscriptPath, @@ -10,7 +11,6 @@ import { resolveRequiredHomeDir } from "../infra/home-dir.js"; import { hasInterSessionUserProvenance } from "../sessions/input-provenance.js"; import { extractToolCallNames, hasToolCall } from "../utils/transcript-tools.js"; import { stripEnvelope } from "./chat-sanitize.js"; -import type { SessionPreviewItem } from "./session-utils.types.js"; type SessionTitleFields = { firstUserMessage: string | null; @@ -197,6 +197,67 @@ export function archiveSessionTranscripts(opts: { return archived; } +function restoreArchiveTimestamp(raw: string): string { + const [datePart, timePart] = raw.split("T"); + if (!datePart || !timePart) { + return raw; + } + return `${datePart}T${timePart.replace(/-/g, ":")}`; +} + +function parseArchivedTimestamp(fileName: string, reason: ArchiveFileReason): number | null { + const marker = `.${reason}.`; + const index = fileName.lastIndexOf(marker); + if (index < 0) { + return null; + } + const raw = fileName.slice(index + marker.length); + if (!raw) { + return null; + } + const timestamp = Date.parse(restoreArchiveTimestamp(raw)); + return Number.isNaN(timestamp) ? null : timestamp; +} + +export async function cleanupArchivedSessionTranscripts(opts: { + directories: string[]; + olderThanMs: number; + reason?: "deleted"; + nowMs?: number; +}): Promise<{ removed: number; scanned: number }> { + if (!Number.isFinite(opts.olderThanMs) || opts.olderThanMs < 0) { + return { removed: 0, scanned: 0 }; + } + const now = opts.nowMs ?? Date.now(); + const reason: ArchiveFileReason = opts.reason ?? "deleted"; + const directories = Array.from(new Set(opts.directories.map((dir) => path.resolve(dir)))); + let removed = 0; + let scanned = 0; + + for (const dir of directories) { + const entries = await fs.promises.readdir(dir).catch(() => []); + for (const entry of entries) { + const timestamp = parseArchivedTimestamp(entry, reason); + if (timestamp == null) { + continue; + } + scanned += 1; + if (now - timestamp <= opts.olderThanMs) { + continue; + } + const fullPath = path.join(dir, entry); + const stat = await fs.promises.stat(fullPath).catch(() => null); + if (!stat?.isFile()) { + continue; + } + await fs.promises.rm(fullPath).catch(() => undefined); + removed += 1; + } + } + + return { removed, scanned }; +} + function jsonUtf8Bytes(value: unknown): number { try { return Buffer.byteLength(JSON.stringify(value), "utf8");