From faaa7efef050dc19e42fa3273521bac352e036ea Mon Sep 17 00:00:00 2001 From: "clawsweeper[bot]" <274271284+clawsweeper[bot]@users.noreply.github.com> Date: Wed, 13 May 2026 16:31:04 +0800 Subject: [PATCH] fix(security): inline redact into appendSessionTranscriptMessage (#79645) Merged via squash. Prepared head SHA: da91ab6cf111db8ee9417f6717689604095f378e Co-authored-by: app/clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: hxy91819 <8814856+hxy91819@users.noreply.github.com> Reviewed-by: @hxy91819 --- CHANGELOG.md | 1 + .../telegram/src/bot-message-dispatch.test.ts | 52 ++ .../telegram/src/bot-message-dispatch.ts | 4 +- src/agents/pi-embedded-runner.guard.test.ts | 28 +- .../session-tool-result-guard-wrapper.ts | 67 +-- src/agents/session-tool-result-guard.test.ts | 48 ++ src/agents/transcript-redact.test.ts | 407 +++++++++++++++ src/agents/transcript-redact.ts | 108 ++++ .../sessions/transcript-append-redact.test.ts | 490 ++++++++++++++++++ src/config/sessions/transcript-append.ts | 46 +- src/config/sessions/transcript.ts | 26 +- .../server-methods/chat-transcript-inject.ts | 10 +- .../chat.inject.parentid.test.ts | 32 ++ src/logging/redact.ts | 18 +- 14 files changed, 1217 insertions(+), 120 deletions(-) create mode 100644 src/agents/transcript-redact.test.ts create mode 100644 src/agents/transcript-redact.ts create mode 100644 src/config/sessions/transcript-append-redact.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 65c69e2c516..113bac189b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Docs: https://docs.openclaw.ai - Agents: escalate LLM idle watchdog timeouts through profile rotation and configured model fallback instead of leaving agent turns stuck after a silent model stream. Fixes #76877. (#80449) Thanks @jimdawdy-hub. - ACPX: stop forwarding unsupported timeout config options to Claude ACP while preserving OpenClaw's own turn timeout. (#80812) Thanks @sxxtony. +- Session transcripts: redact sensitive message content in the centralized JSONL append path so CLI turns, gateway transcript injection, transcript mirrors, and guarded tool results use the same configured redaction behavior. Fixes #73565. Refs #73563. (#79645) Thanks @app/clawsweeper. - Channels/iMessage: ignore Apple link-preview plugin payload attachments when users paste URLs, keeping the URL text while avoiding phantom media context. (#79374) Thanks @homer-byte. - Telegram: detect polling stalls from `getUpdates` liveness only, so outbound API calls no longer mask dead inbound polling; log polling-cycle starts after transport rebuilds. Fixes #78473. - fix(plugins): scan installed dependency runtime code [AI]. (#81066) Thanks @pgondhi987. diff --git a/extensions/telegram/src/bot-message-dispatch.test.ts b/extensions/telegram/src/bot-message-dispatch.test.ts index 2e756cf93ba..31b361710ff 100644 --- a/extensions/telegram/src/bot-message-dispatch.test.ts +++ b/extensions/telegram/src/bot-message-dispatch.test.ts @@ -959,6 +959,58 @@ describe("dispatchTelegramMessage draft streaming", () => { }); }); + it("emits the redacted appended message in transcript updates", async () => { + setupDraftStreams({ answerMessageId: 2001 }); + const context = createContext(); + context.ctxPayload.SessionKey = "agent:default:telegram:direct:123"; + loadSessionStore.mockReturnValue({ + "agent:default:telegram:direct:123": { sessionId: "s1" }, + }); + appendSessionTranscriptMessage.mockImplementationOnce(async ({ message }) => ({ + messageId: "m1", + message: { + ...(message as Record), + content: [{ type: "text", text: "Final sk-abc…0xyz" }], + }, + })); + dispatchReplyWithBufferedBlockDispatcher.mockImplementation(async ({ dispatcherOptions }) => { + await dispatcherOptions.deliver({ text: "Final sk-abcdef1234567890xyz" }, { kind: "final" }); + return { queuedFinal: true }; + }); + + await dispatchWithContext({ context }); + + expectRecordFields(mockCallArg(emitSessionTranscriptUpdate), { + sessionFile: "/tmp/session.jsonl", + sessionKey: "agent:default:telegram:direct:123", + messageId: "m1", + message: { + role: "assistant", + content: [{ type: "text", text: "Final sk-abc…0xyz" }], + api: "openai-responses", + provider: "openclaw", + model: "delivery-mirror", + usage: { + input: 0, + output: 0, + total: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + cache: { + read: 0, + write: 0, + cacheRead: 0, + cacheWrite: 0, + total: 0, + }, + }, + stopReason: "stop", + timestamp: expect.any(Number), + }, + }); + }); + it("streams block and final text through the same answer message", async () => { const { answerDraftStream } = setupDraftStreams({ answerMessageId: 2001 }); dispatchReplyWithBufferedBlockDispatcher.mockImplementation( diff --git a/extensions/telegram/src/bot-message-dispatch.ts b/extensions/telegram/src/bot-message-dispatch.ts index 20c6c1ed4fa..d2f5699f24b 100644 --- a/extensions/telegram/src/bot-message-dispatch.ts +++ b/extensions/telegram/src/bot-message-dispatch.ts @@ -337,7 +337,7 @@ async function mirrorTelegramAssistantReplyToTranscript(params: { stopReason: "stop" as const, timestamp: Date.now(), }; - const { messageId } = await appendSessionTranscriptMessage({ + const { messageId, message: appendedMessage } = await appendSessionTranscriptMessage({ transcriptPath: sessionFile, message, config: params.cfg, @@ -345,7 +345,7 @@ async function mirrorTelegramAssistantReplyToTranscript(params: { emitSessionTranscriptUpdate({ sessionFile, sessionKey: params.sessionKey, - message, + message: appendedMessage, messageId, }); } diff --git a/src/agents/pi-embedded-runner.guard.test.ts b/src/agents/pi-embedded-runner.guard.test.ts index a29ee49fe48..35c5ba2b556 100644 --- a/src/agents/pi-embedded-runner.guard.test.ts +++ b/src/agents/pi-embedded-runner.guard.test.ts @@ -128,23 +128,15 @@ describe("guardSessionManager integration", () => { .filter((e) => e.type === "message") .map((e) => (e as { message: AgentMessage }).message); - expect(messages).toEqual([ - { - role: "assistant", - content: [ - { type: "thinking", thinking: "the email is peter@d***.io", thinkingSignature: "sig" }, - { type: "text", text: "contact peter@d***.io" }, - { type: "toolCall", id: "call_1", name: "read", arguments: { path: "/tmp/peter@dc.io" } }, - ], - stopReason: "toolUse", - }, - { - role: "toolResult", - toolCallId: "call_1", - toolName: "read", - content: [{ type: "text", text: "peter@d***.io\n" }], - isError: false, - }, - ]); + const serialized = JSON.stringify(messages); + + expect(serialized).not.toContain("the email is peter@dc.io"); + expect(serialized).not.toContain("contact peter@dc.io"); + expect(serialized).not.toContain("peter@dc.io\\n"); + expect(serialized).not.toContain('"/tmp/peter@dc.io"'); + expect(serialized).toContain('"thinking":"the email is peter@d***.io"'); + expect(serialized).toContain('"text":"contact peter@d***.io"'); + expect(serialized).toContain('"text":"peter@d***.io\\n"'); + expect(serialized).toContain('"/tmp/peter@d***.io"'); }); }); diff --git a/src/agents/session-tool-result-guard-wrapper.ts b/src/agents/session-tool-result-guard-wrapper.ts index b7f16234b09..46c5eacbcf5 100644 --- a/src/agents/session-tool-result-guard-wrapper.ts +++ b/src/agents/session-tool-result-guard-wrapper.ts @@ -1,7 +1,6 @@ import type { AgentMessage } from "@earendil-works/pi-agent-core"; import type { SessionManager } from "@earendil-works/pi-coding-agent"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { redactSensitiveText } from "../logging/redact.js"; import { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; import { applyInputProvenanceToUserMessage, @@ -9,6 +8,7 @@ import { } from "../sessions/input-provenance.js"; import { resolveLiveToolResultMaxChars } from "./pi-embedded-runner/tool-result-truncation.js"; import { installSessionToolResultGuard } from "./session-tool-result-guard.js"; +import { redactTranscriptMessage } from "./transcript-redact.js"; type GuardedSessionManager = SessionManager & { /** Flush any synthetic tool results for pending tool calls. Idempotent. */ @@ -17,71 +17,6 @@ type GuardedSessionManager = SessionManager & { clearPendingToolResults?: () => void; }; -function redactTranscriptText(value: string, cfg?: OpenClawConfig): string { - if (cfg?.logging?.redactSensitive === "off") { - return value; - } - return redactSensitiveText(value, { - mode: cfg?.logging?.redactSensitive, - patterns: cfg?.logging?.redactPatterns, - }); -} - -function redactTranscriptContentBlock(block: unknown, cfg?: OpenClawConfig): unknown { - if (!block || typeof block !== "object" || Array.isArray(block)) { - return block; - } - const source = block as Record; - let next: Record | null = null; - const assign = (key: string, value: string) => { - const redacted = redactTranscriptText(value, cfg); - if (redacted === value) { - return; - } - next ??= { ...source }; - next[key] = redacted; - }; - - if (typeof source.text === "string") { - assign("text", source.text); - } - if (typeof source.thinking === "string") { - assign("thinking", source.thinking); - } - if (typeof source.partialJson === "string") { - assign("partialJson", source.partialJson); - } - return next ?? block; -} - -function redactTranscriptContent(content: unknown, cfg?: OpenClawConfig): unknown { - if (typeof content === "string") { - return redactTranscriptText(content, cfg); - } - if (!Array.isArray(content)) { - return content; - } - let changed = false; - const redacted = content.map((block) => { - const next = redactTranscriptContentBlock(block, cfg); - changed ||= next !== block; - return next; - }); - return changed ? redacted : content; -} - -function redactTranscriptMessage(message: AgentMessage, cfg?: OpenClawConfig): AgentMessage { - const source = message as unknown as Record; - const redactedContent = redactTranscriptContent(source.content, cfg); - if (redactedContent === source.content) { - return message; - } - return { - ...source, - content: redactedContent, - } as unknown as AgentMessage; -} - /** * Apply the tool-result guard to a SessionManager exactly once and expose * a flush method on the instance for easy teardown handling. diff --git a/src/agents/session-tool-result-guard.test.ts b/src/agents/session-tool-result-guard.test.ts index a48145567a6..c4f14c7b296 100644 --- a/src/agents/session-tool-result-guard.test.ts +++ b/src/agents/session-tool-result-guard.test.ts @@ -3,6 +3,7 @@ import { SessionManager } from "@earendil-works/pi-coding-agent"; import { describe, expect, it } from "vitest"; import { installSessionToolResultGuard } from "./session-tool-result-guard.js"; import { castAgentMessage } from "./test-helpers/agent-message-fixtures.js"; +import { redactTranscriptMessage } from "./transcript-redact.js"; type AppendMessage = Parameters[0]; @@ -452,6 +453,53 @@ describe("installSessionToolResultGuard", () => { expect(text).toBe("rewritten by hook"); }); + it("applies before_message_write redaction to tool-result details before persistence", () => { + const sm = SessionManager.inMemory(); + installSessionToolResultGuard(sm, { + beforeMessageWriteHook: ({ message }) => ({ + message: redactTranscriptMessage(message, { logging: { redactSensitive: "tools" } }), + }), + }); + + sm.appendMessage(toolCallMessage); + sm.appendMessage( + asAppendMessage({ + role: "toolResult", + toolCallId: "call_1", + toolName: "read", + content: [{ type: "text", text: "result sk-abcdef1234567890xyz" }], + details: { + apiKey: "plainsecretvalue123", + password: "hunter2", + nested: { accessToken: ["nestedplainsecret123"] }, + safe: "visible", + }, + isError: false, + timestamp: Date.now(), + }), + ); + + const messages = getPersistedMessages(sm); + const toolResult = messages.find((m) => m.role === "toolResult") as unknown as { + content: Array<{ text: string }>; + details: { + apiKey: string; + password: string; + nested: { accessToken: string[] }; + safe: string; + }; + }; + const serializedToolResult = JSON.stringify(toolResult); + expect(toolResult.content[0].text).not.toContain("sk-abcdef1234567890xyz"); + expect(serializedToolResult).not.toContain("plainsecretvalue123"); + expect(serializedToolResult).not.toContain("hunter2"); + expect(serializedToolResult).not.toContain("nestedplainsecret123"); + expect(toolResult.details.apiKey).toBe("***"); + expect(toolResult.details.password).toBe("***"); + expect(toolResult.details.nested.accessToken[0]).toBe("***"); + expect(serializedToolResult).toContain("visible"); + }); + it("applies before_message_write to synthetic tool-result flushes", () => { const sm = SessionManager.inMemory(); const guard = installSessionToolResultGuard(sm, { diff --git a/src/agents/transcript-redact.test.ts b/src/agents/transcript-redact.test.ts new file mode 100644 index 00000000000..fb65e5e475d --- /dev/null +++ b/src/agents/transcript-redact.test.ts @@ -0,0 +1,407 @@ +import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import { describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { redactTranscriptMessage } from "./transcript-redact.js"; + +/** Typed accessor for `content` on AgentMessage. + * AgentMessage is a union that includes custom message types (e.g. BashExecutionMessage) + * which have no `content` field. Direct `.content` access fails tsgo's strict union check. + */ +function msgContent(msg: AgentMessage): unknown { + return (msg as unknown as { content: unknown }).content; +} + +function textMessage(text: string): AgentMessage { + return { + role: "assistant", + content: [{ type: "text", text }], + } as unknown as AgentMessage; +} + +function cfg(mode: "tools" | "off", patterns?: string[]): OpenClawConfig { + return { + logging: { + redactSensitive: mode, + ...(patterns ? { redactPatterns: patterns } : {}), + }, + } satisfies OpenClawConfig; +} + +const EMAIL_PATTERN = String.raw`([\w]|[-.])+@([\w]|[-.])+\.\w+`; + +describe("redactTranscriptMessage", () => { + it("redacts text block matching default patterns (sk- token)", () => { + const msg = textMessage("key is sk-abcdef1234567890xyz end"); + const result = redactTranscriptMessage(msg, cfg("tools")); + const text = (msgContent(result) as Array<{ text: string }>)[0].text; + expect(text).not.toContain("sk-abcdef1234567890xyz"); + expect(text).toContain("end"); + }); + + it("redacts thinking block", () => { + const msg = { + role: "assistant", + content: [ + { type: "thinking", thinking: "secret sk-abcdef1234567890xyz", thinkingSignature: "sig" }, + ], + } as unknown as AgentMessage; + const result = redactTranscriptMessage(msg, cfg("tools")); + const block = (msgContent(result) as Array<{ thinking: string }>)[0]; + expect(block.thinking).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("redacts partialJson block", () => { + const msg = { + role: "assistant", + content: [{ type: "toolCallDelta", partialJson: '{"key":"sk-abcdef1234567890xyz"}' }], + } as unknown as AgentMessage; + const result = redactTranscriptMessage(msg, cfg("tools")); + const block = (msgContent(result) as Array<{ partialJson: string }>)[0]; + expect(block.partialJson).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("redacts nested strings in assistant tool-call arguments", () => { + const msg = { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_1", + name: "shell", + arguments: { + command: "OPENAI_API_KEY=sk-abcdef1234567890xyz openclaw health", + env: { nested: ["token sk-abcdef1234567890xyz"] }, + count: 1, + }, + }, + ], + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")); + const block = (msgContent(result) as Array<{ arguments: unknown }>)[0]; + const argumentsValue = block.arguments as { + command: string; + env: { nested: string[] }; + count: number; + }; + const serializedArguments = JSON.stringify(block.arguments); + expect(serializedArguments).not.toContain("sk-abcdef1234567890xyz"); + expect(argumentsValue.command).toBe("OPENAI_API_KEY=sk-abc…0xyz openclaw health"); + expect(argumentsValue.env.nested[0]).toBe("token sk-abc…0xyz"); + expect(argumentsValue.count).toBe(1); + expect(serializedArguments).toContain("openclaw health"); + expect(block.arguments).not.toBe( + (msgContent(msg) as Array<{ arguments: unknown }>)[0].arguments, + ); + }); + + it("redacts structured secret fields in assistant tool-call arguments", () => { + const msg = { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_1", + name: "send_request", + arguments: { + apiKey: "plainsecretvalue123", + password: "hunter2", + nested: { accessToken: ["nestedplainsecret123"] }, + safe: "visible", + }, + }, + ], + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")); + const block = (msgContent(result) as Array<{ arguments: unknown }>)[0]; + const argumentsValue = block.arguments as { + apiKey: string; + password: string; + nested: { accessToken: string[] }; + safe: string; + }; + const serializedArguments = JSON.stringify(block.arguments); + expect(serializedArguments).not.toContain("plainsecretvalue123"); + expect(serializedArguments).not.toContain("hunter2"); + expect(serializedArguments).not.toContain("nestedplainsecret123"); + expect(argumentsValue.apiKey).toBe("plains…e123"); + expect(argumentsValue.password).toBe("***"); + expect(argumentsValue.nested.accessToken[0]).toBe("nested…t123"); + expect(serializedArguments).toContain("visible"); + }); + + it("redacts structured tool-use input payloads", () => { + const msg = { + role: "assistant", + content: [ + { + type: "toolUse", + id: "call_1", + name: "send_request", + input: { + apiKey: "plainsecretvalue123", + nested: { accessToken: ["nestedplainsecret123"] }, + command: "OPENAI_API_KEY=sk-abcdef1234567890xyz openclaw health", + safe: "visible", + }, + }, + ], + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")); + const block = (msgContent(result) as Array<{ input: unknown }>)[0]; + const inputValue = block.input as { + apiKey: string; + nested: { accessToken: string[] }; + command: string; + safe: string; + }; + const serializedInput = JSON.stringify(block.input); + expect(serializedInput).not.toContain("plainsecretvalue123"); + expect(serializedInput).not.toContain("nestedplainsecret123"); + expect(serializedInput).not.toContain("sk-abcdef1234567890xyz"); + expect(inputValue.apiKey).toBe("plains…e123"); + expect(inputValue.nested.accessToken[0]).toBe("nested…t123"); + expect(inputValue.command).toBe("OPENAI_API_KEY=sk-abc…0xyz openclaw health"); + expect(serializedInput).toContain("visible"); + }); + + it("redacts defensive function-call input payloads", () => { + const msg = { + role: "assistant", + content: [ + { + type: "functionCall", + id: "call_1", + name: "send_request", + input: { + password: "hunter2", + nested: { accessToken: ["nestedplainsecret123"] }, + }, + }, + ], + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")); + const block = (msgContent(result) as Array<{ input: unknown }>)[0]; + const inputValue = block.input as { + password: string; + nested: { accessToken: string[] }; + }; + const serializedInput = JSON.stringify(block.input); + expect(serializedInput).not.toContain("hunter2"); + expect(serializedInput).not.toContain("nestedplainsecret123"); + expect(inputValue.password).toBe("***"); + expect(inputValue.nested.accessToken[0]).toBe("nested…t123"); + }); + + it("redacts arbitrary gateway/custom content-block fields recursively", () => { + const msg = { + role: "assistant", + content: [ + { + type: "gatewayCustom", + source: { + url: "https://example.com/callback?token=sk-abcdef1234567890xyz", + }, + data: { + apiKey: "plainsecretvalue123", + nested: { + accessToken: "nestedplainsecret123", + }, + }, + safe: "visible", + }, + ], + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")); + const block = (msgContent(result) as Array>)[0]; + const serializedBlock = JSON.stringify(block); + expect(serializedBlock).not.toContain("sk-abcdef1234567890xyz"); + expect(serializedBlock).not.toContain("plainsecretvalue123"); + expect(serializedBlock).not.toContain("nestedplainsecret123"); + expect(serializedBlock).toContain("visible"); + }); + + it("redacts circular structured payloads without throwing", () => { + const details: Record = { + apiKey: "plainsecretvalue123", + }; + details.self = details; + const msg = { + role: "toolResult", + toolCallId: "call_1", + toolName: "send_request", + content: [{ type: "text", text: "result" }], + details, + isError: false, + timestamp: Date.now(), + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")) as unknown as { + details: Record; + }; + expect(result.details.apiKey).toBe("plains…e123"); + expect(result.details.self).toBe("[Circular]"); + }); + + it("redacts structured secret fields in tool-result details", () => { + const msg = { + role: "toolResult", + toolCallId: "call_1", + toolName: "send_request", + content: [{ type: "text", text: "result sk-abcdef1234567890xyz" }], + details: { + apiKey: "plainsecretvalue123", + password: "hunter2", + nested: { accessToken: ["nestedplainsecret123"] }, + safe: "visible", + }, + isError: false, + timestamp: Date.now(), + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")) as unknown as { + content: Array<{ text: string }>; + details: unknown; + }; + const serializedDetails = JSON.stringify(result.details); + const details = result.details as { + apiKey: string; + password: string; + nested: { accessToken: string[] }; + safe: string; + }; + expect(result.content[0].text).not.toContain("sk-abcdef1234567890xyz"); + expect(serializedDetails).not.toContain("plainsecretvalue123"); + expect(serializedDetails).not.toContain("hunter2"); + expect(serializedDetails).not.toContain("nestedplainsecret123"); + expect(details.apiKey).toBe("plains…e123"); + expect(details.password).toBe("***"); + expect(details.nested.accessToken[0]).toBe("nested…t123"); + expect(serializedDetails).toContain("visible"); + }); + + it("redacts string-form content", () => { + const msg = { + role: "user", + content: "my key is sk-abcdef1234567890xyz", + } as unknown as AgentMessage; + const result = redactTranscriptMessage(msg, cfg("tools")); + expect(msgContent(result) as string).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("redacts documented transcript text fields on content-less message types", () => { + const msg = { + role: "bashExecution", + command: "OPENAI_API_KEY=sk-abcdef1234567890xyz openclaw health", + output: "failed with sk-abcdef1234567890xyz", + exitCode: 1, + cancelled: false, + truncated: false, + timestamp: Date.now(), + } as unknown as AgentMessage; + + const result = redactTranscriptMessage(msg, cfg("tools")) as unknown as { + command: string; + output: string; + }; + expect(result.command).not.toContain("sk-abcdef1234567890xyz"); + expect(result.output).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("redacts assistant error and summary transcript fields", () => { + const assistant = { + role: "assistant", + content: [{ type: "text", text: "safe" }], + errorMessage: "provider rejected sk-abcdef1234567890xyz", + } as unknown as AgentMessage; + const summary = { + role: "compactionSummary", + summary: "summary mentions sk-abcdef1234567890xyz", + tokensBefore: 10, + timestamp: Date.now(), + } as unknown as AgentMessage; + + const assistantResult = redactTranscriptMessage(assistant, cfg("tools")) as unknown as { + errorMessage: string; + }; + const summaryResult = redactTranscriptMessage(summary, cfg("tools")) as unknown as { + summary: string; + }; + expect(assistantResult.errorMessage).not.toContain("sk-abcdef1234567890xyz"); + expect(summaryResult.summary).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("redacts using custom pattern without dropping default patterns", () => { + const msg = textMessage("email peter@dc.io and key sk-abcdef1234567890xyz ok"); + const result = redactTranscriptMessage(msg, cfg("tools", [EMAIL_PATTERN])); + const text = (msgContent(result) as Array<{ text: string }>)[0].text; + expect(text).not.toContain("peter@dc.io"); + expect(text).not.toContain("sk-abcdef1234567890xyz"); + expect(text).toContain("ok"); + }); + + it("passes through unchanged when redactSensitive is off", () => { + const msg = textMessage("key is sk-abcdef1234567890xyz"); + const result = redactTranscriptMessage(msg, cfg("off")); + expect(result).toBe(msg); // same reference; nothing changed + }); + + it("leaves structured tool-call secrets unchanged when redactSensitive is off", () => { + const msg = { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_1", + name: "send_request", + arguments: { apiKey: "plainsecretvalue123", password: "hunter2" }, + }, + ], + } as unknown as AgentMessage; + const result = redactTranscriptMessage(msg, cfg("off")); + expect(result).toBe(msg); + expect(JSON.stringify(msgContent(result))).toContain("plainsecretvalue123"); + expect(JSON.stringify(msgContent(result))).toContain("hunter2"); + }); + + it("leaves structured tool-result details unchanged when redactSensitive is off", () => { + const msg = { + role: "toolResult", + toolCallId: "call_1", + toolName: "send_request", + content: [{ type: "text", text: "result" }], + details: { apiKey: "plainsecretvalue123", password: "hunter2" }, + isError: false, + timestamp: Date.now(), + } as unknown as AgentMessage; + const result = redactTranscriptMessage(msg, cfg("off")) as unknown as { details: unknown }; + expect(result).toBe(msg); + expect(JSON.stringify(result.details)).toContain("plainsecretvalue123"); + expect(JSON.stringify(result.details)).toContain("hunter2"); + }); + + it("returns same object reference when nothing matches", () => { + const msg = textMessage("nothing sensitive here"); + const result = redactTranscriptMessage(msg, cfg("tools")); + expect(result).toBe(msg); + }); + + it("redacts with cfg=undefined (falls back to default patterns)", () => { + const msg = textMessage("key is sk-abcdef1234567890xyz"); + const result = redactTranscriptMessage(msg, undefined); + const text = (msgContent(result) as Array<{ text: string }>)[0].text; + expect(text).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("passes through non-object and null blocks without throwing", () => { + const msg = { + role: "assistant", + content: [null, 42, "raw string"], + } as unknown as AgentMessage; + expect(() => redactTranscriptMessage(msg, cfg("tools"))).not.toThrow(); + }); +}); diff --git a/src/agents/transcript-redact.ts b/src/agents/transcript-redact.ts new file mode 100644 index 00000000000..53746a34df2 --- /dev/null +++ b/src/agents/transcript-redact.ts @@ -0,0 +1,108 @@ +import type { AgentMessage } from "@earendil-works/pi-agent-core"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { readLoggingConfig } from "../logging/config.js"; +import { + getDefaultRedactPatterns, + redactSensitiveFieldValue, + redactSensitiveText, +} from "../logging/redact.js"; + +function resolveTranscriptRedactPatterns(patterns?: string[]) { + return patterns && patterns.length > 0 ? [...patterns, ...getDefaultRedactPatterns()] : undefined; +} + +function redactTranscriptOptions(cfg?: OpenClawConfig) { + const configuredLogging = readLoggingConfig(); + const mode = cfg?.logging?.redactSensitive ?? configuredLogging?.redactSensitive; + const patterns = resolveTranscriptRedactPatterns( + cfg?.logging?.redactPatterns ?? configuredLogging?.redactPatterns, + ); + if (mode === undefined && patterns === undefined) { + return undefined; + } + return { + ...(mode !== undefined ? { mode } : {}), + ...(patterns !== undefined ? { patterns } : {}), + }; +} + +function redactTranscriptText(value: string, cfg?: OpenClawConfig): string { + if (cfg?.logging?.redactSensitive === "off") { + return value; + } + return redactSensitiveText(value, redactTranscriptOptions(cfg)); +} + +function redactTranscriptStructuredFieldValue( + key: string, + value: string, + cfg?: OpenClawConfig, +): string { + if (cfg?.logging?.redactSensitive === "off") { + return value; + } + return redactSensitiveFieldValue(key, value, redactTranscriptOptions(cfg)); +} + +function isPlainTranscriptObject(value: object): value is Record { + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function redactTranscriptStructuredValue( + value: unknown, + cfg?: OpenClawConfig, + fieldKey?: string, + seen: WeakSet = new WeakSet(), +): unknown { + if (typeof value === "string") { + if (fieldKey) { + return redactTranscriptStructuredFieldValue(fieldKey, value, cfg); + } + return redactTranscriptText(value, cfg); + } + if (Array.isArray(value)) { + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + let changed = false; + const redacted = value.map((item) => { + const next = redactTranscriptStructuredValue(item, cfg, fieldKey, seen); + changed ||= next !== item; + return next; + }); + seen.delete(value); + return changed ? redacted : value; + } + if (!value || typeof value !== "object") { + return value; + } + if (seen.has(value)) { + return "[Circular]"; + } + if (!isPlainTranscriptObject(value)) { + return value; + } + + seen.add(value); + const source = value; + let next: Record | null = null; + for (const [key, item] of Object.entries(source)) { + const redacted = redactTranscriptStructuredValue(item, cfg, key, seen); + if (redacted === item) { + continue; + } + next ??= { ...source }; + next[key] = redacted; + } + seen.delete(value); + return next ?? value; +} + +export function redactTranscriptMessage(message: AgentMessage, cfg?: OpenClawConfig): AgentMessage { + if (cfg?.logging?.redactSensitive === "off") { + return message; + } + return redactTranscriptStructuredValue(message, cfg) as AgentMessage; +} diff --git a/src/config/sessions/transcript-append-redact.test.ts b/src/config/sessions/transcript-append-redact.test.ts new file mode 100644 index 00000000000..d2b1904b329 --- /dev/null +++ b/src/config/sessions/transcript-append-redact.test.ts @@ -0,0 +1,490 @@ +import fs from "node:fs"; +import path from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; +import { resolveSessionTranscriptPathInDir } from "./paths.js"; +import { useTempSessionsFixture } from "./test-helpers.js"; +import { appendSessionTranscriptMessage } from "./transcript-append.js"; +import { + appendAssistantMessageToSessionTranscript, + appendExactAssistantMessageToSessionTranscript, +} from "./transcript.js"; + +const readLoggingConfig = vi.hoisted(() => vi.fn()); + +vi.mock("../../logging/config.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readLoggingConfig, + }; +}); + +const EMAIL_PATTERN = String.raw`([\w]|[-.])+@([\w]|[-.])+\.\w+`; + +function readMessages(sessionFile: string) { + return fs + .readFileSync(sessionFile, "utf-8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as { type?: string; message?: unknown }) + .filter((r) => r.type === "message") + .map((r) => r.message); +} + +describe("appendSessionTranscriptMessage - redaction", () => { + const fixture = useTempSessionsFixture("transcript-redact-test-"); + + beforeEach(() => { + readLoggingConfig.mockReset(); + readLoggingConfig.mockReturnValue(undefined); + }); + + it("masks secrets in message content before writing to disk", async () => { + const sessionFile = resolveSessionTranscriptPathInDir("redact-on", fixture.sessionsDir()); + const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { + role: "user", + content: [{ type: "text", text: "my key is sk-abcdef1234567890xyz ok" }], + }, + config, + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).not.toContain("sk-abcdef1234567890xyz"); + expect(raw).toContain("ok"); // safe text preserved + + const [msg] = readMessages(sessionFile) as Array<{ + content: Array<{ text: string }>; + }>; + expect(msg.content[0].text).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("writes content unchanged when redactSensitive is off", async () => { + const sessionFile = resolveSessionTranscriptPathInDir("redact-off", fixture.sessionsDir()); + const config: OpenClawConfig = { logging: { redactSensitive: "off" } }; + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { + role: "user", + content: [{ type: "text", text: "my key is sk-abcdef1234567890xyz" }], + }, + config, + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).toContain("sk-abcdef1234567890xyz"); + }); + + it("masks secrets when config is undefined (default patterns)", async () => { + const sessionFile = resolveSessionTranscriptPathInDir("redact-undef", fixture.sessionsDir()); + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { + role: "user", + content: [{ type: "text", text: "my key is sk-abcdef1234567890xyz" }], + }, + // config intentionally omitted + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).not.toContain("sk-abcdef1234567890xyz"); + }); + + it("masks secrets in string payloads without role before writing to disk", async () => { + const sessionFile = resolveSessionTranscriptPathInDir( + "redact-string-payload", + fixture.sessionsDir(), + ); + const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: "my key is sk-abcdef1234567890xyz ok", + config, + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).not.toContain("sk-abcdef1234567890xyz"); + expect(raw).toContain("ok"); + + const [msg] = readMessages(sessionFile) as string[]; + expect(msg).not.toContain("sk-abcdef1234567890xyz"); + expect(msg).toContain("ok"); + }); + + it("masks secrets in structured payloads without role before writing to disk", async () => { + const sessionFile = resolveSessionTranscriptPathInDir( + "redact-structured-no-role", + fixture.sessionsDir(), + ); + const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { + apiKey: "plainsecretvalue123", + password: "hunter2", + nested: { accessToken: ["nestedplainsecret123"] }, + command: "OPENAI_API_KEY=sk-abcdef1234567890xyz openclaw health", + safe: "visible", + }, + config, + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).not.toContain("plainsecretvalue123"); + expect(raw).not.toContain("hunter2"); + expect(raw).not.toContain("nestedplainsecret123"); + expect(raw).not.toContain("sk-abcdef1234567890xyz"); + expect(raw).toContain("visible"); + + const [msg] = readMessages(sessionFile) as Array<{ + apiKey: string; + password: string; + nested: { accessToken: string[] }; + command: string; + safe: string; + }>; + expect(msg.apiKey).toBe("plains…e123"); + expect(msg.password).toBe("***"); + expect(msg.nested.accessToken[0]).toBe("nested…t123"); + expect(msg.command).toBe("OPENAI_API_KEY=sk-abc…0xyz openclaw health"); + expect(msg.safe).toBe("visible"); + }); + + it("uses configured custom patterns when cfg omits logging", async () => { + const sessionFile = resolveSessionTranscriptPathInDir( + "redact-config-pattern-fallback", + fixture.sessionsDir(), + ); + readLoggingConfig.mockReturnValue({ + redactSensitive: "tools", + redactPatterns: [EMAIL_PATTERN], + }); + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { + role: "user", + content: [{ type: "text", text: "email peter@dc.io and key sk-abcdef1234567890xyz ok" }], + }, + config: { + session: { + writeLock: { + acquireTimeoutMs: 25_000, + }, + }, + }, + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).not.toContain("peter@dc.io"); + expect(raw).not.toContain("sk-abcdef1234567890xyz"); + expect(raw).toContain("ok"); + }); + + it("masks secrets in assistant tool-call arguments before writing to disk", async () => { + const sessionFile = resolveSessionTranscriptPathInDir( + "redact-tool-call-args", + fixture.sessionsDir(), + ); + const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_1", + name: "shell", + arguments: { + command: "OPENAI_API_KEY=sk-abcdef1234567890xyz openclaw health", + env: { nested: ["token sk-abcdef1234567890xyz"] }, + apiKey: "plainsecretvalue123", + password: "hunter2", + }, + }, + ], + }, + config, + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).not.toContain("sk-abcdef1234567890xyz"); + expect(raw).not.toContain("plainsecretvalue123"); + expect(raw).not.toContain("hunter2"); + expect(raw).toContain("OPENAI_API_KEY=sk-abc…0xyz openclaw health"); + expect(raw).toContain("openclaw health"); + + const [msg] = readMessages(sessionFile) as Array<{ + content: Array<{ + arguments: { + command: string; + env: { nested: string[] }; + apiKey: string; + password: string; + }; + }>; + }>; + expect(JSON.stringify(msg.content[0].arguments)).not.toContain("sk-abcdef1234567890xyz"); + expect(msg.content[0].arguments.command).toBe("OPENAI_API_KEY=sk-abc…0xyz openclaw health"); + expect(msg.content[0].arguments.env.nested[0]).toBe("token sk-abc…0xyz"); + expect(msg.content[0].arguments.apiKey).toBe("plains…e123"); + expect(msg.content[0].arguments.password).toBe("***"); + }); + + it("masks secrets in tool-result details before writing to disk", async () => { + const sessionFile = resolveSessionTranscriptPathInDir( + "redact-tool-result-details", + fixture.sessionsDir(), + ); + const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; + + await appendSessionTranscriptMessage({ + transcriptPath: sessionFile, + message: { + role: "toolResult", + toolCallId: "call_1", + toolName: "send_request", + content: [{ type: "text", text: "result sk-abcdef1234567890xyz" }], + details: { + apiKey: "plainsecretvalue123", + password: "hunter2", + nested: { accessToken: ["nestedplainsecret123"] }, + safe: "visible", + }, + isError: false, + timestamp: Date.now(), + }, + config, + }); + + const raw = fs.readFileSync(sessionFile, "utf-8"); + expect(raw).not.toContain("sk-abcdef1234567890xyz"); + expect(raw).not.toContain("plainsecretvalue123"); + expect(raw).not.toContain("hunter2"); + expect(raw).not.toContain("nestedplainsecret123"); + expect(raw).toContain("visible"); + + const [msg] = readMessages(sessionFile) as Array<{ + content: Array<{ text: string }>; + details: { + apiKey: string; + password: string; + nested: { accessToken: string[] }; + safe: string; + }; + }>; + expect(msg.content[0].text).not.toContain("sk-abcdef1234567890xyz"); + expect(JSON.stringify(msg.details)).not.toContain("plainsecretvalue123"); + expect(msg.details.apiKey).toBe("plains…e123"); + expect(msg.details.password).toBe("***"); + expect(msg.details.nested.accessToken[0]).toBe("nested…t123"); + }); +}); + +describe("appendExactAssistantMessageToSessionTranscript - redaction", () => { + const fixture = useTempSessionsFixture("exact-assistant-redact-test-"); + + it("does not redact when config.logging.redactSensitive is off", async () => { + // Set up a minimal session store so the function can resolve the session file. + const sessionsDir = fixture.sessionsDir(); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionId = "test-session-redact-off"; + const sessionKey = "test-channel:test-user"; + const store = { + [sessionKey]: { sessionId, updatedAt: Date.now() }, + }; + fs.writeFileSync(storePath, JSON.stringify(store, null, 2), { encoding: "utf-8", mode: 0o600 }); + + const fakeApiKey = "sk-proj-FAKEKEYFORTESTINGONLY1234567890"; + const config: OpenClawConfig = { logging: { redactSensitive: "off" } }; + + const result = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath, + config, + message: { + role: "assistant", + content: [{ type: "text", text: `Here is your key: ${fakeApiKey}` }], + api: "openai-responses", + provider: "openclaw", + model: "test-model", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) { + return; + } + + const raw = fs.readFileSync(result.sessionFile, "utf-8"); + expect(raw).toContain(fakeApiKey); + }); + + it("emits the redacted assistant message for inline transcript updates", async () => { + const sessionsDir = fixture.sessionsDir(); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionId = "test-session-redact-event"; + const sessionKey = "test-channel:test-redact-event"; + fs.writeFileSync( + storePath, + JSON.stringify({ [sessionKey]: { sessionId, updatedAt: Date.now() } }, null, 2), + { encoding: "utf-8", mode: 0o600 }, + ); + + const fakeApiKey = "sk-proj-FAKEKEYFORTESTINGONLY1234567890"; + const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; + const updates: Array<{ message?: unknown }> = []; + const unsubscribe = onSessionTranscriptUpdate((update) => updates.push(update)); + + try { + const result = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath, + config, + message: { + role: "assistant", + content: [{ type: "text", text: `Here is your key: ${fakeApiKey}` }], + api: "openai-responses", + provider: "openclaw", + model: "test-model", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }, + }); + + expect(result.ok).toBe(true); + if (!result.ok) { + return; + } + + const [diskMessage] = readMessages(result.sessionFile); + expect(JSON.stringify(diskMessage)).not.toContain(fakeApiKey); + expect(updates).toHaveLength(1); + expect(updates[0]?.message).toEqual(diskMessage); + expect(JSON.stringify(updates[0]?.message)).not.toContain(fakeApiKey); + } finally { + unsubscribe(); + } + }); + + it("dedupes delivery mirrors against the redacted persisted text", async () => { + const sessionsDir = fixture.sessionsDir(); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionId = "test-session-redact-dedupe"; + const sessionKey = "test-channel:test-redact-dedupe"; + fs.writeFileSync( + storePath, + JSON.stringify({ [sessionKey]: { sessionId, updatedAt: Date.now() } }, null, 2), + { encoding: "utf-8", mode: 0o600 }, + ); + + const fakeApiKey = "sk-proj-FAKEKEYFORTESTINGONLY1234567890"; + const config: OpenClawConfig = { logging: { redactSensitive: "tools" } }; + + const first = await appendAssistantMessageToSessionTranscript({ + sessionKey, + storePath, + config, + text: `Here is your key: ${fakeApiKey}`, + }); + const second = await appendAssistantMessageToSessionTranscript({ + sessionKey, + storePath, + config, + text: `Here is your key: ${fakeApiKey}`, + }); + + expect(first.ok).toBe(true); + expect(second.ok).toBe(true); + if (!first.ok || !second.ok) { + return; + } + expect(second.messageId).toBe(first.messageId); + + const raw = fs.readFileSync(second.sessionFile, "utf-8"); + expect(raw).not.toContain(fakeApiKey); + expect(readMessages(second.sessionFile)).toHaveLength(1); + }); + + it("dedupes delivery mirrors against older unredacted assistant entries", async () => { + const sessionsDir = fixture.sessionsDir(); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionId = "test-session-redact-upgrade-dedupe"; + const sessionKey = "test-channel:test-redact-upgrade-dedupe"; + fs.writeFileSync( + storePath, + JSON.stringify({ [sessionKey]: { sessionId, updatedAt: Date.now() } }, null, 2), + { encoding: "utf-8", mode: 0o600 }, + ); + + const fakeApiKey = "sk-proj-OLDERUNREDACTEDTRANSCRIPT1234567890"; + const unredacted = await appendExactAssistantMessageToSessionTranscript({ + sessionKey, + storePath, + config: { logging: { redactSensitive: "off" } }, + message: { + role: "assistant", + content: [{ type: "text", text: `Here is your key: ${fakeApiKey}` }], + api: "openai-responses", + provider: "openclaw", + model: "legacy-assistant", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + }, + }); + const deduped = await appendAssistantMessageToSessionTranscript({ + sessionKey, + storePath, + config: { logging: { redactSensitive: "tools" } }, + text: `Here is your key: ${fakeApiKey}`, + }); + + expect(unredacted.ok).toBe(true); + expect(deduped.ok).toBe(true); + if (!unredacted.ok || !deduped.ok) { + return; + } + expect(deduped.messageId).toBe(unredacted.messageId); + + const raw = fs.readFileSync(deduped.sessionFile, "utf-8"); + expect(raw).toContain(fakeApiKey); + expect(readMessages(deduped.sessionFile)).toHaveLength(1); + }); +}); diff --git a/src/config/sessions/transcript-append.ts b/src/config/sessions/transcript-append.ts index 4abb19c7444..498c38763a1 100644 --- a/src/config/sessions/transcript-append.ts +++ b/src/config/sessions/transcript-append.ts @@ -2,11 +2,13 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { StringDecoder } from "node:string_decoder"; +import type { AgentMessage } from "@earendil-works/pi-agent-core"; import { acquireSessionWriteLock, - type SessionWriteLockAcquireTimeoutConfig, resolveSessionWriteLockAcquireTimeoutMs, } from "../../agents/session-write-lock.js"; +import { redactTranscriptMessage } from "../../agents/transcript-redact.js"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { redactSecrets } from "../../logging/redact.js"; const TRANSCRIPT_APPEND_SCAN_CHUNK_BYTES = 64 * 1024; @@ -230,29 +232,36 @@ async function withTranscriptAppendQueue( } } -export async function appendSessionTranscriptMessage(params: { +type AppendSessionTranscriptMessageParams = { transcriptPath: string; - message: unknown; + message: TMessage; now?: number; sessionId?: string; cwd?: string; useRawWhenLinear?: boolean; - config?: SessionWriteLockAcquireTimeoutConfig; -}): Promise<{ messageId: string }> { + config?: OpenClawConfig; +}; + +function isTranscriptAgentMessage(value: unknown): value is AgentMessage { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + typeof (value as { role?: unknown }).role === "string" + ); +} + +export async function appendSessionTranscriptMessage( + params: AppendSessionTranscriptMessageParams, +): Promise<{ messageId: string; message: TMessage }> { return await withTranscriptAppendQueue(params.transcriptPath, () => appendSessionTranscriptMessageLocked(params), ); } -async function appendSessionTranscriptMessageLocked(params: { - transcriptPath: string; - message: unknown; - now?: number; - sessionId?: string; - cwd?: string; - useRawWhenLinear?: boolean; - config?: SessionWriteLockAcquireTimeoutConfig; -}): Promise<{ messageId: string }> { +async function appendSessionTranscriptMessageLocked( + params: AppendSessionTranscriptMessageParams, +): Promise<{ messageId: string; message: TMessage }> { const lock = await acquireSessionWriteLock({ sessionFile: params.transcriptPath, timeoutMs: resolveSessionWriteLockAcquireTimeoutMs(params.config), @@ -286,15 +295,20 @@ async function appendSessionTranscriptMessageLocked(params: { nonSessionEntryCount: leafInfo.nonSessionEntryCount, }; } + const finalMessage = ( + isTranscriptAgentMessage(params.message) + ? redactTranscriptMessage(params.message, params.config) + : redactSecrets(params.message) + ) as TMessage; const entry = { type: "message", id: messageId, ...(shouldRawAppend ? {} : { parentId: leafInfo.leafId ?? null }), timestamp: new Date(now).toISOString(), - message: redactSecrets(params.message), + message: finalMessage, }; await fs.appendFile(params.transcriptPath, `${JSON.stringify(entry)}\n`, "utf-8"); - return { messageId }; + return { messageId, message: finalMessage }; } finally { await lock.release(); } diff --git a/src/config/sessions/transcript.ts b/src/config/sessions/transcript.ts index de5f0bc5803..950c389156f 100644 --- a/src/config/sessions/transcript.ts +++ b/src/config/sessions/transcript.ts @@ -1,10 +1,12 @@ import fs from "node:fs"; import path from "node:path"; +import type { AgentMessage } from "@earendil-works/pi-agent-core"; import type { SessionManager } from "@earendil-works/pi-coding-agent"; -import type { SessionWriteLockAcquireTimeoutConfig } from "../../agents/session-write-lock.js"; +import { redactTranscriptMessage } from "../../agents/transcript-redact.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { emitSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { extractAssistantVisibleText } from "../../shared/chat-message-content.js"; +import type { OpenClawConfig } from "../types.openclaw.js"; import { resolveDefaultSessionStorePath, resolveSessionFilePath, @@ -186,7 +188,7 @@ export async function appendAssistantMessageToSessionTranscript(params: { /** Optional override for store path (mostly for tests). */ storePath?: string; updateMode?: SessionTranscriptUpdateMode; - config?: SessionWriteLockAcquireTimeoutConfig; + config?: OpenClawConfig; }): Promise { const sessionKey = params.sessionKey.trim(); if (!sessionKey) { @@ -241,7 +243,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { idempotencyKey?: string; storePath?: string; updateMode?: SessionTranscriptUpdateMode; - config?: SessionWriteLockAcquireTimeoutConfig; + config?: OpenClawConfig; }): Promise { const sessionKey = params.sessionKey.trim(); if (!sessionKey) { @@ -295,7 +297,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { } const latestEquivalentAssistantId = isRedundantDeliveryMirror(params.message) - ? await findLatestEquivalentAssistantMessageId(sessionFile, params.message) + ? await findLatestEquivalentAssistantMessageId(sessionFile, params.message, params.config) : undefined; if (latestEquivalentAssistantId) { return { ok: true, sessionFile, messageId: latestEquivalentAssistantId }; @@ -305,7 +307,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { ...params.message, ...(explicitIdempotencyKey ? { idempotencyKey: explicitIdempotencyKey } : {}), } as Parameters[0]; - const { messageId } = await appendSessionTranscriptMessage({ + const { messageId, message: appendedMessage } = await appendSessionTranscriptMessage({ transcriptPath: sessionFile, message, config: params.config, @@ -313,7 +315,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: { switch (params.updateMode ?? "inline") { case "inline": - emitSessionTranscriptUpdate({ sessionFile, sessionKey, message, messageId }); + emitSessionTranscriptUpdate({ sessionFile, sessionKey, message: appendedMessage, messageId }); break; case "file-only": emitSessionTranscriptUpdate({ sessionFile, sessionKey }); @@ -381,8 +383,11 @@ function extractAssistantMessageText(message: SessionTranscriptAssistantMessage) async function findLatestEquivalentAssistantMessageId( transcriptPath: string, message: SessionTranscriptAssistantMessage, + config?: OpenClawConfig, ): Promise { - const expectedText = extractAssistantMessageText(message); + const expectedText = extractAssistantMessageText( + redactTranscriptMessage(message, config) as unknown as SessionTranscriptAssistantMessage, + ); if (!expectedText) { return undefined; } @@ -397,7 +402,12 @@ async function findLatestEquivalentAssistantMessageId( if (!candidate || candidate.role !== "assistant") { continue; } - const candidateText = extractAssistantMessageText(candidate); + const candidateText = extractAssistantMessageText( + redactTranscriptMessage( + candidate as AgentMessage, + config, + ) as unknown as SessionTranscriptAssistantMessage, + ); if (candidateText !== expectedText) { return undefined; } diff --git a/src/gateway/server-methods/chat-transcript-inject.ts b/src/gateway/server-methods/chat-transcript-inject.ts index a04187a1cf3..00cade5d730 100644 --- a/src/gateway/server-methods/chat-transcript-inject.ts +++ b/src/gateway/server-methods/chat-transcript-inject.ts @@ -1,6 +1,6 @@ import type { SessionManager } from "@earendil-works/pi-coding-agent"; -import type { SessionWriteLockAcquireTimeoutConfig } from "../../agents/session-write-lock.js"; import { appendSessionTranscriptMessage } from "../../config/sessions/transcript-append.js"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { emitSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; @@ -52,7 +52,7 @@ export async function appendInjectedAssistantMessageToTranscript(params: { idempotencyKey?: string; abortMeta?: GatewayInjectedAbortMeta; now?: number; - config?: SessionWriteLockAcquireTimeoutConfig; + config?: OpenClawConfig; }): Promise { const now = params.now ?? Date.now(); const usage = { @@ -103,7 +103,7 @@ export async function appendInjectedAssistantMessageToTranscript(params: { }; try { - const { messageId } = await appendSessionTranscriptMessage({ + const { messageId, message: appendedMessage } = await appendSessionTranscriptMessage({ transcriptPath: params.transcriptPath, message: messageBody, now, @@ -112,10 +112,10 @@ export async function appendInjectedAssistantMessageToTranscript(params: { }); emitSessionTranscriptUpdate({ sessionFile: params.transcriptPath, - message: messageBody, + message: appendedMessage, messageId, }); - return { ok: true, messageId, message: messageBody }; + return { ok: true, messageId, message: appendedMessage as unknown as Record }; } catch (err) { return { ok: false, error: formatErrorMessage(err) }; } diff --git a/src/gateway/server-methods/chat.inject.parentid.test.ts b/src/gateway/server-methods/chat.inject.parentid.test.ts index 7fc0f3b76d6..cc1db17ea37 100644 --- a/src/gateway/server-methods/chat.inject.parentid.test.ts +++ b/src/gateway/server-methods/chat.inject.parentid.test.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import { describe, expect, it } from "vitest"; +import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { appendInjectedAssistantMessageToTranscript } from "./chat-transcript-inject.js"; import { createTranscriptFixtureSync } from "./chat.test-helpers.js"; @@ -93,4 +94,35 @@ describe("gateway chat.inject transcript writes", () => { fs.rmSync(dir, { recursive: true, force: true }); } }); + + it("emits and returns the redacted injected assistant message", async () => { + const { dir, transcriptPath } = createTranscriptFixtureSync({ + prefix: "openclaw-chat-inject-redact-", + sessionId: "sess-redact", + }); + const fakeApiKey = "sk-proj-FAKEKEYFORTESTINGONLY1234567890"; + const updates: Array<{ message?: unknown }> = []; + const unsubscribe = onSessionTranscriptUpdate((update) => updates.push(update)); + + try { + const appended = await appendInjectedAssistantMessageToTranscript({ + transcriptPath, + message: `Here is your key: ${fakeApiKey}`, + config: { logging: { redactSensitive: "tools" } }, + }); + + expect(appended.ok).toBe(true); + expect(JSON.stringify(appended.message)).not.toContain(fakeApiKey); + expect(updates).toHaveLength(1); + + const lines = readTranscriptLines(transcriptPath); + const last = JSON.parse(lines.at(-1) as string) as { message?: unknown }; + expect(JSON.stringify(last.message)).not.toContain(fakeApiKey); + expect(updates[0]?.message).toEqual(last.message); + expect(JSON.stringify(updates[0]?.message)).not.toContain(fakeApiKey); + } finally { + unsubscribe(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/src/logging/redact.ts b/src/logging/redact.ts index 44698030d31..52bc94d69c7 100644 --- a/src/logging/redact.ts +++ b/src/logging/redact.ts @@ -4,7 +4,7 @@ import { readLoggingConfig } from "./config.js"; import { replacePatternBounded } from "./redact-bounded.js"; export type RedactSensitiveMode = "off" | "tools"; -type RedactPattern = string | RegExp; +export type RedactPattern = string | RegExp; type LoggingConfig = OpenClawConfig["logging"]; const DEFAULT_REDACT_MODE: RedactSensitiveMode = "tools"; @@ -85,7 +85,7 @@ const DEFAULT_REDACT_PATTERNS: string[] = [ String.raw`\b(\d{6,}:[A-Za-z0-9_-]{20,})\b`, ]; -type RedactOptions = { +export type RedactOptions = { mode?: RedactSensitiveMode; patterns?: RedactPattern[]; }; @@ -253,7 +253,11 @@ function redactSensitiveFieldValueWithOptions( value: string, options: RedactOptions, ): string { - const redacted = redactSensitiveText(value, options); + const resolved = resolveRedactOptions(options); + if (resolved.mode === "off") { + return value; + } + const redacted = redactText(value, resolved.patterns); const shouldRedactAppPassword = redacted !== value || STRUCTURED_APP_PASSWORD_FIELD_RE.test(key); if (shouldRedactAppPassword) { const appRedacted = redactAppSpecificPasswords(redacted); @@ -270,8 +274,12 @@ function redactSensitiveFieldValueWithOptions( return value; } -export function redactSensitiveFieldValue(key: string, value: string): string { - return redactSensitiveFieldValueWithConfig(key, value, readLoggingConfig()); +export function redactSensitiveFieldValue( + key: string, + value: string, + options?: RedactOptions, +): string { + return redactSensitiveFieldValueWithOptions(key, value, options ?? resolveToolPayloadRedaction()); } export function redactSensitiveFieldValueWithConfig(