diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0e69fe21c..0407d03ad03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -767,6 +767,7 @@ Docs: https://docs.openclaw.ai - Discord/groups: tell Discord-channel agents to wrap bare URLs as `` so link previews do not expand into uninvited embeds. (#78614) - Agents/fallback: fail fast on session write-lock timeouts instead of trying fallback models for local file contention. Fixes #66646. Thanks @sallyom. - Browser/SSRF: stop closing user-owned Chrome tabs when a read-only operation (snapshot/screenshot/interactions) is rejected by the SSRF guard — only OpenClaw-initiated navigations now close on policy denial. Thanks @scotthuang. +- iMessage: stage native inbound attachments into OpenClaw-managed media and convert HEIC/HEIF images to JPEG before dispatch, so image tools can read photos sent over native iMessage without requiring BlueBubbles. - Agents/Gateway: throttle and cap live exec command-output events so noisy tool runs cannot flood Gateway WebSocket clients or starve RPC handling. (#78645) Thanks @joshavant. - Memory Wiki: skip empty and whitespace-only source pages when refreshing generated Related blocks, preventing blank pages from being rewritten into Related-only stubs. Fixes #78121. Thanks @amknight. - Telegram: keep duplicate message-tool-only Codex turns from posting generic silent-reply fallback text, so private finals stay private after inbound dedupe. Thanks @rubencu. diff --git a/extensions/imessage/src/monitor.media-policy.test.ts b/extensions/imessage/src/monitor.media-policy.test.ts new file mode 100644 index 00000000000..85113e6e8df --- /dev/null +++ b/extensions/imessage/src/monitor.media-policy.test.ts @@ -0,0 +1,115 @@ +import type { waitForTransportReady } from "openclaw/plugin-sdk/transport-ready-runtime"; +import { describe, expect, it, vi } from "vitest"; +import type { createIMessageRpcClient } from "./client.js"; +import { monitorIMessageProvider } from "./monitor.js"; +import type { stageIMessageAttachments } from "./monitor/media-staging.js"; + +const waitForTransportReadyMock = vi.hoisted(() => + vi.fn(async () => {}), +); +const createIMessageRpcClientMock = vi.hoisted(() => vi.fn()); +const stageIMessageAttachmentsMock = vi.hoisted(() => vi.fn()); +const readChannelAllowFromStoreMock = vi.hoisted(() => vi.fn(async () => [] as string[])); + +vi.mock("openclaw/plugin-sdk/transport-ready-runtime", () => ({ + waitForTransportReady: waitForTransportReadyMock, +})); + +vi.mock("openclaw/plugin-sdk/conversation-runtime", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readChannelAllowFromStore: readChannelAllowFromStoreMock, + recordInboundSession: vi.fn(), + upsertChannelPairingRequest: vi.fn(), + }; +}); + +vi.mock("openclaw/plugin-sdk/channel-inbound", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createChannelInboundDebouncer: vi.fn((opts) => ({ + debouncer: { + enqueue: async (entry: unknown) => await opts.onFlush([entry]), + }, + })), + shouldDebounceTextInbound: vi.fn(() => false), + }; +}); + +vi.mock("./client.js", () => ({ + createIMessageRpcClient: createIMessageRpcClientMock, +})); + +vi.mock("./monitor/abort-handler.js", () => ({ + attachIMessageMonitorAbortHandler: vi.fn(() => () => {}), +})); + +vi.mock("./monitor/media-staging.js", () => ({ + stageIMessageAttachments: stageIMessageAttachmentsMock, +})); + +describe("iMessage monitor attachment policy", () => { + it("does not stage local attachments for messages dropped by inbound policy", async () => { + stageIMessageAttachmentsMock.mockResolvedValue([]); + readChannelAllowFromStoreMock.mockResolvedValue([]); + + const attachmentPath = "/Users/openclaw/Library/Messages/Attachments/AA/BB/photo.heic"; + let onNotification: ((message: { method: string; params: unknown }) => void) | undefined; + const client = { + request: vi.fn(async () => ({ subscription: 1 })), + waitForClose: vi.fn(async () => { + onNotification?.({ + method: "message", + params: { + message: { + id: 1, + chat_id: 123, + sender: "+15550001111", + is_from_me: false, + is_group: true, + text: "no mention here", + attachments: [ + { + original_path: attachmentPath, + mime_type: "image/heic", + missing: false, + }, + ], + }, + }, + }); + await Promise.resolve(); + await Promise.resolve(); + }), + stop: vi.fn(async () => {}), + }; + createIMessageRpcClientMock.mockImplementation(async (params) => { + if (!params?.onNotification) { + throw new Error("expected iMessage notification handler"); + } + onNotification = params.onNotification; + return client as never; + }); + + await monitorIMessageProvider({ + config: { + channels: { + imessage: { + includeAttachments: true, + attachmentRoots: ["/Users/*/Library/Messages/Attachments"], + dmPolicy: "open", + groupPolicy: "open", + groups: { "*": { requireMention: true } }, + }, + }, + messages: { groupChat: { mentionPatterns: ["@openclaw"] } }, + session: { mainKey: "main" }, + } as never, + }); + + expect(readChannelAllowFromStoreMock).toHaveBeenCalled(); + expect(stageIMessageAttachmentsMock).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/imessage/src/monitor/media-staging.test.ts b/extensions/imessage/src/monitor/media-staging.test.ts new file mode 100644 index 00000000000..8d93e1961fd --- /dev/null +++ b/extensions/imessage/src/monitor/media-staging.test.ts @@ -0,0 +1,120 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { stageIMessageAttachments } from "./media-staging.js"; + +let tempDir: string; + +async function writeTempFile(name: string, contents: Buffer | string): Promise { + const filePath = path.join(tempDir, name); + await fs.writeFile(filePath, contents); + return filePath; +} + +describe("stageIMessageAttachments", () => { + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-imessage-media-")); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it("copies allowed iMessage attachments into the inbound media store", async () => { + const sourcePath = await writeTempFile("photo.png", Buffer.from("png-bytes")); + const saveMediaBuffer = vi.fn(async () => ({ + id: "saved.png", + path: "/state/media/inbound/saved.png", + size: 9, + contentType: "image/png", + })); + + await expect( + stageIMessageAttachments( + [{ original_path: sourcePath, mime_type: "image/png", missing: false }], + { maxBytes: 1024, allowedRoots: [tempDir], deps: { saveMediaBuffer } }, + ), + ).resolves.toEqual([{ path: "/state/media/inbound/saved.png", contentType: "image/png" }]); + + expect(saveMediaBuffer).toHaveBeenCalledWith( + Buffer.from("png-bytes"), + "image/png", + "inbound", + 1024, + "photo.png", + ); + }); + + it("drops attachments whose canonical path escapes the allowed root", async () => { + const allowedRoot = path.join(tempDir, "allowed"); + const outsideRoot = path.join(tempDir, "outside"); + await fs.mkdir(allowedRoot, { recursive: true }); + await fs.mkdir(outsideRoot, { recursive: true }); + const outsidePath = path.join(outsideRoot, "secret.png"); + await fs.writeFile(outsidePath, Buffer.from("secret-bytes")); + await fs.symlink(outsideRoot, path.join(allowedRoot, "link"), "dir"); + + const saveMediaBuffer = vi.fn(); + const logVerbose = vi.fn(); + + await expect( + stageIMessageAttachments( + [ + { + original_path: path.join(allowedRoot, "link", "secret.png"), + mime_type: "image/png", + missing: false, + }, + ], + { maxBytes: 1024, allowedRoots: [allowedRoot], deps: { saveMediaBuffer, logVerbose } }, + ), + ).resolves.toEqual([]); + + expect(saveMediaBuffer).not.toHaveBeenCalled(); + expect(logVerbose).toHaveBeenCalledWith( + expect.stringContaining("attachment path resolves outside allowed roots"), + ); + }); + + it("converts HEIC iMessage attachments to JPEG before staging", async () => { + const sourcePath = await writeTempFile("IMG_0001.HEIC", Buffer.from("heic-bytes")); + const saveMediaBuffer = vi.fn(async () => ({ + id: "saved.jpg", + path: "/state/media/inbound/saved.jpg", + size: 10, + contentType: "image/jpeg", + })); + const convertHeicToJpeg = vi.fn(async () => Buffer.from("jpeg-bytes")); + + await stageIMessageAttachments( + [{ original_path: sourcePath, mime_type: "image/heic", missing: false }], + { maxBytes: 1024, deps: { saveMediaBuffer, convertHeicToJpeg } }, + ); + + expect(convertHeicToJpeg).toHaveBeenCalledWith(sourcePath, 1024); + expect(saveMediaBuffer).toHaveBeenCalledWith( + Buffer.from("jpeg-bytes"), + "image/jpeg", + "inbound", + 1024, + "IMG_0001.jpg", + ); + }); + + it("drops attachments over the inbound media limit", async () => { + const sourcePath = await writeTempFile("huge.png", Buffer.from("too large")); + const saveMediaBuffer = vi.fn(); + const logVerbose = vi.fn(); + + await expect( + stageIMessageAttachments( + [{ original_path: sourcePath, mime_type: "image/png", missing: false }], + { maxBytes: 4, deps: { saveMediaBuffer, logVerbose } }, + ), + ).resolves.toEqual([]); + + expect(saveMediaBuffer).not.toHaveBeenCalled(); + expect(logVerbose).toHaveBeenCalledWith(expect.stringContaining("failed to stage")); + }); +}); diff --git a/extensions/imessage/src/monitor/media-staging.ts b/extensions/imessage/src/monitor/media-staging.ts new file mode 100644 index 00000000000..08db64d2c0e --- /dev/null +++ b/extensions/imessage/src/monitor/media-staging.ts @@ -0,0 +1,205 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { isInboundPathAllowed } from "openclaw/plugin-sdk/media-runtime"; +import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store"; +import { buildRandomTempFilePath } from "openclaw/plugin-sdk/temp-path"; +import type { IMessageAttachment } from "./types.js"; + +const execFileAsync = promisify(execFile); + +const HEIC_CONVERSION_TIMEOUT_MS = 15_000; +const HEIC_CONVERSION_MAX_BUFFER_BYTES = 64 * 1024; + +export type StagedIMessageAttachment = { + path: string; + contentType?: string; +}; + +type SaveMediaBufferImpl = typeof saveMediaBuffer; + +type StageIMessageAttachmentsDeps = { + saveMediaBuffer?: SaveMediaBufferImpl; + convertHeicToJpeg?: (sourcePath: string, maxBytes: number) => Promise; + logVerbose?: (message: string) => void; +}; + +function isHeicAttachment(attachmentPath: string, mimeType?: string | null): boolean { + const normalizedMime = mimeType?.toLowerCase(); + if (normalizedMime === "image/heic" || normalizedMime === "image/heif") { + return true; + } + const ext = path.extname(attachmentPath).toLowerCase(); + return ext === ".heic" || ext === ".heif"; +} + +function jpegFilenameForAttachment(attachmentPath: string): string { + const parsed = path.parse(attachmentPath); + return `${parsed.name || "imessage-attachment"}.jpg`; +} + +function hasWildcardSegment(root: string): boolean { + return root.replaceAll("\\", "/").split("/").includes("*"); +} + +async function canonicalizeAllowedRoots(roots: readonly string[]): Promise { + const canonicalRoots: string[] = []; + for (const root of roots) { + canonicalRoots.push(root); + if (hasWildcardSegment(root)) { + continue; + } + const canonicalRoot = await fs.realpath(root).catch(() => undefined); + if (canonicalRoot && canonicalRoot !== root) { + canonicalRoots.push(canonicalRoot); + } + } + return canonicalRoots; +} + +async function resolveAllowedCanonicalAttachmentPath(params: { + attachmentPath: string; + allowedRoots?: readonly string[]; +}): Promise { + if (!params.allowedRoots) { + return params.attachmentPath; + } + const canonicalPath = await fs.realpath(params.attachmentPath); + const canonicalRoots = await canonicalizeAllowedRoots(params.allowedRoots); + if (!isInboundPathAllowed({ filePath: canonicalPath, roots: canonicalRoots })) { + throw new Error("attachment path resolves outside allowed roots"); + } + return canonicalPath; +} + +async function convertHeicToJpegWithSips(sourcePath: string, maxBytes: number): Promise { + const tempPath = buildRandomTempFilePath({ + prefix: "openclaw-imessage", + extension: "jpg", + }); + try { + await execFileAsync( + "sips", + [ + "-s", + "format", + "jpeg", + "-s", + "formatOptions", + "90", + "-Z", + "4096", + sourcePath, + "--out", + tempPath, + ], + { + timeout: HEIC_CONVERSION_TIMEOUT_MS, + maxBuffer: HEIC_CONVERSION_MAX_BUFFER_BYTES, + killSignal: "SIGKILL", + }, + ); + const stat = await fs.stat(tempPath); + if (stat.size > maxBytes) { + throw new Error(`converted media exceeds ${Math.round(maxBytes / (1024 * 1024))}MB limit`); + } + return await fs.readFile(tempPath); + } finally { + await fs.rm(tempPath, { force: true }).catch(() => {}); + } +} + +async function readAttachmentBuffer(params: { + attachmentPath: string; + mimeType?: string | null; + maxBytes: number; + allowedRoots?: readonly string[]; + deps: StageIMessageAttachmentsDeps; +}): Promise<{ buffer: Buffer; contentType?: string; originalFilename?: string }> { + const stat = await fs.lstat(params.attachmentPath); + if (stat.isSymbolicLink()) { + throw new Error("attachment path is a symlink"); + } + if (!stat.isFile()) { + throw new Error("attachment path is not a file"); + } + if (stat.size > params.maxBytes) { + throw new Error(`attachment exceeds ${Math.round(params.maxBytes / (1024 * 1024))}MB limit`); + } + + const canonicalPath = await resolveAllowedCanonicalAttachmentPath({ + attachmentPath: params.attachmentPath, + allowedRoots: params.allowedRoots, + }); + const canonicalStat = await fs.stat(canonicalPath); + if (!canonicalStat.isFile()) { + throw new Error("attachment path is not a file"); + } + if (canonicalStat.size > params.maxBytes) { + throw new Error(`attachment exceeds ${Math.round(params.maxBytes / (1024 * 1024))}MB limit`); + } + + if (isHeicAttachment(params.attachmentPath, params.mimeType)) { + try { + const convert = params.deps.convertHeicToJpeg ?? convertHeicToJpegWithSips; + return { + buffer: await convert(canonicalPath, params.maxBytes), + contentType: "image/jpeg", + originalFilename: jpegFilenameForAttachment(params.attachmentPath), + }; + } catch (err) { + params.deps.logVerbose?.( + `imessage: HEIC attachment conversion failed; staging original instead: ${String(err)}`, + ); + } + } + + return { + buffer: await fs.readFile(canonicalPath), + contentType: params.mimeType ?? undefined, + originalFilename: path.basename(params.attachmentPath), + }; +} + +export async function stageIMessageAttachments( + attachments: IMessageAttachment[], + params: { + maxBytes: number; + allowedRoots?: readonly string[]; + deps?: StageIMessageAttachmentsDeps; + }, +): Promise { + const deps = params.deps ?? {}; + const save = deps.saveMediaBuffer ?? saveMediaBuffer; + const staged: StagedIMessageAttachment[] = []; + + for (const attachment of attachments) { + const attachmentPath = attachment.original_path?.trim(); + if (!attachmentPath || attachment.missing) { + continue; + } + + try { + const media = await readAttachmentBuffer({ + attachmentPath, + mimeType: attachment.mime_type, + maxBytes: params.maxBytes, + allowedRoots: params.allowedRoots, + deps, + }); + const saved = await save( + media.buffer, + media.contentType, + "inbound", + params.maxBytes, + media.originalFilename, + ); + staged.push({ path: saved.path, contentType: saved.contentType }); + } catch (err) { + deps.logVerbose?.(`imessage: failed to stage inbound attachment: ${String(err)}`); + } + } + + return staged; +} diff --git a/extensions/imessage/src/monitor/monitor-provider.ts b/extensions/imessage/src/monitor/monitor-provider.ts index e99a9b48023..eb5c1c0326e 100644 --- a/extensions/imessage/src/monitor/monitor-provider.ts +++ b/extensions/imessage/src/monitor/monitor-provider.ts @@ -65,6 +65,7 @@ import { resolveIMessageInboundDecision, } from "./inbound-processing.js"; import { createLoopRateLimiter } from "./loop-rate-limiter.js"; +import { stageIMessageAttachments } from "./media-staging.js"; import { parseIMessageNotification } from "./parse-notification.js"; import { enqueueIMessageReactionSystemEvent } from "./reaction-system-event.js"; import { normalizeAllowList, resolveRuntime } from "./runtime.js"; @@ -371,13 +372,14 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P logVerbose(`imessage: dropping inbound attachment outside allowed roots: ${attachmentPath}`); return false; }); - const firstAttachment = validAttachments[0]; - const mediaPath = firstAttachment?.original_path ?? undefined; - const mediaType = firstAttachment?.mime_type ?? undefined; - // Build arrays for all attachments (for multi-image support) - const mediaPaths = validAttachments.map((a) => a.original_path).filter(Boolean) as string[]; - const mediaTypes = validAttachments.map((a) => a.mime_type ?? undefined); - const kind = kindFromMime(mediaType ?? undefined); + const rawMediaAttachments = validAttachments.flatMap((a) => { + const attachmentPath = a.original_path?.trim(); + return attachmentPath + ? [{ path: attachmentPath, contentType: a.mime_type ?? undefined }] + : []; + }); + const placeholderMediaType = rawMediaAttachments[0]?.contentType; + const kind = kindFromMime(placeholderMediaType ?? undefined); const placeholder = kind ? `` : validAttachments.length @@ -501,6 +503,20 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P const storePath = resolveStorePath(cfg.session?.store, { agentId: decision.route.agentId, }); + const stagedAttachments = remoteHost + ? [] + : await stageIMessageAttachments(validAttachments, { + maxBytes: mediaMaxBytes, + allowedRoots: effectiveAttachmentRoots, + deps: { logVerbose }, + }); + const mediaAttachments = remoteHost ? rawMediaAttachments : stagedAttachments; + const firstAttachment = mediaAttachments[0]; + const mediaPath = firstAttachment?.path ?? undefined; + const mediaType = firstAttachment?.contentType ?? undefined; + // Build arrays for all attachments (for multi-image support) + const mediaPaths = mediaAttachments.map((a) => a.path).filter(Boolean); + const mediaTypes = mediaAttachments.map((a) => a.contentType ?? undefined); const previousTimestamp = readSessionUpdatedAt({ storePath, sessionKey: decision.route.sessionKey, diff --git a/extensions/imessage/src/monitor/types.ts b/extensions/imessage/src/monitor/types.ts index 992130f8b3d..1a309716606 100644 --- a/extensions/imessage/src/monitor/types.ts +++ b/extensions/imessage/src/monitor/types.ts @@ -1,7 +1,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env"; -type IMessageAttachment = { +export type IMessageAttachment = { original_path?: string | null; mime_type?: string | null; missing?: boolean | null;