fix(imessage): make inbound image attachments readable by agents (#78580)

Stage native iMessage inbound attachments into managed media and convert HEIC/HEIF images to JPEG before dispatch. Thanks @homer-byte.
This commit is contained in:
homer-byte
2026-05-13 11:35:52 -04:00
committed by GitHub
parent 58591c37a4
commit 1d6e5f7a3e
6 changed files with 465 additions and 8 deletions

View File

@@ -767,6 +767,7 @@ Docs: https://docs.openclaw.ai
- Discord/groups: tell Discord-channel agents to wrap bare URLs as `<https://example.com>` so link previews do not expand into uninvited embeds. (#78614) - Discord/groups: tell Discord-channel agents to wrap bare URLs as `<https://example.com>` so link previews do not expand into uninvited embeds. (#78614)
- Agents/fallback: fail fast on session write-lock timeouts instead of trying fallback models for local file contention. Fixes #66646. Thanks @sallyom. - Agents/fallback: fail fast on session write-lock timeouts instead of trying fallback models for local file contention. Fixes #66646. Thanks @sallyom.
- Browser/SSRF: stop closing user-owned Chrome tabs when a read-only operation (snapshot/screenshot/interactions) is rejected by the SSRF guard — only OpenClaw-initiated navigations now close on policy denial. Thanks @scotthuang. - Browser/SSRF: stop closing user-owned Chrome tabs when a read-only operation (snapshot/screenshot/interactions) is rejected by the SSRF guard — only OpenClaw-initiated navigations now close on policy denial. Thanks @scotthuang.
- iMessage: stage native inbound attachments into OpenClaw-managed media and convert HEIC/HEIF images to JPEG before dispatch, so image tools can read photos sent over native iMessage without requiring BlueBubbles.
- Agents/Gateway: throttle and cap live exec command-output events so noisy tool runs cannot flood Gateway WebSocket clients or starve RPC handling. (#78645) Thanks @joshavant. - Agents/Gateway: throttle and cap live exec command-output events so noisy tool runs cannot flood Gateway WebSocket clients or starve RPC handling. (#78645) Thanks @joshavant.
- Memory Wiki: skip empty and whitespace-only source pages when refreshing generated Related blocks, preventing blank pages from being rewritten into Related-only stubs. Fixes #78121. Thanks @amknight. - Memory Wiki: skip empty and whitespace-only source pages when refreshing generated Related blocks, preventing blank pages from being rewritten into Related-only stubs. Fixes #78121. Thanks @amknight.
- Telegram: keep duplicate message-tool-only Codex turns from posting generic silent-reply fallback text, so private finals stay private after inbound dedupe. Thanks @rubencu. - Telegram: keep duplicate message-tool-only Codex turns from posting generic silent-reply fallback text, so private finals stay private after inbound dedupe. Thanks @rubencu.

View File

@@ -0,0 +1,115 @@
import type { waitForTransportReady } from "openclaw/plugin-sdk/transport-ready-runtime";
import { describe, expect, it, vi } from "vitest";
import type { createIMessageRpcClient } from "./client.js";
import { monitorIMessageProvider } from "./monitor.js";
import type { stageIMessageAttachments } from "./monitor/media-staging.js";
const waitForTransportReadyMock = vi.hoisted(() =>
vi.fn<typeof waitForTransportReady>(async () => {}),
);
const createIMessageRpcClientMock = vi.hoisted(() => vi.fn<typeof createIMessageRpcClient>());
const stageIMessageAttachmentsMock = vi.hoisted(() => vi.fn<typeof stageIMessageAttachments>());
const readChannelAllowFromStoreMock = vi.hoisted(() => vi.fn(async () => [] as string[]));
vi.mock("openclaw/plugin-sdk/transport-ready-runtime", () => ({
waitForTransportReady: waitForTransportReadyMock,
}));
vi.mock("openclaw/plugin-sdk/conversation-runtime", async (importOriginal) => {
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/conversation-runtime")>();
return {
...actual,
readChannelAllowFromStore: readChannelAllowFromStoreMock,
recordInboundSession: vi.fn(),
upsertChannelPairingRequest: vi.fn(),
};
});
vi.mock("openclaw/plugin-sdk/channel-inbound", async (importOriginal) => {
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/channel-inbound")>();
return {
...actual,
createChannelInboundDebouncer: vi.fn((opts) => ({
debouncer: {
enqueue: async (entry: unknown) => await opts.onFlush([entry]),
},
})),
shouldDebounceTextInbound: vi.fn(() => false),
};
});
vi.mock("./client.js", () => ({
createIMessageRpcClient: createIMessageRpcClientMock,
}));
vi.mock("./monitor/abort-handler.js", () => ({
attachIMessageMonitorAbortHandler: vi.fn(() => () => {}),
}));
vi.mock("./monitor/media-staging.js", () => ({
stageIMessageAttachments: stageIMessageAttachmentsMock,
}));
describe("iMessage monitor attachment policy", () => {
it("does not stage local attachments for messages dropped by inbound policy", async () => {
stageIMessageAttachmentsMock.mockResolvedValue([]);
readChannelAllowFromStoreMock.mockResolvedValue([]);
const attachmentPath = "/Users/openclaw/Library/Messages/Attachments/AA/BB/photo.heic";
let onNotification: ((message: { method: string; params: unknown }) => void) | undefined;
const client = {
request: vi.fn(async () => ({ subscription: 1 })),
waitForClose: vi.fn(async () => {
onNotification?.({
method: "message",
params: {
message: {
id: 1,
chat_id: 123,
sender: "+15550001111",
is_from_me: false,
is_group: true,
text: "no mention here",
attachments: [
{
original_path: attachmentPath,
mime_type: "image/heic",
missing: false,
},
],
},
},
});
await Promise.resolve();
await Promise.resolve();
}),
stop: vi.fn(async () => {}),
};
createIMessageRpcClientMock.mockImplementation(async (params) => {
if (!params?.onNotification) {
throw new Error("expected iMessage notification handler");
}
onNotification = params.onNotification;
return client as never;
});
await monitorIMessageProvider({
config: {
channels: {
imessage: {
includeAttachments: true,
attachmentRoots: ["/Users/*/Library/Messages/Attachments"],
dmPolicy: "open",
groupPolicy: "open",
groups: { "*": { requireMention: true } },
},
},
messages: { groupChat: { mentionPatterns: ["@openclaw"] } },
session: { mainKey: "main" },
} as never,
});
expect(readChannelAllowFromStoreMock).toHaveBeenCalled();
expect(stageIMessageAttachmentsMock).not.toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,120 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { stageIMessageAttachments } from "./media-staging.js";
let tempDir: string;
async function writeTempFile(name: string, contents: Buffer | string): Promise<string> {
const filePath = path.join(tempDir, name);
await fs.writeFile(filePath, contents);
return filePath;
}
describe("stageIMessageAttachments", () => {
beforeEach(async () => {
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-imessage-media-"));
});
afterEach(async () => {
await fs.rm(tempDir, { recursive: true, force: true });
});
it("copies allowed iMessage attachments into the inbound media store", async () => {
const sourcePath = await writeTempFile("photo.png", Buffer.from("png-bytes"));
const saveMediaBuffer = vi.fn(async () => ({
id: "saved.png",
path: "/state/media/inbound/saved.png",
size: 9,
contentType: "image/png",
}));
await expect(
stageIMessageAttachments(
[{ original_path: sourcePath, mime_type: "image/png", missing: false }],
{ maxBytes: 1024, allowedRoots: [tempDir], deps: { saveMediaBuffer } },
),
).resolves.toEqual([{ path: "/state/media/inbound/saved.png", contentType: "image/png" }]);
expect(saveMediaBuffer).toHaveBeenCalledWith(
Buffer.from("png-bytes"),
"image/png",
"inbound",
1024,
"photo.png",
);
});
it("drops attachments whose canonical path escapes the allowed root", async () => {
const allowedRoot = path.join(tempDir, "allowed");
const outsideRoot = path.join(tempDir, "outside");
await fs.mkdir(allowedRoot, { recursive: true });
await fs.mkdir(outsideRoot, { recursive: true });
const outsidePath = path.join(outsideRoot, "secret.png");
await fs.writeFile(outsidePath, Buffer.from("secret-bytes"));
await fs.symlink(outsideRoot, path.join(allowedRoot, "link"), "dir");
const saveMediaBuffer = vi.fn();
const logVerbose = vi.fn();
await expect(
stageIMessageAttachments(
[
{
original_path: path.join(allowedRoot, "link", "secret.png"),
mime_type: "image/png",
missing: false,
},
],
{ maxBytes: 1024, allowedRoots: [allowedRoot], deps: { saveMediaBuffer, logVerbose } },
),
).resolves.toEqual([]);
expect(saveMediaBuffer).not.toHaveBeenCalled();
expect(logVerbose).toHaveBeenCalledWith(
expect.stringContaining("attachment path resolves outside allowed roots"),
);
});
it("converts HEIC iMessage attachments to JPEG before staging", async () => {
const sourcePath = await writeTempFile("IMG_0001.HEIC", Buffer.from("heic-bytes"));
const saveMediaBuffer = vi.fn(async () => ({
id: "saved.jpg",
path: "/state/media/inbound/saved.jpg",
size: 10,
contentType: "image/jpeg",
}));
const convertHeicToJpeg = vi.fn(async () => Buffer.from("jpeg-bytes"));
await stageIMessageAttachments(
[{ original_path: sourcePath, mime_type: "image/heic", missing: false }],
{ maxBytes: 1024, deps: { saveMediaBuffer, convertHeicToJpeg } },
);
expect(convertHeicToJpeg).toHaveBeenCalledWith(sourcePath, 1024);
expect(saveMediaBuffer).toHaveBeenCalledWith(
Buffer.from("jpeg-bytes"),
"image/jpeg",
"inbound",
1024,
"IMG_0001.jpg",
);
});
it("drops attachments over the inbound media limit", async () => {
const sourcePath = await writeTempFile("huge.png", Buffer.from("too large"));
const saveMediaBuffer = vi.fn();
const logVerbose = vi.fn();
await expect(
stageIMessageAttachments(
[{ original_path: sourcePath, mime_type: "image/png", missing: false }],
{ maxBytes: 4, deps: { saveMediaBuffer, logVerbose } },
),
).resolves.toEqual([]);
expect(saveMediaBuffer).not.toHaveBeenCalled();
expect(logVerbose).toHaveBeenCalledWith(expect.stringContaining("failed to stage"));
});
});

View File

@@ -0,0 +1,205 @@
import { execFile } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { isInboundPathAllowed } from "openclaw/plugin-sdk/media-runtime";
import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store";
import { buildRandomTempFilePath } from "openclaw/plugin-sdk/temp-path";
import type { IMessageAttachment } from "./types.js";
const execFileAsync = promisify(execFile);
const HEIC_CONVERSION_TIMEOUT_MS = 15_000;
const HEIC_CONVERSION_MAX_BUFFER_BYTES = 64 * 1024;
export type StagedIMessageAttachment = {
path: string;
contentType?: string;
};
type SaveMediaBufferImpl = typeof saveMediaBuffer;
type StageIMessageAttachmentsDeps = {
saveMediaBuffer?: SaveMediaBufferImpl;
convertHeicToJpeg?: (sourcePath: string, maxBytes: number) => Promise<Buffer>;
logVerbose?: (message: string) => void;
};
function isHeicAttachment(attachmentPath: string, mimeType?: string | null): boolean {
const normalizedMime = mimeType?.toLowerCase();
if (normalizedMime === "image/heic" || normalizedMime === "image/heif") {
return true;
}
const ext = path.extname(attachmentPath).toLowerCase();
return ext === ".heic" || ext === ".heif";
}
function jpegFilenameForAttachment(attachmentPath: string): string {
const parsed = path.parse(attachmentPath);
return `${parsed.name || "imessage-attachment"}.jpg`;
}
function hasWildcardSegment(root: string): boolean {
return root.replaceAll("\\", "/").split("/").includes("*");
}
async function canonicalizeAllowedRoots(roots: readonly string[]): Promise<string[]> {
const canonicalRoots: string[] = [];
for (const root of roots) {
canonicalRoots.push(root);
if (hasWildcardSegment(root)) {
continue;
}
const canonicalRoot = await fs.realpath(root).catch(() => undefined);
if (canonicalRoot && canonicalRoot !== root) {
canonicalRoots.push(canonicalRoot);
}
}
return canonicalRoots;
}
async function resolveAllowedCanonicalAttachmentPath(params: {
attachmentPath: string;
allowedRoots?: readonly string[];
}): Promise<string> {
if (!params.allowedRoots) {
return params.attachmentPath;
}
const canonicalPath = await fs.realpath(params.attachmentPath);
const canonicalRoots = await canonicalizeAllowedRoots(params.allowedRoots);
if (!isInboundPathAllowed({ filePath: canonicalPath, roots: canonicalRoots })) {
throw new Error("attachment path resolves outside allowed roots");
}
return canonicalPath;
}
async function convertHeicToJpegWithSips(sourcePath: string, maxBytes: number): Promise<Buffer> {
const tempPath = buildRandomTempFilePath({
prefix: "openclaw-imessage",
extension: "jpg",
});
try {
await execFileAsync(
"sips",
[
"-s",
"format",
"jpeg",
"-s",
"formatOptions",
"90",
"-Z",
"4096",
sourcePath,
"--out",
tempPath,
],
{
timeout: HEIC_CONVERSION_TIMEOUT_MS,
maxBuffer: HEIC_CONVERSION_MAX_BUFFER_BYTES,
killSignal: "SIGKILL",
},
);
const stat = await fs.stat(tempPath);
if (stat.size > maxBytes) {
throw new Error(`converted media exceeds ${Math.round(maxBytes / (1024 * 1024))}MB limit`);
}
return await fs.readFile(tempPath);
} finally {
await fs.rm(tempPath, { force: true }).catch(() => {});
}
}
async function readAttachmentBuffer(params: {
attachmentPath: string;
mimeType?: string | null;
maxBytes: number;
allowedRoots?: readonly string[];
deps: StageIMessageAttachmentsDeps;
}): Promise<{ buffer: Buffer; contentType?: string; originalFilename?: string }> {
const stat = await fs.lstat(params.attachmentPath);
if (stat.isSymbolicLink()) {
throw new Error("attachment path is a symlink");
}
if (!stat.isFile()) {
throw new Error("attachment path is not a file");
}
if (stat.size > params.maxBytes) {
throw new Error(`attachment exceeds ${Math.round(params.maxBytes / (1024 * 1024))}MB limit`);
}
const canonicalPath = await resolveAllowedCanonicalAttachmentPath({
attachmentPath: params.attachmentPath,
allowedRoots: params.allowedRoots,
});
const canonicalStat = await fs.stat(canonicalPath);
if (!canonicalStat.isFile()) {
throw new Error("attachment path is not a file");
}
if (canonicalStat.size > params.maxBytes) {
throw new Error(`attachment exceeds ${Math.round(params.maxBytes / (1024 * 1024))}MB limit`);
}
if (isHeicAttachment(params.attachmentPath, params.mimeType)) {
try {
const convert = params.deps.convertHeicToJpeg ?? convertHeicToJpegWithSips;
return {
buffer: await convert(canonicalPath, params.maxBytes),
contentType: "image/jpeg",
originalFilename: jpegFilenameForAttachment(params.attachmentPath),
};
} catch (err) {
params.deps.logVerbose?.(
`imessage: HEIC attachment conversion failed; staging original instead: ${String(err)}`,
);
}
}
return {
buffer: await fs.readFile(canonicalPath),
contentType: params.mimeType ?? undefined,
originalFilename: path.basename(params.attachmentPath),
};
}
export async function stageIMessageAttachments(
attachments: IMessageAttachment[],
params: {
maxBytes: number;
allowedRoots?: readonly string[];
deps?: StageIMessageAttachmentsDeps;
},
): Promise<StagedIMessageAttachment[]> {
const deps = params.deps ?? {};
const save = deps.saveMediaBuffer ?? saveMediaBuffer;
const staged: StagedIMessageAttachment[] = [];
for (const attachment of attachments) {
const attachmentPath = attachment.original_path?.trim();
if (!attachmentPath || attachment.missing) {
continue;
}
try {
const media = await readAttachmentBuffer({
attachmentPath,
mimeType: attachment.mime_type,
maxBytes: params.maxBytes,
allowedRoots: params.allowedRoots,
deps,
});
const saved = await save(
media.buffer,
media.contentType,
"inbound",
params.maxBytes,
media.originalFilename,
);
staged.push({ path: saved.path, contentType: saved.contentType });
} catch (err) {
deps.logVerbose?.(`imessage: failed to stage inbound attachment: ${String(err)}`);
}
}
return staged;
}

View File

@@ -65,6 +65,7 @@ import {
resolveIMessageInboundDecision, resolveIMessageInboundDecision,
} from "./inbound-processing.js"; } from "./inbound-processing.js";
import { createLoopRateLimiter } from "./loop-rate-limiter.js"; import { createLoopRateLimiter } from "./loop-rate-limiter.js";
import { stageIMessageAttachments } from "./media-staging.js";
import { parseIMessageNotification } from "./parse-notification.js"; import { parseIMessageNotification } from "./parse-notification.js";
import { enqueueIMessageReactionSystemEvent } from "./reaction-system-event.js"; import { enqueueIMessageReactionSystemEvent } from "./reaction-system-event.js";
import { normalizeAllowList, resolveRuntime } from "./runtime.js"; import { normalizeAllowList, resolveRuntime } from "./runtime.js";
@@ -371,13 +372,14 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
logVerbose(`imessage: dropping inbound attachment outside allowed roots: ${attachmentPath}`); logVerbose(`imessage: dropping inbound attachment outside allowed roots: ${attachmentPath}`);
return false; return false;
}); });
const firstAttachment = validAttachments[0]; const rawMediaAttachments = validAttachments.flatMap((a) => {
const mediaPath = firstAttachment?.original_path ?? undefined; const attachmentPath = a.original_path?.trim();
const mediaType = firstAttachment?.mime_type ?? undefined; return attachmentPath
// Build arrays for all attachments (for multi-image support) ? [{ path: attachmentPath, contentType: a.mime_type ?? undefined }]
const mediaPaths = validAttachments.map((a) => a.original_path).filter(Boolean) as string[]; : [];
const mediaTypes = validAttachments.map((a) => a.mime_type ?? undefined); });
const kind = kindFromMime(mediaType ?? undefined); const placeholderMediaType = rawMediaAttachments[0]?.contentType;
const kind = kindFromMime(placeholderMediaType ?? undefined);
const placeholder = kind const placeholder = kind
? `<media:${kind}>` ? `<media:${kind}>`
: validAttachments.length : validAttachments.length
@@ -501,6 +503,20 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
const storePath = resolveStorePath(cfg.session?.store, { const storePath = resolveStorePath(cfg.session?.store, {
agentId: decision.route.agentId, agentId: decision.route.agentId,
}); });
const stagedAttachments = remoteHost
? []
: await stageIMessageAttachments(validAttachments, {
maxBytes: mediaMaxBytes,
allowedRoots: effectiveAttachmentRoots,
deps: { logVerbose },
});
const mediaAttachments = remoteHost ? rawMediaAttachments : stagedAttachments;
const firstAttachment = mediaAttachments[0];
const mediaPath = firstAttachment?.path ?? undefined;
const mediaType = firstAttachment?.contentType ?? undefined;
// Build arrays for all attachments (for multi-image support)
const mediaPaths = mediaAttachments.map((a) => a.path).filter(Boolean);
const mediaTypes = mediaAttachments.map((a) => a.contentType ?? undefined);
const previousTimestamp = readSessionUpdatedAt({ const previousTimestamp = readSessionUpdatedAt({
storePath, storePath,
sessionKey: decision.route.sessionKey, sessionKey: decision.route.sessionKey,

View File

@@ -1,7 +1,7 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env"; import type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";
type IMessageAttachment = { export type IMessageAttachment = {
original_path?: string | null; original_path?: string | null;
mime_type?: string | null; mime_type?: string | null;
missing?: boolean | null; missing?: boolean | null;