diff --git a/AGENTS.md b/AGENTS.md index 44d08ae955..7913ddabd2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,7 @@ ### General Principles - Keep things in one function unless composable or reusable +- Do not extract single-use helpers preemptively. Inline the logic at the call site unless the helper is reused, hides a genuinely complex boundary, or has a clear independent name that improves the caller. - Avoid `try`/`catch` where possible - Avoid using the `any` type - Use Bun APIs when possible, like `Bun.file()` diff --git a/bun.lock b/bun.lock index 9dcad8f126..d98f1b2456 100644 --- a/bun.lock +++ b/bun.lock @@ -412,6 +412,7 @@ "@opentui/solid": "catalog:", "@parcel/watcher": "2.5.1", "@pierre/diffs": "catalog:", + "@silvia-odwyer/photon-node": "0.3.4", "@solid-primitives/event-bus": "1.1.2", "@solid-primitives/scheduled": "1.5.2", "@standard-schema/spec": "1.0.0", @@ -677,6 +678,7 @@ "solid-js@1.9.10": "patches/solid-js@1.9.10.patch", "@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch", "@npmcli/agent@4.0.0": "patches/@npmcli%2Fagent@4.0.0.patch", + "@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch", }, "overrides": { "@types/bun": "catalog:", @@ -2035,6 +2037,8 @@ "@sigstore/verify": ["@sigstore/verify@3.1.0", "", { "dependencies": { "@sigstore/bundle": "^4.0.0", "@sigstore/core": "^3.1.0", "@sigstore/protobuf-specs": "^0.5.0" } }, "sha512-mNe0Iigql08YupSOGv197YdHpPPr+EzDZmfCgMc7RPNaZTw5aLN01nBl6CHJOh3BGtnMIj83EeN4butBchc8Ag=="], + "@silvia-odwyer/photon-node": ["@silvia-odwyer/photon-node@0.3.4", "", {}, "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA=="], + "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], "@slack/bolt": ["@slack/bolt@3.22.0", "", { "dependencies": { "@slack/logger": "^4.0.0", "@slack/oauth": "^2.6.3", "@slack/socket-mode": "^1.3.6", "@slack/types": "^2.13.0", "@slack/web-api": "^6.13.0", "@types/express": "^4.16.1", "@types/promise.allsettled": "^1.0.3", "@types/tsscmp": "^1.0.0", "axios": "^1.7.4", "express": "^4.21.0", "path-to-regexp": "^8.1.0", "promise.allsettled": "^1.0.2", "raw-body": "^2.3.3", "tsscmp": "^1.0.6" } }, "sha512-iKDqGPEJDnrVwxSVlFW6OKTkijd7s4qLBeSufoBsTM0reTyfdp/5izIQVkxNfzjHi3o6qjdYbRXkYad5HBsBog=="], diff --git a/package.json b/package.json index 27a3597553..5faf8be920 100644 --- a/package.json +++ b/package.json @@ -133,6 +133,7 @@ }, "patchedDependencies": { "@npmcli/agent@4.0.0": "patches/@npmcli%2Fagent@4.0.0.patch", + "@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch", "@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch", "solid-js@1.9.10": "patches/solid-js@1.9.10.patch" } diff --git a/packages/opencode/package.json b/packages/opencode/package.json index dbd766f934..f6977296f9 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -119,6 +119,7 @@ "@opentui/solid": "catalog:", "@parcel/watcher": "2.5.1", "@pierre/diffs": "catalog:", + "@silvia-odwyer/photon-node": "0.3.4", "@solid-primitives/event-bus": "1.1.2", "@solid-primitives/scheduled": "1.5.2", "@standard-schema/spec": "1.0.0", diff --git a/packages/opencode/src/audio.d.ts b/packages/opencode/src/audio.d.ts index 54a86efa30..c7c947450d 100644 --- a/packages/opencode/src/audio.d.ts +++ b/packages/opencode/src/audio.d.ts @@ -2,3 +2,8 @@ declare module "*.wav" { const file: string export default file } + +declare module "*.wasm" { + const file: string + export default file +} diff --git a/packages/opencode/src/config/attachment.ts b/packages/opencode/src/config/attachment.ts new file mode 100644 index 0000000000..7af429afde --- /dev/null +++ b/packages/opencode/src/config/attachment.ts @@ -0,0 +1,30 @@ +export * as ConfigAttachment from "./attachment" + +import { Schema } from "effect" +import { zod } from "@opencode-ai/core/effect-zod" +import { PositiveInt, withStatics } from "@opencode-ai/core/schema" + +export const Image = Schema.Struct({ + auto_resize: Schema.optional(Schema.Boolean).annotate({ + description: "Resize images before sending them to the model when they exceed configured limits (default: true)", + }), + max_width: Schema.optional(PositiveInt).annotate({ + description: "Maximum image width before resizing or rejecting the attachment (default: 2000)", + }), + max_height: Schema.optional(PositiveInt).annotate({ + description: "Maximum image height before resizing or rejecting the attachment (default: 2000)", + }), + max_base64_bytes: Schema.optional(PositiveInt).annotate({ + description: "Maximum base64 payload bytes for an image attachment (default: 4718592)", + }), +}) + .annotate({ identifier: "ImageAttachmentConfig" }) + .pipe(withStatics((s) => ({ zod: zod(s) }))) +export type Image = Schema.Schema.Type + +export const Info = Schema.Struct({ + image: Schema.optional(Image).annotate({ description: "Image attachment configuration" }), +}) + .annotate({ identifier: "AttachmentConfig" }) + .pipe(withStatics((s) => ({ zod: zod(s) }))) +export type Info = Schema.Schema.Type diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index 073001e191..41ccac7492 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -25,6 +25,7 @@ import { containsPath } from "../project/instance-context" import { zod } from "@opencode-ai/core/effect-zod" import { NonNegativeInt, PositiveInt, withStatics, type DeepMutable } from "@opencode-ai/core/schema" import { ConfigAgent } from "./agent" +import { ConfigAttachment } from "./attachment" import { ConfigCommand } from "./command" import { ConfigFormatter } from "./formatter" import { ConfigLayout } from "./layout" @@ -241,6 +242,9 @@ export const Info = Schema.Struct({ layout: Schema.optional(ConfigLayout.Layout).annotate({ description: "@deprecated Always uses stretch layout." }), permission: Schema.optional(ConfigPermission.Info), tools: Schema.optional(Schema.Record(Schema.String, Schema.Boolean)), + attachment: Schema.optional(ConfigAttachment.Info).annotate({ + description: "Attachment processing configuration, including image size limits and resizing behavior", + }), enterprise: Schema.optional( Schema.Struct({ url: Schema.optional(Schema.String).annotate({ description: "Enterprise URL" }), diff --git a/packages/opencode/src/image/image.ts b/packages/opencode/src/image/image.ts new file mode 100644 index 0000000000..2115e19198 --- /dev/null +++ b/packages/opencode/src/image/image.ts @@ -0,0 +1,180 @@ +import { Config } from "@/config/config" +import type { MessageV2 } from "@/session/message-v2" +import * as Log from "@opencode-ai/core/util/log" +import { Context, Effect, Layer, Schema } from "effect" + +const MAX_BASE64_BYTES = 4.5 * 1024 * 1024 +const MAX_WIDTH = 2000 +const MAX_HEIGHT = 2000 +const AUTO_RESIZE = true +const JPEG_QUALITIES = [80, 85, 70, 55, 40] +const log = Log.create({ service: "image" }) + +export class PhotonUnavailableError extends Schema.TaggedErrorClass()( + "ImagePhotonUnavailableError", + {}, +) { + override get message() { + return "Photon image processor is unavailable" + } +} + +export class InvalidDataUrlError extends Schema.TaggedErrorClass()("ImageInvalidDataUrlError", { + url: Schema.String, +}) { + override get message() { + return "Image URL must be a base64 data URL" + } +} + +export class DecodeError extends Schema.TaggedErrorClass()("ImageDecodeError", {}) { + override get message() { + return "Image could not be decoded" + } +} + +export class SizeError extends Schema.TaggedErrorClass()("ImageSizeError", { + bytes: Schema.Number, + max: Schema.Number, + width: Schema.Number, + height: Schema.Number, + max_width: Schema.Number, + max_height: Schema.Number, +}) { + override get message() { + return `Image ${this.width}x${this.height} with base64 size ${this.bytes} exceeds configured limits and could not be resized below ${this.max_width}x${this.max_height}/${this.max} bytes` + } +} + +export type Error = PhotonUnavailableError | InvalidDataUrlError | DecodeError | SizeError + +export interface Interface { + readonly normalize: (input: MessageV2.FilePart) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/Image") {} + +export const layer = Layer.effect( + Service, + Effect.gen(function* () { + const config = yield* Config.Service + const loadPhoton = yield* Effect.cached( + Effect.promise(async () => { + try { + const photonWasm = (await import("@silvia-odwyer/photon-node/photon_rs_bg.wasm", { with: { type: "file" } })) + .default + // Patched photon-node reads this during module init so Bun compiled binaries use the embedded wasm path. + ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH = + photonWasm + return await import("@silvia-odwyer/photon-node") + } catch { + return null + } + }), + ) + + const normalize = Effect.fn("Image.normalize")(function* (input: MessageV2.FilePart) { + const image = (yield* config.get()).attachment?.image + const info = { + autoResize: image?.auto_resize ?? AUTO_RESIZE, + maxWidth: image?.max_width ?? MAX_WIDTH, + maxHeight: image?.max_height ?? MAX_HEIGHT, + maxBase64Bytes: image?.max_base64_bytes ?? MAX_BASE64_BYTES, + } + if (!input.url.startsWith("data:") || !input.url.includes(";base64,")) + return yield* new InvalidDataUrlError({ url: input.url }) + + const base64 = input.url.slice(input.url.indexOf(";base64,") + ";base64,".length) + const photon = yield* loadPhoton + if (!photon) return yield* new PhotonUnavailableError() + + const decoded = yield* Effect.sync(() => { + try { + return photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64")) + } catch { + return undefined + } + }) + if (!decoded) return yield* new DecodeError() + + try { + const originalWidth = decoded.get_width() + const originalHeight = decoded.get_height() + if ( + originalWidth <= info.maxWidth && + originalHeight <= info.maxHeight && + Buffer.byteLength(base64, "utf8") <= info.maxBase64Bytes + ) + return input + if (!info.autoResize) + return yield* new SizeError({ + bytes: Buffer.byteLength(base64, "utf8"), + max: info.maxBase64Bytes, + width: originalWidth, + height: originalHeight, + max_width: info.maxWidth, + max_height: info.maxHeight, + }) + + const scale = Math.min(1, info.maxWidth / originalWidth, info.maxHeight / originalHeight) + for (const size of Array.from({ length: 32 }).reduce>((acc) => { + const previous = acc.at(-1) ?? { + width: Math.max(1, Math.round(originalWidth * scale)), + height: Math.max(1, Math.round(originalHeight * scale)), + } + const next = + acc.length === 0 + ? previous + : { + width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)), + height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)), + } + return acc.some((item) => item.width === next.width && item.height === next.height) ? acc : [...acc, next] + }, [])) { + const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3) + const candidate = [ + { data: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" }, + ...JPEG_QUALITIES.map((quality) => ({ + data: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"), + mime: "image/jpeg", + })), + ] + .map((item) => ({ ...item, bytes: Buffer.byteLength(item.data, "utf8") })) + .find((item) => item.bytes <= info.maxBase64Bytes) + resized.free() + + if (candidate) { + log.info("using resized image", { + from_mime: input.mime, + to_mime: candidate.mime, + from: `${originalWidth}x${originalHeight}`, + to: `${size.width}x${size.height}`, + }) + return { + ...input, + mime: candidate.mime, + url: `data:${candidate.mime};base64,${candidate.data}`, + } + } + } + + return yield* new SizeError({ + bytes: Buffer.byteLength(base64, "utf8"), + max: info.maxBase64Bytes, + width: originalWidth, + height: originalHeight, + max_width: info.maxWidth, + max_height: info.maxHeight, + }) + } finally { + decoded.free() + } + }) + + return Service.of({ normalize }) + }), +) + +export const defaultLayer = layer.pipe(Layer.provide(Config.defaultLayer)) + +export * as Image from "./image" diff --git a/packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts b/packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts index 2328375b97..e64ad6d504 100644 --- a/packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts +++ b/packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts @@ -203,13 +203,15 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", params: { sessionID: SessionID } payload: typeof InitPayload.Type }) { - yield* promptSvc.command({ - sessionID: ctx.params.sessionID, - messageID: ctx.payload.messageID, - model: `${ctx.payload.providerID}/${ctx.payload.modelID}`, - command: Command.Default.INIT, - arguments: "", - }) + yield* promptSvc + .command({ + sessionID: ctx.params.sessionID, + messageID: ctx.payload.messageID, + model: `${ctx.payload.providerID}/${ctx.payload.modelID}`, + command: Command.Default.INIT, + arguments: "", + }) + .pipe(Effect.mapError(() => new HttpApiError.BadRequest({}))) return true }) @@ -258,18 +260,18 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", }) { const instance = yield* InstanceState.context const workspace = yield* InstanceState.workspaceID + const message = yield* promptSvc + .prompt({ + ...ctx.payload, + sessionID: ctx.params.sessionID, + }) + .pipe( + Effect.provideService(InstanceRef, instance), + Effect.provideService(WorkspaceRef, workspace), + Effect.mapError(() => new HttpApiError.BadRequest({})), + ) return HttpServerResponse.stream( - Stream.fromEffect( - promptSvc - .prompt({ - ...ctx.payload, - sessionID: ctx.params.sessionID, - }) - .pipe(Effect.provideService(InstanceRef, instance), Effect.provideService(WorkspaceRef, workspace)), - ).pipe( - Stream.map((message) => JSON.stringify(message)), - Stream.encodeText, - ), + Stream.make(JSON.stringify(message)).pipe(Stream.encodeText), { contentType: "application/json" }, ) }) @@ -297,7 +299,9 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", params: { sessionID: SessionID } payload: typeof CommandPayload.Type }) { - return yield* promptSvc.command({ ...ctx.payload, sessionID: ctx.params.sessionID }) + return yield* promptSvc + .command({ ...ctx.payload, sessionID: ctx.params.sessionID }) + .pipe(Effect.mapError(() => new HttpApiError.BadRequest({}))) }) const shell = Effect.fn("SessionHttpApi.shell")(function* (ctx: { diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index 6e84db16e2..d87f04f888 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -1,4 +1,4 @@ -import { Cause, Deferred, Effect, Layer, Context, Scope } from "effect" +import { Cause, Deferred, Effect, Exit, Layer, Context, Scope } from "effect" import * as Stream from "effect/Stream" import { Agent } from "@/agent/agent" import { Bus } from "@/bus" @@ -9,6 +9,7 @@ import { Snapshot } from "@/snapshot" import * as Session from "./session" import { LLM } from "./llm" import { MessageV2 } from "./message-v2" +import { Image } from "@/image/image" import { isOverflow } from "./overflow" import { PartID } from "./schema" import type { SessionID } from "./schema" @@ -92,6 +93,7 @@ export const layer: Layer.Layer< | LLM.Service | Permission.Service | Plugin.Service + | Image.Service | SessionSummary.Service | SessionStatus.Service > = Layer.effect( @@ -108,6 +110,7 @@ export const layer: Layer.Layer< const summary = yield* SessionSummary.Service const scope = yield* Scope.Scope const status = yield* SessionStatus.Service + const image = yield* Image.Service const create = Effect.fn("SessionProcessor.create")(function* (input: Input) { // Pre-capture snapshot before the LLM stream starts. The AI SDK @@ -377,17 +380,43 @@ export const layer: Layer.Layer< case "tool-result": { const toolCall = yield* readToolCall(value.toolCallId) + const toolAttachments: MessageV2.FilePart[] = ( + Array.isArray(value.output.attachments) ? value.output.attachments : [] + ).filter( + (attachment: unknown): attachment is MessageV2.FilePart => + isRecord(attachment) && + attachment.type === "file" && + typeof attachment.mime === "string" && + typeof attachment.url === "string", + ) + const normalized = yield* Effect.forEach( + toolAttachments, + (attachment) => + attachment.mime.startsWith("image/") + ? image.normalize(attachment).pipe(Effect.exit) + : Effect.succeed(Exit.succeed(attachment)), + ) + const omitted = normalized.filter(Exit.isFailure).length + const attachments = normalized.filter(Exit.isSuccess).map((item) => item.value) + const output = { + ...value.output, + output: + omitted === 0 + ? value.output.output + : `${value.output.output}\n\n[${omitted} image${omitted === 1 ? "" : "s"} omitted: could not be resized below the inline image size limit.]`, + attachments: attachments?.length ? attachments : undefined, + } // TODO(v2): Temporary dual-write while migrating session messages to v2 events. EventV2.run(SessionEvent.Tool.Success.Sync, { sessionID: ctx.sessionID, callID: value.toolCallId, - structured: value.output.metadata, + structured: output.metadata, content: [ { type: "text", - text: value.output.output, + text: output.output, }, - ...(value.output.attachments?.map((item: MessageV2.FilePart) => ({ + ...(output.attachments?.map((item: MessageV2.FilePart) => ({ type: "file", uri: item.url, mime: item.mime, @@ -399,7 +428,7 @@ export const layer: Layer.Layer< }, timestamp: DateTime.makeUnsafe(Date.now()), }) - yield* completeToolCall(value.toolCallId, value.output) + yield* completeToolCall(value.toolCallId, output) return } @@ -758,6 +787,7 @@ export const defaultLayer = Layer.suspend(() => Layer.provide(Plugin.defaultLayer), Layer.provide(SessionSummary.defaultLayer), Layer.provide(SessionStatus.defaultLayer), + Layer.provide(Image.defaultLayer), Layer.provide(Bus.layer), Layer.provide(Config.defaultLayer), ), diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 1d5e4055c8..1fd61d23e0 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -43,6 +43,7 @@ import { Shell } from "@/shell/shell" import { ShellID } from "@/tool/shell/id" import { AppFileSystem } from "@opencode-ai/core/filesystem" import { Truncate } from "@/tool/truncate" +import { Image } from "@/image/image" import { decodeDataUrl } from "@/util/data-url" import { Process } from "@/util/process" import { Cause, Effect, Exit, Latch, Layer, Option, Scope, Context, Schema, Types } from "effect" @@ -80,10 +81,10 @@ const elog = EffectLogger.create({ service: "session.prompt" }) export interface Interface { readonly cancel: (sessionID: SessionID) => Effect.Effect - readonly prompt: (input: PromptInput) => Effect.Effect + readonly prompt: (input: PromptInput) => Effect.Effect readonly loop: (input: LoopInput) => Effect.Effect readonly shell: (input: ShellInput) => Effect.Effect - readonly command: (input: CommandInput) => Effect.Effect + readonly command: (input: CommandInput) => Effect.Effect readonly resolvePromptParts: (template: string) => Effect.Effect } @@ -108,6 +109,7 @@ export const layer = Layer.effect( const lsp = yield* LSP.Service const registry = yield* ToolRegistry.Service const truncate = yield* Truncate.Service + const image = yield* Image.Service const spawner = yield* ChildProcessSpawner.ChildProcessSpawner const scope = yield* Scope.Scope const instruction = yield* Instruction.Service @@ -123,7 +125,7 @@ export const layer = Layer.effect( return { cancel: (sessionID: SessionID) => cancel(sessionID), resolvePromptParts: (template: string) => resolvePromptParts(template), - prompt: (input: PromptInput) => prompt(input), + prompt: (input: PromptInput) => prompt(input).pipe(Effect.catch(Effect.die)), } satisfies TaskPromptOps }) @@ -1259,7 +1261,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the return [{ ...part, messageID: info.id, sessionID: input.sessionID }] }) - const parts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe( + const resolvedParts = yield* Effect.forEach(input.parts, resolvePart, { concurrency: "unbounded" }).pipe( Effect.map((x) => x.flat().map(assign)), ) @@ -1272,7 +1274,13 @@ NOTE: At any point in time through this workflow you should feel free to ask the messageID: input.messageID, variant: input.variant, }, - { message: info, parts }, + { message: info, parts: resolvedParts }, + ) + + const parts = yield* Effect.forEach(resolvedParts, (part) => + part.type === "file" && part.mime.startsWith("image/") + ? image.normalize(part) + : Effect.succeed(part), ) const parsed = MessageV2.Info.zod.safeParse(info) @@ -1368,7 +1376,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the return { info, parts } }, Effect.scoped) - const prompt: (input: PromptInput) => Effect.Effect = Effect.fn("SessionPrompt.prompt")( + const prompt: (input: PromptInput) => Effect.Effect = Effect.fn("SessionPrompt.prompt")( function* (input: PromptInput) { const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie) yield* revert.cleanup(session) @@ -1788,6 +1796,7 @@ export const defaultLayer = Layer.suspend(() => Layer.provide(Session.defaultLayer), Layer.provide(SessionRevert.defaultLayer), Layer.provide(SessionSummary.defaultLayer), + Layer.provide(Image.defaultLayer), Layer.provide( Layer.mergeAll( Agent.defaultLayer, diff --git a/packages/opencode/test/image/image.test.ts b/packages/opencode/test/image/image.test.ts new file mode 100644 index 0000000000..67e6977e38 --- /dev/null +++ b/packages/opencode/test/image/image.test.ts @@ -0,0 +1,78 @@ +import { describe, expect } from "bun:test" +import { Cause, Effect, Exit, Layer } from "effect" +import { Image } from "@/image/image" +import { MessageID, PartID, SessionID } from "@/session/schema" +import { TestConfig } from "../fixture/config" +import { testEffect } from "../lib/effect" + +const it = testEffect(Layer.mergeAll(Image.layer.pipe(Layer.provide(TestConfig.layer())))) +const tiny = testEffect( + Layer.mergeAll( + Image.layer.pipe( + Layer.provide(TestConfig.layer({ get: () => Effect.succeed({ attachment: { image: { max_base64_bytes: 1 } } }) })), + ), + ), +) + +function part(mime: string, data: string) { + return { + id: PartID.ascending(), + messageID: MessageID.ascending(), + sessionID: SessionID.make("ses_test"), + type: "file" as const, + mime, + url: `data:${mime};base64,${data}`, + } +} + +describe("Image", () => { + it.effect("normalizes generated png and jpeg attachments", () => + Effect.gen(function* () { + const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node")) + const source = new photon.PhotonImage( + new Uint8Array(Array.from({ length: 64 * 64 * 4 }, (_, index) => (index % 4 === 3 ? 255 : index % 251))), + 64, + 64, + ) + const image = yield* Image.Service + const results = yield* Effect.all([ + image.normalize(part("image/png", Buffer.from(source.get_bytes()).toString("base64"))), + image.normalize(part("image/jpeg", Buffer.from(source.get_bytes_jpeg(90)).toString("base64"))), + ]) + + source.free() + expect(results.map((result) => result.url.startsWith(`data:${result.mime};base64,`))).toEqual([true, true]) + expect(results.every((result) => result.mime === "image/png" || result.mime === "image/jpeg")).toBe(true) + }), + ) + + it.effect("accepts webp attachments that are already within limits", () => + Effect.gen(function* () { + const image = yield* Image.Service + const input = part("image/webp", "UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA") + + expect(yield* image.normalize(input)).toEqual(input) + }), + ) + + tiny.effect("fails with a typed size error when no resized candidate fits", () => + Effect.gen(function* () { + const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node")) + const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 4 }, () => 255)), 1, 1) + const image = yield* Image.Service + const exit = yield* image.normalize(part("image/png", Buffer.from(source.get_bytes()).toString("base64"))).pipe(Effect.exit) + + source.free() + expect(Exit.isFailure(exit)).toBe(true) + if (Exit.isFailure(exit)) { + const error = Cause.squash(exit.cause) + expect(error).toBeInstanceOf(Image.SizeError) + if (error instanceof Image.SizeError) { + expect(error.width).toBe(1) + expect(error.height).toBe(1) + expect(error.max).toBe(1) + } + } + }), + ) +}) diff --git a/packages/opencode/test/session/compaction.test.ts b/packages/opencode/test/session/compaction.test.ts index cde9c1397f..03b2576154 100644 --- a/packages/opencode/test/session/compaction.test.ts +++ b/packages/opencode/test/session/compaction.test.ts @@ -5,6 +5,7 @@ import * as Stream from "effect/Stream" import z from "zod" import { Bus } from "../../src/bus" import { Config } from "@/config/config" +import { Image } from "@/image/image" import { Agent } from "../../src/agent/agent" import { LLM } from "../../src/session/llm" import { SessionCompaction } from "../../src/session/compaction" @@ -278,7 +279,7 @@ function llm() { function liveRuntime(layer: Layer.Layer, provider = ProviderTest.fake(), config = Config.defaultLayer) { const bus = Bus.layer const status = SessionStatus.layer.pipe(Layer.provide(bus)) - const processor = SessionProcessorModule.SessionProcessor.layer.pipe(Layer.provide(summary)) + const processor = SessionProcessorModule.SessionProcessor.layer.pipe(Layer.provide(summary), Layer.provide(Image.defaultLayer)) return ManagedRuntime.make( Layer.mergeAll(SessionCompaction.layer.pipe(Layer.provide(processor)), processor, bus, status).pipe( Layer.provide(provider.layer), diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 226bab9864..a0736b459b 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -6,6 +6,7 @@ import type { Agent } from "../../src/agent/agent" import { Agent as AgentSvc } from "../../src/agent/agent" import { Bus } from "../../src/bus" import { Config } from "@/config/config" +import { Image } from "@/image/image" import { Permission } from "../../src/permission" import { Plugin } from "../../src/plugin" import { Provider } from "@/provider/provider" @@ -168,7 +169,7 @@ const deps = Layer.mergeAll( ).pipe(Layer.provideMerge(infra)) const env = Layer.mergeAll( TestLLMServer.layer, - SessionProcessor.layer.pipe(Layer.provide(summary), Layer.provideMerge(deps)), + SessionProcessor.layer.pipe(Layer.provide(summary), Layer.provide(Image.defaultLayer), Layer.provideMerge(deps)), ) const it = testEffect(env) diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 3b0009d2b3..bf3811113c 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -16,6 +16,7 @@ import { Plugin } from "../../src/plugin" import { Provider as ProviderSvc } from "@/provider/provider" import { Env } from "../../src/env" import { Git } from "../../src/git" +import { Image } from "../../src/image/image" import { ModelID, ProviderID } from "../../src/provider/schema" import { Question } from "../../src/question" import { Todo } from "../../src/session/todo" @@ -187,12 +188,13 @@ function makeHttp() { Layer.provideMerge(deps), ) const trunc = Truncate.layer.pipe(Layer.provideMerge(deps)) - const proc = SessionProcessor.layer.pipe(Layer.provide(summary), Layer.provideMerge(deps)) + const proc = SessionProcessor.layer.pipe(Layer.provide(summary), Layer.provide(Image.defaultLayer), Layer.provideMerge(deps)) const compact = SessionCompaction.layer.pipe(Layer.provideMerge(proc), Layer.provideMerge(deps)) return Layer.mergeAll( TestLLMServer.layer, SessionPrompt.layer.pipe( Layer.provide(SessionRevert.defaultLayer), + Layer.provide(Image.defaultLayer), Layer.provide(summary), Layer.provideMerge(run), Layer.provideMerge(compact), diff --git a/packages/opencode/test/session/snapshot-tool-race.test.ts b/packages/opencode/test/session/snapshot-tool-race.test.ts index 671f62145c..727b75cfee 100644 --- a/packages/opencode/test/session/snapshot-tool-race.test.ts +++ b/packages/opencode/test/session/snapshot-tool-race.test.ts @@ -41,6 +41,7 @@ import { Plugin } from "../../src/plugin" import { Provider as ProviderSvc } from "@/provider/provider" import { Env } from "../../src/env" import { Question } from "../../src/question" +import { Image } from "../../src/image/image" import { Skill } from "../../src/skill" import { SystemPrompt } from "../../src/session/system" import { Todo } from "../../src/session/todo" @@ -137,13 +138,18 @@ function makeHttp() { Layer.provideMerge(deps), ) const trunc = Truncate.layer.pipe(Layer.provideMerge(deps)) - const proc = SessionProcessor.layer.pipe(Layer.provide(SessionSummary.defaultLayer), Layer.provideMerge(deps)) + const proc = SessionProcessor.layer.pipe( + Layer.provide(SessionSummary.defaultLayer), + Layer.provide(Image.defaultLayer), + Layer.provideMerge(deps), + ) const compact = SessionCompaction.layer.pipe(Layer.provideMerge(proc), Layer.provideMerge(deps)) return Layer.mergeAll( TestLLMServer.layer, SessionSummary.defaultLayer, SessionPrompt.layer.pipe( Layer.provide(SessionRevert.defaultLayer), + Layer.provide(Image.defaultLayer), Layer.provide(SessionSummary.defaultLayer), Layer.provideMerge(run), Layer.provideMerge(compact), diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 4779f7cebd..ae7e9767ce 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -1132,6 +1132,17 @@ export type McpRemoteConfig = { */ export type LayoutConfig = "auto" | "stretch" +export type ImageAttachmentConfig = { + auto_resize?: boolean + max_width?: number + max_height?: number + max_base64_bytes?: number +} + +export type AttachmentConfig = { + image?: ImageAttachmentConfig +} + export type Config = { $schema?: string shell?: string @@ -1246,6 +1257,7 @@ export type Config = { tools?: { [key: string]: boolean } + attachment?: AttachmentConfig enterprise?: { url?: string } diff --git a/patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch b/patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch new file mode 100644 index 0000000000..2e43225562 --- /dev/null +++ b/patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch @@ -0,0 +1,14 @@ +diff --git a/photon_rs.js b/photon_rs.js +index 8f4144d..b83e9a9 100644 +--- a/photon_rs.js ++++ b/photon_rs.js +@@ -4509,7 +4509,8 @@ module.exports.__wbindgen_init_externref_table = function() { + ; + }; + +-const path = require('path').join(__dirname, 'photon_rs_bg.wasm'); ++// Allow opencode's Bun compiled binary to point photon-node at its embedded wasm asset. ++const path = globalThis.__OPENCODE_PHOTON_WASM_PATH || require('path').join(__dirname, 'photon_rs_bg.wasm'); + const bytes = require('fs').readFileSync(path); + + const wasmModule = new WebAssembly.Module(bytes);