diff --git a/bun.lock b/bun.lock index 3341e3f480..00cb6ee20d 100644 --- a/bun.lock +++ b/bun.lock @@ -463,6 +463,7 @@ "@babel/core": "7.28.4", "@octokit/webhooks-types": "7.6.1", "@opencode-ai/core": "workspace:*", + "@opencode-ai/http-recorder": "workspace:*", "@opencode-ai/script": "workspace:*", "@parcel/watcher-darwin-arm64": "2.5.1", "@parcel/watcher-darwin-x64": "2.5.1", diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 46f7fc3606..bbafcd126e 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -42,8 +42,9 @@ "devDependencies": { "@babel/core": "7.28.4", "@octokit/webhooks-types": "7.6.1", - "@opencode-ai/script": "workspace:*", "@opencode-ai/core": "workspace:*", + "@opencode-ai/http-recorder": "workspace:*", + "@opencode-ai/script": "workspace:*", "@parcel/watcher-darwin-arm64": "2.5.1", "@parcel/watcher-darwin-x64": "2.5.1", "@parcel/watcher-linux-arm64-glibc": "2.5.1", diff --git a/packages/opencode/test/fixtures/recordings/session/native-openai-tool-call.json b/packages/opencode/test/fixtures/recordings/session/native-openai-tool-call.json new file mode 100644 index 0000000000..b6670d58aa --- /dev/null +++ b/packages/opencode/test/fixtures/recordings/session/native-openai-tool-call.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "metadata": { + "name": "session/native-openai-tool-call", + "recordedAt": "2026-05-13T00:27:15.166Z", + "provider": "openai", + "protocol": "openai-responses", + "route": "openai-responses", + "tags": ["opencode", "native", "tool-call"] + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4.1-mini\",\"input\":[{\"role\":\"system\",\"content\":\"Call tools exactly as instructed.\\nYou must call the lookup tool exactly once with query weather. Do not answer in text.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Use lookup.\"}]}],\"tools\":[{\"type\":\"function\",\"name\":\"lookup\",\"description\":\"Lookup data.\",\"parameters\":{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\"}},\"required\":[\"query\"],\"additionalProperties\":false}}],\"tool_choice\":\"required\",\"store\":false,\"prompt_cache_key\":\"session-recorded-native-tool\",\"temperature\":0,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_09354e0087427e4e016a03c56273e481a1842a9d4d6e5c3434\",\"object\":\"response\",\"created_at\":1778632034,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":\"session-recorded-native-tool\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"required\",\"tools\":[{\"type\":\"function\",\"description\":\"Lookup data.\",\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\"}},\"required\":[\"query\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_09354e0087427e4e016a03c56273e481a1842a9d4d6e5c3434\",\"object\":\"response\",\"created_at\":1778632034,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":\"session-recorded-native-tool\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"required\",\"tools\":[{\"type\":\"function\",\"description\":\"Lookup data.\",\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\"}},\"required\":[\"query\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_0bqJ0EdThTwv5g1VILLkf9bo\",\"name\":\"lookup\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"obfuscation\":\"sDHc7xGP1uQu4v\",\"output_index\":0,\"sequence_number\":3}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"query\",\"item_id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"obfuscation\":\"wGG9bOcTCVa\",\"output_index\":0,\"sequence_number\":4}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"obfuscation\":\"i3uIOqQeUw5x4\",\"output_index\":0,\"sequence_number\":5}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"weather\",\"item_id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"obfuscation\":\"Y6emvEwAT\",\"output_index\":0,\"sequence_number\":6}\n\nevent: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"obfuscation\":\"e5oTX3Ry6hrVEC\",\"output_index\":0,\"sequence_number\":7}\n\nevent: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"query\\\":\\\"weather\\\"}\",\"item_id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"output_index\":0,\"sequence_number\":8}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"query\\\":\\\"weather\\\"}\",\"call_id\":\"call_0bqJ0EdThTwv5g1VILLkf9bo\",\"name\":\"lookup\"},\"output_index\":0,\"sequence_number\":9}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_09354e0087427e4e016a03c56273e481a1842a9d4d6e5c3434\",\"object\":\"response\",\"created_at\":1778632034,\"status\":\"completed\",\"background\":false,\"completed_at\":1778632034,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"moderation\":null,\"output\":[{\"id\":\"fc_09354e0087427e4e016a03c562e5e881a1af75085c0cd7b52f\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"query\\\":\\\"weather\\\"}\",\"call_id\":\"call_0bqJ0EdThTwv5g1VILLkf9bo\",\"name\":\"lookup\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":\"session-recorded-native-tool\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"required\",\"tools\":[{\"type\":\"function\",\"description\":\"Lookup data.\",\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\"}},\"required\":[\"query\"],\"additionalProperties\":false},\"strict\":true}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":72,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":6,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":78},\"user\":null,\"metadata\":{}},\"sequence_number\":10}\n\n" + } + } + ] +} diff --git a/packages/opencode/test/session/llm-native-recorded.test.ts b/packages/opencode/test/session/llm-native-recorded.test.ts new file mode 100644 index 0000000000..b79e12957d --- /dev/null +++ b/packages/opencode/test/session/llm-native-recorded.test.ts @@ -0,0 +1,173 @@ +import { NodeFileSystem } from "@effect/platform-node" +import { HttpRecorder } from "@opencode-ai/http-recorder" +import { describe, expect, test } from "bun:test" +import { tool } from "ai" +import { Effect, Layer, Stream } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import path from "node:path" +import z from "zod" +import { Auth } from "@/auth" +import { Config } from "@/config/config" +import { AppRuntime } from "@/effect/app-runtime" +import { attach } from "@/effect/run-service" +import { Plugin } from "@/plugin" +import { Provider } from "@/provider/provider" +import { ModelID, ProviderID } from "@/provider/schema" +import { Filesystem } from "@/util/filesystem" +import { LLMClient, RequestExecutor } from "@opencode-ai/llm/route" +import type { Agent } from "../../src/agent/agent" +import { WithInstance } from "../../src/project/with-instance" +import { LLM } from "../../src/session/llm" +import { MessageV2 } from "../../src/session/message-v2" +import { MessageID, SessionID } from "../../src/session/schema" +import type { ModelsDev } from "../../src/provider/models" +import { tmpdir } from "../fixture/fixture" + +const CASSETTE = "session/native-openai-tool-call" +const FIXTURES_DIR = path.join(import.meta.dir, "../fixtures/recordings") +const OPENAI_API_KEY = process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY + +const shouldRecord = process.env.RECORD === "true" +const canRun = shouldRecord + ? Boolean(OPENAI_API_KEY) + : HttpRecorder.hasCassetteSync(CASSETTE, { directory: FIXTURES_DIR }) +const recordedTest = canRun ? test : test.skip + +async function loadFixture(providerID: string, modelID: string) { + const data = await Filesystem.readJson>( + path.join(import.meta.dir, "../tool/fixtures/models-api.json"), + ) + const provider = data[providerID] + if (!provider) throw new Error(`Missing provider in fixture: ${providerID}`) + const model = provider.models[modelID] + if (!model) throw new Error(`Missing model in fixture: ${modelID}`) + return model +} + +async function getModel(providerID: ProviderID, modelID: ModelID) { + return AppRuntime.runPromise( + Effect.gen(function* () { + const provider = yield* Provider.Service + return yield* provider.getModel(providerID, modelID) + }), + ) +} + +const openAIConfig = (model: ModelsDev.Provider["models"][string]): Partial => ({ + enabled_providers: ["openai"], + provider: { + openai: { + name: "OpenAI", + env: ["OPENAI_API_KEY"], + npm: "@ai-sdk/openai", + api: "https://api.openai.com/v1", + models: { + [model.id]: JSON.parse(JSON.stringify(model)) as NonNullable< + NonNullable[string]["models"] + >[string], + }, + options: { + apiKey: OPENAI_API_KEY ?? "fixture-openai-key", + baseURL: "https://api.openai.com/v1", + }, + }, + }, +}) + +function recordedLLMLayer() { + const cassetteService = HttpRecorder.Cassette.fileSystem({ directory: FIXTURES_DIR }).pipe( + Layer.provide(NodeFileSystem.layer), + ) + const recorder = HttpRecorder.recordingLayer(CASSETTE, { + mode: shouldRecord ? "record" : "replay", + metadata: { + provider: "openai", + protocol: "openai-responses", + route: "openai-responses", + tags: ["opencode", "native", "tool-call"], + }, + }).pipe(Layer.provide(FetchHttpClient.layer)) + const executor = RequestExecutor.layer.pipe(Layer.provide(recorder)) + const client = LLMClient.layer.pipe(Layer.provide(executor)) + + return LLM.layer.pipe( + Layer.provide(Auth.defaultLayer), + Layer.provide(Config.defaultLayer), + Layer.provide(Provider.defaultLayer), + Layer.provide(Plugin.defaultLayer), + Layer.provide(client), + Layer.provide(cassetteService), + ) +} + +async function collect(layer: Layer.Layer, input: LLM.StreamInput) { + return Array.from( + await Effect.runPromise( + attach(LLM.Service.use((svc) => svc.stream(input).pipe(Stream.runCollect))).pipe(Effect.provide(layer)), + ), + ) +} + +describe("session.llm native recorded", () => { + recordedTest("uses real RequestExecutor with HTTP recorder for native OpenAI tools", async () => { + const model = await loadFixture("openai", "gpt-4.1-mini") + let executed: unknown + + await using tmp = await tmpdir({ config: openAIConfig(model) }) + + await WithInstance.provide({ + directory: tmp.path, + fn: async () => { + const previous = process.env.OPENCODE_LLM_RUNTIME + process.env.OPENCODE_LLM_RUNTIME = "native" + try { + const sessionID = SessionID.make("session-recorded-native-tool") + const agent = { + name: "test", + mode: "primary", + prompt: "Call tools exactly as instructed.", + options: {}, + permission: [{ permission: "*", pattern: "*", action: "allow" }], + temperature: 0, + } satisfies Agent.Info + + const resolved = await getModel(ProviderID.openai, ModelID.make(model.id)) + const events = await collect(recordedLLMLayer(), { + user: { + id: MessageID.make("msg_user-recorded-native-tool"), + sessionID, + role: "user", + time: { created: 0 }, + agent: agent.name, + model: { providerID: ProviderID.make("openai"), modelID: ModelID.make(model.id) }, + } satisfies MessageV2.User, + sessionID, + model: resolved, + agent, + system: ["You must call the lookup tool exactly once with query weather. Do not answer in text."], + messages: [{ role: "user", content: "Use lookup." }], + toolChoice: "required", + tools: { + lookup: tool({ + description: "Lookup data.", + inputSchema: z.object({ query: z.string() }), + execute: async (args, options) => { + executed = { args, toolCallId: options.toolCallId } + return { output: "looked up" } + }, + }), + }, + }) + + expect(events.filter((event) => event.type === "step-finish")).toHaveLength(1) + expect(events.filter((event) => event.type === "finish")).toHaveLength(1) + expect(events.some((event) => event.type === "tool-result")).toBe(true) + expect(executed).toMatchObject({ args: { query: "weather" }, toolCallId: expect.any(String) }) + } finally { + if (previous === undefined) delete process.env.OPENCODE_LLM_RUNTIME + else process.env.OPENCODE_LLM_RUNTIME = previous + } + }, + }) + }) +})