feat(native-llm): route Anthropic API-key models through native runtime (#28271)

This commit is contained in:
Kit Langton
2026-05-19 08:20:27 -04:00
committed by GitHub
parent cb15b3ad84
commit 6618e2bce2
7 changed files with 355 additions and 297 deletions

View File

@@ -37,13 +37,16 @@ type StreamInput = {
}
export function status(input: Pick<StreamInput, "model" | "provider" | "auth">): RuntimeStatus {
if (input.model.providerID !== "openai" && !input.model.providerID.startsWith("opencode"))
return { type: "unsupported", reason: "provider is not openai or opencode" }
if (input.model.api.npm !== "@ai-sdk/openai") return { type: "unsupported", reason: "provider package is not OpenAI" }
const providerID = input.model.providerID
if (providerID !== "openai" && providerID !== "anthropic" && !providerID.startsWith("opencode"))
return { type: "unsupported", reason: "provider is not openai, opencode, or anthropic" }
const npm = input.model.api.npm
if (npm !== "@ai-sdk/openai" && npm !== "@ai-sdk/anthropic")
return { type: "unsupported", reason: "provider package is not OpenAI or Anthropic" }
if (input.auth?.type === "oauth") return { type: "unsupported", reason: "OAuth auth is not supported" }
const apiKey = typeof input.provider.options.apiKey === "string" ? input.provider.options.apiKey : input.provider.key
if (!apiKey) return { type: "unsupported", reason: "OpenAI API key is not configured" }
if (!apiKey) return { type: "unsupported", reason: "API key is not configured" }
return {
type: "supported",

View File

@@ -0,0 +1,53 @@
{
"version": 1,
"metadata": {
"name": "session/native-anthropic-tool-loop",
"recordedAt": "2026-05-19T01:40:12.788Z",
"provider": "anthropic",
"protocol": "anthropic-messages",
"route": "anthropic-messages",
"tags": [
"opencode",
"native",
"tool-loop"
]
},
"interactions": [
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.anthropic.com/v1/messages",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Answer using tools when appropriate.\\nUse the get_weather tool exactly once to look up Paris, then reply with exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"input_schema\":{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"max_tokens\":32000,\"temperature\":0}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream; charset=utf-8"
},
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01KSRzhxWxF38x5yYVYvktbc\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":622,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":54,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01A8pEqifk2HVQfq1ZDNP6iY\",\"name\":\"get_weather\",\"input\":{},\"caller\":{\"type\":\"direct\"}} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"city\\\": \\\"P\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"aris\\\"}\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":622,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":54} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
}
},
{
"transport": "http",
"request": {
"method": "POST",
"url": "https://api.anthropic.com/v1/messages",
"headers": {
"content-type": "application/json"
},
"body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Answer using tools when appropriate.\\nUse the get_weather tool exactly once to look up Paris, then reply with exactly: Paris is sunny.\",\"cache_control\":{\"type\":\"ephemeral\"}}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"What is the weather in Paris?\",\"cache_control\":{\"type\":\"ephemeral\"}}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01A8pEqifk2HVQfq1ZDNP6iY\",\"name\":\"get_weather\",\"input\":{\"city\":{}}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_01A8pEqifk2HVQfq1ZDNP6iY\",\"content\":\"{\\\"temperature\\\":22,\\\"condition\\\":\\\"sunny\\\"}\"}]}],\"tools\":[{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"input_schema\":{\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"additionalProperties\":false},\"cache_control\":{\"type\":\"ephemeral\"}}],\"stream\":true,\"max_tokens\":32000,\"temperature\":0}"
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream; charset=utf-8"
},
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01UyghbuSVecMVozDny14vCD\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":697,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\" is sunny.\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":697,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":7} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n"
}
}
]
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1,7 +1,7 @@
import { NodeFileSystem } from "@effect/platform-node"
import { HttpRecorder, Redactor } from "@opencode-ai/http-recorder"
import { describe, expect } from "bun:test"
import { tool } from "ai"
import { tool, type ModelMessage, type JSONValue } from "ai"
import { Effect, Layer, Stream } from "effect"
import { FetchHttpClient } from "effect/unstable/http"
import path from "node:path"
@@ -12,6 +12,7 @@ import { Plugin } from "@/plugin"
import { Provider } from "@/provider/provider"
import { ModelID, ProviderID } from "@/provider/schema"
import { Filesystem } from "@/util/filesystem"
import { LLMEvent, LLMResponse } from "@opencode-ai/llm"
import { LLMClient, RequestExecutor } from "@opencode-ai/llm/route"
import { RuntimeFlags } from "@/effect/runtime-flags"
import type { Agent } from "../../src/agent/agent"
@@ -22,22 +23,105 @@ import type { ModelsDev } from "@opencode-ai/core/models-dev"
import { TestInstance } from "../fixture/fixture"
import { testEffect } from "../lib/effect"
const OPENAI_CASSETTE = "session/native-openai-tool-call"
const ZEN_CASSETTE = "session/native-zen-tool-call"
const FIXTURES_DIR = path.join(import.meta.dir, "../fixtures/recordings")
const OPENAI_API_KEY = process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY
const CONSOLE_TOKEN = process.env.OPENCODE_RECORD_CONSOLE_TOKEN
const ZEN_ORG_ID = process.env.OPENCODE_RECORD_ZEN_ORG_ID
const ZEN_API_URL =
process.env.OPENCODE_RECORD_ZEN_API_URL ?? "https://console.opencode.ai/proxy/connections/fixture/v1"
const zenURL = (connection: string) => `https://console.opencode.ai/proxy/connections/${connection}/v1`
type ProviderSpec = {
readonly providerID: ProviderID
readonly modelID: string
readonly cassette: string
readonly protocol: string
readonly tags: ReadonlyArray<string>
readonly canRecord: boolean
readonly config: (model: ModelsDev.Provider["models"][string]) => Partial<Config.Info>
}
const cloneModel = (model: ModelsDev.Provider["models"][string]) =>
structuredClone(model) as NonNullable<NonNullable<Config.Info["provider"]>[string]["models"]>[string]
const PROVIDERS = {
openai: {
providerID: ProviderID.openai,
modelID: "gpt-4.1-mini",
cassette: "session/native-openai-tool-loop",
protocol: "openai-responses",
tags: ["opencode", "native", "tool-loop"],
canRecord: Boolean(process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY),
config: (model) => ({
enabled_providers: ["openai"],
provider: {
openai: {
name: "OpenAI",
env: ["OPENAI_API_KEY"],
npm: "@ai-sdk/openai",
api: "https://api.openai.com/v1",
models: { [model.id]: cloneModel(model) },
options: {
apiKey: process.env.OPENCODE_RECORD_OPENAI_API_KEY ?? process.env.OPENAI_API_KEY ?? "fixture-openai-key",
baseURL: "https://api.openai.com/v1",
},
},
},
}),
},
opencode: {
providerID: ProviderID.opencode,
modelID: "gpt-5.2-codex",
cassette: "session/native-zen-tool-loop",
protocol: "openai-responses",
tags: ["opencode", "zen", "native", "tool-loop"],
canRecord: Boolean(process.env.OPENCODE_RECORD_CONSOLE_TOKEN && process.env.OPENCODE_RECORD_ZEN_ORG_ID),
config: (model) => ({
enabled_providers: ["opencode"],
provider: {
opencode: {
name: "OpenCode Zen",
env: ["OPENCODE_CONSOLE_TOKEN"],
npm: "@ai-sdk/openai-compatible",
// The connection slug is account-specific; the cassette redactor
// normalizes it to {connection} for replay. Set during recording.
api: zenURL(process.env.OPENCODE_RECORD_ZEN_CONNECTION ?? "fixture"),
models: { [model.id]: cloneModel(model) },
options: {
apiKey: process.env.OPENCODE_RECORD_CONSOLE_TOKEN ?? "fixture-console-token",
headers: { "x-org-id": process.env.OPENCODE_RECORD_ZEN_ORG_ID ?? "fixture-org" },
},
},
},
}),
},
anthropic: {
providerID: ProviderID.anthropic,
modelID: "claude-haiku-4-5-20251001",
cassette: "session/native-anthropic-tool-loop",
protocol: "anthropic-messages",
tags: ["opencode", "native", "tool-loop"],
canRecord: Boolean(process.env.OPENCODE_RECORD_ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_API_KEY),
config: (model) => ({
enabled_providers: ["anthropic"],
provider: {
anthropic: {
name: "Anthropic",
env: ["ANTHROPIC_API_KEY"],
npm: "@ai-sdk/anthropic",
api: "https://api.anthropic.com/v1",
models: { [model.id]: cloneModel(model) },
options: {
apiKey:
process.env.OPENCODE_RECORD_ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_API_KEY ?? "fixture-anthropic-key",
baseURL: "https://api.anthropic.com/v1",
},
},
},
}),
},
} satisfies Record<string, ProviderSpec>
const shouldRecord = process.env.RECORD === "true"
const canRunOpenAI = shouldRecord
? Boolean(OPENAI_API_KEY)
: HttpRecorder.hasCassetteSync(OPENAI_CASSETTE, { directory: FIXTURES_DIR })
const canRunZen = shouldRecord
? Boolean(CONSOLE_TOKEN && ZEN_ORG_ID)
: HttpRecorder.hasCassetteSync(ZEN_CASSETTE, { directory: FIXTURES_DIR })
const canRun = (spec: ProviderSpec) =>
shouldRecord ? spec.canRecord : HttpRecorder.hasCassetteSync(spec.cassette, { directory: FIXTURES_DIR })
async function loadFixture(providerID: string, modelID: string) {
const data = await Filesystem.readJson<Record<string, ModelsDev.Provider>>(
@@ -50,234 +134,140 @@ async function loadFixture(providerID: string, modelID: string) {
return model
}
const openAIConfig = (model: ModelsDev.Provider["models"][string]): Partial<Config.Info> => ({
enabled_providers: ["openai"],
provider: {
openai: {
name: "OpenAI",
env: ["OPENAI_API_KEY"],
npm: "@ai-sdk/openai",
api: "https://api.openai.com/v1",
models: {
[model.id]: JSON.parse(JSON.stringify(model)) as NonNullable<
NonNullable<Config.Info["provider"]>[string]["models"]
>[string],
},
options: {
apiKey: OPENAI_API_KEY ?? "fixture-openai-key",
baseURL: "https://api.openai.com/v1",
},
},
},
})
const zenConfig = (model: ModelsDev.Provider["models"][string]): Partial<Config.Info> => ({
enabled_providers: ["opencode"],
provider: {
opencode: {
name: "OpenCode Zen",
env: ["OPENCODE_CONSOLE_TOKEN"],
npm: "@ai-sdk/openai-compatible",
api: ZEN_API_URL,
models: {
[model.id]: JSON.parse(JSON.stringify(model)) as NonNullable<
NonNullable<Config.Info["provider"]>[string]["models"]
>[string],
},
options: {
apiKey: CONSOLE_TOKEN ?? "fixture-console-token",
headers: {
"x-org-id": ZEN_ORG_ID ?? "fixture-org",
},
},
},
},
})
function recordedNativeLLMLayer(cassette: string, metadata: Record<string, unknown>) {
const cassetteService = HttpRecorder.Cassette.fileSystem({ directory: FIXTURES_DIR }).pipe(
Layer.provide(NodeFileSystem.layer),
)
function recordedNativeLLMLayer(spec: ProviderSpec) {
// Only the HTTP client is recorded; RequestExecutor and the opencode LLM stack remain real.
const recorder = HttpRecorder.recordingLayer(cassette, {
mode: shouldRecord ? "record" : "replay",
metadata,
redactor: Redactor.compose(
Redactor.defaults({
url: {
transform: (url) => url.replace(/\/proxy\/connections\/[^/]+\/v1/, "/proxy/connections/{connection}/v1"),
},
}),
{
response: (snapshot) => ({ ...snapshot, body: snapshot.body.replace(/wrk_[A-Z0-9]+/g, "wrk_redacted") }),
},
const recordedClient = LLMClient.layer.pipe(
Layer.provide(RequestExecutor.layer),
Layer.provide(
HttpRecorder.recordingLayer(spec.cassette, {
mode: shouldRecord ? "record" : "replay",
metadata: { provider: spec.providerID, protocol: spec.protocol, route: spec.protocol, tags: spec.tags },
redactor: Redactor.compose(
Redactor.defaults({
url: {
transform: (url) => url.replace(/\/proxy\/connections\/[^/]+\/v1/, "/proxy/connections/{connection}/v1"),
},
}),
{
response: (snapshot) => ({ ...snapshot, body: snapshot.body.replace(/wrk_[A-Z0-9]+/g, "wrk_redacted") }),
},
),
}).pipe(Layer.provide(FetchHttpClient.layer)),
),
}).pipe(Layer.provide(FetchHttpClient.layer))
const executor = RequestExecutor.layer.pipe(Layer.provide(recorder))
const client = LLMClient.layer.pipe(Layer.provide(executor))
const providerLayer = Provider.defaultLayer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Plugin.defaultLayer),
)
const llmLayer = LLM.layer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Provider.defaultLayer),
Layer.provide(Plugin.defaultLayer),
Layer.provide(client),
Layer.provide(cassetteService),
Layer.provide(RuntimeFlags.layer({ experimentalNativeLlm: true })),
)
return Layer.mergeAll(providerLayer, llmLayer)
return Layer.mergeAll(
Provider.defaultLayer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Plugin.defaultLayer),
),
LLM.layer.pipe(
Layer.provide(Auth.defaultLayer),
Layer.provide(Config.defaultLayer),
Layer.provide(Provider.defaultLayer),
Layer.provide(Plugin.defaultLayer),
Layer.provide(recordedClient),
Layer.provide(HttpRecorder.Cassette.fileSystem({ directory: FIXTURES_DIR }).pipe(Layer.provide(NodeFileSystem.layer))),
Layer.provide(RuntimeFlags.layer({ experimentalNativeLlm: true })),
),
)
}
const openAIIt = testEffect(
recordedNativeLLMLayer(OPENAI_CASSETTE, {
provider: "openai",
protocol: "openai-responses",
route: "openai-responses",
tags: ["opencode", "native", "tool-call"],
}),
)
const zenIt = testEffect(
recordedNativeLLMLayer(ZEN_CASSETTE, {
provider: "opencode",
protocol: "openai-responses",
route: "openai-responses",
tags: ["opencode", "zen", "native", "tool-call"],
}),
)
const recordedOpenAIInstance = canRunOpenAI ? openAIIt.instance : openAIIt.instance.skip
const recordedZenInstance = canRunZen ? zenIt.instance : zenIt.instance.skip
const writeConfig = (
directory: string,
model: ModelsDev.Provider["models"][string],
config: (model: ModelsDev.Provider["models"][string]) => Partial<Config.Info> = openAIConfig,
) =>
const writeConfig = (directory: string, spec: ProviderSpec, model: ModelsDev.Provider["models"][string]) =>
Effect.promise(() =>
Bun.write(
path.join(directory, "opencode.json"),
JSON.stringify({ $schema: "https://opencode.ai/config.json", ...config(model) }),
JSON.stringify({ $schema: "https://opencode.ai/config.json", ...spec.config(model) }),
),
)
const getModel = (providerID: ProviderID, modelID: ModelID) =>
Effect.gen(function* () {
const provider = yield* Provider.Service
return yield* provider.getModel(providerID, modelID)
})
const collect = (input: LLM.StreamInput) =>
Effect.gen(function* () {
const llm = yield* LLM.Service
return Array.from(yield* llm.stream(input).pipe(Stream.runCollect))
})
describe("session.llm native recorded", () => {
recordedOpenAIInstance("uses real RequestExecutor with HTTP recorder for native OpenAI tools", () =>
Effect.gen(function* () {
const test = yield* TestInstance
const model = yield* Effect.promise(() => loadFixture("openai", "gpt-4.1-mini"))
yield* writeConfig(test.directory, model)
const WEATHER_RESULT = { temperature: 22, condition: "sunny" } as const
const WEATHER_SYSTEM =
"Use the get_weather tool exactly once to look up Paris, then reply with exactly: Paris is sunny."
const WEATHER_USER = "What is the weather in Paris?"
const sessionID = SessionID.make("session-recorded-native-tool")
const agent = {
name: "test",
mode: "primary",
prompt: "Call tools exactly as instructed.",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0,
} satisfies Agent.Info
const resolved = yield* getModel(ProviderID.openai, ModelID.make(model.id))
let executed: unknown
const events = yield* collect({
user: {
id: MessageID.make("msg_user-recorded-native-tool"),
sessionID,
role: "user",
time: { created: 0 },
agent: agent.name,
model: { providerID: ProviderID.make("openai"), modelID: ModelID.make(model.id) },
} satisfies MessageV2.User,
sessionID,
model: resolved,
agent,
system: ["You must call the lookup tool exactly once with query weather. Do not answer in text."],
messages: [{ role: "user", content: "Use lookup." }],
toolChoice: "required",
tools: {
lookup: tool({
description: "Lookup data.",
inputSchema: z.object({ query: z.string() }),
execute: async (args, options) => {
executed = { args, toolCallId: options.toolCallId }
return { output: "looked up" }
},
}),
},
})
expect(events.filter((event) => event.type === "step-finish")).toHaveLength(1)
expect(events.filter((event) => event.type === "finish")).toHaveLength(1)
expect(events.some((event) => event.type === "tool-result")).toBe(true)
expect(executed).toMatchObject({ args: { query: "weather" }, toolCallId: expect.any(String) })
}),
)
recordedZenInstance("uses console-managed Zen config with native OpenAI-compatible tools", () =>
Effect.gen(function* () {
const test = yield* TestInstance
const model = yield* Effect.promise(() => loadFixture("opencode", "gpt-5.2-codex"))
yield* writeConfig(test.directory, model, zenConfig)
const sessionID = SessionID.make("session-recorded-native-zen-tool")
const agent = {
name: "test",
mode: "primary",
prompt: "Call tools exactly as instructed.",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
} satisfies Agent.Info
const resolved = yield* getModel(ProviderID.opencode, ModelID.make(model.id))
let executed: unknown
const events = yield* collect({
user: {
id: MessageID.make("msg_user-recorded-native-zen-tool"),
sessionID,
role: "user",
time: { created: 0 },
agent: agent.name,
model: { providerID: ProviderID.opencode, modelID: ModelID.make(model.id) },
} satisfies MessageV2.User,
sessionID,
model: resolved,
agent,
system: ["You must call the lookup tool exactly once with query weather. Do not answer in text."],
messages: [{ role: "user", content: "Use lookup." }],
toolChoice: "required",
tools: {
lookup: tool({
description: "Lookup data.",
inputSchema: z.object({ query: z.string() }),
execute: async (args, options) => {
executed = { args, toolCallId: options.toolCallId }
return { output: "looked up" }
},
}),
},
})
expect(events.filter((event) => event.type === "step-finish")).toHaveLength(1)
expect(events.filter((event) => event.type === "finish")).toHaveLength(1)
expect(events.some((event) => event.type === "tool-result")).toBe(true)
expect(executed).toMatchObject({ args: { query: "weather" }, toolCallId: expect.any(String) })
}),
)
const weatherTool = tool({
description: "Get the current weather for a city.",
inputSchema: z.object({ city: z.string() }),
execute: async () => WEATHER_RESULT,
})
const toolRoundtrip = (
call: { readonly id: string; readonly name: string; readonly input: unknown },
result: JSONValue,
): ModelMessage[] => [
{ role: "assistant", content: [{ type: "tool-call", toolCallId: call.id, toolName: call.name, input: call.input }] },
{
role: "tool",
content: [{ type: "tool-result", toolCallId: call.id, toolName: call.name, output: { type: "json", value: result } }],
},
]
const driveToolLoop = (spec: ProviderSpec) =>
Effect.gen(function* () {
const test = yield* TestInstance
const model = yield* Effect.promise(() => loadFixture(spec.providerID, spec.modelID))
yield* writeConfig(test.directory, spec, model)
const sessionID = SessionID.make(`session-recorded-${spec.providerID}-loop`)
const modelID = ModelID.make(model.id)
const agent = {
name: "test",
mode: "primary",
prompt: "Answer using tools when appropriate.",
options: {},
permission: [{ permission: "*", pattern: "*", action: "allow" }],
temperature: 0,
} satisfies Agent.Info
const provider = yield* Provider.Service
const resolved = yield* provider.getModel(spec.providerID, modelID)
const userMessage = { role: "user", content: WEATHER_USER } satisfies ModelMessage
const base = {
user: {
id: MessageID.make(`msg_user-recorded-${spec.providerID}-loop`),
sessionID,
role: "user",
time: { created: 0 },
agent: agent.name,
model: { providerID: spec.providerID, modelID },
} satisfies MessageV2.User,
sessionID,
model: resolved,
agent,
system: [WEATHER_SYSTEM],
tools: { get_weather: weatherTool },
}
const turn1 = yield* collect({ ...base, messages: [userMessage] })
const toolCall = turn1.find(LLMEvent.is.toolCall)
expect(toolCall).toBeDefined()
expect(turn1.find(LLMEvent.is.toolResult)).toBeDefined()
expect(toolCall!.name).toBe("get_weather")
expect(toolCall!.input).toMatchObject({ city: expect.stringMatching(/Paris/i) })
expect(turn1.filter(LLMEvent.is.stepFinish)).toHaveLength(1)
const turn2 = yield* collect({
...base,
messages: [userMessage, ...toolRoundtrip(toolCall!, WEATHER_RESULT)],
})
expect(LLMResponse.text({ events: turn2 })).toMatch(/Paris is sunny/i)
expect(turn2.filter(LLMEvent.is.finish)).toHaveLength(1)
expect(turn2.filter(LLMEvent.is.toolCall)).toHaveLength(0)
})
describe("session.llm native recorded", () => {
for (const [name, spec] of Object.entries(PROVIDERS)) {
const it = testEffect(recordedNativeLLMLayer(spec))
const instance = canRun(spec) ? it.instance : it.instance.skip
instance(`${name}: drives a tool loop to a final text answer`, () => driveToolLoop(spec))
}
})

View File

@@ -262,11 +262,11 @@ describe("session.llm-native.request", () => {
})
expect(
LLMNativeRuntime.status({
model: { ...baseModel, providerID: ProviderID.make("anthropic") },
provider: { ...providerInfo, id: ProviderID.make("anthropic") },
model: { ...baseModel, providerID: ProviderID.make("google") },
provider: { ...providerInfo, id: ProviderID.make("google") },
auth: undefined,
}),
).toEqual({ type: "unsupported", reason: "provider is not openai or opencode" })
).toEqual({ type: "unsupported", reason: "provider is not openai, opencode, or anthropic" })
expect(
LLMNativeRuntime.status({
model: baseModel,
@@ -277,11 +277,11 @@ describe("session.llm-native.request", () => {
expect(
LLMNativeRuntime.status({
model: { ...baseModel, api: { ...baseModel.api, npm: "@ai-sdk/anthropic" } },
model: { ...baseModel, api: { ...baseModel.api, npm: "@ai-sdk/google" } },
provider: providerInfo,
auth: undefined,
}),
).toEqual({ type: "unsupported", reason: "provider package is not OpenAI" })
).toEqual({ type: "unsupported", reason: "provider package is not OpenAI or Anthropic" })
expect(
LLMNativeRuntime.status({
@@ -289,7 +289,27 @@ describe("session.llm-native.request", () => {
provider: { ...providerInfo, options: {} },
auth: undefined,
}),
).toEqual({ type: "unsupported", reason: "OpenAI API key is not configured" })
).toEqual({ type: "unsupported", reason: "API key is not configured" })
})
test("enables native runtime for Anthropic API-key models", () => {
expect(
LLMNativeRuntime.status({
model: {
...baseModel,
providerID: ProviderID.make("anthropic"),
api: { ...baseModel.api, npm: "@ai-sdk/anthropic", url: "https://api.anthropic.com/v1" },
},
provider: {
...providerInfo,
id: ProviderID.make("anthropic"),
name: "Anthropic",
env: ["ANTHROPIC_API_KEY"],
options: { apiKey: "test-anthropic-key" },
},
auth: undefined,
}),
).toMatchObject({ type: "supported", apiKey: "test-anthropic-key" })
})
test("prefers console provider api key over stored opencode auth", () => {