From c5b4bbd0a71a6737fbdf86b937cd44f43add1341 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 8 May 2026 17:15:34 -0400 Subject: [PATCH] test(server): harden HttpApi exercise coverage --- .github/workflows/test.yml | 5 + packages/opencode/package.json | 1 + .../opencode/src/control-plane/workspace.ts | 3 +- .../test/control-plane/workspace.test.ts | 35 ++ .../test/server/httpapi-exercise/backend.ts | 55 ++- .../test/server/httpapi-exercise/dsl.ts | 32 +- .../test/server/httpapi-exercise/index.ts | 369 +++++++++++------- .../test/server/httpapi-exercise/report.ts | 2 +- .../test/server/httpapi-exercise/routing.ts | 20 + .../test/server/httpapi-exercise/runner.ts | 96 ++++- .../test/server/httpapi-exercise/types.ts | 8 + 11 files changed, 452 insertions(+), 174 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69a3a1a2d1..f226d3483a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,6 +68,11 @@ jobs: env: OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: ${{ runner.os == 'Windows' && 'true' || 'false' }} + - name: Run HttpApi exerciser gates + if: runner.os == 'Linux' + working-directory: packages/opencode + run: bun run test:httpapi + - name: Publish unit reports if: always() uses: mikepenz/action-junit-report@v6 diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 06c1ac7371..04a9cf0271 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -9,6 +9,7 @@ "typecheck": "tsgo --noEmit", "test": "bun test --timeout 30000", "test:ci": "mkdir -p .artifacts/unit && bun test --timeout 30000 --reporter=junit --reporter-outfile=.artifacts/unit/junit.xml", + "test:httpapi": "bun run script/httpapi-exercise.ts --mode coverage --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode auth --fail-on-missing --fail-on-skip", "build": "bun run script/build.ts", "fix-node-pty": "bun run script/fix-node-pty.ts", "dev": "bun run --conditions=browser ./src/index.ts", diff --git a/packages/opencode/src/control-plane/workspace.ts b/packages/opencode/src/control-plane/workspace.ts index 03640576d6..b30536ec02 100644 --- a/packages/opencode/src/control-plane/workspace.ts +++ b/packages/opencode/src/control-plane/workspace.ts @@ -5,7 +5,6 @@ import { asc } from "drizzle-orm" import { eq } from "drizzle-orm" import { inArray } from "drizzle-orm" import { Project } from "@/project/project" -import { Instance } from "@/project/instance" import { BusEvent } from "@/bus/bus-event" import { GlobalBus } from "@/bus/global" import { Auth } from "@/auth" @@ -646,7 +645,7 @@ export const layer = Layer.effect( // "claim" this session so any future events coming from // the old workspace are ignored - SyncEvent.claim(input.sessionID, input.workspaceID ?? Instance.project.id) + SyncEvent.claim(input.sessionID, input.workspaceID ?? previous.projectID) } } diff --git a/packages/opencode/test/control-plane/workspace.test.ts b/packages/opencode/test/control-plane/workspace.test.ts index e3de9cae71..8333d9573f 100644 --- a/packages/opencode/test/control-plane/workspace.test.ts +++ b/packages/opencode/test/control-plane/workspace.test.ts @@ -848,6 +848,41 @@ describe("workspace CRUD", () => { }) }) + test("sessionWarp detaches to the source project when invoked from a workspace instance", async () => { + await withInstance(async () => { + const projectID = Instance.project.id + await using workspaceTmp = await tmpdir({ git: true }) + const previousType = unique("warp-detach-workspace-instance") + const previous = workspaceInfo(projectID, previousType) + insertWorkspace(previous) + registerAdapter(projectID, previousType, localAdapter(workspaceTmp.path, { createDir: false }).adapter) + const session = await AppRuntime.runPromise(SessionNs.Service.use((svc) => svc.create({}))) + attachSessionToWorkspace(session.id, previous.id) + + const workspaceProjectID = await WithInstance.provide({ + directory: workspaceTmp.path, + fn: async () => { + const id = Instance.project.id + expect(id).not.toBe(projectID) + await warpWorkspaceSession({ workspaceID: null, sessionID: session.id }) + return id + }, + }) + + expect( + Database.use((db) => + db + .select({ workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(eq(SessionTable.id, session.id)) + .get(), + )?.workspaceID, + ).toBeNull() + expect(sessionSequenceOwner(session.id)).toBe(projectID) + expect(sessionSequenceOwner(session.id)).not.toBe(workspaceProjectID) + }) + }) + it.live("sessionWarp syncs previous remote history, replays it, steals, and claims the sequence", () => { const calls: FetchCall[] = [] let historySessionID: SessionID | undefined diff --git a/packages/opencode/test/server/httpapi-exercise/backend.ts b/packages/opencode/test/server/httpapi-exercise/backend.ts index c393383e03..f66d15a3ee 100644 --- a/packages/opencode/test/server/httpapi-exercise/backend.ts +++ b/packages/opencode/test/server/httpapi-exercise/backend.ts @@ -23,13 +23,31 @@ export function call( ) } -export function callAuthProbe(backend: Backend, scenario: ActiveScenario) { - return Effect.promise(async () => - capture( - await app(await runtime(), backend, { auth: { password: "secret" } }).request(toAuthProbeRequest(scenario)), - scenario.capture, - ), - ) +export function callAuthProbe( + backend: Backend, + scenario: ActiveScenario, + credentials: "missing" | "valid" = "missing", +) { + return Effect.promise(async () => { + const controller = new AbortController() + return Promise.race([ + Promise.resolve( + app(await runtime(), backend, { auth: { password: "secret" } }).request( + toAuthProbeRequest(scenario, credentials, controller.signal), + ), + ).then((response) => capture(response, scenario.capture)), + Bun.sleep(1_000).then(() => { + controller.abort("auth probe timed out") + return { + status: 0, + contentType: "", + text: "auth probe timed out", + body: undefined, + timedOut: true, + } + }), + ]) + }) } const appCache: Partial> = {} @@ -78,14 +96,28 @@ function toRequest(scenario: ActiveScenario, ctx: SeededContext) { }) } -function toAuthProbeRequest(scenario: ActiveScenario) { - return new Request(new URL(authProbePath(scenario.path), "http://localhost"), { +function toAuthProbeRequest(scenario: ActiveScenario, credentials: "missing" | "valid", signal: AbortSignal) { + const spec = scenario.authProbe ?? { + path: authProbePath(scenario.path), + body: scenario.method === "GET" ? undefined : {}, + } + const headers = { + ...(spec.body === undefined ? {} : { "content-type": "application/json" }), + ...spec.headers, + ...(credentials === "valid" ? { authorization: basic("opencode", "secret") } : {}), + } + return new Request(new URL(spec.path, "http://localhost"), { method: scenario.method, - headers: scenario.method === "GET" ? undefined : { "content-type": "application/json" }, - body: scenario.method === "GET" ? undefined : JSON.stringify({}), + headers, + body: spec.body === undefined ? undefined : JSON.stringify(spec.body), + signal, }) } +function basic(username: string, password: string) { + return `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}` +} + function authProbePath(path: string) { return path .replace(/\{([^}]+)\}/g, (_match, key: string) => `auth_${key}`) @@ -99,6 +131,7 @@ async function capture(response: Response, mode: CaptureMode): Promise { private readonly state: BuilderState - constructor(method: Method, path: string, name: string) { + constructor(method: Method, path: string, name: string, auth: AuthPolicy) { this.state = { method, path, @@ -25,10 +26,11 @@ class ScenarioBuilder { // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- The unseeded builder state is intentionally undefined until `.seeded(...)` narrows it. seed: () => Effect.succeed(undefined as S), request: (ctx) => ({ path, headers: ctx.headers() }), + authProbe: undefined, capture: "full", mutates: false, reset: true, - auth: "protected", + auth, } } @@ -48,6 +50,10 @@ class ScenarioBuilder { return this.clone({ request }) } + probe(authProbe: RequestSpec) { + return this.clone({ authProbe }) + } + mutating() { return this.clone({ mutates: true }) } @@ -124,7 +130,7 @@ class ScenarioBuilder { } private clone(next: Partial>) { - const builder = new ScenarioBuilder(this.state.method, this.state.path, this.state.name) + const builder = new ScenarioBuilder(this.state.method, this.state.path, this.state.name, this.state.auth) Object.assign(builder.state, this.state, next) return builder } @@ -134,7 +140,7 @@ class ScenarioBuilder { * for `.at(...)` and assertions, giving stateful route tests type-safe setup. */ seeded(seed: (ctx: ScenarioContext) => Effect.Effect) { - const builder = new ScenarioBuilder(this.state.method, this.state.path, this.state.name) + const builder = new ScenarioBuilder(this.state.method, this.state.path, this.state.name, this.state.auth) Object.assign(builder.state, this.state, { seed }) return builder } @@ -151,6 +157,7 @@ class ScenarioBuilder { name: state.name, project: state.project, seed: state.seed, + authProbe: state.authProbe, // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- `.seeded(...)` preserves the paired request/state type inside the builder. request: (ctx, seeded) => state.request({ ...ctx, state: seeded as S }), // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- `.seeded(...)` preserves the paired assertion/state type inside the builder. @@ -164,12 +171,19 @@ class ScenarioBuilder { } } +const routes = (auth: AuthPolicy) => ({ + get: (path: string, name: string) => new ScenarioBuilder("GET", path, name, auth), + post: (path: string, name: string) => new ScenarioBuilder("POST", path, name, auth), + put: (path: string, name: string) => new ScenarioBuilder("PUT", path, name, auth), + patch: (path: string, name: string) => new ScenarioBuilder("PATCH", path, name, auth), + delete: (path: string, name: string) => new ScenarioBuilder("DELETE", path, name, auth), +}) + export const http = { - get: (path: string, name: string) => new ScenarioBuilder("GET", path, name), - post: (path: string, name: string) => new ScenarioBuilder("POST", path, name), - put: (path: string, name: string) => new ScenarioBuilder("PUT", path, name), - patch: (path: string, name: string) => new ScenarioBuilder("PATCH", path, name), - delete: (path: string, name: string) => new ScenarioBuilder("DELETE", path, name), + protected: routes("protected"), + public: routes("public"), + publicBypass: routes("public-bypass"), + ticketBypass: routes("ticket-bypass"), } export const pending = (method: Method, path: string, name: string, reason: string): TodoScenario => ({ diff --git a/packages/opencode/test/server/httpapi-exercise/index.ts b/packages/opencode/test/server/httpapi-exercise/index.ts index 32d9af464b..bc876d9f07 100644 --- a/packages/opencode/test/server/httpapi-exercise/index.ts +++ b/packages/opencode/test/server/httpapi-exercise/index.ts @@ -11,7 +11,7 @@ * so this must never point at a developer's real session database. * * DSL shape: - * - `http.get/post/...` starts a scenario for one OpenAPI route key. + * - `http.protected.get/post/...` starts a scenario for one OpenAPI route key. * - `.seeded(...)` creates typed per-scenario state using Effect helpers on `ctx`. * - `.at(...)` builds the request from that typed state. * - `.json(...)` / `.jsonEffect(...)` assert response shape and optional side effects. @@ -32,7 +32,7 @@ import { exerciseGlobalRoot, } from "./environment" import { color, printHeader, printResults } from "./report" -import { coverageResult, matches, parseOptions, routeKey, routeKeys } from "./routing" +import { coverageResult, parseOptions, routeKey, routeKeys, selectedScenarios } from "./routing" import { runScenario } from "./runner" import { runtime } from "./runtime" import { type Scenario } from "./types" @@ -40,14 +40,14 @@ import { type Scenario } from "./types" void (await import("@opencode-ai/core/util/log")).init({ print: false }) const scenarios: Scenario[] = [ - http + http.protected .get("/global/health", "global.health") .global() .json(200, (body) => { object(body) check(body.healthy === true, "server should report healthy") }), - http + http.protected .get("/global/event", "global.event") .global() .stream() @@ -60,8 +60,8 @@ const scenarios: Scenario[] = [ }), "status", ), - http.get("/global/config", "global.config.get").global().json(), - http + http.protected.get("/global/config", "global.config.get").global().json(), + http.protected .patch("/global/config", "global.config.update") .global() .seeded(() => @@ -86,7 +86,7 @@ const scenarios: Scenario[] = [ }), "status", ), - http + http.protected .post("/global/dispose", "global.dispose") .global() .mutating() @@ -97,23 +97,37 @@ const scenarios: Scenario[] = [ }, "status", ), - http.get("/path", "path.get").json(200, (body, ctx) => { + http.protected.get("/path", "path.get").json(200, (body, ctx) => { object(body) check(body.directory === ctx.directory, "directory should resolve from x-opencode-directory") check(body.worktree === ctx.directory, "worktree should resolve from x-opencode-directory") }), - http.get("/vcs", "vcs.get").json(), - http + http.protected.get("/vcs", "vcs.get").json(), + http.protected.get("/vcs/status", "vcs.status").json(200, array), + http.protected .get("/vcs/diff", "vcs.diff") .at((ctx) => ({ path: "/vcs/diff?mode=git", headers: ctx.headers() })) .json(200, array), - http.get("/command", "command.list").json(200, array, "status"), - http.get("/agent", "app.agents").json(200, array, "status"), - http.get("/skill", "app.skills").json(200, array, "status"), - http.get("/lsp", "lsp.status").json(200, array), - http.get("/formatter", "formatter.status").json(200, array), - http.get("/config", "config.get").json(200, undefined, "status"), - http + http.protected.get("/vcs/diff/raw", "vcs.diff.raw").status( + 200, + (_ctx, result) => + Effect.sync(() => { + check(typeof result.text === "string", "raw VCS diff should return text") + }), + "status", + ), + http.protected + .post("/vcs/apply", "vcs.apply") + .inProject({ git: false }) + .at((ctx) => ({ path: "/vcs/apply", headers: ctx.headers(), body: { patch: "" } })) + .status(400, undefined, "status"), + http.protected.get("/command", "command.list").json(200, array, "status"), + http.protected.get("/agent", "app.agents").json(200, array, "status"), + http.protected.get("/skill", "app.skills").json(200, array, "status"), + http.protected.get("/lsp", "lsp.status").json(200, array), + http.protected.get("/formatter", "formatter.status").json(200, array), + http.protected.get("/config", "config.get").json(200, undefined, "status"), + http.protected .patch("/config", "config.update") .mutating() .at((ctx) => ({ path: "/config", headers: ctx.headers(), body: { username: "httpapi-local" } })) @@ -125,13 +139,13 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .patch("/config", "config.update.invalid") .at((ctx) => ({ path: "/config", headers: ctx.headers(), body: { username: 1 } })) .status(400), - http.get("/config/providers", "config.providers").json(), - http.get("/project", "project.list").json(200, array, "status"), - http.get("/project/current", "project.current").json( + http.protected.get("/config/providers", "config.providers").json(), + http.protected.get("/project", "project.list").json(200, array, "status"), + http.protected.get("/project/current", "project.current").json( 200, (body, ctx) => { object(body) @@ -139,7 +153,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .patch("/project/{projectID}", "project.update") .mutating() .seeded((ctx) => ctx.project()) @@ -160,7 +174,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/project/git/init", "project.initGit") .mutating() .inProject({ git: false }) @@ -173,9 +187,9 @@ const scenarios: Scenario[] = [ }, "status", ), - http.get("/provider", "provider.list").json(), - http.get("/provider/auth", "provider.auth").json(), - http + http.protected.get("/provider", "provider.list").json(), + http.protected.get("/provider/auth", "provider.auth").json(), + http.protected .post("/provider/{providerID}/oauth/authorize", "provider.oauth.authorize") .at((ctx) => ({ path: route("/provider/{providerID}/oauth/authorize", { providerID: "httpapi" }), @@ -183,7 +197,7 @@ const scenarios: Scenario[] = [ body: { method: "bad" }, })) .status(400), - http + http.protected .post("/provider/{providerID}/oauth/callback", "provider.oauth.callback") .at((ctx) => ({ path: route("/provider/{providerID}/oauth/callback", { providerID: "httpapi" }), @@ -191,8 +205,8 @@ const scenarios: Scenario[] = [ body: { method: "bad" }, })) .status(400), - http.get("/permission", "permission.list").json(200, array), - http + http.protected.get("/permission", "permission.list").json(200, array), + http.protected .post("/permission/{requestID}/reply", "permission.reply.invalid") .at((ctx) => ({ path: route("/permission/{requestID}/reply", { requestID: "per_httpapi" }), @@ -200,7 +214,7 @@ const scenarios: Scenario[] = [ body: { reply: "bad" }, })) .status(400), - http + http.protected .post("/permission/{requestID}/reply", "permission.reply") .at((ctx) => ({ path: route("/permission/{requestID}/reply", { requestID: "per_httpapi" }), @@ -210,8 +224,8 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "permission reply should return true even when request is no longer pending") }), - http.get("/question", "question.list").json(200, array), - http + http.protected.get("/question", "question.list").json(200, array), + http.protected .post("/question/{requestID}/reply", "question.reply.invalid") .at((ctx) => ({ path: route("/question/{requestID}/reply", { requestID: "que_httpapi_reply" }), @@ -219,7 +233,7 @@ const scenarios: Scenario[] = [ body: { answers: "Yes" }, })) .status(400), - http + http.protected .post("/question/{requestID}/reply", "question.reply") .at((ctx) => ({ path: route("/question/{requestID}/reply", { requestID: "que_httpapi_reply" }), @@ -229,7 +243,7 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "question reply should return true even when request is no longer pending") }), - http + http.protected .post("/question/{requestID}/reject", "question.reject") .at((ctx) => ({ path: route("/question/{requestID}/reject", { requestID: "que_httpapi_reject" }), @@ -238,12 +252,12 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "question reject should return true even when request is no longer pending") }), - http + http.protected .get("/file", "file.list") .seeded((ctx) => ctx.file("hello.txt", "hello\n")) .at((ctx) => ({ path: `/file?${new URLSearchParams({ path: "." })}`, headers: ctx.headers() })) .json(200, array), - http + http.protected .get("/file/content", "file.read") .seeded((ctx) => ctx.file("hello.txt", "hello\n")) .at((ctx) => ({ path: `/file/content?${new URLSearchParams({ path: "hello.txt" })}`, headers: ctx.headers() })) @@ -251,20 +265,20 @@ const scenarios: Scenario[] = [ object(body) check(body.content === "hello", `content should match seeded file: ${JSON.stringify(body)}`) }), - http + http.protected .get("/file/content", "file.read.missing") .at((ctx) => ({ path: `/file/content?${new URLSearchParams({ path: "missing.txt" })}`, headers: ctx.headers() })) .json(200, (body) => { object(body) check(body.type === "text" && body.content === "", "missing file content should return an empty text result") }), - http.get("/file/status", "file.status").json(200, array), - http + http.protected.get("/file/status", "file.status").json(200, array), + http.protected .get("/find", "find.text") .seeded((ctx) => ctx.file("hello.txt", "hello\n")) .at((ctx) => ({ path: `/find?${new URLSearchParams({ pattern: "hello" })}`, headers: ctx.headers() })) .json(200, array), - http + http.protected .get("/find/file", "find.files") .seeded((ctx) => ctx.file("hello.txt", "hello\n")) .at((ctx) => ({ @@ -272,12 +286,12 @@ const scenarios: Scenario[] = [ headers: ctx.headers(), })) .json(200, array), - http + http.protected .get("/find/symbol", "find.symbols") .seeded((ctx) => ctx.file("hello.ts", "export const hello = 1\n")) .at((ctx) => ({ path: `/find/symbol?${new URLSearchParams({ query: "hello" })}`, headers: ctx.headers() })) .json(200, array), - http + http.protected .get("/event", "event.stream") .stream() .status( @@ -289,8 +303,8 @@ const scenarios: Scenario[] = [ }), "status", ), - http.get("/mcp", "mcp.status").json(), - http + http.protected.get("/mcp", "mcp.status").json(), + http.protected .post("/mcp", "mcp.add") .mutating() .at((ctx) => ({ @@ -307,7 +321,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/mcp", "mcp.add.invalid") .at((ctx) => ({ path: "/mcp", @@ -315,7 +329,7 @@ const scenarios: Scenario[] = [ body: { name: "httpapi-invalid", config: { type: "invalid" } }, })) .status(400), - http + http.protected .post("/mcp/{name}/auth", "mcp.auth.start") .at((ctx) => ({ path: route("/mcp/{name}/auth", { name: "httpapi-missing" }), headers: ctx.headers() })) .json( @@ -326,7 +340,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .delete("/mcp/{name}/auth", "mcp.auth.remove") .mutating() .at((ctx) => ({ path: route("/mcp/{name}/auth", { name: "httpapi-missing" }), headers: ctx.headers() })) @@ -334,7 +348,7 @@ const scenarios: Scenario[] = [ object(body) check(body.success === true, "MCP auth removal should return success") }), - http + http.protected .post("/mcp/{name}/auth/authenticate", "mcp.auth.authenticate") .at((ctx) => ({ path: route("/mcp/{name}/auth/authenticate", { name: "httpapi-missing" }), @@ -348,7 +362,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/mcp/{name}/auth/callback", "mcp.auth.callback") .at((ctx) => ({ path: route("/mcp/{name}/auth/callback", { name: "httpapi-missing" }), @@ -356,23 +370,23 @@ const scenarios: Scenario[] = [ body: { code: 1 }, })) .status(400), - http + http.protected .post("/mcp/{name}/connect", "mcp.connect") .mutating() .at((ctx) => ({ path: route("/mcp/{name}/connect", { name: "httpapi-missing" }), headers: ctx.headers() })) .json(200, (body) => { check(body === true, "missing MCP connect should remain a no-op success") }), - http + http.protected .post("/mcp/{name}/disconnect", "mcp.disconnect") .mutating() .at((ctx) => ({ path: route("/mcp/{name}/disconnect", { name: "httpapi-missing" }), headers: ctx.headers() })) .json(200, (body) => { check(body === true, "missing MCP disconnect should remain a no-op success") }), - http.get("/pty/shells", "pty.shells").json(200, array), - http.get("/pty", "pty.list").json(200, array), - http + http.protected.get("/pty/shells", "pty.shells").json(200, array), + http.protected.get("/pty", "pty.list").json(200, array), + http.protected .post("/pty", "pty.create") .mutating() .at((ctx) => ({ path: "/pty", headers: ctx.headers(), body: controlledPtyInput("HTTP API PTY") })) @@ -386,15 +400,22 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/pty", "pty.create.invalid") .at((ctx) => ({ path: "/pty", headers: ctx.headers(), body: { command: 1 } })) .status(400), - http + http.protected + .post("/pty/{ptyID}/connect-token", "pty.connectToken.invalid") + .at((ctx) => ({ + path: route("/pty/{ptyID}/connect-token", { ptyID: "pty_httpapi_missing" }), + headers: ctx.headers(), + })) + .status(403, undefined, "status"), + http.protected .get("/pty/{ptyID}", "pty.get") .at((ctx) => ({ path: route("/pty/{ptyID}", { ptyID: "pty_httpapi_missing" }), headers: ctx.headers() })) .status(404), - http + http.protected .put("/pty/{ptyID}", "pty.update") .mutating() .at((ctx) => ({ @@ -403,20 +424,20 @@ const scenarios: Scenario[] = [ body: { size: { rows: 0, cols: 0 } }, })) .status(400), - http + http.protected .delete("/pty/{ptyID}", "pty.remove") .mutating() .at((ctx) => ({ path: route("/pty/{ptyID}", { ptyID: "pty_httpapi_missing" }), headers: ctx.headers() })) .json(200, (body) => { check(body === true, "PTY remove should return true") }), - http + http.protected .get("/pty/{ptyID}/connect", "pty.connect") .at((ctx) => ({ path: route("/pty/{ptyID}/connect", { ptyID: "pty_httpapi_missing" }), headers: ctx.headers() })) .status(404, undefined, "none"), - http.get("/experimental/console", "experimental.console.get").json(), - http.get("/experimental/console/orgs", "experimental.console.listOrgs").json(), - http + http.protected.get("/experimental/console", "experimental.console.get").json(), + http.protected.get("/experimental/console/orgs", "experimental.console.listOrgs").json(), + http.protected .post("/experimental/console/switch", "experimental.console.switchOrg") .at((ctx) => ({ path: "/experimental/console/switch", @@ -424,14 +445,17 @@ const scenarios: Scenario[] = [ body: { accountID: "httpapi-account", orgID: "httpapi-org" }, })) .status(400, undefined, "none"), - http.get("/experimental/workspace/adapter", "experimental.workspace.adapter.list").json(200, array), - http.get("/experimental/workspace", "experimental.workspace.list").json(200, array), - http.get("/experimental/workspace/status", "experimental.workspace.status").json(200, array), - http + http.protected.get("/experimental/workspace/adapter", "experimental.workspace.adapter.list").json(200, array), + http.protected.get("/experimental/workspace", "experimental.workspace.list").json(200, array), + http.protected.get("/experimental/workspace/status", "experimental.workspace.status").json(200, array), + http.protected .post("/experimental/workspace", "experimental.workspace.create") .at((ctx) => ({ path: "/experimental/workspace", headers: ctx.headers(), body: {} })) .status(400), - http + http.protected + .post("/experimental/workspace/sync-list", "experimental.workspace.syncList") + .status(204, undefined, "status"), + http.protected .delete("/experimental/workspace/{id}", "experimental.workspace.remove") .mutating() .at((ctx) => ({ @@ -439,7 +463,7 @@ const scenarios: Scenario[] = [ headers: ctx.headers(), })) .status(200), - http + http.protected .post("/experimental/workspace/warp", "experimental.workspace.warp") .at((ctx) => ({ path: "/experimental/workspace/warp", @@ -447,16 +471,16 @@ const scenarios: Scenario[] = [ body: {}, })) .status(400), - http + http.protected .get("/experimental/tool", "tool.list") .at((ctx) => ({ path: `/experimental/tool?${new URLSearchParams({ provider: "opencode", model: "test" })}`, headers: ctx.headers(), })) .json(200, array, "status"), - http.get("/experimental/tool/ids", "tool.ids").json(200, array), - http.get("/experimental/worktree", "worktree.list").json(200, array), - http + http.protected.get("/experimental/tool/ids", "tool.ids").json(200, array), + http.protected.get("/experimental/worktree", "worktree.list").json(200, array), + http.protected .post("/experimental/worktree", "worktree.create") .mutating() .at((ctx) => ({ path: "/experimental/worktree", headers: ctx.headers(), body: { name: "api-dsl" } })) @@ -470,11 +494,11 @@ const scenarios: Scenario[] = [ }), "status", ), - http + http.protected .post("/experimental/worktree", "worktree.create.invalid") .at((ctx) => ({ path: "/experimental/worktree", headers: ctx.headers(), body: { name: 1 } })) .status(400), - http + http.protected .delete("/experimental/worktree", "worktree.remove") .mutating() .seeded((ctx) => ctx.worktree({ name: "api-remove" })) @@ -482,7 +506,7 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "worktree remove should return true") }), - http + http.protected .post("/experimental/worktree/reset", "worktree.reset") .mutating() .seeded((ctx) => ctx.worktree({ name: "api-reset" })) @@ -497,37 +521,41 @@ const scenarios: Scenario[] = [ yield* ctx.worktreeRemove(ctx.state.directory) }), ), - http.get("/experimental/session", "experimental.session.list").json(200, array), - http.get("/experimental/resource", "experimental.resource.list").json(), - http + http.protected.get("/experimental/session", "experimental.session.list").json(200, array), + http.protected.get("/experimental/resource", "experimental.resource.list").json(), + http.protected .post("/sync/history", "sync.history.list") .at((ctx) => ({ path: "/sync/history", headers: ctx.headers(), body: {} })) .json(200, array), - http + http.protected .post("/sync/replay", "sync.replay") .at((ctx) => ({ path: "/sync/replay", headers: ctx.headers(), body: { directory: ctx.directory, events: [] } })) .status(400), - http + http.protected + .post("/sync/steal", "sync.steal.invalid") + .at((ctx) => ({ path: "/sync/steal", headers: ctx.headers(), body: {} })) + .status(400, undefined, "status"), + http.protected .post("/sync/start", "sync.start") .mutating() .preserveDatabase() .json(200, (body) => { check(body === true, "sync start should return true when no workspace sessions exist") }), - http + http.protected .post("/instance/dispose", "instance.dispose") .mutating() .json(200, (body) => { check(body === true, "instance dispose should return true") }), - http + http.protected .post("/log", "app.log") .global() .at(() => ({ path: "/log", body: { service: "httpapi-exercise", level: "info", message: "route coverage" } })) .json(200, (body) => { check(body === true, "log route should return true") }), - http + http.protected .put("/auth/{providerID}", "auth.set") .global() .at(() => ({ path: route("/auth/{providerID}", { providerID: "test" }), body: { type: "api", key: "test-key" } })) @@ -539,7 +567,7 @@ const scenarios: Scenario[] = [ check(isRecord(auth.test) && auth.test.key === "test-key", "auth set should write isolated auth file") }), ), - http + http.protected .delete("/auth/{providerID}", "auth.remove") .global() .seeded(() => @@ -559,7 +587,63 @@ const scenarios: Scenario[] = [ check(auth.test === undefined, "auth remove should delete provider from isolated auth file") }), ), - http + http.protected + .get("/api/session", "v2.session.list") + .at((ctx) => ({ path: "/api/session?roots=true", headers: ctx.headers() })) + .json( + 200, + (body) => { + object(body) + array(body.items) + object(body.cursor) + }, + "none", + ), + http.protected + .get("/api/session/{sessionID}/context", "v2.session.context") + .at((ctx) => ({ + path: route("/api/session/{sessionID}/context", { sessionID: "ses_httpapi_missing" }), + headers: ctx.headers(), + })) + .json(200, array, "none"), + http.protected + .get("/api/session/{sessionID}/message", "v2.session.messages") + .at((ctx) => ({ + path: route("/api/session/{sessionID}/message", { sessionID: "ses_httpapi_missing" }), + headers: ctx.headers(), + })) + .json( + 200, + (body) => { + object(body) + array(body.items) + object(body.cursor) + }, + "none", + ), + http.protected + .post("/api/session/{sessionID}/prompt", "v2.session.prompt.invalid") + .at((ctx) => ({ + path: route("/api/session/{sessionID}/prompt", { sessionID: "ses_httpapi_missing" }), + headers: ctx.headers(), + body: {}, + })) + .status(400, undefined, "none"), + http.protected + .post("/api/session/{sessionID}/compact", "v2.session.compact") + .at((ctx) => ({ + path: route("/api/session/{sessionID}/compact", { sessionID: "ses_httpapi_missing" }), + headers: ctx.headers(), + })) + .status(204, undefined, "none"), + http.protected + .post("/api/session/{sessionID}/wait", "v2.session.wait") + .at((ctx) => ({ + path: route("/api/session/{sessionID}/wait", { sessionID: "ses_httpapi_missing" }), + headers: ctx.headers(), + })) + .status(204, undefined, "none"), + http.protected .get("/session", "session.list") .seeded((ctx) => ctx.session({ title: "List me" })) .at((ctx) => ({ path: "/session?roots=true", headers: ctx.headers() })) @@ -570,11 +654,11 @@ const scenarios: Scenario[] = [ "seeded session should be listed", ) }), - http + http.protected .get("/session/status", "session.status") .seeded((ctx) => ctx.session({ title: "Status session" })) .json(200, object), - http + http.protected .post("/session", "session.create") .mutating() .at((ctx) => ({ path: "/session", headers: ctx.headers(), body: { title: "Created session" } })) @@ -587,7 +671,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .get("/session/{sessionID}", "session.get") .seeded((ctx) => ctx.session({ title: "Get me" })) .at((ctx) => ({ path: route("/session/{sessionID}", { sessionID: ctx.state.id }), headers: ctx.headers() })) @@ -596,14 +680,14 @@ const scenarios: Scenario[] = [ check(body.id === ctx.state.id, "should return requested session") check(body.title === "Get me", "should preserve seeded title") }), - http + http.protected .get("/session/{sessionID}", "session.get.missing") .at((ctx) => ({ path: route("/session/{sessionID}", { sessionID: "ses_httpapi_missing" }), headers: ctx.headers(), })) .status(404), - http + http.protected .patch("/session/{sessionID}", "session.update") .mutating() .seeded((ctx) => ctx.session({ title: "Before rename" })) @@ -620,7 +704,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .patch("/session/{sessionID}", "session.update.invalid") .mutating() .at((ctx) => ({ @@ -629,7 +713,7 @@ const scenarios: Scenario[] = [ body: { title: 1 }, })) .status(400), - http + http.protected .delete("/session/{sessionID}", "session.delete") .mutating() .seeded((ctx) => ctx.session({ title: "Delete me" })) @@ -640,7 +724,7 @@ const scenarios: Scenario[] = [ check((yield* ctx.sessionGet(ctx.state.id)) === undefined, "deleted session should not remain in storage") }), ), - http + http.protected .get("/session/{sessionID}/children", "session.children") .seeded((ctx) => Effect.gen(function* () { @@ -660,7 +744,7 @@ const scenarios: Scenario[] = [ "children should include seeded child", ) }), - http + http.protected .get("/session/{sessionID}/todo", "session.todo") .seeded((ctx) => Effect.gen(function* () { @@ -677,12 +761,12 @@ const scenarios: Scenario[] = [ .json(200, (body, ctx) => { check(stable(body) === stable(ctx.state.todos), "todos should match seeded state") }), - http + http.protected .get("/session/{sessionID}/diff", "session.diff") .seeded((ctx) => ctx.session({ title: "Diff session" })) .at((ctx) => ({ path: route("/session/{sessionID}/diff", { sessionID: ctx.state.id }), headers: ctx.headers() })) .json(200, array), - http + http.protected .get("/session/{sessionID}/message", "session.messages") .seeded((ctx) => ctx.session({ title: "Messages session" })) .at((ctx) => ({ path: route("/session/{sessionID}/message", { sessionID: ctx.state.id }), headers: ctx.headers() })) @@ -690,7 +774,7 @@ const scenarios: Scenario[] = [ array(body) check(body.length === 0, "new session should have no messages") }), - http + http.protected .get("/session/{sessionID}/message/{messageID}", "session.message") .seeded((ctx) => Effect.gen(function* () { @@ -714,7 +798,7 @@ const scenarios: Scenario[] = [ "message should include seeded part", ) }), - http + http.protected .patch("/session/{sessionID}/message/{messageID}/part/{partID}", "part.update") .mutating() .seeded((ctx) => @@ -741,7 +825,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .delete("/session/{sessionID}/message/{messageID}/part/{partID}", "part.delete") .mutating() .seeded((ctx) => @@ -766,7 +850,7 @@ const scenarios: Scenario[] = [ check(messages[0]?.parts.length === 0, "deleted part should not remain on message") }), ), - http + http.protected .delete("/session/{sessionID}/message/{messageID}", "session.deleteMessage") .mutating() .seeded((ctx) => @@ -789,7 +873,7 @@ const scenarios: Scenario[] = [ check((yield* ctx.messages(ctx.state.session.id)).length === 0, "deleted message should not remain") }), ), - http + http.protected .post("/session/{sessionID}/fork", "session.fork") .mutating() .seeded((ctx) => ctx.session({ title: "Fork source" })) @@ -806,7 +890,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/session/{sessionID}/abort", "session.abort") .mutating() .seeded((ctx) => ctx.session({ title: "Abort session" })) @@ -814,7 +898,7 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "abort should return true") }), - http + http.protected .post("/session/{sessionID}/abort", "session.abort.missing") .at((ctx) => ({ path: route("/session/{sessionID}/abort", { sessionID: "ses_httpapi_missing" }), @@ -823,7 +907,7 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "missing session abort should remain a no-op success") }), - http + http.protected .post("/session/{sessionID}/init", "session.init") .preserveDatabase() .withLlm() @@ -847,7 +931,7 @@ const scenarios: Scenario[] = [ yield* ctx.llmWait(1) }), ), - http + http.protected .post("/session/{sessionID}/message", "session.prompt") .preserveDatabase() .withLlm() @@ -882,7 +966,7 @@ const scenarios: Scenario[] = [ }), "status", ), - http + http.protected .post("/session/{sessionID}/prompt_async", "session.prompt_async") .preserveDatabase() .withLlm() @@ -908,7 +992,7 @@ const scenarios: Scenario[] = [ yield* ctx.llmWait(1) }), ), - http + http.protected .post("/session/{sessionID}/command", "session.command") .preserveDatabase() .withLlm() @@ -935,7 +1019,7 @@ const scenarios: Scenario[] = [ }), "status", ), - http + http.protected .post("/session/{sessionID}/shell", "session.shell") .preserveDatabase() .mutating() @@ -957,7 +1041,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/session/{sessionID}/summarize", "session.summarize") .preserveDatabase() .withLlm() @@ -1018,7 +1102,7 @@ const scenarios: Scenario[] = [ }), "status", ), - http + http.protected .post("/session/{sessionID}/revert", "session.revert") .mutating() .seeded((ctx) => @@ -1045,7 +1129,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/session/{sessionID}/unrevert", "session.unrevert") .mutating() .seeded((ctx) => ctx.session({ title: "Unrevert session" })) @@ -1061,7 +1145,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/session/{sessionID}/permissions/{permissionID}", "permission.respond") .seeded((ctx) => ctx.session({ title: "Deprecated permission session" })) .at((ctx) => ({ @@ -1075,7 +1159,7 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "deprecated permission response should return true") }), - http + http.protected .post("/session/{sessionID}/share", "session.share") .mutating() .seeded((ctx) => ctx.session({ title: "Share session" })) @@ -1088,7 +1172,7 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .delete("/session/{sessionID}/share", "session.unshare") .mutating() .seeded((ctx) => ctx.session({ title: "Unshare session" })) @@ -1101,25 +1185,25 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/tui/append-prompt", "tui.appendPrompt") .at((ctx) => ({ path: "/tui/append-prompt", headers: ctx.headers(), body: { text: "hello" } })) .json(200, boolean, "status"), - http + http.protected .post("/tui/select-session", "tui.selectSession.invalid") .at((ctx) => ({ path: "/tui/select-session", headers: ctx.headers(), body: { sessionID: "invalid" } })) .status(400), - http.post("/tui/open-help", "tui.openHelp").json(200, boolean, "status"), - http.post("/tui/open-sessions", "tui.openSessions").json(200, boolean, "status"), - http.post("/tui/open-themes", "tui.openThemes").json(200, boolean, "status"), - http.post("/tui/open-models", "tui.openModels").json(200, boolean, "status"), - http.post("/tui/submit-prompt", "tui.submitPrompt").json(200, boolean, "status"), - http.post("/tui/clear-prompt", "tui.clearPrompt").json(200, boolean, "status"), - http + http.protected.post("/tui/open-help", "tui.openHelp").json(200, boolean, "status"), + http.protected.post("/tui/open-sessions", "tui.openSessions").json(200, boolean, "status"), + http.protected.post("/tui/open-themes", "tui.openThemes").json(200, boolean, "status"), + http.protected.post("/tui/open-models", "tui.openModels").json(200, boolean, "status"), + http.protected.post("/tui/submit-prompt", "tui.submitPrompt").json(200, boolean, "status"), + http.protected.post("/tui/clear-prompt", "tui.clearPrompt").json(200, boolean, "status"), + http.protected .post("/tui/execute-command", "tui.executeCommand") .at((ctx) => ({ path: "/tui/execute-command", headers: ctx.headers(), body: { command: "agent_cycle" } })) .json(200, boolean, "status"), - http + http.protected .post("/tui/show-toast", "tui.showToast") .at((ctx) => ({ path: "/tui/show-toast", @@ -1127,7 +1211,7 @@ const scenarios: Scenario[] = [ body: { title: "Exercise", message: "covered", variant: "info", duration: 1000 }, })) .json(200, boolean, "status"), - http + http.protected .post("/tui/publish", "tui.publish") .at((ctx) => ({ path: "/tui/publish", @@ -1135,16 +1219,16 @@ const scenarios: Scenario[] = [ body: { type: "tui.prompt.append", properties: { text: "published" } }, })) .json(200, boolean, "status"), - http + http.protected .post("/tui/select-session", "tui.selectSession") .seeded((ctx) => ctx.session({ title: "TUI select" })) .at((ctx) => ({ path: "/tui/select-session", headers: ctx.headers(), body: { sessionID: ctx.state.id } })) .json(200, boolean, "status"), - http + http.protected .post("/tui/control/response", "tui.control.response") .at((ctx) => ({ path: "/tui/control/response", headers: ctx.headers(), body: { ok: true } })) .json(200, boolean, "status"), - http + http.protected .get("/tui/control/next", "tui.control.next") .mutating() .seeded((ctx) => ctx.tuiRequest({ path: "/tui/exercise", body: { text: "queued" } })) @@ -1158,23 +1242,38 @@ const scenarios: Scenario[] = [ }, "status", ), - http + http.protected .post("/global/upgrade", "global.upgrade") .global() + .probe({ path: "/global/upgrade", body: { target: 1 } }) .at(() => ({ path: "/global/upgrade", body: { target: 1 } })) .status(400), ] +const llmScenarios = new Set([ + "session.init", + "session.prompt", + "session.prompt_async", + "session.command", + "session.summarize", +]) + const main = Effect.gen(function* () { yield* Effect.addFinalizer(() => cleanupExercisePaths) const options = parseOptions(Bun.argv.slice(2)) const modules = yield* Effect.promise(() => runtime()) const effectRoutes = routeKeys(OpenApi.fromApi(modules.PublicApi)) const honoRoutes = routeKeys(yield* Effect.promise(() => modules.Server.openapiHono())) - const selected = scenarios.filter((scenario) => matches(options, scenario)) + const selected = selectedScenarios(options, scenarios) const missing = effectRoutes.filter((route) => !scenarios.some((scenario) => route === routeKey(scenario))) const extra = scenarios.filter((scenario) => !effectRoutes.includes(routeKey(scenario))) + for (const scenario of scenarios) { + if (scenario.kind === "active" && llmScenarios.has(scenario.name) && !scenario.project?.llm) { + return yield* Effect.fail(new Error(`${scenario.name} must use TestLLMServer via .withLlm()`)) + } + } + printHeader(options, effectRoutes, honoRoutes, selected, missing, extra, { database: exerciseDatabasePath, global: exerciseGlobalRoot, @@ -1183,7 +1282,15 @@ const main = Effect.gen(function* () { const results = options.mode === "coverage" ? selected.map(coverageResult) - : yield* Effect.forEach(selected, runScenario(options), { concurrency: 1 }) + : yield* Effect.forEach( + selected, + (scenario) => + Effect.gen(function* () { + if (options.progress) console.log(`${color.dim}RUN ${routeKey(scenario)} ${scenario.name}${color.reset}`) + return yield* runScenario(options)(scenario) + }), + { concurrency: 1 }, + ) printResults(results, missing, extra) if (results.some((result) => result.status === "fail")) diff --git a/packages/opencode/test/server/httpapi-exercise/report.ts b/packages/opencode/test/server/httpapi-exercise/report.ts index 9ed764f4db..844534d040 100644 --- a/packages/opencode/test/server/httpapi-exercise/report.ts +++ b/packages/opencode/test/server/httpapi-exercise/report.ts @@ -23,7 +23,7 @@ export function printHeader( console.log(`${color.dim}db=${paths.database}${color.reset}`) console.log(`${color.dim}global=${paths.global}${color.reset}`) console.log( - `${color.dim}mode=${options.mode} selected=${selected.length} effectRoutes=${effectRoutes.length} missing=${missing.length} extra=${extra.length} onlyEffect=${effectRoutes.filter((route) => !honoRoutes.includes(route)).length} onlyHono=${honoRoutes.filter((route) => !effectRoutes.includes(route)).length}${color.reset}`, + `${color.dim}mode=${options.mode} selected=${selected.length} scenarioTimeout=${options.scenarioTimeout} effectRoutes=${effectRoutes.length} missing=${missing.length} extra=${extra.length} onlyEffect=${effectRoutes.filter((route) => !honoRoutes.includes(route)).length} onlyHono=${honoRoutes.filter((route) => !effectRoutes.includes(route)).length}${color.reset}`, ) console.log("") } diff --git a/packages/opencode/test/server/httpapi-exercise/routing.ts b/packages/opencode/test/server/httpapi-exercise/routing.ts index 39bda11209..cdb58841bd 100644 --- a/packages/opencode/test/server/httpapi-exercise/routing.ts +++ b/packages/opencode/test/server/httpapi-exercise/routing.ts @@ -24,8 +24,13 @@ export function parseOptions(args: string[]): Options { return { mode, include: option(args, "--include"), + startAt: option(args, "--start-at"), + stopAt: option(args, "--stop-at"), failOnMissing: args.includes("--fail-on-missing"), failOnSkip: args.includes("--fail-on-skip"), + scenarioTimeout: option(args, "--scenario-timeout") ?? "30 seconds", + progress: args.includes("--progress"), + trace: args.includes("--trace"), } } @@ -38,6 +43,21 @@ export function matches(options: Options, scenario: Scenario) { ) } +export function selectedScenarios(options: Options, scenarios: Scenario[]) { + const included = scenarios.filter((scenario) => matches(options, scenario)) + const start = options.startAt ? included.findIndex((scenario) => matchesName(options.startAt!, scenario)) : 0 + const end = options.stopAt + ? included.findIndex((scenario) => matchesName(options.stopAt!, scenario)) + : included.length - 1 + if (start === -1) throw new Error(`--start-at matched no scenario: ${options.startAt}`) + if (end === -1) throw new Error(`--stop-at matched no scenario: ${options.stopAt}`) + return included.slice(start, end + 1) +} + +function matchesName(value: string, scenario: Scenario) { + return scenario.name.includes(value) || scenario.path.includes(value) || scenario.method.includes(value.toUpperCase()) +} + function option(args: string[], name: string) { const index = args.indexOf(name) if (index === -1) return undefined diff --git a/packages/opencode/test/server/httpapi-exercise/runner.ts b/packages/opencode/test/server/httpapi-exercise/runner.ts index 18ef991807..2793de75ef 100644 --- a/packages/opencode/test/server/httpapi-exercise/runner.ts +++ b/packages/opencode/test/server/httpapi-exercise/runner.ts @@ -24,6 +24,10 @@ export function runScenario(options: Options) { return (scenario: Scenario) => { if (scenario.kind === "todo") return Effect.succeed({ status: "skip", scenario } as Result) return runActive(options, scenario).pipe( + Effect.timeoutOrElse({ + duration: options.scenarioTimeout, + orElse: () => Effect.die(new Error(`scenario timed out after ${options.scenarioTimeout}`)), + }), Effect.as({ status: "pass", scenario } as Result), Effect.catchCause((cause) => Effect.succeed({ status: "fail" as const, scenario, message: Cause.pretty(cause) })), Effect.scoped, @@ -36,20 +40,32 @@ function runActive(options: Options, scenario: ActiveScenario) { if (options.mode === "parity" && scenario.mutates && scenario.compare !== "none") { return Effect.gen(function* () { - const effect = yield* runBackend("effect", scenario) - const legacy = yield* runBackend("legacy", scenario) + const effect = yield* runBackend(options, "effect", scenario) + const legacy = yield* runBackend(options, "legacy", scenario) + yield* trace(options, scenario, "compare start") yield* compare(scenario, effect, legacy) + yield* trace(options, scenario, "compare done") }) } - return withContext(scenario, (ctx) => + return withContext(options, scenario, "shared", (ctx) => Effect.gen(function* () { + yield* trace(options, scenario, "effect request start") const effect = yield* call("effect", scenario, ctx) + yield* trace(options, scenario, `effect response ${effect.status}`) + yield* trace(options, scenario, "effect expect start") yield* scenario.expect(ctx, ctx.state, effect) + yield* trace(options, scenario, "effect expect done") if (options.mode === "parity" && scenario.compare !== "none") { + yield* trace(options, scenario, "legacy request start") const legacy = yield* call("legacy", scenario, ctx) + yield* trace(options, scenario, `legacy response ${legacy.status}`) + yield* trace(options, scenario, "legacy expect start") yield* scenario.expect(ctx, ctx.state, legacy) + yield* trace(options, scenario, "legacy expect done") + yield* trace(options, scenario, "compare start") yield* compare(scenario, effect, legacy) + yield* trace(options, scenario, "compare done") } }), ) @@ -57,61 +73,89 @@ function runActive(options: Options, scenario: ActiveScenario) { function runAuth(scenario: ActiveScenario) { return Effect.gen(function* () { - const effect = yield* callAuthProbe("effect", scenario) - const legacy = yield* callAuthProbe("legacy", scenario) + const effect = yield* callAuthProbe("effect", scenario, "missing") + const legacy = yield* callAuthProbe("legacy", scenario, "missing") if (scenario.auth === "protected") { if (effect.status !== 401) throw new Error(`effect auth expected 401, got ${effect.status}`) if (legacy.status !== 401) throw new Error(`legacy auth expected 401, got ${legacy.status}`) + const effectAuthed = yield* callAuthProbe("effect", scenario, "valid") + const legacyAuthed = yield* callAuthProbe("legacy", scenario, "valid") + if (effectAuthed.status === 401) throw new Error("effect auth rejected valid credentials") + if (legacyAuthed.status === 401) throw new Error("legacy auth rejected valid credentials") return } if (effect.status === 401) throw new Error("effect auth expected public access, got 401") if (legacy.status === 401) throw new Error("legacy auth expected public access, got 401") + if (effect.timedOut) throw new Error("effect auth expected public access, probe timed out") + if (legacy.timedOut) throw new Error("legacy auth expected public access, probe timed out") }) } -function runBackend(backend: "effect" | "legacy", scenario: ActiveScenario) { - return withContext(scenario, (ctx) => +function runBackend(options: Options, backend: "effect" | "legacy", scenario: ActiveScenario) { + return withContext(options, scenario, backend, (ctx) => Effect.gen(function* () { + yield* trace(options, scenario, `${backend} request start`) const result = yield* call(backend, scenario, ctx) + yield* trace(options, scenario, `${backend} response ${result.status}`) + yield* trace(options, scenario, `${backend} expect start`) yield* scenario.expect(ctx, ctx.state, result) + yield* trace(options, scenario, `${backend} expect done`) return result }), ) } -function withContext(scenario: ActiveScenario, use: (ctx: SeededContext) => Effect.Effect) { +function withContext( + options: Options, + scenario: ActiveScenario, + label: string, + use: (ctx: SeededContext) => Effect.Effect, +) { return Effect.acquireRelease( Effect.gen(function* () { + yield* trace(options, scenario, `${label} context acquire start`) const llm = scenario.project?.llm ? yield* TestLLMServer : undefined const project = scenario.project const dir = project ? yield* Effect.promise(async () => (await runtime()).tmpdir(projectOptions(project, llm?.url))) : undefined + yield* trace(options, scenario, `${label} context acquire done`) return { dir, llm } }), (ctx) => - Effect.promise(async () => { - await ctx.dir?.[Symbol.asyncDispose]() - }).pipe(Effect.ignore), + Effect.gen(function* () { + yield* trace(options, scenario, `${label} tmpdir cleanup start`) + yield* Effect.promise(async () => { + await ctx.dir?.[Symbol.asyncDispose]() + }).pipe(Effect.ignore) + yield* trace(options, scenario, `${label} tmpdir cleanup done`) + }), ).pipe( Effect.flatMap((context) => Effect.gen(function* () { + yield* trace(options, scenario, `${label} runtime start`) const modules = yield* Effect.promise(() => runtime()) + yield* trace(options, scenario, `${label} runtime done`) const path = context.dir?.path const instance = path - ? yield* modules.InstanceStore.Service.use((store) => store.load({ directory: path })).pipe( - Effect.provide(modules.AppLayer), - Effect.catchCause((cause) => - Effect.sleep("100 millis").pipe( - Effect.andThen( - modules.InstanceStore.Service.use((store) => store.load({ directory: path })).pipe( - Effect.provide(modules.AppLayer), + ? yield* trace(options, scenario, `${label} instance load start`).pipe( + Effect.andThen( + modules.InstanceStore.Service.use((store) => store.load({ directory: path })).pipe( + Effect.provide(modules.AppLayer), + Effect.catchCause((cause) => + Effect.sleep("100 millis").pipe( + Effect.andThen( + modules.InstanceStore.Service.use((store) => store.load({ directory: path })).pipe( + Effect.provide(modules.AppLayer), + ), + ), + Effect.catchCause(() => Effect.failCause(cause)), ), ), - Effect.catchCause(() => Effect.failCause(cause)), ), ), + Effect.tap(() => trace(options, scenario, `${label} instance load done`)), ) : undefined const run = (effect: Effect.Effect) => @@ -184,14 +228,26 @@ function withContext(scenario: ActiveScenario, use: (ctx: SeededContext Effect.suspend(() => llm().wait(count)), tuiRequest: (request) => Effect.sync(() => modules.Tui.submitTuiRequest(request)), } + yield* trace(options, scenario, `${label} seed start`) const state = yield* scenario.seed(base) - return yield* use({ ...base, state }) + yield* trace(options, scenario, `${label} seed done`) + yield* trace(options, scenario, `${label} use start`) + const result = yield* use({ ...base, state }) + yield* trace(options, scenario, `${label} use done`) + return result }).pipe(Effect.ensuring(context.llm ? context.llm.reset : Effect.void)), ), Effect.ensuring(scenario.reset ? resetState : Effect.void), ) } +function trace(options: Options, scenario: ActiveScenario, phase: string) { + return Effect.sync(() => { + if (!options.trace) return + console.log(`[trace] ${scenario.name}: ${phase}`) + }) +} + function projectOptions( project: ProjectOptions, llmUrl: string | undefined, diff --git a/packages/opencode/test/server/httpapi-exercise/types.ts b/packages/opencode/test/server/httpapi-exercise/types.ts index c725739b4e..886ee61833 100644 --- a/packages/opencode/test/server/httpapi-exercise/types.ts +++ b/packages/opencode/test/server/httpapi-exercise/types.ts @@ -22,8 +22,13 @@ export type JsonObject = Record export type Options = { mode: Mode include: string | undefined + startAt: string | undefined + stopAt: string | undefined failOnMissing: boolean failOnSkip: boolean + scenarioTimeout: string + progress: boolean + trace: boolean } export type RequestSpec = { @@ -37,6 +42,7 @@ export type CallResult = { contentType: string body: unknown text: string + timedOut: boolean } export type BackendApp = { @@ -75,6 +81,7 @@ export type ActiveScenario = { project: ProjectOptions | undefined seed: (ctx: ScenarioContext) => Effect.Effect request: (ctx: ScenarioContext, state: unknown) => RequestSpec + authProbe: RequestSpec | undefined expect: (ctx: ScenarioContext, state: unknown, result: CallResult) => Effect.Effect compare: Comparison capture: CaptureMode @@ -90,6 +97,7 @@ export type BuilderState = { project: ProjectOptions | undefined seed: (ctx: ScenarioContext) => Effect.Effect request: (ctx: SeededContext) => RequestSpec + authProbe: RequestSpec | undefined capture: CaptureMode mutates: boolean reset: boolean