diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index 17ad23339f3..8ba0ed5e5f1 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -265,175 +265,155 @@ describe("matrix live qa runtime", () => { }); it("records default and per-scenario Matrix config snapshots in the summary", () => { - expect( - liveTesting.buildMatrixQaSummary({ - artifactPaths: { - observedEvents: "/tmp/observed.json", - report: "/tmp/report.md", - summary: "/tmp/summary.json", - }, - checks: [{ name: "Matrix harness ready", status: "pass" }], - config: { - default: liveTesting.buildMatrixQaConfigSnapshot({ - driverUserId: "@driver:matrix-qa.test", - observerUserId: "@observer:matrix-qa.test", - sutUserId: "@sut:matrix-qa.test", - topology: { - defaultRoomId: "!room:matrix-qa.test", - defaultRoomKey: "main", - rooms: [ - { - key: "main", - kind: "group", - memberRoles: ["driver", "observer", "sut"], - memberUserIds: [ - "@driver:matrix-qa.test", - "@observer:matrix-qa.test", - "@sut:matrix-qa.test", - ], - name: "Matrix QA", - requireMention: true, - roomId: "!room:matrix-qa.test", - }, - ], - }, - }), - scenarios: [ - { - id: "matrix-room-thread-reply-override", - title: "Matrix threadReplies always keeps room replies threaded", - config: liveTesting.buildMatrixQaConfigSnapshot({ - driverUserId: "@driver:matrix-qa.test", - observerUserId: "@observer:matrix-qa.test", - overrides: { - threadReplies: "always", - }, - sutUserId: "@sut:matrix-qa.test", - topology: { - defaultRoomId: "!room:matrix-qa.test", - defaultRoomKey: "main", - rooms: [ - { - key: "main", - kind: "group", - memberRoles: ["driver", "observer", "sut"], - memberUserIds: [ - "@driver:matrix-qa.test", - "@observer:matrix-qa.test", - "@sut:matrix-qa.test", - ], - name: "Matrix QA", - requireMention: true, - roomId: "!room:matrix-qa.test", - }, - ], - }, - }), - }, - ], - }, - finishedAt: "2026-04-10T10:05:00.000Z", - harness: { - baseUrl: "http://127.0.0.1:28008/", - composeFile: "/tmp/docker-compose.yml", - dmRoomIds: [], - image: "ghcr.io/matrix-construct/tuwunel:v1.5.1", - roomId: "!room:matrix-qa.test", - roomIds: ["!room:matrix-qa.test"], - serverName: "matrix-qa.test", - }, - observedEventCount: 0, - scenarios: [], - startedAt: "2026-04-10T10:00:00.000Z", - sutAccountId: "sut", - timings: { - artifactWriteMs: 5, - canaryMs: 40, - harnessBootMs: 100, - initialGatewayBootMs: 200, - provisioningMs: 300, - scenarioGatewayBootMs: 50, - scenarioRestartGatewayMs: 60, - scenarioTransportInterruptMs: 70, - scenarios: [], - totalMs: 825, - }, - userIds: { - driver: "@driver:matrix-qa.test", - observer: "@observer:matrix-qa.test", - sut: "@sut:matrix-qa.test", - }, - }).config, - ).toMatchObject({ - default: { - replyToMode: "off", - threadReplies: "inbound", + const summary = liveTesting.buildMatrixQaSummary({ + artifactPaths: { + observedEvents: "/tmp/observed.json", + report: "/tmp/report.md", + summary: "/tmp/summary.json", }, - scenarios: [ - { - id: "matrix-room-thread-reply-override", - config: { - threadReplies: "always", + checks: [{ name: "Matrix harness ready", status: "pass" }], + config: { + default: liveTesting.buildMatrixQaConfigSnapshot({ + driverUserId: "@driver:matrix-qa.test", + observerUserId: "@observer:matrix-qa.test", + sutUserId: "@sut:matrix-qa.test", + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Matrix QA", + requireMention: true, + roomId: "!room:matrix-qa.test", + }, + ], }, - }, - ], + }), + scenarios: [ + { + id: "matrix-room-thread-reply-override", + title: "Matrix threadReplies always keeps room replies threaded", + config: liveTesting.buildMatrixQaConfigSnapshot({ + driverUserId: "@driver:matrix-qa.test", + observerUserId: "@observer:matrix-qa.test", + overrides: { + threadReplies: "always", + }, + sutUserId: "@sut:matrix-qa.test", + topology: { + defaultRoomId: "!room:matrix-qa.test", + defaultRoomKey: "main", + rooms: [ + { + key: "main", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Matrix QA", + requireMention: true, + roomId: "!room:matrix-qa.test", + }, + ], + }, + }), + }, + ], + }, + finishedAt: "2026-04-10T10:05:00.000Z", + harness: { + baseUrl: "http://127.0.0.1:28008/", + composeFile: "/tmp/docker-compose.yml", + dmRoomIds: [], + image: "ghcr.io/matrix-construct/tuwunel:v1.5.1", + roomId: "!room:matrix-qa.test", + roomIds: ["!room:matrix-qa.test"], + serverName: "matrix-qa.test", + }, + observedEventCount: 0, + scenarios: [], + startedAt: "2026-04-10T10:00:00.000Z", + sutAccountId: "sut", + timings: { + artifactWriteMs: 5, + canaryMs: 40, + harnessBootMs: 100, + initialGatewayBootMs: 200, + provisioningMs: 300, + scenarioGatewayBootMs: 50, + scenarioRestartGatewayMs: 60, + scenarioTransportInterruptMs: 70, + scenarios: [], + totalMs: 825, + }, + userIds: { + driver: "@driver:matrix-qa.test", + observer: "@observer:matrix-qa.test", + sut: "@sut:matrix-qa.test", + }, }); + const config = summary.config; + expect(config.default.replyToMode).toBe("off"); + expect(config.default.threadReplies).toBe("inbound"); + expect(config.scenarios).toHaveLength(1); + expect(config.scenarios[0]?.id).toBe("matrix-room-thread-reply-override"); + expect(config.scenarios[0]?.config.threadReplies).toBe("always"); }); it("preserves negative-scenario artifacts in the Matrix summary", () => { - expect( - liveTesting.buildMatrixQaSummary( - buildMatrixQaSummaryInput({ + const summary = liveTesting.buildMatrixQaSummary( + buildMatrixQaSummaryInput({ + scenarios: [ + { + id: "matrix-mention-gating", + title: "Matrix room message without mention does not trigger", + status: "pass", + details: "no reply", + artifacts: { + actorUserId: "@driver:matrix-qa.test", + driverEventId: "$driver", + expectedNoReplyWindowMs: 8_000, + token: "MATRIX_QA_NOMENTION_TOKEN", + triggerBody: "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN", + }, + }, + ], + timings: { scenarios: [ { + durationMs: 80, + gatewayBootMs: 0, + gatewayRestartMs: 0, id: "matrix-mention-gating", title: "Matrix room message without mention does not trigger", - status: "pass", - details: "no reply", - artifacts: { - actorUserId: "@driver:matrix-qa.test", - driverEventId: "$driver", - expectedNoReplyWindowMs: 8_000, - token: "MATRIX_QA_NOMENTION_TOKEN", - triggerBody: "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN", - }, + transportInterruptMs: 0, }, ], - timings: { - scenarios: [ - { - durationMs: 80, - gatewayBootMs: 0, - gatewayRestartMs: 0, - id: "matrix-mention-gating", - title: "Matrix room message without mention does not trigger", - transportInterruptMs: 0, - }, - ], - totalMs: 905, - }, - }), - ), - ).toMatchObject({ - counts: { - total: 2, - passed: 2, - failed: 0, - }, - scenarios: [ - { - id: "matrix-mention-gating", - artifacts: { - actorUserId: "@driver:matrix-qa.test", - expectedNoReplyWindowMs: 8_000, - triggerBody: "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN", - }, + totalMs: 905, }, - ], - timings: { - totalMs: 905, - }, - }); + }), + ); + expect(summary.counts.total).toBe(2); + expect(summary.counts.passed).toBe(2); + expect(summary.counts.failed).toBe(0); + expect(summary.scenarios[0]?.id).toBe("matrix-mention-gating"); + expect(summary.scenarios[0]?.artifacts?.actorUserId).toBe("@driver:matrix-qa.test"); + expect(summary.scenarios[0]?.artifacts?.expectedNoReplyWindowMs).toBe(8_000); + expect(summary.scenarios[0]?.artifacts?.triggerBody).toBe( + "reply with only this exact marker: MATRIX_QA_NOMENTION_TOKEN", + ); + expect(summary.timings.totalMs).toBe(905); }); it("keeps failing Matrix scenario details and timings complete in summary + report output", () => { @@ -468,28 +448,14 @@ describe("matrix live qa runtime", () => { }), ); - expect(summary).toMatchObject({ - counts: { - total: 2, - passed: 1, - failed: 1, - }, - scenarios: [ - { - id: "matrix-reaction-not-a-reply", - status: "fail", - details: expect.stringContaining("reaction event: $reaction"), - }, - ], - timings: { - scenarios: [ - { - id: "matrix-reaction-not-a-reply", - durationMs: 8_000, - }, - ], - }, - }); + expect(summary.counts.total).toBe(2); + expect(summary.counts.passed).toBe(1); + expect(summary.counts.failed).toBe(1); + expect(summary.scenarios[0]?.id).toBe("matrix-reaction-not-a-reply"); + expect(summary.scenarios[0]?.status).toBe("fail"); + expect(summary.scenarios[0]?.details).toContain("reaction event: $reaction"); + expect(summary.timings.scenarios[0]?.id).toBe("matrix-reaction-not-a-reply"); + expect(summary.timings.scenarios[0]?.durationMs).toBe(8_000); const report = renderQaMarkdownReport({ title: "Matrix QA Report", diff --git a/extensions/qa-matrix/src/substrate/client.test.ts b/extensions/qa-matrix/src/substrate/client.test.ts index 4037fabea10..75791c91dbd 100644 --- a/extensions/qa-matrix/src/substrate/client.test.ts +++ b/extensions/qa-matrix/src/substrate/client.test.ts @@ -154,18 +154,15 @@ describe("matrix driver client", () => { fetchImpl, }); - await expect( - client.loginWithPassword({ - deviceName: "OpenClaw Matrix QA Stale Device", - password: "driver-password", - userId: "@qa-driver:matrix-qa.test", - }), - ).resolves.toMatchObject({ - accessToken: "secondary-token", - deviceId: "SECONDARYDEVICE", + const login = await client.loginWithPassword({ + deviceName: "OpenClaw Matrix QA Stale Device", password: "driver-password", userId: "@qa-driver:matrix-qa.test", }); + expect(login.accessToken).toBe("secondary-token"); + expect(login.deviceId).toBe("SECONDARYDEVICE"); + expect(login.password).toBe("driver-password"); + expect(login.userId).toBe("@qa-driver:matrix-qa.test"); expect(requests).toEqual([ { @@ -306,12 +303,11 @@ describe("matrix driver client", () => { expect(requests[0]?.url).toContain( "/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/send/m.room.message/", ); - expect(requests[0]?.body).toMatchObject({ - "m.relates_to": { - rel_type: "m.replace", - event_id: "$msg-1", - }, - }); + const relation = requests[0]?.body?.["m.relates_to"] as + | { event_id?: string; rel_type?: string } + | undefined; + expect(relation?.rel_type).toBe("m.replace"); + expect(relation?.event_id).toBe("$msg-1"); expect(requests[1]?.url).toMatch( /^http:\/\/127\.0\.0\.1:28008\/_matrix\/client\/v3\/rooms\/!room%3Amatrix-qa\.test\/redact\/%24reaction-1\/[0-9a-f-]{36}$/, ); @@ -376,21 +372,15 @@ describe("matrix driver client", () => { expect(requests[1]?.url).toContain( "/_matrix/client/v3/rooms/!room%3Amatrix-qa.test/send/m.room.message/", ); - expect( - typeof requests[1]?.body === "string" ? JSON.parse(requests[1].body) : requests[1]?.body, - ).toMatchObject({ - body: "@sut:matrix-qa.test Image understanding check", - msgtype: "m.image", - filename: "red-top-blue-bottom.png", - url: "mxc://matrix-qa.test/red-top-blue-bottom", - info: { - mimetype: "image/png", - size: "png-bytes".length, - }, - "m.mentions": { - user_ids: ["@sut:matrix-qa.test"], - }, - }); + const messageBody = + typeof requests[1]?.body === "string" ? JSON.parse(requests[1].body) : requests[1]?.body; + expect(messageBody.body).toBe("@sut:matrix-qa.test Image understanding check"); + expect(messageBody.msgtype).toBe("m.image"); + expect(messageBody.filename).toBe("red-top-blue-bottom.png"); + expect(messageBody.url).toBe("mxc://matrix-qa.test/red-top-blue-bottom"); + expect(messageBody.info?.mimetype).toBe("image/png"); + expect(messageBody.info?.size).toBe("png-bytes".length); + expect(messageBody["m.mentions"]?.user_ids).toEqual(["@sut:matrix-qa.test"]); }); it("adds Matrix room encryption state when provisioning encrypted QA rooms", async () => {