Files
shivammittal274 d383b5e344 feat(eval): add claude-generated run report artifact (#892)
* feat(eval): add claude-generated run report artifact

* fix(eval): install claude code cli for CI evals

* fix(eval): bypass claude code tool permissions

* Eval metrics configs (#932)

* feat(eval): add agisdk comparison metrics configs

* fix(eval): keep cdp crashes from aborting run
2026-05-04 21:09:06 +05:30

199 lines
5.7 KiB
TypeScript

import { afterEach, beforeEach, describe, it } from 'bun:test'
import assert from 'node:assert'
import { TIMEOUTS } from '@browseros/shared/constants/timeouts'
import { CdpBackend } from '../../../src/browser/backends/cdp'
class MockWebSocket {
static instances: MockWebSocket[] = []
onopen: (() => void) | null = null
onerror: ((event: unknown) => void) | null = null
onclose: (() => void) | null = null
onmessage: ((event: { data: string }) => void) | null = null
readonly url: string
sent: string[] = []
constructor(url: string) {
this.url = url
MockWebSocket.instances.push(this)
}
send(data: string): void {
this.sent.push(data)
}
close(): void {
this.onclose?.()
}
open(): void {
this.onopen?.()
}
}
async function waitFor(
predicate: () => boolean,
timeoutMs = 1_000,
): Promise<void> {
const start = Date.now()
while (Date.now() - start < timeoutMs) {
if (predicate()) return
await Bun.sleep(1)
}
throw new Error('Timed out waiting for condition')
}
describe('CdpBackend', () => {
const originalFetch = globalThis.fetch
const originalWebSocket = globalThis.WebSocket
const originalConnectTimeout = TIMEOUTS.CDP_CONNECT
const originalReconnectDelay = TIMEOUTS.CDP_RECONNECT_DELAY
let fetchUrls: string[] = []
let failIpv4Discovery = false
let failAllDiscovery = false
let wsHost = '127.0.0.1'
let originalExit: typeof process.exit
beforeEach(() => {
MockWebSocket.instances = []
fetchUrls = []
failIpv4Discovery = false
failAllDiscovery = false
wsHost = '127.0.0.1'
originalExit = process.exit
;(TIMEOUTS as unknown as { CDP_CONNECT: number }).CDP_CONNECT = 200
;(
TIMEOUTS as unknown as { CDP_RECONNECT_DELAY: number }
).CDP_RECONNECT_DELAY = 1
globalThis.fetch = (async (input: string | URL | Request) => {
const url = String(input)
fetchUrls.push(url)
if (failAllDiscovery) {
throw new Error('Unable to connect')
}
if (failIpv4Discovery && url.includes('127.0.0.1')) {
throw new Error('Unable to connect')
}
const id = fetchUrls.length
return {
ok: true,
status: 200,
statusText: 'OK',
json: async () => ({
webSocketDebuggerUrl: `ws://${wsHost}:9222/devtools/browser/${id}`,
}),
} as Response
}) as typeof fetch
globalThis.WebSocket = MockWebSocket as unknown as typeof WebSocket
})
afterEach(() => {
globalThis.fetch = originalFetch
globalThis.WebSocket = originalWebSocket
process.exit = originalExit
;(TIMEOUTS as unknown as { CDP_CONNECT: number }).CDP_CONNECT =
originalConnectTimeout
;(
TIMEOUTS as unknown as { CDP_RECONNECT_DELAY: number }
).CDP_RECONNECT_DELAY = originalReconnectDelay
})
it('falls back from 127.0.0.1 to localhost for /json/version', async () => {
failIpv4Discovery = true
wsHost = 'localhost'
const cdp = new CdpBackend({ port: 9222 })
const connectPromise = cdp.connect()
await waitFor(() => MockWebSocket.instances.length === 1)
MockWebSocket.instances[0]?.open()
await connectPromise
assert.strictEqual(fetchUrls[0], 'http://127.0.0.1:9222/json/version')
assert.strictEqual(fetchUrls[1], 'http://localhost:9222/json/version')
assert.strictEqual(
MockWebSocket.instances[0]?.url,
'ws://localhost:9222/devtools/browser/2',
)
await cdp.disconnect()
})
it('prefers the last successful discovery host during reconnect', async () => {
failIpv4Discovery = true
wsHost = 'localhost'
const cdp = new CdpBackend({ port: 9222 })
const connectPromise = cdp.connect()
await waitFor(() => MockWebSocket.instances.length === 1)
const ws1 = MockWebSocket.instances[0]
ws1?.open()
await connectPromise
assert.strictEqual(fetchUrls[0], 'http://127.0.0.1:9222/json/version')
assert.strictEqual(fetchUrls[1], 'http://localhost:9222/json/version')
ws1?.close()
await waitFor(() => fetchUrls.length >= 3)
assert.strictEqual(fetchUrls[2], 'http://localhost:9222/json/version')
await cdp.disconnect()
})
it('reconnects again when a socket closes during reconnect', async () => {
const cdp = new CdpBackend({ port: 9222 })
const connectPromise = cdp.connect()
await waitFor(() => MockWebSocket.instances.length === 1)
const ws1 = MockWebSocket.instances[0]
ws1?.open()
await connectPromise
assert.strictEqual(cdp.isConnected(), true)
ws1?.close()
await waitFor(() => MockWebSocket.instances.length >= 2)
const ws2 = MockWebSocket.instances[1]
ws2?.open()
ws2?.close()
await waitFor(() => MockWebSocket.instances.length >= 3)
const ws3 = MockWebSocket.instances[2]
ws3?.open()
await waitFor(() => cdp.isConnected())
assert.strictEqual(cdp.isConnected(), true)
assert(fetchUrls.length >= 3)
await cdp.disconnect()
})
it('can disable process exit when reconnect retries are exhausted', async () => {
let exitCalled = false
process.exit = (() => {
exitCalled = true
throw new Error('process.exit should not be called')
}) as unknown as typeof process.exit
const cdp = new CdpBackend({ port: 9222, exitOnReconnectFailure: false })
const connectPromise = cdp.connect()
await waitFor(() => MockWebSocket.instances.length === 1)
const ws1 = MockWebSocket.instances[0]
ws1?.open()
await connectPromise
assert.strictEqual(cdp.isConnected(), true)
failAllDiscovery = true
ws1?.close()
await waitFor(() => fetchUrls.length >= 10)
await Bun.sleep(5)
assert.strictEqual(exitCalled, false)
assert.strictEqual(cdp.isConnected(), false)
await cdp.disconnect()
})
})