feat: make server tests use dynamic browser runtime allocation (#416)

* feat: use dynamic runtime allocation for server test browser startup

* fix: address PR review comments for sdk_test_dev_runner_migration

apps/server/tests/__helpers__/browser.ts

@@ -7,20 +7,20 @@
  */
 import type { ChildProcess } from 'node:child_process'
 import { spawn } from 'node:child_process'
-import { mkdtempSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
+import { rmSync } from 'node:fs'
 
 export interface BrowserConfig {
   cdpPort: number
   serverPort: number
   extensionPort: number
   binaryPath: string
+  userDataDir: string
+  headless: boolean
 }
 
 interface BrowserState {
   process: ChildProcess
-  tempUserDataDir: string
+  userDataDir: string
   config: BrowserConfig
 }
 
@@ -66,11 +66,6 @@ export async function spawnBrowser(
     await killBrowser()
   }
 
-  const tempUserDataDir = mkdtempSync(join(tmpdir(), 'browseros-test-'))
-  console.log(`Created temp profile: ${tempUserDataDir}`)
-
-  const headless = process.env.BROWSEROS_TEST_HEADLESS === 'true'
-
   console.log(`Starting BrowserOS on CDP port ${config.cdpPort}...`)
   const browserProcess = spawn(
     config.binaryPath,
@@ -78,8 +73,8 @@ export async function spawnBrowser(
       '--use-mock-keychain',
       '--show-component-extension-options',
       '--enable-logging=stderr',
-      ...(headless ? ['--headless=new'] : []),
-      `--user-data-dir=${tempUserDataDir}`,
+      ...(config.headless ? ['--headless=new'] : []),
+      `--user-data-dir=${config.userDataDir}`,
       // TODO: replace with --browseros-cdp-port once we fix the browseros bug
       `--remote-debugging-port=${config.cdpPort}`,
       `--browseros-mcp-port=${config.serverPort}`,
@@ -109,7 +104,11 @@ export async function spawnBrowser(
   await waitForCdp(config.cdpPort)
   console.log('CDP is ready')
 
-  browserState = { process: browserProcess, tempUserDataDir, config }
+  browserState = {
+    process: browserProcess,
+    userDataDir: config.userDataDir,
+    config,
+  }
   return browserState
 }
 
@@ -135,10 +134,10 @@ export async function killBrowser(): Promise<void> {
 
   console.log('BrowserOS stopped')
 
-  if (browserState.tempUserDataDir) {
-    console.log(`Cleaning up temp profile: ${browserState.tempUserDataDir}`)
+  if (browserState.userDataDir) {
+    console.log(`Cleaning up temp profile: ${browserState.userDataDir}`)
     try {
-      rmSync(browserState.tempUserDataDir, { recursive: true, force: true })
+      rmSync(browserState.userDataDir, { recursive: true, force: true })
     } catch (error) {
       console.error('Failed to clean up temp directory:', error)
     }

apps/server/tests/__helpers__/index.ts

@@ -10,4 +10,8 @@ export {
   killProcessOnPort,
   withMcpServer,
 } from './utils'
-export { type WithBrowserContext, withBrowser } from './with-browser'
+export {
+  cleanupWithBrowser,
+  type WithBrowserContext,
+  withBrowser,
+} from './with-browser'

apps/server/tests/__helpers__/setup.ts

@@ -5,8 +5,6 @@
  * Unified test environment orchestrator.
  * Ensures server + browser + extension are all ready.
  */
-import { TEST_PORTS } from '@browseros/shared/constants/ports'
-
 import {
   type BrowserConfig,
   getBrowserState,
@@ -14,6 +12,7 @@ import {
   spawnBrowser,
 } from './browser'
 import { getServerState, killServer, spawnServer } from './server'
+import { createTestRuntimePlan, type TestRuntimePlan } from './test-runtime'
 import { killProcessOnPort } from './utils'
 
 export interface TestEnvironmentConfig {
@@ -23,24 +22,7 @@ export interface TestEnvironmentConfig {
   skipExtension?: boolean
 }
 
-const DEFAULT_CONFIG: TestEnvironmentConfig = {
-  cdpPort: Number.parseInt(
-    process.env.BROWSEROS_CDP_PORT || String(TEST_PORTS.cdp),
-    10,
-  ),
-  serverPort: Number.parseInt(
-    process.env.BROWSEROS_SERVER_PORT || String(TEST_PORTS.server),
-    10,
-  ),
-  extensionPort: Number.parseInt(
-    process.env.BROWSEROS_EXTENSION_PORT || String(TEST_PORTS.extension),
-    10,
-  ),
-}
-
-const DEFAULT_BINARY_PATH =
-  process.env.BROWSEROS_BINARY ??
-  '/Applications/BrowserOS.app/Contents/MacOS/BrowserOS'
+let runtimePlan: TestRuntimePlan | null = null
 
 async function isExtensionConnected(port: number): Promise<boolean> {
   try {
@@ -94,10 +76,14 @@ function configsMatch(
 export async function ensureBrowserOS(
   options?: Partial<TestEnvironmentConfig>,
 ): Promise<TestEnvironmentConfig> {
+  if (!runtimePlan) {
+    runtimePlan = await createTestRuntimePlan()
+  }
+
   const config: TestEnvironmentConfig = {
-    cdpPort: options?.cdpPort ?? DEFAULT_CONFIG.cdpPort,
-    serverPort: options?.serverPort ?? DEFAULT_CONFIG.serverPort,
-    extensionPort: options?.extensionPort ?? DEFAULT_CONFIG.extensionPort,
+    cdpPort: options?.cdpPort ?? runtimePlan.ports.cdp,
+    serverPort: options?.serverPort ?? runtimePlan.ports.server,
+    extensionPort: options?.extensionPort ?? runtimePlan.ports.extension,
     skipExtension: options?.skipExtension ?? false,
   }
 
@@ -140,16 +126,18 @@ export async function ensureBrowserOS(
   await killProcessOnPort(config.extensionPort)
   await killProcessOnPort(config.cdpPort)
 
-  // 2. Start server first (WebSocket ready for extension)
-  await spawnServer(config)
-
-  // 3. Start browser (extension will connect to server)
+  // 2. Start browser first so CDP is available before server startup.
   const browserConfig: BrowserConfig = {
     ...config,
-    binaryPath: DEFAULT_BINARY_PATH,
+    binaryPath: runtimePlan.binaryPath,
+    userDataDir: runtimePlan.userDataDir,
+    headless: runtimePlan.headless,
   }
   await spawnBrowser(browserConfig)
 
+  // 3. Start server once CDP is available.
+  await spawnServer(config)
+
   // 4. Wait for extension to connect (unless skipped for CDP-only tests)
   if (!config.skipExtension) {
     console.log('Waiting for extension to connect...')
@@ -170,5 +158,6 @@ export async function cleanupBrowserOS(): Promise<void> {
   console.log('\n=== Cleaning up BrowserOS test environment ===')
   await killBrowser()
   await killServer()
+  runtimePlan = null
   console.log('=== Cleanup complete ===\n')
 }

apps/server/tests/__helpers__/test-runtime.ts

@@ -0,0 +1,148 @@
+import { mkdtempSync } from 'node:fs'
+import { createServer } from 'node:net'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { TEST_PORTS } from '@browseros/shared/constants/ports'
+
+const DEFAULT_BINARY_PATH =
+  process.env.BROWSEROS_BINARY ??
+  '/Applications/BrowserOS.app/Contents/MacOS/BrowserOS'
+const PORT_SCAN_RANGE = 100
+
+export interface RuntimePorts {
+  cdp: number
+  server: number
+  extension: number
+}
+
+export interface TestRuntimePlan {
+  ports: RuntimePorts
+  userDataDir: string
+  binaryPath: string
+  headless: boolean
+  usesFixedPorts: boolean
+}
+
+function parsePort(
+  value: string | undefined,
+  envName: string,
+): number | undefined {
+  if (!value) {
+    return undefined
+  }
+
+  const parsed = Number.parseInt(value, 10)
+  if (!Number.isInteger(parsed) || parsed <= 0 || parsed > 65535) {
+    throw new Error(`Invalid ${envName} value: ${value}`)
+  }
+  return parsed
+}
+
+async function isPortAvailable(port: number): Promise<boolean> {
+  return await new Promise<boolean>((resolve) => {
+    const server = createServer()
+    server.unref()
+    server.once('error', () => resolve(false))
+    server.listen(port, () => {
+      server.close(() => resolve(true))
+    })
+  })
+}
+
+async function findAvailablePort(
+  startPort: number,
+  reserved: Set<number>,
+): Promise<number> {
+  for (let port = startPort; port < startPort + PORT_SCAN_RANGE; port++) {
+    if (reserved.has(port)) {
+      continue
+    }
+    if (await isPortAvailable(port)) {
+      reserved.add(port)
+      return port
+    }
+  }
+  throw new Error(`Failed to find available port near ${startPort}`)
+}
+
+function resolveFixedPort(
+  testEnvName:
+    | 'BROWSEROS_TEST_CDP_PORT'
+    | 'BROWSEROS_TEST_SERVER_PORT'
+    | 'BROWSEROS_TEST_EXTENSION_PORT',
+  baseEnvName:
+    | 'BROWSEROS_CDP_PORT'
+    | 'BROWSEROS_SERVER_PORT'
+    | 'BROWSEROS_EXTENSION_PORT',
+): number | undefined {
+  const testPort = parsePort(process.env[testEnvName], testEnvName)
+  if (testPort !== undefined) {
+    return testPort
+  }
+  if (process.env.BROWSEROS_TEST_USE_ENV_PORTS === 'true') {
+    return parsePort(process.env[baseEnvName], baseEnvName)
+  }
+  return undefined
+}
+
+function assertUniquePorts(ports: RuntimePorts): void {
+  const values = new Set([ports.cdp, ports.server, ports.extension])
+  if (values.size !== 3) {
+    throw new Error(
+      `Port conflict detected: cdp=${ports.cdp} server=${ports.server} extension=${ports.extension}`,
+    )
+  }
+}
+
+export async function resolveRuntimePorts(): Promise<{
+  ports: RuntimePorts
+  usesFixedPorts: boolean
+}> {
+  const cdpOverride = resolveFixedPort(
+    'BROWSEROS_TEST_CDP_PORT',
+    'BROWSEROS_CDP_PORT',
+  )
+  const serverOverride = resolveFixedPort(
+    'BROWSEROS_TEST_SERVER_PORT',
+    'BROWSEROS_SERVER_PORT',
+  )
+  const extensionOverride = resolveFixedPort(
+    'BROWSEROS_TEST_EXTENSION_PORT',
+    'BROWSEROS_EXTENSION_PORT',
+  )
+
+  const reserved = new Set<number>()
+  const cdp = cdpOverride ?? (await findAvailablePort(TEST_PORTS.cdp, reserved))
+  reserved.add(cdp)
+  const server =
+    serverOverride ?? (await findAvailablePort(TEST_PORTS.server, reserved))
+  reserved.add(server)
+  const extension =
+    extensionOverride ??
+    (await findAvailablePort(TEST_PORTS.extension, reserved))
+
+  const ports = { cdp, server, extension }
+  assertUniquePorts(ports)
+
+  return {
+    ports,
+    usesFixedPorts:
+      cdpOverride !== undefined ||
+      serverOverride !== undefined ||
+      extensionOverride !== undefined,
+  }
+}
+
+export async function createTestRuntimePlan(): Promise<TestRuntimePlan> {
+  const resolvedPorts = await resolveRuntimePorts()
+  const userDataDir = mkdtempSync(join(tmpdir(), 'browseros-test-'))
+  const headless = process.env.BROWSEROS_TEST_HEADLESS === 'true'
+
+  return {
+    ports: resolvedPorts.ports,
+    userDataDir,
+    binaryPath: DEFAULT_BINARY_PATH,
+    headless,
+    usesFixedPorts: resolvedPorts.usesFixedPorts,
+  }
+}

apps/server/tests/__helpers__/with-browser.ts

@@ -1,4 +1,4 @@
-import { TEST_PORTS } from '@browseros/shared/constants/ports'
+import { existsSync } from 'node:fs'
 import { Mutex } from 'async-mutex'
 import { CdpBackend } from '../../src/browser/backends/cdp'
 import type { ControllerBackend } from '../../src/browser/backends/types'
@@ -6,28 +6,14 @@ import { Browser } from '../../src/browser/browser'
 import type { ToolDefinition } from '../../src/tools/framework'
 import { executeTool } from '../../src/tools/framework'
 import type { ToolResult } from '../../src/tools/response'
-import { type BrowserConfig, spawnBrowser } from './browser'
+import { type BrowserConfig, killBrowser, spawnBrowser } from './browser'
+import { createTestRuntimePlan, type TestRuntimePlan } from './test-runtime'
 import { killProcessOnPort } from './utils'
 
-const cdpPort = Number.parseInt(
-  process.env.BROWSEROS_CDP_PORT || String(TEST_PORTS.cdp),
-  10,
-)
-const serverPort = Number.parseInt(
-  process.env.BROWSEROS_SERVER_PORT || String(TEST_PORTS.server),
-  10,
-)
-const extensionPort = Number.parseInt(
-  process.env.BROWSEROS_EXTENSION_PORT || String(TEST_PORTS.extension),
-  10,
-)
-const binaryPath =
-  process.env.BROWSEROS_BINARY ??
-  '/Applications/BrowserOS.app/Contents/MacOS/BrowserOS'
-
 const mutex = new Mutex()
 let cachedCdp: CdpBackend | null = null
 let cachedBrowser: Browser | null = null
+let runtimePlan: TestRuntimePlan | null = null
 
 const stubController: ControllerBackend = {
   start: async () => {},
@@ -41,23 +27,44 @@ const stubController: ControllerBackend = {
 async function getOrCreateBrowser(): Promise<Browser> {
   if (cachedBrowser && cachedCdp?.isConnected()) return cachedBrowser
 
-  await killProcessOnPort(cdpPort)
+  if (runtimePlan && !existsSync(runtimePlan.userDataDir)) {
+    runtimePlan = null
+  }
+
+  if (!runtimePlan) {
+    runtimePlan = await createTestRuntimePlan()
+  }
+
+  if (runtimePlan.usesFixedPorts) {
+    await killProcessOnPort(runtimePlan.ports.cdp)
+  }
 
   const config: BrowserConfig = {
-    cdpPort,
-    serverPort,
-    extensionPort,
-    binaryPath,
+    cdpPort: runtimePlan.ports.cdp,
+    serverPort: runtimePlan.ports.server,
+    extensionPort: runtimePlan.ports.extension,
+    binaryPath: runtimePlan.binaryPath,
+    userDataDir: runtimePlan.userDataDir,
+    headless: runtimePlan.headless,
   }
   await spawnBrowser(config)
 
-  cachedCdp = new CdpBackend({ port: cdpPort })
+  cachedCdp = new CdpBackend({ port: runtimePlan.ports.cdp })
   await cachedCdp.connect()
 
   cachedBrowser = new Browser(cachedCdp, stubController)
   return cachedBrowser
 }
 
+export async function cleanupWithBrowser(): Promise<void> {
+  await mutex.runExclusive(async () => {
+    await killBrowser()
+    cachedCdp = null
+    cachedBrowser = null
+    runtimePlan = null
+  })
+}
+
 export interface WithBrowserContext {
   browser: Browser
   execute: (tool: ToolDefinition, args: unknown) => Promise<ToolResult>