From ba7892322b42f84304cb655ca0e697102bfb10b7 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sat, 21 Mar 2026 09:46:36 -0700 Subject: [PATCH] ci: run BrowserOS test suites on PRs (#514) * ci: run browseros tests on pull requests * refactor: rework 0320-github_action_for_tests based on feedback * refactor: rework 0320-github_action_for_tests based on feedback * chore: add CI artifacts to .gitignore Co-Authored-By: Claude Opus 4.6 (1M context) * fix: remove mikepenz/action-junit-report to fix check suite misattribution The JUnit report action creates check runs that GitHub associates with the CLA check suite instead of the Tests check suite, causing test reports to appear under "CLA Assistant" in the PR checks UI. Remove the action and rely on job status + step summary + artifact upload for test result visibility. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/test.yml | 127 +++++++++++++++++- packages/browseros-agent/.gitignore | 4 + .../apps/server/tests/__helpers__/browser.ts | 24 +++- .../apps/server/tests/__helpers__/setup.ts | 1 + .../server/tests/__helpers__/test-runtime.ts | 13 ++ .../server/tests/__helpers__/with-browser.ts | 1 + 6 files changed, 157 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 310ba191..fa88fb92 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,15 +1,44 @@ name: Tests -on: [] +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + paths: + - .github/workflows/test.yml + - packages/browseros-agent/** + workflow_dispatch: + +permissions: + contents: read + +env: + BROWSEROS_APPIMAGE_URL: https://files.browseros.com/download/BrowserOS.AppImage jobs: test: - name: Run Tests - runs-on: macos-latest - timeout-minutes: 10 + name: Tests / ${{ matrix.suite }} + runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: packages/browseros-agent + strategy: + fail-fast: false + matrix: + include: + - suite: tools + test_path: tests/tools + junit_path: test-results/tools.xml + - suite: integration + test_path: tests/server.integration.test.ts + junit_path: test-results/integration.xml + - suite: sdk + test_path: tests/sdk + junit_path: test-results/sdk.xml steps: - name: Checkout code @@ -21,7 +50,91 @@ jobs: - name: Install dependencies run: bun ci - - name: Run all tests - run: bun test:all + - name: Resolve BrowserOS cache key + id: browseros-cache-key + run: | + set -euo pipefail + headers="$(curl -fsSI "$BROWSEROS_APPIMAGE_URL")" + etag="$(printf '%s\n' "$headers" | awk 'BEGIN{IGNORECASE=1} /^etag:/ {sub(/\r$/, "", $2); gsub(/"/, "", $2); print $2; exit}')" + last_modified="$(printf '%s\n' "$headers" | awk 'BEGIN{IGNORECASE=1} /^last-modified:/ {$1=""; sub(/^ /, ""); sub(/\r$/, ""); print; exit}')" + raw_key="${etag:-$last_modified}" + if [ -z "$raw_key" ]; then + raw_key="$BROWSEROS_APPIMAGE_URL" + fi + cache_key="$(printf '%s' "$raw_key" | shasum -a 256 | awk '{print $1}')" + echo "key=browseros-appimage-${{ runner.os }}-$cache_key" >> "$GITHUB_OUTPUT" + + - name: Restore BrowserOS cache + id: browseros-cache + uses: actions/cache@v4 + with: + path: packages/browseros-agent/.ci/bin/BrowserOS.AppImage + key: ${{ steps.browseros-cache-key.outputs.key }} + + - name: Download BrowserOS + if: steps.browseros-cache.outputs.cache-hit != 'true' + run: | + mkdir -p .ci/bin + curl -fsSL "$BROWSEROS_APPIMAGE_URL" -o .ci/bin/BrowserOS.AppImage + chmod +x .ci/bin/BrowserOS.AppImage + + - name: Prepare BrowserOS wrapper + run: | + mkdir -p .ci/bin + cat > .ci/bin/browseros <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + export APPIMAGE_EXTRACT_AND_RUN=1 + exec "$(dirname "$0")/BrowserOS.AppImage" "$@" + EOF + chmod +x .ci/bin/browseros + + - name: Create server env file + working-directory: packages/browseros-agent/apps/server + run: cp .env.example .env.development + + - name: Run ${{ matrix.suite }} tests + id: test env: - PUPPETEER_EXECUTABLE_PATH: /Applications/Google Chrome.app/Contents/MacOS/Google Chrome + BROWSEROS_BINARY: ${{ github.workspace }}/packages/browseros-agent/.ci/bin/browseros + BROWSEROS_TEST_HEADLESS: "true" + BROWSEROS_TEST_EXTRA_ARGS: --no-sandbox --disable-dev-shm-usage + run: | + set +e + mkdir -p test-results + cd apps/server + bun run test:cleanup + bun --env-file=.env.development test "${{ matrix.test_path }}" --reporter=junit --reporter-outfile="../../${{ matrix.junit_path }}" + exit_code=$? + cd ../.. + if [ ! -f "${{ matrix.junit_path }}" ]; then + cat > "${{ matrix.junit_path }}" < + + + + See workflow logs for details. + + + + EOF + fi + echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" + + - name: Upload JUnit XML + if: always() + uses: actions/upload-artifact@v4 + with: + name: junit-${{ matrix.suite }} + path: packages/browseros-agent/${{ matrix.junit_path }} + + - name: Summarize suite result + if: always() + run: | + if [ "${{ steps.test.outputs.exit_code }}" = "0" ]; then + echo "### :white_check_mark: ${{ matrix.suite }} suite passed" >> "$GITHUB_STEP_SUMMARY" + else + echo "### :x: ${{ matrix.suite }} suite failed (exit code ${{ steps.test.outputs.exit_code }})" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "See the uploaded \`junit-${{ matrix.suite }}\` artifact for details." >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/packages/browseros-agent/.gitignore b/packages/browseros-agent/.gitignore index da255e57..67cacb88 100644 --- a/packages/browseros-agent/.gitignore +++ b/packages/browseros-agent/.gitignore @@ -187,6 +187,10 @@ log.txt # Testing iteration temp files tmp/ +# CI artifacts +.ci/ +test-results/ + # Coding agent artifacts .agent/ .llm/ diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts b/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts index d6d45b92..f4c98c7f 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts @@ -16,6 +16,7 @@ export interface BrowserConfig { binaryPath: string userDataDir: string headless: boolean + extraArgs: string[] } interface BrowserState { @@ -26,6 +27,12 @@ interface BrowserState { let browserState: BrowserState | null = null +function shouldLogBrowserOutput(): boolean { + return ( + process.env.CI === 'true' || process.env.BROWSEROS_TEST_DEBUG === 'true' + ) +} + export async function isBrowserRunning(cdpPort: number): Promise { try { const response = await fetch(`http://127.0.0.1:${cdpPort}/json/version`, { @@ -74,6 +81,7 @@ export async function spawnBrowser( '--show-component-extension-options', '--enable-logging=stderr', ...(config.headless ? ['--headless=new'] : []), + ...config.extraArgs, `--user-data-dir=${config.userDataDir}`, // TODO: replace with --browseros-cdp-port once we fix the browseros bug `--remote-debugging-port=${config.cdpPort}`, @@ -86,14 +94,18 @@ export async function spawnBrowser( }, ) - browserProcess.stdout?.on('data', (_data) => { - // Uncomment for debugging - // console.log(`[BROWSER] ${_data.toString().trim()}`) + browserProcess.stdout?.on('data', (data) => { + if (!shouldLogBrowserOutput()) { + return + } + console.log(`[BROWSER] ${data.toString().trim()}`) }) - browserProcess.stderr?.on('data', (_data) => { - // Uncomment for debugging - // console.log(`[BROWSER] ${_data.toString().trim()}`) + browserProcess.stderr?.on('data', (data) => { + if (!shouldLogBrowserOutput()) { + return + } + console.error(`[BROWSER] ${data.toString().trim()}`) }) browserProcess.on('error', (error) => { diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts b/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts index 2872345b..9f43b8ba 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts @@ -132,6 +132,7 @@ export async function ensureBrowserOS( binaryPath: runtimePlan.binaryPath, userDataDir: runtimePlan.userDataDir, headless: runtimePlan.headless, + extraArgs: runtimePlan.extraArgs, } await spawnBrowser(browserConfig) diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts b/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts index 47b80d5e..85f10abc 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts @@ -20,9 +20,20 @@ export interface TestRuntimePlan { userDataDir: string binaryPath: string headless: boolean + extraArgs: string[] usesFixedPorts: boolean } +function parseExtraArgs(value: string | undefined): string[] { + if (!value) { + return [] + } + return value + .split(/\s+/) + .map((part) => part.trim()) + .filter((part) => part.length > 0) +} + function parsePort( value: string | undefined, envName: string, @@ -137,12 +148,14 @@ export async function createTestRuntimePlan(): Promise { const resolvedPorts = await resolveRuntimePorts() const userDataDir = mkdtempSync(join(tmpdir(), 'browseros-test-')) const headless = process.env.BROWSEROS_TEST_HEADLESS === 'true' + const extraArgs = parseExtraArgs(process.env.BROWSEROS_TEST_EXTRA_ARGS) return { ports: resolvedPorts.ports, userDataDir, binaryPath: DEFAULT_BINARY_PATH, headless, + extraArgs, usesFixedPorts: resolvedPorts.usesFixedPorts, } } diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts b/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts index b8556cdd..7afec4ab 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts @@ -46,6 +46,7 @@ async function getOrCreateBrowser(): Promise { binaryPath: runtimePlan.binaryPath, userDataDir: runtimePlan.userDataDir, headless: runtimePlan.headless, + extraArgs: runtimePlan.extraArgs, } await spawnBrowser(config)