diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 310ba191f..fa88fb925 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,15 +1,44 @@ name: Tests -on: [] +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + paths: + - .github/workflows/test.yml + - packages/browseros-agent/** + workflow_dispatch: + +permissions: + contents: read + +env: + BROWSEROS_APPIMAGE_URL: https://files.browseros.com/download/BrowserOS.AppImage jobs: test: - name: Run Tests - runs-on: macos-latest - timeout-minutes: 10 + name: Tests / ${{ matrix.suite }} + runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: packages/browseros-agent + strategy: + fail-fast: false + matrix: + include: + - suite: tools + test_path: tests/tools + junit_path: test-results/tools.xml + - suite: integration + test_path: tests/server.integration.test.ts + junit_path: test-results/integration.xml + - suite: sdk + test_path: tests/sdk + junit_path: test-results/sdk.xml steps: - name: Checkout code @@ -21,7 +50,91 @@ jobs: - name: Install dependencies run: bun ci - - name: Run all tests - run: bun test:all + - name: Resolve BrowserOS cache key + id: browseros-cache-key + run: | + set -euo pipefail + headers="$(curl -fsSI "$BROWSEROS_APPIMAGE_URL")" + etag="$(printf '%s\n' "$headers" | awk 'BEGIN{IGNORECASE=1} /^etag:/ {sub(/\r$/, "", $2); gsub(/"/, "", $2); print $2; exit}')" + last_modified="$(printf '%s\n' "$headers" | awk 'BEGIN{IGNORECASE=1} /^last-modified:/ {$1=""; sub(/^ /, ""); sub(/\r$/, ""); print; exit}')" + raw_key="${etag:-$last_modified}" + if [ -z "$raw_key" ]; then + raw_key="$BROWSEROS_APPIMAGE_URL" + fi + cache_key="$(printf '%s' "$raw_key" | shasum -a 256 | awk '{print $1}')" + echo "key=browseros-appimage-${{ runner.os }}-$cache_key" >> "$GITHUB_OUTPUT" + + - name: Restore BrowserOS cache + id: browseros-cache + uses: actions/cache@v4 + with: + path: packages/browseros-agent/.ci/bin/BrowserOS.AppImage + key: ${{ steps.browseros-cache-key.outputs.key }} + + - name: Download BrowserOS + if: steps.browseros-cache.outputs.cache-hit != 'true' + run: | + mkdir -p .ci/bin + curl -fsSL "$BROWSEROS_APPIMAGE_URL" -o .ci/bin/BrowserOS.AppImage + chmod +x .ci/bin/BrowserOS.AppImage + + - name: Prepare BrowserOS wrapper + run: | + mkdir -p .ci/bin + cat > .ci/bin/browseros <<'EOF' + #!/usr/bin/env bash + set -euo pipefail + export APPIMAGE_EXTRACT_AND_RUN=1 + exec "$(dirname "$0")/BrowserOS.AppImage" "$@" + EOF + chmod +x .ci/bin/browseros + + - name: Create server env file + working-directory: packages/browseros-agent/apps/server + run: cp .env.example .env.development + + - name: Run ${{ matrix.suite }} tests + id: test env: - PUPPETEER_EXECUTABLE_PATH: /Applications/Google Chrome.app/Contents/MacOS/Google Chrome + BROWSEROS_BINARY: ${{ github.workspace }}/packages/browseros-agent/.ci/bin/browseros + BROWSEROS_TEST_HEADLESS: "true" + BROWSEROS_TEST_EXTRA_ARGS: --no-sandbox --disable-dev-shm-usage + run: | + set +e + mkdir -p test-results + cd apps/server + bun run test:cleanup + bun --env-file=.env.development test "${{ matrix.test_path }}" --reporter=junit --reporter-outfile="../../${{ matrix.junit_path }}" + exit_code=$? + cd ../.. + if [ ! -f "${{ matrix.junit_path }}" ]; then + cat > "${{ matrix.junit_path }}" < + + + + See workflow logs for details. + + + + EOF + fi + echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" + + - name: Upload JUnit XML + if: always() + uses: actions/upload-artifact@v4 + with: + name: junit-${{ matrix.suite }} + path: packages/browseros-agent/${{ matrix.junit_path }} + + - name: Summarize suite result + if: always() + run: | + if [ "${{ steps.test.outputs.exit_code }}" = "0" ]; then + echo "### :white_check_mark: ${{ matrix.suite }} suite passed" >> "$GITHUB_STEP_SUMMARY" + else + echo "### :x: ${{ matrix.suite }} suite failed (exit code ${{ steps.test.outputs.exit_code }})" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "See the uploaded \`junit-${{ matrix.suite }}\` artifact for details." >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/packages/browseros-agent/.gitignore b/packages/browseros-agent/.gitignore index da255e579..67cacb882 100644 --- a/packages/browseros-agent/.gitignore +++ b/packages/browseros-agent/.gitignore @@ -187,6 +187,10 @@ log.txt # Testing iteration temp files tmp/ +# CI artifacts +.ci/ +test-results/ + # Coding agent artifacts .agent/ .llm/ diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts b/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts index d6d45b925..f4c98c7ff 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/browser.ts @@ -16,6 +16,7 @@ export interface BrowserConfig { binaryPath: string userDataDir: string headless: boolean + extraArgs: string[] } interface BrowserState { @@ -26,6 +27,12 @@ interface BrowserState { let browserState: BrowserState | null = null +function shouldLogBrowserOutput(): boolean { + return ( + process.env.CI === 'true' || process.env.BROWSEROS_TEST_DEBUG === 'true' + ) +} + export async function isBrowserRunning(cdpPort: number): Promise { try { const response = await fetch(`http://127.0.0.1:${cdpPort}/json/version`, { @@ -74,6 +81,7 @@ export async function spawnBrowser( '--show-component-extension-options', '--enable-logging=stderr', ...(config.headless ? ['--headless=new'] : []), + ...config.extraArgs, `--user-data-dir=${config.userDataDir}`, // TODO: replace with --browseros-cdp-port once we fix the browseros bug `--remote-debugging-port=${config.cdpPort}`, @@ -86,14 +94,18 @@ export async function spawnBrowser( }, ) - browserProcess.stdout?.on('data', (_data) => { - // Uncomment for debugging - // console.log(`[BROWSER] ${_data.toString().trim()}`) + browserProcess.stdout?.on('data', (data) => { + if (!shouldLogBrowserOutput()) { + return + } + console.log(`[BROWSER] ${data.toString().trim()}`) }) - browserProcess.stderr?.on('data', (_data) => { - // Uncomment for debugging - // console.log(`[BROWSER] ${_data.toString().trim()}`) + browserProcess.stderr?.on('data', (data) => { + if (!shouldLogBrowserOutput()) { + return + } + console.error(`[BROWSER] ${data.toString().trim()}`) }) browserProcess.on('error', (error) => { diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts b/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts index 2872345ba..9f43b8bab 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/setup.ts @@ -132,6 +132,7 @@ export async function ensureBrowserOS( binaryPath: runtimePlan.binaryPath, userDataDir: runtimePlan.userDataDir, headless: runtimePlan.headless, + extraArgs: runtimePlan.extraArgs, } await spawnBrowser(browserConfig) diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts b/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts index 47b80d5e6..85f10abc2 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/test-runtime.ts @@ -20,9 +20,20 @@ export interface TestRuntimePlan { userDataDir: string binaryPath: string headless: boolean + extraArgs: string[] usesFixedPorts: boolean } +function parseExtraArgs(value: string | undefined): string[] { + if (!value) { + return [] + } + return value + .split(/\s+/) + .map((part) => part.trim()) + .filter((part) => part.length > 0) +} + function parsePort( value: string | undefined, envName: string, @@ -137,12 +148,14 @@ export async function createTestRuntimePlan(): Promise { const resolvedPorts = await resolveRuntimePorts() const userDataDir = mkdtempSync(join(tmpdir(), 'browseros-test-')) const headless = process.env.BROWSEROS_TEST_HEADLESS === 'true' + const extraArgs = parseExtraArgs(process.env.BROWSEROS_TEST_EXTRA_ARGS) return { ports: resolvedPorts.ports, userDataDir, binaryPath: DEFAULT_BINARY_PATH, headless, + extraArgs, usesFixedPorts: resolvedPorts.usesFixedPorts, } } diff --git a/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts b/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts index b8556cdd6..7afec4abc 100644 --- a/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts +++ b/packages/browseros-agent/apps/server/tests/__helpers__/with-browser.ts @@ -46,6 +46,7 @@ async function getOrCreateBrowser(): Promise { binaryPath: runtimePlan.binaryPath, userDataDir: runtimePlan.userDataDir, headless: runtimePlan.headless, + extraArgs: runtimePlan.extraArgs, } await spawnBrowser(config)