mirror of
https://github.com/browseros-ai/BrowserOS.git
synced 2026-05-18 19:16:22 +00:00
84a79ba0a1
* feat(eval): add suite variant config bridge * feat(eval): add stable run artifacts * refactor(eval): add shared grader contract * feat(eval): persist grader artifacts * refactor(eval): rename runner layers * refactor(eval): add executor backend boundary * refactor(eval): split clado backend * feat(eval): add workflow compatible cli * feat(eval): add r2 publisher module * ci(eval): migrate weekly workflow to eval cli * docs(eval): document suite pipeline * chore(eval): verify pipeline refactor * fix: address review feedback for PR #875 * docs(eval): add env example * docs(eval): explain suites and variants * chore(eval): organize config layouts * chore(eval): colocate grader python evaluators
114 lines
4.4 KiB
YAML
114 lines
4.4 KiB
YAML
name: Weekly Eval
|
|
|
|
on:
|
|
schedule:
|
|
# Every Saturday at 06:00 UTC
|
|
- cron: '0 6 * * 6'
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'packages/browseros-agent/apps/server/src/agent/**'
|
|
- 'packages/browseros-agent/apps/server/src/tools/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
config:
|
|
description: 'Eval config file (relative to apps/eval/)'
|
|
required: false
|
|
default: 'configs/legacy/browseros-agent-weekly.json'
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
eval:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 360
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install BrowserOS
|
|
run: |
|
|
# Rolling stable channel — see https://cdn.browseros.com/download/BrowserOS.deb
|
|
wget -q -O BrowserOS.deb https://cdn.browseros.com/download/BrowserOS.deb
|
|
sudo dpkg -i BrowserOS.deb
|
|
browseros --version || echo "BrowserOS installed at $(which browseros)"
|
|
|
|
- name: Install Bun
|
|
uses: oven-sh/setup-bun@v2
|
|
with:
|
|
bun-version: latest
|
|
|
|
- name: Install dependencies
|
|
working-directory: packages/browseros-agent
|
|
run: bun install --ignore-scripts
|
|
|
|
- name: Install Python eval dependencies
|
|
# agisdk pinned so silent upstream releases can't shift task definitions
|
|
# or grader behavior. Bump intentionally with a documented re-baseline.
|
|
run: pip install agisdk==0.3.5 requests
|
|
|
|
- name: Clone WebArena-Infinity
|
|
run: git clone --depth 1 https://github.com/web-arena-x/webarena-infinity.git /tmp/webarena-infinity
|
|
|
|
- name: Install xvfb
|
|
run: sudo apt-get update && sudo apt-get install -y xvfb
|
|
|
|
- name: Install captcha solver extension
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
run: |
|
|
mkdir -p extensions
|
|
curl -sL -o /tmp/nopecha.zip https://github.com/NopeCHALLC/nopecha-extension/releases/latest/download/chromium_automation.zip
|
|
unzip -qo /tmp/nopecha.zip -d extensions/nopecha
|
|
|
|
- name: Run eval and publish to R2
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
env:
|
|
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
|
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
|
NOPECHA_API_KEY: ${{ secrets.NOPECHA_API_KEY }}
|
|
EVAL_R2_ACCOUNT_ID: ${{ secrets.EVAL_R2_ACCOUNT_ID }}
|
|
EVAL_R2_ACCESS_KEY_ID: ${{ secrets.EVAL_R2_ACCESS_KEY_ID }}
|
|
EVAL_R2_SECRET_ACCESS_KEY: ${{ secrets.EVAL_R2_SECRET_ACCESS_KEY }}
|
|
EVAL_R2_BUCKET: ${{ secrets.EVAL_R2_BUCKET }}
|
|
EVAL_R2_CDN_BASE_URL: ${{ secrets.EVAL_R2_CDN_BASE_URL }}
|
|
BROWSEROS_BINARY: /usr/bin/browseros
|
|
WEBARENA_INFINITY_DIR: /tmp/webarena-infinity
|
|
# OpenClaw container runtime is macOS-only; opt the Linux runner
|
|
# into the no-op stub so the server can boot and the eval can run.
|
|
BROWSEROS_SKIP_OPENCLAW: '1'
|
|
EVAL_CONFIG: ${{ github.event.inputs.config || 'configs/legacy/browseros-agent-weekly.json' }}
|
|
run: |
|
|
echo "Running eval with config: $EVAL_CONFIG"
|
|
xvfb-run --auto-servernum --server-args="-screen 0 1440x900x24" bun run src/index.ts suite --config "$EVAL_CONFIG" --publish r2
|
|
|
|
- name: Generate trend report
|
|
if: success()
|
|
timeout-minutes: 5
|
|
continue-on-error: true
|
|
working-directory: packages/browseros-agent
|
|
env:
|
|
EVAL_R2_ACCOUNT_ID: ${{ secrets.EVAL_R2_ACCOUNT_ID }}
|
|
EVAL_R2_ACCESS_KEY_ID: ${{ secrets.EVAL_R2_ACCESS_KEY_ID }}
|
|
EVAL_R2_SECRET_ACCESS_KEY: ${{ secrets.EVAL_R2_SECRET_ACCESS_KEY }}
|
|
EVAL_R2_BUCKET: ${{ secrets.EVAL_R2_BUCKET }}
|
|
EVAL_R2_CDN_BASE_URL: ${{ secrets.EVAL_R2_CDN_BASE_URL }}
|
|
run: bun apps/eval/scripts/weekly-report.ts /tmp/eval-report.html
|
|
|
|
- name: Upload report as artifact
|
|
if: success()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: eval-report-${{ github.run_id }}
|
|
path: /tmp/eval-report.html
|
|
|
|
- name: Upload server stderr logs (for post-mortem on startup failures)
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: browseros-server-logs-${{ github.run_id }}
|
|
path: /tmp/browseros-server-logs/
|
|
if-no-files-found: ignore
|