mirror of
https://github.com/browseros-ai/BrowserOS.git
synced 2026-05-13 15:46:22 +00:00
4a3b9ff294
* feat: add deterministic eval graders (AGI SDK + WebArena-Infinity) Two new benchmark integrations with programmatic grading — no LLM judge. AGI SDK / REAL Bench (52 tasks): - 11 React/Next.js clones of consumer apps (DoorDash, Amazon, Gmail, etc.) - Grader navigates browser to /finish, extracts state diff from <pre> tag - Python verifier checks exact values via jmespath queries WebArena-Infinity (50 hard tasks): - 13 LLM-generated SaaS clones (Gmail, GitLab, Linear, Figma, etc.) - InfinityAppManager starts fresh app server per task per worker - Python verifier calls /api/state and asserts on JSON state Infrastructure: - GraderInput extended with mcpUrl + infinityAppUrl for parallel workers - Each worker gets isolated ports (no cross-worker state contamination) - CI workflow: pip install agisdk, clone webarena-infinity repo * chore: switch eval configs back to kimi-k2p5 * fix: register deterministic graders in pass rate calculation Add agisdk_state_diff and infinity_state to PASS_FAIL_GRADER_ORDER in both runner types and weekly report script, so scores show correctly in the dashboard. * chore: temp switch to opus 4.6 for eval run * chore: restore kimi-k2p5 as default eval config * ci: add timeout and continue-on-error for trend report step
109 lines
4.0 KiB
YAML
109 lines
4.0 KiB
YAML
name: Weekly Eval
|
|
|
|
on:
|
|
schedule:
|
|
# Every Saturday at 06:00 UTC
|
|
- cron: '0 6 * * 6'
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'packages/browseros-agent/apps/server/src/agent/**'
|
|
- 'packages/browseros-agent/apps/server/src/tools/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
config:
|
|
description: 'Eval config file (relative to apps/eval/)'
|
|
required: false
|
|
default: 'configs/browseros-agent-weekly.json'
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
eval:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 360
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install BrowserOS
|
|
run: |
|
|
wget -q https://github.com/browseros-ai/BrowserOS/releases/download/v0.44.0.1/BrowserOS_v0.44.0.1_amd64.deb
|
|
sudo dpkg -i BrowserOS_v0.44.0.1_amd64.deb
|
|
browseros --version || echo "BrowserOS installed at $(which browseros)"
|
|
|
|
- name: Install Bun
|
|
uses: oven-sh/setup-bun@v2
|
|
with:
|
|
bun-version: latest
|
|
|
|
- name: Install dependencies
|
|
working-directory: packages/browseros-agent
|
|
run: bun install --ignore-scripts && bun run build:agent-sdk
|
|
|
|
- name: Install Python eval dependencies
|
|
run: pip install agisdk requests
|
|
|
|
- name: Clone WebArena-Infinity
|
|
run: git clone --depth 1 https://github.com/web-arena-x/webarena-infinity.git /tmp/webarena-infinity
|
|
|
|
- name: Install xvfb
|
|
run: sudo apt-get update && sudo apt-get install -y xvfb
|
|
|
|
- name: Install captcha solver extension
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
run: |
|
|
mkdir -p extensions
|
|
curl -sL -o /tmp/nopecha.zip https://github.com/NopeCHALLC/nopecha-extension/releases/latest/download/chromium_automation.zip
|
|
unzip -qo /tmp/nopecha.zip -d extensions/nopecha
|
|
|
|
- name: Run eval
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
env:
|
|
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
|
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
|
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
|
NOPECHA_API_KEY: ${{ secrets.NOPECHA_API_KEY }}
|
|
BROWSEROS_BINARY: /usr/bin/browseros
|
|
WEBARENA_INFINITY_DIR: /tmp/webarena-infinity
|
|
EVAL_CONFIG: ${{ github.event.inputs.config || 'configs/browseros-agent-weekly.json' }}
|
|
run: |
|
|
echo "Running eval with config: $EVAL_CONFIG"
|
|
xvfb-run --auto-servernum --server-args="-screen 0 1440x900x24" bun run src/index.ts -c "$EVAL_CONFIG"
|
|
|
|
- name: Upload runs to R2
|
|
if: success()
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
env:
|
|
EVAL_R2_ACCOUNT_ID: ${{ secrets.EVAL_R2_ACCOUNT_ID }}
|
|
EVAL_R2_ACCESS_KEY_ID: ${{ secrets.EVAL_R2_ACCESS_KEY_ID }}
|
|
EVAL_R2_SECRET_ACCESS_KEY: ${{ secrets.EVAL_R2_SECRET_ACCESS_KEY }}
|
|
EVAL_R2_BUCKET: ${{ secrets.EVAL_R2_BUCKET }}
|
|
EVAL_R2_CDN_BASE_URL: ${{ secrets.EVAL_R2_CDN_BASE_URL }}
|
|
EVAL_CONFIG: ${{ github.event.inputs.config || 'configs/browseros-agent-weekly.json' }}
|
|
run: |
|
|
CONFIG_NAME=$(basename "$EVAL_CONFIG" .json)
|
|
bun scripts/upload-run.ts "results/$CONFIG_NAME"
|
|
|
|
- name: Generate trend report
|
|
if: success()
|
|
timeout-minutes: 5
|
|
continue-on-error: true
|
|
working-directory: packages/browseros-agent
|
|
env:
|
|
EVAL_R2_ACCOUNT_ID: ${{ secrets.EVAL_R2_ACCOUNT_ID }}
|
|
EVAL_R2_ACCESS_KEY_ID: ${{ secrets.EVAL_R2_ACCESS_KEY_ID }}
|
|
EVAL_R2_SECRET_ACCESS_KEY: ${{ secrets.EVAL_R2_SECRET_ACCESS_KEY }}
|
|
EVAL_R2_BUCKET: ${{ secrets.EVAL_R2_BUCKET }}
|
|
EVAL_R2_CDN_BASE_URL: ${{ secrets.EVAL_R2_CDN_BASE_URL }}
|
|
run: bun apps/eval/scripts/weekly-report.ts /tmp/eval-report.html
|
|
|
|
- name: Upload report as artifact
|
|
if: success()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: eval-report-${{ github.run_id }}
|
|
path: /tmp/eval-report.html
|