mirror of
https://github.com/browseros-ai/BrowserOS.git
synced 2026-05-13 23:53:25 +00:00
0babc05077
* feat(eval): show mean score instead of pass/fail in report and viewer * feat(eval): integrate NopeCHA CAPTCHA solver into eval pipeline Add CAPTCHA detection and waiting so screenshots capture post-solve state. Run headed with xvfb on CI since headless breaks extension content scripts. - Add CaptchaWaiter module (detect reCAPTCHA/hCaptcha/Turnstile, poll until solved) - Add optional `captcha` config block to EvalConfigSchema - Wait for CAPTCHA solve before screenshot in single-agent and orchestrator-executor - Patch NopeCHA manifest with API key before launching workers - Fix CAPTCHA_EXT_DIR path (was pointing one level too high) - Remove --incognito (extensions don't run in incognito; fresh user-data-dir isolates) - CI: install xvfb, run headed via xvfb-run, pass NOPECHA_API_KEY secret
99 lines
3.6 KiB
YAML
99 lines
3.6 KiB
YAML
name: Weekly Eval
|
|
|
|
on:
|
|
schedule:
|
|
# Every Saturday at 06:00 UTC
|
|
- cron: '0 6 * * 6'
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'packages/browseros-agent/apps/server/src/agent/**'
|
|
- 'packages/browseros-agent/apps/server/src/tools/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
config:
|
|
description: 'Eval config file (relative to apps/eval/)'
|
|
required: false
|
|
default: 'configs/browseros-agent-weekly.json'
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
eval:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 360
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Install BrowserOS
|
|
run: |
|
|
wget -q https://github.com/browseros-ai/BrowserOS/releases/download/v0.44.0.1/BrowserOS_v0.44.0.1_amd64.deb
|
|
sudo dpkg -i BrowserOS_v0.44.0.1_amd64.deb
|
|
browseros --version || echo "BrowserOS installed at $(which browseros)"
|
|
|
|
- name: Install Bun
|
|
uses: oven-sh/setup-bun@v2
|
|
with:
|
|
bun-version: latest
|
|
|
|
- name: Install dependencies
|
|
working-directory: packages/browseros-agent
|
|
run: bun install --ignore-scripts && bun run build:agent-sdk
|
|
|
|
- name: Install xvfb
|
|
run: sudo apt-get update && sudo apt-get install -y xvfb
|
|
|
|
- name: Install captcha solver extension
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
run: |
|
|
mkdir -p extensions
|
|
curl -sL -o /tmp/nopecha.zip https://github.com/NopeCHALLC/nopecha-extension/releases/latest/download/chromium_automation.zip
|
|
unzip -qo /tmp/nopecha.zip -d extensions/nopecha
|
|
|
|
- name: Run eval
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
env:
|
|
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
|
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
|
NOPECHA_API_KEY: ${{ secrets.NOPECHA_API_KEY }}
|
|
BROWSEROS_BINARY: /usr/bin/browseros
|
|
EVAL_CONFIG: ${{ github.event.inputs.config || 'configs/browseros-agent-weekly.json' }}
|
|
run: |
|
|
echo "Running eval with config: $EVAL_CONFIG"
|
|
xvfb-run --auto-servernum --server-args="-screen 0 1440x900x24" bun run src/index.ts -c "$EVAL_CONFIG"
|
|
|
|
- name: Upload runs to R2
|
|
if: success()
|
|
working-directory: packages/browseros-agent/apps/eval
|
|
env:
|
|
EVAL_R2_ACCOUNT_ID: ${{ secrets.EVAL_R2_ACCOUNT_ID }}
|
|
EVAL_R2_ACCESS_KEY_ID: ${{ secrets.EVAL_R2_ACCESS_KEY_ID }}
|
|
EVAL_R2_SECRET_ACCESS_KEY: ${{ secrets.EVAL_R2_SECRET_ACCESS_KEY }}
|
|
EVAL_R2_BUCKET: ${{ secrets.EVAL_R2_BUCKET }}
|
|
EVAL_R2_CDN_BASE_URL: ${{ secrets.EVAL_R2_CDN_BASE_URL }}
|
|
EVAL_CONFIG: ${{ github.event.inputs.config || 'configs/browseros-agent-weekly.json' }}
|
|
run: |
|
|
CONFIG_NAME=$(basename "$EVAL_CONFIG" .json)
|
|
bun scripts/upload-run.ts "results/$CONFIG_NAME"
|
|
|
|
- name: Generate trend report
|
|
if: success()
|
|
working-directory: packages/browseros-agent
|
|
env:
|
|
EVAL_R2_ACCOUNT_ID: ${{ secrets.EVAL_R2_ACCOUNT_ID }}
|
|
EVAL_R2_ACCESS_KEY_ID: ${{ secrets.EVAL_R2_ACCESS_KEY_ID }}
|
|
EVAL_R2_SECRET_ACCESS_KEY: ${{ secrets.EVAL_R2_SECRET_ACCESS_KEY }}
|
|
EVAL_R2_BUCKET: ${{ secrets.EVAL_R2_BUCKET }}
|
|
EVAL_R2_CDN_BASE_URL: ${{ secrets.EVAL_R2_CDN_BASE_URL }}
|
|
run: bun apps/eval/scripts/weekly-report.ts /tmp/eval-report.html
|
|
|
|
- name: Upload report as artifact
|
|
if: success()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: eval-report-${{ github.run_id }}
|
|
path: /tmp/eval-report.html
|