Compare commits
14 Commits
feat/defau
...
agent-test
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
11aca93cf5 | ||
|
|
bd619d1a29 | ||
|
|
f18ae0237c | ||
|
|
81744417ab | ||
|
|
9ec4445804 | ||
|
|
02bbb3fdd5 | ||
|
|
f11d16c674 | ||
|
|
64ddcc2a07 | ||
|
|
8b0e6dbfd3 | ||
|
|
07a2d13f16 | ||
|
|
46031ed573 | ||
|
|
ecd31efcb0 | ||
|
|
c79c775fb8 | ||
|
|
4bee76253d |
1
.gitattributes
vendored
@@ -9,5 +9,4 @@ packages/browseros/chromium_patches/**/*.py linguist-generated
|
||||
scripts/*.py linguist-generated
|
||||
# Mark build directories as generated
|
||||
build/* linguist-generated
|
||||
docs/images/** filter=lfs diff=lfs merge=lfs -text
|
||||
docs/videos/** filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 302 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 843 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 892 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 139 KiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.5 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.3 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.5 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.8 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.6 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.8 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.5 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 6.4 MiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 496 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 525 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 562 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 702 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 360 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 456 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 157 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 130 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 191 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 100 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 604 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 416 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 852 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 102 KiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.2 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.9 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 2.5 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 4.0 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 2.5 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 3.7 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 2.5 MiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 235 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 183 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 226 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 221 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 257 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 188 KiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 2.4 MiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 102 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 100 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 122 KiB |
|
Before Width: | Height: | Size: 130 B After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 314 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 270 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 289 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 247 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 566 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 217 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 100 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 244 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 342 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 556 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 490 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 488 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 457 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 438 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 419 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 434 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 721 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 418 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 228 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 275 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 514 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 182 KiB |
|
Before Width: | Height: | Size: 130 B After Width: | Height: | Size: 13 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 135 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 583 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 110 KiB |
|
Before Width: | Height: | Size: 130 B After Width: | Height: | Size: 84 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 452 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 437 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 275 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 116 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 155 KiB |
|
Before Width: | Height: | Size: 130 B After Width: | Height: | Size: 59 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 233 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 204 KiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 195 KiB |
|
Before Width: | Height: | Size: 130 B After Width: | Height: | Size: 74 KiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 1.4 MiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 1.4 MiB |
|
Before Width: | Height: | Size: 131 B After Width: | Height: | Size: 948 KiB |
|
Before Width: | Height: | Size: 132 B After Width: | Height: | Size: 2.5 MiB |
@@ -6,7 +6,7 @@ description: "How to fix connection issues when the BrowserOS agent is not respo
|
||||
If you see **"Failed to Fetch"** or **"Unable to connect to BrowserOS agent"**, your agent has been killed.
|
||||
|
||||
<Frame>
|
||||
<img src="/images/troubleshooting/failed-to-fetch-error.png" alt="Failed to fetch error" />
|
||||
<img src="/troubleshooting/failed-to-fetch-error.png" alt="Failed to fetch error" />
|
||||
</Frame>
|
||||
|
||||
## Windows: Did you allow the BrowserOS Agent?
|
||||
@@ -14,7 +14,7 @@ If you see **"Failed to Fetch"** or **"Unable to connect to BrowserOS agent"**,
|
||||
When BrowserOS starts for the first time on Windows, a popup will ask if you want to allow **BrowserOS Agent** to access the network. Click **Allow** — the agent runs as a separate process alongside the browser and needs this permission to work. If you clicked **Cancel** or missed this prompt, the agent won't be able to connect.
|
||||
|
||||
<Frame>
|
||||
<img src="/images/windows-installation/windows-installation-allow-browseros-agent.png" alt="Allow BrowserOS Agent network access prompt" />
|
||||
<img src="/troubleshooting/windows-installation-allow-browseros-agent.png" alt="Allow BrowserOS Agent network access prompt" />
|
||||
</Frame>
|
||||
|
||||
To fix this, go to **Windows Security → Allow an app through your network settings**, and make sure **BrowserOS Agent** is enabled.
|
||||
@@ -26,7 +26,7 @@ To fix this, go to **Windows Security → Allow an app through your network sett
|
||||
Open **Settings** from the sidebar, scroll to the MCP Server section, and click **Restart**. Wait a couple of minutes.
|
||||
|
||||
<Frame>
|
||||
<img src="/images/troubleshooting/mcp-server-restart.png" alt="Click Restart in MCP Server Settings" />
|
||||
<img src="/troubleshooting/mcp-server-restart.png" alt="Click Restart in MCP Server Settings" />
|
||||
</Frame>
|
||||
</Step>
|
||||
|
||||
@@ -39,7 +39,7 @@ To fix this, go to **Windows Security → Allow an app through your network sett
|
||||
4. Reopen **BrowserOS** — the agent will restart automatically.
|
||||
|
||||
<Frame>
|
||||
<img src="/images/windows-installation/troubleshoot-search-restart-agent-task-manager.png" alt="Search for BrowserOS Agent in Task Manager and restart it" />
|
||||
<img src="/troubleshooting/troubleshoot-search-restart-agent-task-manager.png" alt="Search for BrowserOS Agent in Task Manager and restart it" />
|
||||
</Frame>
|
||||
|
||||
<Note>If the agent still doesn't reconnect, end both **BrowserOS** and **BrowserOS Agent** in Task Manager, then reopen the app.</Note>
|
||||
@@ -100,7 +100,7 @@ If you're still experiencing issues, reach out:
|
||||
<Card title="Discord" icon="discord" href="https://discord.gg/YKwjt5vuKr">
|
||||
Get help from the community
|
||||
</Card>
|
||||
<Card title="Slack" icon="slack" href="SLACK_LINK_HERE">
|
||||
<Card title="Slack" icon="slack" href="https://dub.sh/browserOS-slack">
|
||||
Join our Slack
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
BIN
docs/troubleshooting/failed-to-fetch-error.png
Normal file
|
After Width: | Height: | Size: 13 KiB |
BIN
docs/troubleshooting/mcp-server-restart.png
Normal file
|
After Width: | Height: | Size: 135 KiB |
|
After Width: | Height: | Size: 1.4 MiB |
|
After Width: | Height: | Size: 1.4 MiB |
286
packages/browseros-agent/.claude/skills/test-ui/SKILL.md
Normal file
@@ -0,0 +1,286 @@
|
||||
---
|
||||
name: test-ui
|
||||
description: Test the BrowserOS agent extension UI by starting the dev environment and visually verifying changes via CDP. Covers the new tab page (left sidebar — Home, Scheduled Tasks, Settings, etc.) and the right side panel (chat interface). Use after making UI changes to apps/agent/.
|
||||
argument-hint: [what to test, e.g. "verify the new settings page renders correctly"]
|
||||
---
|
||||
|
||||
# Test Agent UI
|
||||
|
||||
Visually test the BrowserOS agent extension UI — both the new tab page (left sidebar) and the right side panel (chat) — by starting the dev environment and inspecting via CDP.
|
||||
|
||||
## When to use
|
||||
|
||||
After making code changes to `apps/agent/` (the Chrome extension), use this skill to:
|
||||
- Verify new UI components render correctly
|
||||
- Check navigation between views works
|
||||
- Confirm layout/styling changes look right
|
||||
- Test interactive elements (buttons, inputs, forms)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Go** must be installed (`brew install go`) — the dev tool is written in Go
|
||||
- **BrowserOS.app** must be installed at `/Applications/BrowserOS.app/`
|
||||
- The `scripts/dev/inspect-ui.ts` utility must exist (CDP inspector script)
|
||||
|
||||
## Step 1: Start the dev environment
|
||||
|
||||
```bash
|
||||
bun run dev:watch -- --new
|
||||
```
|
||||
|
||||
This single command handles everything:
|
||||
- Builds the Go dev CLI tool
|
||||
- Picks random available ports (avoids conflicts)
|
||||
- Creates a fresh browser profile
|
||||
- Builds controller-ext
|
||||
- Runs GraphQL codegen if `apps/agent/generated/graphql/` doesn't exist
|
||||
- Starts the agent extension with WXT HMR (hot module replacement)
|
||||
- Waits for CDP to be ready
|
||||
- Starts the MCP server
|
||||
|
||||
Run it in the background and **read the output to find the CDP port**:
|
||||
|
||||
```
|
||||
[info] Ports: CDP=9552 Server=9065 Extension=9929
|
||||
```
|
||||
|
||||
The CDP port is randomized. You MUST extract it from the output and set it for all subsequent commands:
|
||||
|
||||
```bash
|
||||
export BROWSEROS_CDP_PORT=<port from output>
|
||||
```
|
||||
|
||||
Wait for these messages before proceeding:
|
||||
1. `[server] CDP ready`
|
||||
2. `[server] HTTP server listening`
|
||||
|
||||
## Step 2: Discover targets
|
||||
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts targets
|
||||
```
|
||||
|
||||
You will see targets like:
|
||||
- `[service_worker]` — extension background scripts (not directly testable for UI)
|
||||
- `[page] chrome-extension://bflpfmnmnokmjhmgnolecpppdbdophmk/app.html#/...` — **New tab page (left sidebar)**
|
||||
- `[page] sidepanel.html` — **Right side panel (chat)**
|
||||
|
||||
The two main testable surfaces:
|
||||
- **`app.html`** — the new tab page with left sidebar (Home, Connect Apps, Scheduled Tasks, Skills, Memory, Soul, Settings)
|
||||
- **`sidepanel.html`** — the right side panel chat interface
|
||||
|
||||
## Step 3: Navigate to the main UI
|
||||
|
||||
A fresh profile opens the **onboarding page** (`app.html#/onboarding`). Navigate to the home page first:
|
||||
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts eval app.html "window.location.hash = '#/home'"
|
||||
```
|
||||
|
||||
Verify with a snapshot (not screenshot — snapshot is faster and sufficient for structural checks):
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts snapshot app.html
|
||||
```
|
||||
|
||||
## Snapshot vs Screenshot
|
||||
|
||||
**Prefer `snapshot` for most checks** — it's fast, text-based, and tells you what elements exist, their text, and their IDs. Use it after every navigation or interaction to verify state.
|
||||
|
||||
**Use `screenshot` only when you need visual verification** — layout changes, CSS/styling, colors, images, or a final "does it look right" check. Screenshots are expensive (capture → save → read image).
|
||||
|
||||
| Check | Use |
|
||||
|-------|-----|
|
||||
| Did the page navigate? | `snapshot` — look for new elements |
|
||||
| Does my new component render? | `snapshot` — look for its text/role |
|
||||
| Did a click change state? | `snapshot` — check element names/values |
|
||||
| Is the layout correct? | `screenshot` — visual check needed |
|
||||
| Do CSS changes look right? | `screenshot` — visual check needed |
|
||||
| Final verification before committing | `screenshot` — one visual confirmation |
|
||||
|
||||
## Step 4: Test the new tab page (left sidebar)
|
||||
|
||||
### Get element IDs
|
||||
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts snapshot app.html
|
||||
```
|
||||
|
||||
Output shows interactive elements with IDs:
|
||||
```
|
||||
[52] link "Home"
|
||||
[57] link "Connect Apps"
|
||||
[65] link "Scheduled Tasks"
|
||||
[74] link "Skills"
|
||||
[103] link "Settings"
|
||||
```
|
||||
|
||||
### Navigate via click or hash routing
|
||||
|
||||
**Click-based** (use element IDs from snapshot):
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts click app.html 65 # Click "Scheduled Tasks"
|
||||
```
|
||||
|
||||
**Hash routing** (faster, no snapshot needed):
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts eval app.html "window.location.hash = '#/settings'"
|
||||
bun scripts/dev/inspect-ui.ts eval app.html "window.location.hash = '#/scheduled-tasks'"
|
||||
bun scripts/dev/inspect-ui.ts eval app.html "window.location.hash = '#/home'"
|
||||
```
|
||||
|
||||
### Verify navigation
|
||||
|
||||
```bash
|
||||
# Snapshot to confirm the page changed (fast, preferred)
|
||||
bun scripts/dev/inspect-ui.ts snapshot app.html
|
||||
|
||||
# Screenshot only if you need to check visual layout
|
||||
bun scripts/dev/inspect-ui.ts screenshot app.html /tmp/settings.png
|
||||
```
|
||||
|
||||
### CRITICAL: Re-snapshot after every navigation
|
||||
|
||||
React re-renders change element IDs. **Always run snapshot again** before clicking/filling after navigating to a new view. Using stale IDs will fail.
|
||||
|
||||
## Step 5: Open and test the right side panel
|
||||
|
||||
The side panel starts **disabled** in a fresh profile. Open it using BrowserOS-specific APIs:
|
||||
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts open-sidepanel
|
||||
```
|
||||
|
||||
Wait 2 seconds for it to appear as a target, then:
|
||||
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts screenshot sidepanel /tmp/panel.png
|
||||
bun scripts/dev/inspect-ui.ts snapshot sidepanel
|
||||
```
|
||||
|
||||
### Interact with the side panel
|
||||
|
||||
```bash
|
||||
# Get element IDs
|
||||
bun scripts/dev/inspect-ui.ts snapshot sidepanel
|
||||
# Output: [37] textbox "What should I do?"
|
||||
# [124] button "Send"
|
||||
# [60] link "Chat history"
|
||||
# [99] button "Agent Mode ON"
|
||||
|
||||
# Fill the chat input and press Enter to send
|
||||
bun scripts/dev/inspect-ui.ts fill sidepanel 37 "Hello world"
|
||||
bun scripts/dev/inspect-ui.ts press_key sidepanel Enter
|
||||
|
||||
# Or click the Send button
|
||||
bun scripts/dev/inspect-ui.ts click sidepanel 124
|
||||
|
||||
# Wait for a response to appear
|
||||
bun scripts/dev/inspect-ui.ts wait_for sidepanel text "response text"
|
||||
|
||||
# Scroll down to see more content
|
||||
bun scripts/dev/inspect-ui.ts scroll sidepanel down 3
|
||||
|
||||
# Hover over an element to test hover states
|
||||
bun scripts/dev/inspect-ui.ts hover sidepanel 99
|
||||
|
||||
# Snapshot to verify state changed (fast, preferred)
|
||||
bun scripts/dev/inspect-ui.ts snapshot sidepanel
|
||||
|
||||
# Screenshot only for visual/layout verification
|
||||
bun scripts/dev/inspect-ui.ts screenshot sidepanel /tmp/result.png
|
||||
```
|
||||
|
||||
## Step 6: Verify and iterate
|
||||
|
||||
### The core loop
|
||||
|
||||
```
|
||||
snapshot → identify element IDs → click/fill/press_key → snapshot → verify
|
||||
```
|
||||
|
||||
Use `screenshot` only when visual layout verification is needed (CSS changes, final check).
|
||||
|
||||
### After making code changes
|
||||
|
||||
1. Fix the code in `apps/agent/`
|
||||
2. WXT HMR will hot-reload the extension automatically (watch mode)
|
||||
3. Wait 2-3 seconds for the reload to complete
|
||||
4. **Re-snapshot** — element IDs WILL change after HMR reload
|
||||
5. Verify the fix with snapshot (or screenshot if visual)
|
||||
|
||||
### Check server logs
|
||||
|
||||
The dev server output (running in background) contains useful diagnostics:
|
||||
- `[agent]` — WXT build/HMR status, compilation errors
|
||||
- `[server]` — MCP server logs, tool execution, errors
|
||||
- `[build]` — Extension build output
|
||||
|
||||
If the UI isn't rendering, check for build errors in the `[agent]` output.
|
||||
|
||||
### Check for JavaScript errors
|
||||
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts eval sidepanel "JSON.stringify(window.__errors || 'no errors')"
|
||||
```
|
||||
|
||||
Or check the console for React errors:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts eval app.html "document.querySelector('#root')?.innerHTML?.substring(0, 200)"
|
||||
```
|
||||
|
||||
### Verify API connectivity
|
||||
|
||||
The extension talks to the MCP server. Verify the server is reachable:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts eval sidepanel "fetch('http://127.0.0.1:<serverPort>/health').then(r => r.ok).catch(() => false)"
|
||||
```
|
||||
|
||||
### Common issues
|
||||
|
||||
| Symptom | Cause | Fix |
|
||||
|---------|-------|-----|
|
||||
| Blank page after navigation | React render error | Check `eval` for JS errors |
|
||||
| Element IDs don't match | Page re-rendered (HMR/navigation) | Re-run `snapshot` before interacting |
|
||||
| `open-sidepanel` fails | Extension not fully loaded | Wait longer after dev server starts |
|
||||
| Click does nothing | Element not visible (below fold) | Use `scroll` first, then re-snapshot |
|
||||
| `wait_for` times out | Content hasn't loaded yet | Check server logs for API errors |
|
||||
|
||||
## Available commands reference
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `targets` | List all CDP targets, marks extension pages with `[EXTENSION]` |
|
||||
| `screenshot <target> [file]` | Capture PNG screenshot (default: `screenshot.png`) |
|
||||
| `snapshot <target>` | Print accessibility tree with `[elementId] role "name"` |
|
||||
| `click <target> <elementId>` | Click element by ID (3-tier coordinate fallback + JS click) |
|
||||
| `fill <target> <elementId> <text>` | Focus element, clear, type text |
|
||||
| `press_key <target> <key>` | Press key or combo: `Enter`, `Escape`, `Tab`, `Control+A`, `Meta+Shift+P` |
|
||||
| `scroll <target> <dir> [amount]` | Scroll `up`/`down`/`left`/`right`, amount in ticks (default 3) |
|
||||
| `hover <target> <elementId>` | Hover over element (for tooltips, hover states) |
|
||||
| `select_option <target> <id> <val>` | Select dropdown option by value or visible text |
|
||||
| `wait_for <target> text\|selector <v>` | Wait up to 10s for text or CSS selector to appear |
|
||||
| `eval <target> <expression>` | Run JavaScript in the target's context |
|
||||
| `open-sidepanel` | Enable and open the right side panel |
|
||||
|
||||
`<target>` is a URL substring (e.g., `sidepanel`, `app.html`) or numeric index from `targets` output.
|
||||
|
||||
## Known app.html routes
|
||||
|
||||
These can be used with `eval app.html "window.location.hash = '#/<route>'"`:
|
||||
|
||||
| Route | View |
|
||||
|-------|------|
|
||||
| `/home` | Home page with search bar and top sites |
|
||||
| `/settings` | Settings (LLM providers, customization, workflows, MCP) |
|
||||
| `/scheduled-tasks` | Scheduled Tasks management |
|
||||
| `/onboarding` | Onboarding flow (first-run experience) |
|
||||
|
||||
## Gotchas learned from real testing
|
||||
|
||||
1. **Ports are randomized** with `--new` — always extract from dev server output
|
||||
2. **Fresh profile = onboarding page** — navigate to `#/home` to see the main UI
|
||||
3. **Element IDs change after navigation** — always re-snapshot before clicking
|
||||
4. **Side panel starts disabled** — `open-sidepanel` handles the BrowserOS-specific enable + toggle API
|
||||
5. **`Input.enable` does not exist** — the CDP Input domain has no enable method (already handled in the script)
|
||||
6. **`DOM.getDocument` required** — must be called before DOM operations like `pushNodesByBackendIdsToFrontend` (already handled in the script)
|
||||
7. **Settings sub-navigation** — the settings page has its own left sidebar (BrowserOS AI, Chat & Council Provider, Search Provider, Customize BrowserOS, BrowserOS as MCP, Workflows) — use snapshot + click to navigate within settings
|
||||
@@ -165,3 +165,68 @@ Tests are in `apps/server/tests/`:
|
||||
- `agent/` - Agent tests (compaction, rate limiter)
|
||||
- `sdk/` - Agent SDK tests
|
||||
- `__helpers__/` - Test utilities and fixtures
|
||||
|
||||
## Self-Testing UI Changes
|
||||
|
||||
After making UI changes to the agent extension (`apps/agent/`), you can visually verify them using the CDP inspector script. This connects directly to the browser via Chrome DevTools Protocol and can inspect extension pages (side panel, new tab, etc.) that the agent's own tools cannot see.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
The dev server must be running:
|
||||
```bash
|
||||
bun run dev:watch -- --new
|
||||
```
|
||||
Read the output to find the randomized CDP port, then:
|
||||
```bash
|
||||
export BROWSEROS_CDP_PORT=<port from output>
|
||||
```
|
||||
|
||||
### Workflow
|
||||
|
||||
1. **List all targets** to see what's available:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts targets
|
||||
```
|
||||
|
||||
2. **Open the side panel** if it's not already open:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts open-sidepanel
|
||||
```
|
||||
|
||||
3. **Take a screenshot** of the side panel:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts screenshot sidepanel /tmp/panel.png
|
||||
```
|
||||
Then read `/tmp/panel.png` to view the result.
|
||||
|
||||
4. **Get the accessibility tree** for structural verification:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts snapshot sidepanel
|
||||
```
|
||||
|
||||
5. **Click an element** by its ID from the snapshot:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts click sidepanel 142
|
||||
```
|
||||
|
||||
6. **Fill a text input** by its ID from the snapshot:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts fill sidepanel 85 "search query"
|
||||
```
|
||||
|
||||
7. **Evaluate JavaScript** in the extension context:
|
||||
```bash
|
||||
bun scripts/dev/inspect-ui.ts eval sidepanel "document.title"
|
||||
```
|
||||
|
||||
### Interaction workflow
|
||||
|
||||
The typical loop is: snapshot → identify element IDs → click/fill → screenshot to verify.
|
||||
Element IDs come from the `[number]` in snapshot output (these are `backendDOMNodeId` values).
|
||||
This uses the same element resolution as the server's MCP tools — no coordinate guessing.
|
||||
|
||||
### Target selection
|
||||
|
||||
The `<target>` argument can be:
|
||||
- An **index** from the `targets` output (e.g., `3`)
|
||||
- A **URL substring** (e.g., `sidepanel`, `newtab`, `chrome-extension://`)
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import { formatConversationHistory } from '@/lib/conversations/formatConversationHistory'
|
||||
import { declinedAppsStorage } from '@/lib/declined-apps/storage'
|
||||
import { useGraphqlQuery } from '@/lib/graphql/useGraphqlQuery'
|
||||
import { createDefaultBrowserOSProvider } from '@/lib/llm-providers/storage'
|
||||
import { useLlmProviders } from '@/lib/llm-providers/useLlmProviders'
|
||||
import { track } from '@/lib/metrics/track'
|
||||
import { searchActionsStorage } from '@/lib/search-actions/searchActionsStorage'
|
||||
@@ -208,7 +209,7 @@ export const useChatSession = (options?: ChatSessionOptions) => {
|
||||
})
|
||||
const activeTab = activeTabsList?.[0] ?? undefined
|
||||
const message = getLastMessageText(messages)
|
||||
const provider = selectedLlmProviderRef.current
|
||||
const provider = selectedLlmProviderRef.current ?? createDefaultBrowserOSProvider()
|
||||
const currentMode = modeRef.current
|
||||
const enabledMcpServers = enabledMcpServersRef.current
|
||||
const customMcpServers = enabledCustomServersRef.current
|
||||
|
||||
@@ -60,6 +60,15 @@ export function useLlmProviders(): UseLlmProvidersReturn {
|
||||
await defaultProviderIdStorage.setValue(loadedDefaultId)
|
||||
}
|
||||
|
||||
// Repair stale default ID that doesn't match any provider
|
||||
const defaultExists = loadedProviders.some(
|
||||
(p) => p.id === loadedDefaultId,
|
||||
)
|
||||
if (!defaultExists && loadedProviders.length > 0) {
|
||||
loadedDefaultId = loadedProviders[0].id
|
||||
await defaultProviderIdStorage.setValue(loadedDefaultId)
|
||||
}
|
||||
|
||||
setProviders(loadedProviders)
|
||||
setDefaultProviderId(loadedDefaultId)
|
||||
} catch {
|
||||
@@ -146,8 +155,12 @@ export function useLlmProviders(): UseLlmProvidersReturn {
|
||||
await providersStorage.setValue(updatedProviders)
|
||||
}
|
||||
|
||||
// Fall back to first provider if defaultProviderId is stale/invalid
|
||||
const selectedProvider = useMemo(
|
||||
() => providers.find((p) => p.id === defaultProviderId) ?? null,
|
||||
() =>
|
||||
providers.find((p) => p.id === defaultProviderId) ??
|
||||
providers[0] ??
|
||||
null,
|
||||
[providers, defaultProviderId],
|
||||
)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { createParser, type EventSourceMessage } from 'eventsource-parser'
|
||||
import type { ChatMode } from '@/entrypoints/sidepanel/index/chatTypes'
|
||||
import { getAgentServerUrl } from '@/lib/browseros/helpers'
|
||||
import {
|
||||
createDefaultBrowserOSProvider,
|
||||
defaultProviderIdStorage,
|
||||
providersStorage,
|
||||
} from '@/lib/llm-providers/storage'
|
||||
@@ -78,7 +79,7 @@ export async function getChatServerResponse(
|
||||
request: ChatServerRequest,
|
||||
): Promise<ChatServerResponse> {
|
||||
const agentServerUrl = await getAgentServerUrl()
|
||||
const provider = await getDefaultProvider()
|
||||
const provider = (await getDefaultProvider()) ?? createDefaultBrowserOSProvider()
|
||||
const conversationId = request.conversationId ?? crypto.randomUUID()
|
||||
const personalization = await personalizationStorage.getValue()
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 BrowserOS
|
||||
* SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
*/
|
||||
|
||||
import type { UIMessage } from 'ai'
|
||||
|
||||
/**
|
||||
* Checks whether a UIMessage has meaningful content that can be sent
|
||||
* to the AI provider without causing validation errors.
|
||||
*
|
||||
* Two layers of validation can reject messages:
|
||||
*
|
||||
* 1. **AI SDK** (`validate-ui-messages.ts`):
|
||||
* - `parts` array must be `.nonempty()` — rejects `parts: []`
|
||||
*
|
||||
* 2. **Provider API** (e.g. Gemini `generateContent`, Anthropic, OpenAI):
|
||||
* - Assistant messages with only empty-string text are rejected
|
||||
* as semantically empty, even though the SDK schema allows it
|
||||
*
|
||||
* This function guards against both layers so callers can filter
|
||||
* messages before passing them to `createAgentUIStreamResponse`.
|
||||
*/
|
||||
export function hasMessageContent(message: UIMessage): boolean {
|
||||
if (message.parts.length === 0) return false
|
||||
|
||||
// A message that contains any non-text part (tool invocation, reasoning,
|
||||
// file, step-start, etc.) is always considered valid — those part types
|
||||
// carry meaning regardless of text content.
|
||||
const hasNonTextPart = message.parts.some((p) => p.type !== 'text')
|
||||
if (hasNonTextPart) return true
|
||||
|
||||
// All parts are text — at least one must have non-whitespace content.
|
||||
return message.parts.some(
|
||||
(p) => p.type === 'text' && p.text.trim().length > 0,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters a UIMessage array, removing messages that would fail
|
||||
* SDK validation or provider-level content checks.
|
||||
*/
|
||||
export function filterValidMessages(messages: UIMessage[]): UIMessage[] {
|
||||
return messages.filter(hasMessageContent)
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { mkdir, utimes } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { createAgentUIStreamResponse, type UIMessage } from 'ai'
|
||||
import { AiSdkAgent } from '../../agent/ai-sdk-agent'
|
||||
import { filterValidMessages } from '../../agent/message-validation'
|
||||
import { formatUserMessage } from '../../agent/format-message'
|
||||
import type { SessionStore } from '../../agent/session-store'
|
||||
import type { ResolvedAgentConfig } from '../../agent/types'
|
||||
@@ -139,6 +140,7 @@ export class ChatService {
|
||||
|
||||
if (isNewSession && request.previousConversation?.length) {
|
||||
for (const msg of request.previousConversation) {
|
||||
if (!msg.content.trim()) continue
|
||||
session.agent.messages.push({
|
||||
id: crypto.randomUUID(),
|
||||
role: msg.role === 'assistant' ? 'assistant' : 'user',
|
||||
@@ -168,10 +170,10 @@ export class ChatService {
|
||||
|
||||
return createAgentUIStreamResponse({
|
||||
agent: session.agent.toolLoopAgent,
|
||||
uiMessages: session.agent.messages,
|
||||
uiMessages: filterValidMessages(session.agent.messages),
|
||||
abortSignal,
|
||||
onFinish: async ({ messages }: { messages: UIMessage[] }) => {
|
||||
session.agent.messages = messages
|
||||
session.agent.messages = filterValidMessages(messages)
|
||||
logger.info('Agent execution complete', {
|
||||
conversationId: request.conversationId,
|
||||
totalMessages: messages.length,
|
||||
|
||||
1060
packages/browseros-agent/scripts/dev/inspect-ui.ts
Normal file
@@ -127,6 +127,14 @@
|
||||
"created_at": "2026-03-06T16:18:27Z",
|
||||
"repoId": 985839104,
|
||||
"pullRequestNo": 400
|
||||
},
|
||||
{
|
||||
"name": "shivammittal274",
|
||||
"id": 56757235,
|
||||
"comment_id": 4062893235,
|
||||
"created_at": "2026-03-15T12:26:54Z",
|
||||
"repoId": 985839104,
|
||||
"pullRequestNo": 448
|
||||
}
|
||||
]
|
||||
}
|
||||