LLM/BrowserOS
1
0
Fork 0
mirror of https://github.com/browseros-ai/BrowserOS.git synced 2026-05-13 23:53:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add structured MCP tool outputs and schemas (#420)

Browse Source 
* feat: add structured MCP outputs for browser tools

* fix: address PR review comments for mcp_structured_content
This commit is contained in:
Nikhil
2026-03-05 13:19:01 -08:00
committed by GitHub
parent 88cb227444
commit e37d19da51
25 changed files with 971 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/server/package.json
View File

@@ -55,7 +55,7 @@
"@hono/mcp": "^0.2.3",
"@hono/node-server": "^1.19.6",
"@hono/zod-validator": "^0.4.3",
"@modelcontextprotocol/sdk": "^1.25.2",
"@modelcontextprotocol/sdk": "^1.27.1",
"@openrouter/ai-sdk-provider": "^2.2.3",
"@sentry/bun": "^10.31.0",
"ai": "^6.0.94",

2
apps/server/src/api/services/mcp/register-mcp.ts
View File

@@ -30,6 +30,7 @@ export function registerTools(
return {
content: result.content,
isError: result.isError,
structuredContent: result.structuredContent,
}
} catch (error) {
const errorText = error instanceof Error ? error.message : String(error)
@@ -53,6 +54,7 @@ export function registerTools(
{
description: tool.description,
inputSchema: tool.input as unknown as Record<string, never>,
outputSchema: tool.output as unknown as Record<string, never>,
},
handler,
)

48
apps/server/src/tools/bookmarks.ts
View File

@@ -2,6 +2,17 @@ import { z } from 'zod'
import type { BookmarkNode } from '../browser/bookmarks'
import { defineTool } from './framework'
const bookmarkNodeSchema = z.object({
id: z.string(),
title: z.string(),
url: z.string().optional(),
parentId: z.string().optional(),
type: z.enum(['url', 'folder']),
index: z.number().optional(),
dateAdded: z.number(),
dateLastUsed: z.number().optional(),
})
function formatBookmarkTree(nodes: BookmarkNode[]): string {
const lines: string[] = []
for (const node of nodes) {
@@ -19,15 +30,21 @@ export const get_bookmarks = defineTool({
name: 'get_bookmarks',
description: 'List all bookmarks in the browser',
input: z.object({}),
output: z.object({
bookmarks: z.array(bookmarkNodeSchema),
count: z.number(),
}),
handler: async (_args, ctx, response) => {
const bookmarks = await ctx.browser.getBookmarks()
if (bookmarks.length === 0) {
response.text('No bookmarks found.')
response.data({ bookmarks: [], count: 0 })
return
}
response.text(
`Found ${bookmarks.length} bookmarks:\n\n${formatBookmarkTree(bookmarks)}`,
)
response.data({ bookmarks, count: bookmarks.length })
},
})
@@ -42,6 +59,10 @@ export const create_bookmark = defineTool({
.describe('URL to bookmark (omit to create a folder)'),
parentId: z.string().optional().describe('Folder ID to create bookmark in'),
}),
output: z.object({
action: z.literal('create_bookmark'),
bookmark: bookmarkNodeSchema,
}),
handler: async (args, ctx, response) => {
const bookmark = await ctx.browser.createBookmark(args)
if (bookmark.url) {
@@ -51,6 +72,7 @@ export const create_bookmark = defineTool({
} else {
response.text(`Created folder: ${bookmark.title}\nID: ${bookmark.id}`)
}
response.data({ action: 'create_bookmark', bookmark })
},
})
@@ -60,9 +82,14 @@ export const remove_bookmark = defineTool({
input: z.object({
id: z.string().describe('Bookmark or folder ID to remove'),
}),
output: z.object({
action: z.literal('remove_bookmark'),
id: z.string(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.removeBookmark(args.id)
response.text(`Removed bookmark ${args.id}`)
response.data({ action: 'remove_bookmark', id: args.id })
},
})
@@ -74,12 +101,17 @@ export const update_bookmark = defineTool({
title: z.string().optional().describe('New title for the bookmark'),
url: z.string().optional().describe('New URL for the bookmark'),
}),
output: z.object({
action: z.literal('update_bookmark'),
bookmark: bookmarkNodeSchema,
}),
handler: async (args, ctx, response) => {
const bookmark = await ctx.browser.updateBookmark(args.id, {
title: args.title,
url: args.url,
})
response.text(`Updated bookmark: ${bookmark.title}\nID: ${bookmark.id}`)
response.data({ action: 'update_bookmark', bookmark })
},
})
@@ -96,12 +128,17 @@ export const move_bookmark = defineTool({
.optional()
.describe('Position within parent (0-based)'),
}),
output: z.object({
action: z.literal('move_bookmark'),
bookmark: bookmarkNodeSchema,
}),
handler: async (args, ctx, response) => {
const bookmark = await ctx.browser.moveBookmark(args.id, {
parentId: args.parentId,
index: args.index,
})
response.text(`Moved: ${bookmark.title}`)
response.data({ action: 'move_bookmark', bookmark })
},
})
@@ -113,14 +150,25 @@ export const search_bookmarks = defineTool({
.string()
.describe('Search query to find bookmarks by title or URL'),
}),
output: z.object({
query: z.string(),
bookmarks: z.array(bookmarkNodeSchema),
count: z.number(),
}),
handler: async (args, ctx, response) => {
const bookmarks = await ctx.browser.searchBookmarks(args.query)
if (bookmarks.length === 0) {
response.text(`No bookmarks found matching "${args.query}".`)
response.data({ query: args.query, bookmarks: [], count: 0 })
return
}
response.text(
`Found ${bookmarks.length} bookmarks matching "${args.query}":\n\n${formatBookmarkTree(bookmarks)}`,
)
response.data({
query: args.query,
bookmarks,
count: bookmarks.length,
})
},
})

5
apps/server/src/tools/browseros-info.ts
View File

@@ -117,8 +117,13 @@ export const browseros_info = defineTool({
'Specific topic to get info about. Use "overview" for general questions.',
),
}),
output: z.object({
topic: z.enum(VALID_TOPICS),
content: z.string(),
}),
handler: async (args, _ctx, response) => {
const content = args.topic ? getTopicContent(args.topic) : BROWSEROS_INFO
response.text(content)
response.data({ topic: args.topic, content })
},
})

46
apps/server/src/tools/dom.ts
View File

@@ -19,6 +19,12 @@ export const get_dom = defineTool({
"CSS selector to scope (e.g. 'main', '#content', 'form.login')",
),
}),
output: z.object({
html: z.string(),
selector: z.string().optional(),
truncated: z.boolean(),
totalLength: z.number(),
}),
handler: async (args, ctx, response) => {
const html = await ctx.browser.getDom(args.page, {
selector: args.selector,
@@ -34,13 +40,26 @@ export const get_dom = defineTool({
}
if (html.length > MAX_DOM_HTML_LENGTH) {
const truncatedHtml = html.substring(0, MAX_DOM_HTML_LENGTH)
response.text(
`${html.substring(0, MAX_DOM_HTML_LENGTH)}\n\n[Truncated — ${html.length} chars total. Use a CSS selector to scope to a specific element.]`,
`${truncatedHtml}\n\n[Truncated — ${html.length} chars total. Use a CSS selector to scope to a specific element.]`,
)
response.data({
html: truncatedHtml,
selector: args.selector,
truncated: true,
totalLength: html.length,
})
return
}
response.text(html)
response.data({
html,
selector: args.selector,
truncated: false,
totalLength: html.length,
})
},
})
@@ -61,6 +80,19 @@ export const search_dom = defineTool({
.default(25)
.describe('Maximum number of results to return (1–200)'),
}),
output: z.object({
query: z.string(),
totalCount: z.number(),
shownCount: z.number(),
results: z.array(
z.object({
tag: z.string(),
nodeId: z.number(),
backendNodeId: z.number(),
attributes: z.record(z.string()),
}),
),
}),
handler: async (args, ctx, response) => {
const { results, totalCount } = await ctx.browser.searchDom(
args.page,
@@ -70,6 +102,12 @@ export const search_dom = defineTool({
if (results.length === 0) {
response.text(`No elements matching "${args.query}" found.`)
response.data({
query: args.query,
totalCount,
shownCount: 0,
results: [],
})
return
}
@@ -81,5 +119,11 @@ export const search_dom = defineTool({
response.text(
`Found ${totalCount} matching elements:\n\n${lines.join('\n\n')}${suffix}`,
)
response.data({
query: args.query,
totalCount,
shownCount: results.length,
results,
})
},
})

11
apps/server/src/tools/framework.ts
View File

@@ -6,6 +6,7 @@ export interface ToolDefinition {
name: string
description: string
input: z.ZodType
output?: z.ZodType
handler: ToolHandler
}
@@ -19,12 +20,16 @@ export type ToolContext = {
browser: Browser
}
export function defineTool<T extends z.ZodType>(config: {
export function defineTool<
TInput extends z.ZodType,
TOutput extends z.ZodType | undefined = undefined,
>(config: {
name: string
description: string
input: T
input: TInput
output?: TOutput
handler: (
args: z.infer<T>,
args: z.infer<TInput>,
ctx: ToolContext,
response: ToolResponse,
) => Promise<void>

48
apps/server/src/tools/history.ts
View File

@@ -1,6 +1,15 @@
import { z } from 'zod'
import { defineTool } from './framework'
const historyItemSchema = z.object({
id: z.string(),
url: z.string(),
title: z.string(),
lastVisitTime: z.number(),
visitCount: z.number().optional(),
typedCount: z.number().optional(),
})
export const search_history = defineTool({
name: 'search_history',
description: 'Search browser history by text query',
@@ -11,11 +20,17 @@ export const search_history = defineTool({
.optional()
.describe('Maximum number of results to return (default: 100)'),
}),
output: z.object({
query: z.string(),
items: z.array(historyItemSchema),
count: z.number(),
}),
handler: async (args, ctx, response) => {
const items = await ctx.browser.searchHistory(args.query, args.maxResults)
if (items.length === 0) {
response.text(`No history items found matching "${args.query}".`)
response.data({ query: args.query, items: [], count: 0 })
return
}
@@ -38,6 +53,11 @@ export const search_history = defineTool({
}
response.text(lines.join('\n'))
response.data({
query: args.query,
items,
count: items.length,
})
},
})
@@ -50,11 +70,16 @@ export const get_recent_history = defineTool({
.optional()
.describe('Number of recent items to retrieve (default: 20)'),
}),
output: z.object({
items: z.array(historyItemSchema),
count: z.number(),
}),
handler: async (args, ctx, response) => {
const items = await ctx.browser.getRecentHistory(args.maxResults)
if (items.length === 0) {
response.text('No recent history items.')
response.data({ items: [], count: 0 })
return
}
@@ -77,6 +102,10 @@ export const get_recent_history = defineTool({
}
response.text(lines.join('\n'))
response.data({
items,
count: items.length,
})
},
})
@@ -86,9 +115,14 @@ export const delete_history_url = defineTool({
input: z.object({
url: z.string().describe('URL to delete from history'),
}),
output: z.object({
action: z.literal('delete_history_url'),
url: z.string(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.deleteHistoryUrl(args.url)
response.text(`Deleted ${args.url} from history`)
response.data({ action: 'delete_history_url', url: args.url })
},
})
@@ -99,10 +133,24 @@ export const delete_history_range = defineTool({
startTime: z.number().describe('Start time as epoch ms'),
endTime: z.number().describe('End time as epoch ms'),
}),
output: z.object({
action: z.literal('delete_history_range'),
startTime: z.number(),
endTime: z.number(),
startIso: z.string(),
endIso: z.string(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.deleteHistoryRange(args.startTime, args.endTime)
const start = new Date(args.startTime).toISOString()
const end = new Date(args.endTime).toISOString()
response.text(`Deleted history from ${start} to ${end}`)
response.data({
action: 'delete_history_range',
startTime: args.startTime,
endTime: args.endTime,
startIso: start,
endIso: end,
})
},
})

150
apps/server/src/tools/input.ts
View File

@@ -21,12 +21,26 @@ export const click = defineTool({
.default(1)
.describe('Number of clicks (2 for double-click)'),
}),
output: z.object({
action: z.literal('click'),
page: z.number(),
element: z.number(),
button: z.enum(['left', 'right', 'middle']),
clickCount: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.click(args.page, args.element, {
button: args.button,
clickCount: args.clickCount,
})
response.text(`Clicked [${args.element}]`)
response.data({
action: 'click',
page: args.page,
element: args.element,
button: args.button,
clickCount: args.clickCount,
})
response.includeSnapshot(args.page)
},
})
@@ -44,12 +58,28 @@ export const click_at = defineTool({
.describe('Mouse button'),
clickCount: z.number().default(1).describe('Number of clicks'),
}),
output: z.object({
action: z.literal('click_at'),
page: z.number(),
x: z.number(),
y: z.number(),
button: z.enum(['left', 'right', 'middle']),
clickCount: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.clickAt(args.page, args.x, args.y, {
button: args.button,
clickCount: args.clickCount,
})
response.text(`Clicked at (${args.x}, ${args.y})`)
response.data({
action: 'click_at',
page: args.page,
x: args.x,
y: args.y,
button: args.button,
clickCount: args.clickCount,
})
response.includeSnapshot(args.page)
},
})
@@ -61,9 +91,15 @@ export const hover = defineTool({
page: pageParam,
element: elementParam,
}),
output: z.object({
action: z.literal('hover'),
page: z.number(),
element: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.hover(args.page, args.element)
response.text(`Hovered over [${args.element}]`)
response.data({ action: 'hover', page: args.page, element: args.element })
},
})
@@ -74,9 +110,15 @@ export const clear = defineTool({
page: pageParam,
element: elementParam,
}),
output: z.object({
action: z.literal('clear'),
page: z.number(),
element: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.fill(args.page, args.element, '', true)
response.text(`Cleared [${args.element}]`)
response.data({ action: 'clear', page: args.page, element: args.element })
response.includeSnapshot(args.page)
},
})
@@ -94,9 +136,23 @@ export const fill = defineTool({
.default(true)
.describe('Clear existing text before typing'),
}),
output: z.object({
action: z.literal('fill'),
page: z.number(),
element: z.number(),
textLength: z.number(),
clear: z.boolean(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.fill(args.page, args.element, args.text, args.clear)
response.text(`Typed ${args.text.length} characters into [${args.element}]`)
response.data({
action: 'fill',
page: args.page,
element: args.element,
textLength: args.text.length,
clear: args.clear,
})
response.includeSnapshot(args.page)
},
})
@@ -111,9 +167,15 @@ export const press_key = defineTool({
.string()
.describe("Key or combo like 'Enter', 'Control+A', 'ArrowDown'"),
}),
output: z.object({
action: z.literal('press_key'),
page: z.number(),
key: z.string(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.pressKey(args.page, args.key)
response.text(`Pressed ${args.key}`)
response.data({ action: 'press_key', page: args.page, key: args.key })
},
})
@@ -134,6 +196,14 @@ export const drag = defineTool({
.optional()
.describe('Target Y coordinate (if not using targetElement)'),
}),
output: z.object({
action: z.literal('drag'),
page: z.number(),
sourceElement: z.number(),
targetElement: z.number().optional(),
targetX: z.number().optional(),
targetY: z.number().optional(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.drag(args.page, args.sourceElement, {
element: args.targetElement,
@@ -145,6 +215,14 @@ export const drag = defineTool({
? `[${args.targetElement}]`
: `(${args.targetX}, ${args.targetY})`
response.text(`Dragged [${args.sourceElement}] \u2192 ${target}`)
response.data({
action: 'drag',
page: args.page,
sourceElement: args.sourceElement,
targetElement: args.targetElement,
targetX: args.targetX,
targetY: args.targetY,
})
response.includeSnapshot(args.page)
},
})
@@ -164,6 +242,13 @@ export const scroll = defineTool({
.optional()
.describe('Element ID to scroll at (scrolls page center if omitted)'),
}),
output: z.object({
action: z.literal('scroll'),
page: z.number(),
direction: z.enum(['up', 'down', 'left', 'right']),
amount: z.number(),
element: z.number().optional(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.scroll(
args.page,
@@ -172,6 +257,13 @@ export const scroll = defineTool({
args.element,
)
response.text(`Scrolled ${args.direction} by ${args.amount}`)
response.data({
action: 'scroll',
page: args.page,
direction: args.direction,
amount: args.amount,
element: args.element,
})
},
})
@@ -186,9 +278,21 @@ export const handle_dialog = defineTool({
.optional()
.describe('Text to enter for prompt dialogs'),
}),
output: z.object({
action: z.literal('handle_dialog'),
page: z.number(),
accept: z.boolean(),
promptText: z.string().optional(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.handleDialog(args.page, args.accept, args.promptText)
response.text(args.accept ? 'Dialog accepted' : 'Dialog dismissed')
response.data({
action: 'handle_dialog',
page: args.page,
accept: args.accept,
promptText: args.promptText,
})
},
})
@@ -199,9 +303,15 @@ export const focus = defineTool({
page: pageParam,
element: elementParam,
}),
output: z.object({
action: z.literal('focus'),
page: z.number(),
element: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.focus(args.page, args.element)
response.text(`Focused [${args.element}]`)
response.data({ action: 'focus', page: args.page, element: args.element })
},
})
@@ -212,9 +322,15 @@ export const check = defineTool({
page: pageParam,
element: elementParam,
}),
output: z.object({
action: z.literal('check'),
page: z.number(),
element: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.check(args.page, args.element)
response.text(`Checked [${args.element}]`)
response.data({ action: 'check', page: args.page, element: args.element })
response.includeSnapshot(args.page)
},
})
@@ -226,9 +342,15 @@ export const uncheck = defineTool({
page: pageParam,
element: elementParam,
}),
output: z.object({
action: z.literal('uncheck'),
page: z.number(),
element: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.uncheck(args.page, args.element)
response.text(`Unchecked [${args.element}]`)
response.data({ action: 'uncheck', page: args.page, element: args.element })
response.includeSnapshot(args.page)
},
})
@@ -244,9 +366,23 @@ export const upload_file = defineTool({
),
files: z.array(z.string()).describe('Absolute file paths to upload'),
}),
output: z.object({
action: z.literal('upload_file'),
page: z.number(),
element: z.number(),
files: z.array(z.string()),
fileCount: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.uploadFile(args.page, args.element, args.files)
response.text(`Set ${args.files.length} file(s) on [${args.element}]`)
response.data({
action: 'upload_file',
page: args.page,
element: args.element,
files: args.files,
fileCount: args.files.length,
})
response.includeSnapshot(args.page)
},
})
@@ -260,6 +396,13 @@ export const select_option = defineTool({
element: elementParam.describe('Element ID of the <select> element'),
value: z.string().describe('Option value or visible text to select'),
}),
output: z.object({
action: z.literal('select_option'),
page: z.number(),
element: z.number(),
value: z.string(),
selected: z.string(),
}),
handler: async (args, ctx, response) => {
const selected = await ctx.browser.selectOption(
args.page,
@@ -273,6 +416,13 @@ export const select_option = defineTool({
return
}
response.text(`Selected "${selected}" in [${args.element}]`)
response.data({
action: 'select_option',
page: args.page,
element: args.element,
value: args.value,
selected,
})
response.includeSnapshot(args.page)
},
})

88
apps/server/src/tools/navigation.ts
View File

@@ -2,11 +2,27 @@ import { z } from 'zod'
import { defineTool } from './framework'
const pageParam = z.number().describe('Page ID (from list_pages)')
const pageInfoSchema = z.object({
pageId: z.number(),
targetId: z.string(),
tabId: z.number(),
url: z.string(),
title: z.string(),
isActive: z.boolean(),
isLoading: z.boolean(),
loadProgress: z.number(),
isPinned: z.boolean(),
isHidden: z.boolean(),
windowId: z.number().optional(),
index: z.number().optional(),
groupId: z.string().optional(),
})
export const get_active_page = defineTool({
name: 'get_active_page',
description: 'Get the currently active (focused) page in the browser',
input: z.object({}),
output: z.object({ page: pageInfoSchema }),
handler: async (_args, ctx, response) => {
const page = await ctx.browser.getActivePage()
if (!page) {
@@ -16,6 +32,7 @@ export const get_active_page = defineTool({
response.text(
`Active page: ${page.pageId} (tab ${page.tabId})\n${page.title}\n${page.url}`,
)
response.data({ page })
},
})
@@ -23,11 +40,16 @@ export const list_pages = defineTool({
name: 'list_pages',
description: 'List all pages (tabs) currently open in the browser',
input: z.object({}),
output: z.object({
pages: z.array(pageInfoSchema),
count: z.number(),
}),
handler: async (_args, ctx, response) => {
const pages = await ctx.browser.listPages()
if (pages.length === 0) {
response.text('No pages open.')
response.data({ pages: [], count: 0 })
return
}
@@ -35,6 +57,7 @@ export const list_pages = defineTool({
(p) => `${p.pageId}. ${p.title} (tab ${p.tabId})\n ${p.url}`,
)
response.text(lines.join('\n\n'))
response.data({ pages, count: pages.length })
},
})
@@ -52,6 +75,11 @@ export const navigate_page = defineTool({
.optional()
.describe("URL to navigate to (required when action is 'url')"),
}),
output: z.object({
page: z.number(),
action: z.enum(['url', 'back', 'forward', 'reload']),
url: z.string().optional(),
}),
handler: async (args, ctx, response) => {
if (args.action === 'url' && !args.url) {
response.error(
@@ -82,6 +110,11 @@ export const navigate_page = defineTool({
reload: 'Page reloaded',
}
response.text(messages[args.action] ?? 'Done')
response.data({
page: args.page,
action: args.action,
url: args.url,
})
response.includeSnapshot(args.page)
},
})
@@ -98,6 +131,13 @@ export const new_page = defineTool({
.describe('Open in background without activating'),
windowId: z.number().optional().describe('Window ID to create tab in'),
}),
output: z.object({
pageId: z.number(),
url: z.string(),
hidden: z.boolean(),
background: z.boolean(),
windowId: z.number().optional(),
}),
handler: async (args, ctx, response) => {
const pageId = await ctx.browser.newPage(args.url, {
hidden: args.hidden || undefined,
@@ -105,6 +145,13 @@ export const new_page = defineTool({
windowId: args.windowId,
})
response.text(`Opened new page: ${args.url}\nPage ID: ${pageId}`)
response.data({
pageId,
url: args.url,
hidden: args.hidden,
background: args.background,
windowId: args.windowId,
})
response.includePages()
},
})
@@ -117,6 +164,13 @@ export const new_hidden_page = defineTool({
url: z.string().describe('URL to open'),
windowId: z.number().optional().describe('Window ID to create tab in'),
}),
output: z.object({
pageId: z.number(),
url: z.string(),
hidden: z.literal(true),
background: z.literal(true),
windowId: z.number().optional(),
}),
handler: async (args, ctx, response) => {
const pageId = await ctx.browser.newPage(args.url, {
hidden: true,
@@ -124,6 +178,13 @@ export const new_hidden_page = defineTool({
windowId: args.windowId,
})
response.text(`Opened hidden page: ${args.url}\nPage ID: ${pageId}`)
response.data({
pageId,
url: args.url,
hidden: true,
background: true,
windowId: args.windowId,
})
response.includePages()
},
})
@@ -147,6 +208,7 @@ export const show_page = defineTool({
.default(true)
.describe('Activate (focus) the tab after showing'),
}),
output: z.object({ page: pageInfoSchema }),
handler: async (args, ctx, response) => {
const updated = await ctx.browser.showPage(args.page, {
windowId: args.windowId,
@@ -156,6 +218,7 @@ export const show_page = defineTool({
response.text(
`Page ${args.page} is now visible in window ${updated.windowId}`,
)
response.data({ page: updated })
response.includePages()
},
})
@@ -175,6 +238,7 @@ export const move_page = defineTool({
.optional()
.describe('Tab position index within the target window'),
}),
output: z.object({ page: pageInfoSchema }),
handler: async (args, ctx, response) => {
const updated = await ctx.browser.movePage(args.page, {
windowId: args.windowId,
@@ -183,6 +247,7 @@ export const move_page = defineTool({
response.text(
`Moved page ${args.page} to window ${updated.windowId} at index ${updated.index}`,
)
response.data({ page: updated })
response.includePages()
},
})
@@ -193,9 +258,14 @@ export const close_page = defineTool({
input: z.object({
page: pageParam,
}),
output: z.object({
page: z.number(),
action: z.literal('close_page'),
}),
handler: async (args, ctx, response) => {
await ctx.browser.closePage(args.page)
response.text(`Closed page ${args.page}`)
response.data({ page: args.page, action: 'close_page' })
response.includePages()
},
})
@@ -213,6 +283,12 @@ export const wait_for = defineTool({
.default(10000)
.describe('Maximum wait time in milliseconds'),
}),
output: z.object({
page: z.number(),
found: z.boolean(),
target: z.string(),
timeout: z.number(),
}),
handler: async (args, ctx, response) => {
if (!args.text && !args.selector) {
response.error('Provide either text or selector to wait for.')
@@ -230,11 +306,23 @@ export const wait_for = defineTool({
? `text "${args.text}"`
: `selector "${args.selector}"`
response.text(`Found ${target} on page.`)
response.data({
page: args.page,
found,
target,
timeout: args.timeout,
})
response.includeSnapshot(args.page)
} else {
const target = args.text
? `text "${args.text}"`
: `selector "${args.selector}"`
response.data({
page: args.page,
found,
target,
timeout: args.timeout,
})
response.error(`Timed out after ${args.timeout}ms waiting for ${target}.`)
}
},

42
apps/server/src/tools/page-actions.ts
View File

@@ -20,11 +20,21 @@ export const save_pdf = defineTool({
.optional()
.describe('Working directory to resolve relative paths against'),
}),
output: z.object({
action: z.literal('save_pdf'),
page: z.number(),
path: z.string(),
}),
handler: async (args, ctx, response) => {
const resolvedPath = resolve(args.cwd ?? process.cwd(), args.path)
const { data } = await ctx.browser.printToPDF(args.page)
await Bun.write(resolvedPath, Buffer.from(data, 'base64'))
response.text(`Saved PDF to ${resolvedPath}`)
response.data({
action: 'save_pdf',
page: args.page,
path: resolvedPath,
})
},
})
@@ -55,6 +65,14 @@ export const save_screenshot = defineTool({
.default(false)
.describe('Capture full scrollable page'),
}),
output: z.object({
action: z.literal('save_screenshot'),
page: z.number(),
path: z.string(),
format: z.enum(['png', 'jpeg', 'webp']),
quality: z.number().optional(),
fullPage: z.boolean(),
}),
handler: async (args, ctx, response) => {
const resolvedPath = resolve(args.cwd ?? process.cwd(), args.path)
const { data } = await ctx.browser.screenshot(args.page, {
@@ -64,6 +82,14 @@ export const save_screenshot = defineTool({
})
await Bun.write(resolvedPath, Buffer.from(data, 'base64'))
response.text(`Saved screenshot to ${resolvedPath}`)
response.data({
action: 'save_screenshot',
page: args.page,
path: resolvedPath,
format: args.format,
quality: args.quality,
fullPage: args.fullPage,
})
},
})
@@ -80,6 +106,14 @@ export const download_file = defineTool({
.optional()
.describe('Working directory to resolve relative paths against'),
}),
output: z.object({
action: z.literal('download_file'),
page: z.number(),
element: z.number(),
directory: z.string(),
suggestedFilename: z.string(),
destinationPath: z.string(),
}),
handler: async (args, ctx, response) => {
const resolvedDir = resolve(args.cwd ?? process.cwd(), args.path)
const tempDir = await mkdtemp(join(tmpdir(), 'browseros-dl-'))
@@ -92,6 +126,14 @@ export const download_file = defineTool({
await rename(filePath, destPath)
response.text(`Downloaded "${suggestedFilename}" to ${destPath}`)
response.data({
action: 'download_file',
page: args.page,
element: args.element,
directory: resolvedDir,
suggestedFilename,
destinationPath: destPath,
})
} finally {
await rm(tempDir, { recursive: true, force: true }).catch(() => {})
}

14
apps/server/src/tools/response.ts
View File

@@ -18,6 +18,7 @@ export interface ToolResult {
content: ContentItem[]
isError?: boolean
metadata?: ToolResultMetadata
structuredContent?: Record<string, unknown>
}
interface ToolResponseOptions {
@@ -27,6 +28,7 @@ interface ToolResponseOptions {
export class ToolResponse {
private content: ContentItem[] = []
private hasError = false
private structured: Record<string, unknown> = {}
private postActions: PostAction[] = []
private postActionTimeoutMs: number
@@ -48,6 +50,16 @@ export class ToolResponse {
this.content.push({ type: 'text', text: message })
}
data(key: string, value: unknown): void
data(obj: Record<string, unknown>): void
data(keyOrObj: string | Record<string, unknown>, value?: unknown): void {
if (typeof keyOrObj === 'string') {
this.structured[keyOrObj] = value
return
}
Object.assign(this.structured, keyOrObj)
}
includeSnapshot(page: number): void {
this.postActions.push({ type: 'snapshot', page })
}
@@ -127,9 +139,11 @@ export class ToolResponse {
}
toResult(): ToolResult {
const hasStructured = Object.keys(this.structured).length > 0
return {
content: this.content,
...(this.hasError && { isError: true }),
...(hasStructured && { structuredContent: this.structured }),
}
}
}

57
apps/server/src/tools/snapshot.ts
View File

@@ -10,9 +10,13 @@ export const take_snapshot = defineTool({
input: z.object({
page: pageParam,
}),
output: z.object({
snapshot: z.string(),
}),
handler: async (args, ctx, response) => {
const tree = await ctx.browser.snapshot(args.page)
response.text(tree || 'Page has no interactive elements.')
response.data({ snapshot: tree || '' })
},
})
@@ -23,9 +27,13 @@ export const take_enhanced_snapshot = defineTool({
input: z.object({
page: pageParam,
}),
output: z.object({
snapshot: z.string(),
}),
handler: async (args, ctx, response) => {
const tree = await ctx.browser.enhancedSnapshot(args.page)
response.text(tree || 'Page has no visible content.')
response.data({ snapshot: tree || '' })
},
})
@@ -54,6 +62,13 @@ export const get_page_content = defineTool({
.default(false)
.describe('Include image references as ![alt](src)'),
}),
output: z.object({
content: z.string(),
selector: z.string().optional(),
viewportOnly: z.boolean(),
includeLinks: z.boolean(),
includeImages: z.boolean(),
}),
handler: async (args, ctx, response) => {
const text = await ctx.browser.contentAsMarkdown(args.page, {
selector: args.selector,
@@ -62,6 +77,13 @@ export const get_page_content = defineTool({
includeImages: args.includeImages,
})
response.text(text || 'No text content found.')
response.data({
content: text || '',
selector: args.selector,
viewportOnly: args.viewportOnly,
includeLinks: args.includeLinks,
includeImages: args.includeImages,
})
},
})
@@ -85,6 +107,9 @@ export const take_screenshot = defineTool({
.default(false)
.describe('Capture full scrollable page'),
}),
output: z.object({
mimeType: z.string(),
}),
handler: async (args, ctx, response) => {
const { data, mimeType } = await ctx.browser.screenshot(args.page, {
format: args.format,
@@ -92,6 +117,7 @@ export const take_screenshot = defineTool({
fullPage: args.fullPage,
})
response.image(data, mimeType)
response.data({ mimeType })
},
})
@@ -102,16 +128,27 @@ export const get_page_links = defineTool({
input: z.object({
page: pageParam,
}),
output: z.object({
links: z.array(
z.object({
text: z.string(),
href: z.string(),
}),
),
count: z.number(),
}),
handler: async (args, ctx, response) => {
const links = await ctx.browser.getPageLinks(args.page)
if (links.length === 0) {
response.text('No links found on the page.')
response.data({ links: [], count: 0 })
return
}
const lines = links.map((l) => (l.text ? `[${l.text}](${l.href})` : l.href))
response.text(lines.join('\n'))
response.data({ links, count: links.length })
},
})
@@ -123,6 +160,11 @@ export const evaluate_script = defineTool({
page: pageParam,
expression: z.string().describe('JavaScript expression to evaluate'),
}),
output: z.object({
text: z.string(),
value: z.unknown().optional(),
description: z.string().optional(),
}),
handler: async (args, ctx, response) => {
const result = await ctx.browser.evaluate(args.page, args.expression)
@@ -132,12 +174,21 @@ export const evaluate_script = defineTool({
}
const val = result.value
let text: string
if (val === undefined) {
response.text(result.description ?? 'undefined')
text = result.description ?? 'undefined'
response.text(text)
} else if (typeof val === 'string') {
response.text(val)
text = val
response.text(text)
} else {
response.text(JSON.stringify(val, null, 2))
text = JSON.stringify(val, null, 2)
response.text(text)
}
response.data({
text,
value: result.value,
description: result.description,
})
},
})

56
apps/server/src/tools/tab-groups.ts
View File

@@ -13,15 +13,29 @@ const TAB_GROUP_COLORS = [
'orange',
] as const
const tabGroupWithPageIdsSchema = z.object({
groupId: z.string(),
windowId: z.number(),
title: z.string(),
color: z.string(),
collapsed: z.boolean(),
pageIds: z.array(z.number()),
})
export const list_tab_groups = defineTool({
name: 'list_tab_groups',
description: 'List all tab groups in the browser',
input: z.object({}),
output: z.object({
groups: z.array(tabGroupWithPageIdsSchema),
count: z.number(),
}),
handler: async (_args, ctx, response) => {
const groups = await ctx.browser.listTabGroups()
if (groups.length === 0) {
response.text('No tab groups found.')
response.data({ groups: [], count: 0 })
return
}
@@ -37,6 +51,7 @@ export const list_tab_groups = defineTool({
}
response.text(lines.join('\n'))
response.data({ groups, count: groups.length })
},
})
@@ -51,6 +66,10 @@ export const group_tabs = defineTool({
title: z.string().optional().describe('Title for the group'),
groupId: z.string().optional().describe('Existing group ID to add tabs to'),
}),
output: z.object({
group: tabGroupWithPageIdsSchema,
groupedCount: z.number(),
}),
handler: async (args, ctx, response) => {
const group = await ctx.browser.groupTabs(args.pageIds, {
title: args.title,
@@ -59,6 +78,10 @@ export const group_tabs = defineTool({
response.text(
`Grouped ${args.pageIds.length} pages into "${group.title || '(unnamed)'}" (${group.color})\nGroup ID: ${group.groupId}`,
)
response.data({
group,
groupedCount: args.pageIds.length,
})
},
})
@@ -77,15 +100,33 @@ export const update_tab_group = defineTool({
.optional()
.describe('Whether to collapse (hide) the group tabs'),
}),
output: z.object({
group: tabGroupWithPageIdsSchema,
}),
handler: async (args, ctx, response) => {
const group = await ctx.browser.updateTabGroup(args.groupId, {
title: args.title,
color: args.color,
collapsed: args.collapsed,
})
const tabToPage = await ctx.browser.resolveTabIds(group.tabIds)
const pageIds = group.tabIds
.map((tabId) => tabToPage.get(tabId))
.filter((id): id is number => id !== undefined)
const groupWithPageIds = {
groupId: group.groupId,
windowId: group.windowId,
title: group.title,
color: group.color,
collapsed: group.collapsed,
pageIds,
}
response.text(
`Updated group ${group.groupId}: "${group.title || '(unnamed)'}" (${group.color})${group.collapsed ? ' [COLLAPSED]' : ''}`,
)
response.data({
group: groupWithPageIds,
})
},
})
@@ -97,9 +138,19 @@ export const ungroup_tabs = defineTool({
.array(z.number())
.describe('Array of page IDs to remove from their groups'),
}),
output: z.object({
action: z.literal('ungroup_tabs'),
pageIds: z.array(z.number()),
count: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.ungroupTabs(args.pageIds)
response.text(`Ungrouped ${args.pageIds.length} pages`)
response.data({
action: 'ungroup_tabs',
pageIds: args.pageIds,
count: args.pageIds.length,
})
},
})
@@ -111,9 +162,14 @@ export const close_tab_group = defineTool({
.string()
.describe('ID of the group to close (closes all tabs in group)'),
}),
output: z.object({
action: z.literal('close_tab_group'),
groupId: z.string(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.closeTabGroup(args.groupId)
response.text(`Closed tab group ${args.groupId} and all its tabs`)
response.data({ action: 'close_tab_group', groupId: args.groupId })
response.includePages()
},
})

49
apps/server/src/tools/windows.ts
View File

@@ -1,15 +1,45 @@
import { z } from 'zod'
import { defineTool } from './framework'
const windowInfoSchema = z.object({
windowId: z.number(),
windowType: z.enum([
'normal',
'popup',
'app',
'devtools',
'app_popup',
'picture_in_picture',
]),
bounds: z.object({
left: z.number().optional(),
top: z.number().optional(),
width: z.number().optional(),
height: z.number().optional(),
windowState: z
.enum(['normal', 'minimized', 'maximized', 'fullscreen'])
.optional(),
}),
isActive: z.boolean(),
isVisible: z.boolean(),
tabCount: z.number(),
activeTabId: z.number().optional(),
})
export const list_windows = defineTool({
name: 'list_windows',
description: 'List all browser windows',
input: z.object({}),
output: z.object({
windows: z.array(windowInfoSchema),
count: z.number(),
}),
handler: async (_args, ctx, response) => {
const windows = await ctx.browser.listWindows()
if (windows.length === 0) {
response.text('No windows found.')
response.data({ windows: [], count: 0 })
return
}
@@ -25,6 +55,7 @@ export const list_windows = defineTool({
}
response.text(lines.join('\n'))
response.data({ windows, count: windows.length })
},
})
@@ -34,10 +65,14 @@ export const create_window = defineTool({
input: z.object({
hidden: z.boolean().optional().describe('Create as hidden window'),
}),
output: z.object({
window: windowInfoSchema,
}),
handler: async (args, ctx, response) => {
const window = await ctx.browser.createWindow(args)
const hiddenMarker = !window.isVisible ? ' (hidden)' : ''
response.text(`Created window ${window.windowId}${hiddenMarker}`)
response.data({ window })
},
})
@@ -46,9 +81,13 @@ export const create_hidden_window = defineTool({
description:
'Create a new hidden browser window. Hidden windows are not visible to the user and useful for background automation. Note: take_screenshot is not supported on hidden windows.',
input: z.object({}),
output: z.object({
window: windowInfoSchema,
}),
handler: async (_args, ctx, response) => {
const window = await ctx.browser.createWindow({ hidden: true })
response.text(`Created hidden window ${window.windowId}`)
response.data({ window })
},
})
@@ -58,9 +97,14 @@ export const close_window = defineTool({
input: z.object({
windowId: z.number().describe('Window ID to close'),
}),
output: z.object({
action: z.literal('close_window'),
windowId: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.closeWindow(args.windowId)
response.text(`Closed window ${args.windowId}`)
response.data({ action: 'close_window', windowId: args.windowId })
response.includePages()
},
})
@@ -71,8 +115,13 @@ export const activate_window = defineTool({
input: z.object({
windowId: z.number().describe('Window ID to activate'),
}),
output: z.object({
action: z.literal('activate_window'),
windowId: z.number(),
}),
handler: async (args, ctx, response) => {
await ctx.browser.activateWindow(args.windowId)
response.text(`Activated window ${args.windowId}`)
response.data({ action: 'activate_window', windowId: args.windowId })
},
})

57
apps/server/tests/tools/bookmarks.test.ts
View File

@@ -19,6 +19,11 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
describe('bookmark tools', () => {
it('full CRUD lifecycle', async () => {
await withBrowser(async ({ execute }) => {
@@ -28,23 +33,35 @@ describe('bookmark tools', () => {
url: 'https://example.com/test-bookmark',
})
assert.ok(!createResult.isError, textOf(createResult))
const createText = textOf(createResult)
assert.ok(createText.includes('Test Bookmark'))
const idMatch = createText.match(/ID:\s*(\S+)/)
assert.ok(idMatch, 'Could not extract bookmark ID')
const bookmarkId = idMatch?.[1]
const createData = structuredOf<{
bookmark: { id: string; title: string }
}>(createResult)
assert.strictEqual(createData.bookmark.title, 'Test Bookmark')
const bookmarkId = createData.bookmark.id
// Get
const getResult = await execute(get_bookmarks, {})
assert.ok(!getResult.isError, textOf(getResult))
assert.ok(textOf(getResult).includes('Test Bookmark'))
const getData = structuredOf<{ bookmarks: Array<{ title: string }> }>(
getResult,
)
assert.ok(
getData.bookmarks.some(
(bookmark) => bookmark.title === 'Test Bookmark',
),
)
// Search
const searchResult = await execute(search_bookmarks, {
query: 'Test Bookmark',
})
assert.ok(!searchResult.isError, textOf(searchResult))
assert.ok(textOf(searchResult).includes('Test Bookmark'))
const searchData = structuredOf<{ bookmarks: Array<{ id: string }> }>(
searchResult,
)
assert.ok(
searchData.bookmarks.some((bookmark) => bookmark.id === bookmarkId),
)
// Update
const updateResult = await execute(update_bookmark, {
@@ -52,12 +69,19 @@ describe('bookmark tools', () => {
title: 'Updated Bookmark',
})
assert.ok(!updateResult.isError, textOf(updateResult))
assert.ok(textOf(updateResult).includes('Updated Bookmark'))
const updateData = structuredOf<{ bookmark: { title: string } }>(
updateResult,
)
assert.strictEqual(updateData.bookmark.title, 'Updated Bookmark')
// Remove
const removeResult = await execute(remove_bookmark, { id: bookmarkId })
assert.ok(!removeResult.isError, textOf(removeResult))
assert.ok(textOf(removeResult).includes('Removed'))
const removeData = structuredOf<{ action: string; id: string }>(
removeResult,
)
assert.strictEqual(removeData.action, 'remove_bookmark')
assert.strictEqual(removeData.id, bookmarkId)
})
}, 60_000)
@@ -68,15 +92,19 @@ describe('bookmark tools', () => {
title: 'Test Folder',
})
assert.ok(!folderResult.isError, textOf(folderResult))
assert.ok(textOf(folderResult).includes('folder'))
const folderId = textOf(folderResult).match(/ID:\s*(\S+)/)?.[1]
const folderData = structuredOf<{
bookmark: { id: string; type: string }
}>(folderResult)
assert.strictEqual(folderData.bookmark.type, 'folder')
const folderId = folderData.bookmark.id
// Create bookmark
const bmResult = await execute(create_bookmark, {
title: 'Movable Bookmark',
url: 'https://example.com/movable',
})
const bmId = textOf(bmResult).match(/ID:\s*(\S+)/)?.[1]
const bmId = structuredOf<{ bookmark: { id: string } }>(bmResult).bookmark
.id
// Move into folder
const moveResult = await execute(move_bookmark, {
@@ -84,7 +112,10 @@ describe('bookmark tools', () => {
parentId: folderId,
})
assert.ok(!moveResult.isError, textOf(moveResult))
assert.ok(textOf(moveResult).includes('Moved'))
const moveData = structuredOf<{ bookmark: { parentId?: string } }>(
moveResult,
)
assert.strictEqual(moveData.bookmark.parentId, folderId)
// Cleanup
await execute(remove_bookmark, { id: folderId })

68
apps/server/tests/tools/dom.test.ts
View File

@@ -14,6 +14,20 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
function pageIdOf(result: {
content: { type: string; text?: string }[]
structuredContent?: unknown
}): number {
const data = result.structuredContent as { pageId?: number } | undefined
if (typeof data?.pageId === 'number') return data.pageId
return Number(textOf(result).match(/Page ID:\s*(\d+)/)?.[1])
}
const RICH_PAGE = `data:text/html,${encodeURIComponent(`<!DOCTYPE html>
<html><head><title>DOM Test Page</title></head><body>
<header id="main-header">
@@ -53,7 +67,7 @@ describe('get_dom', () => {
it('returns full page HTML', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(get_dom, { page: pageId })
assert.ok(!result.isError, textOf(result))
@@ -67,6 +81,13 @@ describe('get_dom', () => {
'Should contain heading text',
)
assert.ok(html.includes('<head>'), 'Full page should include <head>')
const data = structuredOf<{
html: string
truncated: boolean
totalLength: number
}>(result)
assert.strictEqual(data.truncated, false)
assert.ok(data.totalLength >= data.html.length)
await execute(close_page, { page: pageId })
})
@@ -75,7 +96,7 @@ describe('get_dom', () => {
it('scopes to a CSS selector', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(get_dom, {
page: pageId,
@@ -107,7 +128,7 @@ describe('get_dom', () => {
it('scopes to a nested CSS selector', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(get_dom, {
page: pageId,
@@ -128,7 +149,7 @@ describe('get_dom', () => {
it('returns error for non-matching selector', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(get_dom, {
page: pageId,
@@ -147,7 +168,7 @@ describe('get_dom', () => {
it('returns HTML for about:blank', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(get_dom, { page: pageId })
assert.ok(!result.isError, textOf(result))
@@ -164,7 +185,7 @@ describe('get_dom', () => {
it('truncates very large DOM with message', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
// Generate a large DOM: 10000 divs each with 20 chars ~ 200KB
await execute(evaluate_script, {
@@ -198,7 +219,7 @@ describe('get_dom', () => {
it('preserves element attributes in output', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(get_dom, {
page: pageId,
@@ -228,7 +249,7 @@ describe('search_dom', () => {
it('finds elements by plain text', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -241,6 +262,13 @@ describe('search_dom', () => {
text.includes('matching elements'),
'Should include match count',
)
const data = structuredOf<{
query: string
shownCount: number
totalCount: number
}>(result)
assert.strictEqual(data.query, 'Welcome to Test Page')
assert.ok(data.totalCount >= data.shownCount)
await execute(close_page, { page: pageId })
})
@@ -249,7 +277,7 @@ describe('search_dom', () => {
it('finds elements by CSS selector', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -271,7 +299,7 @@ describe('search_dom', () => {
it('finds elements by XPath', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -289,7 +317,7 @@ describe('search_dom', () => {
it('finds multiple elements with CSS class selector', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -306,7 +334,7 @@ describe('search_dom', () => {
it('finds elements by ID selector', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -328,7 +356,7 @@ describe('search_dom', () => {
it('returns no-match message for non-existent content', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -347,7 +375,7 @@ describe('search_dom', () => {
it('respects limit parameter', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -377,7 +405,7 @@ describe('search_dom', () => {
it('returns element attributes in search results', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -400,7 +428,7 @@ describe('search_dom', () => {
it('finds elements on dynamically modified page', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
await execute(evaluate_script, {
page: pageId,
@@ -432,7 +460,7 @@ describe('search_dom', () => {
it('finds elements using attribute selector', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -450,7 +478,7 @@ describe('search_dom', () => {
it('finds text across multiple elements', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -472,7 +500,7 @@ describe('search_dom', () => {
it('includes nodeId in search results for element reference', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: RICH_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,
@@ -499,7 +527,7 @@ describe('search_dom', () => {
it('handles empty page gracefully', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(search_dom, {
page: pageId,

35
apps/server/tests/tools/history.test.ts
View File

@@ -18,12 +18,26 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
function pageIdOf(result: {
content: { type: string; text?: string }[]
structuredContent?: unknown
}): number {
const data = result.structuredContent as { pageId?: number } | undefined
if (typeof data?.pageId === 'number') return data.pageId
return Number(textOf(result).match(/Page ID:\s*(\d+)/)?.[1])
}
describe('history tools', () => {
it('get_recent_history returns items', async () => {
await withBrowser(async ({ execute }) => {
// Navigate somewhere to ensure history exists
const newResult = await execute(new_page, { url: 'https://example.com' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(get_recent_history, { maxResults: 10 })
assert.ok(!result.isError, textOf(result))
@@ -32,6 +46,8 @@ describe('history tools', () => {
text.includes('history') || text.includes('Retrieved'),
'Expected history response',
)
const data = structuredOf<{ items: unknown[]; count: number }>(result)
assert.strictEqual(data.items.length, data.count)
await execute(close_page, { page: pageId })
})
@@ -44,6 +60,13 @@ describe('history tools', () => {
maxResults: 10,
})
assert.ok(!result.isError, textOf(result))
const data = structuredOf<{
query: string
items: unknown[]
count: number
}>(result)
assert.strictEqual(data.query, 'example')
assert.strictEqual(data.items.length, data.count)
})
}, 60_000)
@@ -54,6 +77,8 @@ describe('history tools', () => {
})
assert.ok(!result.isError, textOf(result))
assert.ok(textOf(result).includes('Deleted'))
const data = structuredOf<{ action: string; url: string }>(result)
assert.strictEqual(data.action, 'delete_history_url')
})
}, 60_000)
@@ -66,6 +91,14 @@ describe('history tools', () => {
})
assert.ok(!result.isError, textOf(result))
assert.ok(textOf(result).includes('Deleted history from'))
const data = structuredOf<{
action: string
startTime: number
endTime: number
}>(result)
assert.strictEqual(data.action, 'delete_history_range')
assert.strictEqual(data.startTime, now - 1000)
assert.strictEqual(data.endTime, now)
})
}, 60_000)
})

42
apps/server/tests/tools/input.test.ts
View File

@@ -23,6 +23,20 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
function pageIdOf(result: {
content: { type: string; text?: string }[]
structuredContent?: unknown
}): number {
const data = result.structuredContent as { pageId?: number } | undefined
if (typeof data?.pageId === 'number') return data.pageId
return Number(textOf(result).match(/Page ID:\s*(\d+)/)?.[1])
}
function escapeRegex(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}
@@ -75,7 +89,7 @@ describe('input tools', () => {
it('fill types text into an input', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snap = await execute(take_snapshot, { page: pageId })
const snapText = textOf(snap)
@@ -87,6 +101,11 @@ describe('input tools', () => {
text: 'John Doe',
})
assert.ok(!fillResult.isError, textOf(fillResult))
const fillData = structuredOf<{ action: string; textLength: number }>(
fillResult,
)
assert.strictEqual(fillData.action, 'fill')
assert.strictEqual(fillData.textLength, 'John Doe'.length)
const val = await execute(evaluate_script, {
page: pageId,
@@ -101,7 +120,7 @@ describe('input tools', () => {
it('click triggers a button', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
// Fill the input first
const snap = await execute(take_snapshot, { page: pageId })
@@ -116,6 +135,11 @@ describe('input tools', () => {
element: btnId,
})
assert.ok(!clickResult.isError, textOf(clickResult))
const clickData = structuredOf<{ action: string; element: number }>(
clickResult,
)
assert.strictEqual(clickData.action, 'click')
assert.strictEqual(clickData.element, btnId)
const output = await execute(evaluate_script, {
page: pageId,
@@ -130,7 +154,7 @@ describe('input tools', () => {
it('check and uncheck toggle a checkbox', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snap = await execute(take_snapshot, { page: pageId })
const snapText = textOf(snap)
@@ -167,7 +191,7 @@ describe('input tools', () => {
it('select_option selects a dropdown value', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
// Use evaluate_script to get the select element's backendNodeId directly
const nodeId = await execute(evaluate_script, {
@@ -203,7 +227,7 @@ describe('input tools', () => {
it('press_key sends a keystroke', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snap = await execute(take_snapshot, { page: pageId })
const inputId = findElementId(textOf(snap), 'Enter name')
@@ -230,7 +254,7 @@ describe('input tools', () => {
it('press_key Enter fires keypress event', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snap = await execute(take_snapshot, { page: pageId })
const inputId = findElementId(textOf(snap), 'Enter name')
@@ -272,7 +296,7 @@ describe('input tools', () => {
it('press_key normalizes case-insensitive key names', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snap = await execute(take_snapshot, { page: pageId })
const inputId = findElementId(textOf(snap), 'Enter name')
@@ -300,7 +324,7 @@ describe('input tools', () => {
const newResult = await execute(new_page, {
url: FORM_PAGE,
})
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const before = await execute(evaluate_script, {
page: pageId,
@@ -331,7 +355,7 @@ describe('input tools', () => {
it('hover moves cursor over element', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: FORM_PAGE })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snap = await execute(take_snapshot, { page: pageId })
const btnId = findElementId(textOf(snap), 'Submit')

73
apps/server/tests/tools/navigation.test.ts
View File

@@ -23,13 +23,22 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
describe('navigation tools', () => {
it('list_pages returns at least one page', async () => {
await withBrowser(async ({ execute }) => {
const result = await execute(list_pages, {})
assert.ok(!result.isError, textOf(result))
const text = textOf(result)
assert.ok(text.length > 0, 'Expected non-empty page list')
const data = structuredOf<{
pages: Array<{ pageId: number }>
count: number
}>(result)
assert.ok(data.count > 0, 'Expected non-empty page list')
assert.ok(data.pages[0]?.pageId !== undefined, 'Expected page IDs')
})
}, 60_000)
@@ -37,26 +46,25 @@ describe('navigation tools', () => {
await withBrowser(async ({ execute }) => {
const result = await execute(get_active_page, {})
assert.ok(!result.isError, textOf(result))
assert.ok(textOf(result).includes('Active page:'))
const data = structuredOf<{ page: { pageId: number; tabId: number } }>(
result,
)
assert.ok(data.page.pageId > 0)
assert.ok(data.page.tabId > 0)
})
}, 60_000)
it('new_page opens a tab and close_page removes it', async () => {
await withBrowser(async ({ execute }) => {
const beforeResult = await execute(list_pages, {})
const beforeCount = textOf(beforeResult).split(/\n\n/).length
const beforeCount = structuredOf<{ count: number }>(beforeResult).count
const newResult = await execute(new_page, { url: 'about:blank' })
assert.ok(!newResult.isError, textOf(newResult))
const newText = textOf(newResult)
assert.ok(newText.includes('Page ID:'), 'Expected page ID in response')
const pageIdMatch = newText.match(/Page ID:\s*(\d+)/)
assert.ok(pageIdMatch, 'Could not extract page ID')
const pageId = Number(pageIdMatch?.[1])
const pageId = structuredOf<{ pageId: number }>(newResult).pageId
const afterResult = await execute(list_pages, {})
const afterCount = textOf(afterResult).split(/\n\n/).length
const afterCount = structuredOf<{ count: number }>(afterResult).count
assert.ok(afterCount > beforeCount, 'Expected more pages after new_page')
const closeResult = await execute(close_page, { page: pageId })
@@ -68,7 +76,7 @@ describe('navigation tools', () => {
it('navigate_page navigates to a URL', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = structuredOf<{ pageId: number }>(newResult).pageId
const navResult = await execute(navigate_page, {
page: pageId,
@@ -77,6 +85,9 @@ describe('navigation tools', () => {
})
assert.ok(!navResult.isError, textOf(navResult))
assert.ok(textOf(navResult).includes('Navigated to'))
const data = structuredOf<{ action: string; page: number }>(navResult)
assert.strictEqual(data.action, 'url')
assert.strictEqual(data.page, pageId)
await execute(close_page, { page: pageId })
})
@@ -85,7 +96,7 @@ describe('navigation tools', () => {
it('wait_for finds text on page', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'https://example.com' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = structuredOf<{ pageId: number }>(newResult).pageId
const waitResult = await execute(wait_for, {
page: pageId,
@@ -94,6 +105,9 @@ describe('navigation tools', () => {
})
assert.ok(!waitResult.isError, textOf(waitResult))
assert.ok(textOf(waitResult).includes('Found'))
const data = structuredOf<{ found: boolean; page: number }>(waitResult)
assert.strictEqual(data.found, true)
assert.strictEqual(data.page, pageId)
await execute(close_page, { page: pageId })
})
@@ -102,7 +116,7 @@ describe('navigation tools', () => {
it('wait_for times out for missing text', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = structuredOf<{ pageId: number }>(newResult).pageId
const waitResult = await execute(wait_for, {
page: pageId,
@@ -122,11 +136,9 @@ describe('navigation tools', () => {
url: 'about:blank',
})
assert.ok(!result.isError, textOf(result))
const text = textOf(result)
assert.ok(text.includes('Opened hidden page'), 'Expected hidden page')
assert.ok(text.includes('Page ID:'), 'Expected page ID')
const pageId = Number(text.match(/Page ID:\s*(\d+)/)?.[1])
const data = structuredOf<{ pageId: number; hidden: boolean }>(result)
assert.strictEqual(data.hidden, true)
const pageId = data.pageId
await execute(close_page, { page: pageId })
})
}, 60_000)
@@ -136,11 +148,16 @@ describe('navigation tools', () => {
const hiddenResult = await execute(new_hidden_page, {
url: 'about:blank',
})
const pageId = Number(textOf(hiddenResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = structuredOf<{ pageId: number }>(hiddenResult).pageId
const showResult = await execute(show_page, { page: pageId })
assert.ok(!showResult.isError, textOf(showResult))
assert.ok(textOf(showResult).includes('now visible'))
const data = structuredOf<{
page: { pageId: number; isHidden: boolean }
}>(showResult)
assert.strictEqual(data.page.pageId, pageId)
assert.strictEqual(data.page.isHidden, false)
await execute(close_page, { page: pageId })
})
@@ -149,7 +166,7 @@ describe('navigation tools', () => {
it('show_page errors on an already-visible page', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = structuredOf<{ pageId: number }>(newResult).pageId
const showResult = await execute(show_page, { page: pageId })
assert.ok(showResult.isError, 'Expected error for visible page')
@@ -161,20 +178,22 @@ describe('navigation tools', () => {
it('move_page moves a tab to a different window', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = structuredOf<{ pageId: number }>(newResult).pageId
const winResult = await execute(create_window, {})
const windowId = Number(
textOf(winResult).match(/Created window\s+(\d+)/)?.[1],
)
const windowId = structuredOf<{ window: { windowId: number } }>(winResult)
.window.windowId
const moveResult = await execute(move_page, {
page: pageId,
windowId,
})
assert.ok(!moveResult.isError, textOf(moveResult))
assert.ok(textOf(moveResult).includes('Moved page'))
assert.ok(textOf(moveResult).includes(`window ${windowId}`))
const moveData = structuredOf<{
page: { pageId: number; windowId?: number }
}>(moveResult)
assert.strictEqual(moveData.page.pageId, pageId)
assert.strictEqual(moveData.page.windowId, windowId)
await execute(close_page, { page: pageId })
await execute(close_window, { windowId })

45
apps/server/tests/tools/observation.test.ts
View File

@@ -20,16 +20,32 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
function pageIdOf(result: {
content: { type: string; text?: string }[]
structuredContent?: unknown
}): number {
const data = result.structuredContent as { pageId?: number } | undefined
if (typeof data?.pageId === 'number') return data.pageId
return Number(textOf(result).match(/Page ID:\s*(\d+)/)?.[1])
}
describe('observation tools', () => {
it('take_snapshot returns element IDs', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'https://example.com' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snapResult = await execute(take_snapshot, { page: pageId })
assert.ok(!snapResult.isError, textOf(snapResult))
const text = textOf(snapResult)
assert.ok(text.length > 0, 'Snapshot should not be empty')
const data = structuredOf<{ snapshot: string }>(snapResult)
assert.ok(data.snapshot.length > 0, 'Expected structured snapshot')
await execute(close_page, { page: pageId })
})
@@ -38,7 +54,7 @@ describe('observation tools', () => {
it('take_enhanced_snapshot returns structural context', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'https://example.com' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const snapResult = await execute(take_enhanced_snapshot, { page: pageId })
assert.ok(!snapResult.isError, textOf(snapResult))
@@ -52,7 +68,7 @@ describe('observation tools', () => {
it('take_screenshot returns an image', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'https://example.com' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const result = await execute(take_screenshot, { page: pageId })
assert.ok(!result.isError)
@@ -63,6 +79,8 @@ describe('observation tools', () => {
)
assert.ok(imageItem, 'Expected an image content item')
assert.ok(imageItem.data.length > 0, 'Image data should not be empty')
const data = structuredOf<{ mimeType: string }>(result)
assert.ok(data.mimeType.startsWith('image/'))
await execute(close_page, { page: pageId })
})
@@ -71,7 +89,7 @@ describe('observation tools', () => {
it('evaluate_script returns values', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const evalResult = await execute(evaluate_script, {
page: pageId,
@@ -87,7 +105,7 @@ describe('observation tools', () => {
it('evaluate_script returns strings', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const evalResult = await execute(evaluate_script, {
page: pageId,
@@ -103,7 +121,7 @@ describe('observation tools', () => {
it('evaluate_script reports errors', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const evalResult = await execute(evaluate_script, {
page: pageId,
@@ -119,7 +137,7 @@ describe('observation tools', () => {
it('get_page_content returns markdown text', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'https://example.com' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const contentResult = await execute(get_page_content, { page: pageId })
assert.ok(!contentResult.isError, textOf(contentResult))
@@ -133,7 +151,7 @@ describe('observation tools', () => {
it('get_page_links extracts links from constructed HTML', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const html = `
<a href="https://example.com/one">First Link</a>
@@ -163,6 +181,10 @@ describe('observation tools', () => {
const linksResult = await execute(get_page_links, { page: pageId })
assert.ok(!linksResult.isError, textOf(linksResult))
const text = textOf(linksResult)
const linksData = structuredOf<{
links: Array<{ href: string }>
count: number
}>(linksResult)
assert.ok(text.includes('First Link'), 'Expected first link text')
assert.ok(text.includes('Second Link'), 'Expected second link text')
@@ -173,6 +195,7 @@ describe('observation tools', () => {
// should deduplicate by URL
const oneCount = (text.match(/example\.com\/one/g) || []).length
assert.strictEqual(oneCount, 1, 'Expected deduplication of same URL')
assert.ok(linksData.count >= 3, 'Expected structured links count')
// should skip javascript: links
assert.ok(
@@ -190,11 +213,15 @@ describe('observation tools', () => {
it('get_page_links returns empty message for pages with no links', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'about:blank' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = pageIdOf(newResult)
const linksResult = await execute(get_page_links, { page: pageId })
assert.ok(!linksResult.isError, textOf(linksResult))
assert.ok(textOf(linksResult).includes('No links found'))
const linksData = structuredOf<{ links: unknown[]; count: number }>(
linksResult,
)
assert.strictEqual(linksData.count, 0)
await execute(close_page, { page: pageId })
})

10
apps/server/tests/tools/page-actions.test.ts
View File

@@ -16,11 +16,16 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
describe('page action tools', () => {
it('save_pdf writes a PDF file to disk', async () => {
await withBrowser(async ({ execute }) => {
const newResult = await execute(new_page, { url: 'https://example.com' })
const pageId = Number(textOf(newResult).match(/Page ID:\s*(\d+)/)?.[1])
const pageId = structuredOf<{ pageId: number }>(newResult).pageId
const pdfPath = join(tmpdir(), `browseros-test-${Date.now()}.pdf`)
@@ -31,6 +36,9 @@ describe('page action tools', () => {
})
assert.ok(!pdfResult.isError, textOf(pdfResult))
assert.ok(textOf(pdfResult).includes('Saved PDF'))
const data = structuredOf<{ action: string; path: string }>(pdfResult)
assert.strictEqual(data.action, 'save_pdf')
assert.strictEqual(data.path, pdfPath)
assert.ok(existsSync(pdfPath), 'PDF file should exist on disk')
const stat = Bun.file(pdfPath)

23
apps/server/tests/tools/response.test.ts
View File

@@ -13,6 +13,29 @@ function textOf(result: {
}
describe('ToolResponse', () => {
it('accumulates structured content from data()', () => {
const response = new ToolResponse()
response.data('action', 'click')
response.data({ page: 1, element: 42 })
const result = response.toResult()
assert.deepStrictEqual(result.structuredContent, {
action: 'click',
page: 1,
element: 42,
})
})
it('overwrites keys on repeated data() writes', () => {
const response = new ToolResponse()
response.data('count', 1)
response.data({ count: 2 })
response.data('count', 3)
const result = response.toResult()
assert.deepStrictEqual(result.structuredContent, { count: 3 })
})
it('times out slow post-actions without failing tool output', async () => {
const response = new ToolResponse({ postActionTimeoutMs: 25 })
response.text('ok')

61
apps/server/tests/tools/tab-groups.test.ts
View File

@@ -19,11 +19,19 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
describe('tab group tools', () => {
it('list_tab_groups returns without error', async () => {
await withBrowser(async ({ execute }) => {
const result = await execute(list_tab_groups, {})
assert.ok(!result.isError, textOf(result))
const data = structuredOf<{ groups: unknown[]; count: number }>(result)
assert.ok(Array.isArray(data.groups))
assert.ok(data.count >= 0)
})
}, 60_000)
@@ -31,14 +39,10 @@ describe('tab group tools', () => {
await withBrowser(async ({ execute }) => {
// Create two tabs to group
const tab1Result = await execute(new_page, { url: 'about:blank' })
const tab1PageId = Number(
textOf(tab1Result).match(/Page ID:\s*(\d+)/)?.[1],
)
const tab1PageId = structuredOf<{ pageId: number }>(tab1Result).pageId
const tab2Result = await execute(new_page, { url: 'about:blank' })
const tab2PageId = Number(
textOf(tab2Result).match(/Page ID:\s*(\d+)/)?.[1],
)
const tab2PageId = structuredOf<{ pageId: number }>(tab2Result).pageId
const pageIds = [tab1PageId, tab2PageId]
@@ -48,11 +52,12 @@ describe('tab group tools', () => {
title: 'Test Group',
})
assert.ok(!groupResult.isError, textOf(groupResult))
const groupText = textOf(groupResult)
assert.ok(groupText.includes('Test Group'))
const groupIdMatch = groupText.match(/Group ID:\s*(\S+)/)
assert.ok(groupIdMatch, 'Could not extract group ID')
const groupId = groupIdMatch?.[1]
const groupData = structuredOf<{
group: { groupId: string; title: string; pageIds: number[] }
}>(groupResult)
assert.strictEqual(groupData.group.title, 'Test Group')
assert.deepStrictEqual(groupData.group.pageIds.sort(), pageIds.sort())
const groupId = groupData.group.groupId
// Update group
const updateResult = await execute(update_tab_group, {
@@ -61,17 +66,34 @@ describe('tab group tools', () => {
color: 'blue',
})
assert.ok(!updateResult.isError, textOf(updateResult))
assert.ok(textOf(updateResult).includes('Renamed Group'))
const updateData = structuredOf<{
group: { groupId: string; title: string; color: string }
}>(updateResult)
assert.strictEqual(updateData.group.groupId, groupId)
assert.strictEqual(updateData.group.title, 'Renamed Group')
assert.strictEqual(updateData.group.color, 'blue')
// Verify in list
const listResult = await execute(list_tab_groups, {})
assert.ok(!listResult.isError, textOf(listResult))
assert.ok(textOf(listResult).includes('Renamed Group'))
const listData = structuredOf<{
groups: Array<{ groupId: string; title: string }>
}>(listResult)
assert.ok(
listData.groups.some(
(group) =>
group.groupId === groupId && group.title === 'Renamed Group',
),
)
// Ungroup
const ungroupResult = await execute(ungroup_tabs, { pageIds })
assert.ok(!ungroupResult.isError, textOf(ungroupResult))
assert.ok(textOf(ungroupResult).includes('Ungrouped'))
const ungroupData = structuredOf<{ action: string; count: number }>(
ungroupResult,
)
assert.strictEqual(ungroupData.action, 'ungroup_tabs')
assert.strictEqual(ungroupData.count, 2)
// Cleanup
await execute(close_page, { page: tab1PageId })
@@ -82,7 +104,7 @@ describe('tab group tools', () => {
it('close_tab_group closes group and tabs', async () => {
await withBrowser(async ({ execute }) => {
const tabResult = await execute(new_page, { url: 'about:blank' })
const tabPageId = Number(textOf(tabResult).match(/Page ID:\s*(\d+)/)?.[1])
const tabPageId = structuredOf<{ pageId: number }>(tabResult).pageId
// Group
const groupResult = await execute(group_tabs, {
@@ -90,12 +112,17 @@ describe('tab group tools', () => {
title: 'Disposable',
})
assert.ok(!groupResult.isError, textOf(groupResult))
const groupId = textOf(groupResult).match(/Group ID:\s*(\S+)/)?.[1]
const groupId = structuredOf<{ group: { groupId: string } }>(groupResult)
.group.groupId
// Close group (also closes the tab)
const closeResult = await execute(close_tab_group, { groupId })
assert.ok(!closeResult.isError, textOf(closeResult))
assert.ok(textOf(closeResult).includes('Closed tab group'))
const closeData = structuredOf<{ action: string; groupId: string }>(
closeResult,
)
assert.strictEqual(closeData.action, 'close_tab_group')
assert.strictEqual(closeData.groupId, groupId)
})
}, 60_000)
})

57
apps/server/tests/tools/windows.test.ts
View File

@@ -18,12 +18,22 @@ function textOf(result: {
.join('\n')
}
function structuredOf<T>(result: { structuredContent?: unknown }): T {
assert.ok(result.structuredContent, 'Expected structuredContent')
return result.structuredContent as T
}
describe('window tools', () => {
it('list_windows returns at least one window', async () => {
await withBrowser(async ({ execute }) => {
const result = await execute(list_windows, {})
assert.ok(!result.isError, textOf(result))
assert.ok(textOf(result).includes('Window'))
const data = structuredOf<{
windows: Array<{ windowId: number }>
count: number
}>(result)
assert.ok(data.count > 0, 'Expected at least one window')
assert.ok(data.windows[0]?.windowId !== undefined, 'Expected window IDs')
})
}, 60_000)
@@ -31,29 +41,36 @@ describe('window tools', () => {
await withBrowser(async ({ execute }) => {
const createResult = await execute(create_window, {})
assert.ok(!createResult.isError, textOf(createResult))
const text = textOf(createResult)
assert.ok(text.includes('Created window'))
const windowIdMatch = text.match(/Created window\s+(\d+)/)
assert.ok(windowIdMatch, 'Could not extract window ID')
const windowId = Number(windowIdMatch?.[1])
const windowId = structuredOf<{ window: { windowId: number } }>(
createResult,
).window.windowId
const closeResult = await execute(close_window, { windowId })
assert.ok(!closeResult.isError, textOf(closeResult))
assert.ok(textOf(closeResult).includes('Closed window'))
const closeData = structuredOf<{ action: string; windowId: number }>(
closeResult,
)
assert.strictEqual(closeData.action, 'close_window')
assert.strictEqual(closeData.windowId, windowId)
})
}, 60_000)
it('activate_window focuses a window', async () => {
await withBrowser(async ({ execute }) => {
const listResult = await execute(list_windows, {})
const windowIdMatch = textOf(listResult).match(/Window\s+(\d+)/)
assert.ok(windowIdMatch, 'No window found')
const windowId = Number(windowIdMatch?.[1])
const listData = structuredOf<{ windows: Array<{ windowId: number }> }>(
listResult,
)
const windowId = listData.windows[0]?.windowId
assert.ok(windowId !== undefined, 'No window found')
const activateResult = await execute(activate_window, { windowId })
assert.ok(!activateResult.isError, textOf(activateResult))
assert.ok(textOf(activateResult).includes('Activated'))
const activateData = structuredOf<{ action: string; windowId: number }>(
activateResult,
)
assert.strictEqual(activateData.action, 'activate_window')
assert.strictEqual(activateData.windowId, windowId)
})
}, 60_000)
@@ -61,16 +78,18 @@ describe('window tools', () => {
await withBrowser(async ({ execute }) => {
const createResult = await execute(create_hidden_window, {})
assert.ok(!createResult.isError, textOf(createResult))
const text = textOf(createResult)
assert.ok(text.includes('Created hidden window'))
const windowIdMatch = text.match(/Created hidden window\s+(\d+)/)
assert.ok(windowIdMatch, 'Could not extract window ID')
const windowId = Number(windowIdMatch?.[1])
const windowData = structuredOf<{
window: { windowId: number; isVisible: boolean }
}>(createResult)
assert.strictEqual(windowData.window.isVisible, false)
const windowId = windowData.window.windowId
const listResult = await execute(list_windows, {})
const listData = structuredOf<{ windows: Array<{ windowId: number }> }>(
listResult,
)
assert.ok(
textOf(listResult).includes(`Window ${windowId}`),
listData.windows.some((w) => w.windowId === windowId),
'Hidden window should appear in list',
)

8
bun.lock
View File

@@ -162,7 +162,7 @@
"@hono/mcp": "^0.2.3",
"@hono/node-server": "^1.19.6",
"@hono/zod-validator": "^0.4.3",
"@modelcontextprotocol/sdk": "^1.25.2",
"@modelcontextprotocol/sdk": "^1.27.1",
"@openrouter/ai-sdk-provider": "^2.2.3",
"@sentry/bun": "^10.31.0",
"ai": "^6.0.94",
@@ -1770,7 +1770,7 @@
"chrome-devtools-frontend": ["chrome-devtools-frontend@1.0.1577886", "", {}, "sha512-B9hY3o/0RuVCDWNYh9YnkEbRrPUMCY+NaOgBxvZRzGvqbGSMNckkVSdO67SwWR8bm4fo/qplXbUj0cSr229V6w=="],
"chrome-devtools-mcp": ["chrome-devtools-mcp@0.18.1", "", { "bin": { "chrome-devtools-mcp": "build/src/index.js" } }, "sha512-foYSH4oXtMCteFaKOggpcwYcU1R3RYvU1Bq8qNIOWjpP4GyroNn8mLKSYo+u1kcuujmoJ80sgBBIibRo68w3aQ=="],
"chrome-devtools-mcp": ["chrome-devtools-mcp@0.19.0", "", { "bin": { "chrome-devtools-mcp": "build/src/index.js" } }, "sha512-LfqjOxdUjWvCQrfeI5V3ZBJCUIDKGNmexSbSAgsrjVggN4X1OSObLxleSlX2zwcXRZYxqy209cww0MXcXuN1zw=="],
"chrome-launcher": ["chrome-launcher@1.2.0", "", { "dependencies": { "@types/node": "*", "escape-string-regexp": "^4.0.0", "is-wsl": "^2.2.0", "lighthouse-logger": "^2.0.1" }, "bin": { "print-chrome-path": "bin/print-chrome-path.cjs" } }, "sha512-JbuGuBNss258bvGil7FT4HKdC3SC2K7UAEUqiPy3ACS3Yxo3hAW6bvFpCu2HsIJLgTqxgEX6BkujvzZfLpUD0Q=="],
@@ -3894,6 +3894,8 @@
"@browseros/agent/zod": ["zod@4.3.5", "", {}, "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g=="],
"@browseros/server/@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.27.1", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA=="],
"@browseros/server/@types/bun": ["@types/bun@1.3.5", "", { "dependencies": { "bun-types": "1.3.5" } }, "sha512-RnygCqNrd3srIPEWBd5LFeUYG7plCoH2Yw9WaZGyNmdTEei+gWaHqydbaIRkIkcbXwhBT94q78QljxN0Sk838w=="],
"@devicefarmer/adbkit/commander": ["commander@9.5.0", "", {}, "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ=="],
@@ -4588,6 +4590,8 @@
"@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="],
"@browseros/server/@modelcontextprotocol/sdk/zod": ["zod@4.3.5", "", {}, "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g=="],
"@browseros/server/@types/bun/bun-types": ["bun-types@1.3.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-inmAYe2PFLs0SUbFOWSVD24sg1jFlMPxOjOSSCYqUgn4Hsc3rDc7dFvfVYjFPNHtov6kgUeulV4SxbuIV/stPw=="],
"@google/gemini-cli-core/@opentelemetry/exporter-logs-otlp-http/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.203.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-9B9RU0H7Ya1Dx/Rkyc4stuBZSGVQF27WigitInx2QQoj6KUpEFYPKoWjdFTunJYxmXmh17HeBvbMa1EhGyPmqQ=="],