import { afterAll, describe, it } from 'bun:test' import assert from 'node:assert' import type { Browser } from '../../src/browser/browser' import { disposeSemanticPipeline } from '../../src/tools/acl/acl-embeddings' import { executeTool, type ToolContext } from '../../src/tools/framework' import { check, click, click_at, fill, hover, press_key, scroll, select_option, type_at, uncheck, } from '../../src/tools/input' import { close_page, navigate_page, new_page } from '../../src/tools/navigation' import { evaluate_script, take_snapshot } from '../../src/tools/snapshot' import { cleanupWithBrowser, withBrowser } from '../__helpers__/with-browser' process.env.ACL_EMBEDDING_DISABLE = 'true' function textOf(result: { content: { type: string; text?: string }[] }): string { return result.content .filter((c) => c.type === 'text') .map((c) => c.text) .join('\n') } function structuredOf(result: { structuredContent?: unknown }): T { assert.ok(result.structuredContent, 'Expected structuredContent') return result.structuredContent as T } function pageIdOf(result: { content: { type: string; text?: string }[] structuredContent?: unknown }): number { const data = result.structuredContent as { pageId?: number } | undefined if (typeof data?.pageId === 'number') return data.pageId return Number(textOf(result).match(/Page ID:\s*(\d+)/)?.[1]) } function escapeRegex(s: string): string { return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') } function findElementId(snapshotText: string, label: string): number { const regex = new RegExp(`\\[(\\d+)\\].*?${escapeRegex(label)}`) const match = snapshotText.match(regex) if (!match) throw new Error(`Element "${label}" not found in snapshot`) return Number.parseInt(match[1], 10) } async function pointInsideElement( ctx: ToolContext, pageId: number, elementDomId: string, ): Promise<{ x: number; y: number }> { const pointResult = await executeTool( evaluate_script, { page: pageId, expression: `(() => { const el = document.getElementById(${JSON.stringify(elementDomId)}); if (!el) return null; const rect = el.getBoundingClientRect(); const insetX = Math.max(1, Math.min(10, Math.floor(rect.width / 4))); const insetY = Math.max(1, Math.min(10, Math.floor(rect.height / 4))); const candidates = [ { x: Math.round(rect.left + rect.width / 2), y: Math.round(rect.top + rect.height / 2), }, { x: Math.round(rect.left + insetX), y: Math.round(rect.top + insetY), }, { x: Math.round(rect.right - insetX), y: Math.round(rect.top + insetY), }, { x: Math.round(rect.left + insetX), y: Math.round(rect.bottom - insetY), }, { x: Math.round(rect.right - insetX), y: Math.round(rect.bottom - insetY), }, ]; for (const candidate of candidates) { const target = document.elementFromPoint(candidate.x, candidate.y); if (target && (target === el || el.contains(target))) { return { ...candidate, matched: true, hitId: target.id || null }; } } const fallback = candidates[0]; const fallbackTarget = document.elementFromPoint(fallback.x, fallback.y); return { ...fallback, matched: false, hitId: fallbackTarget instanceof Element ? fallbackTarget.id || null : null, }; })()`, }, ctx, AbortSignal.timeout(30_000), ) const point = structuredOf<{ value: { x: number; y: number; matched: boolean; hitId: string | null } } | null>(pointResult)?.value assert.ok(point, `Expected a point for #${elementDomId}`) assert.ok( point.matched, `Expected coordinates inside #${elementDomId}, got ${point.hitId ?? 'null'}`, ) return { x: point.x, y: point.y } } const FORM_PAGE = `data:text/html,${encodeURIComponent(`

Test Form

Bottom of page

`)}` afterAll(async () => { await disposeSemanticPipeline() await cleanupWithBrowser() }) describe('input tools', () => { it('fill types text into an input', async () => { await withBrowser(async ({ execute }) => { const newResult = await execute(new_page, { url: FORM_PAGE }) const pageId = pageIdOf(newResult) const snap = await execute(take_snapshot, { page: pageId }) const snapText = textOf(snap) const inputId = findElementId(snapText, 'Enter name') const fillResult = await execute(fill, { page: pageId, element: inputId, text: 'John Doe', }) assert.ok(!fillResult.isError, textOf(fillResult)) const fillData = structuredOf<{ action: string; textLength: number }>( fillResult, ) assert.strictEqual(fillData.action, 'fill') assert.strictEqual(fillData.textLength, 'John Doe'.length) const val = await execute(evaluate_script, { page: pageId, expression: 'document.getElementById("name").value', }) assert.strictEqual(textOf(val), 'John Doe') await execute(close_page, { page: pageId }) }) }, 60_000) it('click triggers a button', async () => { await withBrowser(async ({ execute }) => { const newResult = await execute(new_page, { url: FORM_PAGE }) const pageId = pageIdOf(newResult) // Fill the input first const snap = await execute(take_snapshot, { page: pageId }) const snapText = textOf(snap) const inputId = findElementId(snapText, 'Enter name') await execute(fill, { page: pageId, element: inputId, text: 'Alice' }) // Click submit const btnId = findElementId(snapText, 'Submit') const clickResult = await execute(click, { page: pageId, element: btnId, }) assert.ok(!clickResult.isError, textOf(clickResult)) const clickData = structuredOf<{ action: string; element: number }>( clickResult, ) assert.strictEqual(clickData.action, 'click') assert.strictEqual(clickData.element, btnId) const output = await execute(evaluate_script, { page: pageId, expression: 'document.getElementById("output").textContent', }) assert.strictEqual(textOf(output), 'clicked:Alice') await execute(close_page, { page: pageId }) }) }, 60_000) it('check and uncheck toggle a checkbox', async () => { await withBrowser(async ({ execute }) => { const newResult = await execute(new_page, { url: FORM_PAGE }) const pageId = pageIdOf(newResult) const snap = await execute(take_snapshot, { page: pageId }) const snapText = textOf(snap) const checkboxId = findElementId(snapText, 'I agree') const checkResult = await execute(check, { page: pageId, element: checkboxId, }) assert.ok(!checkResult.isError, textOf(checkResult)) const checked = await execute(evaluate_script, { page: pageId, expression: 'document.getElementById("agree").checked', }) assert.strictEqual(textOf(checked), 'true') const uncheckResult = await execute(uncheck, { page: pageId, element: checkboxId, }) assert.ok(!uncheckResult.isError, textOf(uncheckResult)) const unchecked = await execute(evaluate_script, { page: pageId, expression: 'document.getElementById("agree").checked', }) assert.strictEqual(textOf(unchecked), 'false') await execute(close_page, { page: pageId }) }) }, 60_000) it('select_option selects a dropdown value', async () => { await withBrowser(async ({ execute }) => { const newResult = await execute(new_page, { url: FORM_PAGE }) const pageId = pageIdOf(newResult) // Use evaluate_script to get the select element's backendNodeId directly const nodeId = await execute(evaluate_script, { page: pageId, expression: '(() => { const el = document.getElementById("color"); return el ? el.getAttribute("id") : null })()', }) assert.strictEqual(textOf(nodeId), 'color') // Get the select element ID from the snapshot const snap = await execute(take_snapshot, { page: pageId }) const snapText = textOf(snap) // Find the combobox/listbox element (the