fix: address review feedback and add upload_file tests

- Only fall through to file chooser fallback when error is specifically about the node not being a file input; re-throw other errors (e.g. invalid file paths) immediately - Reject with clear error when fileChooserOpened event lacks backendNodeId instead of falling back to the wrong node - Add tests for direct file input upload and button-triggered file chooser Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
fix: handle file upload for non-file-input elements via file chooser interception
2026-05-13 23:53:25 +00:00 · 2026-03-16 18:15:59 -07:00 · 2026-03-16 18:03:38 -07:00
3 changed files with 149 additions and 3 deletions
--- a/packages/browseros-agent/apps/server/src/browser/browser.ts
+++ b/packages/browseros-agent/apps/server/src/browser/browser.ts
@@ -1015,7 +1015,56 @@ export class Browser {
    files: string[],
  ): Promise<void> {
    const session = await this.resolveSession(page)
-    await session.DOM.setFileInputFiles({ files, backendNodeId: element })
+
+    // Fast path: element is a <input type="file">
+    try {
+      await session.DOM.setFileInputFiles({ files, backendNodeId: element })
+      return
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err)
+      if (!msg.includes('file input')) throw err
+    }
+
+    // Fallback: intercept the native file chooser dialog triggered by clicking the element
+    await session.Page.setInterceptFileChooserDialog({ enabled: true })
+    try {
+      const fileChooserPromise = new Promise<number>((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          unsubscribe()
+          reject(
+            new Error(
+              'File chooser did not open within 3s after clicking the element. Ensure the element triggers a file upload dialog.',
+            ),
+          )
+        }, 3000)
+
+        const unsubscribe = session.Page.on(
+          'fileChooserOpened',
+          (event) => {
+            clearTimeout(timeout)
+            unsubscribe()
+            if (event.backendNodeId == null) {
+              reject(
+                new Error(
+                  'fileChooserOpened event did not include a backendNodeId; cannot set files.',
+                ),
+              )
+              return
+            }
+            resolve(event.backendNodeId)
+          },
+        )
+      })
+
+      await this.click(page, element)
+      const fileInputNodeId = await fileChooserPromise
+      await session.DOM.setFileInputFiles({
+        files,
+        backendNodeId: fileInputNodeId,
+      })
+    } finally {
+      await session.Page.setInterceptFileChooserDialog({ enabled: false })
+    }
  }

  // --- File operations ---
--- a/packages/browseros-agent/apps/server/src/tools/input.ts
+++ b/packages/browseros-agent/apps/server/src/tools/input.ts
@@ -431,11 +431,11 @@ export const uncheck = defineTool({
 export const upload_file = defineTool({
  name: 'upload_file',
  description:
-    'Set file(s) on a file input element. Files must be absolute paths on disk.',
+    'Upload file(s) via a file input or any element that triggers a file chooser dialog (e.g. buttons, dropzones). Files must be absolute paths on disk.',
  input: z.object({
    page: pageParam,
    element: elementParam.describe(
-      'Element ID of the <input type="file"> element',
+      'Element ID of the file input, upload button, or dropzone that triggers a file chooser',
    ),
    files: z.array(z.string()).describe('Absolute file paths to upload'),
  }),
--- a/packages/browseros-agent/apps/server/tests/tools/input.test.ts
+++ b/packages/browseros-agent/apps/server/tests/tools/input.test.ts
@@ -9,6 +9,7 @@ import {
  scroll,
  select_option,
  uncheck,
+  upload_file,
 } from '../../src/tools/input'
 import { close_page, new_page } from '../../src/tools/navigation'
 import { evaluate_script, take_snapshot } from '../../src/tools/snapshot'
@@ -371,3 +372,99 @@ describe('input tools', () => {
    })
  }, 60_000)
 })
+
+const UPLOAD_PAGE = `data:text/html,${encodeURIComponent(`<!DOCTYPE html>
+<html><body>
+  <h1>Upload Test</h1>
+  <label for="file-input">Direct Upload</label>
+  <input id="file-input" type="file" aria-label="Direct Upload" />
+  <button id="upload-btn" type="button">Proxy Upload</button>
+  <input id="hidden-file" type="file" style="display:none" />
+  <div id="result"></div>
+  <script>
+    document.getElementById('file-input').addEventListener('change', function(e) {
+      document.getElementById('result').textContent = 'direct:' + e.target.files[0].name;
+    });
+    document.getElementById('upload-btn').addEventListener('click', function() {
+      document.getElementById('hidden-file').click();
+    });
+    document.getElementById('hidden-file').addEventListener('change', function(e) {
+      document.getElementById('result').textContent = 'button:' + e.target.files[0].name;
+    });
+  </script>
+</body></html>`)}`
+
+describe('upload_file tool', () => {
+  it('uploads a file via a direct <input type="file">', async () => {
+    await withBrowser(async ({ execute }) => {
+      const newResult = await execute(new_page, { url: UPLOAD_PAGE })
+      const pageId = pageIdOf(newResult)
+
+      // Get the file input's backendNodeId from the snapshot
+      const snap = await execute(take_snapshot, { page: pageId })
+      const snapText = textOf(snap)
+      const fileInputId = findElementId(snapText, 'Direct Upload')
+
+      const tmpPath = '/tmp/browseros-test-upload.txt'
+      await Bun.write(tmpPath, 'test file content')
+
+      const uploadResult = await execute(upload_file, {
+        page: pageId,
+        element: fileInputId,
+        files: [tmpPath],
+      })
+      assert.ok(!uploadResult.isError, textOf(uploadResult))
+      assert.ok(textOf(uploadResult).includes('file(s)'))
+
+      const data = structuredOf<{
+        action: string
+        fileCount: number
+      }>(uploadResult)
+      assert.strictEqual(data.action, 'upload_file')
+      assert.strictEqual(data.fileCount, 1)
+
+      const result = await execute(evaluate_script, {
+        page: pageId,
+        expression: 'document.getElementById("result").textContent',
+      })
+      assert.strictEqual(
+        textOf(result),
+        'direct:browseros-test-upload.txt',
+      )
+
+      await execute(close_page, { page: pageId })
+    })
+  }, 60_000)
+
+  it('uploads a file via a button that triggers a file chooser', async () => {
+    await withBrowser(async ({ execute }) => {
+      const newResult = await execute(new_page, { url: UPLOAD_PAGE })
+      const pageId = pageIdOf(newResult)
+
+      const snap = await execute(take_snapshot, { page: pageId })
+      const snapText = textOf(snap)
+      const btnId = findElementId(snapText, 'Proxy Upload')
+
+      const tmpPath = '/tmp/browseros-test-upload-btn.txt'
+      await Bun.write(tmpPath, 'button upload content')
+
+      const uploadResult = await execute(upload_file, {
+        page: pageId,
+        element: btnId,
+        files: [tmpPath],
+      })
+      assert.ok(!uploadResult.isError, textOf(uploadResult))
+
+      const result = await execute(evaluate_script, {
+        page: pageId,
+        expression: 'document.getElementById("result").textContent',
+      })
+      assert.strictEqual(
+        textOf(result),
+        'button:browseros-test-upload-btn.txt',
+      )
+
+      await execute(close_page, { page: pageId })
+    })
+  }, 60_000)
+})