Compare commits

...

4 Commits

Author SHA1 Message Date
shivammittal274
9d9cc97016 feat(cli): add strata commands for Klavis MCP integrations
Expose the 7 Klavis Strata MCP tools as CLI subcommands under
`browseros-cli strata`, so CLI users (claude-code, gemini-cli) can
discover and execute actions on 40+ external services.

Commands: check, discover, actions, details, exec, search, auth.
Includes discovery flow guidance in help text, integration tests,
and an "Integrations:" group in the root help output.
2026-04-14 15:29:26 +05:30
Felarof
f78068bb9d chore: add .omc/ to gitignore (#682)
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-11 20:53:24 -07:00
github-actions[bot]
6b18ebb1d8 docs: update agent extension changelog for v0.0.99 (#660)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-04-10 09:53:44 -07:00
shivammittal274
1f2e783ab9 fix: enable agent interaction with elements inside iframes (#667)
* fix: enable agent interaction with elements inside iframes

Fetch accessibility trees from all frames via Page.getFrameTree() +
per-frame Accessibility.getFullAXTree(frameId), so iframe elements
appear in snapshots with valid backendNodeIds. Pages without iframes
take the original single-call path with zero overhead.

Update snapshot tree builders to walk multiple RootWebArea roots from
merged multi-frame trees. Extract same-origin iframe content in the
markdown walker; show [iframe: url] placeholder for cross-origin.

* fix: namespace AX nodeIds by frameId to prevent cross-frame collisions

CDP AXNodeId values are frame-scoped — each frame's accessibility tree
starts its own counter from 1. Prefix nodeId and childIds with frameId
before merging so the nodeMap in snapshot builders never overwrites
nodes from a different frame.
2026-04-09 23:14:53 +05:30
9 changed files with 411 additions and 18 deletions

1
.gitignore vendored
View File

@@ -29,3 +29,4 @@ packages/browseros/build/tools/
# AI SDK DevTools traces
.devtools/
.omc/

View File

@@ -1,5 +1,16 @@
# BrowserOS Agent Extension
## v0.0.99 (2026-04-08)
## What's Changed
- chore: bump server and extension version (#659)
- chore(agent): remove workflows feature (#656)
- feat: replace model picker with shadcn Combobox + fuse.js fuzzy search (#617)
- feat: clean-up - remove obsolete controller extension (#610)
- docs: update agent extension changelog for v0.0.98 (#609)
## v0.0.98 (2026-03-27)
## What's Changed

View File

@@ -56,6 +56,7 @@ var groupOrder = []string{
"Observe:",
"Input:",
"Resources:",
"Integrations:",
"Setup:",
}

View File

@@ -33,6 +33,7 @@ func TestCommandName(t *testing.T) {
{"known command", []string{"health"}, "browseros-cli health"},
{"unknown command", []string{"nonexistent"}, "unknown"},
{"subcommand", []string{"bookmark", "search"}, "browseros-cli bookmark search"},
{"strata subcommand", []string{"strata", "check"}, "browseros-cli strata check"},
{"known with extra args", []string{"snap", "--enhanced"}, "browseros-cli snap"},
}
for _, tt := range tests {

View File

@@ -0,0 +1,235 @@
package cmd
import (
"browseros-cli/output"
"github.com/spf13/cobra"
)
func init() {
strataCmd := &cobra.Command{
Use: "strata",
Annotations: map[string]string{"group": "Integrations:"},
Short: "Manage Strata MCP integrations (Gmail, Slack, GitHub, etc.)",
Long: `Interact with 40+ external services via Strata MCP integrations.
Supported services:
gmail, google calendar, google docs, google drive, google sheets, slack,
linkedin, notion, airtable, confluence, github, gitlab, linear, jira,
figma, salesforce, hubspot, stripe, discord, asana, clickup, zendesk,
monday, shopify, dropbox, onedrive, box, youtube, whatsapp, resend,
posthog, mixpanel, vercel, supabase, cloudflare, wordpress, postman,
intercom, cal.com, brave search, microsoft teams, outlook mail,
outlook calendar, google forms, mem0
Discovery flow — do not guess action names:
1. check → verify the service is connected (get auth URL if not)
2. discover → find categories or actions for a service
3. actions → expand categories into specific actions
4. details → get the parameter schema before executing
5. exec → execute the action with parameters
6. search → fallback keyword search if discover doesn't find it
Authentication:
If a service is not connected, "check" returns an authUrl.
Open that URL in a browser to authenticate, then retry.
If "exec" fails with an auth error, use "auth" to get a fresh authUrl.
Example — search Gmail:
browseros-cli strata check gmail
browseros-cli strata discover "search emails" gmail
browseros-cli strata actions GMAIL_EMAIL
browseros-cli strata details GMAIL_EMAIL gmail_search_emails
browseros-cli strata exec gmail GMAIL_EMAIL gmail_search_emails \
--body '{"query":"from:user@example.com","maxResults":5}'`,
}
checkCmd := &cobra.Command{
Use: "check <server-name>",
Short: "Check if a service is connected and ready",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
c := newClient()
result, err := c.CallTool("connector_mcp_servers", map[string]any{
"server_name": args[0],
})
if err != nil {
output.Error(err.Error(), 1)
}
if jsonOut {
output.JSON(result)
} else {
output.Text(result)
}
},
}
discoverCmd := &cobra.Command{
Use: "discover <query> <server> [servers...]",
Short: "Discover available categories or actions for servers",
Args: cobra.MinimumNArgs(2),
Run: func(cmd *cobra.Command, args []string) {
c := newClient()
result, err := c.CallTool("discover_server_categories_or_actions", map[string]any{
"user_query": args[0],
"server_names": args[1:],
})
if err != nil {
output.Error(err.Error(), 1)
}
if jsonOut {
output.JSON(result)
} else {
output.Text(result)
}
},
}
actionsCmd := &cobra.Command{
Use: "actions <category> [categories...]",
Short: "Get actions within categories",
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
c := newClient()
result, err := c.CallTool("get_category_actions", map[string]any{
"category_names": args,
})
if err != nil {
output.Error(err.Error(), 1)
}
if jsonOut {
output.JSON(result)
} else {
output.Text(result)
}
},
}
detailsCmd := &cobra.Command{
Use: "details <category> <action>",
Short: "Get parameter schema for an action",
Args: cobra.ExactArgs(2),
Run: func(cmd *cobra.Command, args []string) {
c := newClient()
result, err := c.CallTool("get_action_details", map[string]any{
"category_name": args[0],
"action_name": args[1],
})
if err != nil {
output.Error(err.Error(), 1)
}
if jsonOut {
output.JSON(result)
} else {
output.Text(result)
}
},
}
execCmd := &cobra.Command{
Use: "exec <server> <category> <action>",
Short: "Execute an action on a connected service",
Long: `Execute an action on a connected service.
Pass request body as a JSON string with --body.
Use --query and --path for query/path parameters.
Use --output-field to limit response fields.
Example:
browseros-cli strata exec gmail GMAIL_EMAIL gmail_search_emails \
--body '{"query":"from:user@example.com","maxResults":5}'`,
Args: cobra.ExactArgs(3),
Run: func(cmd *cobra.Command, args []string) {
bodySchema, _ := cmd.Flags().GetString("body")
queryParams, _ := cmd.Flags().GetString("query")
pathParams, _ := cmd.Flags().GetString("path")
outputFields, _ := cmd.Flags().GetStringArray("output-field")
maxChars, _ := cmd.Flags().GetInt("max-chars")
toolArgs := map[string]any{
"server_name": args[0],
"category_name": args[1],
"action_name": args[2],
}
if bodySchema != "" {
toolArgs["body_schema"] = bodySchema
}
if queryParams != "" {
toolArgs["query_params"] = queryParams
}
if pathParams != "" {
toolArgs["path_params"] = pathParams
}
if len(outputFields) > 0 {
toolArgs["include_output_fields"] = outputFields
}
if cmd.Flags().Changed("max-chars") {
toolArgs["maximum_output_characters"] = maxChars
}
c := newClient()
result, err := c.CallTool("execute_action", toolArgs)
if err != nil {
output.Error(err.Error(), 1)
}
if jsonOut {
output.JSON(result)
} else {
output.Text(result)
}
},
}
execCmd.Flags().String("body", "", "Request body as JSON string")
execCmd.Flags().String("query", "", "Query parameters as JSON string")
execCmd.Flags().String("path", "", "Path parameters as JSON string")
execCmd.Flags().StringArray("output-field", nil, "Limit response to these fields (repeatable)")
execCmd.Flags().Int("max-chars", 0, "Maximum output characters")
searchCmd := &cobra.Command{
Use: "search <query> <server>",
Short: "Search documentation for a service",
Args: cobra.ExactArgs(2),
Run: func(cmd *cobra.Command, args []string) {
c := newClient()
result, err := c.CallTool("search_documentation", map[string]any{
"query": args[0],
"server_name": args[1],
})
if err != nil {
output.Error(err.Error(), 1)
}
if jsonOut {
output.JSON(result)
} else {
output.Text(result)
}
},
}
authCmd := &cobra.Command{
Use: "auth <server-name>",
Short: "Handle authentication failure for a service",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
intention, _ := cmd.Flags().GetString("intention")
c := newClient()
result, err := c.CallTool("handle_auth_failure", map[string]any{
"server_name": args[0],
"intention": intention,
})
if err != nil {
output.Error(err.Error(), 1)
}
if jsonOut {
output.JSON(result)
} else {
output.Text(result)
}
},
}
authCmd.Flags().String("intention", "get_auth_url", "Auth intention")
strataCmd.AddCommand(checkCmd, discoverCmd, actionsCmd, detailsCmd, execCmd, searchCmd, authCmd)
rootCmd.AddCommand(strataCmd)
}

View File

@@ -270,3 +270,84 @@ func TestInvalidPage(t *testing.T) {
t.Errorf("expected snap with invalid page ID to exit non-zero")
}
}
func TestStrataCheck(t *testing.T) {
r := run(t, "--json", "strata", "check", "Gmail")
// Klavis may not be configured — accept success or structured error
out := strings.TrimSpace(r.Stdout + r.Stderr)
if out == "" {
t.Fatal("strata check produced no output")
}
if r.ExitCode == 0 {
var data map[string]any
if err := json.Unmarshal([]byte(strings.TrimSpace(r.Stdout)), &data); err != nil {
t.Fatalf("strata check returned non-JSON: %s", r.Stdout)
}
}
}
func TestStrataDiscover(t *testing.T) {
r := run(t, "--json", "strata", "discover", "send email", "Gmail")
out := strings.TrimSpace(r.Stdout + r.Stderr)
if out == "" {
t.Fatal("strata discover produced no output")
}
if r.ExitCode == 0 {
var data map[string]any
if err := json.Unmarshal([]byte(strings.TrimSpace(r.Stdout)), &data); err != nil {
t.Fatalf("strata discover returned non-JSON: %s", r.Stdout)
}
}
}
func TestStrataSearch(t *testing.T) {
r := run(t, "--json", "strata", "search", "send email", "Gmail")
out := strings.TrimSpace(r.Stdout + r.Stderr)
if out == "" {
t.Fatal("strata search produced no output")
}
if r.ExitCode == 0 {
var data map[string]any
if err := json.Unmarshal([]byte(strings.TrimSpace(r.Stdout)), &data); err != nil {
t.Fatalf("strata search returned non-JSON: %s", r.Stdout)
}
}
}
func TestStrataActions(t *testing.T) {
r := run(t, "--json", "strata", "actions", "Gmail")
out := strings.TrimSpace(r.Stdout + r.Stderr)
if out == "" {
t.Fatal("strata actions produced no output")
}
}
func TestStrataDetails(t *testing.T) {
r := run(t, "--json", "strata", "details", "Gmail", "send_email")
out := strings.TrimSpace(r.Stdout + r.Stderr)
if out == "" {
t.Fatal("strata details produced no output")
}
}
func TestStrataAuth(t *testing.T) {
r := run(t, "--json", "strata", "auth", "Gmail")
out := strings.TrimSpace(r.Stdout + r.Stderr)
if out == "" {
t.Fatal("strata auth produced no output")
}
}
func TestStrataExecMissingArgs(t *testing.T) {
r := run(t, "strata", "exec")
if r.ExitCode == 0 {
t.Error("expected strata exec without args to exit non-zero")
}
}
func TestStrataCheckMissingArgs(t *testing.T) {
r := run(t, "strata", "check")
if r.ExitCode == 0 {
t.Error("expected strata check without args to exit non-zero")
}
}

View File

@@ -392,9 +392,48 @@ export class Browser {
// --- Observation ---
private async getFrameIds(session: ProtocolApi): Promise<string[]> {
try {
const result = await session.Page.getFrameTree()
const ids: string[] = []
type Tree = { frame: { id: string }; childFrames?: Tree[] }
function collect(tree: Tree) {
ids.push(tree.frame.id)
if (tree.childFrames)
for (const child of tree.childFrames) collect(child)
}
collect(result.frameTree as Tree)
return ids
} catch {
return []
}
}
private async fetchAXTree(session: ProtocolApi): Promise<AXNode[]> {
const result = await session.Accessibility.getFullAXTree()
return (result.nodes as AXNode[]) ?? []
const frameIds = await this.getFrameIds(session)
if (frameIds.length <= 1) {
const result = await session.Accessibility.getFullAXTree()
return (result.nodes as AXNode[]) ?? []
}
const allNodes: AXNode[] = []
for (const frameId of frameIds) {
try {
const result = await session.Accessibility.getFullAXTree({ frameId })
const nodes = (result.nodes as AXNode[]) ?? []
for (const node of nodes) {
allNodes.push({
...node,
nodeId: `${frameId}:${node.nodeId}`,
childIds: node.childIds?.map((id) => `${frameId}:${id}`),
})
}
} catch {
// Cross-origin or detached frames may fail — skip
}
}
return allNodes
}
async snapshot(page: number): Promise<string> {

View File

@@ -20,7 +20,7 @@ export function buildContentMarkdownExpression(
// Uses var + ES5 style for consistency with other injected scripts.
// Context object: { pre: bool, ld: listDepth, lt: listType, td: tableDepth }
const DOM_WALKER_SCRIPT = `(function(o) {
var SKIP = {SCRIPT:1,STYLE:1,NOSCRIPT:1,SVG:1,TEMPLATE:1,IFRAME:1,CANVAS:1,VIDEO:1,AUDIO:1,OBJECT:1,EMBED:1};
var SKIP = {SCRIPT:1,STYLE:1,NOSCRIPT:1,SVG:1,TEMPLATE:1,CANVAS:1,VIDEO:1,AUDIO:1,OBJECT:1,EMBED:1};
var FORM = {INPUT:1,SELECT:1,TEXTAREA:1,BUTTON:1};
var vh = window.innerHeight, vw = window.innerWidth;
var root = o.selector ? document.querySelector(o.selector) : document.body;
@@ -219,6 +219,15 @@ function walk(node, ctx) {
t = kids(el, ctx).trim();
return t ? '\\n*' + t + '*\\n' : '';
case 'IFRAME':
try {
var idoc = el.contentDocument;
if (idoc && idoc.body) return walk(idoc.body, ctx);
} catch(e) {}
var isrc = el.src || el.getAttribute('src');
if (isrc) return '\\n\\n[iframe: ' + isrc + ']\\n\\n';
return '';
default:
return kids(el, ctx);
}

View File

@@ -100,11 +100,16 @@ export function buildInteractiveTree(nodes: AXNode[]): string[] {
if (node.childIds) for (const childId of node.childIds) walk(childId)
}
const root =
nodes.find(
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
) ?? nodes[0]
if (root?.childIds) for (const childId of root.childIds) walk(childId)
const roots = nodes.filter(
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
)
if (roots.length === 0 && nodes[0]?.childIds) {
for (const childId of nodes[0].childIds) walk(childId)
} else {
for (const root of roots) {
if (root.childIds) for (const childId of root.childIds) walk(childId)
}
}
return lines
}
@@ -160,11 +165,16 @@ export function buildEnhancedTree(nodes: AXNode[]): string[] {
for (const childId of node.childIds) walk(childId, depth + 1)
}
const root =
nodes.find(
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
) ?? nodes[0]
if (root?.childIds) for (const childId of root.childIds) walk(childId, 0)
const roots = nodes.filter(
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
)
if (roots.length === 0 && nodes[0]?.childIds) {
for (const childId of nodes[0].childIds) walk(childId, 0)
} else {
for (const root of roots) {
if (root.childIds) for (const childId of root.childIds) walk(childId, 0)
}
}
return lines
}
@@ -292,11 +302,16 @@ export function extractLinkNodes(nodes: AXNode[]): LinkNode[] {
if (node.childIds) for (const childId of node.childIds) walk(childId)
}
const root =
nodes.find(
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
) ?? nodes[0]
if (root?.childIds) for (const childId of root.childIds) walk(childId)
const roots = nodes.filter(
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
)
if (roots.length === 0 && nodes[0]?.childIds) {
for (const childId of nodes[0].childIds) walk(childId)
} else {
for (const root of roots) {
if (root.childIds) for (const childId of root.childIds) walk(childId)
}
}
return links
}