mirror of
https://github.com/browseros-ai/BrowserOS.git
synced 2026-05-14 16:14:28 +00:00
Compare commits
4 Commits
fix/iframe
...
klavis-cli
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d9cc97016 | ||
|
|
f78068bb9d | ||
|
|
6b18ebb1d8 | ||
|
|
1f2e783ab9 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -29,3 +29,4 @@ packages/browseros/build/tools/
|
||||
|
||||
# AI SDK DevTools traces
|
||||
.devtools/
|
||||
.omc/
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
# BrowserOS Agent Extension
|
||||
|
||||
## v0.0.99 (2026-04-08)
|
||||
|
||||
## What's Changed
|
||||
|
||||
- chore: bump server and extension version (#659)
|
||||
- chore(agent): remove workflows feature (#656)
|
||||
- feat: replace model picker with shadcn Combobox + fuse.js fuzzy search (#617)
|
||||
- feat: clean-up - remove obsolete controller extension (#610)
|
||||
- docs: update agent extension changelog for v0.0.98 (#609)
|
||||
|
||||
|
||||
## v0.0.98 (2026-03-27)
|
||||
|
||||
## What's Changed
|
||||
|
||||
@@ -56,6 +56,7 @@ var groupOrder = []string{
|
||||
"Observe:",
|
||||
"Input:",
|
||||
"Resources:",
|
||||
"Integrations:",
|
||||
"Setup:",
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ func TestCommandName(t *testing.T) {
|
||||
{"known command", []string{"health"}, "browseros-cli health"},
|
||||
{"unknown command", []string{"nonexistent"}, "unknown"},
|
||||
{"subcommand", []string{"bookmark", "search"}, "browseros-cli bookmark search"},
|
||||
{"strata subcommand", []string{"strata", "check"}, "browseros-cli strata check"},
|
||||
{"known with extra args", []string{"snap", "--enhanced"}, "browseros-cli snap"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
|
||||
235
packages/browseros-agent/apps/cli/cmd/strata.go
Normal file
235
packages/browseros-agent/apps/cli/cmd/strata.go
Normal file
@@ -0,0 +1,235 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"browseros-cli/output"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
strataCmd := &cobra.Command{
|
||||
Use: "strata",
|
||||
Annotations: map[string]string{"group": "Integrations:"},
|
||||
Short: "Manage Strata MCP integrations (Gmail, Slack, GitHub, etc.)",
|
||||
Long: `Interact with 40+ external services via Strata MCP integrations.
|
||||
|
||||
Supported services:
|
||||
gmail, google calendar, google docs, google drive, google sheets, slack,
|
||||
linkedin, notion, airtable, confluence, github, gitlab, linear, jira,
|
||||
figma, salesforce, hubspot, stripe, discord, asana, clickup, zendesk,
|
||||
monday, shopify, dropbox, onedrive, box, youtube, whatsapp, resend,
|
||||
posthog, mixpanel, vercel, supabase, cloudflare, wordpress, postman,
|
||||
intercom, cal.com, brave search, microsoft teams, outlook mail,
|
||||
outlook calendar, google forms, mem0
|
||||
|
||||
Discovery flow — do not guess action names:
|
||||
1. check → verify the service is connected (get auth URL if not)
|
||||
2. discover → find categories or actions for a service
|
||||
3. actions → expand categories into specific actions
|
||||
4. details → get the parameter schema before executing
|
||||
5. exec → execute the action with parameters
|
||||
6. search → fallback keyword search if discover doesn't find it
|
||||
|
||||
Authentication:
|
||||
If a service is not connected, "check" returns an authUrl.
|
||||
Open that URL in a browser to authenticate, then retry.
|
||||
If "exec" fails with an auth error, use "auth" to get a fresh authUrl.
|
||||
|
||||
Example — search Gmail:
|
||||
browseros-cli strata check gmail
|
||||
browseros-cli strata discover "search emails" gmail
|
||||
browseros-cli strata actions GMAIL_EMAIL
|
||||
browseros-cli strata details GMAIL_EMAIL gmail_search_emails
|
||||
browseros-cli strata exec gmail GMAIL_EMAIL gmail_search_emails \
|
||||
--body '{"query":"from:user@example.com","maxResults":5}'`,
|
||||
}
|
||||
|
||||
checkCmd := &cobra.Command{
|
||||
Use: "check <server-name>",
|
||||
Short: "Check if a service is connected and ready",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
c := newClient()
|
||||
result, err := c.CallTool("connector_mcp_servers", map[string]any{
|
||||
"server_name": args[0],
|
||||
})
|
||||
if err != nil {
|
||||
output.Error(err.Error(), 1)
|
||||
}
|
||||
if jsonOut {
|
||||
output.JSON(result)
|
||||
} else {
|
||||
output.Text(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
discoverCmd := &cobra.Command{
|
||||
Use: "discover <query> <server> [servers...]",
|
||||
Short: "Discover available categories or actions for servers",
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
c := newClient()
|
||||
result, err := c.CallTool("discover_server_categories_or_actions", map[string]any{
|
||||
"user_query": args[0],
|
||||
"server_names": args[1:],
|
||||
})
|
||||
if err != nil {
|
||||
output.Error(err.Error(), 1)
|
||||
}
|
||||
if jsonOut {
|
||||
output.JSON(result)
|
||||
} else {
|
||||
output.Text(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
actionsCmd := &cobra.Command{
|
||||
Use: "actions <category> [categories...]",
|
||||
Short: "Get actions within categories",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
c := newClient()
|
||||
result, err := c.CallTool("get_category_actions", map[string]any{
|
||||
"category_names": args,
|
||||
})
|
||||
if err != nil {
|
||||
output.Error(err.Error(), 1)
|
||||
}
|
||||
if jsonOut {
|
||||
output.JSON(result)
|
||||
} else {
|
||||
output.Text(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
detailsCmd := &cobra.Command{
|
||||
Use: "details <category> <action>",
|
||||
Short: "Get parameter schema for an action",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
c := newClient()
|
||||
result, err := c.CallTool("get_action_details", map[string]any{
|
||||
"category_name": args[0],
|
||||
"action_name": args[1],
|
||||
})
|
||||
if err != nil {
|
||||
output.Error(err.Error(), 1)
|
||||
}
|
||||
if jsonOut {
|
||||
output.JSON(result)
|
||||
} else {
|
||||
output.Text(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
execCmd := &cobra.Command{
|
||||
Use: "exec <server> <category> <action>",
|
||||
Short: "Execute an action on a connected service",
|
||||
Long: `Execute an action on a connected service.
|
||||
|
||||
Pass request body as a JSON string with --body.
|
||||
Use --query and --path for query/path parameters.
|
||||
Use --output-field to limit response fields.
|
||||
|
||||
Example:
|
||||
browseros-cli strata exec gmail GMAIL_EMAIL gmail_search_emails \
|
||||
--body '{"query":"from:user@example.com","maxResults":5}'`,
|
||||
Args: cobra.ExactArgs(3),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
bodySchema, _ := cmd.Flags().GetString("body")
|
||||
queryParams, _ := cmd.Flags().GetString("query")
|
||||
pathParams, _ := cmd.Flags().GetString("path")
|
||||
outputFields, _ := cmd.Flags().GetStringArray("output-field")
|
||||
maxChars, _ := cmd.Flags().GetInt("max-chars")
|
||||
|
||||
toolArgs := map[string]any{
|
||||
"server_name": args[0],
|
||||
"category_name": args[1],
|
||||
"action_name": args[2],
|
||||
}
|
||||
|
||||
if bodySchema != "" {
|
||||
toolArgs["body_schema"] = bodySchema
|
||||
}
|
||||
if queryParams != "" {
|
||||
toolArgs["query_params"] = queryParams
|
||||
}
|
||||
if pathParams != "" {
|
||||
toolArgs["path_params"] = pathParams
|
||||
}
|
||||
if len(outputFields) > 0 {
|
||||
toolArgs["include_output_fields"] = outputFields
|
||||
}
|
||||
if cmd.Flags().Changed("max-chars") {
|
||||
toolArgs["maximum_output_characters"] = maxChars
|
||||
}
|
||||
|
||||
c := newClient()
|
||||
result, err := c.CallTool("execute_action", toolArgs)
|
||||
if err != nil {
|
||||
output.Error(err.Error(), 1)
|
||||
}
|
||||
if jsonOut {
|
||||
output.JSON(result)
|
||||
} else {
|
||||
output.Text(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
execCmd.Flags().String("body", "", "Request body as JSON string")
|
||||
execCmd.Flags().String("query", "", "Query parameters as JSON string")
|
||||
execCmd.Flags().String("path", "", "Path parameters as JSON string")
|
||||
execCmd.Flags().StringArray("output-field", nil, "Limit response to these fields (repeatable)")
|
||||
execCmd.Flags().Int("max-chars", 0, "Maximum output characters")
|
||||
|
||||
searchCmd := &cobra.Command{
|
||||
Use: "search <query> <server>",
|
||||
Short: "Search documentation for a service",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
c := newClient()
|
||||
result, err := c.CallTool("search_documentation", map[string]any{
|
||||
"query": args[0],
|
||||
"server_name": args[1],
|
||||
})
|
||||
if err != nil {
|
||||
output.Error(err.Error(), 1)
|
||||
}
|
||||
if jsonOut {
|
||||
output.JSON(result)
|
||||
} else {
|
||||
output.Text(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
authCmd := &cobra.Command{
|
||||
Use: "auth <server-name>",
|
||||
Short: "Handle authentication failure for a service",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
intention, _ := cmd.Flags().GetString("intention")
|
||||
c := newClient()
|
||||
result, err := c.CallTool("handle_auth_failure", map[string]any{
|
||||
"server_name": args[0],
|
||||
"intention": intention,
|
||||
})
|
||||
if err != nil {
|
||||
output.Error(err.Error(), 1)
|
||||
}
|
||||
if jsonOut {
|
||||
output.JSON(result)
|
||||
} else {
|
||||
output.Text(result)
|
||||
}
|
||||
},
|
||||
}
|
||||
authCmd.Flags().String("intention", "get_auth_url", "Auth intention")
|
||||
|
||||
strataCmd.AddCommand(checkCmd, discoverCmd, actionsCmd, detailsCmd, execCmd, searchCmd, authCmd)
|
||||
rootCmd.AddCommand(strataCmd)
|
||||
}
|
||||
@@ -270,3 +270,84 @@ func TestInvalidPage(t *testing.T) {
|
||||
t.Errorf("expected snap with invalid page ID to exit non-zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataCheck(t *testing.T) {
|
||||
r := run(t, "--json", "strata", "check", "Gmail")
|
||||
// Klavis may not be configured — accept success or structured error
|
||||
out := strings.TrimSpace(r.Stdout + r.Stderr)
|
||||
if out == "" {
|
||||
t.Fatal("strata check produced no output")
|
||||
}
|
||||
if r.ExitCode == 0 {
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(r.Stdout)), &data); err != nil {
|
||||
t.Fatalf("strata check returned non-JSON: %s", r.Stdout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataDiscover(t *testing.T) {
|
||||
r := run(t, "--json", "strata", "discover", "send email", "Gmail")
|
||||
out := strings.TrimSpace(r.Stdout + r.Stderr)
|
||||
if out == "" {
|
||||
t.Fatal("strata discover produced no output")
|
||||
}
|
||||
if r.ExitCode == 0 {
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(r.Stdout)), &data); err != nil {
|
||||
t.Fatalf("strata discover returned non-JSON: %s", r.Stdout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataSearch(t *testing.T) {
|
||||
r := run(t, "--json", "strata", "search", "send email", "Gmail")
|
||||
out := strings.TrimSpace(r.Stdout + r.Stderr)
|
||||
if out == "" {
|
||||
t.Fatal("strata search produced no output")
|
||||
}
|
||||
if r.ExitCode == 0 {
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(r.Stdout)), &data); err != nil {
|
||||
t.Fatalf("strata search returned non-JSON: %s", r.Stdout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataActions(t *testing.T) {
|
||||
r := run(t, "--json", "strata", "actions", "Gmail")
|
||||
out := strings.TrimSpace(r.Stdout + r.Stderr)
|
||||
if out == "" {
|
||||
t.Fatal("strata actions produced no output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataDetails(t *testing.T) {
|
||||
r := run(t, "--json", "strata", "details", "Gmail", "send_email")
|
||||
out := strings.TrimSpace(r.Stdout + r.Stderr)
|
||||
if out == "" {
|
||||
t.Fatal("strata details produced no output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataAuth(t *testing.T) {
|
||||
r := run(t, "--json", "strata", "auth", "Gmail")
|
||||
out := strings.TrimSpace(r.Stdout + r.Stderr)
|
||||
if out == "" {
|
||||
t.Fatal("strata auth produced no output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataExecMissingArgs(t *testing.T) {
|
||||
r := run(t, "strata", "exec")
|
||||
if r.ExitCode == 0 {
|
||||
t.Error("expected strata exec without args to exit non-zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrataCheckMissingArgs(t *testing.T) {
|
||||
r := run(t, "strata", "check")
|
||||
if r.ExitCode == 0 {
|
||||
t.Error("expected strata check without args to exit non-zero")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -392,9 +392,48 @@ export class Browser {
|
||||
|
||||
// --- Observation ---
|
||||
|
||||
private async getFrameIds(session: ProtocolApi): Promise<string[]> {
|
||||
try {
|
||||
const result = await session.Page.getFrameTree()
|
||||
const ids: string[] = []
|
||||
type Tree = { frame: { id: string }; childFrames?: Tree[] }
|
||||
function collect(tree: Tree) {
|
||||
ids.push(tree.frame.id)
|
||||
if (tree.childFrames)
|
||||
for (const child of tree.childFrames) collect(child)
|
||||
}
|
||||
collect(result.frameTree as Tree)
|
||||
return ids
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchAXTree(session: ProtocolApi): Promise<AXNode[]> {
|
||||
const result = await session.Accessibility.getFullAXTree()
|
||||
return (result.nodes as AXNode[]) ?? []
|
||||
const frameIds = await this.getFrameIds(session)
|
||||
|
||||
if (frameIds.length <= 1) {
|
||||
const result = await session.Accessibility.getFullAXTree()
|
||||
return (result.nodes as AXNode[]) ?? []
|
||||
}
|
||||
|
||||
const allNodes: AXNode[] = []
|
||||
for (const frameId of frameIds) {
|
||||
try {
|
||||
const result = await session.Accessibility.getFullAXTree({ frameId })
|
||||
const nodes = (result.nodes as AXNode[]) ?? []
|
||||
for (const node of nodes) {
|
||||
allNodes.push({
|
||||
...node,
|
||||
nodeId: `${frameId}:${node.nodeId}`,
|
||||
childIds: node.childIds?.map((id) => `${frameId}:${id}`),
|
||||
})
|
||||
}
|
||||
} catch {
|
||||
// Cross-origin or detached frames may fail — skip
|
||||
}
|
||||
}
|
||||
return allNodes
|
||||
}
|
||||
|
||||
async snapshot(page: number): Promise<string> {
|
||||
|
||||
@@ -20,7 +20,7 @@ export function buildContentMarkdownExpression(
|
||||
// Uses var + ES5 style for consistency with other injected scripts.
|
||||
// Context object: { pre: bool, ld: listDepth, lt: listType, td: tableDepth }
|
||||
const DOM_WALKER_SCRIPT = `(function(o) {
|
||||
var SKIP = {SCRIPT:1,STYLE:1,NOSCRIPT:1,SVG:1,TEMPLATE:1,IFRAME:1,CANVAS:1,VIDEO:1,AUDIO:1,OBJECT:1,EMBED:1};
|
||||
var SKIP = {SCRIPT:1,STYLE:1,NOSCRIPT:1,SVG:1,TEMPLATE:1,CANVAS:1,VIDEO:1,AUDIO:1,OBJECT:1,EMBED:1};
|
||||
var FORM = {INPUT:1,SELECT:1,TEXTAREA:1,BUTTON:1};
|
||||
var vh = window.innerHeight, vw = window.innerWidth;
|
||||
var root = o.selector ? document.querySelector(o.selector) : document.body;
|
||||
@@ -219,6 +219,15 @@ function walk(node, ctx) {
|
||||
t = kids(el, ctx).trim();
|
||||
return t ? '\\n*' + t + '*\\n' : '';
|
||||
|
||||
case 'IFRAME':
|
||||
try {
|
||||
var idoc = el.contentDocument;
|
||||
if (idoc && idoc.body) return walk(idoc.body, ctx);
|
||||
} catch(e) {}
|
||||
var isrc = el.src || el.getAttribute('src');
|
||||
if (isrc) return '\\n\\n[iframe: ' + isrc + ']\\n\\n';
|
||||
return '';
|
||||
|
||||
default:
|
||||
return kids(el, ctx);
|
||||
}
|
||||
|
||||
@@ -100,11 +100,16 @@ export function buildInteractiveTree(nodes: AXNode[]): string[] {
|
||||
if (node.childIds) for (const childId of node.childIds) walk(childId)
|
||||
}
|
||||
|
||||
const root =
|
||||
nodes.find(
|
||||
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
|
||||
) ?? nodes[0]
|
||||
if (root?.childIds) for (const childId of root.childIds) walk(childId)
|
||||
const roots = nodes.filter(
|
||||
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
|
||||
)
|
||||
if (roots.length === 0 && nodes[0]?.childIds) {
|
||||
for (const childId of nodes[0].childIds) walk(childId)
|
||||
} else {
|
||||
for (const root of roots) {
|
||||
if (root.childIds) for (const childId of root.childIds) walk(childId)
|
||||
}
|
||||
}
|
||||
|
||||
return lines
|
||||
}
|
||||
@@ -160,11 +165,16 @@ export function buildEnhancedTree(nodes: AXNode[]): string[] {
|
||||
for (const childId of node.childIds) walk(childId, depth + 1)
|
||||
}
|
||||
|
||||
const root =
|
||||
nodes.find(
|
||||
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
|
||||
) ?? nodes[0]
|
||||
if (root?.childIds) for (const childId of root.childIds) walk(childId, 0)
|
||||
const roots = nodes.filter(
|
||||
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
|
||||
)
|
||||
if (roots.length === 0 && nodes[0]?.childIds) {
|
||||
for (const childId of nodes[0].childIds) walk(childId, 0)
|
||||
} else {
|
||||
for (const root of roots) {
|
||||
if (root.childIds) for (const childId of root.childIds) walk(childId, 0)
|
||||
}
|
||||
}
|
||||
|
||||
return lines
|
||||
}
|
||||
@@ -292,11 +302,16 @@ export function extractLinkNodes(nodes: AXNode[]): LinkNode[] {
|
||||
if (node.childIds) for (const childId of node.childIds) walk(childId)
|
||||
}
|
||||
|
||||
const root =
|
||||
nodes.find(
|
||||
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
|
||||
) ?? nodes[0]
|
||||
if (root?.childIds) for (const childId of root.childIds) walk(childId)
|
||||
const roots = nodes.filter(
|
||||
(n) => n.role?.value === 'RootWebArea' || n.role?.value === 'WebArea',
|
||||
)
|
||||
if (roots.length === 0 && nodes[0]?.childIds) {
|
||||
for (const childId of nodes[0].childIds) walk(childId)
|
||||
} else {
|
||||
for (const root of roots) {
|
||||
if (root.childIds) for (const childId of root.childIds) walk(childId)
|
||||
}
|
||||
}
|
||||
|
||||
return links
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user