From 38e2bd7e503468b25fa496e9a620a2df1da04a7a Mon Sep 17 00:00:00 2001 From: Felarof Date: Wed, 21 Jan 2026 13:04:22 -0800 Subject: [PATCH] feat: created new Chat Mode pill and exclude tools when in chatMode (#263) * feat: agent mode on or off * fix: cleaner whitelist for chat mode * fix: cleaner whitelist for chat mode * feat: agent mode with tooltip * feat: agent mode chat mode final UI * feat: previous conversation history * fix: re-enable the DELETE endpoint * fix: make bun run start:server show lgos * fix: minor text change * fix: keep 16k context window size * fix: use message ref to get access to full restored messages (when create prev conversation history) * fix: don't run watchdog in dev-mode * Revert "fix: re-enable the DELETE endpoint" This reverts commit 9cbbbab6768c7c412c8f65bd88643e2856fa5169. --------- Co-authored-by: Nikhil Sonti --- .../sidepanel/index/ChatModeToggle.tsx | 73 +++++++++++-------- .../sidepanel/index/useChatSession.ts | 13 +++- .../formatConversationHistory.ts | 32 ++++++++ apps/server/src/agent/gemini-agent.ts | 57 ++++++++++++++- apps/server/src/agent/types.ts | 2 + apps/server/src/api/services/chat-service.ts | 2 + apps/server/src/api/types.ts | 2 + package.json | 2 +- 8 files changed, 149 insertions(+), 34 deletions(-) create mode 100644 apps/agent/lib/conversations/formatConversationHistory.ts diff --git a/apps/agent/entrypoints/sidepanel/index/ChatModeToggle.tsx b/apps/agent/entrypoints/sidepanel/index/ChatModeToggle.tsx index d16f3416..ed4f2e6e 100644 --- a/apps/agent/entrypoints/sidepanel/index/ChatModeToggle.tsx +++ b/apps/agent/entrypoints/sidepanel/index/ChatModeToggle.tsx @@ -1,5 +1,11 @@ -import { MessageSquare, Zap } from 'lucide-react' +import { MessageSquare, MousePointer2 } from 'lucide-react' import type { FC } from 'react' +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@/components/ui/tooltip' import { cn } from '@/lib/utils' import type { ChatMode } from './chatTypes' @@ -12,36 +18,41 @@ export const ChatModeToggle: FC = ({ mode, onModeChange, }) => { + const isAgentMode = mode === 'agent' + return ( -
- - -
+ + + + + + + {isAgentMode + ? 'AI can browse, click, and navigate' + : 'AI can only read, cannot click or navigate'} + + + ) } diff --git a/apps/agent/entrypoints/sidepanel/index/useChatSession.ts b/apps/agent/entrypoints/sidepanel/index/useChatSession.ts index cbaf32dd..06954728 100644 --- a/apps/agent/entrypoints/sidepanel/index/useChatSession.ts +++ b/apps/agent/entrypoints/sidepanel/index/useChatSession.ts @@ -18,6 +18,7 @@ import { conversationStorage, useConversations, } from '@/lib/conversations/conversationStorage' +import { formatConversationHistory } from '@/lib/conversations/formatConversationHistory' import { useLlmProviders } from '@/lib/llm-providers/useLlmProviders' import { track } from '@/lib/metrics/track' import { searchActionsStorage } from '@/lib/search-actions/searchActionsStorage' @@ -90,7 +91,7 @@ export const useChatSession = () => { type: p.type, })) - const [mode, setMode] = useState('chat') + const [mode, setMode] = useState('agent') const [textToAction, setTextToAction] = useState>( new Map(), ) @@ -139,6 +140,7 @@ export const useChatSession = () => { const modeRef = useRef(mode) const textToActionRef = useRef>(textToAction) const workingDirRef = useRef(undefined) + const messagesRef = useRef([]) useDeepCompareEffect(() => { modeRef.current = mode @@ -231,6 +233,13 @@ export const useChatSession = () => { }[] } + // Format previous messages from ref (messagesRef doesn't include current message yet) + const previousMessages = messagesRef.current + const previousConversation = + previousMessages.length > 0 + ? formatConversationHistory(previousMessages) + : undefined + return { api: `${agentUrlRef.current}/chat`, body: { @@ -256,6 +265,7 @@ export const useChatSession = () => { userSystemPrompt: personalizationRef.current, userWorkingDir: workingDirRef.current, supportsImages: provider?.supportsImages, + previousConversation, }, } }, @@ -292,6 +302,7 @@ export const useChatSession = () => { // biome-ignore lint/correctness/useExhaustiveDependencies: only need to run when messages change useEffect(() => { + messagesRef.current = messages if (messages.length > 0) { saveConversation(conversationIdRef.current, messages) } diff --git a/apps/agent/lib/conversations/formatConversationHistory.ts b/apps/agent/lib/conversations/formatConversationHistory.ts new file mode 100644 index 00000000..c721b26a --- /dev/null +++ b/apps/agent/lib/conversations/formatConversationHistory.ts @@ -0,0 +1,32 @@ +import type { UIMessage } from 'ai' + +const MAX_MESSAGES = 10 +const MAX_MESSAGE_CHARS = 65536 // 16K context window size + +export function formatConversationHistory(messages: UIMessage[]): string { + if (messages.length === 0) return '' + + const recentMessages = messages.slice(-MAX_MESSAGES) + + const formatted = recentMessages + .map((msg) => { + const role = msg.role === 'user' ? 'user' : 'assistant' + const textContent = msg.parts + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join('\n') + + if (!textContent.trim()) return null + + const truncatedContent = + textContent.length > MAX_MESSAGE_CHARS + ? `${textContent.slice(0, MAX_MESSAGE_CHARS)}... [truncated]` + : textContent + + return `<${role}>${truncatedContent}` + }) + .filter(Boolean) + .join('\n\n') + + return formatted +} diff --git a/apps/server/src/agent/gemini-agent.ts b/apps/server/src/agent/gemini-agent.ts index e95be9bf..d1d4c455 100644 --- a/apps/server/src/agent/gemini-agent.ts +++ b/apps/server/src/agent/gemini-agent.ts @@ -19,6 +19,8 @@ import type { Content, Part } from '@google/genai' import type { BrowserContext } from '../api/types' import { logger } from '../lib/logger' import { Sentry } from '../lib/sentry' +import { allCdpTools } from '../tools/cdp-based/registry' +import { allControllerTools } from '../tools/controller-based/registry' import { AgentExecutionError } from './errors' import { buildSystemPrompt } from './prompt' import { VercelAIContentGenerator } from './provider-adapter/index' @@ -26,6 +28,17 @@ import type { HonoSSEStream } from './provider-adapter/types' import { UIMessageStreamWriter } from './provider-adapter/ui-message-stream' import type { ResolvedAgentConfig } from './types' +const CHAT_MODE_ALLOWED_TOOLS = new Set([ + 'browser_get_active_tab', + 'browser_list_tabs', + 'browser_get_page_content', + 'browser_scroll_down', + 'browser_scroll_up', + 'browser_get_screenshot', + 'browser_get_interactive_elements', + 'browser_execute_javascript', +]) + export interface ToolExecutionResult { parts: Part[] isError: boolean @@ -102,6 +115,7 @@ export class GeminiAgent { // Build excluded tools list - always exclude save_memory and google_web_search // Conditionally exclude screenshot tools if model doesn't support images // Exclude window management tools unless in eval mode + // In chat mode, only allow read-only tools for page content extraction const excludedTools = ['save_memory', 'google_web_search'] if (config.supportsImages === false) { excludedTools.push( @@ -114,6 +128,21 @@ export class GeminiAgent { excludedTools.push('browser_create_window', 'browser_close_window') } + // Chat mode: restrict to read-only tools only (no browser automation) + if (config.chatMode === true) { + const allToolNames = [ + ...allControllerTools.map((t) => t.name), + ...allCdpTools.map((t) => t.name), + ] + const chatModeExcludedTools = allToolNames.filter( + (name) => !CHAT_MODE_ALLOWED_TOOLS.has(name), + ) + excludedTools.push(...chatModeExcludedTools) + logger.info('Chat mode enabled, restricting to read-only tools', { + allowedTools: Array.from(CHAT_MODE_ALLOWED_TOOLS), + }) + } + const geminiConfig = new GeminiConfig({ sessionId: config.conversationId, targetDir: config.sessionExecutionDir, @@ -371,12 +400,38 @@ export class GeminiAgent { honoStream: HonoSSEStream, signal?: AbortSignal, browserContext?: BrowserContext, + previousConversation?: string, ): Promise { const abortSignal = signal || new AbortController().signal const promptId = `${this.conversationId}-${Date.now()}` const contextPrefix = this.formatBrowserContext(browserContext) - let currentParts: Part[] = [{ text: contextPrefix + message }] + + // User query + const userQuery = ` +${message} +` + + // Inject previous conversation if resuming (no server-side history) + let fullMessage = userQuery + const hasHistory = this.client.getHistory().length > 0 + if (previousConversation && !hasHistory) { + fullMessage = ` +The user is resuming a previous conversation. Here is the conversation history for context: + +${previousConversation} + + +Continue the conversation based on the above context. Here is the user's new message: + +${userQuery}` + logger.info('Injecting previous conversation for resume', { + conversationId: this.conversationId, + historyLength: previousConversation.length, + }) + } + + let currentParts: Part[] = [{ text: contextPrefix + fullMessage }] let turnCount = 0 const uiStream = honoStream diff --git a/apps/server/src/agent/types.ts b/apps/server/src/agent/types.ts index 72ec7cc9..3e80c4c7 100644 --- a/apps/server/src/agent/types.ts +++ b/apps/server/src/agent/types.ts @@ -37,4 +37,6 @@ export interface ResolvedAgentConfig { supportsImages?: boolean /** Eval mode - enables window management tools. Defaults to false. */ evalMode?: boolean + /** Chat mode - restricts to read-only tools (no browser automation). Defaults to false. */ + chatMode?: boolean } diff --git a/apps/server/src/api/services/chat-service.ts b/apps/server/src/api/services/chat-service.ts index 9ae71c62..8c2e9e0c 100644 --- a/apps/server/src/api/services/chat-service.ts +++ b/apps/server/src/api/services/chat-service.ts @@ -92,6 +92,7 @@ export class ChatService { userSystemPrompt: request.userSystemPrompt, sessionExecutionDir, supportsImages: request.supportsImages, + chatMode: request.mode === 'chat', } const agent = await sessionManager.getOrCreate(agentConfig, mcpServers) @@ -100,6 +101,7 @@ export class ChatService { rawStream, abortSignal, request.browserContext, + request.previousConversation, ) } diff --git a/apps/server/src/api/types.ts b/apps/server/src/api/types.ts index 0d2f6b52..ab70ac9b 100644 --- a/apps/server/src/api/types.ts +++ b/apps/server/src/api/types.ts @@ -41,6 +41,8 @@ export const ChatRequestSchema = VercelAIConfigSchema.extend({ isScheduledTask: z.boolean().optional().default(false), userWorkingDir: z.string().min(1).optional(), supportsImages: z.boolean().optional().default(true), + mode: z.enum(['chat', 'agent']).optional().default('agent'), + previousConversation: z.string().optional(), }) export type ChatRequest = z.infer diff --git a/package.json b/package.json index 22a95a02..0d645663 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "scripts": { "postinstall": "bun run build:agent-sdk", "start:dev": "bun ./scripts/build/start-all.ts", - "start:server": "bun run --filter @browseros/server start", + "start:server": "bun run --filter @browseros/server --elide-lines=0 start", "start:agent": "bun ./scripts/build/controller-ext.ts && bun run --filter @browseros/agent dev", "build:server": "FORCE_COLOR=1 bun run --filter @browseros/server --elide-lines=0 build", "build:agent": "bun run --filter @browseros/agent build",