feat: created new Chat Mode pill and exclude tools when in chatMode (#263)

* feat: agent mode on or off

* fix: cleaner whitelist for chat mode

* fix: cleaner whitelist for chat mode

* feat: agent mode with tooltip

* feat: agent mode chat mode final UI

* feat: previous conversation history

* fix: re-enable the DELETE endpoint

* fix: make bun run start:server show lgos

* fix: minor text change

* fix: keep 16k context window size

* fix: use message ref to get access to full restored messages (when create prev conversation history)

* fix: don't run watchdog in dev-mode

* Revert "fix: re-enable the DELETE endpoint"

This reverts commit 9cbbbab6768c7c412c8f65bd88643e2856fa5169.

---------

Co-authored-by: Nikhil Sonti <nikhilsv92@gmail.com>

apps/agent/entrypoints/sidepanel/index/ChatModeToggle.tsx

@@ -1,5 +1,11 @@
-import { MessageSquare, Zap } from 'lucide-react'
+import { MessageSquare, MousePointer2 } from 'lucide-react'
 import type { FC } from 'react'
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from '@/components/ui/tooltip'
 import { cn } from '@/lib/utils'
 import type { ChatMode } from './chatTypes'
 
@@ -12,36 +18,41 @@ export const ChatModeToggle: FC<ChatModeToggleProps> = ({
   mode,
   onModeChange,
 }) => {
+  const isAgentMode = mode === 'agent'
+
   return (
-    <div className="flex items-center rounded-full border border-border/50 bg-muted p-0.5">
-      <button
-        type="button"
-        onClick={() => onModeChange('chat')}
-        className={cn(
-          'flex items-center gap-1.5 rounded-full px-2.5 py-1.5 font-medium text-xs transition-all',
-          mode === 'chat'
-            ? 'bg-background text-foreground shadow-sm'
-            : 'text-muted-foreground',
-        )}
-        title="Chat Mode"
-      >
-        <MessageSquare className="h-3 w-3" />
-        <span>Chat</span>
-      </button>
-      <button
-        type="button"
-        onClick={() => onModeChange('agent')}
-        className={cn(
-          'flex items-center gap-1.5 rounded-full px-2.5 py-1.5 font-medium text-xs transition-all',
-          mode === 'agent'
-            ? 'bg-background text-foreground shadow-sm'
-            : 'text-muted-foreground',
-        )}
-        title="Agent Mode"
-      >
-        <Zap className="h-3 w-3" />
-        <span>Agent</span>
-      </button>
-    </div>
+    <TooltipProvider delayDuration={0}>
+      <Tooltip>
+        <TooltipTrigger asChild>
+          <button
+            type="button"
+            onClick={() => onModeChange(isAgentMode ? 'chat' : 'agent')}
+            className={cn(
+              'flex items-center gap-1.5 rounded-full border px-2.5 py-1.5 font-medium text-xs transition-all',
+              isAgentMode
+                ? 'border-border/50 bg-muted text-muted-foreground hover:text-foreground'
+                : 'border-[var(--accent-orange)]/30 bg-[var(--accent-orange)]/10 text-[var(--accent-orange)]',
+            )}
+          >
+            {isAgentMode ? (
+              <>
+                <MousePointer2 className="h-3 w-3" />
+                <span>Agent Mode ON</span>
+              </>
+            ) : (
+              <>
+                <MessageSquare className="h-3 w-3" />
+                <span>Chat Mode ON</span>
+              </>
+            )}
+          </button>
+        </TooltipTrigger>
+        <TooltipContent side="top" className="max-w-[220px]">
+          {isAgentMode
+            ? 'AI can browse, click, and navigate'
+            : 'AI can only read, cannot click or navigate'}
+        </TooltipContent>
+      </Tooltip>
+    </TooltipProvider>
   )
 }

apps/agent/entrypoints/sidepanel/index/useChatSession.ts

@@ -18,6 +18,7 @@ import {
   conversationStorage,
   useConversations,
 } from '@/lib/conversations/conversationStorage'
+import { formatConversationHistory } from '@/lib/conversations/formatConversationHistory'
 import { useLlmProviders } from '@/lib/llm-providers/useLlmProviders'
 import { track } from '@/lib/metrics/track'
 import { searchActionsStorage } from '@/lib/search-actions/searchActionsStorage'
@@ -90,7 +91,7 @@ export const useChatSession = () => {
     type: p.type,
   }))
 
-  const [mode, setMode] = useState<ChatMode>('chat')
+  const [mode, setMode] = useState<ChatMode>('agent')
   const [textToAction, setTextToAction] = useState<Map<string, ChatAction>>(
     new Map(),
   )
@@ -139,6 +140,7 @@ export const useChatSession = () => {
   const modeRef = useRef<ChatMode>(mode)
   const textToActionRef = useRef<Map<string, ChatAction>>(textToAction)
   const workingDirRef = useRef<string | undefined>(undefined)
+  const messagesRef = useRef<UIMessage[]>([])
 
   useDeepCompareEffect(() => {
     modeRef.current = mode
@@ -231,6 +233,13 @@ export const useChatSession = () => {
           }[]
         }
 
+        // Format previous messages from ref (messagesRef doesn't include current message yet)
+        const previousMessages = messagesRef.current
+        const previousConversation =
+          previousMessages.length > 0
+            ? formatConversationHistory(previousMessages)
+            : undefined
+
         return {
           api: `${agentUrlRef.current}/chat`,
           body: {
@@ -256,6 +265,7 @@ export const useChatSession = () => {
             userSystemPrompt: personalizationRef.current,
             userWorkingDir: workingDirRef.current,
             supportsImages: provider?.supportsImages,
+            previousConversation,
           },
         }
       },
@@ -292,6 +302,7 @@ export const useChatSession = () => {
 
   // biome-ignore lint/correctness/useExhaustiveDependencies: only need to run when messages change
   useEffect(() => {
+    messagesRef.current = messages
     if (messages.length > 0) {
       saveConversation(conversationIdRef.current, messages)
     }

apps/agent/lib/conversations/formatConversationHistory.ts

@@ -0,0 +1,32 @@
+import type { UIMessage } from 'ai'
+
+const MAX_MESSAGES = 10
+const MAX_MESSAGE_CHARS = 65536 // 16K context window size
+
+export function formatConversationHistory(messages: UIMessage[]): string {
+  if (messages.length === 0) return ''
+
+  const recentMessages = messages.slice(-MAX_MESSAGES)
+
+  const formatted = recentMessages
+    .map((msg) => {
+      const role = msg.role === 'user' ? 'user' : 'assistant'
+      const textContent = msg.parts
+        .filter((part) => part.type === 'text')
+        .map((part) => part.text)
+        .join('\n')
+
+      if (!textContent.trim()) return null
+
+      const truncatedContent =
+        textContent.length > MAX_MESSAGE_CHARS
+          ? `${textContent.slice(0, MAX_MESSAGE_CHARS)}... [truncated]`
+          : textContent
+
+      return `<${role}>${truncatedContent}</${role}>`
+    })
+    .filter(Boolean)
+    .join('\n\n')
+
+  return formatted
+}

apps/server/src/agent/gemini-agent.ts

@@ -19,6 +19,8 @@ import type { Content, Part } from '@google/genai'
 import type { BrowserContext } from '../api/types'
 import { logger } from '../lib/logger'
 import { Sentry } from '../lib/sentry'
+import { allCdpTools } from '../tools/cdp-based/registry'
+import { allControllerTools } from '../tools/controller-based/registry'
 import { AgentExecutionError } from './errors'
 import { buildSystemPrompt } from './prompt'
 import { VercelAIContentGenerator } from './provider-adapter/index'
@@ -26,6 +28,17 @@ import type { HonoSSEStream } from './provider-adapter/types'
 import { UIMessageStreamWriter } from './provider-adapter/ui-message-stream'
 import type { ResolvedAgentConfig } from './types'
 
+const CHAT_MODE_ALLOWED_TOOLS = new Set([
+  'browser_get_active_tab',
+  'browser_list_tabs',
+  'browser_get_page_content',
+  'browser_scroll_down',
+  'browser_scroll_up',
+  'browser_get_screenshot',
+  'browser_get_interactive_elements',
+  'browser_execute_javascript',
+])
+
 export interface ToolExecutionResult {
   parts: Part[]
   isError: boolean
@@ -102,6 +115,7 @@ export class GeminiAgent {
     // Build excluded tools list - always exclude save_memory and google_web_search
     // Conditionally exclude screenshot tools if model doesn't support images
     // Exclude window management tools unless in eval mode
+    // In chat mode, only allow read-only tools for page content extraction
     const excludedTools = ['save_memory', 'google_web_search']
     if (config.supportsImages === false) {
       excludedTools.push(
@@ -114,6 +128,21 @@ export class GeminiAgent {
       excludedTools.push('browser_create_window', 'browser_close_window')
     }
 
+    // Chat mode: restrict to read-only tools only (no browser automation)
+    if (config.chatMode === true) {
+      const allToolNames = [
+        ...allControllerTools.map((t) => t.name),
+        ...allCdpTools.map((t) => t.name),
+      ]
+      const chatModeExcludedTools = allToolNames.filter(
+        (name) => !CHAT_MODE_ALLOWED_TOOLS.has(name),
+      )
+      excludedTools.push(...chatModeExcludedTools)
+      logger.info('Chat mode enabled, restricting to read-only tools', {
+        allowedTools: Array.from(CHAT_MODE_ALLOWED_TOOLS),
+      })
+    }
+
     const geminiConfig = new GeminiConfig({
       sessionId: config.conversationId,
       targetDir: config.sessionExecutionDir,
@@ -371,12 +400,38 @@ export class GeminiAgent {
     honoStream: HonoSSEStream,
     signal?: AbortSignal,
     browserContext?: BrowserContext,
+    previousConversation?: string,
   ): Promise<void> {
     const abortSignal = signal || new AbortController().signal
     const promptId = `${this.conversationId}-${Date.now()}`
 
     const contextPrefix = this.formatBrowserContext(browserContext)
-    let currentParts: Part[] = [{ text: contextPrefix + message }]
+
+    // User query
+    const userQuery = `<USER_QUERY>
+${message}
+</USER_QUERY>`
+
+    // Inject previous conversation if resuming (no server-side history)
+    let fullMessage = userQuery
+    const hasHistory = this.client.getHistory().length > 0
+    if (previousConversation && !hasHistory) {
+      fullMessage = `<previous_conversation>
+The user is resuming a previous conversation. Here is the conversation history for context:
+
+${previousConversation}
+</previous_conversation>
+
+Continue the conversation based on the above context. Here is the user's new message:
+
+${userQuery}`
+      logger.info('Injecting previous conversation for resume', {
+        conversationId: this.conversationId,
+        historyLength: previousConversation.length,
+      })
+    }
+
+    let currentParts: Part[] = [{ text: contextPrefix + fullMessage }]
     let turnCount = 0
 
     const uiStream = honoStream

apps/server/src/agent/types.ts

@@ -37,4 +37,6 @@ export interface ResolvedAgentConfig {
   supportsImages?: boolean
   /** Eval mode - enables window management tools. Defaults to false. */
   evalMode?: boolean
+  /** Chat mode - restricts to read-only tools (no browser automation). Defaults to false. */
+  chatMode?: boolean
 }

apps/server/src/api/services/chat-service.ts

@@ -92,6 +92,7 @@ export class ChatService {
       userSystemPrompt: request.userSystemPrompt,
       sessionExecutionDir,
       supportsImages: request.supportsImages,
+      chatMode: request.mode === 'chat',
     }
 
     const agent = await sessionManager.getOrCreate(agentConfig, mcpServers)
@@ -100,6 +101,7 @@ export class ChatService {
       rawStream,
       abortSignal,
       request.browserContext,
+      request.previousConversation,
     )
   }
 

apps/server/src/api/types.ts

@@ -41,6 +41,8 @@ export const ChatRequestSchema = VercelAIConfigSchema.extend({
   isScheduledTask: z.boolean().optional().default(false),
   userWorkingDir: z.string().min(1).optional(),
   supportsImages: z.boolean().optional().default(true),
+  mode: z.enum(['chat', 'agent']).optional().default('agent'),
+  previousConversation: z.string().optional(),
 })
 
 export type ChatRequest = z.infer<typeof ChatRequestSchema>

package.json

@@ -11,7 +11,7 @@
   "scripts": {
     "postinstall": "bun run build:agent-sdk",
     "start:dev": "bun ./scripts/build/start-all.ts",
-    "start:server": "bun run --filter @browseros/server start",
+    "start:server": "bun run --filter @browseros/server --elide-lines=0 start",
     "start:agent": "bun ./scripts/build/controller-ext.ts && bun run --filter @browseros/agent dev",
     "build:server": "FORCE_COLOR=1 bun run --filter @browseros/server --elide-lines=0 build",
     "build:agent": "bun run --filter @browseros/agent build",