feat: display selected text from page in sidepanel (#496)

* feat: select text and pass to sidepanel * fix: lint issues * fix: persist selection across tabs * fix: review comments * fix: change when the selection is cleared * feat: sanitize url
2026-05-13 15:46:22 +00:00 · 2026-03-19 20:21:31 +05:30
parent f4d4b73a24
commit 5bb6143373
9 changed files with 256 additions and 3 deletions
--- a/packages/browseros-agent/apps/agent/entrypoints/background/index.ts
+++ b/packages/browseros-agent/apps/agent/entrypoints/background/index.ts
@@ -18,6 +18,7 @@ import {
  syncScheduledJobs,
 } from '@/lib/schedules/scheduleStorage'
 import { searchActionsStorage } from '@/lib/search-actions/searchActionsStorage'
+import { selectedTextStorage } from '@/lib/selected-text/selectedTextStorage'
 import { stopAgentStorage } from '@/lib/stop-agent/stop-agent-storage'
 import { scheduledJobRuns } from './scheduledJobRuns'

@@ -66,7 +67,12 @@ export default defineBackground(() => {
    }
  })

-  chrome.runtime.onMessage.addListener((message, sender) => {
+  chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
+    if (message?.type === 'get-tab-id') {
+      sendResponse({ tabId: sender.tab?.id })
+      return true
+    }
+
    if (message?.type === 'AUTH_SUCCESS' && sender.tab?.id) {
      const tabId = sender.tab.id
      authRedirectPathStorage
@@ -93,6 +99,17 @@ export default defineBackground(() => {
    }
  })

+  // Clean up selected text storage when a tab is closed
+  chrome.tabs.onRemoved.addListener((tabId) => {
+    const key = String(tabId)
+    selectedTextStorage.getValue().then((map) => {
+      if (map[key]) {
+        const { [key]: _, ...rest } = map
+        selectedTextStorage.setValue(rest)
+      }
+    })
+  })
+
  sessionStorage.watch(async (newSession) => {
    if (newSession?.user?.id) {
      try {
--- a/packages/browseros-agent/apps/agent/entrypoints/selection.content.ts
+++ b/packages/browseros-agent/apps/agent/entrypoints/selection.content.ts
@@ -0,0 +1,42 @@
+import { selectedTextStorage } from '@/lib/selected-text/selectedTextStorage'
+
+const MAX_SELECTED_TEXT_LENGTH = 5000
+
+export default defineContentScript({
+  matches: ['*://*/*'],
+  runAt: 'document_idle',
+  async main() {
+    const response = await chrome.runtime.sendMessage({ type: 'get-tab-id' })
+    const tabId: number | undefined = response?.tabId
+    if (!tabId) return
+
+    const key = String(tabId)
+
+    document.addEventListener('mouseup', () => {
+      const text = window.getSelection()?.toString().trim()
+
+      if (text && text.length > 0) {
+        selectedTextStorage.getValue().then((map) => {
+          selectedTextStorage.setValue({
+            ...map,
+            [key]: {
+              text: text.slice(0, MAX_SELECTED_TEXT_LENGTH),
+              pageUrl: window.location.href,
+              pageTitle: document.title,
+              tabId,
+              timestamp: Date.now(),
+            },
+          })
+        })
+      } else {
+        // User clicked without selecting — clear this tab's entry only
+        selectedTextStorage.getValue().then((map) => {
+          if (map[key]) {
+            const { [key]: _, ...rest } = map
+            selectedTextStorage.setValue(rest)
+          }
+        })
+      }
+    })
+  },
+})
--- a/packages/browseros-agent/apps/agent/entrypoints/sidepanel/index/ChatFooter.tsx
+++ b/packages/browseros-agent/apps/agent/entrypoints/sidepanel/index/ChatFooter.tsx
@@ -8,12 +8,17 @@ import { useGetUserMCPIntegrations } from '@/entrypoints/app/connect-mcp/useGetU
 import { Feature } from '@/lib/browseros/capabilities'
 import { useCapabilities } from '@/lib/browseros/useCapabilities'
 import { useMcpServers } from '@/lib/mcp/mcpServerStorage'
+import {
+  type SelectedTextData,
+  selectedTextStorage,
+} from '@/lib/selected-text/selectedTextStorage'
 import { cn } from '@/lib/utils'
 import type { VoiceInputState } from '@/lib/voice/useVoiceInput'
 import { useWorkspace } from '@/lib/workspace/use-workspace'
 import { ChatAttachedTabs } from './ChatAttachedTabs'
 import { ChatInput, type ChatInputHandle } from './ChatInput'
 import { ChatModeToggle } from './ChatModeToggle'
+import { ChatSelectedText } from './ChatSelectedText'
 import type { ChatMode } from './chatTypes'

 interface ChatFooterProps {
@@ -48,6 +53,33 @@ export const ChatFooter: FC<ChatFooterProps> = ({
  const { servers: mcpServers } = useMcpServers()
  const { data: userMCPIntegrations } = useGetUserMCPIntegrations()
  const chatInputRef = useRef<ChatInputHandle>(null)
+  const [selectionMap, setSelectionMap] = useState<
+    Record<string, SelectedTextData>
+  >({})
+  const [activeTabId, setActiveTabId] = useState<number | undefined>()
+
+  // Track active tab for tab-scoped selection display
+  useEffect(() => {
+    chrome.tabs
+      .query({ active: true, currentWindow: true })
+      .then((tabs) => setActiveTabId(tabs[0]?.id))
+    const listener = (activeInfo: { tabId: number }) => {
+      setActiveTabId(activeInfo.tabId)
+    }
+    chrome.tabs.onActivated.addListener(listener)
+    return () => chrome.tabs.onActivated.removeListener(listener)
+  }, [])
+
+  // Watch selected text storage (per-tab map)
+  useEffect(() => {
+    selectedTextStorage.getValue().then(setSelectionMap)
+    const unwatch = selectedTextStorage.watch(setSelectionMap)
+    return () => unwatch()
+  }, [])
+
+  const visibleSelectedText = activeTabId
+    ? (selectionMap[String(activeTabId)] ?? null)
+    : null
  const [isTabMentionOpen, setIsTabMentionOpen] = useState(false)

  useEffect(() => {
@@ -81,6 +113,19 @@ export const ChatFooter: FC<ChatFooterProps> = ({
  return (
    <footer className="border-border/40 border-t bg-background/80 backdrop-blur-md">
      <ChatAttachedTabs tabs={attachedTabs} onRemoveTab={onRemoveTab} />
+      {visibleSelectedText && (
+        <ChatSelectedText
+          selectedText={visibleSelectedText}
+          onDismiss={() => {
+            if (!activeTabId) return
+            const key = String(activeTabId)
+            selectedTextStorage.getValue().then((map) => {
+              const { [key]: _, ...rest } = map
+              selectedTextStorage.setValue(rest)
+            })
+          }}
+        />
+      )}

      <div className="p-3">
        <div className="flex items-center gap-2">
--- a/packages/browseros-agent/apps/agent/entrypoints/sidepanel/index/ChatSelectedText.tsx
+++ b/packages/browseros-agent/apps/agent/entrypoints/sidepanel/index/ChatSelectedText.tsx
@@ -0,0 +1,46 @@
+import { FileText, X } from 'lucide-react'
+import type { FC } from 'react'
+import type { SelectedTextData } from '@/lib/selected-text/selectedTextStorage'
+
+const MAX_DISPLAY_LENGTH = 200
+
+interface ChatSelectedTextProps {
+  selectedText: SelectedTextData
+  onDismiss: () => void
+}
+
+export const ChatSelectedText: FC<ChatSelectedTextProps> = ({
+  selectedText,
+  onDismiss,
+}) => {
+  const truncated =
+    selectedText.text.length > MAX_DISPLAY_LENGTH
+      ? `${selectedText.text.slice(0, MAX_DISPLAY_LENGTH)}...`
+      : selectedText.text
+
+  return (
+    <div className="px-3 pt-2">
+      <div className="relative rounded-lg border border-[var(--accent-orange)]/30 bg-accent/30">
+        <div className="flex items-start gap-2 px-3 py-2">
+          <FileText className="mt-0.5 h-3.5 w-3.5 flex-shrink-0 text-[var(--accent-orange)]" />
+          <div className="min-w-0 flex-1">
+            <div className="mb-0.5 truncate font-medium text-[10px] text-muted-foreground">
+              {selectedText.pageTitle}
+            </div>
+            <div className="line-clamp-3 text-foreground text-xs leading-relaxed">
+              &ldquo;{truncated}&rdquo;
+            </div>
+          </div>
+          <button
+            type="button"
+            onClick={onDismiss}
+            className="flex-shrink-0 rounded p-0.5 transition-colors hover:bg-background"
+            title="Remove selected text"
+          >
+            <X className="h-3 w-3 text-muted-foreground" />
+          </button>
+        </div>
+      </div>
+    </div>
+  )
+}
--- a/packages/browseros-agent/apps/agent/entrypoints/sidepanel/index/useChatSession.ts
+++ b/packages/browseros-agent/apps/agent/entrypoints/sidepanel/index/useChatSession.ts
@@ -27,6 +27,7 @@ import { createDefaultBrowserOSProvider } from '@/lib/llm-providers/storage'
 import { useLlmProviders } from '@/lib/llm-providers/useLlmProviders'
 import { track } from '@/lib/metrics/track'
 import { searchActionsStorage } from '@/lib/search-actions/searchActionsStorage'
+import { selectedTextStorage } from '@/lib/selected-text/selectedTextStorage'
 import { stopAgentStorage } from '@/lib/stop-agent/stop-agent-storage'
 import { selectedWorkspaceStorage } from '@/lib/workspace/workspace-storage'
 import type { ChatMode } from './chatTypes'
@@ -165,8 +166,34 @@ export const useChatSession = (options?: ChatSessionOptions) => {
  const modeRef = useRef<ChatMode>(mode)
  const textToActionRef = useRef<Map<string, ChatAction>>(textToAction)
  const workingDirRef = useRef<string | undefined>(undefined)
+  const selectionMapRef = useRef<
+    Record<string, { text: string; url: string; title: string }>
+  >({})
+  const pendingSelectionTabKeyRef = useRef<string | null>(null)
  const messagesRef = useRef<UIMessage[]>([])

+  useEffect(() => {
+    const toRef = (
+      map: Record<string, { text: string; pageUrl: string; pageTitle: string }>,
+    ) => {
+      const result: Record<
+        string,
+        { text: string; url: string; title: string }
+      > = {}
+      for (const [k, v] of Object.entries(map)) {
+        result[k] = { text: v.text, url: v.pageUrl, title: v.pageTitle }
+      }
+      return result
+    }
+    selectedTextStorage.getValue().then((map) => {
+      selectionMapRef.current = toRef(map)
+    })
+    const unwatchText = selectedTextStorage.watch((map) => {
+      selectionMapRef.current = toRef(map)
+    })
+    return () => unwatchText()
+  }, [])
+
  useEffect(() => {
    selectedWorkspaceStorage.getValue().then((folder) => {
      workingDirRef.current = folder?.path
@@ -210,6 +237,9 @@ export const useChatSession = (options?: ChatSessionOptions) => {
          currentWindow: true,
        })
        const activeTab = activeTabsList?.[0] ?? undefined
+        const activeTabSelection = activeTab?.id
+          ? (selectionMapRef.current[String(activeTab.id)] ?? null)
+          : null
        const message = getLastMessageText(messages)
        const provider =
          selectedLlmProviderRef.current ?? createDefaultBrowserOSProvider()
@@ -287,7 +317,7 @@ export const useChatSession = (options?: ChatSessionOptions) => {
            : history.map((m) => `${m.role}: ${m.content}`).join('\n')
          : undefined

-        return {
+        const result = {
          api: `${agentUrlRef.current}/chat`,
          body: {
            message,
@@ -322,8 +352,21 @@ export const useChatSession = (options?: ChatSessionOptions) => {
            supportsImages: provider?.supportsImages,
            previousConversation,
            declinedApps: declinedApps.length > 0 ? declinedApps : undefined,
+            selectedText: activeTabSelection?.text,
+            selectedTextSource: activeTabSelection
+              ? {
+                  url: activeTabSelection.url,
+                  title: activeTabSelection.title,
+                }
+              : undefined,
          },
        }
+
+        // Track which tab's selection was sent so we can clear it on success
+        pendingSelectionTabKeyRef.current =
+          activeTabSelection && activeTab?.id ? String(activeTab.id) : null
+
+        return result
      },
    }),
  })
@@ -417,6 +460,19 @@ export const useChatSession = (options?: ChatSessionOptions) => {

    if (!justFinished) return

+    // Clear the selected text that was sent with this request
+    const tabKey = pendingSelectionTabKeyRef.current
+    if (tabKey) {
+      pendingSelectionTabKeyRef.current = null
+      delete selectionMapRef.current[tabKey]
+      selectedTextStorage.getValue().then((map) => {
+        if (map[tabKey]) {
+          const { [tabKey]: _, ...rest } = map
+          selectedTextStorage.setValue(rest)
+        }
+      })
+    }
+
    const messagesToSave = messages.filter((m) => m.parts?.length > 0)
    if (messagesToSave.length === 0) return

--- a/packages/browseros-agent/apps/agent/lib/selected-text/selectedTextStorage.ts
+++ b/packages/browseros-agent/apps/agent/lib/selected-text/selectedTextStorage.ts
@@ -0,0 +1,14 @@
+import { storage } from '@wxt-dev/storage'
+
+export interface SelectedTextData {
+  text: string
+  pageUrl: string
+  pageTitle: string
+  tabId: number
+  timestamp: number
+}
+
+/** Map of tabId → selected text. Each tab's selection is independent. */
+export const selectedTextStorage = storage.defineItem<
+  Record<string, SelectedTextData>
+>('local:selectedTextMap', { defaultValue: {} })
--- a/packages/browseros-agent/apps/server/src/agent/format-message.ts
+++ b/packages/browseros-agent/apps/server/src/agent/format-message.ts
@@ -38,10 +38,34 @@ export function formatBrowserContext(browserContext?: BrowserContext): string {
  return `${lines.join('\n')}\n\n---\n\n`
 }

+/** Strip XML-like tags that match our prompt delimiters to prevent injection. */
+function sanitizeForPrompt(s: string): string {
+  return s.replace(
+    /<\/?(?:selected_text|USER_QUERY|page_context|AGENT_PROMPT|soul|memory_and_identity|security|workspace)[^>]*>/gi,
+    '',
+  )
+}
+
 export function formatUserMessage(
  message: string,
  browserContext?: BrowserContext,
+  selectedText?: string,
+  selectedTextSource?: { url: string; title: string },
 ): string {
  const contextPrefix = formatBrowserContext(browserContext)
-  return `${contextPrefix}<USER_QUERY>\n${message}\n</USER_QUERY>`
+
+  let selectedTextBlock = ''
+  if (selectedText) {
+    const sanitizedText = sanitizeForPrompt(selectedText)
+    const title = selectedTextSource?.title
+      ? sanitizeForPrompt(selectedTextSource.title).replace(/"/g, "'")
+      : ''
+    const url = selectedTextSource?.url
+      ? sanitizeForPrompt(selectedTextSource.url)
+      : ''
+    const source = title ? ` (from "${title}"${url ? ` — ${url}` : ''})` : ''
+    selectedTextBlock = `<selected_text${source}>\n${sanitizedText}\n</selected_text>\n\n`
+  }
+
+  return `${contextPrefix}${selectedTextBlock}<USER_QUERY>\n${message}\n</USER_QUERY>`
 }
--- a/packages/browseros-agent/apps/server/src/api/services/chat-service.ts
+++ b/packages/browseros-agent/apps/server/src/api/services/chat-service.ts
@@ -171,6 +171,8 @@ export class ChatService {
    const userContent = formatUserMessage(
      request.message,
      resolvedMessageContext,
+      request.selectedText,
+      request.selectedTextSource,
    )
    session.agent.appendUserMessage(userContent)

--- a/packages/browseros-agent/apps/server/src/api/types.ts
+++ b/packages/browseros-agent/apps/server/src/api/types.ts
@@ -47,6 +47,13 @@ export const ChatRequestSchema = AgentLLMConfigSchema.extend({
  supportsImages: z.boolean().optional().default(true),
  mode: z.enum(['chat', 'agent']).optional().default('agent'),
  declinedApps: z.array(z.string()).optional(),
+  selectedText: z.string().optional(),
+  selectedTextSource: z
+    .object({
+      url: z.string(),
+      title: z.string(),
+    })
+    .optional(),
  previousConversation: z
    .union([
      z.array(