mirror of
https://github.com/browseros-ai/BrowserOS.git
synced 2026-05-14 16:14:28 +00:00
Compare commits
6 Commits
feat/cli-l
...
multi-tab-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d9e249bb23 | ||
|
|
9c949a3014 | ||
|
|
ef2be606af | ||
|
|
f29d596c6d | ||
|
|
30178d6e07 | ||
|
|
092003c90c |
@@ -1,3 +1,4 @@
|
||||
import { activeStreamsStorage } from '@/lib/active-stream/active-stream-storage'
|
||||
import { sessionStorage } from '@/lib/auth/sessionStorage'
|
||||
import { Capabilities } from '@/lib/browseros/capabilities'
|
||||
import { getHealthCheckUrl, getMcpServerUrl } from '@/lib/browseros/helpers'
|
||||
@@ -93,6 +94,30 @@ export default defineBackground(() => {
|
||||
}
|
||||
})
|
||||
|
||||
// Auto-open side panel on tabs the agent interacts with during streaming.
|
||||
const openedFollowerTabs = new Set<number>()
|
||||
activeStreamsStorage.watch((map) => {
|
||||
const streams = Object.values(map)
|
||||
if (streams.length === 0) {
|
||||
openedFollowerTabs.clear()
|
||||
return
|
||||
}
|
||||
for (const state of streams) {
|
||||
const isActive =
|
||||
state.status === 'streaming' || state.status === 'submitted'
|
||||
if (!isActive || !state.followerTabIds?.length) continue
|
||||
|
||||
for (const tabId of state.followerTabIds) {
|
||||
if (openedFollowerTabs.has(tabId)) continue
|
||||
openedFollowerTabs.add(tabId)
|
||||
openSidePanel(tabId).catch(() => {})
|
||||
}
|
||||
}
|
||||
})
|
||||
chrome.tabs.onRemoved.addListener((tabId) => {
|
||||
openedFollowerTabs.delete(tabId)
|
||||
})
|
||||
|
||||
sessionStorage.watch(async (newSession) => {
|
||||
if (newSession?.user?.id) {
|
||||
try {
|
||||
|
||||
@@ -42,6 +42,7 @@ export const Chat = () => {
|
||||
disliked,
|
||||
onClickDislike,
|
||||
isRestoringConversation,
|
||||
isFollowing,
|
||||
} = useChatSessionContext()
|
||||
|
||||
const {
|
||||
@@ -202,7 +203,7 @@ export const Chat = () => {
|
||||
<div className="flex flex-1 items-center justify-center">
|
||||
<Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />
|
||||
</div>
|
||||
) : messages.length === 0 ? (
|
||||
) : messages.length === 0 && !isFollowing ? (
|
||||
<ChatEmptyState
|
||||
mode={mode}
|
||||
mounted={mounted}
|
||||
@@ -227,19 +228,36 @@ export const Chat = () => {
|
||||
{chatError && <ChatError error={chatError} />}
|
||||
</main>
|
||||
|
||||
<ChatFooter
|
||||
mode={mode}
|
||||
onModeChange={handleModeChange}
|
||||
input={input}
|
||||
onInputChange={setInput}
|
||||
onSubmit={handleSubmit}
|
||||
status={status}
|
||||
onStop={handleStop}
|
||||
attachedTabs={attachedTabs}
|
||||
onToggleTab={toggleTabSelection}
|
||||
onRemoveTab={removeTab}
|
||||
voice={voiceState}
|
||||
/>
|
||||
{isFollowing ? (
|
||||
<footer className="border-border/40 border-t bg-background/80 backdrop-blur-md">
|
||||
<div className="flex items-center justify-between p-3">
|
||||
<span className="text-muted-foreground text-sm">
|
||||
Following active task...
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleStop}
|
||||
className="cursor-pointer rounded-full bg-red-600 px-3 py-1.5 font-medium text-white text-xs shadow-sm transition-all duration-200 hover:bg-red-900"
|
||||
>
|
||||
Stop
|
||||
</button>
|
||||
</div>
|
||||
</footer>
|
||||
) : (
|
||||
<ChatFooter
|
||||
mode={mode}
|
||||
onModeChange={handleModeChange}
|
||||
input={input}
|
||||
onInputChange={setInput}
|
||||
onSubmit={handleSubmit}
|
||||
status={status}
|
||||
onStop={handleStop}
|
||||
attachedTabs={attachedTabs}
|
||||
onToggleTab={toggleTabSelection}
|
||||
onRemoveTab={removeTab}
|
||||
voice={voiceState}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import { useChat } from '@ai-sdk/react'
|
||||
import { DefaultChatTransport, type UIMessage } from 'ai'
|
||||
import { type ChatStatus, DefaultChatTransport, type UIMessage } from 'ai'
|
||||
import { compact } from 'es-toolkit/array'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { useSearchParams } from 'react-router'
|
||||
import useDeepCompareEffect from 'use-deep-compare-effect'
|
||||
import type { Provider } from '@/components/chat/chatComponentTypes'
|
||||
import {
|
||||
activeStreamsStorage,
|
||||
clearActiveStream,
|
||||
extractToolTabIds,
|
||||
getAllActiveStreams,
|
||||
setActiveStream,
|
||||
} from '@/lib/active-stream/active-stream-storage'
|
||||
import { Capabilities, Feature } from '@/lib/browseros/capabilities'
|
||||
import { useAgentServerUrl } from '@/lib/browseros/useBrowserOSProviders'
|
||||
import type { ChatAction } from '@/lib/chat-actions/types'
|
||||
@@ -129,6 +136,14 @@ export const useChatSession = (options?: ChatSessionOptions) => {
|
||||
conversationIdRef.current = conversationId
|
||||
}, [conversationId])
|
||||
|
||||
// Multi-tab stream sync: leader broadcasts, followers display
|
||||
const isLeaderRef = useRef(false)
|
||||
const isFollowingRef = useRef(false)
|
||||
const optedOutRef = useRef(false)
|
||||
const [isFollowing, setIsFollowing] = useState(false)
|
||||
const [followedMessages, setFollowedMessages] = useState<UIMessage[]>([])
|
||||
const [followedStatus, setFollowedStatus] = useState<ChatStatus>('ready')
|
||||
|
||||
const onClickLike = (messageId: string) => {
|
||||
const { responseText, queryText } = getResponseAndQueryFromMessageId(
|
||||
messages,
|
||||
@@ -335,11 +350,139 @@ export const useChatSession = (options?: ChatSessionOptions) => {
|
||||
}, [messages, status, setMessages])
|
||||
|
||||
useNotifyActiveTab({
|
||||
messages,
|
||||
status,
|
||||
messages: isFollowing ? followedMessages : messages,
|
||||
status: isFollowing ? followedStatus : status,
|
||||
conversationId: conversationIdRef.current,
|
||||
})
|
||||
|
||||
// Leader: broadcast stream state to shared storage (debounced)
|
||||
const writeTimeoutRef = useRef<ReturnType<typeof setTimeout>>()
|
||||
useEffect(() => {
|
||||
if (!isLeaderRef.current) return
|
||||
|
||||
const isStreaming = status === 'streaming' || status === 'submitted'
|
||||
const isFinished = status === 'ready' || status === 'error'
|
||||
const followerTabIds = extractToolTabIds(messages)
|
||||
|
||||
if (isStreaming) {
|
||||
clearTimeout(writeTimeoutRef.current)
|
||||
writeTimeoutRef.current = setTimeout(() => {
|
||||
setActiveStream({
|
||||
conversationId,
|
||||
messages,
|
||||
status,
|
||||
lastUpdated: Date.now(),
|
||||
followerTabIds,
|
||||
})
|
||||
}, 300)
|
||||
return () => clearTimeout(writeTimeoutRef.current)
|
||||
}
|
||||
|
||||
if (isFinished && messages.length > 0) {
|
||||
clearTimeout(writeTimeoutRef.current)
|
||||
setActiveStream({
|
||||
conversationId,
|
||||
messages,
|
||||
status,
|
||||
lastUpdated: Date.now(),
|
||||
followerTabIds,
|
||||
})
|
||||
isLeaderRef.current = false
|
||||
// Clean up after followers have had time to read the final state
|
||||
setTimeout(() => {
|
||||
clearActiveStream(conversationId)
|
||||
}, 2000)
|
||||
}
|
||||
}, [messages, status, conversationId])
|
||||
|
||||
// Follower: if this panel opened with no messages and there's an active
|
||||
// stream, follow it. Background only opens side panels on agent-interacted
|
||||
// tabs, so any fresh panel during streaming is a follower.
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: must run once — re-runs tear down the watcher
|
||||
useEffect(() => {
|
||||
const STALE_THRESHOLD_MS = 10_000
|
||||
let staleCheckTimer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const check = async () => {
|
||||
if (isLeaderRef.current || optedOutRef.current) return
|
||||
|
||||
const streams = await getAllActiveStreams()
|
||||
|
||||
// Find an active stream, or if we're already following, find the
|
||||
// completed stream we were following (to adopt its final messages)
|
||||
const activeStream = streams.find(
|
||||
(s) => s.status === 'streaming' || s.status === 'submitted',
|
||||
)
|
||||
const completedStream =
|
||||
!activeStream && isFollowingRef.current
|
||||
? streams.find(
|
||||
(s) =>
|
||||
s.status === 'ready' &&
|
||||
s.conversationId === conversationIdRef.current,
|
||||
)
|
||||
: undefined
|
||||
|
||||
const state = activeStream ?? completedStream
|
||||
|
||||
if (!state) {
|
||||
if (isFollowingRef.current) {
|
||||
isFollowingRef.current = false
|
||||
setIsFollowing(false)
|
||||
}
|
||||
clearTimeout(staleCheckTimer)
|
||||
return
|
||||
}
|
||||
|
||||
// Stream completed — adopt final messages and exit follower mode
|
||||
if (state.status === 'ready') {
|
||||
isFollowingRef.current = false
|
||||
setIsFollowing(false)
|
||||
setMessages(state.messages)
|
||||
setConversationId(
|
||||
state.conversationId as ReturnType<typeof crypto.randomUUID>,
|
||||
)
|
||||
clearTimeout(staleCheckTimer)
|
||||
return
|
||||
}
|
||||
|
||||
// Stale leader detection
|
||||
if (Date.now() - state.lastUpdated > STALE_THRESHOLD_MS) {
|
||||
if (isFollowingRef.current) {
|
||||
isFollowingRef.current = false
|
||||
setIsFollowing(false)
|
||||
}
|
||||
clearTimeout(staleCheckTimer)
|
||||
return
|
||||
}
|
||||
|
||||
isFollowingRef.current = true
|
||||
setIsFollowing(true)
|
||||
setFollowedMessages(state.messages)
|
||||
setFollowedStatus(state.status)
|
||||
setConversationId(
|
||||
state.conversationId as ReturnType<typeof crypto.randomUUID>,
|
||||
)
|
||||
clearTimeout(staleCheckTimer)
|
||||
staleCheckTimer = setTimeout(check, STALE_THRESHOLD_MS + 500)
|
||||
}
|
||||
|
||||
// Only auto-follow if this panel has no conversation of its own
|
||||
if (messagesRef.current.length === 0) {
|
||||
check()
|
||||
}
|
||||
|
||||
const unwatchStreams = activeStreamsStorage.watch(() => {
|
||||
if (isFollowingRef.current || messagesRef.current.length === 0) {
|
||||
check()
|
||||
}
|
||||
})
|
||||
|
||||
return () => {
|
||||
unwatchStreams()
|
||||
clearTimeout(staleCheckTimer)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const {
|
||||
data: remoteConversationData,
|
||||
isFetched: isRemoteConversationFetched,
|
||||
@@ -452,6 +595,11 @@ export const useChatSession = (options?: ChatSessionOptions) => {
|
||||
}, [isIntegrationsSynced, baseSendMessage])
|
||||
|
||||
const sendMessage = (params: { text: string; action?: ChatAction }) => {
|
||||
isLeaderRef.current = true
|
||||
isFollowingRef.current = false
|
||||
optedOutRef.current = false
|
||||
setIsFollowing(false)
|
||||
|
||||
track(MESSAGE_SENT_EVENT, {
|
||||
mode,
|
||||
provider_type: selectedLlmProvider?.type,
|
||||
@@ -518,6 +666,14 @@ export const useChatSession = (options?: ChatSessionOptions) => {
|
||||
const resetConversation = () => {
|
||||
track(CONVERSATION_RESET_EVENT, { message_count: messages.length })
|
||||
stop()
|
||||
if (isLeaderRef.current) {
|
||||
clearActiveStream(conversationIdRef.current)
|
||||
isLeaderRef.current = false
|
||||
}
|
||||
isFollowingRef.current = false
|
||||
optedOutRef.current = true
|
||||
setIsFollowing(false)
|
||||
setFollowedMessages([])
|
||||
setConversationId(crypto.randomUUID())
|
||||
setMessages([])
|
||||
setTextToAction(new Map())
|
||||
@@ -530,18 +686,26 @@ export const useChatSession = (options?: ChatSessionOptions) => {
|
||||
const isRestoringConversation =
|
||||
!!conversationIdParam && restoredConversationId !== conversationIdParam
|
||||
|
||||
const stopFollowedStream = () => {
|
||||
stopAgentStorage.setValue({
|
||||
conversationId: conversationIdRef.current,
|
||||
timestamp: Date.now(),
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
mode,
|
||||
setMode,
|
||||
messages,
|
||||
messages: isFollowing ? followedMessages : messages,
|
||||
sendMessage,
|
||||
status,
|
||||
stop,
|
||||
status: isFollowing ? followedStatus : status,
|
||||
stop: isFollowing ? stopFollowedStream : stop,
|
||||
providers,
|
||||
selectedProvider,
|
||||
isLoading: isLoadingProviders || isLoadingAgentUrl,
|
||||
isSyncing: !isIntegrationsSynced,
|
||||
isRestoringConversation,
|
||||
isFollowing,
|
||||
agentUrlError,
|
||||
chatError,
|
||||
handleSelectProvider,
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { storage } from '@wxt-dev/storage'
|
||||
import type { ChatStatus, UIMessage } from 'ai'
|
||||
|
||||
export interface ActiveStreamState {
|
||||
conversationId: string
|
||||
messages: UIMessage[]
|
||||
status: ChatStatus
|
||||
lastUpdated: number
|
||||
followerTabIds: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Single storage item holding the active stream state.
|
||||
* Uses local storage for reliable cross-context access (background + sidepanel).
|
||||
* Keyed by conversationId inside the map for parallel agent support.
|
||||
*/
|
||||
export type ActiveStreamsMap = Record<string, ActiveStreamState>
|
||||
|
||||
export const activeStreamsStorage = storage.defineItem<ActiveStreamsMap>(
|
||||
'local:active-streams',
|
||||
{ fallback: {} },
|
||||
)
|
||||
|
||||
/** Write a conversation's stream state. */
|
||||
export async function setActiveStream(state: ActiveStreamState): Promise<void> {
|
||||
const map = await activeStreamsStorage.getValue()
|
||||
map[state.conversationId] = state
|
||||
await activeStreamsStorage.setValue(map)
|
||||
}
|
||||
|
||||
/** Remove a conversation's stream state. */
|
||||
export async function clearActiveStream(conversationId: string): Promise<void> {
|
||||
const map = await activeStreamsStorage.getValue()
|
||||
delete map[conversationId]
|
||||
await activeStreamsStorage.setValue(map)
|
||||
}
|
||||
|
||||
/** Read all active streams. */
|
||||
export async function getAllActiveStreams(): Promise<ActiveStreamState[]> {
|
||||
const map = await activeStreamsStorage.getValue()
|
||||
return Object.values(map)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all unique tabIds from tool output metadata in messages.
|
||||
* The server attaches metadata.tabId to every tool that operates on or creates a page.
|
||||
*/
|
||||
export function extractToolTabIds(messages: UIMessage[]): number[] {
|
||||
const tabIds = new Set<number>()
|
||||
for (const message of messages) {
|
||||
if (!message.parts) continue
|
||||
for (const part of message.parts) {
|
||||
const typedPart = part as { type?: string; output?: unknown }
|
||||
if (!typedPart.type?.startsWith('tool-')) continue
|
||||
|
||||
const output = typedPart.output as
|
||||
| { metadata?: { tabId?: number } }
|
||||
| undefined
|
||||
if (output?.metadata?.tabId) {
|
||||
tabIds.add(output.metadata.tabId)
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...tabIds]
|
||||
}
|
||||
@@ -74,8 +74,12 @@ export async function executeTool(
|
||||
|
||||
const result = await response.build(ctx.browser)
|
||||
|
||||
// TODO: nikhil -- maybe add to tool context instead of ugly args casting
|
||||
const pageId = (args as Record<string, unknown>).page
|
||||
// Resolve tabId for the page this tool operated on.
|
||||
// First check the `page` input param (tools that act on existing pages),
|
||||
// then fall back to `structuredContent.pageId` (tools that create new pages
|
||||
// like new_page / new_hidden_page).
|
||||
const pageId =
|
||||
(args as Record<string, unknown>).page ?? result.structuredContent?.pageId
|
||||
if (typeof pageId === 'number') {
|
||||
const tabId = ctx.browser.getTabIdForPage(pageId)
|
||||
if (tabId !== undefined) {
|
||||
|
||||
Reference in New Issue
Block a user