mirror of
https://github.com/AIPexStudio/AIPex.git
synced 2026-05-13 18:51:35 +00:00
feat: enhance chat adapter and settings components
- Updated ChatAdapter to handle new assistant messages after tool calls, improving message management. - Added connection tab support in SettingsPage, allowing for better organization of settings. - Introduced connectionContent prop in SettingsPageProps for dynamic content rendering. - Enhanced message-item component to include onLogin handler for improved user interaction. - Refactored useChat and useChatConfig hooks for better state management and silent reload capabilities. - Implemented WebSocket MCP bridge in background script for real-time communication. This update improves user experience and functionality across chat and settings components.
This commit is contained in:
@@ -140,6 +140,14 @@ export class ChatAdapter {
|
||||
break;
|
||||
|
||||
case "content_delta":
|
||||
// When text arrives after tool calls, start a new assistant message.
|
||||
// This mirrors aipex's behavior where each model response after tool
|
||||
// execution becomes a separate message, enabling the turn-based
|
||||
// collapsing logic in the message list.
|
||||
if (this.toolsAddedSinceLastText) {
|
||||
this.state.currentAssistantMessageId = null;
|
||||
this.toolsAddedSinceLastText = false;
|
||||
}
|
||||
this.ensureAssistantMessage();
|
||||
this.updateStatus("streaming");
|
||||
this.appendContentDelta(event.delta);
|
||||
@@ -179,11 +187,13 @@ export class ChatAdapter {
|
||||
case "execution_complete":
|
||||
this.updateStatus("idle");
|
||||
this.state.currentAssistantMessageId = null;
|
||||
this.toolsAddedSinceLastText = false;
|
||||
break;
|
||||
|
||||
case "error":
|
||||
this.updateStatus("error");
|
||||
this.state.currentAssistantMessageId = null;
|
||||
this.toolsAddedSinceLastText = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,6 +257,7 @@ export function DefaultMessageItem({
|
||||
{message.metadata.needLogin && (
|
||||
<LoginPrompt
|
||||
showByokOption
|
||||
onLogin={slots.onLogin}
|
||||
onOpenSettings={() => chrome.runtime?.openOptionsPage?.()}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
Mic,
|
||||
Package,
|
||||
Palette,
|
||||
Plug,
|
||||
Plus,
|
||||
Search,
|
||||
Settings,
|
||||
@@ -221,6 +222,7 @@ export function SettingsPage({
|
||||
onSave,
|
||||
onTestConnection,
|
||||
skillsContent,
|
||||
connectionContent,
|
||||
sttConfig,
|
||||
initialTab,
|
||||
initialSkill: _initialSkill,
|
||||
@@ -826,9 +828,14 @@ export function SettingsPage({
|
||||
className="w-full"
|
||||
>
|
||||
<TabsList
|
||||
className={`grid w-full mb-6 ${
|
||||
skillsContent ? "grid-cols-3" : "grid-cols-2"
|
||||
}`}
|
||||
className={cn(
|
||||
"grid w-full mb-6",
|
||||
skillsContent && connectionContent
|
||||
? "grid-cols-4"
|
||||
: skillsContent || connectionContent
|
||||
? "grid-cols-3"
|
||||
: "grid-cols-2",
|
||||
)}
|
||||
>
|
||||
<TabsTrigger value="general" className="flex items-center gap-2">
|
||||
<Settings className="h-4 w-4" />
|
||||
@@ -844,6 +851,12 @@ export function SettingsPage({
|
||||
{t("settings.skillsTab")}
|
||||
</TabsTrigger>
|
||||
)}
|
||||
{connectionContent && (
|
||||
<TabsTrigger value="connection" className="flex items-center gap-2">
|
||||
<Plug className="h-4 w-4" />
|
||||
{language === "zh" ? "连接" : "Connection"}
|
||||
</TabsTrigger>
|
||||
)}
|
||||
</TabsList>
|
||||
|
||||
{/* General Tab */}
|
||||
@@ -1680,6 +1693,13 @@ export function SettingsPage({
|
||||
{skillsContent}
|
||||
</TabsContent>
|
||||
)}
|
||||
|
||||
{/* Connection Tab */}
|
||||
{connectionContent && (
|
||||
<TabsContent value="connection" className="space-y-6">
|
||||
{connectionContent}
|
||||
</TabsContent>
|
||||
)}
|
||||
</Tabs>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -22,6 +22,8 @@ export interface SettingsPageProps {
|
||||
onSave?: (settings: AppSettings) => void;
|
||||
onTestConnection?: (settings: AppSettings) => Promise<boolean>;
|
||||
skillsContent?: ReactNode;
|
||||
/** Optional content for a "Connection" tab (e.g. MCP bridge panel). */
|
||||
connectionContent?: ReactNode;
|
||||
/** Optional ElevenLabs STT config adapter; when provided the STT card is shown. */
|
||||
sttConfig?: STTConfigAdapter;
|
||||
/** Pre-select a tab on mount (e.g. from URL params). */
|
||||
@@ -38,7 +40,7 @@ export interface ProviderConfig {
|
||||
|
||||
export type ProviderConfigs = Record<AIProviderKey, ProviderConfig>;
|
||||
|
||||
export type SettingsTab = "general" | "ai" | "skills";
|
||||
export type SettingsTab = "general" | "ai" | "skills" | "connection";
|
||||
|
||||
export interface SaveStatus {
|
||||
type: "success" | "error" | "info" | "";
|
||||
|
||||
@@ -83,25 +83,46 @@ export function useChatConfig(
|
||||
});
|
||||
const [isLoading, setIsLoading] = useState(autoLoad);
|
||||
|
||||
const applyStoredSettings = useCallback(
|
||||
(stored: unknown) => {
|
||||
setSettings((prev: AppSettings) => ({
|
||||
...prev,
|
||||
...(stored as Partial<AppSettings>),
|
||||
customModels: (stored as AppSettings).customModels ?? [],
|
||||
providerType: (stored as AppSettings).providerType ?? "openai",
|
||||
providerEnabled: (stored as AppSettings).providerEnabled ?? false,
|
||||
}));
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const loadSettings = useCallback(async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const stored = await storageAdapter.load(STORAGE_KEYS.SETTINGS);
|
||||
if (stored) {
|
||||
setSettings((prev: AppSettings) => ({
|
||||
...prev,
|
||||
...stored,
|
||||
customModels: (stored as AppSettings).customModels ?? [],
|
||||
providerType: (stored as AppSettings).providerType ?? "openai",
|
||||
providerEnabled: (stored as AppSettings).providerEnabled ?? false,
|
||||
}));
|
||||
applyStoredSettings(stored);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to load chat settings:", error);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}, [storageAdapter]);
|
||||
}, [storageAdapter, applyStoredSettings]);
|
||||
|
||||
// Silent reload that does NOT touch isLoading, so existing UI stays mounted.
|
||||
// Used by the storage watcher to pick up changes (e.g. model switch)
|
||||
// without unmounting ChatBot.
|
||||
const reloadSettingsSilently = useCallback(async () => {
|
||||
try {
|
||||
const stored = await storageAdapter.load(STORAGE_KEYS.SETTINGS);
|
||||
if (stored) {
|
||||
applyStoredSettings(stored);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to reload chat settings:", error);
|
||||
}
|
||||
}, [storageAdapter, applyStoredSettings]);
|
||||
|
||||
const saveSettings = useCallback(
|
||||
async (newSettings: AppSettings) => {
|
||||
@@ -119,16 +140,16 @@ export function useChatConfig(
|
||||
void loadSettings();
|
||||
}
|
||||
|
||||
// Set up storage change listener for real-time sync
|
||||
// Set up storage change listener for real-time sync.
|
||||
// Uses silent reload so the UI does not unmount (preserving chat state).
|
||||
const unwatch = storageAdapter.watch(STORAGE_KEYS.SETTINGS, () => {
|
||||
// Reload settings when storage changes (e.g., from settings page)
|
||||
void loadSettings();
|
||||
void reloadSettingsSilently();
|
||||
});
|
||||
|
||||
return () => {
|
||||
unwatch();
|
||||
};
|
||||
}, [autoLoad, loadSettings, storageAdapter]);
|
||||
}, [autoLoad, loadSettings, reloadSettingsSilently, storageAdapter]);
|
||||
|
||||
const updateSetting = useCallback(
|
||||
async <K extends keyof AppSettings>(
|
||||
|
||||
@@ -91,9 +91,6 @@ export function useChat(
|
||||
const [sessionId, setSessionId] = useState<string | null>(null);
|
||||
const [metrics, setMetrics] = useState<AgentMetrics | null>(null);
|
||||
|
||||
// Cumulative session-level metrics (sum across all runs)
|
||||
const cumulativeMetricsRef = useRef<AgentMetrics | null>(null);
|
||||
|
||||
// Refs for stable callbacks
|
||||
const handlersRef = useRef(handlers);
|
||||
handlersRef.current = handlers;
|
||||
@@ -102,6 +99,18 @@ export function useChat(
|
||||
configRef.current = config;
|
||||
|
||||
const activeGeneratorRef = useRef<AsyncGenerator<AgentEvent> | null>(null);
|
||||
const prevAgentRef = useRef<AIPex | undefined>(agent);
|
||||
|
||||
// When the agent instance changes (e.g. model switch), reset the sessionId
|
||||
// so subsequent messages create a fresh session on the new agent, but
|
||||
// preserve existing UI messages so the conversation history stays visible.
|
||||
useEffect(() => {
|
||||
if (agent && prevAgentRef.current && agent !== prevAgentRef.current) {
|
||||
setSessionId(null);
|
||||
setMetrics(null);
|
||||
}
|
||||
prevAgentRef.current = agent;
|
||||
}, [agent]);
|
||||
|
||||
// Create adapter with callbacks
|
||||
const adapter = useMemo(() => {
|
||||
@@ -156,24 +165,13 @@ export function useChat(
|
||||
handlersRef.current?.onError?.(event.error);
|
||||
}
|
||||
|
||||
// Handle metrics update – accumulate across the session
|
||||
// Handle metrics update – show latest completion metrics (not cumulative)
|
||||
if (event.type === "metrics_update") {
|
||||
const prev = cumulativeMetricsRef.current;
|
||||
const cumulative: AgentMetrics = {
|
||||
tokensUsed: (prev?.tokensUsed ?? 0) + event.metrics.tokensUsed,
|
||||
promptTokens:
|
||||
(prev?.promptTokens ?? 0) + event.metrics.promptTokens,
|
||||
completionTokens:
|
||||
(prev?.completionTokens ?? 0) + event.metrics.completionTokens,
|
||||
// Non-cumulative fields: use latest values
|
||||
itemCount: event.metrics.itemCount,
|
||||
maxTurns: event.metrics.maxTurns,
|
||||
duration: (prev?.duration ?? 0) + event.metrics.duration,
|
||||
startTime: prev?.startTime ?? event.metrics.startTime,
|
||||
};
|
||||
cumulativeMetricsRef.current = cumulative;
|
||||
setMetrics(cumulative);
|
||||
handlersRef.current?.onMetricsUpdate?.(cumulative, event.sessionId);
|
||||
setMetrics(event.metrics);
|
||||
handlersRef.current?.onMetricsUpdate?.(
|
||||
event.metrics,
|
||||
event.sessionId,
|
||||
);
|
||||
}
|
||||
|
||||
// Process the event through adapter
|
||||
@@ -277,7 +275,6 @@ export function useChat(
|
||||
activeGeneratorRef.current = null;
|
||||
setSessionId(null);
|
||||
setMetrics(null);
|
||||
cumulativeMetricsRef.current = null;
|
||||
adapter.reset(configRef.current?.initialMessages ?? []);
|
||||
}, [adapter, agent, sessionId]);
|
||||
|
||||
|
||||
@@ -128,6 +128,8 @@ export interface ChatbotSlots {
|
||||
afterMessages?: () => ReactNode;
|
||||
/** Extra content rendered inside PromptInput (e.g., context/skill loaders) */
|
||||
promptExtras?: () => ReactNode;
|
||||
/** Handler called when user clicks Login in a LoginPrompt inside a message */
|
||||
onLogin?: () => void;
|
||||
}
|
||||
|
||||
// ============ Components Configuration ============
|
||||
|
||||
@@ -81,6 +81,7 @@
|
||||
"debugger",
|
||||
"cookies",
|
||||
"webNavigation",
|
||||
"audioCapture"
|
||||
"audioCapture",
|
||||
"alarms"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -637,4 +637,87 @@ chrome.runtime.onMessageExternal.addListener(
|
||||
},
|
||||
);
|
||||
|
||||
// =============================================================================
|
||||
// WebSocket MCP Bridge
|
||||
// =============================================================================
|
||||
import { wsMcpServer } from "@aipexstudio/browser-runtime";
|
||||
|
||||
// Handle MCP bridge messages
|
||||
chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
|
||||
if (message.request === "ws-bridge-connect") {
|
||||
const url = message.url as string;
|
||||
wsMcpServer
|
||||
.connect(url)
|
||||
.then(() => {
|
||||
updateMcpBadge(true);
|
||||
sendResponse({ success: true });
|
||||
})
|
||||
.catch((error) => {
|
||||
sendResponse({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
if (message.request === "ws-bridge-disconnect") {
|
||||
wsMcpServer
|
||||
.disconnect()
|
||||
.then(() => {
|
||||
updateMcpBadge(false);
|
||||
sendResponse({ success: true });
|
||||
})
|
||||
.catch((error) => {
|
||||
sendResponse({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
if (message.request === "ws-bridge-status") {
|
||||
sendResponse(wsMcpServer.getStatus());
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
|
||||
// Update badge to show MCP connection status
|
||||
function updateMcpBadge(connected: boolean) {
|
||||
if (connected) {
|
||||
chrome.action.setBadgeText({ text: "ON" });
|
||||
chrome.action.setBadgeBackgroundColor({ color: "#22c55e" });
|
||||
} else {
|
||||
chrome.action.setBadgeText({ text: "" });
|
||||
}
|
||||
}
|
||||
|
||||
// Handle keepalive alarms for the WebSocket connection
|
||||
chrome.alarms.onAlarm.addListener((alarm) => {
|
||||
wsMcpServer.handleAlarm(alarm);
|
||||
});
|
||||
|
||||
// Track MCP connection status for badge updates
|
||||
wsMcpServer.onStatusChange((state) => {
|
||||
updateMcpBadge(state.status === "connected");
|
||||
});
|
||||
|
||||
// Auto-connect to saved URL on startup
|
||||
wsMcpServer
|
||||
.getSavedUrl()
|
||||
.then((url) => {
|
||||
if (url) {
|
||||
console.log("[WsMcpServer] Auto-connecting to saved URL:", url);
|
||||
wsMcpServer.connect(url).catch(() => {
|
||||
// connect() handles its own retry logic
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
// Ignore storage errors on startup
|
||||
});
|
||||
|
||||
console.log("AIPex background service worker started");
|
||||
|
||||
@@ -5,13 +5,6 @@
|
||||
|
||||
import { useChatContext } from "@aipexstudio/aipex-react/components/chatbot";
|
||||
import { Button } from "@aipexstudio/aipex-react/components/ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuSeparator,
|
||||
DropdownMenuTrigger,
|
||||
} from "@aipexstudio/aipex-react/components/ui/dropdown-menu";
|
||||
import { useTranslation } from "@aipexstudio/aipex-react/i18n/context";
|
||||
import { getRuntime } from "@aipexstudio/aipex-react/lib/runtime";
|
||||
import { cn } from "@aipexstudio/aipex-react/lib/utils";
|
||||
@@ -20,15 +13,11 @@ import { conversationStorage } from "@aipexstudio/browser-runtime";
|
||||
import {
|
||||
KeyboardIcon,
|
||||
MicIcon,
|
||||
MoreHorizontalIcon,
|
||||
PlusIcon,
|
||||
SettingsIcon,
|
||||
Share2Icon,
|
||||
SparklesIcon,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { UserProfile, useAuth } from "../auth";
|
||||
import { shareConversation } from "../services/share-conversation";
|
||||
import { ConversationHistory } from "./conversation-history";
|
||||
import { useInputMode } from "./input-mode-context";
|
||||
import { fromStorageFormat, toStorageFormat } from "./message-adapter";
|
||||
@@ -43,7 +32,7 @@ export function BrowserChatHeader({
|
||||
}: HeaderProps) {
|
||||
const { t } = useTranslation();
|
||||
const runtime = getRuntime();
|
||||
const { messages, setMessages, interrupt, sendMessage } = useChatContext();
|
||||
const { messages, setMessages, interrupt } = useChatContext();
|
||||
const { user, login, isLoading: isAuthLoading } = useAuth();
|
||||
|
||||
const [currentConversationId, setCurrentConversationId] = useState<
|
||||
@@ -149,34 +138,6 @@ export function BrowserChatHeader({
|
||||
setInputMode(inputMode === "voice" ? "text" : "voice");
|
||||
}, [inputMode, setInputMode]);
|
||||
|
||||
// Share conversation
|
||||
const [isSharing, setIsSharing] = useState(false);
|
||||
|
||||
const handleShare = useCallback(async () => {
|
||||
if (isSharing) return;
|
||||
|
||||
const nonSystemMessages = messages.filter((m) => m.role !== "system");
|
||||
if (nonSystemMessages.length === 0) return;
|
||||
|
||||
setIsSharing(true);
|
||||
try {
|
||||
const { url } = await shareConversation(messages);
|
||||
chrome.tabs.create({ url });
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"[Share] Failed:",
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
} finally {
|
||||
setIsSharing(false);
|
||||
}
|
||||
}, [messages, isSharing]);
|
||||
|
||||
// Save as Skill — instructs the AI to create a skill from the conversation
|
||||
const handleSaveAsSkill = useCallback(() => {
|
||||
sendMessage("use skill-creator skill to save the conversation");
|
||||
}, [sendMessage]);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
@@ -224,39 +185,8 @@ export function BrowserChatHeader({
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Right side - More menu, New Chat, User Profile */}
|
||||
{/* Right side - New Chat, User Profile */}
|
||||
<div className="flex items-center gap-1">
|
||||
{/* More actions dropdown (Share, Save as Skill) */}
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button variant="ghost" size="icon" className="size-8">
|
||||
<MoreHorizontalIcon className="size-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="w-48">
|
||||
<DropdownMenuItem
|
||||
onClick={handleShare}
|
||||
disabled={
|
||||
isSharing ||
|
||||
messages.filter((m) => m.role !== "system").length === 0
|
||||
}
|
||||
>
|
||||
<Share2Icon className="mr-2 size-4" />
|
||||
{isSharing ? "Sharing..." : "Share Conversation"}
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem
|
||||
onClick={handleSaveAsSkill}
|
||||
disabled={
|
||||
messages.filter((m) => m.role !== "system").length === 0
|
||||
}
|
||||
>
|
||||
<SparklesIcon className="mr-2 size-4" />
|
||||
Save as Skill
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
|
||||
87
packages/browser-ext/src/lib/browser-message-actions.tsx
Normal file
87
packages/browser-ext/src/lib/browser-message-actions.tsx
Normal file
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* BrowserMessageActions
|
||||
* Custom message actions for the browser extension.
|
||||
* Renders Retry, Copy, Share Conversation, and Save as Skill inline
|
||||
* with each last assistant message (matching aipex behavior).
|
||||
*/
|
||||
|
||||
import {
|
||||
Action,
|
||||
Actions,
|
||||
} from "@aipexstudio/aipex-react/components/ai-elements/actions";
|
||||
import { useChatContext } from "@aipexstudio/aipex-react/components/chatbot";
|
||||
import type { MessageActionsSlotProps } from "@aipexstudio/aipex-react/types";
|
||||
import {
|
||||
CopyIcon,
|
||||
PuzzleIcon,
|
||||
RefreshCcwIcon,
|
||||
Share2Icon,
|
||||
} from "lucide-react";
|
||||
import { useCallback, useState } from "react";
|
||||
import { useConfigContext } from "@aipexstudio/aipex-react/components/chatbot";
|
||||
import { useAuth } from "../auth";
|
||||
import { isByokConfigured } from "./ai-provider";
|
||||
import { shareConversation } from "../services/share-conversation";
|
||||
|
||||
export function BrowserMessageActions({
|
||||
onRegenerate,
|
||||
onCopy,
|
||||
}: MessageActionsSlotProps) {
|
||||
const { messages, sendMessage } = useChatContext();
|
||||
const { user } = useAuth();
|
||||
const { settings } = useConfigContext();
|
||||
const [isSharing, setIsSharing] = useState(false);
|
||||
|
||||
const isByok = isByokConfigured(settings);
|
||||
|
||||
const handleShare = useCallback(async () => {
|
||||
if (isSharing) return;
|
||||
|
||||
const nonSystemMessages = messages.filter((m) => m.role !== "system");
|
||||
if (nonSystemMessages.length === 0) return;
|
||||
|
||||
setIsSharing(true);
|
||||
try {
|
||||
const { url } = await shareConversation(messages);
|
||||
chrome.tabs.create({ url });
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"[Share] Failed:",
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
} finally {
|
||||
setIsSharing(false);
|
||||
}
|
||||
}, [messages, isSharing]);
|
||||
|
||||
const handleSaveAsSkill = useCallback(() => {
|
||||
sendMessage("use skill-creator skill to save the conversation");
|
||||
}, [sendMessage]);
|
||||
|
||||
return (
|
||||
<Actions className="mt-2">
|
||||
{onCopy && (
|
||||
<Action onClick={onCopy} label="Copy">
|
||||
<CopyIcon className="size-3" />
|
||||
</Action>
|
||||
)}
|
||||
{!isByok && user && (
|
||||
<Action
|
||||
onClick={handleShare}
|
||||
disabled={isSharing}
|
||||
label={isSharing ? "Sharing..." : "Share"}
|
||||
>
|
||||
<Share2Icon className="size-3" />
|
||||
</Action>
|
||||
)}
|
||||
<Action onClick={handleSaveAsSkill} label="Save as Skill">
|
||||
<PuzzleIcon className="size-3" />
|
||||
</Action>
|
||||
{onRegenerate && (
|
||||
<Action onClick={onRegenerate} label="Retry">
|
||||
<RefreshCcwIcon className="size-3" />
|
||||
</Action>
|
||||
)}
|
||||
</Actions>
|
||||
);
|
||||
}
|
||||
@@ -15,7 +15,7 @@ import type { AuthCheckResult } from "@aipexstudio/aipex-react/types";
|
||||
import { ChromeStorageAdapter } from "@aipexstudio/browser-runtime";
|
||||
import React, { useCallback, useEffect, useRef, useState } from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { AuthProvider } from "../../auth";
|
||||
import { AuthProvider, useAuth } from "../../auth";
|
||||
import { chromeStorageAdapter } from "../../hooks";
|
||||
import { isByokConfigured } from "../../lib/ai-provider";
|
||||
import { AutomationModeInputToolbar } from "../../lib/automation-mode-toolbar";
|
||||
@@ -34,6 +34,7 @@ import { ChatImagesListener } from "../../lib/chat-images-listener";
|
||||
import { InputModeProvider } from "../../lib/input-mode-context";
|
||||
import { InterventionModeProvider } from "../../lib/intervention-mode-context";
|
||||
import { InterventionUI } from "../../lib/intervention-ui";
|
||||
import { BrowserMessageActions } from "../../lib/browser-message-actions";
|
||||
import { UpdateBannerWrapper } from "../../lib/update-banner-wrapper";
|
||||
|
||||
const i18nStorageAdapter = new ChromeStorageAdapter<Language>();
|
||||
@@ -242,6 +243,7 @@ function ChatApp() {
|
||||
...BROWSER_AGENT_CONFIG,
|
||||
});
|
||||
|
||||
const { login } = useAuth();
|
||||
const pendingInput = usePendingPrompt();
|
||||
const heartbeat = useConversationHeartbeat();
|
||||
useReplaySetup();
|
||||
@@ -337,8 +339,10 @@ function ChatApp() {
|
||||
<ChatImagesListener />
|
||||
</>
|
||||
),
|
||||
messageActions: (props) => <BrowserMessageActions {...props} />,
|
||||
inputToolbar: (props) => <AutomationModeInputToolbar {...props} />,
|
||||
promptExtras: () => <BrowserContextLoader />,
|
||||
onLogin: login,
|
||||
}}
|
||||
/>
|
||||
</InterventionModeProvider>
|
||||
|
||||
@@ -12,16 +12,17 @@ import React, { useCallback, useMemo } from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { chromeStorageAdapter } from "../../hooks";
|
||||
import { createAIProvider } from "../../lib/ai-provider";
|
||||
import { McpBridgePanel } from "./mcp-bridge-panel";
|
||||
import { SkillsOptionsTab } from "./skills-tab";
|
||||
|
||||
/** Parse and validate URL params for deep-linking. */
|
||||
function parseUrlParams() {
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const tabAllowlist = new Set(["general", "ai", "skills"]);
|
||||
const tabAllowlist = new Set(["general", "ai", "skills", "connection"]);
|
||||
const rawTab = params.get("tab");
|
||||
const tab =
|
||||
rawTab && tabAllowlist.has(rawTab)
|
||||
? (rawTab as "general" | "ai" | "skills")
|
||||
? (rawTab as "general" | "ai" | "skills" | "connection")
|
||||
: undefined;
|
||||
const rawSkill = params.get("skill");
|
||||
// Bound skill name length to prevent abuse
|
||||
@@ -77,16 +78,15 @@ function OptionsPageContent() {
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-background">
|
||||
<SettingsPage
|
||||
storageAdapter={chromeStorageAdapter}
|
||||
onTestConnection={handleTestConnection}
|
||||
skillsContent={<SkillsOptionsTab initialSkill={initialSkill} />}
|
||||
sttConfig={chromeSttAdapter}
|
||||
initialTab={initialTab}
|
||||
initialSkill={initialSkill}
|
||||
/>
|
||||
</div>
|
||||
<SettingsPage
|
||||
storageAdapter={chromeStorageAdapter}
|
||||
onTestConnection={handleTestConnection}
|
||||
skillsContent={<SkillsOptionsTab initialSkill={initialSkill} />}
|
||||
connectionContent={<McpBridgePanel />}
|
||||
sttConfig={chromeSttAdapter}
|
||||
initialTab={initialTab}
|
||||
initialSkill={initialSkill}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
150
packages/browser-ext/src/pages/options/mcp-bridge-panel.tsx
Normal file
150
packages/browser-ext/src/pages/options/mcp-bridge-panel.tsx
Normal file
@@ -0,0 +1,150 @@
|
||||
/**
|
||||
* MCP WebSocket Bridge Panel
|
||||
* UI for connecting/disconnecting the extension to the aipex-mcp-bridge.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import type { WsMcpServerState } from "@aipexstudio/browser-runtime";
|
||||
|
||||
const DEFAULT_URL = "ws://localhost:9223";
|
||||
|
||||
type ConnectionStatus = WsMcpServerState["status"];
|
||||
|
||||
export function McpBridgePanel() {
|
||||
const [url, setUrl] = useState(DEFAULT_URL);
|
||||
const [status, setStatus] = useState<ConnectionStatus>("disconnected");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [connectedAt, setConnectedAt] = useState<number | null>(null);
|
||||
const [reconnectAttempt, setReconnectAttempt] = useState(0);
|
||||
|
||||
const refreshStatus = useCallback(async () => {
|
||||
try {
|
||||
const state = await chrome.runtime.sendMessage({
|
||||
request: "ws-bridge-status",
|
||||
});
|
||||
if (state) {
|
||||
setStatus(state.status);
|
||||
setError(state.error);
|
||||
setConnectedAt(state.connectedAt);
|
||||
setReconnectAttempt(state.reconnectAttempt);
|
||||
if (state.url) setUrl(state.url);
|
||||
}
|
||||
} catch {
|
||||
// Background may not be ready
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
refreshStatus();
|
||||
const interval = setInterval(refreshStatus, 3000);
|
||||
return () => clearInterval(interval);
|
||||
}, [refreshStatus]);
|
||||
|
||||
const handleConnect = async () => {
|
||||
setError(null);
|
||||
try {
|
||||
const response = await chrome.runtime.sendMessage({
|
||||
request: "ws-bridge-connect",
|
||||
url,
|
||||
});
|
||||
if (!response.success) {
|
||||
setError(response.error || "Connection failed");
|
||||
}
|
||||
await refreshStatus();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
};
|
||||
|
||||
const handleDisconnect = async () => {
|
||||
try {
|
||||
await chrome.runtime.sendMessage({ request: "ws-bridge-disconnect" });
|
||||
await refreshStatus();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
};
|
||||
|
||||
const statusColor = {
|
||||
disconnected: "bg-gray-400",
|
||||
connecting: "bg-yellow-400 animate-pulse",
|
||||
connected: "bg-green-500",
|
||||
error: "bg-red-500",
|
||||
}[status];
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border bg-card p-6 shadow-sm">
|
||||
<h3 className="text-lg font-semibold mb-4">
|
||||
MCP WebSocket Bridge
|
||||
</h3>
|
||||
<p className="text-sm text-muted-foreground mb-4">
|
||||
Connect to an external MCP client (e.g. Claude, Cursor) via the
|
||||
aipex-mcp-bridge. The bridge exposes AIPex browser tools to external AI
|
||||
agents.
|
||||
</p>
|
||||
|
||||
{/* Status indicator */}
|
||||
<div className="flex items-center gap-2 mb-4">
|
||||
<span className={`inline-block h-3 w-3 rounded-full ${statusColor}`} />
|
||||
<span className="text-sm font-medium capitalize">{status}</span>
|
||||
{connectedAt && status === "connected" && (
|
||||
<span className="text-xs text-muted-foreground ml-2">
|
||||
since {new Date(connectedAt).toLocaleTimeString()}
|
||||
</span>
|
||||
)}
|
||||
{reconnectAttempt > 0 && status !== "connected" && (
|
||||
<span className="text-xs text-muted-foreground ml-2">
|
||||
(reconnect attempt {reconnectAttempt})
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* URL input */}
|
||||
<div className="flex gap-2 mb-4">
|
||||
<input
|
||||
type="text"
|
||||
value={url}
|
||||
onChange={(e) => setUrl(e.target.value)}
|
||||
placeholder="ws://localhost:9223"
|
||||
disabled={status === "connected" || status === "connecting"}
|
||||
className="flex-1 rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50"
|
||||
/>
|
||||
{status === "connected" ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleDisconnect}
|
||||
className="inline-flex items-center rounded-md bg-destructive px-4 py-2 text-sm font-medium text-destructive-foreground hover:bg-destructive/90"
|
||||
>
|
||||
Disconnect
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleConnect}
|
||||
disabled={status === "connecting" || !url.trim()}
|
||||
className="inline-flex items-center rounded-md bg-primary px-4 py-2 text-sm font-medium text-primary-foreground hover:bg-primary/90 disabled:pointer-events-none disabled:opacity-50"
|
||||
>
|
||||
{status === "connecting" ? "Connecting..." : "Connect"}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Error display */}
|
||||
{error && (
|
||||
<div className="rounded-md bg-destructive/10 border border-destructive/20 p-3 text-sm text-destructive">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Info */}
|
||||
<div className="mt-4 text-xs text-muted-foreground space-y-1">
|
||||
<p>
|
||||
The bridge exposes <code>tools/list</code> and{" "}
|
||||
<code>tools/call</code> over the MCP protocol, allowing external
|
||||
agents to use AIPex browser automation tools.
|
||||
</p>
|
||||
<p>Only localhost connections (127.0.0.1, ::1) are allowed.</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -37,3 +37,5 @@ export * from "./storage/index.js";
|
||||
export * from "./tools/index.js";
|
||||
// Voice
|
||||
// export * from "./voice/index.js";
|
||||
// WebSocket MCP Bridge
|
||||
export * from "./ws-bridge/index.js";
|
||||
|
||||
@@ -401,7 +401,7 @@ export class SkillStorage {
|
||||
// Check if skills directory exists
|
||||
const skillsDirExists = await zenfs.exists(skillsPath);
|
||||
if (!skillsDirExists) {
|
||||
console.log("[SkillStorage] /skills directory does not exist yet");
|
||||
console.debug("[SkillStorage] /skills directory does not exist yet");
|
||||
return skills;
|
||||
}
|
||||
|
||||
@@ -445,7 +445,7 @@ export class SkillStorage {
|
||||
enabled: true,
|
||||
});
|
||||
|
||||
console.log(`[SkillStorage] Found skill in ZenFS: ${entry}`);
|
||||
console.debug(`[SkillStorage] Found skill in ZenFS: ${entry}`);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[SkillStorage] Failed to process skill ${entry}:`,
|
||||
@@ -494,7 +494,7 @@ export class SkillStorage {
|
||||
const stats = { added: 0, skipped: 0, failed: 0 };
|
||||
|
||||
try {
|
||||
console.log("[SkillStorage] Starting sync from ZenFS...");
|
||||
console.debug("[SkillStorage] Starting sync from ZenFS...");
|
||||
|
||||
// Scan ZenFS for all skills
|
||||
const zenfsSkills = await this.scanZenFSForSkills();
|
||||
@@ -528,7 +528,7 @@ export class SkillStorage {
|
||||
try {
|
||||
await this.saveToIndexedDB(skill);
|
||||
stats.added++;
|
||||
console.log(`[SkillStorage] Added skill from ZenFS: ${skill.name}`);
|
||||
console.debug(`[SkillStorage] Added skill from ZenFS: ${skill.name}`);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[SkillStorage] Failed to add skill ${skill.name}:`,
|
||||
@@ -543,7 +543,7 @@ export class SkillStorage {
|
||||
// Update last sync time
|
||||
this.lastSyncTime = Date.now();
|
||||
|
||||
console.log(
|
||||
console.debug(
|
||||
`[SkillStorage] Sync completed: ${stats.added} added, ${stats.skipped} skipped, ${stats.failed} failed`,
|
||||
);
|
||||
} catch (error) {
|
||||
|
||||
@@ -52,8 +52,16 @@ async function compressImage(
|
||||
|
||||
export const captureScreenshotTool = tool({
|
||||
name: "capture_screenshot",
|
||||
description:
|
||||
"Capture screenshot of current visible tab and return as base64 data URL. When sendToLLM=true, the screenshot will be sent to the LLM for visual analysis AND visual coordinate tools (computer) will be unlocked for subsequent interactions. NOTE: This tool requires focus mode.",
|
||||
description: `[HIGH-COST FALLBACK] Capture screenshot of current visible tab.
|
||||
|
||||
TRY search_elements FIRST: For most interactions (clicking, filling, reading), use search_elements + UID-based tools. They are faster and don't send images to LLM.
|
||||
|
||||
USE THIS ONLY WHEN:
|
||||
- search_elements cannot find the target after 2 query attempts
|
||||
- You need to see visual layout, images, charts, or canvas content
|
||||
- The page uses non-standard rendering that snapshots miss
|
||||
|
||||
When sendToLLM=true: Sends image to LLM (higher latency/cost, may capture sensitive on-screen data) and enables the computer tool for coordinate-based actions. NOTE: This tool requires focus mode.`,
|
||||
parameters: z.object({
|
||||
sendToLLM: z
|
||||
.boolean()
|
||||
@@ -61,7 +69,7 @@ export const captureScreenshotTool = tool({
|
||||
.optional()
|
||||
.default(false)
|
||||
.describe(
|
||||
"Whether to send the screenshot to LLM for visual analysis. When true, visual coordinate tools will be enabled.",
|
||||
"Whether to send the screenshot to LLM for visual analysis. When true, enables computer tool for coordinate actions. Use sparingly - adds latency and token cost.",
|
||||
),
|
||||
}),
|
||||
execute: async ({ sendToLLM = false }) => {
|
||||
@@ -202,8 +210,16 @@ export const captureScreenshotTool = tool({
|
||||
|
||||
export const captureTabScreenshotTool = tool({
|
||||
name: "capture_tab_screenshot",
|
||||
description:
|
||||
"Capture screenshot of a specific tab by ID. When sendToLLM=true, the screenshot will be sent to the LLM for visual analysis AND visual coordinate tools (computer) will be unlocked for subsequent interactions. NOTE: This tool requires focus mode.",
|
||||
description: `[HIGH-COST FALLBACK] Capture screenshot of a specific tab by ID.
|
||||
|
||||
TRY search_elements FIRST: Visual verification is expensive. Use search_elements + UID-based tools for most interactions.
|
||||
|
||||
USE THIS ONLY WHEN:
|
||||
- Visual verification is essential
|
||||
- search_elements failed to find the target
|
||||
- You need to see images, charts, or canvas content
|
||||
|
||||
When sendToLLM=true: Sends image to LLM (higher latency/cost) and enables coordinate-based actions. NOTE: This tool requires focus mode.`,
|
||||
parameters: z.object({
|
||||
tabId: z.number().describe("The tab ID to capture"),
|
||||
sendToLLM: z
|
||||
@@ -341,8 +357,9 @@ const MAX_SELECTOR_LENGTH = 500;
|
||||
|
||||
export const captureScreenshotWithHighlightTool = tool({
|
||||
name: "capture_screenshot_with_highlight",
|
||||
description:
|
||||
"Capture screenshot of the current visible tab, optionally highlighting and cropping to a specific element identified by CSS selector. The screenshot is always sent to the LLM for visual analysis. NOTE: This tool requires focus mode.",
|
||||
description: `[HIGH-COST] Capture screenshot of the current visible tab, optionally highlighting and cropping to a specific element identified by CSS selector. The screenshot is always sent to the LLM for visual analysis.
|
||||
|
||||
PREFER search_elements for finding/interacting with elements. Use this only when you need to visually verify element appearance or layout. NOTE: This tool requires focus mode.`,
|
||||
parameters: z.object({
|
||||
selector: z
|
||||
.string()
|
||||
|
||||
13
packages/browser-runtime/src/ws-bridge/index.ts
Normal file
13
packages/browser-runtime/src/ws-bridge/index.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
export {
|
||||
wsMcpServer,
|
||||
WsMcpServer,
|
||||
type ConnectionStatus,
|
||||
type WsMcpServerState,
|
||||
} from "./ws-mcp-server.js";
|
||||
export {
|
||||
WebSocketClientTransport,
|
||||
type JSONRPCMessage,
|
||||
type JSONRPCRequest,
|
||||
type JSONRPCResponse,
|
||||
type JSONRPCNotification,
|
||||
} from "./ws-transport.js";
|
||||
466
packages/browser-runtime/src/ws-bridge/ws-mcp-server.ts
Normal file
466
packages/browser-runtime/src/ws-bridge/ws-mcp-server.ts
Normal file
@@ -0,0 +1,466 @@
|
||||
/**
|
||||
* WebSocket command executor for the AIPex extension.
|
||||
*
|
||||
* Connects as a WebSocket client to the aipex-mcp-bridge and listens
|
||||
* for tool execution commands. The bridge is the true MCP server — this
|
||||
* class simply executes tools and returns results.
|
||||
*
|
||||
* Handles:
|
||||
* - tools/call -- executes a tool from allBrowserTools
|
||||
* - ping -- responds with {} for keepalive
|
||||
*
|
||||
* No MCP protocol negotiation (initialize, tools/list) is needed on this
|
||||
* side — the bridge handles that entirely with static tool schemas.
|
||||
*/
|
||||
|
||||
import type { FunctionTool } from "@aipexstudio/aipex-core";
|
||||
import { allBrowserTools } from "../tools/index.js";
|
||||
import {
|
||||
WebSocketClientTransport,
|
||||
type JSONRPCMessage,
|
||||
type JSONRPCRequest,
|
||||
} from "./ws-transport.js";
|
||||
|
||||
export type ConnectionStatus =
|
||||
| "disconnected"
|
||||
| "connecting"
|
||||
| "connected"
|
||||
| "error";
|
||||
|
||||
export interface WsMcpServerState {
|
||||
status: ConnectionStatus;
|
||||
url: string | null;
|
||||
error: string | null;
|
||||
connectedAt: number | null;
|
||||
reconnectAttempt: number;
|
||||
}
|
||||
|
||||
type StatusListener = (state: WsMcpServerState) => void;
|
||||
|
||||
const KEEPALIVE_ALARM_NAME = "ws-mcp-keepalive";
|
||||
const KEEPALIVE_INTERVAL_MINUTES = 0.4;
|
||||
const TOOL_CALL_TIMEOUT_MS = 60_000;
|
||||
|
||||
const STORAGE_KEY_WS_URL = "ws-mcp-url";
|
||||
|
||||
function getReconnectDelayMs(attempt: number): number {
|
||||
const withJitter = (base: number) =>
|
||||
Math.round(base * (0.7 + Math.random() * 0.6));
|
||||
return withJitter(Math.min(500 * Math.pow(2, attempt), 10_000));
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a tool by name from the registered browser tools.
|
||||
*/
|
||||
function findBrowserTool(name: string): FunctionTool | undefined {
|
||||
return allBrowserTools.find((t) => t.name === name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a tool execution result into MCP content blocks.
|
||||
*/
|
||||
function buildMcpContent(
|
||||
data: unknown,
|
||||
): Array<{ type: string; text?: string; data?: string; mimeType?: string }> {
|
||||
if (data === null || data === undefined) {
|
||||
return [{ type: "text", text: "null" }];
|
||||
}
|
||||
|
||||
if (typeof data === "string") {
|
||||
// Check if it's a base64 data URL (screenshot)
|
||||
if (data.startsWith("data:image/")) {
|
||||
const commaIndex = data.indexOf(",");
|
||||
if (commaIndex > 0) {
|
||||
const mimeType = data.slice(5, data.indexOf(";"));
|
||||
const base64Data = data.slice(commaIndex + 1);
|
||||
return [{ type: "image", data: base64Data, mimeType }];
|
||||
}
|
||||
}
|
||||
return [{ type: "text", text: data }];
|
||||
}
|
||||
|
||||
return [{ type: "text", text: JSON.stringify(data, null, 2) }];
|
||||
}
|
||||
|
||||
export class WsMcpServer {
|
||||
private transport: WebSocketClientTransport | null = null;
|
||||
private state: WsMcpServerState = {
|
||||
status: "disconnected",
|
||||
url: null,
|
||||
error: null,
|
||||
connectedAt: null,
|
||||
reconnectAttempt: 0,
|
||||
};
|
||||
private listeners: Set<StatusListener> = new Set();
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private autoReconnectEnabled = true;
|
||||
|
||||
private validateUrl(url: string): void {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error(`Invalid WebSocket URL: ${url}`);
|
||||
}
|
||||
|
||||
const hostname = parsed.hostname;
|
||||
const allowedHosts = ["localhost", "127.0.0.1", "[::1]", "::1"];
|
||||
if (!allowedHosts.includes(hostname)) {
|
||||
throw new Error(
|
||||
`Only localhost connections are allowed. Got: ${hostname}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "ws:" && parsed.protocol !== "wss:") {
|
||||
throw new Error(
|
||||
`URL must use ws:// or wss:// protocol. Got: ${parsed.protocol}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async connect(url: string): Promise<void> {
|
||||
this.validateUrl(url);
|
||||
this.cancelReconnect();
|
||||
|
||||
if (
|
||||
this.state.status === "connected" ||
|
||||
this.state.status === "connecting"
|
||||
) {
|
||||
await this.disconnect();
|
||||
}
|
||||
|
||||
this.updateState({
|
||||
status: "connecting",
|
||||
url,
|
||||
error: null,
|
||||
connectedAt: null,
|
||||
});
|
||||
|
||||
try {
|
||||
const transport = new WebSocketClientTransport(url);
|
||||
this.transport = transport;
|
||||
|
||||
transport.onclose = () => {
|
||||
this.handleDisconnect();
|
||||
};
|
||||
|
||||
transport.onerror = (error: Error) => {
|
||||
console.error("[WsMcpServer] Transport error:", error.message);
|
||||
};
|
||||
|
||||
transport.onmessage = (message: JSONRPCMessage) => {
|
||||
this.handleMessage(message);
|
||||
};
|
||||
|
||||
await transport.start();
|
||||
this.updateState({
|
||||
status: "connected",
|
||||
connectedAt: Date.now(),
|
||||
reconnectAttempt: 0,
|
||||
});
|
||||
this.startKeepalive();
|
||||
this.persistUrl(url);
|
||||
this.autoReconnectEnabled = true;
|
||||
console.log(`[WsMcpServer] Connected to ${url}`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
this.updateState({ status: "error", error: message, connectedAt: null });
|
||||
this.transport = null;
|
||||
this.scheduleReconnect(url);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
this.autoReconnectEnabled = false;
|
||||
this.cancelReconnect();
|
||||
this.stopKeepalive();
|
||||
|
||||
if (this.transport) {
|
||||
try {
|
||||
await this.transport.close();
|
||||
} catch {
|
||||
// ignore close errors
|
||||
}
|
||||
this.transport = null;
|
||||
}
|
||||
|
||||
this.updateState({
|
||||
status: "disconnected",
|
||||
url: null,
|
||||
error: null,
|
||||
connectedAt: null,
|
||||
reconnectAttempt: 0,
|
||||
});
|
||||
this.clearPersistedUrl();
|
||||
console.log("[WsMcpServer] Disconnected");
|
||||
}
|
||||
|
||||
isConnected(): boolean {
|
||||
return this.state.status === "connected";
|
||||
}
|
||||
|
||||
getStatus(): WsMcpServerState {
|
||||
return { ...this.state };
|
||||
}
|
||||
|
||||
onStatusChange(listener: StatusListener): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
async getSavedUrl(): Promise<string | null> {
|
||||
try {
|
||||
const result = await chrome.storage.local.get(STORAGE_KEY_WS_URL);
|
||||
return (result[STORAGE_KEY_WS_URL] as string) || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// -- Auto-reconnect with exponential backoff --
|
||||
|
||||
private scheduleReconnect(url: string): void {
|
||||
if (!this.autoReconnectEnabled) return;
|
||||
this.cancelReconnect();
|
||||
|
||||
const attempt = this.state.reconnectAttempt;
|
||||
const delay = getReconnectDelayMs(attempt);
|
||||
console.log(
|
||||
`[WsMcpServer] Scheduling reconnect attempt ${attempt + 1} in ${delay}ms`,
|
||||
);
|
||||
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
this.updateState({ reconnectAttempt: attempt + 1 });
|
||||
this.connect(url).catch(() => {
|
||||
// connect() itself schedules next retry on failure
|
||||
});
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private cancelReconnect(): void {
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private persistUrl(url: string): void {
|
||||
try {
|
||||
chrome.storage.local.set({ [STORAGE_KEY_WS_URL]: url });
|
||||
} catch {
|
||||
// ignore storage errors
|
||||
}
|
||||
}
|
||||
|
||||
private clearPersistedUrl(): void {
|
||||
try {
|
||||
chrome.storage.local.remove(STORAGE_KEY_WS_URL);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
// -- Message handling --
|
||||
|
||||
private handleMessage(message: JSONRPCMessage): void {
|
||||
if (!("method" in message) || !("id" in message)) {
|
||||
return;
|
||||
}
|
||||
const request = message as JSONRPCRequest;
|
||||
this.handleRequest(request).catch((error) => {
|
||||
console.error("[WsMcpServer] Unhandled error processing request:", error);
|
||||
this.sendError(request.id, -32603, "Internal error");
|
||||
});
|
||||
}
|
||||
|
||||
private async handleRequest(request: JSONRPCRequest): Promise<void> {
|
||||
switch (request.method) {
|
||||
case "tools/call":
|
||||
return this.handleToolsCall(request);
|
||||
case "ping":
|
||||
return this.sendResult(request.id, {});
|
||||
default:
|
||||
return this.sendError(
|
||||
request.id,
|
||||
-32601,
|
||||
`Method not found: ${request.method}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async handleToolsCall(request: JSONRPCRequest): Promise<void> {
|
||||
const params = (request.params || {}) as Record<string, unknown>;
|
||||
const name = params.name as string | undefined;
|
||||
const args = (params.arguments || {}) as Record<string, unknown>;
|
||||
|
||||
if (!name) {
|
||||
return this.sendError(
|
||||
request.id,
|
||||
-32602,
|
||||
"Missing required parameter: name",
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const toolExecution = this.executeTool(name, args);
|
||||
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error(
|
||||
`Tool '${name}' timed out after ${TOOL_CALL_TIMEOUT_MS}ms`,
|
||||
),
|
||||
),
|
||||
TOOL_CALL_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
|
||||
const result = await Promise.race([toolExecution, timeoutPromise]);
|
||||
|
||||
await this.sendResult(request.id, {
|
||||
content: buildMcpContent(result),
|
||||
});
|
||||
} catch (error) {
|
||||
await this.sendResult(request.id, {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `Error: ${error instanceof Error ? error.message : String(error)}`,
|
||||
},
|
||||
],
|
||||
isError: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async executeTool(
|
||||
name: string,
|
||||
args: Record<string, unknown>,
|
||||
): Promise<unknown> {
|
||||
const browserTool = findBrowserTool(name);
|
||||
if (!browserTool) {
|
||||
throw new Error(`Tool not found: ${name}`);
|
||||
}
|
||||
|
||||
// FunctionTool.invoke(runContext, inputJsonString, details?)
|
||||
// We pass an empty object as RunContext since we're outside the agent loop.
|
||||
return await (browserTool as any).invoke(
|
||||
{} as any,
|
||||
JSON.stringify(args),
|
||||
);
|
||||
}
|
||||
|
||||
private async sendResult(
|
||||
id: string | number,
|
||||
result: unknown,
|
||||
): Promise<void> {
|
||||
if (!this.transport) return;
|
||||
await this.transport.send({
|
||||
jsonrpc: "2.0",
|
||||
id,
|
||||
result,
|
||||
} as JSONRPCMessage);
|
||||
}
|
||||
|
||||
private async sendError(
|
||||
id: string | number,
|
||||
code: number,
|
||||
message: string,
|
||||
): Promise<void> {
|
||||
if (!this.transport) return;
|
||||
await this.transport.send({
|
||||
jsonrpc: "2.0",
|
||||
id,
|
||||
error: { code, message },
|
||||
} as JSONRPCMessage);
|
||||
}
|
||||
|
||||
// -- Connection lifecycle --
|
||||
|
||||
private handleDisconnect(): void {
|
||||
this.stopKeepalive();
|
||||
const lastUrl = this.state.url;
|
||||
this.transport = null;
|
||||
if (this.state.status !== "disconnected") {
|
||||
this.updateState({
|
||||
status: "disconnected",
|
||||
error: null,
|
||||
connectedAt: null,
|
||||
});
|
||||
console.log("[WsMcpServer] Connection closed by remote");
|
||||
if (lastUrl && this.autoReconnectEnabled) {
|
||||
this.scheduleReconnect(lastUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private updateState(partial: Partial<WsMcpServerState>): void {
|
||||
this.state = { ...this.state, ...partial };
|
||||
for (const listener of this.listeners) {
|
||||
try {
|
||||
listener(this.getStatus());
|
||||
} catch {
|
||||
// don't let listener errors break us
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -- Service Worker Keepalive --
|
||||
|
||||
private startKeepalive(): void {
|
||||
try {
|
||||
chrome.alarms.create(KEEPALIVE_ALARM_NAME, {
|
||||
periodInMinutes: KEEPALIVE_INTERVAL_MINUTES,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("[WsMcpServer] Failed to create keepalive alarm:", error);
|
||||
}
|
||||
}
|
||||
|
||||
private stopKeepalive(): void {
|
||||
try {
|
||||
chrome.alarms.clear(KEEPALIVE_ALARM_NAME);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
handleAlarm(alarm: chrome.alarms.Alarm): void {
|
||||
if (alarm.name !== KEEPALIVE_ALARM_NAME) return;
|
||||
|
||||
if (this.state.status === "connected" && this.transport?.isOpen) {
|
||||
this.sendPing();
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.state.status === "connecting") {
|
||||
return;
|
||||
}
|
||||
|
||||
const url = this.state.url;
|
||||
if (url && this.autoReconnectEnabled) {
|
||||
this.connect(url).catch(() => {
|
||||
// connect() schedules its own retry on failure
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async sendPing(): Promise<void> {
|
||||
if (!this.transport?.isOpen) return;
|
||||
try {
|
||||
await this.transport.send({
|
||||
jsonrpc: "2.0",
|
||||
id: `ping-${Date.now()}`,
|
||||
method: "ping",
|
||||
} as JSONRPCMessage);
|
||||
} catch {
|
||||
console.warn("[WsMcpServer] Ping send failed, closing connection");
|
||||
this.handleDisconnect();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const wsMcpServer = new WsMcpServer();
|
||||
129
packages/browser-runtime/src/ws-bridge/ws-transport.ts
Normal file
129
packages/browser-runtime/src/ws-bridge/ws-transport.ts
Normal file
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* WebSocket client transport for MCP-over-WebSocket.
|
||||
*
|
||||
* Uses the browser-native WebSocket API. AIPex connects as a WebSocket
|
||||
* client to the host agent's WebSocket server, then JSON-RPC 2.0 messages
|
||||
* flow bidirectionally over that connection.
|
||||
*
|
||||
* No dependency on @modelcontextprotocol/sdk to avoid pulling ajv
|
||||
* (which uses `new Function()`) into the MV3 service worker bundle.
|
||||
*/
|
||||
|
||||
export interface JSONRPCRequest {
|
||||
jsonrpc: "2.0";
|
||||
id: string | number;
|
||||
method: string;
|
||||
params?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface JSONRPCResponse {
|
||||
jsonrpc: "2.0";
|
||||
id: string | number;
|
||||
result?: unknown;
|
||||
error?: { code: number; message: string; data?: unknown };
|
||||
}
|
||||
|
||||
export interface JSONRPCNotification {
|
||||
jsonrpc: "2.0";
|
||||
method: string;
|
||||
params?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type JSONRPCMessage =
|
||||
| JSONRPCRequest
|
||||
| JSONRPCResponse
|
||||
| JSONRPCNotification;
|
||||
|
||||
function isValidJsonRpcMessage(data: unknown): data is JSONRPCMessage {
|
||||
if (typeof data !== "object" || data === null) return false;
|
||||
const msg = data as Record<string, unknown>;
|
||||
return msg.jsonrpc === "2.0";
|
||||
}
|
||||
|
||||
const WS_CLOSE_NORMAL = 1000;
|
||||
|
||||
export class WebSocketClientTransport {
|
||||
private socket: WebSocket | null = null;
|
||||
private url: string;
|
||||
private closeFired = false;
|
||||
|
||||
onmessage?: (message: JSONRPCMessage) => void;
|
||||
onerror?: (error: Error) => void;
|
||||
onclose?: () => void;
|
||||
|
||||
constructor(url: string) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
start(): Promise<void> {
|
||||
if (this.socket) {
|
||||
throw new Error("WebSocketClientTransport already started");
|
||||
}
|
||||
|
||||
this.closeFired = false;
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const ws = new WebSocket(this.url);
|
||||
this.socket = ws;
|
||||
|
||||
ws.onopen = () => {
|
||||
resolve();
|
||||
};
|
||||
|
||||
ws.onerror = () => {
|
||||
const error = new Error("WebSocket error");
|
||||
reject(error);
|
||||
this.onerror?.(error);
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
this.socket = null;
|
||||
if (!this.closeFired) {
|
||||
this.closeFired = true;
|
||||
this.onclose?.();
|
||||
}
|
||||
};
|
||||
|
||||
ws.onmessage = (event: MessageEvent) => {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(String(event.data));
|
||||
} catch (error) {
|
||||
this.onerror?.(
|
||||
error instanceof Error ? error : new Error(String(error)),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!isValidJsonRpcMessage(parsed)) {
|
||||
this.onerror?.(new Error("Received non-JSON-RPC message"));
|
||||
return;
|
||||
}
|
||||
this.onmessage?.(parsed);
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async send(message: JSONRPCMessage): Promise<void> {
|
||||
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("WebSocket not connected");
|
||||
}
|
||||
this.socket.send(JSON.stringify(message));
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
const ws = this.socket;
|
||||
if (!ws) return;
|
||||
|
||||
this.socket = null;
|
||||
if (
|
||||
ws.readyState === WebSocket.OPEN ||
|
||||
ws.readyState === WebSocket.CONNECTING
|
||||
) {
|
||||
ws.close(WS_CLOSE_NORMAL);
|
||||
}
|
||||
}
|
||||
|
||||
get isOpen(): boolean {
|
||||
return this.socket?.readyState === WebSocket.OPEN;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user