Feature adjust (#172)

This commit is contained in:
ropzislaw
2026-03-08 15:56:20 +08:00
committed by GitHub
35 changed files with 1195 additions and 252 deletions

View File

@@ -140,6 +140,14 @@ export class ChatAdapter {
break;
case "content_delta":
// When text arrives after tool calls, start a new assistant message.
// This mirrors aipex's behavior where each model response after tool
// execution becomes a separate message, enabling the turn-based
// collapsing logic in the message list.
if (this.toolsAddedSinceLastText) {
this.state.currentAssistantMessageId = null;
this.toolsAddedSinceLastText = false;
}
this.ensureAssistantMessage();
this.updateStatus("streaming");
this.appendContentDelta(event.delta);
@@ -179,11 +187,13 @@ export class ChatAdapter {
case "execution_complete":
this.updateStatus("idle");
this.state.currentAssistantMessageId = null;
this.toolsAddedSinceLastText = false;
break;
case "error":
this.updateStatus("error");
this.state.currentAssistantMessageId = null;
this.toolsAddedSinceLastText = false;
break;
}
}

View File

@@ -257,6 +257,7 @@ export function DefaultMessageItem({
{message.metadata.needLogin && (
<LoginPrompt
showByokOption
onLogin={slots.onLogin}
onOpenSettings={() => chrome.runtime?.openOptionsPage?.()}
/>
)}

View File

@@ -22,6 +22,7 @@ import {
Mic,
Package,
Palette,
Plug,
Plus,
Search,
Settings,
@@ -221,6 +222,7 @@ export function SettingsPage({
onSave,
onTestConnection,
skillsContent,
connectionContent,
sttConfig,
initialTab,
initialSkill: _initialSkill,
@@ -826,9 +828,14 @@ export function SettingsPage({
className="w-full"
>
<TabsList
className={`grid w-full mb-6 ${
skillsContent ? "grid-cols-3" : "grid-cols-2"
}`}
className={cn(
"grid w-full mb-6",
skillsContent && connectionContent
? "grid-cols-4"
: skillsContent || connectionContent
? "grid-cols-3"
: "grid-cols-2",
)}
>
<TabsTrigger value="general" className="flex items-center gap-2">
<Settings className="h-4 w-4" />
@@ -844,6 +851,15 @@ export function SettingsPage({
{t("settings.skillsTab")}
</TabsTrigger>
)}
{connectionContent && (
<TabsTrigger
value="connection"
className="flex items-center gap-2"
>
<Plug className="h-4 w-4" />
{language === "zh" ? "连接" : "Connection"}
</TabsTrigger>
)}
</TabsList>
{/* General Tab */}
@@ -1680,6 +1696,13 @@ export function SettingsPage({
{skillsContent}
</TabsContent>
)}
{/* Connection Tab */}
{connectionContent && (
<TabsContent value="connection" className="space-y-6">
{connectionContent}
</TabsContent>
)}
</Tabs>
</div>
</div>

View File

@@ -22,6 +22,8 @@ export interface SettingsPageProps {
onSave?: (settings: AppSettings) => void;
onTestConnection?: (settings: AppSettings) => Promise<boolean>;
skillsContent?: ReactNode;
/** Optional content for a "Connection" tab (e.g. MCP bridge panel). */
connectionContent?: ReactNode;
/** Optional ElevenLabs STT config adapter; when provided the STT card is shown. */
sttConfig?: STTConfigAdapter;
/** Pre-select a tab on mount (e.g. from URL params). */
@@ -38,7 +40,7 @@ export interface ProviderConfig {
export type ProviderConfigs = Record<AIProviderKey, ProviderConfig>;
export type SettingsTab = "general" | "ai" | "skills";
export type SettingsTab = "general" | "ai" | "skills" | "connection";
export interface SaveStatus {
type: "success" | "error" | "info" | "";

View File

@@ -83,25 +83,43 @@ export function useChatConfig(
});
const [isLoading, setIsLoading] = useState(autoLoad);
const applyStoredSettings = useCallback((stored: unknown) => {
setSettings((prev: AppSettings) => ({
...prev,
...(stored as Partial<AppSettings>),
customModels: (stored as AppSettings).customModels ?? [],
providerType: (stored as AppSettings).providerType ?? "openai",
providerEnabled: (stored as AppSettings).providerEnabled ?? false,
}));
}, []);
const loadSettings = useCallback(async () => {
setIsLoading(true);
try {
const stored = await storageAdapter.load(STORAGE_KEYS.SETTINGS);
if (stored) {
setSettings((prev: AppSettings) => ({
...prev,
...stored,
customModels: (stored as AppSettings).customModels ?? [],
providerType: (stored as AppSettings).providerType ?? "openai",
providerEnabled: (stored as AppSettings).providerEnabled ?? false,
}));
applyStoredSettings(stored);
}
} catch (error) {
console.error("Failed to load chat settings:", error);
} finally {
setIsLoading(false);
}
}, [storageAdapter]);
}, [storageAdapter, applyStoredSettings]);
// Silent reload that does NOT touch isLoading, so existing UI stays mounted.
// Used by the storage watcher to pick up changes (e.g. model switch)
// without unmounting ChatBot.
const reloadSettingsSilently = useCallback(async () => {
try {
const stored = await storageAdapter.load(STORAGE_KEYS.SETTINGS);
if (stored) {
applyStoredSettings(stored);
}
} catch (error) {
console.error("Failed to reload chat settings:", error);
}
}, [storageAdapter, applyStoredSettings]);
const saveSettings = useCallback(
async (newSettings: AppSettings) => {
@@ -119,16 +137,16 @@ export function useChatConfig(
void loadSettings();
}
// Set up storage change listener for real-time sync
// Set up storage change listener for real-time sync.
// Uses silent reload so the UI does not unmount (preserving chat state).
const unwatch = storageAdapter.watch(STORAGE_KEYS.SETTINGS, () => {
// Reload settings when storage changes (e.g., from settings page)
void loadSettings();
void reloadSettingsSilently();
});
return () => {
unwatch();
};
}, [autoLoad, loadSettings, storageAdapter]);
}, [autoLoad, loadSettings, reloadSettingsSilently, storageAdapter]);
const updateSetting = useCallback(
async <K extends keyof AppSettings>(

View File

@@ -91,9 +91,6 @@ export function useChat(
const [sessionId, setSessionId] = useState<string | null>(null);
const [metrics, setMetrics] = useState<AgentMetrics | null>(null);
// Cumulative session-level metrics (sum across all runs)
const cumulativeMetricsRef = useRef<AgentMetrics | null>(null);
// Refs for stable callbacks
const handlersRef = useRef(handlers);
handlersRef.current = handlers;
@@ -102,6 +99,18 @@ export function useChat(
configRef.current = config;
const activeGeneratorRef = useRef<AsyncGenerator<AgentEvent> | null>(null);
const prevAgentRef = useRef<AIPex | undefined>(agent);
// When the agent instance changes (e.g. model switch), reset the sessionId
// so subsequent messages create a fresh session on the new agent, but
// preserve existing UI messages so the conversation history stays visible.
useEffect(() => {
if (agent && prevAgentRef.current && agent !== prevAgentRef.current) {
setSessionId(null);
setMetrics(null);
}
prevAgentRef.current = agent;
}, [agent]);
// Create adapter with callbacks
const adapter = useMemo(() => {
@@ -156,24 +165,13 @@ export function useChat(
handlersRef.current?.onError?.(event.error);
}
// Handle metrics update accumulate across the session
// Handle metrics update show latest completion metrics (not cumulative)
if (event.type === "metrics_update") {
const prev = cumulativeMetricsRef.current;
const cumulative: AgentMetrics = {
tokensUsed: (prev?.tokensUsed ?? 0) + event.metrics.tokensUsed,
promptTokens:
(prev?.promptTokens ?? 0) + event.metrics.promptTokens,
completionTokens:
(prev?.completionTokens ?? 0) + event.metrics.completionTokens,
// Non-cumulative fields: use latest values
itemCount: event.metrics.itemCount,
maxTurns: event.metrics.maxTurns,
duration: (prev?.duration ?? 0) + event.metrics.duration,
startTime: prev?.startTime ?? event.metrics.startTime,
};
cumulativeMetricsRef.current = cumulative;
setMetrics(cumulative);
handlersRef.current?.onMetricsUpdate?.(cumulative, event.sessionId);
setMetrics(event.metrics);
handlersRef.current?.onMetricsUpdate?.(
event.metrics,
event.sessionId,
);
}
// Process the event through adapter
@@ -277,7 +275,6 @@ export function useChat(
activeGeneratorRef.current = null;
setSessionId(null);
setMetrics(null);
cumulativeMetricsRef.current = null;
adapter.reset(configRef.current?.initialMessages ?? []);
}, [adapter, agent, sessionId]);

View File

@@ -128,6 +128,8 @@ export interface ChatbotSlots {
afterMessages?: () => ReactNode;
/** Extra content rendered inside PromptInput (e.g., context/skill loaders) */
promptExtras?: () => ReactNode;
/** Handler called when user clicks Login in a LoginPrompt inside a message */
onLogin?: () => void;
}
// ============ Components Configuration ============

View File

@@ -81,6 +81,7 @@
"debugger",
"cookies",
"webNavigation",
"audioCapture"
"audioCapture",
"alarms"
]
}

View File

@@ -637,4 +637,87 @@ chrome.runtime.onMessageExternal.addListener(
},
);
// =============================================================================
// WebSocket MCP Bridge
// =============================================================================
import { wsMcpServer } from "@aipexstudio/browser-runtime";
// Handle MCP bridge messages
chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
if (message.request === "ws-bridge-connect") {
const url = message.url as string;
wsMcpServer
.connect(url)
.then(() => {
updateMcpBadge(true);
sendResponse({ success: true });
})
.catch((error) => {
sendResponse({
success: false,
error: error instanceof Error ? error.message : String(error),
});
});
return true;
}
if (message.request === "ws-bridge-disconnect") {
wsMcpServer
.disconnect()
.then(() => {
updateMcpBadge(false);
sendResponse({ success: true });
})
.catch((error) => {
sendResponse({
success: false,
error: error instanceof Error ? error.message : String(error),
});
});
return true;
}
if (message.request === "ws-bridge-status") {
sendResponse(wsMcpServer.getStatus());
return true;
}
return false;
});
// Update badge to show MCP connection status
function updateMcpBadge(connected: boolean) {
if (connected) {
chrome.action.setBadgeText({ text: "ON" });
chrome.action.setBadgeBackgroundColor({ color: "#22c55e" });
} else {
chrome.action.setBadgeText({ text: "" });
}
}
// Handle keepalive alarms for the WebSocket connection
chrome.alarms.onAlarm.addListener((alarm) => {
wsMcpServer.handleAlarm(alarm);
});
// Track MCP connection status for badge updates
wsMcpServer.onStatusChange((state) => {
updateMcpBadge(state.status === "connected");
});
// Auto-connect to saved URL on startup
wsMcpServer
.getSavedUrl()
.then((url) => {
if (url) {
console.log("[WsMcpServer] Auto-connecting to saved URL:", url);
wsMcpServer.connect(url).catch(() => {
// connect() handles its own retry logic
});
}
})
.catch(() => {
// Ignore storage errors on startup
});
console.log("AIPex background service worker started");

View File

@@ -5,30 +5,14 @@
import { useChatContext } from "@aipexstudio/aipex-react/components/chatbot";
import { Button } from "@aipexstudio/aipex-react/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@aipexstudio/aipex-react/components/ui/dropdown-menu";
import { useTranslation } from "@aipexstudio/aipex-react/i18n/context";
import { getRuntime } from "@aipexstudio/aipex-react/lib/runtime";
import { cn } from "@aipexstudio/aipex-react/lib/utils";
import type { HeaderProps } from "@aipexstudio/aipex-react/types";
import { conversationStorage } from "@aipexstudio/browser-runtime";
import {
KeyboardIcon,
MicIcon,
MoreHorizontalIcon,
PlusIcon,
SettingsIcon,
Share2Icon,
SparklesIcon,
} from "lucide-react";
import { KeyboardIcon, MicIcon, PlusIcon, SettingsIcon } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { UserProfile, useAuth } from "../auth";
import { shareConversation } from "../services/share-conversation";
import { ConversationHistory } from "./conversation-history";
import { useInputMode } from "./input-mode-context";
import { fromStorageFormat, toStorageFormat } from "./message-adapter";
@@ -43,7 +27,7 @@ export function BrowserChatHeader({
}: HeaderProps) {
const { t } = useTranslation();
const runtime = getRuntime();
const { messages, setMessages, interrupt, sendMessage } = useChatContext();
const { messages, setMessages, interrupt } = useChatContext();
const { user, login, isLoading: isAuthLoading } = useAuth();
const [currentConversationId, setCurrentConversationId] = useState<
@@ -149,34 +133,6 @@ export function BrowserChatHeader({
setInputMode(inputMode === "voice" ? "text" : "voice");
}, [inputMode, setInputMode]);
// Share conversation
const [isSharing, setIsSharing] = useState(false);
const handleShare = useCallback(async () => {
if (isSharing) return;
const nonSystemMessages = messages.filter((m) => m.role !== "system");
if (nonSystemMessages.length === 0) return;
setIsSharing(true);
try {
const { url } = await shareConversation(messages);
chrome.tabs.create({ url });
} catch (error) {
console.error(
"[Share] Failed:",
error instanceof Error ? error.message : String(error),
);
} finally {
setIsSharing(false);
}
}, [messages, isSharing]);
// Save as Skill — instructs the AI to create a skill from the conversation
const handleSaveAsSkill = useCallback(() => {
sendMessage("use skill-creator skill to save the conversation");
}, [sendMessage]);
return (
<div
className={cn(
@@ -224,39 +180,8 @@ export function BrowserChatHeader({
/>
</div>
{/* Right side - More menu, New Chat, User Profile */}
{/* Right side - New Chat, User Profile */}
<div className="flex items-center gap-1">
{/* More actions dropdown (Share, Save as Skill) */}
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="ghost" size="icon" className="size-8">
<MoreHorizontalIcon className="size-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-48">
<DropdownMenuItem
onClick={handleShare}
disabled={
isSharing ||
messages.filter((m) => m.role !== "system").length === 0
}
>
<Share2Icon className="mr-2 size-4" />
{isSharing ? "Sharing..." : "Share Conversation"}
</DropdownMenuItem>
<DropdownMenuSeparator />
<DropdownMenuItem
onClick={handleSaveAsSkill}
disabled={
messages.filter((m) => m.role !== "system").length === 0
}
>
<SparklesIcon className="mr-2 size-4" />
Save as Skill
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
<Button
variant="ghost"
size="sm"

View File

@@ -0,0 +1,90 @@
/**
* BrowserMessageActions
* Custom message actions for the browser extension.
* Renders Retry, Copy, Share Conversation, and Save as Skill inline
* with each last assistant message (matching aipex behavior).
*/
import {
Action,
Actions,
} from "@aipexstudio/aipex-react/components/ai-elements/actions";
import {
useChatContext,
useConfigContext,
} from "@aipexstudio/aipex-react/components/chatbot";
import type { MessageActionsSlotProps } from "@aipexstudio/aipex-react/types";
import { CopyIcon, PuzzleIcon, RefreshCcwIcon, Share2Icon } from "lucide-react";
import { useCallback, useState } from "react";
import { useAuth } from "../auth";
import { shareConversation } from "../services/share-conversation";
import { isByokConfigured } from "./ai-provider";
export function BrowserMessageActions({
message,
onRegenerate,
onCopy,
}: MessageActionsSlotProps) {
const { messages, sendMessage } = useChatContext();
const textContent = message.parts
.filter((p) => p.type === "text")
.map((p) => (p.type === "text" ? p.text : ""))
.join("\n");
const { user } = useAuth();
const { settings } = useConfigContext();
const [isSharing, setIsSharing] = useState(false);
const isByok = isByokConfigured(settings);
const handleShare = useCallback(async () => {
if (isSharing) return;
const nonSystemMessages = messages.filter((m) => m.role !== "system");
if (nonSystemMessages.length === 0) return;
setIsSharing(true);
try {
const { url } = await shareConversation(messages);
chrome.tabs.create({ url });
} catch (error) {
console.error(
"[Share] Failed:",
error instanceof Error ? error.message : String(error),
);
} finally {
setIsSharing(false);
}
}, [messages, isSharing]);
const handleSaveAsSkill = useCallback(() => {
sendMessage("use skill-creator skill to save the conversation");
}, [sendMessage]);
return (
<Actions className="mt-2">
{onCopy && textContent && (
<Action onClick={() => onCopy(textContent)} label="Copy">
<CopyIcon className="size-3" />
</Action>
)}
{!isByok && user && (
<Action
onClick={handleShare}
disabled={isSharing}
label={isSharing ? "Sharing..." : "Share"}
>
<Share2Icon className="size-3" />
</Action>
)}
<Action onClick={handleSaveAsSkill} label="Save as Skill">
<PuzzleIcon className="size-3" />
</Action>
{onRegenerate && (
<Action onClick={onRegenerate} label="Retry">
<RefreshCcwIcon className="size-3" />
</Action>
)}
</Actions>
);
}

View File

@@ -15,7 +15,7 @@ import type { AuthCheckResult } from "@aipexstudio/aipex-react/types";
import { ChromeStorageAdapter } from "@aipexstudio/browser-runtime";
import React, { useCallback, useEffect, useRef, useState } from "react";
import ReactDOM from "react-dom/client";
import { AuthProvider } from "../../auth";
import { AuthProvider, useAuth } from "../../auth";
import { chromeStorageAdapter } from "../../hooks";
import { isByokConfigured } from "../../lib/ai-provider";
import { AutomationModeInputToolbar } from "../../lib/automation-mode-toolbar";
@@ -29,6 +29,7 @@ import {
import { BrowserChatHeader } from "../../lib/browser-chat-header";
import { BrowserChatInputArea } from "../../lib/browser-chat-input-area";
import { BrowserContextLoader } from "../../lib/browser-context-loader";
import { BrowserMessageActions } from "../../lib/browser-message-actions";
import { BrowserMessageList } from "../../lib/browser-message-list";
import { ChatImagesListener } from "../../lib/chat-images-listener";
import { InputModeProvider } from "../../lib/input-mode-context";
@@ -242,6 +243,7 @@ function ChatApp() {
...BROWSER_AGENT_CONFIG,
});
const { login } = useAuth();
const pendingInput = usePendingPrompt();
const heartbeat = useConversationHeartbeat();
useReplaySetup();
@@ -337,8 +339,10 @@ function ChatApp() {
<ChatImagesListener />
</>
),
messageActions: (props) => <BrowserMessageActions {...props} />,
inputToolbar: (props) => <AutomationModeInputToolbar {...props} />,
promptExtras: () => <BrowserContextLoader />,
onLogin: login,
}}
/>
</InterventionModeProvider>

View File

@@ -12,16 +12,17 @@ import React, { useCallback, useMemo } from "react";
import ReactDOM from "react-dom/client";
import { chromeStorageAdapter } from "../../hooks";
import { createAIProvider } from "../../lib/ai-provider";
import { McpBridgePanel } from "./mcp-bridge-panel";
import { SkillsOptionsTab } from "./skills-tab";
/** Parse and validate URL params for deep-linking. */
function parseUrlParams() {
const params = new URLSearchParams(window.location.search);
const tabAllowlist = new Set(["general", "ai", "skills"]);
const tabAllowlist = new Set(["general", "ai", "skills", "connection"]);
const rawTab = params.get("tab");
const tab =
rawTab && tabAllowlist.has(rawTab)
? (rawTab as "general" | "ai" | "skills")
? (rawTab as "general" | "ai" | "skills" | "connection")
: undefined;
const rawSkill = params.get("skill");
// Bound skill name length to prevent abuse
@@ -77,16 +78,15 @@ function OptionsPageContent() {
}, []);
return (
<div className="min-h-screen bg-background">
<SettingsPage
storageAdapter={chromeStorageAdapter}
onTestConnection={handleTestConnection}
skillsContent={<SkillsOptionsTab initialSkill={initialSkill} />}
sttConfig={chromeSttAdapter}
initialTab={initialTab}
initialSkill={initialSkill}
/>
</div>
<SettingsPage
storageAdapter={chromeStorageAdapter}
onTestConnection={handleTestConnection}
skillsContent={<SkillsOptionsTab initialSkill={initialSkill} />}
connectionContent={<McpBridgePanel />}
sttConfig={chromeSttAdapter}
initialTab={initialTab}
initialSkill={initialSkill}
/>
);
}

View File

@@ -0,0 +1,148 @@
/**
* MCP WebSocket Bridge Panel
* UI for connecting/disconnecting the extension to the aipex-mcp-bridge.
*/
import type { WsMcpServerState } from "@aipexstudio/browser-runtime";
import { useCallback, useEffect, useState } from "react";
const DEFAULT_URL = "ws://localhost:9223";
type ConnectionStatus = WsMcpServerState["status"];
export function McpBridgePanel() {
const [url, setUrl] = useState(DEFAULT_URL);
const [status, setStatus] = useState<ConnectionStatus>("disconnected");
const [error, setError] = useState<string | null>(null);
const [connectedAt, setConnectedAt] = useState<number | null>(null);
const [reconnectAttempt, setReconnectAttempt] = useState(0);
const refreshStatus = useCallback(async () => {
try {
const state = await chrome.runtime.sendMessage({
request: "ws-bridge-status",
});
if (state) {
setStatus(state.status);
setError(state.error);
setConnectedAt(state.connectedAt);
setReconnectAttempt(state.reconnectAttempt);
if (state.url) setUrl(state.url);
}
} catch {
// Background may not be ready
}
}, []);
useEffect(() => {
refreshStatus();
const interval = setInterval(refreshStatus, 3000);
return () => clearInterval(interval);
}, [refreshStatus]);
const handleConnect = async () => {
setError(null);
try {
const response = await chrome.runtime.sendMessage({
request: "ws-bridge-connect",
url,
});
if (!response.success) {
setError(response.error || "Connection failed");
}
await refreshStatus();
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
}
};
const handleDisconnect = async () => {
try {
await chrome.runtime.sendMessage({ request: "ws-bridge-disconnect" });
await refreshStatus();
} catch (err) {
setError(err instanceof Error ? err.message : String(err));
}
};
const statusColor = {
disconnected: "bg-gray-400",
connecting: "bg-yellow-400 animate-pulse",
connected: "bg-green-500",
error: "bg-red-500",
}[status];
return (
<div className="rounded-lg border bg-card p-6 shadow-sm">
<h3 className="text-lg font-semibold mb-4">MCP WebSocket Bridge</h3>
<p className="text-sm text-muted-foreground mb-4">
Connect to an external MCP client (e.g. Claude, Cursor) via the
aipex-mcp-bridge. The bridge exposes AIPex browser tools to external AI
agents.
</p>
{/* Status indicator */}
<div className="flex items-center gap-2 mb-4">
<span className={`inline-block h-3 w-3 rounded-full ${statusColor}`} />
<span className="text-sm font-medium capitalize">{status}</span>
{connectedAt && status === "connected" && (
<span className="text-xs text-muted-foreground ml-2">
since {new Date(connectedAt).toLocaleTimeString()}
</span>
)}
{reconnectAttempt > 0 && status !== "connected" && (
<span className="text-xs text-muted-foreground ml-2">
(reconnect attempt {reconnectAttempt})
</span>
)}
</div>
{/* URL input */}
<div className="flex gap-2 mb-4">
<input
type="text"
value={url}
onChange={(e) => setUrl(e.target.value)}
placeholder="ws://localhost:9223"
disabled={status === "connected" || status === "connecting"}
className="flex-1 rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50"
/>
{status === "connected" ? (
<button
type="button"
onClick={handleDisconnect}
className="inline-flex items-center rounded-md bg-destructive px-4 py-2 text-sm font-medium text-destructive-foreground hover:bg-destructive/90"
>
Disconnect
</button>
) : (
<button
type="button"
onClick={handleConnect}
disabled={status === "connecting" || !url.trim()}
className="inline-flex items-center rounded-md bg-primary px-4 py-2 text-sm font-medium text-primary-foreground hover:bg-primary/90 disabled:pointer-events-none disabled:opacity-50"
>
{status === "connecting" ? "Connecting..." : "Connect"}
</button>
)}
</div>
{/* Error display */}
{error && (
<div className="rounded-md bg-destructive/10 border border-destructive/20 p-3 text-sm text-destructive">
{error}
</div>
)}
{/* Info */}
<div className="mt-4 text-xs text-muted-foreground space-y-1">
<p>
The bridge exposes <code>tools/list</code> and <code>tools/call</code>{" "}
over the MCP protocol, allowing external agents to use AIPex browser
automation tools.
</p>
<p>Only localhost connections (127.0.0.1, ::1) are allowed.</p>
</div>
</div>
);
}

View File

@@ -37,3 +37,5 @@ export * from "./storage/index.js";
export * from "./tools/index.js";
// Voice
// export * from "./voice/index.js";
// WebSocket MCP Bridge
export * from "./ws-bridge/index.js";

View File

@@ -4,14 +4,10 @@
* between QuickJS VM and the host environment
*/
import type { FileStats } from "./types";
import { zenfs } from "./zenfs-manager";
export interface FileStats {
isFile: boolean;
isDirectory: boolean;
size: number;
mtime: Date;
}
export type { FileStats };
export interface ToolDefinition {
name: string;

View File

@@ -0,0 +1,6 @@
export interface FileStats {
isFile: boolean;
isDirectory: boolean;
size: number;
mtime: Date;
}

View File

@@ -5,7 +5,7 @@
import { configure, fs } from "@zenfs/core";
import { IndexedDB } from "@zenfs/dom";
import type { FileStats } from "./skill-api";
import type { FileStats } from "./types";
type FsPromises = {
mkdir: (...args: any[]) => Promise<any>;

View File

@@ -1,30 +1,17 @@
import type { ContextProvider } from "@aipexstudio/aipex-core";
import type { RuntimeAddon } from "./runtime-addon.js";
import type { RuntimeBroadcastMessage } from "./types.js";
import type {
AutomationTarget,
RuntimeAddon,
RuntimeBroadcastMessage,
SnapshotCaptureOptions,
SnapshotResult,
} from "./types.js";
export interface AutomationTarget {
tabId: number;
frameId?: number;
windowId?: number;
}
export interface SnapshotCaptureOptions {
includeDom?: boolean;
includeScreenshot?: boolean;
includeContext?: boolean;
reason?: string;
tabId?: number;
}
export interface SnapshotResult {
id: string;
capturedAt: number;
screenshot?: string;
dom?: string;
title?: string;
url?: string;
metadata?: Record<string, unknown>;
}
export type {
AutomationTarget,
SnapshotCaptureOptions,
SnapshotResult,
} from "./types.js";
export interface CaptureSessionOptions {
target: AutomationTarget;

View File

@@ -1,22 +1,4 @@
import type {
AutomationTarget,
SnapshotCaptureOptions,
SnapshotResult,
} from "./browser-automation-host.js";
import type { RuntimeBroadcastMessage } from "./types.js";
export interface SnapshotHookContext {
target: AutomationTarget;
options?: SnapshotCaptureOptions;
}
export interface RuntimeAddon {
id: string;
initialize?(): Promise<void> | void;
onMessage?(message: RuntimeBroadcastMessage): Promise<void> | void;
onBeforeSnapshot?(ctx: SnapshotHookContext): Promise<void> | void;
onAfterSnapshot?(
result: SnapshotResult,
ctx: SnapshotHookContext,
): Promise<void> | void;
}
export type {
RuntimeAddon,
SnapshotHookContext,
} from "./types.js";

View File

@@ -8,3 +8,43 @@ export interface RuntimeBroadcastMessage<TPayload = unknown> {
export interface RuntimeAddonCleanup {
dispose(): Promise<void> | void;
}
export interface AutomationTarget {
tabId: number;
frameId?: number;
windowId?: number;
}
export interface SnapshotCaptureOptions {
includeDom?: boolean;
includeScreenshot?: boolean;
includeContext?: boolean;
reason?: string;
tabId?: number;
}
export interface SnapshotResult {
id: string;
capturedAt: number;
screenshot?: string;
dom?: string;
title?: string;
url?: string;
metadata?: Record<string, unknown>;
}
export interface SnapshotHookContext {
target: AutomationTarget;
options?: SnapshotCaptureOptions;
}
export interface RuntimeAddon {
id: string;
initialize?(): Promise<void> | void;
onMessage?(message: RuntimeBroadcastMessage): Promise<void> | void;
onBeforeSnapshot?(ctx: SnapshotHookContext): Promise<void> | void;
onAfterSnapshot?(
result: SnapshotResult,
ctx: SnapshotHookContext,
): Promise<void> | void;
}

View File

@@ -401,7 +401,7 @@ export class SkillStorage {
// Check if skills directory exists
const skillsDirExists = await zenfs.exists(skillsPath);
if (!skillsDirExists) {
console.log("[SkillStorage] /skills directory does not exist yet");
console.debug("[SkillStorage] /skills directory does not exist yet");
return skills;
}
@@ -445,7 +445,7 @@ export class SkillStorage {
enabled: true,
});
console.log(`[SkillStorage] Found skill in ZenFS: ${entry}`);
console.debug(`[SkillStorage] Found skill in ZenFS: ${entry}`);
} catch (error) {
console.warn(
`[SkillStorage] Failed to process skill ${entry}:`,
@@ -494,7 +494,7 @@ export class SkillStorage {
const stats = { added: 0, skipped: 0, failed: 0 };
try {
console.log("[SkillStorage] Starting sync from ZenFS...");
console.debug("[SkillStorage] Starting sync from ZenFS...");
// Scan ZenFS for all skills
const zenfsSkills = await this.scanZenFSForSkills();
@@ -528,7 +528,7 @@ export class SkillStorage {
try {
await this.saveToIndexedDB(skill);
stats.added++;
console.log(`[SkillStorage] Added skill from ZenFS: ${skill.name}`);
console.debug(`[SkillStorage] Added skill from ZenFS: ${skill.name}`);
} catch (error) {
console.error(
`[SkillStorage] Failed to add skill ${skill.name}:`,
@@ -543,7 +543,7 @@ export class SkillStorage {
// Update last sync time
this.lastSyncTime = Date.now();
console.log(
console.debug(
`[SkillStorage] Sync completed: ${stats.added} added, ${stats.skipped} skipped, ${stats.failed} failed`,
);
} catch (error) {

View File

@@ -128,47 +128,8 @@ export function registerDefaultBrowserTools<T extends ToolRegistryLike>(
return registry;
}
/**
* Get the currently active tab
* @throws Error if no active tab is found
*/
export async function getActiveTab(): Promise<chrome.tabs.Tab> {
const [tab] = await chrome.tabs.query({
active: true,
currentWindow: true,
});
if (!tab?.id) {
throw new Error("No active tab found");
}
return tab;
}
/**
* Execute a script in a specific tab
*/
export async function executeScriptInTab<T, Args extends any[]>(
tabId: number,
func: (...args: Args) => T,
args: Args,
): Promise<T> {
const results = await chrome.scripting.executeScript({
target: { tabId },
func,
args,
});
return results[0]?.result as T;
}
/**
* Execute a script in the active tab
*/
export async function executeScriptInActiveTab<T, Args extends any[]>(
func: (...args: Args) => T,
args: Args,
): Promise<T> {
const tab = await getActiveTab();
return await executeScriptInTab(tab.id!, func, args);
}
export {
executeScriptInActiveTab,
executeScriptInTab,
getActiveTab,
} from "./tab-utils";

View File

@@ -1,6 +1,6 @@
import { tool } from "@aipexstudio/aipex-core";
import { z } from "zod";
import { getActiveTab } from "./index";
import { getActiveTab } from "./tab-utils";
/**
* Get page metadata including title, description, keywords, etc.

View File

@@ -3,11 +3,11 @@ import { z } from "zod";
import { cacheScreenshotMetadata } from "../automation/computer";
import { RuntimeScreenshotStorage } from "../lib/screenshot-storage";
import { getAutomationMode } from "../runtime/automation-mode";
import { getActiveTab } from "./index";
import {
captureVisibleTabWithElementCrop,
MAX_PADDING,
} from "./screenshot-helpers.js";
import { getActiveTab } from "./tab-utils";
// Re-export the shared helper types/function so existing consumers aren't broken
export type {
@@ -52,8 +52,16 @@ async function compressImage(
export const captureScreenshotTool = tool({
name: "capture_screenshot",
description:
"Capture screenshot of current visible tab and return as base64 data URL. When sendToLLM=true, the screenshot will be sent to the LLM for visual analysis AND visual coordinate tools (computer) will be unlocked for subsequent interactions. NOTE: This tool requires focus mode.",
description: `[HIGH-COST FALLBACK] Capture screenshot of current visible tab.
TRY search_elements FIRST: For most interactions (clicking, filling, reading), use search_elements + UID-based tools. They are faster and don't send images to LLM.
USE THIS ONLY WHEN:
- search_elements cannot find the target after 2 query attempts
- You need to see visual layout, images, charts, or canvas content
- The page uses non-standard rendering that snapshots miss
When sendToLLM=true: Sends image to LLM (higher latency/cost, may capture sensitive on-screen data) and enables the computer tool for coordinate-based actions. NOTE: This tool requires focus mode.`,
parameters: z.object({
sendToLLM: z
.boolean()
@@ -61,7 +69,7 @@ export const captureScreenshotTool = tool({
.optional()
.default(false)
.describe(
"Whether to send the screenshot to LLM for visual analysis. When true, visual coordinate tools will be enabled.",
"Whether to send the screenshot to LLM for visual analysis. When true, enables computer tool for coordinate actions. Use sparingly - adds latency and token cost.",
),
}),
execute: async ({ sendToLLM = false }) => {
@@ -202,8 +210,16 @@ export const captureScreenshotTool = tool({
export const captureTabScreenshotTool = tool({
name: "capture_tab_screenshot",
description:
"Capture screenshot of a specific tab by ID. When sendToLLM=true, the screenshot will be sent to the LLM for visual analysis AND visual coordinate tools (computer) will be unlocked for subsequent interactions. NOTE: This tool requires focus mode.",
description: `[HIGH-COST FALLBACK] Capture screenshot of a specific tab by ID.
TRY search_elements FIRST: Visual verification is expensive. Use search_elements + UID-based tools for most interactions.
USE THIS ONLY WHEN:
- Visual verification is essential
- search_elements failed to find the target
- You need to see images, charts, or canvas content
When sendToLLM=true: Sends image to LLM (higher latency/cost) and enables coordinate-based actions. NOTE: This tool requires focus mode.`,
parameters: z.object({
tabId: z.number().describe("The tab ID to capture"),
sendToLLM: z
@@ -341,8 +357,9 @@ const MAX_SELECTOR_LENGTH = 500;
export const captureScreenshotWithHighlightTool = tool({
name: "capture_screenshot_with_highlight",
description:
"Capture screenshot of the current visible tab, optionally highlighting and cropping to a specific element identified by CSS selector. The screenshot is always sent to the LLM for visual analysis. NOTE: This tool requires focus mode.",
description: `[HIGH-COST] Capture screenshot of the current visible tab, optionally highlighting and cropping to a specific element identified by CSS selector. The screenshot is always sent to the LLM for visual analysis.
PREFER search_elements for finding/interacting with elements. Use this only when you need to visually verify element appearance or layout. NOTE: This tool requires focus mode.`,
parameters: z.object({
selector: z
.string()

View File

@@ -1,7 +1,7 @@
import { tool } from "@aipexstudio/aipex-core";
import { z } from "zod";
import * as snapshotProvider from "../automation/snapshot-provider";
import { getActiveTab } from "./index";
import { getActiveTab } from "./tab-utils";
export const takeSnapshotTool = tool({
name: "take_snapshot",

View File

@@ -0,0 +1,44 @@
/**
* Get the currently active tab
* @throws Error if no active tab is found
*/
export async function getActiveTab(): Promise<chrome.tabs.Tab> {
const [tab] = await chrome.tabs.query({
active: true,
currentWindow: true,
});
if (!tab?.id) {
throw new Error("No active tab found");
}
return tab;
}
/**
* Execute a script in a specific tab
*/
export async function executeScriptInTab<T, Args extends any[]>(
tabId: number,
func: (...args: Args) => T,
args: Args,
): Promise<T> {
const results = await chrome.scripting.executeScript({
target: { tabId },
func,
args,
});
return results[0]?.result as T;
}
/**
* Execute a script in the active tab
*/
export async function executeScriptInActiveTab<T, Args extends any[]>(
func: (...args: Args) => T,
args: Args,
): Promise<T> {
const tab = await getActiveTab();
return await executeScriptInTab(tab.id!, func, args);
}

View File

@@ -1,7 +1,7 @@
import { tool } from "@aipexstudio/aipex-core";
import { z } from "zod";
import { getAutomationMode } from "../runtime/automation-mode";
import { getActiveTab } from "./index";
import { getActiveTab } from "./tab-utils";
/**
* Get all open tabs across all windows

View File

@@ -0,0 +1,13 @@
export {
type ConnectionStatus,
WsMcpServer,
type WsMcpServerState,
wsMcpServer,
} from "./ws-mcp-server.js";
export {
type JSONRPCMessage,
type JSONRPCNotification,
type JSONRPCRequest,
type JSONRPCResponse,
WebSocketClientTransport,
} from "./ws-transport.js";

View File

@@ -0,0 +1,463 @@
/**
* WebSocket command executor for the AIPex extension.
*
* Connects as a WebSocket client to the aipex-mcp-bridge and listens
* for tool execution commands. The bridge is the true MCP server — this
* class simply executes tools and returns results.
*
* Handles:
* - tools/call -- executes a tool from allBrowserTools
* - ping -- responds with {} for keepalive
*
* No MCP protocol negotiation (initialize, tools/list) is needed on this
* side — the bridge handles that entirely with static tool schemas.
*/
import type { FunctionTool } from "@aipexstudio/aipex-core";
import { allBrowserTools } from "../tools/index.js";
import {
type JSONRPCMessage,
type JSONRPCRequest,
WebSocketClientTransport,
} from "./ws-transport.js";
export type ConnectionStatus =
| "disconnected"
| "connecting"
| "connected"
| "error";
export interface WsMcpServerState {
status: ConnectionStatus;
url: string | null;
error: string | null;
connectedAt: number | null;
reconnectAttempt: number;
}
type StatusListener = (state: WsMcpServerState) => void;
const KEEPALIVE_ALARM_NAME = "ws-mcp-keepalive";
const KEEPALIVE_INTERVAL_MINUTES = 0.4;
const TOOL_CALL_TIMEOUT_MS = 60_000;
const STORAGE_KEY_WS_URL = "ws-mcp-url";
function getReconnectDelayMs(attempt: number): number {
const withJitter = (base: number) =>
Math.round(base * (0.7 + Math.random() * 0.6));
return withJitter(Math.min(500 * 2 ** attempt, 10_000));
}
/**
* Find a tool by name from the registered browser tools.
*/
function findBrowserTool(name: string): FunctionTool | undefined {
return allBrowserTools.find((t) => t.name === name);
}
/**
* Format a tool execution result into MCP content blocks.
*/
function buildMcpContent(
data: unknown,
): Array<{ type: string; text?: string; data?: string; mimeType?: string }> {
if (data === null || data === undefined) {
return [{ type: "text", text: "null" }];
}
if (typeof data === "string") {
// Check if it's a base64 data URL (screenshot)
if (data.startsWith("data:image/")) {
const commaIndex = data.indexOf(",");
if (commaIndex > 0) {
const mimeType = data.slice(5, data.indexOf(";"));
const base64Data = data.slice(commaIndex + 1);
return [{ type: "image", data: base64Data, mimeType }];
}
}
return [{ type: "text", text: data }];
}
return [{ type: "text", text: JSON.stringify(data, null, 2) }];
}
export class WsMcpServer {
private transport: WebSocketClientTransport | null = null;
private state: WsMcpServerState = {
status: "disconnected",
url: null,
error: null,
connectedAt: null,
reconnectAttempt: 0,
};
private listeners: Set<StatusListener> = new Set();
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private autoReconnectEnabled = true;
private validateUrl(url: string): void {
let parsed: URL;
try {
parsed = new URL(url);
} catch {
throw new Error(`Invalid WebSocket URL: ${url}`);
}
const hostname = parsed.hostname;
const allowedHosts = ["localhost", "127.0.0.1", "[::1]", "::1"];
if (!allowedHosts.includes(hostname)) {
throw new Error(
`Only localhost connections are allowed. Got: ${hostname}`,
);
}
if (parsed.protocol !== "ws:" && parsed.protocol !== "wss:") {
throw new Error(
`URL must use ws:// or wss:// protocol. Got: ${parsed.protocol}`,
);
}
}
async connect(url: string): Promise<void> {
this.validateUrl(url);
this.cancelReconnect();
if (
this.state.status === "connected" ||
this.state.status === "connecting"
) {
await this.disconnect();
}
this.updateState({
status: "connecting",
url,
error: null,
connectedAt: null,
});
try {
const transport = new WebSocketClientTransport(url);
this.transport = transport;
transport.onclose = () => {
this.handleDisconnect();
};
transport.onerror = (error: Error) => {
console.error("[WsMcpServer] Transport error:", error.message);
};
transport.onmessage = (message: JSONRPCMessage) => {
this.handleMessage(message);
};
await transport.start();
this.updateState({
status: "connected",
connectedAt: Date.now(),
reconnectAttempt: 0,
});
this.startKeepalive();
this.persistUrl(url);
this.autoReconnectEnabled = true;
console.log(`[WsMcpServer] Connected to ${url}`);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
this.updateState({ status: "error", error: message, connectedAt: null });
this.transport = null;
this.scheduleReconnect(url);
throw error;
}
}
async disconnect(): Promise<void> {
this.autoReconnectEnabled = false;
this.cancelReconnect();
this.stopKeepalive();
if (this.transport) {
try {
await this.transport.close();
} catch {
// ignore close errors
}
this.transport = null;
}
this.updateState({
status: "disconnected",
url: null,
error: null,
connectedAt: null,
reconnectAttempt: 0,
});
this.clearPersistedUrl();
console.log("[WsMcpServer] Disconnected");
}
isConnected(): boolean {
return this.state.status === "connected";
}
getStatus(): WsMcpServerState {
return { ...this.state };
}
onStatusChange(listener: StatusListener): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
async getSavedUrl(): Promise<string | null> {
try {
const result = await chrome.storage.local.get(STORAGE_KEY_WS_URL);
return (result[STORAGE_KEY_WS_URL] as string) || null;
} catch {
return null;
}
}
// -- Auto-reconnect with exponential backoff --
private scheduleReconnect(url: string): void {
if (!this.autoReconnectEnabled) return;
this.cancelReconnect();
const attempt = this.state.reconnectAttempt;
const delay = getReconnectDelayMs(attempt);
console.log(
`[WsMcpServer] Scheduling reconnect attempt ${attempt + 1} in ${delay}ms`,
);
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
this.updateState({ reconnectAttempt: attempt + 1 });
this.connect(url).catch(() => {
// connect() itself schedules next retry on failure
});
}, delay);
}
private cancelReconnect(): void {
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
}
private persistUrl(url: string): void {
try {
chrome.storage.local.set({ [STORAGE_KEY_WS_URL]: url });
} catch {
// ignore storage errors
}
}
private clearPersistedUrl(): void {
try {
chrome.storage.local.remove(STORAGE_KEY_WS_URL);
} catch {
// ignore
}
}
// -- Message handling --
private handleMessage(message: JSONRPCMessage): void {
if (!("method" in message) || !("id" in message)) {
return;
}
const request = message as JSONRPCRequest;
this.handleRequest(request).catch((error) => {
console.error("[WsMcpServer] Unhandled error processing request:", error);
this.sendError(request.id, -32603, "Internal error");
});
}
private async handleRequest(request: JSONRPCRequest): Promise<void> {
switch (request.method) {
case "tools/call":
return this.handleToolsCall(request);
case "ping":
return this.sendResult(request.id, {});
default:
return this.sendError(
request.id,
-32601,
`Method not found: ${request.method}`,
);
}
}
private async handleToolsCall(request: JSONRPCRequest): Promise<void> {
const params = (request.params || {}) as Record<string, unknown>;
const name = params.name as string | undefined;
const args = (params.arguments || {}) as Record<string, unknown>;
if (!name) {
return this.sendError(
request.id,
-32602,
"Missing required parameter: name",
);
}
try {
const toolExecution = this.executeTool(name, args);
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(
() =>
reject(
new Error(
`Tool '${name}' timed out after ${TOOL_CALL_TIMEOUT_MS}ms`,
),
),
TOOL_CALL_TIMEOUT_MS,
);
});
const result = await Promise.race([toolExecution, timeoutPromise]);
await this.sendResult(request.id, {
content: buildMcpContent(result),
});
} catch (error) {
await this.sendResult(request.id, {
content: [
{
type: "text",
text: `Error: ${error instanceof Error ? error.message : String(error)}`,
},
],
isError: true,
});
}
}
private async executeTool(
name: string,
args: Record<string, unknown>,
): Promise<unknown> {
const browserTool = findBrowserTool(name);
if (!browserTool) {
throw new Error(`Tool not found: ${name}`);
}
// FunctionTool.invoke(runContext, inputJsonString, details?)
// We pass an empty object as RunContext since we're outside the agent loop.
return await (browserTool as any).invoke({} as any, JSON.stringify(args));
}
private async sendResult(
id: string | number,
result: unknown,
): Promise<void> {
if (!this.transport) return;
await this.transport.send({
jsonrpc: "2.0",
id,
result,
} as JSONRPCMessage);
}
private async sendError(
id: string | number,
code: number,
message: string,
): Promise<void> {
if (!this.transport) return;
await this.transport.send({
jsonrpc: "2.0",
id,
error: { code, message },
} as JSONRPCMessage);
}
// -- Connection lifecycle --
private handleDisconnect(): void {
this.stopKeepalive();
const lastUrl = this.state.url;
this.transport = null;
if (this.state.status !== "disconnected") {
this.updateState({
status: "disconnected",
error: null,
connectedAt: null,
});
console.log("[WsMcpServer] Connection closed by remote");
if (lastUrl && this.autoReconnectEnabled) {
this.scheduleReconnect(lastUrl);
}
}
}
private updateState(partial: Partial<WsMcpServerState>): void {
this.state = { ...this.state, ...partial };
for (const listener of this.listeners) {
try {
listener(this.getStatus());
} catch {
// don't let listener errors break us
}
}
}
// -- Service Worker Keepalive --
private startKeepalive(): void {
try {
chrome.alarms.create(KEEPALIVE_ALARM_NAME, {
periodInMinutes: KEEPALIVE_INTERVAL_MINUTES,
});
} catch (error) {
console.warn("[WsMcpServer] Failed to create keepalive alarm:", error);
}
}
private stopKeepalive(): void {
try {
chrome.alarms.clear(KEEPALIVE_ALARM_NAME);
} catch {
// ignore
}
}
handleAlarm(alarm: chrome.alarms.Alarm): void {
if (alarm.name !== KEEPALIVE_ALARM_NAME) return;
if (this.state.status === "connected" && this.transport?.isOpen) {
this.sendPing();
return;
}
if (this.state.status === "connecting") {
return;
}
const url = this.state.url;
if (url && this.autoReconnectEnabled) {
this.connect(url).catch(() => {
// connect() schedules its own retry on failure
});
}
}
private async sendPing(): Promise<void> {
if (!this.transport?.isOpen) return;
try {
await this.transport.send({
jsonrpc: "2.0",
id: `ping-${Date.now()}`,
method: "ping",
} as JSONRPCMessage);
} catch {
console.warn("[WsMcpServer] Ping send failed, closing connection");
this.handleDisconnect();
}
}
}
export const wsMcpServer = new WsMcpServer();

View File

@@ -0,0 +1,129 @@
/**
* WebSocket client transport for MCP-over-WebSocket.
*
* Uses the browser-native WebSocket API. AIPex connects as a WebSocket
* client to the host agent's WebSocket server, then JSON-RPC 2.0 messages
* flow bidirectionally over that connection.
*
* No dependency on @modelcontextprotocol/sdk to avoid pulling ajv
* (which uses `new Function()`) into the MV3 service worker bundle.
*/
export interface JSONRPCRequest {
jsonrpc: "2.0";
id: string | number;
method: string;
params?: Record<string, unknown>;
}
export interface JSONRPCResponse {
jsonrpc: "2.0";
id: string | number;
result?: unknown;
error?: { code: number; message: string; data?: unknown };
}
export interface JSONRPCNotification {
jsonrpc: "2.0";
method: string;
params?: Record<string, unknown>;
}
export type JSONRPCMessage =
| JSONRPCRequest
| JSONRPCResponse
| JSONRPCNotification;
function isValidJsonRpcMessage(data: unknown): data is JSONRPCMessage {
if (typeof data !== "object" || data === null) return false;
const msg = data as Record<string, unknown>;
return msg.jsonrpc === "2.0";
}
const WS_CLOSE_NORMAL = 1000;
export class WebSocketClientTransport {
private socket: WebSocket | null = null;
private url: string;
private closeFired = false;
onmessage?: (message: JSONRPCMessage) => void;
onerror?: (error: Error) => void;
onclose?: () => void;
constructor(url: string) {
this.url = url;
}
start(): Promise<void> {
if (this.socket) {
throw new Error("WebSocketClientTransport already started");
}
this.closeFired = false;
return new Promise<void>((resolve, reject) => {
const ws = new WebSocket(this.url);
this.socket = ws;
ws.onopen = () => {
resolve();
};
ws.onerror = () => {
const error = new Error("WebSocket error");
reject(error);
this.onerror?.(error);
};
ws.onclose = () => {
this.socket = null;
if (!this.closeFired) {
this.closeFired = true;
this.onclose?.();
}
};
ws.onmessage = (event: MessageEvent) => {
let parsed: unknown;
try {
parsed = JSON.parse(String(event.data));
} catch (error) {
this.onerror?.(
error instanceof Error ? error : new Error(String(error)),
);
return;
}
if (!isValidJsonRpcMessage(parsed)) {
this.onerror?.(new Error("Received non-JSON-RPC message"));
return;
}
this.onmessage?.(parsed);
};
});
}
async send(message: JSONRPCMessage): Promise<void> {
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
throw new Error("WebSocket not connected");
}
this.socket.send(JSON.stringify(message));
}
async close(): Promise<void> {
const ws = this.socket;
if (!ws) return;
this.socket = null;
if (
ws.readyState === WebSocket.OPEN ||
ws.readyState === WebSocket.CONNECTING
) {
ws.close(WS_CLOSE_NORMAL);
}
}
get isOpen(): boolean {
return this.socket?.readyState === WebSocket.OPEN;
}
}

View File

@@ -1,4 +1,4 @@
import type { ProviderType } from "./settings.js";
import type { ProviderType } from "./types.js";
export interface AIProviderConfig {
name: string;

View File

@@ -1,6 +1,7 @@
import type { AIProviderKey } from "./ai-providers.js";
import type { ProviderType } from "./types.js";
export type ProviderType = "google" | "openai" | "claude";
export type { ProviderType };
export interface CustomModelConfig {
id: string;

View File

@@ -0,0 +1 @@
export type ProviderType = "google" | "openai" | "claude";

View File

@@ -4,10 +4,7 @@ import type {
Agent as OpenAIAgent,
} from "@openai/agents";
import type { AiSdkModel } from "@openai/agents-extensions";
import type { AIPex } from "./agent/aipex.js";
import type { Context, ContextManager } from "./context/index.js";
import type { ConversationManager } from "./conversation/manager.js";
import type { Session } from "./conversation/session.js";
import type { AgentError } from "./utils/errors.js";
// Re-export types from @openai/agents for convenient access
@@ -73,7 +70,7 @@ export interface AIPexOptions<
* Fully custom ConversationManager instance.
* When provided, storage and compression options are ignored.
*/
conversationManager?: ConversationManager;
conversationManager?: import("./conversation/manager.js").ConversationManager;
/**
* Context manager for providing additional context to the agent.
@@ -161,7 +158,7 @@ export interface MetricsPayload {
}
export interface AgentPluginContext {
agent: AIPex;
agent: import("./agent/aipex.js").AIPex;
}
export interface AgentPluginHooks {
@@ -240,8 +237,8 @@ export interface SessionTree {
}
export interface SessionStorageAdapter {
save(session: Session): Promise<void>;
load(id: string): Promise<Session | null>;
save(session: import("./conversation/session.js").Session): Promise<void>;
load(id: string): Promise<import("./conversation/session.js").Session | null>;
delete(id: string): Promise<void>;
listAll(): Promise<SessionSummary[]>;
getSessionTree(rootId?: string): Promise<SessionTree[]>;