feat: enhance chatbot functionality with skill management and input mode toggle

- Introduced a new context for managing skills within the PromptInput component, allowing users to add, remove, and clear skills.
- Added SkillItem type and related components for displaying skill tags in the chatbot interface.
- Implemented initial input value support in the Chatbot component for better user experience.
- Enhanced the BrowserChatHeader to include a toggle for switching between voice and text input modes.
- Updated various components to support new skill management features and improved overall code organization.
This commit is contained in:
ropzislaw
2026-02-14 00:10:49 +08:00
parent dc8d5f21ec
commit a938e4b28d
26 changed files with 1507 additions and 115 deletions

View File

@@ -12,6 +12,7 @@ import {
Loader2Icon,
PaperclipIcon,
PlusIcon,
PuzzleIcon,
SendIcon,
SquareIcon,
XIcon,
@@ -211,6 +212,35 @@ export const usePromptInputContexts = () => {
return context;
};
// ============ Skill Items Context ============
export type SkillItem = {
id: string;
name: string;
description?: string;
};
type SkillItemsContext = {
items: SkillItem[];
add: (item: SkillItem) => void;
remove: (id: string) => void;
clear: () => void;
availableSkills: SkillItem[];
setAvailableSkills: (items: SkillItem[]) => void;
};
const SkillItemsContext = createContext<SkillItemsContext | null>(null);
export const usePromptInputSkills = () => {
const context = useContext(SkillItemsContext);
if (!context) {
throw new Error("usePromptInputSkills must be used within a PromptInput");
}
return context;
};
export type PromptInputAttachmentProps = HTMLAttributes<HTMLDivElement> & {
data: FileUIPart & { id: string };
className?: string;
@@ -421,10 +451,103 @@ export function PromptInputContextTags({
);
}
// ============ Skill Items Components ============
export type PromptInputSkillTagProps = HTMLAttributes<HTMLDivElement> & {
data: SkillItem;
className?: string;
};
export function PromptInputSkillTag({
data,
className,
...props
}: PromptInputSkillTagProps) {
const skills = usePromptInputSkills();
return (
<div
className={cn(
"group inline-flex items-center gap-1.5 px-2 py-1 text-sm rounded-md",
"bg-primary/10 hover:bg-primary/20 transition-colors",
"border border-primary/30",
className,
)}
{...props}
>
<span className="text-primary">
<PuzzleIcon className="size-4" />
</span>
<span className="max-w-[200px] truncate text-primary">{data.name}</span>
<Button
aria-label="Remove skill"
className="h-4 w-4 p-0 opacity-0 group-hover:opacity-100 transition-opacity"
onClick={() => skills.remove(data.id)}
size="icon"
type="button"
variant="ghost"
>
<XIcon className="h-3 w-3" />
</Button>
</div>
);
}
export type PromptInputSkillTagsProps = Omit<
HTMLAttributes<HTMLDivElement>,
"children"
> & {
children?: (item: SkillItem) => ReactNode;
};
export function PromptInputSkillTags({
className,
children,
...props
}: PromptInputSkillTagsProps) {
const skills = usePromptInputSkills();
const [height, setHeight] = useState(0);
const contentRef = useRef<HTMLDivElement>(null);
useLayoutEffect(() => {
const el = contentRef.current;
if (!el) {
return;
}
const ro = new ResizeObserver(() => {
setHeight(el.getBoundingClientRect().height);
});
ro.observe(el);
setHeight(el.getBoundingClientRect().height);
return () => ro.disconnect();
}, []);
return (
<div
aria-live="polite"
className={cn(
"overflow-hidden transition-[height] duration-200 ease-out",
className,
)}
style={{ height: skills.items.length ? height : 0 }}
{...props}
>
<div className="flex flex-wrap gap-2 p-3 pb-0" ref={contentRef}>
{skills.items.map((item) => (
<Fragment key={item.id}>
{children ? children(item) : <PromptInputSkillTag data={item} />}
</Fragment>
))}
</div>
</div>
);
}
export type PromptInputMessage = {
text?: string;
files?: FileUIPart[];
contexts?: ContextItem[];
skills?: SkillItem[];
};
export type PromptInputProps = Omit<
@@ -466,6 +589,8 @@ export const PromptInput = ({
const [items, setItems] = useState<(FileUIPart & { id: string })[]>([]);
const [contextItems, setContextItems] = useState<ContextItem[]>([]);
const [availableContexts, setAvailableContexts] = useState<ContextItem[]>([]);
const [skillItems, setSkillItems] = useState<SkillItem[]>([]);
const [availableSkills, setAvailableSkills] = useState<SkillItem[]>([]);
const inputRef = useRef<HTMLInputElement | null>(null);
const anchorRef = useRef<HTMLSpanElement>(null);
const formRef = useRef<HTMLFormElement | null>(null);
@@ -586,6 +711,25 @@ export const PromptInput = ({
setContextItems([]);
}, []);
// Skill management callbacks
const addSkill = useCallback((skill: SkillItem) => {
setSkillItems((prev) => {
// Avoid duplicates
if (prev.some((item) => item.id === skill.id)) {
return prev;
}
return [...prev, skill];
});
}, []);
const removeSkill = useCallback((id: string) => {
setSkillItems((prev) => prev.filter((item) => item.id !== id));
}, []);
const clearSkills = useCallback(() => {
setSkillItems([]);
}, []);
// Note: File input cannot be programmatically set for security reasons
// The syncHiddenInput prop is no longer functional
useEffect(() => {
@@ -684,9 +828,13 @@ export const PromptInput = ({
return item;
}),
).then((files: FileUIPart[]) => {
onSubmit({ text, files, contexts: contextItems }, event);
onSubmit(
{ text, files, contexts: contextItems, skills: skillItems },
event,
);
clear();
clearContexts();
clearSkills();
});
};
@@ -714,28 +862,42 @@ export const PromptInput = ({
[contextItems, addContext, removeContext, clearContexts, availableContexts],
);
const skillsCtx = useMemo<SkillItemsContext>(
() => ({
items: skillItems,
add: addSkill,
remove: removeSkill,
clear: clearSkills,
availableSkills,
setAvailableSkills,
}),
[skillItems, addSkill, removeSkill, clearSkills, availableSkills],
);
return (
<AttachmentsContext.Provider value={ctx}>
<ContextItemsContext.Provider value={contextsCtx}>
<span aria-hidden="true" className="hidden" ref={anchorRef} />
<input
accept={accept}
className="hidden"
multiple={multiple}
onChange={handleChange}
ref={inputRef}
type="file"
/>
<form
className={cn(
"w-full divide-y overflow-hidden rounded-xl border bg-background shadow-sm",
className,
)}
onSubmit={handleSubmit}
{...props}
>
{children}
</form>
<SkillItemsContext.Provider value={skillsCtx}>
<span aria-hidden="true" className="hidden" ref={anchorRef} />
<input
accept={accept}
className="hidden"
multiple={multiple}
onChange={handleChange}
ref={inputRef}
type="file"
/>
<form
className={cn(
"w-full divide-y overflow-hidden rounded-xl border bg-background shadow-sm",
className,
)}
onSubmit={handleSubmit}
{...props}
>
{children}
</form>
</SkillItemsContext.Provider>
</ContextItemsContext.Provider>
</AttachmentsContext.Provider>
);
@@ -781,6 +943,7 @@ export const PromptInputTextarea = ({
}: PromptInputTextareaProps) => {
const attachments = usePromptInputAttachments();
const contexts = usePromptInputContexts();
const skills = usePromptInputSkills();
const [isFocused, setIsFocused] = useState(false);
const [hasValue, setHasValue] = useState(false);
const [showContextMenu, setShowContextMenu] = useState(false);
@@ -796,6 +959,13 @@ export const PromptInputTextarea = ({
const scrollContainerRef = useRef<HTMLDivElement>(null);
const selectedItemRef = useRef<HTMLButtonElement>(null);
// Skill slash command state
const [showSkillMenu, setShowSkillMenu] = useState(false);
const [slashSearchQuery, setSlashSearchQuery] = useState("");
const [slashPosition, setSlashPosition] = useState<number | null>(null);
const [selectedSkillIndex, setSelectedSkillIndex] = useState(0);
const selectedSkillItemRef = useRef<HTMLButtonElement>(null);
// Sync hasValue with external value prop (for controlled components)
useEffect(() => {
setHasValue(!!props.value);
@@ -820,6 +990,41 @@ export const PromptInputTextarea = ({
: placeholder;
const handleKeyDown: KeyboardEventHandler<HTMLTextAreaElement> = (e) => {
// Handle skill menu navigation
if (showSkillMenu && filteredSkills.length > 0) {
if (e.key === "ArrowDown") {
e.preventDefault();
setSelectedSkillIndex((prev) => (prev + 1) % filteredSkills.length);
return;
}
if (e.key === "ArrowUp") {
e.preventDefault();
setSelectedSkillIndex(
(prev) =>
(prev - 1 + filteredSkills.length) % filteredSkills.length,
);
return;
}
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
const selectedSkill = filteredSkills[selectedSkillIndex];
if (selectedSkill) {
handleSkillSelect(selectedSkill);
}
return;
}
if (e.key === "Escape") {
e.preventDefault();
setShowSkillMenu(false);
setSlashSearchQuery("");
setSlashPosition(null);
return;
}
}
// Handle context menu navigation
if (showContextMenu && filteredContexts.length > 0) {
if (e.key === "ArrowDown") {
@@ -917,6 +1122,50 @@ export const PromptInputTextarea = ({
[atPosition, contexts, props.value, onChange],
);
// Handle skill selection
const handleSkillSelect = useCallback(
(skill: SkillItem) => {
if (!textareaRef.current || slashPosition === null) return;
// Add skill to the skills context
skills.add({
id: skill.id,
name: skill.name,
description: skill.description,
});
// Remove /xxx from textarea (similar to @ handling)
const currentValue = String(props.value || "");
const beforeSlash = currentValue.slice(0, slashPosition);
const afterSearch = currentValue.slice(
textareaRef.current.selectionStart,
);
const newValue = beforeSlash + afterSearch;
// Trigger onChange with new value
const syntheticEvent = {
target: { value: newValue },
currentTarget: { value: newValue },
} as ChangeEvent<HTMLTextAreaElement>;
onChange?.(syntheticEvent);
// Close menu
setShowSkillMenu(false);
setSlashSearchQuery("");
setSlashPosition(null);
// Refocus textarea
setTimeout(() => {
if (textareaRef.current) {
textareaRef.current.focus();
textareaRef.current.selectionStart = beforeSlash.length;
textareaRef.current.selectionEnd = beforeSlash.length;
}
}, 0);
},
[slashPosition, skills, props.value, onChange],
);
const handlePaste: ClipboardEventHandler<HTMLTextAreaElement> = (event) => {
const items = event.clipboardData?.items;
@@ -964,6 +1213,19 @@ export const PromptInputTextarea = ({
setAtPosition(null);
}
// Detect / command for skills
const slashMatch = beforeCursor.match(/\/([^\s]*)$/);
if (slashMatch) {
const slashQuery = slashMatch[1]; // Text after /
setSlashPosition(beforeCursor.lastIndexOf("/"));
setSlashSearchQuery(slashQuery ?? "");
setShowSkillMenu(true);
} else {
setShowSkillMenu(false);
setSlashSearchQuery("");
setSlashPosition(null);
}
onChange?.(e);
};
@@ -1011,11 +1273,47 @@ export const PromptInputTextarea = ({
});
}, [contexts.availableContexts, searchQuery]);
// Filter skills based on search query with fuzzy matching
const filteredSkills = useMemo(() => {
if (!slashSearchQuery) return skills.availableSkills;
const query = slashSearchQuery.toLowerCase();
return skills.availableSkills.filter((skill) => {
// Match against name
if (skill.name.toLowerCase().includes(query)) return true;
// Match against description
if (skill.description?.toLowerCase().includes(query)) return true;
// Fuzzy match: check if query characters appear in order in name
const nameLower = skill.name.toLowerCase();
let queryIndex = 0;
for (
let i = 0;
i < nameLower.length && queryIndex < query.length;
i++
) {
if (nameLower[i] === query[queryIndex]) {
queryIndex++;
}
}
if (queryIndex === query.length) return true;
return false;
});
}, [skills.availableSkills, slashSearchQuery]);
// Reset selected index when filtered contexts change
useEffect(() => {
setSelectedIndex(filteredContexts.length ? 0 : -1);
}, [filteredContexts]);
// Reset selected skill index when filtered skills change
useEffect(() => {
setSelectedSkillIndex(0);
}, [filteredSkills]);
// Auto-scroll to selected item when navigating with keyboard
useEffect(() => {
if (selectedIndex < 0) {
@@ -1030,10 +1328,20 @@ export const PromptInputTextarea = ({
}
}, [selectedIndex]);
// Calculate menu position when showing context menu
// Auto-scroll to selected skill item when navigating with keyboard
useEffect(() => {
if (selectedSkillItemRef.current) {
selectedSkillItemRef.current.scrollIntoView({
block: "nearest",
behavior: "smooth",
});
}
}, [selectedSkillIndex]);
// Calculate menu position when showing context menu or skill menu
useEffect(() => {
const updatePosition = () => {
if (showContextMenu && textareaRef.current) {
if ((showContextMenu || showSkillMenu) && textareaRef.current) {
const rect = textareaRef.current.getBoundingClientRect();
const windowHeight = window.innerHeight;
setMenuPosition({
@@ -1047,7 +1355,7 @@ export const PromptInputTextarea = ({
updatePosition();
// Update position on scroll and resize
if (!showContextMenu) {
if (!showContextMenu && !showSkillMenu) {
return;
}
@@ -1058,7 +1366,7 @@ export const PromptInputTextarea = ({
window.removeEventListener("scroll", updatePosition, true);
window.removeEventListener("resize", updatePosition);
};
}, [showContextMenu]);
}, [showContextMenu, showSkillMenu]);
return (
<div className="relative">
@@ -1144,6 +1452,78 @@ export const PromptInputTextarea = ({
</div>,
document.body,
)}
{/* Skill Command Menu - Portal Implementation */}
{showSkillMenu &&
filteredSkills.length > 0 &&
typeof document !== "undefined" &&
createPortal(
<div
className="fixed z-[9999] animate-in fade-in-0 zoom-in-95 slide-in-from-bottom-2"
style={{
bottom: `${menuPosition.bottom}px`,
left: `${menuPosition.left}px`,
width: `${menuPosition.width}px`,
}}
>
<div className="bg-popover border rounded-lg shadow-xl max-h-[400px] overflow-hidden mb-2">
{/* Search hint */}
<div className="px-3 py-2 text-xs text-muted-foreground border-b bg-muted/50">
{slashSearchQuery ? (
<>
Searching skills:{" "}
<span className="font-medium">/{slashSearchQuery}</span>
{filteredSkills.length > 0 && (
<span className="ml-2">
({filteredSkills.length} found)
</span>
)}
</>
) : (
<span>{filteredSkills.length} skills available</span>
)}
</div>
{/* Skills Results */}
<div className="max-h-[350px] overflow-y-auto">
{filteredSkills.map((skill, index) => (
<button
key={skill.id}
ref={
index === selectedSkillIndex
? selectedSkillItemRef
: null
}
type="button"
className={cn(
"w-full flex flex-col gap-1 px-3 py-2 text-sm transition-colors text-left min-w-0 border-b last:border-b-0",
index === selectedSkillIndex
? "bg-accent text-accent-foreground"
: "hover:bg-accent/50",
)}
onClick={() => handleSkillSelect(skill)}
onMouseEnter={() => setSelectedSkillIndex(index)}
>
<div className="flex items-center gap-2 w-full">
<span className="font-medium truncate flex-1 min-w-0">
{skill.name}
</span>
<span className="text-xs text-muted-foreground shrink-0 px-1.5 py-0.5 rounded bg-background/50">
skill
</span>
</div>
{skill.description && (
<span className="text-xs text-muted-foreground line-clamp-2">
{skill.description}
</span>
)}
</button>
))}
</div>
</div>
</div>,
document.body,
)}
</div>
);
};

View File

@@ -139,6 +139,8 @@ export interface ChatbotProps extends Omit<ChatbotProviderProps, "children"> {
placeholderTexts?: string[];
/** Header title */
title?: string;
/** Initial input value to pre-fill the text area */
initialInput?: string;
}
/**
@@ -177,6 +179,7 @@ export function Chatbot({
models = DEFAULT_MODELS,
placeholderTexts,
title = "AIPex",
initialInput,
}: ChatbotProps) {
return (
<ChatbotProvider
@@ -195,6 +198,7 @@ export function Chatbot({
models={models}
placeholderTexts={placeholderTexts}
title={title}
initialInput={initialInput}
/>
</ChatbotProvider>
);
@@ -207,10 +211,12 @@ function ChatbotContent({
models,
placeholderTexts,
title,
initialInput: initialInputProp,
}: {
models: Array<{ name: string; value: string }>;
placeholderTexts?: string[];
title: string;
initialInput?: string;
}) {
const themeCtx = useContext(ThemeContext);
const chatCtx = useContext(ChatContext);
@@ -221,7 +227,7 @@ function ChatbotContent({
chatCtx || {};
const { isReady: isAgentReady } = agentCtx || {};
const [input, setInput] = useState("");
const [input, setInput] = useState(initialInputProp ?? "");
const [inputResetCount, setInputResetCount] = useState(0);
const handleSubmit = useCallback(

View File

@@ -22,6 +22,8 @@ import {
PromptInputModelSelectItem,
PromptInputModelSelectTrigger,
PromptInputModelSelectValue,
PromptInputSkillTag,
PromptInputSkillTags,
PromptInputSubmit,
PromptInputTextarea,
PromptInputToolbar,
@@ -150,6 +152,14 @@ export function DefaultInputArea({
{(context) => <PromptInputContextTag data={context} />}
</PromptInputContextTags>
{/* Skill Tags */}
<PromptInputSkillTags>
{(skill) => <PromptInputSkillTag data={skill} />}
</PromptInputSkillTags>
{/* Platform-specific extras (e.g., context/skill data loaders) */}
{slots.promptExtras?.()}
{/* Attachments */}
<PromptInputAttachments>
{(attachment) => <PromptInputAttachment data={attachment} />}

View File

@@ -17,6 +17,9 @@ import {
SourcesTrigger,
} from "../../ai-elements/sources";
import { useComponentsContext } from "../context";
import { BuyTokenPrompt } from "./buy-token-prompt";
import { LoginPrompt } from "./login-prompt";
import { ModelChangePrompt } from "./model-change-prompt";
import { DefaultToolDisplay } from "./slots/tool-display";
/**
@@ -208,6 +211,34 @@ export function DefaultMessageItem({
return null;
}
})}
{/* Metadata-driven prompts for assistant error messages */}
{message.role === "assistant" && message.metadata && (
<>
{message.metadata.needLogin && (
<LoginPrompt
showByokOption
onOpenSettings={() =>
chrome.runtime?.openOptionsPage?.()
}
/>
)}
{message.metadata.needBuyToken && (
<BuyTokenPrompt
currentCredits={message.metadata.currentCredits}
requiredCredits={message.metadata.requiredCredits}
/>
)}
{message.metadata.needChangeModel && (
<ModelChangePrompt
supportedModels={message.metadata.supportedModels || []}
onModelChange={(modelId) => {
chrome.storage?.local?.set?.({ aiModel: modelId });
}}
/>
)}
</>
)}
</div>
);
}

View File

@@ -33,6 +33,8 @@ export function DefaultMessageList({
<div className={cn("flex-1 overflow-hidden", className)} {...props}>
<Conversation className="h-full">
<ConversationContent>
{/* Before messages slot - for banners, announcements */}
{slots.beforeMessages?.()}
{displayMessages.length === 0 ? (
<WelcomeScreen
onSuggestionClick={(text) => {

View File

@@ -1,53 +1,73 @@
import {
DollarSignIcon,
CameraIcon,
FileTextIcon,
LayersIcon,
ScanSearchIcon,
SearchIcon,
} from "lucide-react";
import { useMemo } from "react";
import { useTranslation } from "../../../i18n/context";
import { cn } from "../../../lib/utils";
import type { WelcomeScreenProps, WelcomeSuggestion } from "../../../types";
import { Suggestion, Suggestions } from "../../ai-elements/suggestion";
import { useComponentsContext } from "../context";
/**
* Default suggestions for the welcome screen
* Build i18n-driven default suggestions matching legacy AIPex layout.
*/
const DEFAULT_SUGGESTIONS: WelcomeSuggestion[] = [
{
icon: LayersIcon,
text: "Help me organize my browser tabs by topic",
iconColor: "text-blue-600",
bgColor: "bg-blue-100",
},
{
icon: FileTextIcon,
text: "Summarize this page for me",
iconColor: "text-green-600",
bgColor: "bg-green-100",
},
{
icon: SearchIcon,
text: "Research a topic across multiple tabs",
iconColor: "text-purple-600",
bgColor: "bg-purple-100",
},
{
icon: DollarSignIcon,
text: "Compare prices across shopping tabs",
iconColor: "text-orange-600",
bgColor: "bg-orange-100",
},
];
function useDefaultSuggestions(): WelcomeSuggestion[] {
const { t } = useTranslation();
return useMemo(
() => [
{
icon: FileTextIcon,
text: t("welcome.analyzePage"),
iconColor: "text-green-600",
bgColor: "bg-green-100",
},
{
icon: LayersIcon,
text: t("welcome.organizeTabs"),
iconColor: "text-blue-600",
bgColor: "bg-blue-100",
},
{
icon: SearchIcon,
text: t("welcome.research"),
iconColor: "text-purple-600",
bgColor: "bg-purple-100",
},
{
icon: CameraIcon,
text: t("welcome.screenRecording"),
iconColor: "text-orange-600",
bgColor: "bg-orange-100",
},
{
icon: ScanSearchIcon,
text: t("welcome.uxAuditGoal"),
iconColor: "text-cyan-600",
bgColor: "bg-cyan-100",
},
],
[t],
);
}
/**
* Default WelcomeScreen component
*/
export function DefaultWelcomeScreen({
onSuggestionClick,
suggestions = DEFAULT_SUGGESTIONS,
suggestions,
className,
...props
}: WelcomeScreenProps) {
const { t } = useTranslation();
const defaultSuggestions = useDefaultSuggestions();
const effectiveSuggestions = suggestions ?? defaultSuggestions;
return (
<div
className={cn(
@@ -58,16 +78,16 @@ export function DefaultWelcomeScreen({
>
<div className="text-center mb-6 sm:mb-8">
<h3 className="text-xl font-semibold text-gray-900 dark:text-gray-100 mb-2">
Welcome to AIPex
{t("welcome.title")}
</h3>
<p className="text-sm text-gray-600 dark:text-gray-400">
Your AI-powered browser assistant
{t("welcome.subtitle")}
</p>
</div>
<div className="w-full max-w-2xl">
<Suggestions className="grid gap-3 sm:gap-4 sm:grid-cols-2 w-full">
{suggestions.map((suggestion) => {
{effectiveSuggestions.map((suggestion) => {
const Icon = suggestion.icon;
return (
<Suggestion

View File

@@ -39,6 +39,7 @@ export type {
} from "../../types";
// Individual component exports
export {
BuyTokenPrompt,
ConfigurationGuide,
type ConfigurationGuideProps,
DefaultHeader,
@@ -49,10 +50,17 @@ export {
type ExtendedInputAreaProps,
Header,
InputArea,
LoginPrompt,
MessageItem,
MessageList,
type AutomationModeValue,
ModeIndicator,
ModelChangePrompt,
type ModelInfo,
TokenUsageIndicator,
type TokenUsageIndicatorProps,
UpdateBanner,
type VersionCheckResult,
WelcomeScreen,
} from "./components";
// Default export for backward compatibility

View File

@@ -18,6 +18,7 @@ import {
Info,
Mail,
MessageCircle,
Mic,
MessageSquare,
Package,
Palette,
@@ -220,6 +221,7 @@ export function SettingsPage({
onSave,
onTestConnection,
skillsContent,
sttConfig,
}: SettingsPageProps) {
const { t, language, changeLanguage } = useTranslation();
const { theme, changeTheme, effectiveTheme } = useTheme();
@@ -250,6 +252,12 @@ export function SettingsPage({
const [searchTerm, setSearchTerm] = useState("");
const [dataSharingEnabled, setDataSharingEnabled] = useState(true);
// ElevenLabs STT state (independent of main settings blob)
const [sttApiKey, setSttApiKey] = useState("");
const [sttModelId, setSttModelId] = useState("");
const [showSttKey, setShowSttKey] = useState(false);
const [isSavingStt, setIsSavingStt] = useState(false);
useEffect(() => {
const loadSettings = async () => {
try {
@@ -316,6 +324,15 @@ export function SettingsPage({
loadSettings();
}, [storageAdapter, storageKey]);
// Load ElevenLabs STT config when adapter is provided
useEffect(() => {
if (!sttConfig) return;
sttConfig.load().then(({ apiKey, modelId }) => {
setSttApiKey(apiKey);
setSttModelId(modelId);
});
}, [sttConfig]);
const updateSettingsFromModel = useCallback((model: CustomModelConfig) => {
const providerKey = resolveProviderKey(model);
setSettings((prev: AppSettings) => ({
@@ -677,6 +694,21 @@ export function SettingsPage({
[storageAdapter, storageKey, settings],
);
const handleSaveStt = useCallback(async () => {
if (!sttConfig) return;
setIsSavingStt(true);
try {
await sttConfig.save({ apiKey: sttApiKey, modelId: sttModelId });
setSaveStatus({ type: "success", message: t("settings.saveSuccess") });
setTimeout(() => setSaveStatus({ type: "", message: "" }), 3000);
} catch (error) {
console.error("Error saving STT settings:", error);
setSaveStatus({ type: "error", message: t("settings.saveError") });
} finally {
setIsSavingStt(false);
}
}, [sttConfig, sttApiKey, sttModelId, t]);
const filteredModels = useMemo(() => {
const term = searchTerm.toLowerCase();
if (!term) return customModels;
@@ -931,6 +963,142 @@ export function SettingsPage({
</CardContent>
</Card>
{/* ElevenLabs STT Configuration (shown when adapter provided) */}
{sttConfig && (
<Card>
<CardHeader>
<CardTitle className="flex items-center gap-2">
<Mic className="h-5 w-5" />
{language === "zh"
? "ElevenLabs 语音转文本"
: "ElevenLabs Speech-to-Text"}
</CardTitle>
<CardDescription>
{language === "zh"
? "配置 ElevenLabs API 密钥以启用语音注释功能"
: "Configure ElevenLabs API key to enable voice annotation feature"}
</CardDescription>
</CardHeader>
<CardContent className="space-y-4">
<div className="space-y-2">
<Label htmlFor="sttApiKey">
{language === "zh" ? "API 密钥" : "API Key"}
<span className="text-destructive ml-1">*</span>
</Label>
<div className="relative">
<Input
id="sttApiKey"
type={showSttKey ? "text" : "password"}
value={sttApiKey}
onChange={(e) => setSttApiKey(e.target.value)}
placeholder="xi-..."
className="pr-10"
/>
<Button
type="button"
variant="ghost"
size="icon"
onClick={() => setShowSttKey(!showSttKey)}
className="absolute right-0 top-0 h-full px-3 py-2 hover:bg-transparent"
>
{showSttKey ? (
<EyeOff className="h-4 w-4" />
) : (
<Eye className="h-4 w-4" />
)}
</Button>
</div>
<p className="text-xs text-muted-foreground">
{language === "zh"
? "在 ElevenLabs 获取 API 密钥:"
: "Get your API key from ElevenLabs:"}{" "}
<a
href="https://elevenlabs.io/app/developers/api-keys"
target="_blank"
rel="noopener noreferrer"
className="text-primary hover:underline inline-flex items-center gap-1"
>
elevenlabs.io
<ExternalLink className="h-3 w-3" />
</a>
</p>
</div>
<div className="space-y-2">
<Label htmlFor="sttModelId">
{language === "zh"
? "模型 ID可选"
: "Model ID (Optional)"}
</Label>
<Input
id="sttModelId"
type="text"
value={sttModelId}
onChange={(e) => setSttModelId(e.target.value)}
placeholder={
language === "zh"
? "留空使用默认模型"
: "Leave blank to use default model"
}
/>
<p className="text-xs text-muted-foreground">
{language === "zh"
? "默认使用通用多语言模型。如需指定特定模型,请输入模型 ID。"
: "Default uses the general multilingual model. Specify a model ID if needed."}
</p>
</div>
<div className="flex gap-2">
<Button
onClick={handleSaveStt}
disabled={isSavingStt}
size="sm"
>
{isSavingStt ? (
<>
<div className="animate-spin rounded-full h-4 w-4 border-2 border-white border-t-transparent mr-2" />
{language === "zh" ? "保存中..." : "Saving..."}
</>
) : (
<>{language === "zh" ? "保存配置" : "Save Configuration"}</>
)}
</Button>
{sttApiKey && (
<Button
variant="outline"
size="sm"
onClick={() => {
setSttApiKey("");
setSaveStatus({
type: "info",
message:
language === "zh"
? "已清空,点击保存以生效"
: "Cleared. Click Save to apply.",
});
}}
>
{language === "zh" ? "清空" : "Clear"}
</Button>
)}
</div>
{sttApiKey && (
<Alert>
<AlertDescription className="flex items-center gap-2">
<CheckCircle className="h-4 w-4 text-green-600" />
<span className="text-sm">
{language === "zh"
? "API 密钥已配置"
: "API key is configured"}
</span>
</AlertDescription>
</Alert>
)}
</CardContent>
</Card>
)}
{/* About Us Section */}
<Card>
<CardHeader>
@@ -1472,4 +1640,4 @@ export function SettingsPage({
);
}
export type { SettingsPageProps } from "./types";
export type { SettingsPageProps, STTConfigAdapter } from "./types";

View File

@@ -5,6 +5,16 @@ import type {
} from "@aipexstudio/aipex-core";
import type { ReactNode } from "react";
/**
* Callbacks for loading / saving ElevenLabs Speech-to-Text configuration.
* Keys are stored separately from the main settings blob so that VoiceInput
* can read them without loading the full settings.
*/
export interface STTConfigAdapter {
load: () => Promise<{ apiKey: string; modelId: string }>;
save: (values: { apiKey: string; modelId: string }) => Promise<void>;
}
export interface SettingsPageProps {
storageAdapter: KeyValueStorage<unknown>;
storageKey?: string;
@@ -12,6 +22,8 @@ export interface SettingsPageProps {
onSave?: (settings: AppSettings) => void;
onTestConnection?: (settings: AppSettings) => Promise<boolean>;
skillsContent?: ReactNode;
/** Optional ElevenLabs STT config adapter; when provided the STT card is shown. */
sttConfig?: STTConfigAdapter;
}
export interface ProviderConfig {

View File

@@ -66,7 +66,9 @@
"newChat": "New Chat",
"settings": "Settings",
"close": "Close",
"stopResponse": "Stop AI response"
"stopResponse": "Stop AI response",
"switchToVoice": "Switch to voice mode",
"switchToText": "Switch to text mode"
},
"mode": {
"focus": "Focus Mode",
@@ -101,10 +103,11 @@
"welcome": {
"title": "Welcome to AIpex",
"subtitle": "Choose a quick action or ask anything to get started",
"analyzePage": "Summarize this page",
"organizeTabs": "Please organize my open tabs by topic and purpose",
"analyzePage": "Summarize this page and save key points to clipboard",
"research": "Please use Google to research topic 'MCP'",
"comparePrice": "Compare the price of iPhone 17"
"screenRecording": "Start User Manual Guide creation",
"uxAuditGoal": "Deep audit your UX goals"
},
"config": {
"title": "Setup Required",

View File

@@ -66,7 +66,9 @@
"newChat": "新对话",
"settings": "设置",
"close": "关闭",
"stopResponse": "停止 AI 回应"
"stopResponse": "停止 AI 回应",
"switchToVoice": "切换到语音模式",
"switchToText": "切换到文字模式"
},
"mode": {
"focus": "聚焦模式",
@@ -101,10 +103,11 @@
"welcome": {
"title": "欢迎使用 AIpex",
"subtitle": "选择一个快捷操作或询问任何问题来开始",
"analyzePage": "总结此页面",
"organizeTabs": "请按主题和用途整理我的打开标签页",
"analyzePage": "总结此页面并将关键点保存到剪贴板",
"research": "请使用 Google 研究主题 'MCP'",
"comparePrice": "比较 iPhone 17 的价格"
"screenRecording": "开始创建用户操作手册",
"uxAuditGoal": "深度审计您的 UX 目标"
},
"config": {
"title": "需要配置",

View File

@@ -67,6 +67,8 @@ export interface TranslationResources {
settings: string;
close: string;
stopResponse: string;
switchToVoice: string;
switchToText: string;
};
mode: {
focus: string;
@@ -95,10 +97,11 @@ export interface TranslationResources {
welcome: {
title: string;
subtitle: string;
organizeTabs: string;
analyzePage: string;
organizeTabs: string;
research: string;
comparePrice: string;
screenRecording: string;
uxAuditGoal: string;
};
config: {
title: string;
@@ -244,6 +247,8 @@ export type BaseTranslationKey =
| "tooltip.settings"
| "tooltip.close"
| "tooltip.stopResponse"
| "tooltip.switchToVoice"
| "tooltip.switchToText"
| "mode.focus"
| "mode.background"
| "mode.selectMode"
@@ -262,10 +267,11 @@ export type BaseTranslationKey =
| "input.placeholder3"
| "welcome.title"
| "welcome.subtitle"
| "welcome.organizeTabs"
| "welcome.analyzePage"
| "welcome.organizeTabs"
| "welcome.research"
| "welcome.comparePrice"
| "welcome.screenRecording"
| "welcome.uxAuditGoal"
| "config.title"
| "config.description"
| "config.apiTokenRequired"

View File

@@ -120,8 +120,12 @@ export interface ChatbotSlots {
emptyState?: (props: WelcomeScreenProps) => ReactNode;
/** Custom loading indicator */
loadingIndicator?: () => ReactNode;
/** Content to render before all messages (e.g., update banners, announcements) */
beforeMessages?: () => ReactNode;
/** Content to render after all messages (for platform-specific features like interventions) */
afterMessages?: () => ReactNode;
/** Extra content rendered inside PromptInput (e.g., context/skill loaders) */
promptExtras?: () => ReactNode;
}
// ============ Components Configuration ============

View File

@@ -32,6 +32,7 @@ export type {
UIContextPart,
UIFilePart,
UIMessage,
UIMessageMetadata,
UIPart,
UIReasoningPart,
UIRole,

View File

@@ -65,11 +65,22 @@ export type UIPart =
| UIToolPart
| UIContextPart;
export interface UIMessageMetadata {
needLogin?: boolean;
needBuyToken?: boolean;
needChangeModel?: boolean;
supportedModels?: string[];
currentCredits?: number;
requiredCredits?: number;
errorCode?: string;
}
export interface UIMessage {
id: string;
role: UIRole;
parts: UIPart[];
timestamp?: number;
metadata?: UIMessageMetadata;
}
// ============ Context Item Types ============

View File

@@ -44,6 +44,14 @@
"matches": ["<all_urls>"]
}
],
"externally_connectable": {
"matches": [
"https://www.claudechrome.com/*",
"https://claudechrome.com/*",
"https://aipex.ing/*",
"http://localhost:*/*"
]
},
"host_permissions": ["https://*/*", "http://*/*", "<all_urls>"],
"commands": {
"open-aipex": {

View File

@@ -39,7 +39,37 @@ chrome.runtime.onInstalled.addListener((details) => {
}
});
// Handle messages for element capture relay
// =============================================================================
// Sidepanel port lifecycle
// =============================================================================
// Track whether a recording is active so we can clean up on disconnect
let isRecording = false;
chrome.runtime.onConnect.addListener((port) => {
if (port.name === "sidepanel") {
port.onDisconnect.addListener(() => {
// When sidepanel closes, stop capture on all tabs if recording was active
if (isRecording) {
isRecording = false;
chrome.tabs.query({}).then((tabs) => {
for (const tab of tabs) {
if (tab.id) {
chrome.tabs
.sendMessage(tab.id, { request: "stop-capture" })
.catch(() => {
/* tab may not have content script */
});
}
}
});
}
});
}
});
// =============================================================================
// Internal message router
// =============================================================================
chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
// Echo capture events to all extension contexts
if (message.request === "capture-click-event") {
@@ -70,7 +100,109 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
return true;
}
// Relay a message to the active tab's content script
if (message.request === "relay-to-active-tab") {
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
const tabId = tabs[0]?.id;
if (tabId && message.message) {
chrome.tabs
.sendMessage(tabId, message.message)
.then(() => sendResponse({ success: true }))
.catch((err) => {
sendResponse({
success: false,
error: err instanceof Error ? err.message : String(err),
});
});
} else {
sendResponse({ success: false, error: "No active tab" });
}
});
return true;
}
// Recording lifecycle markers
if (message.request === "start-recording") {
isRecording = true;
sendResponse({ success: true });
return true;
}
if (message.request === "stop-recording") {
isRecording = false;
sendResponse({ success: true });
return true;
}
return false;
});
// =============================================================================
// External Message Listener - Website Integration
// =============================================================================
// Origin verification is handled by manifest.json's externally_connectable
chrome.runtime.onMessageExternal.addListener(
(message, sender, sendResponse) => {
// Handle "openWithPrompt" action from website
if (message.action === "openWithPrompt") {
const prompt = message.prompt;
if (!prompt || typeof prompt !== "string") {
sendResponse({ success: false, error: "Invalid prompt" });
return true;
}
// Save prompt to chrome.storage.local with timestamp
chrome.storage.local.set(
{
"aipex-pending-prompt": prompt,
"aipex-pending-prompt-timestamp": Date.now(),
},
() => {
if (chrome.runtime.lastError) {
sendResponse({
success: false,
error: chrome.runtime.lastError.message,
});
return;
}
// Open sidepanel
const windowId = sender.tab?.windowId;
if (!windowId) {
chrome.windows
.getCurrent()
.then((window) => {
if (window.id) {
return chrome.sidePanel.open({ windowId: window.id });
}
throw new Error("No window ID available");
})
.then(() => {
sendResponse({ success: true });
})
.catch((error) => {
sendResponse({ success: false, error: error.message });
});
} else {
chrome.sidePanel
.open({ windowId })
.then(() => {
sendResponse({ success: true });
})
.catch((error) => {
sendResponse({ success: false, error: error.message });
});
}
},
);
return true; // Keep message channel open for async response
}
sendResponse({ success: false, error: "Unknown action" });
return true;
},
);
console.log("AIPex background service worker started");

View File

@@ -10,10 +10,16 @@ import { getRuntime } from "@aipexstudio/aipex-react/lib/runtime";
import { cn } from "@aipexstudio/aipex-react/lib/utils";
import type { HeaderProps } from "@aipexstudio/aipex-react/types";
import { conversationStorage } from "@aipexstudio/browser-runtime";
import { PlusIcon, SettingsIcon } from "lucide-react";
import {
KeyboardIcon,
MicIcon,
PlusIcon,
SettingsIcon,
} from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { UserProfile, useAuth } from "../auth";
import { ConversationHistory } from "./conversation-history";
import { useInputMode } from "./input-mode-context";
import { fromStorageFormat, toStorageFormat } from "./message-adapter";
export function BrowserChatHeader({
@@ -126,6 +132,12 @@ export function BrowserChatHeader({
onNewChat?.();
}, [onNewChat]);
const { inputMode, setInputMode } = useInputMode();
const toggleInputMode = useCallback(() => {
setInputMode(inputMode === "voice" ? "text" : "voice");
}, [inputMode, setInputMode]);
return (
<div
className={cn(
@@ -134,26 +146,47 @@ export function BrowserChatHeader({
)}
{...props}
>
{/* Left side - Settings */}
<Button
variant="ghost"
size="sm"
onClick={handleOpenOptions}
className="gap-2"
>
<SettingsIcon className="size-4" />
{t("common.settings")}
</Button>
{/* Left side - Settings + Voice/Text toggle + History */}
<div className="flex items-center gap-1">
<Button
variant="ghost"
size="icon"
onClick={handleOpenOptions}
title={t("tooltip.settings")}
className="size-8"
>
<SettingsIcon className="size-4" />
</Button>
{/* Center - History */}
<ConversationHistory
currentConversationId={currentConversationId}
onConversationSelect={handleConversationSelect}
onNewConversation={handleNewChat}
/>
{/* Voice / Text toggle */}
<Button
variant="ghost"
size="icon"
onClick={toggleInputMode}
title={
inputMode === "voice"
? t("tooltip.switchToText")
: t("tooltip.switchToVoice")
}
className="size-8"
>
{inputMode === "voice" ? (
<KeyboardIcon className="size-4" />
) : (
<MicIcon className="size-4" />
)}
</Button>
{/* Conversation History */}
<ConversationHistory
currentConversationId={currentConversationId}
onConversationSelect={handleConversationSelect}
onNewConversation={handleNewChat}
/>
</div>
{/* Right side - New Chat and User Profile */}
<div className="flex items-center gap-2">
<div className="flex items-center gap-1">
<Button
variant="ghost"
size="sm"

View File

@@ -0,0 +1,47 @@
/**
* BrowserChatInputArea
* Renders VoiceInput when inputMode is "voice", otherwise the default text InputArea.
*/
import {
DefaultInputArea,
type ExtendedInputAreaProps,
} from "@aipexstudio/aipex-react/components/chatbot/components";
import { VoiceInput } from "@aipexstudio/aipex-react/components/voice";
import type { InputAreaProps } from "@aipexstudio/aipex-react/types";
import { useCallback } from "react";
import { useInputMode } from "./input-mode-context";
export function BrowserChatInputArea(props: InputAreaProps) {
const { inputMode, setInputMode } = useInputMode();
const handleTranscript = useCallback(
(text: string) => {
// Send the transcribed text as a message
props.onSubmit(text);
},
[props.onSubmit],
);
const handleSwitchToText = useCallback(() => {
setInputMode("text");
}, [setInputMode]);
if (inputMode === "voice") {
const isStreaming =
props.status === "streaming" || props.status === "submitted";
return (
<div className="flex-1 overflow-hidden">
<VoiceInput
onTranscript={handleTranscript}
isPaused={isStreaming}
onSwitchToText={handleSwitchToText}
/>
</div>
);
}
// Text mode: render the default input area, forwarding all props
return <DefaultInputArea {...(props as ExtendedInputAreaProps)} />;
}

View File

@@ -0,0 +1,90 @@
/**
* BrowserContextLoader
* Rendered inside PromptInput (via the promptExtras slot) to populate
* available contexts (tabs, bookmarks, current page) and available skills.
*
* This component renders nothing visible; it only syncs data from
* browser-runtime providers into the PromptInput context hooks.
*/
import {
usePromptInputContexts,
usePromptInputSkills,
type SkillItem,
} from "@aipexstudio/aipex-react/components/ai-elements/prompt-input";
import { skillManager, skillStorage } from "@aipexstudio/browser-runtime";
import type { SkillMetadata } from "@aipexstudio/browser-runtime";
import { useEffect } from "react";
import { useTabsSync } from "../hooks/use-tabs-sync";
export function BrowserContextLoader() {
const contexts = usePromptInputContexts();
const skills = usePromptInputSkills();
// Sync contexts from tab/bookmark/page providers
useTabsSync({
onContextsUpdate: (availableContexts) => {
contexts.setAvailableContexts(availableContexts);
},
onContextRemove: (contextId) => {
contexts.remove(contextId);
},
getSelectedContexts: () => {
return contexts.items;
},
debounceDelay: 300,
});
// Load skills and subscribe to skill changes
useEffect(() => {
const loadSkills = async () => {
try {
const allSkills: SkillMetadata[] = await skillStorage.listSkills();
const enabledSkills = allSkills.filter(
(skill: SkillMetadata) => skill.enabled,
);
const skillItems: SkillItem[] = enabledSkills.map(
(skill: SkillMetadata) => ({
id: skill.id,
name: skill.name,
description: skill.description,
}),
);
skills.setAvailableSkills(skillItems);
} catch (error) {
console.error("[BrowserContextLoader] Failed to load skills:", error);
}
};
// Initial load
void loadSkills();
// Subscribe to skill changes
const unsubscribeLoaded = skillManager.subscribe(
"skill_loaded",
() => void loadSkills(),
);
const unsubscribeUnloaded = skillManager.subscribe(
"skill_unloaded",
() => void loadSkills(),
);
const unsubscribeEnabled = skillManager.subscribe(
"skill_enabled",
() => void loadSkills(),
);
const unsubscribeDisabled = skillManager.subscribe(
"skill_disabled",
() => void loadSkills(),
);
return () => {
unsubscribeLoaded();
unsubscribeUnloaded();
unsubscribeEnabled();
unsubscribeDisabled();
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
return null;
}

View File

@@ -0,0 +1,22 @@
/**
* BrowserMessageList
* Wraps the default MessageList and hides it when inputMode is "voice",
* matching aipex's behaviour where messages are hidden in voice mode.
*/
import { DefaultMessageList } from "@aipexstudio/aipex-react/components/chatbot/components";
import type { MessageListProps } from "@aipexstudio/aipex-react/types";
import { useInputMode } from "./input-mode-context";
export function BrowserMessageList(
props: MessageListProps & { onSuggestionClick?: (text: string) => void },
) {
const { inputMode } = useInputMode();
// In voice mode, hide the message list (matching aipex full-screen voice behaviour)
if (inputMode === "voice") {
return null;
}
return <DefaultMessageList {...props} />;
}

View File

@@ -0,0 +1,78 @@
/**
* InputModeContext
* Shared context for voice/text input mode toggle, persisted in chrome.storage.local.
*/
import React, {
createContext,
useCallback,
useContext,
useEffect,
useState,
} from "react";
export type InputMode = "voice" | "text";
interface InputModeContextValue {
inputMode: InputMode;
setInputMode: (mode: InputMode) => void;
}
const InputModeContext = createContext<InputModeContextValue>({
inputMode: "text",
setInputMode: () => {},
});
const STORAGE_KEY = "aipex-input-mode";
export function InputModeProvider({
children,
}: {
children: React.ReactNode;
}) {
const [inputMode, setInputModeState] = useState<InputMode>("text");
// Load persisted value on mount
useEffect(() => {
chrome.storage.local.get(STORAGE_KEY).then((result) => {
const stored = result[STORAGE_KEY];
if (stored === "voice" || stored === "text") {
setInputModeState(stored);
}
}).catch(() => {
// storage may not be available yet
});
}, []);
// Listen for external changes (e.g. another instance)
useEffect(() => {
const listener = (
changes: Record<string, chrome.storage.StorageChange>,
areaName: string,
) => {
if (areaName === "local" && changes[STORAGE_KEY]) {
const newValue = changes[STORAGE_KEY].newValue;
if (newValue === "voice" || newValue === "text") {
setInputModeState(newValue);
}
}
};
chrome.storage.onChanged.addListener(listener);
return () => chrome.storage.onChanged.removeListener(listener);
}, []);
const setInputMode = useCallback((mode: InputMode) => {
setInputModeState(mode);
chrome.storage.local.set({ [STORAGE_KEY]: mode }).catch(() => {});
}, []);
return (
<InputModeContext.Provider value={{ inputMode, setInputMode }}>
{children}
</InputModeContext.Provider>
);
}
export function useInputMode() {
return useContext(InputModeContext);
}

View File

@@ -0,0 +1,45 @@
/**
* UpdateBannerWrapper
* Connects the platform-agnostic UpdateBanner component to Chrome extension
* version-checking services.
*/
import { useCallback } from "react";
import { UpdateBanner } from "@aipexstudio/aipex-react/components/chatbot";
import {
checkVersion,
dismissUpdate,
isUpdateDismissed,
openChangelog,
openUpdatePage,
requestUpdate,
} from "../services/version-checker";
export function UpdateBannerWrapper() {
const handleCheckVersion = useCallback(() => checkVersion(), []);
const handleIsUpdateDismissed = useCallback(
(version: string) => isUpdateDismissed(version),
[],
);
const handleDismissUpdate = useCallback(
(version: string) => dismissUpdate(version),
[],
);
const handleRequestUpdate = useCallback(() => requestUpdate(), []);
const handleOpenChangelog = useCallback(
(url: string) => openChangelog(url),
[],
);
const handleOpenUpdatePage = useCallback(() => openUpdatePage(), []);
return (
<UpdateBanner
onCheckVersion={handleCheckVersion}
onIsUpdateDismissed={handleIsUpdateDismissed}
onDismissUpdate={handleDismissUpdate}
onRequestUpdate={handleRequestUpdate}
onOpenChangelog={handleOpenChangelog}
onOpenUpdatePage={handleOpenUpdatePage}
/>
);
}

View File

@@ -11,11 +11,12 @@ import type { Language } from "@aipexstudio/aipex-react/i18n/types";
import { ThemeProvider } from "@aipexstudio/aipex-react/theme/context";
import type { Theme } from "@aipexstudio/aipex-react/theme/types";
import { ChromeStorageAdapter } from "@aipexstudio/browser-runtime";
import React, { useState } from "react";
import React, { useCallback, useEffect, useRef, useState } from "react";
import ReactDOM from "react-dom/client";
import { AuthProvider } from "../../auth";
import { chromeStorageAdapter } from "../../hooks";
import { AutomationModeInputToolbar } from "../../lib/automation-mode-toolbar";
import { UpdateBannerWrapper } from "../../lib/update-banner-wrapper";
import {
BROWSER_AGENT_CONFIG,
useBrowserContextProviders,
@@ -24,12 +25,100 @@ import {
useBrowserTools,
} from "../../lib/browser-agent-config";
import { BrowserChatHeader } from "../../lib/browser-chat-header";
import { BrowserChatInputArea } from "../../lib/browser-chat-input-area";
import { BrowserContextLoader } from "../../lib/browser-context-loader";
import { BrowserMessageList } from "../../lib/browser-message-list";
import { InputModeProvider } from "../../lib/input-mode-context";
import { InterventionModeProvider } from "../../lib/intervention-mode-context";
import { InterventionUI } from "../../lib/intervention-ui";
const i18nStorageAdapter = new ChromeStorageAdapter<Language>();
const themeStorageAdapter = new ChromeStorageAdapter<Theme>();
/**
* Reads and consumes a pending prompt saved by the openWithPrompt external
* message handler in the background service worker. Prompts older than 5 s
* are treated as expired and silently discarded.
*/
function usePendingPrompt() {
const [pendingInput, setPendingInput] = useState<string | undefined>(
undefined,
);
useEffect(() => {
const check = async () => {
try {
const result = await chrome.storage.local.get([
"aipex-pending-prompt",
"aipex-pending-prompt-timestamp",
]);
const prompt = result["aipex-pending-prompt"];
const timestamp = result["aipex-pending-prompt-timestamp"];
if (prompt && typeof prompt === "string") {
const now = Date.now();
// Only use prompts that are less than 5 seconds old
if (timestamp && now - timestamp < 5000) {
setPendingInput(prompt);
}
}
// Always clear storage regardless of expiry
if (prompt) {
chrome.storage.local.remove([
"aipex-pending-prompt",
"aipex-pending-prompt-timestamp",
]);
}
} catch {
// Silently ignore storage may not be available yet
}
};
check();
}, []);
return pendingInput;
}
/**
* Manages the "aipex-conversation-active" heartbeat in chrome.storage.local
* so content scripts can show the breathing border overlay while the AI is
* actively generating a response.
*/
function useConversationHeartbeat() {
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const start = useCallback(() => {
// Avoid duplicate intervals
if (intervalRef.current) return;
const tick = () => {
chrome.storage.local
.set({ "aipex-conversation-active": Date.now() })
.catch(() => {});
};
tick(); // Immediate first tick
intervalRef.current = setInterval(tick, 2000);
}, []);
const stop = useCallback(() => {
if (intervalRef.current) {
clearInterval(intervalRef.current);
intervalRef.current = null;
}
chrome.storage.local
.remove("aipex-conversation-active")
.catch(() => {});
}, []);
// Cleanup on unmount
useEffect(() => stop, [stop]);
return { start, stop };
}
function ChatApp() {
const { settings, isLoading } = useChatConfig({
storageAdapter: chromeStorageAdapter,
@@ -51,9 +140,56 @@ function ChatApp() {
...BROWSER_AGENT_CONFIG,
});
const pendingInput = usePendingPrompt();
const heartbeat = useConversationHeartbeat();
const handleStatusChange = useCallback(
(status: string) => {
if (status === "streaming" || status === "submitted") {
heartbeat.start();
} else {
heartbeat.stop();
}
},
[heartbeat],
);
const [interventionMode, setInterventionMode] =
useState<InterventionMode>("passive");
// Sidepanel lifecycle: port connection + cleanup on hide/close
useEffect(() => {
// Long-lived port so the background can detect sidepanel disconnect
const port = chrome.runtime.connect({ name: "sidepanel" });
const handleVisibilityChange = () => {
if (document.hidden) {
// Stop any active recording
chrome.runtime
.sendMessage({ request: "stop-recording" })
.catch(() => {
/* background may be busy */
});
// Stop element capture on the active tab
chrome.runtime
.sendMessage({
request: "relay-to-active-tab",
message: { request: "stop-capture" },
})
.catch(() => {
/* tab may be closed */
});
}
};
document.addEventListener("visibilitychange", handleVisibilityChange);
return () => {
document.removeEventListener("visibilitychange", handleVisibilityChange);
port.disconnect();
};
}, []);
if (isLoading) {
return (
<div className="flex h-full items-center justify-center">
@@ -63,29 +199,39 @@ function ChatApp() {
}
return (
<InterventionModeProvider
mode={interventionMode}
setMode={setInterventionMode}
>
<ChatBot
agent={agent}
configError={error}
initialSettings={settings}
storageAdapter={chromeStorageAdapter}
components={{
Header: BrowserChatHeader,
}}
slots={{
afterMessages: () => (
<InterventionUI
mode={interventionMode}
onModeChange={setInterventionMode}
/>
),
inputToolbar: (props) => <AutomationModeInputToolbar {...props} />,
}}
/>
</InterventionModeProvider>
<InputModeProvider>
<InterventionModeProvider
mode={interventionMode}
setMode={setInterventionMode}
>
<ChatBot
agent={agent}
configError={error}
initialSettings={settings}
storageAdapter={chromeStorageAdapter}
initialInput={pendingInput}
handlers={{
onStatusChange: handleStatusChange,
}}
components={{
Header: BrowserChatHeader,
MessageList: BrowserMessageList,
InputArea: BrowserChatInputArea,
}}
slots={{
beforeMessages: () => <UpdateBannerWrapper />,
afterMessages: () => (
<InterventionUI
mode={interventionMode}
onModeChange={setInterventionMode}
/>
),
inputToolbar: (props) => <AutomationModeInputToolbar {...props} />,
promptExtras: () => <BrowserContextLoader />,
}}
/>
</InterventionModeProvider>
</InputModeProvider>
);
}

View File

@@ -339,6 +339,102 @@ const ContentApp = () => {
);
};
// ============================================================================
// Breathing Border Overlay — mounted OUTSIDE shadow DOM so z-index works
// against page elements. Driven by the "aipex-conversation-active" storage key
// which the sidepanel writes as a heartbeat.
// ============================================================================
const HEARTBEAT_KEY = "aipex-conversation-active";
const HEARTBEAT_TTL_MS = 6_000; // Hide overlay if heartbeat is stale (>6 s)
function BorderOverlayApp() {
const [visible, setVisible] = React.useState(false);
const handleConversationState = React.useCallback(
(timestamp: unknown) => {
if (typeof timestamp === "number" && Date.now() - timestamp < HEARTBEAT_TTL_MS) {
setVisible(true);
} else {
setVisible(false);
}
},
[],
);
React.useEffect(() => {
// Check on mount
chrome.storage.local.get(HEARTBEAT_KEY, (result) => {
handleConversationState(result[HEARTBEAT_KEY]);
});
// Listen for changes
const onChange = (
changes: Record<string, chrome.storage.StorageChange>,
area: string,
) => {
if (area === "local" && changes[HEARTBEAT_KEY]) {
handleConversationState(changes[HEARTBEAT_KEY].newValue);
}
};
chrome.storage.onChanged.addListener(onChange);
// Poll heartbeat staleness every 3 s
const interval = setInterval(() => {
chrome.storage.local.get(HEARTBEAT_KEY, (result) => {
handleConversationState(result[HEARTBEAT_KEY]);
});
}, 3000);
return () => {
chrome.storage.onChanged.removeListener(onChange);
clearInterval(interval);
};
}, [handleConversationState]);
if (!visible) return null;
return (
<>
<div
style={{
position: "fixed",
top: 0,
left: 0,
width: "100vw",
height: "100vh",
zIndex: 999998,
pointerEvents: "none",
animation: "aipexBreathe 2.5s ease-in-out infinite",
boxShadow: `
inset 0 0 15px 3px rgba(37, 99, 235, 0.5),
inset 0 0 25px 5px rgba(59, 130, 246, 0.4),
inset 0 0 35px 7px rgba(96, 165, 250, 0.3),
inset 0 0 45px 9px rgba(147, 197, 253, 0.2)
`,
}}
/>
<style>{`
@keyframes aipexBreathe {
0%, 100% {
box-shadow:
inset 0 0 12px 3px rgba(37,99,235,0.35),
inset 0 0 20px 5px rgba(59,130,246,0.28),
inset 0 0 28px 6px rgba(96,165,250,0.22),
inset 0 0 35px 8px rgba(147,197,253,0.15);
}
50% {
box-shadow:
inset 0 0 20px 5px rgba(37,99,235,0.7),
inset 0 0 30px 7px rgba(59,130,246,0.6),
inset 0 0 40px 9px rgba(96,165,250,0.5),
inset 0 0 50px 11px rgba(147,197,253,0.35);
}
}
`}</style>
</>
);
}
// Wait for DOM to be ready
if (document.readyState === "loading") {
document.addEventListener("DOMContentLoaded", initContentScript);
@@ -347,17 +443,15 @@ if (document.readyState === "loading") {
}
function initContentScript() {
// Mount the content script
// Mount the content script (shadow DOM for isolation)
const container = document.createElement("div");
container.id = "aipex-content-root";
document.body.appendChild(container);
// Create shadow DOM to isolate styles
const shadowRoot = container.attachShadow({ mode: "open" });
const shadowContainer = document.createElement("div");
shadowRoot.appendChild(shadowContainer);
// Inject Tailwind CSS into shadow DOM
const style = document.createElement("style");
style.textContent = `
:host {
@@ -367,11 +461,22 @@ function initContentScript() {
`;
shadowRoot.appendChild(style);
// Render the app
const root = ReactDOM.createRoot(shadowContainer);
root.render(
<React.StrictMode>
<ContentApp />
</React.StrictMode>,
);
// Mount breathing border overlay OUTSIDE shadow DOM so z-index works
const borderContainer = document.createElement("div");
borderContainer.id = "aipex-border-overlay";
document.body.appendChild(borderContainer);
const borderRoot = ReactDOM.createRoot(borderContainer);
borderRoot.render(
<React.StrictMode>
<BorderOverlayApp />
</React.StrictMode>,
);
}

View File

@@ -1,5 +1,6 @@
import type { AppSettings } from "@aipexstudio/aipex-core";
import { SettingsPage } from "@aipexstudio/aipex-react";
import type { STTConfigAdapter } from "@aipexstudio/aipex-react";
import { I18nProvider } from "@aipexstudio/aipex-react/i18n/context";
import type { Language } from "@aipexstudio/aipex-react/i18n/types";
import { ThemeProvider } from "@aipexstudio/aipex-react/theme/context";
@@ -18,6 +19,25 @@ import "../tailwind.css";
const i18nStorageAdapter = new ChromeStorageAdapter<Language>();
const themeStorageAdapter = new ChromeStorageAdapter<Theme>();
const chromeSttAdapter: STTConfigAdapter = {
load: async () => {
const result = await chrome.storage.local.get([
"elevenlabsApiKey",
"elevenlabsModelId",
]);
return {
apiKey: (result.elevenlabsApiKey as string) || "",
modelId: (result.elevenlabsModelId as string) || "",
};
},
save: async ({ apiKey, modelId }) => {
await chrome.storage.local.set({
elevenlabsApiKey: apiKey,
elevenlabsModelId: modelId,
});
},
};
function OptionsPageContent() {
const handleTestConnection = useCallback(async (settings: AppSettings) => {
try {
@@ -45,6 +65,7 @@ function OptionsPageContent() {
storageAdapter={chromeStorageAdapter}
onTestConnection={handleTestConnection}
skillsContent={<SkillsOptionsTab />}
sttConfig={chromeSttAdapter}
/>
</div>
);