feat: add local workflow skill suggestions (bosmain-nj6)

This commit is contained in:
flint
2026-05-08 18:31:14 -07:00
committed by Nikhil Sonti
parent d7e1125db3
commit 1af0378d98
7 changed files with 579 additions and 4 deletions

View File

@@ -1,4 +1,4 @@
import { useEffect, useRef, useState } from 'react'
import { useCallback, useEffect, useRef, useState } from 'react'
import {
type AgentHarnessStreamEvent,
attachToHarnessTurn,
@@ -16,8 +16,13 @@ import type {
} from '@/lib/agent-conversations/types'
import { useInvalidateAgentOutputs } from '@/lib/agent-files'
import type { ServerAttachmentPayload } from '@/lib/attachments'
import { sentry } from '@/lib/sentry/sentry'
import { consumeSSEStream } from '@/lib/sse'
import { buildToolLabel } from '@/lib/tool-labels'
import {
createWorkflowUsageRecord,
recordWorkflowUsage,
} from '@/lib/workflow-usage/storage'
import { mapAgentHarnessToolStatus } from './agent-stream-events'
export interface SendInput {
@@ -68,6 +73,8 @@ export function useAgentConversation(
const streamAbortRef = useRef<AbortController | null>(null)
const onCompleteRef = useRef(options.onComplete)
const onSessionKeyChangeRef = useRef(options.onSessionKeyChange)
const workflowToolNamesRef = useRef<string[]>([])
const workflowToolIdsRef = useRef(new Set<string>())
// Per-turn resume bookkeeping. `turnId` is captured from the response
// header; `lastSeq` advances with every SSE event so a reconnect can
// resume via Last-Event-ID.
@@ -112,6 +119,35 @@ export function useAgentConversation(
})
}
const resetWorkflowUsageCapture = useCallback(() => {
workflowToolNamesRef.current = []
workflowToolIdsRef.current = new Set()
}, [])
const persistWorkflowUsageCapture = useCallback(
(turnId?: string | null) => {
const toolNames = workflowToolNamesRef.current
if (toolNames.length === 0) return
void recordWorkflowUsage(
createWorkflowUsageRecord({
id: `agent-harness-turn:${turnId ?? crypto.randomUUID()}`,
source: 'agent-harness-chat',
toolNames,
}),
).catch((error) => {
sentry.captureException(error, {
extra: {
message: 'Failed to persist agent workflow usage pattern',
agentId,
turnId,
},
})
})
},
[agentId],
)
const appendTextDelta = (delta: string) => {
textAccRef.current += delta
const text = textAccRef.current
@@ -174,6 +210,11 @@ export function useAgentConversation(
const upsertAgentHarnessTool = (event: AgentHarnessStreamEvent) => {
if (event.type !== 'tool_call') return
const rawName = event.title || event.rawType || 'tool call'
const toolId = event.id ?? rawName
if (!workflowToolIdsRef.current.has(toolId)) {
workflowToolIdsRef.current.add(toolId)
workflowToolNamesRef.current.push(rawName)
}
const { label, subject } = buildToolLabel(
rawName,
event.text ? { description: event.text } : undefined,
@@ -295,6 +336,7 @@ export function useAgentConversation(
streamAbortRef.current = abortController
setStreaming(true)
weStartedStream = true
resetWorkflowUsageCapture()
const response = await attachToHarnessTurn(agentId, {
turnId: active.turnId,
@@ -328,6 +370,7 @@ export function useAgentConversation(
// itself, so resetting here would only cause a brief flicker.
if (!cancelled && weStartedStream) {
const finishedTurnId = turnIdRef.current
persistWorkflowUsageCapture(finishedTurnId)
turnIdRef.current = null
lastSeqRef.current = null
setStreaming(false)
@@ -344,7 +387,12 @@ export function useAgentConversation(
cancelled = true
abortController.abort()
}
}, [agentId, activeTurnIdDep])
}, [
agentId,
activeTurnIdDep,
persistWorkflowUsageCapture,
resetWorkflowUsageCapture,
])
/**
* Send the chat request and follow the 409-active-turn redirect
@@ -422,6 +470,7 @@ export function useAgentConversation(
}
setTurns((prev) => [...prev, turn])
setStreaming(true)
resetWorkflowUsageCapture()
textAccRef.current = ''
thinkAccRef.current = ''
const abortController = new AbortController()
@@ -466,6 +515,7 @@ export function useAgentConversation(
// useAgentTurnFiles consumers also flush, not just the agent-wide
// rail query.
const finishedTurnId = turnIdRef.current
persistWorkflowUsageCapture(finishedTurnId)
turnIdRef.current = null
lastSeqRef.current = null
onCompleteRef.current?.()

View File

@@ -48,6 +48,18 @@ import {
normalizeToolApprovalConfig,
toolApprovalConfigStorage,
} from '@/lib/tool-approvals/storage'
import {
analyzeWorkflowUsage,
detectWorkflowAdvisorCommand,
formatWorkflowAnalysisResponse,
formatWorkflowUsageClearedResponse,
formatWorkflowUsageDataResponse,
type WorkflowAdvisorCommand,
} from '@/lib/workflow-usage/advisor'
import {
clearWorkflowUsageRecords,
getWorkflowUsageRecords,
} from '@/lib/workflow-usage/storage'
import { selectedWorkspaceStorage } from '@/lib/workspace/workspace-storage'
import type { ChatMode } from './chatTypes'
import { GetConversationWithMessagesDocument } from './graphql/chatSessionDocument'
@@ -133,6 +145,7 @@ export interface ChatSessionOptions {
}
const NEWTAB_SYSTEM_PROMPT = `IMPORTANT: The user is chatting from the New Tab page. When performing browser actions, ALWAYS open content in a NEW TAB rather than navigating the current tab. The user's new tab page should remain accessible.`
const WORKFLOW_ADVISOR_LOCAL_ONLY = 'workflow-advisor'
const getUserSystemPrompt = (
origin: ChatOrigin | undefined,
@@ -142,6 +155,25 @@ const getUserSystemPrompt = (
? [personalization, NEWTAB_SYSTEM_PROMPT].filter(Boolean).join('\n\n')
: personalization
const createTextMessage = (
role: 'user' | 'assistant',
text: string,
options?: { localOnly?: boolean },
): UIMessage =>
({
id: crypto.randomUUID(),
role,
parts: [{ type: 'text', text }],
metadata: options?.localOnly
? { browserosLocalOnly: WORKFLOW_ADVISOR_LOCAL_ONLY }
: undefined,
}) as UIMessage
const isWorkflowAdvisorLocalOnlyMessage = (message: UIMessage): boolean => {
const metadata = (message as { metadata?: Record<string, unknown> }).metadata
return metadata?.browserosLocalOnly === WORKFLOW_ADVISOR_LOCAL_ONLY
}
const buildRequestBrowserContext = ({
activeTab,
action,
@@ -376,7 +408,9 @@ export const useChatSession = (options?: ChatSessionOptions) => {
Feature.PREVIOUS_CONVERSATION_ARRAY,
)
const previousMessages = messagesRef.current
const previousMessages = messagesRef.current.filter(
(message) => !isWorkflowAdvisorLocalOnlyMessage(message),
)
const history =
previousMessages.length > 0
? formatConversationHistory(previousMessages)
@@ -559,7 +593,9 @@ export const useChatSession = (options?: ChatSessionOptions) => {
})
}
const messagesToSave = messages.filter((m) => m.parts?.length > 0)
const messagesToSave = messages.filter(
(m) => m.parts?.length > 0 && !isWorkflowAdvisorLocalOnlyMessage(m),
)
if (messagesToSave.length === 0) return
if (isLoggedIn) {
@@ -645,6 +681,54 @@ export const useChatSession = (options?: ChatSessionOptions) => {
action?: ChatAction
} | null>(null)
const appendLocalWorkflowAdvisorExchange = (
userText: string,
responseText: string,
) => {
const nextMessages = [
...messagesRef.current,
createTextMessage('user', userText, { localOnly: true }),
createTextMessage('assistant', responseText, { localOnly: true }),
]
messagesRef.current = nextMessages
setMessages(nextMessages)
}
const handleWorkflowAdvisorCommand = async (
text: string,
command: WorkflowAdvisorCommand,
) => {
try {
if (command === 'clear') {
await clearWorkflowUsageRecords()
appendLocalWorkflowAdvisorExchange(
text,
formatWorkflowUsageClearedResponse(),
)
return
}
const records = await getWorkflowUsageRecords()
const response =
command === 'view'
? formatWorkflowUsageDataResponse(records)
: formatWorkflowAnalysisResponse(analyzeWorkflowUsage(records))
appendLocalWorkflowAdvisorExchange(text, response)
} catch (error) {
sentry.captureException(error, {
extra: {
message: 'Failed to run local workflow advisor command',
command,
},
})
appendLocalWorkflowAdvisorExchange(
text,
"I couldn't read the local workflow usage patterns. Nothing was sent to a model or external service.",
)
}
}
const dispatchMessage = useCallback(
(text: string) => {
startExecutionTask({
@@ -696,6 +780,12 @@ export const useChatSession = (options?: ChatSessionOptions) => {
selectedLlmProvider?.modelId,
})
const workflowAdvisorCommand = detectWorkflowAdvisorCommand(params.text)
if (workflowAdvisorCommand) {
void handleWorkflowAdvisorCommand(params.text, workflowAdvisorCommand)
return
}
if (!isIntegrationsSyncedRef.current) {
// Queue the message — will be sent when sync completes
pendingMessageRef.current = params

View File

@@ -10,6 +10,10 @@ import type {
ExecutionTaskStatus,
} from '@/lib/execution-history/types'
import { sentry } from '@/lib/sentry/sentry'
import {
createWorkflowUsageRecordFromExecutionTask,
recordWorkflowUsage,
} from '@/lib/workflow-usage/storage'
interface StartExecutionTaskInput {
conversationId: string
@@ -145,6 +149,17 @@ export function useExecutionHistoryTracker() {
}
persistTask(nextTask)
void recordWorkflowUsage(
createWorkflowUsageRecordFromExecutionTask(nextTask),
).catch((error) => {
sentry.captureException(error, {
extra: {
message: 'Failed to persist workflow usage pattern',
conversationId: nextTask.conversationId,
taskId: nextTask.id,
},
})
})
activeTaskRef.current = null
},
[persistTask],

View File

@@ -0,0 +1,73 @@
import { describe, expect, it } from 'bun:test'
import {
analyzeWorkflowUsage,
detectWorkflowAdvisorCommand,
formatWorkflowAnalysisResponse,
normalizeToolSequence,
} from './advisor'
import type { WorkflowUsageRecord } from './types'
describe('workflow usage advisor', () => {
it('detects explicit workflow advisor commands only', () => {
expect(detectWorkflowAdvisorCommand('analyze my workflow')).toBe('analyze')
expect(detectWorkflowAdvisorCommand('what patterns do you see?')).toBe(
'analyze',
)
expect(detectWorkflowAdvisorCommand('show workflow usage data')).toBe(
'view',
)
expect(detectWorkflowAdvisorCommand('clear skill suggestion data')).toBe(
'clear',
)
expect(detectWorkflowAdvisorCommand('summarize this page')).toBeNull()
})
it('normalizes command sequences without retaining repeated adjacent tools', () => {
expect(normalizeToolSequence([' new_page ', 'new_page', 'open'])).toEqual([
'new_page',
'open',
])
})
it('suggests repeated local tool-name patterns', () => {
const analysis = analyzeWorkflowUsage([
record('1', ['new_page', 'navigate', 'get_page_content'], 100),
record('2', ['new_page', 'navigate', 'get_page_content'], 200),
record('3', ['search', 'open'], 300),
])
expect(analysis.totalRuns).toBe(3)
expect(analysis.suggestions).toHaveLength(1)
expect(analysis.suggestions[0]).toMatchObject({
runCount: 2,
pattern: ['new_page', 'navigate', 'get_page_content'],
})
})
it('formats concrete suggestions with a privacy note', () => {
const response = formatWorkflowAnalysisResponse(
analyzeWorkflowUsage([
record('1', ['new_page', 'navigate', 'get_page_content'], 100),
record('2', ['new_page', 'navigate', 'get_page_content'], 200),
]),
)
expect(response).toContain('Pattern: Open page -> Navigate -> Read page')
expect(response).toContain('Create a "Open Page to Read Page Skill" skill')
expect(response).toContain('does not include URLs')
expect(response).toContain('tool inputs')
})
})
function record(
id: string,
toolNames: string[],
recordedAt: number,
): WorkflowUsageRecord {
return {
id,
source: 'sidepanel-chat',
recordedAt,
toolNames,
}
}

View File

@@ -0,0 +1,241 @@
import type {
WorkflowSkillSuggestion,
WorkflowUsageAnalysis,
WorkflowUsageRecord,
} from './types'
export type WorkflowAdvisorCommand = 'analyze' | 'view' | 'clear'
const DEFAULT_MIN_RUNS = 2
const DEFAULT_SUGGESTION_LIMIT = 3
const MIN_PATTERN_LENGTH = 2
const MAX_PATTERN_LENGTH = 8
const PRIVACY_NOTE =
'This analysis uses only local tool-name sequences. BrowserOS does not include URLs, page content, prompts, tool inputs, or tool outputs in this workflow pattern data.'
const TOOL_LABELS: Record<string, string> = {
click: 'Click',
extract_data: 'Extract data',
filesystem_read: 'Read file',
filesystem_write: 'Write file',
get_page_content: 'Read page',
new_page: 'Open page',
navigate: 'Navigate',
open: 'Open',
screenshot: 'Screenshot',
search: 'Search',
type: 'Type',
}
function normalizeCommandText(text: string): string {
return text.toLowerCase().replace(/\s+/g, ' ').trim()
}
export function detectWorkflowAdvisorCommand(
text: string,
): WorkflowAdvisorCommand | null {
const normalized = normalizeCommandText(text)
if (!normalized) return null
const mentionsWorkflowData =
normalized.includes('workflow usage') ||
normalized.includes('usage pattern') ||
normalized.includes('workflow pattern') ||
normalized.includes('skill suggestion')
if (
mentionsWorkflowData &&
/\b(clear|delete|reset|forget)\b/.test(normalized)
) {
return 'clear'
}
if (
mentionsWorkflowData &&
/\b(show|view|list|display|what)\b/.test(normalized)
) {
return 'view'
}
if (
normalized.includes('analyze my workflow') ||
normalized.includes('analyse my workflow') ||
normalized.includes('what patterns do you see') ||
normalized.includes('suggest skills') ||
normalized.includes('find skill suggestions') ||
normalized.includes('what can be automated') ||
normalized.includes('analyze workflow patterns') ||
normalized.includes('analyse workflow patterns')
) {
return 'analyze'
}
return null
}
function normalizeToolName(toolName: string): string {
return toolName.trim().toLowerCase()
}
export function normalizeToolSequence(toolNames: string[]): string[] {
const sequence: string[] = []
for (const rawName of toolNames) {
const toolName = normalizeToolName(rawName)
if (!toolName) continue
if (sequence[sequence.length - 1] === toolName) continue
sequence.push(toolName)
}
return sequence.slice(0, MAX_PATTERN_LENGTH)
}
function labelTool(toolName: string): string {
const normalized = normalizeToolName(toolName)
const known = TOOL_LABELS[normalized]
if (known) return known
return normalized
.replace(/^(browser|tool|mcp)[_-]/, '')
.replace(/[_-]+/g, ' ')
.replace(/\b\w/g, (char) => char.toUpperCase())
}
function titleCase(value: string): string {
return value.replace(/\b\w/g, (char) => char.toUpperCase())
}
function buildSuggestionTitle(pattern: string[]): string {
const labels = pattern.map((toolName) => titleCase(labelTool(toolName)))
const uniqueLabels = Array.from(new Set(labels))
if (uniqueLabels.length <= 2) return `${uniqueLabels.join(' + ')} Skill`
return `${uniqueLabels[0]} to ${uniqueLabels.at(-1)} Skill`
}
function buildBenefit(pattern: string[]): string {
const patternLabel = pattern.map(labelTool).join(' -> ')
return `Turn the repeated ${patternLabel} sequence into reusable skill instructions.`
}
function compareSuggestions(
left: WorkflowSkillSuggestion,
right: WorkflowSkillSuggestion,
): number {
return (
right.runCount - left.runCount ||
right.pattern.length - left.pattern.length ||
right.lastUsedAt - left.lastUsedAt
)
}
export function analyzeWorkflowUsage(
records: WorkflowUsageRecord[],
options?: { minRuns?: number; limit?: number },
): WorkflowUsageAnalysis {
const minRuns = options?.minRuns ?? DEFAULT_MIN_RUNS
const limit = options?.limit ?? DEFAULT_SUGGESTION_LIMIT
const groups = new Map<
string,
{ pattern: string[]; runCount: number; lastUsedAt: number }
>()
for (const record of records) {
const pattern = normalizeToolSequence(record.toolNames)
if (pattern.length < MIN_PATTERN_LENGTH) continue
const key = pattern.join('\u001f')
const existing = groups.get(key)
groups.set(key, {
pattern,
runCount: (existing?.runCount ?? 0) + 1,
lastUsedAt: Math.max(existing?.lastUsedAt ?? 0, record.recordedAt),
})
}
const suggestions = Array.from(groups.values())
.filter((group) => group.runCount >= minRuns)
.map((group, index): WorkflowSkillSuggestion => {
const pattern = group.pattern
return {
id: `workflow-${index + 1}`,
title: buildSuggestionTitle(pattern),
runCount: group.runCount,
pattern,
lastUsedAt: group.lastUsedAt,
benefit: buildBenefit(pattern),
}
})
.sort(compareSuggestions)
.slice(0, limit)
return {
totalRuns: records.length,
eligibleRuns: Array.from(groups.values()).reduce(
(count, group) => count + group.runCount,
0,
),
suggestions,
}
}
export function formatWorkflowAnalysisResponse(
analysis: WorkflowUsageAnalysis,
): string {
if (analysis.suggestions.length === 0) {
return [
"I don't have enough repeated local tool patterns to suggest a custom skill yet.",
'',
PRIVACY_NOTE,
'',
`Tracked runs: ${analysis.totalRuns}. Eligible repeated-tool runs: ${analysis.eligibleRuns}.`,
].join('\n')
}
const suggestionLines = analysis.suggestions.map((suggestion, index) =>
[
`${index + 1}. ${suggestion.title} -> ${suggestion.runCount} times`,
` Pattern: ${suggestion.pattern.map(labelTool).join(' -> ')}`,
` Suggestion: Create a "${suggestion.title}" skill for this exact command sequence.`,
` Benefit: ${suggestion.benefit}`,
].join('\n'),
)
return [
`Found ${analysis.suggestions.length} potential automation${analysis.suggestions.length === 1 ? '' : 's'}:`,
'',
...suggestionLines,
'',
PRIVACY_NOTE,
].join('\n')
}
export function formatWorkflowUsageDataResponse(
records: WorkflowUsageRecord[],
): string {
if (records.length === 0) {
return [
'No local workflow usage patterns are stored yet.',
'',
PRIVACY_NOTE,
].join('\n')
}
const recentRecords = records
.slice()
.sort((left, right) => right.recordedAt - left.recordedAt)
.slice(0, 10)
return [
`Stored local workflow runs: ${records.length}`,
'',
...recentRecords.map(
(record, index) =>
`${index + 1}. ${normalizeToolSequence(record.toolNames).map(labelTool).join(' -> ')}`,
),
'',
PRIVACY_NOTE,
].join('\n')
}
export function formatWorkflowUsageClearedResponse(): string {
return 'Cleared the local workflow usage pattern data. No URLs, page content, prompts, tool inputs, or tool outputs were stored in this data.'
}

View File

@@ -0,0 +1,78 @@
import { storage } from '@wxt-dev/storage'
import type { ExecutionTaskRecord } from '@/lib/execution-history/types'
import { normalizeToolSequence } from './advisor'
import type {
WorkflowUsageRecord,
WorkflowUsageSource,
WorkflowUsageStore,
} from './types'
const MAX_WORKFLOW_USAGE_RECORDS = 300
export const workflowUsageStorage = storage.defineItem<WorkflowUsageStore>(
'local:workflowUsagePatterns',
{
fallback: { version: 1, records: [] },
version: 1,
},
)
export function createWorkflowUsageRecord(input: {
id: string
source: WorkflowUsageSource
toolNames: string[]
recordedAt?: number
}): WorkflowUsageRecord | null {
const toolNames = normalizeToolSequence(input.toolNames)
if (toolNames.length === 0) return null
return {
id: input.id,
source: input.source,
recordedAt: input.recordedAt ?? Date.now(),
toolNames,
}
}
export function createWorkflowUsageRecordFromExecutionTask(
task: ExecutionTaskRecord,
): WorkflowUsageRecord | null {
return createWorkflowUsageRecord({
id: `execution-task:${task.id}`,
source: 'sidepanel-chat',
recordedAt: Date.parse(task.completedAt ?? task.startedAt),
toolNames: task.steps.map((step) => step.toolName),
})
}
export async function recordWorkflowUsage(
record: WorkflowUsageRecord | null,
): Promise<void> {
if (!record) return
const current = (await workflowUsageStorage.getValue()) ?? {
version: 1,
records: [],
}
const recordsById = new Map(
current.records.map((existing) => [existing.id, existing]),
)
recordsById.set(record.id, record)
const records = Array.from(recordsById.values())
.sort((left, right) => left.recordedAt - right.recordedAt)
.slice(-MAX_WORKFLOW_USAGE_RECORDS)
await workflowUsageStorage.setValue({ version: 1, records })
}
export async function getWorkflowUsageRecords(): Promise<
WorkflowUsageRecord[]
> {
const current = await workflowUsageStorage.getValue()
return current?.records ?? []
}
export async function clearWorkflowUsageRecords(): Promise<void> {
await workflowUsageStorage.setValue({ version: 1, records: [] })
}

View File

@@ -0,0 +1,28 @@
export type WorkflowUsageSource = 'sidepanel-chat' | 'agent-harness-chat'
export interface WorkflowUsageRecord {
id: string
source: WorkflowUsageSource
recordedAt: number
toolNames: string[]
}
export interface WorkflowUsageStore {
version: 1
records: WorkflowUsageRecord[]
}
export interface WorkflowSkillSuggestion {
id: string
title: string
runCount: number
pattern: string[]
lastUsedAt: number
benefit: string
}
export interface WorkflowUsageAnalysis {
totalRuns: number
eligibleRuns: number
suggestions: WorkflowSkillSuggestion[]
}