Merge pull request #363 from alibaba/feat/ext-controll-all-tabs

fix(ext): MultiPageAgent inside content script can not detect new tabs
feat(ext): experimentalIncludeAllTabs - control all window tabs

packages/extension/src/agent/MultiPageAgent.ts

@@ -11,13 +11,18 @@ function detectLanguage(): 'en-US' | 'zh-CN' {
 	return lang.startsWith('zh') ? 'zh-CN' : 'en-US'
 }
 
+interface MultiPageAgentConfig extends AgentConfig {
+	includeInitialTab?: boolean
+	experimentalIncludeAllTabs?: boolean
+}
+
 /**
  * MultiPageAgent
  * - use with extension
  * - can be used from a side panel or a content script
  */
 export class MultiPageAgent extends PageAgentCore {
-	constructor(config: AgentConfig & { includeInitialTab?: boolean }) {
+	constructor(config: MultiPageAgentConfig) {
 		// multi page controller
 		const tabsController = new TabsController()
 		const pageController = new RemotePageController(tabsController)
@@ -31,8 +36,8 @@ export class MultiPageAgent extends PageAgentCore {
 			`Default working language: **${targetLanguage}**`
 		)
 
-		// include initial tab for controlling
 		const includeInitialTab = config.includeInitialTab ?? true
+		const experimentalIncludeAllTabs = config.experimentalIncludeAllTabs ?? false
 
 		/**
 		 * When the agent is in side-panel and user closed the side-panel.
@@ -50,7 +55,7 @@ export class MultiPageAgent extends PageAgentCore {
 			customSystemPrompt: systemPrompt,
 
 			onBeforeTask: async (agent) => {
-				await tabsController.init(agent.task, includeInitialTab)
+				await tabsController.init(agent.task, { includeInitialTab, experimentalIncludeAllTabs })
 
 				heartBeatInterval = window.setInterval(() => {
 					chrome.storage.local.set({

packages/extension/src/agent/RemotePageController.background.ts

@@ -10,9 +10,7 @@ export function handlePageControlMessage(
 ): true | undefined {
 	const PREFIX = '[RemotePageController.background]'
 
-	function debug(...messages: any[]) {
-		console.debug(`\x1b[90m${PREFIX}\x1b[0m`, ...messages)
-	}
+	const debug = console.debug.bind(console, `\x1b[90m${PREFIX}\x1b[0m`)
 
 	const { action, payload, targetTabId } = message
 

packages/extension/src/agent/RemotePageController.ts

@@ -4,9 +4,7 @@ import type { TabsController } from './TabsController'
 
 const PREFIX = '[RemotePageController]'
 
-function debug(...messages: any[]) {
-	console.debug(`\x1b[90m${PREFIX}\x1b[0m`, ...messages)
-}
+const debug = console.debug.bind(console, `\x1b[90m${PREFIX}\x1b[0m`)
 
 function sendMessage(message: {
 	type: 'PAGE_CONTROL'

packages/extension/src/agent/TabsController.background.ts

@@ -5,9 +5,7 @@ import type { TabAction } from './TabsController'
 
 const PREFIX = '[TabsController.background]'
 
-function debug(...messages: any[]) {
-	console.debug(`\x1b[90m${PREFIX}\x1b[0m`, ...messages)
-}
+const debug = console.debug.bind(console, `\x1b[90m${PREFIX}\x1b[0m`)
 
 export function handleTabControlMessage(
 	message: { type: 'TAB_CONTROL'; action: TabAction; payload: any },
@@ -20,11 +18,10 @@ export function handleTabControlMessage(
 		case 'get_active_tab': {
 			debug('get_active_tab')
 			chrome.tabs
-				.query({ active: true, currentWindow: true })
+				.query({ active: true })
 				.then((tabs) => {
-					const tabId = tabs.length > 0 ? tabs[0].id || null : null
-					debug('get_active_tab: success', tabId)
-					sendResponse({ success: true, tabId })
+					debug('get_active_tab: success', tabs)
+					sendResponse({ success: true, tab: tabs[0] })
 				})
 				.catch((error) => {
 					sendResponse({ error: error instanceof Error ? error.message : String(error) })
@@ -63,7 +60,7 @@ export function handleTabControlMessage(
 		case 'create_tab_group': {
 			debug('create_tab_group', payload)
 			chrome.tabs
-				.group({ tabIds: payload.tabIds })
+				.group({ tabIds: payload.tabIds, createProperties: { windowId: payload.windowId } })
 				.then((groupId) => {
 					debug('create_tab_group: success', groupId)
 					sendResponse({ success: true, groupId })
@@ -114,47 +111,59 @@ export function handleTabControlMessage(
 			return true // async response
 		}
 
+		case 'get_window_tabs': {
+			debug('get_window_tabs', payload)
+			chrome.tabs
+				.query({ windowId: payload.windowId })
+				.then((tabs) => {
+					sendResponse({ success: true, tabs })
+				})
+				.catch((error) => {
+					sendResponse({ error: error instanceof Error ? error.message : String(error) })
+				})
+			return true
+		}
+
 		default:
 			sendResponse({ error: `Unknown action: ${action}` })
 			return
 	}
 }
 
-export function setupTabChangeEvents() {
-	console.log('[TabsController.background] setupTabChangeEvents')
+const tabEventPorts = new Set<chrome.runtime.Port>()
+
+function broadcastTabEvent(message: object) {
+	for (const port of tabEventPorts) {
+		port.postMessage(message)
+	}
+}
+
+/**
+ * Port-based tab events: agents connect via `chrome.runtime.connect({ name: 'tab-events' })`
+ * and receive tab change events through the port. Works for both extension pages and content scripts.
+ */
+export function setupTabEventsPort() {
+	chrome.runtime.onConnect.addListener((port) => {
+		if (port.name !== 'tab-events') return
+
+		debug('port connected', port.sender?.tab?.id ?? port.sender?.url)
+		tabEventPorts.add(port)
+
+		port.onDisconnect.addListener(() => {
+			debug('port disconnected')
+			tabEventPorts.delete(port)
+		})
+	})
 
 	chrome.tabs.onCreated.addListener((tab) => {
-		debug('onCreated', tab)
-		chrome.runtime
-			.sendMessage({ type: 'TAB_CHANGE', action: 'created', payload: { tab } })
-			.catch((error) => {
-				debug('onCreated error:', error)
-			})
+		broadcastTabEvent({ action: 'created', payload: { tab } })
 	})
 
 	chrome.tabs.onRemoved.addListener((tabId, removeInfo) => {
-		debug('onRemoved', tabId, removeInfo)
-		chrome.runtime
-			.sendMessage({
-				type: 'TAB_CHANGE',
-				action: 'removed',
-				payload: { tabId, removeInfo },
-			})
-			.catch((error) => {
-				debug('onRemoved error:', error)
-			})
+		broadcastTabEvent({ action: 'removed', payload: { tabId, removeInfo } })
 	})
 
 	chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
-		debug('onUpdated', tabId, changeInfo)
-		chrome.runtime
-			.sendMessage({
-				type: 'TAB_CHANGE',
-				action: 'updated',
-				payload: { tabId, changeInfo, tab },
-			})
-			.catch((error) => {
-				debug('onUpdated error:', error)
-			})
+		broadcastTabEvent({ action: 'updated', payload: { tabId, changeInfo, tab } })
 	})
 }

packages/extension/src/agent/TabsController.ts

@@ -2,9 +2,7 @@ import { isContentScriptAllowed } from './RemotePageController'
 
 const PREFIX = '[TabsController]'
 
-function debug(...messages: any[]) {
-	console.debug(`\x1b[90m${PREFIX}\x1b[0m`, ...messages)
-}
+const debug = console.debug.bind(console, `\x1b[90m${PREFIX}\x1b[0m`)
 
 function sendMessage(message: {
 	type: 'TAB_CONTROL'
@@ -22,46 +20,91 @@ function sendMessage(message: {
  * - live in the agent env (extension page or content script)
  * - no chrome apis. call sw for tab operations
  */
-export class TabsController extends EventTarget {
+export class TabsController {
 	currentTabId: number | null = null
 
+	private disposed = false
+	private port: chrome.runtime.Port | null = null
+	private portRetries = 0
+
+	private windowId: number | null = null
 	private tabs: TabMeta[] = []
 	private initialTabId: number | null = null
 	private tabGroupId: number | null = null
+	private experimentalIncludeAllTabs = false
 	private task: string = ''
 
-	async init(task: string, includeInitialTab: boolean = true) {
-		debug('init', task, includeInitialTab)
+	async init(task: string, options: TabsInitOptions = {}) {
+		const { includeInitialTab = true, experimentalIncludeAllTabs = false } = options
+		debug('init', task, options)
+
+		if (this.disposed) {
+			throw new Error('TabsController already disposed')
+		}
 
-		this.task = task
-		this.tabs = []
 		this.currentTabId = null
+		this.disposed = false
+		this.port = null
+		this.portRetries = 0
+
+		this.windowId = null
+		this.tabs = []
 		this.tabGroupId = null
 		this.initialTabId = null
+		this.experimentalIncludeAllTabs = experimentalIncludeAllTabs
+		this.task = task
 
-		const result = await sendMessage({
+		const activeTabResult = await sendMessage({
 			type: 'TAB_CONTROL',
 			action: 'get_active_tab',
 		})
 
-		this.initialTabId = result.tabId
+		this.initialTabId = activeTabResult.tab?.id
+		this.windowId = activeTabResult.tab?.windowId
 
-		if (!this.initialTabId) {
-			throw new Error('Failed to get initial tab ID')
+		if (!this.initialTabId || !this.windowId) {
+			if (activeTabResult.error) {
+				throw new Error(activeTabResult.error)
+			} else {
+				throw new Error('Failed to get active tab')
+			}
 		}
 
-		if (includeInitialTab) {
+		this.connectTabEvents()
+
+		if (experimentalIncludeAllTabs) {
+			const allTabs = await sendMessage({
+				type: 'TAB_CONTROL',
+				action: 'get_window_tabs',
+				payload: { windowId: this.windowId },
+			})
+			for (const tab of allTabs.tabs as chrome.tabs.Tab[]) {
+				if (tab.id && !tab.pinned && isContentScriptAllowed(tab.url)) {
+					this.tabs.push({
+						id: tab.id,
+						isInitial: tab.id === this.initialTabId,
+						url: tab.url,
+						title: tab.title,
+						status: tab.status,
+					})
+				}
+			}
+			if (this.tabs.find((t) => t.id === this.initialTabId)) {
+				this.currentTabId = this.initialTabId
+				await this.createTabGroup([this.initialTabId])
+			}
+		} else if (includeInitialTab) {
 			const info = await sendMessage({
 				type: 'TAB_CONTROL',
 				action: 'get_tab_info',
 				payload: { tabId: this.initialTabId },
 			})
 
-			if (isContentScriptAllowed(info.url)) {
+			if (isContentScriptAllowed(info.url) && !info.pinned) {
 				this.currentTabId = this.initialTabId
 
 				this.tabs.push({
-					id: result.tabId,
+					id: this.initialTabId,
 					isInitial: true,
 					url: info.url,
 					title: info.title,
@@ -73,52 +116,6 @@ export class TabsController extends EventTarget {
 		}
 
 		await this.updateCurrentTabId(this.currentTabId)
-
-		const tabChangeHandler = (message: any): void => {
-			if (message.type !== 'TAB_CHANGE') {
-				// throw new Error(`[TabsController]: Invalid message type: ${message.type}`)
-				return
-			}
-
-			if (message.action === 'created') {
-				const tab = message.payload.tab as chrome.tabs.Tab
-				if (tab.groupId === this.tabGroupId && tab.id != null) {
-					// Tab created in our controlled group
-					if (!this.tabs.find((t) => t.id === tab.id)) {
-						this.tabs.push({ id: tab.id, isInitial: false })
-					}
-					this.switchToTab(tab.id)
-				}
-			} else if (message.action === 'removed') {
-				const { tabId } = message.payload as { tabId: number }
-				const targetTab = this.tabs.find((t) => t.id === tabId)
-				if (targetTab) {
-					this.tabs = this.tabs.filter((t) => t.id !== tabId)
-					if (this.currentTabId === tabId) {
-						const newCurrentTab = this.tabs[this.tabs.length - 1] || null
-						if (newCurrentTab) {
-							this.switchToTab(newCurrentTab.id)
-						} else {
-							this.updateCurrentTabId(null)
-						}
-					}
-				}
-			} else if (message.action === 'updated') {
-				const { tabId, tab } = message.payload as { tabId: number; tab: chrome.tabs.Tab }
-				const targetTab = this.tabs.find((t) => t.id === tabId)
-				if (targetTab) {
-					targetTab.url = tab.url
-					targetTab.title = tab.title
-					targetTab.status = tab.status
-				}
-			}
-		}
-
-		chrome.runtime.onMessage.addListener(tabChangeHandler)
-
-		this.addEventListener('dispose', () => {
-			chrome.runtime.onMessage.removeListener(tabChangeHandler)
-		})
 	}
 
 	async openNewTab(url: string): Promise<string> {
@@ -209,7 +206,7 @@ export class TabsController extends EventTarget {
 		const result = await sendMessage({
 			type: 'TAB_CONTROL',
 			action: 'create_tab_group',
-			payload: { tabIds },
+			payload: { tabIds, windowId: this.windowId },
 		})
 
 		if (!result?.success) {
@@ -288,9 +285,79 @@ export class TabsController extends EventTarget {
 		await waitUntil(() => tab.status === 'complete', 4_000)
 	}
 
-	dispose() {
-		this.dispatchEvent(new Event('dispose'))
+	/**
+	 * Connect to background SW via port to receive tab change events.
+	 *
+	 * @note Port is 1:1 (runtime.connect → background SW has no frames),
+	 * so onDisconnect fires exactly once and we can safely reconnect.
+	 * Reconnection may miss events during the gap.
+	 * TODO: refresh this.tabs from background after reconnect to stay consistent.
+	 */
+	private connectTabEvents() {
+		this.port = chrome.runtime.connect({ name: 'tab-events' })
+
+		this.port.onMessage.addListener((message: any) => {
+			if (this.disposed) return
+			this.portRetries = 0
+
+			if (message.action === 'created') {
+				const tab = message.payload.tab as chrome.tabs.Tab
+				const shouldTrack = this.experimentalIncludeAllTabs || tab.groupId === this.tabGroupId
+				if (shouldTrack && tab.id != null) {
+					if (!this.tabs.find((t) => t.id === tab.id)) {
+						this.tabs.push({ id: tab.id, isInitial: false })
+					}
+					this.switchToTab(tab.id)
+				}
+			} else if (message.action === 'removed') {
+				const { tabId } = message.payload as { tabId: number }
+				const targetTab = this.tabs.find((t) => t.id === tabId)
+				if (targetTab) {
+					this.tabs = this.tabs.filter((t) => t.id !== tabId)
+					if (this.currentTabId === tabId) {
+						const newCurrentTab = this.tabs[this.tabs.length - 1] || null
+						if (newCurrentTab) {
+							this.switchToTab(newCurrentTab.id)
+						} else {
+							this.updateCurrentTabId(null)
+						}
+					}
+				}
+			} else if (message.action === 'updated') {
+				const { tabId, tab } = message.payload as { tabId: number; tab: chrome.tabs.Tab }
+				const targetTab = this.tabs.find((t) => t.id === tabId)
+				if (targetTab) {
+					targetTab.url = tab.url
+					targetTab.title = tab.title
+					targetTab.status = tab.status
+				}
+			}
+		})
+
+		this.port.onDisconnect.addListener(() => {
+			this.port = null
+			if (this.disposed) return
+			if (this.portRetries >= 7) {
+				console.error(PREFIX, 'tab events port failed after 3 retries, giving up')
+				return
+			}
+			debug('port disconnected, reconnecting...')
+			this.portRetries++
+			this.connectTabEvents()
+		})
 	}
+
+	dispose() {
+		debug('dispose')
+		this.disposed = true
+		this.port?.disconnect()
+		this.port = null
+	}
+}
+
+export interface TabsInitOptions {
+	includeInitialTab?: boolean
+	experimentalIncludeAllTabs?: boolean
 }
 
 export type TabAction =
@@ -302,6 +369,7 @@ export type TabAction =
 	| 'add_tab_to_group'
 	| 'close_tab'
 	| 'get_tab_title'
+	| 'get_window_tabs'
 
 interface TabMeta {
 	id: number

packages/extension/src/agent/useAgent.ts

@@ -21,6 +21,7 @@ export interface AdvancedConfig {
 	maxSteps?: number
 	systemInstruction?: string
 	experimentalLlmsTxt?: boolean
+	experimentalIncludeAllTabs?: boolean
 	disableNamedToolChoice?: boolean
 }
 
@@ -125,6 +126,7 @@ export function useAgent(): UseAgentResult {
 			maxSteps,
 			systemInstruction,
 			experimentalLlmsTxt,
+			experimentalIncludeAllTabs,
 			disableNamedToolChoice,
 			...llmConfig
 		}: ExtConfig) => {
@@ -138,6 +140,7 @@ export function useAgent(): UseAgentResult {
 				maxSteps,
 				systemInstruction,
 				experimentalLlmsTxt,
+				experimentalIncludeAllTabs,
 				disableNamedToolChoice,
 			}
 			await chrome.storage.local.set({ advancedConfig })

packages/extension/src/components/ConfigPanel.tsx

@@ -36,6 +36,9 @@ export function ConfigPanel({ config, onSave, onClose }: ConfigPanelProps) {
 	const [experimentalLlmsTxt, setExperimentalLlmsTxt] = useState(
 		config?.experimentalLlmsTxt ?? false
 	)
+	const [experimentalIncludeAllTabs, setExperimentalIncludeAllTabs] = useState(
+		config?.experimentalIncludeAllTabs ?? false
+	)
 	const [disableNamedToolChoice, setDisableNamedToolChoice] = useState(
 		config?.disableNamedToolChoice ?? false
 	)
@@ -54,6 +57,7 @@ export function ConfigPanel({ config, onSave, onClose }: ConfigPanelProps) {
 		setMaxSteps(config?.maxSteps)
 		setSystemInstruction(config?.systemInstruction ?? '')
 		setExperimentalLlmsTxt(config?.experimentalLlmsTxt ?? false)
+		setExperimentalIncludeAllTabs(config?.experimentalIncludeAllTabs ?? false)
 		setDisableNamedToolChoice(config?.disableNamedToolChoice ?? false)
 	}, [config])
 
@@ -100,6 +104,7 @@ export function ConfigPanel({ config, onSave, onClose }: ConfigPanelProps) {
 				maxSteps: maxSteps || undefined,
 				systemInstruction: systemInstruction || undefined,
 				experimentalLlmsTxt,
+				experimentalIncludeAllTabs,
 				disableNamedToolChoice,
 			})
 		} finally {
@@ -285,6 +290,14 @@ export function ConfigPanel({ config, onSave, onClose }: ConfigPanelProps) {
 						<span className="text-xs text-muted-foreground">Experimental llms.txt support</span>
 						<Switch checked={experimentalLlmsTxt} onCheckedChange={setExperimentalLlmsTxt} />
 					</label>
+
+					<label className="flex items-center justify-between cursor-pointer">
+						<span className="text-xs text-muted-foreground">Experimental include all tabs</span>
+						<Switch
+							checked={experimentalIncludeAllTabs}
+							onCheckedChange={setExperimentalIncludeAllTabs}
+						/>
+					</label>
 				</>
 			)}
 

packages/extension/src/entrypoints/background.ts

@@ -1,12 +1,12 @@
 import { handlePageControlMessage } from '@/agent/RemotePageController.background'
-import { handleTabControlMessage, setupTabChangeEvents } from '@/agent/TabsController.background'
+import { handleTabControlMessage, setupTabEventsPort } from '@/agent/TabsController.background'
 
 export default defineBackground(() => {
 	console.log('[Background] Service Worker started')
 
 	// tab change events
 
-	setupTabChangeEvents()
+	setupTabEventsPort()
 
 	// generate user auth token
 

packages/extension/src/entrypoints/main-world.ts

@@ -13,6 +13,9 @@ export interface ExecuteConfig {
 	 */
 	includeInitialTab?: boolean
 
+	/** Control all unpinned tabs in the window instead of only the tab group. */
+	experimentalIncludeAllTabs?: boolean
+
 	onStatusChange?: (status: AgentStatus) => void
 	onActivity?: (activity: AgentActivity) => void
 	onHistoryUpdate?: (history: HistoricalEvent[]) => void
@@ -87,6 +90,7 @@ export default defineUnlistedScript(() => {
 						model: config.model,
 						apiKey: config.apiKey,
 						includeInitialTab: config.includeInitialTab,
+						experimentalIncludeAllTabs: config.experimentalIncludeAllTabs,
 					},
 				},
 			},