refactor(PageController): implement PageController

2025-12-05 16:18:01 +08:00
parent ad19a26a57
commit 683602bb6b
33 changed files with 823 additions and 363 deletions
--- a/packages/page-agent/package.json
+++ b/packages/page-agent/package.json
@@ -39,10 +39,10 @@
 	},
 	"homepage": "https://alibaba.github.io/page-agent/",
 	"scripts": {
-		"build": "MODE=lib vite build && MODE=umd vite build",
+		"build": "npm run build:lib && npm run build:umd",
 		"build:lib": "MODE=lib vite build",
 		"build:umd": "MODE=umd vite build",
-		"build:watch": "MODE=lib vite build --watch",
+		"build:watch": "MODE=umd vite build --watch",
 		"prepublishOnly": "node -e \"const fs=require('fs');['README.md','LICENSE'].forEach(f=>fs.copyFileSync('../../'+f,f))\"",
 		"postpublish": "node -e \"['README.md','LICENSE'].forEach(f=>{try{require('fs').unlinkSync(f)}catch{}})\""
 	},
@@ -52,8 +52,6 @@
 		"zod": "^4.1.12"
 	},
 	"devDependencies": {
-		"@microsoft/api-extractor": "^7.55.1",
-		"unplugin-dts": "^1.0.0-beta.6",
-		"vite-plugin-css-injected-by-js": "^3.5.2"
+		"@page-agent/page-controller": "*"
 	}
 }
--- a/packages/page-agent/src/PageAgent.ts
+++ b/packages/page-agent/src/PageAgent.ts
@@ -2,17 +2,14 @@
 * Copyright (C) 2025 Alibaba Group Holding Limited
 * All rights reserved.
 */
+import { PageController } from '@page-agent/page-controller'
 import chalk from 'chalk'
 import zod from 'zod'

 import type { PageAgentConfig } from './config'
-import { MAX_STEPS, VIEWPORT_EXPANSION } from './config/constants'
-import * as dom from './dom'
-import { FlatDomTree, InteractiveElementDomNode } from './dom/dom_tree/type'
-import { getPageInfo } from './dom/getPageInfo'
+import { MAX_STEPS } from './config/constants'
 import { I18n } from './i18n'
 import { LLM, type Tool } from './llms'
-import { patchReact } from './patches/react'
 import SYSTEM_PROMPT from './prompts/system_prompt.md?raw'
 import { tools } from './tools'
 import { Panel, getToolCompletedText, getToolExecutingText } from './ui/Panel'
@@ -87,19 +84,8 @@ export class PageAgent extends EventTarget {
 	#totalWaitTime = 0
 	#abortController = new AbortController()

-	/** Corresponds to eval_page in browser-use */
-	flatTree: FlatDomTree | null = null
-	/**
-	 * All highlighted index-mapped interactive elements
-	 * Corresponds to DOMState.selector_map in browser-use
-	 */
-	selectorMap = new Map<number, InteractiveElementDomNode>()
-	/** highlight index -> element text */
-	elementTextMap = new Map<number, string>()
-	/** Corresponds to clickable_elements_to_string in browser-use */
-	simplifiedHTML = '<EMPTY>'
-	/** last time the tree was updated */
-	lastTimeUpdate = 0
+	/** PageController for DOM operations */
+	pageController: PageController

 	/** Fullscreen mask */
 	mask = new SimulatorMask()
@@ -115,6 +101,9 @@ export class PageAgent extends EventTarget {
 		this.panel = new Panel(this)
 		this.tools = new Map(tools)

+		// Initialize PageController with config
+		this.pageController = new PageController(this.config)
+
 		if (this.config.customTools) {
 			for (const [name, tool] of Object.entries(this.config.customTools)) {
 				if (tool === null) {
@@ -129,8 +118,6 @@ export class PageAgent extends EventTarget {
 			this.tools.delete('execute_javascript')
 		}

-		patchReact(this)
-
 		window.addEventListener('beforeunload', (e) => {
 			if (!this.disposed) this.dispose('PAGE_UNLOADING')
 		})
@@ -175,7 +162,7 @@ export class PageAgent extends EventTarget {
 			while (true) {
 				await onBeforeStep.call(this, step)

-				console.group(`step: ${step + 1}`)
+				console.group(`step: ${step}`)

 				// abort
 				if (this.#abortController.signal.aborted) throw new Error('AbortError')
@@ -197,7 +184,7 @@ export class PageAgent extends EventTarget {
 						},
 						{
 							role: 'user',
-							content: this.#assembleUserPrompt(),
+							content: await this.#assembleUserPrompt(),
 						},
 					],
 					{ AgentOutput: this.#packMacroTool() },
@@ -392,7 +379,7 @@ export class PageAgent extends EventTarget {
 		return systemPrompt
 	}

-	#assembleUserPrompt(): string {
+	async #assembleUserPrompt(): Promise<string> {
 		let prompt = ''

 		// <agent_history>
@@ -430,13 +417,13 @@ export class PageAgent extends EventTarget {

 		// <browser_state>

-		prompt += this.#getBrowserState()
+		prompt += await this.#getBrowserState()

 		return trimLines(prompt)
 	}

 	#onDone(text: string, success = true) {
-		dom.cleanUpHighlights()
+		this.pageController.cleanUpHighlights()

 		// Update panel status
 		this.bus.emit('panel:update', {
@@ -455,37 +442,42 @@ export class PageAgent extends EventTarget {
 		this.#abortController.abort()
 	}

-	#getBrowserState(): string {
-		const pageUrl = window.location.href
-		const pageTitle = document.title
-		const pi = getPageInfo()
+	async #getBrowserState(): Promise<string> {
+		const pageUrl = await this.pageController.getCurrentUrl()
+		const pageTitle = await this.pageController.getPageTitle()
+		const pi = await this.pageController.getPageInfo()
+		const viewportExpansion = await this.pageController.getViewportExpansion()

-		this.#updateTree()
+		this.mask.wrapper.style.pointerEvents = 'none'
+		await this.pageController.updateTree()
+		this.mask.wrapper.style.pointerEvents = 'auto'
+
+		const simplifiedHTML = await this.pageController.getSimplifiedHTML()

 		let prompt = trimLines(`<browser_state>
 			Current Page: [${pageTitle}](${pageUrl})

 			Page info: ${pi.viewport_width}x${pi.viewport_height}px viewport, ${pi.page_width}x${pi.page_height}px total page size, ${pi.pages_above.toFixed(1)} pages above, ${pi.pages_below.toFixed(1)} pages below, ${pi.total_pages.toFixed(1)} total pages, at ${(pi.current_page_position * 100).toFixed(0)}% of page

-			${VIEWPORT_EXPANSION === -1 ? 'Interactive elements from top layer of the current page (full page):' : 'Interactive elements from top layer of the current page inside the viewport:'}
+			${viewportExpansion === -1 ? 'Interactive elements from top layer of the current page (full page):' : 'Interactive elements from top layer of the current page inside the viewport:'}

 		`)

 		// Page header info
 		const has_content_above = pi.pixels_above > 4
-		if (has_content_above && VIEWPORT_EXPANSION !== -1) {
+		if (has_content_above && viewportExpansion !== -1) {
 			prompt += `... ${pi.pixels_above} pixels above (${pi.pages_above.toFixed(1)} pages) - scroll to see more ...\n`
 		} else {
 			prompt += `[Start of page]\n`
 		}

 		// Current viewport info
-		prompt += this.simplifiedHTML
+		prompt += simplifiedHTML
 		prompt += `\n`

 		// Page footer info
 		const has_content_below = pi.pixels_below > 4
-		if (has_content_below && VIEWPORT_EXPANSION !== -1) {
+		if (has_content_below && viewportExpansion !== -1) {
 			prompt += `... ${pi.pixels_below} pixels below (${pi.pages_below.toFixed(1)} pages) - scroll to see more ...\n`
 		} else {
 			prompt += `[End of page]\n`
@@ -496,37 +488,10 @@ export class PageAgent extends EventTarget {
 		return prompt
 	}

-	/**
-	 * Update document tree
-	 */
-	#updateTree() {
-		this.dispatchEvent(new Event('beforeUpdate'))
-		this.lastTimeUpdate = Date.now()
-		dom.cleanUpHighlights()
-		this.mask.wrapper.style.pointerEvents = 'none'
-		this.flatTree = dom.getFlatTree({
-			...this.config,
-			interactiveBlacklist: [
-				...(this.config.interactiveBlacklist || []),
-				...document.querySelectorAll('[data-page-agent-not-interactive]').values(),
-			],
-		})
-		this.mask.wrapper.style.pointerEvents = 'auto'
-		this.simplifiedHTML = dom.flatTreeToString(this.flatTree, this.config.include_attributes)
-		this.selectorMap.clear()
-		this.selectorMap = dom.getSelectorMap(this.flatTree)
-		this.elementTextMap.clear()
-		this.elementTextMap = dom.getElementTextMap(this.simplifiedHTML)
-		this.dispatchEvent(new Event('afterUpdate'))
-	}
-
 	dispose(reason?: string) {
 		console.log('Disposing PageAgent...')
 		this.disposed = true
-		dom.cleanUpHighlights()
-		this.flatTree = null
-		this.selectorMap.clear()
-		this.elementTextMap.clear()
+		this.pageController.dispose()
 		this.panel.dispose()
 		this.mask.dispose()
 		this.history = []
--- a/packages/page-agent/src/config/constants.ts
+++ b/packages/page-agent/src/config/constants.ts
@@ -1,10 +1,3 @@
-/**
- * @note Since isTopElement depends on elementFromPoint,
- * it returns null when out of viewport, this feature has no practical use, only differ between -1 and 0
- */
-// export const VIEWPORT_EXPANSION = 100
-export const VIEWPORT_EXPANSION = -1
-
 // Dev environment: use .env config if available, otherwise fallback to testing api
 export const DEFAULT_MODEL_NAME: string =
 	import.meta.env.DEV && import.meta.env.LLM_MODEL_NAME
--- a/packages/page-agent/src/config/index.ts
+++ b/packages/page-agent/src/config/index.ts
@@ -1,5 +1,6 @@
+import type { PageControllerConfig } from '@page-agent/page-controller'
+
 import type { AgentHistory, ExecutionResult, PageAgent } from '../PageAgent'
-import type { DomConfig } from '../dom'
 import type { SupportedLanguage } from '../i18n'
 import type { PageAgentTool } from '../tools'
 import {
@@ -94,7 +95,7 @@ export interface AgentConfig {
 	experimentalPreventNewPage?: boolean
 }

-export type PageAgentConfig = LLMConfig & AgentConfig & DomConfig
+export type PageAgentConfig = LLMConfig & AgentConfig & PageControllerConfig

 export function parseLLMConfig(config: LLMConfig): Required<LLMConfig> {
 	return {
--- a/packages/page-agent/src/env.d.ts
+++ b/packages/page-agent/src/env.d.ts
@@ -1,5 +1,5 @@
 /// <reference types="vite/client" />
-import type { PageAgent } from './src/PageAgent'
+import type { PageAgent } from './PageAgent'

 declare module '*.module.css' {
 	const classes: Record<string, string>
--- a/packages/page-agent/src/patches/antd.ts
+++ b/packages/page-agent/src/patches/antd.ts
@@ -1,20 +0,0 @@
-import type { PageAgent } from '../PageAgent'
-
-const clearFunctions = [] as (() => void)[]
-
-/**
- * antd 的 select 是 div 包 input 的结构，所有信息都在 input 标签上，
- * 但是 input 不可见，也不会出现在清洗后的树里，因此这里把他提上来
- */
-function fixAntdSelect() {
-	const selects = [...document.querySelectorAll('input[role="combobox"]')]
-	// for (const select of selects) {}
-}
-
-export function patchAntd(pageAgent: PageAgent) {
-	pageAgent.addEventListener('beforeUpdate', fixAntdSelect)
-	pageAgent.addEventListener('afterUpdate', () => {
-		for (const fn of clearFunctions) fn()
-		clearFunctions.length = 0
-	})
-}
--- a/packages/page-agent/src/patches/react.ts
+++ b/packages/page-agent/src/patches/react.ts
@@ -1,16 +0,0 @@
-import type { PageAgent } from '../PageAgent'
-
-// Find common React root elements and add data-page-agent-not-interactive attribute
-export function patchReact(pageAgent: PageAgent) {
-	const reactRootElements = document.querySelectorAll(
-		'[data-reactroot], [data-reactid], [data-react-checksum], #root, #app, [id^="root-"], [id^="app-"], #adex-wrapper, #adex-root'
-	)
-
-	for (const element of reactRootElements) {
-		element.setAttribute('data-page-agent-not-interactive', 'true')
-	}
-}
-
-/**
- * @todo (Heavy, might have false negatives) Interaction detection, if element width/height equals body offsetWidth/Height, consider it root element and non-interactive (React often attaches many events to root elements, causing false positives)
- */
--- a/packages/page-agent/src/tools/index.ts
+++ b/packages/page-agent/src/tools/index.ts
@@ -5,21 +5,7 @@
 import zod, { type z } from 'zod'

 import type { PageAgent } from '../PageAgent'
-import {
-	clickElement,
-	getElementByIndex,
-	getSystemInfo,
-	inputTextElement,
-	scrollHorizontally,
-	scrollVertically,
-	selectOptionElement,
-	waitFor,
-} from './actions'
-// debug
-import * as utils from './actions'
-
-// @ts-expect-error debug only
-window.utils = utils
+import { waitFor } from '../utils'

 /**
 * Internal tool definition that has access to PageAgent `this` context
@@ -41,18 +27,6 @@ export function tool<TParams>(options: PageAgentTool<TParams>): PageAgentTool<TP
 */
 export const tools = new Map<string, PageAgentTool>()

-// tools.set(
-// 	'get_current_html',
-// 	tool({
-// 		description: 'Get the current (updated) simplified HTML of the page',
-// 		inputSchema: zod.object({}),
-// 		execute: function (this: PageAgent) {
-// 			this.updateTree()
-// 			return this.simplifiedHTML
-// 		},
-// 	})
-// )
-
 tools.set(
 	'done',
 	tool({
@@ -79,11 +53,11 @@ tools.set(
 			seconds: zod.number().min(1).max(10).default(1),
 		}),
 		execute: async function (this: PageAgent, input) {
-			const lastTimeUpdate = this.lastTimeUpdate
+			const lastTimeUpdate = await this.pageController.getLastUpdateTime()
 			const actualWaitTime = Math.max(0, input.seconds - (Date.now() - lastTimeUpdate) / 1000)
 			console.log(`actualWaitTime: ${actualWaitTime} seconds`)
 			await waitFor(actualWaitTime)
-			return `✅ Waited for ${input.seconds} seconds.` + (await getSystemInfo())
+			return `✅ Waited for ${input.seconds} seconds.`
 		},
 	})
 )
@@ -98,7 +72,7 @@ tools.set(
 		}),
 		execute: async function (this: PageAgent, input) {
 			const answer = await this.panel.askUser(input.question)
-			return `✅ Received user answer: ${answer}` + (await getSystemInfo())
+			return `✅ Received user answer: ${answer}`
 		},
 	})
 )
@@ -111,16 +85,8 @@ tools.set(
 			index: zod.int().min(0),
 		}),
 		execute: async function (this: PageAgent, input) {
-			const element = getElementByIndex(this, input.index)
-			const elemText = this.elementTextMap.get(input.index)
-			await clickElement(element)
-
-			// @workaround: Handle links that open in new tabs
-			if (element instanceof HTMLAnchorElement && element.target === '_blank') {
-				return `⚠️ Clicked link that opens in a new tab (${elemText ? elemText : input.index}). You are not capable of reading new tabs.`
-			}
-
-			return `✅ Clicked element (${elemText ? elemText : input.index}).` + (await getSystemInfo())
+			const result = await this.pageController.clickElement(input.index)
+			return result.message
 		},
 	})
 )
@@ -134,13 +100,8 @@ tools.set(
 			text: zod.string(),
 		}),
 		execute: async function (this: PageAgent, input) {
-			const element = getElementByIndex(this, input.index)
-			const elemText = this.elementTextMap.get(input.index)
-			await inputTextElement(element, input.text)
-			return (
-				`✅ Input text (${input.text}) into element (${elemText ? elemText : input.index}).` +
-				(await getSystemInfo())
-			)
+			const result = await this.pageController.inputText(input.index, input.text)
+			return result.message
 		},
 	})
 )
@@ -155,13 +116,8 @@ tools.set(
 			text: zod.string(),
 		}),
 		execute: async function (this: PageAgent, input) {
-			const element = getElementByIndex(this, input.index)
-			const elemText = this.elementTextMap.get(input.index)
-			await selectOptionElement(element as HTMLSelectElement, input.text)
-			return (
-				`✅ Selected option (${input.text}) in element (${elemText ? elemText : input.index}).` +
-				(await getSystemInfo())
-			)
+			const result = await this.pageController.selectOption(input.index, input.text)
+			return result.message
 		},
 	})
 )
@@ -181,13 +137,11 @@ tools.set(
 			index: zod.number().int().min(0).optional(),
 		}),
 		execute: async function (this: PageAgent, input) {
-			const { down, num_pages, index, pixels } = input
-
-			const scroll_amount = pixels ? pixels : num_pages * (down ? 1 : -1) * window.innerHeight
-
-			const element = index !== undefined ? getElementByIndex(this, index) : null
-
-			return (await scrollVertically(down, scroll_amount, element)) + (await getSystemInfo())
+			const result = await this.pageController.scroll({
+				...input,
+				numPages: input.num_pages,
+			})
+			return result.message
 		},
 	})
 )
@@ -203,13 +157,8 @@ tools.set(
 			index: zod.number().int().min(0).optional(),
 		}),
 		execute: async function (this: PageAgent, input) {
-			const { right, pixels, index } = input
-
-			const scroll_amount = pixels * (right ? 1 : -1)
-
-			const element = index !== undefined ? getElementByIndex(this, index) : null
-
-			return (await scrollHorizontally(right, scroll_amount, element)) + (await getSystemInfo())
+			const result = await this.pageController.scrollHorizontally(input)
+			return result.message
 		},
 	})
 )
@@ -223,14 +172,8 @@ tools.set(
 			script: zod.string(),
 		}),
 		execute: async function (this: PageAgent, input) {
-			try {
-				// Wrap script in async function to support await
-				const asyncFunction = eval(`(async () => { ${input.script} })`)
-				const result = await asyncFunction()
-				return `✅ Executed JavaScript. Result: ${result}` + (await getSystemInfo())
-			} catch (error) {
-				return `❌ Error executing JavaScript: ${error}` + (await getSystemInfo())
-			}
+			const result = await this.pageController.executeJavascript(input.script)
+			return result.message
 		},
 	})
 )
--- a/packages/page-agent/src/utils/bus.ts
+++ b/packages/page-agent/src/utils/bus.ts
@@ -23,8 +23,6 @@ export interface PageAgentEventMap {
 	'panel:collapse': { params: undefined }

 	// PageAgent status events
-	// 'agent:beforeUpdate': { params: undefined }
-	// 'agent:afterUpdate': { params: undefined }
 	// 'agent:execute': { params: { task: string } }
 	// 'agent:done': { params: { text: string; success: boolean } }
 	// 'agent:paused': { params: undefined }
--- a/packages/page-agent/src/utils/index.ts
+++ b/packages/page-agent/src/utils/index.ts
@@ -20,6 +20,10 @@ export async function waitUntil(check: () => boolean, timeout = 60 * 60_1000): P
 	})
 }

+export async function waitFor(seconds: number): Promise<void> {
+	await new Promise((resolve) => setTimeout(resolve, seconds * 1000))
+}
+
 //

 export function truncate(text: string, maxLength: number): string {
--- a/packages/page-agent/tsconfig.dts.json
+++ b/packages/page-agent/tsconfig.dts.json
@@ -0,0 +1,9 @@
+{
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		// @workaround DTS bug
+		// dts do not work with monorepo path mapping
+		// disable path mapping for it
+		"paths": {}
+	}
+}
--- a/packages/page-agent/tsconfig.json
+++ b/packages/page-agent/tsconfig.json
@@ -1,10 +1,16 @@
 {
-	"extends": "../../tsconfig.json",
+	"extends": "../../tsconfig.base.json",
 	"compilerOptions": {
-		"composite": true,
+		"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",
 		"noEmit": false,
-		"outDir": "./dist",
-		"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo"
+		"allowImportingTsExtensions": false,
+		"baseUrl": ".",
+		"outDir": "dist",
+		"paths": {
+			"@page-agent/page-controller": ["../page-controller/src/PageController.ts"]
+		}
 	},
-	"include": ["src", "env.d.ts"]
+	"include": ["**/*.ts"],
+	"exclude": ["dist", "node_modules"],
+	"references": [{ "path": "../page-controller" }]
 }
--- a/packages/page-agent/vite.config.js
+++ b/packages/page-agent/vite.config.js
@@ -17,7 +17,8 @@ const __dirname = dirname(fileURLToPath(import.meta.url))
 const libConfig = {
 	clearScreen: false,
 	plugins: [
-		dts({ tsconfigPath: './tsconfig.json', bundleTypes: true }),
+		dts({ tsconfigPath: './tsconfig.dts.json', bundleTypes: true }),
+		// dts({ tsconfigPath: './tsconfig.json', bundleTypes: true, compilerOptions: { paths: {} } }),
 		cssInjectedByJsPlugin({ relativeCSSInjection: true }),
 	],
 	publicDir: false,
@@ -33,7 +34,7 @@ const libConfig = {
 		},
 		outDir: resolve(__dirname, 'dist', 'lib'),
 		rollupOptions: {
-			external: ['ai', 'ai-motion', 'chalk', 'zod'],
+			external: ['ai', 'ai-motion', 'chalk', 'zod', '@page-agent/*'],
 		},
 		minify: false,
 		sourcemap: true,
@@ -54,6 +55,11 @@ const umdConfig = {
 	esbuild: {
 		keepNames: true,
 	},
+	resolve: {
+		alias: {
+			'@page-agent/page-controller': resolve(__dirname, '../page-controller/src/PageController.ts'),
+		},
+	},
 	build: {
 		lib: {
 			entry: resolve(__dirname, 'src/entry.ts'),