From b6232d4e217a427721337b601e7519c493aecd30 Mon Sep 17 00:00:00 2001 From: Simon <10131203+gaomeng1900@users.noreply.github.com> Date: Tue, 13 Jan 2026 17:33:38 +0800 Subject: [PATCH] refactor: mv `SimulatorMask` from `ui` to `page-controller` --- AGENTS.md | 11 ++--- README-zh.md | 4 +- README.md | 4 +- packages/page-agent/src/PageAgent.ts | 18 ++++---- packages/page-controller/package.json | 3 ++ .../page-controller/src/PageController.ts | 43 +++++++++++++++++++ .../src/SimulatorMask.module.css | 0 .../src/SimulatorMask.ts | 0 .../src/checkDarkMode.ts | 0 .../src/cursor.module.css | 0 packages/page-controller/src/env.d.ts | 6 +++ packages/page-controller/vite.config.js | 2 +- packages/ui/package.json | 5 +-- packages/ui/src/index.ts | 1 - packages/ui/vite.config.js | 2 +- .../docs/integration/configuration/page.tsx | 7 +++ 16 files changed, 80 insertions(+), 26 deletions(-) rename packages/{ui => page-controller}/src/SimulatorMask.module.css (100%) rename packages/{ui => page-controller}/src/SimulatorMask.ts (100%) rename packages/{ui => page-controller}/src/checkDarkMode.ts (100%) rename packages/{ui => page-controller}/src/cursor.module.css (100%) create mode 100644 packages/page-controller/src/env.d.ts diff --git a/AGENTS.md b/AGENTS.md index 4016f29..bcc9f4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,8 +10,8 @@ This is a **monorepo** with npm workspaces: Internal packages: - **LLMs** (`packages/llms/`) - LLM client with reflection-before-action mental model -- **Page Controller** (`packages/page-controller/`) - DOM operations, independent of LLM -- **UI** (`packages/ui/`) - Panel, SimulatorMask, i18n. Decoupled from PageAgent +- **Page Controller** (`packages/page-controller/`) - DOM operations and visual feedback (SimulatorMask), independent of LLM +- **UI** (`packages/ui/`) - Panel and i18n. Decoupled from PageAgent ## Development Commands @@ -43,8 +43,8 @@ packages/ - **Page Agent**: Core lib. Imports from `@page-agent/llms`, `@page-agent/page-controller`, `@page-agent/ui` - **LLMs**: LLM client with MacroToolInput contract. No dependency on page-agent -- **UI**: Panel, Mask, i18n. No dependency on page-agent -- **Page Controller**: Pure DOM operations. No LLM or UI dependency +- **UI**: Panel and i18n. No dependency on page-agent +- **Page Controller**: DOM operations with optional visual feedback (SimulatorMask). No LLM dependency. Enable mask via `enableMask: true` config ### PageController ↔ PageAgent Communication @@ -101,7 +101,8 @@ Query params configure `PageAgentConfig` in `src/umd.ts`. | File | Description | |------|-------------| -| `src/PageController.ts` | ⭐ Main controller class | +| `src/PageController.ts` | ⭐ Main controller class with optional mask support | +| `src/SimulatorMask.ts` | Visual overlay blocking user interaction during automation | | `src/actions.ts` | Element interactions (click, input, scroll) | | `src/dom/dom_tree/index.js` | Core DOM extraction engine | diff --git a/README-zh.md b/README-zh.md index 0d6f873..f673736 100644 --- a/README-zh.md +++ b/README-zh.md @@ -79,8 +79,8 @@ PageAgent adopts a simplified monorepo structure: packages/ ├── page-agent/ # AI agent (npm: page-agent) ├── llms/ # LLM 客户端 (npm: @page-agent/llms) -├── page-controller/ # DOM 操作 (npm: @page-agent/page-controller) -├── ui/ # 面板 & 蒙层 & 模拟鼠标 (npm: @page-agent/ui) +├── page-controller/ # DOM 操作 & 蒙层 & 模拟鼠标 (npm: @page-agent/page-controller) +├── ui/ # 面板 & i18n (npm: @page-agent/ui) └── website/ # 文档站点 ``` diff --git a/README.md b/README.md index a9db07d..452c068 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,8 @@ PageAgent adopts a simplified monorepo structure: packages/ ├── page-agent/ # AI agent (npm: page-agent) ├── llms/ # LLM client (npm: @page-agent/llms) -├── page-controller/ # DOM operations (npm: @page-agent/page-controller) -├── ui/ # Panel & Mask & Mouse Animation (npm: @page-agent/ui) +├── page-controller/ # DOM operations & Visual Mask (npm: @page-agent/page-controller) +├── ui/ # Panel & i18n (npm: @page-agent/ui) └── website/ # Demo & Documentation site ``` diff --git a/packages/page-agent/src/PageAgent.ts b/packages/page-agent/src/PageAgent.ts index c50f422..1c6ca3c 100644 --- a/packages/page-agent/src/PageAgent.ts +++ b/packages/page-agent/src/PageAgent.ts @@ -4,7 +4,7 @@ */ import { LLM, type Tool } from '@page-agent/llms' import { PageController } from '@page-agent/page-controller' -import { Panel, SimulatorMask } from '@page-agent/ui' +import { Panel } from '@page-agent/ui' import chalk from 'chalk' import zod from 'zod' @@ -92,8 +92,6 @@ export class PageAgent extends EventTarget { /** PageController for DOM operations */ pageController: PageController - /** Fullscreen mask */ - mask = new SimulatorMask() /** History records */ history: AgentHistory[] = [] @@ -114,8 +112,11 @@ export class PageAgent extends EventTarget { }) this.tools = new Map(tools) - // Initialize PageController with config - this.pageController = new PageController(this.config) + // Initialize PageController with config (mask enabled by default) + this.pageController = new PageController({ + ...this.config, + enableMask: this.config.enableMask ?? true, + }) // Listen to LLM events this.#llmRetryListener = (e) => { @@ -162,7 +163,7 @@ export class PageAgent extends EventTarget { await onBeforeTask.call(this) // Show mask and panel - this.mask.show() + this.pageController.showMask() this.panel.show() this.panel.reset() @@ -485,7 +486,7 @@ export class PageAgent extends EventTarget { // Task completed this.panel.update({ type: 'completed' }) - this.mask.hide() + this.pageController.hideMask() this.#abortController.abort() } @@ -496,9 +497,7 @@ export class PageAgent extends EventTarget { const pi = await this.pageController.getPageInfo() const viewportExpansion = await this.pageController.getViewportExpansion() - this.mask.wrapper.style.pointerEvents = 'none' await this.pageController.updateTree() - this.mask.wrapper.style.pointerEvents = 'auto' let simplifiedHTML = await this.pageController.getSimplifiedHTML() @@ -545,7 +544,6 @@ export class PageAgent extends EventTarget { this.disposed = true this.pageController.dispose() this.panel.dispose() - this.mask.dispose() this.history = [] this.#abortController.abort(reason ?? 'PageAgent disposed') diff --git a/packages/page-controller/package.json b/packages/page-controller/package.json index 34a39ae..05ec7ef 100644 --- a/packages/page-controller/package.json +++ b/packages/page-controller/package.json @@ -34,5 +34,8 @@ "build": "vite build", "prepublishOnly": "node -e \"const fs=require('fs');['LICENSE'].forEach(f=>fs.copyFileSync('../../'+f,f))\"", "postpublish": "node -e \"['LICENSE'].forEach(f=>{try{require('fs').unlinkSync(f)}catch{}})\"" + }, + "dependencies": { + "ai-motion": "^0.4.8" } } diff --git a/packages/page-controller/src/PageController.ts b/packages/page-controller/src/PageController.ts index 83dfe7b..d2a8277 100644 --- a/packages/page-controller/src/PageController.ts +++ b/packages/page-controller/src/PageController.ts @@ -6,6 +6,7 @@ * Designed to be independent of LLM and can be tested in unit tests. * All public methods are async for potential remote calling support. */ +import { SimulatorMask } from './SimulatorMask' import { clickElement, getElementByIndex, @@ -20,11 +21,15 @@ import type { FlatDomTree, InteractiveElementDomNode } from './dom/dom_tree/type import { getPageInfo } from './dom/getPageInfo' import { patchReact } from './patches/react' +export { SimulatorMask } + /** * Configuration for PageController */ export interface PageControllerConfig extends dom.DomConfig { viewportExpansion?: number + /** Enable visual mask overlay during operations (default: false) */ + enableMask?: boolean } interface ActionResult { @@ -64,12 +69,19 @@ export class PageController extends EventTarget { /** last time the tree was updated */ private lastTimeUpdate = 0 + /** Visual mask overlay for blocking user interaction during automation */ + private mask: SimulatorMask | null = null + constructor(config: PageControllerConfig = {}) { super() this.config = config patchReact(this) + + if (config.enableMask) { + this.mask = new SimulatorMask() + } } // ======= State Queries ======= @@ -136,12 +148,18 @@ export class PageController extends EventTarget { /** * Update DOM tree, returns simplified HTML for LLM. * This is the main method to refresh the page state. + * Automatically bypasses mask during DOM extraction if enabled. */ async updateTree(): Promise { this.dispatchEvent(new Event('beforeUpdate')) this.lastTimeUpdate = Date.now() + // Temporarily bypass mask to allow DOM extraction + if (this.mask) { + this.mask.wrapper.style.pointerEvents = 'none' + } + dom.cleanUpHighlights() const blacklist = [ @@ -162,6 +180,11 @@ export class PageController extends EventTarget { this.elementTextMap.clear() this.elementTextMap = dom.getElementTextMap(this.simplifiedHTML) + // Restore mask blocking + if (this.mask) { + this.mask.wrapper.style.pointerEvents = 'auto' + } + this.dispatchEvent(new Event('afterUpdate')) return this.simplifiedHTML @@ -326,6 +349,24 @@ export class PageController extends EventTarget { } } + // ======= Mask Operations ======= + + /** + * Show the visual mask overlay. + * Only works if enableMask was set to true in config. + */ + showMask(): void { + this.mask?.show() + } + + /** + * Hide the visual mask overlay. + * Only works if enableMask was set to true in config. + */ + hideMask(): void { + this.mask?.hide() + } + /** * Dispose and clean up resources */ @@ -335,5 +376,7 @@ export class PageController extends EventTarget { this.selectorMap.clear() this.elementTextMap.clear() this.simplifiedHTML = '' + this.mask?.dispose() + this.mask = null } } diff --git a/packages/ui/src/SimulatorMask.module.css b/packages/page-controller/src/SimulatorMask.module.css similarity index 100% rename from packages/ui/src/SimulatorMask.module.css rename to packages/page-controller/src/SimulatorMask.module.css diff --git a/packages/ui/src/SimulatorMask.ts b/packages/page-controller/src/SimulatorMask.ts similarity index 100% rename from packages/ui/src/SimulatorMask.ts rename to packages/page-controller/src/SimulatorMask.ts diff --git a/packages/ui/src/checkDarkMode.ts b/packages/page-controller/src/checkDarkMode.ts similarity index 100% rename from packages/ui/src/checkDarkMode.ts rename to packages/page-controller/src/checkDarkMode.ts diff --git a/packages/ui/src/cursor.module.css b/packages/page-controller/src/cursor.module.css similarity index 100% rename from packages/ui/src/cursor.module.css rename to packages/page-controller/src/cursor.module.css diff --git a/packages/page-controller/src/env.d.ts b/packages/page-controller/src/env.d.ts new file mode 100644 index 0000000..10798a0 --- /dev/null +++ b/packages/page-controller/src/env.d.ts @@ -0,0 +1,6 @@ +/// + +declare module '*.module.css' { + const classes: Record + export default classes +} diff --git a/packages/page-controller/vite.config.js b/packages/page-controller/vite.config.js index c2243af..5b1c7fc 100644 --- a/packages/page-controller/vite.config.js +++ b/packages/page-controller/vite.config.js @@ -29,7 +29,7 @@ export default defineConfig({ }, outDir: resolve(__dirname, 'dist', 'lib'), rollupOptions: { - external: ['@page-agent/*'], + external: ['@page-agent/*', 'ai-motion'], }, minify: false, sourcemap: true, diff --git a/packages/ui/package.json b/packages/ui/package.json index f912614..ee7f3a9 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -15,7 +15,7 @@ "files": [ "dist/" ], - "description": "UI components for page-agent - Panel, SimulatorMask, and i18n", + "description": "UI components for page-agent - Panel and i18n", "keywords": [ "page-agent", "ui", @@ -34,8 +34,5 @@ "build": "vite build", "prepublishOnly": "node -e \"const fs=require('fs');['LICENSE'].forEach(f=>fs.copyFileSync('../../'+f,f))\"", "postpublish": "node -e \"['LICENSE'].forEach(f=>{try{require('fs').unlinkSync(f)}catch{}})\"" - }, - "dependencies": { - "ai-motion": "^0.4.8" } } diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index 5a1608f..7e42775 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -1,4 +1,3 @@ export { Panel, type PanelConfig, type PanelUpdate } from './Panel' -export { SimulatorMask } from './SimulatorMask' export { UIState, type Step, type AgentStatus } from './UIState' export { I18n, type SupportedLanguage, type TranslationKey } from './i18n' diff --git a/packages/ui/vite.config.js b/packages/ui/vite.config.js index 7828dae..fee1231 100644 --- a/packages/ui/vite.config.js +++ b/packages/ui/vite.config.js @@ -29,7 +29,7 @@ export default defineConfig({ }, outDir: resolve(__dirname, 'dist', 'lib'), rollupOptions: { - external: ['ai-motion'], + external: [], }, minify: false, sourcemap: true, diff --git a/packages/website/src/pages/docs/integration/configuration/page.tsx b/packages/website/src/pages/docs/integration/configuration/page.tsx index e64798e..9cc380a 100644 --- a/packages/website/src/pages/docs/integration/configuration/page.tsx +++ b/packages/website/src/pages/docs/integration/configuration/page.tsx @@ -121,6 +121,13 @@ export default function Configuration() { /** Viewport expansion in pixels (-1 for full page) */ viewportExpansion?: number + + /** + * Enable visual mask overlay during automation. + * Blocks user interaction while agent is running. + * Default: false for PageController, true for PageAgent. + */ + enableMask?: boolean }`} />