refactor: mv SimulatorMask from ui to page-controller

packages/page-controller/package.json

@@ -34,5 +34,8 @@
 		"build": "vite build",
 		"prepublishOnly": "node -e \"const fs=require('fs');['LICENSE'].forEach(f=>fs.copyFileSync('../../'+f,f))\"",
 		"postpublish": "node -e \"['LICENSE'].forEach(f=>{try{require('fs').unlinkSync(f)}catch{}})\""
+	},
+	"dependencies": {
+		"ai-motion": "^0.4.8"
 	}
 }

packages/page-controller/src/PageController.ts

@@ -6,6 +6,7 @@
  * Designed to be independent of LLM and can be tested in unit tests.
  * All public methods are async for potential remote calling support.
  */
+import { SimulatorMask } from './SimulatorMask'
 import {
 	clickElement,
 	getElementByIndex,
@@ -20,11 +21,15 @@ import type { FlatDomTree, InteractiveElementDomNode } from './dom/dom_tree/type
 import { getPageInfo } from './dom/getPageInfo'
 import { patchReact } from './patches/react'
 
+export { SimulatorMask }
+
 /**
  * Configuration for PageController
  */
 export interface PageControllerConfig extends dom.DomConfig {
 	viewportExpansion?: number
+	/** Enable visual mask overlay during operations (default: false) */
+	enableMask?: boolean
 }
 
 interface ActionResult {
@@ -64,12 +69,19 @@ export class PageController extends EventTarget {
 	/** last time the tree was updated */
 	private lastTimeUpdate = 0
 
+	/** Visual mask overlay for blocking user interaction during automation */
+	private mask: SimulatorMask | null = null
+
 	constructor(config: PageControllerConfig = {}) {
 		super()
 
 		this.config = config
 
 		patchReact(this)
+
+		if (config.enableMask) {
+			this.mask = new SimulatorMask()
+		}
 	}
 
 	// ======= State Queries =======
@@ -136,12 +148,18 @@ export class PageController extends EventTarget {
 	/**
 	 * Update DOM tree, returns simplified HTML for LLM.
 	 * This is the main method to refresh the page state.
+	 * Automatically bypasses mask during DOM extraction if enabled.
 	 */
 	async updateTree(): Promise<string> {
 		this.dispatchEvent(new Event('beforeUpdate'))
 
 		this.lastTimeUpdate = Date.now()
 
+		// Temporarily bypass mask to allow DOM extraction
+		if (this.mask) {
+			this.mask.wrapper.style.pointerEvents = 'none'
+		}
+
 		dom.cleanUpHighlights()
 
 		const blacklist = [
@@ -162,6 +180,11 @@ export class PageController extends EventTarget {
 		this.elementTextMap.clear()
 		this.elementTextMap = dom.getElementTextMap(this.simplifiedHTML)
 
+		// Restore mask blocking
+		if (this.mask) {
+			this.mask.wrapper.style.pointerEvents = 'auto'
+		}
+
 		this.dispatchEvent(new Event('afterUpdate'))
 
 		return this.simplifiedHTML
@@ -326,6 +349,24 @@ export class PageController extends EventTarget {
 		}
 	}
 
+	// ======= Mask Operations =======
+
+	/**
+	 * Show the visual mask overlay.
+	 * Only works if enableMask was set to true in config.
+	 */
+	showMask(): void {
+		this.mask?.show()
+	}
+
+	/**
+	 * Hide the visual mask overlay.
+	 * Only works if enableMask was set to true in config.
+	 */
+	hideMask(): void {
+		this.mask?.hide()
+	}
+
 	/**
 	 * Dispose and clean up resources
 	 */
@@ -335,5 +376,7 @@ export class PageController extends EventTarget {
 		this.selectorMap.clear()
 		this.elementTextMap.clear()
 		this.simplifiedHTML = '<EMPTY>'
+		this.mask?.dispose()
+		this.mask = null
 	}
 }

packages/page-controller/src/SimulatorMask.module.css

@@ -0,0 +1,10 @@
+.wrapper {
+	position: fixed;
+	inset: 0;
+	z-index: 2147483641; /* 确保在所有元素之上，除了 panel */
+	/* pointer-events: none; */
+	cursor: not-allowed;
+	overflow: hidden;
+
+	display: none;
+}

packages/page-controller/src/SimulatorMask.ts

@@ -0,0 +1,172 @@
+import { Motion } from 'ai-motion'
+
+import { isPageDark } from './checkDarkMode'
+
+import styles from './SimulatorMask.module.css'
+import cursorStyles from './cursor.module.css'
+
+export class SimulatorMask {
+	wrapper = document.createElement('div')
+	motion = new Motion({
+		mode: isPageDark() ? 'dark' : 'light',
+		styles: {
+			position: 'absolute',
+			inset: '0',
+		},
+	})
+
+	#cursor = document.createElement('div')
+
+	#currentCursorX = 0
+	#currentCursorY = 0
+
+	#targetCursorX = 0
+	#targetCursorY = 0
+
+	constructor() {
+		this.wrapper.id = 'page-agent-runtime_simulator-mask'
+		this.wrapper.className = styles.wrapper
+		this.wrapper.setAttribute('data-browser-use-ignore', 'true')
+
+		this.wrapper.appendChild(this.motion.element)
+		this.motion.autoResize(this.wrapper)
+
+		// Capture all mouse, keyboard, and wheel events
+		this.wrapper.addEventListener('click', (e) => {
+			e.stopPropagation()
+			e.preventDefault()
+		})
+		this.wrapper.addEventListener('mousedown', (e) => {
+			e.stopPropagation()
+			e.preventDefault()
+		})
+		this.wrapper.addEventListener('mouseup', (e) => {
+			e.stopPropagation()
+			e.preventDefault()
+		})
+		this.wrapper.addEventListener('mousemove', (e) => {
+			e.stopPropagation()
+			e.preventDefault()
+		})
+		this.wrapper.addEventListener('wheel', (e) => {
+			e.stopPropagation()
+			e.preventDefault()
+		})
+		this.wrapper.addEventListener('keydown', (e) => {
+			e.stopPropagation()
+			e.preventDefault()
+		})
+		this.wrapper.addEventListener('keyup', (e) => {
+			e.stopPropagation()
+			e.preventDefault()
+		})
+
+		// Create AI cursor
+		this.#createCursor()
+		// this.show()
+
+		document.body.appendChild(this.wrapper)
+
+		this.#moveCursorToTarget()
+
+		window.addEventListener('PageAgent::MovePointerTo', (event: Event) => {
+			const { x, y } = (event as CustomEvent).detail
+			this.setCursorPosition(x, y)
+		})
+
+		window.addEventListener('PageAgent::ClickPointer', (event: Event) => {
+			this.triggerClickAnimation()
+		})
+	}
+
+	#createCursor() {
+		this.#cursor.className = cursorStyles.cursor
+
+		// Create ripple effect container
+		const rippleContainer = document.createElement('div')
+		rippleContainer.className = cursorStyles.cursorRipple
+		this.#cursor.appendChild(rippleContainer)
+
+		// Create filling layer
+		const fillingLayer = document.createElement('div')
+		fillingLayer.className = cursorStyles.cursorFilling
+		this.#cursor.appendChild(fillingLayer)
+
+		// Create border layer
+		const borderLayer = document.createElement('div')
+		borderLayer.className = cursorStyles.cursorBorder
+		this.#cursor.appendChild(borderLayer)
+
+		this.wrapper.appendChild(this.#cursor)
+	}
+
+	#moveCursorToTarget() {
+		const newX = this.#currentCursorX + (this.#targetCursorX - this.#currentCursorX) * 0.2
+		const newY = this.#currentCursorY + (this.#targetCursorY - this.#currentCursorY) * 0.2
+
+		const xDistance = Math.abs(newX - this.#targetCursorX)
+		if (xDistance > 0) {
+			if (xDistance < 2) {
+				this.#currentCursorX = this.#targetCursorX
+			} else {
+				this.#currentCursorX = newX
+			}
+			this.#cursor.style.left = `${this.#currentCursorX}px`
+		}
+
+		const yDistance = Math.abs(newY - this.#targetCursorY)
+		if (yDistance > 0) {
+			if (yDistance < 2) {
+				this.#currentCursorY = this.#targetCursorY
+			} else {
+				this.#currentCursorY = newY
+			}
+			this.#cursor.style.top = `${this.#currentCursorY}px`
+		}
+
+		requestAnimationFrame(() => this.#moveCursorToTarget())
+	}
+
+	setCursorPosition(x: number, y: number) {
+		this.#targetCursorX = x
+		this.#targetCursorY = y
+	}
+
+	triggerClickAnimation() {
+		this.#cursor.classList.remove(cursorStyles.clicking)
+		// Force reflow to restart animation
+		void this.#cursor.offsetHeight
+		this.#cursor.classList.add(cursorStyles.clicking)
+	}
+
+	show() {
+		this.motion.start()
+		this.motion.fadeIn()
+
+		this.wrapper.style.display = 'block'
+
+		// Initialize cursor position
+		this.#currentCursorX = window.innerWidth / 2
+		this.#currentCursorY = window.innerHeight / 2
+		this.#targetCursorX = this.#currentCursorX
+		this.#targetCursorY = this.#currentCursorY
+		this.#cursor.style.left = `${this.#currentCursorX}px`
+		this.#cursor.style.top = `${this.#currentCursorY}px`
+	}
+
+	hide() {
+		this.motion.fadeOut()
+		this.motion.pause()
+
+		this.#cursor.classList.remove(cursorStyles.clicking)
+
+		setTimeout(() => {
+			this.wrapper.style.display = 'none'
+		}, 800) // Match the animation duration
+	}
+
+	dispose() {
+		this.motion.dispose()
+		this.wrapper.remove()
+	}
+}

packages/page-controller/src/checkDarkMode.ts

@@ -0,0 +1,115 @@
+/**
+ * Checks for common dark mode CSS classes on the html or body elements.
+ * @returns {boolean} - True if a common dark mode class is found.
+ */
+function hasDarkModeClass() {
+	const DEFAULT_DARK_MODE_CLASSES = ['dark', 'dark-mode', 'theme-dark', 'night', 'night-mode']
+
+	const htmlElement = document.documentElement
+	const bodyElement = document.body || document.documentElement // can be null in some cases
+
+	// Check class names on <html> and <body>
+	for (const className of DEFAULT_DARK_MODE_CLASSES) {
+		if (htmlElement.classList.contains(className) || bodyElement?.classList.contains(className)) {
+			return true
+		}
+	}
+
+	// Some sites use data attributes
+	const darkThemeAttribute = htmlElement.getAttribute('data-theme')
+	if (darkThemeAttribute?.toLowerCase().includes('dark')) {
+		return true
+	}
+
+	return false
+}
+
+/**
+ * Parses an RGB or RGBA color string and returns an object with r, g, b properties.
+ * @param {string} colorString - e.g., "rgb(34, 34, 34)" or "rgba(0, 0, 0, 0.5)"
+ * @returns {{r: number, g: number, b: number}|null}
+ */
+function parseRgbColor(colorString: string) {
+	const rgbMatch = /rgba?\((\d+),\s*(\d+),\s*(\d+)/.exec(colorString)
+	if (!rgbMatch) {
+		return null // Not a valid rgb/rgba string
+	}
+	return {
+		r: parseInt(rgbMatch[1]),
+		g: parseInt(rgbMatch[2]),
+		b: parseInt(rgbMatch[3]),
+	}
+}
+
+/**
+ * Determines if a color is "dark" based on its calculated luminance.
+ * @param {string} colorString - The CSS color string (e.g., "rgb(50, 50, 50)").
+ * @param {number} threshold - A value between 0 and 255. Colors with luminance below this will be considered dark. Default is 128.
+ * @returns {boolean} - True if the color is considered dark.
+ */
+function isColorDark(colorString: string, threshold = 128) {
+	if (!colorString || colorString === 'transparent' || colorString.startsWith('rgba(0, 0, 0, 0)')) {
+		return false // Transparent is not dark
+	}
+
+	const rgb = parseRgbColor(colorString)
+	if (!rgb) {
+		return false // Could not parse color
+	}
+
+	// Calculate perceived luminance using the standard formula
+	const luminance = 0.299 * rgb.r + 0.587 * rgb.g + 0.114 * rgb.b
+
+	return luminance < threshold
+}
+
+/**
+ * Checks the background color of the body element to determine if the page is dark.
+ * @returns {boolean}
+ */
+function isBackgroundDark() {
+	// We check both <html> and <body> because some pages set the color on <html>
+	const htmlStyle = window.getComputedStyle(document.documentElement)
+	const bodyStyle = window.getComputedStyle(document.body || document.documentElement)
+
+	// Get background colors
+	const htmlBgColor = htmlStyle.backgroundColor
+	const bodyBgColor = bodyStyle.backgroundColor
+
+	// The body's background might be transparent, in which case we should
+	// fall back to the html element's background.
+	if (isColorDark(bodyBgColor)) {
+		return true
+	} else if (bodyBgColor === 'transparent' || bodyBgColor.startsWith('rgba(0, 0, 0, 0)')) {
+		return isColorDark(htmlBgColor)
+	}
+
+	return false
+}
+
+/**
+ * A comprehensive function to determine if the page is currently in a dark theme.
+ * It combines class checking and background color analysis.
+ * @returns {boolean} - True if the page is likely dark.
+ */
+export function isPageDark() {
+	try {
+		// Strategy 1: Check for common dark mode classes
+		if (hasDarkModeClass()) {
+			return true
+		}
+
+		// Strategy 2: Analyze the computed background color
+		if (isBackgroundDark()) {
+			return true
+		}
+
+		// @TODO add more checks here, e.g., analyzing text color,
+		// or checking the background of major layout elements like <main> or #app.
+
+		return false
+	} catch (error) {
+		console.warn('Error determining if page is dark:', error)
+		return false
+	}
+}

packages/page-controller/src/cursor.module.css

@@ -0,0 +1,91 @@
+/* AI 光标样式 */
+.cursor {
+	position: absolute;
+	width: var(--cursor-size, 75px);
+	height: var(--cursor-size, 75px);
+	pointer-events: none;
+	z-index: 10000;
+	transform: translate(-30%, -30%);
+
+	animation: cursor-enter 300ms ease-out forwards;
+}
+
+.cursorBorder {
+	position: absolute;
+	inset: 0;
+	background: linear-gradient(45deg, rgb(57, 182, 255), rgb(189, 69, 251));
+	mask-image: url(https://img.alicdn.com/imgextra/i1/O1CN01YHLVYR1LvqWIyo5kH_!!6000000001362-2-tps-202-202.png);
+	mask-size: 100% 100%;
+	mask-repeat: no-repeat;
+	animation: cursor-breathe 2s ease-in-out infinite;
+}
+
+.cursorFilling {
+	position: absolute;
+	inset: 0;
+	background: url(https://img.alicdn.com/imgextra/i3/O1CN01JZOqOS1Tu1sIKbPLW_!!6000000002441-2-tps-202-202.png);
+	background-size: 100% 100%;
+	background-repeat: no-repeat;
+}
+
+.cursorRipple {
+	position: absolute;
+	inset: 0;
+	pointer-events: none;
+}
+
+.cursor.clicking .cursorRipple::after {
+	content: '';
+	position: absolute;
+	width: 100%;
+	height: 100%;
+	left: -30%;
+	top: -30%;
+	border: 4px solid rgba(57, 182, 255, 1);
+	border-radius: 50%;
+	animation: cursor-ripple 300ms ease-out forwards;
+}
+
+/* 光标动画关键帧 */
+@keyframes cursor-breathe {
+	0%,
+	100% {
+		transform: scale(1);
+		opacity: 0.9;
+	}
+	50% {
+		transform: scale(1.05);
+		opacity: 1;
+	}
+}
+
+@keyframes cursor-rotate {
+	0% {
+		transform: rotate(0deg);
+	}
+	100% {
+		transform: rotate(360deg);
+	}
+}
+
+@keyframes cursor-enter {
+	0% {
+		transform: translate(-30%, -30%) scale(0.5);
+		opacity: 0;
+	}
+	100% {
+		transform: translate(-30%, -30%) scale(1);
+		opacity: 1;
+	}
+}
+
+@keyframes cursor-ripple {
+	0% {
+		transform: scale(0);
+		opacity: 1;
+	}
+	100% {
+		transform: scale(2);
+		opacity: 0;
+	}
+}

packages/page-controller/src/env.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="vite/client" />
+
+declare module '*.module.css' {
+	const classes: Record<string, string>
+	export default classes
+}

packages/page-controller/vite.config.js

@@ -29,7 +29,7 @@ export default defineConfig({
 		},
 		outDir: resolve(__dirname, 'dist', 'lib'),
 		rollupOptions: {
-			external: ['@page-agent/*'],
+			external: ['@page-agent/*', 'ai-motion'],
 		},
 		minify: false,
 		sourcemap: true,