Merge branch 'main' into fix/scroll-direction-pixels

packages/page-controller/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@page-agent/page-controller",
-    "version": "1.6.0",
+    "version": "1.7.0",
     "type": "module",
     "main": "./dist/lib/page-controller.js",
     "module": "./dist/lib/page-controller.js",

packages/page-controller/src/PageController.ts

@@ -218,6 +218,7 @@ export class PageController extends EventTarget {
 	 * Clean up all element highlights
 	 */
 	async cleanUpHighlights(): Promise<void> {
+		console.log('[PageController] cleanUpHighlights')
 		dom.cleanUpHighlights()
 	}
 
@@ -424,3 +425,5 @@ export class PageController extends EventTarget {
 		this.mask = null
 	}
 }
+
+export * from './actions'

packages/page-controller/src/actions.ts

@@ -4,6 +4,9 @@
  */
 import type { InteractiveElementDomNode } from './dom/dom_tree/type'
 import {
+	clickPointer,
+	disablePassThrough,
+	enablePassThrough,
 	getNativeValueSetter,
 	isHTMLElement,
 	isInputElement,
@@ -15,6 +18,7 @@ import {
 
 /**
  * Get the HTMLElement by index from a selectorMap.
+ * @private Internal method, subject to change at any time.
  */
 export function getElementByIndex(
 	selectorMap: Map<number, InteractiveElementDomNode>,
@@ -41,19 +45,21 @@ let lastClickedElement: HTMLElement | null = null
 
 function blurLastClickedElement() {
 	if (lastClickedElement) {
+		lastClickedElement.dispatchEvent(new PointerEvent('pointerout', { bubbles: true }))
+		lastClickedElement.dispatchEvent(new PointerEvent('pointerleave', { bubbles: false }))
+		lastClickedElement.dispatchEvent(new MouseEvent('mouseout', { bubbles: true }))
+		lastClickedElement.dispatchEvent(new MouseEvent('mouseleave', { bubbles: false }))
 		lastClickedElement.blur()
-		lastClickedElement.dispatchEvent(
-			new MouseEvent('mouseout', { bubbles: true, cancelable: true })
-		)
-		lastClickedElement.dispatchEvent(
-			new MouseEvent('mouseleave', { bubbles: false, cancelable: true })
-		)
 		lastClickedElement = null
 	}
 }
 
 /**
- * Simulate a click on the element
+ * Simulate a full click following W3C Pointer Events + UI Events spec order:
+ * pointerover/enter → mouseover/enter → pointerdown → mousedown → [focus] →
+ * pointerup → mouseup → click
+ *
+ * @private Internal method, subject to change at any time.
  */
 export async function clickElement(element: HTMLElement) {
 	blurLastClickedElement()
@@ -61,34 +67,67 @@ export async function clickElement(element: HTMLElement) {
 	lastClickedElement = element
 
 	await scrollIntoViewIfNeeded(element)
-	// Scroll the iframe element itself into view if needed
 	const frame = element.ownerDocument.defaultView?.frameElement
 	if (frame) await scrollIntoViewIfNeeded(frame)
 
-	await movePointerToElement(element)
-	window.dispatchEvent(new CustomEvent('PageAgent::ClickPointer'))
+	const rect = element.getBoundingClientRect()
+	const x = rect.left + rect.width / 2
+	const y = rect.top + rect.height / 2
+
+	await movePointerToElement(element, x, y)
+	await clickPointer()
 
 	await waitFor(0.1)
 
-	// hover it
-	element.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true, cancelable: true }))
-	element.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true }))
+	// Hit-test to find the deepest element at click coordinates, matching
+	// real browser behavior where events target the innermost element.
+	// @note This may hit a element in the blacklist
+	// TODO: This is a temporary workaround. Should have been handled during dom extraction.
+	const doc = element.ownerDocument
+	await enablePassThrough()
+	const hitTarget = doc.elementFromPoint(x, y)
+	await disablePassThrough()
+	const target =
+		hitTarget instanceof HTMLElement && element.contains(hitTarget) ? hitTarget : element
 
-	// dispatch a sequence of events to ensure all listeners are triggered
-	element.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, cancelable: true }))
+	const pointerOpts = {
+		bubbles: true,
+		cancelable: true,
+		clientX: x,
+		clientY: y,
+		pointerType: 'mouse',
+	}
+	const mouseOpts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 }
 
-	// focus it to ensure it gets the click event
-	element.focus()
+	// Hover — pointer events first, then mouse events (spec order)
+	target.dispatchEvent(new PointerEvent('pointerover', pointerOpts))
+	target.dispatchEvent(new PointerEvent('pointerenter', { ...pointerOpts, bubbles: false }))
+	target.dispatchEvent(new MouseEvent('mouseover', mouseOpts))
+	target.dispatchEvent(new MouseEvent('mouseenter', { ...mouseOpts, bubbles: false }))
 
-	element.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, cancelable: true }))
-	element.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }))
+	// Press
+	target.dispatchEvent(new PointerEvent('pointerdown', pointerOpts))
+	target.dispatchEvent(new MouseEvent('mousedown', mouseOpts))
 
-	// dispatch a click event
-	// element.click()
+	// Focus is not part of the standard pointer/mouse event sequence
+	// "undefined and varies between user agents".
+	// We focus the original element (nearest focusable ancestor), not the hit-test target, matching browser behavior.
+	element.focus({ preventScroll: true })
 
-	await waitFor(0.2) // Wait to ensure click event processing completes
+	// Release
+	target.dispatchEvent(new PointerEvent('pointerup', pointerOpts))
+	target.dispatchEvent(new MouseEvent('mouseup', mouseOpts))
+
+	// Click — activation behavior (navigation, form submit, etc.) triggers
+	// via bubbling from target up to the interactive ancestor.
+	target.click()
+
+	await waitFor(0.2)
 }
 
+/**
+ * @private Internal method, subject to change at any time.
+ */
 export async function inputTextElement(element: HTMLElement, text: string) {
 	const isContentEditable = element.isContentEditable
 	if (!isInputElement(element) && !isTextAreaElement(element) && !isContentEditable) {
@@ -196,6 +235,7 @@ export async function inputTextElement(element: HTMLElement, text: string) {
 
 /**
  * @todo browser-use version is very complex and supports menu tags, need to follow up
+ * @private Internal method, subject to change at any time.
  */
 export async function selectOptionElement(selectElement: HTMLSelectElement, optionText: string) {
 	if (!isSelectElement(selectElement)) {
@@ -219,6 +259,9 @@ interface ScrollableElement extends Element {
 	scrollIntoViewIfNeeded?: (centerIfNeeded?: boolean) => void
 }
 
+/**
+ * @private Internal method, subject to change at any time.
+ */
 export async function scrollIntoViewIfNeeded(element: Element) {
 	const el = element as ScrollableElement
 	if (typeof el.scrollIntoViewIfNeeded === 'function') {

packages/page-controller/src/dom/dom_tree/index.js

@@ -18,6 +18,7 @@
  * @edit improve `sampleRect`, filter out rects with 0 area
  * @edit exclude aria-hidden elements
  * @edit make sure attributes exist for interactive candidates.
+ * @edit fix "aria-*" attributes check
  */
 
 export default (
@@ -1143,6 +1144,31 @@ export default (
 	 * @param {HTMLElement} element - The element to check.
 	 * @returns {boolean} Whether the element is an interactive candidate.
 	 */
+
+	// @edit fix "aria-*" attributes check
+	const INTERACTIVE_ARIA_ATTRS = [
+		'aria-expanded',
+		'aria-checked',
+		'aria-selected',
+		'aria-pressed',
+		'aria-haspopup',
+		'aria-controls',
+		'aria-owns',
+		'aria-activedescendant',
+		'aria-valuenow',
+		'aria-valuetext',
+		'aria-valuemax',
+		'aria-valuemin',
+		'aria-autocomplete',
+	]
+
+	function hasInteractiveAria(el) {
+		for (let i = 0; i < INTERACTIVE_ARIA_ATTRS.length; i++) {
+			if (el.hasAttribute(INTERACTIVE_ARIA_ATTRS[i])) return true
+		}
+		return false
+	}
+
 	function isInteractiveCandidate(element) {
 		if (!element || element.nodeType !== Node.ELEMENT_NODE) return false
 
@@ -1167,7 +1193,7 @@ export default (
 			element.hasAttribute('onclick') ||
 			element.hasAttribute('role') ||
 			element.hasAttribute('tabindex') ||
-			element.hasAttribute('aria-') ||
+			hasInteractiveAria(element) ||
 			element.hasAttribute('data-action') ||
 			element.getAttribute('contenteditable') === 'true'
 

packages/page-controller/src/mask/SimulatorMask.ts

@@ -5,7 +5,7 @@ import { isPageDark } from './checkDarkMode'
 import styles from './SimulatorMask.module.css'
 import cursorStyles from './cursor.module.css'
 
-export class SimulatorMask {
+export class SimulatorMask extends EventTarget {
 	shown: boolean = false
 	wrapper = document.createElement('div')
 	motion: Motion | null = null
@@ -19,6 +19,8 @@ export class SimulatorMask {
 	#targetCursorY = 0
 
 	constructor() {
+		super()
+
 		this.wrapper.id = 'page-agent-runtime_simulator-mask'
 		this.wrapper.className = styles.wrapper
 		this.wrapper.setAttribute('data-browser-use-ignore', 'true')
@@ -74,13 +76,34 @@ export class SimulatorMask {
 
 		this.#moveCursorToTarget()
 
-		window.addEventListener('PageAgent::MovePointerTo', (event: Event) => {
+		// global events
+		// @note Mask should be isolated from the rest of the code.
+		// Global events are easier to manage and cleanup.
+
+		const movePointerToListener = (event: Event) => {
 			const { x, y } = (event as CustomEvent).detail
 			this.setCursorPosition(x, y)
-		})
-
-		window.addEventListener('PageAgent::ClickPointer', (event: Event) => {
+		}
+		const clickPointerListener = () => {
 			this.triggerClickAnimation()
+		}
+		const enablePassThroughListener = () => {
+			this.wrapper.style.pointerEvents = 'none'
+		}
+		const disablePassThroughListener = () => {
+			this.wrapper.style.pointerEvents = 'auto'
+		}
+
+		window.addEventListener('PageAgent::MovePointerTo', movePointerToListener)
+		window.addEventListener('PageAgent::ClickPointer', clickPointerListener)
+		window.addEventListener('PageAgent::EnablePassThrough', enablePassThroughListener)
+		window.addEventListener('PageAgent::DisablePassThrough', disablePassThroughListener)
+
+		this.addEventListener('dispose', () => {
+			window.removeEventListener('PageAgent::MovePointerTo', movePointerToListener)
+			window.removeEventListener('PageAgent::ClickPointer', clickPointerListener)
+			window.removeEventListener('PageAgent::EnablePassThrough', enablePassThroughListener)
+			window.removeEventListener('PageAgent::DisablePassThrough', disablePassThroughListener)
 		})
 	}
 
@@ -177,7 +200,9 @@ export class SimulatorMask {
 	}
 
 	dispose() {
+		console.log('dispose SimulatorMask')
 		this.motion?.dispose()
 		this.wrapper.remove()
+		this.dispatchEvent(new Event('dispose'))
 	}
 }

packages/page-controller/src/utils/index.ts

@@ -48,15 +48,33 @@ export async function waitFor(seconds: number): Promise<void> {
 	await new Promise((resolve) => setTimeout(resolve, seconds * 1000))
 }
 
-// ======= dom utils =======
+// ======= mask events =======
 
-export async function movePointerToElement(element: HTMLElement) {
-	const rect = element.getBoundingClientRect()
+/**
+ * Move the visual pointer to a position within an element.
+ * @param x - x coordinate in the element's document viewport
+ * @param y - y coordinate in the element's document viewport
+ */
+export async function movePointerToElement(element: HTMLElement, x: number, y: number) {
 	const offset = getIframeOffset(element)
-	const x = rect.left + rect.width / 2 + offset.x
-	const y = rect.top + rect.height / 2 + offset.y
 
-	window.dispatchEvent(new CustomEvent('PageAgent::MovePointerTo', { detail: { x, y } }))
+	window.dispatchEvent(
+		new CustomEvent('PageAgent::MovePointerTo', {
+			detail: { x: x + offset.x, y: y + offset.y },
+		})
+	)
 
 	await waitFor(0.3)
 }
+
+export async function clickPointer() {
+	window.dispatchEvent(new CustomEvent('PageAgent::ClickPointer'))
+}
+
+export async function enablePassThrough() {
+	window.dispatchEvent(new CustomEvent('PageAgent::EnablePassThrough'))
+}
+
+export async function disablePassThrough() {
+	window.dispatchEvent(new CustomEvent('PageAgent::DisablePassThrough'))
+}