Merge pull request #378 from alibaba/fix/controller-click-action-robust

packages/page-controller/src/PageController.ts

@@ -218,6 +218,7 @@ export class PageController extends EventTarget {
 	 * Clean up all element highlights
 	 */
 	async cleanUpHighlights(): Promise<void> {
+		console.log('[PageController] cleanUpHighlights')
 		dom.cleanUpHighlights()
 	}
 

packages/page-controller/src/actions.ts

@@ -4,6 +4,9 @@
  */
 import type { InteractiveElementDomNode } from './dom/dom_tree/type'
 import {
+	clickPointer,
+	disablePassThrough,
+	enablePassThrough,
 	getNativeValueSetter,
 	isHTMLElement,
 	isInputElement,
@@ -42,19 +45,20 @@ let lastClickedElement: HTMLElement | null = null
 
 function blurLastClickedElement() {
 	if (lastClickedElement) {
+		lastClickedElement.dispatchEvent(new PointerEvent('pointerout', { bubbles: true }))
+		lastClickedElement.dispatchEvent(new PointerEvent('pointerleave', { bubbles: false }))
+		lastClickedElement.dispatchEvent(new MouseEvent('mouseout', { bubbles: true }))
+		lastClickedElement.dispatchEvent(new MouseEvent('mouseleave', { bubbles: false }))
 		lastClickedElement.blur()
-		lastClickedElement.dispatchEvent(
-			new MouseEvent('mouseout', { bubbles: true, cancelable: true })
-		)
-		lastClickedElement.dispatchEvent(
-			new MouseEvent('mouseleave', { bubbles: false, cancelable: true })
-		)
 		lastClickedElement = null
 	}
 }
 
 /**
- * Simulate a click on the element
+ * Simulate a full click following W3C Pointer Events + UI Events spec order:
+ * pointerover/enter → mouseover/enter → pointerdown → mousedown → [focus] →
+ * pointerup → mouseup → click
+ *
  * @private Internal method, subject to change at any time.
  */
 export async function clickElement(element: HTMLElement) {
@@ -63,32 +67,62 @@ export async function clickElement(element: HTMLElement) {
 	lastClickedElement = element
 
 	await scrollIntoViewIfNeeded(element)
-	// Scroll the iframe element itself into view if needed
 	const frame = element.ownerDocument.defaultView?.frameElement
 	if (frame) await scrollIntoViewIfNeeded(frame)
 
-	await movePointerToElement(element)
-	window.dispatchEvent(new CustomEvent('PageAgent::ClickPointer'))
+	const rect = element.getBoundingClientRect()
+	const x = rect.left + rect.width / 2
+	const y = rect.top + rect.height / 2
+
+	await movePointerToElement(element, x, y)
+	await clickPointer()
 
 	await waitFor(0.1)
 
-	// hover it
-	element.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true, cancelable: true }))
-	element.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true }))
+	// Hit-test to find the deepest element at click coordinates, matching
+	// real browser behavior where events target the innermost element.
+	// @note This may hit a element in the blacklist
+	// TODO: This is a temporary workaround. Should have been handled during dom extraction.
+	const doc = element.ownerDocument
+	await enablePassThrough()
+	const hitTarget = doc.elementFromPoint(x, y)
+	await disablePassThrough()
+	const target =
+		hitTarget instanceof HTMLElement && element.contains(hitTarget) ? hitTarget : element
 
-	// dispatch a sequence of events to ensure all listeners are triggered
-	element.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, cancelable: true }))
+	const pointerOpts = {
+		bubbles: true,
+		cancelable: true,
+		clientX: x,
+		clientY: y,
+		pointerType: 'mouse',
+	}
+	const mouseOpts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 }
 
-	// focus it to ensure it gets the click event
-	element.focus()
+	// Hover — pointer events first, then mouse events (spec order)
+	target.dispatchEvent(new PointerEvent('pointerover', pointerOpts))
+	target.dispatchEvent(new PointerEvent('pointerenter', { ...pointerOpts, bubbles: false }))
+	target.dispatchEvent(new MouseEvent('mouseover', mouseOpts))
+	target.dispatchEvent(new MouseEvent('mouseenter', { ...mouseOpts, bubbles: false }))
 
-	element.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, cancelable: true }))
-	element.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }))
+	// Press
+	target.dispatchEvent(new PointerEvent('pointerdown', pointerOpts))
+	target.dispatchEvent(new MouseEvent('mousedown', mouseOpts))
 
-	// dispatch a click event
-	// element.click()
+	// Focus is not part of the standard pointer/mouse event sequence
+	// "undefined and varies between user agents".
+	// We focus the original element (nearest focusable ancestor), not the hit-test target, matching browser behavior.
+	element.focus({ preventScroll: true })
 
-	await waitFor(0.2) // Wait to ensure click event processing completes
+	// Release
+	target.dispatchEvent(new PointerEvent('pointerup', pointerOpts))
+	target.dispatchEvent(new MouseEvent('mouseup', mouseOpts))
+
+	// Click — activation behavior (navigation, form submit, etc.) triggers
+	// via bubbling from target up to the interactive ancestor.
+	target.click()
+
+	await waitFor(0.2)
 }
 
 /**

packages/page-controller/src/mask/SimulatorMask.ts

@@ -5,7 +5,7 @@ import { isPageDark } from './checkDarkMode'
 import styles from './SimulatorMask.module.css'
 import cursorStyles from './cursor.module.css'
 
-export class SimulatorMask {
+export class SimulatorMask extends EventTarget {
 	shown: boolean = false
 	wrapper = document.createElement('div')
 	motion: Motion | null = null
@@ -19,6 +19,8 @@ export class SimulatorMask {
 	#targetCursorY = 0
 
 	constructor() {
+		super()
+
 		this.wrapper.id = 'page-agent-runtime_simulator-mask'
 		this.wrapper.className = styles.wrapper
 		this.wrapper.setAttribute('data-browser-use-ignore', 'true')
@@ -74,13 +76,34 @@ export class SimulatorMask {
 
 		this.#moveCursorToTarget()
 
-		window.addEventListener('PageAgent::MovePointerTo', (event: Event) => {
+		// global events
+		// @note Mask should be isolated from the rest of the code.
+		// Global events are easier to manage and cleanup.
+
+		const movePointerToListener = (event: Event) => {
 			const { x, y } = (event as CustomEvent).detail
 			this.setCursorPosition(x, y)
-		})
-
-		window.addEventListener('PageAgent::ClickPointer', (event: Event) => {
+		}
+		const clickPointerListener = () => {
 			this.triggerClickAnimation()
+		}
+		const enablePassThroughListener = () => {
+			this.wrapper.style.pointerEvents = 'none'
+		}
+		const disablePassThroughListener = () => {
+			this.wrapper.style.pointerEvents = 'auto'
+		}
+
+		window.addEventListener('PageAgent::MovePointerTo', movePointerToListener)
+		window.addEventListener('PageAgent::ClickPointer', clickPointerListener)
+		window.addEventListener('PageAgent::EnablePassThrough', enablePassThroughListener)
+		window.addEventListener('PageAgent::DisablePassThrough', disablePassThroughListener)
+
+		this.addEventListener('dispose', () => {
+			window.removeEventListener('PageAgent::MovePointerTo', movePointerToListener)
+			window.removeEventListener('PageAgent::ClickPointer', clickPointerListener)
+			window.removeEventListener('PageAgent::EnablePassThrough', enablePassThroughListener)
+			window.removeEventListener('PageAgent::DisablePassThrough', disablePassThroughListener)
 		})
 	}
 
@@ -177,7 +200,9 @@ export class SimulatorMask {
 	}
 
 	dispose() {
+		console.log('dispose SimulatorMask')
 		this.motion?.dispose()
 		this.wrapper.remove()
+		this.dispatchEvent(new Event('dispose'))
 	}
 }

packages/page-controller/src/utils/index.ts

@@ -48,15 +48,33 @@ export async function waitFor(seconds: number): Promise<void> {
 	await new Promise((resolve) => setTimeout(resolve, seconds * 1000))
 }
 
-// ======= dom utils =======
+// ======= mask events =======
 
-export async function movePointerToElement(element: HTMLElement) {
-	const rect = element.getBoundingClientRect()
+/**
+ * Move the visual pointer to a position within an element.
+ * @param x - x coordinate in the element's document viewport
+ * @param y - y coordinate in the element's document viewport
+ */
+export async function movePointerToElement(element: HTMLElement, x: number, y: number) {
 	const offset = getIframeOffset(element)
-	const x = rect.left + rect.width / 2 + offset.x
-	const y = rect.top + rect.height / 2 + offset.y
 
-	window.dispatchEvent(new CustomEvent('PageAgent::MovePointerTo', { detail: { x, y } }))
+	window.dispatchEvent(
+		new CustomEvent('PageAgent::MovePointerTo', {
+			detail: { x: x + offset.x, y: y + offset.y },
+		})
+	)
 
 	await waitFor(0.3)
 }
+
+export async function clickPointer() {
+	window.dispatchEvent(new CustomEvent('PageAgent::ClickPointer'))
+}
+
+export async function enablePassThrough() {
+	window.dispatchEvent(new CustomEvent('PageAgent::EnablePassThrough'))
+}
+
+export async function disablePassThrough() {
+	window.dispatchEvent(new CustomEvent('PageAgent::DisablePassThrough'))
+}