diff --git a/packages/page-controller/src/PageController.ts b/packages/page-controller/src/PageController.ts index a8392ae..2439691 100644 --- a/packages/page-controller/src/PageController.ts +++ b/packages/page-controller/src/PageController.ts @@ -218,6 +218,7 @@ export class PageController extends EventTarget { * Clean up all element highlights */ async cleanUpHighlights(): Promise { + console.log('[PageController] cleanUpHighlights') dom.cleanUpHighlights() } diff --git a/packages/page-controller/src/actions.ts b/packages/page-controller/src/actions.ts index 92c7388..8a86fa7 100644 --- a/packages/page-controller/src/actions.ts +++ b/packages/page-controller/src/actions.ts @@ -4,6 +4,9 @@ */ import type { InteractiveElementDomNode } from './dom/dom_tree/type' import { + clickPointer, + disablePassThrough, + enablePassThrough, getNativeValueSetter, isHTMLElement, isInputElement, @@ -42,19 +45,20 @@ let lastClickedElement: HTMLElement | null = null function blurLastClickedElement() { if (lastClickedElement) { + lastClickedElement.dispatchEvent(new PointerEvent('pointerout', { bubbles: true })) + lastClickedElement.dispatchEvent(new PointerEvent('pointerleave', { bubbles: false })) + lastClickedElement.dispatchEvent(new MouseEvent('mouseout', { bubbles: true })) + lastClickedElement.dispatchEvent(new MouseEvent('mouseleave', { bubbles: false })) lastClickedElement.blur() - lastClickedElement.dispatchEvent( - new MouseEvent('mouseout', { bubbles: true, cancelable: true }) - ) - lastClickedElement.dispatchEvent( - new MouseEvent('mouseleave', { bubbles: false, cancelable: true }) - ) lastClickedElement = null } } /** - * Simulate a click on the element + * Simulate a full click following W3C Pointer Events + UI Events spec order: + * pointerover/enter → mouseover/enter → pointerdown → mousedown → [focus] → + * pointerup → mouseup → click + * * @private Internal method, subject to change at any time. */ export async function clickElement(element: HTMLElement) { @@ -63,32 +67,62 @@ export async function clickElement(element: HTMLElement) { lastClickedElement = element await scrollIntoViewIfNeeded(element) - // Scroll the iframe element itself into view if needed const frame = element.ownerDocument.defaultView?.frameElement if (frame) await scrollIntoViewIfNeeded(frame) - await movePointerToElement(element) - window.dispatchEvent(new CustomEvent('PageAgent::ClickPointer')) + const rect = element.getBoundingClientRect() + const x = rect.left + rect.width / 2 + const y = rect.top + rect.height / 2 + + await movePointerToElement(element, x, y) + await clickPointer() await waitFor(0.1) - // hover it - element.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true, cancelable: true })) - element.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true })) + // Hit-test to find the deepest element at click coordinates, matching + // real browser behavior where events target the innermost element. + // @note This may hit a element in the blacklist + // TODO: This is a temporary workaround. Should have been handled during dom extraction. + const doc = element.ownerDocument + await enablePassThrough() + const hitTarget = doc.elementFromPoint(x, y) + await disablePassThrough() + const target = + hitTarget instanceof HTMLElement && element.contains(hitTarget) ? hitTarget : element - // dispatch a sequence of events to ensure all listeners are triggered - element.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, cancelable: true })) + const pointerOpts = { + bubbles: true, + cancelable: true, + clientX: x, + clientY: y, + pointerType: 'mouse', + } + const mouseOpts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 } - // focus it to ensure it gets the click event - element.focus() + // Hover — pointer events first, then mouse events (spec order) + target.dispatchEvent(new PointerEvent('pointerover', pointerOpts)) + target.dispatchEvent(new PointerEvent('pointerenter', { ...pointerOpts, bubbles: false })) + target.dispatchEvent(new MouseEvent('mouseover', mouseOpts)) + target.dispatchEvent(new MouseEvent('mouseenter', { ...mouseOpts, bubbles: false })) - element.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, cancelable: true })) - element.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })) + // Press + target.dispatchEvent(new PointerEvent('pointerdown', pointerOpts)) + target.dispatchEvent(new MouseEvent('mousedown', mouseOpts)) - // dispatch a click event - // element.click() + // Focus is not part of the standard pointer/mouse event sequence + // "undefined and varies between user agents". + // We focus the original element (nearest focusable ancestor), not the hit-test target, matching browser behavior. + element.focus({ preventScroll: true }) - await waitFor(0.2) // Wait to ensure click event processing completes + // Release + target.dispatchEvent(new PointerEvent('pointerup', pointerOpts)) + target.dispatchEvent(new MouseEvent('mouseup', mouseOpts)) + + // Click — activation behavior (navigation, form submit, etc.) triggers + // via bubbling from target up to the interactive ancestor. + target.click() + + await waitFor(0.2) } /** diff --git a/packages/page-controller/src/mask/SimulatorMask.ts b/packages/page-controller/src/mask/SimulatorMask.ts index eb13eb6..41cbdb4 100644 --- a/packages/page-controller/src/mask/SimulatorMask.ts +++ b/packages/page-controller/src/mask/SimulatorMask.ts @@ -5,7 +5,7 @@ import { isPageDark } from './checkDarkMode' import styles from './SimulatorMask.module.css' import cursorStyles from './cursor.module.css' -export class SimulatorMask { +export class SimulatorMask extends EventTarget { shown: boolean = false wrapper = document.createElement('div') motion: Motion | null = null @@ -19,6 +19,8 @@ export class SimulatorMask { #targetCursorY = 0 constructor() { + super() + this.wrapper.id = 'page-agent-runtime_simulator-mask' this.wrapper.className = styles.wrapper this.wrapper.setAttribute('data-browser-use-ignore', 'true') @@ -74,13 +76,34 @@ export class SimulatorMask { this.#moveCursorToTarget() - window.addEventListener('PageAgent::MovePointerTo', (event: Event) => { + // global events + // @note Mask should be isolated from the rest of the code. + // Global events are easier to manage and cleanup. + + const movePointerToListener = (event: Event) => { const { x, y } = (event as CustomEvent).detail this.setCursorPosition(x, y) - }) - - window.addEventListener('PageAgent::ClickPointer', (event: Event) => { + } + const clickPointerListener = () => { this.triggerClickAnimation() + } + const enablePassThroughListener = () => { + this.wrapper.style.pointerEvents = 'none' + } + const disablePassThroughListener = () => { + this.wrapper.style.pointerEvents = 'auto' + } + + window.addEventListener('PageAgent::MovePointerTo', movePointerToListener) + window.addEventListener('PageAgent::ClickPointer', clickPointerListener) + window.addEventListener('PageAgent::EnablePassThrough', enablePassThroughListener) + window.addEventListener('PageAgent::DisablePassThrough', disablePassThroughListener) + + this.addEventListener('dispose', () => { + window.removeEventListener('PageAgent::MovePointerTo', movePointerToListener) + window.removeEventListener('PageAgent::ClickPointer', clickPointerListener) + window.removeEventListener('PageAgent::EnablePassThrough', enablePassThroughListener) + window.removeEventListener('PageAgent::DisablePassThrough', disablePassThroughListener) }) } @@ -177,7 +200,9 @@ export class SimulatorMask { } dispose() { + console.log('dispose SimulatorMask') this.motion?.dispose() this.wrapper.remove() + this.dispatchEvent(new Event('dispose')) } } diff --git a/packages/page-controller/src/utils/index.ts b/packages/page-controller/src/utils/index.ts index 7f651a2..885357d 100644 --- a/packages/page-controller/src/utils/index.ts +++ b/packages/page-controller/src/utils/index.ts @@ -48,15 +48,33 @@ export async function waitFor(seconds: number): Promise { await new Promise((resolve) => setTimeout(resolve, seconds * 1000)) } -// ======= dom utils ======= +// ======= mask events ======= -export async function movePointerToElement(element: HTMLElement) { - const rect = element.getBoundingClientRect() +/** + * Move the visual pointer to a position within an element. + * @param x - x coordinate in the element's document viewport + * @param y - y coordinate in the element's document viewport + */ +export async function movePointerToElement(element: HTMLElement, x: number, y: number) { const offset = getIframeOffset(element) - const x = rect.left + rect.width / 2 + offset.x - const y = rect.top + rect.height / 2 + offset.y - window.dispatchEvent(new CustomEvent('PageAgent::MovePointerTo', { detail: { x, y } })) + window.dispatchEvent( + new CustomEvent('PageAgent::MovePointerTo', { + detail: { x: x + offset.x, y: y + offset.y }, + }) + ) await waitFor(0.3) } + +export async function clickPointer() { + window.dispatchEvent(new CustomEvent('PageAgent::ClickPointer')) +} + +export async function enablePassThrough() { + window.dispatchEvent(new CustomEvent('PageAgent::EnablePassThrough')) +} + +export async function disablePassThrough() { + window.dispatchEvent(new CustomEvent('PageAgent::DisablePassThrough')) +}