zoushiyang/page-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: init

Browse Source
This commit is contained in:
Simon
2025-09-29 16:33:15 +08:00
parent e8041e0582
commit 847620b5e8
98 changed files with 20166 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

430
src/tools/actions.ts Normal file
View File

@@ -0,0 +1,430 @@
/**
* Copyright (C) 2025 Alibaba Group Holding Limited
* All rights reserved.
*/
import type { PageAgent } from '../PageAgent'
// ======= general utils =======
export async function waitFor(seconds: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, seconds * 1000))
}
let currentUrl = window.location.href
export async function getSystemInfo() {
// If current URL is already up to date, no need to add message
if (currentUrl === window.location.href) return ''
await waitFor(0.3) // Wait a bit longer for page to load
currentUrl = window.location.href
return `\n<sys> Current URL changed to: ${currentUrl} </sys>`
}
// ======= dom utils =======
export async function movePointerToElement(element: HTMLElement) {
const rect = element.getBoundingClientRect()
const x = rect.left + rect.width / 2
const y = rect.top + rect.height / 2
window.dispatchEvent(new CustomEvent('PageAgent::MovePointerTo', { detail: { x, y } }))
await waitFor(0.3)
}
/**
* Get the HTMLElement by index from the selectorMap in PageAgent.
*/
export function getElementByIndex(pageAgent: PageAgent, index: number): HTMLElement {
const interactiveNode = pageAgent.selectorMap.get(index)
if (!interactiveNode) {
throw new Error(`No interactive element found at index ${index}`)
}
const element = interactiveNode.ref
if (!element) {
throw new Error(`Element at index ${index} does not have a reference`)
}
if (!(element instanceof HTMLElement)) {
throw new Error(`Element at index ${index} is not an HTMLElement`)
}
return element
}
let lastClickedElement: HTMLElement | null = null
function blurLastClickedElement() {
if (lastClickedElement) {
lastClickedElement.blur()
lastClickedElement.dispatchEvent(
new MouseEvent('mouseout', { bubbles: true, cancelable: true })
)
lastClickedElement = null
}
}
/**
* Simulate a click on the element
*/
export async function clickElement(element: HTMLElement) {
blurLastClickedElement()
lastClickedElement = element
await scrollIntoViewIfNeeded(element)
await movePointerToElement(element)
window.dispatchEvent(new CustomEvent('PageAgent::ClickPointer'))
await waitFor(0.1)
// hover it
element.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true, cancelable: true }))
element.dispatchEvent(new MouseEvent('mouseover', { bubbles: true, cancelable: true }))
// dispatch a sequence of events to ensure all listeners are triggered
element.dispatchEvent(new MouseEvent('mousedown', { bubbles: true, cancelable: true }))
// focus it to ensure it gets the click event
element.focus()
element.dispatchEvent(new MouseEvent('mouseup', { bubbles: true, cancelable: true }))
element.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }))
// dispatch a click event
// element.click()
await waitFor(0.1) // Wait to ensure click event processing completes
}
// eslint-disable-next-line @typescript-eslint/unbound-method
const nativeInputValueSetter = Object.getOwnPropertyDescriptor(
window.HTMLInputElement.prototype,
'value'
)!.set!
// eslint-disable-next-line @typescript-eslint/unbound-method
const nativeTextAreaValueSetter = Object.getOwnPropertyDescriptor(
window.HTMLTextAreaElement.prototype,
'value'
)!.set!
/**
* create a synthetic keyboard event
* with key keycode code
*/
export async function createSyntheticInputEvent(elem: HTMLElement, key: string) {
elem.dispatchEvent(new KeyboardEvent('keydown', { bubbles: true, cancelable: true, key }))
await waitFor(0.01)
if (elem instanceof HTMLInputElement || elem instanceof HTMLTextAreaElement) {
elem.dispatchEvent(new Event('beforeinput', { bubbles: true }))
await waitFor(0.01)
elem.dispatchEvent(new Event('input', { bubbles: true }))
await waitFor(0.01)
}
elem.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, cancelable: true, key }))
}
export async function inputTextElement(element: HTMLElement, text: string) {
if (!(element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement)) {
throw new Error('Element is not an input or textarea')
}
await clickElement(element)
if (element instanceof HTMLTextAreaElement) {
nativeTextAreaValueSetter.call(element, text)
} else {
nativeInputValueSetter.call(element, text)
}
const inputEvent = new Event('input', { bubbles: true })
element.dispatchEvent(inputEvent)
await waitFor(0.1) // Wait to ensure input event processing completes
blurLastClickedElement()
}
/**
* @todo browser-use version is very complex and supports menu tags, need to follow up
*/
export async function selectOptionElement(selectElement: HTMLSelectElement, optionText: string) {
if (!(selectElement instanceof HTMLSelectElement)) {
throw new Error('Element is not a select element')
}
const options = Array.from(selectElement.options)
const option = options.find((opt) => opt.textContent?.trim() === optionText.trim())
if (!option) {
throw new Error(`Option with text "${optionText}" not found in select element`)
}
selectElement.value = option.value
selectElement.dispatchEvent(new Event('change', { bubbles: true }))
await waitFor(0.1) // Wait to ensure change event processing completes
}
// eslint-disable-next-line @typescript-eslint/require-await
export async function scrollIntoViewIfNeeded(element: HTMLElement) {
const el = element as any
if (el.scrollIntoViewIfNeeded) {
el.scrollIntoViewIfNeeded()
// await waitFor(0.5) // Animation playback
} else {
// @todo visibility check
el.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'nearest' })
// await waitFor(0.5) // Animation playback
}
}
export async function scrollVertically(
down: boolean,
scroll_amount: number,
element?: HTMLElement | null
) {
// Element-specific scrolling if element is provided
if (element) {
const targetElement = element
console.log(
'[SCROLL DEBUG] Starting direct container scroll for element:',
targetElement.tagName
)
let currentElement = targetElement as HTMLElement | null
let scrollSuccess = false
let scrolledElement: HTMLElement | null = null
let scrollDelta = 0
let attempts = 0
const dy = scroll_amount
while (currentElement && attempts < 10) {
const computedStyle = window.getComputedStyle(currentElement)
const hasScrollableY = /(auto|scroll|overlay)/.test(computedStyle.overflowY)
const canScrollVertically = currentElement.scrollHeight > currentElement.clientHeight
console.log(
'[SCROLL DEBUG] Checking element:',
currentElement.tagName,
'hasScrollableY:',
hasScrollableY,
'canScrollVertically:',
canScrollVertically,
'scrollHeight:',
currentElement.scrollHeight,
'clientHeight:',
currentElement.clientHeight
)
if (hasScrollableY && canScrollVertically) {
const beforeScroll = currentElement.scrollTop
const maxScroll = currentElement.scrollHeight - currentElement.clientHeight
let scrollAmount = dy / 3
if (scrollAmount > 0) {
scrollAmount = Math.min(scrollAmount, maxScroll - beforeScroll)
} else {
scrollAmount = Math.max(scrollAmount, -beforeScroll)
}
currentElement.scrollTop = beforeScroll + scrollAmount
const afterScroll = currentElement.scrollTop
const actualScrollDelta = afterScroll - beforeScroll
console.log(
'[SCROLL DEBUG] Scroll attempt:',
currentElement.tagName,
'before:',
beforeScroll,
'after:',
afterScroll,
'delta:',
actualScrollDelta
)
if (Math.abs(actualScrollDelta) > 0.5) {
scrollSuccess = true
scrolledElement = currentElement
scrollDelta = actualScrollDelta
console.log(
'[SCROLL DEBUG] Successfully scrolled container:',
currentElement.tagName,
'delta:',
actualScrollDelta
)
break
}
}
if (currentElement === document.body || currentElement === document.documentElement) {
break
}
currentElement = currentElement.parentElement
attempts++
}
if (scrollSuccess) {
return `Scrolled container (${scrolledElement?.tagName}) by ${scrollDelta}px`
} else {
return `No scrollable container found for element (${targetElement.tagName})`
}
}
// Page-level scrolling (default or fallback)
const dy = scroll_amount
const bigEnough = (el: HTMLElement) => el.clientHeight >= window.innerHeight * 0.5
const canScroll = (el: HTMLElement | null) =>
el &&
/(auto|scroll|overlay)/.test(getComputedStyle(el).overflowY) &&
el.scrollHeight > el.clientHeight &&
bigEnough(el)
let el: HTMLElement | null = document.activeElement as HTMLElement | null
while (el && !canScroll(el) && el !== document.body) el = el.parentElement
el = canScroll(el)
? el
: Array.from(document.querySelectorAll<HTMLElement>('*')).find(canScroll) ||
(document.scrollingElement as HTMLElement) ||
(document.documentElement as HTMLElement)
if (el === document.scrollingElement || el === document.documentElement || el === document.body) {
window.scrollBy(0, dy)
return `✅ Scrolled page by ${dy}px.`
} else {
el!.scrollBy({ top: dy, behavior: 'smooth' })
await waitFor(0.1) // Animation playback
return `✅ Scrolled container (${el!.tagName}) by ${dy}px.`
}
}
export async function scrollHorizontally(
right: boolean,
scroll_amount: number,
element?: HTMLElement | null
) {
// Element-specific scrolling if element is provided
if (element) {
const targetElement = element
console.log(
'[SCROLL DEBUG] Starting direct container scroll for element:',
targetElement.tagName
)
let currentElement = targetElement as HTMLElement | null
let scrollSuccess = false
let scrolledElement: HTMLElement | null = null
let scrollDelta = 0
let attempts = 0
const dx = right ? scroll_amount : -scroll_amount
while (currentElement && attempts < 10) {
const computedStyle = window.getComputedStyle(currentElement)
const hasScrollableX = /(auto|scroll|overlay)/.test(computedStyle.overflowX)
const canScrollHorizontally = currentElement.scrollWidth > currentElement.clientWidth
console.log(
'[SCROLL DEBUG] Checking element:',
currentElement.tagName,
'hasScrollableX:',
hasScrollableX,
'canScrollHorizontally:',
canScrollHorizontally,
'scrollWidth:',
currentElement.scrollWidth,
'clientWidth:',
currentElement.clientWidth
)
if (hasScrollableX && canScrollHorizontally) {
const beforeScroll = currentElement.scrollLeft
const maxScroll = currentElement.scrollWidth - currentElement.clientWidth
let scrollAmount = dx / 3
if (scrollAmount > 0) {
scrollAmount = Math.min(scrollAmount, maxScroll - beforeScroll)
} else {
scrollAmount = Math.max(scrollAmount, -beforeScroll)
}
currentElement.scrollLeft = beforeScroll + scrollAmount
const afterScroll = currentElement.scrollLeft
const actualScrollDelta = afterScroll - beforeScroll
console.log(
'[SCROLL DEBUG] Scroll attempt:',
currentElement.tagName,
'before:',
beforeScroll,
'after:',
afterScroll,
'delta:',
actualScrollDelta
)
if (Math.abs(actualScrollDelta) > 0.5) {
scrollSuccess = true
scrolledElement = currentElement
scrollDelta = actualScrollDelta
console.log(
'[SCROLL DEBUG] Successfully scrolled container:',
currentElement.tagName,
'delta:',
actualScrollDelta
)
break
}
}
if (currentElement === document.body || currentElement === document.documentElement) {
break
}
currentElement = currentElement.parentElement
attempts++
}
if (scrollSuccess) {
return `Scrolled container (${scrolledElement?.tagName}) horizontally by ${scrollDelta}px`
} else {
return `No horizontally scrollable container found for element (${targetElement.tagName})`
}
}
// Page-level scrolling (default or fallback)
const dx = right ? scroll_amount : -scroll_amount
const bigEnough = (el: HTMLElement) => el.clientWidth >= window.innerWidth * 0.5
const canScroll = (el: HTMLElement | null) =>
el &&
/(auto|scroll|overlay)/.test(getComputedStyle(el).overflowX) &&
el.scrollWidth > el.clientWidth &&
bigEnough(el)
let el: HTMLElement | null = document.activeElement as HTMLElement | null
while (el && !canScroll(el) && el !== document.body) el = el.parentElement
el = canScroll(el)
? el
: Array.from(document.querySelectorAll<HTMLElement>('*')).find(canScroll) ||
(document.scrollingElement as HTMLElement) ||
(document.documentElement as HTMLElement)
if (el === document.scrollingElement || el === document.documentElement || el === document.body) {
window.scrollBy(dx, 0)
return `✅ Scrolled page horizontally by ${dx}px`
} else {
el!.scrollBy({ left: dx, behavior: 'smooth' })
await waitFor(0.1) // Animation playback
return `✅ Scrolled container (${el!.tagName}) horizontally by ${dx}px`
}
}

208
src/tools/index.ts Normal file
View File

@@ -0,0 +1,208 @@
/**
* Internal tools for PageAgent.
* @note Adapted from browser-use
*/
import { Tool, tool } from 'ai'
import zod from 'zod'
import type { PageAgent } from '@/PageAgent'
import {
clickElement,
getElementByIndex,
getSystemInfo,
inputTextElement,
scrollHorizontally,
scrollVertically,
selectOptionElement,
waitFor,
} from './actions'
// debug
import * as utils from './actions'
// @ts-expect-error debug only
window.utils = utils
/**
* Internal tools for PageAgent.
*/
export const tools = new Map<string, Tool>()
// tools.set(
// 'get_current_html',
// tool({
// description: 'Get the current (updated) simplified HTML of the page',
// inputSchema: zod.object({}),
// execute: function (this: PageAgent) {
// this.updateTree()
// return this.simplifiedHTML
// },
// })
// )
tools.set(
'done',
tool({
description:
'Complete task - provide a summary of results for the user. Set success=True if task completed successfully, false otherwise. Text should be your response to the user summarizing results.',
inputSchema: zod.object({
text: zod.string(),
success: zod.boolean().default(true),
}),
execute: function (this: PageAgent, input) {
// @note main loop will handle this one
// this.onDone(input.text, input.success)
},
})
)
tools.set(
'wait',
tool({
description:
'Wait for x seconds. default 1s (max 10 seconds, min 1 second). This can be used to wait until the page or data is fully loaded.',
inputSchema: zod.object({
seconds: zod.number().min(1).max(10).default(1),
}),
execute: async function (this: PageAgent, input) {
const lastTimeUpdate = this.lastTimeUpdate
const actualWaitTime = Math.max(0, input.seconds - (Date.now() - lastTimeUpdate) / 1000)
console.log(`actualWaitTime: ${actualWaitTime} seconds`)
await waitFor(actualWaitTime)
return `✅ Waited for ${input.seconds} seconds.` + (await getSystemInfo())
},
})
)
tools.set(
'ask_user',
tool({
description:
'Ask the user a question and wait for their answer. Use this if you need more information or clarification.',
inputSchema: zod.object({
question: zod.string(),
}),
execute: async function (this: PageAgent, input) {
const answer = await this.panel.askUser(input.question)
return `✅ Received user answer: ${answer}` + (await getSystemInfo())
},
})
)
tools.set(
'click_element_by_index',
tool({
description: 'Click element by index',
inputSchema: zod.object({
index: zod.int().min(0),
}),
execute: async function (this: PageAgent, input) {
const element = getElementByIndex(this, input.index)
const elemText = this.elementTextMap.get(input.index)
await clickElement(element)
// @workaround: Handle links that open in new tabs
if (element instanceof HTMLAnchorElement && element.target === '_blank') {
return `⚠️ Clicked link that opens in a new tab (${elemText ? elemText : input.index}). You are not capable of reading new tabs.`
}
return `✅ Clicked element (${elemText ? elemText : input.index}).` + (await getSystemInfo())
},
})
)
tools.set(
'input_text',
tool({
description: 'Click and input text into a input interactive element',
inputSchema: zod.object({
index: zod.int().min(0),
text: zod.string(),
}),
execute: async function (this: PageAgent, input) {
const element = getElementByIndex(this, input.index)
const elemText = this.elementTextMap.get(input.index)
await inputTextElement(element, input.text)
return (
`✅ Input text (${input.text}) into element (${elemText ? elemText : input.index}).` +
(await getSystemInfo())
)
},
})
)
tools.set(
'select_dropdown_option',
tool({
description:
'Select dropdown option for interactive element index by the text of the option you want to select',
inputSchema: zod.object({
index: zod.int().min(0),
text: zod.string(),
}),
execute: async function (this: PageAgent, input) {
const element = getElementByIndex(this, input.index)
const elemText = this.elementTextMap.get(input.index)
await selectOptionElement(element as any, input.text)
return (
`✅ Selected option (${input.text}) in element (${elemText ? elemText : input.index}).` +
(await getSystemInfo())
)
},
})
)
/**
* @note Reference from browser-use
*/
tools.set(
'scroll',
tool({
description:
'Scroll the page by specified number of pages (set down=True to scroll down, down=False to scroll up, num_pages=number of pages to scroll like 0.5 for half page, 1.0 for one page, etc.). Optional index parameter to scroll within a specific element or its scroll container (works well for dropdowns and custom UI components). Optional pixels parameter to scroll by a specific number of pixels instead of pages.',
inputSchema: zod.object({
down: zod.boolean().default(true),
num_pages: zod.number().min(0).max(10).optional().default(0.1),
pixels: zod.number().int().min(0).optional(),
index: zod.number().int().min(0).optional(),
}),
execute: async function (this: PageAgent, input) {
const { down, num_pages, index, pixels } = input
const scroll_amount = pixels ? pixels : num_pages * (down ? 1 : -1) * window.innerHeight
const element = index !== undefined ? getElementByIndex(this, index) : null
return (await scrollVertically(down, scroll_amount, element)) + (await getSystemInfo())
},
})
)
tools.set(
'scroll_horizontally',
tool({
description:
'Scroll the page or element horizontally (set right=True to scroll right, right=False to scroll left, pixels=number of pixels to scroll). Optional index parameter to scroll within a specific element or its scroll container (works well for wide tables).',
inputSchema: zod.object({
right: zod.boolean().default(true),
pixels: zod.number().int().min(0),
index: zod.number().int().min(0).optional(),
}),
execute: async function (this: PageAgent, input) {
const { right, pixels, index } = input
const scroll_amount = pixels * (right ? 1 : -1)
const element = index !== undefined ? getElementByIndex(this, index) : null
return (await scrollHorizontally(right, scroll_amount, element)) + (await getSystemInfo())
},
})
)
// @todo get_dropdown_options
// @todo select_dropdown_option
// @todo send_keys
// @todo upload_file
// @todo go_back
// @todo extract_structured_data