44 * SPDX-License-Identifier: Apache-2.0
55 */
66
7+ import { logger } from './logger.js' ;
78import type {
89 Dialog ,
910 ElementHandle ,
@@ -13,7 +14,7 @@ import type {
1314} from './third_party/index.js' ;
1415import type { ToolGroup , ToolDefinition } from './tools/inPage.js' ;
1516import { takeSnapshot } from './tools/snapshot.js' ;
16- import type { ContextPage } from './tools/ToolDefinition.js' ;
17+ import type { ContextPage , Context , Response } from './tools/ToolDefinition.js' ;
1718import type {
1819 EmulationSettings ,
1920 GeolocationOptions ,
@@ -40,7 +41,7 @@ export class McpPage implements ContextPage {
4041 // Snapshot
4142 textSnapshot : TextSnapshot | null = null ;
4243 uniqueBackendNodeIdToMcpId = new Map < string , string > ( ) ;
43- extraHandles ? : ElementHandle [ ] ;
44+ extraHandles : ElementHandle [ ] = [ ] ;
4445
4546 // Emulation
4647 emulationSettings : EmulationSettings = { } ;
@@ -131,6 +132,198 @@ export class McpPage implements ContextPage {
131132 this . pptrPage . off ( 'dialog' , this . #dialogHandler) ;
132133 }
133134
135+ async executeInPageTool (
136+ toolName : string ,
137+ params : Record < string , unknown > ,
138+ response : Response ,
139+ context : Context ,
140+ ) : Promise < void > {
141+ // Creates array of ElementHandles from the UIDs in the params.
142+ // We do not replace the uids with the ElementsHandles yet, because
143+ // the `evaluate` function only turns them into DOM elements if they
144+ // are passed as non-nested arguments.
145+ const handles : ElementHandle [ ] = [ ] ;
146+ for ( const value of Object . values ( params ) ) {
147+ if (
148+ value instanceof Object &&
149+ 'uid' in value &&
150+ typeof value . uid === 'string' &&
151+ Object . keys ( value ) . length === 1
152+ ) {
153+ handles . push ( await this . getElementByUid ( value . uid ) ) ;
154+ }
155+ }
156+
157+ const result = await this . pptrPage . evaluate (
158+ async ( name , args , ...elements ) => {
159+ // Replace the UIDs with DOM elements.
160+ for ( const [ key , value ] of Object . entries ( args ) ) {
161+ if (
162+ value instanceof Object &&
163+ 'uid' in value &&
164+ typeof value . uid === 'string' &&
165+ Object . keys ( value ) . length === 1
166+ ) {
167+ args [ key ] = elements . shift ( ) ;
168+ }
169+ }
170+
171+ if ( ! window . __dtmcp ?. executeTool ) {
172+ throw new Error ( 'No tools found on the page' ) ;
173+ }
174+ const toolResult = await window . __dtmcp . executeTool ( name , args ) ;
175+
176+ const stashDOMElement = ( el : Element ) => {
177+ if ( ! window . __dtmcp ) {
178+ window . __dtmcp = { } ;
179+ }
180+ if ( window . __dtmcp . stashedElements === undefined ) {
181+ window . __dtmcp . stashedElements = [ ] ;
182+ }
183+ window . __dtmcp . stashedElements . push ( el ) ;
184+ return {
185+ stashedId : `stashed-${ window . __dtmcp . stashedElements . length - 1 } ,
186+ } ;
187+ } ;
188+
189+ const ancestors : unknown [ ] = [ ] ;
190+ // Recursively walks the tool result:
191+ // - Replaces DOM elements with an ID and stashes the DOM element on the window object
192+ // - Replaces non-plain objects with a string representation of the object
193+ // - Replaces circular references with the string '<Circular reference>'
194+ // - Replaces functions with the string '<Function object>'
195+ const processToolResult = (
196+ data : unknown ,
197+ parentEl ?: unknown ,
198+ ) : unknown => {
199+ // 1. Handle DOM Elements
200+ if ( data instanceof Element ) {
201+ return stashDOMElement ( data ) ;
202+ }
203+
204+ // 2. Handle Arrays
205+ if ( Array . isArray ( data ) ) {
206+ return data . map ( ( item : unknown ) =>
207+ processToolResult ( item , parentEl ) ,
208+ ) ;
209+ }
210+
211+ // 3. Handle Objects
212+ if ( data !== null && typeof data === 'object' ) {
213+ while ( ancestors . length > 0 && ancestors . at ( - 1 ) !== parentEl ) {
214+ ancestors . pop ( ) ;
215+ }
216+ if ( ancestors . includes ( data ) ) {
217+ return '<Circular reference>' ;
218+ }
219+ ancestors . push ( data ) ;
220+
221+ // If not a plain object, return a string representation of the object
222+ if ( Object . getPrototypeOf ( data ) !== Object . prototype ) {
223+ return `<${ data . constructor . name } ;
224+ }
225+
226+ const processedObj : Record < string , unknown > = { } ;
227+ for ( const [ key , value ] of Object . entries ( data ) ) {
228+ processedObj [ key ] = processToolResult ( value , data ) ;
229+ }
230+ return processedObj ;
231+ }
232+
233+ // 4. Handle Functions
234+ if ( typeof data === 'function' ) {
235+ return '<Function object>' ;
236+ }
237+
238+ // 5. Return primitives (strings, numbers, booleans) as-is
239+ return data ;
240+ } ;
241+
242+ return {
243+ result : processToolResult ( toolResult ) ,
244+ stashed : window . __dtmcp ?. stashedElements ?. length ?? 0 ,
245+ } ;
246+ } ,
247+ toolName ,
248+ params ,
249+ ...handles ,
250+ ) ;
251+
252+ const elementHandles : ElementHandle [ ] = [ ] ;
253+ for ( let i = 0 ; i < ( result . stashed ?? 0 ) ; i ++ ) {
254+ const elementHandle = await this . pptrPage . evaluateHandle ( index => {
255+ const el = window . __dtmcp ?. stashedElements ?. [ index ] ;
256+ if ( ! el ) {
257+ throw new Error ( `Stashed element at index ${ index } ) ;
258+ }
259+ return el ;
260+ } , i ) ;
261+ elementHandles . push ( elementHandle ) ;
262+ }
263+ const resultWithStashedElements = result . result ;
264+
265+ let isPageSnapshotUpdated = false ;
266+
267+ const stashedToUid = async ( index : number ) => {
268+ const backendNodeId = await elementHandles [ index ] . backendNodeId ( ) ;
269+ if ( ! backendNodeId ) {
270+ logger ( `No backendNodeId for stashed DOM element with index ${ index } ) ;
271+ return { uid : `stashed-${ index } } ;
272+ }
273+ let cdpElementId = context . resolveCdpElementId ( this , backendNodeId ) ;
274+ if ( ! cdpElementId ) {
275+ await context . createTextSnapshot (
276+ this ,
277+ false ,
278+ undefined ,
279+ elementHandles ,
280+ ) ;
281+ isPageSnapshotUpdated = true ;
282+ cdpElementId = context . resolveCdpElementId ( this , backendNodeId ) ;
283+ }
284+ if ( ! cdpElementId ) {
285+ logger ( `Could not get cdpElementId for backend node ${ backendNodeId } ) ;
286+ return { uid : `stashed-${ index } } ;
287+ }
288+ return { uid : cdpElementId } ;
289+ } ;
290+
291+ const recursivelyReplaceStashedElements = async (
292+ node : unknown ,
293+ ) : Promise < unknown > => {
294+ if ( Array . isArray ( node ) ) {
295+ return await Promise . all (
296+ node . map ( async x => await recursivelyReplaceStashedElements ( x ) ) ,
297+ ) ;
298+ }
299+ if ( node !== null && typeof node === 'object' ) {
300+ if (
301+ 'stashedId' in node &&
302+ typeof node . stashedId === 'string' &&
303+ node . stashedId . startsWith ( 'stashed-' ) &&
304+ Object . keys ( node ) . length === 1
305+ ) {
306+ const index = parseInt ( node . stashedId . split ( '-' ) [ 1 ] ) ;
307+ return stashedToUid ( index ) ;
308+ }
309+ const resultObj : Record < string , unknown > = { } ;
310+ for ( const [ key , value ] of Object . entries ( node ) ) {
311+ resultObj [ key ] = await recursivelyReplaceStashedElements ( value ) ;
312+ }
313+ return resultObj ;
314+ }
315+ return node ;
316+ } ;
317+
318+ const resultWithUids = await recursivelyReplaceStashedElements (
319+ resultWithStashedElements ,
320+ ) ;
321+ response . appendResponseLine ( JSON . stringify ( resultWithUids , null , 2 ) ) ;
322+ if ( isPageSnapshotUpdated ) {
323+ response . includeSnapshot ( ) ;
324+ }
325+ }
326+
134327 async getElementByUid ( uid : string ) : Promise < ElementHandle < Element > > {
135328 if ( ! this . textSnapshot ) {
136329 throw new Error (
@@ -165,20 +358,4 @@ export class McpPage implements ContextPage {
165358 getAXNodeByUid ( uid : string ) {
166359 return this . textSnapshot ?. idToNode . get ( uid ) ;
167360 }
168-
169- getSnapshot ( ) : TextSnapshot | null {
170- return this . textSnapshot ;
171- }
172-
173- setSnapshot ( snapshot : TextSnapshot ) : void {
174- this . textSnapshot = snapshot ;
175- }
176-
177- getExtraHandles ( ) : ElementHandle [ ] | undefined {
178- return this . extraHandles ;
179- }
180-
181- setExtraHandles ( extraHandles : ElementHandle [ ] ) : void {
182- this . extraHandles = extraHandles ;
183- }
184361}
0 commit comments