From 06c6124b58085d0e14db67ff5d97b47c4a2c1c2d Mon Sep 17 00:00:00 2001 From: Innocent Nortey Date: Sun, 31 May 2026 00:25:40 -0500 Subject: [PATCH] input: allow drag drop-point biasing within target bounding box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the `drag` tool with optional `to_offset_x` / `to_offset_y` / `to_fraction_x` / `to_fraction_y` so callers can bias the drop point inside the target element's bounding rectangle instead of always dropping at the geometric center. When none of the new parameters are provided, the tool keeps the existing centroid-to-centroid behaviour (`fromHandle.drag(toHandle)` + 50 ms pause + `toHandle.drop(fromHandle)`) so every current caller is unaffected. When any of the new parameters are provided, the tool computes a custom point inside the target's bounding box and replicates the existing HTML5-drag emulation path (hover source → mouse.down → mouse.move to custom point → 150 ms pause → mouse.up). This avoids needing `setDragInterception(true)`, which would conflict with the default code path used by every other call site. Motivation: some drag-and-drop UIs expose a single visible drop zone whose geometric centre is not the safest or correct drop hotspot (e.g. Zoho Analytics Pivot View construction, where dropping a measure onto the visible Data shelf container can land in a neighbouring Rows shelf because the shelf centre overlaps it). In those layouts, dropping onto a child element deeper inside the intended shelf works reliably, which points at target-point geometry rather than source selection or timing as the failure mode. Adds a regression test that verifies the offset path lands in the expected sub-region of the target. Regenerated tool reference docs, CLI options, and tool-call metrics via `npm run gen`. --- docs/tool-reference.md | 4 + src/bin/chrome-devtools-cli-options.ts | 28 +++++ src/telemetry/tool_call_metrics.json | 16 +++ src/tools/input.ts | 138 ++++++++++++++++++++++++- tests/tools/input.test.ts | 60 +++++++++++ 5 files changed, 243 insertions(+), 3 deletions(-) diff --git a/docs/tool-reference.md b/docs/tool-reference.md index 747193452..f1a4f7a97 100644 --- a/docs/tool-reference.md +++ b/docs/tool-reference.md @@ -81,6 +81,10 @@ - **from_uid** (string) **(required)**: The uid of the element to [`drag`](#drag) - **to_uid** (string) **(required)**: The uid of the element to drop into - **includeSnapshot** (boolean) _(optional)_: Whether to include a snapshot in the response. Default is false. +- **to_fraction_x** (number) _(optional)_: Optional x fraction within the target element bounding rect. 0 is the left edge and 1 is the right edge. Cannot be combined with to_offset_x. +- **to_fraction_y** (number) _(optional)_: Optional y fraction within the target element bounding rect. 0 is the top edge and 1 is the bottom edge. Cannot be combined with to_offset_y. +- **to_offset_x** (number) _(optional)_: Optional x offset in CSS pixels from the target element bounding rect top-left corner. Cannot be combined with to_fraction_x. +- **to_offset_y** (number) _(optional)_: Optional y offset in CSS pixels from the target element bounding rect top-left corner. Cannot be combined with to_fraction_y. --- diff --git a/src/bin/chrome-devtools-cli-options.ts b/src/bin/chrome-devtools-cli-options.ts index af92dccd5..f401fdad7 100644 --- a/src/bin/chrome-devtools-cli-options.ts +++ b/src/bin/chrome-devtools-cli-options.ts @@ -111,6 +111,34 @@ export const commands: Commands = { description: 'The uid of the element to drop into', required: true, }, + to_offset_x: { + name: 'to_offset_x', + type: 'number', + description: + 'Optional x offset in CSS pixels from the target element bounding rect top-left corner. Cannot be combined with to_fraction_x.', + required: false, + }, + to_offset_y: { + name: 'to_offset_y', + type: 'number', + description: + 'Optional y offset in CSS pixels from the target element bounding rect top-left corner. Cannot be combined with to_fraction_y.', + required: false, + }, + to_fraction_x: { + name: 'to_fraction_x', + type: 'number', + description: + 'Optional x fraction within the target element bounding rect. 0 is the left edge and 1 is the right edge. Cannot be combined with to_offset_x.', + required: false, + }, + to_fraction_y: { + name: 'to_fraction_y', + type: 'number', + description: + 'Optional y fraction within the target element bounding rect. 0 is the top edge and 1 is the bottom edge. Cannot be combined with to_offset_y.', + required: false, + }, includeSnapshot: { name: 'includeSnapshot', type: 'boolean', diff --git a/src/telemetry/tool_call_metrics.json b/src/telemetry/tool_call_metrics.json index 597bfa7a3..785f0d5c6 100644 --- a/src/telemetry/tool_call_metrics.json +++ b/src/telemetry/tool_call_metrics.json @@ -56,6 +56,22 @@ { "name": "include_snapshot", "argType": "boolean" + }, + { + "name": "to_offset_x", + "argType": "number" + }, + { + "name": "to_offset_y", + "argType": "number" + }, + { + "name": "to_fraction_x", + "argType": "number" + }, + { + "name": "to_fraction_y", + "argType": "number" } ] }, diff --git a/src/tools/input.ts b/src/tools/input.ts index ee4973437..2010d068b 100644 --- a/src/tools/input.ts +++ b/src/tools/input.ts @@ -33,6 +33,108 @@ const submitKeySchema = zod 'Optional key to press after typing. E.g., "Enter", "Tab", "Escape"', ); +const dragOffsetXSchema = zod + .number() + .optional() + .describe( + 'Optional x offset in CSS pixels from the target element bounding rect top-left corner. Cannot be combined with to_fraction_x.', + ); + +const dragOffsetYSchema = zod + .number() + .optional() + .describe( + 'Optional y offset in CSS pixels from the target element bounding rect top-left corner. Cannot be combined with to_fraction_y.', + ); + +const dragFractionXSchema = zod + .number() + .min(0) + .max(1) + .optional() + .describe( + 'Optional x fraction within the target element bounding rect. 0 is the left edge and 1 is the right edge. Cannot be combined with to_offset_x.', + ); + +const dragFractionYSchema = zod + .number() + .min(0) + .max(1) + .optional() + .describe( + 'Optional y fraction within the target element bounding rect. 0 is the top edge and 1 is the bottom edge. Cannot be combined with to_offset_y.', + ); + +const CUSTOM_DRAG_DELAY_MS = 150; + +interface DragCustomDropParams { + to_offset_x?: number; + to_offset_y?: number; + to_fraction_x?: number; + to_fraction_y?: number; +} + +function hasCustomDropPoint(params: DragCustomDropParams): boolean { + return ( + params.to_offset_x !== undefined || + params.to_offset_y !== undefined || + params.to_fraction_x !== undefined || + params.to_fraction_y !== undefined + ); +} + +function assertValidCustomDropPoint(params: DragCustomDropParams): void { + if (params.to_offset_x !== undefined && params.to_fraction_x !== undefined) { + throw new Error( + 'Specify only one of to_offset_x or to_fraction_x for drag().', + ); + } + if (params.to_offset_y !== undefined && params.to_fraction_y !== undefined) { + throw new Error( + 'Specify only one of to_offset_y or to_fraction_y for drag().', + ); + } +} + +function resolveDropAxisCoordinate( + origin: number, + size: number, + offset?: number, + fraction?: number, +): number { + if (offset !== undefined) { + return origin + offset; + } + if (fraction !== undefined) { + return origin + size * fraction; + } + return origin + size / 2; +} + +async function resolveCustomDropPoint( + handle: ElementHandle, + params: DragCustomDropParams, +): Promise<{x: number; y: number}> { + const box = await handle.boundingBox(); + if (!box) { + throw new Error('Failed to compute the drag drop target bounding box.'); + } + return { + x: resolveDropAxisCoordinate( + box.x, + box.width, + params.to_offset_x, + params.to_fraction_x, + ), + y: resolveDropAxisCoordinate( + box.y, + box.height, + params.to_offset_y, + params.to_fraction_y, + ), + }; +} + function handleActionError(error: unknown, uid: string) { logger('failed to act using a locator', error); throw new Error( @@ -378,6 +480,10 @@ export const drag = definePageTool({ schema: { from_uid: zod.string().describe('The uid of the element to drag'), to_uid: zod.string().describe('The uid of the element to drop into'), + to_offset_x: dragOffsetXSchema, + to_offset_y: dragOffsetYSchema, + to_fraction_x: dragFractionXSchema, + to_fraction_y: dragFractionYSchema, includeSnapshot: includeSnapshotSchema, }, blockedByDialog: true, @@ -387,11 +493,37 @@ export const drag = definePageTool({ request.params.from_uid, ); const toHandle = await request.page.getElementByUid(request.params.to_uid); + const customDropParams: DragCustomDropParams = { + to_offset_x: request.params.to_offset_x, + to_offset_y: request.params.to_offset_y, + to_fraction_x: request.params.to_fraction_x, + to_fraction_y: request.params.to_fraction_y, + }; try { const result = await request.page.waitForEventsAfterAction(async () => { - await fromHandle.drag(toHandle); - await new Promise(resolve => setTimeout(resolve, 50)); - await toHandle.drop(fromHandle); + if (!hasCustomDropPoint(customDropParams)) { + await fromHandle.drag(toHandle); + await new Promise(resolve => setTimeout(resolve, 50)); + await toHandle.drop(fromHandle); + return; + } + + assertValidCustomDropPoint(customDropParams); + + await fromHandle.scrollIntoView(); + await toHandle.scrollIntoView(); + + const targetPoint = await resolveCustomDropPoint( + toHandle, + customDropParams, + ); + const mouse = request.page.pptrPage.mouse; + + await fromHandle.hover(); + await mouse.down(); + await mouse.move(targetPoint.x, targetPoint.y); + await new Promise(resolve => setTimeout(resolve, CUSTOM_DRAG_DELAY_MS)); + await mouse.up(); }); response.appendResponseLine(`Successfully dragged an element`); response.attachWaitForResult(result); diff --git a/tests/tools/input.test.ts b/tests/tools/input.test.ts index 29c81e513..bbc12db5f 100644 --- a/tests/tools/input.test.ts +++ b/tests/tools/input.test.ts @@ -1053,6 +1053,66 @@ describe('input', () => { assert.ok(await page.$('text/dropped')); }); }); + + it('drags onto a biased point inside the target bounding box', async () => { + await withMcpContext(async (response, context) => { + const page = context.getSelectedPptrPage(); + await page.setContent( + html`
drag me
+
+ `, + ); + context.getSelectedMcpPage().textSnapshot = await TextSnapshot.create( + context.getSelectedMcpPage(), + ); + await drag.handler( + { + params: { + from_uid: '1_1', + to_uid: '1_2', + to_offset_y: 10, + }, + page: context.getSelectedMcpPage(), + }, + response, + context, + ); + assert.strictEqual( + response.responseLines[0], + 'Successfully dragged an element', + ); + assert.strictEqual( + await page.$eval('#drop', el => el.getAttribute('data-region')), + 'top', + ); + }); + }); }); describe('fill form', () => {