From 315084a602c8a9a071b69e544484de2221b3a694 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Wed, 4 Mar 2026 12:29:27 +0100 Subject: [PATCH 1/4] Update webdriver remote actions descriptions --- .../alayer/actions/WdRemoteClickAction.java | 7 ++-- .../actions/WdRemoteScrollClickAction.java | 7 ++-- .../actions/WdRemoteScrollTypeAction.java | 7 ++-- .../alayer/actions/WdRemoteTypeAction.java | 18 +++++---- .../webdriver/TestWebdriverRemoteActions.java | 40 +++++++------------ 5 files changed, 34 insertions(+), 45 deletions(-) diff --git a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteClickAction.java b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteClickAction.java index ee911bc4e..964ec9230 100644 --- a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteClickAction.java +++ b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteClickAction.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -80,8 +80,7 @@ public void run(State state, Canvas canvas, Pen pen) { public WdRemoteClickAction(WdWidget widget) { this.widget = widget; this.mapOriginWidget(widget); - this.set(Tags.Desc, "Remote click " + widget.element.getElementDescription() + " : " - + widget.element.remoteWebElement.getId()); + this.set(Tags.Desc, "Remote click " + widget.get(Tags.Desc, widget.element.getElementDescription())); this.set(Tags.Role, WdActionRoles.RemoteClick); Role role = widget.get(Tags.Role, Roles.Widget); if (!role.equals(WdRoles.WdOPTION)) { diff --git a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollClickAction.java b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollClickAction.java index 47ac02a21..fe59c3457 100644 --- a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollClickAction.java +++ b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollClickAction.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,8 +45,7 @@ public class WdRemoteScrollClickAction extends WdRemoteClickAction { public WdRemoteScrollClickAction(WdWidget widget) { super(widget); - this.set(Tags.Desc, "Remote scroll and click " + widget.element.getElementDescription() + " : " - + widget.element.remoteWebElement.getId()); + this.set(Tags.Desc, "Remote scroll and click " + widget.get(Tags.Desc, widget.element.getElementDescription())); this.set(Tags.Role, WdActionRoles.RemoteScrollClick); } diff --git a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollTypeAction.java b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollTypeAction.java index b5eca29b0..dc0c94db1 100644 --- a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollTypeAction.java +++ b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteScrollTypeAction.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2023 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2023 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -43,8 +43,7 @@ public class WdRemoteScrollTypeAction extends WdRemoteTypeAction { public WdRemoteScrollTypeAction(WdWidget widget, CharSequence keys) { super(widget, keys); - this.set(Tags.Desc, "Remote scroll and type " + keys + " to widget " - + widget.element.getElementDescription() + " : " + widget.element.remoteWebElement.getId()); + this.set(Tags.Desc, "Remote scroll and type " + keys + " to widget " + widget.get(Tags.Desc, widget.element.getElementDescription())); this.set(Tags.Role, WdActionRoles.RemoteScrollType); } diff --git a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteTypeAction.java b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteTypeAction.java index eba4cf5fc..0ff7cf2bf 100644 --- a/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteTypeAction.java +++ b/webdriver/src/org/testar/monkey/alayer/actions/WdRemoteTypeAction.java @@ -1,6 +1,6 @@ /** - * Copyright (c) 2021 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2021 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2021 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2021 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -84,13 +84,17 @@ public void run(State state, Canvas cv, Pen pen) { public WdRemoteTypeAction(WdWidget widget, CharSequence keys) { this.widget = widget; - this.keys = keys; - this.mapOriginWidget(widget); - this.set(Tags.Desc, "Remote type " + keys + " to widget " - + widget.element.getElementDescription() + " : " + widget.element.remoteWebElement.getId()); this.set(Tags.Role, WdActionRoles.RemoteType); - this.set(Tags.Visualizer, new TypeVisualizer(widget.get(WdTags.WebBoundingRectangle), keys.toString(), TypePen)); + setKeys(keys); + } + + public void setKeys(CharSequence keys) { + String keyText = keys == null ? "" : keys.toString(); + this.keys = keyText; + this.set(Tags.InputText, keyText); + this.set(Tags.Desc, "Remote type " + keyText + " to widget " + widget.get(Tags.Desc, widget.element.getElementDescription())); + this.set(Tags.Visualizer, new TypeVisualizer(widget.get(WdTags.WebBoundingRectangle), keyText, TypePen)); } @Override diff --git a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverRemoteActions.java b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverRemoteActions.java index 5fe380c3a..e45d87f52 100644 --- a/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverRemoteActions.java +++ b/webdriver/test/org/testar/monkey/alayer/webdriver/TestWebdriverRemoteActions.java @@ -26,6 +26,8 @@ public class TestWebdriverRemoteActions { private WdElement childWdElement; private WdWidget childWdWidget; + private RemoteWebElement remoteWebElement; + // Before each JUnit test prepare the new objects @Before public void prepareWebdriverElements() { @@ -34,6 +36,10 @@ public void prepareWebdriverElements() { childWdElement = new WdElement(rootWdElement, rootWdElement); childWdWidget = new WdWidget(rootWdState, rootWdState, childWdElement); + + remoteWebElement = Mockito.mock(RemoteWebElement.class); + Mockito.when(remoteWebElement.getWrappedDriver()).thenReturn(Mockito.mock(RemoteWebDriver.class)); + childWdElement.remoteWebElement = remoteWebElement; } @Test @@ -42,18 +48,14 @@ public void test_create_WdRemoteClickAction() { childWdElement.name = "custom_name_value"; childWdElement.rect = Rect.from(0, 0, 100, 100); childWdWidget.set(Tags.Role, WdRoles.WdBUTTON); - - // Mock a remoteWebElement with getId - RemoteWebElement remoteWebElement = Mockito.mock(RemoteWebElement.class); - Mockito.when(remoteWebElement.getId()).thenReturn("remote_id_value"); - childWdElement.remoteWebElement = remoteWebElement; + childWdWidget.set(Tags.Desc, "Widget Description"); WdRemoteClickAction remoteClickAction = new WdRemoteClickAction(childWdWidget); // Verify the action and the Tags were created Assert.isTrue(remoteClickAction.get(Tags.OriginWidget).equals(childWdWidget)); Assert.isTrue(remoteClickAction.get(Tags.Role).equals(WdActionRoles.RemoteClick)); - Assert.isTrue(remoteClickAction.get(Tags.Desc).equals("Remote click " + childWdElement.name + " : " + "remote_id_value")); + Assert.isTrue(remoteClickAction.get(Tags.Desc).equals("Remote click " + "Widget Description")); Assert.isTrue(remoteClickAction.toShortString().equals("Remote click " + childWdElement.name)); Assert.isTrue(remoteClickAction.toParametersString().equals("Remote click " + childWdElement.name)); Assert.isTrue(remoteClickAction.toString(new Role[0]).equals("Remote click " + childWdElement.name)); @@ -70,19 +72,14 @@ public void test_create_WdRemoteScrollClickAction() { childWdElement.name = "custom_name_value"; childWdElement.rect = Rect.from(0, 0, 100, 100); childWdWidget.set(Tags.Role, WdRoles.WdBUTTON); - - // Mock a remoteWebElement with getId and getWrappedDriver - RemoteWebElement remoteWebElement = Mockito.mock(RemoteWebElement.class); - Mockito.when(remoteWebElement.getId()).thenReturn("remote_id_value"); - Mockito.when(remoteWebElement.getWrappedDriver()).thenReturn(Mockito.mock(RemoteWebDriver.class)); - childWdElement.remoteWebElement = remoteWebElement; + childWdWidget.set(Tags.Desc, "Widget Description"); WdRemoteScrollClickAction remoteScrollClickAction = new WdRemoteScrollClickAction(childWdWidget); // Verify the action and the Tags were created Assert.isTrue(remoteScrollClickAction.get(Tags.OriginWidget).equals(childWdWidget)); Assert.isTrue(remoteScrollClickAction.get(Tags.Role).equals(WdActionRoles.RemoteScrollClick)); - Assert.isTrue(remoteScrollClickAction.get(Tags.Desc).equals("Remote scroll and click " + childWdElement.name + " : " + "remote_id_value")); + Assert.isTrue(remoteScrollClickAction.get(Tags.Desc).equals("Remote scroll and click " + "Widget Description")); Assert.isTrue(remoteScrollClickAction.toShortString().equals("Remote scroll and click " + childWdElement.name)); Assert.isTrue(remoteScrollClickAction.toParametersString().equals("Remote scroll and click " + childWdElement.name)); Assert.isTrue(remoteScrollClickAction.toString(new Role[0]).equals("Remote scroll and click " + childWdElement.name)); @@ -99,11 +96,7 @@ public void test_create_WdRemoteTypeAction() { childWdElement.name = "custom_name_value"; childWdElement.rect = Rect.from(0, 0, 100, 100); childWdWidget.set(Tags.Role, WdRoles.WdBUTTON); - - // Mock a remoteWebElement with getId - RemoteWebElement remoteWebElement = Mockito.mock(RemoteWebElement.class); - Mockito.when(remoteWebElement.getId()).thenReturn("remote_id_value"); - childWdElement.remoteWebElement = remoteWebElement; + childWdWidget.set(Tags.Desc, "Widget Description"); String typedText = "input_text"; WdRemoteTypeAction remoteTypeAction = new WdRemoteTypeAction(childWdWidget, typedText); @@ -111,7 +104,7 @@ public void test_create_WdRemoteTypeAction() { // Verify the action and the Tags were created Assert.isTrue(remoteTypeAction.get(Tags.OriginWidget).equals(childWdWidget)); Assert.isTrue(remoteTypeAction.get(Tags.Role).equals(WdActionRoles.RemoteType)); - Assert.isTrue(remoteTypeAction.get(Tags.Desc).equals("Remote type " + typedText + " to widget " + childWdElement.name + " : " + "remote_id_value")); + Assert.isTrue(remoteTypeAction.get(Tags.Desc).equals("Remote type " + typedText + " to widget " + "Widget Description")); Assert.isTrue(remoteTypeAction.toShortString().equals("Remote type " + typedText + " " + childWdElement.name)); Assert.isTrue(remoteTypeAction.toParametersString().equals("Remote type " + typedText + " " + childWdElement.name)); Assert.isTrue(remoteTypeAction.toString(new Role[0]).equals("Remote type " + typedText + " " + childWdElement.name)); @@ -128,12 +121,7 @@ public void test_create_WdRemoteScrollTypeAction() { childWdElement.name = "custom_name_value"; childWdElement.rect = Rect.from(0, 0, 100, 100); childWdWidget.set(Tags.Role, WdRoles.WdBUTTON); - - // Mock a remoteWebElement with getId and getWrappedDriver - RemoteWebElement remoteWebElement = Mockito.mock(RemoteWebElement.class); - Mockito.when(remoteWebElement.getId()).thenReturn("remote_id_value"); - Mockito.when(remoteWebElement.getWrappedDriver()).thenReturn(Mockito.mock(RemoteWebDriver.class)); - childWdElement.remoteWebElement = remoteWebElement; + childWdWidget.set(Tags.Desc, "Widget Description"); String typedText = "input_scroll_text"; WdRemoteScrollTypeAction remoteScrollTypeAction = new WdRemoteScrollTypeAction(childWdWidget, typedText); @@ -141,7 +129,7 @@ public void test_create_WdRemoteScrollTypeAction() { // Verify the action and the Tags were created Assert.isTrue(remoteScrollTypeAction.get(Tags.OriginWidget).equals(childWdWidget)); Assert.isTrue(remoteScrollTypeAction.get(Tags.Role).equals(WdActionRoles.RemoteScrollType)); - Assert.isTrue(remoteScrollTypeAction.get(Tags.Desc).equals("Remote scroll and type " + typedText + " to widget " + childWdElement.name + " : " + "remote_id_value")); + Assert.isTrue(remoteScrollTypeAction.get(Tags.Desc).equals("Remote scroll and type " + typedText + " to widget " + "Widget Description")); Assert.isTrue(remoteScrollTypeAction.toShortString().equals("Remote scroll and type " + typedText + " " + childWdElement.name)); Assert.isTrue(remoteScrollTypeAction.toParametersString().equals("Remote scroll and type " + typedText + " " + childWdElement.name)); Assert.isTrue(remoteScrollTypeAction.toString(new Role[0]).equals("Remote scroll and type " + typedText + " " + childWdElement.name)); From 297b28f5d1883754488e7b8d79df12f3a55ff61b Mon Sep 17 00:00:00 2001 From: ferpasri Date: Wed, 4 Mar 2026 12:59:16 +0100 Subject: [PATCH 2/4] Add web remote actions to LLM conversation --- .../action/priorization/llm/ActionHistory.java | 14 ++++++++++++-- .../priorization/llm/LlmParseActionResponse.java | 10 ++++++++-- .../llm/prompt/ActionWebPromptGenerator.java | 8 ++++++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/testar/src/org/testar/action/priorization/llm/ActionHistory.java b/testar/src/org/testar/action/priorization/llm/ActionHistory.java index 3d3715762..ecd094bcf 100644 --- a/testar/src/org/testar/action/priorization/llm/ActionHistory.java +++ b/testar/src/org/testar/action/priorization/llm/ActionHistory.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,6 +40,7 @@ import org.testar.monkey.alayer.actions.CompoundAction; import org.testar.monkey.alayer.actions.PasteText; import org.testar.monkey.alayer.actions.Type; +import org.testar.monkey.alayer.actions.WdRemoteTypeAction; import org.testar.monkey.alayer.actions.WdSelectListAction; import java.util.ArrayList; @@ -122,11 +123,15 @@ public String toString() { switch(type) { case "ClickTypeInto": case "PasteTextInto": + case "RemoteType": + case "RemoteScrollType": String input = getCompoundActionInputText(action); // TODO: Differentiate between types of input fields (numeric, password, etc.) builder.append(String.format("%s: Typed '%s' in TextField '%s'", actionId, input, description)); break; case "LeftClickAt": + case "RemoteClick": + case "RemoteScrollClick": builder.append(String.format("%s: Clicked on '%s'", actionId, description)); break; case "HistoryBackScript": @@ -176,6 +181,11 @@ private String getCompoundActionInputText(Action action) { } } + if(action instanceof WdRemoteTypeAction) { + CharSequence input = ((WdRemoteTypeAction) action).getKeys(); + return input == null ? "Unknown Action Input" : input.toString(); + } + return action.get(Tags.InputText, "Unknown Action Input"); } } diff --git a/testar/src/org/testar/action/priorization/llm/LlmParseActionResponse.java b/testar/src/org/testar/action/priorization/llm/LlmParseActionResponse.java index d964ddae9..1b0b6e37c 100644 --- a/testar/src/org/testar/action/priorization/llm/LlmParseActionResponse.java +++ b/testar/src/org/testar/action/priorization/llm/LlmParseActionResponse.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2025 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2025 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -46,6 +46,7 @@ import org.testar.monkey.alayer.actions.NOP; import org.testar.monkey.alayer.actions.PasteText; import org.testar.monkey.alayer.actions.Type; +import org.testar.monkey.alayer.actions.WdRemoteTypeAction; import org.testar.monkey.alayer.actions.WdSelectListAction; import org.testar.monkey.alayer.visualizers.TextVisualizer; import org.testar.monkey.alayer.webdriver.enums.WdTags; @@ -178,6 +179,11 @@ private boolean setCompoundActionInputText(Action action, String inputText) { } private boolean updateTextAction(Action action, Action innerAction, String inputText) { + if(action instanceof WdRemoteTypeAction) { + ((WdRemoteTypeAction) action).setKeys(inputText); + return true; + } + innerAction.set(Tags.InputText, inputText); String widgetDesc = action.get(Tags.OriginWidget).get(Tags.Desc, ""); diff --git a/testar/src/org/testar/llm/prompt/ActionWebPromptGenerator.java b/testar/src/org/testar/llm/prompt/ActionWebPromptGenerator.java index cec8b52ff..8d8f4a131 100644 --- a/testar/src/org/testar/llm/prompt/ActionWebPromptGenerator.java +++ b/testar/src/org/testar/llm/prompt/ActionWebPromptGenerator.java @@ -1,7 +1,7 @@ /*************************************************************************************************** * - * Copyright (c) 2024 - 2025 Open Universiteit - www.ou.nl - * Copyright (c) 2024 - 2025 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2024 - 2026 Open Universiteit - www.ou.nl + * Copyright (c) 2024 - 2026 Universitat Politecnica de Valencia - www.upv.es * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -160,11 +160,15 @@ public String generateActionSelectionPrompt(Set actions, State state, Ac switch (type) { case "ClickTypeInto": case "PasteTextInto": + case "RemoteType": + case "RemoteScrollType": // Differentiate between types of input fields. Example: password -> Password Field String fieldType = StringUtils.capitalize(widget.get(WdTags.WebType, "text")); builder.append(String.format("%s: Type in %sField '%s' ", actionId, fieldType, description)); break; case "LeftClickAt": + case "RemoteClick": + case "RemoteScrollClick": builder.append(String.format("%s: Click on '%s' ", actionId, description)); break; case "HistoryBackScript": From 830502e6053ec2272fac8ae3cc185f970bdbbbf2 Mon Sep 17 00:00:00 2001 From: ferpasri Date: Wed, 4 Mar 2026 13:02:27 +0100 Subject: [PATCH 3/4] Update LLM conversation unit tests --- .../priorization/llm/TestActionHistory.java | 295 +++++++++++++----- .../llm/TestLlmParseActionResponse.java | 222 ++++++++----- .../prompt/TestActionWebPromptGenerator.java | 226 +++++++++----- .../alayer/webdriver/stub/WdWidgetStub.java | 133 ++++++++ 4 files changed, 644 insertions(+), 232 deletions(-) create mode 100644 webdriver/src/org/testar/monkey/alayer/webdriver/stub/WdWidgetStub.java diff --git a/testar/test/org/testar/action/priorization/llm/TestActionHistory.java b/testar/test/org/testar/action/priorization/llm/TestActionHistory.java index fa3786a4c..f05dea6c1 100644 --- a/testar/test/org/testar/action/priorization/llm/TestActionHistory.java +++ b/testar/test/org/testar/action/priorization/llm/TestActionHistory.java @@ -2,7 +2,6 @@ import static org.junit.Assert.assertTrue; -import org.junit.BeforeClass; import org.junit.Test; import org.testar.monkey.alayer.Action; import org.testar.monkey.alayer.Rect; @@ -11,12 +10,17 @@ import org.testar.monkey.alayer.actions.CompoundAction; import org.testar.monkey.alayer.actions.StdActionCompiler; import org.testar.monkey.alayer.actions.Type; +import org.testar.monkey.alayer.actions.WdRemoteClickAction; +import org.testar.monkey.alayer.actions.WdRemoteScrollClickAction; +import org.testar.monkey.alayer.actions.WdRemoteScrollTypeAction; +import org.testar.monkey.alayer.actions.WdRemoteTypeAction; import org.testar.monkey.alayer.actions.WdSelectListAction; import org.testar.monkey.alayer.android.actions.AndroidActionType; import org.testar.monkey.alayer.android.enums.AndroidRoles; import org.testar.monkey.alayer.android.enums.AndroidTags; import org.testar.monkey.alayer.webdriver.enums.WdRoles; import org.testar.monkey.alayer.webdriver.enums.WdTags; +import org.testar.monkey.alayer.webdriver.stub.WdWidgetStub; import org.testar.stub.StateStub; import org.testar.stub.WidgetStub; @@ -24,90 +28,18 @@ public class TestActionHistory { private static StdActionCompiler ac = new AnnotatingActionCompiler(); - private static Action web_click_action; - private static Action web_type_action; - private static Action web_select_action; - private static Action android_type_action; - - @BeforeClass - public static void setup() { - StateStub state = new StateStub(); - state.set(WdTags.WebTitle, "Page Title | State"); - - // Derive a click action - WidgetStub clickable_widget = new WidgetStub(); - state.addChild(clickable_widget); - clickable_widget.setParent(state); - clickable_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - clickable_widget.set(Tags.Role, WdRoles.WdA); - clickable_widget.set(Tags.Path, "[0,0,1]"); - clickable_widget.set(WdTags.WebId, "clickable_widget_web_id"); - clickable_widget.set(Tags.Desc, "clickable_widget_desc"); - clickable_widget.set(Tags.ConcreteID, "CID_clickable_widget"); - clickable_widget.set(Tags.AbstractID, "AID_clickable_widget"); - web_click_action = ac.leftClickAt(clickable_widget); - web_click_action.set(Tags.ConcreteID, "CID_click"); - web_click_action.set(Tags.AbstractID, "AID_click"); - - // Derive a Type action - WidgetStub typeable_widget = new WidgetStub(); - state.addChild(typeable_widget); - typeable_widget.setParent(state); - typeable_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - typeable_widget.set(Tags.Role, WdRoles.WdTEXTAREA); - typeable_widget.set(Tags.Path, "[0,0,1]"); - typeable_widget.set(WdTags.WebId, "typeable_widget_web_id"); - typeable_widget.set(Tags.Desc, "typeable_widget_desc"); - typeable_widget.set(Tags.ConcreteID, "CID_typeable_widget"); - typeable_widget.set(Tags.AbstractID, "AID_typeable_widget"); - web_type_action = ac.clickTypeInto(typeable_widget, "input_text", false); - web_type_action.set(Tags.ConcreteID, "CID_type"); - web_type_action.set(Tags.AbstractID, "AID_type"); - // Right now, TESTAR creates compound actions which contains Type actions - for(Action innerAction : ((CompoundAction)web_type_action).getActions()) { - if(innerAction instanceof Type) { - ((Type)innerAction).set(Tags.InputText, "LLM_text"); - } - } - - // Derive a select combobox action - WidgetStub combobox_widget = new WidgetStub(); - state.addChild(combobox_widget); - combobox_widget.setParent(state); - combobox_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - combobox_widget.set(Tags.Role, WdRoles.WdSELECT); - combobox_widget.set(WdTags.WebTagName, "select"); - combobox_widget.set(Tags.Path, "[0,0,1]"); - combobox_widget.set(WdTags.WebId, "combobox_widget_web_id"); - combobox_widget.set(Tags.Desc, "combobox_widget_desc"); - combobox_widget.set(WdTags.WebInnerHTML, ""); - combobox_widget.set(Tags.ConcreteID, "CID_combobox_widget"); - combobox_widget.set(Tags.AbstractID, "AID_combobox_widget"); - web_select_action = new WdSelectListAction("combobox_widget_web_id", "Saab", combobox_widget, WdSelectListAction.JsTargetMethod.ID); - web_select_action.set(Tags.ConcreteID, "CID_select"); - web_select_action.set(Tags.AbstractID, "AID_select"); - - // Derive an AndroidActionType action - WidgetStub android_edit_widget = new WidgetStub(); - state.addChild(android_edit_widget); - android_edit_widget.setParent(state); - android_edit_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - android_edit_widget.set(Tags.Role, AndroidRoles.AndroidWidget); - android_edit_widget.set(Tags.Path, "[0,0,1]"); - android_edit_widget.set(AndroidTags.AndroidXpath, "//path[0]"); - android_edit_widget.set(Tags.Desc, "android_edit_widget_desc"); - android_edit_widget.set(Tags.ConcreteID, "CID_android_edit_widget"); - android_edit_widget.set(Tags.AbstractID, "AID_android_edit_widget"); - android_edit_widget.set(AndroidTags.AndroidAccessibilityId, "accessibilityId"); - android_edit_widget.set(AndroidTags.AndroidClassName, "className"); - android_type_action = new AndroidActionType(state, android_edit_widget, "default"); - android_type_action.set(Tags.ConcreteID, "CID_android_action_type"); - android_type_action.set(Tags.AbstractID, "AID_android_action_type"); - android_type_action.set(Tags.InputText, "LLM_text"); - } - @Test public void test_history_web_click_action() { + Action web_click_action = createClickAction( + createState(), + "clickable_widget_desc", + "clickable_widget_web_id", + "CID_clickable_widget", + "AID_clickable_widget", + "CID_click", + "AID_click" + ); + ActionHistory actionHistory = new ActionHistory(1); actionHistory.addToHistory(web_click_action); assertTrue(actionHistory.getActions().contains(web_click_action)); @@ -117,6 +49,17 @@ public void test_history_web_click_action() { @Test public void test_history_web_type_action() { + Action web_type_action = createTypeAction( + createState(), + "typeable_widget_desc", + "typeable_widget_web_id", + "CID_typeable_widget", + "AID_typeable_widget", + "CID_type", + "AID_type", + "input_text", + "LLM_text"); + ActionHistory actionHistory = new ActionHistory(1); actionHistory.addToHistory(web_type_action); assertTrue(actionHistory.getActions().contains(web_type_action)); @@ -126,6 +69,16 @@ public void test_history_web_type_action() { @Test public void test_history_web_select_action() { + Action web_select_action = createSelectAction( + createState(), + "combobox_widget_desc", + "combobox_widget_web_id", + "CID_combobox_widget", + "AID_combobox_widget", + "CID_select", + "AID_select", + "Saab"); + ActionHistory actionHistory = new ActionHistory(1); actionHistory.addToHistory(web_select_action); assertTrue(actionHistory.getActions().contains(web_select_action)); @@ -135,6 +88,8 @@ public void test_history_web_select_action() { @Test public void test_history_android_type_action() { + Action android_type_action = createAndroidTypeAction(createState()); + ActionHistory actionHistory = new ActionHistory(1); actionHistory.addToHistory(android_type_action); assertTrue(actionHistory.getActions().contains(android_type_action)); @@ -142,8 +97,79 @@ public void test_history_android_type_action() { assertTrue(actionHistory.toString().contains("Typed 'LLM_text' in TextField 'android_edit_widget_desc'")); } + @Test + public void test_history_web_remote_click_action() { + Action web_remote_click_action = createRemoteClickAction( + "remote_click_widget", + "remote_click_id", + "CID_remote_click", + "AID_remote_click"); + + ActionHistory actionHistory = new ActionHistory(1); + actionHistory.addToHistory(web_remote_click_action); + assertTrue(actionHistory.getActions().contains(web_remote_click_action)); + assertTrue(actionHistory.toString().contains("This is the last action we executed: AID_remote_click:")); + assertTrue(actionHistory.toString().contains("Clicked on 'remote_click_widget'")); + } + + @Test + public void test_history_web_remote_scroll_click_action() { + Action web_remote_scroll_click_action = createRemoteScrollClickAction( + "remote_scroll_click_widget", + "remote_scroll_click_id", + "CID_remote_scroll_click", + "AID_remote_scroll_click"); + + ActionHistory actionHistory = new ActionHistory(1); + actionHistory.addToHistory(web_remote_scroll_click_action); + assertTrue(actionHistory.getActions().contains(web_remote_scroll_click_action)); + assertTrue(actionHistory.toString().contains("This is the last action we executed: AID_remote_scroll_click:")); + assertTrue(actionHistory.toString().contains("Clicked on 'remote_scroll_click_widget'")); + } + + @Test + public void test_history_web_remote_type_action() { + Action web_remote_type_action = createRemoteTypeAction( + "remote_type_widget", + "remote_type_id", + "CID_remote_type", + "AID_remote_type", + "remote_text"); + + ActionHistory actionHistory = new ActionHistory(1); + actionHistory.addToHistory(web_remote_type_action); + assertTrue(actionHistory.getActions().contains(web_remote_type_action)); + assertTrue(actionHistory.toString().contains("This is the last action we executed: AID_remote_type:")); + assertTrue(actionHistory.toString().contains("Typed 'remote_text' in TextField 'remote_type_widget'")); + } + + @Test + public void test_history_web_remote_scroll_type_action() { + Action web_remote_scroll_type_action = createRemoteScrollTypeAction( + "remote_scroll_type_widget", + "remote_scroll_type_id", + "CID_remote_scroll_type", + "AID_remote_scroll_type", + "remote_scroll_text"); + + ActionHistory actionHistory = new ActionHistory(1); + actionHistory.addToHistory(web_remote_scroll_type_action); + assertTrue(actionHistory.getActions().contains(web_remote_scroll_type_action)); + assertTrue(actionHistory.toString().contains("This is the last action we executed: AID_remote_scroll_type:")); + assertTrue(actionHistory.toString().contains("Typed 'remote_scroll_text' in TextField 'remote_scroll_type_widget'")); + } + @Test public void test_history_multiple_actions() { + StateStub state = createState(); + Action web_click_action = createClickAction(state, "clickable_widget_desc", "clickable_widget_web_id", + "CID_clickable_widget", "AID_clickable_widget", "CID_click", "AID_click"); + Action web_type_action = createTypeAction(state, "typeable_widget_desc", "typeable_widget_web_id", + "CID_typeable_widget", "AID_typeable_widget", "CID_type", "AID_type", "input_text", "LLM_text"); + Action web_select_action = createSelectAction(state, "combobox_widget_desc", "combobox_widget_web_id", + "CID_combobox_widget", "AID_combobox_widget", "CID_select", "AID_select", "Saab"); + Action android_type_action = createAndroidTypeAction(state); + ActionHistory actionHistory = new ActionHistory(3); actionHistory.addToHistory(web_click_action); actionHistory.addToHistory(web_type_action); @@ -163,4 +189,107 @@ public void test_history_multiple_actions() { assertTrue(actionHistory.toString().contains("Typed 'LLM_text' in TextField 'android_edit_widget_desc'")); } + private static StateStub createState() { + StateStub createdState = new StateStub(); + createdState.set(WdTags.WebTitle, "Page Title | State"); + return createdState; + } + + private static WidgetStub createWebWidget(StateStub parentState, org.testar.monkey.alayer.Role role, String description, + String webId, String concreteId, String abstractId) { + WidgetStub widget = new WidgetStub(); + parentState.addChild(widget); + widget.setParent(parentState); + widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); + widget.set(Tags.Role, role); + widget.set(Tags.Path, "[0,0,1]"); + widget.set(WdTags.WebId, webId); + widget.set(Tags.Desc, description); + widget.set(Tags.ConcreteID, concreteId); + widget.set(Tags.AbstractID, abstractId); + return widget; + } + + private static Action createClickAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId) { + Action action = ac.leftClickAt(createWebWidget(parentState, WdRoles.WdA, description, webId, widgetConcreteId, widgetAbstractId)); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + return action; + } + + private static Action createTypeAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId, String inputText, String llmText) { + Action action = ac.clickTypeInto(createWebWidget(parentState, WdRoles.WdTEXTAREA, description, webId, widgetConcreteId, widgetAbstractId), + inputText, false); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + for(Action innerAction : ((CompoundAction) action).getActions()) { + if(innerAction instanceof Type) { + ((Type) innerAction).set(Tags.InputText, llmText); + } + } + return action; + } + + private static Action createSelectAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId, String value) { + WidgetStub widget = createWebWidget(parentState, WdRoles.WdSELECT, description, webId, widgetConcreteId, widgetAbstractId); + widget.set(WdTags.WebTagName, "select"); + widget.set(WdTags.WebInnerHTML, ""); + Action action = new WdSelectListAction(webId, value, widget, WdSelectListAction.JsTargetMethod.ID); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + return action; + } + + private static Action createRemoteClickAction(String description, String remoteId, String concreteId, String abstractId) { + Action action = new WdRemoteClickAction(new WdWidgetStub(description, remoteId, WdRoles.WdA, "a")); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createRemoteScrollClickAction(String description, String remoteId, String concreteId, String abstractId) { + Action action = new WdRemoteScrollClickAction(new WdWidgetStub(description, remoteId, WdRoles.WdA, "a")); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createRemoteTypeAction(String description, String remoteId, String concreteId, String abstractId, String text) { + Action action = new WdRemoteTypeAction(new WdWidgetStub(description, remoteId, WdRoles.WdTEXTAREA, "textarea"), text); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createRemoteScrollTypeAction(String description, String remoteId, String concreteId, String abstractId, String text) { + Action action = new WdRemoteScrollTypeAction(new WdWidgetStub(description, remoteId, WdRoles.WdTEXTAREA, "textarea"), text); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createAndroidTypeAction(StateStub parentState) { + WidgetStub widget = new WidgetStub(); + parentState.addChild(widget); + widget.setParent(parentState); + widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); + widget.set(Tags.Role, AndroidRoles.AndroidWidget); + widget.set(Tags.Path, "[0,0,1]"); + widget.set(AndroidTags.AndroidXpath, "//path[0]"); + widget.set(Tags.Desc, "android_edit_widget_desc"); + widget.set(Tags.ConcreteID, "CID_android_edit_widget"); + widget.set(Tags.AbstractID, "AID_android_edit_widget"); + widget.set(AndroidTags.AndroidAccessibilityId, "accessibilityId"); + widget.set(AndroidTags.AndroidClassName, "className"); + + Action action = new AndroidActionType(parentState, widget, "default"); + action.set(Tags.ConcreteID, "CID_android_action_type"); + action.set(Tags.AbstractID, "AID_android_action_type"); + action.set(Tags.InputText, "LLM_text"); + return action; + } + } diff --git a/testar/test/org/testar/action/priorization/llm/TestLlmParseActionResponse.java b/testar/test/org/testar/action/priorization/llm/TestLlmParseActionResponse.java index 20167b50d..0f4fbc591 100644 --- a/testar/test/org/testar/action/priorization/llm/TestLlmParseActionResponse.java +++ b/testar/test/org/testar/action/priorization/llm/TestLlmParseActionResponse.java @@ -3,7 +3,6 @@ import java.util.HashSet; import java.util.Set; -import org.junit.BeforeClass; import org.junit.Test; import org.testar.monkey.Assert; import org.testar.monkey.alayer.Action; @@ -13,6 +12,8 @@ import org.testar.monkey.alayer.actions.CompoundAction; import org.testar.monkey.alayer.actions.StdActionCompiler; import org.testar.monkey.alayer.actions.Type; +import org.testar.monkey.alayer.actions.WdRemoteScrollTypeAction; +import org.testar.monkey.alayer.actions.WdRemoteTypeAction; import org.testar.monkey.alayer.actions.WdSelectListAction; import org.testar.monkey.alayer.android.actions.AndroidActionType; import org.testar.monkey.alayer.android.enums.AndroidRoles; @@ -20,93 +21,18 @@ import org.testar.monkey.alayer.visualizers.TextVisualizer; import org.testar.monkey.alayer.webdriver.enums.WdRoles; import org.testar.monkey.alayer.webdriver.enums.WdTags; +import org.testar.monkey.alayer.webdriver.stub.WdWidgetStub; import org.testar.stub.StateStub; import org.testar.stub.WidgetStub; public class TestLlmParseActionResponse { - private static StateStub state; - private static Set derivedActions; private static StdActionCompiler ac = new AnnotatingActionCompiler(); - @BeforeClass - public static void setup() { - state = new StateStub(); - state.set(WdTags.WebTitle, "Page Title | State"); - derivedActions = new HashSet<>(); - - // Derive a click action - WidgetStub clickable_widget = new WidgetStub(); - state.addChild(clickable_widget); - clickable_widget.setParent(state); - clickable_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - clickable_widget.set(Tags.Role, WdRoles.WdA); - clickable_widget.set(Tags.Path, "[0,0,1]"); - clickable_widget.set(WdTags.WebId, "clickable_widget_web_id"); - clickable_widget.set(Tags.Desc, "clickable_widget_desc"); - clickable_widget.set(Tags.ConcreteID, "CID_clickable_widget"); - clickable_widget.set(Tags.AbstractID, "AID_clickable_widget"); - Action click_action = ac.leftClickAt(clickable_widget); - click_action.set(Tags.ConcreteID, "CID_click"); - click_action.set(Tags.AbstractID, "AID_click"); - derivedActions.add(click_action); - - // Derive a Type action - WidgetStub typeable_widget = new WidgetStub(); - state.addChild(typeable_widget); - typeable_widget.setParent(state); - typeable_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - typeable_widget.set(Tags.Role, WdRoles.WdTEXTAREA); - typeable_widget.set(Tags.Path, "[0,0,1]"); - typeable_widget.set(WdTags.WebId, "typeable_widget_web_id"); - typeable_widget.set(Tags.Desc, "typeable_widget_desc"); - typeable_widget.set(Tags.ConcreteID, "CID_typeable_widget"); - typeable_widget.set(Tags.AbstractID, "AID_typeable_widget"); - Action type_action = ac.clickTypeInto(typeable_widget, "random_text", false); - type_action.set(Tags.ConcreteID, "CID_type"); - type_action.set(Tags.AbstractID, "AID_type"); - derivedActions.add(type_action); - - // Derive a select combobox action - WidgetStub combobox_widget = new WidgetStub(); - state.addChild(combobox_widget); - combobox_widget.setParent(state); - combobox_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - combobox_widget.set(Tags.Role, WdRoles.WdSELECT); - combobox_widget.set(WdTags.WebTagName, "select"); - combobox_widget.set(Tags.Path, "[0,0,1]"); - combobox_widget.set(WdTags.WebId, "combobox_widget_web_id"); - combobox_widget.set(Tags.Desc, "combobox_widget_desc"); - combobox_widget.set(WdTags.WebInnerHTML, ""); - combobox_widget.set(Tags.ConcreteID, "CID_combobox_widget"); - combobox_widget.set(Tags.AbstractID, "AID_combobox_widget"); - Action select_action = new WdSelectListAction("combobox_widget_web_id", "Saab", combobox_widget, WdSelectListAction.JsTargetMethod.ID); - select_action.set(Tags.ConcreteID, "CID_select"); - select_action.set(Tags.AbstractID, "AID_select"); - derivedActions.add(select_action); - - // Derive an AndroidActionType action - WidgetStub android_edit_widget = new WidgetStub(); - state.addChild(android_edit_widget); - android_edit_widget.setParent(state); - android_edit_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - android_edit_widget.set(Tags.Role, AndroidRoles.AndroidWidget); - android_edit_widget.set(Tags.Path, "[0,0,1]"); - android_edit_widget.set(AndroidTags.AndroidXpath, "//path[0]"); - android_edit_widget.set(Tags.Desc, "android_edit_widget_desc"); - android_edit_widget.set(Tags.ConcreteID, "CID_android_edit_widget"); - android_edit_widget.set(Tags.AbstractID, "AID_android_edit_widget"); - android_edit_widget.set(AndroidTags.AndroidAccessibilityId, "accessibilityId"); - android_edit_widget.set(AndroidTags.AndroidClassName, "className"); - Action android_action_type = new AndroidActionType(state, android_edit_widget, "default"); - android_action_type.set(Tags.ConcreteID, "CID_android_action_type"); - android_action_type.set(Tags.AbstractID, "AID_android_action_type"); - derivedActions.add(android_action_type); - } - @Test public void test_llm_selects_click_action() { String llmResponse = "{\"actionId\":\"AID_click\",\"input\":\"\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -118,6 +44,7 @@ public void test_llm_selects_click_action() { @Test public void test_parsing_click_action() { String llmResponse = "```json{\"actionId\":\"AID_click\",\"input\":\"\"}```"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -129,6 +56,7 @@ public void test_parsing_click_action() { @Test public void test_llm_selects_type_action() { String llmResponse = "{\"actionId\":\"AID_type\",\"input\":\"testar\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -153,6 +81,7 @@ public void test_llm_selects_type_action() { @Test public void test_parsing_type_action() { String llmResponse = "```json{\"actionId\":\"AID_type\",\"input\":\"testar\"}```"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -174,9 +103,41 @@ public void test_parsing_type_action() { Assert.isTrue(innerText.equals("testar")); } + @Test + public void test_llm_selects_remote_type_action() { + String llmResponse = "{\"actionId\":\"AID_remote_type\",\"input\":\"testar_remote\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); + + LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); + LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); + + Assert.isTrue(llmParseResult.getParseResult().equals(LlmParseActionResult.ParseResult.SUCCESS)); + Assert.isTrue(llmParseResult.getActionToExecute().get(Tags.AbstractID).equals("AID_remote_type")); + Assert.isTrue(llmParseResult.getActionToExecute() instanceof WdRemoteTypeAction); + Assert.isTrue(((WdRemoteTypeAction) llmParseResult.getActionToExecute()).getKeys().equals("testar_remote")); + Assert.isTrue(llmParseResult.getActionToExecute().get(Tags.InputText, "").equals("testar_remote")); + } + + @Test + public void test_llm_selects_remote_scroll_type_action() { + String llmResponse = "{\"actionId\":\"AID_remote_scroll_type\",\"input\":\"testar_remote_scroll\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); + + LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); + LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); + + Assert.isTrue(llmParseResult.getParseResult().equals(LlmParseActionResult.ParseResult.SUCCESS)); + Assert.isTrue(llmParseResult.getActionToExecute().get(Tags.AbstractID).equals("AID_remote_scroll_type")); + Assert.isTrue(llmParseResult.getActionToExecute() instanceof WdRemoteTypeAction); + Assert.isTrue(llmParseResult.getActionToExecute() instanceof WdRemoteScrollTypeAction); + Assert.isTrue(((WdRemoteScrollTypeAction) llmParseResult.getActionToExecute()).getKeys().equals("testar_remote_scroll")); + Assert.isTrue(llmParseResult.getActionToExecute().get(Tags.InputText, "").equals("testar_remote_scroll")); + } + @Test public void test_llm_creates_combobox_action() { String llmResponse = "{\"actionId\":\"AID_select\",\"input\":\"Saab\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -190,6 +151,7 @@ public void test_llm_creates_combobox_action() { @Test public void test_llm_missing_actionid() { String llmResponse = "{\"input\":\"\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -200,6 +162,7 @@ public void test_llm_missing_actionid() { @Test public void test_llm_selects_invalid_action() { String llmResponse = "{\"actionId\":\"ABCDEF\",\"input\":\"\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -210,6 +173,7 @@ public void test_llm_selects_invalid_action() { @Test public void test_llm_missing_select_input() { String llmResponse = "{\"actionId\":\"AID_select\",\"input\":\"\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -220,6 +184,7 @@ public void test_llm_missing_select_input() { @Test public void test_invalid_response_format() { String llmResponse = "\"actionId\":\"AID_click\",\"input\":\"\""; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -230,6 +195,7 @@ public void test_invalid_response_format() { @Test public void test_null_response() { String llmResponse = null; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -240,6 +206,7 @@ public void test_null_response() { @Test public void test_llm_selects_android_type_action() { String llmResponse = "{\"actionId\":\"AID_android_action_type\",\"input\":\"testar\"}"; + Set derivedActions = createDefaultDerivedActions(createState()); LlmParseActionResponse llmParseResponse = new LlmParseActionResponse(); LlmParseActionResult llmParseResult = llmParseResponse.parseLlmResponse(derivedActions, llmResponse); @@ -255,4 +222,103 @@ public void test_llm_selects_android_type_action() { Assert.isTrue(innerText.equals("testar")); } + private static StateStub createState() { + StateStub createdState = new StateStub(); + createdState.set(WdTags.WebTitle, "Page Title | State"); + return createdState; + } + + private static Set createDefaultDerivedActions(StateStub state) { + Set derivedActions = new HashSet<>(); + derivedActions.add(createClickAction(state, "clickable_widget_desc", "clickable_widget_web_id", + "CID_clickable_widget", "AID_clickable_widget", "CID_click", "AID_click")); + derivedActions.add(createTypeAction(state, "typeable_widget_desc", "typeable_widget_web_id", + "CID_typeable_widget", "AID_typeable_widget", "CID_type", "AID_type", "random_text")); + derivedActions.add(createRemoteTypeAction("remote_type_widget", "remote_type_id", + "CID_remote_type", "AID_remote_type", "random_remote_text")); + derivedActions.add(createRemoteScrollTypeAction("remote_scroll_type_widget", "remote_scroll_type_id", + "CID_remote_scroll_type", "AID_remote_scroll_type", "random_remote_scroll_text")); + derivedActions.add(createSelectAction(state, "combobox_widget_desc", "combobox_widget_web_id", + "CID_combobox_widget", "AID_combobox_widget", "CID_select", "AID_select", "Saab")); + derivedActions.add(createAndroidTypeAction(state)); + return derivedActions; + } + + private static WidgetStub createWebWidget(StateStub parentState, org.testar.monkey.alayer.Role role, String description, + String webId, String concreteId, String abstractId) { + WidgetStub widget = new WidgetStub(); + parentState.addChild(widget); + widget.setParent(parentState); + widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); + widget.set(Tags.Role, role); + widget.set(Tags.Path, "[0,0,1]"); + widget.set(WdTags.WebId, webId); + widget.set(Tags.Desc, description); + widget.set(Tags.ConcreteID, concreteId); + widget.set(Tags.AbstractID, abstractId); + return widget; + } + + private static Action createClickAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId) { + Action action = ac.leftClickAt(createWebWidget(parentState, WdRoles.WdA, description, webId, widgetConcreteId, widgetAbstractId)); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + return action; + } + + private static Action createTypeAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId, String inputText) { + Action action = ac.clickTypeInto(createWebWidget(parentState, WdRoles.WdTEXTAREA, description, webId, widgetConcreteId, widgetAbstractId), + inputText, false); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + return action; + } + + private static Action createSelectAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId, String value) { + WidgetStub widget = createWebWidget(parentState, WdRoles.WdSELECT, description, webId, widgetConcreteId, widgetAbstractId); + widget.set(WdTags.WebTagName, "select"); + widget.set(WdTags.WebInnerHTML, ""); + Action action = new WdSelectListAction(webId, value, widget, WdSelectListAction.JsTargetMethod.ID); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + return action; + } + + private static Action createRemoteTypeAction(String description, String remoteId, String concreteId, String abstractId, String text) { + Action action = new WdRemoteTypeAction(new WdWidgetStub(description, remoteId, WdRoles.WdTEXTAREA, "textarea"), text); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createRemoteScrollTypeAction(String description, String remoteId, String concreteId, String abstractId, String text) { + Action action = new WdRemoteScrollTypeAction(new WdWidgetStub(description, remoteId, WdRoles.WdTEXTAREA, "textarea"), text); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createAndroidTypeAction(StateStub parentState) { + WidgetStub widget = new WidgetStub(); + parentState.addChild(widget); + widget.setParent(parentState); + widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); + widget.set(Tags.Role, AndroidRoles.AndroidWidget); + widget.set(Tags.Path, "[0,0,1]"); + widget.set(AndroidTags.AndroidXpath, "//path[0]"); + widget.set(Tags.Desc, "android_edit_widget_desc"); + widget.set(Tags.ConcreteID, "CID_android_edit_widget"); + widget.set(Tags.AbstractID, "AID_android_edit_widget"); + widget.set(AndroidTags.AndroidAccessibilityId, "accessibilityId"); + widget.set(AndroidTags.AndroidClassName, "className"); + + Action action = new AndroidActionType(parentState, widget, "default"); + action.set(Tags.ConcreteID, "CID_android_action_type"); + action.set(Tags.AbstractID, "AID_android_action_type"); + return action; + } + } diff --git a/testar/test/org/testar/llm/prompt/TestActionWebPromptGenerator.java b/testar/test/org/testar/llm/prompt/TestActionWebPromptGenerator.java index 8ee4c43b5..b806ed546 100644 --- a/testar/test/org/testar/llm/prompt/TestActionWebPromptGenerator.java +++ b/testar/test/org/testar/llm/prompt/TestActionWebPromptGenerator.java @@ -4,7 +4,6 @@ import java.util.HashSet; import java.util.Set; -import org.junit.BeforeClass; import org.junit.Test; import org.testar.action.priorization.llm.ActionHistory; import org.testar.monkey.Assert; @@ -17,89 +16,26 @@ import org.testar.monkey.alayer.actions.Type; import org.testar.monkey.alayer.actions.WdCloseTabAction; import org.testar.monkey.alayer.actions.WdHistoryBackAction; +import org.testar.monkey.alayer.actions.WdRemoteClickAction; +import org.testar.monkey.alayer.actions.WdRemoteScrollClickAction; +import org.testar.monkey.alayer.actions.WdRemoteScrollTypeAction; +import org.testar.monkey.alayer.actions.WdRemoteTypeAction; import org.testar.monkey.alayer.actions.WdSelectListAction; import org.testar.monkey.alayer.webdriver.enums.WdRoles; import org.testar.monkey.alayer.webdriver.enums.WdTags; +import org.testar.monkey.alayer.webdriver.stub.WdWidgetStub; import org.testar.stub.StateStub; import org.testar.stub.WidgetStub; public class TestActionWebPromptGenerator { - private static StateStub state; - private static Set derivedActions; - private static ActionHistory actionHistory; private static StdActionCompiler ac = new AnnotatingActionCompiler(); - @BeforeClass - public static void setup() { - state = new StateStub(); - state.set(WdTags.WebTitle, "Page Title | State"); - derivedActions = new HashSet<>(); - actionHistory = new ActionHistory(1); - - // Derive a click action - WidgetStub clickable_widget = new WidgetStub(); - state.addChild(clickable_widget); - clickable_widget.setParent(state); - clickable_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - clickable_widget.set(Tags.Role, WdRoles.WdA); - clickable_widget.set(Tags.Path, "[0,0,1]"); - clickable_widget.set(WdTags.WebId, "clickable_widget_web_id"); - clickable_widget.set(Tags.Desc, "clickable_widget_desc"); - clickable_widget.set(Tags.ConcreteID, "CID_clickable_widget"); - clickable_widget.set(Tags.AbstractID, "AID_clickable_widget"); - Action click_action = ac.leftClickAt(clickable_widget); - click_action.set(Tags.ConcreteID, "CID_click"); - click_action.set(Tags.AbstractID, "AID_click"); - derivedActions.add(click_action); - actionHistory.addToHistory(click_action); - - // Derive a Type action - WidgetStub typeable_widget = new WidgetStub(); - state.addChild(typeable_widget); - typeable_widget.setParent(state); - typeable_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - typeable_widget.set(Tags.Role, WdRoles.WdTEXTAREA); - typeable_widget.set(Tags.Path, "[0,0,1]"); - typeable_widget.set(WdTags.WebId, "typeable_widget_web_id"); - typeable_widget.set(Tags.Desc, "typeable_widget_desc"); - typeable_widget.set(Tags.ConcreteID, "CID_typeable_widget"); - typeable_widget.set(Tags.AbstractID, "AID_typeable_widget"); - Action type_action = ac.clickTypeInto(typeable_widget, "input_text", false); - type_action.set(Tags.ConcreteID, "CID_type"); - type_action.set(Tags.AbstractID, "AID_type"); - // Right now, TESTAR creates compound actions which contains Type actions - for(Action innerAction : ((CompoundAction)type_action).getActions()) { - if(innerAction instanceof Type) { - ((Type)innerAction).set(Tags.InputText, "LLM_text"); - } - } - derivedActions.add(type_action); - actionHistory.addToHistory(type_action); - - // Derive a select combobox action - WidgetStub combobox_widget = new WidgetStub(); - state.addChild(combobox_widget); - combobox_widget.setParent(state); - combobox_widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); - combobox_widget.set(Tags.Role, WdRoles.WdSELECT); - combobox_widget.set(WdTags.WebTagName, "select"); - combobox_widget.set(Tags.Path, "[0,0,1]"); - combobox_widget.set(WdTags.WebId, "combobox_widget_web_id"); - combobox_widget.set(Tags.Desc, "combobox_widget_desc"); - combobox_widget.set(WdTags.WebInnerHTML, ""); - combobox_widget.set(Tags.ConcreteID, "CID_combobox_widget"); - combobox_widget.set(Tags.AbstractID, "AID_combobox_widget"); - Action select_action = new WdSelectListAction("combobox_widget_web_id", "Saab", combobox_widget, WdSelectListAction.JsTargetMethod.ID); - select_action.set(Tags.ConcreteID, "CID_select"); - select_action.set(Tags.AbstractID, "AID_select"); - derivedActions.add(select_action); - actionHistory.addToHistory(select_action); - } - @Test public void test_prompt_generator_default_empty_history() { IPromptActionGenerator promptActionGenerator = new ActionWebPromptGenerator(); + StateStub state = createState(); + Set derivedActions = createDefaultDerivedActions(state); String prompt = promptActionGenerator.generateActionSelectionPrompt( derivedActions, @@ -126,6 +62,9 @@ public void test_prompt_generator_default_empty_history() { @Test public void test_prompt_generator_webid_with_history() { IPromptActionGenerator promptActionGenerator = new ActionWebPromptGenerator(WdTags.WebId); + StateStub state = createState(); + Set derivedActions = createDefaultDerivedActions(state); + ActionHistory actionHistory = createDefaultActionHistory(state); String prompt = promptActionGenerator.generateActionSelectionPrompt( derivedActions, @@ -152,6 +91,7 @@ public void test_prompt_generator_webid_with_history() { @Test public void test_prompt_generator_history_back() { IPromptActionGenerator promptActionGenerator = new ActionWebPromptGenerator(); + StateStub state = createState(); Action actionHistoryBack = new WdHistoryBackAction(state); actionHistoryBack.set(Tags.ConcreteID, "CID_history"); @@ -187,6 +127,7 @@ public void test_prompt_generator_history_back() { @Test public void test_prompt_generator_close_tab() { IPromptActionGenerator promptActionGenerator = new ActionWebPromptGenerator(); + StateStub state = createState(); Action actionHistoryBack = new WdHistoryBackAction(state); actionHistoryBack.set(Tags.ConcreteID, "CID_history"); @@ -218,4 +159,147 @@ public void test_prompt_generator_close_tab() { Assert.isTrue(prompt.contains("This is the last action we executed: AID_history: Go History back in the browser")); Assert.isTrue(prompt.contains("Which action should be executed to accomplish the test goal?")); } + + @Test + public void test_prompt_generator_supports_remote_actions() throws Exception { + IPromptActionGenerator promptActionGenerator = new ActionWebPromptGenerator(); + StateStub state = createState(); + + Set actions = new HashSet<>(); + + Action remoteClickAction = createRemoteClickAction("remote_click_widget", "remote_click_id", + "CID_remote_click", "AID_remote_click"); + actions.add(remoteClickAction); + + Action remoteTypeAction = createRemoteTypeAction("remote_type_widget", "remote_type_id", + "CID_remote_type", "AID_remote_type", "remote_text"); + actions.add(remoteTypeAction); + + Action remoteScrollClickAction = createRemoteScrollClickAction("remote_scroll_click_widget", "remote_scroll_click_id", + "CID_remote_scroll_click", "AID_remote_scroll_click"); + actions.add(remoteScrollClickAction); + + Action remoteScrollTypeAction = createRemoteScrollTypeAction("remote_scroll_type_widget", "remote_scroll_type_id", + "CID_remote_scroll_type", "AID_remote_scroll_type", "remote_scroll_text"); + actions.add(remoteScrollTypeAction); + + String prompt = promptActionGenerator.generateActionSelectionPrompt( + actions, + state, + new ActionHistory(5), + "GUI_app", + "achieve this web goal", + "" + ); + + Assert.isTrue(!prompt.isEmpty()); + Assert.isTrue(prompt.contains("AID_remote_click: Click on 'remote_click_widget'")); + Assert.isTrue(prompt.contains("AID_remote_type: Type in TextField 'remote_type_widget'")); + Assert.isTrue(prompt.contains("AID_remote_scroll_click: Click on 'remote_scroll_click_widget'")); + Assert.isTrue(prompt.contains("AID_remote_scroll_type: Type in TextField 'remote_scroll_type_widget'")); + } + + private static StateStub createState() { + StateStub createdState = new StateStub(); + createdState.set(WdTags.WebTitle, "Page Title | State"); + return createdState; + } + + private static Set createDefaultDerivedActions(StateStub state) { + Set derivedActions = new HashSet<>(); + derivedActions.add(createClickAction(state, "clickable_widget_desc", "clickable_widget_web_id", + "CID_clickable_widget", "AID_clickable_widget", "CID_click", "AID_click")); + derivedActions.add(createTypeAction(state, "typeable_widget_desc", "typeable_widget_web_id", + "CID_typeable_widget", "AID_typeable_widget", "CID_type", "AID_type", "input_text", "LLM_text")); + derivedActions.add(createSelectAction(state, "combobox_widget_desc", "combobox_widget_web_id", + "CID_combobox_widget", "AID_combobox_widget", "CID_select", "AID_select", "Saab")); + return derivedActions; + } + + private static ActionHistory createDefaultActionHistory(StateStub state) { + ActionHistory actionHistory = new ActionHistory(1); + actionHistory.addToHistory(createClickAction(state, "clickable_widget_desc", "clickable_widget_web_id", + "CID_clickable_widget", "AID_clickable_widget", "CID_click", "AID_click")); + actionHistory.addToHistory(createTypeAction(state, "typeable_widget_desc", "typeable_widget_web_id", + "CID_typeable_widget", "AID_typeable_widget", "CID_type", "AID_type", "input_text", "LLM_text")); + actionHistory.addToHistory(createSelectAction(state, "combobox_widget_desc", "combobox_widget_web_id", + "CID_combobox_widget", "AID_combobox_widget", "CID_select", "AID_select", "Saab")); + return actionHistory; + } + + private static WidgetStub createWebWidget(StateStub parentState, org.testar.monkey.alayer.Role role, String description, + String webId, String concreteId, String abstractId) { + WidgetStub widget = new WidgetStub(); + parentState.addChild(widget); + widget.setParent(parentState); + widget.set(Tags.Shape, Rect.fromCoordinates(1, 1, 1, 1)); + widget.set(Tags.Role, role); + widget.set(Tags.Path, "[0,0,1]"); + widget.set(WdTags.WebId, webId); + widget.set(Tags.Desc, description); + widget.set(Tags.ConcreteID, concreteId); + widget.set(Tags.AbstractID, abstractId); + return widget; + } + + private static Action createClickAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId) { + Action action = ac.leftClickAt(createWebWidget(parentState, WdRoles.WdA, description, webId, widgetConcreteId, widgetAbstractId)); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + return action; + } + + private static Action createTypeAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId, String inputText, String llmText) { + Action action = ac.clickTypeInto(createWebWidget(parentState, WdRoles.WdTEXTAREA, description, webId, widgetConcreteId, widgetAbstractId), + inputText, false); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + for(Action innerAction : ((CompoundAction) action).getActions()) { + if(innerAction instanceof Type) { + ((Type) innerAction).set(Tags.InputText, llmText); + } + } + return action; + } + + private static Action createSelectAction(StateStub parentState, String description, String webId, String widgetConcreteId, + String widgetAbstractId, String actionConcreteId, String actionAbstractId, String value) { + WidgetStub widget = createWebWidget(parentState, WdRoles.WdSELECT, description, webId, widgetConcreteId, widgetAbstractId); + widget.set(WdTags.WebTagName, "select"); + widget.set(WdTags.WebInnerHTML, ""); + Action action = new WdSelectListAction(webId, value, widget, WdSelectListAction.JsTargetMethod.ID); + action.set(Tags.ConcreteID, actionConcreteId); + action.set(Tags.AbstractID, actionAbstractId); + return action; + } + + private static Action createRemoteClickAction(String description, String remoteId, String concreteId, String abstractId) { + Action action = new WdRemoteClickAction(new WdWidgetStub(description, remoteId, WdRoles.WdA, "a")); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createRemoteScrollClickAction(String description, String remoteId, String concreteId, String abstractId) { + Action action = new WdRemoteScrollClickAction(new WdWidgetStub(description, remoteId, WdRoles.WdA, "a")); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createRemoteTypeAction(String description, String remoteId, String concreteId, String abstractId, String text) { + Action action = new WdRemoteTypeAction(new WdWidgetStub(description, remoteId, WdRoles.WdTEXTAREA, "textarea"), text); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } + + private static Action createRemoteScrollTypeAction(String description, String remoteId, String concreteId, String abstractId, String text) { + Action action = new WdRemoteScrollTypeAction(new WdWidgetStub(description, remoteId, WdRoles.WdTEXTAREA, "textarea"), text); + action.set(Tags.ConcreteID, concreteId); + action.set(Tags.AbstractID, abstractId); + return action; + } } diff --git a/webdriver/src/org/testar/monkey/alayer/webdriver/stub/WdWidgetStub.java b/webdriver/src/org/testar/monkey/alayer/webdriver/stub/WdWidgetStub.java new file mode 100644 index 000000000..0a11fecb6 --- /dev/null +++ b/webdriver/src/org/testar/monkey/alayer/webdriver/stub/WdWidgetStub.java @@ -0,0 +1,133 @@ +/** + * Copyright (c) 2026 Universitat Politecnica de Valencia - www.upv.es + * Copyright (c) 2026 Open Universiteit - www.ou.nl + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +package org.testar.monkey.alayer.webdriver.stub; + +import org.openqa.selenium.remote.RemoteWebElement; +import org.testar.monkey.alayer.Rect; +import org.testar.monkey.alayer.Role; +import org.testar.monkey.alayer.Roles; +import org.testar.monkey.alayer.Tags; +import org.testar.monkey.alayer.webdriver.WdElement; +import org.testar.monkey.alayer.webdriver.WdState; +import org.testar.monkey.alayer.webdriver.WdWidget; + +public class WdWidgetStub extends WdWidget { + private static final long serialVersionUID = 2469259954952714389L; + + public WdWidgetStub() { + this(createState(), "", "", Roles.Widget, "div", "text", Rect.fromCoordinates(0, 0, 1, 1), createRemoteWebElement("")); + } + + public WdWidgetStub(String description, String remoteId, Role role, String tagName) { + this(createState(), description, remoteId, role, tagName, "text", Rect.fromCoordinates(0, 0, 1, 1)); + } + + public WdWidgetStub(WdState state, String description, String remoteId, Role role, String tagName) { + this(state, description, remoteId, role, tagName, "text", Rect.fromCoordinates(0, 0, 1, 1)); + } + + public WdWidgetStub(WdState state, String description, String remoteId, Role role, String tagName, String type, Rect rect) { + this(state, description, remoteId, role, tagName, type, rect, createRemoteWebElement(remoteId)); + } + + public WdWidgetStub(WdState state, String description, String remoteId, Role role, String tagName, String type, Rect rect, RemoteWebElement remoteWebElement) { + super(state, state, createElement(state, description, remoteId, tagName, type, rect, remoteWebElement)); + setDescription(description); + setRemoteId(remoteId); + setRole(role); + setTagName(tagName); + setType(type); + setRect(rect); + } + + public static WdState createState() { + return new WdState(new WdElement(null, null)); + } + + public WdWidgetStub setDescription(String description) { + this.element.name = description; + this.set(Tags.Desc, description); + return this; + } + + public WdWidgetStub setRemoteId(String remoteId) { + this.element.id = remoteId; + if (this.element.remoteWebElement == null) { + this.element.remoteWebElement = createRemoteWebElement(remoteId); + } else { + this.element.remoteWebElement.setId(remoteId); + } + return this; + } + + public WdWidgetStub setRole(Role role) { + this.set(Tags.Role, role); + return this; + } + + public WdWidgetStub setTagName(String tagName) { + this.element.tagName = tagName; + return this; + } + + public WdWidgetStub setType(String type) { + this.element.type = type; + return this; + } + + public WdWidgetStub setRect(Rect rect) { + this.element.rect = rect; + return this; + } + + public WdWidgetStub setRemoteWebElement(RemoteWebElement remoteWebElement) { + this.element.remoteWebElement = remoteWebElement; + return this; + } + + private static WdElement createElement(WdState state, String description, String remoteId, String tagName, String type, Rect rect, RemoteWebElement remoteWebElement) { + WdElement element = new WdElement(null, state.element); + element.name = description; + element.id = remoteId; + element.tagName = tagName; + element.type = type; + element.rect = rect; + element.remoteWebElement = remoteWebElement; + + return element; + } + + private static RemoteWebElement createRemoteWebElement(String remoteId) { + RemoteWebElement remoteWebElement = new RemoteWebElement(); + remoteWebElement.setId(remoteId); + return remoteWebElement; + } +} From f07f7613146153d5250e5ee308f07f466cdf3fab Mon Sep 17 00:00:00 2001 From: ferpasri Date: Thu, 5 Mar 2026 11:23:38 +0100 Subject: [PATCH 4/4] Update TESTAR version to v2.7.23 --- CHANGELOG | 6 +++++- VERSION | 2 +- testar/src/org/testar/monkey/Main.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3155fc8e8..5fa55bfe6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,8 @@ -#TESTAR v2.7.21 (24-Feb-2026) +#TESTAR v2.7.23 (5-Mar-2026) +- Update webdriver remote actions descriptions +- Add webdriver remote actions to LLM conversation + +#TESTAR v2.7.22 (24-Feb-2026) - Bump org.seleniumhq.selenium:selenium-java from 4.40.0 to 4.41.0 - Update devtools dependencies to v145 - Ignore dynamic numbers when deduplicating AndroidLogcatOracle messages diff --git a/VERSION b/VERSION index af14db8ef..ab20351de 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.7.22 \ No newline at end of file +2.7.23 \ No newline at end of file diff --git a/testar/src/org/testar/monkey/Main.java b/testar/src/org/testar/monkey/Main.java index b3db5f6a7..7dcc69d6d 100644 --- a/testar/src/org/testar/monkey/Main.java +++ b/testar/src/org/testar/monkey/Main.java @@ -64,7 +64,7 @@ public class Main { - public static final String TESTAR_VERSION = "v2.7.22 (24-Feb-2026)"; + public static final String TESTAR_VERSION = "v2.7.23 (5-Mar-2026)"; //public static final String TESTAR_DIR_PROPERTY = "DIRNAME"; //Use the OS environment to obtain TESTAR directory public static final String SETTINGS_FILE = "test.settings";