From 42e405d0cd9209d965eba26a506b9d6567f396fc Mon Sep 17 00:00:00 2001 From: nickiovets Date: Mon, 2 Feb 2026 14:58:30 +0100 Subject: [PATCH 1/4] more required fields for open ai models --- src/ai/tester.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 9bb8ad3..22ec460 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -546,8 +546,8 @@ export class Tester extends TaskAgent implements Agent { const schema = z.object({ assessment: z.string().describe('Short review of current progress toward the main scenario goal'), suggestion: z.string().describe('Specific next action recommendation'), - recommendReset: z.boolean().optional().describe('Recommend reset() if persistent failures suggest navigation issues'), - recommendStop: z.boolean().optional().describe('Recommend stop() if test is fundamentally incompatible or cannot proceed'), + recommendReset: z.boolean().describe('Recommend reset() if persistent failures suggest navigation issues'), + recommendStop: z.boolean().describe('Recommend stop() if test is fundamentally incompatible or cannot proceed'), }); let problemContext = ''; @@ -664,7 +664,7 @@ export class Tester extends TaskAgent implements Agent { const schema = z.object({ summary: z.string().describe('Concise overview of the test findings'), scenarioAchieved: z.boolean().describe('Indicates if the scenario goal appears satisfied'), - recommendation: z.string().optional().describe('Follow-up suggestion if needed'), + recommendation: z.string().describe('Follow-up suggestion if needed'), }); const model = this.provider.getModelForAgent('tester'); From bcf2c4033d58d1f6a212e5fb6f67a0b90c8d00f7 Mon Sep 17 00:00:00 2001 From: nickiovets Date: Mon, 2 Feb 2026 16:43:39 +0100 Subject: [PATCH 2/4] add deep interactive exploration mode for researcher --- src/ai/researcher.ts | 138 +++++++++++++++++++++++++++++++ src/commands/research-command.ts | 4 +- src/config.ts | 7 +- 3 files changed, 147 insertions(+), 2 deletions(-) diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 8ee86d2..9c8dcc0 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -18,6 +18,7 @@ import type { Agent } from './agent.js'; import type { Conversation } from './conversation.js'; import type { Provider } from './provider.js'; import { locatorRule as generalLocatorRuleText, multipleLocatorRule, sectionUiMapRule } from './rules.js'; +import { collectInteractiveNodes, detectFocusArea } from '../utils/aria.ts'; const debugLog = createDebug('explorbot:researcher'); @@ -252,9 +253,146 @@ export class Researcher implements Agent { } ); + const explorationResults = await this.performInteractiveExploration(state); + if (explorationResults) { + additionalResearch += explorationResults; + } + return additionalResearch; } + private async performInteractiveExploration(state: WebPageState): Promise { + debugLog('Starting interactive exploration of page elements'); + tag('step').log('Exploring interactive elements...'); + + const currentState = this.stateManager.getCurrentState(); + if (!currentState?.ariaSnapshot) { + debugLog('No ARIA snapshot available for exploration'); + return ''; + } + + const interactiveNodes = collectInteractiveNodes(currentState.ariaSnapshot); + const clickableRoles = new Set(['button', 'link', 'tab', 'menuitem', 'switch', 'checkbox']); + const defaultExcluded = ['close', 'cancel', 'dismiss', 'x', 'back', 'previous', 'escape', 'exit']; + const configExcluded = ConfigParser.getInstance().getConfig().research?.skipElements || []; + const excludedNames = new Set([...defaultExcluded, ...configExcluded.map((s) => s.toLowerCase())]); + + const targets = interactiveNodes.filter((node) => { + const role = String(node.role || '').toLowerCase(); + if (!clickableRoles.has(role)) return false; + + const name = String(node.name || '') + .toLowerCase() + .trim(); + if (!name) return false; + if (excludedNames.has(name)) return false; + const matchesExcluded = [...excludedNames].some((pattern) => name.includes(pattern)); + if (matchesExcluded) return false; + if (name.length > 50) return false; + + return true; + }); + + if (targets.length === 0) { + debugLog('No clickable elements found for exploration'); + return ''; + } + + debugLog(`Found ${targets.length} elements to explore`); + tag('substep').log(`Found ${targets.length} elements to explore`); + + const results: Array<{ element: string; role: string; result: string }> = []; + const originalUrl = state.url; + const maxElements = Math.min(targets.length, 10); + + for (let i = 0; i < maxElements; i++) { + const target = targets[i]; + const role = String(target.role || ''); + const name = String(target.name || ''); + const locator = JSON.stringify({ role, text: name }); + + tag('substep').log(`Exploring: ${role} "${name}"`); + + const action = this.explorer.createAction(); + const beforeState = await action.capturePageState({}); + + try { + await action.execute(`I.click(${locator})`); + const afterState = await action.capturePageState({}); + + let resultDescription = 'no visible effect'; + + const urlChanged = afterState.url !== beforeState.url; + if (urlChanged) { + resultDescription = `navigates to ${afterState.url}`; + await this.navigateTo(originalUrl); + } else { + const focusArea = detectFocusArea(afterState.ariaSnapshot); + if (focusArea.detected) { + const modalName = focusArea.name ? ` "${focusArea.name}"` : ''; + resultDescription = `opens ${focusArea.type}${modalName}`; + await action.execute("I.pressKey('Escape')"); + } else if (afterState.ariaSnapshot !== beforeState.ariaSnapshot) { + resultDescription = 'reveals/changes content'; + } + } + + results.push({ element: name, role, result: resultDescription }); + } catch (error) { + debugLog(`Failed to click ${name}:`, error); + results.push({ element: name, role, result: 'element not clickable' }); + } + + const postState = this.stateManager.getCurrentState(); + if (postState?.url !== originalUrl) { + await this.navigateTo(originalUrl); + } + } + + if (results.length === 0) { + return ''; + } + + const buttonResults = results.filter((r) => r.role === 'button'); + const linkResults = results.filter((r) => r.role === 'link'); + const otherResults = results.filter((r) => r.role !== 'button' && r.role !== 'link'); + + let output = '\n\n## Interactive Exploration\n\n'; + + if (buttonResults.length > 0) { + output += '### Buttons\n\n'; + output += '| Element | Result |\n'; + output += '|---------|--------|\n'; + for (const r of buttonResults) { + output += `| "${r.element}" | ${r.result} |\n`; + } + output += '\n'; + } + + if (linkResults.length > 0) { + output += '### Links\n\n'; + output += '| Element | Result |\n'; + output += '|---------|--------|\n'; + for (const r of linkResults) { + output += `| "${r.element}" | ${r.result} |\n`; + } + output += '\n'; + } + + if (otherResults.length > 0) { + output += '### Other Elements\n\n'; + output += '| Element | Type | Result |\n'; + output += '|---------|------|--------|\n'; + for (const r of otherResults) { + output += `| "${r.element}" | ${r.role} | ${r.result} |\n`; + } + output += '\n'; + } + + tag('success').log(`Explored ${results.length} elements`); + return output; + } + private async ensureNavigated(url: string, screenshot?: boolean): Promise { if (!this.actionResult) { debugLog('No action result, navigating to URL'); diff --git a/src/commands/research-command.ts b/src/commands/research-command.ts index d0a92d4..db620f2 100644 --- a/src/commands/research-command.ts +++ b/src/commands/research-command.ts @@ -6,7 +6,8 @@ export class ResearchCommand extends BaseCommand { async execute(args: string): Promise { const includeData = args.includes('--data'); - const target = args.replace('--data', '').trim(); + const includeDeep = args.includes('--deep'); + const target = args.replace('--data', '').replace('--deep', '').trim(); if (target) { await this.explorBot.getExplorer().visit(target); @@ -21,6 +22,7 @@ export class ResearchCommand extends BaseCommand { screenshot: true, force: true, data: includeData, + deep: includeDeep, }); } } diff --git a/src/config.ts b/src/config.ts index 0fec6b5..5fa8482 100644 --- a/src/config.ts +++ b/src/config.ts @@ -88,11 +88,16 @@ interface ActionConfig { retries?: number; } +interface ResearchConfig { + skipElements?: string[]; +} + interface ExplorbotConfig { playwright: PlaywrightConfig; ai: AIConfig; html?: HtmlConfig; action?: ActionConfig; + research?: ResearchConfig; dirs?: { knowledge: string; experience: string; @@ -112,7 +117,7 @@ const config: ExplorbotConfig = { }, }; -export type { ExplorbotConfig, PlaywrightConfig, AIConfig, HtmlConfig, ActionConfig, AgentConfig, AgentsConfig }; +export type { ExplorbotConfig, PlaywrightConfig, AIConfig, HtmlConfig, ActionConfig, AgentConfig, AgentsConfig, ResearchConfig }; export class ConfigParser { private static instance: ConfigParser; From bf43441f487456b0fcd02ab96de334d2fffcbbca Mon Sep 17 00:00:00 2001 From: nickiovets Date: Mon, 2 Feb 2026 20:20:46 +0100 Subject: [PATCH 3/4] includeElements researcher config added --- CONFIGURATION.md | 42 ++++++++++++++++++++++++++++++++++++++++ src/ai/researcher.ts | 46 +++++++++++++++++++++++++++++++++----------- src/config.ts | 6 +++++- 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index ba0ba0d..f66e7c4 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -35,6 +35,7 @@ interface ExplorbotConfig { test: TestConfig; html?: HtmlConfig; action?: ActionConfig; + research?: ResearchConfig; dirs?: DirsConfig; } ``` @@ -140,6 +141,47 @@ interface TestConfig { } ``` +### Research Configuration + +Controls the Researcher agent's interactive exploration during `/research --deep`. + +```typescript +interface ResearchConfig { + skipElements?: string[]; + includeElements?: string[]; +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `skipElements` | `string[]` | Patterns or selectors for elements to skip during exploration. | +| `includeElements` | `string[]` | Patterns or selectors for elements to always explore (overrides skipElements). | + +**Pattern types (both arrays support):** +- **Name patterns** - plain text, partial match, case-insensitive (e.g., `'more'`, `'cookie'`) +- **CSS selectors** - starts with `[`, `.`, or `#` (e.g., `'[aria-haspopup="true"]'`, `'.menu-btn'`) +- **XPath selectors** - starts with `//` (e.g., `'//button[@data-action]'`) + +**Recommended starting values:** +- `skipElements`: `['close', 'cancel', 'dismiss', 'x', 'back', 'previous', 'escape', 'exit']` +- `includeElements`: `['more', 'options', 'actions', 'kebab', 'ellipsis', 'dots', 'overflow', '[aria-haspopup="true"]']` + +**Filtering priority:** +1. `includeElements` patterns/selectors always get explored (highest priority) +2. `skipElements` patterns/selectors are never explored + +Example: +```typescript +research: { + skipElements: ['cookie', 'consent', 'newsletter', '.chat-widget'], + includeElements: ['edit', 'delete', 'settings', '[aria-haspopup="true"]', '[data-testid="menu-btn"]'], +} +``` + +**Workflow:** Run `/research --deep`, inspect output, then update config to skip unwanted elements or include missed ones. + +**Implementation:** `src/ai/researcher.ts` (`performInteractiveExploration` method) + ## Example Configuration ```typescript diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 9c8dcc0..d4f45a4 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -273,9 +273,13 @@ export class Researcher implements Agent { const interactiveNodes = collectInteractiveNodes(currentState.ariaSnapshot); const clickableRoles = new Set(['button', 'link', 'tab', 'menuitem', 'switch', 'checkbox']); - const defaultExcluded = ['close', 'cancel', 'dismiss', 'x', 'back', 'previous', 'escape', 'exit']; - const configExcluded = ConfigParser.getInstance().getConfig().research?.skipElements || []; - const excludedNames = new Set([...defaultExcluded, ...configExcluded.map((s) => s.toLowerCase())]); + const researchConfig = ConfigParser.getInstance().getConfig().research!; + + const isSelector = (s: string) => /^[\[.#]|^\/\//.test(s); + const skipPatterns = researchConfig.skipElements!.filter((s) => !isSelector(s)).map((s) => s.toLowerCase()); + const skipSelectors = researchConfig.skipElements!.filter(isSelector); + const includePatterns = researchConfig.includeElements!.filter((s) => !isSelector(s)).map((s) => s.toLowerCase()); + const includeSelectors = researchConfig.includeElements!.filter(isSelector); const targets = interactiveNodes.filter((node) => { const role = String(node.role || '').toLowerCase(); @@ -284,10 +288,17 @@ export class Researcher implements Agent { const name = String(node.name || '') .toLowerCase() .trim(); + const hasPopup = node.haspopup === true || node.haspopup === 'true' || node.haspopup === 'menu'; + + const matchesIncludePattern = name && includePatterns.some((pattern) => name.includes(pattern)); + const matchesIncludeSelector = includeSelectors.length > 0 && hasPopup && role === 'button'; + if (matchesIncludePattern || matchesIncludeSelector) return true; + + const matchesSkipPattern = name && skipPatterns.some((pattern) => name.includes(pattern)); + const matchesSkipSelector = skipSelectors.length > 0 && hasPopup && role === 'button'; + if (matchesSkipPattern || matchesSkipSelector) return false; + if (!name) return false; - if (excludedNames.has(name)) return false; - const matchesExcluded = [...excludedNames].some((pattern) => name.includes(pattern)); - if (matchesExcluded) return false; if (name.length > 50) return false; return true; @@ -309,9 +320,22 @@ export class Researcher implements Agent { const target = targets[i]; const role = String(target.role || ''); const name = String(target.name || ''); - const locator = JSON.stringify({ role, text: name }); + const hasPopup = target.haspopup === true || target.haspopup === 'true' || target.haspopup === 'menu'; + + let locator: string; + let displayName: string; + + if (name) { + locator = JSON.stringify({ role, text: name }); + displayName = name; + } else if (hasPopup && includeSelectors.length > 0) { + locator = `'${includeSelectors[0]}'`; + displayName = '[menu trigger]'; + } else { + continue; + } - tag('substep').log(`Exploring: ${role} "${name}"`); + tag('substep').log(`Exploring: ${role} "${displayName}"`); const action = this.explorer.createAction(); const beforeState = await action.capturePageState({}); @@ -337,10 +361,10 @@ export class Researcher implements Agent { } } - results.push({ element: name, role, result: resultDescription }); + results.push({ element: displayName, role, result: resultDescription }); } catch (error) { - debugLog(`Failed to click ${name}:`, error); - results.push({ element: name, role, result: 'element not clickable' }); + debugLog(`Failed to click ${displayName}:`, error); + results.push({ element: displayName, role, result: 'element not clickable' }); } const postState = this.stateManager.getCurrentState(); diff --git a/src/config.ts b/src/config.ts index 5fa8482..58b4600 100644 --- a/src/config.ts +++ b/src/config.ts @@ -90,6 +90,7 @@ interface ActionConfig { interface ResearchConfig { skipElements?: string[]; + includeElements?: string[]; } interface ExplorbotConfig { @@ -323,12 +324,15 @@ export class ConfigParser { const defaults = { playwright: { browser: 'chromium', - show: false, // we need headless }, action: { delay: 1000, retries: 3, }, + research: { + skipElements: ['close', 'cancel', 'dismiss', 'x', 'back', 'previous', 'escape', 'exit'], + includeElements: ['more', 'options', 'actions', 'kebab', 'ellipsis', 'dots', 'overflow', '[aria-haspopup="true"]'], + }, dirs: { knowledge: 'knowledge', experience: 'experience', From d85be6166aa9b89267e8c40fcae5a541f8830073 Mon Sep 17 00:00:00 2001 From: nickiovets Date: Mon, 2 Feb 2026 20:50:07 +0100 Subject: [PATCH 4/4] remove default values from config and add knowledge to gitignore --- .gitignore | 3 +++ src/config.ts | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6725f99..12e48fd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ out/ # Example folder example/ +# Knowledge (may contain credentials) +knowledge/ + # Output directories output/ output/logs/ diff --git a/src/config.ts b/src/config.ts index 58b4600..609c2a3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -324,14 +324,15 @@ export class ConfigParser { const defaults = { playwright: { browser: 'chromium', + show: false, // we need headless }, action: { delay: 1000, retries: 3, }, research: { - skipElements: ['close', 'cancel', 'dismiss', 'x', 'back', 'previous', 'escape', 'exit'], - includeElements: ['more', 'options', 'actions', 'kebab', 'ellipsis', 'dots', 'overflow', '[aria-haspopup="true"]'], + skipElements: [], + includeElements: [], }, dirs: { knowledge: 'knowledge',