From d0842c3aa56817b79e379bfa7c47cc867321821b Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Fri, 5 Jun 2026 16:12:31 +0200 Subject: [PATCH 1/6] updated workflow --- .github/skills/cli-skill/SKILL.md | 1 + .../workflows/cli-skill-review-reusable.yml | 52 +---- .pi/skills/cli-skill/SKILL.md | 21 -- README.md | 4 +- cli-skill/adapters/README.md | 6 +- cli-skill/adapters/claude-code/commands.yaml | 1 + cli-skill/adapters/copilot/SKILL.md | 1 + cli-skill/adapters/opencode/commands.json | 3 +- cli-skill/adapters/pi-coding-agent/SKILL.md | 1 + cli-skill/schemas/commands.manifest.yaml | 4 + knowledge_base/ARCHITECTURE.md | 2 +- scripts/check-agents.js | 80 ------- scripts/generate-reports.js | 198 ------------------ scripts/run-agents.js | 142 ------------- scripts/watch-and-report.js | 52 ----- 15 files changed, 21 insertions(+), 547 deletions(-) delete mode 100644 .pi/skills/cli-skill/SKILL.md delete mode 100644 scripts/check-agents.js delete mode 100644 scripts/generate-reports.js delete mode 100644 scripts/run-agents.js delete mode 100644 scripts/watch-and-report.js diff --git a/.github/skills/cli-skill/SKILL.md b/.github/skills/cli-skill/SKILL.md index eab69ab..cf7d01a 100644 --- a/.github/skills/cli-skill/SKILL.md +++ b/.github/skills/cli-skill/SKILL.md @@ -20,3 +20,4 @@ This is an adapter entrypoint. The canonical implementation lives in `cli-skill/ - `/cli-semantic-analysis` - `/cli-heuristic-analysis` - `/cli-check-help` +- `/cli-behavioral-analysis` diff --git a/.github/workflows/cli-skill-review-reusable.yml b/.github/workflows/cli-skill-review-reusable.yml index 08b9c5c..266e035 100644 --- a/.github/workflows/cli-skill-review-reusable.yml +++ b/.github/workflows/cli-skill-review-reusable.yml @@ -99,6 +99,11 @@ jobs: - name: Sync cli-skill adapters run: node scripts/sync-cli-skill-adapters.js + - name: Install cli-skill for Pi discovery + run: | + mkdir -p ~/.pi/skills + cp -R cli-skill ~/.pi/skills/ + - name: Run Pi agent id: pi uses: shaftoe/pi-coding-agent-action@v2 @@ -190,53 +195,6 @@ jobs: print(f"Wrote {p}") PY - - name: Upsert PR comment - if: ${{ inputs.post_pr_comment && inputs.pr_number > 0 }} - uses: actions/github-script@v7 - env: - PR_NUMBER: ${{ inputs.pr_number }} - REPORT_PATH: pi-report-comment.md - MARKER: "" - with: - github-token: ${{ secrets.gh_token }} - script: | - const fs = require('node:fs'); - - const owner = context.repo.owner; - const repo = context.repo.repo; - const issue_number = Number(process.env.PR_NUMBER); - const marker = process.env.MARKER; - const body = fs.readFileSync(process.env.REPORT_PATH, 'utf8'); - - const comments = await github.paginate(github.rest.issues.listComments, { - owner, - repo, - issue_number, - per_page: 100, - }); - - const existing = comments.find((c) => - c.user?.type === 'Bot' && typeof c.body === 'string' && c.body.includes(marker) - ); - - if (existing) { - await github.rest.issues.updateComment({ - owner, - repo, - comment_id: existing.id, - body, - }); - core.info(`Updated existing report comment: ${existing.id}`); - } else { - const created = await github.rest.issues.createComment({ - owner, - repo, - issue_number, - body, - }); - core.info(`Created report comment: ${created.data.id}`); - } - - name: Fail when agent run is unsuccessful if: ${{ inputs.fail_on_agent_error && steps.pi.outputs.success != 'true' }} run: | diff --git a/.pi/skills/cli-skill/SKILL.md b/.pi/skills/cli-skill/SKILL.md deleted file mode 100644 index cf8e063..0000000 --- a/.pi/skills/cli-skill/SKILL.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: cli-skill -description: "Pi entrypoint for cross-agent CLI command workflows." ---- - -# Entrypoint - -Canonical files are in `cli-skill/`. - -## Dispatch - -- Manifest: `cli-skill/schemas/commands.manifest.yaml` -- Shared preflight: `cli-skill/shared/cli-discovery-preflight.md` -- Commands: `cli-skill/commands/*.md` - -## Supported Commands - -- `/cli-review` -- `/cli-semantic-analysis` -- `/cli-heuristic-analysis` -- `/cli-check-help` diff --git a/README.md b/README.md index 773f054..67ec86e 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Future scaffolds: ## Use with Different Agents -1. Sync adapters from the manifest: +1. Sync adapters from the manifest (single maintained helper script): node scripts/sync-cli-skill-adapters.js @@ -68,7 +68,7 @@ On pull requests (opened, synchronize, reopened, ready_for_review), the workflow - command: /cli-review - provider: openrouter -- model: gpt-5.4 +- model: moonshotai/kimi-k2.6 It publishes a markdown report in PR comments and updates the same report on subsequent PR updates. diff --git a/cli-skill/adapters/README.md b/cli-skill/adapters/README.md index 2430daf..64df225 100644 --- a/cli-skill/adapters/README.md +++ b/cli-skill/adapters/README.md @@ -28,10 +28,10 @@ These are scaffolds for portability. If a specific runtime needs strict schema f ## Regeneration -When command mappings change, regenerate all adapter files from the canonical manifest: +When command mappings change, regenerate all adapter files from the canonical manifest using the single maintained helper script: ```bash -node /project/scripts/sync-cli-skill-adapters.js +node scripts/sync-cli-skill-adapters.js ``` Source of truth: @@ -46,7 +46,7 @@ A pre-commit hook is available at: The hook: -1. Runs `node /project/scripts/sync-cli-skill-adapters.js` +1. Runs `node scripts/sync-cli-skill-adapters.js` 2. Fails the commit if adapter files changed during sync 3. Prompts you to stage regenerated adapter files before retrying commit diff --git a/cli-skill/adapters/claude-code/commands.yaml b/cli-skill/adapters/claude-code/commands.yaml index 6bf4806..2a50e51 100644 --- a/cli-skill/adapters/claude-code/commands.yaml +++ b/cli-skill/adapters/claude-code/commands.yaml @@ -11,6 +11,7 @@ routes: /cli-semantic-analysis: cli-skill/commands/cli-semantic-analysis.md /cli-heuristic-analysis: cli-skill/commands/cli-heuristic-analysis.md /cli-check-help: cli-skill/commands/cli-check-help.md + /cli-behavioral-analysis: cli-skill/commands/cli-behavioral-analysis.md future: /cli-propose-command: cli-skill/future-commands/cli-propose-command.md diff --git a/cli-skill/adapters/copilot/SKILL.md b/cli-skill/adapters/copilot/SKILL.md index bdbb1f7..84f85ff 100644 --- a/cli-skill/adapters/copilot/SKILL.md +++ b/cli-skill/adapters/copilot/SKILL.md @@ -20,6 +20,7 @@ This adapter maps slash-style command intents to command files in `cli-skill/`. - `/cli-semantic-analysis` -> `cli-skill/commands/cli-semantic-analysis.md` - `/cli-heuristic-analysis` -> `cli-skill/commands/cli-heuristic-analysis.md` - `/cli-check-help` -> `cli-skill/commands/cli-check-help.md` +- `/cli-behavioral-analysis` -> `cli-skill/commands/cli-behavioral-analysis.md` ## Future Command Stubs diff --git a/cli-skill/adapters/opencode/commands.json b/cli-skill/adapters/opencode/commands.json index 9ab9dfb..75579e6 100644 --- a/cli-skill/adapters/opencode/commands.json +++ b/cli-skill/adapters/opencode/commands.json @@ -8,7 +8,8 @@ "/cli-review": "cli-skill/commands/cli-review.md", "/cli-semantic-analysis": "cli-skill/commands/cli-semantic-analysis.md", "/cli-heuristic-analysis": "cli-skill/commands/cli-heuristic-analysis.md", - "/cli-check-help": "cli-skill/commands/cli-check-help.md" + "/cli-check-help": "cli-skill/commands/cli-check-help.md", + "/cli-behavioral-analysis": "cli-skill/commands/cli-behavioral-analysis.md" }, "future": { "/cli-propose-command": "cli-skill/future-commands/cli-propose-command.md", diff --git a/cli-skill/adapters/pi-coding-agent/SKILL.md b/cli-skill/adapters/pi-coding-agent/SKILL.md index d9ea4c6..6741cd4 100644 --- a/cli-skill/adapters/pi-coding-agent/SKILL.md +++ b/cli-skill/adapters/pi-coding-agent/SKILL.md @@ -19,3 +19,4 @@ This adapter is intentionally thin and delegates all behavior to files under `cl - `/cli-semantic-analysis` -> `cli-skill/commands/cli-semantic-analysis.md` - `/cli-heuristic-analysis` -> `cli-skill/commands/cli-heuristic-analysis.md` - `/cli-check-help` -> `cli-skill/commands/cli-check-help.md` +- `/cli-behavioral-analysis` -> `cli-skill/commands/cli-behavioral-analysis.md` diff --git a/cli-skill/schemas/commands.manifest.yaml b/cli-skill/schemas/commands.manifest.yaml index fe8343d..642b007 100644 --- a/cli-skill/schemas/commands.manifest.yaml +++ b/cli-skill/schemas/commands.manifest.yaml @@ -24,6 +24,10 @@ commands: file: cli-skill/commands/cli-check-help.md stage: active description: Help-system focused CLI audit. + - name: /cli-behavioral-analysis + file: cli-skill/commands/cli-behavioral-analysis.md + stage: active + description: Analyze runtime behavior, output contracts, errors, and safety properties. futureCommands: - name: /cli-propose-command diff --git a/knowledge_base/ARCHITECTURE.md b/knowledge_base/ARCHITECTURE.md index ebc0d25..9378566 100644 --- a/knowledge_base/ARCHITECTURE.md +++ b/knowledge_base/ARCHITECTURE.md @@ -40,5 +40,5 @@ - Reusable workflow uses `gh_token` (not `github_token`) for `workflow_call` secret interface to avoid reserved-name collision. 11. Repository-root path policy for scripts -- Operational scripts resolve repository root at runtime via git (`rev-parse --show-toplevel`) with fallback to script-relative root. +- The maintained helper script resolves repository root at runtime via git (`rev-parse --show-toplevel`) with fallback to script-relative root. - Hard-coded absolute workspace paths are intentionally avoided. diff --git a/scripts/check-agents.js b/scripts/check-agents.js deleted file mode 100644 index 540cd8a..0000000 --- a/scripts/check-agents.js +++ /dev/null @@ -1,80 +0,0 @@ -const fs = require('node:fs'); -const path = require('node:path'); -const readline = require('node:readline'); -const { execSync } = require('node:child_process'); - -function getRepoRoot() { - try { - return execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim(); - } catch { - return path.resolve(__dirname, '..'); - } -} - -const ROOT = getRepoRoot(); -const AGENTS_ROOT = path.join(ROOT, 'agents'); - -const AGENTS = [ - 'kimi-k2.6-juju', 'glm-5-juju', 'deepseek-v4-pro-juju', - 'kimi-k2.6-qwen36-snap', 'glm-5-qwen36-snap', 'deepseek-v4-pro-qwen36-snap', -]; - -function streamLines(filePath) { - return new Promise((resolve) => { - if (!fs.existsSync(filePath)) { resolve(0); return; } - let count = 0; - const rl = readline.createInterface({ input: fs.createReadStream(filePath), crlfDelay: Infinity }); - rl.on('line', () => count++); - rl.on('close', () => resolve(count)); - rl.on('error', () => resolve(0)); - }); -} - -async function streamStats(agentName) { - const dir = path.join(AGENTS_ROOT, agentName); - const sessionFile = path.join(dir, 'session.jsonl'); - const analysisDir = path.join(dir, '0-analysis'); - const designDir = path.join(dir, '1-command-design'); - - const sessionLines = await streamLines(sessionFile); - const analysisFiles = fs.existsSync(analysisDir) ? fs.readdirSync(analysisDir).filter(f => f.endsWith('.md')).length : 0; - const designFiles = fs.existsSync(designDir) ? fs.readdirSync(designDir).filter(f => f.endsWith('.md')).length : 0; - - const orchestratorPath = path.join(AGENTS_ROOT, 'orchestrator.log'); - const orchestratorLog = fs.existsSync(orchestratorPath) ? fs.readFileSync(orchestratorPath, 'utf-8') : ''; - const isDone = orchestratorLog.includes(`[DONE] ${agentName}`); - - let toolCalls = 0; - let toolErrors = 0; - - if (fs.existsSync(sessionFile)) { - const rl = readline.createInterface({ input: fs.createReadStream(sessionFile), crlfDelay: Infinity }); - for await (const line of rl) { - if (!line.trim()) continue; - try { - const ev = JSON.parse(line); - if (ev.type === 'tool_execution_start') toolCalls++; - if (ev.type === 'tool_execution_end' && ev.isError) toolErrors++; - } catch {} - } - } - - return { agentName, sessionLines, analysisFiles, designFiles, isDone, toolCalls, toolErrors }; -} - -async function status() { - console.log('=== Agent Status ===\n'); - let done = 0; - let running = 0; - for (const name of AGENTS) { - const r = await streamStats(name); - const icon = r.isDone ? '✅' : '🔄'; - console.log(`${icon} ${r.agentName}`); - console.log(` events: ${r.sessionLines.toLocaleString()} | tools: ${r.toolCalls} | errors: ${r.toolErrors}`); - console.log(` 0-analysis: ${r.analysisFiles} files | 1-command-design: ${r.designFiles} files`); - if (r.isDone) done++; else running++; - } - console.log(`\nDone: ${done} | Running: ${running}`); -} - -status().catch(console.error); diff --git a/scripts/generate-reports.js b/scripts/generate-reports.js deleted file mode 100644 index 28684fe..0000000 --- a/scripts/generate-reports.js +++ /dev/null @@ -1,198 +0,0 @@ -const fs = require('node:fs'); -const path = require('node:path'); -const readline = require('node:readline'); -const { execSync } = require('node:child_process'); - -function getRepoRoot() { - try { - return execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim(); - } catch { - return path.resolve(__dirname, '..'); - } -} - -const ROOT = getRepoRoot(); -const AGENTS_ROOT = path.join(ROOT, 'agents'); - -const AGENTS = [ - 'kimi-k2.6-juju', 'glm-5-juju', 'deepseek-v4-pro-juju', - 'kimi-k2.6-qwen36-snap', 'glm-5-qwen36-snap', 'deepseek-v4-pro-qwen36-snap', -]; - -async function streamStats(agentName) { - const dir = path.join(AGENTS_ROOT, agentName); - const sessionFile = path.join(dir, 'session.jsonl'); - const analysisDir = path.join(dir, '0-analysis'); - const designDir = path.join(dir, '1-command-design'); - - let toolCalls = 0; - let toolErrors = 0; - let events = 0; - let lastUsage = null; - - if (fs.existsSync(sessionFile)) { - const rl = readline.createInterface({ input: fs.createReadStream(sessionFile), crlfDelay: Infinity }); - for await (const line of rl) { - if (!line.trim()) continue; - events++; - try { - const ev = JSON.parse(line); - if (ev.type === 'tool_execution_start') toolCalls++; - if (ev.type === 'tool_execution_end' && ev.isError) toolErrors++; - if (ev.type === 'message_end' && ev.message?.role === 'assistant' && ev.message?.usage) { - lastUsage = ev.message.usage; - } - } catch {} - } - } - - const analysisFiles = fs.existsSync(analysisDir) ? fs.readdirSync(analysisDir).filter(f => f.endsWith('.md')) : []; - const designFiles = fs.existsSync(designDir) ? fs.readdirSync(designDir).filter(f => f.endsWith('.md')) : []; - - const orchestratorPath = path.join(AGENTS_ROOT, 'orchestrator.log'); - const orchestratorLog = fs.existsSync(orchestratorPath) ? fs.readFileSync(orchestratorPath, 'utf-8') : ''; - const isDone = orchestratorLog.includes(`[DONE] ${agentName}`); - - return { agentName, events, toolCalls, toolErrors, analysisFiles, designFiles, isDone, lastUsage }; -} - -async function generateFeedback() { - const results = []; - for (const name of AGENTS) results.push(await streamStats(name)); - - let md = '# Agent Interview Protocol\n\n'; - md += 'Interviews conducted with each sub-agent after completing the cli-review skill workflows.\n\n'; - md += '---\n\n'; - - for (const r of results) { - md += `## ${r.agentName}\n\n`; - - md += '### 1. Agent Profile\n\n'; - md += '- **Knowledge Level**: Autonomous LLM agent with full tool access (read, bash, edit, write).\n'; - md += '- **Product Familiarity**: First exposure to the target project; no prior context.\n'; - md += '- **Context**: Stateless subprocess with isolated context window.\n\n'; - - md += '### 2. Target Tasks\n\n'; - md += '- **Core Goal**: Execute analyze-cli followed by discuss-commandset from the cli-review skill.\n'; - md += '- **Happy Path**: Read project files → infer CLI structure → write 0-analysis/*.md → write 1-command-design/*.md.\n'; - const phase1 = r.analysisFiles.length >= 9 ? 'completed' : r.analysisFiles.length > 0 ? 'partial' : 'incomplete'; - const phase2 = r.designFiles.length > 0 ? 'completed' : 'incomplete'; - md += `- **Sequence**: Analysis phase ${phase1}, design phase ${phase2}.\n\n`; - - md += '### 3. Four Core Questions\n\n'; - - md += '#### Intent\n\n'; - if (r.analysisFiles.length >= 9) { - md += '✅ The agent correctly identified the need to produce all required analysis files before proceeding to commandset discussion.\n\n'; - } else if (r.analysisFiles.length > 0) { - md += `⚠️ The agent produced ${r.analysisFiles.length} analysis files. May have skipped some or not yet finished.\n\n`; - } else { - md += '❌ The agent produced no analysis files. It may be stuck in exploration or failed to locate the project CLI surface.\n\n'; - } - - md += '#### Visibility\n\n'; - if (r.toolCalls > 0) { - md += `✅ The agent made ${r.toolCalls} tool calls, indicating it could see and use the available actions.\n\n`; - } else { - md += '❌ The agent made zero tool calls. It may not have recognized the tool surface or encountered an initialization error.\n\n'; - } - - md += '#### Matching\n\n'; - md += '✅ The agent used bash and read tools to explore the project, connecting skill instructions to physical files.\n\n'; - - md += '#### Feedback\n\n'; - if (r.isDone) { - md += `✅ Agent exited with completion status. Produced ${r.analysisFiles.length} analysis files and ${r.designFiles.length} design files.\n\n`; - } else { - md += '⚠️ Agent process exited without clear completion signal in orchestrator log. Output files were still produced.\n\n'; - } - - md += '### 4. Problems and Fixes\n\n'; - if (r.toolErrors > 0) { - md += `- **Medium friction**: ${r.toolErrors} tool execution errors encountered during the run.\n`; - } - if (r.designFiles.length === 0) { - md += '- **High friction**: No commandset design files produced. The agent may have stopped after analysis phase.\n'; - } - if (r.events > 20000) { - md += '- **Low friction**: Very high event count suggests extensive exploration, which may be inefficient for structured tasks.\n'; - } - md += '\n'; - } - - const feedbackPath = path.join(AGENTS_ROOT, 'feedback.md'); - fs.writeFileSync(feedbackPath, md); - console.log(`Wrote ${feedbackPath}`); -} - -async function generateInsights() { - const results = []; - for (const name of AGENTS) results.push(await streamStats(name)); - - const done = results.filter(r => r.isDone); - const withAnalysis = results.filter(r => r.analysisFiles.length >= 5); - const withDesign = results.filter(r => r.designFiles.length >= 1); - const totalToolCalls = results.reduce((s, r) => s + r.toolCalls, 0); - const totalErrors = results.reduce((s, r) => s + r.toolErrors, 0); - - let md = '# Agent Swarm Insights\n\n'; - md += `Generated from ${results.length} parallel agent runs across 2 projects and 3 models.\n\n`; - md += '---\n\n'; - - md += '## Aggregate Metrics\n\n'; - md += `| Metric | Value |\n|---|---|\n`; - md += `| Agents launched | ${results.length} |\n`; - md += `| Agents with orchestrator completion | ${done.length} |\n`; - md += `| Agents with substantial analysis | ${withAnalysis.length} |\n`; - md += `| Agents with commandset design | ${withDesign.length} |\n`; - md += `| Total tool calls | ${totalToolCalls} |\n`; - md += `| Total tool errors | ${totalErrors} |\n`; - md += `| Error rate | ${totalToolCalls > 0 ? ((totalErrors / totalToolCalls) * 100).toFixed(1) : 0}% |\n\n`; - - md += '## Model Comparison\n\n'; - const byModel = {}; - for (const r of results) { - const parts = r.agentName.split('-'); - const model = parts.slice(0, -1).join('-'); - if (!byModel[model]) byModel[model] = []; - byModel[model].push(r); - } - for (const [model, runs] of Object.entries(byModel)) { - const avgTools = runs.reduce((s, r) => s + r.toolCalls, 0) / runs.length; - const avgFiles = runs.reduce((s, r) => s + r.analysisFiles.length, 0) / runs.length; - const completed = runs.filter(r => r.isDone).length; - const avgEvents = runs.reduce((s, r) => s + r.events, 0) / runs.length; - md += `### ${model}\n\n`; - md += `- Runs: ${runs.length}\n`; - md += `- Orchestrator-completed: ${completed}\n`; - md += `- Avg tool calls: ${avgTools.toFixed(0)}\n`; - md += `- Avg analysis files: ${avgFiles.toFixed(1)}\n`; - md += `- Avg events: ${avgEvents.toFixed(0)}\n\n`; - } - - md += '## Key Findings\n\n'; - md += '1. **All agents produced analysis output**: Every agent wrote at least some 0-analysis files, indicating the skill instructions were sufficiently clear for project exploration.\n\n'; - md += '2. **GLM-5 most efficient**: GLM-5 agents had the lowest event counts while still completing both phases, suggesting more focused exploration.\n\n'; - md += '3. **Kimi K2.6 most verbose**: kimi-k2.6-juju generated over 24,000 events (2GB session log), indicating extensive but potentially unfocused exploration.\n\n'; - md += '4. **DeepSeek V4 Pro inconsistent**: deepseek-v4-pro-juju stalled in analysis phase initially but eventually completed; deepseek-v4-pro-qwen36-snap produced the most design files (6).\n\n'; - md += '5. **Orchestrator reliability issues**: 3 of 6 exit events were missed by the Node.js orchestrator due to large stdout streams. This is a technical limitation, not a model issue.\n\n'; - - md += '## Recommendations for the Skill\n\n'; - md += '1. **Chunk the workflow**: Split analyze-cli into smaller sub-tasks to prevent context overflow and excessive exploration.\n\n'; - md += '2. **Add file-count validation**: Instruct agents to verify they have written all required files before declaring completion.\n\n'; - md += '3. **Provide project size hints**: Large projects like juju should include a "known entry points" list to reduce exploratory tool calls.\n\n'; - md += '4. **Model-specific tuning**: Kimi K2.6 benefits from tighter step-by-step constraints; GLM-5 works well with high-level goals.\n\n'; - - const insightsPath = path.join(AGENTS_ROOT, 'insights.md'); - fs.writeFileSync(insightsPath, md); - console.log(`Wrote ${insightsPath}`); -} - -async function main() { - console.log('Generating reports from current agent state...'); - await generateFeedback(); - await generateInsights(); - console.log('Done.'); -} - -main().catch(console.error); diff --git a/scripts/run-agents.js b/scripts/run-agents.js deleted file mode 100644 index 839655b..0000000 --- a/scripts/run-agents.js +++ /dev/null @@ -1,142 +0,0 @@ -const { spawn } = require('node:child_process'); -const fs = require('node:fs'); -const path = require('node:path'); -const os = require('node:os'); -const { execSync } = require('node:child_process'); - -function getRepoRoot() { - try { - return execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim(); - } catch { - return path.resolve(__dirname, '..'); - } -} - -const ROOT = getRepoRoot(); - -const PROJECTS = [ - { name: 'juju', dir: path.join(ROOT, 'juju') }, - { name: 'qwen36-snap', dir: path.join(ROOT, 'qwen36-snap') }, -]; - -const MODELS = [ - { name: 'kimi-k2.6', modelId: 'moonshotai/kimi-k2.6' }, - { name: 'glm-5', modelId: 'z-ai/glm-5' }, - { name: 'deepseek-v4-pro', modelId: 'deepseek/deepseek-v4-pro' }, -]; - -const AGENTS_DIR = path.join(ROOT, 'agents'); - -dirname = path.dirname; - -// Ensure skill is available in each project -PROJECTS.forEach(p => { - const skillDir = path.join(p.dir, '.pi', 'skills', 'cli-review'); - const skillSrc = path.join(ROOT, '.github', 'skills', 'cli-review'); - if (!fs.existsSync(skillDir)) { - fs.mkdirSync(skillDir, { recursive: true }); - fs.copyFileSync(path.join(skillSrc, 'SKILL.md'), path.join(skillDir, 'SKILL.md')); - fs.mkdirSync(path.join(skillDir, 'standard'), { recursive: true }); - fs.copyFileSync(path.join(skillSrc, 'standard', 'README.md'), path.join(skillDir, 'standard', 'README.md')); - fs.mkdirSync(path.join(skillDir, 'deprecation'), { recursive: true }); - fs.copyFileSync(path.join(skillSrc, 'deprecation', 'README.md'), path.join(skillDir, 'deprecation', 'README.md')); - } -}); - -// Create agent output directories and run -const agents = []; -for (const proj of PROJECTS) { - for (const mod of MODELS) { - const agentName = `${mod.name}-${proj.name}`; - const agentDir = path.join(AGENTS_DIR, agentName); - fs.mkdirSync(agentDir, { recursive: true }); - fs.mkdirSync(path.join(agentDir, '0-analysis'), { recursive: true }); - fs.mkdirSync(path.join(agentDir, '1-command-design'), { recursive: true }); - - const sysPrompt = `You are an autonomous CLI analysis agent named "${agentName}".\n\nYour task:\n1. Run the cli-review skill workflow for the project at ${proj.dir}.\n2. Execute analyze-cli in THREE phases. Complete each phase fully before starting the next:\n - Phase 1 (Structure Discovery): Write architecture.md, commandset.md, argument-structure.md to 0-analysis/. Verify all 3 exist before continuing.\n - Phase 2 (Behavioral Analysis): Re-read your Phase 1 files, then write configuration-model.md, output-contracts.md, error-model-and-exit-codes.md, safety-model.md. Verify all 4 exist before continuing.\n - Phase 3 (Meta-Analysis): Re-read your Phase 1+2 files, then write extensibility-model.md, documentation-quality-gaps.md. Verify all 9 files exist.\n3. Then, execute discuss-commandset: create a 1-command-design/commandset-shape.md with all six sections.\n4. Write ALL outputs to your agent directory: ${agentDir}/\n5. Do NOT reuse or reference any pre-existing analysis files. Do your own independent work.\n6. Be thorough and complete all sections.\n\nThe cli-review skill has been copied into this project at .pi/skills/cli-review/. It will be auto-discovered.\n`; - - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pi-agent-')); - const promptFile = path.join(tmpDir, 'prompt.md'); - fs.writeFileSync(promptFile, sysPrompt, { mode: 0o600 }); - - const args = [ - '--provider', 'openrouter', - '--mode', 'json', - '-p', - '--no-session', - '--model', mod.modelId, - '--append-system-prompt', promptFile, - `Run cli-review for ${proj.name}. Create all analysis and commandset files in ${agentDir}/. Do independent fresh work.`, - ]; - - agents.push({ - name: agentName, - dir: agentDir, - tmpDir, - promptFile, - project: proj, - model: mod, - args, - process: null, - exitCode: null, - stdout: '', - stderr: '', - }); - } -} - -function runAgent(agent) { - return new Promise((resolve) => { - console.log(`[START] ${agent.name} -> ${agent.dir}`); - const sessionPath = path.join(agent.dir, 'session.jsonl'); - const stdoutStream = fs.createWriteStream(sessionPath); - const proc = spawn('pi', agent.args, { - cwd: agent.project.dir, - shell: false, - stdio: ['ignore', 'pipe', 'pipe'], - }); - agent.process = proc; - proc.stdout.pipe(stdoutStream); - - proc.stderr.on('data', (data) => { - agent.stderr += data.toString(); - }); - proc.on('close', (code) => { - stdoutStream.end(); - agent.exitCode = code ?? 0; - fs.writeFileSync(path.join(agent.dir, 'stderr.log'), agent.stderr, 'utf-8'); - // Cleanup temp - try { fs.unlinkSync(agent.promptFile); } catch {} - try { fs.rmdirSync(agent.tmpDir); } catch {} - console.log(`[DONE] ${agent.name} exit=${agent.exitCode}`); - resolve(); - }); - proc.on('error', (err) => { - stdoutStream.end(); - agent.exitCode = 1; - fs.writeFileSync(path.join(agent.dir, 'stderr.log'), agent.stderr + '\n' + err.message, 'utf-8'); - try { fs.unlinkSync(agent.promptFile); } catch {} - try { fs.rmdirSync(agent.tmpDir); } catch {} - console.log(`[ERROR] ${agent.name} ${err.message}`); - resolve(); - }); - }); -} - -async function main() { - console.log(`Spawning ${agents.length} agents...`); - await Promise.all(agents.map(runAgent)); - - // Summary - console.log('\n=== SUMMARY ==='); - for (const a of agents) { - const hasOutput = fs.existsSync(path.join(a.dir, '0-analysis')) && - fs.readdirSync(path.join(a.dir, '0-analysis')).length > 0; - console.log(`${a.name}: exit=${a.exitCode}, produced_files=${hasOutput ? 'yes' : 'no'}`); - } -} - -main().catch(err => { - console.error('Orchestrator failed:', err); - process.exit(1); -}); diff --git a/scripts/watch-and-report.js b/scripts/watch-and-report.js deleted file mode 100644 index 5a17df9..0000000 --- a/scripts/watch-and-report.js +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env node -const fs = require('node:fs'); -const path = require('node:path'); -const { spawn } = require('node:child_process'); -const { execSync } = require('node:child_process'); - -function getRepoRoot() { - try { - return execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim(); - } catch { - return path.resolve(__dirname, '..'); - } -} - -const ROOT = getRepoRoot(); -const AGENTS_ROOT = path.join(ROOT, 'agents'); -const GENERATE_REPORTS_SCRIPT = path.join(ROOT, 'scripts', 'generate-reports.js'); - -const AGENTS = [ - 'kimi-k2.6-juju', 'glm-5-juju', 'deepseek-v4-pro-juju', - 'kimi-k2.6-qwen36-snap', 'glm-5-qwen36-snap', 'deepseek-v4-pro-qwen36-snap', -]; - -function isDone(agentName) { - const orchestratorPath = path.join(AGENTS_ROOT, 'orchestrator.log'); - const log = fs.existsSync(orchestratorPath) ? fs.readFileSync(orchestratorPath, 'utf-8') : ''; - return log.includes(`[DONE] ${agentName}`); -} - -let lastDoneCount = 0; - -function poll() { - const done = AGENTS.filter(isDone); - if (done.length > lastDoneCount) { - lastDoneCount = done.length; - console.log(`[${new Date().toISOString()}] ${done.length}/${AGENTS.length} agents done. Regenerating reports...`); - const child = spawn('node', [GENERATE_REPORTS_SCRIPT], { - stdio: ['ignore', 'pipe', 'pipe'], - }); - child.stdout.on('data', d => process.stdout.write(d)); - child.stderr.on('data', d => process.stderr.write(d)); - } - if (done.length < AGENTS.length) { - setTimeout(poll, 30000); // check every 30s - } else { - console.log(`[${new Date().toISOString()}] All agents complete. Final reports generated.`); - process.exit(0); - } -} - -console.log('Watcher started. Checking every 30 seconds...'); -poll(); From 186cfaa9f2f64e471bec3cd52bde750708afadd3 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Fri, 5 Jun 2026 16:17:01 +0200 Subject: [PATCH 2/6] updated workflow --- cli-skill/shared/cli-discovery-preflight.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli-skill/shared/cli-discovery-preflight.md b/cli-skill/shared/cli-discovery-preflight.md index 69921dc..584b0cc 100644 --- a/cli-skill/shared/cli-discovery-preflight.md +++ b/cli-skill/shared/cli-discovery-preflight.md @@ -30,7 +30,7 @@ Use multiple signals. Positive matches increase confidence: ### Failure Policy -- If no CLI is found, stop the analysis here. Do not write any files, and return the result "No CLI present. No CLI review necessary". +- If no CLI is found, stop the analysis here. Do not write any files, and return the result "No CLI present. No CLI review necessary". Do not try harder. The contents of a scripts/ directory are NOT considered CLIs. Just stop here. ## Phase 0: Structure Discovery **Goal: Map the CLI surface area — architecture, commands, and arguments.** From b21f563ad2b53578aebffa8507eafa7c5e930427 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Fri, 5 Jun 2026 16:20:22 +0200 Subject: [PATCH 3/6] looking for clis --- cli-skill/shared/cli-discovery-preflight.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli-skill/shared/cli-discovery-preflight.md b/cli-skill/shared/cli-discovery-preflight.md index 584b0cc..ea17ff6 100644 --- a/cli-skill/shared/cli-discovery-preflight.md +++ b/cli-skill/shared/cli-discovery-preflight.md @@ -4,7 +4,8 @@ Run this module before any analysis command. ## Purpose -- Identify whether the current repository contains a CLI +- Identify whether the current repository contains a CLI - the skill and simple scripts are NOT considered CLIs. +- If there is no CLI present, stop and return "No CLI found." - Determine which files should be parsed for command discovery and behavior analysis - Produce interim discovery artifacts for downstream commands @@ -30,7 +31,7 @@ Use multiple signals. Positive matches increase confidence: ### Failure Policy -- If no CLI is found, stop the analysis here. Do not write any files, and return the result "No CLI present. No CLI review necessary". Do not try harder. The contents of a scripts/ directory are NOT considered CLIs. Just stop here. +- If no CLI is found, stop the analysis here. Do not write any files, and return the result "No CLI found." Do not try harder. The contents of a scripts/ directory are NOT considered CLIs. Just stop here. ## Phase 0: Structure Discovery **Goal: Map the CLI surface area — architecture, commands, and arguments.** From a1c510d5798116180297e5080b4f75349a9cb473 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Wed, 10 Jun 2026 14:10:56 +0200 Subject: [PATCH 4/6] small gh action improvement --- .pi/skills/cli-skill/SKILL.md | 22 +++++++++++++++++++++ cli-skill/shared/cli-discovery-preflight.md | 6 +----- knowledge_base/CLI.md | 1 + 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 .pi/skills/cli-skill/SKILL.md create mode 100644 knowledge_base/CLI.md diff --git a/.pi/skills/cli-skill/SKILL.md b/.pi/skills/cli-skill/SKILL.md new file mode 100644 index 0000000..e6af09a --- /dev/null +++ b/.pi/skills/cli-skill/SKILL.md @@ -0,0 +1,22 @@ +--- +name: cli-skill +description: "Pi entrypoint for cross-agent CLI command workflows." +--- + +# Entrypoint + +Canonical files are in `cli-skill/`. + +## Dispatch + +- Manifest: `cli-skill/schemas/commands.manifest.yaml` +- Shared preflight: `cli-skill/shared/cli-discovery-preflight.md` +- Commands: `cli-skill/commands/*.md` + +## Supported Commands + +- `/cli-review` +- `/cli-semantic-analysis` +- `/cli-heuristic-analysis` +- `/cli-check-help` +- `/cli-behavioral-analysis` diff --git a/cli-skill/shared/cli-discovery-preflight.md b/cli-skill/shared/cli-discovery-preflight.md index ea17ff6..6d85bc0 100644 --- a/cli-skill/shared/cli-discovery-preflight.md +++ b/cli-skill/shared/cli-discovery-preflight.md @@ -5,7 +5,7 @@ Run this module before any analysis command. ## Purpose - Identify whether the current repository contains a CLI - the skill and simple scripts are NOT considered CLIs. -- If there is no CLI present, stop and return "No CLI found." +- If there is no CLI present, stop and return "No CLI found." This is an expected, valid, useful output for the user triggering this skill. - Determine which files should be parsed for command discovery and behavior analysis - Produce interim discovery artifacts for downstream commands @@ -29,10 +29,6 @@ Use multiple signals. Positive matches increase confidence: - README usage blocks and examples - Packaging metadata referencing command binaries -### Failure Policy - -- If no CLI is found, stop the analysis here. Do not write any files, and return the result "No CLI found." Do not try harder. The contents of a scripts/ directory are NOT considered CLIs. Just stop here. - ## Phase 0: Structure Discovery **Goal: Map the CLI surface area — architecture, commands, and arguments.** diff --git a/knowledge_base/CLI.md b/knowledge_base/CLI.md new file mode 100644 index 0000000..2d4b44b --- /dev/null +++ b/knowledge_base/CLI.md @@ -0,0 +1 @@ +This project has no CLI. \ No newline at end of file From af703b3633ca7a0344a1ebaf7077ec93e184df13 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Wed, 10 Jun 2026 14:28:42 +0200 Subject: [PATCH 5/6] small gh action improvement --- .../workflows/cli-skill-review-reusable.yml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-skill-review-reusable.yml b/.github/workflows/cli-skill-review-reusable.yml index 266e035..a6e2f0f 100644 --- a/.github/workflows/cli-skill-review-reusable.yml +++ b/.github/workflows/cli-skill-review-reusable.yml @@ -66,12 +66,23 @@ jobs: with: fetch-depth: 0 + - name: Check CLI metadata and changed paths + id: cli_scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + python3 scripts/check-cli-scope.py + - name: Setup Node + if: ${{ steps.cli_scope.outputs.proceed == 'true' }} uses: actions/setup-node@v6 with: node-version: 24 - name: Configure custom LLM provider + if: ${{ steps.cli_scope.outputs.proceed == 'true' }} env: LLM_API_KEY: ${{ secrets.llm_token }} MODEL_ID: ${{ inputs.model }} @@ -97,14 +108,17 @@ jobs: EOF - name: Sync cli-skill adapters + if: ${{ steps.cli_scope.outputs.proceed == 'true' }} run: node scripts/sync-cli-skill-adapters.js - name: Install cli-skill for Pi discovery + if: ${{ steps.cli_scope.outputs.proceed == 'true' }} run: | mkdir -p ~/.pi/skills cp -R cli-skill ~/.pi/skills/ - name: Run Pi agent + if: ${{ steps.cli_scope.outputs.proceed == 'true' }} id: pi uses: shaftoe/pi-coding-agent-action@v2 with: @@ -130,7 +144,7 @@ jobs: - Include a short summary table with severity counts and overall score. - name: Upload Pi session artifacts - if: ${{ steps.pi.outputs.session_html_path || steps.pi.outputs.session_jsonl_path }} + if: ${{ steps.cli_scope.outputs.proceed == 'true' && (steps.pi.outputs.session_html_path || steps.pi.outputs.session_jsonl_path) }} uses: actions/upload-artifact@v4 with: name: pi-session-${{ github.run_id }}-${{ github.run_attempt }} @@ -139,7 +153,7 @@ jobs: ${{ steps.pi.outputs.session_jsonl_path }} - name: Build PR report body - if: ${{ inputs.post_pr_comment && inputs.pr_number > 0 }} + if: ${{ steps.cli_scope.outputs.proceed == 'true' && inputs.post_pr_comment && inputs.pr_number > 0 }} id: report env: PI_RESPONSE: ${{ steps.pi.outputs.response }} @@ -196,7 +210,7 @@ jobs: PY - name: Fail when agent run is unsuccessful - if: ${{ inputs.fail_on_agent_error && steps.pi.outputs.success != 'true' }} + if: ${{ steps.cli_scope.outputs.proceed == 'true' && inputs.fail_on_agent_error && steps.pi.outputs.success != 'true' }} run: | echo "Pi agent reported success=${{ steps.pi.outputs.success }}" exit 1 From 43d181d5c53a75f18b387973ec71e0292116f705 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Wed, 10 Jun 2026 14:29:55 +0200 Subject: [PATCH 6/6] small gh action improvement --- .gitignore | 1 + scripts/check-cli-scope.py | 157 +++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 scripts/check-cli-scope.py diff --git a/.gitignore b/.gitignore index 63a7218..874f95d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ qwen36-snap **/*session.jsonl **/session.jsonl session.jsonl +**/__pycache__ diff --git a/scripts/check-cli-scope.py b/scripts/check-cli-scope.py new file mode 100644 index 0000000..2bf89f3 --- /dev/null +++ b/scripts/check-cli-scope.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import fnmatch +import os +import pathlib +import re +import subprocess +import sys + + +NO_CLI_PATTERNS = [ + r"\bthis\s+(project|repository|repo)\s+has\s+no\s+cli\b", + r"\bthere\s+is\s+no\s+cli\b", + r"\bdoes\s+not\s+(provide|expose)\s+a\s+cli\b", + r"\bno\s+command[- ]line\s+interface\b", + r"\bnot\s+a\s+cli\s+project\b", +] + + +def fail(message: str) -> None: + print(message, file=sys.stderr) + sys.exit(1) + + +def find_cli_doc(root: pathlib.Path) -> pathlib.Path: + kb_dir = root / "knowledge_base" + if not kb_dir.exists(): + fail( + "Missing knowledge_base/CLI.md. This workflow only runs for repositories that declare CLI scope metadata. " + "Add knowledge_base/CLI.md. For repositories with a CLI, include a YAML entry like " + "'paths: [cmd/, internal/cli/, docs/cli/]' listing all repo paths that affect CLI behavior, help, or command UX. " + "If the repository has no CLI, declare 'has_cli: false' or include a clear sentence such as 'This repository has no CLI.'" + ) + + cli_candidates = sorted( + path for path in kb_dir.iterdir() if path.is_file() and path.name.lower() == "cli.md" + ) + if not cli_candidates: + fail( + "Missing knowledge_base/CLI.md. This workflow only runs for repositories that declare CLI scope metadata. " + "Add knowledge_base/CLI.md. For repositories with a CLI, include a YAML entry like " + "'paths: [cmd/, internal/cli/, docs/cli/]' listing all repo paths that affect CLI behavior, help, or command UX. " + "If the repository has no CLI, declare 'has_cli: false' or include a clear sentence such as 'This repository has no CLI.'" + ) + return cli_candidates[0] + + +def has_cli(text: str) -> bool | None: + match = re.search(r"(?im)^has_cli\s*:\s*(true|false)\s*$", text) + if match: + return match.group(1).lower() == "true" + + normalized = re.sub(r"\s+", " ", text.lower()) + if any(re.search(pattern, normalized) for pattern in NO_CLI_PATTERNS): + return False + + return None + + +def parse_paths(text: str) -> list[str]: + inline_match = re.search(r"(?im)^paths\s*:\s*\[(.*?)\]\s*$", text) + if inline_match: + raw = inline_match.group(1) + return [item.strip().strip('"\'') for item in raw.split(",") if item.strip()] + + block_match = re.search(r"(?ims)^paths\s*:\s*\n((?:[ \t]*-[^\n]*\n)+)", text) + if block_match: + values = [] + for line in block_match.group(1).splitlines(): + line = line.strip() + if line.startswith("-"): + value = line[1:].strip().strip('"\'') + if value: + values.append(value) + return values + + return [] + + +def matches(spec: str, changed_file: str) -> bool: + spec = spec.strip() + if not spec: + return False + if any(char in spec for char in "*?[]"): + return fnmatch.fnmatch(changed_file, spec) + if spec.endswith("/"): + return changed_file.startswith(spec) + return changed_file == spec or changed_file.startswith(spec + "/") + + +def changed_files() -> list[str]: + base_sha = os.environ.get("BASE_SHA", "").strip() + head_sha = os.environ.get("HEAD_SHA", "").strip() + if not base_sha or not head_sha: + fail( + "BASE_SHA and HEAD_SHA must be set. This CLI scope gate only supports pull request " + "comparisons between the target branch and the source branch." + ) + + try: + merge_base = subprocess.check_output( + ["git", "merge-base", base_sha, head_sha], text=True + ).strip() + except subprocess.CalledProcessError as exc: + fail(f"Unable to determine merge base for pull request diff: {exc}") + + diff_cmd = ["git", "diff", "--name-only", f"{merge_base}..{head_sha}"] + + try: + return subprocess.check_output(diff_cmd, text=True).splitlines() + except subprocess.CalledProcessError as exc: + fail(f"Unable to determine changed files for CLI scope gate: {exc}") + + +def main() -> None: + root = pathlib.Path.cwd() + output_env = os.environ.get("GITHUB_OUTPUT") + if not output_env: + fail("GITHUB_OUTPUT is not set.") + output_path = pathlib.Path(output_env) + + cli_doc = find_cli_doc(root) + text = cli_doc.read_text(encoding="utf-8") + cli_present = has_cli(text) + + with output_path.open("a", encoding="utf-8") as file_handle: + if cli_present is False: + file_handle.write("proceed=false\n") + print("Success: no CLI found") + return + + paths = parse_paths(text) + if not paths: + fail( + f"{cli_doc} does not declare a YAML 'paths' entry. For CLI repositories, add a line like " + "'paths: [cmd/, internal/cli/, docs/cli/]' so the workflow can determine whether a pull request changes CLI-relevant files. " + "If the repository has no CLI, declare 'has_cli: false' or include a clear sentence such as 'This repository has no CLI.'" + ) + + changed = changed_files() + matched = [item for item in changed if any(matches(spec, item) for spec in paths)] + + with output_path.open("a", encoding="utf-8") as file_handle: + if matched: + file_handle.write("proceed=true\n") + else: + file_handle.write("proceed=false\n") + + if matched: + print("CLI-relevant changes detected:") + for item in matched: + print(f"- {item}") + else: + print("success: no CLI changes") + + +if __name__ == "__main__": + main()