From bd6fbf8bb415e0cb9447a27dec3c6e4c2aa43174 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Wed, 10 Jun 2026 14:34:14 +0200 Subject: [PATCH 1/4] small gh action improvement --- .../workflows/cli-skill-review-reusable.yml | 42 +---------------- scripts/build-pr-report.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 scripts/build-pr-report.py diff --git a/.github/workflows/cli-skill-review-reusable.yml b/.github/workflows/cli-skill-review-reusable.yml index a6e2f0f..9638b35 100644 --- a/.github/workflows/cli-skill-review-reusable.yml +++ b/.github/workflows/cli-skill-review-reusable.yml @@ -167,47 +167,7 @@ jobs: COMMAND: ${{ inputs.command }} run: | set -euo pipefail - - python3 - << 'PY' - import os - from pathlib import Path - - marker = os.environ.get('MARKER', '') - command = os.environ.get('COMMAND', '/cli-review') - success = os.environ.get('PI_SUCCESS', '') - cost = os.environ.get('PI_COST', '') - duration = os.environ.get('PI_DURATION', '') - input_tokens = os.environ.get('PI_INPUT_TOKENS', '') - output_tokens = os.environ.get('PI_OUTPUT_TOKENS', '') - run_url = os.environ.get('RUN_URL', '') - response = os.environ.get('PI_RESPONSE', '') - - if not response.strip(): - response = '_No response was returned by Pi._' - - # Keep body safely below GitHub comment max size. - max_response_len = 50000 - if len(response) > max_response_len: - response = response[:max_response_len] + "\n\n... _truncated_" - - body = f"""{marker} - ## CLI Skill Report ({command}) - - | Metric | Value | - |---|---| - | Success | {success or 'unknown'} | - | Duration (s) | {duration or 'n/a'} | - | Cost (USD) | {cost or 'n/a'} | - | Tokens (in/out) | {input_tokens or 'n/a'} / {output_tokens or 'n/a'} | - | Workflow run | [view run]({run_url}) | - - {response} - """ - - p = Path('pi-report-comment.md') - p.write_text(body.strip() + "\n", encoding='utf-8') - print(f"Wrote {p}") - PY + python3 scripts/build-pr-report.py - name: Fail when agent run is unsuccessful if: ${{ steps.cli_scope.outputs.proceed == 'true' && inputs.fail_on_agent_error && steps.pi.outputs.success != 'true' }} diff --git a/scripts/build-pr-report.py b/scripts/build-pr-report.py new file mode 100644 index 0000000..0b1ee86 --- /dev/null +++ b/scripts/build-pr-report.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import os +from pathlib import Path + + +def main() -> None: + marker = os.environ.get("MARKER", "") + command = os.environ.get("COMMAND", "/cli-review") + success = os.environ.get("PI_SUCCESS", "") + cost = os.environ.get("PI_COST", "") + duration = os.environ.get("PI_DURATION", "") + input_tokens = os.environ.get("PI_INPUT_TOKENS", "") + output_tokens = os.environ.get("PI_OUTPUT_TOKENS", "") + run_url = os.environ.get("RUN_URL", "") + response = os.environ.get("PI_RESPONSE", "") + + if not response.strip(): + response = "_No response was returned by Pi._" + + # Keep body safely below GitHub comment max size. + max_response_len = 50000 + if len(response) > max_response_len: + response = response[:max_response_len] + "\n\n... _truncated_" + + body = f"""{marker} +## CLI Skill Report ({command}) + +| Metric | Value | +|---|---| +| Success | {success or 'unknown'} | +| Duration (s) | {duration or 'n/a'} | +| Cost (USD) | {cost or 'n/a'} | +| Tokens (in/out) | {input_tokens or 'n/a'} / {output_tokens or 'n/a'} | +| Workflow run | [view run]({run_url}) | + +{response} +""" + + report_path = Path("pi-report-comment.md") + report_path.write_text(body.strip() + "\n", encoding="utf-8") + print(f"Wrote {report_path}") + + +if __name__ == "__main__": + main() From dc9091e3568ea2f884566ed50e403155a4088ea8 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Wed, 10 Jun 2026 14:39:59 +0200 Subject: [PATCH 2/4] Mock CLI added --- knowledge_base/CLI.md | 5 ++- scripts/demo_cli.py | 82 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 scripts/demo_cli.py diff --git a/knowledge_base/CLI.md b/knowledge_base/CLI.md index 2d4b44b..f65975e 100644 --- a/knowledge_base/CLI.md +++ b/knowledge_base/CLI.md @@ -1 +1,4 @@ -This project has no CLI. \ No newline at end of file +# CLI scope + +This repository includes a mock task-management CLI fixture for standards-review testing. +paths: [scripts/demo_cli.py] \ No newline at end of file diff --git a/scripts/demo_cli.py b/scripts/demo_cli.py new file mode 100644 index 0000000..0a5ba82 --- /dev/null +++ b/scripts/demo_cli.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Intentionally non-compliant demo CLI for task-management review tests.""" + +import argparse + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="taskThing", + description="A wildly inconsistent task manager CLI that is intentionally bad for standards testing.", + epilog="Try basically anything. Some commands probably look right.", + ) + parser.add_argument("-v", "--verbose", action="store_true", help="be extra talkative") + parser.add_argument("-q", "--quiet", action="store_true", help="say less maybe") + parser.add_argument("--format", "-f", choices=["txt", "json", "table"], default="txt") + + commands = parser.add_subparsers(dest="what") + + add = commands.add_parser( + "addTodo", + help="Add a todo thing into the todo area", + description="Create one todo item in a very flexible and not especially predictable way.", + ) + add.add_argument("title", help="todo name") + add.add_argument("place", nargs="?", help="where it lives") + add.add_argument("owner", nargs="?", help="person maybe") + add.add_argument("-p", "--priority", dest="priority") + add.add_argument("--prio", dest="priority") + add.add_argument("-t", "--tag", action="append") + add.add_argument("--tags", help="comma separated tags") + + remove = commands.add_parser( + "remove-todo-item", + help="Remove, delete, or otherwise get rid of a todo", + description="Deletes a task by whichever identifier feels useful at the time.", + ) + remove.add_argument("todo") + remove.add_argument("extra", nargs="?", help="second positional just because") + remove.add_argument("-i", "--id") + remove.add_argument("--name") + remove.add_argument("-y", "--yes", action="store_true") + remove.add_argument("--force", "-F", action="store_true") + + update = commands.add_parser( + "update", + help="Update something about todos", + description="A two-level command for changing todo-ish data in a not very disciplined way.", + ) + update_commands = update.add_subparsers(dest="update_target") + update_todos = update_commands.add_parser( + "todos", + help="Update todo records in bulk or singular form", + ) + update_todos.add_argument("todo") + update_todos.add_argument("field", nargs="?", help="what to change") + update_todos.add_argument("value", nargs="?", help="new value maybe") + update_todos.add_argument("-s", "--status") + update_todos.add_argument("--set-status") + update_todos.add_argument("--label") + update_todos.add_argument("--labels") + + read = commands.add_parser( + "readTodos", + help="Read todo information in one of several inconsistent ways", + description="Shows one todo, many todos, or maybe all todos depending on the inputs you try.", + ) + read.add_argument("todo", nargs="?", help="todo name or id or blank for everything") + read.add_argument("-a", "--all", action="store_true") + read.add_argument("--id") + read.add_argument("--sort") + read.add_argument("--order") + read.add_argument("--no-headers", action="store_true") + + return parser + + +def main() -> None: + build_parser().parse_args() + + +if __name__ == "__main__": + main() From 59a9316dc1ad44f1deaf442e8989f94ffdb44cd7 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Wed, 10 Jun 2026 14:48:07 +0200 Subject: [PATCH 3/4] update scoring --- cli-skill/commands/cli-review.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli-skill/commands/cli-review.md b/cli-skill/commands/cli-review.md index 30e1737..8c535f5 100644 --- a/cli-skill/commands/cli-review.md +++ b/cli-skill/commands/cli-review.md @@ -16,8 +16,8 @@ CLI standard compliance review only. - Non-compliance and compliant evidence based on observed CLI behavior/docs/code - Use these rules to determine severity: * High: violations of command structure and naming, use of positional parameters, accessibility/color violations - * Medium: use if non-standard verbs for commands, inconsistent flag names, extremely high complexity (eg. created by >20 commands) - * Low: formatting violations + * Medium: use if non-standard verbs for commands, inconsistent flag names or usage, extremely high complexity (eg. created by >20 commands) + * Low: formatting violations, duplicate short/long flags ## Out Of Scope @@ -43,7 +43,7 @@ Required sections: ## Summary Requirements The summary should list the number of violations, and their severity. It shall include these in a table. It shall give an overall score (Excellent = >95%, Very Good = >90%, Good = >80%, Room for Improvement = >60%, Need for Action = <=60%) -Start with a score of 100%, the number of commands N, and the weight of a single command W=1/N. For each High violation, reduce the score by 5*W, for each Medium violation by 2*W, and for each Small violation by 0.5*W. +The score is calculated based on violations and set into relation with the size of the command set. Start with a score of 100%, the number of commands N, and the weight of a single command W=100/N. First, make a list of all the violations sorted by command. For each High violation, reduce the score by 5*W, for each Medium violation by 2*W, and for each Small violation by 0.5*W. USE THIS ALGORITHM, DO NOT REASON ABOUT IT, OR FIND ALTERNATIVE WAYS TO CALCULATE A SCORE. ## Compliance Matrix Requirements From 7d906da8987e5fcc4ef24584dccb6d8730edc5c2 Mon Sep 17 00:00:00 2001 From: Hartmut Obendorf Date: Wed, 10 Jun 2026 14:50:02 +0200 Subject: [PATCH 4/4] better skill install? --- .github/workflows/cli-skill-review-reusable.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/cli-skill-review-reusable.yml b/.github/workflows/cli-skill-review-reusable.yml index 9638b35..6d1125c 100644 --- a/.github/workflows/cli-skill-review-reusable.yml +++ b/.github/workflows/cli-skill-review-reusable.yml @@ -107,10 +107,6 @@ jobs: } EOF - - name: Sync cli-skill adapters - if: ${{ steps.cli_scope.outputs.proceed == 'true' }} - run: node scripts/sync-cli-skill-adapters.js - - name: Install cli-skill for Pi discovery if: ${{ steps.cli_scope.outputs.proceed == 'true' }} run: |