From 9295820244fadb9f3687e27bfe05386e8fd17663 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 21 Jun 2026 00:45:30 +0000 Subject: [PATCH 1/4] feat: add AI workflow instruction drift pack (#214) Co-authored-by: ColumbusLabs --- .github/workflows/ci.yml | 2 + CHANGELOG.md | 3 + action.yml | 2 +- docs/ai-workflow-drift-rfc.md | 74 +++++++ docs/example-report.md | 2 +- docs/examples.md | 1 + docs/pack-chooser.md | 4 +- docs/rule-packs.md | 3 + docs/rules.md | 38 ++++ .../ai-workflow/.cursor/rules/testing.mdc | 7 + .../.github/copilot-instructions.md | 5 + examples/ai-workflow/AGENTS.md | 9 + examples/ai-workflow/CLAUDE.md | 9 + examples/ai-workflow/README.md | 14 ++ .../ai-workflow/.cursor/rules/review.mdc | 7 + .../false-positives/ai-workflow/AGENTS.md | 9 + .../false-positives/ai-workflow/CLAUDE.md | 9 + schema/debtlens.config.schema.json | 35 +++- src/config/mergeConfig.ts | 3 + src/config/packs.ts | 15 ++ src/detectors/aiWorkflow/index.ts | 9 + .../aiWorkflow/instructionContradiction.ts | 102 ++++++++++ .../aiWorkflow/instructionDuplication.ts | 62 ++++++ src/detectors/aiWorkflow/parse.ts | 191 ++++++++++++++++++ src/detectors/index.ts | 3 + tests/action/actionYml.test.ts | 2 +- tests/action/ciWorkflow.test.ts | 3 +- tests/config/packs.test.ts | 7 +- tests/config/schema.test.ts | 1 + tests/detectors/aiWorkflow.test.ts | 124 ++++++++++++ 30 files changed, 745 insertions(+), 10 deletions(-) create mode 100644 docs/ai-workflow-drift-rfc.md create mode 100644 examples/ai-workflow/.cursor/rules/testing.mdc create mode 100644 examples/ai-workflow/.github/copilot-instructions.md create mode 100644 examples/ai-workflow/AGENTS.md create mode 100644 examples/ai-workflow/CLAUDE.md create mode 100644 examples/ai-workflow/README.md create mode 100644 examples/false-positives/ai-workflow/.cursor/rules/review.mdc create mode 100644 examples/false-positives/ai-workflow/AGENTS.md create mode 100644 examples/false-positives/ai-workflow/CLAUDE.md create mode 100644 src/detectors/aiWorkflow/index.ts create mode 100644 src/detectors/aiWorkflow/instructionContradiction.ts create mode 100644 src/detectors/aiWorkflow/instructionDuplication.ts create mode 100644 src/detectors/aiWorkflow/parse.ts create mode 100644 tests/detectors/aiWorkflow.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb42119..49c1fac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ jobs: node dist/cli/index.js scan examples/rails --pack rails --format json --output debtlens-rails-report.json node dist/cli/index.js scan examples/compose --pack compose --format json --output debtlens-compose-report.json node dist/cli/index.js scan examples/swiftui --pack swiftui --format json --output debtlens-swiftui-report.json + node dist/cli/index.js scan examples/ai-workflow --pack ai-workflow-drift --format json --output debtlens-ai-workflow-report.json - name: Benchmark regression gate run: npm run benchmark:ci env: @@ -55,3 +56,4 @@ jobs: - run: node dist/cli/index.js scan examples/rails --pack rails --min-severity info --format markdown --output debtlens-rails-report.md - run: node dist/cli/index.js scan examples/compose --pack compose --min-severity info --format markdown --output debtlens-compose-report.md - run: node dist/cli/index.js scan examples/swiftui --pack swiftui --min-severity info --format markdown --output debtlens-swiftui-report.md + - run: node dist/cli/index.js scan examples/ai-workflow --pack ai-workflow-drift --min-severity info --format markdown --output debtlens-ai-workflow-report.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8801520..d8b8c56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ All notable changes to DebtLens are documented here. This project adheres to `ruby-dead-abstraction`, and `ruby-todo-comment` ([#198](https://github.com/ColumbusLabs/DebtLens/issues/198)). - **Rails framework pack** (`rails`) combining Ruby core rules with `rails-route-sprawl` and `rails-controller-sprawl` ([#198](https://github.com/ColumbusLabs/DebtLens/issues/198)). +- **`ai-workflow-drift` pack** with `ai-instruction-duplication` and + `ai-instruction-contradiction` for repository-local assistant instruction files. + The pack does not detect AI-authored code ([#214](https://github.com/ColumbusLabs/DebtLens/issues/214)). ### Changed diff --git a/action.yml b/action.yml index 97b526b..d84a555 100644 --- a/action.yml +++ b/action.yml @@ -37,7 +37,7 @@ inputs: description: Compare against a git ref and report only introduced findings. default: "" pack: - description: Built-in rule pack preset (core, react, react-native, next, expo, node, python, python-web, vue, svelte, kotlin, swift, ruby, rails, compose, swiftui, ai-assisted-maintainer, oss-maintainer). + description: Built-in rule pack preset (core, react, react-native, next, expo, node, python, python-web, vue, svelte, kotlin, swift, ruby, rails, compose, swiftui, ai-assisted-maintainer, oss-maintainer, ai-workflow-drift). default: "" package: description: Workspace package name to scan. diff --git a/docs/ai-workflow-drift-rfc.md b/docs/ai-workflow-drift-rfc.md new file mode 100644 index 0000000..e89e048 --- /dev/null +++ b/docs/ai-workflow-drift-rfc.md @@ -0,0 +1,74 @@ +# AI Workflow Instruction Drift RFC + +Status: **Prototype shipped (`ai-workflow-drift` pack)** + +DebtLens teams increasingly maintain parallel instruction files for coding assistants: +`AGENTS.md`, `CLAUDE.md`, `.cursor/rules/**`, and `.github/copilot-instructions.md`. +Those files drift apart quickly, producing contradictory guidance for humans and agents. + +This pack adds conservative, local checks for duplicated and conflicting instruction text. +It is a maintainability scanner extension, not an authorship detector. + +## Target files + +The pack scans repository-local instruction surfaces only: + +| Path pattern | Purpose | +| --- | --- | +| `AGENTS.md` | Repository agent instructions (Cursor, Codex, and similar tools) | +| `CLAUDE.md` | Claude Code / Claude project instructions | +| `.github/copilot-instructions.md` | GitHub Copilot repository instructions | +| `.cursor/rules/**` (`*.md`, `*.mdc`) | Cursor rule files | + +Detectors resolve files from the scan `context.files` list and, when needed, walk the +scan target on disk (similar to `config-drift` JSON discovery). + +## Non-goals + +- **Does not detect AI-authored code.** The pack inspects instruction markdown only. +- **No external telemetry.** Analysis stays on the local filesystem during `debtlens scan`. +- **No semantic policy enforcement.** The MVP uses normalized text duplication and a small + set of conservative contradiction patterns (for example, "always run tests" vs "skip tests"). +- **No secret scanning.** Instruction files may contain credentials; this pack does not + upload, index, or phone home file contents. + +## Prototype rules + +| Rule id | Signal | +| --- | --- | +| `ai-instruction-duplication` | The same normalized instruction block appears in two or more target files | +| `ai-instruction-contradiction` | Conservative opposing directives appear across instruction files | + +## File discovery model + +The `ai-workflow-drift` pack declares `includeGlobs` for the instruction paths above. +`mergeConfig()` unions those globs into scan discovery when the pack is selected, while +detectors still filter `context.files` by instruction path patterns inside `detect()`. + +Example: + +```bash +debtlens scan examples/ai-workflow --pack ai-workflow-drift +``` + +## Privacy and security considerations + +- **Local-only analysis:** DebtLens reads files from the scan target and optional git + staged overrides; it does not transmit instruction text to third-party services. +- **Sensitive content risk:** Instruction files sometimes embed API keys, internal URLs, + or customer names. Treat scan artifacts (JSON, SARIF, Markdown reports) like source + code output and restrict CI artifact retention when needed. +- **Conservative contradictions:** The contradiction rule intentionally under-reports to + avoid blocking legitimate tool-specific nuance. Review findings; do not treat them as + policy violations without human confirmation. +- **No provenance claims:** Findings describe instruction drift, not whether code was + written by an assistant. + +## Future direction + +- Optional `includeGlobs` overrides per repository config +- Richer contradiction templates with configurable vocabulary +- Cross-link suggestions to a canonical `AGENTS.md` +- Pack composition with `ai-assisted-maintainer` for code maintainability plus instruction drift + +Track implementation in [#214](https://github.com/ColumbusLabs/DebtLens/issues/214). diff --git a/docs/example-report.md b/docs/example-report.md index bf5a3c6..6e4e92f 100644 --- a/docs/example-report.md +++ b/docs/example-report.md @@ -1,6 +1,6 @@ # DebtLens Report -Scanned **3** files with **26** rules in **174ms**. +Scanned **3** files with **28** rules in **174ms**. ## Summary diff --git a/docs/examples.md b/docs/examples.md index 6570828..a4a10ba 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -18,6 +18,7 @@ these commands when evaluating output, writing docs, or checking a reporter chan | SwiftUI screen | `debtlens scan examples/swiftui --pack swiftui --min-severity info` | SwiftUI large-view and state-sprawl rules. | | Ruby service | `debtlens scan examples/ruby --pack ruby --min-severity info` | Ruby duplicate, large-method, thin-wrapper, and TODO rules. | | Rails app | `debtlens scan examples/rails --pack rails --min-severity info` | Rails route/controller sprawl plus Ruby core rules. | +| AI workflow instructions | `debtlens scan examples/ai-workflow --pack ai-workflow-drift --min-severity info` | Duplicated and contradictory assistant instruction files. | | Jetpack Compose screen | `debtlens scan examples/compose --pack compose --min-severity info` | Compose large-composable and state-hoisting rules. | | Local plugin | `debtlens scan examples/plugin --config examples/plugin/debtlens.config.json --min-severity info` | Trusted local plugin loading and plugin rule output. | | False-positive playground | `debtlens scan examples/false-positives --pack react --min-severity info` | Calibrated near-misses that should stay quiet. | diff --git a/docs/pack-chooser.md b/docs/pack-chooser.md index 1c6245f..cda74ec 100644 --- a/docs/pack-chooser.md +++ b/docs/pack-chooser.md @@ -23,6 +23,6 @@ Action behavior are language-neutral; only file discovery and detectors are pack | Mixed TS/Python/SFC/Kotlin/Swift monorepo | `core,python,vue,svelte,kotlin,swift` | `debtlens scan . --pack core,python,vue,svelte,kotlin,swift --format json` | Use package or path-scoped baselines; add `compose` or `swiftui` for UI modules. | | Open-source library | `oss-maintainer` | `debtlens scan . --pack oss-maintainer --min-severity medium` | Prefer reports and issues before hard CI gates. | | Ruby service or Rails app | `ruby` / `rails` | `debtlens scan . --pack rails --min-severity low` | Advisory first; review route and controller ownership before gating. | +| Agent-heavy repo with instruction files | `ai-workflow-drift` | `debtlens scan . --pack ai-workflow-drift --min-severity medium` | Review duplicated/contradictory guidance; not authorship detection. | -Future packs such as AI workflow instruction drift should reuse this same chooser shape -once their MVPs land. +Future packs should reuse this same chooser shape once their MVPs land. diff --git a/docs/rule-packs.md b/docs/rule-packs.md index 8f21c9e..8f2930e 100644 --- a/docs/rule-packs.md +++ b/docs/rule-packs.md @@ -79,6 +79,8 @@ For a user-facing selection table, see [`pack-chooser.md`](./pack-chooser.md). | `ruby-todo-comment` | **ruby** | TODO/FIXME/HACK/temporary implementation comments in Ruby files | Low | | `rails-route-sprawl` | **rails** | Rails `routes.rb` modules registering too many routes | Medium | | `rails-controller-sprawl` | **rails** | Rails controllers with too many public actions | Medium | +| `ai-instruction-duplication` | **ai-workflow-drift** | The same normalized instruction block repeated across assistant instruction files | Medium | +| `ai-instruction-contradiction` | **ai-workflow-drift** | Conservative opposing directives across assistant instruction files | High | | `compose-large-composable` | **compose** | Oversized or branch-heavy Jetpack Compose functions | Medium | | `compose-state-hoisting` | **compose** | Composables that own many local state holders instead of hoisting state | Medium | @@ -234,6 +236,7 @@ and Compose UI debt. | `swiftui` | SwiftUI oversized views and local state sprawl | **Shipped** | | `ruby` | Ruby duplicate methods, large methods, thin wrappers, and TODO debt | **Shipped** | | `rails` | Ruby core rules plus Rails route and controller sprawl | **Shipped** | +| `ai-workflow-drift` | Duplicated or contradictory AI assistant instruction files | **Shipped** | | `compose` | Jetpack Compose oversized composables and state-hoisting smells | **Shipped** | | `expo` | Expo Router and RN app shell boundaries | **Shipped** (React Native tuning plus barrel tolerance) | | `ai-assisted-maintainer` | Maintainability signals common in assistant-heavy codebases | **Shipped** | diff --git a/docs/rules.md b/docs/rules.md index f4a7e8e..30df550 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -1028,3 +1028,41 @@ When this is a false positive: - the competing names belong to separate concepts rather than one overloaded domain term Confidence: **0.62**. Co-occurring domain synonyms are often legitimate vocabulary, so this rule stays advisory. + +## `ai-instruction-duplication` + +Flags the same normalized instruction block repeated across assistant instruction files such as `AGENTS.md`, `CLAUDE.md`, `.cursor/rules/**`, and `.github/copilot-instructions.md`. + +Why it matters: duplicated guidance creates maintenance overhead and makes it unclear which file is canonical when assistants load multiple instruction sources. + +Good fixes: + +- keep one canonical instruction file and link to it from tool-specific files +- tailor each file to tool-specific context instead of copy-pasting shared blocks + +When this is a false positive: + +- the repeated text is intentionally mirrored while one file is being migrated +- short boilerplate headers are duplicated but substantive guidance differs elsewhere in the file + +Confidence: **0.82**. Normalized text equality is direct evidence, but some duplication may be deliberate during transitions. + +## `ai-instruction-contradiction` + +Flags conservative opposing directives across assistant instruction files, such as "always run tests" versus "skip tests". + +Why it matters: contradictory assistant guidance produces inconsistent edits, failed CI expectations, and review churn. + +Good fixes: + +- reconcile policies into one canonical instruction +- scope exceptions explicitly ("skip tests only for docs-only edits") in the same section as the base rule + +When this is a false positive: + +- the policies apply to different contexts and are not actually opposing +- one file is deprecated and scheduled for removal + +Confidence: **0.80**. Pattern matching is intentionally conservative and may miss nuanced conflicts or over-flag during migrations. + +This pack does **not** detect AI-authored code. It inspects instruction markdown only and performs local analysis without external telemetry. diff --git a/examples/ai-workflow/.cursor/rules/testing.mdc b/examples/ai-workflow/.cursor/rules/testing.mdc new file mode 100644 index 0000000..262e478 --- /dev/null +++ b/examples/ai-workflow/.cursor/rules/testing.mdc @@ -0,0 +1,7 @@ +--- +description: Testing policy for documentation-only edits +--- + +# Testing policy + +Skip tests when making documentation-only changes. diff --git a/examples/ai-workflow/.github/copilot-instructions.md b/examples/ai-workflow/.github/copilot-instructions.md new file mode 100644 index 0000000..ac8931f --- /dev/null +++ b/examples/ai-workflow/.github/copilot-instructions.md @@ -0,0 +1,5 @@ +# Copilot instructions + +## Formatting + +Always run prettier on changed files before opening a pull request. diff --git a/examples/ai-workflow/AGENTS.md b/examples/ai-workflow/AGENTS.md new file mode 100644 index 0000000..bd75745 --- /dev/null +++ b/examples/ai-workflow/AGENTS.md @@ -0,0 +1,9 @@ +# Agent instructions + +## Testing + +Always run the full test suite before committing changes. + +## Style + +Use TypeScript strict mode for all new files. diff --git a/examples/ai-workflow/CLAUDE.md b/examples/ai-workflow/CLAUDE.md new file mode 100644 index 0000000..2f4a30a --- /dev/null +++ b/examples/ai-workflow/CLAUDE.md @@ -0,0 +1,9 @@ +# Claude project instructions + +## Testing + +Always run the full test suite before committing changes. + +## Documentation + +Update README examples when public APIs change. diff --git a/examples/ai-workflow/README.md b/examples/ai-workflow/README.md new file mode 100644 index 0000000..fabfdf1 --- /dev/null +++ b/examples/ai-workflow/README.md @@ -0,0 +1,14 @@ +# AI workflow drift examples + +This fixture demonstrates duplicated and contradictory AI assistant instruction files. + +Run: + +```bash +debtlens scan examples/ai-workflow --pack ai-workflow-drift +``` + +Expected signals: + +- `ai-instruction-duplication` for the repeated testing block in `AGENTS.md` and `CLAUDE.md` +- `ai-instruction-contradiction` for "always run tests" vs "skip tests" diff --git a/examples/false-positives/ai-workflow/.cursor/rules/review.mdc b/examples/false-positives/ai-workflow/.cursor/rules/review.mdc new file mode 100644 index 0000000..d3215d4 --- /dev/null +++ b/examples/false-positives/ai-workflow/.cursor/rules/review.mdc @@ -0,0 +1,7 @@ +--- +description: Review checklist for assistant edits +--- + +# Review checklist + +Request human review before merging risky authentication changes. diff --git a/examples/false-positives/ai-workflow/AGENTS.md b/examples/false-positives/ai-workflow/AGENTS.md new file mode 100644 index 0000000..d685316 --- /dev/null +++ b/examples/false-positives/ai-workflow/AGENTS.md @@ -0,0 +1,9 @@ +# Agent instructions + +## Unit testing + +Run unit tests for all application code changes. + +## Type safety + +Enable TypeScript strict mode in new modules. diff --git a/examples/false-positives/ai-workflow/CLAUDE.md b/examples/false-positives/ai-workflow/CLAUDE.md new file mode 100644 index 0000000..69f657c --- /dev/null +++ b/examples/false-positives/ai-workflow/CLAUDE.md @@ -0,0 +1,9 @@ +# Claude project instructions + +## Integration testing + +Run integration tests when API contracts change. + +## Documentation + +Keep public README examples aligned with exported APIs. diff --git a/schema/debtlens.config.schema.json b/schema/debtlens.config.schema.json index 8394cbc..6be9384 100644 --- a/schema/debtlens.config.schema.json +++ b/schema/debtlens.config.schema.json @@ -53,12 +53,13 @@ "rails", "compose", "ai-assisted-maintainer", - "oss-maintainer" + "oss-maintainer", + "ai-workflow-drift" ] }, { "type": "string", - "pattern": "^\\s*(?:core|react|react-native|next|expo|node|python|python-web|vue|svelte|kotlin|swift|swiftui|ruby|rails|compose|ai-assisted-maintainer|oss-maintainer)(?:\\s*,\\s*(?:core|react|react-native|next|expo|node|python|python-web|vue|svelte|kotlin|swift|swiftui|ruby|rails|compose|ai-assisted-maintainer|oss-maintainer))*\\s*$" + "pattern": "^\\s*(?:core|react|react-native|next|expo|node|python|python-web|vue|svelte|kotlin|swift|swiftui|ruby|rails|compose|ai-assisted-maintainer|oss-maintainer|ai-workflow-drift)(?:\\s*,\\s*(?:core|react|react-native|next|expo|node|python|python-web|vue|svelte|kotlin|swift|swiftui|ruby|rails|compose|ai-assisted-maintainer|oss-maintainer|ai-workflow-drift))*\\s*$" } ], "description": "Built-in rule pack preset, or a comma-separated list of presets. Explicit rules override the pack." @@ -125,7 +126,9 @@ "rails-route-sprawl", "rails-controller-sprawl", "compose-large-composable", - "compose-state-hoisting" + "compose-state-hoisting", + "ai-instruction-duplication", + "ai-instruction-contradiction" ] }, { @@ -801,6 +804,22 @@ "medium", "high" ] + }, + "ai-instruction-duplication": { + "enum": [ + "info", + "low", + "medium", + "high" + ] + }, + "ai-instruction-contradiction": { + "enum": [ + "info", + "low", + "medium", + "high" + ] } }, "additionalProperties": { @@ -1095,6 +1114,16 @@ "type": "number", "minimum": 0, "maximum": 1 + }, + "ai-instruction-duplication": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "ai-instruction-contradiction": { + "type": "number", + "minimum": 0, + "maximum": 1 } }, "additionalProperties": { diff --git a/src/config/mergeConfig.ts b/src/config/mergeConfig.ts index 9851f9e..71e013f 100644 --- a/src/config/mergeConfig.ts +++ b/src/config/mergeConfig.ts @@ -95,9 +95,12 @@ function resolveIncludeGlobs( if (cliOptions.include?.length) return cliOptions.include; const base = resolveBaseIncludeGlobs(fileConfig, languageDiscovery); const discoveryLanguages = languageDiscovery.languages.filter((language) => language !== DEFAULT_SOURCE_LANGUAGE); + const packIncludeGlobs = parsePackIds(cliOptions.pack ?? fileConfig.pack) + .flatMap((packId) => getRulePack(packId).includeGlobs ?? []); return unique([ ...base, ...includeGlobsForLanguages(discoveryLanguages), + ...packIncludeGlobs, ]); } diff --git a/src/config/packs.ts b/src/config/packs.ts index 96c7f06..5028e3b 100644 --- a/src/config/packs.ts +++ b/src/config/packs.ts @@ -1,10 +1,13 @@ import type { SourceLanguage } from "../core/types.js"; +import { INSTRUCTION_FILE_GLOBS } from "../detectors/aiWorkflow/parse.js"; export interface RulePack { id: string; description: string; rules: string[]; languages: SourceLanguage[]; + /** Extra include globs merged into scan discovery when this pack is selected. */ + includeGlobs?: string[]; thresholds?: Record; duplicatedLiteral?: { ignoreStrings?: string[]; @@ -56,6 +59,11 @@ const NODE_RULES = [ "route-sprawl", ] as const; +const AI_WORKFLOW_DRIFT_RULES = [ + "ai-instruction-duplication", + "ai-instruction-contradiction", +] as const; + const AI_ASSISTED_MAINTAINER_RULES = [ "duplicate-logic", "duplicated-literal", @@ -304,6 +312,13 @@ export const RULE_PACKS: Record = { "weak-test-boundary.allowTypeOnly": 1, }, }, + "ai-workflow-drift": { + id: "ai-workflow-drift", + description: "Flags duplicated or contradictory AI assistant instruction files; does not detect AI-authored code.", + rules: [...AI_WORKFLOW_DRIFT_RULES], + languages: ["tsjs"], + includeGlobs: [...INSTRUCTION_FILE_GLOBS], + }, }; export const RULE_PACK_IDS = Object.keys(RULE_PACKS); diff --git a/src/detectors/aiWorkflow/index.ts b/src/detectors/aiWorkflow/index.ts new file mode 100644 index 0000000..7b1570c --- /dev/null +++ b/src/detectors/aiWorkflow/index.ts @@ -0,0 +1,9 @@ +export { instructionContradictionDetector } from "./instructionContradiction.js"; +export { instructionDuplicationDetector } from "./instructionDuplication.js"; +export { + extractInstructionBlocks, + INSTRUCTION_FILE_GLOBS, + isInstructionFile, + normalizeInstructionBlock, + resolveInstructionFiles, +} from "./parse.js"; diff --git a/src/detectors/aiWorkflow/instructionContradiction.ts b/src/detectors/aiWorkflow/instructionContradiction.ts new file mode 100644 index 0000000..2e2d329 --- /dev/null +++ b/src/detectors/aiWorkflow/instructionContradiction.ts @@ -0,0 +1,102 @@ +import type { DebtIssue, Detector, DetectorContext } from "../../core/types.js"; +import { createIssue } from "../../utils/createIssue.js"; +import { extractInstructionBlocks, resolveInstructionFiles } from "./parse.js"; + +interface DirectiveMatch { + file: string; + text: string; + startLine: number; + side: "left" | "right"; +} + +interface ContradictionPair { + label: string; + left: RegExp; + right: RegExp; +} + +const CONTRADICTION_PAIRS: ContradictionPair[] = [ + { + label: "test execution policy", + left: /\balways\s+run\b[^.!\n]{0,40}\btests?\b/i, + right: /\b(skip|do not run|don't run|never run)\b[^.!\n]{0,40}\btests?\b/i, + }, + { + label: "test execution policy", + left: /\brun\b[^.!\n]{0,30}\btests?\b[^.!\n]{0,30}\bbefore\b/i, + right: /\bskip\s+tests?\b/i, + }, + { + label: "formatting policy", + left: /\balways\s+(run|apply)\b[^.!\n]{0,40}\b(format|prettier|lint)\b/i, + right: /\b(skip|do not run|don't run|never run)\b[^.!\n]{0,40}\b(format|prettier|lint)\b/i, + }, + { + label: "commit review policy", + left: /\bnever\s+commit\b[^.!\n]{0,40}\bwithout\b/i, + right: /\bcommit\b[^.!\n]{0,40}\bwithout\b[^.!\n]{0,40}\b(review|approval)\b/i, + }, +]; + +export const instructionContradictionDetector: Detector = { + id: "ai-instruction-contradiction", + name: "AI instruction contradiction", + description: "Flags conservative opposing directives across AI workflow instruction files.", + defaultSeverity: "high", + tags: ["ai-workflow", "maintainability", "documentation"], + languages: ["tsjs"], + detect(context: DetectorContext): DebtIssue[] { + const maxFiles = context.getThreshold("ai-instruction-contradiction.maxInstructionFiles", 50); + const minBlockLength = context.getThreshold("ai-instruction-contradiction.minBlockLength", 24); + const issues: DebtIssue[] = []; + const seen = new Set(); + + const blocks = resolveInstructionFiles(context, maxFiles).flatMap((file) => + extractInstructionBlocks(file.content, minBlockLength).map((block) => ({ + file: file.relativePath, + text: block.text, + startLine: block.startLine, + }))); + + for (const pair of CONTRADICTION_PAIRS) { + const leftMatches = blocks.filter((block) => pair.left.test(block.text)); + const rightMatches = blocks.filter((block) => pair.right.test(block.text)); + if (leftMatches.length === 0 || rightMatches.length === 0) continue; + + for (const left of leftMatches) { + for (const right of rightMatches) { + if (left.file === right.file && left.startLine === right.startLine) continue; + const fingerprint = [ + pair.label, + left.file, + left.startLine, + right.file, + right.startLine, + ].join("|"); + if (seen.has(fingerprint)) continue; + seen.add(fingerprint); + + issues.push(createIssue({ + detector: instructionContradictionDetector, + confidence: 0.8, + file: left.file, + location: { startLine: left.startLine }, + message: `Conflicting AI workflow directives detected for ${pair.label}.`, + evidence: [ + `${left.file}:${left.startLine} — ${summarize(left.text)}`, + `${right.file}:${right.startLine} — ${summarize(right.text)}`, + ], + suggestion: "Reconcile the instruction files so assistants receive one consistent policy.", + })); + } + } + } + + return issues.slice(0, 50); + }, +}; + +function summarize(text: string): string { + const singleLine = text.replace(/\s+/g, " ").trim(); + return singleLine.length <= 100 ? singleLine : `${singleLine.slice(0, 97)}...`; +} diff --git a/src/detectors/aiWorkflow/instructionDuplication.ts b/src/detectors/aiWorkflow/instructionDuplication.ts new file mode 100644 index 0000000..abd7044 --- /dev/null +++ b/src/detectors/aiWorkflow/instructionDuplication.ts @@ -0,0 +1,62 @@ +import type { DebtIssue, Detector, DetectorContext } from "../../core/types.js"; +import { createIssue } from "../../utils/createIssue.js"; +import { extractInstructionBlocks, resolveInstructionFiles } from "./parse.js"; + +interface BlockOccurrence { + file: string; + text: string; + startLine: number; +} + +export const instructionDuplicationDetector: Detector = { + id: "ai-instruction-duplication", + name: "AI instruction duplication", + description: "Flags the same normalized instruction block repeated across multiple AI workflow files.", + defaultSeverity: "medium", + tags: ["ai-workflow", "maintainability", "documentation"], + languages: ["tsjs"], + detect(context: DetectorContext): DebtIssue[] { + const maxFiles = context.getThreshold("ai-instruction-duplication.maxInstructionFiles", 50); + const minBlockLength = context.getThreshold("ai-instruction-duplication.minBlockLength", 24); + const occurrences = new Map(); + + for (const file of resolveInstructionFiles(context, maxFiles)) { + for (const block of extractInstructionBlocks(file.content, minBlockLength)) { + const entries = occurrences.get(block.normalized) ?? []; + entries.push({ + file: file.relativePath, + text: block.text, + startLine: block.startLine, + }); + occurrences.set(block.normalized, entries); + } + } + + const issues: DebtIssue[] = []; + for (const [normalized, entries] of occurrences) { + const files = new Set(entries.map((entry) => entry.file)); + if (files.size < 2) continue; + const first = entries[0]; + if (!first) continue; + issues.push(createIssue({ + detector: instructionDuplicationDetector, + severity: files.size > 2 ? "high" : "medium", + confidence: 0.82, + file: first.file, + location: { startLine: first.startLine }, + message: "The same AI workflow instruction appears in multiple files.", + evidence: [ + ...entries.map((entry) => `${entry.file}:${entry.startLine}`), + `normalized: ${truncate(normalized, 120)}`, + ], + suggestion: "Keep one canonical instruction file and link to it from the others, or tailor each file to its tool-specific context.", + })); + } + + return issues.slice(0, 50); + }, +}; + +function truncate(value: string, maxLength: number): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 3)}...`; +} diff --git a/src/detectors/aiWorkflow/parse.ts b/src/detectors/aiWorkflow/parse.ts new file mode 100644 index 0000000..b738a36 --- /dev/null +++ b/src/detectors/aiWorkflow/parse.ts @@ -0,0 +1,191 @@ +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { basename, isAbsolute, relative, resolve } from "node:path"; +import type { DetectorContext, SourceFileInfo } from "../../core/types.js"; + +export const INSTRUCTION_FILE_GLOBS = [ + "**/AGENTS.md", + "**/CLAUDE.md", + "**/.github/copilot-instructions.md", + "**/.cursor/rules/**/*.md", + "**/.cursor/rules/**/*.mdc", +] as const; + +export interface InstructionBlock { + text: string; + normalized: string; + startLine: number; +} + +export interface InstructionFile { + relativePath: string; + content: string; +} + +const INSTRUCTION_FILE_PATTERNS = [ + /(?:^|\/)AGENTS\.md$/i, + /(?:^|\/)CLAUDE\.md$/i, + /(?:^|\/)\.github\/copilot-instructions\.md$/i, + /(?:^|\/)\.cursor\/rules\/.+\.(?:md|mdc)$/i, +]; + +export function isInstructionFile(relativePath: string): boolean { + const normalized = relativePath.replaceAll("\\", "/"); + return INSTRUCTION_FILE_PATTERNS.some((pattern) => pattern.test(normalized)); +} + +export function resolveInstructionFiles( + context: DetectorContext, + maxFiles = 50, +): InstructionFile[] { + const fromContext = context.files + .filter((file) => isInstructionFile(file.relativePath)) + .map((file) => ({ relativePath: file.relativePath, content: file.content })) + .slice(0, maxFiles); + + if (fromContext.length >= maxFiles) return fromContext; + + const seen = new Set(fromContext.map((file) => file.relativePath)); + const discovered = discoverInstructionFiles(context, maxFiles - fromContext.length); + for (const file of discovered) { + if (seen.has(file.relativePath)) continue; + seen.add(file.relativePath); + fromContext.push(file); + if (fromContext.length >= maxFiles) break; + } + + return fromContext; +} + +function discoverInstructionFiles(context: DetectorContext, limit: number): InstructionFile[] { + if (limit <= 0) return []; + if (!isAbsolute(context.options.target) || !existsSync(context.options.target)) return []; + + const stats = statSync(context.options.target); + const absolutePaths = stats.isFile() + ? [context.options.target] + : collectInstructionPaths(context.options.target, context.options.exclude, limit); + + return absolutePaths + .map((absolutePath) => { + const relativePath = stats.isFile() + ? basename(absolutePath) + : relative(context.options.target, absolutePath).replaceAll("\\", "/"); + if (!isInstructionFile(relativePath)) return undefined; + return { + relativePath, + content: readFileSync(resolve(absolutePath), "utf8"), + }; + }) + .filter((file): file is InstructionFile => file !== undefined); +} + +function collectInstructionPaths(root: string, exclude: string[], limit: number): string[] { + const paths: string[] = []; + + const visit = (directory: string) => { + if (paths.length >= limit) return; + + const entries = readdirSync(directory, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + if (paths.length >= limit) break; + const absolutePath = resolve(directory, entry.name); + const relativePath = relative(root, absolutePath).replaceAll("\\", "/"); + if (isExcluded(relativePath, exclude)) continue; + + if (entry.isDirectory()) { + visit(absolutePath); + } else if (entry.isFile()) { + if (isInstructionFile(relativePath)) paths.push(absolutePath); + } + } + }; + + visit(root); + return paths; +} + +function isExcluded(path: string, exclude: string[]): boolean { + return exclude.some((glob) => { + if (glob.endsWith("/**") && path === glob.slice(0, -3)) return true; + return globMatches(path, glob); + }); +} + +function globMatches(path: string, glob: string): boolean { + let expression = ""; + for (let index = 0; index < glob.length; index += 1) { + const char = glob[index]; + const next = glob[index + 1]; + if (char === "*" && next === "*") { + expression += ".*"; + index += 1; + } else if (char === "*") { + expression += "[^/]*"; + } else { + expression += escapeRegExp(char ?? ""); + } + } + return new RegExp(`^${expression}$`).test(path); +} + +function escapeRegExp(value: string): string { + return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&"); +} + +export function normalizeInstructionBlock(text: string): string { + return text + .replace(/```[\s\S]*?```/g, " ") + .replace(/^#{1,6}\s+/gm, "") + .replace(/^\s*[-*+]\s+/gm, "") + .replace(/`([^`]+)`/g, "$1") + .replace(/\s+/g, " ") + .trim() + .toLowerCase(); +} + +export function extractInstructionBlocks(content: string, minLength = 24): InstructionBlock[] { + const blocks: InstructionBlock[] = []; + const lines = content.split(/\r?\n/); + let current: string[] = []; + let startLine = 1; + + const flush = (endLine: number) => { + const text = current.join("\n").trim(); + current = []; + if (!text) return; + const normalized = normalizeInstructionBlock(text); + if (normalized.length < minLength) return; + blocks.push({ text, normalized, startLine }); + startLine = endLine + 1; + }; + + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + const lineNumber = index + 1; + const isBoundary = line.trim() === "" || /^#{1,6}\s+/.test(line); + + if (isBoundary) { + if (current.length > 0) flush(lineNumber); + if (/^#{1,6}\s+/.test(line)) { + startLine = lineNumber; + current.push(line); + } else { + startLine = lineNumber + 1; + } + continue; + } + + if (current.length === 0) startLine = lineNumber; + current.push(line); + } + + if (current.length > 0) flush(lines.length); + return blocks; +} + +export function instructionFilesFromContext(files: SourceFileInfo[]): InstructionFile[] { + return files + .filter((file) => isInstructionFile(file.relativePath)) + .map((file) => ({ relativePath: file.relativePath, content: file.content })); +} diff --git a/src/detectors/index.ts b/src/detectors/index.ts index 7becdc3..6dd8c48 100644 --- a/src/detectors/index.ts +++ b/src/detectors/index.ts @@ -1,4 +1,5 @@ import type { Detector } from "../core/types.js"; +import { instructionContradictionDetector, instructionDuplicationDetector } from "./aiWorkflow/index.js"; import { composeLargeComposableDetector, composeStateHoistingDetector } from "./compose/index.js"; import { apiSurfaceSprawlDetector } from "./apiSurfaceSprawl.js"; import { barrelFileDetector } from "./barrelFile.js"; @@ -104,6 +105,8 @@ export const allDetectors: Detector[] = [ railsControllerSprawlDetector, composeLargeComposableDetector, composeStateHoistingDetector, + instructionDuplicationDetector, + instructionContradictionDetector, ]; export const detectorIds = allDetectors.map((detector) => detector.id); diff --git a/tests/action/actionYml.test.ts b/tests/action/actionYml.test.ts index 0a2acc8..86c0001 100644 --- a/tests/action/actionYml.test.ts +++ b/tests/action/actionYml.test.ts @@ -111,7 +111,7 @@ describe("GitHub Action metadata", () => { }); it("documents supported packs and bootstraps tagged release assets before source fallback", () => { - assert.match(actionYml, /core, react, react-native, next, expo, node, python, python-web, vue, svelte, kotlin, swift, ruby, rails, compose, swiftui, ai-assisted-maintainer, oss-maintainer/); + assert.match(actionYml, /core, react, react-native, next, expo, node, python, python-web, vue, svelte, kotlin, swift, ruby, rails, compose, swiftui, ai-assisted-maintainer, oss-maintainer, ai-workflow-drift/); assert.match(actionYml, /DL_ACTION_REF: \$\{\{ github\.action_ref \}\}/); assert.match(actionYml, /DL_ACTION_REPOSITORY: \$\{\{ github\.action_repository \}\}/); assert.match(actionYml, /scripts\/prepare-action-runtime\.sh/); diff --git a/tests/action/ciWorkflow.test.ts b/tests/action/ciWorkflow.test.ts index faa6034..480f6c6 100644 --- a/tests/action/ciWorkflow.test.ts +++ b/tests/action/ciWorkflow.test.ts @@ -26,7 +26,8 @@ describe("CI workflow drift guards", () => { assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/ruby --pack ruby --min-severity info --format markdown/); assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/rails --pack rails --min-severity info --format markdown/); assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/compose --pack compose --format json/); - assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/swiftui --pack swiftui --format json/); + assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/ai-workflow --pack ai-workflow-drift --format json/); + assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/ai-workflow --pack ai-workflow-drift --min-severity info --format markdown/); assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/swiftui --pack swiftui --min-severity info --format markdown/); assert.match(ciWorkflow, /dist\/cli\/index\.js scan examples\/compose --pack compose --min-severity info --format markdown/); }); diff --git a/tests/config/packs.test.ts b/tests/config/packs.test.ts index f6f611f..1a5623a 100644 --- a/tests/config/packs.test.ts +++ b/tests/config/packs.test.ts @@ -6,7 +6,7 @@ import { getRulePack, listRulePacks } from "../../src/config/packs.js"; describe("rule packs", () => { it("lists built-in packs with expected rule counts", () => { const packs = listRulePacks(); - assert.equal(packs.length, 18); + assert.equal(packs.length, 19); assert.equal(getRulePack("core").rules.length, 13); assert.deepEqual(getRulePack("core").languages, ["tsjs"]); assert.equal(getRulePack("react").rules.length, 20); @@ -95,6 +95,11 @@ describe("rule packs", () => { assert.deepEqual(getRulePack("rails").languages, ["ruby"]); assert.equal(getRulePack("rails").thresholds?.["rails-route-sprawl.maxRoutes"], 8); assert.equal(getRulePack("rails").thresholds?.["rails-controller-sprawl.maxActions"], 8); + assert.deepEqual(getRulePack("ai-workflow-drift").rules, [ + "ai-instruction-duplication", + "ai-instruction-contradiction", + ]); + assert.ok(getRulePack("ai-workflow-drift").includeGlobs?.includes("**/AGENTS.md")); assert.ok(getRulePack("ai-assisted-maintainer").rules.includes("duplicated-literal")); assert.ok(getRulePack("oss-maintainer").rules.includes("api-surface-sprawl")); }); diff --git a/tests/config/schema.test.ts b/tests/config/schema.test.ts index e9e0b74..2f655ed 100644 --- a/tests/config/schema.test.ts +++ b/tests/config/schema.test.ts @@ -127,6 +127,7 @@ describe("config JSON schema", () => { "compose", "ai-assisted-maintainer", "oss-maintainer", + "ai-workflow-drift", ]); assert.match(built.properties.pack?.anyOf[1]?.pattern ?? "", /compose/); assert.match(built.properties.pack?.anyOf[1]?.pattern ?? "", /svelte/); diff --git a/tests/detectors/aiWorkflow.test.ts b/tests/detectors/aiWorkflow.test.ts new file mode 100644 index 0000000..2bec4d9 --- /dev/null +++ b/tests/detectors/aiWorkflow.test.ts @@ -0,0 +1,124 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "node:test"; +import { mergeConfig } from "../../src/config/mergeConfig.js"; +import { getRulePack } from "../../src/config/packs.js"; +import { + extractInstructionBlocks, + instructionContradictionDetector, + instructionDuplicationDetector, + INSTRUCTION_FILE_GLOBS, + isInstructionFile, + normalizeInstructionBlock, +} from "../../src/detectors/aiWorkflow/index.js"; +import { runDetector } from "../helpers/runDetector.js"; + +describe("ai workflow instruction parsing", () => { + it("recognizes supported instruction file paths", () => { + assert.equal(isInstructionFile("AGENTS.md"), true); + assert.equal(isInstructionFile(".github/copilot-instructions.md"), true); + assert.equal(isInstructionFile(".cursor/rules/testing.mdc"), true); + assert.equal(isInstructionFile("src/app.ts"), false); + }); + + it("normalizes and extracts substantive instruction blocks", () => { + const blocks = extractInstructionBlocks(`## Testing + +Always run the full test suite before committing changes. + +## Notes + +Short. +`); + assert.equal(blocks.length, 1); + assert.equal( + normalizeInstructionBlock(" - Always run tests "), + "always run tests", + ); + assert.match(blocks[0]?.normalized ?? "", /always run the full test suite/); + }); +}); + +describe("ai-instruction-duplication detector", () => { + it("flags the same normalized block across multiple instruction files", async () => { + const shared = "Always run the full test suite before committing changes."; + const issues = await runDetector(instructionDuplicationDetector, { + "AGENTS.md": `## Testing\n\n${shared}`, + "CLAUDE.md": `## Testing\n\n${shared}`, + }, { language: "tsjs" }); + + assert.equal(issues.length, 1); + assert.equal(issues[0]?.ruleId, "ai-instruction-duplication"); + assert.ok(issues[0]?.evidence?.some((entry) => entry.includes("AGENTS.md"))); + assert.ok(issues[0]?.evidence?.some((entry) => entry.includes("CLAUDE.md"))); + }); + + it("does not flag complementary instructions in different files", async () => { + const issues = await runDetector(instructionDuplicationDetector, { + "AGENTS.md": "## Unit testing\n\nRun unit tests for all application code changes.", + "CLAUDE.md": "## Integration testing\n\nRun integration tests when API contracts change.", + }, { language: "tsjs" }); + + assert.equal(issues.length, 0); + }); +}); + +describe("ai-instruction-contradiction detector", () => { + it("flags conservative opposing test directives", async () => { + const issues = await runDetector(instructionContradictionDetector, { + "AGENTS.md": "## Testing\n\nAlways run the full test suite before committing changes.", + ".cursor/rules/testing.mdc": "## Testing policy\n\nSkip tests when making documentation-only changes.", + }, { language: "tsjs" }); + + assert.ok(issues.some((issue) => issue.ruleId === "ai-instruction-contradiction")); + assert.ok(issues.some((issue) => issue.message.includes("test execution policy"))); + }); + + it("does not flag complementary non-contradictory policies", async () => { + const issues = await runDetector(instructionContradictionDetector, { + "AGENTS.md": "## Unit testing\n\nRun unit tests for all application code changes.", + "CLAUDE.md": "## Integration testing\n\nRun integration tests when API contracts change.", + ".cursor/rules/review.mdc": "## Review\n\nRequest human review before merging risky authentication changes.", + }, { language: "tsjs" }); + + assert.equal(issues.length, 0); + }); + + it("discovers instruction files from the scan target when they are absent from context", async () => { + const dir = mkdtempSync(join(tmpdir(), "debtlens-ai-workflow-scope-")); + try { + writeFileSync(join(dir, "AGENTS.md"), "## Testing\n\nAlways run tests before committing.\n", "utf8"); + mkdirSync(join(dir, ".cursor", "rules"), { recursive: true }); + writeFileSync( + join(dir, ".cursor", "rules", "docs.mdc"), + "## Docs\n\nSkip tests for documentation-only edits.\n", + "utf8", + ); + + const issues = await runDetector(instructionContradictionDetector, {}, { + target: dir, + language: "tsjs", + }); + + assert.ok(issues.some((issue) => issue.ruleId === "ai-instruction-contradiction")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe("ai-workflow-drift pack discovery", () => { + it("adds instruction file globs when the pack is selected", () => { + const options = mergeConfig(".", {}, { cwd: process.cwd(), pack: "ai-workflow-drift" }); + + assert.deepEqual(getRulePack("ai-workflow-drift").rules, [ + "ai-instruction-duplication", + "ai-instruction-contradiction", + ]); + for (const glob of INSTRUCTION_FILE_GLOBS) { + assert.ok(options.include.includes(glob), `missing include glob ${glob}`); + } + }); +}); From ddb5c15175a862801699eada9cd5635465ff48c0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 21 Jun 2026 01:14:32 +0000 Subject: [PATCH 2/4] fix: harden AI workflow drift pack discovery and contradiction matching (#214) - Use empty pack languages so ai-workflow-drift does not widen TS/JS discovery - Respect changedFiles scope and fileContents overrides in instruction resolver - Ignore negated skip-tests guidance in contradiction heuristics - Extend runDetector helper for scoped scan tests Co-authored-by: ColumbusLabs --- src/config/packs.ts | 2 +- .../aiWorkflow/instructionContradiction.ts | 9 ++++++- src/detectors/aiWorkflow/parse.ts | 20 ++++++++++++--- tests/detectors/aiWorkflow.test.ts | 25 +++++++++++++++++++ tests/helpers/runDetector.ts | 6 +++++ 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/config/packs.ts b/src/config/packs.ts index 5028e3b..a88f322 100644 --- a/src/config/packs.ts +++ b/src/config/packs.ts @@ -316,7 +316,7 @@ export const RULE_PACKS: Record = { id: "ai-workflow-drift", description: "Flags duplicated or contradictory AI assistant instruction files; does not detect AI-authored code.", rules: [...AI_WORKFLOW_DRIFT_RULES], - languages: ["tsjs"], + languages: [], includeGlobs: [...INSTRUCTION_FILE_GLOBS], }, }; diff --git a/src/detectors/aiWorkflow/instructionContradiction.ts b/src/detectors/aiWorkflow/instructionContradiction.ts index 2e2d329..1cf22ca 100644 --- a/src/detectors/aiWorkflow/instructionContradiction.ts +++ b/src/detectors/aiWorkflow/instructionContradiction.ts @@ -60,7 +60,7 @@ export const instructionContradictionDetector: Detector = { for (const pair of CONTRADICTION_PAIRS) { const leftMatches = blocks.filter((block) => pair.left.test(block.text)); - const rightMatches = blocks.filter((block) => pair.right.test(block.text)); + const rightMatches = blocks.filter((block) => matchesContradictionSide(pair.right, block.text)); if (leftMatches.length === 0 || rightMatches.length === 0) continue; for (const left of leftMatches) { @@ -100,3 +100,10 @@ function summarize(text: string): string { const singleLine = text.replace(/\s+/g, " ").trim(); return singleLine.length <= 100 ? singleLine : `${singleLine.slice(0, 97)}...`; } + +function matchesContradictionSide(pattern: RegExp, text: string): boolean { + const match = pattern.exec(text); + if (!match) return false; + const prefix = text.slice(0, match.index).trimEnd(); + return !/\b(?:do not|don't|never|not)\s*$/i.test(prefix); +} diff --git a/src/detectors/aiWorkflow/parse.ts b/src/detectors/aiWorkflow/parse.ts index b738a36..e1218af 100644 --- a/src/detectors/aiWorkflow/parse.ts +++ b/src/detectors/aiWorkflow/parse.ts @@ -37,12 +37,17 @@ export function resolveInstructionFiles( context: DetectorContext, maxFiles = 50, ): InstructionFile[] { + const scopedPaths = context.options.changedFiles; const fromContext = context.files .filter((file) => isInstructionFile(file.relativePath)) - .map((file) => ({ relativePath: file.relativePath, content: file.content })) + .filter((file) => !scopedPaths?.length || scopedPaths.includes(file.relativePath)) + .map((file) => ({ + relativePath: file.relativePath, + content: resolveInstructionContent(context, file.relativePath, file.content), + })) .slice(0, maxFiles); - if (fromContext.length >= maxFiles) return fromContext; + if (fromContext.length >= maxFiles || scopedPaths?.length) return fromContext; const seen = new Set(fromContext.map((file) => file.relativePath)); const discovered = discoverInstructionFiles(context, maxFiles - fromContext.length); @@ -56,6 +61,14 @@ export function resolveInstructionFiles( return fromContext; } +function resolveInstructionContent( + context: DetectorContext, + relativePath: string, + fallback: string, +): string { + return context.options.fileContents?.[relativePath] ?? fallback; +} + function discoverInstructionFiles(context: DetectorContext, limit: number): InstructionFile[] { if (limit <= 0) return []; if (!isAbsolute(context.options.target) || !existsSync(context.options.target)) return []; @@ -71,9 +84,10 @@ function discoverInstructionFiles(context: DetectorContext, limit: number): Inst ? basename(absolutePath) : relative(context.options.target, absolutePath).replaceAll("\\", "/"); if (!isInstructionFile(relativePath)) return undefined; + const override = context.options.fileContents?.[relativePath]; return { relativePath, - content: readFileSync(resolve(absolutePath), "utf8"), + content: override ?? readFileSync(resolve(absolutePath), "utf8"), }; }) .filter((file): file is InstructionFile => file !== undefined); diff --git a/tests/detectors/aiWorkflow.test.ts b/tests/detectors/aiWorkflow.test.ts index 2bec4d9..9da6c5e 100644 --- a/tests/detectors/aiWorkflow.test.ts +++ b/tests/detectors/aiWorkflow.test.ts @@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, it } from "node:test"; +import { defaultConfig } from "../../src/config/defaults.js"; import { mergeConfig } from "../../src/config/mergeConfig.js"; import { getRulePack } from "../../src/config/packs.js"; import { @@ -86,6 +87,15 @@ describe("ai-instruction-contradiction detector", () => { assert.equal(issues.length, 0); }); + it("does not treat negated skip guidance as a contradictory skip-tests policy", async () => { + const issues = await runDetector(instructionContradictionDetector, { + "AGENTS.md": "## Testing\n\nAlways run the full test suite before committing changes.", + "CLAUDE.md": "## Testing\n\nDo not skip tests for documentation-only edits.", + }, { language: "tsjs" }); + + assert.equal(issues.length, 0); + }); + it("discovers instruction files from the scan target when they are absent from context", async () => { const dir = mkdtempSync(join(tmpdir(), "debtlens-ai-workflow-scope-")); try { @@ -117,8 +127,23 @@ describe("ai-workflow-drift pack discovery", () => { "ai-instruction-duplication", "ai-instruction-contradiction", ]); + assert.deepEqual(getRulePack("ai-workflow-drift").languages, []); for (const glob of INSTRUCTION_FILE_GLOBS) { assert.ok(options.include.includes(glob), `missing include glob ${glob}`); } + for (const glob of defaultConfig.include) { + assert.ok(!options.include.includes(glob), `unexpected TS/JS include glob ${glob}`); + } + }); + + it("respects changedFiles scope without disk discovery", async () => { + const issues = await runDetector(instructionContradictionDetector, { + "AGENTS.md": "## Testing\n\nAlways run the full test suite before committing changes.", + }, { + language: "tsjs", + changedFiles: ["AGENTS.md"], + }); + + assert.equal(issues.length, 0); }); }); diff --git a/tests/helpers/runDetector.ts b/tests/helpers/runDetector.ts index 091e815..05b1fa4 100644 --- a/tests/helpers/runDetector.ts +++ b/tests/helpers/runDetector.ts @@ -26,6 +26,10 @@ export interface RunDetectorOptions { todoCommentMarkers?: Array<{ pattern: string; severity?: Severity; label?: string }>; /** Override inferred source language for all files. */ language?: SourceLanguage; + /** Limit detectors to these relative paths when set. */ + changedFiles?: string[]; + /** Override file contents keyed by relative path. */ + fileContents?: Record; } function inferSourceLanguage(relativePath: string, override?: SourceLanguage): SourceLanguage { @@ -92,6 +96,8 @@ export async function runDetector( todoCommentMarkers: options.todoCommentMarkers ? compileTodoCommentMarkers(options.todoCommentMarkers) : undefined, + changedFiles: options.changedFiles, + fileContents: options.fileContents, }; const issues = await detector.detect({ From fb766698e83373c62986c4d0c31fa62d0be6ce54 Mon Sep 17 00:00:00 2001 From: ColumbusLabs <287001685+ColumbusLabs@users.noreply.github.com> Date: Sat, 20 Jun 2026 23:27:44 -0400 Subject: [PATCH 3/4] fix: respect AI workflow scan scope --- README.md | 3 ++- src/detectors/aiWorkflow/parse.ts | 38 ++++++++++++++++++++++++------ tests/detectors/aiWorkflow.test.ts | 36 ++++++++++++++++++++++++++++ tests/helpers/runDetector.ts | 8 +++++-- 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 82cb68a..13547aa 100644 --- a/README.md +++ b/README.md @@ -477,6 +477,7 @@ also declare their discovery metadata, so selecting `python`, `vue`, `svelte`, ` | `expo` | React Native tuning for Expo Router projects | | `ai-assisted-maintainer` | high-signal maintainability checks for assistant-heavy codebases; no authorship claims | | `oss-maintainer` | library API surface, barrels, duplication, tests, and TODO debt | +| `ai-workflow-drift` | duplicated or contradictory AI assistant instruction files; no AI-authorship claims | ```json { @@ -932,7 +933,7 @@ and PR comment upsert, and `--diff-base` branch comparisons. The architecture stays intentionally simple: a language-agnostic scan and reporting layer with pluggable rule packs on top. Current shipped packs cover core TS/JS, React, -React Native, Next.js, Expo, Node, Python, Python web, Vue/Svelte SFC scripts, Kotlin, Swift, Ruby/Rails, Jetpack Compose, SwiftUI, and maintainer workflows. Additional +React Native, Next.js, Expo, Node, Python, Python web, Vue/Svelte SFC scripts, Kotlin, Swift, Ruby/Rails, Jetpack Compose, SwiftUI, maintainer workflows, and AI workflow instruction drift. Additional packs expand from the same scan/reporting contract. See [`ROADMAP.md`](./ROADMAP.md) and [`docs/rule-packs.md`](./docs/rule-packs.md). diff --git a/src/detectors/aiWorkflow/parse.ts b/src/detectors/aiWorkflow/parse.ts index e1218af..ce264c4 100644 --- a/src/detectors/aiWorkflow/parse.ts +++ b/src/detectors/aiWorkflow/parse.ts @@ -37,17 +37,17 @@ export function resolveInstructionFiles( context: DetectorContext, maxFiles = 50, ): InstructionFile[] { - const scopedPaths = context.options.changedFiles; + const scopedPaths = normalizeScopedPaths(context); const fromContext = context.files .filter((file) => isInstructionFile(file.relativePath)) - .filter((file) => !scopedPaths?.length || scopedPaths.includes(file.relativePath)) + .filter((file) => !scopedPaths || scopedPaths.has(file.relativePath)) .map((file) => ({ relativePath: file.relativePath, content: resolveInstructionContent(context, file.relativePath, file.content), })) .slice(0, maxFiles); - if (fromContext.length >= maxFiles || scopedPaths?.length) return fromContext; + if (fromContext.length >= maxFiles || scopedPaths) return fromContext; const seen = new Set(fromContext.map((file) => file.relativePath)); const discovered = discoverInstructionFiles(context, maxFiles - fromContext.length); @@ -76,7 +76,7 @@ function discoverInstructionFiles(context: DetectorContext, limit: number): Inst const stats = statSync(context.options.target); const absolutePaths = stats.isFile() ? [context.options.target] - : collectInstructionPaths(context.options.target, context.options.exclude, limit); + : collectInstructionPaths(context.options.target, context.options.include, context.options.exclude, limit); return absolutePaths .map((absolutePath) => { @@ -84,6 +84,7 @@ function discoverInstructionFiles(context: DetectorContext, limit: number): Inst ? basename(absolutePath) : relative(context.options.target, absolutePath).replaceAll("\\", "/"); if (!isInstructionFile(relativePath)) return undefined; + if (!isIncluded(relativePath, context.options.include)) return undefined; const override = context.options.fileContents?.[relativePath]; return { relativePath, @@ -93,7 +94,22 @@ function discoverInstructionFiles(context: DetectorContext, limit: number): Inst .filter((file): file is InstructionFile => file !== undefined); } -function collectInstructionPaths(root: string, exclude: string[], limit: number): string[] { +function normalizeScopedPaths(context: DetectorContext): Set | undefined { + const changedFiles = context.options.changedFiles; + if (changedFiles === undefined) return undefined; + + const scoped = new Set(); + for (const path of changedFiles) { + const normalized = path.replaceAll("\\", "/"); + scoped.add(normalized); + if (isAbsolute(path) && isAbsolute(context.options.target)) { + scoped.add(relative(context.options.target, path).replaceAll("\\", "/")); + } + } + return scoped; +} + +function collectInstructionPaths(root: string, include: string[], exclude: string[], limit: number): string[] { const paths: string[] = []; const visit = (directory: string) => { @@ -110,7 +126,7 @@ function collectInstructionPaths(root: string, exclude: string[], limit: number) if (entry.isDirectory()) { visit(absolutePath); } else if (entry.isFile()) { - if (isInstructionFile(relativePath)) paths.push(absolutePath); + if (isInstructionFile(relativePath) && isIncluded(relativePath, include)) paths.push(absolutePath); } } }; @@ -119,6 +135,10 @@ function collectInstructionPaths(root: string, exclude: string[], limit: number) return paths; } +function isIncluded(path: string, include: string[]): boolean { + return include.length === 0 || include.some((glob) => globMatches(path, glob)); +} + function isExcluded(path: string, exclude: string[]): boolean { return exclude.some((glob) => { if (glob.endsWith("/**") && path === glob.slice(0, -3)) return true; @@ -131,7 +151,11 @@ function globMatches(path: string, glob: string): boolean { for (let index = 0; index < glob.length; index += 1) { const char = glob[index]; const next = glob[index + 1]; - if (char === "*" && next === "*") { + const nextNext = glob[index + 2]; + if (char === "*" && next === "*" && nextNext === "/") { + expression += "(?:.*/)?"; + index += 2; + } else if (char === "*" && next === "*") { expression += ".*"; index += 1; } else if (char === "*") { diff --git a/tests/detectors/aiWorkflow.test.ts b/tests/detectors/aiWorkflow.test.ts index 9da6c5e..377802e 100644 --- a/tests/detectors/aiWorkflow.test.ts +++ b/tests/detectors/aiWorkflow.test.ts @@ -146,4 +146,40 @@ describe("ai-workflow-drift pack discovery", () => { assert.equal(issues.length, 0); }); + + it("treats an empty changedFiles scope as empty instead of falling back to disk discovery", async () => { + const dir = mkdtempSync(join(tmpdir(), "debtlens-ai-workflow-empty-scope-")); + try { + writeFileSync(join(dir, "AGENTS.md"), "## Testing\n\nAlways run tests before committing.\n", "utf8"); + writeFileSync(join(dir, "CLAUDE.md"), "## Testing\n\nSkip tests for documentation-only edits.\n", "utf8"); + + const issues = await runDetector(instructionContradictionDetector, {}, { + target: dir, + language: "tsjs", + changedFiles: [], + }); + + assert.equal(issues.length, 0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not discover instruction files excluded by explicit include globs", async () => { + const dir = mkdtempSync(join(tmpdir(), "debtlens-ai-workflow-include-scope-")); + try { + writeFileSync(join(dir, "AGENTS.md"), "## Testing\n\nAlways run tests before committing.\n", "utf8"); + writeFileSync(join(dir, "CLAUDE.md"), "## Testing\n\nSkip tests for documentation-only edits.\n", "utf8"); + + const issues = await runDetector(instructionContradictionDetector, {}, { + target: dir, + language: "tsjs", + include: ["**/*.ts"], + }); + + assert.equal(issues.length, 0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/tests/helpers/runDetector.ts b/tests/helpers/runDetector.ts index 05b1fa4..c24ad42 100644 --- a/tests/helpers/runDetector.ts +++ b/tests/helpers/runDetector.ts @@ -10,6 +10,10 @@ export interface RunDetectorOptions { thresholds?: ScanThresholds; /** Minimum severity passed through on the synthetic ScanOptions. */ minSeverity?: ScanOptions["minSeverity"]; + /** Include globs passed through on the synthetic ScanOptions. */ + include?: string[]; + /** Exclude globs passed through on the synthetic ScanOptions. */ + exclude?: string[]; /** Naming-drift vocabulary override. */ vocabulary?: Record; /** Naming-drift: skip built-in concept groups. */ @@ -81,8 +85,8 @@ export async function runDetector( const scanOptions: ScanOptions = { cwd: "/", target: options.target ?? ".", - include: [], - exclude: [], + include: options.include ?? [], + exclude: options.exclude ?? [], minSeverity: options.minSeverity ?? "info", thresholds, rules: undefined, From b83c4ee3205a553de5f61b8141d133793f3604c5 Mon Sep 17 00:00:00 2001 From: ColumbusLabs <287001685+ColumbusLabs@users.noreply.github.com> Date: Sat, 20 Jun 2026 23:50:32 -0400 Subject: [PATCH 4/4] test: harden watch SIGINT timeout --- tests/cli/watch.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/cli/watch.test.ts b/tests/cli/watch.test.ts index b45a705..d92a7e8 100644 --- a/tests/cli/watch.test.ts +++ b/tests/cli/watch.test.ts @@ -81,13 +81,16 @@ describe("debtlens watch", () => { stderr += chunk; }); - const timeout = setTimeout(() => child.kill("SIGTERM"), 10000); + const timeout = setTimeout(() => child.kill("SIGTERM"), 30000); const close = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { child.on("close", (code, signal) => resolve({ code, signal })); }); clearTimeout(timeout); - assert.ok(close.code === 0 || close.signal === "SIGINT", stderr); + assert.ok( + close.code === 0 || close.signal === "SIGINT", + `watch exited with code=${close.code} signal=${close.signal}\nstdout:\n${stdout}\nstderr:\n${stderr}`, + ); assert.match(stdout, /DebtLens watch: watching/); }); });