From 3f89b08b24b5ef28cf1d12bbf739e8ade4eaab1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A0nh=20Nh=C3=A2n?= <60387689+NhanAZ@users.noreply.github.com> Date: Tue, 9 Jun 2026 23:27:21 +0700 Subject: [PATCH] test(scanner): add comprehensive test fixtures and CLI integration tests --- ROADMAP.md | 2 +- tests/cli/cli.test.ts | 73 +++++++++++++++++++ .../fixtures/placeholder-code/python-pass.py | 2 + tests/rules/opk-001-ai-credentials.test.ts | 7 ++ tests/rules/opk-003-placeholder-code.test.ts | 10 +++ 5 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 tests/cli/cli.test.ts create mode 100644 tests/fixtures/placeholder-code/python-pass.py diff --git a/ROADMAP.md b/ROADMAP.md index beb4bea..ebb2d68 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,7 +28,7 @@ - [x] Severity filtering (`--min-severity`) - [x] File/path exclusion patterns - [x] Performance optimization for large repositories -- [ ] More comprehensive test fixtures +- [x] More comprehensive test fixtures ## Future (not committed) diff --git a/tests/cli/cli.test.ts b/tests/cli/cli.test.ts new file mode 100644 index 0000000..b89547d --- /dev/null +++ b/tests/cli/cli.test.ts @@ -0,0 +1,73 @@ +import test from 'node:test'; +import assert from 'node:assert'; +import { execSync } from 'child_process'; +import * as path from 'path'; + +const CLI_PATH = path.resolve(__dirname, '..', '..', '..', 'bin', 'opk'); +const FIXTURES_DIR = path.resolve(__dirname, '..', '..', '..', 'tests', 'fixtures'); + +function runCli(args: string[]): { stdout: string; stderr: string; status: number | null } { + try { + const stdout = execSync(`node ${CLI_PATH} ${args.join(' ')}`, { encoding: 'utf-8' }); + return { stdout, stderr: '', status: 0 }; + } catch (error: any) { + return { + stdout: error.stdout ? error.stdout.toString() : '', + stderr: error.stderr ? error.stderr.toString() : '', + status: error.status, + }; + } +} + +test('CLI Integration', async (t) => { + await t.test('should exit with 0 when scanning a clean directory', () => { + const targetDir = path.join(FIXTURES_DIR, 'scanner-config'); + const result = runCli(['scan', targetDir]); + if (result.status !== 0) { + console.log('CLEAN DIRECTORY DEBUG:', result); + } + assert.strictEqual(result.status, 0); + assert.ok(result.stdout.includes('scanned') || result.stdout.includes('No findings')); + }); + + await t.test('should exit with 1 when findings are found', () => { + const targetDir = path.join(FIXTURES_DIR, 'ai-credentials'); + const result = runCli(['scan', targetDir]); + if (result.status !== 1) { + console.log('FINDINGS DEBUG:', result); + } + assert.strictEqual(result.status, 1); + assert.ok(result.stdout.includes('OPK-001')); + }); + + await t.test('should output JSON when --json flag is used', () => { + const targetDir = path.join(FIXTURES_DIR, 'prompt-artifacts'); + const result = runCli(['scan', targetDir, '--json']); + + // Attempt to parse stdout as JSON + const parsed = JSON.parse(result.stdout); + assert.ok(Array.isArray(parsed.findings)); + assert.ok(parsed.findings.length > 0); + assert.strictEqual(parsed.findings[0].ruleId, 'OPK-002'); + }); + + await t.test('should exit with 2 when an invalid --min-severity is provided', () => { + const result = runCli(['scan', '--min-severity', 'invalid_level']); + assert.strictEqual(result.status, 2); + assert.ok(result.stderr.includes('Invalid --min-severity value')); + }); + + await t.test('should exit with 2 when --min-severity is missing value', () => { + const result = runCli(['scan', '--min-severity']); + assert.strictEqual(result.status, 2); + assert.ok(result.stderr.includes('requires a value')); + }); + + await t.test('should filter out warnings when --min-severity error is provided', () => { + const targetDir = path.join(FIXTURES_DIR, 'prompt-artifacts'); + // prompt-artifacts only contains OPK-002 which is a warning. + const result = runCli(['scan', targetDir, '--min-severity', 'error']); + // Since only warnings exist, they are filtered out, so exit code should be 0. + assert.strictEqual(result.status, 0); + }); +}); diff --git a/tests/fixtures/placeholder-code/python-pass.py b/tests/fixtures/placeholder-code/python-pass.py new file mode 100644 index 0000000..4849c74 --- /dev/null +++ b/tests/fixtures/placeholder-code/python-pass.py @@ -0,0 +1,2 @@ +def my_function(): + pass # TODO: implement this diff --git a/tests/rules/opk-001-ai-credentials.test.ts b/tests/rules/opk-001-ai-credentials.test.ts index 05baf4b..311fa20 100644 --- a/tests/rules/opk-001-ai-credentials.test.ts +++ b/tests/rules/opk-001-ai-credentials.test.ts @@ -59,6 +59,13 @@ describe('OPK-001: Hardcoded AI Credentials', () => { assert.strictEqual(findings.length, 0, 'Expected no findings for env var reference'); }); + it('should NOT flag credentials in .env files', async () => { + const context = makeContext(['.env', '.env.local', '.env.example']); + const findings = await rule.check(context); + + assert.strictEqual(findings.length, 0, 'Expected no findings for skipped .env files'); + }); + it('should NOT flag clean files without credentials', async () => { const context = makeContext(['clean-file.ts']); const findings = await rule.check(context); diff --git a/tests/rules/opk-003-placeholder-code.test.ts b/tests/rules/opk-003-placeholder-code.test.ts index eb6ac08..7e0ca59 100644 --- a/tests/rules/opk-003-placeholder-code.test.ts +++ b/tests/rules/opk-003-placeholder-code.test.ts @@ -58,6 +58,16 @@ describe('OPK-003: Placeholder Implementation', () => { assert.strictEqual(findings.length, 0, 'Expected no findings for clean file'); }); + it('should detect Python pass with TODO', async () => { + const context = makeContext(['python-pass.py']); + const findings = await rule.check(context); + + assert.ok(findings.length > 0, 'Expected finding for Python pass with TODO'); + assert.strictEqual(findings[0].ruleId, 'OPK-003'); + // It may match the "TODO implement" pattern before the Python-specific pattern + assert.ok(findings[0].message.includes('TODO')); + }); + it('should have correct rule metadata', () => { assert.strictEqual(rule.id, 'OPK-003'); assert.strictEqual(rule.name, 'Placeholder Implementation');