From 018f622b2e378f06724492c469d5c789b7651e08 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Fri, 12 Jun 2026 21:30:46 -0700 Subject: [PATCH] test: Level 0 negative conformance cases Add tests/test_level0_negative.py with 19 negative tests covering: - Bad digest format (sha512 prefix, wrong hex length) on build_provenance.digest and tool_transcript.hash at both schema and conformance levels - iat in the future (now+3600) rejected by TR-ENV-002 - Unknown platform value ("quantum-chip") rejected by schema enum and TR-RTE-001 - slsa_level out of range (4 and -1) rejected by schema and TR-SCA-001 - transparency with http:// rejected by TR-ANC-001 - Missing build_provenance and missing transparency rejected by schema/conformance (missing runtime already covered in test_schema.py, not duplicated) All 86 existing tests still pass; total suite is 105 passed. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_level0_negative.py | 268 ++++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 tests/test_level0_negative.py diff --git a/tests/test_level0_negative.py b/tests/test_level0_negative.py new file mode 100644 index 0000000..6cedd01 --- /dev/null +++ b/tests/test_level0_negative.py @@ -0,0 +1,268 @@ +"""Level 0 negative conformance tests. + +Each test mutates one field of a valid fixture and asserts that either the +JSON Schema validator or the relevant conformance module rejects it. All +assertions check for FAIL / ValidationError -- never PASS. + +Coverage: +- Bad digest format on build_provenance.digest (sha512 prefix; wrong hex length) +- Bad digest format on tool_transcript.hash (sha512 prefix; wrong hex length) +- iat in the future (now + 3600) rejected by TR-ENV-002 +- Unknown platform value rejected by schema enum and TR-RTE-001 +- slsa_level out of range (4 and -1) rejected by schema and TR-SCA-001 +- transparency with http:// rejected by TR-ANC-001 +- Missing build_provenance rejected by schema (runtime already covered in test_schema.py) +""" + +from __future__ import annotations + +import copy +import time + +import jsonschema +import pytest + +from trace_tests.modules import tr_anc, tr_env, tr_rte, tr_sca +from trace_tests.result import Status + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _mutate(base: dict, *path_and_value) -> dict: + """Return a deep copy of *base* with one nested key replaced. + + Usage: _mutate(base, "runtime", "platform", "bad-value") + The last argument is the value; all preceding arguments are keys. + """ + *keys, value = path_and_value + record = copy.deepcopy(base) + node = record + for k in keys[:-1]: + node = node[k] + node[keys[-1]] = value + return record + + +def _delete(base: dict, *keys) -> dict: + """Return a deep copy of *base* with the nested key at *keys removed.""" + record = copy.deepcopy(base) + node = record + for k in keys[:-1]: + node = node[k] + del node[keys[-1]] + return record + + +# --------------------------------------------------------------------------- +# Bad digest format +# --------------------------------------------------------------------------- + +@pytest.mark.level0 +@pytest.mark.negative +class TestBadDigestFormat: + """Digest fields must match sha256:<64hex> or sha384:<96hex> exactly.""" + + # -- build_provenance.digest -- + + def test_build_digest_sha512_prefix_fails_schema(self, schema, valid_level0): + bad = _mutate(valid_level0, "build_provenance", "digest", + "sha512:" + "a" * 128) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + def test_build_digest_wrong_hex_length_fails_schema(self, schema, valid_level0): + # sha256 prefix but only 32 hex chars (should be 64) + bad = _mutate(valid_level0, "build_provenance", "digest", + "sha256:" + "a" * 32) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + def test_build_digest_sha512_prefix_fails_conformance(self, valid_level0): + bad = _mutate(valid_level0, "build_provenance", "digest", + "sha512:" + "a" * 128) + findings = tr_sca.check(bad) + digest_findings = [f for f in findings if f.code == "TR-SCA-002"] + assert digest_findings, "TR-SCA-002 finding expected" + assert all(f.failed() for f in digest_findings), ( + f"sha512-prefixed digest must fail TR-SCA-002; got {digest_findings}" + ) + + def test_build_digest_wrong_hex_length_fails_conformance(self, valid_level0): + bad = _mutate(valid_level0, "build_provenance", "digest", + "sha256:" + "a" * 32) + findings = tr_sca.check(bad) + digest_findings = [f for f in findings if f.code == "TR-SCA-002"] + assert digest_findings, "TR-SCA-002 finding expected" + assert all(f.failed() for f in digest_findings), ( + f"Wrong-length digest must fail TR-SCA-002; got {digest_findings}" + ) + + # -- tool_transcript.hash -- + + def test_transcript_hash_sha512_prefix_fails_schema(self, schema, valid_level0_with_transcript): + bad = _mutate(valid_level0_with_transcript, "tool_transcript", "hash", + "sha512:" + "a" * 128) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + def test_transcript_hash_wrong_hex_length_fails_schema(self, schema, valid_level0_with_transcript): + bad = _mutate(valid_level0_with_transcript, "tool_transcript", "hash", + "sha256:" + "a" * 32) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + +# --------------------------------------------------------------------------- +# iat in the future +# --------------------------------------------------------------------------- + +@pytest.mark.level0 +@pytest.mark.negative +class TestFutureIat: + """iat more than 60 seconds in the future must be rejected by TR-ENV-002.""" + + def test_future_iat_fails_conformance(self, valid_level0): + future_iat = int(time.time()) + 3600 + bad = _mutate(valid_level0, "iat", future_iat) + findings = tr_env.check(bad) + iat_findings = [f for f in findings if f.code == "TR-ENV-002"] + assert iat_findings, "TR-ENV-002 finding expected" + assert all(f.failed() for f in iat_findings), ( + f"Future iat must fail TR-ENV-002; got {iat_findings}" + ) + + def test_future_iat_finding_mentions_future(self, valid_level0): + future_iat = int(time.time()) + 3600 + bad = _mutate(valid_level0, "iat", future_iat) + findings = tr_env.check(bad) + iat_findings = [f for f in findings if f.code == "TR-ENV-002" and f.failed()] + assert iat_findings, "Expected a TR-ENV-002 FAIL finding" + assert any("future" in f.message.lower() for f in iat_findings), ( + "TR-ENV-002 failure message must mention 'future'" + ) + + +# --------------------------------------------------------------------------- +# Unknown platform +# --------------------------------------------------------------------------- + +@pytest.mark.level0 +@pytest.mark.negative +class TestUnknownPlatform: + """runtime.platform must be one of the registered values.""" + + def test_unknown_platform_fails_schema(self, schema, valid_level0): + bad = _mutate(valid_level0, "runtime", "platform", "quantum-chip") + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + def test_unknown_platform_fails_conformance(self, valid_level0): + bad = _mutate(valid_level0, "runtime", "platform", "quantum-chip") + findings = tr_rte.check(bad) + platform_findings = [f for f in findings if f.code == "TR-RTE-001"] + assert platform_findings, "TR-RTE-001 finding expected" + assert all(f.failed() for f in platform_findings), ( + f"Unknown platform must fail TR-RTE-001; got {platform_findings}" + ) + + +# --------------------------------------------------------------------------- +# slsa_level out of range +# --------------------------------------------------------------------------- + +@pytest.mark.level0 +@pytest.mark.negative +class TestSlsaLevelOutOfRange: + """build_provenance.slsa_level must be 1, 2, or 3.""" + + def test_slsa_level_4_fails_schema(self, schema, valid_level0): + bad = _mutate(valid_level0, "build_provenance", "slsa_level", 4) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + def test_slsa_level_negative_fails_schema(self, schema, valid_level0): + bad = _mutate(valid_level0, "build_provenance", "slsa_level", -1) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + def test_slsa_level_4_fails_conformance(self, valid_level0): + bad = _mutate(valid_level0, "build_provenance", "slsa_level", 4) + findings = tr_sca.check(bad) + slsa_findings = [f for f in findings if f.code == "TR-SCA-001"] + assert slsa_findings, "TR-SCA-001 finding expected" + assert all(f.failed() for f in slsa_findings), ( + f"slsa_level=4 must fail TR-SCA-001; got {slsa_findings}" + ) + + def test_slsa_level_negative_fails_conformance(self, valid_level0): + bad = _mutate(valid_level0, "build_provenance", "slsa_level", -1) + findings = tr_sca.check(bad) + slsa_findings = [f for f in findings if f.code == "TR-SCA-001"] + assert slsa_findings, "TR-SCA-001 finding expected" + assert all(f.failed() for f in slsa_findings), ( + f"slsa_level=-1 must fail TR-SCA-001; got {slsa_findings}" + ) + + +# --------------------------------------------------------------------------- +# transparency URI with http:// +# --------------------------------------------------------------------------- + +@pytest.mark.level0 +@pytest.mark.negative +class TestTransparencyHttpUri: + """transparency must be an https URI; bare http must be rejected.""" + + def test_http_transparency_fails_conformance(self, valid_level0): + bad = _mutate(valid_level0, "transparency", + "http://scitt.example.org/receipts/abc123") + findings = tr_anc.check(bad) + anc_findings = [f for f in findings if f.code == "TR-ANC-001"] + assert anc_findings, "TR-ANC-001 finding expected" + assert all(f.failed() for f in anc_findings), ( + f"http transparency URI must fail TR-ANC-001; got {anc_findings}" + ) + + def test_http_transparency_finding_mentions_scheme(self, valid_level0): + bad = _mutate(valid_level0, "transparency", + "http://scitt.example.org/receipts/abc123") + findings = tr_anc.check(bad) + fail_findings = [f for f in findings if f.code == "TR-ANC-001" and f.failed()] + assert fail_findings, "Expected a TR-ANC-001 FAIL finding" + assert any("http" in f.message.lower() for f in fail_findings), ( + "TR-ANC-001 failure message must reference the invalid scheme" + ) + + +# --------------------------------------------------------------------------- +# Missing required field: build_provenance +# (runtime is already covered in test_schema.py::test_missing_runtime_rejected) +# --------------------------------------------------------------------------- + +@pytest.mark.level0 +@pytest.mark.negative +class TestMissingRequiredFields: + """Records missing required top-level fields must fail schema validation.""" + + def test_missing_build_provenance_fails_schema(self, schema, valid_level0): + bad = _delete(valid_level0, "build_provenance") + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(bad, schema) + + def test_missing_build_provenance_fails_conformance(self, valid_level0): + bad = _delete(valid_level0, "build_provenance") + findings = tr_sca.check(bad) + assert findings, "TR-SCA findings expected when build_provenance is absent" + assert all(f.failed() for f in findings), ( + f"Missing build_provenance must produce only FAIL findings; got {findings}" + ) + + def test_missing_transparency_fails_conformance(self, valid_level0): + bad = _delete(valid_level0, "transparency") + findings = tr_anc.check(bad) + assert findings, "TR-ANC findings expected when transparency is absent" + assert all(f.failed() for f in findings), ( + f"Missing transparency must produce only FAIL findings; got {findings}" + )