diff --git a/Justfile b/Justfile index 39041974066..1851cfc9995 100644 --- a/Justfile +++ b/Justfile @@ -181,6 +181,7 @@ test-tests *args: -n {{ xdist_workers }} \ --basetemp="{{ output_dir }}/test-tests/tmp" \ --ignore=src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py \ + --ignore=src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint_e2e.py \ "$@" \ src @@ -192,6 +193,7 @@ test-tests-pypy *args: -n auto --maxprocesses 6 \ --basetemp="{{ output_dir }}/test-tests-pypy/tmp" \ --ignore=src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py \ + --ignore=src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint_e2e.py \ "$@" \ src @@ -205,6 +207,31 @@ test-tests-bench *args: "$@" \ packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py +# Scan tests for gas assertions and apply @pytest.mark.gas_check to them +[group('gas check')] +mark_gas_tests *args: + @mkdir -p "{{ output_dir }}/mark-gas-tests/tmp" + uv run fill \ + --detect-gas-checks \ + --gas-check-report="{{ output_dir }}/mark-gas-tests/gas_check_report.json" \ + -m "not slow" \ + -n {{ xdist_workers }} --dist=loadgroup \ + --clean \ + --skip-index \ + --output="{{ output_dir }}/mark-gas-tests/fixtures" \ + "$@" + uv run python scripts/mark_tests.py \ + "{{ output_dir }}/mark-gas-tests/gas_check_report.json" + +# Run the gas_taint plugin end-to-end (pytester) tests +[group('gas check')] +mark_gas_tests_test *args: + @mkdir -p "{{ output_dir }}/mark-gas-tests-test/tmp" + uv run pytest \ + --basetemp="{{ output_dir }}/mark-gas-tests-test/tmp" \ + "$@" \ + packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint_e2e.py + # Run CI release script integration tests [group('unit tests')] test-ci-scripts *args: diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py index 5a733c09ad5..9a48c5191a7 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py @@ -1700,6 +1700,18 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super(BaseTestWrapper, self).__init__(*args, **kwargs) + if getattr(request.config, "_gas_taint_enabled", False): + from .gas_taint import collect_taint_hits + + hits = collect_taint_hits(self, request.node) + if hits: + request.config._gas_taint_results[ # type: ignore[attr-defined] + request.node.nodeid + ] = hits + request.node.user_properties.append( + ("gas_taint_hits", hits) + ) + # Get the filling session from config session: FillingSession = request.config.filling_session # type: ignore assert isinstance(session, FillingSession) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/gas_taint.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/gas_taint.py new file mode 100644 index 00000000000..0d6db824ac3 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/gas_taint.py @@ -0,0 +1,520 @@ +""" +Pytest plugin that detects tests which positively assert specific gas values. + +Enabled with ``--detect-gas-checks``. After each test body constructs its +``StateTest`` / ``BlockchainTest`` (but before the t8n runs), the walker +inspects two categories of sinks: + +1. **Field-name implies gas assertion** — checked by presence (``is not + None``). No taint needed because the field name itself is the signal: + + - ``tx.expected_receipt.{cumulative_gas_used, gas_used, blob_gas_used}`` + - ``block.header_verify.{gas_used, blob_gas_used}`` + - ``block.expected_gas_used`` + - ``self.expected_benchmark_gas_used`` + +2. **Storage slots in ``post``** — checked via taint propagation, since + a storage slot can hold any value (counter, identifier, etc.) and the + field alone doesn't tell us whether it's gas-related. + +For the storage sink the plugin wraps gas-source functions +(``Bytecode.gas_cost``, ``fork.gas_costs()``, ``opcode_gas_calculator``, +``transaction_intrinsic_cost_calculator``) so they return a ``GasTainted`` +int subclass carrying provenance, and patches ``Number.__new__`` / +``FixedSizeHexNumber.__new__`` so taint survives construction of +``HashInt`` / ``HexNumber`` / ``ZeroPaddedHexNumber``. + +Tests with any sink hit get a ``gas_check`` marker attached and an entry +written to the JSON report. Tests that expect a transaction or block +exception (OOG-style) are skipped — their gas value lives on +``tx.gas_limit`` (an input, not a sink) and would never appear here. +""" + +from __future__ import annotations + +import json +from dataclasses import fields, replace +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + List, + Set, + Tuple, +) + +import pytest + +from execution_testing.specs.base import BaseTest +from execution_testing.specs.benchmark import BenchmarkTest +from execution_testing.specs.blockchain import Block, BlockchainTest +from execution_testing.specs.state import StateTest +from execution_testing.test_types.account_types import Alloc +from execution_testing.test_types.transaction_types import Transaction +from execution_testing.test_types.utils import Removable +from execution_testing.vm.bases import OpcodeBase + +if TYPE_CHECKING: + from xdist.workermanage import WorkerController + +# --------------------------------------------------------------------------- +# Taint carrier +# --------------------------------------------------------------------------- + + +class GasTainted(int): + """An int that remembers it was computed from a gas-cost source.""" + + origins: Tuple[str, ...] + + def __new__( + cls, value: int, origins: Tuple[str, ...] = () + ) -> "GasTainted": + """Build a tainted int with attached provenance tuple.""" + inst = super().__new__(cls, int(value)) + inst.origins = origins or ("?",) + return inst + + def _merge(self, other: Any) -> Tuple[str, ...]: + o = getattr(other, "origins", ()) + return tuple(dict.fromkeys((*self.origins, *o))) + + def _propagate(self, raw: Any, other: Any) -> Any: + if raw is NotImplemented: + return NotImplemented + return GasTainted(raw, self._merge(other)) + + def __add__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__add__(self, o), o) + + def __radd__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__radd__(self, o), o) + + def __sub__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__sub__(self, o), o) + + def __rsub__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__rsub__(self, o), o) + + def __mul__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__mul__(self, o), o) + + def __rmul__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__rmul__(self, o), o) + + def __floordiv__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__floordiv__(self, o), o) + + def __rfloordiv__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__rfloordiv__(self, o), o) + + def __mod__(self, o: Any) -> Any: # noqa: D105 + return self._propagate(int.__mod__(self, o), o) + + def __neg__(self) -> "GasTainted": # noqa: D105 + return GasTainted(int.__neg__(self), self.origins) + + +def _origins_of(value: Any) -> Tuple[str, ...] | None: + return getattr(value, "origins", None) + + +def _is_tainted(value: Any) -> bool: + return _origins_of(value) is not None + + +# --------------------------------------------------------------------------- +# Source instrumentation +# --------------------------------------------------------------------------- + +_installed = False +_originals: List[Callable[[], None]] = [] + + +def _all_subclasses(cls: type) -> Set[type]: + out: Set[type] = set() + for sub in cls.__subclasses__(): + out.add(sub) + out.update(_all_subclasses(sub)) + return out + + +def _patch_classmethod_if_local( + cls: type, name: str, wrap: Callable[[type, Callable, tuple, dict], Any] +) -> None: + """Wrap ``cls.name`` only if defined on ``cls`` itself (not inherited).""" + if name not in cls.__dict__: + return + original_cm = cls.__dict__[name] + if not isinstance(original_cm, classmethod): + return + original_fn = original_cm.__func__ + + def wrapped(c: type, *args: Any, **kwargs: Any) -> Any: + return wrap(c, original_fn, args, kwargs) + + setattr(cls, name, classmethod(wrapped)) + + def revert() -> None: + setattr(cls, name, original_cm) + + _originals.append(revert) + + +def _wrap_gas_costs(c: type, fn: Callable, args: tuple, kwargs: dict) -> Any: + gc = fn(c, *args, **kwargs) + patched: Dict[str, Any] = {} + for f in fields(gc): + v = getattr(gc, f.name) + if isinstance(v, int) and not _is_tainted(v): + patched[f.name] = GasTainted(v, (f"gas_costs.{f.name}",)) + return replace(gc, **patched) if patched else gc + + +def _wrap_intrinsic(c: type, fn: Callable, args: tuple, kwargs: dict) -> Any: + inner = fn(c, *args, **kwargs) + + def calc(*a: Any, **kw: Any) -> Any: + result = inner(*a, **kw) + if isinstance(result, int) and not _is_tainted(result): + return GasTainted(result, ("intrinsic_gas",)) + return result + + return calc + + +def _wrap_opcode_calc(c: type, fn: Callable, args: tuple, kwargs: dict) -> Any: + inner = fn(c, *args, **kwargs) + + def calc(opcode: OpcodeBase) -> int: + result = inner(opcode) + if isinstance(result, int) and not _is_tainted(result): + name = getattr(opcode, "_name_", None) or str(opcode) + return GasTainted(result, (f"opcode_gas[{name}]",)) + return result + + return calc + + +def install_taint() -> None: + """Install all gas-source patches. Idempotent.""" + global _installed + if _installed: + return + _installed = True + + # Bytecode.gas_cost - taint the aggregate result + from execution_testing.vm.bytecode import Bytecode + + original_bc = Bytecode.gas_cost + + def gas_cost(self: Any, fork: Any) -> Any: + result = original_bc(self, fork) + if isinstance(result, int) and not _is_tainted(result): + return GasTainted(result, ("Bytecode.gas_cost",)) + return result + + Bytecode.gas_cost = gas_cost # type: ignore[method-assign] + _originals.append(lambda: setattr(Bytecode, "gas_cost", original_bc)) + + # Concrete forks: walk subclasses and wrap any locally-defined + # gas_costs / transaction_intrinsic_cost_calculator / opcode_gas_calculator + from execution_testing.forks.base_fork import BaseFork + + for cls in _all_subclasses(BaseFork): + _patch_classmethod_if_local(cls, "gas_costs", _wrap_gas_costs) + _patch_classmethod_if_local( + cls, "transaction_intrinsic_cost_calculator", _wrap_intrinsic + ) + _patch_classmethod_if_local( + cls, "opcode_gas_calculator", _wrap_opcode_calc + ) + + # Number.__new__ / FixedSizeHexNumber.__new__ - preserve origins through + # HexNumber, ZeroPaddedHexNumber, HashInt, etc. (two separate hierarchies). + from execution_testing.base_types.base_types import ( + FixedSizeHexNumber, + Number, + ) + + def _wrap_new(target: type) -> None: + orig = target.__new__ + + def new(cls: type, input_number: Any) -> Any: + inst: Any = orig(cls, input_number) + origins = getattr(input_number, "origins", None) + if origins is not None: + try: + inst.origins = origins + except (AttributeError, TypeError): + pass + return inst + + target.__new__ = new # type: ignore[assignment,method-assign] + + def revert() -> None: + target.__new__ = orig # type: ignore[method-assign] + + _originals.append(revert) + + _wrap_new(Number) + _wrap_new(FixedSizeHexNumber) + + +def uninstall_taint() -> None: + """Revert all patches in LIFO order. Used by tests of the plugin.""" + global _installed + while _originals: + try: + _originals.pop()() + except Exception: + pass + _installed = False + + +# --------------------------------------------------------------------------- +# Sink walker +# --------------------------------------------------------------------------- + + +def _record_tainted( + hits: List[dict], kind: str, location: str, value: int | None +) -> None: + """Record only if value carries gas-source taint (used for storage).""" + if value is None: + return + origins = _origins_of(value) + if origins is None: + return + hits.append( + { + "kind": kind, + "location": location, + "value": int(value), + "origins": list(origins), + } + ) + + +def _record_present( + hits: List[dict], kind: str, location: str, value: int | None +) -> None: + """Record if value is set (field-name-implies-gas-assertion sinks).""" + if value is None: + return + hit: Dict[str, object] = { + "kind": kind, + "location": location, + "value": int(value), + } + origins = _origins_of(value) + if origins is not None: + hit["origins"] = list(origins) + hits.append(hit) + + +def _walk_storage(hits: List[dict], post: Alloc | None) -> None: + """Storage slots are general-purpose; only flag tainted values.""" + if post is None: + return + for address, account in post.items(): + if account is None: + continue + for slot, value in account.storage.root.items(): + _record_tainted( + hits, + "storage", + f"{address}:{int(slot)}", + value, + ) + + +def _walk_receipt(hits: List[dict], tx: Transaction | None) -> None: + """Receipt gas fields are self-identifying — flag on presence.""" + if tx is None: + return + receipt = tx.expected_receipt + if receipt is None: + return + _record_present( + hits, "receipt", "cumulative_gas_used", receipt.cumulative_gas_used + ) + _record_present(hits, "receipt", "gas_used", receipt.gas_used) + _record_present(hits, "receipt", "blob_gas_used", receipt.blob_gas_used) + + +def _walk_header_and_block(hits: List[dict], block: Block, i: int) -> None: + """Header gas fields and expected_gas_used are self-identifying.""" + header_verify = block.header_verify + if header_verify is not None: + _record_present( + hits, "header", f"block[{i}].gas_used", header_verify.gas_used + ) + # blob_gas_used is ``Removable | HexNumber | None`` — the + # ``Removable`` sentinel means "delete this from the verified + # header" and is not a gas assertion. + blob_gas_used = header_verify.blob_gas_used + if not isinstance(blob_gas_used, Removable): + _record_present( + hits, + "header", + f"block[{i}].blob_gas_used", + blob_gas_used, + ) + _record_present( + hits, + "block_expected_gas_used", + f"block[{i}].expected_gas_used", + block.expected_gas_used, + ) + + +def _is_oog_test(test: BaseTest, node: pytest.Item | None) -> bool: + if isinstance(test, StateTest) and test.tx.error is not None: + return True + if isinstance(test, StateTest) and test.block_exception is not None: + return True + if ( + node is not None + and node.get_closest_marker("exception_test") is not None + ): + return True + return False + + +def collect_taint_hits(test: BaseTest, node: pytest.Item | None) -> List[dict]: + """Walk all known gas-assertion sinks on the test object.""" + if _is_oog_test(test, node): + return [] + + hits: List[dict] = [] + if isinstance(test, StateTest): + _walk_storage(hits, test.post) + _walk_receipt(hits, test.tx) + elif isinstance(test, BlockchainTest): + _walk_storage(hits, test.post) + for i, block in enumerate(test.blocks): + _walk_header_and_block(hits, block, i) + for tx in block.txs: + _walk_receipt(hits, tx) + + # ``expected_benchmark_gas_used`` is auto-defaulted on every test by + # the filler, so only treat it as a sink on actual BenchmarkTest + # instances. + if isinstance(test, BenchmarkTest): + _record_present( + hits, + "benchmark", + "expected_benchmark_gas_used", + test.expected_benchmark_gas_used, + ) + + return hits + + +# --------------------------------------------------------------------------- +# Pytest hooks +# --------------------------------------------------------------------------- + + +_RESULTS_ATTR = "_gas_taint_results" +_ENABLED_ATTR = "_gas_taint_enabled" + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add ``--detect-gas-checks`` / ``--gas-check-report`` options.""" + group = parser.getgroup( + "gas-taint", "Detect tests that assert specific gas values" + ) + group.addoption( + "--detect-gas-checks", + action="store_true", + dest="detect_gas_checks", + default=False, + help=( + "Instrument gas-source functions, propagate provenance, and " + "emit a JSON report listing tests whose post-state asserts a " + "gas-derived value." + ), + ) + group.addoption( + "--gas-check-report", + action="store", + dest="gas_check_report", + default="gas_check_report.json", + help="Output path for the gas-check JSON report.", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + """Install taint and initialize result storage when enabled.""" + config.addinivalue_line( + "markers", + "gas_check: test asserts a specific gas value (auto-applied).", + ) + if not config.getoption("detect_gas_checks", default=False): + return + install_taint() + setattr(config, _ENABLED_ATTR, True) + setattr(config, _RESULTS_ATTR, {}) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, call: pytest.CallInfo +) -> Generator[None, None, None]: + """Attach a ``gas_check`` marker if the item has taint hits.""" + yield + if call.when != "call": + return + for key, value in item.user_properties: + if key == "gas_taint_hits" and value: + item.add_marker(pytest.mark.gas_check) + break + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + """Forward worker results to master, or write JSON on master.""" + del exitstatus + config = session.config + if not getattr(config, _ENABLED_ATTR, False): + return + results = getattr(config, _RESULTS_ATTR, {}) + + try: + import xdist + + is_worker = xdist.is_xdist_worker(session) + except ImportError: + is_worker = False + + if is_worker: + # Send worker's results to master via workeroutput. + config.workeroutput["gas_taint_results"] = results # type: ignore[attr-defined] + return + + path = Path(config.getoption("gas_check_report")) + path.write_text(json.dumps(results, indent=2, sort_keys=True)) + + +def pytest_testnodedown( + node: "WorkerController", error: object | None +) -> None: + """Aggregate worker results into the master's results dict.""" + del error + config = node.config + if not getattr(config, _ENABLED_ATTR, False): + return + worker_results = getattr(node, "workeroutput", {}).get( + "gas_taint_results", {} + ) + results = getattr(config, _RESULTS_ATTR, None) + if results is None: + return + results.update(worker_results) + path = Path(config.getoption("gas_check_report")) + path.write_text(json.dumps(results, indent=2, sort_keys=True)) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint.py new file mode 100644 index 00000000000..b712cda217c --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint.py @@ -0,0 +1,524 @@ +"""Unit tests for the gas_taint plugin.""" + +from __future__ import annotations + +from typing import Any, Callable, Iterator, cast +from unittest.mock import MagicMock + +import pytest + +from execution_testing.base_types import Account, Address +from execution_testing.base_types.base_types import ( + HashInt, + HexNumber, + ZeroPaddedHexNumber, +) +from execution_testing.base_types.composite_types import Storage +from execution_testing.cli.pytest_commands.plugins.filler.gas_taint import ( + GasTainted, + _is_tainted, + _origins_of, + collect_taint_hits, + install_taint, + uninstall_taint, +) +from execution_testing.exceptions.exceptions.transaction import ( + TransactionException, +) +from execution_testing.forks.forks.forks import Cancun +from execution_testing.specs.benchmark import BenchmarkTest +from execution_testing.specs.blockchain import Block, BlockchainTest, Header +from execution_testing.specs.state import StateTest +from execution_testing.test_types.account_types import Alloc +from execution_testing.test_types.receipt_types import TransactionReceipt +from execution_testing.test_types.transaction_types import Transaction + +# --------------------------------------------------------------------------- +# GasTainted carrier +# --------------------------------------------------------------------------- + + +class TestGasTaintedCarrier: + """The int subclass that carries provenance through arithmetic.""" + + def test_construction_attaches_origins(self) -> None: + """Test construction attaches origins.""" + t = GasTainted(42, ("source.X",)) + assert int(t) == 42 + assert t.origins == ("source.X",) + assert isinstance(t, int) + + def test_default_origin_when_empty(self) -> None: + """Test default origin when empty.""" + # An empty tuple gets replaced with the "?" placeholder so callers + # can always rely on ``origins`` being non-empty. + t = GasTainted(7) + assert t.origins == ("?",) + + def test_is_tainted_and_origins_of(self) -> None: + """Test is tainted and origins of.""" + assert _is_tainted(GasTainted(1, ("X",))) + assert not _is_tainted(1) + assert _origins_of(GasTainted(5, ("Y",))) == ("Y",) + assert _origins_of(5) is None + + @pytest.mark.parametrize( + "op,expected", + [ + (lambda a, b: a + b, 13), + (lambda a, b: a - b, 7), + (lambda a, b: a * b, 30), + (lambda a, b: a // b, 3), + (lambda a, b: a % b, 1), + ], + ) + def test_arithmetic_preserves_taint( + self, op: Callable[[int, int], int], expected: int + ) -> None: + """Test arithmetic preserves taint.""" + a = GasTainted(10, ("A",)) + result = op(a, 3) + assert isinstance(result, GasTainted) + assert int(result) == expected + assert result.origins == ("A",) + + def test_negation_preserves_taint(self) -> None: + """Test negation preserves taint.""" + n = -GasTainted(5, ("X",)) + assert isinstance(n, GasTainted) + assert int(n) == -5 + assert n.origins == ("X",) + + def test_reflected_ops_preserve_taint(self) -> None: + """Test reflected ops preserve taint.""" + # ``5 + GasTainted(3)`` triggers __radd__ on the tainted side. + result = 5 + GasTainted(3, ("R",)) + assert isinstance(result, GasTainted) + assert int(result) == 8 + assert result.origins == ("R",) + + def test_merge_origins_unions(self) -> None: + """Test merge origins unions.""" + a = GasTainted(2, ("A",)) + b = GasTainted(3, ("B",)) + s = a + b + # Order preserved (a's origins first, then b's). + assert s.origins == ("A", "B") + + def test_merge_origins_deduplicates(self) -> None: + """Test merge origins deduplicates.""" + a = GasTainted(2, ("X", "Y")) + b = GasTainted(3, ("Y", "Z")) + s = a + b + assert s.origins == ("X", "Y", "Z") + + def test_rmul_with_bytes_returns_notimplemented_gracefully(self) -> None: + """Test rmul with bytes returns notimplemented gracefully.""" + # ``b"\x01" * GasTainted(n)`` is a real expression in tests that + # compute calldata sizes from gas. ``bytes.__mul__`` doesn't accept + # an arbitrary int subclass, so Python falls back to + # ``GasTainted.__rmul__(n, b"...")``; ``int.__rmul__`` returns + # ``NotImplemented``, which the dunder must forward — wrapping it + # in a new GasTainted blows up on ``int(NotImplemented)``. + result = b"\x01" * GasTainted(5, ("X",)) + assert result == b"\x01\x01\x01\x01\x01" + assert type(result) is bytes + + +# --------------------------------------------------------------------------- +# install_taint / uninstall_taint +# --------------------------------------------------------------------------- + + +@pytest.fixture +def taint_installed() -> Iterator[None]: + """Install taint patches for one test, then revert.""" + install_taint() + try: + yield + finally: + uninstall_taint() + + +@pytest.mark.usefixtures("taint_installed") +class TestTaintInstallation: + """The monkey-patches that make gas values carry provenance.""" + + def test_bytecode_gas_cost_is_tainted(self) -> None: + """Test bytecode gas cost is tainted.""" + from execution_testing import Op + from execution_testing.forks.forks.forks import Cancun + + result = Op.PUSH1[1].gas_cost(Cancun) + assert _is_tainted(result) + # Origin can be either the per-opcode label (when the wrapped + # opcode_gas_calculator already returns tainted gas_costs.X) or + # the outer Bytecode.gas_cost label (when summing untainted + # constants). The important guarantee is that *some* gas origin + # is recorded. + assert any("gas" in o for o in cast(GasTainted, result).origins) + + def test_fork_gas_costs_fields_are_tainted(self) -> None: + """Test fork gas costs fields are tainted.""" + from execution_testing.forks.forks.forks import Cancun + + gc = Cancun.gas_costs() + assert _is_tainted(gc.STORAGE_SET) + assert ( + "gas_costs.STORAGE_SET" in cast(GasTainted, gc.STORAGE_SET).origins + ) + + def test_hashint_preserves_taint(self) -> None: + """Test hashint preserves taint.""" + # HashInt inherits from FixedSizeHexNumber, *not* from Number. + # This is the case that motivated patching both __new__ methods. + tainted = GasTainted(123, ("src",)) + h = HashInt(tainted) + assert _is_tainted(h) + assert cast(GasTainted, h).origins == ("src",) + + def test_hexnumber_preserves_taint(self) -> None: + """Test hexnumber preserves taint.""" + # HexNumber inherits from Number — covered by the Number.__new__ + # patch. + tainted = GasTainted(456, ("src",)) + x = HexNumber(tainted) + assert _is_tainted(x) + + def test_zero_padded_hexnumber_preserves_taint(self) -> None: + """Test zero padded hexnumber preserves taint.""" + tainted = GasTainted(789, ("src",)) + x = ZeroPaddedHexNumber(tainted) + assert _is_tainted(x) + + def test_plain_value_through_constructors_is_not_tainted(self) -> None: + """Test plain value through constructors is not tainted.""" + # The patch must be opt-in: an ordinary int passed through these + # constructors must NOT become tainted, or every storage slot + # would be flagged. + assert not _is_tainted(HashInt(42)) + assert not _is_tainted(HexNumber(42)) + + def test_storage_dict_preserves_taint(self) -> None: + """Test storage dict preserves taint.""" + # Storage is a Pydantic RootModel; the value goes through HashInt + # coercion. This is the critical end-to-end taint path. + tainted = GasTainted(99, ("from_test",)) + s = Storage(cast(Any, {0: tainted})) + stored = next(iter(s.root.values())) + assert _is_tainted(stored) + assert cast(GasTainted, stored).origins == ("from_test",) + + def test_install_is_idempotent(self) -> None: + """Test install is idempotent.""" + install_taint() + try: + install_taint() # second call must not double-wrap. + from execution_testing import Op + from execution_testing.forks.forks.forks import Cancun + + result = Op.PUSH1[1].gas_cost(Cancun) + # If install double-wrapped, origins would contain + # 'Bytecode.gas_cost' twice. The carrier dedupes within a + # tuple, but the double-wrap would still produce nested + # GasTainted instances and confused arithmetic. + assert _is_tainted(result) + # Sanity check: it's a flat int subclass, not a nested + # GasTainted-of-GasTainted. + assert type(result).__name__ == "GasTainted" + assert type(int(result)) is int + finally: + uninstall_taint() + + def test_uninstall_reverts_patches(self) -> None: + """Test uninstall reverts patches.""" + install_taint() + uninstall_taint() + from execution_testing import Op + from execution_testing.forks.forks.forks import Cancun + + result = Op.PUSH1[1].gas_cost(Cancun) + assert not _is_tainted(result) + gc = Cancun.gas_costs() + assert not _is_tainted(gc.STORAGE_SET) + + +# --------------------------------------------------------------------------- +# collect_taint_hits walker +# +# These tests instantiate real ``StateTest`` / ``BlockchainTest`` / +# ``BenchmarkTest`` Pydantic models so that the walker's typed signature +# guards the field surface. Renaming or retyping a sink field becomes a +# compile-time error rather than a silent miss. +# --------------------------------------------------------------------------- + + +_ADDR = Address(0x1234) + + +def _state_test_with_storage(value: int | None) -> StateTest: + storage = ( + Storage(cast(Any, {0: value})) if value is not None else Storage() + ) + return StateTest( + fork=Cancun, + pre=Alloc(), + post=Alloc(cast(Any, {_ADDR: Account(storage=storage)})), + tx=Transaction(), + ) + + +def _node_with_marker(name: str | None) -> pytest.Item: + """Mock ``pytest.Item`` whose ``get_closest_marker`` returns truthy.""" + node = MagicMock(spec=pytest.Item) + node.get_closest_marker.side_effect = lambda n: ( + object() if name is not None and n == name else None + ) + return cast(pytest.Item, node) + + +@pytest.mark.usefixtures("taint_installed") +class TestStorageSink: + """ + ``post[addr].storage[slot]`` requires taint to flag. + + The ``taint_installed`` fixture is needed so that ``GasTainted`` + values survive the ``Storage`` Pydantic validator (which constructs + a ``HashInt`` for each value); the walker itself doesn't depend on + installation. + """ + + def test_tainted_storage_value_is_recorded(self) -> None: + """Test tainted storage value is recorded.""" + test = _state_test_with_storage(GasTainted(42, ("Y",))) + hits = collect_taint_hits(test, None) + assert len(hits) == 1 + assert hits[0]["kind"] == "storage" + assert hits[0]["value"] == 42 + assert hits[0]["origins"] == ["Y"] + assert str(_ADDR) in hits[0]["location"] + + def test_untainted_storage_value_is_skipped(self) -> None: + """Test untainted storage value is skipped.""" + # A plain int in storage — the whole point of using taint here is + # that storage slots can hold anything. + test = _state_test_with_storage(42) + assert collect_taint_hits(test, None) == [] + + def test_account_with_no_storage_is_skipped(self) -> None: + """Test account with no storage is skipped.""" + test = StateTest( + fork=Cancun, + pre=Alloc(), + post=Alloc(cast(Any, {_ADDR: Account()})), + tx=Transaction(), + ) + assert collect_taint_hits(test, None) == [] + + def test_empty_post_is_skipped(self) -> None: + """Test empty post is skipped.""" + test = StateTest( + fork=Cancun, pre=Alloc(), post=Alloc(), tx=Transaction() + ) + assert collect_taint_hits(test, None) == [] + + +class TestReceiptSink: + """``expected_receipt`` fields are flagged on presence, not taint.""" + + def test_cumulative_gas_used_recorded(self) -> None: + """Test cumulative gas used recorded.""" + tx = Transaction() + tx.expected_receipt = TransactionReceipt(cumulative_gas_used=21000) + test = StateTest(fork=Cancun, pre=Alloc(), post=Alloc(), tx=tx) + hits = collect_taint_hits(test, None) + assert len(hits) == 1 + assert hits[0]["kind"] == "receipt" + assert hits[0]["location"] == "cumulative_gas_used" + assert hits[0]["value"] == 21000 + # No taint involved — origins absent from the hit dict. + assert "origins" not in hits[0] + + def test_all_three_receipt_fields(self) -> None: + """Test all three receipt fields.""" + tx = Transaction() + tx.expected_receipt = TransactionReceipt( + cumulative_gas_used=100, gas_used=200, blob_gas_used=300 + ) + test = StateTest(fork=Cancun, pre=Alloc(), post=Alloc(), tx=tx) + hits = collect_taint_hits(test, None) + locations = {h["location"] for h in hits} + assert locations == { + "cumulative_gas_used", + "gas_used", + "blob_gas_used", + } + + def test_no_expected_receipt_skipped(self) -> None: + """Test no expected receipt skipped.""" + test = StateTest( + fork=Cancun, pre=Alloc(), post=Alloc(), tx=Transaction() + ) + assert collect_taint_hits(test, None) == [] + + +class TestHeaderAndBlockSinks: + """``header_verify`` and ``expected_gas_used`` flagged on presence.""" + + def test_header_verify_blob_gas_used(self) -> None: + """Test header verify blob gas used.""" + block = Block(header_verify=Header(blob_gas_used=131072)) + test = BlockchainTest( + fork=Cancun, pre=Alloc(), post=Alloc(), blocks=[block] + ) + hits = collect_taint_hits(test, None) + assert len(hits) == 1 + assert hits[0]["kind"] == "header" + assert hits[0]["location"] == "block[0].blob_gas_used" + assert hits[0]["value"] == 131072 + + def test_block_expected_gas_used(self) -> None: + """Test block expected gas used.""" + block = Block(expected_gas_used=HexNumber(199_156)) + test = BlockchainTest( + fork=Cancun, pre=Alloc(), post=Alloc(), blocks=[block] + ) + hits = collect_taint_hits(test, None) + # Block inherits gas_used / blob_gas_used from Header (both + # default None), so only the explicit expected_gas_used is + # recorded. + assert len(hits) == 1 + assert hits[0]["kind"] == "block_expected_gas_used" + assert hits[0]["value"] == 199_156 + + def test_multiple_blocks_indexed(self) -> None: + """Test multiple blocks indexed.""" + blocks = [ + Block(expected_gas_used=HexNumber(100)), + Block(expected_gas_used=HexNumber(200)), + ] + test = BlockchainTest( + fork=Cancun, pre=Alloc(), post=Alloc(), blocks=blocks + ) + hits = collect_taint_hits(test, None) + assert {h["location"] for h in hits} == { + "block[0].expected_gas_used", + "block[1].expected_gas_used", + } + + +class TestBenchmarkSink: + """``expected_benchmark_gas_used`` is recorded only on BenchmarkTest.""" + + def test_skipped_on_non_benchmark_test(self) -> None: + """Test skipped on non benchmark test.""" + # The filler auto-defaults expected_benchmark_gas_used on every + # test; without the isinstance(test, BenchmarkTest) gate, this + # would emit a benchmark hit even on a plain StateTest. + test = StateTest( + fork=Cancun, pre=Alloc(), post=Alloc(), tx=Transaction() + ) + # Sanity: set it just like the filler would for non-benchmark + # tests. + test.expected_benchmark_gas_used = HexNumber(120_000_000) + assert collect_taint_hits(test, None) == [] + + def test_recorded_on_benchmark_test(self) -> None: + """Test recorded on benchmark test.""" + bench = BenchmarkTest( + fork=Cancun, + pre=Alloc(), + tx=Transaction(), + expected_benchmark_gas_used=HexNumber(99_999_999), + ) + hits = collect_taint_hits(bench, None) + assert len(hits) == 1 + assert hits[0]["kind"] == "benchmark" + assert hits[0]["value"] == 99_999_999 + + +@pytest.mark.usefixtures("taint_installed") +class TestOOGExclusion: + """OOG-style tests are excluded — they don't write to positive sinks.""" + + def test_tx_error_set_excludes_test(self) -> None: + """Test tx error set excludes test.""" + # Even with a tainted storage value, a tx.error means the test + # expects rejection — drop it. + tx = Transaction(error=TransactionException.INTRINSIC_GAS_TOO_LOW) + test = StateTest( + fork=Cancun, + pre=Alloc(), + post=Alloc( + cast( + Any, + { + _ADDR: Account( + storage=Storage( + cast(Any, {0: GasTainted(1, ("X",))}) + ) + ) + }, + ) + ), + tx=tx, + ) + assert collect_taint_hits(test, None) == [] + + def test_block_exception_excludes_test(self) -> None: + """Test block exception excludes test.""" + test = StateTest( + fork=Cancun, + pre=Alloc(), + post=Alloc( + cast( + Any, + { + _ADDR: Account( + storage=Storage( + cast(Any, {0: GasTainted(1, ("X",))}) + ) + ) + }, + ) + ), + tx=Transaction(), + block_exception=TransactionException.INTRINSIC_GAS_TOO_LOW, + ) + assert collect_taint_hits(test, None) == [] + + def test_exception_test_marker_excludes_test(self) -> None: + """Test exception test marker excludes test.""" + test = _state_test_with_storage(GasTainted(1, ("X",))) + node = _node_with_marker("exception_test") + assert collect_taint_hits(test, node) == [] + + +@pytest.mark.usefixtures("taint_installed") +class TestMixedSinks: + """Multiple sinks can fire for the same test.""" + + def test_storage_and_receipt(self) -> None: + """Test storage and receipt.""" + tx = Transaction() + tx.expected_receipt = TransactionReceipt(cumulative_gas_used=21000) + test = StateTest( + fork=Cancun, + pre=Alloc(), + post=Alloc( + cast( + Any, + { + _ADDR: Account( + storage=Storage( + cast(Any, {0: GasTainted(50, ("X",))}) + ) + ) + }, + ) + ), + tx=tx, + ) + hits = collect_taint_hits(test, None) + kinds = {h["kind"] for h in hits} + assert kinds == {"storage", "receipt"} diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint_e2e.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint_e2e.py new file mode 100644 index 00000000000..aaede425e4c --- /dev/null +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_gas_taint_e2e.py @@ -0,0 +1,293 @@ +""" +End-to-end pytester tests for the gas_taint plugin. + +These tests load the real fill plugin chain into a pytester subprocess +and verify that ``--detect-gas-checks`` produces the expected JSON +report. They cover the integration that the unit tests in +``test_gas_taint.py`` deliberately don't: + +- ``pytest_addoption`` / ``pytest_configure`` parse and propagate the + flag. +- ``install_taint`` runs once at session start. +- The hook inside ``BaseTestWrapper.__init__`` actually fires for each + test and pushes hits into ``request.config._gas_taint_results``. +- ``pytest_sessionfinish`` / ``pytest_testnodedown`` write the JSON + report to the path given to ``--gas-check-report``. + +Each test runs a full ``fill`` invocation against a synthetic test +module, so an actual t8n binary must be available (defaults to ``evm``; +override with ``EVM_BIN`` like the benchmarking tests). +""" + +import json +import os +import textwrap +from pathlib import Path + +import pytest + +GAS_TAINT_EVM_T8N = os.environ.get("EVM_BIN", "evm") + + +# A synthetic test module that asserts a gas-derived value in post +# storage via ``CodeGasMeasure``. Picks up via the ``storage`` sink kind +# in the report. +GAS_CHECK_TEST_MODULE = textwrap.dedent( + """\ + import pytest + from execution_testing import ( + Account, + Alloc, + CodeGasMeasure, + Environment, + Op, + StateTestFiller, + Transaction, + ) + + @pytest.mark.valid_at("Cancun") + def test_dummy_gas_check( + state_test: StateTestFiller, pre: Alloc, fork + ) -> None: + gas_measure = CodeGasMeasure(code=Op.PUSH1[1] + Op.POP) + expected_gas = (Op.PUSH1[1] + Op.POP).gas_cost(fork) + contract = pre.deploy_contract(code=gas_measure) + sender = pre.fund_eoa() + state_test( + env=Environment(), + pre=pre, + post={contract: Account(storage={0: expected_gas})}, + tx=Transaction(sender=sender, to=contract, gas_limit=200_000), + ) + """ +) + + +# A plain non-gas test: deploys a tiny contract, asserts a literal +# storage value (``1`` — not derived from any gas constant), and sets no +# expected receipt or header field. The walker should produce no hits. +PLAIN_TEST_MODULE = textwrap.dedent( + """\ + import pytest + from execution_testing import ( + Account, + Alloc, + Environment, + Op, + StateTestFiller, + Transaction, + ) + + @pytest.mark.valid_at("Cancun") + def test_dummy_non_gas( + state_test: StateTestFiller, pre: Alloc + ) -> None: + contract = pre.deploy_contract(code=Op.SSTORE(0, 1) + Op.STOP) + sender = pre.fund_eoa() + state_test( + env=Environment(), + pre=pre, + post={contract: Account(storage={0: 1})}, + tx=Transaction(sender=sender, to=contract, gas_limit=100_000), + ) + """ +) + + +# A synthetic OOG-style test marked with ``exception_test``. The walker +# must skip it even though it ends up running through the same hook. +OOG_TEST_MODULE = textwrap.dedent( + """\ + import pytest + from execution_testing import ( + Account, + Alloc, + Environment, + Op, + StateTestFiller, + Transaction, + TransactionException, + ) + + @pytest.mark.valid_at("Cancun") + @pytest.mark.exception_test + def test_dummy_oog( + state_test: StateTestFiller, pre: Alloc + ) -> None: + contract = pre.deploy_contract(code=Op.STOP) + sender = pre.fund_eoa() + state_test( + env=Environment(), + pre=pre, + post={}, + tx=Transaction( + sender=sender, + to=contract, + gas_limit=20_999, + error=TransactionException.INTRINSIC_GAS_TOO_LOW, + ), + ) + """ +) + + +def _setup_pytester( + pytester: pytest.Pytester, test_content: str, filename: str +) -> Path: + """Drop a synthetic test module under ``tests/`` and copy fill.ini.""" + tests_dir = pytester.mkdir("tests") + dummy_dir = tests_dir / "dummy_module" + dummy_dir.mkdir() + module_path = dummy_dir / filename + module_path.write_text(test_content) + pytester.copy_example( + name="src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini" + ) + return module_path + + +def test_detect_gas_checks_option_added(pytester: pytest.Pytester) -> None: + """``--detect-gas-checks`` appears in ``fill --help``.""" + pytester.copy_example( + name="src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini" + ) + result = pytester.runpytest("-c", "pytest-fill.ini", "--help") + assert result.ret == 0 + assert any("--detect-gas-checks" in line for line in result.outlines), ( + "expected --detect-gas-checks in help output" + ) + assert any("--gas-check-report" in line for line in result.outlines), ( + "expected --gas-check-report in help output" + ) + + +def test_gas_check_report_records_storage_hit( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + """A synthetic CodeGasMeasure test ends up in the JSON report.""" + _setup_pytester(pytester, GAS_CHECK_TEST_MODULE, "test_dummy_gas.py") + report_path = tmp_path / "gas_check_report.json" + output_dir = tmp_path / "fixtures" + + result = pytester.runpytest( + "-c", + "pytest-fill.ini", + "--fork", + "Cancun", + "--detect-gas-checks", + f"--gas-check-report={report_path}", + "--no-html", + "--skip-index", + f"--output={output_dir}", + "tests/dummy_module/", + "-q", + ) + assert result.ret == 0, f"fill failed:\n{result.outlines}" + assert report_path.exists(), "expected report file to be written" + + report = json.loads(report_path.read_text()) + storage_entries = [ + (nodeid, hit) + for nodeid, hits in report.items() + for hit in hits + if hit["kind"] == "storage" + ] + assert storage_entries, ( + f"expected at least one storage hit; got {report!r}" + ) + for _, hit in storage_entries: + # Origins should mention either Bytecode.gas_cost or one of the + # underlying gas_costs.* constants. + assert any("gas" in origin for origin in hit["origins"]), ( + f"unexpected origins for hit {hit!r}" + ) + + +def test_gas_check_report_excludes_oog_test( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + """Tests marked ``exception_test`` are excluded from the report.""" + _setup_pytester(pytester, OOG_TEST_MODULE, "test_dummy_oog.py") + report_path = tmp_path / "gas_check_report.json" + output_dir = tmp_path / "fixtures" + + result = pytester.runpytest( + "-c", + "pytest-fill.ini", + "--fork", + "Cancun", + "--detect-gas-checks", + f"--gas-check-report={report_path}", + "--no-html", + "--skip-index", + f"--output={output_dir}", + "tests/dummy_module/", + "-q", + ) + assert result.ret == 0, f"fill failed:\n{result.outlines}" + assert report_path.exists() + + report = json.loads(report_path.read_text()) + # No entry should reference the OOG test. + assert not any("test_dummy_oog" in nodeid for nodeid in report), ( + f"OOG test should be excluded; got report keys: {list(report)!r}" + ) + + +def test_gas_check_report_excludes_non_gas_test( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + """A passing test that doesn't assert any gas value isn't flagged.""" + _setup_pytester(pytester, PLAIN_TEST_MODULE, "test_dummy_plain.py") + report_path = tmp_path / "gas_check_report.json" + output_dir = tmp_path / "fixtures" + + result = pytester.runpytest( + "-c", + "pytest-fill.ini", + "--fork", + "Cancun", + "--detect-gas-checks", + f"--gas-check-report={report_path}", + "--no-html", + "--skip-index", + f"--output={output_dir}", + "tests/dummy_module/", + "-q", + ) + assert result.ret == 0, f"fill failed:\n{result.outlines}" + assert report_path.exists() + + report = json.loads(report_path.read_text()) + # The synthetic test stores a literal 1 with no expected_receipt / + # header_verify / expected_gas_used — none of the detector's + # triggers should fire. + assert report == {}, ( + f"plain test must produce empty report; got {report!r}" + ) + + +def test_no_report_written_without_flag( + pytester: pytest.Pytester, tmp_path: Path +) -> None: + """Without ``--detect-gas-checks`` no report file is produced.""" + _setup_pytester(pytester, GAS_CHECK_TEST_MODULE, "test_dummy_gas.py") + report_path = tmp_path / "gas_check_report.json" + output_dir = tmp_path / "fixtures" + + result = pytester.runpytest( + "-c", + "pytest-fill.ini", + "--fork", + "Cancun", + f"--gas-check-report={report_path}", # path supplied but flag off + "--no-html", + "--skip-index", + f"--output={output_dir}", + "tests/dummy_module/", + "-q", + ) + assert result.ret == 0, f"fill failed:\n{result.outlines}" + assert not report_path.exists(), ( + "report must not be written when detector is disabled" + ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini index 179ac2082f0..a34102e6c37 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini +++ b/packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini @@ -13,6 +13,7 @@ addopts = -p execution_testing.cli.pytest_commands.plugins.filler.pre_alloc -p execution_testing.cli.pytest_commands.plugins.filler.ported_tests -p execution_testing.cli.pytest_commands.plugins.filler.static_filler + -p execution_testing.cli.pytest_commands.plugins.filler.gas_taint -p execution_testing.cli.pytest_commands.plugins.shared.benchmarking -p execution_testing.cli.pytest_commands.plugins.shared.transaction_fixtures -p execution_testing.cli.pytest_commands.plugins.help.help diff --git a/packages/testing/src/execution_testing/specs/base.py b/packages/testing/src/execution_testing/specs/base.py index 8aee027f097..59b38f502c0 100644 --- a/packages/testing/src/execution_testing/specs/base.py +++ b/packages/testing/src/execution_testing/specs/base.py @@ -20,7 +20,7 @@ from pydantic import BaseModel, ConfigDict, Field from typing_extensions import Self -from execution_testing.base_types import to_hex +from execution_testing.base_types import HexNumber, to_hex from execution_testing.client_clis import Result, TransitionTool from execution_testing.client_clis.cli_types import OpcodeCount from execution_testing.execution import ( @@ -113,7 +113,7 @@ class BaseTest(BaseModel): ) operation_mode: OpMode | None = None gas_optimization_max_gas_limit: int | None = None - expected_benchmark_gas_used: int | None = None + expected_benchmark_gas_used: HexNumber | None = None skip_gas_used_validation: bool = False expected_receipt_status: int | None = None is_tx_gas_heavy_test: bool = False @@ -278,7 +278,9 @@ def validate_benchmark_gas( if not self.skip_gas_used_validation: # Verify that the total gas consumed in the last block # matches expectations - expected_benchmark_gas_used = self.expected_benchmark_gas_used + expected_benchmark_gas_used: HexNumber | int | None = ( + self.expected_benchmark_gas_used + ) if expected_benchmark_gas_used is None: expected_benchmark_gas_used = gas_benchmark_value diff = benchmark_gas_used - expected_benchmark_gas_used diff --git a/packages/testing/src/execution_testing/specs/benchmark.py b/packages/testing/src/execution_testing/specs/benchmark.py index 197cb5063eb..097247e6d6c 100644 --- a/packages/testing/src/execution_testing/specs/benchmark.py +++ b/packages/testing/src/execution_testing/specs/benchmark.py @@ -300,7 +300,7 @@ class BenchmarkTest(BaseTest): | None ) = None env: Environment = Field(default_factory=Environment) - expected_benchmark_gas_used: int | None = None + expected_benchmark_gas_used: HexNumber | None = None gas_benchmark_value: int = Field( default_factory=lambda: int(Environment().gas_limit) ) diff --git a/packages/testing/src/execution_testing/specs/blockchain.py b/packages/testing/src/execution_testing/specs/blockchain.py index 39fc641c3fb..41332741a6c 100644 --- a/packages/testing/src/execution_testing/specs/blockchain.py +++ b/packages/testing/src/execution_testing/specs/blockchain.py @@ -306,7 +306,7 @@ class Block(Header): """Post state for verification after block execution in BlockchainTest""" block_access_list: Bytes | None = Field(None) """EIP-7928: Block-level access lists (serialized).""" - expected_gas_used: int | None = None + expected_gas_used: HexNumber | None = None """Expected gas used for the block.""" def set_environment(self, env: Environment) -> Environment: diff --git a/packages/testing/stubs/xdist/workermanage.pyi b/packages/testing/stubs/xdist/workermanage.pyi new file mode 100644 index 00000000000..334839b5fe1 --- /dev/null +++ b/packages/testing/stubs/xdist/workermanage.pyi @@ -0,0 +1,7 @@ +from typing import Any + +import pytest + +class WorkerController: + config: pytest.Config + workeroutput: dict[str, Any] diff --git a/scripts/mark_tests.py b/scripts/mark_tests.py new file mode 100644 index 00000000000..bc0fa3fe699 --- /dev/null +++ b/scripts/mark_tests.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python3 +""" +Add a ``@pytest.mark.`` decorator to test functions listed in a +JSON file. Idempotent — re-running leaves already-marked tests alone. + +INPUT FORMATS +------------- + +The input JSON may be in any of these shapes. Parametrize brackets +(``[…]``) are stripped from node IDs since markers attach to the +function, not to individual parameter cases. + +1. **Bare mapping** (the format emitted by ``--detect-gas-checks``):: + + { + "tests/foo/test_bar.py::test_baz[case_id]": [ ... hits ... ], + "tests/foo/test_bar.py::TestClass::test_method[other]": [ ... ] + } + + Marker name must be supplied via ``--marker``. + +2. **Bare list of node IDs**:: + + [ + "tests/foo/test_bar.py::test_baz", + "tests/foo/test_bar.py::TestClass::test_method" + ] + + Marker name must be supplied via ``--marker``. + +3. **Wrapped (self-describing)** — the JSON declares which marker it + maps to, so producers can ship a single file without remembering to + pass ``--marker``:: + + { + "marker": "gas_check", + "tests": { + "tests/foo/test_bar.py::test_baz[case]": [ ... ] + } + } + + or:: + + { + "marker": "slow", + "nodeids": [ + "tests/foo/test_bar.py::test_baz", + "tests/foo/test_bar.py::test_qux" + ] + } + + ``--marker`` on the command line overrides the value in the file. + +USAGE +----- + +:: + + uv run python scripts/mark_tests.py REPORT.json --marker gas_check + uv run python scripts/mark_tests.py --dry-run wrapped_report.json +""" + +from __future__ import annotations + +import argparse +import ast +import json +import sys +from collections import defaultdict +from pathlib import Path +from typing import Dict, Iterable, List, Set, Tuple + +DEFAULT_MARKER = "gas_check" + + +# --------------------------------------------------------------------------- +# Input parsing +# --------------------------------------------------------------------------- + + +def _coerce_nodeids(raw: object) -> Iterable[str]: + if isinstance(raw, dict): + return raw.keys() + if isinstance(raw, list): + return [n for n in raw if isinstance(n, str)] + raise ValueError( + f"expected dict or list of node IDs, got {type(raw).__name__}" + ) + + +def load_input( + payload: object, cli_marker: str | None +) -> Tuple[str, Dict[str, Set[str]]]: + """ + Resolve the marker name and the ``{module: {qualname, ...}}`` map. + + Format detection: + - ``dict`` with a ``"marker"`` key → wrapped form; nodeids come + from ``"tests"`` or ``"nodeids"``. + - any other ``dict`` → bare mapping (keys are nodeids). + - ``list`` → bare list of nodeids. + """ + marker_in_file: str | None = None + if isinstance(payload, dict) and "marker" in payload: + marker_in_file = payload["marker"] + body = payload.get("tests") + if body is None: + body = payload.get("nodeids") + if body is None: + raise ValueError( + "wrapped JSON must have a 'tests' or 'nodeids' field" + ) + nodeids = list(_coerce_nodeids(body)) + else: + nodeids = list(_coerce_nodeids(payload)) + + marker = cli_marker or marker_in_file or DEFAULT_MARKER + if not marker.isidentifier(): + raise ValueError( + f"marker name {marker!r} is not a valid Python identifier" + ) + + targets: Dict[str, Set[str]] = defaultdict(set) + for nodeid in nodeids: + module, sep, rest = nodeid.partition("::") + if not sep: + continue + bracket = rest.find("[") + if bracket != -1: + rest = rest[:bracket] + targets[module].add(rest) + + return marker, targets + + +# --------------------------------------------------------------------------- +# AST navigation +# --------------------------------------------------------------------------- + + +def _is_marker(node: ast.AST, name: str) -> bool: + """Match ``@pytest.mark.NAME`` and ``@pytest.mark.NAME(...)``.""" + if isinstance(node, ast.Call): + node = node.func + if not isinstance(node, ast.Attribute) or node.attr != name: + return False + mark = node.value + if not isinstance(mark, ast.Attribute) or mark.attr != "mark": + return False + pytest_node = mark.value + return isinstance(pytest_node, ast.Name) and pytest_node.id == "pytest" + + +def _has_marker( + func: ast.FunctionDef | ast.AsyncFunctionDef, name: str +) -> bool: + return any(_is_marker(d, name) for d in func.decorator_list) + + +def _find_function( + body: List[ast.stmt], qualname: str +) -> ast.FunctionDef | ast.AsyncFunctionDef | None: + """Resolve ``Class::Method::…`` (or just ``test_foo``) inside ``body``.""" + parts = qualname.split("::") + current_body: List[ast.stmt] = body + for i, part in enumerate(parts): + last = i == len(parts) - 1 + for node in current_body: + if ( + last + and isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == part + ): + return node + if ( + not last + and isinstance(node, ast.ClassDef) + and node.name == part + ): + current_body = node.body + break + else: + return None + return None + + +# --------------------------------------------------------------------------- +# Source modification +# --------------------------------------------------------------------------- + + +def _decorator_insert_line( + func: ast.FunctionDef | ast.AsyncFunctionDef, +) -> int: + """ + 1-based line where a new decorator should be inserted. + + Above any existing decorators; above the ``def`` if none. + """ + if func.decorator_list: + return func.decorator_list[0].lineno + return func.lineno + + +def _line_indent(line: str) -> str: + return line[: len(line) - len(line.lstrip())] + + +def insert_markers( + source: str, marker: str, targets: Set[str] +) -> Tuple[str, List[str], List[str]]: + """ + Return ``(new_source, added_qualnames, missing_qualnames)``. + + Targets already carrying the marker are silently skipped. + """ + tree = ast.parse(source) + found: List[Tuple[str, ast.FunctionDef | ast.AsyncFunctionDef, bool]] = [] + missing: List[str] = [] + + for qn in sorted(targets): + func = _find_function(tree.body, qn) + if func is None: + missing.append(qn) + continue + found.append((qn, func, _has_marker(func, marker))) + + to_add = [(qn, func) for qn, func, present in found if not present] + if not to_add: + return source, [], missing + + lines = source.splitlines(keepends=True) + # Insert in descending order so earlier insertions don't shift later + # line numbers. + to_add.sort(key=lambda pair: _decorator_insert_line(pair[1]), reverse=True) + added: List[str] = [] + decorator_line = f"@pytest.mark.{marker}\n" + for qn, func in to_add: + idx = _decorator_insert_line(func) - 1 # 1-based -> 0-based + if idx < 0 or idx >= len(lines): + missing.append(qn) + continue + indent = _line_indent(lines[idx]) + lines.insert(idx, indent + decorator_line) + added.append(qn) + + return "".join(lines), added, missing + + +# --------------------------------------------------------------------------- +# Driver +# --------------------------------------------------------------------------- + + +def main(argv: List[str] | None = None) -> int: + """Parse command-line arguments and mark the listed test functions.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "report", + type=Path, + help="Path to JSON file listing node IDs (see module docstring " + "for accepted shapes).", + ) + parser.add_argument( + "--marker", + default=None, + help=( + "Pytest marker to apply. Overrides any 'marker' field in " + f"the JSON. Defaults to {DEFAULT_MARKER!r} when neither is " + "set." + ), + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Report what would change without writing any file.", + ) + parser.add_argument( + "--root", + type=Path, + default=Path.cwd(), + help=( + "Repository root; node ID module paths are resolved against " + "this directory (default: current directory)." + ), + ) + args = parser.parse_args(argv) + + try: + payload = json.loads(args.report.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"error: failed to read {args.report}: {exc}", file=sys.stderr) + return 2 + + try: + marker, targets_by_module = load_input(payload, args.marker) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + print(f"Marker: @pytest.mark.{marker}") + + total_added = 0 + total_already = 0 + total_missing = 0 + files_changed = 0 + files_skipped: List[Path] = [] + + for module_rel, qualnames in sorted(targets_by_module.items()): + module_path = args.root / module_rel + if not module_path.is_file(): + files_skipped.append(module_path) + total_missing += len(qualnames) + continue + try: + original = module_path.read_text() + except OSError as exc: + print(f" skip {module_rel}: {exc}", file=sys.stderr) + continue + + try: + new_source, added, missing = insert_markers( + original, marker, qualnames + ) + except SyntaxError as exc: + print( + f" skip {module_rel}: syntax error at line " + f"{exc.lineno}: {exc.msg}", + file=sys.stderr, + ) + continue + + already = len(qualnames) - len(added) - len(missing) + total_added += len(added) + total_already += already + total_missing += len(missing) + + if added: + files_changed += 1 + verb = "would mark" if args.dry_run else "marked" + print(f" {module_rel}: {verb} {len(added)}") + for qn in added: + print(f" + {qn}") + if not args.dry_run: + module_path.write_text(new_source) + elif missing: + print(f" {module_rel}: nothing to add") + for qn in missing: + print(f" ? {qn} (not found)") + + total_targets = sum(len(s) for s in targets_by_module.values()) + print() + print(f"Targets in input : {total_targets}") + print(f"Markers added : {total_added}") + print(f"Already present : {total_already}") + print(f"Could not locate : {total_missing}") + print(f"Files changed : {files_changed}") + if files_skipped: + print(f"Files skipped : {len(files_skipped)} (not on disk)") + if args.dry_run: + print("(dry run — no files modified)") + return 0 if total_missing == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/amsterdam/eip7778_block_gas_accounting_without_refunds/test_gas_accounting.py b/tests/amsterdam/eip7778_block_gas_accounting_without_refunds/test_gas_accounting.py index d1b9f9121af..e3dfbc937e8 100644 --- a/tests/amsterdam/eip7778_block_gas_accounting_without_refunds/test_gas_accounting.py +++ b/tests/amsterdam/eip7778_block_gas_accounting_without_refunds/test_gas_accounting.py @@ -168,6 +168,7 @@ def build_refund_tx( ) +@pytest.mark.gas_check @pytest.mark.parametrize( "refund_tx_reverts", [ @@ -215,6 +216,7 @@ def test_simple_gas_accounting( ) +@pytest.mark.gas_check @pytest.mark.parametrize( "refund_tx_reverts", [ @@ -356,6 +358,7 @@ class CallDataTestType(Enum): """calldata_floor > tx_gas_before_refund.""" +@pytest.mark.gas_check @pytest.mark.parametrize( "refund_tx_reverts", [ @@ -487,6 +490,7 @@ def test_varying_calldata_costs( ) +@pytest.mark.gas_check @pytest.mark.parametrize( "refund_tx_reverts", [ @@ -533,6 +537,7 @@ def test_multiple_refund_types_in_one_tx( ) +@pytest.mark.gas_check @pytest.mark.execute(pytest.mark.skip(reason="Requires specific gas price")) @pytest.mark.valid_from("EIP7778") def test_mixed_gas_regimes( diff --git a/tests/amsterdam/eip7954_increase_max_contract_size/test_max_code_size.py b/tests/amsterdam/eip7954_increase_max_contract_size/test_max_code_size.py index 85ed98451c9..47a26d35cde 100644 --- a/tests/amsterdam/eip7954_increase_max_contract_size/test_max_code_size.py +++ b/tests/amsterdam/eip7954_increase_max_contract_size/test_max_code_size.py @@ -273,6 +273,7 @@ def test_max_code_size_self_opcodes( state_test(pre=pre, tx=tx, post=post) +@pytest.mark.gas_check @pytest.mark.parametrize( "create_opcode", [ diff --git a/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_additional_coverage.py b/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_additional_coverage.py index c62550daa9d..b1645a14f3a 100644 --- a/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_additional_coverage.py +++ b/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_additional_coverage.py @@ -49,6 +49,7 @@ def to(self, pre: Alloc) -> Address: """Deploy a simple contract that does nothing.""" return pre.deploy_contract(Op.STOP) + @pytest.mark.gas_check @pytest.mark.parametrize( "calldata,expected_standard_tokens,description", [ @@ -185,6 +186,7 @@ def to(self, pre: Alloc) -> Address: """Deploy a simple contract.""" return pre.deploy_contract(Op.STOP) + @pytest.mark.gas_check def test_maximum_calldata_size( self, state_test: StateTestFiller, @@ -535,6 +537,7 @@ def to(self, pre: Alloc) -> Address: """Deploy a simple contract.""" return pre.deploy_contract(Op.STOP) + @pytest.mark.gas_check @pytest.mark.parametrize( "access_list,authorization_list", [ @@ -667,6 +670,7 @@ def to(self, pre: Alloc) -> Address: """Deploy a simple contract.""" return pre.deploy_contract(Op.STOP) + @pytest.mark.gas_check @pytest.mark.parametrize( "num_authorizations", [ @@ -767,6 +771,7 @@ def sender(self, pre: Alloc) -> Address: """Create sender account.""" return pre.fund_eoa() + @pytest.mark.gas_check def test_refund_cap_at_one_fifth( self, state_test: StateTestFiller, @@ -854,6 +859,7 @@ def test_refund_cap_at_one_fifth( tx=tx, ) + @pytest.mark.gas_check def test_floor_cost_not_reduced_by_refunds( self, state_test: StateTestFiller, diff --git a/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_execution_gas.py b/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_execution_gas.py index a46c9d5daf6..160e5f4ad30 100644 --- a/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_execution_gas.py +++ b/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_execution_gas.py @@ -55,6 +55,7 @@ def to( """ return pre.deploy_contract(Op.INVALID) + @pytest.mark.gas_check @pytest.mark.parametrize( "ty,protected,authorization_list", [ @@ -133,6 +134,7 @@ def to( (Op.JUMPDEST * (execution_gas - 1)) + Op.STOP ) + @pytest.mark.gas_check @pytest.mark.parametrize( "ty,protected,authorization_list", [ diff --git a/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_refunds.py b/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_refunds.py index 4fcddcd316c..0cdbd338caa 100644 --- a/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_refunds.py +++ b/tests/amsterdam/eip7976_increase_calldata_floor_cost/test_refunds.py @@ -271,6 +271,7 @@ def tx_gas_limit( return tx_gas_limit +@pytest.mark.gas_check @pytest.mark.parametrize( "refund_test_type", [ diff --git a/tests/amsterdam/eip7981_increase_access_list_cost/test_access_list_cost.py b/tests/amsterdam/eip7981_increase_access_list_cost/test_access_list_cost.py index e9479b276c9..a102b6bdd99 100644 --- a/tests/amsterdam/eip7981_increase_access_list_cost/test_access_list_cost.py +++ b/tests/amsterdam/eip7981_increase_access_list_cost/test_access_list_cost.py @@ -134,6 +134,7 @@ def test_access_list_token_calculation( ) +@pytest.mark.gas_check @pytest.mark.with_all_tx_types(selector=lambda tx_type: tx_type >= 1) @pytest.mark.parametrize( "access_list,tx_data", diff --git a/tests/berlin/eip2929_gas_cost_increases/test_call.py b/tests/berlin/eip2929_gas_cost_increases/test_call.py index 7322e1b1d2f..8776b386b5f 100644 --- a/tests/berlin/eip2929_gas_cost_increases/test_call.py +++ b/tests/berlin/eip2929_gas_cost_increases/test_call.py @@ -16,6 +16,7 @@ REFERENCE_SPEC_VERSION = "0e11417265a623adb680c527b15d0cb6701b870b" +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") @pytest.mark.eels_base_coverage def test_call_insufficient_balance( diff --git a/tests/berlin/eip2929_gas_cost_increases/test_create.py b/tests/berlin/eip2929_gas_cost_increases/test_create.py index f216a39242e..8a2c333b436 100644 --- a/tests/berlin/eip2929_gas_cost_increases/test_create.py +++ b/tests/berlin/eip2929_gas_cost_increases/test_create.py @@ -41,6 +41,7 @@ REFERENCE_SPEC_VERSION = "0e11417265a623adb680c527b15d0cb6701b870b" +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") @pytest.mark.parametrize( "create_opcode", @@ -176,6 +177,7 @@ def test_create_insufficient_balance( ) +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") @pytest.mark.parametrize( "create_opcode", diff --git a/tests/berlin/eip2929_gas_cost_increases/test_warm_status_revert.py b/tests/berlin/eip2929_gas_cost_increases/test_warm_status_revert.py index 3925cd89958..980f032a2f5 100644 --- a/tests/berlin/eip2929_gas_cost_increases/test_warm_status_revert.py +++ b/tests/berlin/eip2929_gas_cost_increases/test_warm_status_revert.py @@ -19,6 +19,7 @@ REFERENCE_SPEC_VERSION = "0e11417265a623adb680c527b15d0cb6701b870b" +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") def test_storage_warm_status_reverted_by_subcall( state_test: StateTestFiller, @@ -89,6 +90,7 @@ def test_storage_warm_status_reverted_by_subcall( ) +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") def test_account_warm_status_reverted_by_subcall( state_test: StateTestFiller, diff --git a/tests/berlin/eip2930_access_list/test_acl.py b/tests/berlin/eip2930_access_list/test_acl.py index 7f0f3a82498..347d22a6ed3 100644 --- a/tests/berlin/eip2930_access_list/test_acl.py +++ b/tests/berlin/eip2930_access_list/test_acl.py @@ -24,6 +24,7 @@ pytestmark = pytest.mark.valid_from("Berlin") +@pytest.mark.gas_check @pytest.mark.parametrize( "account_warm,storage_key_warm", [ @@ -278,6 +279,7 @@ def test_transaction_intrinsic_gas_cost( state_test(env=env, pre=pre, post=post, tx=tx) +@pytest.mark.gas_check def test_repeated_address_acl( state_test: StateTestFiller, pre: Alloc, diff --git a/tests/byzantium/eip196_ec_add_mul/test_gas.py b/tests/byzantium/eip196_ec_add_mul/test_gas.py index bcf5012bc25..405d920feef 100644 --- a/tests/byzantium/eip196_ec_add_mul/test_gas.py +++ b/tests/byzantium/eip196_ec_add_mul/test_gas.py @@ -65,6 +65,7 @@ def test_gas_costs( state_test(pre=pre, post=post, tx=tx) +@pytest.mark.gas_check @pytest.mark.valid_from("Byzantium") @pytest.mark.parametrize( "precompile_address, invalid_input", diff --git a/tests/byzantium/eip197_ec_pairing/test_gas.py b/tests/byzantium/eip197_ec_pairing/test_gas.py index 1a1108c37f0..edabde581c0 100644 --- a/tests/byzantium/eip197_ec_pairing/test_gas.py +++ b/tests/byzantium/eip197_ec_pairing/test_gas.py @@ -62,6 +62,7 @@ def test_gas_costs( state_test(pre=pre, post=post, tx=tx) +@pytest.mark.gas_check @pytest.mark.valid_from("Byzantium") @pytest.mark.parametrize( "input_data", diff --git a/tests/cancun/eip1153_tstore/test_basic_tload.py b/tests/cancun/eip1153_tstore/test_basic_tload.py index 6cd9eddd647..72c8bbd0cc9 100644 --- a/tests/cancun/eip1153_tstore/test_basic_tload.py +++ b/tests/cancun/eip1153_tstore/test_basic_tload.py @@ -186,6 +186,7 @@ def test_basic_tload_other_after_tstore( state_test(env=Environment(), pre=pre, post=post, tx=tx) +@pytest.mark.gas_check @pytest.mark.ported_from( [ "https://github.com/ethereum/tests/blob/v13.3/src/GeneralStateTestsFiller/Cancun/stEIP1153-transientStorage/16_tloadGasFiller.yml", diff --git a/tests/cancun/eip1153_tstore/test_tstorage.py b/tests/cancun/eip1153_tstore/test_tstorage.py index a32bb0504df..27f68ece331 100644 --- a/tests/cancun/eip1153_tstore/test_tstorage.py +++ b/tests/cancun/eip1153_tstore/test_tstorage.py @@ -239,6 +239,7 @@ class GasMeasureTestCases(PytestParameterEnum): } +@pytest.mark.gas_check @pytest.mark.ported_from( [ "https://github.com/ethereum/tests/blob/v13.3/src/GeneralStateTestsFiller/Cancun/stEIP1153-transientStorage/17_tstoreGasFiller.yml", # noqa: E501 diff --git a/tests/cancun/eip4844_blobs/test_blob_txs.py b/tests/cancun/eip4844_blobs/test_blob_txs.py index 50be25f9524..2d0071ba453 100644 --- a/tests/cancun/eip4844_blobs/test_blob_txs.py +++ b/tests/cancun/eip4844_blobs/test_blob_txs.py @@ -390,6 +390,7 @@ def block( ) +@pytest.mark.gas_check @pytest.mark.parametrize_by_fork( "blobs_per_tx", SpecHelpers.all_valid_blob_combinations, @@ -729,6 +730,7 @@ def test_sufficient_balance_blob_tx( ) +@pytest.mark.gas_check @pytest.mark.parametrize_by_fork( "blobs_per_tx", lambda fork: [ diff --git a/tests/cancun/eip4844_blobs/test_blobhash_opcode.py b/tests/cancun/eip4844_blobs/test_blobhash_opcode.py index fdf1d87e1b7..9b551bab386 100644 --- a/tests/cancun/eip4844_blobs/test_blobhash_opcode.py +++ b/tests/cancun/eip4844_blobs/test_blobhash_opcode.py @@ -172,6 +172,7 @@ def generate_blobhash_bytecode( return scenario +@pytest.mark.gas_check @pytest.mark.parametrize("blobhash_index", blobhash_index_values) @pytest.mark.with_all_tx_types def test_blobhash_gas_cost( diff --git a/tests/cancun/eip4844_blobs/test_excess_blob_gas.py b/tests/cancun/eip4844_blobs/test_excess_blob_gas.py index 5d6c14a10e8..1f9e1e2acd7 100644 --- a/tests/cancun/eip4844_blobs/test_excess_blob_gas.py +++ b/tests/cancun/eip4844_blobs/test_excess_blob_gas.py @@ -301,6 +301,7 @@ def post( # noqa: D103 } +@pytest.mark.gas_check @pytest.mark.parametrize_by_fork( "parent_blobs", lambda fork: range(0, fork.max_blobs_per_block() + 1), @@ -365,6 +366,7 @@ def generator_function(fork: Fork) -> List[int]: return generator_function +@pytest.mark.gas_check @pytest.mark.parametrize_by_fork( "parent_excess_blobs", generate_blob_gas_cost_increases_tests(-1), @@ -401,6 +403,7 @@ def test_correct_increasing_blob_gas_costs( ) +@pytest.mark.gas_check @pytest.mark.parametrize_by_fork( "parent_excess_blobs", generate_blob_gas_cost_increases_tests(0), diff --git a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py index 952b41069ea..547e814e4e6 100644 --- a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py +++ b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile.py @@ -534,6 +534,7 @@ def test_call_opcode_types( ) +@pytest.mark.gas_check @pytest.mark.parametrize( "call_gas", [ diff --git a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py index 231f084a7f9..733bad52fb8 100644 --- a/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py +++ b/tests/cancun/eip4844_blobs/test_point_evaluation_precompile_gas.py @@ -188,6 +188,7 @@ def post( } +@pytest.mark.gas_check @pytest.mark.parametrize( "call_type", [Op.CALL, Op.DELEGATECALL, Op.CALLCODE, Op.STATICCALL], diff --git a/tests/cancun/eip5656_mcopy/test_mcopy_memory_expansion.py b/tests/cancun/eip5656_mcopy/test_mcopy_memory_expansion.py index 23f32896ed1..a4cff355b04 100644 --- a/tests/cancun/eip5656_mcopy/test_mcopy_memory_expansion.py +++ b/tests/cancun/eip5656_mcopy/test_mcopy_memory_expansion.py @@ -151,6 +151,7 @@ def post( # noqa: D103 } +@pytest.mark.gas_check @pytest.mark.parametrize( "dest,src,length", [ @@ -210,6 +211,7 @@ def test_mcopy_memory_expansion( ) +@pytest.mark.gas_check @pytest.mark.parametrize( "dest,src,length", [ diff --git a/tests/frontier/opcodes/test_all_opcodes.py b/tests/frontier/opcodes/test_all_opcodes.py index c13bb4f1563..6bdec21fb47 100644 --- a/tests/frontier/opcodes/test_all_opcodes.py +++ b/tests/frontier/opcodes/test_all_opcodes.py @@ -305,6 +305,7 @@ def constant_gas_opcodes(fork: Fork) -> Generator[ParameterSet, None, None]: ) +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") @pytest.mark.parametrize_by_fork("opcode", constant_gas_opcodes) @pytest.mark.eels_base_coverage diff --git a/tests/frontier/opcodes/test_call.py b/tests/frontier/opcodes/test_call.py index 95ad749a868..b0cdd94e956 100644 --- a/tests/frontier/opcodes/test_call.py +++ b/tests/frontier/opcodes/test_call.py @@ -16,6 +16,7 @@ # TODO: There's an issue with gas definitions on forks previous to Berlin, # remove this when fixed. https://github.com/ethereum/execution-spec- # tests/pull/1952#discussion_r2237634275 +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") def test_call_large_offset_mstore( state_test: StateTestFiller, @@ -83,6 +84,7 @@ def test_call_large_offset_mstore( # TODO: There's an issue with gas definitions on forks previous to Berlin, # remove this when fixed. https://github.com/ethereum/execution-spec- # tests/pull/1952#discussion_r2237634275 +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") def test_call_memory_expands_on_early_revert( state_test: StateTestFiller, @@ -170,6 +172,7 @@ def test_call_memory_expands_on_early_revert( # TODO: There's an issue with gas definitions on forks previous to Berlin, # remove this when fixed. https://github.com/ethereum/execution-spec- # tests/pull/1952#discussion_r2237634275 +@pytest.mark.gas_check @pytest.mark.with_all_call_opcodes @pytest.mark.valid_from("Berlin") def test_call_large_args_offset_size_zero( diff --git a/tests/frontier/opcodes/test_exp.py b/tests/frontier/opcodes/test_exp.py index 2fcbbe9f990..1cab6ba6338 100644 --- a/tests/frontier/opcodes/test_exp.py +++ b/tests/frontier/opcodes/test_exp.py @@ -15,6 +15,7 @@ REFERENCE_SPEC_VERSION = "N/A" +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") @pytest.mark.parametrize( "a", [0, 1, pytest.param(2**256 - 1, id="a2to256minus1")] diff --git a/tests/frontier/opcodes/test_log.py b/tests/frontier/opcodes/test_log.py index e3cfe61c4fc..e25130673d0 100644 --- a/tests/frontier/opcodes/test_log.py +++ b/tests/frontier/opcodes/test_log.py @@ -15,6 +15,7 @@ REFERENCE_SPEC_VERSION = "N/A" +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") @pytest.mark.parametrize( "opcode,topics", diff --git a/tests/osaka/eip7918_blob_reserve_price/test_blob_base_fee.py b/tests/osaka/eip7918_blob_reserve_price/test_blob_base_fee.py index cd6b444f227..3b778c96fd5 100644 --- a/tests/osaka/eip7918_blob_reserve_price/test_blob_base_fee.py +++ b/tests/osaka/eip7918_blob_reserve_price/test_blob_base_fee.py @@ -133,6 +133,7 @@ def post( } +@pytest.mark.gas_check @pytest.mark.parametrize( "block_base_fee_per_gas", [1, 7, 15, 16, 17, 100, 1000, 10000], @@ -228,6 +229,7 @@ def get_boundary_scenarios(fork: Fork) -> Iterator[Any]: yield pytest.param(excess_blobs, delta) +@pytest.mark.gas_check @pytest.mark.parametrize_by_fork( "parent_excess_blobs,block_base_fee_per_gas_delta", get_boundary_scenarios, diff --git a/tests/osaka/eip7939_count_leading_zeros/test_count_leading_zeros.py b/tests/osaka/eip7939_count_leading_zeros/test_count_leading_zeros.py index 171b8255431..daf0762cc12 100644 --- a/tests/osaka/eip7939_count_leading_zeros/test_count_leading_zeros.py +++ b/tests/osaka/eip7939_count_leading_zeros/test_count_leading_zeros.py @@ -125,6 +125,7 @@ def test_clz_opcode_scenarios( state_test(pre=pre, post=post, tx=tx) +@pytest.mark.gas_check @pytest.mark.valid_from("Osaka") def test_clz_gas_cost( state_test: StateTestFiller, pre: Alloc, fork: Fork diff --git a/tests/prague/eip7623_increase_calldata_cost/test_execution_gas.py b/tests/prague/eip7623_increase_calldata_cost/test_execution_gas.py index 7e8bc2c80f6..0ba75d781a3 100644 --- a/tests/prague/eip7623_increase_calldata_cost/test_execution_gas.py +++ b/tests/prague/eip7623_increase_calldata_cost/test_execution_gas.py @@ -57,6 +57,7 @@ def to( """ return pre.deploy_contract(Op.INVALID) + @pytest.mark.gas_check @pytest.mark.parametrize( "ty,protected,authorization_list", [ @@ -135,6 +136,7 @@ def to( (Op.JUMPDEST * (execution_gas - 1)) + Op.STOP ) + @pytest.mark.gas_check @pytest.mark.parametrize( "ty,protected,authorization_list", [ diff --git a/tests/prague/eip7623_increase_calldata_cost/test_refunds.py b/tests/prague/eip7623_increase_calldata_cost/test_refunds.py index fdbe63e0c03..57eb183444c 100644 --- a/tests/prague/eip7623_increase_calldata_cost/test_refunds.py +++ b/tests/prague/eip7623_increase_calldata_cost/test_refunds.py @@ -279,6 +279,7 @@ def tx_gas_limit( return tx_gas_limit +@pytest.mark.gas_check @pytest.mark.parametrize( "refund_test_type", [ diff --git a/tests/prague/eip7702_set_code_tx/test_set_code_txs.py b/tests/prague/eip7702_set_code_tx/test_set_code_txs.py index 0a08b2cbfb9..f38488e3374 100644 --- a/tests/prague/eip7702_set_code_tx/test_set_code_txs.py +++ b/tests/prague/eip7702_set_code_tx/test_set_code_txs.py @@ -3113,6 +3113,7 @@ def test_set_code_to_precompile( ) +@pytest.mark.gas_check @pytest.mark.with_all_precompiles def test_set_code_to_precompile_not_enough_gas_for_precompile_execution( state_test: StateTestFiller, diff --git a/tests/prague/eip7702_set_code_tx/test_set_code_txs_2.py b/tests/prague/eip7702_set_code_tx/test_set_code_txs_2.py index c92a3c34ff6..b6a47ebb83a 100644 --- a/tests/prague/eip7702_set_code_tx/test_set_code_txs_2.py +++ b/tests/prague/eip7702_set_code_tx/test_set_code_txs_2.py @@ -656,6 +656,7 @@ class AccessListTo(Enum): CONTRACT_ADDRESS = 2 +@pytest.mark.gas_check @pytest.mark.parametrize( "access_list_rule", [ @@ -876,6 +877,7 @@ def test_gas_diff_pointer_vs_direct_call( ) +@pytest.mark.gas_check @pytest.mark.valid_from("Prague") def test_pointer_call_followed_by_direct_call( state_test: StateTestFiller, @@ -2096,6 +2098,7 @@ def test_set_code_type_tx_pre_fork( ) +@pytest.mark.gas_check @pytest.mark.valid_from("Prague") @pytest.mark.xdist_group(name="bigmem") def test_delegation_replacement_call_previous_contract( diff --git a/tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py b/tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py index 1a33c174470..9188777851f 100644 --- a/tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py +++ b/tests/shanghai/eip3651_warm_coinbase/test_warm_coinbase.py @@ -136,6 +136,7 @@ def test_warm_coinbase_call_out_of_gas( ] +@pytest.mark.gas_check @pytest.mark.valid_from("Berlin") # these tests fill for fork >= Berlin @pytest.mark.parametrize( "opcode,measured_code,extra_stack_items",