diff --git a/python-stdlib/unittest-discover/unittest/__main__.py b/python-stdlib/unittest-discover/unittest/__main__.py index 09dfd03b9..eb30a53be 100644 --- a/python-stdlib/unittest-discover/unittest/__main__.py +++ b/python-stdlib/unittest-discover/unittest/__main__.py @@ -146,8 +146,8 @@ def discover_main(): # Ensure an appropriate output is printed if no tests are found. runner.run(TestSuite()) - # Terminate with non zero return code in case of failures. - sys.exit(result.failuresNum + result.errorsNum) + # Non-zero exit on failures; an unexpected success counts as a failure too. + sys.exit(result.failuresNum + result.errorsNum + result.unexpectedSuccessesNum) discover_main() diff --git a/python-stdlib/unittest/tests/test_expectedfailure.py b/python-stdlib/unittest/tests/test_expectedfailure.py new file mode 100644 index 000000000..3db7ca1e9 --- /dev/null +++ b/python-stdlib/unittest/tests/test_expectedfailure.py @@ -0,0 +1,84 @@ +# Tests for @unittest.expectedFailure: a decorated test that fails is an expected +# failure (run stays OK); one that passes is an unexpected success (run fails). +# Each case runs through its own TestResult so it can't fail the outer run. +import unittest + + +def _run(test_cls): + result = unittest.TestResult() + suite = unittest.TestSuite() + suite.addTest(test_cls) + suite.run(result) + return result + + +class TestExpectedFailure(unittest.TestCase): + def test_expected_failure_is_not_a_failure(self): + class Inner(unittest.TestCase): + @unittest.expectedFailure + def test_x(self): + self.assertEqual(1, 0) # fails as expected + + r = _run(Inner) + self.assertEqual(r.testsRun, 1) + self.assertEqual(r.expectedFailuresNum, 1) + self.assertEqual(r.failuresNum, 0) + self.assertEqual(r.errorsNum, 0) + self.assertTrue(r.wasSuccessful()) + + def test_unexpected_success_is_tracked_separately(self): + class Inner(unittest.TestCase): + @unittest.expectedFailure + def test_x(self): + self.assertEqual(1, 1) # passes -> unexpected success + + r = _run(Inner) + self.assertEqual(r.testsRun, 1) + self.assertEqual(r.unexpectedSuccessesNum, 1) + # Not a failure or error, but still makes the run unsuccessful. + self.assertEqual(r.failuresNum, 0) + self.assertEqual(r.errorsNum, 0) + self.assertFalse(r.wasSuccessful()) + + def test_regular_failure_still_counts_as_failure(self): + class Inner(unittest.TestCase): + def test_x(self): + self.assertEqual(1, 0) + + r = _run(Inner) + self.assertEqual(r.failuresNum, 1) + self.assertEqual(r.expectedFailuresNum, 0) + self.assertEqual(r.unexpectedSuccessesNum, 0) + self.assertFalse(r.wasSuccessful()) + + def test_error_is_unaffected(self): + class Inner(unittest.TestCase): + def test_x(self): + raise ValueError("boom") + + r = _run(Inner) + self.assertEqual(r.errorsNum, 1) + self.assertEqual(r.expectedFailuresNum, 0) + self.assertEqual(r.unexpectedSuccessesNum, 0) + self.assertFalse(r.wasSuccessful()) + + def test_results_aggregate_across_modules(self): + # Results combined with "+" must carry the new counters across. + class ExpFail(unittest.TestCase): + @unittest.expectedFailure + def test_x(self): + self.assertEqual(1, 0) + + class UnexpSucc(unittest.TestCase): + @unittest.expectedFailure + def test_x(self): + self.assertEqual(1, 1) + + combined = _run(ExpFail) + _run(UnexpSucc) + self.assertEqual(combined.expectedFailuresNum, 1) + self.assertEqual(combined.unexpectedSuccessesNum, 1) + self.assertFalse(combined.wasSuccessful()) + + +if __name__ == "__main__": + unittest.main() diff --git a/python-stdlib/unittest/tests/unexpected_success.py b/python-stdlib/unittest/tests/unexpected_success.py new file mode 100644 index 000000000..0e6a92c5b --- /dev/null +++ b/python-stdlib/unittest/tests/unexpected_success.py @@ -0,0 +1,14 @@ +# @expectedFailure test that unexpectedly passes -> the run must exit non-zero. +# Not named "test_*.py" so discovery skips it; tools/ci.sh runs it explicitly. + +import unittest + + +class TestUnexpectedSuccess(unittest.TestCase): + @unittest.expectedFailure + def test_unexpectedly_passes(self): + self.assertEqual(1, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/python-stdlib/unittest/unittest/__init__.py b/python-stdlib/unittest/unittest/__init__.py index 5f4c21a29..ea7634a34 100644 --- a/python-stdlib/unittest/unittest/__init__.py +++ b/python-stdlib/unittest/unittest/__init__.py @@ -196,14 +196,23 @@ def skipUnless(cond, msg): return skipIf(not cond, msg) +# Sentinels for the two @expectedFailure outcomes: a test that raised +# (expected failure) vs one that passed (unexpected success). +class _ExpFail(Exception): + pass + + +class _UnexpSucc(Exception): + pass + + def expectedFailure(test): def test_exp_fail(*args, **kwargs): try: test(*args, **kwargs) - except: - pass - else: - raise AssertionError("unexpected success") + except Exception: + raise _ExpFail + raise _UnexpSucc return test_exp_fail @@ -238,13 +247,18 @@ def run(self, suite: TestSuite): res.printErrors() print("----------------------------------------------------------------------") print("Ran %d tests\n" % res.testsRun) - if res.failuresNum > 0 or res.errorsNum > 0: - print("FAILED (failures=%d, errors=%d)" % (res.failuresNum, res.errorsNum)) - else: - msg = "OK" - if res.skippedNum > 0: - msg += " (skipped=%d)" % res.skippedNum - print(msg) + info = [] + for num, label in ( + (res.failuresNum, "failures"), + (res.errorsNum, "errors"), + (res.skippedNum, "skipped"), + (res.expectedFailuresNum, "expected failures"), + (res.unexpectedSuccessesNum, "unexpected successes"), + ): + if num: + info.append("%s=%d" % (label, num)) + status = "OK" if res.wasSuccessful() else "FAILED" + print("%s (%s)" % (status, ", ".join(info)) if info else status) return res @@ -257,6 +271,8 @@ def __init__(self): self.errorsNum = 0 self.failuresNum = 0 self.skippedNum = 0 + self.expectedFailuresNum = 0 + self.unexpectedSuccessesNum = 0 self.testsRun = 0 self.errors = [] self.failures = [] @@ -264,7 +280,8 @@ def __init__(self): self._newFailures = 0 def wasSuccessful(self): - return self.errorsNum == 0 and self.failuresNum == 0 + # An unexpected success isn't a failure/error but still fails the run. + return self.errorsNum == 0 and self.failuresNum == 0 and self.unexpectedSuccessesNum == 0 def printErrors(self): if self.errors or self.failures: @@ -293,6 +310,8 @@ def __add__(self, other): self.errorsNum += other.errorsNum self.failuresNum += other.failuresNum self.skippedNum += other.skippedNum + self.expectedFailuresNum += other.expectedFailuresNum + self.unexpectedSuccessesNum += other.unexpectedSuccessesNum self.testsRun += other.testsRun self.errors.extend(other.errors) self.failures.extend(other.failures) @@ -309,6 +328,16 @@ def _handle_test_exception( test_result.skipped.append((current_test, reason)) print(" skipped:", reason) return + if isinstance(exc, _ExpFail): + test_result.expectedFailuresNum += 1 + if verbose: + print(" expected failure") + return + if isinstance(exc, _UnexpSucc): + test_result.unexpectedSuccessesNum += 1 + if verbose: + print(" unexpected success") + return buf = io.StringIO() sys.print_exception(exc, buf) ex_str = buf.getvalue() diff --git a/tools/ci.sh b/tools/ci.sh index efbd4d186..6dbf5f989 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -111,6 +111,15 @@ function ci_package_tests_run { if [ $? -ne 0 ]; then false; return; fi done + # unexpected_success.py isn't named "test_*.py" so discovery skips it; run it + # explicitly and assert an unexpected success exits non-zero (like CPython). + echo "Running test python-stdlib/unittest/tests/unexpected_success.py (expecting failure)" + if (cd python-stdlib/unittest/tests && "${MICROPYTHON}" -m unittest unexpected_success.py); then + echo "Error: an unexpected success did not produce a non-zero exit code" + false + return + fi + (cd micropython/usb/usb-device && "${MICROPYTHON}" -m tests.test_core_buffer) if [ $? -ne 0 ]; then false; return; fi