diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 3cbc23af578d10..c404bc519300e2 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -20,7 +20,7 @@ jobs: issues: write timeout-minutes: 5 steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5fd5778e28fdbb..392451b79e6855 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 persist-credentials: false @@ -101,10 +101,10 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.x' - name: Runner image version @@ -269,7 +269,7 @@ jobs: OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Runner image version @@ -285,7 +285,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -321,7 +321,7 @@ jobs: OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Runner image version @@ -337,7 +337,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore AWS-LC build' id: cache-aws-lc - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/aws-lc/${{ matrix.awslc_ver }} key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} @@ -386,7 +386,7 @@ jobs: runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Build and test @@ -399,7 +399,7 @@ jobs: timeout-minutes: 60 runs-on: macos-14 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false @@ -431,7 +431,7 @@ jobs: OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Register gcc problem matcher @@ -445,7 +445,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -495,7 +495,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -522,7 +522,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 if: always() with: name: hypothesis-example-db @@ -543,7 +543,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Runner image version @@ -553,7 +553,7 @@ jobs: - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 + uses: egor-tensin/setup-gcc@v2 with: version: 10 - name: Configure OpenSSL env vars @@ -563,7 +563,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -613,7 +613,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Runner image version @@ -670,13 +670,13 @@ jobs: sanitizer: ${{ matrix.sanitizer }} - name: Upload crash if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ matrix.sanitizer }}-artifacts path: ./out/artifacts - name: Upload SARIF if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: cifuzz-sarif/results.sarif checkout_path: cifuzz-sarif diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 62325250bd368e..a6bade2c044f80 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -38,7 +38,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Build tier two interpreter @@ -92,10 +92,10 @@ jobs: architecture: aarch64 runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.11' @@ -140,10 +140,10 @@ jobs: llvm: - 21 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.11' - name: Build with JIT enabled and GIL disabled @@ -168,10 +168,10 @@ jobs: llvm: - 21 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.11' - name: Build with JIT @@ -195,10 +195,10 @@ jobs: llvm: - 21 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.11' - name: Build with JIT and tailcall diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 12fad966845dea..0ded53b00da0ef 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - uses: j178/prek-action@v1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 8810730e193bb6..db363bef7a45ae 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -65,10 +65,10 @@ jobs: "Tools/peg_generator", ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 9f1a8a824e5f19..b25750f0897de2 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v7 + uses: actions/github-script@v8 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index ce5562f2d51fbb..aa2ee275a57fa9 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -66,14 +66,14 @@ jobs: run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3" - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false ref: >- diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 65154aae4c41d5..fc68c040fca059 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -27,7 +27,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: persist-credentials: false ref: >- @@ -52,7 +52,7 @@ jobs: git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3' cache: 'pip' @@ -82,10 +82,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/cache@v4 + - uses: actions/cache@v5 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} @@ -108,11 +108,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3' cache: 'pip' diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 98d557ba1eab84..7eef66bd9d9324 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -28,7 +28,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Runner image version diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index c601d0b73380d4..49876cf49260d9 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Runner image version @@ -99,7 +99,7 @@ jobs: run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 - name: Archive logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: >- ${{ inputs.sanitizer }}-logs-${{ diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 0c1ebe29ae322f..ad725e92f2b20f 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -31,7 +31,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Register gcc problem matcher @@ -51,7 +51,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 91d76fd1b5f8c5..4b03712eb1ee08 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -18,7 +18,7 @@ jobs: CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. @@ -28,7 +28,7 @@ jobs: version: ${{ env.WASMTIME_VERSION }} - name: "Restore WASI SDK" id: cache-wasi-sdk - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ env.WASI_SDK_PATH }} key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} @@ -41,7 +41,7 @@ jobs: - name: "Add ccache to PATH" run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.x' - name: "Runner image version" diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index c95e40a38095f9..c7611804369600 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -23,7 +23,7 @@ jobs: ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Build CPython installer diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 0648b770753255..82ea819867ef6d 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -26,7 +26,7 @@ jobs: env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Register MSVC problem matcher diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 1bc1bf20de0e06..335e1a93dce4ea 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -72,10 +72,10 @@ jobs: architecture: x86_64 runner: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3.11' diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 463e7bf3355cc3..135979078710cc 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,10 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 5e90d9b7bc91ed..6886cd85b09a7d 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -222,6 +222,14 @@ complete listing. Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function be inlined. +.. c:macro:: Py_LOCAL_SYMBOL + + Macro used to declare a symbol as local to the shared library (hidden). + On supported platforms, it ensures the symbol is not exported. + + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("hidden")))``. + .. c:macro:: Py_MAX(x, y) Return the maximum value between ``x`` and ``y``. @@ -376,6 +384,38 @@ complete listing. sizeof(array) / sizeof((array)[0]) +.. c:macro:: Py_EXPORTED_SYMBOL + + Macro used to declare a symbol (function or data) as exported. + On Windows, this expands to ``__declspec(dllexport)``. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("default")))``. + This macro is for defining the C API itself; extension modules should not use it. + + +.. c:macro:: Py_IMPORTED_SYMBOL + + Macro used to declare a symbol as imported. + On Windows, this expands to ``__declspec(dllimport)``. + This macro is for defining the C API itself; extension modules should not use it. + + +.. c:macro:: PyAPI_FUNC(type) + + Macro used by CPython to declare a function as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. + + +.. c:macro:: PyAPI_DATA(type) + + Macro used by CPython to declare a public global variable as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. + + .. _api-objects: Objects, Types and Reference Counts diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 3a4631e7c657fe..2c15fed8dd5e4d 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -515,7 +515,7 @@ Directory and files operations .. exception:: Error - This exception collects exceptions that are raised during a multi-file + Subclass of :exc:`OSError` collecting exceptions raised during a multi-file operation. For :func:`copytree`, the exception argument is a list of 3-tuples (*srcname*, *dstname*, *exception*). diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 73236413cbb80f..eaa0ba54af18e7 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2369,7 +2369,7 @@ These functions and classes should not be used directly as annotations. Their intended purpose is to be building blocks for creating and declaring types. -.. function:: NamedTuple +.. class:: NamedTuple Typed version of :func:`collections.namedtuple`. @@ -2589,7 +2589,7 @@ types. for more details. -.. function:: TypedDict +.. class:: TypedDict(dict) Special construct to add type hints to a dictionary. At runtime ":class:`!TypedDict` instances" are simply :class:`dicts `. diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index 1037feb691f6bc..0ce8e42ddf3b0c 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -12,8 +12,17 @@ The notation used here is the same as in the preceding docs, and is described in the :ref:`notation ` section, except for an extra complication: -* ``~`` ("cut"): commit to the current alternative and fail the rule - even if this fails to parse +* ``~`` ("cut"): commit to the current alternative; fail the rule + if the alternative fails to parse + + Python mainly uses cuts for optimizations or improved error + messages. They often appear to be useless in the listing below. + + .. see gh-143054, and CutValidator in the source, if you want to change this: + + Cuts currently don't appear inside parentheses, brackets, lookaheads + and similar. + Their behavior in these contexts is deliberately left unspecified. .. literalinclude:: ../../Grammar/python.gram :language: peg diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index 97f078ff911b5a..f9ee2f4f68150a 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -129,5 +129,25 @@ def test_popen(self): pass +class OverlappedRefleakTests(unittest.TestCase): + + def test_wsasendto_failure(self): + ov = _overlapped.Overlapped() + buf = bytearray(4096) + with self.assertRaises(OSError): + ov.WSASendTo(0x1234, buf, 0, ("127.0.0.1", 1)) + + def test_wsarecvfrom_failure(self): + ov = _overlapped.Overlapped() + with self.assertRaises(OSError): + ov.WSARecvFrom(0x1234, 1024, 0) + + def test_wsarecvfrominto_failure(self): + ov = _overlapped.Overlapped() + buf = bytearray(4096) + with self.assertRaises(OSError): + ov.WSARecvFromInto(0x1234, buf, len(buf), 0) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_free_threading/test_lzma.py b/Lib/test/test_free_threading/test_lzma.py index 38d7e5db489426..3beacf38576017 100644 --- a/Lib/test/test_free_threading/test_lzma.py +++ b/Lib/test/test_free_threading/test_lzma.py @@ -45,11 +45,24 @@ def worker(): data = lzd.decompress(compressed, chunk_size) self.assertEqual(len(data), chunk_size) output.append(data) + # Read attributes concurrently with other threads decompressing + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertIsInstance(lzd.eof, bool) + self.assertIsInstance(lzd.needs_input, bool) + self.assertIsInstance(lzd.unused_data, bytes) run_concurrently(worker_func=worker, nthreads=NTHREADS) self.assertEqual(len(output), NTHREADS) # Verify the expected chunks (order doesn't matter due to append race) self.assertSetEqual(set(output), set(chunks)) + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertTrue(lzd.eof) + self.assertFalse(lzd.needs_input) + # Each thread added full compressed data to the buffer, but only 1 copy + # is consumed to produce the output. The rest remains as unused_data. + self.assertEqual( + len(lzd.unused_data), len(compressed) * (NTHREADS - 1) + ) if __name__ == "__main__": diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py index c7f20e1de802ce..857aced8ae5dcf 100644 --- a/Lib/test/test_peg_generator/test_grammar_validator.py +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -4,7 +4,8 @@ test_tools.skip_if_missing("peg_generator") with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser - from pegen.validator import SubRuleValidator, ValidationError, RaiseRuleValidator + from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import RaiseRuleValidator, CutValidator from pegen.testutil import parse_string from pegen.grammar import Grammar @@ -59,3 +60,18 @@ def test_raising_valid_rule(self) -> None: with self.assertRaises(ValidationError): for rule_name, rule in grammar.rules.items(): validator.validate_rule(rule_name, rule) + + def test_cut_validator(self) -> None: + grammar_source = """ + star: (OP ~ OP)* + plus: (OP ~ OP)+ + bracket: [OP ~ OP] + gather: OP.(OP ~ OP)+ + nested: [OP | NAME ~ OP] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = CutValidator(grammar) + for rule_name, rule in grammar.rules.items(): + with self.subTest(rule_name): + with self.assertRaises(ValidationError): + validator.validate_rule(rule_name, rule) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index d03ba07975a616..f39fcc2e0d8daf 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -755,6 +755,30 @@ def test_cut(self) -> None: ], ) + def test_cut_is_local_in_rule(self) -> None: + grammar = """ + start: + | inner + | 'x' { "ok" } + inner: + | 'x' ~ 'y' + | 'x' + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + + def test_cut_is_local_in_parens(self) -> None: + # we currently don't guarantee this behavior, see gh-143054 + grammar = """ + start: + | ('x' ~ 'y' | 'x') + | 'x' { "ok" } + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + def test_dangling_reference(self) -> None: grammar = """ start: foo ENDMARKER diff --git a/Misc/NEWS.d/next/Library/2025-12-19-12-38-01.gh-issue-116738.iMt3Ol.rst b/Misc/NEWS.d/next/Library/2025-12-19-12-38-01.gh-issue-116738.iMt3Ol.rst new file mode 100644 index 00000000000000..5d697a54517119 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-19-12-38-01.gh-issue-116738.iMt3Ol.rst @@ -0,0 +1,2 @@ +Make the attributes in :mod:`lzma` thread-safe on the :term:`free threaded +` build. diff --git a/Misc/NEWS.d/next/Library/2025-12-28-14-41-02.gh-issue-143249.K4vEp4.rst b/Misc/NEWS.d/next/Library/2025-12-28-14-41-02.gh-issue-143249.K4vEp4.rst new file mode 100644 index 00000000000000..d50d9e3db850bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-28-14-41-02.gh-issue-143249.K4vEp4.rst @@ -0,0 +1 @@ +Fix possible buffer leaks in Windows overlapped I/O on error handling. diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 5876623399837b..cd0d09682fac69 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -20,6 +20,7 @@ #include "pycore_long.h" // _PyLong_UInt32_Converter() // Blocks output buffer wrappers #include "pycore_blocks_output_buffer.h" +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_*_RELAXED #if OUTPUT_BUFFER_MAX_BLOCK_SIZE > SIZE_MAX #error "The maximum block size accepted by liblzma is SIZE_MAX." @@ -948,10 +949,10 @@ decompress_buf(Decompressor *d, Py_ssize_t max_length) goto error; } if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) { - d->check = lzma_get_check(&d->lzs); + FT_ATOMIC_STORE_INT_RELAXED(d->check, lzma_get_check(&d->lzs)); } if (lzret == LZMA_STREAM_END) { - d->eof = 1; + FT_ATOMIC_STORE_CHAR_RELAXED(d->eof, 1); break; } else if (lzs->avail_out == 0) { /* Need to check lzs->avail_out before lzs->avail_in. @@ -1038,13 +1039,14 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) } if (d->eof) { - d->needs_input = 0; + FT_ATOMIC_STORE_CHAR_RELAXED(d->needs_input, 0); if (lzs->avail_in > 0) { - Py_XSETREF(d->unused_data, - PyBytes_FromStringAndSize((char *)lzs->next_in, lzs->avail_in)); - if (d->unused_data == NULL) { + PyObject *unused_data = PyBytes_FromStringAndSize( + (char *)lzs->next_in, lzs->avail_in); + if (unused_data == NULL) { goto error; } + Py_XSETREF(d->unused_data, unused_data); } } else if (lzs->avail_in == 0) { @@ -1054,17 +1056,17 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) /* (avail_in==0 && avail_out==0) Maybe lzs's internal state still have a few bytes can be output, try to output them next time. */ - d->needs_input = 0; + FT_ATOMIC_STORE_CHAR_RELAXED(d->needs_input, 0); /* If max_length < 0, lzs->avail_out always > 0 */ assert(max_length >= 0); } else { /* Input buffer exhausted, output buffer has space. */ - d->needs_input = 1; + FT_ATOMIC_STORE_CHAR_RELAXED(d->needs_input, 1); } } else { - d->needs_input = 0; + FT_ATOMIC_STORE_CHAR_RELAXED(d->needs_input, 0); /* If we did not use the input buffer, we now have to copy the tail from the caller's buffer into the @@ -1314,6 +1316,26 @@ PyDoc_STRVAR(Decompressor_needs_input_doc, PyDoc_STRVAR(Decompressor_unused_data_doc, "Data found after the end of the compressed stream."); +static PyObject * +Decompressor_unused_data_get(PyObject *op, void *Py_UNUSED(closure)) +{ + Decompressor *self = Decompressor_CAST(op); + if (!FT_ATOMIC_LOAD_CHAR_RELAXED(self->eof)) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + PyMutex_Lock(&self->mutex); + assert(self->unused_data != NULL); + PyObject *result = Py_NewRef(self->unused_data); + PyMutex_Unlock(&self->mutex); + return result; +} + +static PyGetSetDef Decompressor_getset[] = { + {"unused_data", Decompressor_unused_data_get, NULL, + Decompressor_unused_data_doc}, + {NULL}, +}; + static PyMemberDef Decompressor_members[] = { {"check", Py_T_INT, offsetof(Decompressor, check), Py_READONLY, Decompressor_check_doc}, @@ -1321,8 +1343,6 @@ static PyMemberDef Decompressor_members[] = { Decompressor_eof_doc}, {"needs_input", Py_T_BOOL, offsetof(Decompressor, needs_input), Py_READONLY, Decompressor_needs_input_doc}, - {"unused_data", Py_T_OBJECT_EX, offsetof(Decompressor, unused_data), Py_READONLY, - Decompressor_unused_data_doc}, {NULL} }; @@ -1332,6 +1352,7 @@ static PyType_Slot lzma_decompressor_type_slots[] = { {Py_tp_new, _lzma_LZMADecompressor}, {Py_tp_doc, (char *)_lzma_LZMADecompressor__doc__}, {Py_tp_members, Decompressor_members}, + {Py_tp_getset, Decompressor_getset}, {0, 0} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index ca16b06f83010a..985706737c5a36 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -3173,10 +3173,7 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, ERROR_MORE_DATA); WINAPI_CONSTANT(F_DWORD, ERROR_NETNAME_DELETED); WINAPI_CONSTANT(F_DWORD, ERROR_NO_SYSTEM_RESOURCES); - WINAPI_CONSTANT(F_DWORD, ERROR_MORE_DATA); - WINAPI_CONSTANT(F_DWORD, ERROR_NETNAME_DELETED); WINAPI_CONSTANT(F_DWORD, ERROR_NO_DATA); - WINAPI_CONSTANT(F_DWORD, ERROR_NO_SYSTEM_RESOURCES); WINAPI_CONSTANT(F_DWORD, ERROR_OPERATION_ABORTED); WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 29b7b356648a53..09b57ce4b9773a 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1806,7 +1806,7 @@ _overlapped_Overlapped_WSASendTo_impl(OverlappedObject *self, HANDLE handle, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1873,7 +1873,7 @@ _overlapped_Overlapped_WSARecvFrom_impl(OverlappedObject *self, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1940,7 +1940,7 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 524d3066fbffa7..c491f06e9968fe 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -19,14 +19,16 @@ from collections.abc import Set GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH"] -GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") GITHUB_WORKFLOWS_PATH = Path(".github/workflows") -CONFIGURATION_FILE_NAMES = frozenset({ - ".pre-commit-config.yaml", - ".ruff.toml", - "mypy.ini", +RUN_TESTS_IGNORE = frozenset({ + Path("Tools/check-c-api-docs/ignored_c_api.txt"), + Path(".github/CODEOWNERS"), + Path(".pre-commit-config.yaml"), + Path(".ruff.toml"), + Path("mypy.ini"), }) + UNIX_BUILD_SYSTEM_FILE_NAMES = frozenset({ Path("aclocal.m4"), Path("config.guess"), @@ -172,11 +174,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: if file.name == "reusable-wasi.yml": platforms_changed.add("wasi") - if not ( - doc_file - or file == GITHUB_CODEOWNERS_PATH - or file.name in CONFIGURATION_FILE_NAMES - ): + if not doc_file and file not in RUN_TESTS_IGNORE: run_tests = True platform = get_file_platform(file) diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index ebc0b5a8710ab5..096a14a3cc4869 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -18,11 +18,6 @@ Py_HasFileSystemDefaultEncoding Py_UTF8Mode # pyhash.h Py_HASH_EXTERNAL -# exports.h -PyAPI_DATA -Py_EXPORTED_SYMBOL -Py_IMPORTED_SYMBOL -Py_LOCAL_SYMBOL # modsupport.h PyABIInfo_FREETHREADING_AGNOSTIC # moduleobject.h diff --git a/Tools/peg_generator/pegen/validator.py b/Tools/peg_generator/pegen/validator.py index 635eb398b41808..5e2bc238a1e966 100644 --- a/Tools/peg_generator/pegen/validator.py +++ b/Tools/peg_generator/pegen/validator.py @@ -1,3 +1,5 @@ +from typing import Any + from pegen import grammar from pegen.grammar import Alt, GrammarVisitor, Rhs, Rule @@ -44,6 +46,37 @@ def visit_Alt(self, node: Alt) -> None: ) +class CutValidator(GrammarValidator): + """Fail if Cut is not directly in a rule. + + For simplicity, we currently document that a Cut affects alternatives + of the *rule* it is in. + However, the implementation makes cuts local to enclosing Rhs + (e.g. parenthesized list of choices). + Additionally, in academic papers about PEG, repeats and optional items + are "desugared" to choices with an empty alternative, and thus contain + a Cut's effect. + + Please update documentation and tests when adding this cut, + then get rid of this validator. + + See gh-143054. + """ + + def visit(self, node: Any, parents: tuple[Any, ...] = ()) -> None: + super().visit(node, parents=(*parents, node)) + + def visit_Cut(self, node: Alt, parents: tuple[Any, ...] = ()) -> None: + parent_types = [type(p).__name__ for p in parents] + if parent_types != ['Rule', 'Rhs', 'Alt', 'NamedItem', 'Cut']: + raise ValidationError( + f"Rule {self.rulename!r} contains cut that's not on the " + "top level. " + "The intended semantics of such cases need " + "to be clarified; see the CutValidator docstring." + f"\nThe cut is inside: {parent_types}" + ) + def validate_grammar(the_grammar: grammar.Grammar) -> None: for validator_cls in GrammarValidator.__subclasses__(): validator = validator_cls(the_grammar)