diff --git a/.gitattributes b/.gitattributes index eca25b6050c..bc9328a2b05 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,6 +33,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 3f60cadb77f..09da61a470f 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -12,7 +12,8 @@ on: # Only ever run once - opened -permissions: {} +permissions: + contents: read jobs: add-header: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3c918f7128b..a74642b5762 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,8 @@ on: - 'main' - '3.*' -permissions: {} +permissions: + contents: read concurrency: # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency @@ -290,7 +291,7 @@ jobs: # Keep 1.1.1w in our list despite it being upstream EOL and otherwise # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs # supported by important vendors such as AWS-LC. - openssl_ver: [1.1.1w, 3.0.19, 3.3.6, 3.4.4, 3.5.5, 3.6.1] + openssl_ver: [1.1.1w, 3.0.21, 3.4.6, 3.5.7, 3.6.3] # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -365,7 +366,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.0.18 + OPENSSL_VER: 3.0.21 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -472,7 +473,7 @@ jobs: matrix: os: [ubuntu-24.04] env: - OPENSSL_VER: 3.0.18 + OPENSSL_VER: 3.0.21 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: @@ -540,6 +541,7 @@ jobs: needs.build-context.outputs.run-ci-fuzz == 'true' || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: + contents: read security-events: write strategy: fail-fast: false diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index 19314dd0c88..00000000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read the Docs PR preview -# Automatically edits a pull request's descriptions with a link -# to the documentation's preview on Read the Docs. - -on: - pull_request_target: - types: - - opened - paths: - - 'Doc/**' - - '.github/workflows/doc.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - documentation-links: - runs-on: ubuntu-latest - permissions: - pull-requests: write - timeout-minutes: 5 - - steps: - - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1.5 - with: - project-slug: "cpython-previews" - single-version: "true" diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 6be64c24629..9d91fc740d7 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -18,7 +18,8 @@ on: - '!**/*.ini' workflow_dispatch: -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 73fc7f37d84..e3f86d72ed8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,8 @@ name: Lint on: [push, pull_request, workflow_dispatch] -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b09ab433d21..590e3a8ec18 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -30,7 +30,8 @@ on: - "Tools/requirements-dev.txt" workflow_dispatch: -permissions: {} +permissions: + contents: read env: PIP_DISABLE_PIP_VERSION_CHECK: 1 @@ -64,11 +65,11 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: python-version: "3.13" - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index d15a8cdc6a4..5a072ad990a 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -5,7 +5,8 @@ on: types: - opened -permissions: {} +permissions: + contents: read jobs: notify-new-bugs-announce: diff --git a/.github/workflows/regen.yml b/.github/workflows/regen.yml index 1916e338fd0..85a09ae2e47 100644 --- a/.github/workflows/regen.yml +++ b/.github/workflows/regen.yml @@ -3,12 +3,17 @@ name: Regenerate CPython3 Android Artifacts on: workflow_dispatch: # Allows manual trigger from the GitHub Actions UI +permissions: + contents: read + jobs: regen-linux-x86_64: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Install Dependencies run: | @@ -58,7 +63,9 @@ jobs: runs-on: macos-14 # Native Apple Silicon ARM64 runner steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Install Dependencies run: | @@ -91,7 +98,9 @@ jobs: runs-on: ubuntu-24.04-arm # Native ARM64 runner steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Install Dependencies run: | @@ -126,12 +135,10 @@ jobs: git config core.sparseCheckout true echo "sysroots/*" >> .git/info/sparse-checkout git pull --depth=1 origin main - - name: Run Regeneration run: | # Inject -static into LDFLAGS (targets only the linux_arm64 musl target LDFLAGS line) sed -i "s/unwindlib=none/unwindlib=none -static/g" android/regen.sh - # Prepend Google prebuilt compiler path to PATH export PATH="/home/runner/prebuilts/clang/host/linux-arm64/clang-r596125/bin:$PATH" ./android/regen.sh @@ -149,7 +156,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Download and Merge All Artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -162,8 +171,7 @@ jobs: run: | mkdir -p dist/ cp -r android dist/ - cp *.bp dist/ - + cp ./*.bp dist/ echo "Verified files inside staging folder (dist/):" ls -R dist/ diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index edb38d511c6..a6b1fba69da 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,7 +4,8 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] -permissions: {} +permissions: + contents: read jobs: label: diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml index d5029a858a9..440f3e0ea98 100644 --- a/.github/workflows/reusable-cifuzz.yml +++ b/.github/workflows/reusable-cifuzz.yml @@ -13,7 +13,8 @@ on: required: true type: string -permissions: {} +permissions: + contents: read jobs: cifuzz: @@ -23,12 +24,12 @@ jobs: steps: - name: Build fuzzers (${{ inputs.sanitizer }}) id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@53a5831aa1a7651c2484877b7c4325e07e262779 # master + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@2dd789c49ecc2baa41fc0409a86928e61d85315f # master with: oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} sanitizer: ${{ inputs.sanitizer }} - name: Run fuzzers (${{ inputs.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@53a5831aa1a7651c2484877b7c4325e07e262779 # master + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@2dd789c49ecc2baa41fc0409a86928e61d85315f # master with: fuzz-seconds: 600 oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} @@ -42,7 +43,7 @@ jobs: path: ./out/artifacts - name: Upload SARIF if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@87557b9c84dde89fdd9b10e88954ac2f4248e463 # v4.36.1 + uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 with: sarif_file: cifuzz-sarif/results.sarif checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index a1772745b63..15b5c998f8b 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -48,7 +48,8 @@ on: # yamllint disable-line rule:truthy description: Whether to run the Windows tests value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool -permissions: {} +permissions: + contents: read jobs: compute-changes: diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index db1bad6c96e..e2a05ff6910 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -4,7 +4,8 @@ on: workflow_call: workflow_dispatch: -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 4d92e924194..d2339ab7a7e 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -12,7 +12,8 @@ on: required: true type: string -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -37,7 +38,7 @@ jobs: run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 + brew bundle --file=Misc/Brewfile # Because alternate versions are not symlinked into place by default: brew link --overwrite tcl-tk@8 - name: Configure CPython diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index b3fe4e99f6f..3ce3665d9c1 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -12,7 +12,8 @@ on: type: boolean default: false -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -39,17 +40,15 @@ jobs: # Install clang wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh + sudo ./llvm.sh 20 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 + sudo update-alternatives --set clang /usr/bin/clang-20 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-20 if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 # Reduce ASLR to avoid TSan crashing sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 fi - name: Sanitizer option setup diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 03ca8e61245..6a61a67f270 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -9,7 +9,8 @@ on: type: boolean default: false -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -24,7 +25,7 @@ jobs: matrix: os: [ubuntu-24.04, ubuntu-24.04-arm] env: - OPENSSL_VER: 3.0.18 + OPENSSL_VER: 3.0.21 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 24ca0593fe7..5597e6fd9dc 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -3,7 +3,8 @@ name: Reusable WASI on: workflow_call: -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index 6fc67d7095c..a4793349683 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -8,7 +8,8 @@ on: required: true type: string -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 0632540a8f6..3fd72bb265f 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -17,7 +17,8 @@ on: type: boolean default: false -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fe0e0f8b49e..07edf73c717 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,7 +4,8 @@ on: schedule: - cron: "0 */6 * * *" -permissions: {} +permissions: + contents: read jobs: stale: diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 82b14615ed5..d4b91d9a9e3 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -13,7 +13,8 @@ on: - '.github/workflows/verify-ensurepip-wheels.yml' - 'Tools/build/verify_ensurepip_wheels.py' -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml index 67e3941a691..8b2463b944c 100644 --- a/.github/workflows/verify-expat.yml +++ b/.github/workflows/verify-expat.yml @@ -11,7 +11,8 @@ on: - 'Modules/expat/**' - '.github/workflows/verify-expat.yml' -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/.gitignore b/.gitignore index 9345fe95cd5..661f7c8ec4b 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,7 @@ Tools/unicode/data/ /.ccache /cross-build/ /jit_stencils*.h +/jit_unwind_info*.h /platform /profile-clean-stamp /profile-run-stamp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1679bcf40a1..98e933cb147 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,6 +79,9 @@ repos: exclude: Lib/test/tokenizedata/coding20731.py - id: end-of-file-fixer files: '^\.github/CODEOWNERS$' + - id: mixed-line-ending + args: [--fix=auto] + exclude: '^Lib/test/.*data/' - id: trailing-whitespace types_or: [c, inc, python, rst, yaml] - id: trailing-whitespace diff --git a/Android/android.py b/Android/android.py index 60d062aa66f..7c316079854 100755 --- a/Android/android.py +++ b/Android/android.py @@ -213,8 +213,13 @@ def make_build_python(context): def unpack_deps(host, prefix_dir, cache_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.19-1", - "sqlite-3.50.4-0", "xz-5.4.6-1"]: + for name_ver in [ + "bzip2-1.0.8-3", + "libffi-3.4.4-3", + "openssl-3.0.21-0", + "sqlite-3.50.4-0", + "xz-5.4.6-1" + ]: filename = f"{name_ver}-{host}.tar.gz" out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir) shutil.unpack_archive(out_path) @@ -393,17 +398,6 @@ def setup_testbed(): os.chmod(out_path, 0o755) -# run_testbed will build the app automatically, but it's useful to have this as -# a separate command to allow running the app outside of this script. -def build_testbed(context): - setup_sdk() - setup_testbed() - run( - [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], - cwd=TESTBED_DIR, - ) - - # Work around a bug involving sys.exit and TaskGroups # (https://github.com/python/cpython/issues/101515). def exit(*args): @@ -645,6 +639,10 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected + # Ensure that CROSS_BUILD_DIR is in the Gradle environment, regardless + # of whether it was set by environment variable or `--cross-build-dir`. + env["CROSS_BUILD_DIR"] = CROSS_BUILD_DIR + if context.ci_mode: context.args[0:0] = [ # See _add_ci_python_opts in libregrtest/main.py. @@ -871,6 +869,18 @@ def parse_args(): def add_parser(*args, **kwargs): parser = subcommands.add_parser(*args, **kwargs) + parser.add_argument( + "--cross-build-dir", + action="store", + default=os.environ.get("CROSS_BUILD_DIR"), + dest="cross_build_dir", + type=Path, + help=( + "Path to the cross-build directory " + f"(default: {CROSS_BUILD_DIR}). Can also be set " + "with the CROSS_BUILD_DIR environment variable." + ), + ) parser.add_argument( "-v", "--verbose", action="count", default=0, help="Show verbose output. Use twice to be even more verbose.") @@ -883,7 +893,7 @@ def add_parser(*args, **kwargs): ) configure_build = add_parser( "configure-build", help="Run `configure` for the build Python") - make_build = add_parser( + add_parser( "make-build", help="Run `make` for the build Python") configure_host = add_parser( "configure-host", help="Run `configure` for Android") @@ -895,38 +905,12 @@ def add_parser(*args, **kwargs): help="Delete build directories for the selected target" ) - add_parser("build-testbed", help="Build the testbed app") test = add_parser("test", help="Run the testbed app") package = add_parser("package", help="Make a release package") ci = add_parser("ci", help="Run build, package and test") env = add_parser("env", help="Print environment variables") # Common arguments - # --cross-build-dir argument - for cmd in [ - clean, - configure_build, - make_build, - configure_host, - make_host, - build, - package, - test, - ci, - ]: - cmd.add_argument( - "--cross-build-dir", - action="store", - default=os.environ.get("CROSS_BUILD_DIR"), - dest="cross_build_dir", - type=Path, - help=( - "Path to the cross-build directory " - f"(default: {CROSS_BUILD_DIR}). Can also be set " - "with the CROSS_BUILD_DIR environment variable." - ), - ) - # --cache-dir option for cmd in [configure_host, build, ci]: cmd.add_argument( @@ -1031,7 +1015,6 @@ def main(): "make-host": make_host_python, "build": build_targets, "clean": clean_targets, - "build-testbed": build_testbed, "test": run_testbed, "package": package, "ci": ci, diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index b58edc04a92..bd8334b64bb 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -8,7 +8,7 @@ plugins { val ANDROID_DIR = file("../..") val PYTHON_DIR = ANDROID_DIR.parentFile!! -val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") +val PYTHON_CROSS_DIR = file(System.getenv("CROSS_BUILD_DIR") ?: "$PYTHON_DIR/cross-build") val inSourceTree = ( ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() ) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 20ae06df64b..b45bb043502 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -118,7 +118,7 @@ Dictionary Objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. .. versionadded:: 3.13 diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8f2ea5902ab..1791f53f30e 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -412,7 +412,7 @@ an error value). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - Function similar to :c:func:`PyErr_WarnEx`, but use + Function similar to :c:func:`PyErr_WarnEx`, but uses :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is an ASCII-encoded string. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 98fe68737de..ec2fba6da8b 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "my_obj is generic over its contained type"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index f22201dc399..561f95cb90a 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -611,9 +611,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) argument, and store in the instance's :c:member:`~PyVarObject.ob_size` field. Note that the :c:member:`~PyVarObject.ob_size` field may later be used for other purposes. For example, :py:type:`int` instances use the bits of - :c:member:`~PyVarObject.ob_size` in an implementation-defined - way; the underlying storage and its size should be accessed using - :c:func:`PyLong_Export`. + :c:member:`~PyVarObject.ob_size` in an implementation-defined way. .. note:: @@ -2745,13 +2743,13 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. + set ``view->obj`` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. (3) Increment an internal counter for the number of exports. - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. + (4) Set ``view->obj`` to *exporter* and increment ``view->obj``. (5) Return ``0``. @@ -2759,10 +2757,10 @@ Buffer Object Structures schemes can be used: * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. + sets ``view->obj`` to a new reference to itself. * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root + tree. Here, ``view->obj`` will be a new reference to the root object. The individual fields of *view* are described in section @@ -2806,7 +2804,7 @@ Buffer Object Structures *view* argument. - This function MUST NOT decrement :c:expr:`view->obj`, since that is + This function MUST NOT decrement ``view->obj``, since that is done automatically in :c:func:`PyBuffer_Release` (this scheme is useful for breaking reference cycles). diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 1aedd7bed1f..feca7a92e27 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -682,7 +682,7 @@ APIs: Fail if *fill_char* is bigger than the string maximum character, or if the string has more than 1 reference. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -1082,7 +1082,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1104,7 +1104,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index c769e081013..ae19d5feea1 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -42,8 +42,8 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referenceable object, or if *callback* is not callable, ``None``, or - ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + weakly referenceable object, this will raise :exc:`TypeError` and return + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly @@ -58,8 +58,8 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referenceable object, or if *callback* is not callable, - ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + weakly referenceable object, this will raise :exc:`TypeError` and return + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly diff --git a/Doc/conf.py b/Doc/conf.py index c62c5fb03f2..1443db83b43 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -8,15 +8,13 @@ import os import sys -from importlib import import_module from importlib.util import find_spec # Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) -# Python specific content from Doc/Tools/extensions/pyspecific.py -from pyspecific import SOURCE_URI +from patchlevel import get_header_version_info, get_version_info # General configuration # --------------------- @@ -75,7 +73,7 @@ # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = import_module('patchlevel').get_version_info() +version, release = get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -547,16 +545,20 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] + # Options for sphinx.ext.extlinks # ------------------------------- +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" + # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), - "source": (SOURCE_URI, "%s"), + "source": (f"https://github.com/python/cpython/tree/{branch}/%s", "%s"), } extlinks_detect_hardcoded_links = True diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 9156c24ef16..923742077ea 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,7 @@ role,name,added,ifdef_note,struct_abi_kind macro,METH_CLASS,3.2,, macro,METH_COEXIST,3.2,, -macro,METH_FASTCALL,3.7,, +macro,METH_FASTCALL,3.10,, macro,METH_METHOD,3.7,, macro,METH_NOARGS,3.2,, macro,METH_O,3.2,, diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index ba6dd4e0505..6b2c8a7d142 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1636,7 +1636,7 @@ by member descriptors: class Member: def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' + 'Emulate PyMemberDef in Include/descrobject.h' # Also see descr_new() in Objects/descrobject.c self.name = name self.clsname = clsname diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 320dcf7eaca..b78be3bbfbf 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1042,7 +1042,7 @@ first calculation. :: >>> functools.reduce(operator.concat, []) Traceback (most recent call last): ... - TypeError: reduce() of empty sequence with no initial value + TypeError: reduce() of empty iterable with no initial value >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 6a66fc4c64b..7a2a27aa164 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -36,11 +36,6 @@ This module defines the following constants and functions: This is now a synonym of the built-in :exc:`RuntimeError`. -.. data:: LockType - - This is the type of lock objects. - - .. function:: start_new_thread(function, args[, kwargs]) Start a new thread and return its identifier. The thread executes the @@ -159,55 +154,63 @@ This module defines the following constants and functions: .. versionadded:: 3.2 -Lock objects have the following methods: +.. raw:: html + + + + + +.. class:: LockType -.. method:: lock.acquire(blocking=True, timeout=-1) + This is the type of lock objects. - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). + Lock objects have the following methods: - If the *blocking* argument is present, the action depends on its - value: if it is false, the lock is only acquired if it can be acquired - immediately without waiting, while if it is true, the lock is acquired - unconditionally as above. + .. method:: acquire(blocking=True, timeout=-1) - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is false. + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. + If the *blocking* argument is present, the action depends on its + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired + unconditionally as above. - .. versionchanged:: 3.2 - The *timeout* parameter is new. + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *blocking* is false. - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. + .. versionchanged:: 3.2 + The *timeout* parameter is new. -.. method:: lock.release() + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. + .. method:: release() + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. -.. method:: lock.locked() + .. method:: locked() - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: + In addition to these methods, lock objects can also be used via the + :keyword:`with` statement, e.g.:: - import _thread + import _thread - a_lock = _thread.allocate_lock() + a_lock = _thread.allocate_lock() - with a_lock: - print("a_lock is locked while this executes") + with a_lock: + print("a_lock is locked while this executes") **Caveats:** diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 84a242d29a0..5d0a13e0c8f 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -423,9 +423,8 @@ is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. .. note:: - Empty lines are treated as empty strings (``''``), which are allowed as values but - not as arguments. Empty lines that are read as arguments will result in an - "unrecognized arguments" error. + Each line is treated as a single argument, so an empty line is read as an + empty string (``''``). :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -953,6 +952,10 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +Because ``nargs='*'`` gathers any supplied values into a list, an absent +positional argument yields an empty list (``[]``). Only a non-``None`` +*default* overrides this (so ``default=None`` still gives ``[]``). + For required_ arguments, the ``default`` value is ignored. For example, this applies to positional arguments with nargs_ values other than ``?`` or ``*``, or optional arguments marked as ``required=True``. @@ -1266,6 +1269,10 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +Multiple arguments may share the same ``dest``. By default, the value from the +last such argument given on the command line wins. Use ``action='append'`` to +collect values from all of them into a list instead. For conflicting *option +strings* rather than ``dest`` names, see conflict_handler_. .. _deprecated: @@ -1661,6 +1668,11 @@ Subcommands present, and when the ``b`` command is specified, only the ``foo`` and ``baz`` attributes are present. + If a subparser defines an argument with the same ``dest`` as the parent + parser, the two share a single namespace attribute, so the parent's value + won't be retained. Users should give them distinct ``dest`` values to + keep both. + Similarly, when a help message is requested from a subparser, only the help for that particular parser will be printed. The help message will not include parent parser or sibling parser messages. (A help message for each @@ -1828,7 +1840,7 @@ FileType objects Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>) .. versionchanged:: 3.4 - Added the *encodings* and *errors* parameters. + Added the *encoding* and *errors* parameters. Argument groups @@ -1888,6 +1900,9 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. + Within each argument group, arguments are displayed in help output in the + order in which they are added. + .. versionchanged:: 3.11 Calling :meth:`add_argument_group` on an argument group is deprecated. This feature was never supported and does not always work correctly. @@ -2085,6 +2100,9 @@ Customizing file parsing def convert_arg_line_to_args(self, arg_line): return arg_line.split() + Note that with this override an argument can no longer contain spaces, since + each space-separated word becomes a separate argument. + Exiting methods ^^^^^^^^^^^^^^^ diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 95fbf25e3be..f4e8d8fee71 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -99,6 +99,8 @@ The module defines the following type: :exc:`TypeError` is raised. Array objects also implement the buffer interface, and may be used wherever :term:`bytes-like objects ` are supported. + Arrays are :ref:`generic ` over the type of their contents. + .. audit-event:: array.__new__ typecode,initializer array.array diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 42728b1394a..3e1d9fde3fc 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -268,18 +268,25 @@ Root nodes Literals ^^^^^^^^ -.. class:: Constant(value) +.. class:: Constant(value, kind) A constant value. The ``value`` attribute of the ``Constant`` literal contains the Python object it represents. The values represented can be instances of :class:`str`, :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, and the constants :data:`None` and :data:`Ellipsis`. + The ``kind`` attribute is an optional string. For string literals with a + ``u`` prefix, ``kind`` is set to ``'u'``. For all other + constants, ``kind`` is ``None``. + .. doctest:: >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) Expression( body=Constant(value=123)) + >>> print(ast.dump(ast.parse("u'hello'", mode='eval'), indent=4)) + Expression( + body=Constant(value='hello', kind='u')) .. class:: FormattedValue(value, conversion, format_spec) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 602e2faf38d..6fdbfce54d0 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -2033,7 +2033,7 @@ Wait until a file descriptor received some data using the Set signal handlers for SIGINT and SIGTERM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(This ``signals`` example only works on Unix.) +(This ``signal`` example only works on Unix.) Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 1a08afc0222..cab9f6513b1 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -100,6 +100,8 @@ Future Object implementations can inject their own optimized implementations of a Future object. + Futures are :ref:`generic ` over the type of their results. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -195,6 +197,10 @@ Future Object Otherwise, change the Future's state to *cancelled*, schedule the callbacks, and return ``True``. + The optional string argument *msg* is passed as the argument to the + :exc:`CancelledError` exception raised when a cancelled Future + is awaited. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 00423357c62..63bb4bae9bd 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1190,6 +1190,9 @@ Task object blocks. If the coroutine returns or raises without blocking, the task will be finished eagerly and will skip scheduling to the event loop. + Tasks are :ref:`generic ` over the return type of their wrapped + coroutines. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 1c2c57448ce..0248fa2f45f 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -189,19 +191,28 @@ POST request. Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -212,17 +223,22 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. *wrapcol* controls whether the output should have newline (``b'\n'``) characters added to it. If this is non-zero, each output line will be at most this many characters long, excluding the trailing newline. - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 @@ -234,10 +250,12 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. *ignorechars* should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by @@ -251,8 +269,11 @@ Refer to the documentation of the individual functions for more information. Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. .. versionadded:: 3.4 @@ -260,8 +281,7 @@ Refer to the documentation of the individual functions for more information. .. function:: b85decode(b) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. .. versionadded:: 3.4 @@ -269,8 +289,12 @@ Refer to the documentation of the individual functions for more information. .. function:: z85encode(s) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. + + The `ZeroMQ specification `_ + requires the length of Z85-encoded data to be a multiple of 5 + bytes. To produce compliant data frames, you must pad the input + data to this function to a multiple of 4 bytes. .. versionadded:: 3.13 @@ -278,8 +302,7 @@ Refer to the documentation of the individual functions for more information. .. function:: z85decode(s) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. .. versionadded:: 3.13 @@ -352,3 +375,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 96d8b54e7d2..5914f5cb91f 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -203,7 +203,7 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score) + >>> def grade(score): ... i = bisect([60, 70, 80, 90], score) ... return "FDCBA"[i] ... diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 086bfcf2e3a..7e276b94f98 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -56,13 +56,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -88,7 +88,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -400,7 +400,7 @@ For simple text calendars this module provides the following functions. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -438,7 +438,7 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. @@ -539,9 +539,14 @@ The :mod:`calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index bbc67ead2ba..8f3a4768c5e 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1144,7 +1144,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1181,7 +1181,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index e6daccb91f2..213c79cc414 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -459,7 +459,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define + inherit from both :class:`Set` and :class:`Hashable`, then define ``__hash__ = Set._hash``. .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index d2ba42a3860..cfae8104062 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -480,6 +480,8 @@ or subtracting from an empty counter. Unix. They are also useful for tracking transactions and other pools of data where only the most recent activity is of interest. + Deques are :ref:`generic ` over the type of their contents. + Deque objects support the following methods: @@ -735,6 +737,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, as if they were passed to the :class:`dict` constructor, including keyword arguments. + :class:`!defaultdict`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the dictionary's keys and values. + :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: @@ -1224,7 +1229,7 @@ variants of :func:`functools.lru_cache`: .. testcode:: from collections import OrderedDict - from time import time + from time import monotonic class TimeBoundedLRU: "LRU Cache that invalidates and refreshes old entries." @@ -1239,10 +1244,10 @@ variants of :func:`functools.lru_cache`: if args in self.cache: self.cache.move_to_end(args) timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: + if monotonic() - timestamp <= self.maxage: return result result = self.func(*args) - self.cache[args] = time(), result + self.cache[args] = monotonic(), result if len(self.cache) > self.maxsize: self.cache.popitem(last=False) return result diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 7d17eb8a62a..6aa703ec259 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,7 +22,7 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. + ``None`` is the sole instance of the :class:`~types.NoneType` type. .. data:: NotImplemented @@ -33,7 +33,7 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :class:`types.NotImplementedType` type. .. note:: @@ -65,7 +65,7 @@ A small number of constants live in the built-in namespace. They are: The same as the ellipsis literal "``...``", an object frequently used to indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but assignment to ``...`` raises a :exc:`SyntaxError`. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. + ``Ellipsis`` is the sole instance of the :class:`types.EllipsisType` type. .. data:: __debug__ diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index c34f439282e..c5d22240e57 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -44,6 +44,9 @@ Context Variables references to context variables which prevents context variables from being properly garbage collected. + :class:`!ContextVar`\s are :ref:`generic ` over the type of + their contained value. + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. @@ -103,6 +106,9 @@ Context Variables the value of the variable to what it was before the corresponding *set*. A single token cannot reset a context variable more than once. + Tokens are :ref:`generic ` over the same type as the + :class:`ContextVar` which created them. + .. attribute:: Token.var A read-only property. Points to the :class:`ContextVar` object diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 210ad718800..9a0772de509 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -72,9 +72,13 @@ file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. +Shallow copies of many collections can be made using the corresponding +:meth:`!copy` method (such as :meth:`list.copy`, :meth:`dict.copy` or +:meth:`set.copy`), and of sequences (such as lists or bytearrays) by making +a slice of the entire sequence (``sequence[:]``). +However, these methods and slicing can create an instance of the base type +when copying an instance of a subclass, whereas :func:`copy.copy` normally +returns an instance of the same type. .. index:: pair: module; pickle diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d9cd5f2e265..29cf2afd33e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,14 @@ data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: @@ -190,10 +198,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -2689,6 +2695,8 @@ Arrays and pointers subscript and slice accesses; for slice reads, the resulting object is *not* itself an :class:`Array`. + Arrays are :ref:`generic ` over the type of their elements. + .. attribute:: _length_ diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 43efb739a77..c163516eae9 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -492,7 +492,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index ff8b601fe3d..c4570b16c3c 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -96,9 +96,10 @@ headers. ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. ``defects`` is an empty list. The parse method should append any detected defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may + values for at least the keys ``decoded``, ``defects`` and ``parse_tree``. + ``decoded`` should be the string value for the header (that is, the header + value fully decoded to unicode). ``parse_tree`` is set to the parse tree obtained + from parsing the header. The parse method should assume that *string* may contain content-transfer-encoded parts, but should correctly handle all valid unicode characters as well so that it can parse un-encoded header values. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 6ddd7faf5a2..b5b44458d63 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -245,7 +245,7 @@ Data types .. method:: EnumType.__len__(cls) - Returns the number of member in *cls*:: + Returns the number of members in *cls*:: >>> len(Color) 3 diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 1e4cdffd358..df80bfaceca 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -961,6 +961,9 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + Exception groups are :ref:`generic ` over the type of their + contained exceptions. + .. impl-detail:: The ``excs`` parameter may be any sequence, but lists and tuples are diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 35a9c1356e1..39f6fc7bb3e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -629,7 +629,7 @@ are always available. They are listed here in alphabetical order. If the given source is a string, then leading and trailing spaces and tabs are stripped. - See :func:`ast.literal_eval` for a function that can safely evaluate strings + See :func:`ast.literal_eval` for a function to evaluate strings with expressions containing only literals. .. audit-event:: exec code_object eval diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index d0de83907eb..31575655bfd 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -54,19 +54,19 @@ class-based API instead. .. index:: single: _ (underscore); gettext -.. function:: gettext(message) +.. function:: gettext(message, /) Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as :func:`!_` in the local namespace (see examples below). -.. function:: dgettext(domain, message) +.. function:: dgettext(domain, message, /) Like :func:`.gettext`, but look the message up in the specified *domain*. -.. function:: ngettext(singular, plural, n) +.. function:: ngettext(singular, plural, n, /) Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some @@ -81,15 +81,15 @@ class-based API instead. formulas for a variety of languages. -.. function:: dngettext(domain, singular, plural, n) +.. function:: dngettext(domain, singular, plural, n, /) Like :func:`ngettext`, but look the message up in the specified *domain*. -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) +.. function:: pgettext(context, message, /) +.. function:: dpgettext(domain, context, message, /) +.. function:: npgettext(context, singular, plural, n, /) +.. function:: dnpgettext(domain, context, singular, plural, n, /) Similar to the corresponding functions without the ``p`` in the prefix (that is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), @@ -226,20 +226,20 @@ are the methods of :class:`!NullTranslations`: translation for a given message. - .. method:: gettext(message) + .. method:: gettext(message, /) If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) If a fallback has been set, forward :meth:`pgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -247,7 +247,7 @@ are the methods of :class:`!NullTranslations`: .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) If a fallback has been set, forward :meth:`npgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -325,7 +325,7 @@ unexpected, or if other problems occur while reading the file, instantiating a The following methods are overridden from the base class implementation: - .. method:: gettext(message) + .. method:: gettext(message, /) Look up the *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no entry in the catalog for the @@ -334,7 +334,7 @@ unexpected, or if other problems occur while reading the file, instantiating a *message* id is returned. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to determine which @@ -355,7 +355,7 @@ unexpected, or if other problems occur while reading the file, instantiating a n) % {'num': n} - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) Look up the *context* and *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no @@ -366,7 +366,7 @@ unexpected, or if other problems occur while reading the file, instantiating a .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 7580c564494..9ce031b0133 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -83,6 +83,11 @@ The :mod:`!glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -106,6 +111,11 @@ The :mod:`!glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 3f075bb8cba..a673fd3df91 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -141,7 +141,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. The *name* will be translated to lower case, and quotes in the *value* have been removed, and character and entity references - have been replaced. + have been replaced. For empty attributes, *value* is ``None``. For instance, for the tag ````, this method would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. @@ -317,6 +317,18 @@ without further parsing: Data : alert("hello! ☺"); End tag : script +Attribute names are converted to lowercase, quotes from attribute values removed, +and ``None`` is returned as *value* for empty attributes (such as ``checked``): + +.. doctest:: + + >>> parser.feed("") + Start tag: input + attr: ('type', 'checkbox') + attr: ('checked', None) + attr: ('required', '') + attr: ('disabled', 'disabled') + Parsing comments: .. doctest:: diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index fcb0e9feb6a..4eadf28d3ef 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -59,7 +59,7 @@ provides three different variants: This class is used to handle the HTTP requests that arrive at the server. By itself, it cannot respond to any actual HTTP requests; it must be subclassed - to handle each request method (e.g. GET or POST). + to handle each request method (for example, ``'GET'`` or ``'POST'``). :class:`BaseHTTPRequestHandler` provides a number of class and instance variables, and methods for use by subclasses. @@ -201,7 +201,7 @@ provides three different variants: request header it responds back with a ``100 Continue`` followed by ``200 OK`` headers. This method can be overridden to raise an error if the server does not - want the client to continue. For e.g. server can choose to send ``417 + want the client to continue. For example, the server can choose to send ``417 Expectation Failed`` as a response header and ``return False``. .. versionadded:: 3.2 @@ -478,7 +478,9 @@ Command-line interface :mod:`http.server` can also be invoked directly using the :option:`-m` switch of the interpreter. The following example illustrates how to serve -files relative to the current directory:: +files relative to the current directory: + +.. code-block:: bash python -m http.server [OPTIONS] [port] @@ -489,7 +491,9 @@ The following options are accepted: .. option:: port The server listens to port 8000 by default. The default can be overridden - by passing the desired port number as an argument:: + by passing the desired port number as an argument: + + .. code-block:: bash python -m http.server 9000 @@ -498,7 +502,9 @@ The following options are accepted: Specifies a specific address to which it should bind. Both IPv4 and IPv6 addresses are supported. By default, the server binds itself to all interfaces. For example, the following command causes the server to bind - to localhost only:: + to localhost only: + + .. code-block:: bash python -m http.server --bind 127.0.0.1 @@ -511,7 +517,9 @@ The following options are accepted: Specifies a directory to which it should serve the files. By default, the server uses the current directory. For example, the following command - uses a specific directory:: + uses a specific directory: + + .. code-block:: bash python -m http.server --directory /tmp/ @@ -521,7 +529,9 @@ The following options are accepted: Specifies the HTTP version to which the server is conformant. By default, the server is conformant to HTTP/1.0. For example, the following command - runs an HTTP/1.1 conformant server:: + runs an HTTP/1.1 conformant server: + + .. code-block:: bash python -m http.server --protocol HTTP/1.1 @@ -530,7 +540,9 @@ The following options are accepted: .. option:: --cgi :class:`CGIHTTPRequestHandler` can be enabled in the command line by passing - the ``--cgi`` option:: + the ``--cgi`` option: + + .. code-block:: bash python -m http.server --cgi @@ -559,7 +571,7 @@ to be served. Methods :meth:`BaseHTTPRequestHandler.send_header` and :meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input -and does not perform input validation such as checking for the presence of CRLF +and do not perform input validation such as checking for the presence of CRLF sequences. Untrusted input may result in HTTP Header injection attacks. Earlier versions of Python did not scrub control characters from the diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 7a11f4fe069..1acbab1e57f 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -31,6 +31,12 @@ not** have to exist as physical files and directories on the file system: for example, a package and its resources can be imported from a zip file using :py:mod:`zipimport`. +.. warning:: + + :mod:`importlib.resources` follows the same security model as the built-in + :func:`open` function. Passing untrusted inputs to the functions + in this module is unsafe. + .. note:: This module provides functionality similar to `pkg_resources @@ -239,7 +245,6 @@ For all the following functions: .. versionchanged:: 3.13 Multiple *path_names* are accepted. - *encoding* and *errors* must be given as keyword arguments. .. function:: is_resource(anchor, *path_names) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8a7cdb04f5f..3e0df312f1c 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -365,6 +365,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) @@ -1512,10 +1515,11 @@ properties, will be invoked and :meth:`~object.__getattr__` and may be called. For cases where you want passive introspection, like documentation tools, this -can be inconvenient. :func:`getattr_static` has the same signature as :func:`getattr` +can be inconvenient. :func:`getattr_static` has a similar signature as :func:`getattr` but avoids executing code when it fetches attributes. -.. function:: getattr_static(obj, attr, default=None) +.. function:: getattr_static(obj, attr) + getattr_static(obj, attr, default) Retrieve attributes without triggering dynamic lookup via the descriptor protocol, :meth:`~object.__getattr__` diff --git a/Doc/library/io.rst b/Doc/library/io.rst index a090cbb70e8..21420b3ca57 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -46,6 +46,7 @@ will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since :exc:`IOError` is now an alias of :exc:`OSError`. +.. _text-io: Text I/O ^^^^^^^^ @@ -67,6 +68,7 @@ In-memory text streams are also available as :class:`StringIO` objects:: The text stream API is described in detail in the documentation of :class:`TextIOBase`. +.. _binary-io: Binary I/O ^^^^^^^^^^ @@ -105,6 +107,13 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. warning:: + Raw I/O is a low-level interface and methods generally must have their return + values checked and be explicitly retried to ensure an operation completes. + For instance :meth:`~RawIOBase.write` returns the number of bytes written + which may be less than the number of bytes provided (a partial write). + High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement + retry behavior. .. _io-text-encoding: @@ -480,8 +489,11 @@ I/O Base Classes Read up to *size* bytes from the object and return them. As a convenience, if *size* is unspecified or -1, all bytes until EOF are returned. - Otherwise, only one system call is ever made. Fewer than *size* bytes may - be returned if the operating system call returns fewer than *size* bytes. + + Attempts to make only one system call but will retry if interrupted and + the signal handler does not raise an exception (see :pep:`475` for the + rationale). This means fewer than *size* bytes may be returned if the + operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, @@ -495,13 +507,19 @@ I/O Base Classes Read and return all the bytes from the stream until EOF, using multiple calls to the stream if necessary. + If ``0`` bytes are returned this indicates end of file. If the object is in + non-blocking mode and the underlying :meth:`read` returns ``None`` + indicating no bytes are available, ``None`` is returned. + .. method:: readinto(b, /) Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the number of bytes read. For example, *b* might be a :class:`bytearray`. - If the object is in non-blocking mode and no bytes - are available, ``None`` is returned. + + If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If + the object is in non-blocking mode and no bytes are available, ``None`` is + returned. .. method:: write(b, /) @@ -515,6 +533,13 @@ I/O Base Classes this method returns, so the implementation should only access *b* during the method call. + .. warning:: + + This function does not ensure all bytes are written or an exception is + thrown. Callers may implement that behavior by checking the return + value and, if it is less than the length of *b*, looping with additional + write calls until all unwritten bytes are written. High-level I/O + objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. .. class:: BufferedIOBase @@ -643,7 +668,11 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase`. + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. The *name* can be one of two things: @@ -663,10 +692,6 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`~RawIOBase.read` (when called with a positive argument), - :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this - class will only make one system call. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*name*, *flags*). *opener* must return an open file descriptor (passing @@ -678,6 +703,13 @@ Raw File I/O See the :func:`open` built-in function for examples on using the *opener* parameter. + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + .. versionchanged:: 3.3 The *opener* parameter was added. The ``'x'`` mode was added. @@ -824,9 +856,9 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) - A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` - and :class:`BufferedWriter`. + A buffered binary stream implementing :class:`BufferedIOBase` interfaces + providing higher-level access to a seekable :class:`RawIOBase` raw binary + stream. The constructor creates a reader and writer for a seekable raw stream, given in the first argument. If the *buffer_size* is omitted it defaults to diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index ed5fffa42bc..e00e1b2cea4 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -361,9 +361,9 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: ipv4_mapped - For addresses that appear to be IPv4 mapped addresses (starting with - ``::FFFF/96``), this property will report the embedded IPv4 address. - For any other address, this property will be ``None``. + For addresses that appear to be IPv4 mapped addresses in the range + ``::FFFF:0:0/96`` as defined by :RFC:`4291`, this property reports the + embedded IPv4 address. For any other address, this property will be ``None``. .. attribute:: scope_id diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 8c352c939e9..47c5326d137 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -855,7 +855,7 @@ and :term:`generators ` which incur interpreter overhead. def running_mean(iterable): "Yield the average of all values seen so far." - # running_mean([8.5, 9.5, 7.5, 6.5]) -> 8.5 9.0 8.5 8.0 + # running_mean([8.5, 9.5, 7.5, 6.5]) → 8.5 9.0 8.5 8.0 return map(truediv, accumulate(iterable), count(1)) def repeatfunc(function, times=None, *args): @@ -934,10 +934,10 @@ and :term:`generators ` which incur interpreter overhead. yield element def unique(iterable, key=None, reverse=False): - "Yield unique elements in sorted order. Supports unhashable inputs." - # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] - sequenced = sorted(iterable, key=key, reverse=reverse) - return unique_justseen(sequenced, key=key) + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + sequenced = sorted(iterable, key=key, reverse=reverse) + return unique_justseen(sequenced, key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." diff --git a/Doc/library/json.rst b/Doc/library/json.rst index e83eefe351c..2a86d2cae6b 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -214,7 +214,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 46d61388f79..ebf0ea3901b 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -321,10 +321,12 @@ Floating point manipulation functions .. function:: frexp(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the mantissa and exponent of *x* as the pair ``(m, e)``. + If *x* is a finite nonzero number, then *m* is a float with + ``0.5 <= abs(m) < 1.0`` and an integer *e* is such that + ``x == m * 2**e`` exactly. Else, return ``(x, 0)``. + This is used to "pick apart" the internal representation of + a float in a portable way. Note that :func:`frexp` has a different call/return pattern than its C equivalents: it takes a single argument and return a pair of diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 0605a0c0378..b8dc7341ba8 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -100,10 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. -The arguments to :class:`Process` usually need to be unpickleable from within -the child process. If you tried typing the above example directly into a REPL it -could lead to an :exc:`AttributeError` in the child process trying to locate the -*f* function in the ``__main__`` module. +The arguments to :class:`Process` usually need to be picklable so they can be +passed to the child process. If you tried typing the above example directly +into a REPL it could lead to an :exc:`AttributeError` in the child process +trying to locate the *f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -899,7 +899,8 @@ For an example of the usage of queues for interprocess communication see standard library's :mod:`queue` module are raised to signal timeouts. :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. + :meth:`~queue.Queue.task_done`, :meth:`~queue.Queue.join`, and + :meth:`~queue.Queue.shutdown`. .. method:: qsize() @@ -1292,12 +1293,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1317,18 +1318,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` @@ -1643,7 +1644,10 @@ inherited by child processes. Note that *lock* is a keyword only argument. Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. + attributes which can both be used to store and retrieve byte strings. + While *raw* allows interaction with a :class:`bytes` object the full size of + the array, reading *value* will terminate after a null byte, like most + programming languages handle strings. The :mod:`multiprocessing.sharedctypes` module @@ -2828,6 +2832,16 @@ between themselves. Suitable authentication keys can also be generated by using :func:`os.urandom`. +This authentication protects :class:`Listener` and :func:`Client` connections, +which are reachable by address. It is not applied to the anonymous pipes +created by :func:`~multiprocessing.Pipe` or used internally by +:class:`~multiprocessing.Queue`. +:mod:`multiprocessing` treats all local processes running as the same user as +trusted; on most operating systems such processes can access each other's pipe +file descriptors regardless. Applications that require isolation between +processes of the same user must arrange it at the operating-system level -- +for example, by running workers under a different user account or in a sandbox. + Logging ^^^^^^^ diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index a9a6026af40..9f291309a75 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -98,7 +98,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: and_(a, b) __and__(a, b) - Return the bitwise and of *a* and *b*. + Return ``a & b``. .. function:: floordiv(a, b) @@ -122,13 +122,13 @@ The mathematical and bitwise operations are the most numerous: __inv__(obj) __invert__(obj) - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. + Return ``~obj``. .. function:: lshift(a, b) __lshift__(a, b) - Return *a* shifted left by *b*. + Return ``a << b``. .. function:: mod(a, b) @@ -140,7 +140,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: mul(a, b) __mul__(a, b) - Return ``a * b``, for *a* and *b* numbers. + Return ``a * b``. .. function:: matmul(a, b) @@ -160,25 +160,25 @@ The mathematical and bitwise operations are the most numerous: .. function:: or_(a, b) __or__(a, b) - Return the bitwise or of *a* and *b*. + Return ``a | b``. .. function:: pos(obj) __pos__(obj) - Return *obj* positive (``+obj``). + Return ``+obj``. .. function:: pow(a, b) __pow__(a, b) - Return ``a ** b``, for *a* and *b* numbers. + Return ``a ** b``. .. function:: rshift(a, b) __rshift__(a, b) - Return *a* shifted right by *b*. + Return ``a >> b``. .. function:: sub(a, b) @@ -197,7 +197,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: xor(a, b) __xor__(a, b) - Return the bitwise exclusive or of *a* and *b*. + Return ``a ^ b``. Operations which work with sequences (some of them with mappings too) include: @@ -391,13 +391,18 @@ Python syntax and the functions in the :mod:`operator` module. +-----------------------+-------------------------+---------------------------------------+ | Division | ``a // b`` | ``floordiv(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | +| Bitwise And, or | ``a & b`` | ``and_(a, b)`` | +| Intersection | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | +| Bitwise Exclusive Or, | ``a ^ b`` | ``xor(a, b)`` | +| or Symmetric | | | +| Difference | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | +| Bitwise Inversion, or | ``~ a`` | ``invert(a)`` | +| Complement | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | +| Bitwise Or, or | ``a | b`` | ``or_(a, b)`` | +| Union | | | +-----------------------+-------------------------+---------------------------------------+ | Exponentiation | ``a ** b`` | ``pow(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 1ffbae43431..7b5cc4be5c0 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2864,6 +2864,9 @@ features: To be directly usable as a :term:`path-like object`, ``os.DirEntry`` implements the :class:`PathLike` interface. + :class:`!DirEntry` objects are :ref:`generic ` over the type of the + path (:class:`str` or :class:`bytes`). + Attributes and methods on a ``os.DirEntry`` instance are as follows: .. attribute:: name diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index cfb9971cec8..11ff4c4bbd4 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1303,6 +1303,11 @@ Reading directories ``False``, this method follows symlinks except when expanding "``**``" wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 @@ -1329,6 +1334,11 @@ Reading directories The paths are returned in no particular order. If you need a specific order, sort the results. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. seealso:: :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 1273c777b73..fd2b5ceffa0 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -59,7 +59,7 @@ The :mod:`pickle` module differs from :mod:`marshal` in several significant ways * :mod:`marshal` cannot be used to serialize user-defined classes and their instances. :mod:`pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support @@ -680,7 +680,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index e8f153ee1b3..a59793c2515 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -71,6 +71,11 @@ will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +.. warning:: + + The :mod:`!pydoc` HTTP server is intended for local use during + development and is not suitable for production use. + :program:`python -m pydoc -n ` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index b655659256b..ea8e3eca52d 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -238,16 +238,71 @@ XMLParser Objects .. versionadded:: 3.13 -:class:`!xmlparser` objects have the following methods to mitigate some -common XML vulnerabilities. +:class:`!xmlparser` objects have the following methods to tune protections +against some common XML vulnerabilities. + +.. method:: xmlparser.SetBillionLaughsAttackProtectionActivationThreshold(threshold, /) + + Sets the number of output bytes needed to activate protection against + `billion laughs`_ attacks. + + The number of output bytes includes amplification from entity expansion + and reading DTD files. + + Parser objects usually have a protection activation threshold of 8 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + .. note:: + + Activation thresholds below 4 MiB are known to break support for DITA 1.3 + payload and are hence not recommended. + + .. versionadded:: 3.13.14 + +.. method:: xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor, /) + + Sets the maximum tolerated amplification factor for protection against + `billion laughs`_ attacks. + + The amplification factor is calculated as ``(direct + indirect) / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``indirect`` is the number of + bytes added by expanding entities and reading of external DTD files. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Peak amplifications of factor 15,000 for the entire payload + and of factor 30,000 in the middle of parsing have been observed with small + benign files in practice. In particular, the activation threshold should be + carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetBillionLaughsAttackProtectionActivationThreshold` + is exceeded. + + .. versionadded:: 3.13.14 .. method:: xmlparser.SetAllocTrackerActivationThreshold(threshold, /) Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM. - By default, parser objects have an allocation activation threshold of 64 MiB, - or equivalently 67,108,864 bytes. + Parser objects usually have an allocation activation threshold of 64 MiB, + but the actual default value depends on the underlying Expat library. An :exc:`ExpatError` is raised if this method is called on a |xml-non-root-parser| parser. @@ -271,7 +326,8 @@ common XML vulnerabilities. near the start of parsing even with benign files in practice. In particular, the activation threshold should be carefully chosen to avoid false positives. - By default, parser objects have a maximum amplification factor of 100.0. + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. An :exc:`ExpatError` is raised if this method is called on a |xml-non-root-parser| parser or if *max_factor* is outside the valid range. @@ -1012,4 +1068,6 @@ The ``errors`` module has the following attributes: not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. + +.. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack .. |xml-non-root-parser| replace:: :ref:`non-root ` diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 1b75582f0cf..c126f31c945 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -76,6 +76,8 @@ The :mod:`queue` module defines the following classes and exceptions: Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. Simple queues lack advanced functionality such as task tracking. + Simple queues are :ref:`generic ` over the type of their items. + .. versionadded:: 3.7 diff --git a/Doc/library/re.rst b/Doc/library/re.rst index efebc819dda..5c7aadb03da 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1238,6 +1238,9 @@ Regular Expression Objects Compiled regular expression object returned by :func:`re.compile`. + Patterns are :ref:`generic ` over the type of string they handle + (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. See :ref:`types-genericalias`. @@ -1381,6 +1384,9 @@ when there is no match, you can test whether there was a match with a simple Match object returned by successful ``match``\ es and ``search``\ es. + Matches are :ref:`generic ` over the type of string which was + matched (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. See :ref:`types-genericalias`. @@ -1880,7 +1886,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 517dbe8c321..44858c2c987 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -119,9 +119,11 @@ Scheduler Objects function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). + If *blocking* is false, immediately executes all events in the queue which have + a time value less than or equal to the current *timefunc* value (if any) and + returns the difference between the current *timefunc* value and the time value + of the next scheduled event in the scheduler's event queue. If the queue is + empty, returns ``None``. Either *action* or *delayfunc* can raise an exception. In either case, the scheduler will maintain a consistent state and propagate the exception. If an diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f7eba22bb60..26fa014c495 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,9 +54,11 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. @@ -89,18 +91,22 @@ The module defines the following: The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. Use :func:`os.set_inheritable` to make the file descriptor inheritable. + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -108,12 +114,16 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + .. function:: select(rlist, wlist, xlist, timeout=None) @@ -182,7 +192,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -274,50 +284,50 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -345,12 +355,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -374,7 +384,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -399,24 +409,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -462,7 +472,7 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -503,7 +513,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -523,66 +533,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permits control() to return the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disables event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -590,19 +600,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -610,22 +620,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -633,19 +643,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 4e4d7927da7..704820e3e1b 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -381,10 +381,14 @@ Directory and files operations If *dst* already exists but is not a directory, it may be overwritten depending on :func:`os.rename` semantics. - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to the destination using *copy_function* - and then removed. In case of symlinks, a new symlink pointing to the target - of *src* will be created as the destination and *src* will be removed. + :func:`os.rename` is preferably used internally when *src* and the destination are on + the same filesystem. In case :func:`os.rename` fails due to :exc:`OSError` + (e.g. the user has write permission to the destination file but not to its parent + directory), this method falls back to using *copy_function*, in which case + *src* is copied to the destination using *copy_function* and then removed. + + In case of symlinks, a new symlink pointing to the target of *src* will be + created in or as the destination, and *src* will be removed. If *copy_function* is given, it must be a callable that takes two arguments, *src* and the destination, and will be used to copy *src* to the destination @@ -673,7 +677,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Register an archiver for the format *name*. - *function* is the callable that will be used to unpack archives. The callable + *function* is the callable that will be used to create archives. The callable will receive the *base_name* of the file to create, followed by the *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. Further arguments are passed as keyword arguments: *owner*, *group*, @@ -728,8 +732,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 88393c8a0dd..96817ff03cb 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -55,7 +55,7 @@ This document includes four main sections: PEP written by Marc-André Lemburg. -.. We use the following practises for SQL code: +.. We use the following practices for SQL code: - UPPERCASE for keywords - snake_case for schema - single quotes for string literals @@ -1436,6 +1436,9 @@ Connection objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: 3.13.14 + Deleting the ``row_factory`` attribute is no longer allowed. + .. attribute:: text_factory A :term:`callable` that accepts a :class:`bytes` parameter @@ -1445,6 +1448,9 @@ Connection objects See :ref:`sqlite3-howto-encoding` for more details. + .. versionchanged:: 3.13.14 + Deleting the ``text_factory`` attribute is no longer allowed. + .. attribute:: total_changes Return the total number of database rows that have been modified, inserted, or @@ -1728,6 +1734,9 @@ Cursor objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: 3.13.14 + Deleting the ``row_factory`` attribute is no longer allowed. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 4f753655102..601b2b44320 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1091,7 +1091,7 @@ SSL sockets :meth:`SSLContext.wrap_socket` to wrap a socket. .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with + :class:`SSLSocket` instances must be created with :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. @@ -1943,7 +1943,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2746,11 +2746,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 95d281811be..6b498343763 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1351,6 +1351,8 @@ application). Many other operations also produce lists, including the :func:`sorted` built-in. + Lists are :ref:`generic ` over the types of their items. + Lists implement all of the :ref:`common ` and :ref:`mutable ` sequence operations. Lists also provide the following additional method: @@ -1437,6 +1439,10 @@ homogeneous data is needed (such as allowing storage in a :class:`set` or Tuples implement all of the :ref:`common ` sequence operations. + Tuples are :ref:`generic ` over the types of their contents. + For more information, refer to + :ref:`the typing documentation on annotating tuples `. + For heterogeneous collections of data where access by name is clearer than access by index, :func:`collections.namedtuple` may be a more appropriate choice than a simple tuple object. @@ -2017,9 +2023,25 @@ expression support in the :mod:`re` module). character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the + like the `Kharosthi numbers `__. + Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + .. doctest:: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isnumeric`. + .. method:: str.isidentifier() @@ -2060,15 +2082,14 @@ expression support in the :mod:`re` module). >>> '0123456789'.isnumeric() True - >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine True >>> '⅕'.isnumeric() # Vulgar fraction one fifth True >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() (False, True, True) - See also :meth:`isdecimal` and :meth:`isdigit`. Numeric characters are - a superset of decimal numbers. + See also :meth:`isdecimal` and :meth:`isdigit`. .. method:: str.isprintable() @@ -2455,7 +2476,9 @@ expression support in the :mod:`re` module). :func:`re.split`). Splitting an empty string with a specified separator returns ``['']``. - For example:: + For example: + + .. doctest:: >>> '1,2,3'.split(',') ['1', '2', '3'] @@ -2473,7 +2496,9 @@ expression support in the :mod:`re` module). string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: + For example: + + .. doctest:: >>> '1 2 3'.split() ['1', '2', '3'] @@ -2485,7 +2510,9 @@ expression support in the :mod:`re` module). If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only leading runs of consecutive whitespace are considered. - For example:: + For example: + + .. doctest:: >>> "".split(None, 0) [] @@ -2494,7 +2521,7 @@ expression support in the :mod:`re` module). >>> " foo ".split(maxsplit=0) ['foo '] - See also :meth:`join`. + See also :meth:`join` and :meth:`rsplit`. .. index:: @@ -2592,6 +2619,8 @@ expression support in the :mod:`re` module). The *chars* argument is not a prefix or suffix; rather, all combinations of its values are stripped. + Whitespace characters are defined by :meth:`str.isspace`. + For example: .. doctest:: @@ -2620,8 +2649,22 @@ expression support in the :mod:`re` module). .. method:: str.swapcase() Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. + vice versa. For example: + + .. doctest:: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that ``s.swapcase().swapcase() == s``. + For example: + + .. doctest:: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also :meth:`str.lower` and :meth:`str.upper`. .. method:: str.title() @@ -4978,6 +5021,7 @@ Note, the *elem* argument to the :meth:`~object.__contains__`, :meth:`~set.discard` methods may be a set. To support searching for an equivalent frozenset, a temporary one is created from *elem*. +Sets and frozensets are :ref:`generic ` over the type of their elements. .. _typesmapping: @@ -5075,6 +5119,9 @@ can be used interchangeably to index the same dictionary entry. Dictionary order is guaranteed to be insertion order. This behavior was an implementation detail of CPython from 3.6. + Dictionaries are :ref:`generic ` over two types, signifying + (respectively) the types of the dictionary's keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -5513,7 +5560,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] @@ -5562,6 +5610,15 @@ creation:: >>> type(l) + +Instances of ``GenericAlias`` are not classes at runtime, even though they behave like classes (they can be instantiated and subclassed):: + + >>> import inspect + >>> inspect.isclass(list[int]) + False + +This is true for :ref:`user-defined generics ` also. + Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> repr(list[int]) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index b480c1414a1..b7b92ecb14f 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -457,7 +457,9 @@ It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -508,9 +510,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -593,10 +600,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 77da141bfbf..56930274914 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -227,32 +227,32 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | +| ``b`` | :c:expr:`signed char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | +| ``B`` | :c:expr:`unsigned char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | +| ``h`` | :c:expr:`short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | +| ``H`` | :c:expr:`unsigned short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | +| ``i`` | :c:expr:`int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | +| ``I`` | :c:expr:`unsigned int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | +| ``l`` | :c:expr:`long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | +| ``L`` | :c:expr:`unsigned long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | +| ``q`` | :c:expr:`long long` | int | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | +| ``Q`` | :c:expr:`unsigned long | int | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ | ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ @@ -264,7 +264,7 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | +| ``P`` | :c:expr:`void \*` | int | | \(2), \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -329,27 +329,31 @@ Notes: The ``'p'`` format character encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to + smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 + ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (9) For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, + byte string, not a repeat count like for the other format characters; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping to or from ten different Python byte objects. (See :ref:`struct-examples` for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is + If a count is not given, it defaults to 1. For packing, the byte string is truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while + unpacking, the resulting :class:`!bytes` object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty byte string (while ``'0c'`` means 0 characters). + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. A format character may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index f8b0350ad3c..ee031bfc4aa 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -10,10 +10,10 @@ .. note:: - :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, - not an independent module, so there is no need to - ``import sys.monitoring``, simply ``import sys`` and then use - ``sys.monitoring``. + :mod:`!sys.monitoring` is a namespace within the :mod:`sys` module, + not an independent module, and ``import sys.monitoring`` would fail + with a :exc:`ModuleNotFoundError`. Instead, simply ``import sys`` + and then use ``sys.monitoring``. This namespace provides access to the functions and constants necessary to diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 7471f9bf507..08d2d89f43d 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -512,7 +512,7 @@ since it is impossible to detect the termination of alien threads. This constructor should always be called with keyword arguments. Arguments are: - *group* should be ``None``; reserved for future extension when a + *group* must be ``None`` as it is reserved for future extension when a :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. diff --git a/Doc/library/tomllib.rst b/Doc/library/tomllib.rst index 521a7a17fb3..77555c0e484 100644 --- a/Doc/library/tomllib.rst +++ b/Doc/library/tomllib.rst @@ -17,6 +17,13 @@ This module provides an interface for parsing TOML 1.0.0 (Tom's Obvious Minimal Language, `https://toml.io `_). This module does not support writing TOML. +.. warning:: + + Be cautious when parsing data from untrusted sources. + A malicious TOML string may cause the decoder to consume considerable + CPU and memory resources. + Limiting the size of data to be parsed is recommended. + .. seealso:: The :pypi:`Tomli-W package ` diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 291c2f1ce75..44b85c3e82f 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -143,9 +143,7 @@ Module-Level Functions :ref:`traceback object ` *tb*. It is useful for alternate formatting of stack traces. The optional *limit* argument has the same meaning as for :func:`print_tb`. A "pre-processed" stack trace - entry is a :class:`FrameSummary` object containing attributes - :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, - :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the + entry is a :class:`FrameSummary` object with attributes representing the information that is usually printed for a stack trace. @@ -177,7 +175,7 @@ Module-Level Functions .. function:: format_exception_only(exc, /[, value], *, show_group=False) Format the exception part of a traceback using an exception value such as - given by :data:`sys.last_value`. The return value is a list of strings, each + given by :data:`sys.last_exc`. The return value is a list of strings, each ending in a newline. The list contains the exception's message, which is normally a single string; however, for :exc:`SyntaxError` exceptions, it contains several lines that (when printed) display detailed information @@ -338,7 +336,7 @@ the module-level functions described above. .. attribute:: exc_type - The class of the original traceback. + The class of the original exception. .. deprecated:: 3.13 @@ -382,7 +380,7 @@ the module-level functions described above. For syntax errors - the compiler error message. - .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False) + .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 439e119461f..252fbb51679 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -143,7 +143,7 @@ If you instantiate any of these types, note that signatures may vary between Pyt Standard names are defined for the following types: -.. data:: NoneType +.. class:: NoneType The type of :data:`None`. @@ -233,7 +233,7 @@ Standard names are defined for the following types: .. versionadded:: 3.7 -.. data:: NotImplementedType +.. class:: NotImplementedType The type of :data:`NotImplemented`. @@ -273,7 +273,7 @@ Standard names are defined for the following types: creating :class:`!ModuleType` instances which ensures the various attributes are set appropriately. -.. data:: EllipsisType +.. class:: EllipsisType The type of :data:`Ellipsis`. @@ -360,6 +360,10 @@ Standard names are defined for the following types: entries, which means that when the mapping changes, the view reflects these changes. + :class:`!MappingProxyType`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the underlying mapping's keys and + values. + .. versionadded:: 3.3 .. versionchanged:: 3.9 diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 250c861f872..b3cde1aca8f 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -719,8 +719,8 @@ The :data:`Any` type ==================== A special kind of type is :data:`Any`. A static type checker will treat -every type as being compatible with :data:`Any` and :data:`Any` as being -compatible with every type. +every type as assignable to :data:`Any` and :data:`Any` as assignable to +every type. This means that it is possible to perform any operation or method call on a value of type :data:`Any` and assign it to any variable:: @@ -785,7 +785,7 @@ it as a return value) of a more specialized type is a type error. For example:: hash_a(42) hash_a("foo") - # Passes type checking, since Any is compatible with all types + # Passes type checking, since Any is assignable to all types hash_b(42) hash_b("foo") @@ -851,8 +851,8 @@ using ``[]``. Special type indicating an unconstrained type. - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. + * Every type is assignable to :data:`Any`. + * :data:`Any` is assignable to every type. .. versionchanged:: 3.11 :data:`Any` can now be used as a base class. This can be useful for @@ -1166,7 +1166,8 @@ These can be used as types in annotations. They all support subscription using or transforms parameters of another callable. Usage is in the form ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :ref:`Callable `. + is valid when used in :ref:`Callable ` type hints + and when instantiating user-defined generic classes with :class:`ParamSpec` parameters. The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or ellipsis (``...``). @@ -1283,10 +1284,10 @@ These can be used as types in annotations. They all support subscription using :data:`ClassVar` accepts only types and cannot be further subscribed. - :data:`ClassVar` is not a class itself, and should not + :data:`ClassVar` is not a class itself, and cannot be used with :func:`isinstance` or :func:`issubclass`. :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker + it can be used by static type checkers. For example, a type checker might flag the following code as an error:: enterprise_d = Starship(3000) @@ -1356,7 +1357,7 @@ These can be used as types in annotations. They all support subscription using def mutate_movie(m: Movie) -> None: m["year"] = 1999 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. @@ -2162,8 +2163,8 @@ without the dedicated syntax, as documented below. * :data:`Concatenate` * :ref:`annotating-callables` -.. data:: ParamSpecArgs - ParamSpecKwargs +.. class:: ParamSpecArgs + ParamSpecKwargs Arguments and keyword arguments attributes of a :class:`ParamSpec`. The ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``, @@ -2282,9 +2283,9 @@ types. Fields with a default value must come after any fields without a default. - The resulting class has an extra attribute ``__annotations__`` giving a - dict that maps the field names to the field types. (The field names are in - the ``_fields`` attribute and the default values are in the + The types for each field name can be retrieved by calling + :func:`inspect.get_annotations` on the resulting class. (The field names + are in the ``_fields`` attribute and the default values are in the ``_field_defaults`` attribute, both of which are part of the :func:`~collections.namedtuple` API.) @@ -2354,7 +2355,7 @@ types. Helper class to create low-overhead :ref:`distinct types `. - A ``NewType`` is considered a distinct type by a typechecker. At runtime, + A ``NewType`` is considered a distinct type by a type checker. At runtime, however, calling a ``NewType`` returns its argument unchanged. Usage:: @@ -2429,7 +2430,7 @@ types. Mark a protocol class as a runtime protocol. Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. - This allows a simple-minded structural check, very similar to "one trick ponies" + This allows a simple-minded structural check, very similar to "one-trick ponies" in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: @runtime_checkable @@ -2620,7 +2621,7 @@ types. key: T group: list[T] - A ``TypedDict`` can be introspected via annotations dicts + A ``TypedDict`` can be introspected via :func:`inspect.get_annotations` (see :ref:`annotations-howto` for more information on annotations best practices), :attr:`__total__`, :attr:`__required_keys__`, and :attr:`__optional_keys__`. @@ -2663,7 +2664,7 @@ types. For backwards compatibility with Python 3.10 and below, it is also possible to use inheritance to declare both required and - non-required keys in the same ``TypedDict`` . This is done by declaring a + non-required keys in the same ``TypedDict``. This is done by declaring a ``TypedDict`` with one value for the ``total`` argument and then inheriting from it in another ``TypedDict`` with a different value for ``total``: @@ -2745,34 +2746,34 @@ with :deco:`runtime_checkable`. .. class:: SupportsAbs - An ABC with one abstract method ``__abs__`` that is covariant + A protocol with one abstract method ``__abs__`` that is covariant in its return type. .. class:: SupportsBytes - An ABC with one abstract method ``__bytes__``. + A protocol with one abstract method ``__bytes__``. .. class:: SupportsComplex - An ABC with one abstract method ``__complex__``. + A protocol with one abstract method ``__complex__``. .. class:: SupportsFloat - An ABC with one abstract method ``__float__``. + A protocol with one abstract method ``__float__``. .. class:: SupportsIndex - An ABC with one abstract method ``__index__``. + A protocol with one abstract method ``__index__``. .. versionadded:: 3.8 .. class:: SupportsInt - An ABC with one abstract method ``__int__``. + A protocol with one abstract method ``__int__``. .. class:: SupportsRound - An ABC with one abstract method ``__round__`` + A protocol with one abstract method ``__round__`` that is covariant in its return type. ABCs for working with IO @@ -3339,14 +3340,27 @@ Introspection helpers Determine if a type is a :class:`Protocol`. - For example:: + For example: + + .. testcode:: class P(Protocol): def a(self) -> str: ... b: int - is_protocol(P) # => True - is_protocol(int) # => False + assert is_protocol(P) + assert not is_protocol(int) + + This function only returns true for ``Protocol`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericP[T](Protocol): + def a(self) -> T: ... + b: int + + assert not is_protocol(GenericP[int]) .. versionadded:: 3.13 @@ -3369,6 +3383,17 @@ Introspection helpers # not a typed dict itself assert not is_typeddict(TypedDict) + This function only returns true for ``TypedDict`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericFilm[T](TypedDict): + title: str + year: T + + assert not is_typeddict(GenericFilm[int]) + .. versionadded:: 3.10 .. class:: ForwardRef @@ -3407,7 +3432,7 @@ Constant .. data:: TYPE_CHECKING - A special constant that is assumed to be ``True`` by 3rd party static + A special constant that is assumed to be ``True`` by static type checkers. It is ``False`` at runtime. Usage:: diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 22316e25c5a..1a02066ff99 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1211,9 +1211,9 @@ Test cases | :meth:`assertNotRegex(s, r) | ``not r.search(s)`` | 3.2 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | - | ` | elements in the same number, | | - | | regardless of their order. | | + | :meth:`assertCountEqual(a, b) | *a* contains the same elements | 3.2 | + | ` | as *b*, regardless of their | | + | | order. | | +---------------------------------------+--------------------------------+--------------+ diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 04f5466c1a2..945c96892f5 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1007,7 +1007,7 @@ AbstractBasicAuthHandler Objects *headers* should be the error headers. *host* is either an authority (e.g. ``"python.org"``) or a URL containing an - authority component (e.g. ``"http://python.org/"``). In either case, the + authority component (e.g. ``"https://python.org/"``). In either case, the authority must not contain a userinfo component (so, ``"python.org"`` and ``"python.org:80"`` are fine, ``"joe:password@python.org"`` is not). @@ -1203,10 +1203,14 @@ This example gets the python.org main page and displays the first 300 bytes of it:: >>> import urllib.request - >>> with urllib.request.urlopen('http://www.python.org/') as f: - ... print(f.read(300)) - ... - b'\n\n\n - >> import urllib.request - >>> f = urllib.request.urlopen('http://www.python.org/') + >>> f = urllib.request.urlopen('https://www.python.org/') >>> try: - ... print(f.read(100).decode('utf-8')) + ... enc = f.headers.get('Content-Encoding') + ... data = f.read() + ... if enc == 'gzip': + ... import gzip + ... data = gzip.decompress(data) + ... print(data[:100].decode('utf-8', errors='replace')) ... finally: ... f.close() - ... - - - - """ % (output_string.replace('"', r'\"')) + """ % (output_encoded,) def OutputString(self, attrs=None): # Build up our result diff --git a/Lib/idlelib/idle_test/test_squeezer.py b/Lib/idlelib/idle_test/test_squeezer.py index 86c5d41b629..86c21f00bb8 100644 --- a/Lib/idlelib/idle_test/test_squeezer.py +++ b/Lib/idlelib/idle_test/test_squeezer.py @@ -170,6 +170,7 @@ def test_write_not_stdout(self): def test_write_stdout(self): """Test Squeezer's overriding of the EditorWindow's write() method.""" + requires('gui') editwin = self.make_mock_editor_window() for text in ['', 'TEXT']: diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 464126e2df0..fc502f7fde1 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -61,6 +61,7 @@ def set_filename_change_hook(self, hook): self.filename_change_hook = hook filename = None + file_timestamp = None dirname = None def set_filename(self, filename): @@ -127,6 +128,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = False except (UnicodeDecodeError, SyntaxError): # Wait for the editor window to appear @@ -142,6 +144,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = True except OSError as err: messagebox.showerror("I/O Error", str(err), parent=self.text) @@ -170,6 +173,7 @@ def loadfile(self, filename): self.text.insert("1.0", chars) self.reset_undo() self.set_filename(filename) + self.file_timestamp = file_timestamp if converted: # We need to save the conversion results first # before being able to execute the code @@ -206,7 +210,26 @@ def save(self, event): if not self.filename: self.save_as(event) else: + # Check the time of most recent content modification so the + # user doesn't accidentally overwrite a newer version of the file. + try: + file_timestamp = self.getmtime(self.filename) + except OSError: + pass + else: + if self.file_timestamp != file_timestamp: + confirm = messagebox.askokcancel( + title="File has changed", + message=( + "The file has changed on disk since reading it!\n\n" + "Do you really want to overwrite it?"), + default=messagebox.CANCEL, + parent=self.text) + if not confirm: + return "break" + if self.writefile(self.filename): + self.file_timestamp = self.getmtime(self.filename) self.set_saved(True) try: self.editwin.store_file_breaks() @@ -219,6 +242,7 @@ def save_as(self, event): filename = self.asksavefile() if filename: if self.writefile(filename): + self.file_timestamp = self.getmtime(filename) self.set_filename(filename) self.set_saved(1) try: @@ -251,6 +275,9 @@ def writefile(self, filename): parent=self.text) return False + def getmtime(self, filename): + return os.stat(filename).st_mtime + def fixnewlines(self): """Return text with os eols. diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 141e6398944..db16f3c802c 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -616,7 +616,7 @@ def login(self, user, password): """ typ, dat = self._simple_command('LOGIN', user, self._quote(password)) if typ != 'OK': - raise self.error(dat[-1]) + raise self.error(dat[-1].decode('UTF-8', 'replace')) self.state = 'AUTH' return typ, dat diff --git a/Lib/inspect.py b/Lib/inspect.py index d74444e27be..ea384ba5b53 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1,7 +1,7 @@ """Get useful information from live Python objects. This module encapsulates the interface provided by the internal special -attributes (co_*, im_*, tb_*, etc.) in a friendlier fashion. +attributes (co_*, tb_*, etc.) in a friendlier fashion. It also provides some help for examining source code and class layout. Here are some of the useful functions provided by this module: diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index c7a6dcdf77e..a29e6bb58b4 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -143,8 +143,8 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is @@ -207,8 +207,8 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is diff --git a/Lib/json/tool.py b/Lib/json/tool.py index fdfc3372bcc..e132fc58900 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -63,7 +63,8 @@ def main(): infile = open(options.infile, encoding='utf-8') try: if options.json_lines: - objs = (json.loads(line) for line in infile) + lines = infile.readlines() + objs = (json.loads(line) for line in lines) else: objs = (json.load(infile),) finally: diff --git a/Lib/locale.py b/Lib/locale.py index db6d0abb26b..bcb6ac0f49a 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -1491,8 +1491,8 @@ def getpreferredencoding(do_setlocale=True): # This maps Windows language identifiers to locale strings. # # This list has been updated from -# http://msdn.microsoft.com/library/default.asp?url=/library/en-us/intl/nls_238z.asp -# to include every locale up to Windows Vista. +# https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/70feba9f-294e-491e-b6eb-56532684c37f +# to include every locale up to protocol revision 16.0 (2024-04-23). # # NOTE: this mapping is incomplete. If your language is missing, please # submit a bug report as detailed in the Python devguide at: @@ -1502,10 +1502,15 @@ def getpreferredencoding(do_setlocale=True): # windows_locale = { - 0x0436: "af_ZA", # Afrikaans - 0x041c: "sq_AL", # Albanian - 0x0484: "gsw_FR",# Alsatian - France + 0x0036: "af", # Afrikaans + 0x0436: "af_ZA", # Afrikaans - South Africa + 0x001c: "sq", # Albanian + 0x041c: "sq_AL", # Albanian - Albania + 0x0084: "gsw", # Alsatian + 0x0484: "gsw_FR", # Alsatian - France + 0x005e: "am", # Amharic 0x045e: "am_ET", # Amharic - Ethiopia + 0x0001: "ar", # Arabic 0x0401: "ar_SA", # Arabic - Saudi Arabia 0x0801: "ar_IQ", # Arabic - Iraq 0x0c01: "ar_EG", # Arabic - Egypt @@ -1519,39 +1524,72 @@ def getpreferredencoding(do_setlocale=True): 0x2c01: "ar_JO", # Arabic - Jordan 0x3001: "ar_LB", # Arabic - Lebanon 0x3401: "ar_KW", # Arabic - Kuwait - 0x3801: "ar_AE", # Arabic - United Arab Emirates + 0x3801: "ar_AE", # Arabic - U.A.E. 0x3c01: "ar_BH", # Arabic - Bahrain 0x4001: "ar_QA", # Arabic - Qatar - 0x042b: "hy_AM", # Armenian + 0x002b: "hy", # Armenian + 0x042b: "hy_AM", # Armenian - Armenia + 0x004d: "as", # Assamese 0x044d: "as_IN", # Assamese - India - 0x042c: "az_AZ", # Azeri - Latin - 0x082c: "az_AZ", # Azeri - Cyrillic - 0x046d: "ba_RU", # Bashkir - 0x042d: "eu_ES", # Basque - Russia - 0x0423: "be_BY", # Belarusian - 0x0445: "bn_IN", # Begali - 0x201a: "bs_BA", # Bosnian - Cyrillic - 0x141a: "bs_BA", # Bosnian - Latin + 0x002c: "az", # Azerbaijani (Latin) + 0x742c: "az", # Azerbaijani (Cyrillic) + 0x782c: "az", # Azerbaijani (Latin) + 0x042c: "az_AZ", # Azerbaijani (Latin) - Azerbaijan + 0x0045: "bn", # Bangla + 0x0445: "bn_IN", # Bangla - India + 0x0845: "bn_BD", # Bangla - Bangladesh + 0x006d: "ba", # Bashkir + 0x046d: "ba_RU", # Bashkir - Russia + 0x002d: "eu", # Basque + 0x042d: "eu_ES", # Basque - Spain + 0x0023: "be", # Belarusian + 0x0423: "be_BY", # Belarusian - Belarus + 0x641a: "bs", # Bosnian (Cyrillic) + 0x681a: "bs", # Bosnian (Latin) + 0x141a: "bs_BA", # Bosnian (Latin) - Bosnia and Herzegovina + 0x201a: "bs_BA", # Bosnian (Cyrillic) - Bosnia and Herzegovina + 0x781a: "bs", # Bosnian (Latin) + 0x007e: "br", # Breton 0x047e: "br_FR", # Breton - France - 0x0402: "bg_BG", # Bulgarian -# 0x0455: "my_MM", # Burmese - Not supported - 0x0403: "ca_ES", # Catalan - 0x0004: "zh_CHS",# Chinese - Simplified - 0x0404: "zh_TW", # Chinese - Taiwan - 0x0804: "zh_CN", # Chinese - PRC - 0x0c04: "zh_HK", # Chinese - Hong Kong S.A.R. - 0x1004: "zh_SG", # Chinese - Singapore - 0x1404: "zh_MO", # Chinese - Macao S.A.R. - 0x7c04: "zh_CHT",# Chinese - Traditional + 0x0002: "bg", # Bulgarian + 0x0402: "bg_BG", # Bulgarian - Bulgaria + 0x0055: "my", # Burmese + 0x0455: "my_MM", # Burmese - Myanmar + 0x0003: "ca", # Catalan + 0x0403: "ca_ES", # Catalan - Spain + 0x0803: "ca_ES", # Valencian - Spain + 0x0092: "ku", # Central Kurdish + 0x7c92: "ku", # Central Kurdish + 0x0492: "ku_IQ", # Central Kurdish - Iraq + 0x005c: "chr", # Cherokee + 0x7c5c: "chr", # Cherokee + 0x045c: "chr_US", # Cherokee - United States + 0x0004: "zh", # Chinese (Simplified) + 0x7804: "zh", # Chinese (Simplified) + 0x7c04: "zh", # Chinese (Traditional) + 0x0404: "zh_TW", # Chinese (Traditional) - Taiwan + 0x0804: "zh_CN", # Chinese (Simplified) - People's Republic of China + 0x0c04: "zh_HK", # Chinese (Traditional) - Hong Kong S.A.R. + 0x1004: "zh_SG", # Chinese (Simplified) - Singapore + 0x1404: "zh_MO", # Chinese (Traditional) - Macao S.A.R. + 0x0083: "co", # Corsican 0x0483: "co_FR", # Corsican - France - 0x041a: "hr_HR", # Croatian - 0x101a: "hr_BA", # Croatian - Bosnia - 0x0405: "cs_CZ", # Czech - 0x0406: "da_DK", # Danish - 0x048c: "gbz_AF",# Dari - Afghanistan - 0x0465: "div_MV",# Divehi - Maldives - 0x0413: "nl_NL", # Dutch - The Netherlands + 0x001a: "hr", # Croatian + 0x041a: "hr_HR", # Croatian - Croatia + 0x101a: "hr_BA", # Croatian (Latin) - Bosnia and Herzegovina + 0x0005: "cs", # Czech + 0x0405: "cs_CZ", # Czech - Czech Republic + 0x0006: "da", # Danish + 0x0406: "da_DK", # Danish - Denmark + 0x008c: "prs", # Dari + 0x048c: "prs_AF", # Dari - Afghanistan + 0x0065: "dv", # Divehi + 0x0465: "dv_MV", # Divehi - Maldives + 0x0013: "nl", # Dutch + 0x0413: "nl_NL", # Dutch - Netherlands 0x0813: "nl_BE", # Dutch - Belgium + 0x0c51: "dz_BT", # Dzongkha - Bhutan + 0x0009: "en", # English 0x0409: "en_US", # English - United States 0x0809: "en_GB", # English - United Kingdom 0x0c09: "en_AU", # English - Australia @@ -1559,122 +1597,248 @@ def getpreferredencoding(do_setlocale=True): 0x1409: "en_NZ", # English - New Zealand 0x1809: "en_IE", # English - Ireland 0x1c09: "en_ZA", # English - South Africa - 0x2009: "en_JA", # English - Jamaica - 0x2409: "en_CB", # English - Caribbean + 0x2009: "en_JM", # English - Jamaica 0x2809: "en_BZ", # English - Belize - 0x2c09: "en_TT", # English - Trinidad + 0x2c09: "en_TT", # English - Trinidad and Tobago 0x3009: "en_ZW", # English - Zimbabwe - 0x3409: "en_PH", # English - Philippines + 0x3409: "en_PH", # English - Republic of the Philippines + 0x3c09: "en_HK", # English - Hong Kong 0x4009: "en_IN", # English - India 0x4409: "en_MY", # English - Malaysia - 0x4809: "en_IN", # English - Singapore - 0x0425: "et_EE", # Estonian - 0x0438: "fo_FO", # Faroese - 0x0464: "fil_PH",# Filipino - 0x040b: "fi_FI", # Finnish + 0x4809: "en_SG", # English - Singapore + 0x4c09: "en_AE", # English - United Arab Emirates + 0x0025: "et", # Estonian + 0x0425: "et_EE", # Estonian - Estonia + 0x0038: "fo", # Faroese + 0x0438: "fo_FO", # Faroese - Faroe Islands + 0x0064: "fil", # Filipino + 0x0464: "fil_PH", # Filipino - Philippines + 0x000b: "fi", # Finnish + 0x040b: "fi_FI", # Finnish - Finland + 0x000c: "fr", # French 0x040c: "fr_FR", # French - France 0x080c: "fr_BE", # French - Belgium 0x0c0c: "fr_CA", # French - Canada 0x100c: "fr_CH", # French - Switzerland 0x140c: "fr_LU", # French - Luxembourg - 0x180c: "fr_MC", # French - Monaco + 0x180c: "fr_MC", # French - Principality of Monaco + 0x1c0c: "fr_029", # French - Caribbean + 0x200c: "fr_RE", # French - Reunion + 0x240c: "fr_CD", # French - Congo, DRC + 0x280c: "fr_SN", # French - Senegal + 0x2c0c: "fr_CM", # French - Cameroon + 0x300c: "fr_CI", # French - Côte d'Ivoire + 0x340c: "fr_ML", # French - Mali + 0x380c: "fr_MA", # French - Morocco + 0x3c0c: "fr_HT", # French - Haiti + 0x0062: "fy", # Frisian 0x0462: "fy_NL", # Frisian - Netherlands - 0x0456: "gl_ES", # Galician - 0x0437: "ka_GE", # Georgian + 0x0067: "ff", # Fulah + 0x7c67: "ff", # Fulah (Latin) + 0x0467: "ff_NG", + 0x0867: "ff_SN", # Fulah - Senegal + 0x0056: "gl", # Galician + 0x0456: "gl_ES", # Galician - Spain + 0x0037: "ka", # Georgian + 0x0437: "ka_GE", # Georgian - Georgia + 0x0007: "de", # German 0x0407: "de_DE", # German - Germany 0x0807: "de_CH", # German - Switzerland 0x0c07: "de_AT", # German - Austria 0x1007: "de_LU", # German - Luxembourg 0x1407: "de_LI", # German - Liechtenstein - 0x0408: "el_GR", # Greek + 0x0008: "el", # Greek + 0x0408: "el_GR", # Greek - Greece + 0x006f: "kl", # Greenlandic 0x046f: "kl_GL", # Greenlandic - Greenland - 0x0447: "gu_IN", # Gujarati - 0x0468: "ha_NG", # Hausa - Latin - 0x040d: "he_IL", # Hebrew - 0x0439: "hi_IN", # Hindi - 0x040e: "hu_HU", # Hungarian - 0x040f: "is_IS", # Icelandic - 0x0421: "id_ID", # Indonesian - 0x045d: "iu_CA", # Inuktitut - Syllabics - 0x085d: "iu_CA", # Inuktitut - Latin + 0x0074: "gn", # Guarani + 0x0474: "gn_PY", # Guarani - Paraguay + 0x0047: "gu", # Gujarati + 0x0447: "gu_IN", # Gujarati - India + 0x0068: "ha", # Hausa (Latin) + 0x7c68: "ha", # Hausa (Latin) + 0x0468: "ha_NG", # Hausa (Latin) - Nigeria + 0x0075: "haw", # Hawaiian + 0x0475: "haw_US", # Hawaiian - United States + 0x000d: "he", # Hebrew + 0x040d: "he_IL", # Hebrew - Israel + 0x0039: "hi", # Hindi + 0x0439: "hi_IN", # Hindi - India + 0x000e: "hu", # Hungarian + 0x040e: "hu_HU", # Hungarian - Hungary + 0x000f: "is", # Icelandic + 0x040f: "is_IS", # Icelandic - Iceland + 0x0070: "ig", # Igbo + 0x0470: "ig_NG", # Igbo - Nigeria + 0x0021: "id", # Indonesian + 0x0421: "id_ID", # Indonesian - Indonesia + 0x005d: "iu", # Inuktitut (Latin) + 0x785d: "iu", # Inuktitut (Syllabics) + 0x7c5d: "iu", # Inuktitut (Latin) + 0x045d: "iu_CA", # Inuktitut (Syllabics) - Canada + 0x085d: "iu_CA", # Inuktitut (Latin) - Canada + 0x003c: "ga", # Irish 0x083c: "ga_IE", # Irish - Ireland + 0x0010: "it", # Italian 0x0410: "it_IT", # Italian - Italy 0x0810: "it_CH", # Italian - Switzerland - 0x0411: "ja_JP", # Japanese + 0x0011: "ja", # Japanese + 0x0411: "ja_JP", # Japanese - Japan + 0x004b: "kn", # Kannada 0x044b: "kn_IN", # Kannada - India - 0x043f: "kk_KZ", # Kazakh - 0x0453: "kh_KH", # Khmer - Cambodia - 0x0486: "qut_GT",# K'iche - Guatemala + 0x0471: "kr_NG", # Kanuri (Latin) - Nigeria + 0x0060: "ks", # Kashmiri + 0x0460: "ks", # Kashmiri - Perso_Arabic + 0x0860: "ks_IN", # Kashmiri (Devanagari) - India + 0x003f: "kk", # Kazakh + 0x043f: "kk_KZ", # Kazakh - Kazakhstan + 0x0053: "km", # Khmer + 0x0453: "km_KH", # Khmer - Cambodia + 0x0087: "rw", # Kinyarwanda 0x0487: "rw_RW", # Kinyarwanda - Rwanda - 0x0457: "kok_IN",# Konkani - 0x0412: "ko_KR", # Korean - 0x0440: "ky_KG", # Kyrgyz - 0x0454: "lo_LA", # Lao - Lao PDR - 0x0426: "lv_LV", # Latvian - 0x0427: "lt_LT", # Lithuanian - 0x082e: "dsb_DE",# Lower Sorbian - Germany - 0x046e: "lb_LU", # Luxembourgish - 0x042f: "mk_MK", # FYROM Macedonian + 0x0041: "sw", # Kiswahili + 0x0441: "sw_KE", # Kiswahili - Kenya + 0x0057: "kok", # Konkani + 0x0457: "kok_IN", # Konkani - India + 0x0012: "ko", # Korean + 0x0412: "ko_KR", # Korean - Korea + 0x0040: "ky", # Kyrgyz + 0x0440: "ky_KG", # Kyrgyz - Kyrgyzstan + 0x0054: "lo", # Lao + 0x0454: "lo_LA", # Lao - Lao P.D.R. + 0x0476: "la_VA", # Latin - Vatican City + 0x0026: "lv", # Latvian + 0x0426: "lv_LV", # Latvian - Latvia + 0x0027: "lt", # Lithuanian + 0x0427: "lt_LT", # Lithuanian - Lithuania + 0x7c2e: "dsb", # Lower Sorbian + 0x082e: "dsb_DE", # Lower Sorbian - Germany + 0x006e: "lb", # Luxembourgish + 0x046e: "lb_LU", # Luxembourgish - Luxembourg + 0x002f: "mk", # Macedonian + 0x042f: "mk_MK", # Macedonian - North Macedonia + 0x003e: "ms", # Malay 0x043e: "ms_MY", # Malay - Malaysia 0x083e: "ms_BN", # Malay - Brunei Darussalam + 0x004c: "ml", # Malayalam 0x044c: "ml_IN", # Malayalam - India - 0x043a: "mt_MT", # Maltese - 0x0481: "mi_NZ", # Maori - 0x047a: "arn_CL",# Mapudungun - 0x044e: "mr_IN", # Marathi - 0x047c: "moh_CA",# Mohawk - Canada - 0x0450: "mn_MN", # Mongolian - Cyrillic - 0x0850: "mn_CN", # Mongolian - PRC - 0x0461: "ne_NP", # Nepali - 0x0414: "nb_NO", # Norwegian - Bokmal - 0x0814: "nn_NO", # Norwegian - Nynorsk + 0x003a: "mt", # Maltese + 0x043a: "mt_MT", # Maltese - Malta + 0x0081: "mi", # Maori + 0x0481: "mi_NZ", # Maori - New Zealand + 0x007a: "arn", # Mapudungun + 0x047a: "arn_CL", # Mapudungun - Chile + 0x004e: "mr", # Marathi + 0x044e: "mr_IN", # Marathi - India + 0x007c: "moh", # Mohawk + 0x047c: "moh_CA", # Mohawk - Canada + 0x0050: "mn", # Mongolian (Cyrillic) + 0x7850: "mn", # Mongolian (Cyrillic) + 0x7c50: "mn", # Mongolian (Traditional Mongolian) + 0x0450: "mn_MN", # Mongolian (Cyrillic) - Mongolia + 0x0c50: "mn_MN", # Mongolian (Traditional Mongolian) - Mongolia + 0x0061: "ne", # Nepali + 0x0461: "ne_NP", # Nepali - Nepal + 0x0861: "ne_IN", # Nepali - India + 0x0014: "no", # Norwegian (Bokmal) + 0x0414: "nb_NO", # Norwegian (Bokmal) - Norway + 0x0814: "nn_NO", # Norwegian (Nynorsk) - Norway + 0x7814: "nn", # Norwegian (Nynorsk) + 0x7c14: "nb", # Norwegian (Bokmal) + 0x0082: "oc", # Occitan 0x0482: "oc_FR", # Occitan - France - 0x0448: "or_IN", # Oriya - India + 0x0048: "or", # Odia + 0x0448: "or_IN", # Odia - India + 0x0072: "om", # Oromo + 0x0472: "om_ET", # Oromo - Ethiopia + 0x0063: "ps", # Pashto 0x0463: "ps_AF", # Pashto - Afghanistan - 0x0429: "fa_IR", # Persian - 0x0415: "pl_PL", # Polish + 0x0029: "fa", # Persian + 0x0429: "fa_IR", # Persian - Iran + 0x0015: "pl", # Polish + 0x0415: "pl_PL", # Polish - Poland + 0x0016: "pt", # Portuguese 0x0416: "pt_BR", # Portuguese - Brazil 0x0816: "pt_PT", # Portuguese - Portugal - 0x0446: "pa_IN", # Punjabi - 0x046b: "quz_BO",# Quechua (Bolivia) - 0x086b: "quz_EC",# Quechua (Ecuador) - 0x0c6b: "quz_PE",# Quechua (Peru) + 0x0046: "pa", # Punjabi + 0x7c46: "pa", # Punjabi + 0x0446: "pa_IN", # Punjabi - India + 0x0846: "pa_PK", # Punjabi - Islamic Republic of Pakistan + 0x006b: "quz", # Quechua + 0x046b: "quz_BO", # Quechua - Bolivia + 0x086b: "quz_EC", # Quechua - Ecuador + 0x0c6b: "quz_PE", # Quechua - Peru + 0x0018: "ro", # Romanian 0x0418: "ro_RO", # Romanian - Romania - 0x0417: "rm_CH", # Romansh - 0x0419: "ru_RU", # Russian - 0x243b: "smn_FI",# Sami Finland - 0x103b: "smj_NO",# Sami Norway - 0x143b: "smj_SE",# Sami Sweden - 0x043b: "se_NO", # Sami Northern Norway - 0x083b: "se_SE", # Sami Northern Sweden - 0x0c3b: "se_FI", # Sami Northern Finland - 0x203b: "sms_FI",# Sami Skolt - 0x183b: "sma_NO",# Sami Southern Norway - 0x1c3b: "sma_SE",# Sami Southern Sweden - 0x044f: "sa_IN", # Sanskrit - 0x0c1a: "sr_SP", # Serbian - Cyrillic - 0x1c1a: "sr_BA", # Serbian - Bosnia Cyrillic - 0x081a: "sr_SP", # Serbian - Latin - 0x181a: "sr_BA", # Serbian - Bosnia Latin + 0x0818: "ro_MD", # Romanian - Moldova + 0x0017: "rm", # Romansh + 0x0417: "rm_CH", # Romansh - Switzerland + 0x0019: "ru", # Russian + 0x0419: "ru_RU", # Russian - Russia + 0x0819: "ru_MD", # Russian - Moldova + 0x0085: "sah", # Sakha + 0x0485: "sah_RU", # Sakha - Russia + 0x003b: "se", # Sami (Northern) + 0x043b: "se_NO", # Sami (Northern) - Norway + 0x083b: "se_SE", # Sami (Northern) - Sweden + 0x0c3b: "se_FI", # Sami (Northern) - Finland + 0x7c3b: "smj", # Sami (Lule) + 0x103b: "smj_NO", # Sami (Lule) - Norway + 0x143b: "smj_SE", # Sami (Lule) - Sweden + 0x783b: "sma", # Sami (Southern) + 0x183b: "sma_NO", # Sami (Southern) - Norway + 0x1c3b: "sma_SE", # Sami (Southern) - Sweden + 0x743b: "sms", # Sami (Skolt) + 0x203b: "sms_FI", # Sami (Skolt) - Finland + 0x703b: "smn", # Sami (Inari) + 0x243b: "smn_FI", # Sami (Inari) - Finland + 0x004f: "sa", # Sanskrit + 0x044f: "sa_IN", # Sanskrit - India + 0x0091: "gd", # Scottish Gaelic + 0x0491: "gd_GB", # Scottish Gaelic - United Kingdom + 0x6c1a: "sr", # Serbian (Cyrillic) + 0x701a: "sr", # Serbian (Latin) + 0x7c1a: "sr", # Serbian (Latin) + 0x081a: "sr_CS", # Serbian (Latin) - Serbia and Montenegro (Former) + 0x0c1a: "sr_CS", # Serbian (Cyrillic) - Serbia and Montenegro (Former) + 0x181a: "sr_BA", # Serbian (Latin) - Bosnia and Herzegovina + 0x1c1a: "sr_BA", # Serbian (Cyrillic) - Bosnia and Herzegovina + 0x241a: "sr_RS", # Serbian (Latin) - Serbia + 0x281a: "sr_RS", # Serbian (Cyrillic) - Serbia + 0x2c1a: "sr_ME", # Serbian (Latin) - Montenegro + 0x301a: "sr_ME", # Serbian (Cyrillic) - Montenegro + 0x006c: "nso", # Sesotho sa Leboa + 0x046c: "nso_ZA", # Sesotho sa Leboa - South Africa + 0x0032: "tn", # Setswana + 0x0432: "tn_ZA", # Setswana - South Africa + 0x0832: "tn_BW", # Setswana - Botswana + 0x0059: "sd", # Sindhi + 0x7c59: "sd", # Sindhi + 0x0859: "sd_PK", # Sindhi - Islamic Republic of Pakistan + 0x005b: "si", # Sinhala 0x045b: "si_LK", # Sinhala - Sri Lanka - 0x046c: "ns_ZA", # Northern Sotho - 0x0432: "tn_ZA", # Setswana - Southern Africa - 0x041b: "sk_SK", # Slovak - 0x0424: "sl_SI", # Slovenian + 0x001b: "sk", # Slovak + 0x041b: "sk_SK", # Slovak - Slovakia + 0x0024: "sl", # Slovenian + 0x0424: "sl_SI", # Slovenian - Slovenia + 0x0477: "so_SO", # Somali - Somalia + 0x0030: "st", # Sotho + 0x0430: "st_ZA", # Sotho - South Africa + 0x000a: "es", # Spanish 0x040a: "es_ES", # Spanish - Spain 0x080a: "es_MX", # Spanish - Mexico - 0x0c0a: "es_ES", # Spanish - Spain (Modern) + 0x0c0a: "es_ES", # Spanish - Spain 0x100a: "es_GT", # Spanish - Guatemala 0x140a: "es_CR", # Spanish - Costa Rica 0x180a: "es_PA", # Spanish - Panama 0x1c0a: "es_DO", # Spanish - Dominican Republic - 0x200a: "es_VE", # Spanish - Venezuela + 0x200a: "es_VE", # Spanish - Bolivarian Republic of Venezuela 0x240a: "es_CO", # Spanish - Colombia 0x280a: "es_PE", # Spanish - Peru 0x2c0a: "es_AR", # Spanish - Argentina 0x300a: "es_EC", # Spanish - Ecuador 0x340a: "es_CL", # Spanish - Chile - 0x380a: "es_UR", # Spanish - Uruguay + 0x380a: "es_UY", # Spanish - Uruguay 0x3c0a: "es_PY", # Spanish - Paraguay 0x400a: "es_BO", # Spanish - Bolivia 0x440a: "es_SV", # Spanish - El Salvador @@ -1682,36 +1846,87 @@ def getpreferredencoding(do_setlocale=True): 0x4c0a: "es_NI", # Spanish - Nicaragua 0x500a: "es_PR", # Spanish - Puerto Rico 0x540a: "es_US", # Spanish - United States -# 0x0430: "", # Sutu - Not supported - 0x0441: "sw_KE", # Swahili + 0x5c0a: "es_CU", # Spanish - Cuba + 0x001d: "sv", # Swedish 0x041d: "sv_SE", # Swedish - Sweden 0x081d: "sv_FI", # Swedish - Finland - 0x045a: "syr_SY",# Syriac - 0x0428: "tg_TJ", # Tajik - Cyrillic - 0x085f: "tmz_DZ",# Tamazight - Latin - 0x0449: "ta_IN", # Tamil - 0x0444: "tt_RU", # Tatar - 0x044a: "te_IN", # Telugu - 0x041e: "th_TH", # Thai - 0x0851: "bo_BT", # Tibetan - Bhutan - 0x0451: "bo_CN", # Tibetan - PRC - 0x041f: "tr_TR", # Turkish - 0x0442: "tk_TM", # Turkmen - Cyrillic - 0x0480: "ug_CN", # Uighur - Arabic - 0x0422: "uk_UA", # Ukrainian - 0x042e: "wen_DE",# Upper Sorbian - Germany - 0x0420: "ur_PK", # Urdu + 0x005a: "syr", # Syriac + 0x045a: "syr_SY", # Syriac - Syria + 0x0028: "tg", # Tajik (Cyrillic) + 0x7c28: "tg", # Tajik (Cyrillic) + 0x0428: "tg_TJ", # Tajik (Cyrillic) - Tajikistan + 0x005f: "tzm", # Tamazight (Latin) + 0x785f: "tzm", + 0x7c5f: "tzm", # Tamazight (Latin) + 0x085f: "tzm_DZ", # Tamazight (Latin) - Algeria + 0x045f: "tzm_MA", # Central Atlas Tamazight (Arabic) - Morocco + 0x105f: "tzm_MA", + 0x0049: "ta", # Tamil + 0x0449: "ta_IN", # Tamil - India + 0x0849: "ta_LK", # Tamil - Sri Lanka + 0x0044: "tt", # Tatar + 0x0444: "tt_RU", # Tatar - Russia + 0x004a: "te", # Telugu + 0x044a: "te_IN", # Telugu - India + 0x001e: "th", # Thai + 0x041e: "th_TH", # Thai - Thailand + 0x0051: "bo", # Tibetan + 0x0451: "bo_CN", # Tibetan - People's Republic of China + 0x0073: "ti", # Tigrinya + 0x0473: "ti_ET", # Tigrinya - Ethiopia + 0x0873: "ti_ER", # Tigrinya - Eritrea + 0x0031: "ts", # Tsonga + 0x0431: "ts_ZA", # Tsonga - South Africa + 0x001f: "tr", # Turkish + 0x041f: "tr_TR", # Turkish - Turkey + 0x0042: "tk", # Turkmen + 0x0442: "tk_TM", # Turkmen - Turkmenistan + 0x0022: "uk", # Ukrainian + 0x0422: "uk_UA", # Ukrainian - Ukraine + 0x002e: "hsb", # Upper Sorbian + 0x042e: "hsb_DE", # Upper Sorbian - Germany + 0x0020: "ur", # Urdu + 0x0420: "ur_PK", # Urdu - Islamic Republic of Pakistan 0x0820: "ur_IN", # Urdu - India - 0x0443: "uz_UZ", # Uzbek - Latin - 0x0843: "uz_UZ", # Uzbek - Cyrillic - 0x042a: "vi_VN", # Vietnamese - 0x0452: "cy_GB", # Welsh + 0x0080: "ug", # Uyghur + 0x0480: "ug_CN", # Uyghur - People's Republic of China + 0x0043: "uz", # Uzbek (Latin) + 0x7843: "uz", # Uzbek (Cyrillic) + 0x7c43: "uz", # Uzbek (Latin) + 0x0443: "uz_UZ", # Uzbek (Latin) - Uzbekistan + 0x0033: "ve", # Venda + 0x0433: "ve_ZA", # Venda - South Africa + 0x002a: "vi", # Vietnamese + 0x042a: "vi_VN", # Vietnamese - Vietnam + 0x0052: "cy", # Welsh + 0x0452: "cy_GB", # Welsh - United Kingdom + 0x0088: "wo", # Wolof 0x0488: "wo_SN", # Wolof - Senegal + 0x0034: "xh", # Xhosa 0x0434: "xh_ZA", # Xhosa - South Africa - 0x0485: "sah_RU",# Yakut - Cyrillic - 0x0478: "ii_CN", # Yi - PRC + 0x0078: "ii", # Yi + 0x0478: "ii_CN", # Yi - People's Republic of China + 0x043d: "yi_001", # Yiddish - World + 0x006a: "yo", # Yoruba 0x046a: "yo_NG", # Yoruba - Nigeria - 0x0435: "zu_ZA", # Zulu + 0x0035: "zu", # Zulu + 0x0435: "zu_ZA", # Zulu - South Africa + 0x0086: "qut", + +# 0x0001007f: "x-IV-mathan", # math alphanumeric sorting + 0x00010407: "de_DE", + 0x0001040e: "hu_HU", + 0x00010437: "ka_GE", + 0x00020804: "zh_CN", + 0x00021004: "zh_SG", + 0x00021404: "zh_MO", + 0x00030404: "zh_TW", + 0x00040404: "zh_TW", + 0x00040411: "ja_JP", + 0x00040c04: "zh_HK", + 0x00041404: "zh_MO", + 0x00050804: "zh_CN", + 0x00051004: "zh_SG", } def _print_locale(): diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index efb9ea95ab4..2a0151dab80 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -75,7 +75,11 @@ def arbitrary_address(family): if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': - return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir()) + # NOTE: util.get_temp_dir() is a 0o700 per-process directory. A + # mktemp-style ToC vs ToU concern is not important; bind() surfaces + # the extremely unlikely collision as EADDRINUSE. + return os.path.join(util.get_temp_dir(), + f'sock-{os.urandom(6).hex()}') elif family == 'AF_PIPE': return (r'\\.\pipe\pyc-%d-%d-%s' % (os.getpid(), next(_mmap_counter), os.urandom(8).hex())) diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index a57ef6bdad5..05593737dbb 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -64,7 +64,17 @@ def _launch(self, process_obj): code = 1 parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() - self.pid = os.fork() + # gh-146313: Tell the resource tracker's at-fork handler to keep + # the inherited pipe fd so this child reuses the parent's tracker + # (gh-80849) rather than closing it and launching its own. + from .resource_tracker import _fork_intent + _fork_intent.preserve_fd = True + try: + self.pid = os.fork() + finally: + # Reset in both parent and child so the flag does not leak + # into a subsequent raw os.fork() or nested Process launch. + _fork_intent.preserve_fd = False if self.pid == 0: try: atexit._clear() diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 22e3bbcf21b..028c1f60e59 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -20,6 +20,7 @@ import signal import sys import threading +import time import warnings from collections import deque @@ -79,6 +80,10 @@ def __init__(self): # The reader should understand all formats. self._use_simple_format = True + # Set to True by _stop_locked() if the waitpid polling loop ran to + # its timeout without reaping the tracker. Exposed for tests. + self._waitpid_timed_out = False + def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker # gets interrupted by a garbage collection, invoking a finalizer (*) @@ -91,16 +96,51 @@ def __del__(self): # making sure child processess are cleaned before ResourceTracker # gets destructed. # see https://github.com/python/cpython/issues/88887 - self._stop(use_blocking_lock=False) + # gh-146313: use a timeout to avoid deadlocking if a forked child + # still holds the pipe's write end open. + self._stop(use_blocking_lock=False, wait_timeout=1.0) + + def _after_fork_in_child(self): + # gh-146313: Called in the child right after os.fork(). + # + # The tracker process is a child of the *parent*, not of us, so we + # could never waitpid() it anyway. Clearing _pid means our __del__ + # becomes a no-op (the early return for _pid is None). + # + # Whether we keep the inherited _fd depends on who forked us: + # + # - multiprocessing.Process with the 'fork' start method sets + # _fork_intent.preserve_fd before forking. The child keeps the + # fd and reuses the parent's tracker (gh-80849). This is safe + # because multiprocessing's atexit handler joins all children + # before the parent's __del__ runs, so by then the fd copies + # are gone and the parent can reap the tracker promptly. + # + # - A raw os.fork() leaves the flag unset. We close the fd in the child after forking so + # the parent's __del__ can reap the tracker without waiting + # for the child to exit. If we later need a tracker, ensure_running() + # will launch a fresh one. + self._lock._at_fork_reinit() + self._reentrant_messages.clear() + self._pid = None + self._exitcode = None + if (self._fd is not None and + not getattr(_fork_intent, 'preserve_fd', False)): + fd = self._fd + self._fd = None + try: + os.close(fd) + except OSError: + pass - def _stop(self, use_blocking_lock=True): + def _stop(self, use_blocking_lock=True, wait_timeout=None): if use_blocking_lock: with self._lock: - self._stop_locked() + self._stop_locked(wait_timeout=wait_timeout) else: acquired = self._lock.acquire(blocking=False) try: - self._stop_locked() + self._stop_locked(wait_timeout=wait_timeout) finally: if acquired: self._lock.release() @@ -110,6 +150,10 @@ def _stop_locked( close=os.close, waitpid=os.waitpid, waitstatus_to_exitcode=os.waitstatus_to_exitcode, + monotonic=time.monotonic, + sleep=time.sleep, + WNOHANG=getattr(os, 'WNOHANG', None), + wait_timeout=None, ): # This shouldn't happen (it might when called by a finalizer) # so we check for it anyway. @@ -126,7 +170,30 @@ def _stop_locked( self._fd = None try: - _, status = waitpid(self._pid, 0) + if wait_timeout is None: + _, status = waitpid(self._pid, 0) + else: + # gh-146313: A forked child may still hold the pipe's write + # end open, preventing the tracker from seeing EOF and + # exiting. Poll with WNOHANG to avoid blocking forever. + deadline = monotonic() + wait_timeout + delay = 0.001 + while True: + result_pid, status = waitpid(self._pid, WNOHANG) + if result_pid != 0: + break + remaining = deadline - monotonic() + if remaining <= 0: + # The tracker is still running; it will be + # reparented to PID 1 (or the nearest subreaper) + # when we exit, and reaped there once all pipe + # holders release their fd. + self._pid = None + self._exitcode = None + self._waitpid_timed_out = True + return + delay = min(delay * 2, remaining, 0.1) + sleep(delay) except ChildProcessError: self._pid = None self._exitcode = None @@ -312,12 +379,24 @@ def _send(self, cmd, name, rtype): self._ensure_running_and_write(msg) +# gh-146313: Per-thread flag set by .popen_fork.Popen._launch() just before +# os.fork(), telling _after_fork_in_child() to keep the inherited pipe fd so +# the child can reuse this tracker (gh-80849). Unset for raw os.fork() calls, +# where the child instead closes the fd so the parent's __del__ can reap the +# tracker. Using threading.local() keeps multiple threads calling +# popen_fork.Popen._launch() at once from clobbering eachothers intent. +_fork_intent = threading.local() + _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running register = _resource_tracker.register unregister = _resource_tracker.unregister getfd = _resource_tracker.getfd +# gh-146313: See _after_fork_in_child docstring. +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_resource_tracker._after_fork_in_child) + def _decode_message(line): if line.startswith(b'{'): diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 01f060e70be..eb127ec2632 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -152,12 +152,14 @@ def splitdrive(p): It is always true that: result[0] + result[1] == p - If the path contained a drive letter, drive_or_unc will contain everything - up to and including the colon. e.g. splitdrive("c:/dir") returns ("c:", "/dir") + If the path contained a drive letter, drive_or_unc will contain + everything up to and including the colon. e.g. splitdrive("c:/dir") + returns ("c:", "/dir") - If the path contained a UNC path, the drive_or_unc will contain the host name - and share up to but not including the fourth directory separator character. - e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir") + If the path contained a UNC path, the drive_or_unc will contain the + host name and share up to but not including the fourth directory + separator character. e.g. splitdrive("//host/computer/dir") returns + ("//host/computer", "/dir") Paths cannot contain both a drive letter and a UNC path. @@ -222,8 +224,8 @@ def splitroot(p): def split(p): """Split a pathname. - Return tuple (head, tail) where tail is everything after the final slash. - Either part may be empty.""" + Return tuple (head, tail) where tail is everything after the final + slash. Either part may be empty.""" p = os.fspath(p) seps = _get_bothseps(p) d, r, p = splitroot(p) diff --git a/Lib/pickle.py b/Lib/pickle.py index 550f8675f2c..12da440b9f8 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -847,17 +847,11 @@ def save_picklebuffer(self, obj): # Write data in-band # XXX The C implementation avoids a copy here buf = m.tobytes() - in_memo = id(buf) in self.memo if m.readonly: - if in_memo: - self._save_bytes_no_memo(buf) - else: - self.save_bytes(buf) + self._save_bytes_no_memo(buf) else: - if in_memo: - self._save_bytearray_no_memo(buf) - else: - self.save_bytearray(buf) + self._save_bytearray_no_memo(buf) + self.memoize(obj) else: # Write data out-of-band self.write(NEXT_BUFFER) diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py index 8c4013606ca..9e456490967 100644 --- a/Lib/pydoc_data/module_docs.py +++ b/Lib/pydoc_data/module_docs.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Apr 7 20:18:56 2026 +# Autogenerated by Sphinx on Wed Jun 10 14:23:59 2026 # as part of the release process. module_docs = { diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index bbbd6a3effd..31d582a90ab 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Apr 7 20:18:56 2026 +# Autogenerated by Sphinx on Wed Jun 10 14:23:59 2026 # as part of the release process. topics = { @@ -2122,9 +2122,9 @@ class attributes; they are shared by instances. Instance attributes The match statement is used for pattern matching. Syntax: match_stmt ::= 'match' subject_expr ":" NEWLINE INDENT case_block+ DEDENT - subject_expr ::= `!star_named_expression` "," `!star_named_expressions`? - | `!named_expression` - case_block ::= 'case' patterns [guard] ":" `!block` + subject_expr ::= flexible_expression "," [flexible_expression_list [',']] + | assignment_expression + case_block ::= 'case' patterns [guard] ":" suite Note: @@ -2215,7 +2215,7 @@ class attributes; they are shared by instances. Instance attributes Guards ------ - guard ::= "if" `!named_expression` + guard ::= "if" assignment_expression A "guard" (which is part of the "case") must succeed for code inside the "case" block to execute. It takes the form: "if" followed by an @@ -4625,11 +4625,6 @@ class of the instance or a *non-virtual base class* thereof. The See also the description of the "try" statement in section The try statement and "raise" statement in section The raise statement. - --[ Footnotes ]- - -[1] This limitation occurs because the code that is executed by these - operations is not available at the time the module is compiled. ''', 'execmodel': r'''Execution model *************** @@ -4976,6 +4971,180 @@ class of the instance or a *non-virtual base class* thereof. The See also the description of the "try" statement in section The try statement and "raise" statement in section The raise statement. + +Runtime Components +================== + + +General Computing Model +----------------------- + +Python’s execution model does not operate in a vacuum. It runs on a +host machine and through that host’s runtime environment, including +its operating system (OS), if there is one. When a program runs, the +conceptual layers of how it runs on the host look something like this: + + **host machine** + **process** (global resources) + **thread** (runs machine code) + +Each process represents a program running on the host. Think of each +process itself as the data part of its program. Think of the process’ +threads as the execution part of the program. This distinction will +be important to understand the conceptual Python runtime. + +The process, as the data part, is the execution context in which the +program runs. It mostly consists of the set of resources assigned to +the program by the host, including memory, signals, file handles, +sockets, and environment variables. + +Processes are isolated and independent from one another. (The same is +true for hosts.) The host manages the process’ access to its assigned +resources, in addition to coordinating between processes. + +Each thread represents the actual execution of the program’s machine +code, running relative to the resources assigned to the program’s +process. It’s strictly up to the host how and when that execution +takes place. + +From the point of view of Python, a program always starts with exactly +one thread. However, the program may grow to run in multiple +simultaneous threads. Not all hosts support multiple threads per +process, but most do. Unlike processes, threads in a process are not +isolated and independent from one another. Specifically, all threads +in a process share all of the process’ resources. + +The fundamental point of threads is that each one does *run* +independently, at the same time as the others. That may be only +conceptually at the same time (“concurrently”) or physically (“in +parallel”). Either way, the threads effectively run at a non- +synchronized rate. + +Note: + + That non-synchronized rate means none of the process’ memory is + guaranteed to stay consistent for the code running in any given + thread. Thus multi-threaded programs must take care to coordinate + access to intentionally shared resources. Likewise, they must take + care to be absolutely diligent about not accessing any *other* + resources in multiple threads; otherwise two threads running at the + same time might accidentally interfere with each other’s use of some + shared data. All this is true for both Python programs and the + Python runtime.The cost of this broad, unstructured requirement is + the tradeoff for the kind of raw concurrency that threads provide. + The alternative to the required discipline generally means dealing + with non-deterministic bugs and data corruption. + + +Python Runtime Model +-------------------- + +The same conceptual layers apply to each Python program, with some +extra data layers specific to Python: + + **host machine** + **process** (global resources) + Python global runtime (*state*) + Python interpreter (*state*) + **thread** (runs Python bytecode and “C-API”) + Python thread *state* + +At the conceptual level: when a Python program starts, it looks +exactly like that diagram, with one of each. The runtime may grow to +include multiple interpreters, and each interpreter may grow to +include multiple thread states. + +Note: + + A Python implementation won’t necessarily implement the runtime + layers distinctly or even concretely. The only exception is places + where distinct layers are directly specified or exposed to users, + like through the "threading" module. + +Note: + + The initial interpreter is typically called the “main” interpreter. + Some Python implementations, like CPython, assign special roles to + the main interpreter.Likewise, the host thread where the runtime was + initialized is known as the “main” thread. It may be different from + the process’ initial thread, though they are often the same. In + some cases “main thread” may be even more specific and refer to the + initial thread state. A Python runtime might assign specific + responsibilities to the main thread, such as handling signals. + +As a whole, the Python runtime consists of the global runtime state, +interpreters, and thread states. The runtime ensures all that state +stays consistent over its lifetime, particularly when used with +multiple host threads. + +The global runtime, at the conceptual level, is just a set of +interpreters. While those interpreters are otherwise isolated and +independent from one another, they may share some data or other +resources. The runtime is responsible for managing these global +resources safely. The actual nature and management of these resources +is implementation-specific. Ultimately, the external utility of the +global runtime is limited to managing interpreters. + +In contrast, an “interpreter” is conceptually what we would normally +think of as the (full-featured) “Python runtime”. When machine code +executing in a host thread interacts with the Python runtime, it calls +into Python in the context of a specific interpreter. + +Note: + + The term “interpreter” here is not the same as the “bytecode + interpreter”, which is what regularly runs in threads, executing + compiled Python code.In an ideal world, “Python runtime” would refer + to what we currently call “interpreter”. However, it’s been called + “interpreter” at least since introduced in 1997 (CPython:a027efa5b). + +Each interpreter completely encapsulates all of the non-process- +global, non-thread-specific state needed for the Python runtime to +work. Notably, the interpreter’s state persists between uses. It +includes fundamental data like "sys.modules". The runtime ensures +multiple threads using the same interpreter will safely share it +between them. + +A Python implementation may support using multiple interpreters at the +same time in the same process. They are independent and isolated from +one another. For example, each interpreter has its own "sys.modules". + +For thread-specific runtime state, each interpreter has a set of +thread states, which it manages, in the same way the global runtime +contains a set of interpreters. It can have thread states for as many +host threads as it needs. It may even have multiple thread states for +the same host thread, though that isn’t as common. + +Each thread state, conceptually, has all the thread-specific runtime +data an interpreter needs to operate in one host thread. The thread +state includes the current raised exception and the thread’s Python +call stack. It may include other thread-specific resources. + +Note: + + The term “Python thread” can sometimes refer to a thread state, but + normally it means a thread created using the "threading" module. + +Each thread state, over its lifetime, is always tied to exactly one +interpreter and exactly one host thread. It will only ever be used in +that thread and with that interpreter. + +Multiple thread states may be tied to the same host thread, whether +for different interpreters or even the same interpreter. However, for +any given host thread, only one of the thread states tied to it can be +used by the thread at a time. + +Thread states are isolated and independent from one another and don’t +share any data, except for possibly sharing an interpreter and objects +or other resources belonging to that interpreter. + +Once a program is running, new Python threads can be created using the +"threading" module (on platforms and Python implementations that +support threads). Additional processes can be created using the "os", +"subprocess", and "multiprocessing" modules. Coroutines (async) can be +run using "asyncio" in each interpreter, typically only in a single +thread (often the main thread). + -[ Footnotes ]- [1] This limitation occurs because the code that is executed by these @@ -5317,7 +5486,8 @@ class of the instance or a *non-virtual base class* thereof. The | | is not supported. | +-----------+------------------------------------------------------------+ -For a locale aware separator, use the "'n'" presentation type instead. +For a locale-aware separator, use the "'n'" float presentation type or +integer presentation type instead. Changed in version 3.1: Added the "','" option (see also **PEP 378**). @@ -5368,7 +5538,10 @@ class of the instance or a *non-virtual base class* thereof. The +-----------+------------------------------------------------------------+ | "'n'" | Number. This is the same as "'d'", except that it uses the | | | current locale setting to insert the appropriate digit | - | | group separators. | + | | group separators. Note that the default locale is not the | + | | system locale. Depending on your use case, you may wish to | + | | set "LC_NUMERIC" with "locale.setlocale()" before using | + | | "'n'". | +-----------+------------------------------------------------------------+ | None | The same as "'d'". | +-----------+------------------------------------------------------------+ @@ -5442,7 +5615,10 @@ class of the instance or a *non-virtual base class* thereof. The +-----------+------------------------------------------------------------+ | "'n'" | Number. This is the same as "'g'", except that it uses the | | | current locale setting to insert the appropriate digit | - | | group separators for the integral part of a number. | + | | group separators for the integral part of a number. Note | + | | that the default locale is not the system locale. | + | | Depending on your use case, you may wish to set | + | | "LC_NUMERIC" with "locale.setlocale()" before using "'n'". | +-----------+------------------------------------------------------------+ | "'%'" | Percentage. Multiplies the number by 100 and displays in | | | fixed ("'f'") format, followed by a percent sign. | @@ -6039,8 +6215,9 @@ def whats_on_the_telly(penguin=None): 1. find a module, loading and initializing it if necessary -2. define a name or names in the local namespace for the scope where - the "import" statement occurs. +2. define a name or names in the current namespace for the scope where + the "import" statement occurs, just as an assignment statement + would (including "global" and "nonlocal" semantics). When the statement contains multiple clauses (separated by commas) the two steps are carried out separately for each clause, just as though @@ -6085,7 +6262,7 @@ def whats_on_the_telly(penguin=None): 3. if the attribute is not found, "ImportError" is raised. - 4. otherwise, a reference to that value is stored in the local + 4. otherwise, a reference to that value is stored in the current namespace, using the name in the "as" clause if it is present, otherwise using the attribute name @@ -9342,9 +9519,22 @@ class is used in a class pattern with positional arguments, each decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, like the Kharosthi - numbers. Formally, a digit is a character that has the property + numbers. Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also "isdecimal()" and "isnumeric()". + str.isidentifier() Return "True" if the string is a valid identifier according to the @@ -9380,15 +9570,14 @@ class is used in a class pattern with positional arguments, each >>> '0123456789'.isnumeric() True - >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine True >>> '⅕'.isnumeric() # Vulgar fraction one fifth True >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() (False, True, True) - See also "isdecimal()" and "isdigit()". Numeric characters are a - superset of decimal numbers. + See also "isdecimal()" and "isdigit()". str.isprintable() @@ -9770,7 +9959,7 @@ class is used in a class pattern with positional arguments, each >>> " foo ".split(maxsplit=0) ['foo '] - See also "join()". + See also "join()" and "rsplit()". str.splitlines(keepends=False) @@ -9860,6 +10049,8 @@ class is used in a class pattern with positional arguments, each not a prefix or suffix; rather, all combinations of its values are stripped. + Whitespace characters are defined by "str.isspace()". + For example: >>> ' spacious '.strip() @@ -9884,8 +10075,18 @@ class is used in a class pattern with positional arguments, each str.swapcase() Return a copy of the string with uppercase characters converted to - lowercase and vice versa. Note that it is not necessarily true that - "s.swapcase().swapcase() == s". + lowercase and vice versa. For example: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that "s.swapcase().swapcase() + == s". For example: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also "str.lower()" and "str.upper()". str.title() @@ -11899,6 +12100,9 @@ class dict(iterable, /, **kwargs) insertion order. This behavior was an implementation detail of CPython from 3.6. + Dictionaries are generic over two types, signifying (respectively) + the types of the dictionary’s keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -12598,6 +12802,8 @@ class list(iterable=(), /) Many other operations also produce lists, including the "sorted()" built-in. + Lists are generic over the types of their items. + Lists implement all of the common and mutable sequence operations. Lists also provide the following additional method: @@ -12683,6 +12889,10 @@ class tuple(iterable=(), /) Tuples implement all of the common sequence operations. + Tuples are generic over the types of their contents. For more + information, refer to the typing documentation on annotating + tuples. + For heterogeneous collections of data where access by name is clearer than access by index, "collections.namedtuple()" may be a more appropriate choice than a simple tuple object. diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 23eb0020f42..e75dd0a9e3d 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -34,6 +34,7 @@ import inspect import keyword import re +import types import __main__ import warnings @@ -178,14 +179,14 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - if isinstance(getattr(type(thisobject), word, None), - property): - # bpo-44752: thisobject.word is a method decorated by - # `@property`. What follows applies a postfix if - # thisobject.word is callable, but know we know that - # this is not callable (because it is a property). - # Also, getattr(thisobject, word) will evaluate the - # property method, which is not desirable. + + class_attr = getattr(type(thisobject), word, None) + if isinstance( + class_attr, + (property, types.GetSetDescriptorType, types.MemberDescriptorType) + ) or (hasattr(class_attr, '__get__') and not callable(class_attr)): + # Avoid evaluating descriptors, which could run + # arbitrary code or raise exceptions. matches.append(match) continue if (value := getattr(thisobject, word, None)) is not None: diff --git a/Lib/runpy.py b/Lib/runpy.py index ef54d3282ee..1d5ecf0cf15 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -103,8 +103,10 @@ def _run_module_code(code, init_globals=None, # Helper to get the full name, spec and code for a module def _get_module_details(mod_name, error=ImportError): + # name= is only accepted by ImportError and its subclasses. + kwargs = {"name": mod_name} if issubclass(error, ImportError) else {} if mod_name.startswith("."): - raise error("Relative module names not supported") + raise error("Relative module names not supported", **kwargs) pkg_name, _, _ = mod_name.rpartition(".") if pkg_name: # Try importing the parent to avoid catching initialization errors @@ -137,12 +139,13 @@ def _get_module_details(mod_name, error=ImportError): if mod_name.endswith(".py"): msg += (f". Try using '{mod_name[:-3]}' instead of " f"'{mod_name}' as the module name.") - raise error(msg.format(mod_name, type(ex).__name__, ex)) from ex + raise error(msg.format(mod_name, type(ex).__name__, ex), + **kwargs) from ex if spec is None: - raise error("No module named %s" % mod_name) + raise error("No module named %s" % mod_name, **kwargs) if spec.submodule_search_locations is not None: if mod_name == "__main__" or mod_name.endswith(".__main__"): - raise error("Cannot use package as __main__ module") + raise error("Cannot use package as __main__ module", **kwargs) try: pkg_main_name = mod_name + ".__main__" return _get_module_details(pkg_main_name, error) @@ -150,17 +153,19 @@ def _get_module_details(mod_name, error=ImportError): if mod_name not in sys.modules: raise # No module loaded; being a package is irrelevant raise error(("%s; %r is a package and cannot " + - "be directly executed") %(e, mod_name)) + "be directly executed") %(e, mod_name), + **kwargs) loader = spec.loader if loader is None: raise error("%r is a namespace package and cannot be executed" - % mod_name) + % mod_name, + **kwargs) try: code = loader.get_code(mod_name) except ImportError as e: - raise error(format(e)) from e + raise error(format(e), **kwargs) from e if code is None: - raise error("No code object available for %s" % mod_name) + raise error("No code object available for %s" % mod_name, **kwargs) return mod_name, spec, code class _Error(Exception): @@ -234,6 +239,7 @@ def _get_main_module_details(error=ImportError): # Also moves the standard __main__ out of the way so that the # preexisting __loader__ entry doesn't cause issues main_name = "__main__" + kwargs = {"name": main_name} if issubclass(error, ImportError) else {} saved_main = sys.modules[main_name] del sys.modules[main_name] try: @@ -241,7 +247,8 @@ def _get_main_module_details(error=ImportError): except ImportError as exc: if main_name in str(exc): raise error("can't find %r module in %r" % - (main_name, sys.path[0])) from exc + (main_name, sys.path[0]), + **kwargs) from exc raise finally: sys.modules[main_name] = saved_main diff --git a/Lib/shutil.py b/Lib/shutil.py index 7df972012c7..86322f05444 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -823,10 +823,14 @@ def move(src, dst, copy_function=copy2): If dst already exists but is not a directory, it may be overwritten depending on os.rename() semantics. - If the destination is on our current filesystem, then rename() is used. - Otherwise, src is copied to the destination and then removed. Symlinks are - recreated under the new name if os.rename() fails because of cross - filesystem renames. + os.rename() is preferably used if the source and destination are on the + same filesystem. In case os.rename() fails due to OSError (e.g. the user + has write permission to *dst* file but not to its parent directory), + this method falls back to using *copy_function* silently. + Symlinks are also recreated under the new name if os.rename() fails + because of cross filesystem renames. + + It's recommended to use os.rename() if atomic move is strictly required. The optional `copy_function` argument is a callable that will be used to copy the source or it will be delegated to `copytree`. @@ -878,8 +882,8 @@ def move(src, dst, copy_function=copy2): return real_dst def _destinsrc(src, dst): - src = os.path.abspath(src) - dst = os.path.abspath(dst) + src = os.path.realpath(src) + dst = os.path.realpath(dst) if not src.endswith(os.path.sep): src += os.path.sep if not dst.endswith(os.path.sep): @@ -1246,27 +1250,9 @@ def _unpack_zipfile(filename, extract_dir): if not zipfile.is_zipfile(filename): raise ReadError("%s is not a zip file" % filename) - zip = zipfile.ZipFile(filename) - try: - for info in zip.infolist(): - name = info.filename - - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - - targetpath = os.path.join(extract_dir, *name.split('/')) - if not targetpath: - continue - - _ensure_directory(targetpath) - if not name.endswith('/'): - # file - with zip.open(name, 'r') as source, \ - open(targetpath, 'wb') as target: - copyfileobj(source, target) - finally: - zip.close() + with zipfile.ZipFile(filename) as zip: + zip._ignore_invalid_names = True + zip.extractall(extract_dir) def _unpack_tarfile(filename, extract_dir, *, filter=None): """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 9bedcc5ff6c..95ad966c393 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -253,7 +253,6 @@ def __init__(self, host='', port=0, local_hostname=None, will be used. """ - self._host = host self.timeout = timeout self.esmtp_features = {} self.command_encoding = 'ascii' @@ -344,6 +343,7 @@ def connect(self, host='localhost', port=0, source_address=None): port = int(port) except ValueError: raise OSError("nonnumeric port") + self._host = host if not port: port = self.default_port sys.audit("smtplib.connect", self, host, port) diff --git a/Lib/socket.py b/Lib/socket.py index 35d87eff34d..55644bd4c17 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -634,18 +634,22 @@ def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): # Authenticating avoids using a connection from something else # able to connect to {host}:{port} instead of us. # We expect only AF_INET and AF_INET6 families. - try: - if ( - ssock.getsockname() != csock.getpeername() - or csock.getsockname() != ssock.getpeername() - ): - raise ConnectionError("Unexpected peer connection") - except: - # getsockname() and getpeername() can fail - # if either socket isn't connected. - ssock.close() - csock.close() - raise + # + # Note that we skip this on WASI because on that platorm the client socket + # may not have finished connecting by the time we've reached this point (gh-146139). + if sys.platform != "wasi": + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise return (ssock, csock) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 533c0cc8736..b22b2b38abb 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -489,7 +489,7 @@ def _init_read_gz(self): if flag & 4: xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) + self.__read(xlen) if flag & 8: while True: s = self.__read(1) @@ -819,16 +819,22 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + # A link member that resolves to the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if target_path == dest_path: + raise OutsideDestinationError(member, target_path) normalized = os.path.normpath(member.linkname) if normalized != member.linkname: new_attrs['linkname'] = normalized if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) + # The symlink is created at `name` with trailing separators + # stripped, so its target is relative to the directory + # containing that path. + link_dir = os.path.dirname(name.rstrip('/' + os.sep)) + target_path = os.path.join(dest_path, link_dir, normalized) else: - target_path = os.path.join(dest_path, - member.linkname) + target_path = os.path.join(dest_path, normalized) target_path = os.path.realpath(target_path, strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: @@ -882,11 +888,14 @@ class TarInfo(object): size = 'Size in bytes.', mtime = 'Time of last modification.', chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), + type = ('File type. type is usually one of these constants: ' + 'REGTYPE,\n' + 'AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' + 'CONTTYPE, CHRTYPE,\n' + 'BLKTYPE, GNUTYPE_SPARSE.'), linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), + 'in TarInfo\n' + 'objects of type LNKTYPE and SYMTYPE.'), uname = 'User name.', gname = 'Group name.', devmajor = 'Device major number.', @@ -894,7 +903,8 @@ class TarInfo(object): offset = 'The tar header starts here.', offset_data = "The file's data starts here.", pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), + 'associated pax\n' + 'extended header.'), sparse = 'Sparse member information.', _tarfile = None, _sparse_structs = None, @@ -2198,10 +2208,11 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. `members' is optional and must be a subset of the - list returned by getmembers(). + """Print a table of contents to sys.stdout. + + If `verbose' is False, only the names of the members are printed. + If it is True, an `ls -l'-like output is produced. `members' is + optional and must be a subset of the list returned by getmembers(). """ # Convert tarinfo type to stat type. type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, @@ -2292,10 +2303,12 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `tarinfo' represents - a non zero-size regular file, the `fileobj' argument should be a binary file, - and tarinfo.size bytes are read from it and added to the archive. - You can create TarInfo objects directly, or by using gettarinfo(). + """Add the TarInfo object `tarinfo' to the archive. + + If `tarinfo' represents a non zero-size regular file, the `fileobj' + argument should be a binary file, and tarinfo.size bytes are read + from it and added to the archive. You can create TarInfo objects + directly, or by using gettarinfo(). """ self._check("awx") diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index f3fd8f0ac30..d35fa12f17c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6007,8 +6007,9 @@ def test_resource_tracker_sigkill(self): def _is_resource_tracker_reused(conn, pid): from multiprocessing.resource_tracker import _resource_tracker _resource_tracker.ensure_running() - # The pid should be None in the child process, expect for the fork - # context. It should not be a new value. + # The pid should be None in the child (the at-fork handler clears + # it for fork; spawn/forkserver children never had it set). It + # should not be a new value. reused = _resource_tracker._pid in (None, pid) reused &= _resource_tracker._check_alive() conn.send(reused) @@ -6093,6 +6094,183 @@ def test_resource_tracker_blocked_signals(self): # restore sigmask to what it was before executing test signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask) + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_fork_deadlock(self): + # gh-146313: ResourceTracker.__del__ used to deadlock if a forked + # child still held the pipe's write end open when the parent + # exited, because the parent would block in waitpid() waiting for + # the tracker to exit, but the tracker would never see EOF. + cmd = '''if 1: + import os, signal + from multiprocessing.resource_tracker import ensure_running + ensure_running() + if os.fork() == 0: + signal.pause() + os._exit(0) + # parent falls through and exits, triggering __del__ + ''' + proc = subprocess.Popen([sys.executable, '-c', cmd], + start_new_session=True) + try: + try: + proc.wait(timeout=support.SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + self.fail( + "Parent process deadlocked in ResourceTracker.__del__" + ) + self.assertEqual(proc.returncode, 0) + finally: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_mp_fork_reuse_and_prompt_reap(self): + # gh-146313 / gh-80849: A child started via multiprocessing.Process + # with the 'fork' start method should reuse the parent's resource + # tracker (the at-fork handler preserves the inherited pipe fd), + # *and* the parent should be able to reap the tracker promptly + # after joining the child, without hitting the waitpid timeout. + cmd = textwrap.dedent(''' + import multiprocessing as mp + from multiprocessing.resource_tracker import _resource_tracker + + def child(conn): + # Prove we can talk to the parent's tracker by registering + # and unregistering a dummy resource over the inherited fd. + # If the fd were closed, ensure_running would launch a new + # tracker and _pid would be non-None. + _resource_tracker.register("x", "dummy") + _resource_tracker.unregister("x", "dummy") + conn.send((_resource_tracker._fd is not None, + _resource_tracker._pid is None, + _resource_tracker._check_alive())) + + if __name__ == "__main__": + mp.set_start_method("fork") + _resource_tracker.ensure_running() + r, w = mp.Pipe(duplex=False) + p = mp.Process(target=child, args=(w,)) + p.start() + child_has_fd, child_pid_none, child_alive = r.recv() + p.join() + w.close(); r.close() + + # Now simulate __del__: the child has exited and released + # its fd copy, so the tracker should see EOF and exit + # promptly -- no timeout. + _resource_tracker._stop(wait_timeout=5.0) + print(child_has_fd, child_pid_none, child_alive, + _resource_tracker._waitpid_timed_out, + _resource_tracker._exitcode) + ''') + rc, out, err = script_helper.assert_python_ok('-c', cmd) + parts = out.decode().split() + self.assertEqual(parts, ['True', 'True', 'True', 'False', '0'], + f"unexpected: {parts!r} stderr={err!r}") + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_raw_fork_prompt_reap(self): + # gh-146313: After a raw os.fork() the at-fork handler closes the + # child's inherited fd, so the parent can reap the tracker + # immediately -- even while the child is still alive -- rather + # than waiting out the 1s timeout. + cmd = textwrap.dedent(''' + import os, signal + from multiprocessing.resource_tracker import _resource_tracker + + _resource_tracker.ensure_running() + r, w = os.pipe() + pid = os.fork() + if pid == 0: + os.close(r) + # Report whether our fd was closed by the at-fork handler. + os.write(w, b"1" if _resource_tracker._fd is None else b"0") + os.close(w) + signal.pause() # stay alive so parent's reap is meaningful + os._exit(0) + os.close(w) + child_fd_closed = os.read(r, 1) == b"1" + os.close(r) + + # Child is still alive and paused. Because it closed its fd + # copy, our close below is the last one and the tracker exits. + _resource_tracker._stop(wait_timeout=5.0) + + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + print(child_fd_closed, + _resource_tracker._waitpid_timed_out, + _resource_tracker._exitcode) + ''') + rc, out, err = script_helper.assert_python_ok('-c', cmd) + parts = out.decode().split() + self.assertEqual(parts, ['True', 'False', '0'], + f"unexpected: {parts!r} stderr={err!r}") + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_lock_reinit_after_fork(self): + # gh-146313: If a parent thread held the tracker's lock at fork + # time, the child would inherit the held lock and deadlock on + # its next ensure_running(). The at-fork handler reinits it. + cmd = textwrap.dedent(''' + import os, threading + from multiprocessing.resource_tracker import _resource_tracker + + held = threading.Event() + release = threading.Event() + def hold(): + with _resource_tracker._lock: + held.set() + release.wait() + t = threading.Thread(target=hold) + t.start() + held.wait() + + pid = os.fork() + if pid == 0: + ok = _resource_tracker._lock.acquire(timeout=5.0) + os._exit(0 if ok else 1) + + release.set() + t.join() + _, status = os.waitpid(pid, 0) + print(os.waitstatus_to_exitcode(status)) + ''') + rc, out, err = script_helper.assert_python_ok( + '-W', 'ignore::DeprecationWarning', '-c', cmd) + self.assertEqual(out.strip(), b'0', + f"child failed to acquire lock: stderr={err!r}") + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_safety_net_timeout(self): + # gh-146313: When an mp.Process(fork) child holds the preserved + # fd and the parent calls _stop() without joining (simulating + # abnormal shutdown), the safety-net timeout should fire rather + # than deadlocking. + cmd = textwrap.dedent(''' + import multiprocessing as mp + import signal + from multiprocessing.resource_tracker import _resource_tracker + + if __name__ == "__main__": + mp.set_start_method("fork") + _resource_tracker.ensure_running() + p = mp.Process(target=signal.pause) + p.start() + # Stop WITHOUT joining -- child still holds preserved fd + _resource_tracker._stop(wait_timeout=0.5) + print(_resource_tracker._waitpid_timed_out) + p.terminate() + p.join() + ''') + rc, out, err = script_helper.assert_python_ok('-c', cmd) + self.assertEqual(out.strip(), b'True', + f"safety-net timeout did not fire: stderr={err!r}") + + class TestSimpleQueue(unittest.TestCase): @classmethod diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 6749fad35a6..ec6317711bd 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2490,6 +2490,9 @@ def test_utcfromtimestamp(self): # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @unittest.skipIf(sys.platform == "android", + "Android/Bionic mktime behavior differs from POSIX: " + "no system tzdata, local UTC offset unreliable") @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') def test_timestamp_naive(self): t = self.theclass(1970, 1, 1) diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 8fc852f86b4..75fe0d7ad3b 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -118,7 +118,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Storage of uncollectable GC objects (gc.garbage) -GC_GARBAGE = [] +GC_GARBAGE: list[object] = [] def _load_run_test(result: TestResult, runtests: RunTests) -> None: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index fa67dec4b5c..a0223689a5a 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2526,6 +2526,51 @@ def test_bytearray_memoization(self): self.assertIsNot(b2a, b2b) self.assert_is_copy(b2a, b2b) + def test_picklebuffer_memoization(self): + if self.py_version < (3, 8): + self.skipTest('not supported in Python < 3.8') + array_types = [bytes, bytearray] + for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): + for array_type in array_types: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = pickle.PickleBuffer(array_type(s)) + p = self.dumps((b, b), proto) + b1, b2 = self.loads(p) + self.assertIs(b1, b2) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b = array_type(s) + b1a = pickle.PickleBuffer(b) + b2a = pickle.PickleBuffer(b) + p = self.dumps((b1a, b2a), proto) + b1b, b2b = self.loads(p) + if array_type is not bytes: + self.assertIsNot(b1b, b2b) + self.assert_is_copy(b1b, b) + self.assert_is_copy(b2b, b) + + def test_empty_picklebuffer_memoization(self): + # gh-148914: Empty writable PickleBuffer memoized an empty bytearray + # with the id of b'' (a singleton in CPython). + if self.py_version < (3, 8): + self.skipTest('not supported in Python < 3.8') + for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): + for readonly in False, True: + with self.subTest(proto=proto, readonly=readonly): + b = b'' + ba = bytearray() + buf = pickle.PickleBuffer(b if readonly else ba) + p = self.dumps((buf, b, ba), proto) + buf, b, ba = self.loads(p) + array_type = bytes if readonly else bytearray + self.assertIsInstance(buf, array_type) + self.assertIsInstance(b, bytes) + self.assertIsInstance(ba, bytearray) + self.assertEqual(buf, b'') + self.assertEqual(b, b'') + self.assertEqual(ba, b'') + def test_ints(self): for proto in protocols: n = sys.maxsize diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 46739f7cecf..737ce94a120 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2863,3 +2863,18 @@ def control_characters_c0() -> list[str]: C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. """ return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] + + +STATUS_DLL_INIT_FAILED = 0xC0000142 +def skip_on_low_desktop_heap_memory_subprocess(returncode): + if sys.platform not in ('win32', 'cygwin'): + return + # On Windows, STATUS_DLL_INIT_FAILED is a generic error code that could + # come from any of the DLLs being loaded when a new Python process is + # created. In practice, it's likely a memory allocation failure in the + # desktop heap memory which caused the DLL init failure, especially on + # process created with CREATE_NEW_CONSOLE creation flag. See the article: + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/desktop-heap-limitation-out-of-memory + if returncode == STATUS_DLL_INIT_FAILED: + raise unittest.SkipTest('gh-150436: DLL init failed, likely because ' + 'of low desktop heap memory') diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 91475180a1d..d118f334c0b 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1047,7 +1047,7 @@ def test_invalid_enum_value_raises_error(self): parser.add_argument('--color', choices=self.Color) self.assertRaisesRegex( argparse.ArgumentError, - r"invalid choice: 'yellow' \(choose from red, green, blue\)", + r"invalid choice: 'yellow' \(choose from 'red', 'green', 'blue'\)", parser.parse_args, ['--color', 'yellow'], ) @@ -2517,7 +2517,7 @@ def test_wrong_argument_subparsers_no_destination_error(self): parser.parse_args(('baz',)) self.assertRegex( excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from foo, bar\)\n$" + r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from 'foo', 'bar'\)\n$" ) def test_optional_subparsers(self): diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index 2509d4382cd..63ac04f5703 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -228,6 +228,61 @@ def test_sock_sendfile_zero_size(self): self.assertEqual(ret, 0) self.assertEqual(self.file.tell(), 0) + def check_sock_sendfile_offset(self, data, offset, force_fallback=False): + sock, proto = self.prepare_socksendfile() + with tempfile.TemporaryFile() as f: + f.write(data) + f.flush() + self.assertEqual(f.tell(), len(data)) + + if force_fallback: + async def _sock_sendfile_fail(sock, file, offset, count): + raise asyncio.exceptions.SendfileNotAvailableError() + with support.swap_attr(self.loop, '_sock_sendfile_native', _sock_sendfile_fail): + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + else: + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + + self.assertEqual(f.tell(), len(data)) + + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(data) - offset) + + + def test_sock_sendfile_offset(self): + data = b'abcdef' + for offset in (0, len(data) // 2, len(data)): + for force_fallback in (False, True): + with self.subTest(offset=offset, force_fallback=force_fallback): + self.check_sock_sendfile_offset(data, offset, force_fallback) + + def check_sendfile_offset(self, offset, fallback): + srv_proto, cli_proto = self.prepare_sendfile() + self.file.seek(123) + coro = self.loop.sendfile(cli_proto.transport, self.file, offset, fallback=fallback) + try: + ret = self.run_loop(coro) + except asyncio.SendfileNotAvailableError: + if fallback: + raise + cli_proto.transport.close() + self.run_loop(srv_proto.done) + return + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA) - offset) + self.assertEqual(srv_proto.nbytes, len(self.DATA) - offset) + self.assertEqual(srv_proto.data, self.DATA[offset:]) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_offset(self): + for offset in (0, len(self.DATA) // 2, len(self.DATA)): + for fallback in (False, True): + with self.subTest(offset=offset, fallback=fallback): + self.check_sendfile_offset(offset, fallback) + def test_sock_sendfile_mix_with_regular_send(self): buf = b"mix_regular_send" * (4 * 1024) # 64 KiB sock, proto = self.prepare_socksendfile() diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index acef24a703b..112ba572f8b 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -427,6 +427,27 @@ def test_recvfrom_into(self): self.loop.run_until_complete( self._basetest_datagram_recvfrom_into(server_address)) + async def _basetest_datagram_recvfrom_into_wrong_size(self, server_address): + # Call sock_sendto() with a size larger than the buffer + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + + buf = bytearray(5000) + data = b'\x01' * 4096 + wrong_size = len(buf) + 1 + await self.loop.sock_sendto(sock, data, server_address) + with self.assertRaises(ValueError): + await self.loop.sock_recvfrom_into( + sock, buf, wrong_size) + + size, addr = await self.loop.sock_recvfrom_into(sock, buf) + self.assertEqual(buf[:size], data) + + def test_recvfrom_into_wrong_size(self): + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_recvfrom_into_wrong_size(server_address)) + async def _basetest_datagram_sendto_blocking(self, server_address): # Sad path, sock.sendto() raises BlockingIOError # This involves patching sock.sendto() to raise BlockingIOError but diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index c84edacfbd5..f622b51356e 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -77,6 +77,30 @@ def test_pipe_handle(self): else: raise RuntimeError('expected ERROR_INVALID_HANDLE') + def test_pipe_handle_close_after_external_close(self): + # gh-149388: PipeHandle.close() must clear ``_handle`` before calling + # CloseHandle so that if CloseHandle raises on a stale handle the + # PipeHandle is still marked closed and __del__ / subsequent close() + # calls are silent no-ops. + h1, h2 = windows_utils.pipe(overlapped=(False, False)) + try: + p = windows_utils.PipeHandle(h1) + # Simulate an external close of the underlying handle (e.g. + # a finalizer race or a concurrent close on the same object). + _winapi.CloseHandle(p.handle) + # First close() still propagates the OSError from CloseHandle, + # but must clear ``_handle`` first. + with self.assertRaises(OSError): + p.close() + self.assertIsNone(p.handle) + # Second close() is a no-op. + p.close() + # __del__ through GC is also a silent no-op — no unraisable. + del p + support.gc_collect() + finally: + _winapi.CloseHandle(h2) + class PopenTests(unittest.TestCase): diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 6e75d7d8f28..6e397df13c4 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -240,6 +240,10 @@ def test_uu(self): self.assertEqual(binascii.a2b_uu(b"\xff"), b"\x00"*31) self.assertRaises(binascii.Error, binascii.a2b_uu, b"\xff\x00") self.assertRaises(binascii.Error, binascii.a2b_uu, b"!!!!") + self.assertRaises(binascii.Error, binascii.a2b_uu, + self.type2test(b"")) + self.assertRaises(binascii.Error, binascii.a2b_uu, + self.type2test(b"#86)C")[:0]) self.assertRaises(binascii.Error, binascii.b2a_uu, 46*b"!") # Issue #7701 (crash on a pydebug build) @@ -447,6 +451,9 @@ def test_empty_string(self): binascii.crc_hqx(empty, 0) continue f = getattr(binascii, func) + if func == 'a2b_uu': + self.assertRaises(binascii.Error, f, empty) + continue try: f(empty) except Exception as err: diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index b5cd202a613..442296ab172 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1032,6 +1032,21 @@ def test_failure(self): # Previously, a second call could crash due to internal inconsistency self.assertRaises(Exception, bzd.decompress, self.BAD_DATA * 30) + def test_decompress_after_data_error(self): + data = bytes.fromhex( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" + ) + bzd = BZ2Decompressor() + with self.assertRaisesRegex(OSError, "Invalid data stream"): + bzd.decompress(data) + # Previously, a second call could crash due to internal inconsistency + self.assertFalse(bzd.needs_input) + self.assertFalse(bzd.eof) + with self.assertRaisesRegex(ValueError, "previous error"): + bzd.decompress(b'\x00' * 18) + @support.refcount_test def test_refleaks_in___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index df102fe1986..c5296f50b3d 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -457,12 +457,17 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, 13) with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, -1) + def test_illegal_month_error_bases(self): + self.assertTrue(issubclass(calendar.IllegalMonthError, ValueError)) + self.assertTrue(issubclass(calendar.IllegalMonthError, IndexError)) + def test_formatmonthname_with_year(self): self.assertEqual( calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=True), diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 703d228f92e..559ca932595 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -14,7 +14,7 @@ try: import _testinternalcapi except ImportError: - _testinternalcapi = NULL + _testinternalcapi = None # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. diff --git a/Lib/test/test_capi/test_weakref.py b/Lib/test/test_capi/test_weakref.py new file mode 100644 index 00000000000..1d5b5d3625b --- /dev/null +++ b/Lib/test/test_capi/test_weakref.py @@ -0,0 +1,121 @@ +import weakref +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +NULL = None + +class Object: + pass + +class Ref(weakref.ReferenceType): + pass + + +class CAPIWeakrefTest(unittest.TestCase): + def test_pyweakref_check(self): + # Test PyWeakref_Check() + check = _testlimitedcapi.pyweakref_check + obj = Object() + self.assertEqual(check(obj), 0) + self.assertEqual(check(weakref.ref(obj)), 1) + self.assertEqual(check(Ref(obj)), 1) + self.assertEqual(check(weakref.proxy(obj)), 1) + + # CRASHES check(NULL) + + def test_pyweakref_checkref(self): + # Test PyWeakref_CheckRef() + checkref = _testlimitedcapi.pyweakref_checkref + obj = Object() + self.assertEqual(checkref(obj), 0) + self.assertEqual(checkref(weakref.ref(obj)), 1) + self.assertEqual(checkref(Ref(obj)), 1) + self.assertEqual(checkref(weakref.proxy(obj)), 0) + + # CRASHES checkref(NULL) + + def test_pyweakref_checkrefexact(self): + # Test PyWeakref_CheckRefExact() + checkrefexact = _testlimitedcapi.pyweakref_checkrefexact + obj = Object() + self.assertEqual(checkrefexact(obj), 0) + self.assertEqual(checkrefexact(weakref.ref(obj)), 1) + self.assertEqual(checkrefexact(Ref(obj)), 0) + self.assertEqual(checkrefexact(weakref.proxy(obj)), 0) + + # CRASHES checkrefexact(NULL) + + def test_pyweakref_checkproxy(self): + # Test PyWeakref_CheckProxy() + checkproxy = _testlimitedcapi.pyweakref_checkproxy + obj = Object() + self.assertEqual(checkproxy(obj), 0) + self.assertEqual(checkproxy(weakref.ref(obj)), 0) + self.assertEqual(checkproxy(Ref(obj)), 0) + self.assertEqual(checkproxy(weakref.proxy(obj)), 1) + + # CRASHES checkproxy(NULL) + + def test_pyweakref_getref(self): + # Test PyWeakref_GetRef() + getref = _testcapi.pyweakref_getref + obj = Object() + wr = weakref.ref(obj) + wp = weakref.proxy(obj) + self.assertEqual(getref(wr), (1, obj)) + self.assertEqual(getref(wp), (1, obj)) + del obj + self.assertEqual(getref(wr), 0) + self.assertEqual(getref(wp), 0) + + self.assertRaises(TypeError, getref, 42) + self.assertRaises(SystemError, getref, NULL) + + def test_pyweakref_newref(self): + # Test PyWeakref_NewRef() + newref = _testlimitedcapi.pyweakref_newref + obj = Object() + wr = newref(obj) + self.assertIs(type(wr), weakref.ReferenceType) + # PyWeakref_NewRef() handles None callback as NULL callback + wr = newref(obj, None) + self.assertIs(type(wr), weakref.ReferenceType) + log = [] + wr = newref(obj, log.append) + self.assertIs(type(wr), weakref.ReferenceType) + self.assertEqual(log, []) + del obj + self.assertEqual(log, [wr]) + + self.assertRaises(TypeError, newref, []) + # CRASHES newref(NULL) + + def test_pyweakref_newproxy(self): + # Test PyWeakref_NewProxy() + newproxy = _testlimitedcapi.pyweakref_newproxy + obj = Object() + wp = newproxy(obj) + self.assertIs(type(wp), weakref.ProxyType) + # PyWeakref_NewProxy() handles None callback as NULL callback + wp = newproxy(obj, None) + self.assertIs(type(wp), weakref.ProxyType) + log = [] + wp = newproxy(obj, log.append) + self.assertIs(type(wp), weakref.ProxyType) + self.assertEqual(log, []) + del obj + self.assertEqual(log, [wp]) + + def func(): + pass + wp = newproxy(func) + self.assertIs(type(wp), weakref.CallableProxyType) + + self.assertRaises(TypeError, newproxy, []) + # CRASHES newproxy(NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index d4fdd92bdc6..a5cf5c3d408 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -1729,6 +1729,19 @@ def test_error(self): self.assertEqual(e1.message, e2.message) self.assertEqual(repr(e1), repr(e2)) + def test_combine_error_linear_complexity(self): + # Ensure that ParsingError.combine() has linear complexity. + # See https://github.com/python/cpython/issues/148370. + n = 50000 + s = '[*]\n' + (err_line := '=\n') * n + p = configparser.ConfigParser(strict=False) + with self.assertRaises(configparser.ParsingError) as cm: + p.read_string(s) + errlines = cm.exception.message.splitlines() + self.assertEqual(len(errlines), n + 1) + self.assertTrue(errlines[0].startswith("Source contains parsing errors: ")) + self.assertEqual(errlines[42], f"\t[line {43:2d}]: {err_line!r}") + def test_nosectionerror(self): import pickle e1 = configparser.NoSectionError('section') @@ -2212,6 +2225,26 @@ def test_multiple_configs(self): self.assertEqual('2', cfg[configparser.UNNAMED_SECTION]['b']) +class ReDoSTestCase(unittest.TestCase): + """Regression tests for quadratic regex backtracking (gh-146333).""" + + def test_option_regex_does_not_backtrack(self): + # A line with many spaces between non-delimiter characters + # should be parsed in linear time, not quadratic. + parser = configparser.RawConfigParser() + content = "[section]\n" + "x" + " " * 40000 + "y" + "\n" + # This should complete almost instantly. Before the fix, + # it would take over a minute due to catastrophic backtracking. + with self.assertRaises(configparser.ParsingError): + parser.read_string(content) + + def test_option_regex_no_value_does_not_backtrack(self): + parser = configparser.RawConfigParser(allow_no_value=True) + content = "[section]\n" + "x" + " " * 40000 + "y" + "\n" + parser.read_string(content) + self.assertTrue(parser.has_option("section", "x" + " " * 40000 + "y")) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index ce5c03659f1..6ea388313bb 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -552,6 +552,33 @@ def test_roundtrip_escaped_unquoted_newlines(self): self.assertEqual(row, rows[i]) + def test_reader_reentrant_iterator(self): + # gh-145105: re-entering the reader from the iterator must not crash. + class ReentrantIter: + def __init__(self): + self.reader = None + self.n = 0 + def __iter__(self): + return self + def __next__(self): + self.n += 1 + if self.n == 1: + try: + next(self.reader) + except StopIteration: + pass + return "a,b" + if self.n == 2: + return "x" + raise StopIteration + + it = ReentrantIter() + reader = csv.reader(it) + it.reader = reader + with self.assertRaises(csv.Error): + next(reader) + + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): self.assertRaises(TypeError, csv.list_dialects, None) diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index ec70f9ceda5..3b7a5dbc320 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -2865,29 +2865,41 @@ class C(base): class TestFrozen(unittest.TestCase): + # Some tests have a subtest with a slotted dataclass. + # See https://github.com/python/cpython/issues/105936 for the reasons. + def test_frozen(self): - @dataclass(frozen=True) - class C: - i: int + for slots in (False, True): + with self.subTest(slots=slots): - c = C(10) - self.assertEqual(c.i, 10) - with self.assertRaises(FrozenInstanceError): - c.i = 5 - self.assertEqual(c.i, 10) + @dataclass(frozen=True, slots=slots) + class C: + i: int + + c = C(10) + self.assertEqual(c.i, 10) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertEqual(c.i, 10) + with self.assertRaises(FrozenInstanceError): + del c.i + self.assertEqual(c.i, 10) def test_frozen_empty(self): - @dataclass(frozen=True) - class C: - pass + for slots in (False, True): + with self.subTest(slots=slots): - c = C() - self.assertFalse(hasattr(c, 'i')) - with self.assertRaises(FrozenInstanceError): - c.i = 5 - self.assertFalse(hasattr(c, 'i')) - with self.assertRaises(FrozenInstanceError): - del c.i + @dataclass(frozen=True, slots=slots) + class C: + pass + + c = C() + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + del c.i def test_inherit(self): @dataclass(frozen=True) @@ -3083,41 +3095,43 @@ class D(I): d.i = 5 def test_non_frozen_normal_derived(self): - # See bpo-32953. - - @dataclass(frozen=True) - class D: - x: int - y: int = 10 - - class S(D): - pass + # See bpo-32953 and https://github.com/python/cpython/issues/105936 + for slots in (False, True): + with self.subTest(slots=slots): - s = S(3) - self.assertEqual(s.x, 3) - self.assertEqual(s.y, 10) - s.cached = True + @dataclass(frozen=True, slots=slots) + class D: + x: int + y: int = 10 - # But can't change the frozen attributes. - with self.assertRaises(FrozenInstanceError): - s.x = 5 - with self.assertRaises(FrozenInstanceError): - s.y = 5 - self.assertEqual(s.x, 3) - self.assertEqual(s.y, 10) - self.assertEqual(s.cached, True) + class S(D): + pass - with self.assertRaises(FrozenInstanceError): - del s.x - self.assertEqual(s.x, 3) - with self.assertRaises(FrozenInstanceError): - del s.y - self.assertEqual(s.y, 10) - del s.cached - self.assertFalse(hasattr(s, 'cached')) - with self.assertRaises(AttributeError) as cm: - del s.cached - self.assertNotIsInstance(cm.exception, FrozenInstanceError) + s = S(3) + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + s.cached = True + + # But can't change the frozen attributes. + with self.assertRaises(FrozenInstanceError): + s.x = 5 + with self.assertRaises(FrozenInstanceError): + s.y = 5 + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + self.assertEqual(s.cached, True) + + with self.assertRaises(FrozenInstanceError): + del s.x + self.assertEqual(s.x, 3) + with self.assertRaises(FrozenInstanceError): + del s.y + self.assertEqual(s.y, 10) + del s.cached + self.assertFalse(hasattr(s, 'cached')) + with self.assertRaises(AttributeError) as cm: + del s.cached + self.assertNotIsInstance(cm.exception, FrozenInstanceError) def test_non_frozen_normal_derived_from_empty_frozen(self): @dataclass(frozen=True) @@ -3757,6 +3771,50 @@ class WithCorrectSuper(CorrectSuper): # that we create internally. self.assertEqual(CorrectSuper.args, ["default", "default"]) + def test_empty_class_cell(self): + # gh-148947: Make sure that we explicitly handle the empty class cell. + def maker(): + if False: + __class__ = 42 + + def method(self): + return __class__ + return method + + from dataclasses import dataclass + + @dataclass(slots=True) + class X: + a: int + + meth = maker() + + with self.assertRaisesRegex(NameError, '__class__'): + X(1).meth() + + def test_class_cell_from_other_class(self): + # This test fails without the "is oldcls" check in + # _update_func_cell_for__class__. + class Base: + def meth(self): + return "Base" + + class Child(Base): + def meth(self): + return super().meth() + " Child" + + @dataclass(slots=True) + class DC(Child): + a: int + + meth = Child.meth + + closure = DC.meth.__closure__ + self.assertEqual(len(closure), 1) + self.assertIs(closure[0].cell_contents, Child) + + self.assertEqual(DC(1).meth(), "Base Child") + class TestDescriptors(unittest.TestCase): def test_set_name(self): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 7535497af79..bf14a2f72cf 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -1060,6 +1060,78 @@ def get_phrase_cfws_only_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_phrase(' (foo) ') + def test_get_phrase_adjacent_ew(self): + # "'linear-white-space' that separates a pair of adjacent + # 'encoded-word's is ignored" (rfc2047 section 6.2) + self._test_get_x(parser.get_phrase, '=?ascii?q?Joi?= \t =?ascii?q?ned?=', 'Joined', 'Joined', [], '') + + def test_get_phrase_adjacent_ew_different_encodings(self): + self._test_get_x( + parser.get_phrase, + '=?utf-8?q?B=C3=A9r?= =?iso-8859-1?q?=E9nice?=', 'Bérénice', 'Bérénice', [], '' + ) + + def test_get_phrase_adjacent_ew_encoded_spaces(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Encoded?= =?ascii?q?_spaces_?= =?ascii?q?preserved?=', + 'Encoded spaces preserved', + 'Encoded spaces preserved', + [], + '' + ) + + def test_get_phrase_adjacent_ew_comment_is_not_linear_white_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Comment?= (is not) =?ascii?q?linear-white-space?=', + 'Comment (is not) linear-white-space', + 'Comment linear-white-space', + [], + '', + comments=['is not'], + ) + + def test_get_phrase_adjacent_ew_no_error_on_defects(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Def?= =?ascii?q?ect still joins?=', + 'Defect still joins', + 'Defect still joins', + [errors.InvalidHeaderDefect], # whitespace inside encoded word + '' + ) + + def test_get_phrase_adjacent_ew_ignore_non_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?join?= for non-ew', + 'No =?join?= for non-ew', + 'No =?join?= for non-ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_ignore_invalid_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_missing_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Joi?==?ascii?q?ned?=', + 'Joined', + 'Joined', + [errors.InvalidHeaderDefect], # missing trailing whitespace + '' + ) + # get_local_part def test_get_local_part_simple(self): @@ -2398,6 +2470,22 @@ def test_get_address_rfc2047_display_name(self): self.assertEqual(address[0].token_type, 'mailbox') + def test_get_address_rfc2047_display_name_adjacent_ews(self): + address = self._test_get_x(parser.get_address, + '=?utf-8?q?B=C3=A9r?= =?utf-8?q?=C3=A9nice?= ', + 'Bérénice ', + 'Bérénice ', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Bérénice') + self.assertEqual(address[0].token_type, + 'mailbox') + def test_get_address_empty_group(self): address = self._test_get_x(parser.get_address, 'Monty Python:;', @@ -2617,7 +2705,7 @@ def test_get_address_list_mailboxes_invalid_addresses(self): '') self.assertEqual(address_list.token_type, 'address-list') self.assertEqual(len(address_list.mailboxes), 1) - self.assertEqual(len(address_list.all_mailboxes), 3) + self.assertEqual(len(address_list.all_mailboxes), 4) self.assertEqual([str(x) for x in address_list.all_mailboxes], [str(x) for x in address_list.addresses]) self.assertEqual(address_list.mailboxes[0].domain, 'example.com') @@ -2626,11 +2714,13 @@ def test_get_address_list_mailboxes_invalid_addresses(self): self.assertEqual(address_list.addresses[1].token_type, 'address') self.assertEqual(len(address_list.addresses[0].mailboxes), 1) self.assertEqual(len(address_list.addresses[1].mailboxes), 0) - self.assertEqual(len(address_list.addresses[1].mailboxes), 0) + self.assertEqual(len(address_list.addresses[2].mailboxes), 0) + self.assertEqual(len(address_list.addresses[3].mailboxes), 0) self.assertEqual( address_list.addresses[1].all_mailboxes[0].local_part, 'Foo x') + self.assertEqual(address_list.addresses[2].all_mailboxes[0].value, '[]') self.assertEqual( - address_list.addresses[2].all_mailboxes[0].display_name, + address_list.addresses[3].all_mailboxes[0].display_name, "Nobody Is. Special") def test_get_address_list_group_empty(self): @@ -2695,6 +2785,14 @@ def test_get_address_list_group_and_mailboxes(self): self.assertEqual(str(address_list.addresses[1]), str(address_list.mailboxes[2])) + def test_get_address_list_trailing_garbage(self): + address_list = self._test_get_x(parser.get_address_list, + 'unlisted-recipients:; (no To-header on input)', + 'unlisted-recipients:; (no To-header on input)', + 'unlisted-recipients:; ', + [errors.InvalidHeaderDefect]*2 + [errors.ObsoleteHeaderDefect], + '') + def test_invalid_content_disposition(self): content_disp = self._test_parse_x( parser.parse_content_disposition_header, diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index 1e0caeeaed0..f9cc8f3dfa9 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -58,6 +58,62 @@ def test_japanese_codecs(self): # TK: full decode comparison eq(str(h).encode(jcode), subject_bytes) + h = Header("Japanese") + s = '\u65e5\u672c\u8a9e' # 日本語 + h.append(s, Charset('euc-jp')) + h.append(s, Charset('iso-2022-jp')) + h.append(s, Charset('shift_jis')) + eq(h.encode(), """\ +Japanese =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= + =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?=""") + eq(decode_header(h.encode()), + [(b'Japanese ', None), + (b'\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B', 'iso-2022-jp'), + ]) + + def test_chinese_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Chinese") + s = '\u4e2d\u6587' # 中文 + h.append(s, Charset('gb2312')) + h.append(s, Charset('gbk')) + h.append(s, Charset('gb18030')) + h.append(s, Charset('hz')) + h.append(s, Charset('big5')) + h.append(s, Charset('big5hkscs')) + eq(h.encode(), """\ +Chinese =?gb2312?b?1tDOxA==?= =?gbk?b?1tDOxA==?= =?gb18030?b?1tDOxA==?= + =?hz?b?fntWUE5Efn0=?= =?big5?b?pKSk5Q==?= =?big5hkscs?b?pKSk5Q==?=""") + eq(decode_header(h.encode()), + [(b'Chinese ', None), + (b'\xd6\xd0\xce\xc4', 'gb2312'), + (b'\xd6\xd0\xce\xc4', 'gbk'), + (b'\xd6\xd0\xce\xc4', 'gb18030'), + (b'~{VPND~}', 'hz'), + (b'\xa4\xa4\xa4\xe5', 'big5'), + (b'\xa4\xa4\xa4\xe5', 'big5hkscs'), + ]) + + def test_korean_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Korean") + s = '\ud55c\uad6d\uc5b4' # 한국어 + h.append(s, Charset('euc-kr')) + h.append(s, Charset('ks_c_5601-1987')) + h.append(s, Charset('cp949')) + h.append(s, Charset('iso-2022-kr')) + h.append(s, Charset('johab')) + eq(h.encode(), """\ +Korean =?euc-kr?b?x9Gxub7u?= =?ks_c_5601-1987?b?x9Gxub7uIMfRsbm+7g==?= + =?iso-2022-kr?b?GyQpQw5HUTE5Pm4P?= =?johab?b?0GWKgrTh?=""") + eq(decode_header(h.encode()), + [(b'Korean ', None), + (b'\xc7\xd1\xb1\xb9\xbe\xee', 'euc-kr'), + (b'\xc7\xd1\xb1\xb9\xbe\xee \xc7\xd1\xb1\xb9\xbe\xee', 'ks_c_5601-1987'), + (b'\x1b$)C\x0eGQ19>n\x0f', 'iso-2022-kr'), + (b'\xd0e\x8a\x82\xb4\xe1', 'johab'), + ]) + def test_payload_encoding_utf8(self): jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp') diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index e40904f7090..8c06de30022 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -4802,6 +4802,15 @@ def test_decode_soft_line_break(self): def test_decode_false_quoting(self): self._test_decode('A=1,B=A ==> A+B==2', 'A=1,B=A ==> A+B==2') + def test_decode_crlf_eol_no_trailing_newline(self): + self._test_decode('abc', 'abc', eol='\r\n') + + def test_decode_crlf_eol_multiline_no_trailing_newline(self): + self._test_decode('a\r\nb', 'a\r\nb', eol='\r\n') + + def test_decode_crlf_eol_with_trailing_newline(self): + self._test_decode('abc\r\n', 'abc\r\n', eol='\r\n') + def _test_encode(self, body, expected_encoded_body, maxlinelen=None, eol=None): kwargs = {} if maxlinelen is None: @@ -4960,15 +4969,8 @@ def test_body_encode(self): # Try the convert argument, where input codec != output codec c = Charset('euc-jp') # With apologies to Tokio Kikuchi ;) - # XXX FIXME -## try: -## eq('\x1b$B5FCO;~IW\x1b(B', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) -## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) -## except LookupError: -## # We probably don't have the Japanese codecs installed -## pass + eq('\x1b$B5FCO;~IW\x1b(B', + c.body_encode('\u83ca\u5730\u6642\u592b')) # Testing SF bug #625509, which we have to fake, since there are no # built-in encodings where the header encoding is QP but the body # encoding is not. diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index c2d7d09d591..3c9a86f3e8c 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -1,13 +1,20 @@ import io import textwrap import unittest +import random +import sys from email import message_from_string, message_from_bytes from email.message import EmailMessage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from email.generator import Generator, BytesGenerator +import email.generator from email.headerregistry import Address from email import policy import email.errors from test.test_email import TestEmailBase, parameterize +import test.support + @parameterize @@ -288,6 +295,36 @@ def test_keep_long_encoded_newlines(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def _test_boundary_detection(self, linesep): + # Generate a boundary token in the same way as _make_boundary + token = random.randrange(sys.maxsize) + + def _patch_random_randrange(*args, **kwargs): + return token + + with test.support.swap_attr( + random, "randrange", _patch_random_randrange + ): + boundary = self.genclass._make_boundary(text=None) + boundary_in_part = ( + "this goes before the boundary\n--" + + boundary + + "\nthis goes after\n" + ) + msg = MIMEMultipart() + msg.attach(MIMEText(boundary_in_part)) + self.genclass(self.ioclass()).flatten(msg, linesep=linesep) + # Generator checks the message content for the string it is about + # to use as a boundary ('token' in this test) and when it finds it + # in our attachment appends .0 to make the boundary it uses unique. + self.assertEqual(msg.get_boundary(), boundary + ".0") + + def test_lf_boundary_detection(self): + self._test_boundary_detection("\n") + + def test_crlf_boundary_detection(self): + self._test_boundary_detection("\r\n") + class TestGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index b2c31b12ee5..ae29ce3426c 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -1263,12 +1263,12 @@ class TestAddressHeader(TestHeaderBase): 'example.com', None), - } - # XXX: Need many more examples, and in particular some with names in # trailing comments, which aren't currently handled. comments in # general are not handled yet. + } + def example_as_address(self, source, defects, decoded, display_name, addr_spec, username, domain, comment): h = self.make_header('sender', source) @@ -1286,6 +1286,43 @@ def example_as_address(self, source, defects, decoded, display_name, # XXX: we have no comment support yet. #self.assertEqual(a.comment, comment) + example_broken_header_params = { + + 'just_dquote': + ('"', + [errors.InvalidHeaderDefect]*2, + '<>', + '', + '<>', + '', + '', + ), + + } + + def example_broken_header_as_address( + self, + source, + defects, + decoded, + display_name, + addr_spec, + username, + domain, + ): + h = self.make_header('sender', source) + self.assertEqual(h, decoded) + self.assertDefectsEqual(h.defects, defects) + a = h.address + self.assertEqual(str(a), decoded) + self.assertEqual(len(h.groups), 1) + self.assertEqual([a], list(h.groups[0].addresses)) + self.assertEqual([a], list(h.addresses)) + self.assertEqual(a.display_name, display_name) + self.assertEqual(a.addr_spec, addr_spec) + self.assertEqual(a.username, username) + self.assertEqual(a.domain, domain) + def example_as_group(self, source, defects, decoded, display_name, addr_spec, username, domain, comment): source = 'foo: {};'.format(source) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index c5ec56bf609..799768d2707 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -4997,8 +4997,8 @@ class Color(enum.Enum) | __members__ | Returns a mapping of member name->value. | - | This mapping lists all enum members, including aliases. Note that this - | is a read-only view of the internal mapping.""" + | This mapping lists all enum members, including aliases. Note that + | this is a read-only view of the internal mapping.""" expected_help_output_without_docs = """\ Help on class Color in module %s: diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c864d401f9e..f1eff9430f7 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -16,7 +16,7 @@ except ImportError: ssl = None -from unittest import TestCase, skipUnless +from unittest import mock, TestCase, skipUnless from test import support from test.support import requires_subprocess from test.support import threading_helper @@ -1145,6 +1145,40 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + def _make_pair(self, *, advertised_host, real_host, trust=False): + source = mock.Mock(spec=ftplib.FTP) + source.trust_server_pasv_ipv4_address = trust + source.sock.getpeername.return_value = (real_host, 21) + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source.sendcmd.side_effect = lambda cmd: ( + f'227 Entering Passive Mode ({advertised},1,2).' + if cmd == 'PASV' else '150 ok') + target = mock.Mock(spec=ftplib.FTP) + target.sendcmd.return_value = '150 ok' + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('198.51.100.7', 258) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('10.0.0.5', 258) + + class MiscTestCase(TestCase): def test__all__(self): not_exported = { diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 00dc9251e0e..685bddfba70 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -56,9 +56,8 @@ from queue import Queue, SimpleQueue from weakref import WeakSet, ReferenceType, ref import typing -from typing import Unpack +from typing import TypeVar, Unpack -from typing import TypeVar T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') @@ -537,6 +536,13 @@ def test_del_iter(self): iter_x = iter(t) del iter_x + def test_gh150146(self): + # It used to crash: + for container in [list, tuple]: + with self.subTest(container=container): + x = container[TypeVar("")] + with self.assertRaises(TypeError): + x[*typing.Mapping[..., ...]] class TypeIterationTests(unittest.TestCase): _UNITERABLE_TYPES = (list, tuple) diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index e52e17b8dc7..ed86802f069 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -1,5 +1,7 @@ """Test script for the grp module.""" +import random +import string import unittest from test.support import import_helper @@ -50,61 +52,51 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, grp.getgrgid) self.assertRaises(TypeError, grp.getgrgid, 3.14) + self.assertRaises(TypeError, grp.getgrgid, 0.0) + self.assertRaises(TypeError, grp.getgrgid, 0, 0) + # should be out of gid_t range + self.assertRaises(OverflowError, grp.getgrgid, 2**128) + self.assertRaises(OverflowError, grp.getgrgid, -2**128) self.assertRaises(TypeError, grp.getgrnam) self.assertRaises(TypeError, grp.getgrnam, 42) - self.assertRaises(TypeError, grp.getgrall, 42) + self.assertRaises(TypeError, grp.getgrnam, b'root') + self.assertRaises(TypeError, grp.getgrnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, grp.getgrnam, 'roo\udc74') + self.assertRaises(KeyError, grp.getgrnam, '') + self.assertRaises(TypeError, grp.getgrall, 42) - # try to get some errors - bynames = {} - bygids = {} - for (n, p, g, mem) in grp.getgrall(): - if not n or n == '+': - continue # skip NIS entries etc. - bynames[n] = g - bygids[g] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue + # Find a non-existent group name. + # getgrall() will not necessarily report all existing groups + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + grp.getgrnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent group name') + + # Find a non-existent gid. + maxgid = 2**31 + for _ in range(30): + fakegid = random.randrange(maxgid) + try: + grp.getgrgid(fakegid) + except KeyError: + break + except OverflowError: + if maxgid == 2**31: + maxgid = 2**16-1 + elif maxgid == 2**16-1: + maxgid = 2**15 else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, grp.getgrnam, fakename) - - # Choose a non-existent gid. - fakegid = 4127 - while fakegid in bygids: - fakegid = (fakegid * 3) % 0x10000 - - self.assertRaises(KeyError, grp.getgrgid, fakegid) - - def test_noninteger_gid(self): - entries = grp.getgrall() - if not entries: - self.skipTest('no groups') - # Choose an existent gid. - gid = entries[0][2] - self.assertRaises(TypeError, grp.getgrgid, float(gid)) - self.assertRaises(TypeError, grp.getgrgid, str(gid)) + raise + else: + self.fail('Cannot find non-existent gid') if __name__ == "__main__": diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 7edb026324e..c48d5d91c2b 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,10 +1,10 @@ # Simple test suite for http/cookies.py - import copy import unittest import doctest from http import cookies import pickle +import urllib.parse from test import support from test.support.testcase import ExtraAssertions @@ -153,17 +153,19 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme; Version=1', safe='', encoding='utf-8') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme', safe='', encoding='utf-8') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -260,17 +262,19 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1', safe='', encoding='utf-8') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme', safe='', encoding='utf-8') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -361,13 +365,17 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) + expected_encoded_cookie = urllib.parse.quote( + "%s=%s; Path=/foo" % (i, "%s_coded_val" % i), + safe='', encoding='utf-8', + ) expected_js_output = """ - """ % (i, "%s_coded_val" % i) + """ % (expected_encoded_cookie,) self.assertEqual(M.js_output(), expected_js_output) for i in ["foo bar", "foo@bar"]: # Try some illegal characters diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 5267d2fe011..75b350163ce 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -370,6 +370,51 @@ def test_invalid_headers(self): with self.assertRaisesRegex(ValueError, 'Invalid header'): conn.putheader(name, value) + def test_invalid_tunnel_headers(self): + cases = ( + ('Invalid\r\nName', 'ValidValue'), + ('Invalid\rName', 'ValidValue'), + ('Invalid\nName', 'ValidValue'), + ('\r\nInvalidName', 'ValidValue'), + ('\rInvalidName', 'ValidValue'), + ('\nInvalidName', 'ValidValue'), + (' InvalidName', 'ValidValue'), + ('\tInvalidName', 'ValidValue'), + ('Invalid:Name', 'ValidValue'), + (':InvalidName', 'ValidValue'), + ('ValidName', 'Invalid\r\nValue'), + ('ValidName', 'Invalid\rValue'), + ('ValidName', 'Invalid\nValue'), + ('ValidName', 'InvalidValue\r\n'), + ('ValidName', 'InvalidValue\r'), + ('ValidName', 'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + conn = client.HTTPConnection('example.com') + conn.set_tunnel('tunnel', headers={ + name: value + }) + conn.sock = FakeSocket('') + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn._tunnel() # Called in .connect() + + def test_invalid_tunnel_host(self): + cases = ( + 'invalid\r.host', + '\ninvalid.host', + 'invalid.host\r\n', + 'invalid.host\x00', + 'invalid host', + ) + for tunnel_host in cases: + with self.subTest(tunnel_host): + conn = client.HTTPConnection('example.com') + conn.set_tunnel(tunnel_host) + conn.sock = FakeSocket('') + with self.assertRaisesRegex(ValueError, 'Tunnel host can\'t contain control characters'): + conn._tunnel() # Called in .connect() + def test_headers_debuglevel(self): body = ( b'HTTP/1.1 200 OK\r\n' diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 9f1f682d02e..6573b4e7f30 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -387,6 +387,16 @@ def cmd_AUTHENTICATE(self, tag, args): r'\[AUTHENTICATIONFAILED\] invalid'): client.authenticate('MYAUTH', lambda x: b'fake') + def test_invalid_login(self): + class MyServer(SimpleIMAPHandler): + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_tagged(tag, 'NO', '[LOGIN] failed') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[LOGIN\] failed'): + client.login('user', 'wrongpass') + def test_valid_authentication_bytes(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 4d2c263bebd..4eb1deaf348 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -379,6 +379,15 @@ def test_import_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): import something_that_should_not_exist_anywhere + def test_import_null_byte_in_name_raises_ModuleNotFoundError(self): + # gh-150633: module names containing null bytes should not + # lead to duplicates in sys.modules + before = set(sys.modules.keys()) + with self.assertRaises(ModuleNotFoundError): + __import__('zipimport\x00junk') + + self.assertEqual(set(sys.modules.keys()), before) + def test_from_import_missing_module_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): from something_that_should_not_exist_anywhere import blah diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 980289f7ee7..add156d0e36 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,5 +1,6 @@ from collections import namedtuple import contextlib +import errno import json import io import logging @@ -55,7 +56,7 @@ def _close_file(file): else: os.close(file) except OSError as exc: - if exc.errno != 9: + if exc.errno != errno.EBADF: raise # re-raise # It was closed already. diff --git a/Lib/test/test_json/json_lines.jsonl b/Lib/test/test_json/json_lines.jsonl new file mode 100644 index 00000000000..d2f29211195 --- /dev/null +++ b/Lib/test/test_json/json_lines.jsonl @@ -0,0 +1,2 @@ +{"ingredients":["frog", "water", "chocolate", "glucose"]} +{"ingredients":["chocolate","steel bolts"]} diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 682014cfd5b..7c2789f8d1f 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -1,4 +1,5 @@ from test.test_json import CTest +from test.support import gc_collect class BadBool: @@ -80,3 +81,63 @@ def test(name): def test_unsortable_keys(self): with self.assertRaises(TypeError): self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'}) + + def test_mutate_dict_items_during_encode(self): + # gh-142831: Clearing the items list via a re-entrant key encoder + # must not cause a use-after-free. BadDict.items() returns a + # mutable list; encode_str clears it while iterating. + items = None + + class BadDict(dict): + def items(self): + nonlocal items + items = [("boom", object())] + return items + + cleared = False + def encode_str(obj): + nonlocal items, cleared + if items is not None: + items.clear() + items = None + cleared = True + gc_collect() + return '"x"' + + encoder = self.json.encoder.c_make_encoder( + None, lambda o: "null", + encode_str, None, + ": ", ", ", False, + False, True + ) + + # Must not crash (use-after-free under ASan before fix) + encoder(BadDict(real=1), 0) + self.assertTrue(cleared) + + def test_mutate_list_during_encode(self): + # gh-142831: Clearing a list mid-iteration via the default + # callback must not cause a use-after-free. + call_count = 0 + lst = [object() for _ in range(10)] + + def default(obj): + nonlocal call_count + call_count += 1 + if call_count == 3: + lst.clear() + gc_collect() + return None + + encoder = self.json.encoder.c_make_encoder( + None, default, + self.json.encoder.c_encode_basestring, None, + ": ", ", ", False, + False, True + ) + + # Must not crash (use-after-free under ASan before fix) + encoder(lst, 0) + # Verify the mutation path was actually hit and the loop + # stopped iterating after the list was cleared. + self.assertEqual(call_count, 3) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 2b63810d539..231fdfd7626 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -1,4 +1,5 @@ import errno +import pathlib import os import sys import textwrap @@ -6,7 +7,7 @@ import subprocess from test import support -from test.support import os_helper +from test.support import force_not_colorized, os_helper from test.support.script_helper import assert_python_ok @@ -147,6 +148,14 @@ def test_jsonlines(self): self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') + @force_not_colorized + def test_jsonlines_from_file(self): + jsonl = pathlib.Path(__file__).parent / 'json_lines.jsonl' + args = sys.executable, '-m', 'json.tool', '--json-lines', jsonl + process = subprocess.run(args, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, self.jsonlines_expect) + self.assertEqual(process.stderr, '') + def test_help_flag(self): rc, out, err = assert_python_ok('-m', 'json.tool', '-h') self.assertEqual(rc, 0) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 4aeaaf6e176..1a4fde1e0a7 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -310,6 +310,101 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) + def test_reference_loop_list(self): + a = [] + a.append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, list) + self.assertIs(b[0], b) + + def test_reference_loop_dict(self): + a = {} + a[None] = a + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, dict) + self.assertIs(b[None], b) + + def test_reference_loop_tuple(self): + a = ([],) + a[0].append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, tuple) + self.assertIsInstance(b[0], list) + self.assertIs(b[0][0], b) + + def test_reference_loop_code(self): + def f(): + return 1234.5 + code = f.__code__ + a = [] + code = code.replace(co_consts=code.co_consts + (a,)) + # This test creates a reference loop which leads to reference leaks, + # so we need to break the loop manually. See gh-148722. + self.addCleanup(a.clear) + a.append(code) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, code, v) + + def test_loads_reference_loop_list(self): + data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] + a = marshal.loads(data) + self.assertIsInstance(a, list) + self.assertIs(a[0], a) + + def test_loads_reference_loop_dict(self): + data = b'\xfbNr\x00\x00\x00\x000' # {None: } + a = marshal.loads(data) + self.assertIsInstance(a, dict) + self.assertIs(a[None], a) + + def test_loads_abnormal_reference_loops(self): + # Indirect self-references of tuples. + data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], list) + self.assertIs(a[0][0], a) + + data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: },) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], dict) + self.assertIs(a[0][None], a) + + # Direct self-reference which cannot be created in Python. + # This creates a reference loop which cannot be collected. + if False: + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) + + # Direct self-references which cannot be created in Python + # because of unhashability. + data = b'\xfbr\x00\x00\x00\x00N0' # {: None} + self.assertRaises(TypeError, marshal.loads, data) + data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {} + self.assertRaises(TypeError, marshal.loads, data) + + for data in [ + # Direct self-references which cannot be created in Python. + b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({}) + ]: + with self.subTest(data=data): + self.assertRaises(ValueError, marshal.loads, data) + def test_exact_type_match(self): # Former bug: # >>> class Int(int): pass diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index f8dd1231bb8..9d1e186308a 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -520,6 +520,28 @@ def test_array_assign(self): m[:] = new_a self.assertEqual(a, new_a) + def test_boolean_format(self): + # Test '?' format (keep all the checks below for UBSan) + # See github.com/python/cpython/issues/148390. + + # m1a and m1b are equivalent to [False, True, False] + m1a = memoryview(b'\0\2\0').cast('?') + self.assertEqual(m1a.tolist(), [False, True, False]) + m1b = memoryview(b'\0\4\0').cast('?') + self.assertEqual(m1b.tolist(), [False, True, False]) + self.assertEqual(m1a, m1b) + + # m2a and m2b are equivalent to [True, True, True] + m2a = memoryview(b'\1\3\5').cast('?') + self.assertEqual(m2a.tolist(), [True, True, True]) + m2b = memoryview(b'\2\4\6').cast('?') + self.assertEqual(m2b.tolist(), [True, True, True]) + self.assertEqual(m2a, m2b) + + allbytes = bytes(range(256)) + allbytes = memoryview(allbytes).cast('?') + self.assertEqual(allbytes.tolist(), [False] + [True] * 255) + class BytesMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseBytesMemoryTests): diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index 1c6905bd1ee..fef86ce323e 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -4,6 +4,7 @@ import unittest from textwrap import dedent +from test import support from test.support import os_helper, requires_resource from test.support.os_helper import TESTFN, TESTFN_ASCII @@ -67,8 +68,12 @@ def run_in_separated_process(self, code): # Run test in a separated process to avoid stdin conflicts. # See: gh-110147 cmd = [sys.executable, '-c', code] - subprocess.run(cmd, check=True, capture_output=True, - creationflags=subprocess.CREATE_NEW_CONSOLE) + try: + subprocess.run(cmd, check=True, capture_output=True, + creationflags=subprocess.CREATE_NEW_CONSOLE) + except subprocess.CalledProcessError as exc: + support.skip_on_low_desktop_heap_memory_subprocess(exc.returncode) + raise def test_kbhit(self): code = dedent(''' diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 268f4f6d01c..9ee4fe56448 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3876,7 +3876,6 @@ async def test_trailers(self): @requires_headers_trailers @requires_32b async def test_headers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, headers=[b"x" * 2**16] * 2**15) @@ -3885,7 +3884,6 @@ async def test_headers_overflow_32bits(self): @requires_headers_trailers @requires_32b async def test_trailers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, trailers=[b"x" * 2**16] * 2**15) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 8c9f5703ca6..a87b965c85c 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,3 +1,4 @@ +import ast import dis from itertools import combinations, product import opcode @@ -6,7 +7,11 @@ import unittest from test import support -from test.support.bytecode_helper import BytecodeTestCase, CfgOptimizationTestCase +from test.support.bytecode_helper import ( + BytecodeTestCase, + CfgOptimizationTestCase, + _testinternalcapi, +) def compile_pattern_with_fast_locals(pattern): @@ -962,6 +967,45 @@ def trace(frame, event, arg): class DirectCfgOptimizerTests(CfgOptimizationTestCase): + def test_optimize_cfg_const_index_out_of_range(self): + insts = [ + ('LOAD_CONST', 2, 0), + ('RETURN_VALUE', None, 0), + ] + seq = self.seq_from_insts(insts) + with self.assertRaisesRegex(ValueError, "out of range"): + _testinternalcapi.optimize_cfg(seq, [0, 1], 0) + + def test_optimize_cfg_consts_must_be_list(self): + insts = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + seq = self.seq_from_insts(insts) + with self.assertRaisesRegex(TypeError, "consts must be a list"): + _testinternalcapi.optimize_cfg(seq, (0,), 0) + + def test_compiler_codegen_metadata_consts_roundtrips_optimize_cfg(self): + tree = ast.parse("x = (1, 2)", mode="exec", optimize=1) + insts, meta = _testinternalcapi.compiler_codegen(tree, "", 0) + consts = meta["consts"] + self.assertIsInstance(consts, list) + _testinternalcapi.optimize_cfg(insts, consts, 0) + + def test_compiler_codegen_consts_include_none_required_for_implicit_return(self): + tree = ast.parse("pass", mode="exec", optimize=1) + insts, meta = _testinternalcapi.compiler_codegen(tree, "", 0) + consts = meta["consts"] + self.assertEqual(consts, [None]) + + load_const = opcode.opmap["LOAD_CONST"] + self.assertEqual( + [t[1] for t in insts.get_instructions() if t[0] == load_const], + [0], + ) + + _testinternalcapi.optimize_cfg(insts, list(consts), 0) + def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index aa090b464a7..bdf57776c82 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -1,3 +1,5 @@ +import random +import string import sys import unittest from test.support import import_helper @@ -56,59 +58,57 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, pwd.getpwuid) self.assertRaises(TypeError, pwd.getpwuid, 3.14) + self.assertRaises(TypeError, pwd.getpwuid, 0.0) + self.assertRaises(TypeError, pwd.getpwuid, 0, 0) + # should be out of uid_t range + self.assertRaises(KeyError, pwd.getpwuid, 2**128) + self.assertRaises(KeyError, pwd.getpwuid, -2**128) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - self.assertRaises(TypeError, pwd.getpwall, 42) + self.assertRaises(TypeError, pwd.getpwnam, b'root') + self.assertRaises(TypeError, pwd.getpwnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, pwd.getpwnam, 'roo\udc74') + self.assertRaises(KeyError, pwd.getpwnam, '') + self.assertRaises(TypeError, pwd.getpwall, 42) - # try to get some errors - bynames = {} - byuids = {} - for (n, p, u, g, gecos, d, s) in pwd.getpwall(): - bynames[n] = u - byuids[u] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] if allnames else "invaliduser" - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue - else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, pwd.getpwnam, fakename) - - # In some cases, byuids isn't a complete list of all users in the - # system, so if we try to pick a value not in byuids (via a perturbing - # loop, say), pwd.getpwuid() might still be able to find data for that - # uid. Using sys.maxint may provoke the same problems, but hopefully - # it will be a more repeatable failure. - fakeuid = sys.maxsize - self.assertNotIn(fakeuid, byuids) - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + # Find a non-existent user name. + # getpwall() will not necessarily report all existing users + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + pwd.getpwnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent user name') + + # Find a non-existent uid. + maxuid = max(e.pw_uid for e in pwd.getpwall()) + if maxuid < 2**15: + maxuid = 2**15 + elif maxuid < 2**16: + maxuid = 2**16-1 + else: + maxuid = 2**31 + for _ in range(30): + fakeuid = random.randrange(maxuid) + try: + pwd.getpwuid(fakeuid) + except KeyError: + break + else: + self.fail('Cannot find non-existent uid') + + # On Cygwin, getpwuid(-1) returns 'Unknown+User' user + if sys.platform != 'cygwin': + # -1 shouldn't be a valid uid because it has a special meaning in many + # uid-related functions + self.assertRaises(KeyError, pwd.getpwuid, -1) - # -1 shouldn't be a valid uid because it has a special meaning in many - # uid-related functions - self.assertRaises(KeyError, pwd.getpwuid, -1) - # should be out of uid_t range - self.assertRaises(KeyError, pwd.getpwuid, 2**128) - self.assertRaises(KeyError, pwd.getpwuid, -2**128) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 2309353503f..ebd8c955dc2 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -227,8 +227,7 @@ def _verify_parse_output(self, operations): "Character data: '\xb5'", "End element: 'root'", ] - for operation, expected_operation in zip(operations, expected_operations): - self.assertEqual(operation, expected_operation) + self.assertEqual(operations, expected_operations) def test_parse_bytes(self): out = self.Outputter() @@ -276,6 +275,119 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + @support.subTests('encoding', [ + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', + 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', + 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', + 'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', + 'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125', + 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + 'cp1256', 'cp1257', 'cp1258', + 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', + 'mac-roman', 'mac-turkish', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', + ]) + def test_supported_encodings(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + data = (f'\n' + f'{c}').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Start element: 'root' {}", + f'Character data: {c!r}', + "End element: 'root'", + ]) + + @support.subTests('encoding', [ + 'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be', + 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic', + ]) + def test_supported_encodings2(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + data = (f'\n' + '' + '<корінь атрибут="значення">зміст').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Comment: ' коментар '", + "Start element: 'корінь' {'атрибут': 'значення'}", + "Character data: 'зміст'", + "End element: 'корінь'", + ]) + + @support.subTests('encoding', [ + 'UTF-7', + "Big5-HKSCS", "Big5", + "cp932", "cp949", "cp950", + "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR", + "GB18030", "GB2312", "GBK", + "ISO-2022-KR", + "johab", + "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213", + ]) + def test_unsupported_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(ValueError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(ValueError): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875', + 'cp1026', 'cp1140', + 'mac_arabic', 'mac_farsi', + ]) + def test_incompatible_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(expat.ExpatError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'hex_codec', 'rot_13', + ]) + def test_non_text_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(LookupError): + parser.Parse(data, True) + + def test_undefined_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(UnicodeError): + parser.Parse(data, True) + + def test_unknown_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(LookupError): + parser.Parse(data, True) + + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value @@ -1053,6 +1165,64 @@ def test_set_maximum_amplification__fail_for_subparser(self): self.assert_root_parser_failure(setter, 123.45) +@unittest.skipIf(expat.version_info < (2, 4, 0), "requires Expat >= 2.4.0") +class ExpansionProtectionTest(AttackProtectionTestBase, unittest.TestCase): + + def assert_rejected(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) hits the allocation limit.""" + msg = ( + r"limit on input amplification factor \(from DTD and entities\) " + r"breached: line \d+, column \d+" + ) + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + def set_activation_threshold(self, parser, threshold): + return parser.SetBillionLaughsAttackProtectionActivationThreshold(threshold) + + def set_maximum_amplification(self, parser, max_factor): + return parser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor) + + def test_set_activation_threshold__threshold_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be always reached. + self.set_activation_threshold(parser, 3) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_activation_threshold__threshold_not_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be never reached. + self.set_activation_threshold(parser, pow(10, 5)) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assertIsNotNone(parser.Parse(payload, True)) + + def test_set_maximum_amplification__amplification_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to always be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + # Craft a payload for which the peak amplification factor is > 1.0. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_maximum_amplification__amplification_not_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to never be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1e4)) + # Craft a payload for which the peak amplification factor is < 1e4. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assertIsNotNone(parser.Parse(payload, True)) + + @unittest.skipIf(not hasattr(expat.XMLParserType, "SetAllocTrackerMaximumAmplification"), "requires Python compiled with Expat >= 2.7.2") diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 8e1f8509075..bb675bfbe33 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2221,11 +2221,8 @@ def test_unload_tests(self): self.check_executed_tests(output, tests, stats=3) def check_add_python_opts(self, option): - # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python - try: - import _testinternalcapi - except ImportError: - raise unittest.SkipTest("requires _testinternalcapi") + # --fast-ci and --slow-ci add "-u -W error -bb -E" options to Python + code = textwrap.dedent(r""" import sys import unittest @@ -2239,25 +2236,27 @@ def check_add_python_opts(self, option): use_environment = (support.is_emscripten or support.is_wasi) class WorkerTests(unittest.TestCase): - @unittest.skipUnless(get_config is None, 'need get_config()') + @unittest.skipIf(get_config is None, 'need get_config()') def test_config(self): - config = get_config()['config'] + config = get_config() # -u option self.assertEqual(config['buffered_stdio'], 0) - # -W default option - self.assertTrue(config['warnoptions'], ['default']) + # -W error option + self.assertEqual(config['warnoptions'], + ['error', 'error::BytesWarning']) # -bb option - self.assertTrue(config['bytes_warning'], 2) + self.assertEqual(config['bytes_warning'], 2) # -E option - self.assertTrue(config['use_environment'], use_environment) + self.assertEqual(config['use_environment'], use_environment) def test_python_opts(self): # -u option self.assertTrue(sys.__stdout__.write_through) self.assertTrue(sys.__stderr__.write_through) - # -W default option - self.assertTrue(sys.warnoptions, ['default']) + # -W error option + self.assertEqual(sys.warnoptions, + ['error', 'error::BytesWarning']) # -bb option self.assertEqual(sys.flags.bytes_warning, 2) @@ -2276,7 +2275,8 @@ def test_python_opts(self): proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) + text=True, + env=support.make_clean_env()) self.assertEqual(proc.returncode, 0, proc) def test_add_python_opts(self): diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 855dca2258d..16109820bee 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -5,6 +5,7 @@ import subprocess import sys import unittest +from contextlib import contextmanager from functools import partial from textwrap import dedent from test import support @@ -67,6 +68,19 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, custom=F spawn_asyncio_repl = partial(spawn_repl, "-m", "asyncio", custom=True) +@contextmanager +def temp_pythonstartup(*, source: str, histfile: str = ".pythonhist"): + """Create environment variables for a PYTHONSTARTUP script in a temporary directory.""" + with os_helper.temp_dir() as tmpdir: + filename = os.path.join(tmpdir, "pythonstartup.py") + with open(filename, "w") as f: + f.write(source) + yield { + "PYTHONSTARTUP": filename, + "PYTHON_HISTORY": os.path.join(tmpdir, histfile) + } + + def run_on_interactive_mode(source): """Spawn a new Python interpreter, pass the given input source code from the stdin and return the @@ -276,8 +290,6 @@ def make_repl(env): """) % script self.assertIn(expected, output) - - def test_runsource_show_syntax_error_location(self): user_input = dedent("""def f(x, x): ... """) @@ -442,6 +454,33 @@ def test_quiet_mode(self): self.assertEqual(p.returncode, 0) self.assertEqual(output[:3], ">>>") + @support.force_not_colorized + @support.subTests( + ("startup_code", "expected_error"), + [ + ("some invalid syntax\n", "SyntaxError: invalid syntax"), + ("1/0\n", "ZeroDivisionError: division by zero"), + ], + ) + def test_pythonstartup_failure(self, startup_code, expected_error): + startup_env = self.enterContext( + temp_pythonstartup(source=startup_code, histfile=".asyncio_history")) + + p = spawn_repl( + "-qm", "asyncio", + env=os.environ | startup_env, + isolated=False, + custom=True) + p.stdin.write("print('user code', 'executed')\n") + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + tb_hint = f'File "{startup_env["PYTHONSTARTUP"]}", line 1' + self.assertIn(tb_hint, output) + self.assertIn(expected_error, output) + + self.assertIn("user code executed", output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d403a0fe96b..76768d05955 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch import builtins +import types import rlcompleter from test.support import MISSING_C_DOCSTRINGS @@ -135,6 +136,57 @@ def bar(self): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertFalse(f.property_called) + def test_released_memoryview_completion_works(self): + mv = memoryview(b"abc") + mv.release() + + self.assertIsInstance(type(mv).shape, types.GetSetDescriptorType) + self.assertIsInstance(type(mv).strides, types.GetSetDescriptorType) + + completer = rlcompleter.Completer(dict(mv=mv)) + matches = completer.attr_matches('mv.') + + # These are getset descriptors on memoryview and should be completed + # without evaluating the released-memoryview getters. + self.assertIn('mv.shape', matches) + self.assertIn('mv.strides', matches) + + def test_member_descriptor_not_evaluated(self): + class Foo: + __slots__ = ("boom",) + boom_accesses = 0 + + def __getattribute__(self, name): + if name == "boom": + type(self).boom_accesses += 1 + raise RuntimeError("boom access should be skipped") + return super().__getattribute__(name) + + self.assertIsInstance(Foo.boom, types.MemberDescriptorType) + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom_accesses, 0) + + def test_raising_descriptor_completion_works(self): + class ExplodingDescriptor: + def __init__(self): + self.instance_get_calls = 0 + + def __get__(self, obj, owner): + if obj is None: + return self + self.instance_get_calls += 1 + raise RuntimeError("descriptor getter exploded") + + class Foo: + boom = ExplodingDescriptor() + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom.instance_get_calls, 0) def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index cdcf0681b44..620e579c6f2 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -15,14 +15,18 @@ class BaseRobotTest: good = [] bad = [] site_maps = None + expected_output = None def __init_subclass__(cls): super().__init_subclass__() # Remove tests that do nothing. - if not cls.good: - cls.test_good_urls = None - if not cls.bad: - cls.test_bad_urls = None + if issubclass(cls, unittest.TestCase): + if not cls.good: + cls.test_good_urls = None + if not cls.bad: + cls.test_bad_urls = None + if cls.expected_output is None: + cls.test_string_formatting = None def setUp(self): lines = io.StringIO(self.robots_txt).readlines() @@ -50,6 +54,8 @@ def test_bad_urls(self): def test_site_maps(self): self.assertEqual(self.parser.site_maps(), self.site_maps) + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -61,6 +67,56 @@ class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): good = ['/', '/test.html'] bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] +class SimpleExampleTest(BaseRobotTest, unittest.TestCase): + # Example from RFC 9309, section 5.1. + robots_txt = """\ +User-Agent: * +Disallow: *.gif$ +Disallow: /example/ +Allow: /publications/ + +User-Agent: foobot +Disallow:/ +Allow:/example/page.html +Allow:/example/allowed.gif + +User-Agent: barbot +User-Agent: bazbot +Disallow: /example/page.html + +User-Agent: quxbot + """ + good = [ + '/', '/publications/', + ('foobot', '/example/page.html'), ('foobot', '/example/allowed.gif'), + ('barbot', '/'), ('barbot', '/example/'), + ('barbot', '/example/allowed.gif'), + ('barbot', '/example/disallowed.gif'), + ('barbot', '/publications/'), + ('barbot', '/publications/allowed.gif'), + ('bazbot', '/'), ('bazbot', '/example/'), + ('bazbot', '/example/allowed.gif'), + ('bazbot', '/example/disallowed.gif'), + ('bazbot', '/publications/'), + ('bazbot', '/publications/allowed.gif'), + ('quxbot', '/'), ('quxbot', '/example/'), + ('quxbot', '/example/page.html'), ('quxbot', '/example/allowed.gif'), + ('quxbot', '/example/disallowed.gif'), + ('quxbot', '/publications/'), + ('quxbot', '/publications/allowed.gif'), + ] + bad = [ + '/example/', '/example/page.html', '/example/allowed.gif', + '/example/disallowed.gif', + '/publications/allowed.gif', + ('foobot', '/'), ('foobot', '/example/'), + ('foobot', '/example/disallowed.gif'), + ('foobot', '/publications/'), + ('foobot', '/publications/allowed.gif'), + ('barbot', '/example/page.html'), + ('bazbot', '/example/page.html'), + ] + class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -102,7 +158,7 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): User-agent: * Disallow: / """ - good = [] + good = ['/robots.txt'] bad = ['/cyberworld/map/index.html', '/', '/tmp/'] @@ -137,6 +193,7 @@ def test_request_rate(self): class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): robots_txt = '' good = ['/foo'] + expected_output = '' class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): @@ -203,35 +260,209 @@ class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase): - # the order of User-agent should be correct. note - # that this file is incorrect because "Googlebot" is a - # substring of "Googlebot-Mobile" + # the order of User-agent should not matter robots_txt = """\ User-agent: Googlebot Disallow: / +Allow: /folder1/ User-agent: Googlebot-Mobile Allow: / +Disallow: /folder1/ """ agent = 'Googlebot' bad = ['/something.jpg'] + good = ['/folder1/myfile.html'] class UserAgentGoogleMobileTest(UserAgentOrderingTest): - agent = 'Googlebot-Mobile' + agent = 'Googlebot-mobile' + bad = ['/folder1/myfile.html'] + good = ['/something.jpg'] -class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase): - # Google also got the order wrong. You need - # to specify the URLs from more specific to more general +class LongestMatchTest(BaseRobotTest, unittest.TestCase): + # Based on example from RFC 9309, section 5.2. robots_txt = """\ -User-agent: Googlebot -Allow: /folder1/myfile.html -Disallow: /folder1/ +User-agent: * +Allow: /example/page/ +Disallow: /example/page/disallowed.gif +Allow: /example/ """ - agent = 'googlebot' - good = ['/folder1/myfile.html'] - bad = ['/folder1/anotherfile.html'] + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif'] + + +class LongestMatchWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Allow: /example/page/ +Disallow: *.gif +Allow: /example/ + """ + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif', '/x.gif'] + + +class AllowWinsEqualMatchTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: /spam +Disallow: /spam + """ + good = ['/spam', '/spam/'] + + +class AllowWinsEqualFullMatchTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: /spam$ +Disallow: /spam +Disallow: /eggs$ +Allow: /eggs +Disallow: /eggs$ + """ + good = ['/spam', '/eggs', '/eggs/'] + bad = ['/spam/'] + + +class AllowWinsEqualMatchWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: *am +Disallow: /spam +Disallow: *gs +Allow: /eggs +Disallow: *gs + """ + good = ['/spam', '/eggs', '/spam/', '/eggs/'] + + +class MergeGroupsTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: spambot +Disallow: /another/path + """ + agent = 'spambot' + bad = ['/some/path', '/another/path'] + + +class UserAgentStartsGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path +User-agent: eggsbot +Disallow: /another/path + """ + good = [('spambot', '/'), ('spambot', '/another/path'), + ('eggsbot', '/'), ('eggsbot', '/some/path')] + bad = [('spambot', '/some/path'), ('eggsbot', '/another/path')] + expected_output = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: eggsbot +Disallow: /another/path\ +""" + +class IgnoreEmptyLinesTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot + +User-agent: eggsbot +Disallow: /some/path + +Disallow: /another/path + """ + good = [('spambot', '/'), ('eggsbot', '/')] + bad = [ + ('spambot', '/some/path'), ('spambot', '/another/path'), + ('eggsbot', '/some/path'), ('eggsbot', '/another/path'), + ] + expected_output = """\ +User-agent: spambot +User-agent: eggsbot +Disallow: /some/path +Disallow: /another/path\ +""" + + +class IgnoreRulesWithoutUserAgentTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +Disallow: /some/path + +User-agent: * +Disallow: /another/path + """ + good = ['/', '/some/path'] + bad = ['/another/path'] + expected_output = """\ +User-agent: * +Disallow: /another/path\ +""" + + +class EmptyGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot + """ + agent = 'spambot' + good = ['/', '/some/path'] + expected_output = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot +Allow:\ +""" + + +class WeirdPathTest(BaseRobotTest, unittest.TestCase): + robots_txt = f"""\ +User-agent: * +Disallow: /a$$$ +Disallow: /b$z +Disallow: /c*** +Disallow: /d***z +Disallow: /e*$**$$ +Disallow: /f*$**$$z +Disallow: /g$*$$** +Disallow: /h$*$$**z + """ + good = ['/ax', '/a$$', '/b', '/bz', '/b$z', '/d', '/f', '/fz', + '/f$$$z', '/fx$y$$z', '/gx', '/g$$$', '/g$x$$y', '/h', '/hz', + '/h$$$z', '/h$x$$yz'] + bad = ['/a', '/c', '/cxy', '/dz', '/dxyz', '/dxzy', '/e', '/exy', + '/e$$', '/ex$y$', '/g'] + expected_output = """\ +User-agent: * +Disallow: /a$ +Disallow: /c* +Disallow: /d*z +Disallow: /e*$ +Disallow: /g$\ +""" + + +class PathWithManyWildcardsTest(BaseRobotTest, unittest.TestCase): + # This test would take many years if use naive translation to regular + # expression (* -> .*). + N = 50 + robots_txt = f"""\ +User-agent: * +Disallow: /{'*a'*N}*b + """ + good = ['/' + 'a'*N + 'a'] + bad = ['/' + 'a'*N + 'b'] class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): @@ -245,25 +476,13 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): good = ['/some/path', '/some/path?', '/some/path%3Fname=value', '/some/path?name%3Dvalue', '/another/path', '/another/path%3F', - '/yet/one/path?name=value%26more'] + '/yet/one/path?name=value%26more', + '/some/pathxname=value'] bad = ['/some/path?name=value' '/another/path?', '/another/path?name=value', '/yet/one/path?name=value&more'] -class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): - # obey first * entry (#4108) - robots_txt = """\ -User-agent: * -Disallow: /some/path - -User-agent: * -Disallow: /another/path - """ - good = ['/another/path'] - bad = ['/some/path'] - - class PercentEncodingTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * @@ -365,17 +584,60 @@ class StringFormattingTest(BaseRobotTest, unittest.TestCase): """ expected_output = """\ -User-agent: cybermapper -Disallow: /some/path - User-agent: * Crawl-delay: 1 Request-rate: 3/15 -Disallow: /cyberworld/map/\ +Disallow: /cyberworld/map/ + +User-agent: cybermapper +Disallow: /some/path\ """ - def test_string_formatting(self): - self.assertEqual(str(self.parser), self.expected_output) + +class ConstructedStringFormattingTest(unittest.TestCase): + def test_empty(self): + parser = urllib.robotparser.RobotFileParser() + self.assertEqual(str(parser), '') + + def test_group_without_rules(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['hambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['eggsbot'] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Allow: + +User-agent: hambot +Disallow: /ham + +User-agent: eggsbot +Allow:\ +""") + + def test_group_without_user_agent(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/spam', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/eggs', False)] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Disallow: /spam\ +""") @unittest.skipUnless( @@ -384,26 +646,23 @@ def test_string_formatting(self): ) class BaseLocalNetworkTestCase: - def setUp(self): + @classmethod + def setUpClass(cls): # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) + cls.addClassCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) + cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) + cls.addClassCleanup(cls.server.server_close) - self.t = threading.Thread( + t = threading.Thread( name='HTTPServer serving', - target=self.server.serve_forever, + target=cls.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() - - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + cls.enterClassContext(threading_helper.start_threads([t])) + cls.addClassCleanup(cls.server.shutdown) SAMPLE_ROBOTS_TXT = b'''\ @@ -425,7 +684,6 @@ def do_GET(self): def log_message(self, format, *args): pass - @threading_helper.reap_threads def testRead(self): # Test that reading a weird robots.txt doesn't fail. addr = self.server.server_address @@ -440,31 +698,79 @@ def testRead(self): self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) - self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) -class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): +class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): - self.send_error(403, "Forbidden access") + self.send_error(self.server.return_code) def log_message(self, format, *args): pass - @threading_helper.reap_threads - def testPasswordProtectedSite(self): + def setUp(self): + # Make sure that a valid code is set in the test. + self.server.return_code = None + + def testUnauthorized(self): + self.server.return_code = 401 addr = self.server.server_address - url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testForbidden(self): + self.server.return_code = 403 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testNotFound(self): + self.server.return_code = 404 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) + + def testTeapot(self): + self.server.return_code = 418 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) + + def testServiceUnavailable(self): + self.server.return_code = 503 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' robots_url = url + "/robots.txt" parser = urllib.robotparser.RobotFileParser() parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() @@ -476,6 +782,7 @@ class NetworkTestCase(unittest.TestCase): @classmethod def setUpClass(cls): support.requires('network') + cls.addClassCleanup(urllib.request.urlcleanup) with socket_helper.transient_internet(cls.base_url): cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) cls.parser.read() @@ -495,7 +802,7 @@ def test_basic(self): def test_can_fetch(self): self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere'))) self.assertFalse(self.parser.can_fetch('Nutch', self.base_url)) - self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian'))) + self.assertTrue(self.parser.can_fetch('Nutch', self.url('brian'))) self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats'))) self.assertFalse(self.parser.can_fetch('*', self.url('webstats'))) self.assertTrue(self.parser.can_fetch('*', self.base_url)) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index ada78ec8e6b..a4addbeaa2b 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -216,6 +216,25 @@ def test_invalid_names(self): # Package without __main__.py self.expect_import_error("multiprocessing") + def test_invalid_names_set_name_attribute(self): + cases = [ + # (mod_name, expected_name) -- comment indicates raise site + ("nonexistent_runpy_test_module", + "nonexistent_runpy_test_module"), # spec is None + ("sys.imp.eric", "sys.imp.eric"), # find_spec error + (".relative_name", ".relative_name"), # relative name rejected + ("sys", "sys"), # builtin: no code object + ("multiprocessing", "multiprocessing"), # package without __main__ + ] + for mod_name, expected_name in cases: + with self.subTest(mod_name=mod_name): + try: + run_module(mod_name) + except ImportError as exc: + self.assertEqual(exc.name, expected_name) + else: + self.fail("Expected ImportError for %r" % mod_name) + def test_library_module(self): self.assertEqual(run_module("runpy")["__name__"], "runpy") @@ -714,6 +733,17 @@ def test_directory_error(self): msg = "can't find '__main__' module in %r" % script_dir self._check_import_error(script_dir, msg) + def test_directory_error_sets_name_attribute(self): + with temp_dir() as script_dir: + self._make_test_script(script_dir, 'not_main') + try: + run_path(script_dir) + except ImportError as exc: + self.assertEqual(exc.name, '__main__') + else: + self.fail("Expected ImportError for directory without " + "__main__.py") + def test_zipfile(self): with temp_dir() as script_dir: mod_name = '__main__' diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 0e973255621..1469b7cc957 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2114,8 +2114,6 @@ def test_make_zipfile_rootdir_nodir(self): def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) - self.check_unpack_archive_with_converter( - format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -2171,6 +2169,71 @@ def test_unpack_archive_zip(self): with self.assertRaises(TypeError): self.check_unpack_archive('zip', filter='data') + def test_unpack_archive_zip_badpaths(self): + srcdir = self.mkdtemp() + zipname = os.path.join(srcdir, 'test.zip') + abspath = os.path.join(srcdir, 'abspath') + with zipfile.ZipFile(zipname, 'w') as zf: + zf.writestr(abspath, 'badfile') + zf.writestr(os.sep + abspath, 'badfile') + zf.writestr('/abspath', 'badfile') + zf.writestr('C:/abspath', 'badfile') + zf.writestr('D:\\abspath', 'badfile') + zf.writestr('E:abspath', 'badfile') + zf.writestr('F:/G:/abspath', 'badfile') + zf.writestr('//server/share/abspath', 'badfile') + zf.writestr('\\\\server2\\share\\abspath', 'badfile') + zf.writestr('../relpath', 'badfile') + zf.writestr(os.pardir + os.sep + 'relpath2', 'badfile') + zf.writestr('good/file', 'goodfile') + zf.writestr('good..file', 'goodfile') + + dstdir = os.path.join(self.mkdtemp(), 'dst') + unpack_archive(zipname, dstdir) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good', 'file'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good..file'))) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'abspath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'G_'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'server'))) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'C:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'D:\\abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'E:abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, '\\\\server2\\share\\abspath'))) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath'))) + else: + self.assertTrue(os.path.isfile(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath2'))) + + dstdir2 = os.path.join(self.mkdtemp(), 'dst') + os.mkdir(dstdir2) + with os_helper.change_cwd(dstdir2): + unpack_archive(zipname, '') + self.assertTrue(os.path.isfile(os.path.join('good', 'file'))) + self.assertTrue(os.path.isfile('good..file')) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists('abspath')) + self.assertFalse(os.path.exists('C_')) + self.assertFalse(os.path.exists('server')) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join('C:', 'abspath'))) + self.assertTrue(os.path.isfile('D:\\abspath')) + self.assertTrue(os.path.isfile('E:abspath')) + self.assertTrue(os.path.isfile(os.path.join('F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile('\\\\server2\\share\\abspath')) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists('relpath')) + else: + self.assertTrue(os.path.isfile(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists('relpath2')) + def test_unpack_registry(self): formats = get_unpack_formats() @@ -2828,6 +2891,23 @@ def test_destinsrc_false_positive(self): finally: os_helper.rmtree(TESTFN) + @os_helper.skip_unless_symlink + def test_destinsrc_symlink_bypass(self): + tmp = self.mkdtemp() + src = os.path.join(tmp, 'src') + os.makedirs(src) + # tmp/link -> tmp (one level up) + link = os.path.join(tmp, 'link') + os.symlink(tmp, link) + # raw path: tmp/link/src/sub - no src prefix in string space + # real path: tmp/src/sub - physically inside src + dst = os.path.join(link, 'src', 'sub') + self.assertTrue( + shutil._destinsrc(src, dst), + msg='_destinsrc failed to detect dst inside src via symlink ' + '(dst=%s, src=%s)' % (dst, src), + ) + @os_helper.skip_unless_symlink @mock_rename def test_move_file_symlink(self): diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index d765746987b..861e7e6a725 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -45,6 +45,59 @@ def test_connect_starttls(self): server.ehlo() server.quit() + def test_connect_host_port_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP(f'{self.testServer}:{self.remotePort}') + try: + server.starttls(context=context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + + def test_explicit_connect_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP() + server.connect(self.testServer, self.remotePort) + try: + server.starttls(context=context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + + def test_explicit_connect_host_port_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP() + server.connect(f'{self.testServer}:{self.remotePort}') + try: + server.starttls(context=context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + class SmtpSSLTest(unittest.TestCase): testServer = SMTP_TEST_SERVER diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 8a5e8582439..c6f6985faef 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -203,6 +203,25 @@ def _have_socket_hyperv(): return True +def _have_udp_lite(): + if not hasattr(socket, "IPPROTO_UDPLITE"): + return False + # Older Android versions block UDPLITE with SELinux. + if support.is_android and platform.android_ver().api_level < 29: + return False + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) + except OSError as exc: + # Linux 7.1 removed UDP Lite support + if exc.errno == errno.EPROTONOSUPPORT: + return False + raise + sock.close() + + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -245,10 +264,7 @@ def downgrade_malformed_data_warning(): HAVE_SOCKET_VSOCK = _have_socket_vsock() -# Older Android versions block UDPLITE with SELinux. -HAVE_SOCKET_UDPLITE = ( - hasattr(socket, "IPPROTO_UDPLITE") - and not (support.is_android and platform.android_ver().api_level < 29)) +HAVE_SOCKET_UDPLITE = _have_udp_lite() HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 4d815bc3d59..841b8dc6abf 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1431,6 +1431,18 @@ def test_blob_set_empty_slice(self): self.blob[0:0] = b"" self.assertEqual(self.blob[:], self.data) + def test_blob_set_empty_slice_wrong_type(self): + with self.assertRaises(TypeError): + self.blob[5:5] = None + + def test_blob_set_empty_slice_wrong_size(self): + with self.assertRaisesRegex(IndexError, "wrong size"): + self.blob[5:5] = b"123" + + def test_blob_set_empty_slice_correct(self): + self.blob[5:5] = b"" + self.assertEqual(self.blob[:], self.data) + def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"12345" actual = self.cx.execute("select b from test").fetchone()[0] diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 48d35b54a2e..adc15485e28 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -155,6 +155,16 @@ def test_sqlite_row_index(self): with self.assertRaises(IndexError): row[complex()] # index must be int or string + def test_delete_connection_row_factory(self): + # gh-149738: deleting row_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.row_factory + + def test_delete_connection_text_factory(self): + # gh-149738: deleting text_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.text_factory + def test_sqlite_row_index_unicode(self): row = self.con.execute("select 1 as \xff").fetchone() self.assertEqual(row["\xff"], 1) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index fbbe918af75..d57664286cb 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -52,6 +52,16 @@ IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') +HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') +requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +CAN_SET_KEYLOG = HAS_KEYLOG and os.name != "nt" +requires_keylog_setter = unittest.skipUnless( + CAN_SET_KEYLOG, + "cannot set 'keylog_filename' on Windows" +) + + PROTOCOL_TO_TLS_VERSION = {} for proto, ver in ( ("PROTOCOL_SSLv3", "SSLv3"), @@ -295,24 +305,35 @@ def make_test_context( cert_reqs=ssl.CERT_NONE, ca_certs=None, certfile=None, keyfile=None, ciphers=None, + min_version=None, max_version=None, ): if server_side: context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) else: context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + if check_hostname is None: if cert_reqs == ssl.CERT_NONE: context.check_hostname = False else: context.check_hostname = check_hostname + if cert_reqs is not None: context.verify_mode = cert_reqs + if ca_certs is not None: context.load_verify_locations(ca_certs) if certfile is not None or keyfile is not None: context.load_cert_chain(certfile, keyfile) + if ciphers is not None: context.set_ciphers(ciphers) + + if min_version is not None: + context.minimum_version = min_version + if max_version is not None: + context.maximum_version = max_version + return context @@ -324,6 +345,7 @@ def test_wrap_socket( cert_reqs=ssl.CERT_NONE, ca_certs=None, certfile=None, keyfile=None, ciphers=None, + min_version=None, max_version=None, **kwargs, ): context = make_test_context( @@ -332,6 +354,7 @@ def test_wrap_socket( cert_reqs=cert_reqs, ca_certs=ca_certs, certfile=certfile, keyfile=keyfile, ciphers=ciphers, + min_version=min_version, max_version=max_version, ) if not server_side: kwargs.setdefault("server_hostname", SIGNED_CERTFILE_HOSTNAME) @@ -1780,6 +1803,39 @@ def test_num_tickest(self): with self.assertRaises(ValueError): ctx.num_tickets = 1 + @support.cpython_only + def test_refcycle_msg_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context() + def msg_callback(*args, _=ctx, **kwargs): ... + ctx._msg_callback = msg_callback + + @support.cpython_only + @requires_keylog_setter + def test_refcycle_keylog_filename(self): + # See https://github.com/python/cpython/issues/142516. + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + ctx = make_test_context() + class KeylogFilename(str): ... + ctx.keylog_filename = KeylogFilename(os_helper.TESTFN) + ctx.keylog_filename._ = ctx + + @support.cpython_only + @unittest.skipUnless(ssl.HAS_PSK, 'requires TLS-PSK') + def test_refcycle_psk_client_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context() + def psk_client_callback(*args, _=ctx, **kwargs): ... + ctx.set_psk_client_callback(psk_client_callback) + + @support.cpython_only + @unittest.skipUnless(ssl.HAS_PSK, 'requires TLS-PSK') + def test_refcycle_psk_server_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context(server_side=True) + def psk_server_callback(*args, _=ctx, **kwargs): ... + ctx.set_psk_server_callback(psk_server_callback) + class SSLErrorTests(unittest.TestCase): @@ -5027,10 +5083,6 @@ def test_internal_chain_server(self): self.assertEqual(res, b'\x02\n') -HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') -requires_keylog = unittest.skipUnless( - HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') - class TestSSLDebug(unittest.TestCase): def keylog_lines(self, fname=os_helper.TESTFN): @@ -5268,15 +5320,27 @@ def non_linux_skip_if_other_okay_error(self, err): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or - re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I)): + re.search( + # Matches the following error messages: + # '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)' + # '[SSL: RECORD_LAYER_FAILURE] record layer failure (_ssl.c:1109)' + # '[SSL: HTTP_REQUEST] http request (_ssl.c:1143)' + r'wrong.version.number|record.layer.failure|http.request', + str(getattr(err, "reason", "")), + re.IGNORECASE, + ) + ): # On Windows the TCP RST leads to a ConnectionResetError # (ECONNRESET) which Linux doesn't appear to surface to userspace. # If wrap_socket() winds up on the "if connected:" path and doing - # the actual wrapping... we get an SSLError from OpenSSL. Typically - # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario - # we're specifically trying to test. The way this test is written - # is known to work on Linux. We'll skip it anywhere else that it - # does not present as doing so. + # the actual wrapping... we get an SSLError from OpenSSL. This is + # typically WRONG_VERSION_NUMBER. The same happens on iOS, but + # RECORD_LAYER_FAILURE or HTTP_REQUEST is the error. + # + # While appropriate, these scenarios aren't what we're specifically + # trying to test. The way this test is written is known to work on + # Linux. We'll skip it anywhere else that it does not present as + # doing so. try: self.skipTest(f"Could not recreate conditions on {sys.platform}:" f" {err=}") diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index a9f0ce2524e..6b541da5231 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -21,6 +21,7 @@ import sysconfig import select import shutil +import socket import threading import gc import textwrap @@ -1043,19 +1044,49 @@ def test_communicate_timeout_large_input(self): # On Windows, stdin writing must also honor the timeout rather than # blocking indefinitely when the pipe buffer fills. - # Input larger than typical pipe buffer (4-64KB on Windows) - input_data = b"x" * (128 * 1024) + input_data = b"x" * (128 * 1024) # > typical pipe buffer + + # Cross-platform wake mechanism: the slow reader connects to a + # loopback TCP socket and blocks in select() on it (capped at 9s + # as a safety net we don't expect to hit). After phase 1 raises + # TimeoutExpired, the parent sends a byte to release the child so + # it drains stdin. A socket (rather than a raw pipe) is required + # because Windows select() only supports sockets, not arbitrary + # file descriptors. + server = socket.create_server(('127.0.0.1', 0), backlog=1) + server.settimeout(10) # bound the accept() if the child fails to start + port = server.getsockname()[1] + # The child sends one byte (low byte of its PID) first so the parent + # can detect the rare case of an unrelated process on the same host + # connecting to our ephemeral port before our child does. A single + # byte gives 1/256 collision odds, which is plenty for flake-prevention. + slow_reader = ( + "import os, socket, sys, select; " + f"s = socket.create_connection(('127.0.0.1', {port}), timeout=9); " + "s.sendall(bytes([os.getpid() & 0xff])); " + "select.select([s], [], [], 9); " + "sys.stdout.buffer.write(sys.stdin.buffer.read())" + ) p = subprocess.Popen( - [sys.executable, "-c", - "import sys, time; " - "time.sleep(30); " # Don't read stdin for a long time - "sys.stdout.buffer.write(sys.stdin.buffer.read())"], + [sys.executable, "-c", slow_reader], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + conn = None try: + conn, _ = server.accept() + server.close() + server = None + + conn.settimeout(5) + peer_byte = conn.recv(1) + conn.settimeout(None) + self.assertEqual(peer_byte, bytes([p.pid & 0xff]), + f"loopback handshake byte {peer_byte!r} != " + f"low byte of child PID {p.pid} ({p.pid & 0xff:#x})") + timeout = 0.2 start = time.monotonic() try: @@ -1064,7 +1095,7 @@ def test_communicate_timeout_large_input(self): elapsed = time.monotonic() - start self.fail( f"TimeoutExpired not raised. communicate() completed in " - f"{elapsed:.2f}s, but subprocess sleeps for 30s. " + f"{elapsed:.2f}s, but slow reader stalls for up to 9s. " "Stdin writing blocked without enforcing timeout.") except subprocess.TimeoutExpired: elapsed = time.monotonic() - start @@ -1072,11 +1103,16 @@ def test_communicate_timeout_large_input(self): # Timeout should occur close to the specified timeout value, # not after waiting for the subprocess to finish sleeping. # Allow generous margin for slow CI, but must be well under - # the subprocess sleep time. + # the slow-reader's stall cap. self.assertLess(elapsed, 5.0, f"TimeoutExpired raised after {elapsed:.2f}s; expected ~{timeout}s. " "Stdin writing blocked without checking timeout.") + # Release the slow reader so it stops blocking and drains stdin. + conn.sendall(b'go') + conn.close() + conn = None + # After timeout, continue communication. The remaining input # should be sent and we should receive all data back. stdout, stderr = p.communicate() @@ -1086,6 +1122,10 @@ def test_communicate_timeout_large_input(self): f"Expected {len(input_data)} bytes output but got {len(stdout)}") self.assertEqual(stdout, input_data) finally: + if conn is not None: + conn.close() + if server is not None: + server.close() p.kill() p.wait() @@ -3721,13 +3761,17 @@ def test_startupinfo_copy(self): self.assertEqual(startupinfo.wShowWindow, subprocess.SW_HIDE) self.assertEqual(startupinfo.lpAttributeList, {"handle_list": []}) + # CREATE_NEW_CONSOLE creates a "popup" window. + @support.requires_resource('gui') def test_creationflags(self): # creationflags argument CREATE_NEW_CONSOLE = 16 sys.stderr.write(" a DOS box should flash briefly ...\n") - subprocess.call(sys.executable + - ' -c "import time; time.sleep(0.25)"', - creationflags=CREATE_NEW_CONSOLE) + rc = subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + support.skip_on_low_desktop_heap_memory_subprocess(rc) + self.assertEqual(rc, 0) def test_invalid_args(self): # invalid arguments should raise ValueError diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index f1f99bb84bf..e8b1eb8bc03 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -884,10 +884,39 @@ def test_extractall_hardlink_on_symlink(self): self._assert_on_file_content(hardlink_filepath, sha256_regtype) +class GzipReadTestBase: + + def test_read_with_extra_field(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 4 + data[10:10] = b'\x05\x00extra' + with open(tmpname, 'wb') as f: + f.write(data) + print(self.mode) + with tarfile.open(tmpname, mode=self.mode): + pass + + def test_read_with_file_comment(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 16 + i = data.index(0, 10) + 1 + data[i:i] = b'comment\x00' + with open(tmpname, 'wb') as f: + f.write(data) + with tarfile.open(tmpname, mode=self.mode): + pass + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None -class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): +class GzipMiscReadTest(GzipTest, GzipReadTestBase, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): @@ -959,7 +988,7 @@ def test_compare_members(self): finally: tar1.close() -class GzipStreamReadTest(GzipTest, StreamReadTest): +class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest): pass class Bz2StreamReadTest(Bz2Test, StreamReadTest): @@ -3649,6 +3678,39 @@ class TestExtractionFilters(unittest.TestCase): # The destination for the extraction, within `outerdir` destdir = outerdir / 'dest' + @classmethod + def setUpClass(cls): + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + cls.dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + cls.dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + else: + cls.dotdot_resolves_early = False + @contextmanager def check_context(self, tar, filter, *, check_flag=True): """Extracts `tar` to `self.destdir` and allows checking the result @@ -3820,10 +3882,19 @@ def test_parent_symlink(self): + "which is outside the destination") with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'parent' would link to ['"].*outerdir['"], """ - + "which is outside the destination") + if self.dotdot_resolves_early: + # 'current/../..' normalises to '..', which is rejected. + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + else: + # 'current/..' normalises to '.'; the rewritten link is + # created and 'parent/evil' lands harmlessly inside the + # destination. + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='.') + self.expect_file('evil') else: # No symlink support. The symlinks are ignored. @@ -3913,35 +3984,6 @@ def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives - # Posix and Windows have different pathname resolution: - # either symlink or a '..' component resolve first. - # Let's see which we are on. - if os_helper.can_symlink(): - testpath = os.path.join(TEMPDIR, 'resolution_test') - os.mkdir(testpath) - - # testpath/current links to `.` which is all of: - # - `testpath` - # - `testpath/current` - # - `testpath/current/current` - # - etc. - os.symlink('.', os.path.join(testpath, 'current')) - - # we'll test where `testpath/current/../file` ends up - with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): - pass - - if os.path.exists(os.path.join(testpath, 'file')): - # Windows collapses 'current\..' to '.' first, leaving - # 'testpath\file' - dotdot_resolves_early = True - elif os.path.exists(os.path.join(testpath, '..', 'file')): - # Posix resolves 'current' to '.' first, leaving - # 'testpath/../file' - dotdot_resolves_early = False - else: - raise AssertionError('Could not determine link resolution') - with ArchiveMaker() as arc: # `current` links to `.` which is both the destination directory @@ -3977,7 +4019,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'data'): if os_helper.can_symlink(): - if dotdot_resolves_early: + if self.dotdot_resolves_early: # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, @@ -4097,6 +4139,76 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + @os_helper.skip_unless_symlink + def test_normpath_realpath_mismatch(self): + # The link-target check must validate the value that will actually + # be written to disk (the normalised linkname), not the original. + # Here 'a' is a symlink to a deep nonexistent path, so realpath() + # of 'a/../../...' stays inside the destination while normpath() + # collapses 'a/..' lexically and escapes. + depth = len(self.destdir.parts) + 5 + deep = '/'.join(f'p{i}' for i in range(depth)) + sneaky = 'a/' + '../' * depth + 'flag' + for kind in 'symlink_to', 'hardlink_to': + with self.subTest(kind): + with ArchiveMaker() as arc: + arc.add('a', symlink_to=deep) + arc.add('escape', **{kind: sneaky}) + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_symlink_trailing_slash(self): + # A trailing slash on a symlink member's name must not cause the + # link target to be resolved relative to the wrong directory. + with ArchiveMaker() as arc: + t = tarfile.TarInfo('x/') + t.type = tarfile.SYMTYPE + t.linkname = '..' + arc.tar_w.addfile(t) + arc.add('x/escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_link_at_destination(self): + # A link member whose name resolves to the destination directory + # itself must be rejected: otherwise the destination is replaced + # by a symlink and later members can be redirected through it. + for name in '', '.', './': + with ArchiveMaker() as arc: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = '.' + arc.tar_w.addfile(t) + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.OutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_empty_name_symlink_chain(self): + # Regression test for a chain of empty-named symlinks that + # incrementally redirects the destination outwards. + with ArchiveMaker() as arc: + for name, target in [('', ''), ('a/', '..'), + ('', 'dummy'), ('', 'a'), + ('b/', '..'), + ('', 'dummy'), ('', 'a/b')]: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = target + arc.tar_w.addfile(t) + arc.add('escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.FilterError) + @symlink_test def test_deep_symlink(self): # Test that symlinks and hardlinks inside a directory diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 290b8d7a8e7..e636f45a470 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -336,7 +336,9 @@ def test_read_only_directory(self): with _inside_empty_temp_dir(): probe = os.path.join(tempfile.tempdir, 'probe') if os.name == 'nt': - cmd = ['icacls', tempfile.tempdir, '/deny', 'Everyone:(W)'] + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/deny', '*S-1-1-0:(W)'] stdout = None if support.verbose > 1 else subprocess.DEVNULL subprocess.run(cmd, check=True, stdout=stdout) else: @@ -359,7 +361,9 @@ def test_read_only_directory(self): self.make_temp() finally: if os.name == 'nt': - cmd = ['icacls', tempfile.tempdir, '/grant:r', 'Everyone:(M)'] + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/grant:r', '*S-1-1-0:(M)'] subprocess.run(cmd, check=True, stdout=stdout) else: os.chmod(tempfile.tempdir, oldmode) diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index 72a104fc1a6..0ff3170d8f8 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -304,7 +304,7 @@ def test_main_help(self): def test_main_verbose(self): s = self.run_main(switches=['-v']) self.assertEqual(s, dedent("""\ - 1 loop -> 1 secs + 1 loop -> 1 sec raw times: 1 sec, 1 sec, 1 sec, 1 sec, 1 sec @@ -314,19 +314,19 @@ def test_main_verbose(self): def test_main_very_verbose(self): s = self.run_main(seconds_per_increment=0.000_030, switches=['-vv']) self.assertEqual(s, dedent("""\ - 1 loop -> 3e-05 secs - 2 loops -> 6e-05 secs - 5 loops -> 0.00015 secs - 10 loops -> 0.0003 secs - 20 loops -> 0.0006 secs - 50 loops -> 0.0015 secs - 100 loops -> 0.003 secs - 200 loops -> 0.006 secs - 500 loops -> 0.015 secs - 1000 loops -> 0.03 secs - 2000 loops -> 0.06 secs - 5000 loops -> 0.15 secs - 10000 loops -> 0.3 secs + 1 loop -> 3e-05 sec + 2 loops -> 6e-05 sec + 5 loops -> 0.00015 sec + 10 loops -> 0.0003 sec + 20 loops -> 0.0006 sec + 50 loops -> 0.0015 sec + 100 loops -> 0.003 sec + 200 loops -> 0.006 sec + 500 loops -> 0.015 sec + 1000 loops -> 0.03 sec + 2000 loops -> 0.06 sec + 5000 loops -> 0.15 sec + 10000 loops -> 0.3 sec raw times: 300 msec, 300 msec, 300 msec, 300 msec, 300 msec diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index 9e677a337a2..e6c9155d1a9 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -113,3 +113,16 @@ def test_inline_table_recursion_limit(self): nest_count=nest_count): recursive_table_toml = nest_count * "key = {" + nest_count * "}" tomllib.loads(recursive_table_toml) + + def test_key_recursion_limit(self): + nest_count = tomllib._parser.MAX_KEY_PARTS - 2 + nested_key_toml = "a." * nest_count + "a = 1" + tomllib.loads(nested_key_toml) + + nest_count = tomllib._parser.MAX_KEY_PARTS + 2 + nested_key_toml = "a." * nest_count + "a = 1" + with self.assertRaisesRegex( + RecursionError, + r"TOML key has more than the allowed [0-9]+ parts", + ): + tomllib.loads(nested_key_toml) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 8e2bb0c2382..a6a6e8cbdc1 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,4 +1,5 @@ """ Tests for the internal type cache in CPython. """ +import collections.abc import unittest import dis from test import support @@ -108,6 +109,25 @@ class HolderSub(Holder): Holder.set_value() HolderSub.value + def test_abc_register_invalidates_subclass_versions(self): + class Parent: + pass + + class Child(Parent): + pass + + type_assign_version(Parent) + type_assign_version(Child) + parent_version = type_get_version(Parent) + child_version = type_get_version(Child) + if parent_version == 0 or child_version == 0: + self.skipTest("Could not assign valid type versions") + + collections.abc.Mapping.register(Parent) + + self.assertEqual(type_get_version(Parent), 0) + self.assertEqual(type_get_version(Child), 0) + @support.cpython_only @requires_specialization class TypeCacheWithSpecializationTests(unittest.TestCase): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index abe78c38439..b3ef0ff4af9 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5583,6 +5583,27 @@ def foo(x: T): foo(42) + def test_genericalias_instance_isclass(self): + # test against user-defined generic classes + T = TypeVar('T') + + class Node(Generic[T]): + def __init__(self, label: T, + left: 'Node[T] | None' = None, + right: 'Node[T] | None' = None): + self.label = label + self.left = left + self.right = right + + self.assertTrue(inspect.isclass(Node)) + self.assertFalse(inspect.isclass(Node[int])) + self.assertFalse(inspect.isclass(Node[str])) + + # test against standard generic classes + self.assertFalse(inspect.isclass(set[int])) + self.assertFalse(inspect.isclass(list[bytes])) + self.assertFalse(inspect.isclass(dict[str, str])) + def test_implicit_any(self): T = TypeVar('T') @@ -6979,6 +7000,24 @@ def test_get_type_hints_wrapped_decoratored_func(self): self.assertEqual(gth(ForRefExample.func), expects) self.assertEqual(gth(ForRefExample.nested), expects) + def test_get_type_hints_wrapped_cycle_self(self): + # gh-146553: __wrapped__ self-reference must raise ValueError, + # not loop forever. + def f(x: int) -> str: ... + f.__wrapped__ = f + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(f) + + def test_get_type_hints_wrapped_cycle_mutual(self): + # gh-146553: mutual __wrapped__ cycle (a -> b -> a) must raise + # ValueError, not loop forever. + def a(): ... + def b(): ... + a.__wrapped__ = b + b.__wrapped__ = a + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(a) + def test_get_type_hints_annotated(self): def foobar(x: List['X']): ... X = Annotated[int, (1, 10)] @@ -10552,6 +10591,10 @@ def test_no_attributes(self): with self.assertRaises(AttributeError): type(NoDefault).foo + def test_no_subclassing(self): + with self.assertRaises(TypeError): + class Test(type(NoDefault)): ... + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 3eeb6694da4..86995b62e3a 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -562,6 +562,34 @@ def test_issue10254(self): b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079: avoid quadratic canonical ordering. + payload = "a" + ("\u0300\u0327" * 32) + nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32) + nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300" + short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10) + short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9) + long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30) + long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' @@ -664,6 +692,22 @@ def test_failed_import_during_compiling(self): "(can't load unicodedata module)" self.assertIn(error, result.err.decode("ascii")) + def test_unicodedata_unload_reload(self): + # gh-149449: dropping unicodedata and running gc must not leave the + # cached _ucnhash_CAPI pointer dangling. + code = ( + "import gc, sys\n" + "assert '\\N{GRINNING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '', 'exec')\n" + "del sys.modules['unicodedata']\n" + "gc.collect()\n" + "assert '\\N{WINKING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '', 'exec')\n" + ) + script_helper.assert_python_ok("-c", code) + def test_decimal_numeric_consistent(self): # Test that decimal and numeric are consistent, # i.e. if a character has a decimal value, diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 56b717d3797..4b233eee4a9 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -658,6 +658,26 @@ def test_deactivate_with_strict_bash_opts(self): self.assertEqual(out, "".encode()) self.assertEqual(err, "".encode()) + # gh-149701: Test exit code is zero even when hashing is disabled + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts_and_hashing_disabled(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_hash_disabled.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + "set +h\n" # disable hashing + f"source {activate}\n" + "deactivate") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') def test_macos_env(self): diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index e8b8b020dea..135f116d46b 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -118,6 +118,15 @@ def test_open_bad_new_parameter(self): arguments=[URL], kw=dict(new=999)) + def test_reject_action_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaises(ValueError): + browser.open('%action--incognito') + # new=1: action is "--new-window", so "%action" itself expands to + # a dash-prefixed flag even with no dash in the original URL. + with self.assertRaises(ValueError): + browser.open('%action', new=1) + class EdgeCommandTest(CommandTestMixin, unittest.TestCase): diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 597ec830612..c306b7d036c 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -976,12 +976,12 @@ def check(encoding, body=''): check("cp437", '\u221a') check("mac-roman", '\u02da') - def xml(encoding): - return "" % encoding - def bxml(encoding): - return xml(encoding).encode(encoding) + def xml(encoding, body=''): + return "%s" % (encoding, body) + def bxml(encoding, body=''): + return xml(encoding, body).encode(encoding) supported_encodings = [ - 'ascii', 'utf-8', 'utf-8-sig', 'utf-16', 'utf-16be', 'utf-16le', + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', @@ -992,13 +992,14 @@ def bxml(encoding): 'cp1256', 'cp1257', 'cp1258', 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', - 'iso2022-jp', 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', - 'iso2022-jp-3', 'iso2022-jp-ext', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', - 'hz', 'ptcp154', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', ] for encoding in supported_encodings: - self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + with self.subTest(encoding=encoding): + self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + self.assertEqual(ET.tostring(ET.XML(bxml(encoding, c))), + ('&#%d;' % ord(c)).encode()) unsupported_ascii_compatible_encodings = [ 'big5', 'big5hkscs', @@ -1010,14 +1011,16 @@ def bxml(encoding): 'utf-7', ] for encoding in unsupported_ascii_compatible_encodings: - self.assertRaises(ValueError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ValueError, ET.XML, bxml(encoding)) unsupported_ascii_incompatible_encodings = [ 'cp037', 'cp424', 'cp500', 'cp864', 'cp875', 'cp1026', 'cp1140', 'utf_32', 'utf_32_be', 'utf_32_le', ] for encoding in unsupported_ascii_incompatible_encodings: - self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) @@ -1245,7 +1248,12 @@ def check(p, expected, namespaces=None): {'': 'http://www.w3.org/2001/XMLSchema', 'ns': 'http://www.w3.org/2001/XMLSchema'}) - def test_processinginstruction(self): + def test_comment_serialization(self): + comm = ET.Comment(' & ham') + # comments are not escaped + self.assertEqual(ET.tostring(comm), b'') + + def test_processinginstruction_serialization(self): # Test ProcessingInstruction directly self.assertEqual(ET.tostring(ET.ProcessingInstruction('test', 'instruction')), @@ -1254,13 +1262,22 @@ def test_processinginstruction(self): b'') # Issue #2746 - + # processing instructions are not escaped self.assertEqual(ET.tostring(ET.PI('test', '')), b'?>') self.assertEqual(ET.tostring(ET.PI('test', '\xe3'), 'latin-1'), b"\n" b"\xe3?>") + @support.subTests('tag', ("script", "style", "xmp", "iframe", "noembed", "noframes")) + def test_html_cdata_elems_serialization(self, tag): + # content of raw text elements is not escaped in html + tag = tag.title() + elem = ET.Element(tag) + elem.text = '&ham' + self.assertEqual(ET.tostring(elem, method='html'), + ('<%s>&ham' % (tag, tag)).encode()) + def test_html_empty_elems_serialization(self): # issue 15970 # from http://www.w3.org/TR/html401/index/elements.html @@ -1275,6 +1292,14 @@ def test_html_empty_elems_serialization(self): method='html') self.assertEqual(serialized, expected) + def test_html_plaintext_serialization(self): + # content of plaintext is not escaped in html + # no end tag for plaintext + elem = ET.Element('PlainText') + elem.text = '&ham' + self.assertEqual(ET.tostring(elem, method='html'), + b'<spam>&ham') + def test_dump_attribute_order(self): # See BPO 34160 e = ET.Element('cirriculum', status='public', company='example') @@ -3099,6 +3124,16 @@ def __deepcopy__(self, memo): self.assertEqual([c.tag for c in children[3:]], [a.tag, b.tag, a.tag, b.tag]) + def test_deeply_nested_deepcopy(self): + # This should raise a RecursionError and not crash. + # See https://github.com/python/cpython/issues/148801. + root = cur = ET.Element('s') + for _ in range(500_000): + cur = ET.SubElement(cur, 'u') + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + copy.deepcopy(root) + class MutationDeleteElementPath(str): def __new__(cls, elem, *args): @@ -3167,6 +3202,16 @@ def test_findtext_with_mutating(self): e.extend([ET.Element('bar')]) e.findtext(cls(e, 'x')) + def test_findtext_with_mutating_non_none_text(self): + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + child = ET.Element('bar') + child.text = str(object()) + e.append(child) + del child + repr(e.findtext(cls(e, 'x'))) + def test_findtext_with_error(self): e = ET.Element('foo') e.extend([ET.Element('bar')]) @@ -3540,6 +3585,32 @@ def test_basic(self): doc = ET.XML("<root>a&amp;<sub>b&amp;</sub>c&amp;</root>") self.assertEqual(''.join(doc.itertext()), 'a&b&c&') + def test_comment(self): + e = ET.Element('root') + e.text = 'before' + comment = ET.Comment('comment') + self.assertEqual(comment.text, 'comment') + comment.tail = 'after' + e.append(comment) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, comment]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(comment.itertext()), '') + self.assertEqual(list(comment.iter()), [comment]) + + def test_processinginstruction(self): + e = ET.Element('root') + e.text = 'before' + pi = ET.PI('test', 'instruction') + self.assertEqual(pi.text, 'test instruction') + pi.tail = 'after' + e.append(pi) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, pi]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(pi.itertext()), '') + self.assertEqual(list(pi.iter()), [pi]) + def test_corners(self): # single root, no subelements a = ET.Element('a') diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 6c4b8384a32..f046b8d08ce 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -208,6 +208,17 @@ def test_dump_encoding(self): self.assertEqual(xmlrpclib.loads(strg)[0][0], value) self.assertEqual(xmlrpclib.loads(strg)[1], methodname) + def test_dump_escape_methodname(self): + payload = 'foo</methodName><injected attr="evil"/><methodName>bar' + s = xmlrpclib.dumps((), methodname=payload) + self.assertIn( + '<methodName>foo&lt;/methodName&gt;&lt;injected attr="evil"/&gt;' + '&lt;methodName&gt;bar</methodName>', s + ) + self.assertNotIn('<injected attr="evil"/>', s) + load, m = xmlrpclib.loads(s) + self.assertEqual(m, payload) + def test_dump_bytes(self): sample = b"my dog has fleas" self.assertEqual(sample, xmlrpclib.Binary(sample)) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 6239fc5a5f7..b97d0bd06dd 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -3521,29 +3521,23 @@ def test_read_with_unsuitable_metadata_encoding(self): def test_read_after_append(self): newname = '\u56db' # Han 'four' - expected_names = [name.encode('shift_jis').decode('cp437') - for name in self.file_names[:2]] + self.file_names[2:] - expected_names.append(newname) - expected_content = (*self.file_content, b"newcontent") + newname2 = 'fünf' # representable in cp437, but still stored as UTF-8 + expected_names = [*self.file_names, newname, newname2] + mojibake_expected_names = [name.encode('shift_jis').decode('cp437') + if i < 2 else name + for i, name in enumerate(expected_names)] + expected_content = (*self.file_content, b"newcontent", b"newcontent2") with zipfile.ZipFile(TESTFN, "a") as zipfp: zipfp.writestr(newname, "newcontent") - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + zipfp.writestr(newname2, "newcontent2") + self.assertEqual(sorted(zipfp.namelist()), sorted(mojibake_expected_names)) with zipfile.ZipFile(TESTFN, "r") as zipfp: - self._test_read(zipfp, expected_names, expected_content) + self._test_read(zipfp, mojibake_expected_names, expected_content) with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) - for i, (name, content) in enumerate(zip(expected_names, expected_content)): - info = zipfp.getinfo(name) - self.assertEqual(info.filename, name) - self.assertEqual(info.file_size, len(content)) - if i < 2: - with self.assertRaises(zipfile.BadZipFile): - zipfp.read(name) - else: - self.assertEqual(zipfp.read(name), content) + self._test_read(zipfp, expected_names, expected_content) def test_write_with_metadata_encoding(self): ZF = zipfile.ZipFile @@ -3552,6 +3546,20 @@ def test_write_with_metadata_encoding(self): "^metadata_encoding is only"): ZF("nonesuch.zip", mode, metadata_encoding="shift_jis") + def test_add_comment(self): + with zipfile.ZipFile(TESTFN, "r") as zipfp: + mojibake_expected_names = zipfp.namelist() + + with zipfile.ZipFile(TESTFN, "a") as zipfp: + zipfp.comment = b'comment' + self.assertEqual(zipfp.namelist(), mojibake_expected_names) + + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, mojibake_expected_names, self.file_content) + + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self._test_read(zipfp, self.file_names, self.file_content) + def test_cli_with_metadata_encoding(self): errmsg = "Non-conforming encodings not supported with -c." args = ["--metadata-encoding=shift_jis", "-c", "nonesuch", "nonesuch"] diff --git a/Lib/timeit.py b/Lib/timeit.py index 02cfafaf36e..f5f75048be1 100755 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -322,7 +322,7 @@ def main(args=None, *, _wrap_timer=None): callback = None if verbose: def callback(number, time_taken): - msg = "{num} loop{s} -> {secs:.{prec}g} secs" + msg = "{num} loop{s} -> {secs:.{prec}g} sec" plural = (number != 1) print(msg.format(num=number, s='s' if plural else '', secs=time_taken, prec=precision)) diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 9c80a6a547d..a1f7ffd6d24 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -6,8 +6,9 @@ from collections.abc import Iterable import string +import sys from types import MappingProxyType -from typing import Any, BinaryIO, NamedTuple +from typing import Any, BinaryIO, NamedTuple, Final from ._re import ( RE_DATETIME, @@ -19,6 +20,13 @@ ) from ._types import Key, ParseFloat, Pos +# Pathologically excessive number of parts in a key runs into quadratic +# behavior (e.g. in Flags.is_). +# Even if keys aren't currently parsed using recursion, they name a +# recursive structure, so it makes sense to limit it using getrecursionlimit() +# and RecursionError. +MAX_KEY_PARTS: Final = sys.getrecursionlimit() + ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) # Neither of these sets include quotation mark or backslash. They are @@ -385,6 +393,10 @@ def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]: pos = skip_chars(src, pos, TOML_WS) pos, key_part = parse_key_part(src, pos) key += (key_part,) + if len(key) > MAX_KEY_PARTS: + raise RecursionError( + f"TOML key has more than the allowed {MAX_KEY_PARTS} parts" + ) pos = skip_chars(src, pos, TOML_WS) diff --git a/Lib/tomllib/mypy.ini b/Lib/tomllib/mypy.ini index 1761dce4556..f7eeffd575c 100644 --- a/Lib/tomllib/mypy.ini +++ b/Lib/tomllib/mypy.ini @@ -12,6 +12,4 @@ pretty = True # Enable most stricter settings enable_error_code = ignore-without-code strict = True -strict_bytes = True -local_partial_types = True warn_unreachable = True diff --git a/Lib/traceback.py b/Lib/traceback.py index b412954bd53..0acb75bbab5 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -70,10 +70,10 @@ def extract_tb(tb, limit=None): This is useful for alternate formatting of stack traces. If 'limit' is omitted or None, all entries are extracted. A pre-processed stack trace entry is a FrameSummary object - containing attributes filename, lineno, name, and line - representing the information that is usually printed for a stack - trace. The line is a string with leading and trailing - whitespace stripped; if the source is not available it is None. + representing the information that is usually printed for a + stack trace. The line attribute is a string with + leading and trailing whitespace stripped; if the source is not + available the corresponding attribute is None. """ return StackSummary._extract_from_extended_frame_gen( _walk_tb_with_full_positions(tb), limit=limit) @@ -251,9 +251,8 @@ def extract_stack(f=None, limit=None): The return value has the same format as for extract_tb(). The optional 'f' and 'limit' arguments have the same meaning as for - print_stack(). Each item in the list is a quadruple (filename, - line number, function name, text), and the entries are in order - from oldest to newest stack frame. + print_stack(). Each item in the list is a FrameSummary object, + and the entries are in order from oldest to newest stack frame. """ if f is None: f = sys._getframe().f_back @@ -281,7 +280,7 @@ class FrameSummary: active when the frame was captured. - :attr:`name` The name of the function or method that was executing when the frame was captured. - - :attr:`line` The text from the linecache module for the + - :attr:`line` The text from the linecache module for the line of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. @@ -951,7 +950,7 @@ def setup_positions(expr, force_valid=True): _WIDE_CHAR_SPECIFIERS = "WF" def _display_width(line, offset=None): - """Calculate the extra amount of width space the given source + """Calculate the amount of width space the given source code segment might take if it were to be displayed on a fixed width output device. Supports wide unicode characters and emojis.""" @@ -1037,7 +1036,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): - # NB: we need to accept exc_traceback, exc_value, exc_traceback to + # NB: we need to accept exc_type, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. # Handle loops in __cause__ or __context__. diff --git a/Lib/types.py b/Lib/types.py index ff474c14144..84622a5b4df 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -176,18 +176,19 @@ class Baz(list[str]): ... class DynamicClassAttribute: """Route attribute access on a class to __getattr__. - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. - - This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name. (Enum used this between Python - versions 3.4 - 3.9 .) - - Subclass from this to use a different method of accessing virtual attributes - and still be treated properly by the inspect module. (Enum uses this since - Python 3.10 .) + This is a descriptor, used to define attributes that act differently + when accessed through an instance and through a class. Instance access + remains normal, but access to an attribute through a class will be + routed to the class's __getattr__ method; this is done by raising + AttributeError. + + This allows one to have properties active on an instance, and have + virtual attributes on the class with the same name. (Enum used this + between Python versions 3.4 - 3.9 .) + + Subclass from this to use a different method of accessing virtual + attributes and still be treated properly by the inspect module. (Enum + uses this since Python 3.10 .) """ def __init__(self, fget=None, fset=None, fdel=None, doc=None): diff --git a/Lib/typing.py b/Lib/typing.py index cbc6d90e13a..381aaf3c7d0 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -5,7 +5,7 @@ * Generic, Protocol, and internal machinery to support generic aliases. All subscripted types like X[int], Union[int, str] are generic aliases. * Various "special forms" that have unique meanings in type annotations: - NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. + Any, Never, ClassVar, Self, Concatenate, Unpack, and others. * Classes whose instances can be type arguments to generic classes and functions: TypeVar, ParamSpec, TypeVarTuple. * Public helper functions: get_type_hints, overload, cast, final, and others. @@ -598,12 +598,12 @@ def __repr__(self): class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. + - Any is assignable to every type. + - Any assumed to have all methods and attributes. + - All values are assignable to Any. Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance + static type checkers. At runtime, Any cannot be used with instance checks. """ @@ -722,7 +722,7 @@ class Starship: ClassVar accepts only types and cannot be further subscribed. - Note that ClassVar is not a class itself, and should not + Note that ClassVar is not a class itself, and cannot be used with isinstance() or issubclass(). """ item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) @@ -805,7 +805,7 @@ def _make_union(left, right): @_SpecialForm def Optional(self, parameters): - """Optional[X] is equivalent to Union[X, None].""" + """Optional[X] is equivalent to X | None.""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] @@ -848,7 +848,7 @@ def open_helper(file: str, mode: MODE) -> str: def TypeAlias(self, parameters): """Special form for marking type aliases. - Use TypeAlias to indicate that an assignment should + TypeAlias can be used to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -1410,31 +1410,35 @@ def __dir__(self): class _GenericAlias(_BaseGenericAlias, _root=True): - # The type of parameterized generics. - # - # That is, for example, `type(List[int])` is `_GenericAlias`. - # - # Objects which are instances of this class include: - # * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. - # * Note that native container types, e.g. `tuple`, `list`, use - # `types.GenericAlias` instead. - # * Parameterized classes: - # class C[T]: pass - # # C[int] is a _GenericAlias - # * `Callable` aliases, generic `Callable` aliases, and - # parameterized `Callable` aliases: - # T = TypeVar('T') - # # _CallableGenericAlias inherits from _GenericAlias. - # A = Callable[[], None] # _CallableGenericAlias - # B = Callable[[T], None] # _CallableGenericAlias - # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: - # # All _GenericAlias - # Final[int] - # ClassVar[float] - # TypeGuard[bool] - # TypeIs[range] - + """The type of parameterized generics. + + That is, for example, `type(List[int])` is `_GenericAlias`. + + Objects which are instances of this class include: + * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. + * Note that native container types, e.g. `tuple`, `list`, use + `types.GenericAlias` instead. + * Parameterized classes: + class C[T]: pass + # C[int] is a _GenericAlias + * `Callable` aliases, generic `Callable` aliases, and + parameterized `Callable` aliases: + T = TypeVar('T') + # _CallableGenericAlias inherits from _GenericAlias. + A = Callable[[], None] # _CallableGenericAlias + B = Callable[[T], None] # _CallableGenericAlias + C = B[int] # _CallableGenericAlias + * Parameterized `Final`, `ClassVar`, `TypeForm`, `TypeGuard`, and `TypeIs`: + # All _GenericAlias + Final[int] + ClassVar[float] + TypeForm[bytearray] + TypeGuard[bool] + TypeIs[range] + + Note that instances of this class are not classes (e.g by `inspect.isclass`), + even though they behave like them. + """ def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): @@ -1466,20 +1470,21 @@ def __ror__(self, left): @_tp_cache def __getitem__(self, args): - # Parameterizes an already-parameterized object. - # - # For example, we arrive here doing something like: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # T3 = TypeVar('T3') - # class A(Generic[T1]): pass - # B = A[T2] # B is a _GenericAlias - # C = B[T3] # Invokes _GenericAlias.__getitem__ - # - # We also arrive here when parameterizing a generic `Callable` alias: - # T = TypeVar('T') - # C = Callable[[T], None] - # C[int] # Invokes _GenericAlias.__getitem__ + """Parameterizes an already-parameterized object. + + For example, we arrive here doing something like: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + T3 = TypeVar('T3') + class A(Generic[T1]): pass + B = A[T2] # B is a _GenericAlias + C = B[T3] # Invokes _GenericAlias.__getitem__ + + We also arrive here when parameterizing a generic `Callable` alias: + T = TypeVar('T') + C = Callable[[T], None] + C[int] # Invokes _GenericAlias.__getitem__ + """ if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. @@ -1496,20 +1501,20 @@ def __getitem__(self, args): return r def _determine_new_args(self, args): - # Determines new __args__ for __getitem__. - # - # For example, suppose we had: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # class A(Generic[T1, T2]): pass - # T3 = TypeVar('T3') - # B = A[int, T3] - # C = B[str] - # `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. - # Unfortunately, this is harder than it looks, because if `T3` is - # anything more exotic than a plain `TypeVar`, we need to consider - # edge cases. - + """Determines new __args__ for __getitem__. + + For example, suppose we had: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + class A(Generic[T1, T2]): pass + T3 = TypeVar('T3') + B = A[int, T3] + C = B[str] + `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. + Unfortunately, this is harder than it looks, because if `T3` is + anything more exotic than a plain `TypeVar`, we need to consider + edge cases. + """ params = self.__parameters__ # In the example above, this would be {T3: str} for param in params: @@ -1881,7 +1886,7 @@ class Movie(TypedDict): def foo(**kwargs: Unpack[Movie]): ... Note that there is only some runtime checking of this operator. Not - everything the runtime allows may be accepted by static type checkers. + everything the runtime allows is accepted by static type checkers. For more information, see PEPs 646 and 692. """ @@ -2347,7 +2352,7 @@ def runtime_checkable(cls): Such protocol can be used with isinstance() and issubclass(). Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to - one trick ponies in collections.abc such as Iterable. + one-trick ponies in collections.abc such as Iterable. For example:: @@ -2418,8 +2423,8 @@ def greet(name: str) -> None: def get_type_hints(obj, globalns=None, localns=None, include_extras=False): """Return type hints for an object. - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals and recursively replaces all + This is often the same as inspect.get_annotations(obj) or obj.__annotations__, + but it handles forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -2483,8 +2488,12 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): else: nsobj = obj # Find globalns for the unwrapped object. + seen = {id(nsobj)} while hasattr(nsobj, '__wrapped__'): nsobj = nsobj.__wrapped__ + if id(nsobj) in seen: + raise ValueError(f'wrapper loop when unwrapping {obj!r}') + seen.add(id(nsobj)) globalns = getattr(nsobj, '__globals__', {}) if localns is None: localns = globalns @@ -2597,7 +2606,7 @@ def get_args(tp): def is_typeddict(tp): - """Check if an annotation is a TypedDict class. + """Check if an object is a TypedDict class. For example:: @@ -2711,10 +2720,10 @@ def _overload_dummy(*args, **kwds): def overload(func): """Decorator for overloaded functions/methods. - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. - - For example:: + In a non-stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload, followed + by an implementation. The implementation should *not* + be decorated with @overload:: @overload def utf8(value: None) -> None: ... @@ -2722,10 +2731,11 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload:: + In a stub file or in an abstract method (for example, in a Protocol definition), + the implementation may be omitted:: @overload def utf8(value: None) -> None: ... @@ -2733,8 +2743,6 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... - def utf8(value): - ... # implementation goes here The overloads for a function can be retrieved at runtime using the get_overloads() function. @@ -2770,7 +2778,7 @@ def final(f): """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated - method cannot be overridden, and decorated class cannot be subclassed. + method cannot be overridden, and the decorated class cannot be subclassed. For example:: @@ -2812,7 +2820,7 @@ class Other(Leaf): # Error reported by type checker V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. -# Internal type variable used for Type[]. +# Internal type bound to class object types. CT_co = TypeVar('CT_co', covariant=True, bound=type) @@ -2903,7 +2911,7 @@ class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - def new_user[U](user_class: Type[U]) -> U: + def new_user[U](user_class: type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user @@ -2916,7 +2924,7 @@ def new_user[U](user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): - """An ABC with one abstract method __int__.""" + """A protocol with one abstract method __int__.""" __slots__ = () @@ -2927,7 +2935,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): - """An ABC with one abstract method __float__.""" + """A protocol with one abstract method __float__.""" __slots__ = () @@ -2938,7 +2946,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): - """An ABC with one abstract method __complex__.""" + """A protocol with one abstract method __complex__.""" __slots__ = () @@ -2949,7 +2957,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): - """An ABC with one abstract method __bytes__.""" + """A protocol with one abstract method __bytes__.""" __slots__ = () @@ -2960,7 +2968,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): - """An ABC with one abstract method __index__.""" + """A protocol with one abstract method __index__.""" __slots__ = () @@ -2971,7 +2979,7 @@ def __index__(self) -> int: @runtime_checkable class SupportsAbs[T](Protocol): - """An ABC with one abstract method __abs__ that is covariant in its return type.""" + """A protocol with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @@ -2982,7 +2990,7 @@ def __abs__(self) -> T: @runtime_checkable class SupportsRound[T](Protocol): - """An ABC with one abstract method __round__ that is covariant in its return type.""" + """A protocol with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @@ -3061,7 +3069,7 @@ def __new__(cls, typename, bases, ns): def NamedTuple(typename, fields=_sentinel, /, **kwargs): - """Typed version of namedtuple. + """Typed version of collections.namedtuple. Usage:: @@ -3073,8 +3081,8 @@ class Employee(NamedTuple): Employee = collections.namedtuple('Employee', ['name', 'id']) - The resulting class has an extra __annotations__ attribute, giving a - dict that maps field names to types. (The field names are also in + The types for each field name can be retrieved by calling + inspect.get_annotations(Employee). (The field names are also in the _fields attribute, which is part of the namedtuple API.) An alternative equivalent functional syntax is also accepted:: @@ -3159,7 +3167,7 @@ def __new__(cls, name, bases, ns, total=True): This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way - TypedDict supports all three syntax forms described in its docstring. + TypedDict classes can be created through both class-based and functional syntax. Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: @@ -3272,14 +3280,22 @@ def TypedDict(typename, fields=_sentinel, /, *, total=True): >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') True - The type info can be accessed via the Point2D.__annotations__ dict, and - the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. + The type info can be accessed by calling inspect.get_annotations(Point2D), and + via the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. TypedDict supports an additional equivalent form:: Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality:: + to override this by using the NotRequired and Required special forms:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + This means that a Point2D TypedDict can have the "y" key omitted, but the "x" key must be present. + Items are required by default, so the Required special form is not necessary in this example. + In addition, the total argument to the TypedDict function can be used to make all items not required:: class Point2D(TypedDict, total=False): x: int @@ -3288,16 +3304,8 @@ class Point2D(TypedDict, total=False): This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the - class body be required. - - The Required and NotRequired special forms can also be used to mark - individual keys as being required or not required:: - - class Point2D(TypedDict): - x: int # the "x" key must always be present (Required is the default) - y: NotRequired[int] # the "y" key can be omitted - - See PEP 655 for more details on Required and NotRequired. + class body be required. The Required special form can be used to mark individual + keys as required in a total=False TypedDict. The ReadOnly special form can be used to mark individual keys as immutable for type checkers:: @@ -3306,6 +3314,7 @@ class DatabaseUser(TypedDict): id: ReadOnly[int] # the "id" key must not be modified username: str # the "username" key can be changed + See PEPs 589, 655, and 705 for more information. """ if fields is _sentinel or fields is None: import warnings @@ -3352,7 +3361,7 @@ class Movie(TypedDict, total=False): year: int m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) @@ -3374,7 +3383,7 @@ class Movie(TypedDict): year: NotRequired[int] m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) """ @@ -3394,7 +3403,7 @@ class Movie(TypedDict): def mutate_movie(m: Movie) -> None: m["year"] = 1992 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. """ @@ -3481,8 +3490,8 @@ class IO(Generic[AnyStr]): classes (text vs. binary, read vs. write vs. read/write, append-only, unbuffered). The TextIO and BinaryIO subclasses below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. + pervasive in the interface. For more precise types, define a custom + Protocol. """ __slots__ = () @@ -3572,7 +3581,7 @@ def __exit__(self, type, value, traceback) -> None: class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" + """Typed approximation of the return of open() in binary mode.""" __slots__ = () @@ -3586,7 +3595,7 @@ def __enter__(self) -> 'BinaryIO': class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" + """Typed approximation of the return of open() in text mode.""" __slots__ = () @@ -3653,7 +3662,7 @@ def dataclass_transform( field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, ) -> _IdentityCallable: - """Decorator to mark an object as providing dataclass-like behaviour. + """Decorator to mark an object as providing dataclass-like behavior. The decorator can be applied to a function, class, or metaclass. diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index b6dd1c27fee..bf32ff6dd3c 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -3102,6 +3102,10 @@ def _mock_call(self, *args, **kwargs): return ret_value + def _increment_mock_call(self, /, *args, **kwargs): + with self._mock_calls_events_lock: + super()._increment_mock_call(*args, **kwargs) + def wait_until_called(self, *, timeout=_timeout_unset): """Wait until the mock object is called. diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 63689816f30..d055fd4e41b 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -7,7 +7,7 @@ 2) PSF license for Python 2.2 The robots.txt Exclusion Protocol is implemented as specified in - http://www.robotstxt.org/norobots-rfc.txt + RFC 9309 """ import collections @@ -21,19 +21,6 @@ RequestRate = collections.namedtuple("RequestRate", "requests seconds") -def normalize(path): - unquoted = urllib.parse.unquote(path, errors='surrogateescape') - return urllib.parse.quote(unquoted, errors='surrogateescape') - -def normalize_path(path): - path, sep, query = path.partition('?') - path = normalize(path) - if sep: - query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) - path += '?' + query - return path - - class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -42,6 +29,7 @@ class RobotFileParser: def __init__(self, url=''): self.entries = [] + self.groups = {} self.sitemaps = [] self.default_entry = None self.disallow_all = False @@ -86,13 +74,13 @@ def read(self): self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) def _add_entry(self, entry): - if "*" in entry.useragents: - # the default entry is considered last - if self.default_entry is None: - # the first default entry wins - self.default_entry = entry - else: - self.entries.append(entry) + self.entries.append(entry) + for agent in entry.useragents: + agent = agent.lower() + if agent not in self.groups: + self.groups[agent] = entry + else: + self.groups[agent] = merge_entries(self.groups[agent], entry) def parse(self, lines): """Parse the input lines from a robots.txt file. @@ -100,6 +88,7 @@ def parse(self, lines): We allow that a user-agent: line is not preceded by one or more blank lines. """ + entries = [] # states: # 0: start state # 1: saw user-agent line @@ -109,14 +98,6 @@ def parse(self, lines): self.modified() for line in lines: - if not line: - if state == 1: - entry = Entry() - state = 0 - elif state == 2: - self._add_entry(entry) - entry = Entry() - state = 0 # remove optional comment and strip line i = line.find('#') if i >= 0: @@ -132,16 +113,23 @@ def parse(self, lines): if state == 2: self._add_entry(entry) entry = Entry() - entry.useragents.append(line[1]) + product_token = line[1] + entry.useragents.append(product_token) state = 1 elif line[0] == "disallow": if state != 0: - entry.rulelines.append(RuleLine(line[1], False)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], False)) + except ValueError: + pass elif line[0] == "allow": if state != 0: - entry.rulelines.append(RuleLine(line[1], True)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], True)) + except ValueError: + pass elif line[0] == "crawl-delay": if state != 0: # before trying to convert to int we need to make @@ -164,9 +152,18 @@ def parse(self, lines): # so it doesn't matter where you place it in your file." # Therefore we do not change the state of the parser. self.sitemaps.append(line[1]) - if state == 2: + if state != 0: self._add_entry(entry) + def _find_entry(self, useragent): + entry = self.groups.get(useragent.lower()) + if entry is not None: + return entry + for entry in self.groups.values(): + if entry.applies_to(useragent): + return entry + return self.groups.get('*') + def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: @@ -183,37 +180,32 @@ def can_fetch(self, useragent, url): # the first match counts parsed_url = urllib.parse.urlsplit(url) url = urllib.parse.urlunsplit(('', '', *parsed_url[2:])) - url = normalize_path(url) + url = normalize_uri(url) if not url: url = "/" - for entry in self.entries: - if entry.applies_to(useragent): - return entry.allowance(url) - # try the default entry last - if self.default_entry: - return self.default_entry.allowance(url) - # agent not found ==> access granted - return True + if url == '/robots.txt': + # The /robots.txt URI is implicitly allowed. + return True + entry = self._find_entry(useragent) + if entry is None: + return True + return entry.allowance(url) def crawl_delay(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.delay - if self.default_entry: - return self.default_entry.delay - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.delay def request_rate(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.req_rate - if self.default_entry: - return self.default_entry.req_rate - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.req_rate def site_maps(self): if not self.sitemaps: @@ -224,7 +216,7 @@ def __str__(self): entries = self.entries if self.default_entry is not None: entries = entries + [self.default_entry] - return '\n\n'.join(map(str, entries)) + return '\n\n'.join(filter(None, map(str, entries))) class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" @@ -233,14 +225,42 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True - self.path = normalize_path(path) + path = re.sub(r'[*]{2,}', '*', path) + path = re.sub(r'[$][$*]+', '$', path) + path = normalize_pattern(path) + self.fullmatch = path.endswith('$') + path = path.rstrip('$') + if '$' in path: + raise ValueError('$ not at the end of path') + self.matcher = None + if '*' in path: + pattern = re.compile(translate_pattern(path), re.DOTALL) + if self.fullmatch: + self.matcher = pattern.fullmatch + else: + self.matcher = pattern.match + self.path = path self.allowance = allowance def applies_to(self, filename): - return self.path == "*" or filename.startswith(self.path) + # If the filename matches the rule, return the matching length plus 1. + # If it does not match, return 0. + if self.matcher is not None: + m = self.matcher(filename) + if m: + return m.end() + 1 + else: + if self.fullmatch: + if filename == self.path: + return len(self.path) + 1 + else: + if filename.startswith(self.path): + return len(self.path) + 1 + return 0 def __str__(self): - return ("Allow" if self.allowance else "Disallow") + ": " + self.path + return (("Allow" if self.allowance else "Disallow") + ": " + self.path + + ('$' if self.fullmatch else '')) class Entry: @@ -252,6 +272,8 @@ def __init__(self): self.req_rate = None def __str__(self): + if not self.useragents: + return '' ret = [] for agent in self.useragents: ret.append(f"User-agent: {agent}") @@ -260,27 +282,74 @@ def __str__(self): if self.req_rate is not None: rate = self.req_rate ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") - ret.extend(map(str, self.rulelines)) + if self.rulelines: + ret.extend(map(str, self.rulelines)) + else: + ret.append("Allow:") return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" + if useragent is None: + return '*' in self.useragents # split the name token and make it lower case useragent = useragent.split("/")[0].lower() for agent in self.useragents: - if agent == '*': - # we have the catch-all agent - return True - agent = agent.lower() - if agent in useragent: - return True + if agent != '*': + agent = agent.lower() + if agent in useragent: + return True return False def allowance(self, filename): """Preconditions: - our agent applies to this entry - - filename is URL encoded""" + - filename is URL encoded + """ + best_match = -1 + allowance = True for line in self.rulelines: - if line.applies_to(filename): - return line.allowance - return True + m = line.applies_to(filename) + if m: + if m > best_match: + best_match = m + allowance = line.allowance + elif m == best_match and not allowance: + allowance = line.allowance + return allowance + + +def normalize(path): + unquoted = urllib.parse.unquote(path, errors='surrogateescape') + return urllib.parse.quote(unquoted, errors='surrogateescape') + +def normalize_uri(path): + path, sep, query = path.partition('?') + path = normalize(path) + if sep: + query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def normalize_pattern(path): + path, sep, query = path.partition('?') + path = re.sub(r'[^*$]+', lambda m: normalize(m[0]), path) + if sep: + query = re.sub(r'[^=&*$]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def translate_pattern(path): + parts = list(map(re.escape, path.split('*'))) + for i in range(1, len(parts)-1): + parts[i] = f'(?>.*?{parts[i]})' + parts[-1] = f'.*{parts[-1]}' + return ''.join(parts) + +def merge_entries(e1, e2): + entry = Entry() + entry.useragents = list(filter(set(e2.useragents).__contains__, e1.useragents)) + entry.rulelines = e1.rulelines + e2.rulelines + entry.delay = e1.delay if e2.delay is None else e2.delay + entry.req_rate = e1.req_rate if e2.req_rate is None else e2.req_rate + return entry diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 70673a265d4..241a8650bda 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -17,7 +17,7 @@ deactivate () { # Call hash to forget past locations. Without forgetting # past locations the $PATH changes we made may not be respected. # See "man bash" for more details. hash is usually a builtin of your shell - hash -r 2> /dev/null + hash -r 2> /dev/null || true if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -73,4 +73,4 @@ fi # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -hash -r 2> /dev/null +hash -r 2> /dev/null || true diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index ee5824100c7..be033d707a9 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -275,7 +275,6 @@ def _invoke(self, args, remote, autoraise, url=None): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) - self._check_url(url) if new == 0: action = self.remote_action elif new == 1: @@ -289,7 +288,9 @@ def open(self, url, new=0, autoraise=True): raise Error("Bad 'new' parameter to open(); " f"expected 0, 1, or 2, got {new}") - args = [arg.replace("%s", url).replace("%action", action) + self._check_url(url.replace("%action", action)) + + args = [arg.replace("%action", action).replace("%s", url) for arg in self.remote_args] args = [arg for arg in args if arg] success = self._invoke(args, True, autoraise, url) diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 9bb09ab5407..ef162a967d8 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -8,8 +8,8 @@ 2. Element represents a single node in this tree. Interactions with the whole document (reading and writing to/from files) are - usually done on the ElementTree level. Interactions with a single XML element - and its sub-elements are done on the Element level. + usually done on the ElementTree level. Interactions with a single XML + element and its sub-elements are done on the Element level. Element is a flexible container object designed to store hierarchical data structures in memory. It can be described as a cross between a list and a @@ -273,7 +273,8 @@ def find(self, path, namespaces=None): """Find first matching element by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -285,7 +286,8 @@ def findtext(self, path, default=None, namespaces=None): *path* is a string having either an element tag or an XPath, *default* is the value to return if the element was not found, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return text content of first matching element, or default value if none was found. Note that if an element is found having no text @@ -298,7 +300,8 @@ def findall(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Returns list containing all matching elements in document order. @@ -309,7 +312,8 @@ def iterfind(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return an iterable yielding all matching elements in document order. @@ -549,8 +553,8 @@ def _setroot(self, element): def parse(self, source, parser=None): """Load external XML document into element tree. - *source* is a file name or file object, *parser* is an optional parser - instance that defaults to XMLParser. + *source* is a file name or file object, *parser* is an optional + parser instance that defaults to XMLParser. ParseError is raised if the parser fails to parse the document. @@ -583,7 +587,8 @@ def parse(self, source, parser=None): def iter(self, tag=None): """Create and return tree iterator for the root element. - The iterator loops over all elements in this tree, in document order. + The iterator loops over all elements in this tree, in document + order. *tag* is a string with the tag name to iterate over (default is to return all elements). @@ -598,7 +603,8 @@ def find(self, path, namespaces=None): Same as getroot().find(path), which is Element.find() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -620,7 +626,8 @@ def findtext(self, path, default=None, namespaces=None): Same as getroot().findtext(path), which is Element.findtext() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -642,7 +649,8 @@ def findall(self, path, namespaces=None): Same as getroot().findall(path), which is Element.findall(). *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return list containing all matching elements in document order. @@ -689,24 +697,26 @@ def write(self, file_or_filename, """Write element tree to a file as XML. Arguments: - *file_or_filename* -- file name or a file object opened for writing + *file_or_filename* -- file name or a file object opened for + writing *encoding* -- the output encoding (default: US-ASCII) - *xml_declaration* -- bool indicating if an XML declaration should be - added to the output. If None, an XML declaration - is added if encoding IS NOT either of: - US-ASCII, UTF-8, or Unicode + *xml_declaration* -- bool indicating if an XML declaration should + be added to the output. If None, an XML + declaration is added if encoding IS NOT + either of: US-ASCII, UTF-8, or Unicode - *default_namespace* -- sets the default XML namespace (for "xmlns") + *default_namespace* -- sets the default XML namespace (for + "xmlns") *method* -- either "xml" (default), "html, "text", or "c14n" *short_empty_elements* -- controls the formatting of elements - that contain no content. If True (default) - they are emitted as a single self-closed - tag, otherwise they are emitted as a pair - of start/end tags + that contain no content. If True + (default) they are emitted as a single + self-closed tag, otherwise they are + emitted as a pair of start/end tags """ if self._root is None: @@ -903,9 +913,12 @@ def _serialize_xml(write, elem, qnames, namespaces, if elem.tail: write(_escape_cdata(elem.tail)) +_CDATA_CONTENT_ELEMENTS = {"script", "style", "xmp", "iframe", "noembed", + "noframes", "plaintext"} + HTML_EMPTY = {"area", "base", "basefont", "br", "col", "embed", "frame", "hr", "img", "input", "isindex", "link", "meta", "param", "source", - "track", "wbr"} + "track", "wbr", "plaintext"} def _serialize_html(write, elem, qnames, namespaces, **kwargs): tag = elem.tag @@ -946,7 +959,7 @@ def _serialize_html(write, elem, qnames, namespaces, **kwargs): write(">") ltag = tag.lower() if text: - if ltag == "script" or ltag == "style": + if ltag in _CDATA_CONTENT_ELEMENTS: write(text) else: write(_escape_cdata(text)) @@ -1079,9 +1092,9 @@ def tostring(element, encoding=None, method=None, *, is returned. Otherwise a bytestring is returned. *element* is an Element instance, *encoding* is an optional output - encoding defaulting to US-ASCII, *method* is an optional output which can - be one of "xml" (default), "html", "text" or "c14n", *default_namespace* - sets the default XML namespace (for "xmlns"). + encoding defaulting to US-ASCII, *method* is an optional output which + can be one of "xml" (default), "html", "text" or "c14n", + *default_namespace* sets the default XML namespace (for "xmlns"). Returns an (optionally) encoded string containing the XML data. @@ -1221,7 +1234,8 @@ def iterparse(source, events=None, parser=None): "end" events are reported. *source* is a filename or file object containing XML data, *events* is - a list of events to report back, *parser* is an optional parser instance. + a list of events to report back, *parser* is an optional parser + instance. Returns an iterator providing (event, elem) pairs. @@ -1753,10 +1767,11 @@ def flush(self): def canonicalize(xml_data=None, *, out=None, from_file=None, **options): """Convert XML to its C14N 2.0 serialised form. - If *out* is provided, it must be a file or file-like object that receives - the serialised canonical XML output (text, not bytes) through its ``.write()`` - method. To write to a file, open it in text mode with encoding "utf-8". - If *out* is not provided, this function returns the output as text string. + If *out* is provided, it must be a file or file-like object that + receives the serialised canonical XML output (text, not bytes) through + its ``.write()`` method. To write to a file, open it in text mode with + encoding "utf-8". If *out* is not provided, this function returns the + output as text string. Either *xml_data* (an XML string) or *from_file* (a file path or file-like object) must be provided as input. @@ -1790,19 +1805,22 @@ class C14NWriterTarget: Serialises parse events to XML C14N 2.0. The *write* function is used for writing out the resulting data stream - as text (not bytes). To write to a file, open it in text mode with encoding - "utf-8" and pass its ``.write`` method. + as text (not bytes). To write to a file, open it in text mode with + encoding "utf-8" and pass its ``.write`` method. Configuration options: - *with_comments*: set to true to include comments - - *strip_text*: set to true to strip whitespace before and after text content - - *rewrite_prefixes*: set to true to replace namespace prefixes by "n{number}" + - *strip_text*: set to true to strip whitespace before and after text + content + - *rewrite_prefixes*: set to true to replace namespace prefixes by + "n{number}" - *qname_aware_tags*: a set of qname aware tag names in which prefixes should be replaced in text content - - *qname_aware_attrs*: a set of qname aware attribute names in which prefixes - should be replaced in text content - - *exclude_attrs*: a set of attribute names that should not be serialised + - *qname_aware_attrs*: a set of qname aware attribute names in which + prefixes should be replaced in text content + - *exclude_attrs*: a set of attribute names that should not be + serialised - *exclude_tags*: a set of tag names that should not be serialised """ def __init__(self, write, *, diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index f441376d09c..84e4e4d11a7 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -965,7 +965,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None, data = ( xmlheader, "<methodCall>\n" - "<methodName>", methodname, "</methodName>\n", + "<methodName>", escape(methodname), "</methodName>\n", data, "</methodCall>\n" ) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 3d889e9c4f1..37555d32d99 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -531,8 +531,12 @@ def FileHeader(self, zip64=None): return header + filename + extra def _encodeFilenameFlags(self): + if self.flag_bits & _MASK_UTF_FILENAME: + encoding = 'ascii' + else: + encoding = 'cp437' try: - return self.filename.encode('ascii'), self.flag_bits + return self.filename.encode(encoding), self.flag_bits & ~_MASK_UTF_FILENAME except UnicodeEncodeError: return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME @@ -585,11 +589,12 @@ def _decodeExtra(self, filename_crc): def from_file(cls, filename, arcname=None, *, strict_timestamps=True): """Construct an appropriate ZipInfo for a file on the filesystem. - filename should be the path to a file or directory on the filesystem. + filename should be the path to a file or directory on the + filesystem. - arcname is the name which it will have within the archive (by default, - this will be the same as filename, but without a drive letter and with - leading path separators removed). + arcname is the name which it will have within the archive (by + default, this will be the same as filename, but without a drive + letter and with leading path separators removed). """ if isinstance(filename, os.PathLike): filename = os.fspath(filename) @@ -1326,25 +1331,26 @@ class ZipFile: mode: The mode can be either read 'r', write 'w', exclusive create 'x', or append 'a'. compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), - ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - compresslevel: None (default for the given compression type) or an integer - specifying the level to pass to the compressor. - When using ZIP_STORED or ZIP_LZMA this keyword has no effect. - When using ZIP_DEFLATED integers 0 through 9 are accepted. - When using ZIP_BZIP2 integers 1 through 9 are accepted. + ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). + allowZip64: if True ZipFile will create files with ZIP64 extensions + when needed, otherwise it will raise an exception when this + would be necessary. + compresslevel: None (default for the given compression type) or + an integer specifying the level to pass to the compressor. + When using ZIP_STORED or ZIP_LZMA this keyword has no effect. + When using ZIP_DEFLATED integers 0 through 9 are accepted. + When using ZIP_BZIP2 integers 1 through 9 are accepted. """ fp = None # Set here since __del__ checks it _windows_illegal_name_trans_table = None + _ignore_invalid_names = False def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): - """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', - or append 'a'.""" + """Open the ZIP file with mode read 'r', write 'w', exclusive create + 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") @@ -1626,10 +1632,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): pwd is the password to decrypt files (only used for reading). - When writing, if the file size is not known in advance but may exceed - 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large - files. If the size is known in advance, it is best to pass a ZipInfo - instance for name, with zinfo.file_size set. + When writing, if the file size is not known in advance but may + exceed 2 GiB, pass force_zip64 to use the ZIP64 format, which can + handle large files. If the size is known in advance, it is best to + pass a ZipInfo instance for name, with zinfo.file_size set. """ if mode not in {"r", "w"}: raise ValueError('open() requires mode "r" or "w"') @@ -1741,7 +1747,7 @@ def _open_to_write(self, zinfo, force_zip64=False): zinfo.compress_size = 0 zinfo.CRC = 0 - zinfo.flag_bits = 0x00 + zinfo.flag_bits = _MASK_UTF_FILENAME if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 @@ -1824,21 +1830,31 @@ def _extract_member(self, member, targetpath, pwd): # build the destination pathname, replacing # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: + arcname = member.filename + if os.path.sep != '/': + arcname = arcname.replace('/', os.path.sep) + if os.path.altsep and os.path.altsep != '/': arcname = arcname.replace(os.path.altsep, os.path.sep) # interpret absolute pathname as relative, remove drive letter or # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] + drive, root, arcname = os.path.splitroot(arcname) + if self._ignore_invalid_names and (drive or root): + return None + if self._ignore_invalid_names and os.path.pardir in arcname.split(os.path.sep): + return None invalid_path_parts = ('', os.path.curdir, os.path.pardir) arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) if x not in invalid_path_parts) if os.path.sep == '\\': # filter illegal characters on Windows - arcname = self._sanitize_windows_name(arcname, os.path.sep) + arcname2 = self._sanitize_windows_name(arcname, os.path.sep) + if self._ignore_invalid_names and arcname2 != arcname: + return None + arcname = arcname2 if not arcname and not member.is_dir(): + if self._ignore_invalid_names: + return None raise ValueError("Empty filename.") targetpath = os.path.join(targetpath, arcname) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index e7732981d38..7018a3d1e5f 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.19", - url="https://github.com/openssl/openssl/releases/download/openssl-3.0.19/openssl-3.0.19.tar.gz", - checksum='fa5a4143b8aae18be53ef2f3caf29a2e0747430b8bc74d32d88335b94ab63072', + name="OpenSSL 3.0.21", + url="https://github.com/openssl/openssl/releases/download/openssl-3.0.21/openssl-3.0.21.tar.gz", + checksum="617e29af8e421f46649484a4937e48c685e47f46488167c982f88bc4ec1d522f", buildrecipe=build_universal_openssl, configure=None, install=None, @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.17' - tcl_checksum='a3903371efcce8a405c5c245d029e9f6850258a60fa3761c4d58995610949b31' + tcl_tk_ver='8.6.18' + tcl_checksum='14f9af32b1767ff718477a8f974ad03c34341097e6b43f4ce54644ee974e268e' - tk_checksum='e4982df6f969c08bf9dd858a6891059b4a3f50dc6c87c10abadbbe2fc4838946' + tk_checksum='95cd528a80f5e4bdb557af9b14a7197d6860793a3894e25e7c9fad2ed05d4c3c' tk_patches = [] @@ -378,9 +378,9 @@ def library_recipes(): install=f"make && ranlib libsqlite3.a && make install DESTDIR={shellQuote(os.path.join(WORKDIR, 'libraries'))}", ), dict( - name="libmpdec 4.0.0", - url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", - checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + name="libmpdec 4.0.1", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.1.tar.gz", + checksum="96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", configure_pre=[ "--disable-cxx", "MACHINE=universal", diff --git a/Makefile.pre.in b/Makefile.pre.in index ecf77bdc41c..8589a28b726 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -714,7 +714,7 @@ check-clean-src: echo "Building Python out of the source tree (in $(abs_builddir)) requires a clean source tree ($(abs_srcdir))" ; \ echo "Build artifacts such as .o files, executables, and Python/frozen_modules/*.h must not exist within $(srcdir)." ; \ echo "Try to run:" ; \ - echo " (cd \"$(srcdir)\" && make clean || git clean -fdx -e Doc/venv)" ; \ + echo " (cd \"$(srcdir)\" && make distclean || git clean -fdx -e Doc/venv)" ; \ exit 1; \ fi @@ -955,7 +955,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION) $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ -all_load $(LIBRARY) \ - -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ + -install_name $(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ -compatibility_version $(VERSION) \ -current_version $(VERSION) \ -framework CoreFoundation $(LIBS); diff --git a/Misc/Brewfile b/Misc/Brewfile new file mode 100644 index 00000000000..37e3acde4a0 --- /dev/null +++ b/Misc/Brewfile @@ -0,0 +1,14 @@ +brew "gdbm" +brew "mpdecimal" +brew "openssl@3.0" +brew "pkg-config" +brew "tcl-tk@8" +brew "xz" +brew "zstd" + +brew "bzip2" if OS.linux? +brew "libedit" if OS.linux? +brew "libffi" if OS.linux? +brew "ncurses" if OS.linux? +brew "unzip" if OS.linux? +brew "zlib-ng-compat" if OS.linux? diff --git a/Misc/NEWS.d/3.13.10.rst b/Misc/NEWS.d/3.13.10.rst index ed835187893..beb68386330 100644 --- a/Misc/NEWS.d/3.13.10.rst +++ b/Misc/NEWS.d/3.13.10.rst @@ -500,9 +500,9 @@ fix: paste zero-width in default repl width is wrong. Add :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerActivationThreshold` and :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerMaximumAmplification` -to :ref:`xmlparser <xmlparser-objects>` objects to prevent use of -disproportional amounts of dynamic memory from within an Expat parser. Patch -by Bénédikt Tran. +to :ref:`xmlparser <xmlparser-objects>` objects to tune protections against +disproportional amounts of dynamic memory usage from within an Expat parser. +Patch by Bénédikt Tran. .. diff --git a/Misc/NEWS.d/3.13.14.rst b/Misc/NEWS.d/3.13.14.rst new file mode 100644 index 00000000000..14d4a624f14 --- /dev/null +++ b/Misc/NEWS.d/3.13.14.rst @@ -0,0 +1,1039 @@ +.. date: 2026-06-09-01-27-48 +.. gh-issue: 124111 +.. nonce: MDDDD6 +.. release date: 2026-06-10 +.. section: macOS + +Update macOS installer to use Tcl/Tk 8.6.18. + +.. + +.. date: 2026-05-31-10-40-00 +.. gh-issue: 150644 +.. nonce: zLWyjj +.. section: macOS + +When system logging is enabled (with ``config.use_system_logger``, messages +are now tagged as public. This allows the macOS 26 system logger to view +messages without special configuration. + +.. + +.. date: 2025-10-14-00-17-48 +.. gh-issue: 115119 +.. nonce: 470I1N +.. section: macOS + +Update macOS installer to use libmpdecimal 4.0.1. + +.. + +.. date: 2026-06-09-12-04-21 +.. gh-issue: 151159 +.. nonce: O2NVrd +.. section: Windows + +Updated bundled version of OpenSSL to 3.0.21. + +.. + +.. date: 2026-06-09-11-40-17 +.. gh-issue: 151159 +.. nonce: 9si8Fo +.. section: Windows + +Update macOS installer to use OpenSSL 3.0.21. + +.. + +.. date: 2026-06-09-11-52-52 +.. gh-issue: 151130 +.. nonce: 1vslPH +.. section: Tests + +Add more tests for ``PyWeakref_*`` C API. + +.. + +.. date: 2026-05-13-14-53-23 +.. gh-issue: 149776 +.. nonce: orqgsn +.. section: Tests + +Fix test_socket on Linux kernel 7.1 and newer: skip UDP Lite tests if it's +not supported. Patch by Victor Stinner. + +.. + +.. date: 2026-06-09-12-27-17 +.. gh-issue: 151159 +.. nonce: ng1cPU +.. section: Security + +Bumps the OpenSSL version to 3.0.21 on Android. + +.. + +.. date: 2026-05-30-09-36-20 +.. gh-issue: 150599 +.. nonce: nlHqU- +.. section: Security + +Fix a possible stack buffer overflow in :mod:`bz2` when a +:class:`bz2.BZ2Decompressor` is reused after a decompression error. The +decompressor now becomes unusable after libbz2 reports an error. + +.. + +.. date: 2026-05-18-17-46-00 +.. gh-issue: 149835 +.. nonce: EebFlk +.. section: Security + +:func:`shutil.move` now resolves symlinks via :func:`os.path.realpath` when +checking whether the destination is inside the source directory, preventing +a symlink-based bypass of that guard. + +.. + +.. date: 2026-05-11-21-15-07 +.. gh-issue: 149698 +.. nonce: OudOcW +.. section: Security + +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.1 +for the fix for :cve:`2026-45186`. + +.. + +.. date: 2026-05-10-18-05-32 +.. gh-issue: 87451 +.. nonce: XkKB6M +.. section: Security + +The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts +the IPv4 address value returned from the source server in response to the +``PASV`` command by default, completing the fix for CVE-2021-4189. As with +:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the +``trust_server_pasv_ipv4_address`` attribute on the source +:class:`ftplib.FTP` instance to ``True``. Thanks to Qi Deng at Aurascape AI +for the report. + +.. + +.. date: 2026-05-03-21-00-00 +.. gh-issue: 149486 +.. nonce: tarflt +.. section: Security + +:func:`tarfile.data_filter` now validates link targets using the same +normalised value that is written to disk, strips trailing separators from +the member name when resolving a symlink's directory, and rejects link +members that would replace the destination directory itself. This closes +several path-traversal bypasses of the ``data`` extraction filter. + +.. + +.. date: 2026-04-27-16-36-11 +.. gh-issue: 149079 +.. nonce: vKl-LM +.. section: Security + +Fix a potential denial of service in :func:`unicodedata.normalize`. The +canonical ordering step of Unicode normalization used a quadratic-time +insertion sort for reordering combining characters, which could be exploited +with crafted input containing many combining characters in non-canonical +order. Replaced with a linear-time counting sort for long runs. + +.. + +.. date: 2026-04-26-19-30-45 +.. gh-issue: 149018 +.. nonce: a9SqWb +.. section: Security + +Improved protection against XML hash-flooding attacks in +:mod:`xml.parsers.expat` and :mod:`xml.etree.ElementTree` when Python is +compiled with libExpat 2.8.0 or later. + +.. + +.. date: 2026-04-26-17-49-58 +.. gh-issue: 149017 +.. nonce: EiVFPo +.. section: Security + +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.0. + +.. + +.. date: 2026-04-21-13-46-30 +.. gh-issue: 90309 +.. nonce: srvj9q +.. section: Security + +Base64-encode values when embedding cookies to JavaScript using the +:meth:`http.cookies.BaseCookie.js_output` method to avoid injection and +escaping. + +.. + +.. date: 2026-04-20-15-31-37 +.. gh-issue: 148808 +.. nonce: _Z8JL0 +.. section: Security + +Added buffer boundary check when using ``nbytes`` parameter with +:meth:`!asyncio.AbstractEventLoop.sock_recvfrom_into`. Only relevant for +Windows and the :class:`asyncio.ProactorEventLoop`. + +.. + +.. date: 2026-04-10-16-28-21 +.. gh-issue: 148395 +.. nonce: kfzm0G +.. section: Security + +Fix a dangling input pointer in :class:`lzma.LZMADecompressor`, +:class:`bz2.BZ2Decompressor`, and internal :class:`!zlib._ZlibDecompressor` +when memory allocation fails with :exc:`MemoryError`, which could let a +subsequent :meth:`!decompress` call read or write through a stale pointer to +the already-released caller buffer. + +.. + +.. date: 2026-03-31-09-15-51 +.. gh-issue: 148169 +.. nonce: EZJzz2 +.. section: Security + +A bypass in :mod:`webbrowser` allowed URLs prefixed with ``%action`` to pass +the dash-prefix safety check. + +.. + +.. date: 2026-03-29-12-51-33 +.. gh-issue: 146581 +.. nonce: 4vZfB0 +.. section: Security + +Fix vulnerability in :func:`shutil.unpack_archive` for ZIP files on Windows +which allowed to write files outside of the destination tree if the patch in +the archive contains a Windows drive prefix. Now such invalid paths will be +skipped. Files containing ".." in the name (like "foo..bar") are no longer +skipped. + +.. + +.. date: 2026-03-25-00-51-03 +.. gh-issue: 146333 +.. nonce: LqdL__bn +.. section: Security + +Fix quadratic backtracking in :class:`configparser.RawConfigParser` option +parsing regexes (``OPTCRE`` and ``OPTCRE_NV``). A crafted configuration line +with many whitespace characters could cause excessive CPU usage. + +.. + +.. date: 2026-03-20-09-29-42 +.. gh-issue: 146211 +.. nonce: PQVbs7 +.. section: Security + +Reject CR/LF characters in tunnel request headers for the +HTTPConnection.set_tunnel() method. + +.. + +.. date: 2026-06-04-21-49-18 +.. gh-issue: 150913 +.. nonce: EmptyBl +.. section: Library + +Fix :class:`sqlite3.Blob` slice assignment to raise :exc:`TypeError` and +:exc:`IndexError` for type and size mismatches respectively, even when the +target slice is empty. + +.. + +.. date: 2026-06-04-18-22-56 +.. gh-issue: 143008 +.. nonce: z5tw-J +.. section: Library + +Fix race conditions when re-initializing a :class:`io.TextIOWrapper` object. + +.. + +.. date: 2026-05-31-17-47-30 +.. gh-issue: 150685 +.. nonce: EBB2mU +.. section: Library + +Update bundled pip to 26.1.2 + +.. + +.. date: 2026-05-25-17-00-00 +.. gh-issue: 150406 +.. nonce: jF3g63 +.. section: Library + +Fix a possible crash occurring during :mod:`socket` module initialization +when the system is out of memory on platforms without a reentrant +``gethostbyname``. + +.. + +.. date: 2026-05-25-07-22-05 +.. gh-issue: 150372 +.. nonce: 9hLqhe +.. section: Library + +:mod:`readline`: Fix a potential crash during tab completion caused by an +out-of-memory error during module initialization. + +.. + +.. date: 2026-05-21-11-25-58 +.. gh-issue: 150175 +.. nonce: 8H4Caz +.. section: Library + +Fix race condition in :class:`unittest.mock.ThreadingMock` where concurrent +calls could lose increments to ``call_count`` and other attributes due to a +missing lock in ``_increment_mock_call``. + +.. + +.. date: 2026-05-19-19-00-49 +.. gh-issue: 84353 +.. nonce: ZU5zaQ +.. section: Library + +Preserve non-UTF-8 encoded filenames when appending to a +:class:`zipfile.ZipFile`. Previously, non-ASCII names stored in a legacy +encoding (without the UTF-8 flag bit set) could be corrupted when the +central directory was rewritten: they were decoded as cp437 and then +re-stored as UTF-8. + +.. + +.. date: 2026-05-18-07-44-46 +.. gh-issue: 149995 +.. nonce: vvtFHn +.. section: Library + +Update various docstrings in :mod:`typing`. + +.. + +.. date: 2026-05-17-22-37-02 +.. gh-issue: 88726 +.. nonce: BAoL6j +.. section: Library + +The :mod:`email` package now uses standard MIME charset names "gb2312" and +"big5" instead of non-standard names "eucgb2312_cn" and "big5_tw". + +.. + +.. date: 2026-05-17-02-25-56 +.. gh-issue: 149571 +.. nonce: LNyuWJ +.. section: Library + +Fix the C implementation of :meth:`xml.etree.ElementTree.Element.itertext`: +it no longer emits text for comments and processing instructions. + +.. + +.. date: 2026-05-16-21-08-33 +.. gh-issue: 149921 +.. nonce: I1yNML +.. section: Library + +Fix reference leaks in error paths of the :mod:`!_interpchannels` and +:mod:`!_interpqueues` extension modules. + +.. + +.. date: 2026-05-13-23-18-39 +.. gh-issue: 149801 +.. nonce: S_FfGr +.. section: Library + +Add IANA registered names and aliases with leading zeros before number (like +IBM00858, CP00858, IBM01140, CP01140) for corresponding codecs. + +.. + +.. date: 2026-05-12-06-24-54 +.. gh-issue: 149701 +.. nonce: 8v9RTm +.. section: Library + +Fix bad return code from Lib/venv/bin/activate if hashing is disabled + +.. + +.. date: 2026-05-08-15-08-35 +.. gh-issue: 112821 +.. nonce: t9T1YD +.. section: Library + +In the REPL, autocompletion might run arbitrary code in the getter of a +descriptor. If that getter raised an exception, autocompletion would fail to +present any options for the entire object. Autocompletion now works as +expected for these objects. + +.. + +.. date: 2026-05-07-21-58-17 +.. gh-issue: 149388 +.. nonce: DDBPeA +.. section: Library + +Make :class:`!asyncio.windows_utils.PipeHandle` closing idempotent. + +.. + +.. date: 2026-05-07-14-18-47 +.. gh-issue: 149489 +.. nonce: bX9iHe +.. section: Library + +Fix :mod:`~xml.etree.ElementTree` serialization to HTML. The content of +elements "xmp", "iframe", "noembed", "noframes", and "plaintext" is no +longer escaped. The "plaintext" element no longer have the closing tag. + +.. + +.. date: 2026-05-04-19-28-48 +.. gh-issue: 149377 +.. nonce: WNlc8Y +.. section: Library + +Update bundled pip to 26.1.1 + +.. + +.. date: 2026-05-01-16-45-31 +.. gh-issue: 149231 +.. nonce: x2nBEE +.. section: Library + +In :mod:`tomllib`, the number of parts in TOML keys is now limited. + +.. + +.. date: 2026-04-29-16-11-27 +.. gh-issue: 149117 +.. nonce: yEeTYd +.. section: Library + +Fix :func:`runpy.run_module` and :func:`runpy.run_path` to set the +:attr:`~ImportError.name` attribute on the :exc:`ImportError` they raise. + +.. + +.. date: 2026-04-29-14-33-42 +.. gh-issue: 149148 +.. nonce: EaiYvk +.. section: Library + +:mod:`ensurepip`: Upgrade bundled pip to 26.1. This version fixes the +:cve:`2026-3219` vulnerability. Patch by Victor Stinner. + +.. + +.. date: 2026-04-27-22-34-09 +.. gh-issue: 148093 +.. nonce: 9pWceM +.. section: Library + +Fix an out-of-bounds read of one byte in :func:`binascii.a2b_uu`. Raise +:exc:`binascii.Error`, instead of reading past the buffer end. + +.. + +.. date: 2026-04-27-17-12-11 +.. gh-issue: 148914 +.. nonce: i5C3kW +.. section: Library + +Fix memoization of in-band :class:`~pickle.PickleBuffer` in the Python +implementation of :mod:`pickle`. Previously, identical +:class:`!PickleBuffer`\ s did not preserve identity, and empty writable +:class:`!PickleBuffer` memoized an empty bytearray object in place of +``b''``, so the following references to ``b''`` were unpickled as an empty +bytearray object. + +.. + +.. date: 2026-04-25-14-11-24 +.. gh-issue: 138907 +.. nonce: u21Wnh +.. section: Library + +Support :rfc:`9309` in :mod:`urllib.robotparser`. + +.. + +.. date: 2026-04-24-19-54-00 +.. gh-issue: 148954 +.. nonce: v1 +.. section: Library + +Fix XML injection vulnerability in :func:`xmlrpc.client.dumps` where the +``methodname`` was not being escaped before interpolation into the XML body. + +.. + +.. date: 2026-04-20-18-29-21 +.. gh-issue: 148801 +.. nonce: ROeNqs +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +<object.__deepcopy__>` on deeply nested trees. + +.. + +.. date: 2026-04-18-21-39-15 +.. gh-issue: 148735 +.. nonce: siw6DG +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a use-after-free in +:meth:`Element.findtext <xml.etree.ElementTree.Element.findtext>` when the +element tree is mutated concurrently during the search. + +.. + +.. date: 2026-04-15-11-00-39 +.. gh-issue: 146553 +.. nonce: VGOsoP +.. section: Library + +Fix infinite loop in :func:`typing.get_type_hints` when ``__wrapped__`` +forms a cycle. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-04-14-09-04-35 +.. gh-issue: 148508 +.. nonce: -GiXml +.. section: Library + +An intermittent timing error when running SSL tests on iOS has been +resolved. + +.. + +.. date: 2026-04-13-15-59-44 +.. gh-issue: 148518 +.. nonce: RQdvsu +.. section: Library + +If an email containing an address header that ended in an open double quote +was parsed with a non-``compat32`` policy, accessing the ``username`` +attribute of the mailbox accessed through that header object would result in +an ``IndexError``. It now correctly returns an empty string as the result. + +.. + +.. date: 2026-04-12-16-40-11 +.. gh-issue: 148370 +.. nonce: 0Li2EK +.. section: Library + +:mod:`configparser`: prevent quadratic behavior when a +:exc:`~configparser.ParsingError` is raised after a parser fails to parse +multiple lines. Patch by Bénédikt Tran. + +.. + +.. date: 2026-04-09-12-42-42 +.. gh-issue: 148254 +.. nonce: Xt7vKs +.. section: Library + +Use singular "sec" instead of "secs" in :mod:`timeit` verbose output for +consistency with other time units. + +.. + +.. date: 2026-04-07-14-13-40 +.. gh-issue: 148192 +.. nonce: 34AUYQ +.. section: Library + +``email.generator.Generator._make_boundary`` could fail to detect a +duplicate boundary string if linesep was not \n. It now correctly detects +boundary strings when linesep is \r\n as well. + +.. + +.. date: 2026-03-22-23-42-22 +.. gh-issue: 146313 +.. nonce: RtDeAd +.. section: Library + +Fix a deadlock in :mod:`multiprocessing`'s resource tracker where the parent +process could hang indefinitely in :func:`os.waitpid` during interpreter +shutdown if a child created via :func:`os.fork` still held the resource +tracker's pipe open. + +.. + +.. date: 2026-03-11-15-09-52 +.. gh-issue: 145831 +.. nonce: _sW94w +.. section: Library + +Fix :func:`!email.quoprimime.decode` leaving a stray ``\r`` when +``eol='\r\n'`` by stripping the full *eol* string instead of one character. + +.. + +.. date: 2026-02-22-00-00-00 +.. gh-issue: 145105 +.. nonce: csv-reader-reentrant +.. section: Library + +Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator +that calls :func:`next` on the same reader from within ``__next__``. + +.. + +.. date: 2026-02-19-04-40-57 +.. gh-issue: 130750 +.. nonce: 0hW52O +.. section: Library + +Restore quoting of choices in :mod:`argparse` error messages for improved +clarity and consistency with documentation. + +.. + +.. date: 2026-01-19-21-23-18 +.. gh-issue: 105936 +.. nonce: dGrzjM +.. section: Library + +Attempting to mutate non-field attributes of :mod:`dataclasses` with both +*frozen* and *slots* being ``True`` now raises +:class:`~dataclasses.FrozenInstanceError` instead of :class:`TypeError`. +Their non-dataclass subclasses can now freely mutate non-field attributes, +and the original non-slotted class can be garbage collected. The fix also +handles the case of an empty ``__class__`` cell on a function found within +the class (gh-148947). + +.. + +.. date: 2026-01-11-13-03-32 +.. gh-issue: 142516 +.. nonce: u7An-s +.. section: Library + +:mod:`ssl`: fix reference leaks in :class:`ssl.SSLContext` objects. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-12-17-04-10-35 +.. gh-issue: 142831 +.. nonce: ee3t4L +.. section: Library + +Fix a crash in the :mod:`json` module where a use-after-free could occur if +the object being encoded is modified during serialization. + +.. + +.. date: 2025-10-18-12-13-39 +.. gh-issue: 140287 +.. nonce: 49iU-4 +.. section: Library + +The :mod:`asyncio` REPL now handles exceptions when executing +:envvar:`PYTHONSTARTUP` scripts. Patch by Bartosz Sławecki. + +.. + +.. date: 2025-09-26-18-04-28 +.. gh-issue: 90949 +.. nonce: YHjSzX +.. section: Library + +Add +:meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold` +and +:meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification` +to :ref:`xmlparser <xmlparser-objects>` objects to tune protections against +`billion laughs <https://en.wikipedia.org/wiki/Billion_laughs_attack>`_ +attacks. Patch by Bénédikt Tran. + +.. + +.. date: 2025-04-17-15-26-35 +.. gh-issue: 132631 +.. nonce: IDFZfb +.. section: Library + +Fix "I/O operation on closed file" when parsing JSON Lines file with +:mod:`JSON CLI <json.tool>`. + +.. + +.. date: 2025-03-01-13-36-02 +.. gh-issue: 128110 +.. nonce: 9wx_G0 +.. section: Library + +Fix bug in the parsing of :mod:`email` address headers that could result in +extraneous spaces in the decoded text when using a modern email policy. +Space between pairs of adjacent :rfc:`2047` encoded-words is now ignored, +per section 6.2 (and consistent with existing parsing of unstructured +headers like *Subject*). + +.. + +.. date: 2024-11-02-02-02-31 +.. gh-issue: 107398 +.. nonce: uUtA6Q +.. section: Library + +Fix :mod:`tarfile` stream mode exception when process the file with the gzip +extra field. + +.. + +.. date: 2024-09-09-12-48-37 +.. gh-issue: 123853 +.. nonce: e-zFxb +.. section: Library + +Update the table of Windows language code identifiers (LCIDs) used by +:func:`locale.getdefaultlocale` on Windows to protocol version 16.0 +(2024-04-23). + +.. + +.. date: 2024-02-10-21-25-22 +.. gh-issue: 70039 +.. nonce: 6wvcAP +.. section: Library + +Fixed bug where :meth:`smtplib.SMTP.starttls` could fail if +:meth:`smtplib.SMTP.connect` is called explicitly rather than implicitly. + +.. + +.. date: 2023-09-08-13-10-32 +.. gh-issue: 83281 +.. nonce: 2Plpcj +.. section: Library + +:mod:`email`: improve handling trailing garbage in address lists to avoid +throwing AttributeError in certain edge cases + +.. + +.. date: 2023-02-26-14-07-18 +.. gh-issue: 91099 +.. nonce: _QPbEL +.. section: Library + +:meth:`imaplib.IMAP4.login` now raises exceptions with :class:`str` instead +of :class:`bytes`. Patch by Florian Best. + +.. + +.. bpo: 6699 +.. date: 2019-12-12-03-18-02 +.. nonce: 1CqJFG +.. section: IDLE + +Warn the user if a file will be overwritten when saving. + +.. + +.. date: 2026-05-23-17-27-41 +.. gh-issue: 150319 +.. nonce: ol9tWK +.. section: Documentation + +Generic builtin and standard library types now document the meaning of their +type parameters. + +.. + +.. date: 2026-04-17-02-28-55 +.. gh-issue: 148663 +.. nonce: MHIbRB +.. section: Documentation + +Document that :class:`calendar.IllegalMonthError` is a subclass of both +:exc:`ValueError` and :exc:`IndexError` since Python 3.12. + +.. + +.. date: 2026-04-02-07-20-00 +.. gh-issue: 146646 +.. nonce: GlobDoc1 +.. section: Documentation + +Document that :func:`glob.glob`, :func:`glob.iglob`, +:meth:`pathlib.Path.glob`, and :meth:`pathlib.Path.rglob` silently suppress +:exc:`OSError` exceptions raised from scanning the filesystem. + +.. + +.. date: 2023-09-16-23-42-27 +.. gh-issue: 109503 +.. nonce: mZ-kdU +.. section: Documentation + +Fix documentation for :func:`shutil.move` on usage of :func:`os.rename` +since nonatomic move might be used even if the files are on the same +filesystem. Patch by Fang Li + +.. + +.. date: 2026-06-09-12-24-35 +.. gh-issue: 151112 +.. nonce: 4RKCkD +.. section: Core and Builtins + +Fix a crash in the compiler that could occur when running out of memory. + +.. + +.. date: 2026-06-09-10-28-30 +.. gh-issue: 151126 +.. nonce: DKa6Sl +.. section: Core and Builtins + +Fix a crash, when there's no memory left on a device, which happened in: + +- code compilation - :func:`!_winapi.CreateProcess` + +Now these places raise proper :exc:`MemoryError` errors. + +.. + +.. date: 2026-05-30-20-19-35 +.. gh-issue: 150633 +.. nonce: XkNul0 +.. section: Core and Builtins + +Fix the frozen importer accepting module names with embedded null bytes, +which caused it to bypass the :data:`sys.modules` cache and create duplicate +module objects. + +.. + +.. date: 2026-05-24-14-45-00 +.. gh-issue: 149156 +.. nonce: NP73rB +.. section: Core and Builtins + +Fix an intermittent crash after :func:`os.fork` when perf trampoline +profiling is enabled and the child returns through trampoline frames +inherited from the parent process. + +.. + +.. date: 2026-05-23-22-08-01 +.. gh-issue: 149449 +.. nonce: 2lhQFF +.. section: Core and Builtins + +Fix a use-after-free crash when the :mod:`unicodedata` module was removed +from :data:`sys.modules` and garbage-collected between calls that decode +``\N{...}`` escapes or use the ``namereplace`` codec error handler. + +.. + +.. date: 2026-05-23-09-55-50 +.. gh-issue: 148450 +.. nonce: 2MEVqH +.. section: Core and Builtins + +Fix ``abc.register()`` so it invalidates type version tags for registered +classes. + +.. + +.. date: 2026-05-22-21-52-38 +.. gh-issue: 150207 +.. nonce: l2BUtI +.. section: Core and Builtins + +Fix a crash when a memory allocation fails during tokenizer initialization. +A proper :exc:`MemoryError` is now raised instead. + +.. + +.. date: 2026-05-22-17-09-28 +.. gh-issue: 150107 +.. nonce: GD72-D +.. section: Core and Builtins + +:mod:`asyncio`: ``sendfile()`` and ``sock_sendfile()`` event loop methods +now call ``file.seek(offset)`` if *file* has a ``seek()`` method, even if +*offset* is ``0`` (default value). + +.. + +.. date: 2026-05-20-13-06-17 +.. gh-issue: 150146 +.. nonce: i5m_SL +.. section: Core and Builtins + +Fix a crash on a complex type variable substitution. + +``from typing import TypeVar; memoryview[TypeVar("")][*typing.Mapping[..., +...]]`` used to fail due to missing ``NULL`` check on ``_unpack_args`` C +function call. + +.. + +.. date: 2026-05-18-13-47-17 +.. gh-issue: 149590 +.. nonce: IPBeQx +.. section: Core and Builtins + +Fix crash when faulthandler is imported more than once. + +.. + +.. date: 2026-05-13-06-54-41 +.. gh-issue: 149738 +.. nonce: 4BLFoH +.. section: Core and Builtins + +:mod:`sqlite3`: Disallow removing ``row_factory`` and ``text_factory`` +attributes of a connection to prevent a crash on a query. + +.. + +.. date: 2026-05-12-16-47-23 +.. gh-issue: 139808 +.. nonce: iIs7_E +.. section: Core and Builtins + +Add branch protections for AArch64 (BTI/PAC) in assembly code used by +:option:`-X perf_jit <-X>` (Linux perf profiler integration). + +.. + +.. date: 2026-04-21-14-36-44 +.. gh-issue: 148820 +.. nonce: XhOGhA +.. section: Core and Builtins + +Fix a race in :c:type:`!_PyRawMutex` on the free-threaded build where a +``Py_PARK_INTR`` return from ``_PySemaphore_Wait`` could let the waiter +destroy its semaphore before the unlocking thread's ``_PySemaphore_Wakeup`` +completed, causing a fatal ``ReleaseSemaphore`` error. + +.. + +.. date: 2026-04-17-20-37-02 +.. gh-issue: 148653 +.. nonce: nbbHMh +.. section: Core and Builtins + +Forbid :mod:`marshalling <marshal>` recursive code objects which cannot be +correctly unmarshalled. + +.. + +.. date: 2026-04-12-17-27-28 +.. gh-issue: 148390 +.. nonce: MAhw7F +.. section: Core and Builtins + +Fix an undefined behavior in :class:`memoryview` when using the native +boolean format (``?``) in :meth:`~memoryview.cast`. Previously, on some +common platforms, calling ``memoryview(b).cast("?").tolist()`` incorrectly +returned ``[False]`` instead of ``[True]`` for any even byte *b*. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-04-12-10-40-57 +.. gh-issue: 148418 +.. nonce: ggA1LZ +.. section: Core and Builtins + +Fix a possible reference leak in a corrupted ``TYPE_CODE`` marshal stream. + +.. + +.. date: 2026-04-07-20-37-23 +.. gh-issue: 148222 +.. nonce: uF4D4E +.. section: Core and Builtins + +Fix vectorcall support in :class:`types.GenericAlias` when the underlying +type does not support the vectorcall protocol. Fix possible leaks in +:class:`types.GenericAlias` and :class:`types.UnionType` in case of memory +error. + +.. + +.. date: 2026-02-28-16-46-17 +.. gh-issue: 145376 +.. nonce: lG5u1a +.. section: Core and Builtins + +Fix reference leaks in various unusual error scenarios. + +.. + +.. date: 2026-06-04-14-26-17 +.. gh-issue: 150907 +.. nonce: CA91_B +.. section: C API + +Fix ``dynamic_annotations.h`` header file when built with C++ and Valgrind: +add ``extern "C++" scope`` for the C++ template. Patch by Victor Stinner. + +.. + +.. date: 2026-05-04-06-03-50 +.. gh-issue: 149351 +.. nonce: hN4sF0 +.. section: Build + +Avoid possible broken macOS framework install names when DESTDIR is +specified during builds. + +.. + +.. date: 2026-04-30-08-43-47 +.. gh-issue: 146475 +.. nonce: 1cL4hX +.. section: Build + +Block Apple Clang from being used to build the JIT as it ships without +required LLVM tools. + +.. + +.. date: 2026-04-14-15-20-29 +.. gh-issue: 148535 +.. nonce: JjKiaa +.. section: Build + +No longer use the ``gcc -fprofile-update=atomic`` flag on i686. The flag has +been added to fix a random GCC internal error on PGO build (:gh:`145801`) +caused by corruption of profile data (.gcda files). The problem is that it +makes the PGO build way slower (up to 47x slower) on i686. Since the GCC +internal error was not seen on i686 so far, don't use +``-fprofile-update=atomic`` on i686 anymore. Patch by Victor Stinner. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 8b8aa776175..a3ab9416f20 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,21 +70,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "c6ea8a5423f3966923060db2089f869017dfb10bcf2037394146e7a74caec0a8" + "checksumValue": "0349aef1ff9ded3eb23be12c2586e026be5cec3fd40df859e388671d7844fced" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.19.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.21.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.19:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.21:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.0.19" + "versionInfo": "3.0.21" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 1f8a57269db..dca0e838b45 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9dfd09a3be37618cbcea380c2374b2b8f0288f57" + "checksumValue": "58101ef0951568acadd3117033bef084fea24cc1" }, { "algorithm": "SHA256", - "checksumValue": "26805a0d1a7a6a5cd8ead9cf7f4da29f63f0547a9ad41e80dba4ed9fe1943140" + "checksumValue": "52d756026bf09befdb211c453e2009a646d6c6b519e6885e971b2550396619fb" } ], "fileName": "Modules/expat/expat.h" @@ -62,11 +62,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "da0328279276800cc747ea7da23886a3f402ccb3" + "checksumValue": "d8f9211d52ff0384e229e4d4d56adae5db2d7f91" }, { "algorithm": "SHA256", - "checksumValue": "15a80e414e9e7c43edba64b1608a77c724387070138693f9e9bcca49c78a2df7" + "checksumValue": "b77f8192baf90aaa41f7023bc68fd1f22ab2552f98758271a1e090544537def5" } ], "fileName": "Modules/expat/expat_external.h" @@ -90,11 +90,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "6a4a232233ba1034c3f2b459159d502e9b2d413b" + "checksumValue": "2555e70b29c1efc0af40879daafd12f8b36aca2c" }, { "algorithm": "SHA256", - "checksumValue": "c803935722f0dbdeeede7f040028fb119135e96dfad949479f8a5304b885bdd6" + "checksumValue": "4feb1df53898a48ae0ae04b5d0352c90395c8e693e5c2675f8ced41903d6fa94" } ], "fileName": "Modules/expat/internal.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0c74fbd48dd515c58eeb65b7e71b29da94be4694" + "checksumValue": "1dad2ab196cdbe37572674c465bd9187fdbe4495" }, { "algorithm": "SHA256", - "checksumValue": "861e7a50ce81f9f16b42d32a9caa4f817d962b274b2929b579511c6f76d348d4" + "checksumValue": "740137e670d2f3b7269364ffb6f60064e6560091850c5d6f2c3bb1b8ca6e3dd1" } ], "fileName": "Modules/expat/xmlparse.c" @@ -188,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7cff4d7210f046144f5fa635113f9c26f30fe3d3" + "checksumValue": "c8769fcb93f00272a6e6ca560be633649c817ff7" }, { "algorithm": "SHA256", - "checksumValue": "eaa6c327f9db4a5cec768d0c01927fea212d3ef4d4f970ebc0a98b9f3602784c" + "checksumValue": "5b81f0eb0e144b611dbd1bc9e6037075a16bff94f823d57a81eb2a3e4999e91a" } ], "fileName": "Modules/expat/xmlrole.c" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "48b7aa6503302d4157c61a8763629f3236c23502" + "checksumValue": "63e4766a09e63760c6518670509198f8d638f4ad" }, { "algorithm": "SHA256", - "checksumValue": "75da65603e99837fd3116f1453372efd556f9f97d8de73364594dd78b3c8ec54" + "checksumValue": "0ad3f915f2748dc91bf4e4b4a50cf40bf2c95769d0eca7e3b293a230d82bb779" } ], "fileName": "Modules/expat/xmltok.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "705842f8a09b09cc021d82a71ab03344bfd07b0a" + "checksumValue": "41b8c8fc275882c76d4210b7d40a18e506b07147" }, { "algorithm": "SHA256", - "checksumValue": "f95a2b4b7efda40f5faf366537cb20a57dddbad9655859d2e304f5e75f6907cc" + "checksumValue": "b2188c7e5fa5b33e355cf6cf342dfb8f6e23859f2a6b1ddf79841d7f84f7b196" } ], "fileName": "Modules/expat/xmltok_ns.c" @@ -1548,14 +1548,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "9931f9860d18e6cf72d183eb8f309bfb96196c00e1d40caa978e95bc9aa978b6" + "checksumValue": "a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_7_5/expat-2.7.5.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_1/expat-2.8.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.7.5:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1563,7 +1563,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.7.5" + "versionInfo": "2.8.1" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index ddae123e1fa..729e05319bb 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1787,7 +1787,6 @@ [const.METH_COEXIST] added = '3.2' # METH_STACKLESS is undocumented -# METH_FASTCALL is not part of limited API. # The following are defined in private headers, but historically # they were exported as part of the stable ABI. @@ -2121,8 +2120,6 @@ # New method flags in 3.7 (PEP 590): -[const.METH_FASTCALL] - added = '3.7' [const.METH_METHOD] added = '3.7' @@ -2272,6 +2269,10 @@ [data.PyStructSequence_UnnamedField] added = '3.11' +# Added in 3.7 but in the Stable ABI from 3.10 +[const.METH_FASTCALL] + added = '3.10' + # Add stable Py_buffer API in Python 3.11 (https://bugs.python.org/issue45459) [struct.Py_buffer] added = '3.11' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 57b90101bbe..971549950a6 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,8 +163,8 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/file.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/weakref.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_abc.c b/Modules/_abc.c index 4f4b24b035d..baffb80ffe6 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -907,14 +907,14 @@ _abc.get_cache_token Returns the current ABC cache token. -The token is an opaque object (supporting equality testing) identifying the -current version of the ABC cache for virtual subclasses. The token changes -with every call to register() on any ABC. +The token is an opaque object (supporting equality testing) identifying +the current version of the ABC cache for virtual subclasses. The token +changes with every call to register() on any ABC. [clinic start generated code]*/ static PyObject * _abc_get_cache_token_impl(PyObject *module) -/*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ +/*[clinic end generated code: output=c7d87841e033dacc input=d87acc04492f6bf3]*/ { _abcmodule_state *state = get_abc_state(module); return PyLong_FromUnsignedLongLong(get_invalidation_counter(state)); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0ed899f5a7c..857cb06d013 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -781,12 +781,13 @@ Return the result this future represents. If the future has been cancelled, raises CancelledError. If the future's result isn't yet available, raises InvalidStateError. If -the future is done and has an exception set, this exception is raised. +the future is done and has an exception set, this exception is +raised. [clinic start generated code]*/ static PyObject * _asyncio_Future_result_impl(FutureObj *self) -/*[clinic end generated code: output=f35f940936a4b1e5 input=49ecf9cf5ec50dc5]*/ +/*[clinic end generated code: output=f35f940936a4b1e5 input=c7361d75834bb8cd]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; @@ -917,15 +918,15 @@ _asyncio.Future.add_done_callback Add a callback to be run when the future becomes done. -The callback is called with a single argument - the future object. If -the future is already done when this is called, the callback is +The callback is called with a single argument - the future object. +If the future is already done when this is called, the callback is scheduled with call_soon. [clinic start generated code]*/ static PyObject * _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context) -/*[clinic end generated code: output=922e9a4cbd601167 input=599261c521458cc2]*/ +/*[clinic end generated code: output=922e9a4cbd601167 input=1b96105434e0a562]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { @@ -1072,15 +1073,15 @@ _asyncio.Future.cancel Cancel the future and schedule callbacks. -If the future is already done or cancelled, return False. Otherwise, -change the future's state to cancelled, schedule the callbacks and -return True. +If the future is already done or cancelled, return False. +Otherwise, change the future's state to cancelled, schedule the +callbacks and return True. [clinic start generated code]*/ static PyObject * _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg) -/*[clinic end generated code: output=074956f35904b034 input=bba8f8b786941a94]*/ +/*[clinic end generated code: output=074956f35904b034 input=4aefb2a5726ba0f5]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -1110,13 +1111,13 @@ _asyncio.Future.done Return True if the future is done. -Done means either that a result / exception are available, or that the -future was cancelled. +Done means either that a result / exception are available, or that +the future was cancelled. [clinic start generated code]*/ static PyObject * _asyncio_Future_done_impl(FutureObj *self) -/*[clinic end generated code: output=244c5ac351145096 input=28d7b23fdb65d2ac]*/ +/*[clinic end generated code: output=244c5ac351145096 input=afad8c9d2283428d]*/ { if (!future_is_alive(self) || self->fut_state == STATE_PENDING) { Py_RETURN_FALSE; @@ -1439,7 +1440,8 @@ static PyMethodDef FutureType_methods[] = { _ASYNCIO_FUTURE_DONE_METHODDEF _ASYNCIO_FUTURE_GET_LOOP_METHODDEF _ASYNCIO_FUTURE__MAKE_CANCELLED_ERROR_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Futures are generic over the type of their results")}, {NULL, NULL} /* Sentinel */ }; @@ -2611,7 +2613,8 @@ static PyMethodDef TaskType_methods[] = { _ASYNCIO_TASK_SET_NAME_METHODDEF _ASYNCIO_TASK_GET_CORO_METHODDEF _ASYNCIO_TASK_GET_CONTEXT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Tasks are generic over the return type of their wrapped coroutines")}, {NULL, NULL} /* Sentinel */ }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9b146265445..16fc5b6a919 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -161,8 +161,8 @@ _bisect.bisect_right -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e <= x, and all e in -a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will -insert just after the rightmost x already there. +a[i:] have e > x. So if x already appears in the list, a.insert(i, x) +will insert just after the rightmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -173,7 +173,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_right_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=3a4bc09cc7c8a73d input=43071869772dd53a]*/ +/*[clinic end generated code: output=3a4bc09cc7c8a73d input=b8951a7bb11516e1]*/ { return internal_bisect_right(a, x, lo, hi, key); } @@ -346,8 +346,8 @@ _bisect.bisect_left -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e < x, and all e in -a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will -insert just before the leftmost x already there. +a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) +will insert just before the leftmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -358,7 +358,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_left_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=70749d6e5cae9284 input=f29c4fe7f9b797c7]*/ +/*[clinic end generated code: output=70749d6e5cae9284 input=d24dc2b6439000f7]*/ { return internal_bisect_left(a, x, lo, hi, key); } diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 661847ad267..dc536835acd 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -116,6 +116,7 @@ typedef struct { typedef struct { PyObject_HEAD bz_stream bzs; + int bzerror; char eof; /* Py_T_BOOL expects a char */ PyObject *unused_data; char needs_input; @@ -455,8 +456,11 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length) d->bzs_avail_in_real += bzs->avail_in; - if (catch_bz2_error(bzret)) + if (catch_bz2_error(bzret)) { + d->bzerror = bzret; + d->needs_input = 0; goto error; + } if (bzret == BZ_STREAM_END) { d->eof = 1; break; @@ -589,6 +593,7 @@ decompress(BZ2Decompressor *d, char *data, size_t len, Py_ssize_t max_length) return result; error: + bzs->next_in = NULL; Py_XDECREF(result); return NULL; } @@ -601,32 +606,40 @@ _bz2.BZ2Decompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=23e41045deb240a3 input=52e1ffc66a8ea624]*/ +/*[clinic end generated code: output=23e41045deb240a3 input=7f68faa9ff7a1b51]*/ { PyObject *result = NULL; ACQUIRE_LOCK(self); - if (self->eof) + if (self->eof) { PyErr_SetString(PyExc_EOFError, "End of stream already reached"); - else + } + else if (self->bzerror) { + // Re-entering BZ2_bzDecompress() after an error can write out of bounds. + PyErr_SetString(PyExc_ValueError, + "Decompressor is unusable after a previous error"); + } + else { result = decompress(self, data->buf, data->len, max_length); + } RELEASE_LOCK(self); return result; } @@ -660,6 +673,7 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) return NULL; } + self->bzerror = 0; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 32373f0799b..4caf2260178 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -54,14 +54,15 @@ _codecs.register Register a codec search function. -Search functions are expected to take one argument, the encoding name in -all lower case letters, and either return None, or a tuple of functions -(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object). +Search functions are expected to take one argument, the encoding +name in all lower case letters, and either return None, or a tuple +of functions (encoder, decoder, stream_reader, stream_writer) (or +a CodecInfo object). [clinic start generated code]*/ static PyObject * _codecs_register(PyObject *module, PyObject *search_function) -/*[clinic end generated code: output=d1bf21e99db7d6d3 input=369578467955cae4]*/ +/*[clinic end generated code: output=d1bf21e99db7d6d3 input=2321d8c8c0420dfc]*/ { if (PyCodec_Register(search_function)) return NULL; @@ -114,16 +115,16 @@ _codecs.encode Encodes obj using the codec registered for encoding. The default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_encode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=385148eb9a067c86 input=cd5b685040ff61f0]*/ +/*[clinic end generated code: output=385148eb9a067c86 input=e5271d443e391d7f]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -141,16 +142,16 @@ _codecs.decode Decodes obj using the codec registered for encoding. Default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_decode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=679882417dc3a0bd input=7702c0cc2fa1add6]*/ +/*[clinic end generated code: output=679882417dc3a0bd input=3e6254628f9ca538]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -965,14 +966,15 @@ _codecs.register_error Register the specified error handler under the name errors. handler must be a callable object, that will be called with an exception -instance containing information about the location of the encoding/decoding -error and must return a (replacement, new position) tuple. +instance containing information about the location of the +encoding/decoding error and must return a (replacement, new position) +tuple. [clinic start generated code]*/ static PyObject * _codecs_register_error_impl(PyObject *module, const char *errors, PyObject *handler) -/*[clinic end generated code: output=fa2f7d1879b3067d input=5e6709203c2e33fe]*/ +/*[clinic end generated code: output=fa2f7d1879b3067d input=5bea01dfe835d9d8]*/ { if (PyCodec_RegisterError(errors, handler)) return NULL; @@ -986,13 +988,13 @@ _codecs.lookup_error lookup_error(errors) -> handler -Return the error handler for the specified error handling name or raise a -LookupError, if no handler exists under this name. +Return the error handler for the specified error handling name or raise +a LookupError, if no handler exists under this name. [clinic start generated code]*/ static PyObject * _codecs_lookup_error_impl(PyObject *module, const char *name) -/*[clinic end generated code: output=087f05dc0c9a98cc input=4775dd65e6235aba]*/ +/*[clinic end generated code: output=087f05dc0c9a98cc input=86cfb6a7a9c67113]*/ { return PyCodec_LookupError(name); } diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index b2c3d0e42be..daf62c940b3 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1824,7 +1824,7 @@ static PyMethodDef deque_methods[] = { DEQUE_ROTATE_METHODDEF DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("deques are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; @@ -2284,6 +2284,12 @@ defdict_reduce(defdictobject *dd, PyObject *Py_UNUSED(ignored)) return result; } + +PyDoc_STRVAR(defdict_class_getitem_doc, +"defaultdicts are generic over two types, signifying (respectively) the types \ +of the dictionary's keys and values"); + + static PyMethodDef defdict_methods[] = { {"__missing__", (PyCFunction)defdict_missing, METH_O, defdict_missing_doc}, @@ -2294,7 +2300,7 @@ static PyMethodDef defdict_methods[] = { {"__reduce__", (PyCFunction)defdict_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + defdict_class_getitem_doc}, {NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index 0b7ff2575fc..755d005f4d8 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -957,6 +957,12 @@ Reader_iternext(ReaderObj *self) Py_DECREF(lineobj); return NULL; } + if (self->fields == NULL) { + PyErr_SetString(module_state->error_obj, + "iterator has already advanced the reader"); + Py_DECREF(lineobj); + return NULL; + } ++self->line_num; kind = PyUnicode_KIND(lineobj); data = PyUnicode_DATA(lineobj); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index aac812813d9..c66d157be75 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4935,7 +4935,7 @@ Array_length(PyObject *myself) static PyMethodDef Array_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("Arrays are generic over the type of their elements")}, { NULL, NULL } }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 8d7f613421e..d802f9cb4f7 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -405,7 +405,7 @@ EXPORT(char *)my_strtok(char *token, const char *delim) return strtok(token, delim); } -EXPORT(char *)my_strchr(const char *s, int c) +EXPORT(const char *) my_strchr(const char *s, int c) { return strchr(s, c); } diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 7241c3d70a7..1d3cf13cb90 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -234,12 +234,13 @@ _curses_panel.panel.hide Hide the panel. -This does not delete the object, it just makes the window on screen invisible. +This does not delete the object, it just makes the window on screen +invisible. [clinic start generated code]*/ static PyObject * _curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc6ab7203cdc1450 input=1bfc741f473e6055]*/ +/*[clinic end generated code: output=cc6ab7203cdc1450 input=805065e45e6fc1cd]*/ { _curses_panel_state *state = PyType_GetModuleState(cls); return PyCursesCheckERR(state, hide_panel(self->pan), "hide"); @@ -689,12 +690,13 @@ _curses_panel.update_panels Updates the virtual screen after changes in the panel stack. -This does not call curses.doupdate(), so you'll have to do this yourself. +This does not call curses.doupdate(), so you'll have to do this +yourself. [clinic start generated code]*/ static PyObject * _curses_panel_update_panels_impl(PyObject *module) -/*[clinic end generated code: output=2f3b4c2e03d90ded input=5299624c9a708621]*/ +/*[clinic end generated code: output=2f3b4c2e03d90ded input=0d0db79f05ec3ef4]*/ { PyCursesInitialised; update_panels(); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 0200f59020f..e6b52713579 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1072,10 +1072,10 @@ _curses.window.border Draw a border around the edges of the window. -Each parameter specifies the character to use for a specific part of the -border. The characters can be specified as integers or as one-character -strings. A 0 value for any parameter will cause the default character to be -used for that parameter. +Each parameter specifies the character to use for a specific part of +the border. The characters can be specified as integers or as +one-character strings. A 0 value for any parameter will cause the +default character to be used for that parameter. [clinic start generated code]*/ static PyObject * @@ -1083,7 +1083,7 @@ _curses_window_border_impl(PyCursesWindowObject *self, PyObject *ls, PyObject *rs, PyObject *ts, PyObject *bs, PyObject *tl, PyObject *tr, PyObject *bl, PyObject *br) -/*[clinic end generated code: output=670ef38d3d7c2aa3 input=e015f735d67a240b]*/ +/*[clinic end generated code: output=670ef38d3d7c2aa3 input=42568c1458221d24]*/ { chtype ch[8]; int i; @@ -1126,14 +1126,15 @@ _curses.window.box Draw a border around the edges of the window. -Similar to border(), but both ls and rs are verch and both ts and bs are -horch. The default corner characters are always used by this function. +Similar to border(), but both ls and rs are verch and both ts and bs +are horch. The default corner characters are always used by this +function. [clinic start generated code]*/ static PyObject * _curses_window_box_impl(PyCursesWindowObject *self, int group_right_1, PyObject *verch, PyObject *horch) -/*[clinic end generated code: output=f3fcb038bb287192 input=f00435f9c8c98f60]*/ +/*[clinic end generated code: output=f3fcb038bb287192 input=e11acb7dbf6790b6]*/ { chtype ch1 = 0, ch2 = 0; if (group_right_1) { @@ -1294,15 +1295,15 @@ _curses.window.derwin Create a sub-window (window-relative coordinates). -derwin() is the same as calling subwin(), except that begin_y and begin_x -are relative to the origin of the window, rather than relative to the entire -screen. +derwin() is the same as calling subwin(), except that begin_y and +begin_x are relative to the origin of the window, rather than +relative to the entire screen. [clinic start generated code]*/ static PyObject * _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=7924b112d9f70d6e input=966d9481f7f5022e]*/ +/*[clinic end generated code: output=7924b112d9f70d6e input=6efb50722be444ba]*/ { WINDOW *win; @@ -1397,15 +1398,16 @@ _curses.window.getch Get a character code from terminal keyboard. -The integer returned does not have to be in ASCII range: function keys, -keypad keys and so on return numbers higher than 256. In no-delay mode, -1 -is returned if there is no input, else getch() waits until a key is pressed. +The integer returned does not have to be in ASCII range: function +keys, keypad keys and so on return numbers higher than 256. In +no-delay mode, -1 is returned if there is no input, else getch() +waits until a key is pressed. [clinic start generated code]*/ static PyObject * _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=e1639e87d545e676 input=73f350336b1ee8c8]*/ +/*[clinic end generated code: output=e1639e87d545e676 input=0dc5ff40e079787a]*/ { int rtn; @@ -1444,15 +1446,16 @@ _curses.window.getkey Get a character (string) from terminal keyboard. -Returning a string instead of an integer, as getch() does. Function keys, -keypad keys and other special keys return a multibyte string containing the -key name. In no-delay mode, an exception is raised if there is no input. +Returning a string instead of an integer, as getch() does. Function +keys, keypad keys and other special keys return a multibyte string +containing the key name. In no-delay mode, an exception is raised +if there is no input. [clinic start generated code]*/ static PyObject * _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=8490a182db46b10f input=be2dee34f5cf57f8]*/ +/*[clinic end generated code: output=8490a182db46b10f input=bd24a7da1ed9c73b]*/ { int rtn; @@ -1678,15 +1681,15 @@ _curses.window.insch Insert a character before the current or specified position. -All characters to the right of the cursor are shifted one position right, with -the rightmost characters on the line being lost. +All characters to the right of the cursor are shifted one position +right, with the rightmost characters on the line being lost. [clinic start generated code]*/ static PyObject * _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *ch, int group_right_1, long attr) -/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=336342756ee19812]*/ +/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=d662a0f96f33e15a]*/ { int rtn; chtype ch_ = 0; @@ -1717,13 +1720,14 @@ _curses.window.inch -> unsigned_long Return the character at the given position in the window. -The bottom 8 bits are the character proper, and upper bits are the attributes. +The bottom 8 bits are the character proper, and upper bits are the +attributes. [clinic start generated code]*/ static unsigned long _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=6c4719fe978fe86a input=fac23ee11e3b3a66]*/ +/*[clinic end generated code: output=6c4719fe978fe86a input=0d862382571c19d9]*/ { unsigned long rtn; @@ -1821,18 +1825,18 @@ _curses.window.insstr Insert the string before the current or specified position. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor. All characters to the right of -the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, -if specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor. All characters to the +right of the cursor are shifted right, with the rightmost characters +on the line being lost. The cursor position does not change (after +moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int group_right_1, long attr) -/*[clinic end generated code: output=c259a5265ad0b777 input=6827cddc6340a7f3]*/ +/*[clinic end generated code: output=c259a5265ad0b777 input=dbfbdd3892155ea6]*/ { int rtn; int strtype; @@ -1905,19 +1909,19 @@ _curses.window.insnstr Insert at most n characters of the string. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor, up to n characters. If n is zero -or negative, the entire string is inserted. All characters to the right -of the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, if -specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor, up to n characters. If +n is zero or negative, the entire string is inserted. All +characters to the right of the cursor are shifted right, with the +rightmost characters on the line being lost. The cursor position +does not change (after moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int n, int group_right_1, long attr) -/*[clinic end generated code: output=971a32ea6328ec8b input=70fa0cd543901a4c]*/ +/*[clinic end generated code: output=971a32ea6328ec8b input=fd0a9b65b84b385f]*/ { int rtn; int strtype; @@ -1975,12 +1979,13 @@ _curses.window.is_linetouched Return True if the specified line was modified, otherwise return False. -Raise a curses.error exception if line is not valid for the given window. +Raise a curses.error exception if line is not valid for the given +window. [clinic start generated code]*/ static PyObject * _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) -/*[clinic end generated code: output=ad4a4edfee2db08c input=a7be0c189f243914]*/ +/*[clinic end generated code: output=ad4a4edfee2db08c input=b5c1149c36e58222]*/ { int erg; erg = is_linetouched(self->win, line); @@ -2008,9 +2013,9 @@ _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * @@ -2018,21 +2023,21 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=809a1f3c6a03e23e input=3e56898388cd739e]*/ +/*[clinic end generated code: output=809a1f3c6a03e23e input=8b4c74bf55008803]*/ #else /*[clinic input] _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * _curses_window_noutrefresh_impl(PyCursesWindowObject *self) -/*[clinic end generated code: output=6ef6dec666643fee input=876902e3fa431dbd]*/ +/*[clinic end generated code: output=6ef6dec666643fee input=a7c6306f8af9d0dd]*/ #endif { int rtn; @@ -2080,14 +2085,15 @@ _curses.window.overlay Overlay the window on top of destwin. -The windows need not be the same size, only the overlapping region is copied. -This copy is non-destructive, which means that the current background -character does not overwrite the old contents of destwin. +The windows need not be the same size, only the overlapping region +is copied. This copy is non-destructive, which means that the +current background character does not overwrite the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overlay() can be used. sminrow and smincol are the upper-left coordinates -of the source window, and the other variables mark a rectangle in the -destination window. +To get fine-grained control over the copied region, the second form +of overlay() can be used. sminrow and smincol are the upper-left +coordinates of the source window, and the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2095,7 +2101,7 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=7edd23ad22cc1984]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=27cb41491121cc17]*/ { int rtn; @@ -2127,14 +2133,15 @@ _curses.window.overwrite Overwrite the window on top of destwin. -The windows need not be the same size, in which case only the overlapping -region is copied. This copy is destructive, which means that the current -background character overwrites the old contents of destwin. +The windows need not be the same size, in which case only the +overlapping region is copied. This copy is destructive, which means +that the current background character overwrites the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overwrite() can be used. sminrow and smincol are the upper-left coordinates -of the source window, the other variables mark a rectangle in the destination -window. +To get fine-grained control over the copied region, the second form +of overwrite() can be used. sminrow and smincol are the upper-left +coordinates of the source window, the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2143,7 +2150,7 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=ea5de1b35cd948e0]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=bca2bfc09d478c3f]*/ { int rtn; @@ -2241,23 +2248,24 @@ _curses.window.refresh Update the display immediately. Synchronize actual screen with previous drawing/deleting methods. -The 6 optional arguments can only be specified when the window is a pad -created with newpad(). The additional parameters are needed to indicate -what part of the pad and screen are involved. pminrow and pmincol specify -the upper left-hand corner of the rectangle to be displayed in the pad. -sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to -be displayed on the screen. The lower right-hand corner of the rectangle to -be displayed in the pad is calculated from the screen coordinates, since the -rectangles must be the same size. Both rectangles must be entirely contained -within their respective structures. Negative values of pminrow, pmincol, -sminrow, or smincol are treated as if they were zero. +The 6 optional arguments can only be specified when the window is +a pad created with newpad(). The additional parameters are needed +to indicate what part of the pad and screen are involved. pminrow +and pmincol specify the upper left-hand corner of the rectangle to +be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol +specify the edges of the rectangle to be displayed on the screen. +The lower right-hand corner of the rectangle to be displayed in the +pad is calculated from the screen coordinates, since the rectangles +must be the same size. Both rectangles must be entirely contained +within their respective structures. Negative values of pminrow, +pmincol, sminrow, or smincol are treated as if they were zero. [clinic start generated code]*/ static PyObject * _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=42199543115e6e63 input=95e01cb5ffc635d0]*/ +/*[clinic end generated code: output=42199543115e6e63 input=ff2e900c6b2696b1]*/ { int rtn; @@ -2325,14 +2333,14 @@ _curses.window.subwin Create a sub-window (screen-relative coordinates). -By default, the sub-window will extend from the specified position to the -lower right corner of the window. +By default, the sub-window will extend from the specified position +to the lower right corner of the window. [clinic start generated code]*/ static PyObject * _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=93e898afc348f59a input=2129fa47fd57721c]*/ +/*[clinic end generated code: output=93e898afc348f59a input=07b5058cb8820595]*/ { WINDOW *win; @@ -2364,13 +2372,14 @@ _curses.window.scroll Scroll the screen or scrolling region. -Scroll upward if the argument is positive and downward if it is negative. +Scroll upward if the argument is positive and downward if it is +negative. [clinic start generated code]*/ static PyObject * _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, int lines) -/*[clinic end generated code: output=4541a8a11852d360 input=c969ca0cfabbdbec]*/ +/*[clinic end generated code: output=4541a8a11852d360 input=d8d81a5b52b9b40f]*/ { if (!group_right_1) { return PyCursesCheckERR(scroll(self->win), "scroll"); @@ -2392,14 +2401,15 @@ _curses.window.touchline Pretend count lines have been changed, starting with line start. -If changed is supplied, it specifies whether the affected lines are marked -as having been changed (changed=True) or unchanged (changed=False). +If changed is supplied, it specifies whether the affected lines are +marked as having been changed (changed=True) or unchanged +(changed=False). [clinic start generated code]*/ static PyObject * _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed) -/*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ +/*[clinic end generated code: output=65d05b3f7438c61d input=e0dc62f90d9dea55]*/ { if (!group_right_1) { return PyCursesCheckERR(touchline(self->win, start, count), "touchline"); @@ -2727,16 +2737,17 @@ _curses.cbreak Enter cbreak mode. -In cbreak mode (sometimes called "rare" mode) normal tty line buffering is -turned off and characters are available to be read one by one. However, -unlike raw mode, special characters (interrupt, quit, suspend, and flow -control) retain their effects on the tty driver and calling program. -Calling first raw() then cbreak() leaves the terminal in cbreak mode. +In cbreak mode (sometimes called "rare" mode) normal tty line buffering +is turned off and characters are available to be read one by one. +However, unlike raw mode, special characters (interrupt, quit, suspend, +and flow control) retain their effects on the tty driver and calling +program. Calling first raw() then cbreak() leaves the terminal in +cbreak mode. [clinic start generated code]*/ static PyObject * _curses_cbreak_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9f9dee9664769751 input=c7d0bddda93016c1]*/ +/*[clinic end generated code: output=9f9dee9664769751 input=42d81687f11ddbf3]*/ NoArgOrFlagNoReturnFunctionBody(cbreak, flag) /*[clinic input] @@ -2748,13 +2759,14 @@ _curses.color_content Return the red, green, and blue (RGB) components of the specified color. -A 3-tuple is returned, containing the R, G, B values for the given color, -which will be between 0 (no component) and 1000 (maximum amount of component). +A 3-tuple is returned, containing the R, G, B values for the given +color, which will be between 0 (no component) and 1000 (maximum amount +of component). [clinic start generated code]*/ static PyObject * _curses_color_content_impl(PyObject *module, int color_number) -/*[clinic end generated code: output=17b466df7054e0de input=03b5ed0472662aea]*/ +/*[clinic end generated code: output=17b466df7054e0de input=c95fb50093fa0be0]*/ { _CURSES_COLOR_VAL_TYPE r,g,b; @@ -2780,12 +2792,13 @@ _curses.color_pair Return the attribute value for displaying text in the specified color. This attribute value can be combined with A_STANDOUT, A_REVERSE, and the -other A_* attributes. pair_number() is the counterpart to this function. +other A_* attributes. pair_number() is the counterpart to this +function. [clinic start generated code]*/ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) -/*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ +/*[clinic end generated code: output=60718abb10ce9feb input=cf74bb81d3cc3370]*/ { PyCursesInitialised; PyCursesInitialisedColor; @@ -2803,14 +2816,14 @@ _curses.curs_set Set the cursor state. If the terminal supports the visibility requested, the previous cursor -state is returned; otherwise, an exception is raised. On many terminals, -the "visible" mode is an underline cursor and the "very visible" mode is -a block cursor. +state is returned; otherwise, an exception is raised. On many +terminals, the "visible" mode is an underline cursor and the "very +visible" mode is a block cursor. [clinic start generated code]*/ static PyObject * _curses_curs_set_impl(PyObject *module, int visibility) -/*[clinic end generated code: output=ee8e62483b1d6cd4 input=81a7924a65d29504]*/ +/*[clinic end generated code: output=ee8e62483b1d6cd4 input=e010767a328f322b]*/ { int erg; @@ -2842,14 +2855,15 @@ _curses.def_shell_mode Save the current terminal mode as the "shell" mode. -The "shell" mode is the mode when the running program is not using curses. +The "shell" mode is the mode when the running program is not using +curses. Subsequent calls to reset_shell_mode() will restore this mode. [clinic start generated code]*/ static PyObject * _curses_def_shell_mode_impl(PyObject *module) -/*[clinic end generated code: output=d6e42f5c768f860f input=5ead21f6f0baa894]*/ +/*[clinic end generated code: output=d6e42f5c768f860f input=3809f85615c0b693]*/ NoArgNoReturnFunctionBody(def_shell_mode) /*[clinic input] @@ -2891,12 +2905,13 @@ _curses.echo Enter echo mode. -In echo mode, each character input is echoed to the screen as it is entered. +In echo mode, each character input is echoed to the screen as it is +entered. [clinic start generated code]*/ static PyObject * _curses_echo_impl(PyObject *module, int flag) -/*[clinic end generated code: output=03acb2ddfa6c8729 input=86cd4d5bb1d569c0]*/ +/*[clinic end generated code: output=03acb2ddfa6c8729 input=b4e9064326da9da4]*/ NoArgOrFlagNoReturnFunctionBody(echo, flag) /*[clinic input] @@ -2934,12 +2949,13 @@ _curses.flash Flash the screen. -That is, change it to reverse-video and then change it back in a short interval. +That is, change it to reverse-video and then change it back in a short +interval. [clinic start generated code]*/ static PyObject * _curses_flash_impl(PyObject *module) -/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=02fdfb06c8fc3171]*/ +/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=90878e305432add9]*/ NoArgNoReturnFunctionBody(flash) /*[clinic input] @@ -2947,13 +2963,13 @@ _curses.flushinp Flush all input buffers. -This throws away any typeahead that has been typed by the user and has not -yet been processed by the program. +This throws away any typeahead that has been typed by the user and has +not yet been processed by the program. [clinic start generated code]*/ static PyObject * _curses_flushinp_impl(PyObject *module) -/*[clinic end generated code: output=7e7a1fc1473960f5 input=59d042e705cef5ec]*/ +/*[clinic end generated code: output=7e7a1fc1473960f5 input=3a63c7213be8043c]*/ NoArgNoReturnVoidFunctionBody(flushinp) #ifdef getsyx @@ -3225,13 +3241,14 @@ _curses.init_pair Change the definition of a color-pair. -If the color-pair was previously initialized, the screen is refreshed and -all occurrences of that color-pair are changed to the new definition. +If the color-pair was previously initialized, the screen is refreshed +and all occurrences of that color-pair are changed to the new +definition. [clinic start generated code]*/ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) -/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ +/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=5486c3a105130dae]*/ { PyCursesInitialised; PyCursesInitialisedColor; @@ -3435,14 +3452,14 @@ _curses.get_escdelay Gets the curses ESCDELAY setting. -Gets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Gets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_get_escdelay_impl(PyObject *module) -/*[clinic end generated code: output=222fa1a822555d60 input=be2d5b3dd974d0a4]*/ +/*[clinic end generated code: output=222fa1a822555d60 input=b39eeae4b8f169ab]*/ { return PyLong_FromLong(ESCDELAY); } @@ -3454,14 +3471,14 @@ _curses.set_escdelay Sets the curses ESCDELAY setting. -Sets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Sets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_set_escdelay_impl(PyObject *module, int ms) -/*[clinic end generated code: output=43818efbf7980ac4 input=7796fe19f111e250]*/ +/*[clinic end generated code: output=43818efbf7980ac4 input=cc2529bcdda3b06c]*/ { if (ms <= 0) { PyErr_SetString(PyExc_ValueError, "ms must be > 0"); @@ -3476,13 +3493,13 @@ _curses.get_tabsize Gets the curses TABSIZE setting. -Gets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Gets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_get_tabsize_impl(PyObject *module) -/*[clinic end generated code: output=7e9e51fb6126fbdf input=74af86bf6c9f5d7e]*/ +/*[clinic end generated code: output=7e9e51fb6126fbdf input=58bdaacb337c103b]*/ { return PyLong_FromLong(TABSIZE); } @@ -3494,13 +3511,13 @@ _curses.set_tabsize Sets the curses TABSIZE setting. -Sets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Sets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_set_tabsize_impl(PyObject *module, int size) -/*[clinic end generated code: output=c1de5a76c0daab1e input=78cba6a3021ad061]*/ +/*[clinic end generated code: output=c1de5a76c0daab1e input=34c1be9a78cd28a2]*/ { if (size <= 0) { PyErr_SetString(PyExc_ValueError, "size must be > 0"); @@ -3611,13 +3628,13 @@ _curses.longname Return the terminfo long name field describing the current terminal. -The maximum length of a verbose description is 128 characters. It is defined -only after the call to initscr(). +The maximum length of a verbose description is 128 characters. It is +defined only after the call to initscr(). [clinic start generated code]*/ static PyObject * _curses_longname_impl(PyObject *module) -/*[clinic end generated code: output=fdf30433727ef568 input=84c3f20201b1098e]*/ +/*[clinic end generated code: output=fdf30433727ef568 input=a924fabba0de78a6]*/ NoArgReturnStringFunctionBody(longname) /*[clinic input] @@ -3652,13 +3669,13 @@ _curses.mouseinterval Set and retrieve the maximum time between press and release in a click. Set the maximum time that can elapse between press and release events in -order for them to be recognized as a click, and return the previous interval -value. +order for them to be recognized as a click, and return the previous +interval value. [clinic start generated code]*/ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) -/*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ +/*[clinic end generated code: output=c4f5ff04354634c5 input=b90249254389c080]*/ { PyCursesInitialised; @@ -3674,14 +3691,15 @@ _curses.mousemask Set the mouse events to be reported, and return a tuple (availmask, oldmask). Return a tuple (availmask, oldmask). availmask indicates which of the -specified mouse events can be reported; on complete failure it returns 0. -oldmask is the previous value of the given window's mouse event mask. -If this function is never called, no mouse events are ever reported. +specified mouse events can be reported; on complete failure it returns +0. oldmask is the previous value of the given window's mouse event +mask. If this function is never called, no mouse events are ever +reported. [clinic start generated code]*/ static PyObject * _curses_mousemask_impl(PyObject *module, unsigned long newmask) -/*[clinic end generated code: output=9406cf1b8a36e485 input=bdf76b7568a3c541]*/ +/*[clinic end generated code: output=9406cf1b8a36e485 input=e0b02b620ab30644]*/ { mmask_t oldmask, availmask; @@ -3762,14 +3780,14 @@ _curses.newwin Return a new window. -By default, the window will extend from the specified position to the lower -right corner of the screen. +By default, the window will extend from the specified position to the +lower right corner of the screen. [clinic start generated code]*/ static PyObject * _curses_newwin_impl(PyObject *module, int nlines, int ncols, int group_right_1, int begin_y, int begin_x) -/*[clinic end generated code: output=c1e0a8dc8ac2826c input=29312c15a72a003d]*/ +/*[clinic end generated code: output=c1e0a8dc8ac2826c input=a1517cbfea4ab24b]*/ { WINDOW *win; @@ -3793,13 +3811,14 @@ _curses.nl Enter newline mode. -This mode translates the return key into newline on input, and translates -newline into return and line-feed on output. Newline mode is initially on. +This mode translates the return key into newline on input, and +translates newline into return and line-feed on output. Newline mode +is initially on. [clinic start generated code]*/ static PyObject * _curses_nl_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b39cc0ffc9015003 input=18e3e9c6e8cfcf6f]*/ +/*[clinic end generated code: output=b39cc0ffc9015003 input=3fb21dcf55521ee4]*/ NoArgOrFlagNoReturnFunctionBody(nl, flag) /*[clinic input] @@ -3833,13 +3852,13 @@ _curses.nonl Leave newline mode. -Disable translation of return into newline on input, and disable low-level -translation of newline into newline/return on output. +Disable translation of return into newline on input, and disable +low-level translation of newline into newline/return on output. [clinic start generated code]*/ static PyObject * _curses_nonl_impl(PyObject *module) -/*[clinic end generated code: output=99e917e9715770c6 input=9d37dd122d3022fc]*/ +/*[clinic end generated code: output=99e917e9715770c6 input=75cce08e4b6b3ef1]*/ NoArgNoReturnFunctionBody(nonl) /*[clinic input] @@ -4046,13 +4065,13 @@ _curses.raw Enter raw mode. In raw mode, normal line buffering and processing of interrupt, quit, -suspend, and flow control keys are turned off; characters are presented to -curses input functions one by one. +suspend, and flow control keys are turned off; characters are presented +to curses input functions one by one. [clinic start generated code]*/ static PyObject * _curses_raw_impl(PyObject *module, int flag) -/*[clinic end generated code: output=a750e4b342be015b input=4b447701389fb4df]*/ +/*[clinic end generated code: output=a750e4b342be015b input=18a7de7eef16987a]*/ NoArgOrFlagNoReturnFunctionBody(raw, flag) /*[clinic input] @@ -4100,13 +4119,13 @@ _curses.resizeterm Resize the standard and current windows to the specified dimensions. -Adjusts other bookkeeping data used by the curses library that record the -window dimensions (in particular the SIGWINCH handler). +Adjusts other bookkeeping data used by the curses library that record +the window dimensions (in particular the SIGWINCH handler). [clinic start generated code]*/ static PyObject * _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ +/*[clinic end generated code: output=4de3abab50c67f02 input=7f0f077df2da1cf5]*/ { PyObject *result; @@ -4137,15 +4156,16 @@ _curses.resize_term Backend function used by resizeterm(), performing most of the work. When resizing the windows, resize_term() blank-fills the areas that are -extended. The calling application should fill in these areas with appropriate -data. The resize_term() function attempts to resize all windows. However, -due to the calling convention of pads, it is not possible to resize these -without additional interaction with the application. +extended. The calling application should fill in these areas with +appropriate data. The resize_term() function attempts to resize all +windows. However, due to the calling convention of pads, it is not +possible to resize these without additional interaction with the +application. [clinic start generated code]*/ static PyObject * _curses_resize_term_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=46c6d749fa291dbd input=276afa43d8ea7091]*/ +/*[clinic end generated code: output=46c6d749fa291dbd input=ff4baaf2320c8ac9]*/ { PyObject *result; @@ -4205,17 +4225,17 @@ _curses.start_color Initializes eight basic colors and global variables COLORS and COLOR_PAIRS. -Must be called if the programmer wants to use colors, and before any other -color manipulation routine is called. It is good practice to call this -routine right after initscr(). +Must be called if the programmer wants to use colors, and before any +other color manipulation routine is called. It is good practice to call +this routine right after initscr(). -It also restores the colors on the terminal to the values they had when the -terminal was just turned on. +It also restores the colors on the terminal to the values they had when +the terminal was just turned on. [clinic start generated code]*/ static PyObject * _curses_start_color_impl(PyObject *module) -/*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ +/*[clinic end generated code: output=8b772b41d8090ede input=0eed9bc743283a8d]*/ { int code; PyObject *c, *cp; @@ -4301,13 +4321,13 @@ _curses.tigetnum Return the value of the numeric capability. -The value -2 is returned if capname is not a numeric capability, or -1 if -it is canceled or absent from the terminal description. +The value -2 is returned if capname is not a numeric capability, or -1 +if it is canceled or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ +/*[clinic end generated code: output=46f8b0a1b5dff42f input=87a64beec16ae077]*/ { PyCursesSetupTermCalled; @@ -4323,13 +4343,13 @@ _curses.tigetstr Return the value of the string capability. -None is returned if capname is not a string capability, or is canceled or -absent from the terminal description. +None is returned if capname is not a string capability, or is canceled +or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ +/*[clinic end generated code: output=f22b576ad60248f3 input=00bf0feda2207724]*/ { PyCursesSetupTermCalled; @@ -4529,19 +4549,19 @@ _curses.use_env Use environment variables LINES and COLUMNS. -If used, this function should be called before initscr() or newterm() are -called. +If used, this function should be called before initscr() or newterm() +are called. -When flag is False, the values of lines and columns specified in the terminfo -database will be used, even if environment variables LINES and COLUMNS (used -by default) are set, or if curses is running in a window (in which case -default behavior would be to use the window size if LINES and COLUMNS are -not set). +When flag is False, the values of lines and columns specified in the +terminfo database will be used, even if environment variables LINES and +COLUMNS (used by default) are set, or if curses is running in a window +(in which case default behavior would be to use the window size if LINES +and COLUMNS are not set). [clinic start generated code]*/ static PyObject * _curses_use_env_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b2c445e435c0b164 input=06ac30948f2d78e4]*/ +/*[clinic end generated code: output=b2c445e435c0b164 input=8e8feed746cf7fc1]*/ { use_env(flag); Py_RETURN_NONE; diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8cfa04f676d..8df5913dfae 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3208,13 +3208,13 @@ datetime.date.fromtimestamp Create a date from a POSIX timestamp. -The timestamp is a number, e.g. created via time.time(), that is interpreted -as local time. +The timestamp is a number, e.g. created via time.time(), that is +interpreted as local time. [clinic start generated code]*/ static PyObject * datetime_date_fromtimestamp(PyTypeObject *type, PyObject *timestamp) -/*[clinic end generated code: output=fd045fda58168869 input=eabb3fe7f40491fe]*/ +/*[clinic end generated code: output=fd045fda58168869 input=15720eef43b169a1]*/ { return date_fromtimestamp((PyObject *) type, timestamp); } diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 1be4234aad3..5b27541f58c 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -374,13 +374,14 @@ _dbm.dbm.setdefault Return the value for key if present, otherwise default. -If key is not in the database, it is inserted with default as the value. +If key is not in the database, it is inserted with default as the +value. [clinic start generated code]*/ static PyObject * _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value) -/*[clinic end generated code: output=9c2f6ea6d0fb576c input=126a3ff15c5f8232]*/ +/*[clinic end generated code: output=9c2f6ea6d0fb576c input=87f3abca9f55f59d]*/ { datum dbm_key, val; Py_ssize_t tmp_size; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index dfa03663c38..f2e2f1237f1 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -16,6 +16,7 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_import.h" // _PyImport_GetModuleAttrString() #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -563,7 +564,7 @@ element_get_attrib(ElementObject* self) LOCAL(PyObject*) element_get_text(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to text attribute */ PyObject *res = self->text; @@ -578,13 +579,13 @@ element_get_text(ElementObject* self) } } - return res; + return Py_NewRef(res); } LOCAL(PyObject*) element_get_tail(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to tail attribute */ PyObject *res = self->tail; @@ -599,7 +600,7 @@ element_get_tail(ElementObject* self) } } - return res; + return Py_NewRef(res); } static PyObject* @@ -801,26 +802,31 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) /*[clinic end generated code: output=eefc3df50465b642 input=a2d40348c0aade10]*/ { Py_ssize_t i; - ElementObject* element; + ElementObject* element = NULL; PyObject* tag; PyObject* attrib; PyObject* text; PyObject* tail; PyObject* id; + if (_Py_EnterRecursiveCall(" in Element.__deepcopy__")) { + return NULL; + } + PyTypeObject *tp = Py_TYPE(self); elementtreestate *st = get_elementtree_state_by_type(tp); // The deepcopy() helper takes care of incrementing the refcount // of the object to copy so to avoid use-after-frees. tag = deepcopy(st, self->tag, memo); - if (!tag) - return NULL; + if (!tag) { + goto error; + } if (self->extra && self->extra->attrib) { attrib = deepcopy(st, self->extra->attrib, memo); if (!attrib) { Py_DECREF(tag); - return NULL; + goto error; } } else { attrib = NULL; @@ -831,8 +837,9 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) Py_DECREF(tag); Py_XDECREF(attrib); - if (!element) - return NULL; + if (!element) { + goto error; + } text = deepcopy(st, JOIN_OBJ(self->text), memo); if (!text) @@ -894,10 +901,12 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) if (i < 0) goto error; + _Py_LeaveRecursiveCall(); return (PyObject*) element; error: - Py_DECREF(element); + _Py_LeaveRecursiveCall(); + Py_XDECREF(element); return NULL; } @@ -1340,9 +1349,9 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject *text = element_get_text((ElementObject *)item); Py_DECREF(item); if (text == Py_None) { + Py_DECREF(text); return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } - Py_XINCREF(text); return text; } Py_DECREF(item); @@ -2042,15 +2051,13 @@ element_tag_getter(ElementObject *self, void *closure) static PyObject* element_text_getter(ElementObject *self, void *closure) { - PyObject *res = element_get_text(self); - return Py_XNewRef(res); + return element_get_text(self); } static PyObject* element_tail_getter(ElementObject *self, void *closure) { - PyObject *res = element_get_tail(self); - return Py_XNewRef(res); + return element_get_tail(self); } static PyObject* @@ -2266,6 +2273,10 @@ elementiter_next(ElementIterObject *it) return NULL; } if (it->gettext) { + if (elem->tag != Py_None && !PyUnicode_Check(elem->tag)) { + Py_DECREF(elem); + continue; + } text = element_get_text(elem); goto gettext; } @@ -2283,16 +2294,14 @@ elementiter_next(ElementIterObject *it) continue; gettext: + Py_DECREF(elem); if (!text) { - Py_DECREF(elem); return NULL; } if (text == Py_None) { - Py_DECREF(elem); + Py_DECREF(text); } else { - Py_INCREF(text); - Py_DECREF(elem); rc = PyObject_IsTrue(text); if (rc > 0) return text; @@ -3687,8 +3696,12 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target, PyErr_NoMemory(); return -1; } - /* expat < 2.1.0 has no XML_SetHashSalt() */ - if (EXPAT(st, SetHashSalt) != NULL) { + // Prefer 16-byte entropy, only expat >= 2.8.0. See gh-149018 + if (EXPAT(st, SetHashSalt16Bytes) != NULL) { + EXPAT(st, SetHashSalt16Bytes)(self->parser, + _Py_HashSecret.expat.hashsalt16); + } + else if (EXPAT(st, SetHashSalt) != NULL) { EXPAT(st, SetHashSalt)(self->parser, (unsigned long)_Py_HashSecret.expat.hashsalt); } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index a1682ec4c0e..08acd2c5ab9 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -522,7 +522,8 @@ static PyMethodDef partial_methods[] = { {"__reduce__", (PyCFunction)partial_reduce, METH_NOARGS}, {"__setstate__", (PyCFunction)partial_setstate, METH_O}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("partial is generic over the wrapped function's return type")}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 3fe08004790..0f49e0a301e 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -463,14 +463,14 @@ _gdbm.gdbm.firstkey Return the starting key for the traversal. -It's possible to loop over every key in the database using this method -and the nextkey() method. The traversal is ordered by GDBM's internal -hash values, and won't be sorted by the key values. +It's possible to loop over every key in the database using this +method and the nextkey() method. The traversal is ordered by GDBM's +internal hash values, and won't be sorted by the key values. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=139275e9c8b60827 input=ed8782a029a5d299]*/ +/*[clinic end generated code: output=139275e9c8b60827 input=27874d56cf39ac72]*/ { PyObject *v; datum key; @@ -498,8 +498,8 @@ _gdbm.gdbm.nextkey Returns the key that follows key in the traversal. -The following code prints every key in the database db, without having -to create a list in memory that contains them all: +The following code prints every key in the database db, without +having to create a list in memory that contains them all: k = db.firstkey() while k is not None: @@ -510,7 +510,7 @@ to create a list in memory that contains them all: static PyObject * _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) -/*[clinic end generated code: output=c81a69300ef41766 input=365e297bc0b3db48]*/ +/*[clinic end generated code: output=c81a69300ef41766 input=143ca0c63b442738]*/ { PyObject *v; datum dbm_key, nextkey; @@ -540,14 +540,14 @@ Reorganize the database. If you have carried out a lot of deletions and would like to shrink the space used by the GDBM file, this routine will reorganize the -database. GDBM will not shorten the length of a database file except -by using this reorganization; otherwise, deleted file space will be -kept and reused as new (key,value) pairs are added. +database. GDBM will not shorten the length of a database file +except by using this reorganization; otherwise, deleted file space +will be kept and reused as new (key,value) pairs are added. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=d77c69e8e3dd644a input=e1359faeef844e46]*/ +/*[clinic end generated code: output=d77c69e8e3dd644a input=3efcec3838a4fdb6]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index bf3894e4ffb..8a4fe473684 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -1870,13 +1870,13 @@ _hashlib.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. -This may be used to exchange the value safely in email or other non-binary -environments. +This may be used to exchange the value safely in email or other +non-binary environments. [clinic start generated code]*/ static PyObject * _hashlib_HMAC_hexdigest_impl(HMACobject *self) -/*[clinic end generated code: output=80d825be1eaae6a7 input=5abc42702874ddcf]*/ +/*[clinic end generated code: output=80d825be1eaae6a7 input=865e6128c7ec0781]*/ { unsigned char digest[EVP_MAX_MD_SIZE]; unsigned int digest_size = _hashlib_hmac_digest_size(self); @@ -2038,8 +2038,8 @@ _hashlib.get_fips_mode -> int Determine the OpenSSL FIPS mode of operation. For OpenSSL 3.0.0 and newer it returns the state of the default provider -in the default OSSL context. It's not quite the same as FIPS_mode() but good -enough for unittests. +in the default OSSL context. It's not quite the same as FIPS_mode() but +good enough for unittests. Effectively any non-zero return value indicates FIPS mode; values other than 1 may have additional significance. @@ -2047,7 +2047,7 @@ values other than 1 may have additional significance. static int _hashlib_get_fips_mode_impl(PyObject *module) -/*[clinic end generated code: output=87eece1bab4d3fa9 input=2db61538c41c6fef]*/ +/*[clinic end generated code: output=87eece1bab4d3fa9 input=a6cdb6901421d122]*/ { #if OPENSSL_VERSION_NUMBER >= 0x30000000L diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 460b4e5f4ea..4d5099f3807 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -2553,6 +2553,7 @@ _channelid_from_xid(_PyCrossInterpreterData *data) { struct _channelid_xid *xid = \ (struct _channelid_xid *)_PyCrossInterpreterData_DATA(data); + PyObject *cidobj = NULL; // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2562,11 +2563,10 @@ _channelid_from_xid(_PyCrossInterpreterData *data) assert(mod != Py_None); module_state *state = get_module_state(mod); if (state == NULL) { - return NULL; + goto done; } // Note that we do not preserve the "resolve" flag. - PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index d84b53d9021..7a89e2f13a1 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1355,6 +1355,7 @@ _queueobj_from_xid(_PyCrossInterpreterData *data) // XXX import it? PyErr_SetString(PyExc_RuntimeError, MODULE_NAME_STR " module not imported yet"); + Py_DECREF(qidobj); return NULL; } diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 269070fe2b0..6c60a3b0b76 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -86,112 +86,114 @@ Open file and return a stream. Raise OSError upon failure. file is either a text or byte string giving the name (and the path if the file isn't in the current working directory) of the file to be opened or an integer file descriptor of the file to be -wrapped. (If a file descriptor is given, it is closed when the +wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) mode is an optional string that specifies the mode in which the file -is opened. It defaults to 'r' which means open for reading in text +is opened. It defaults to 'r' which means open for reading in text mode. Other common values are 'w' for writing (truncating the file if it already exists), 'x' for creating and writing to a new file, and 'a' for appending (which on some Unix systems, means that all writes append to the end of the file regardless of the current seek position). In text mode, if encoding is not specified the encoding used is platform -dependent: locale.getencoding() is called to get the current locale encoding. -(For reading and writing raw bytes use binary mode and leave encoding -unspecified.) The available modes are: +dependent: locale.getencoding() is called to get the current locale +encoding. (For reading and writing raw bytes use binary mode and leave +encoding unspecified.) The available modes are: -========= =============================================================== +========= ========================================================== Character Meaning ---------- --------------------------------------------------------------- +--------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing -'a' open for writing, appending to the end of the file if it exists +'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) -========= =============================================================== +========= ========================================================== -The default mode is 'rt' (open for reading text). For binary random +The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while -'r+b' opens the file without truncation. The 'x' mode implies 'w' and +'r+b' opens the file without truncation. The 'x' mode implies 'w' and raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, -even when the underlying operating system doesn't. Files opened in +even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as -bytes objects without any decoding. In text mode (the default, or when +bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. -Pass 0 to switch buffering off (only allowed in binary mode), 1 to select -line buffering (only usable in text mode), and an integer > 1 to indicate -the size of a fixed-size chunk buffer. When no buffering argument is -given, the default buffering policy works as follows: +Pass 0 to switch buffering off (only allowed in binary mode), 1 to +select line buffering (only usable in text mode), and an integer > 1 to +indicate the size of a fixed-size chunk buffer. When no buffering +argument is given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer - is chosen using a heuristic trying to determine the underlying device's - "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. - On many systems, the buffer will typically be 4096 or 8192 bytes long. + is chosen using a heuristic trying to determine the underlying + device's "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. + On many systems, the buffer will typically be 4096 or 8192 bytes + long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above for binary files. encoding is the name of the encoding used to decode or encode the -file. This should only be used in text mode. The default encoding is +file. This should only be used in text mode. The default encoding is platform dependent, but any encoding supported by Python can be passed. See the codecs module for the list of supported encodings. errors is an optional string that specifies how encoding errors are to -be handled---this argument should not be used in binary mode. Pass +be handled---this argument should not be used in binary mode. Pass 'strict' to raise a ValueError exception if there is an encoding error (the default of None has the same effect), or pass 'ignore' to ignore -errors. (Note that ignoring encoding errors can lead to data loss.) +errors. (Note that ignoring encoding errors can lead to data loss.) See the documentation for codecs.register or run 'help(codecs.Codec)' for a list of the permitted encoding error strings. newline controls how universal newlines works (it only applies to text -mode). It can be None, '', '\n', '\r', and '\r\n'. It works as +mode). It can be None, '', '\n', '\r', and '\r\n'. It works as follows: -* On input, if newline is None, universal newlines mode is - enabled. Lines in the input can end in '\n', '\r', or '\r\n', and - these are translated into '\n' before being returned to the - caller. If it is '', universal newline mode is enabled, but line - endings are returned to the caller untranslated. If it has any of - the other legal values, input lines are only terminated by the given - string, and the line ending is returned to the caller untranslated. +* On input, if newline is None, universal newlines mode is enabled. + Lines in the input can end in '\n', '\r', or '\r\n', and these are + translated into '\n' before being returned to the caller. If it is + '', universal newline mode is enabled, but line endings are returned + to the caller untranslated. If it has any of the other legal values, + input lines are only terminated by the given string, and the line + ending is returned to the caller untranslated. * On output, if newline is None, any '\n' characters written are - translated to the system default line separator, os.linesep. If - newline is '' or '\n', no translation takes place. If newline is any + translated to the system default line separator, os.linesep. If + newline is '' or '\n', no translation takes place. If newline is any of the other legal values, any '\n' characters written are translated to the given string. If closefd is False, the underlying file descriptor will be kept open -when the file is closed. This does not work when a file name is given +when the file is closed. This does not work when a file name is given and must be True in that case. -A custom opener can be used by passing a callable as *opener*. The +A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by -calling *opener* with (*file*, *flags*). *opener* must return an open +calling *opener* with (*file*, *flags*). *opener* must return an open file descriptor (passing os.open as *opener* results in functionality similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing -are performed. When open() is used to open a file in a text mode ('w', -'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open +are performed. When open() is used to open a file in a text mode ('w', +'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open a file in a binary mode, the returned class varies: in read binary mode, it returns a BufferedReader; in write binary and append binary modes, it returns a BufferedWriter, and in read/write mode, it returns a BufferedRandom. It is also possible to use a string or bytearray as a file for both -reading and writing. For strings StringIO can be used like a file +reading and writing. For strings StringIO can be used like a file opened in a text mode, and for bytes a BytesIO can be used like a file opened in a binary mode. [clinic start generated code]*/ @@ -200,7 +202,7 @@ static PyObject * _io_open_impl(PyObject *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/ +/*[clinic end generated code: output=aefafc4ce2b46dc0 input=202d0aefc19a36c5]*/ { unsigned i; @@ -504,14 +506,14 @@ _io.open_code Opens the provided file with the intent to import the contents. -This may perform extra validation beyond open(), but is otherwise interchangeable -with calling open(path, 'rb'). +This may perform extra validation beyond open(), but is otherwise +interchangeable with calling open(path, 'rb'). [clinic start generated code]*/ static PyObject * _io_open_code_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=f5c18e23f4b2ed9f]*/ +/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=2803c35aeb63c719]*/ { return PyFile_OpenCodeObject(path); } diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index e149823afd3..38d47aafa33 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -457,13 +457,13 @@ _io.BytesIO.read1 Read at most size bytes, returned as a bytes object. -If the size argument is negative or omitted, read until EOF is reached. -Return an empty bytes object at EOF. +If the size argument is negative or omitted, read until EOF is +reached. Return an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=d0f843285aa95f1c input=440a395bf9129ef5]*/ +/*[clinic end generated code: output=d0f843285aa95f1c input=418f5968cb4f2a01]*/ { return _io_BytesIO_read_impl(self, size); } @@ -730,13 +730,13 @@ _io.BytesIO.writelines Write lines to the file. Note that newlines are not added. lines can be any iterable object -producing bytes-like objects. This is equivalent to calling write() for -each element. +producing bytes-like objects. This is equivalent to calling write() +for each element. [clinic start generated code]*/ static PyObject * _io_BytesIO_writelines(bytesio *self, PyObject *lines) -/*[clinic end generated code: output=7f33aa3271c91752 input=e972539176fc8fc1]*/ +/*[clinic end generated code: output=7f33aa3271c91752 input=f4e24a931ef1d528]*/ { PyObject *it, *item; diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 112408a95df..356b38f058f 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -18,112 +18,114 @@ PyDoc_STRVAR(_io_open__doc__, "file is either a text or byte string giving the name (and the path\n" "if the file isn\'t in the current working directory) of the file to\n" "be opened or an integer file descriptor of the file to be\n" -"wrapped. (If a file descriptor is given, it is closed when the\n" +"wrapped. (If a file descriptor is given, it is closed when the\n" "returned I/O object is closed, unless closefd is set to False.)\n" "\n" "mode is an optional string that specifies the mode in which the file\n" -"is opened. It defaults to \'r\' which means open for reading in text\n" +"is opened. It defaults to \'r\' which means open for reading in text\n" "mode. Other common values are \'w\' for writing (truncating the file if\n" "it already exists), \'x\' for creating and writing to a new file, and\n" "\'a\' for appending (which on some Unix systems, means that all writes\n" "append to the end of the file regardless of the current seek position).\n" "In text mode, if encoding is not specified the encoding used is platform\n" -"dependent: locale.getencoding() is called to get the current locale encoding.\n" -"(For reading and writing raw bytes use binary mode and leave encoding\n" -"unspecified.) The available modes are:\n" +"dependent: locale.getencoding() is called to get the current locale\n" +"encoding. (For reading and writing raw bytes use binary mode and leave\n" +"encoding unspecified.) The available modes are:\n" "\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "Character Meaning\n" -"--------- ---------------------------------------------------------------\n" +"--------- ----------------------------------------------------------\n" "\'r\' open for reading (default)\n" "\'w\' open for writing, truncating the file first\n" "\'x\' create a new file and open it for writing\n" -"\'a\' open for writing, appending to the end of the file if it exists\n" +"\'a\' open for writing, appending to the end of the file if it\n" +" exists\n" "\'b\' binary mode\n" "\'t\' text mode (default)\n" "\'+\' open a disk file for updating (reading and writing)\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "\n" -"The default mode is \'rt\' (open for reading text). For binary random\n" +"The default mode is \'rt\' (open for reading text). For binary random\n" "access, the mode \'w+b\' opens and truncates the file to 0 bytes, while\n" -"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" +"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" "raises an `FileExistsError` if the file already exists.\n" "\n" "Python distinguishes between files opened in binary and text modes,\n" -"even when the underlying operating system doesn\'t. Files opened in\n" +"even when the underlying operating system doesn\'t. Files opened in\n" "binary mode (appending \'b\' to the mode argument) return contents as\n" -"bytes objects without any decoding. In text mode (the default, or when\n" +"bytes objects without any decoding. In text mode (the default, or when\n" "\'t\' is appended to the mode argument), the contents of the file are\n" "returned as strings, the bytes having been first decoded using a\n" "platform-dependent encoding or using the specified encoding if given.\n" "\n" "buffering is an optional integer used to set the buffering policy.\n" -"Pass 0 to switch buffering off (only allowed in binary mode), 1 to select\n" -"line buffering (only usable in text mode), and an integer > 1 to indicate\n" -"the size of a fixed-size chunk buffer. When no buffering argument is\n" -"given, the default buffering policy works as follows:\n" +"Pass 0 to switch buffering off (only allowed in binary mode), 1 to\n" +"select line buffering (only usable in text mode), and an integer > 1 to\n" +"indicate the size of a fixed-size chunk buffer. When no buffering\n" +"argument is given, the default buffering policy works as follows:\n" "\n" "* Binary files are buffered in fixed-size chunks; the size of the buffer\n" -" is chosen using a heuristic trying to determine the underlying device\'s\n" -" \"block size\" and falling back on `io.DEFAULT_BUFFER_SIZE`.\n" -" On many systems, the buffer will typically be 4096 or 8192 bytes long.\n" +" is chosen using a heuristic trying to determine the underlying\n" +" device\'s \"block size\" and falling back on `io.DEFAULT_BUFFER_SIZE`.\n" +" On many systems, the buffer will typically be 4096 or 8192 bytes\n" +" long.\n" "\n" "* \"Interactive\" text files (files for which isatty() returns True)\n" " use line buffering. Other text files use the policy described above\n" " for binary files.\n" "\n" "encoding is the name of the encoding used to decode or encode the\n" -"file. This should only be used in text mode. The default encoding is\n" +"file. This should only be used in text mode. The default encoding is\n" "platform dependent, but any encoding supported by Python can be\n" "passed. See the codecs module for the list of supported encodings.\n" "\n" "errors is an optional string that specifies how encoding errors are to\n" -"be handled---this argument should not be used in binary mode. Pass\n" +"be handled---this argument should not be used in binary mode. Pass\n" "\'strict\' to raise a ValueError exception if there is an encoding error\n" "(the default of None has the same effect), or pass \'ignore\' to ignore\n" -"errors. (Note that ignoring encoding errors can lead to data loss.)\n" +"errors. (Note that ignoring encoding errors can lead to data loss.)\n" "See the documentation for codecs.register or run \'help(codecs.Codec)\'\n" "for a list of the permitted encoding error strings.\n" "\n" "newline controls how universal newlines works (it only applies to text\n" -"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" +"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" "follows:\n" "\n" -"* On input, if newline is None, universal newlines mode is\n" -" enabled. Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and\n" -" these are translated into \'\\n\' before being returned to the\n" -" caller. If it is \'\', universal newline mode is enabled, but line\n" -" endings are returned to the caller untranslated. If it has any of\n" -" the other legal values, input lines are only terminated by the given\n" -" string, and the line ending is returned to the caller untranslated.\n" +"* On input, if newline is None, universal newlines mode is enabled.\n" +" Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and these are\n" +" translated into \'\\n\' before being returned to the caller. If it is\n" +" \'\', universal newline mode is enabled, but line endings are returned\n" +" to the caller untranslated. If it has any of the other legal values,\n" +" input lines are only terminated by the given string, and the line\n" +" ending is returned to the caller untranslated.\n" "\n" "* On output, if newline is None, any \'\\n\' characters written are\n" -" translated to the system default line separator, os.linesep. If\n" -" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" +" translated to the system default line separator, os.linesep. If\n" +" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" " of the other legal values, any \'\\n\' characters written are translated\n" " to the given string.\n" "\n" "If closefd is False, the underlying file descriptor will be kept open\n" -"when the file is closed. This does not work when a file name is given\n" +"when the file is closed. This does not work when a file name is given\n" "and must be True in that case.\n" "\n" -"A custom opener can be used by passing a callable as *opener*. The\n" +"A custom opener can be used by passing a callable as *opener*. The\n" "underlying file descriptor for the file object is then obtained by\n" -"calling *opener* with (*file*, *flags*). *opener* must return an open\n" +"calling *opener* with (*file*, *flags*). *opener* must return an open\n" "file descriptor (passing os.open as *opener* results in functionality\n" "similar to passing None).\n" "\n" "open() returns a file object whose type depends on the mode, and\n" "through which the standard file operations such as reading and writing\n" -"are performed. When open() is used to open a file in a text mode (\'w\',\n" -"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" +"are performed. When open() is used to open a file in a text mode (\'w\',\n" +"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" "a file in a binary mode, the returned class varies: in read binary\n" "mode, it returns a BufferedReader; in write binary and append binary\n" "modes, it returns a BufferedWriter, and in read/write mode, it returns\n" "a BufferedRandom.\n" "\n" "It is also possible to use a string or bytearray as a file for both\n" -"reading and writing. For strings StringIO can be used like a file\n" +"reading and writing. For strings StringIO can be used like a file\n" "opened in a text mode, and for bytes a BytesIO can be used like a file\n" "opened in a binary mode."); @@ -349,8 +351,8 @@ PyDoc_STRVAR(_io_open_code__doc__, "\n" "Opens the provided file with the intent to import the contents.\n" "\n" -"This may perform extra validation beyond open(), but is otherwise interchangeable\n" -"with calling open(path, \'rb\')."); +"This may perform extra validation beyond open(), but is otherwise\n" +"interchangeable with calling open(path, \'rb\')."); #define _IO_OPEN_CODE_METHODDEF \ {"open_code", _PyCFunction_CAST(_io_open_code), METH_FASTCALL|METH_KEYWORDS, _io_open_code__doc__}, @@ -404,4 +406,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=5d60f4e778a600a4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6fb7864751a4cbd8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 620e9e3b84e..49234d43dd3 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -202,8 +202,8 @@ PyDoc_STRVAR(_io_BytesIO_read1__doc__, "\n" "Read at most size bytes, returned as a bytes object.\n" "\n" -"If the size argument is negative or omitted, read until EOF is reached.\n" -"Return an empty bytes object at EOF."); +"If the size argument is negative or omitted, read until EOF is\n" +"reached. Return an empty bytes object at EOF."); #define _IO_BYTESIO_READ1_METHODDEF \ {"read1", _PyCFunction_CAST(_io_BytesIO_read1), METH_FASTCALL, _io_BytesIO_read1__doc__}, @@ -452,8 +452,8 @@ PyDoc_STRVAR(_io_BytesIO_writelines__doc__, "Write lines to the file.\n" "\n" "Note that newlines are not added. lines can be any iterable object\n" -"producing bytes-like objects. This is equivalent to calling write() for\n" -"each element."); +"producing bytes-like objects. This is equivalent to calling write()\n" +"for each element."); #define _IO_BYTESIO_WRITELINES_METHODDEF \ {"writelines", (PyCFunction)_io_BytesIO_writelines, METH_O, _io_BytesIO_writelines__doc__}, @@ -534,4 +534,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=ef116925b8b9e535 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=502699cd403dd3a0 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 5b5487d63eb..67c9847a5a6 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -15,8 +15,8 @@ PyDoc_STRVAR(_io_FileIO_close__doc__, "\n" "Close the file.\n" "\n" -"A closed file cannot be used for further I/O operations. close() may be\n" -"called more than once without error."); +"A closed file cannot be used for further I/O operations. close()\n" +"may be called more than once without error."); #define _IO_FILEIO_CLOSE_METHODDEF \ {"close", _PyCFunction_CAST(_io_FileIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_close__doc__}, @@ -41,16 +41,19 @@ PyDoc_STRVAR(_io_FileIO___init____doc__, "Open a file.\n" "\n" "The mode can be \'r\' (default), \'w\', \'x\' or \'a\' for reading,\n" -"writing, exclusive creation or appending. The file will be created if it\n" -"doesn\'t exist when opened for writing or appending; it will be truncated\n" -"when opened for writing. A FileExistsError will be raised if it already\n" -"exists when opened for creating. Opening a file for creating implies\n" -"writing so this mode behaves in a similar way to \'w\'.Add a \'+\' to the mode\n" -"to allow simultaneous reading and writing. A custom opener can be used by\n" -"passing a callable as *opener*. The underlying file descriptor for the file\n" -"object is then obtained by calling opener with (*name*, *flags*).\n" -"*opener* must return an open file descriptor (passing os.open as *opener*\n" -"results in functionality similar to passing None)."); +"writing, exclusive creation or appending. The file will be created\n" +"if it doesn\'t exist when opened for writing or appending; it will be\n" +"truncated when opened for writing. A FileExistsError will be raised\n" +"if it already exists when opened for creating. Opening a file for\n" +"creating implies writing so this mode behaves in a similar way to\n" +"\'w\'. Add a \'+\' to the mode to allow simultaneous reading and\n" +"writing.\n" +"\n" +"A custom opener can be used by passing a callable as *opener*.\n" +"The underlying file descriptor for the file object is then obtained\n" +"by calling opener with (*name*, *flags*). *opener* must return\n" +"an open file descriptor (passing os.open as *opener* results in\n" +"functionality similar to passing None)."); static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, @@ -267,7 +270,8 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "Read all data from the file, returned as bytes.\n" "\n" "In non-blocking mode, returns as much as is immediately available,\n" -"or None if no data is available. Return an empty bytes object at EOF."); +"or None if no data is available. Return an empty bytes object at\n" +"EOF."); #define _IO_FILEIO_READALL_METHODDEF \ {"readall", (PyCFunction)_io_FileIO_readall, METH_NOARGS, _io_FileIO_readall__doc__}, @@ -287,9 +291,9 @@ PyDoc_STRVAR(_io_FileIO_read__doc__, "\n" "Read at most size bytes, returned as bytes.\n" "\n" -"Only makes one system call, so less data may be returned than requested.\n" -"In non-blocking mode, returns None if no data is available.\n" -"Return an empty bytes object at EOF."); +"Only makes one system call, so less data may be returned than\n" +"requested. In non-blocking mode, returns None if no data is\n" +"available. Return an empty bytes object at EOF."); #define _IO_FILEIO_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io_FileIO_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_read__doc__}, @@ -341,8 +345,8 @@ PyDoc_STRVAR(_io_FileIO_write__doc__, "Write buffer b to file, return number of bytes written.\n" "\n" "Only makes one system call, so not all of the data may be written.\n" -"The number of bytes actually written is returned. In non-blocking mode,\n" -"returns None if the write would block."); +"The number of bytes actually written is returned. In non-blocking\n" +"mode, returns None if the write would block."); #define _IO_FILEIO_WRITE_METHODDEF \ {"write", _PyCFunction_CAST(_io_FileIO_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_write__doc__}, @@ -394,11 +398,12 @@ PyDoc_STRVAR(_io_FileIO_seek__doc__, "\n" "Move to new file position and return the file position.\n" "\n" -"Argument offset is a byte count. Optional argument whence defaults to\n" -"SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values\n" -"are SEEK_CUR or 1 (move relative to current position, positive or negative),\n" -"and SEEK_END or 2 (move relative to end of file, usually negative, although\n" -"many platforms allow seeking beyond the end of a file).\n" +"Argument offset is a byte count. Optional argument whence defaults\n" +"to SEEK_SET or 0 (offset from start of file, offset should be >= 0);\n" +"other values are SEEK_CUR or 1 (move relative to current position,\n" +"positive or negative), and SEEK_END or 2 (move relative to end of\n" +"file, usually negative, although many platforms allow seeking beyond\n" +"the end of a file).\n" "\n" "Note that not all file objects are seekable."); @@ -528,4 +533,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=e3d9446b4087020e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=287b778e8b575c06 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index a35cac7dc0b..70c03bd1d5a 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -19,11 +19,13 @@ PyDoc_STRVAR(_io__IOBase_seek__doc__, " whence\n" " The relative position to seek from.\n" "\n" -"The offset is interpreted relative to the position indicated by whence.\n" -"Values for whence are:\n" +"The offset is interpreted relative to the position indicated by\n" +"whence. Values for whence are:\n" "\n" -"* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n" -"* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n" +"* os.SEEK_SET or 0 -- start of stream (the default); offset should\n" +" be zero or positive\n" +"* os.SEEK_CUR or 1 -- current stream position; offset may be\n" +" negative\n" "* os.SEEK_END or 2 -- end of stream; offset is usually negative\n" "\n" "Return the new absolute position."); @@ -102,8 +104,8 @@ PyDoc_STRVAR(_io__IOBase_truncate__doc__, "\n" "Truncate file to size bytes.\n" "\n" -"File pointer is left unchanged. Size defaults to the current IO position\n" -"as reported by tell(). Return the new size."); +"File pointer is left unchanged. Size defaults to the current IO\n" +"position as reported by tell(). Return the new size."); #define _IO__IOBASE_TRUNCATE_METHODDEF \ {"truncate", _PyCFunction_CAST(_io__IOBase_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_truncate__doc__}, @@ -441,4 +443,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=dab5e9323d191e32 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=23596f673b4acd03 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index e8259d26e82..cf47c2e8452 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -181,7 +181,8 @@ PyDoc_STRVAR(_io_StringIO_seek__doc__, "\n" "Change stream position.\n" "\n" -"Seek to character offset pos relative to position indicated by whence:\n" +"Seek to character offset pos relative to position indicated by\n" +"whence:\n" " 0 Start of stream (the default). pos should be >= 0;\n" " 1 Current position - pos must be 0;\n" " 2 End of stream - pos must be 0.\n" @@ -549,4 +550,4 @@ _io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=b16edb931e4fa548 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=499d9098fb8b603d input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index efa992fca71..f35538ee75f 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "\n" "Separate the underlying buffer from the TextIOBase and return it.\n" "\n" -"After the underlying buffer has been detached, the TextIO is in an unusable state."); +"After the underlying buffer has been detached, the TextIO is in\n" +"an unusable state."); #define _IO__TEXTIOBASE_DETACH_METHODDEF \ {"detach", _PyCFunction_CAST(_io__TextIOBase_detach), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_detach__doc__}, @@ -40,8 +41,8 @@ PyDoc_STRVAR(_io__TextIOBase_read__doc__, "\n" "Read at most size characters from stream.\n" "\n" -"Read from underlying buffer until we have size characters or we hit EOF.\n" -"If size is negative or omitted, read until EOF."); +"Read from underlying buffer until we have size characters or we hit\n" +"EOF. If size is negative or omitted, read until EOF."); #define _IO__TEXTIOBASE_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io__TextIOBase_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_read__doc__}, @@ -623,7 +624,9 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper___init___impl((textio *)self, buffer, encoding, errors, newline, line_buffering, write_through); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -919,8 +922,8 @@ PyDoc_STRVAR(_io_TextIOWrapper_tell__doc__, "\n" "Return the stream position as an opaque number.\n" "\n" -"The return value of tell() can be given as input to seek(), to restore a\n" -"previous stream position."); +"The return value of tell() can be given as input to seek(), to\n" +"restore a previous stream position."); #define _IO_TEXTIOWRAPPER_TELL_METHODDEF \ {"tell", (PyCFunction)_io_TextIOWrapper_tell, METH_NOARGS, _io_TextIOWrapper_tell__doc__}, @@ -1283,4 +1286,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=6f78a6d99fad4893 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2d73d5adb0ce09a8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 4696ecc5c84..8e4ae031204 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -46,9 +46,9 @@ PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "\n" "Open a console buffer by file descriptor.\n" "\n" -"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n" -"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n" -"omitted. The *opener* parameter is always ignored."); +"The mode can be \'rb\' (default), or \'wb\' for reading or writing\n" +"bytes. All other mode characters will be ignored. Mode \'b\' will be\n" +"assumed if it is omitted. The *opener* parameter is always ignored."); static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, @@ -457,4 +457,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=2c2bc86713b21dd6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2d7f2c99f2ab3e17 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 65957a713b6..95219661575 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -140,13 +140,13 @@ _io.FileIO.close Close the file. -A closed file cannot be used for further I/O operations. close() may be -called more than once without error. +A closed file cannot be used for further I/O operations. close() +may be called more than once without error. [clinic start generated code]*/ static PyObject * _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) -/*[clinic end generated code: output=c30cbe9d1f23ca58 input=70da49e63db7c64d]*/ +/*[clinic end generated code: output=c30cbe9d1f23ca58 input=b405751dc4163da3]*/ { PyObject *res; int rc; @@ -218,22 +218,25 @@ _io.FileIO.__init__ Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, -writing, exclusive creation or appending. The file will be created if it -doesn't exist when opened for writing or appending; it will be truncated -when opened for writing. A FileExistsError will be raised if it already -exists when opened for creating. Opening a file for creating implies -writing so this mode behaves in a similar way to 'w'.Add a '+' to the mode -to allow simultaneous reading and writing. A custom opener can be used by -passing a callable as *opener*. The underlying file descriptor for the file -object is then obtained by calling opener with (*name*, *flags*). -*opener* must return an open file descriptor (passing os.open as *opener* -results in functionality similar to passing None). +writing, exclusive creation or appending. The file will be created +if it doesn't exist when opened for writing or appending; it will be +truncated when opened for writing. A FileExistsError will be raised +if it already exists when opened for creating. Opening a file for +creating implies writing so this mode behaves in a similar way to +'w'. Add a '+' to the mode to allow simultaneous reading and +writing. + +A custom opener can be used by passing a callable as *opener*. +The underlying file descriptor for the file object is then obtained +by calling opener with (*name*, *flags*). *opener* must return +an open file descriptor (passing os.open as *opener* results in +functionality similar to passing None). [clinic start generated code]*/ static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=23413f68e6484bbd input=588aac967e0ba74b]*/ +/*[clinic end generated code: output=23413f68e6484bbd input=bac4efcd8f930bf3]*/ { #ifdef MS_WINDOWS wchar_t *widename = NULL; @@ -700,12 +703,13 @@ _io.FileIO.readall Read all data from the file, returned as bytes. In non-blocking mode, returns as much as is immediately available, -or None if no data is available. Return an empty bytes object at EOF. +or None if no data is available. Return an empty bytes object at +EOF. [clinic start generated code]*/ static PyObject * _io_FileIO_readall_impl(fileio *self) -/*[clinic end generated code: output=faa0292b213b4022 input=dbdc137f55602834]*/ +/*[clinic end generated code: output=faa0292b213b4022 input=92f0b7ef378a9ef4]*/ { struct _Py_stat_struct status; Py_off_t pos, end; @@ -801,14 +805,14 @@ _io.FileIO.read Read at most size bytes, returned as bytes. -Only makes one system call, so less data may be returned than requested. -In non-blocking mode, returns None if no data is available. -Return an empty bytes object at EOF. +Only makes one system call, so less data may be returned than +requested. In non-blocking mode, returns None if no data is +available. Return an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) -/*[clinic end generated code: output=bbd749c7c224143e input=f613d2057e4a1918]*/ +/*[clinic end generated code: output=bbd749c7c224143e input=5a5d6d3651fcbdc8]*/ { char *ptr; Py_ssize_t n; @@ -864,13 +868,13 @@ _io.FileIO.write Write buffer b to file, return number of bytes written. Only makes one system call, so not all of the data may be written. -The number of bytes actually written is returned. In non-blocking mode, -returns None if the write would block. +The number of bytes actually written is returned. In non-blocking +mode, returns None if the write would block. [clinic start generated code]*/ static PyObject * _io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b) -/*[clinic end generated code: output=927e25be80f3b77b input=2776314f043088f5]*/ +/*[clinic end generated code: output=927e25be80f3b77b input=233f1f70f9e8b09e]*/ { Py_ssize_t n; int err; @@ -971,18 +975,19 @@ _io.FileIO.seek Move to new file position and return the file position. -Argument offset is a byte count. Optional argument whence defaults to -SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values -are SEEK_CUR or 1 (move relative to current position, positive or negative), -and SEEK_END or 2 (move relative to end of file, usually negative, although -many platforms allow seeking beyond the end of a file). +Argument offset is a byte count. Optional argument whence defaults +to SEEK_SET or 0 (offset from start of file, offset should be >= 0); +other values are SEEK_CUR or 1 (move relative to current position, +positive or negative), and SEEK_END or 2 (move relative to end of +file, usually negative, although many platforms allow seeking beyond +the end of a file). Note that not all file objects are seekable. [clinic start generated code]*/ static PyObject * _io_FileIO_seek_impl(fileio *self, PyObject *pos, int whence) -/*[clinic end generated code: output=c976acdf054e6655 input=0439194b0774d454]*/ +/*[clinic end generated code: output=c976acdf054e6655 input=f165a1b4f5d494ad]*/ { if (self->fd < 0) return err_closed(); diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 618652b371a..bbf54dda448 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -100,11 +100,13 @@ _io._IOBase.seek Change the stream position to the given byte offset. -The offset is interpreted relative to the position indicated by whence. -Values for whence are: +The offset is interpreted relative to the position indicated by +whence. Values for whence are: -* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive -* os.SEEK_CUR or 1 -- current stream position; offset may be negative +* os.SEEK_SET or 0 -- start of stream (the default); offset should + be zero or positive +* os.SEEK_CUR or 1 -- current stream position; offset may be + negative * os.SEEK_END or 2 -- end of stream; offset is usually negative Return the new absolute position. @@ -113,7 +115,7 @@ Return the new absolute position. static PyObject * _io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(offset), int Py_UNUSED(whence)) -/*[clinic end generated code: output=8bd74ea6538ded53 input=74211232b363363e]*/ +/*[clinic end generated code: output=8bd74ea6538ded53 input=22eaf07a7a0ee289]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); @@ -140,14 +142,14 @@ _io._IOBase.truncate Truncate file to size bytes. -File pointer is left unchanged. Size defaults to the current IO position -as reported by tell(). Return the new size. +File pointer is left unchanged. Size defaults to the current IO +position as reported by tell(). Return the new size. [clinic start generated code]*/ static PyObject * _io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *Py_UNUSED(size)) -/*[clinic end generated code: output=2013179bff1fe8ef input=660ac20936612c27]*/ +/*[clinic end generated code: output=2013179bff1fe8ef input=5b3b6ab3c7abd806]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "truncate"); diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index e4c39a0a905..f0244f0fc56 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -480,7 +480,8 @@ _io.StringIO.seek Change stream position. -Seek to character offset pos relative to position indicated by whence: +Seek to character offset pos relative to position indicated by +whence: 0 Start of stream (the default). pos should be >= 0; 1 Current position - pos must be 0; 2 End of stream - pos must be 0. @@ -489,7 +490,7 @@ Returns the new absolute position. static PyObject * _io_StringIO_seek_impl(stringio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=c75ced09343a00d7]*/ +/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=ffef24668fd71a5d]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index db6c709c00c..d093514aa66 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -58,12 +58,13 @@ _io._TextIOBase.detach Separate the underlying buffer from the TextIOBase and return it. -After the underlying buffer has been detached, the TextIO is in an unusable state. +After the underlying buffer has been detached, the TextIO is in +an unusable state. [clinic start generated code]*/ static PyObject * _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=50915f40c609eaa4 input=987ca3640d0a3776]*/ +/*[clinic end generated code: output=50915f40c609eaa4 input=8099c088abcb87d8]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "detach"); @@ -77,14 +78,14 @@ _io._TextIOBase.read Read at most size characters from stream. -Read from underlying buffer until we have size characters or we hit EOF. -If size is negative or omitted, read until EOF. +Read from underlying buffer until we have size characters or we hit +EOF. If size is negative or omitted, read until EOF. [clinic start generated code]*/ static PyObject * _io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(size)) -/*[clinic end generated code: output=51a5178a309ce647 input=f5e37720f9fc563f]*/ +/*[clinic end generated code: output=51a5178a309ce647 input=c9fd4cc1cf1b4614]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "read"); @@ -1048,6 +1049,7 @@ io_check_errors(PyObject *errors) /*[clinic input] +@critical_section _io.TextIOWrapper.__init__ buffer: object encoding: str(accept={str, NoneType}) = None @@ -1091,7 +1093,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, const char *encoding, PyObject *errors, const char *newline, int line_buffering, int write_through) -/*[clinic end generated code: output=72267c0c01032ed2 input=e6cfaaaf6059d4f5]*/ +/*[clinic end generated code: output=72267c0c01032ed2 input=0f077220214c40a4]*/ { PyObject *raw, *codec_info = NULL; PyObject *res; @@ -2707,13 +2709,13 @@ _io.TextIOWrapper.tell Return the stream position as an opaque number. -The return value of tell() can be given as input to seek(), to restore a -previous stream position. +The return value of tell() can be given as input to seek(), to +restore a previous stream position. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_tell_impl(textio *self) -/*[clinic end generated code: output=4f168c08bf34ad5f input=415d6b4e4f8e6e8c]*/ +/*[clinic end generated code: output=4f168c08bf34ad5f input=aeece020f747fd92]*/ { PyObject *res; PyObject *posobj = NULL; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 008f8bf1188..5337cf4918f 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -313,16 +313,16 @@ _io._WindowsConsoleIO.__init__ Open a console buffer by file descriptor. -The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All -other mode characters will be ignored. Mode 'b' will be assumed if it is -omitted. The *opener* parameter is always ignored. +The mode can be 'rb' (default), or 'wb' for reading or writing +bytes. All other mode characters will be ignored. Mode 'b' will be +assumed if it is omitted. The *opener* parameter is always ignored. [clinic start generated code]*/ static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=3fd9cbcdd8d95429 input=7a3eed6bbe998fd9]*/ +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=f31100e2cd724617]*/ { const char *s; wchar_t *name = NULL; @@ -669,12 +669,13 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { maxlen += 1; Py_BLOCK_THREADS newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t)); - Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; PyErr_NoMemory(); + Py_UNBLOCK_THREADS break; } + Py_UNBLOCK_THREADS buf = newbuf; /* Only advance by n and not BUFSIZ in this case */ off += n; diff --git a/Modules/_json.c b/Modules/_json.c index afefc71bfbd..25cbd9b9bb8 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1602,9 +1602,13 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { PyObject *item = PyList_GET_ITEM(items, i); + // gh-142831: encoder_encode_key_value() can invoke user code + // that mutates the items list, invalidating this borrowed ref. + Py_INCREF(item); if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); + Py_DECREF(item); goto bail; } @@ -1612,18 +1616,30 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, value = PyTuple_GET_ITEM(item, 1); if (encoder_encode_key_value(s, writer, &first, key, value, new_newline_indent, - current_item_separator) < 0) + current_item_separator) < 0) { + Py_DECREF(item); goto bail; + } + Py_DECREF(item); } Py_CLEAR(items); } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { + // gh-142831: encoder_encode_key_value() can invoke user code + // that mutates the dict, invalidating these borrowed refs. + Py_INCREF(key); + Py_INCREF(value); if (encoder_encode_key_value(s, writer, &first, key, value, new_newline_indent, - current_item_separator) < 0) + current_item_separator) < 0) { + Py_DECREF(key); + Py_DECREF(value); goto bail; + } + Py_DECREF(key); + Py_DECREF(value); } } @@ -1712,12 +1728,20 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); + // gh-142831: encoder_listencode_obj() can invoke user code + // that mutates the sequence, invalidating this borrowed ref. + Py_INCREF(obj); if (i) { - if (_PyUnicodeWriter_WriteStr(writer, separator) < 0) + if (_PyUnicodeWriter_WriteStr(writer, separator) < 0) { + Py_DECREF(obj); goto bail; + } } - if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) + if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) { + Py_DECREF(obj); goto bail; + } + Py_DECREF(obj); } if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 97f3a8f03da..c4ad5dfbb3f 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1112,6 +1112,7 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) return result; error: + lzs->next_in = NULL; Py_XDECREF(result); return NULL; } @@ -1124,24 +1125,25 @@ _lzma.LZMADecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ef4e20ec7122241d input=60c1f135820e309d]*/ +/*[clinic end generated code: output=ef4e20ec7122241d input=0eb62669c4315dee]*/ { PyObject *result = NULL; diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 1b894ea4c67..5913c379524 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -50,9 +50,9 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "\n" "Remove a shared memory object (similar to unlink()).\n" "\n" -"Remove a shared memory object name, and, once all processes have unmapped\n" -"the object, de-allocates and destroys the contents of the associated memory\n" -"region."); +"Remove a shared memory object name, and, once all processes have\n" +"unmapped the object, de-allocates and destroys the contents of the\n" +"associated memory region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, @@ -85,4 +85,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=be0661dbed83ea23 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=12fb6a177b317c7f input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index 8214d1c5c54..83d8b2626cb 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -166,15 +166,15 @@ _posixshmem.shm_unlink Remove a shared memory object (similar to unlink()). -Remove a shared memory object name, and, once all processes have unmapped -the object, de-allocates and destroys the contents of the associated memory -region. +Remove a shared memory object name, and, once all processes have +unmapped the object, de-allocates and destroys the contents of the +associated memory region. [clinic start generated code]*/ static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=8dc0f87143e3b300]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=239da7d9ea49e7db]*/ { int rv; int async_err = 0; diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 4bf63932d22..241d68f3b88 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -118,7 +118,6 @@ _opcode_has_const_impl(PyObject *module, int opcode) } /*[clinic input] - _opcode.has_name -> bool opcode: int @@ -128,7 +127,7 @@ Return True if the opcode accesses an attribute by name, False otherwise. static int _opcode_has_name_impl(PyObject *module, int opcode) -/*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ +/*[clinic end generated code: output=b49a83555c2fa517 input=3032469628dd0849]*/ { return _PyCompile_OpcodeIsValid(opcode) && _PyCompile_OpcodeHasName(opcode); diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 0a7ef74e26f..eef2743f392 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -988,15 +988,15 @@ _posixsubprocess.fork_exec as subprocess_fork_exec Spawn a fresh new child process. -Fork a child process, close parent file descriptors as appropriate in the -child and duplicate the few that are needed before calling exec() in the -child process. +Fork a child process, close parent file descriptors as appropriate in +the child and duplicate the few that are needed before calling exec() in +the child process. -If close_fds is True, close file descriptors 3 and higher, except those listed -in the sorted tuple pass_fds. +If close_fds is True, close file descriptors 3 and higher, except those +listed in the sorted tuple pass_fds. -The preexec_fn, if supplied, will be called immediately before closing file -descriptors and exec. +The preexec_fn, if supplied, will be called immediately before closing +file descriptors and exec. WARNING: preexec_fn is NOT SAFE if your application uses threads. It may trigger infrequent, difficult to debug deadlocks. @@ -1021,7 +1021,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, PyObject *preexec_fn, int allow_vfork) -/*[clinic end generated code: output=7ee4f6ee5cf22b5b input=51757287ef266ffa]*/ +/*[clinic end generated code: output=7ee4f6ee5cf22b5b input=53eb802a43d5c5fc]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 9f1c9a1b0d4..4ac43748fdd 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -290,15 +290,16 @@ _queue.SimpleQueue.put Put the item on the queue. -The optional 'block' and 'timeout' arguments are ignored, as this method -never blocks. They are provided for compatibility with the Queue class. +The optional 'block' and 'timeout' arguments are ignored, as this +method never blocks. They are provided for compatibility with the +Queue class. [clinic start generated code]*/ static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) -/*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ +/*[clinic end generated code: output=4333136e88f90d8b input=9f9ff270a74670c3]*/ { HandoffData data = { .handed_off = 0, @@ -358,10 +359,11 @@ _queue.SimpleQueue.get Remove and return an item from the queue. -If optional args 'block' is true and 'timeout' is None (the default), -block if necessary until an item is available. If 'timeout' is -a non-negative number, it blocks at most 'timeout' seconds and raises -the Empty exception if no item was available within that time. +If optional args 'block' is true and 'timeout' is None (the +default), block if necessary until an item is available. If +'timeout' is a non-negative number, it blocks at most 'timeout' +seconds and raises the Empty exception if no item was available +within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). @@ -371,7 +373,7 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) -/*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ +/*[clinic end generated code: output=5c2cca914cd1e55b input=afa0889bbc6b4761]*/ { PyTime_t endtime = 0; @@ -529,7 +531,7 @@ static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_PUT_NOWAIT_METHODDEF _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("SimpleQueues are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 4db1ac474ef..c09c8582f89 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -167,14 +167,14 @@ _sqlite3.Blob.read as blob_read Read data at the current offset position. -If the end of the blob is reached, the data up to end of file will be returned. -When length is not specified, or is negative, Blob.read() will read until the -end of the blob. +If the end of the blob is reached, the data up to end of file will +be returned. When length is not specified, or is negative, +Blob.read() will read until the end of the blob. [clinic start generated code]*/ static PyObject * blob_read_impl(pysqlite_Blob *self, int length) -/*[clinic end generated code: output=1fc99b2541360dde input=f2e4aa4378837250]*/ +/*[clinic end generated code: output=1fc99b2541360dde input=6b745ad37720e556]*/ { if (!check_blob(self)) { return NULL; @@ -234,13 +234,13 @@ _sqlite3.Blob.write as blob_write Write data at the current offset. -This function cannot change the blob length. Writing beyond the end of the -blob will result in an exception being raised. +This function cannot change the blob length. Writing beyond the end +of the blob will result in an exception being raised. [clinic start generated code]*/ static PyObject * blob_write_impl(pysqlite_Blob *self, Py_buffer *data) -/*[clinic end generated code: output=b34cf22601b570b2 input=a84712f24a028e6d]*/ +/*[clinic end generated code: output=b34cf22601b570b2 input=0d372cb0240a5d49]*/ { if (!check_blob(self)) { return NULL; @@ -264,14 +264,15 @@ _sqlite3.Blob.seek as blob_seek Set the current access position to offset. -The origin argument defaults to os.SEEK_SET (absolute blob positioning). -Other values for origin are os.SEEK_CUR (seek relative to the current position) -and os.SEEK_END (seek relative to the blob's end). +The origin argument defaults to os.SEEK_SET (absolute blob +positioning). Other values for origin are os.SEEK_CUR (seek +relative to the current position) and os.SEEK_END (seek relative to +the blob's end). [clinic start generated code]*/ static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int origin) -/*[clinic end generated code: output=854c5a0e208547a5 input=5da9a07e55fe6bb6]*/ +/*[clinic end generated code: output=854c5a0e208547a5 input=84aea1b6b48607dd]*/ { if (!check_blob(self)) { return NULL; @@ -516,21 +517,25 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (len == 0) { - return 0; - } - Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { return -1; } - int rc = -1; if (vbuf.len != len) { PyErr_SetString(PyExc_IndexError, "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; } - else if (step == 1) { + + if (len == 0) { + PyBuffer_Release(&vbuf); + return 0; + } + + int rc = -1; + if (step == 1) { rc = inner_write(self, vbuf.buf, len, start); } else { diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index b95ba948aaf..bfafcf05410 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -31,9 +31,9 @@ PyDoc_STRVAR(blob_read__doc__, " length\n" " Read length in bytes.\n" "\n" -"If the end of the blob is reached, the data up to end of file will be returned.\n" -"When length is not specified, or is negative, Blob.read() will read until the\n" -"end of the blob."); +"If the end of the blob is reached, the data up to end of file will\n" +"be returned. When length is not specified, or is negative,\n" +"Blob.read() will read until the end of the blob."); #define BLOB_READ_METHODDEF \ {"read", _PyCFunction_CAST(blob_read), METH_FASTCALL, blob_read__doc__}, @@ -70,8 +70,8 @@ PyDoc_STRVAR(blob_write__doc__, "\n" "Write data at the current offset.\n" "\n" -"This function cannot change the blob length. Writing beyond the end of the\n" -"blob will result in an exception being raised."); +"This function cannot change the blob length. Writing beyond the end\n" +"of the blob will result in an exception being raised."); #define BLOB_WRITE_METHODDEF \ {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, @@ -105,9 +105,10 @@ PyDoc_STRVAR(blob_seek__doc__, "\n" "Set the current access position to offset.\n" "\n" -"The origin argument defaults to os.SEEK_SET (absolute blob positioning).\n" -"Other values for origin are os.SEEK_CUR (seek relative to the current position)\n" -"and os.SEEK_END (seek relative to the blob\'s end)."); +"The origin argument defaults to os.SEEK_SET (absolute blob\n" +"positioning). Other values for origin are os.SEEK_CUR (seek\n" +"relative to the current position) and os.SEEK_END (seek relative to\n" +"the blob\'s end)."); #define BLOB_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(blob_seek), METH_FASTCALL, blob_seek__doc__}, @@ -211,4 +212,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=31abd55660e0c5af input=a9049054013a1b77]*/ +/*[clinic end generated code: output=45168bcaeea61140 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index c44b761aca6..4eede407f9a 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -799,13 +799,14 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, "\n" " progress_handler\n" " A callable that takes no arguments.\n" -" If the callable returns non-zero, the current query is terminated,\n" -" and an exception is raised.\n" +" If the callable returns non-zero, the current query is\n" +" terminated, and an exception is raised.\n" " n\n" " The number of SQLite virtual machine instructions that are\n" " executed between invocations of \'progress_handler\'.\n" "\n" -"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled.\n" +"If \'progress_handler\' is None or \'n\' is 0, the progress handler is\n" +"disabled.\n" "\n" "Note: Passing keyword argument \'progress_handler\' to\n" "_sqlite3.Connection.set_progress_handler() is deprecated. Parameter\n" @@ -1453,10 +1454,10 @@ PyDoc_STRVAR(serialize__doc__, " name\n" " Which database to serialize.\n" "\n" -"For an ordinary on-disk database file, the serialization is just a copy of the\n" -"disk file. For an in-memory database or a \"temp\" database, the serialization is\n" -"the same sequence of bytes which would be written to disk if that database\n" -"were backed up to disk."); +"For an ordinary on-disk database file, the serialization is just\n" +"a copy of the disk file. For an in-memory database or a \"temp\"\n" +"database, the serialization is the same sequence of bytes which\n" +"would be written to disk if that database were backed up to disk."); #define SERIALIZE_METHODDEF \ {"serialize", _PyCFunction_CAST(serialize), METH_FASTCALL|METH_KEYWORDS, serialize__doc__}, @@ -1539,12 +1540,13 @@ PyDoc_STRVAR(deserialize__doc__, " name\n" " Which database to reopen with the deserialization.\n" "\n" -"The deserialize interface causes the database connection to disconnect from the\n" -"target database, and then reopen it as an in-memory database based on the given\n" -"serialized data.\n" +"The deserialize interface causes the database connection to\n" +"disconnect from the target database, and then reopen it as\n" +"an in-memory database based on the given serialized data.\n" "\n" -"The deserialize interface will fail with SQLITE_BUSY if the database is\n" -"currently in a read transaction or is involved in a backup operation."); +"The deserialize interface will fail with SQLITE_BUSY if the database\n" +"is currently in a read transaction or is involved in a backup\n" +"operation."); #define DESERIALIZE_METHODDEF \ {"deserialize", _PyCFunction_CAST(deserialize), METH_FASTCALL|METH_KEYWORDS, deserialize__doc__}, @@ -1662,7 +1664,8 @@ PyDoc_STRVAR(pysqlite_connection_exit__doc__, "\n" "Called when the connection is used as a context manager.\n" "\n" -"If there was any exception, a rollback takes place; otherwise we commit."); +"If there was any exception, a rollback takes place; otherwise we\n" +"commit."); #define PYSQLITE_CONNECTION_EXIT_METHODDEF \ {"__exit__", _PyCFunction_CAST(pysqlite_connection_exit), METH_FASTCALL, pysqlite_connection_exit__doc__}, @@ -1700,12 +1703,12 @@ PyDoc_STRVAR(setlimit__doc__, " category\n" " The limit category to be set.\n" " limit\n" -" The new limit. If the new limit is a negative number, the limit is\n" -" unchanged.\n" +" The new limit. If the new limit is a negative number, the limit\n" +" is unchanged.\n" "\n" -"Attempts to increase a limit above its hard upper bound are silently truncated\n" -"to the hard upper bound. Regardless of whether or not the limit was changed,\n" -"the prior value of the limit is returned."); +"Attempts to increase a limit above its hard upper bound are silently\n" +"truncated to the hard upper bound. Regardless of whether or not the\n" +"limit was changed, the prior value of the limit is returned."); #define SETLIMIT_METHODDEF \ {"setlimit", _PyCFunction_CAST(setlimit), METH_FASTCALL, setlimit__doc__}, @@ -1866,4 +1869,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=fa34f4c5f8837a7c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e0bcef289eb64cc input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 5f888ac19ec..bf5307ffac8 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -570,6 +570,47 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return cursor; } +static PyObject * +connection_get_row_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->row_factory); +} + +static int +connection_set_row_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete row_factory attribute"); + return -1; + } + Py_XSETREF(self->row_factory, Py_NewRef(value)); + return 0; +} + +static PyObject * +connection_get_text_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->text_factory); +} + +static int +connection_set_text_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete text_factory attribute"); + return -1; + } + Py_XSETREF(self->text_factory, Py_NewRef(value)); + return 0; +} + + /*[clinic input] _sqlite3.Connection.blobopen as blobopen @@ -1590,8 +1631,8 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han cls: defining_class progress_handler as callable: object A callable that takes no arguments. - If the callable returns non-zero, the current query is terminated, - and an exception is raised. + If the callable returns non-zero, the current query is + terminated, and an exception is raised. / [from 3.15] n: int The number of SQLite virtual machine instructions that are @@ -1599,14 +1640,15 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han Set progress handler callback. -If 'progress_handler' is None or 'n' is 0, the progress handler is disabled. +If 'progress_handler' is None or 'n' is 0, the progress handler is +disabled. [clinic start generated code]*/ static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=0739957fd8034a50 input=b4d6e2ef8b4d32f9]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=d46887797f0c0ecf]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2234,15 +2276,15 @@ _sqlite3.Connection.serialize as serialize Serialize a database into a byte string. -For an ordinary on-disk database file, the serialization is just a copy of the -disk file. For an in-memory database or a "temp" database, the serialization is -the same sequence of bytes which would be written to disk if that database -were backed up to disk. +For an ordinary on-disk database file, the serialization is just +a copy of the disk file. For an in-memory database or a "temp" +database, the serialization is the same sequence of bytes which +would be written to disk if that database were backed up to disk. [clinic start generated code]*/ static PyObject * serialize_impl(pysqlite_Connection *self, const char *name) -/*[clinic end generated code: output=97342b0e55239dd3 input=d2eb5194a65abe2b]*/ +/*[clinic end generated code: output=97342b0e55239dd3 input=7e48654e8e082fa8]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2287,18 +2329,19 @@ _sqlite3.Connection.deserialize as deserialize Load a serialized database. -The deserialize interface causes the database connection to disconnect from the -target database, and then reopen it as an in-memory database based on the given -serialized data. +The deserialize interface causes the database connection to +disconnect from the target database, and then reopen it as +an in-memory database based on the given serialized data. -The deserialize interface will fail with SQLITE_BUSY if the database is -currently in a read transaction or is involved in a backup operation. +The deserialize interface will fail with SQLITE_BUSY if the database +is currently in a read transaction or is involved in a backup +operation. [clinic start generated code]*/ static PyObject * deserialize_impl(pysqlite_Connection *self, Py_buffer *data, const char *name) -/*[clinic end generated code: output=e394c798b98bad89 input=1be4ca1faacf28f2]*/ +/*[clinic end generated code: output=e394c798b98bad89 input=5d20e028d98c0686]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2371,13 +2414,14 @@ _sqlite3.Connection.__exit__ as pysqlite_connection_exit Called when the connection is used as a context manager. -If there was any exception, a rollback takes place; otherwise we commit. +If there was any exception, a rollback takes place; otherwise we +commit. [clinic start generated code]*/ static PyObject * pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=0705200e9321202a input=bd66f1532c9c54a7]*/ +/*[clinic end generated code: output=0705200e9321202a input=8fdb0392ee6f3466]*/ { int commit = 0; PyObject* result; @@ -2417,20 +2461,20 @@ _sqlite3.Connection.setlimit as setlimit category: int The limit category to be set. limit: int - The new limit. If the new limit is a negative number, the limit is - unchanged. + The new limit. If the new limit is a negative number, the limit + is unchanged. / Set connection run-time limits. -Attempts to increase a limit above its hard upper bound are silently truncated -to the hard upper bound. Regardless of whether or not the limit was changed, -the prior value of the limit is returned. +Attempts to increase a limit above its hard upper bound are silently +truncated to the hard upper bound. Regardless of whether or not the +limit was changed, the prior value of the limit is returned. [clinic start generated code]*/ static PyObject * setlimit_impl(pysqlite_Connection *self, int category, int limit) -/*[clinic end generated code: output=0d208213f8d68ccd input=9bd469537e195635]*/ +/*[clinic end generated code: output=0d208213f8d68ccd input=5c2e430091206677]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2628,6 +2672,10 @@ static PyGetSetDef connection_getset[] = { {"in_transaction", (getter)pysqlite_connection_get_in_transaction, (setter)0}, {"autocommit", (getter)get_autocommit, (setter)set_autocommit}, {"__text_signature__", get_sig, (setter)0}, + {"row_factory", connection_get_row_factory, + connection_set_row_factory}, + {"text_factory", connection_get_text_factory, + connection_set_text_factory}, {NULL} }; @@ -2675,8 +2723,6 @@ static struct PyMemberDef connection_members[] = {"InternalError", _Py_T_OBJECT, offsetof(pysqlite_Connection, InternalError), Py_READONLY}, {"ProgrammingError", _Py_T_OBJECT, offsetof(pysqlite_Connection, ProgrammingError), Py_READONLY}, {"NotSupportedError", _Py_T_OBJECT, offsetof(pysqlite_Connection, NotSupportedError), Py_READONLY}, - {"row_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, row_factory)}, - {"text_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, text_factory)}, {NULL} }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index c0cc8268d2d..2a2a5f8ef9e 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -3150,7 +3150,7 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Patterns are generic over the type of string they handle (str or bytes)")}, {NULL, NULL} }; @@ -3206,7 +3206,7 @@ static PyMethodDef match_methods[] = { _SRE_SRE_MATCH___COPY___METHODDEF _SRE_SRE_MATCH___DEEPCOPY___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Matches are generic over the type of string which was matched (str or bytes)")}, {NULL, NULL} }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2d94fd98578..c3471676618 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -296,7 +296,7 @@ typedef struct { int post_handshake_auth; #endif PyObject *msg_cb; - PyObject *keylog_filename; + PyObject *keylog_filename; // can be anything accepted by Py_fopen() BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; @@ -324,7 +324,7 @@ typedef struct { PySSLContext *ctx; /* weakref to SSL context */ char shutdown_seen_zero; enum py_ssl_server_or_client socket_type; - PyObject *owner; /* Python level "owner" passed to servername callback */ + PyObject *owner; /* weakref to Python level "owner" passed to servername callback */ PyObject *server_hostname; _PySSLError err; /* last seen error from various sources */ /* Some SSL callbacks don't have error reporting. Callback wrappers @@ -2173,14 +2173,15 @@ _ssl._SSLSocket.context This changes the context associated with the SSLSocket. -This is typically used from within a callback function set by the sni_callback -on the SSLContext to change the certificate information associated with the -SSLSocket before the cryptographic exchange handshake messages. +This is typically used from within a callback function set by the +sni_callback on the SSLContext to change the certificate information +associated with the SSLSocket before the cryptographic exchange +handshake messages. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_context_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=d23e82f72f32e3d7 input=7cbb97407c2ace30]*/ +/*[clinic end generated code: output=d23e82f72f32e3d7 input=b845dea1f9710ebe]*/ { return Py_NewRef(self->ctx); } @@ -2294,6 +2295,10 @@ PySSL_traverse(PySSLSocket *self, visitproc visit, void *arg) static int PySSL_clear(PySSLSocket *self) { + Py_CLEAR(self->Socket); + Py_CLEAR(self->ctx); + Py_CLEAR(self->owner); + Py_CLEAR(self->server_hostname); Py_CLEAR(self->exc); return 0; } @@ -2317,10 +2322,7 @@ PySSL_dealloc(PySSLSocket *self) SSL_set_shutdown(self->ssl, SSL_SENT_SHUTDOWN | SSL_get_shutdown(self->ssl)); SSL_free(self->ssl); } - Py_XDECREF(self->Socket); - Py_XDECREF(self->ctx); - Py_XDECREF(self->server_hostname); - Py_XDECREF(self->owner); + (void)PySSL_clear(self); PyObject_GC_Del(self); Py_DECREF(tp); } @@ -2807,15 +2809,16 @@ _ssl._SSLSocket.get_channel_binding Get channel binding data for current connection. -Raise ValueError if the requested `cb_type` is not supported. Return bytes -of the data or None if the data is not available (e.g. before the handshake). +Raise ValueError if the requested `cb_type` is not supported. +Return bytes of the data or None if the data is not available (e.g. +before the handshake). Only 'tls-unique' channel binding data from RFC 5929 is supported. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type) -/*[clinic end generated code: output=34bac9acb6a61d31 input=e008004fc08744db]*/ +/*[clinic end generated code: output=34bac9acb6a61d31 input=bed81ef7936535a0]*/ { char buf[PySSL_CB_MAXLEN]; size_t len; @@ -3257,6 +3260,11 @@ context_traverse(PySSLContext *self, visitproc visit, void *arg) { Py_VISIT(self->set_sni_cb); Py_VISIT(self->msg_cb); + Py_VISIT(self->keylog_filename); +#ifndef OPENSSL_NO_PSK + Py_VISIT(self->psk_client_callback); + Py_VISIT(self->psk_server_callback); +#endif Py_VISIT(Py_TYPE(self)); return 0; } @@ -4707,15 +4715,16 @@ _ssl._SSLContext.sni_callback Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension. -If the argument is None then the callback is disabled. The method is called -with the SSLSocket, the server name as a string, and the SSLContext object. +If the argument is None then the callback is disabled. The method +is called with the SSLSocket, the server name as a string, and the +SSLContext object. See RFC 6066 for details of the SNI extension. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) -/*[clinic end generated code: output=961e6575cdfaf036 input=9b2473c5e984cfe6]*/ +/*[clinic end generated code: output=961e6575cdfaf036 input=280120c01d089dc3]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -4807,16 +4816,16 @@ _ssl._SSLContext.cert_store_stats Returns quantities of loaded X.509 certificates. -X.509 certificates with a CA extension and certificate revocation lists -inside the context's cert store. +X.509 certificates with a CA extension and certificate revocation +lists inside the context's cert store. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=5f356f4d9cca874d input=d13c6e3f2b48539b]*/ +/*[clinic end generated code: output=5f356f4d9cca874d input=9e5094e094b892a3]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -4859,16 +4868,16 @@ _ssl._SSLContext.get_ca_certs Returns a list of dicts with information of loaded CA certs. -If the optional argument is True, returns a DER-encoded copy of the CA -certificate. +If the optional argument is True, returns a DER-encoded copy of the +CA certificate. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) -/*[clinic end generated code: output=0d58f148f37e2938 input=eb0592909c9ad6e7]*/ +/*[clinic end generated code: output=0d58f148f37e2938 input=9f71af5aa4e67076]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -5756,13 +5765,13 @@ _ssl.RAND_status Returns True if the OpenSSL PRNG has been seeded with enough data and False if not. -It is necessary to seed the PRNG with RAND_add() on some platforms before -using the ssl() function. +It is necessary to seed the PRNG with RAND_add() on some platforms +before using the ssl() function. [clinic start generated code]*/ static PyObject * _ssl_RAND_status_impl(PyObject *module) -/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=636fb5659ea2e727]*/ +/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=844b0dc0f2165e87]*/ { return PyBool_FromLong(RAND_status()); } @@ -6024,16 +6033,16 @@ _ssl.enum_certificates Retrieve certificates from Windows' cert store. -store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide -more cert storages, too. The function returns a list of (bytes, -encoding_type, trust) tuples. The encoding_type flag can be interpreted -with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either -a set of OIDs or the boolean True. +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may +provide more cert storages, too. The function returns a list of +(bytes, encoding_type, trust) tuples. The encoding_type flag can be +interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The +trust setting is either a set of OIDs or the boolean True. [clinic start generated code]*/ static PyObject * _ssl_enum_certificates_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=5134dc8bb3a3c893 input=263c22e6c6988cf3]*/ +/*[clinic end generated code: output=5134dc8bb3a3c893 input=ef81b4bd1b7ab8e9]*/ { HCERTSTORE hCollectionStore = NULL; PCCERT_CONTEXT pCertCtx = NULL; diff --git a/Modules/_struct.c b/Modules/_struct.c index f5e1c62bb24..8ca52b23fd1 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1,7 +1,7 @@ /* struct module -- pack values into and (out of) bytes objects */ /* New version supporting byte order, alignment and size options, - character strings, and unsigned numbers */ + byte strings, and unsigned numbers */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 @@ -1947,7 +1947,7 @@ Struct_iter_unpack(PyStructObject *self, PyObject *buffer) * * Takes a struct object, a tuple of arguments, and offset in that tuple of * argument for where to start processing the arguments for packing, and a - * character buffer for writing the packed string. The caller must insure + * character buffer for writing the packed data. The caller must ensure * that the buffer may contain the required length for packing the arguments. * 0 is returned on success, 1 is returned if there is an error. * @@ -2484,8 +2484,8 @@ static struct PyMethodDef module_functions[] = { PyDoc_STRVAR(module_doc, "Functions to convert between Python values and C structs.\n\ -Python bytes objects are used to hold the data representing the C struct\n\ -and also as format strings (explained below) to describe the layout of data\n\ +Python bytes objects are used to hold the data representing the C struct.\n\ +The format string (explained below) describes the layout of data\n\ in the C struct.\n\ \n\ The optional first format char indicates byte order, size and alignment:\n\ @@ -2495,18 +2495,17 @@ The optional first format char indicates byte order, size and alignment:\n\ >: big-endian, std. size & alignment\n\ !: same as >\n\ \n\ -The remaining chars indicate types of args and must match exactly;\n\ +The remaining characters indicate types of args and must match exactly;\n\ these can be preceded by a decimal repeat count:\n\ - x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n\ - ?:_Bool; h:short; H:unsigned short; i:int; I:unsigned int;\n\ - l:long; L:unsigned long; f:float; d:double; e:half-float.\n\ + x: pad byte (no data); c: char; b: signed byte; B: unsigned byte;\n\ + ?: _Bool; h: short; H: unsigned short; i: int; I: unsigned int;\n\ + l: long; L: unsigned long; q: long long; Q: unsigned long long;\n\ + f: float; d: double; e: half-float;\n\ Special cases (preceding decimal count indicates length):\n\ - s:string (array of char); p: pascal string (with count byte).\n\ + s: byte string (array of char); p: Pascal string (with count byte).\n\ Special cases (only available in native format):\n\ - n:ssize_t; N:size_t;\n\ - P:an integer type that is wide enough to hold a pointer.\n\ -Special case (not in native mode unless 'long long' in platform C):\n\ - q:long long; Q:unsigned long long\n\ + n: ssize_t; N: size_t;\n\ + P: an integer type that is wide enough to hold a pointer.\n\ Whitespace between formats is ignored.\n\ \n\ The variable struct.error is an exception raised on errors.\n"); diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 41d190961c6..e348587b920 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -60,5 +60,6 @@ int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); int _PyTestCapi_Init_Monitoring(PyObject *module); int _PyTestCapi_Init_Object(PyObject *module); +int _PyTestCapi_Init_Weakref(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/weakref.c b/Modules/_testcapi/weakref.c new file mode 100644 index 00000000000..71378956491 --- /dev/null +++ b/Modules/_testcapi/weakref.c @@ -0,0 +1,34 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +pyweakref_getref(PyObject *module, PyObject *ref) +{ + NULLABLE(ref); + PyObject *obj = UNINITIALIZED_PTR; + int rc = PyWeakref_GetRef(ref, &obj); + if (rc == -1 && PyErr_Occurred()) { + assert(obj == NULL); + return NULL; + } + if (obj == NULL) { + return Py_BuildValue("i", rc); + } + else { + assert(obj != UNINITIALIZED_PTR); + return Py_BuildValue("iN", rc, obj); + } +} + + +static PyMethodDef test_methods[] = { + {"pyweakref_getref", pyweakref_getref, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Weakref(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 51baf1e8ef6..c18b9713b0c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4329,6 +4329,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Object(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Weakref(m) < 0) { + return NULL; + } PyState_AddModule(m, &_testcapimodule); return m; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index b183df7751d..880d8836d1a 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -86,5 +86,8 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_File(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Weakref(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h index ebaeb53921a..f9852aba266 100644 --- a/Modules/_testlimitedcapi/clinic/long.c.h +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -84,8 +84,8 @@ PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, "\n" "Test the PyLong_As{Size,Ssize}_t API.\n" "\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); +"At present this just tests that non-integer arguments are handled\n" +"correctly. It should be extended to test overflow handling."); #define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, @@ -140,4 +140,4 @@ PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, #define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb5c95bd0a4bdad8 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c index e2af09d0840..35505ef33f4 100644 --- a/Modules/_testlimitedcapi/long.c +++ b/Modules/_testlimitedcapi/long.c @@ -451,13 +451,13 @@ _testlimitedcapi.test_long_as_size_t Test the PyLong_As{Size,Ssize}_t API. -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. +At present this just tests that non-integer arguments are handled +correctly. It should be extended to test overflow handling. [clinic start generated code]*/ static PyObject * _testlimitedcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +/*[clinic end generated code: output=297a9f14a42f55af input=692e73744b35bf6e]*/ { size_t out_u; Py_ssize_t out_s; diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 11b2e5c6b83..4058df7fdef 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -41,5 +41,6 @@ int _PyTestLimitedCAPI_Init_Tuple(PyObject *module); int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); int _PyTestLimitedCAPI_Init_File(PyObject *module); +int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/weakref.c b/Modules/_testlimitedcapi/weakref.c new file mode 100644 index 00000000000..e7f9d54d1a0 --- /dev/null +++ b/Modules/_testlimitedcapi/weakref.c @@ -0,0 +1,78 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 +#endif + +#include "parts.h" +#include "util.h" + + +static PyObject * +pyweakref_check(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_Check(obj)); +} + +static PyObject * +pyweakref_checkref(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckRef(obj)); +} + +static PyObject * +pyweakref_checkrefexact(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckRefExact(obj)); +} + +static PyObject * +pyweakref_checkproxy(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckProxy(obj)); +} + +static PyObject * +pyweakref_newref(PyObject *module, PyObject *args) +{ + PyObject *obj; + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "O|O", &obj, &callback)) { + return NULL; + } + NULLABLE(obj); + return PyWeakref_NewRef(obj, callback); +} + +static PyObject * +pyweakref_newproxy(PyObject *module, PyObject *args) +{ + PyObject *obj; + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "O|O", &obj, &callback)) { + return NULL; + } + NULLABLE(obj); + return PyWeakref_NewProxy(obj, callback); +} + + +static PyMethodDef test_methods[] = { + {"pyweakref_check", pyweakref_check, METH_O}, + {"pyweakref_checkref", pyweakref_checkref, METH_O}, + {"pyweakref_checkrefexact", pyweakref_checkrefexact, METH_O}, + {"pyweakref_checkproxy", pyweakref_checkproxy, METH_O}, + {"pyweakref_newref", pyweakref_newref, METH_VARARGS}, + {"pyweakref_newproxy", pyweakref_newproxy, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Weakref(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 3f456e1f40d..56ecbea1460 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -135,14 +135,14 @@ _testmultiphase.StateAccessType.get_defining_module Return the module of the defining class. -Also tests that result of PyType_GetModuleByDef matches defining_class's -module. +Also tests that result of PyType_GetModuleByDef matches +defining_class's module. [clinic start generated code]*/ static PyObject * _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=ba2a14284a5d0921 input=d2c7245c8a9d06f8]*/ +/*[clinic end generated code: output=ba2a14284a5d0921 input=903e7f66555d65ae]*/ { PyObject *retval; retval = PyType_GetModule(cls); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 14efe18db5a..de93994aa00 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3212,12 +3212,13 @@ _tkinter.setbusywaitinterval Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter. -It should be set to a divisor of the maximum time between frames in an animation. +It should be set to a divisor of the maximum time between frames in +an animation. [clinic start generated code]*/ static PyObject * _tkinter_setbusywaitinterval_impl(PyObject *module, int new_val) -/*[clinic end generated code: output=42bf7757dc2d0ab6 input=deca1d6f9e6dae47]*/ +/*[clinic end generated code: output=42bf7757dc2d0ab6 input=984bbb86a3f916b7]*/ { if (new_val < 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Modules/_winapi.c b/Modules/_winapi.c index d51586e6025..4d486400a09 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1184,8 +1184,10 @@ gethandlelist(PyObject *mapping, const char *name, Py_ssize_t *size) } ret = PyMem_Malloc(*size); - if (ret == NULL) + if (ret == NULL) { + PyErr_NoMemory(); goto cleanup; + } for (i = 0; i < PySequence_Fast_GET_SIZE(value_fast); i++) { ret[i] = PYNUM_TO_HANDLE(PySequence_Fast_GET_ITEM(value_fast, i)); @@ -1268,6 +1270,7 @@ getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) attribute_list->attribute_list = PyMem_Malloc(attribute_list_size); if (attribute_list->attribute_list == NULL) { ret = -1; + PyErr_NoMemory(); goto cleanup; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 4c7cc4d47c0..25694e9cf33 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1415,13 +1415,13 @@ array.array.buffer_info Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents. -The length should be multiplied by the itemsize attribute to calculate -the buffer length in bytes. +The length should be multiplied by the itemsize attribute to +calculate the buffer length in bytes. [clinic start generated code]*/ static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=a58bae5c6e1ac6a6]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=f4d34398d5dfc856]*/ { PyObject *retval = NULL, *v; @@ -1467,13 +1467,13 @@ array.array.byteswap Byteswap all items of the array. -If the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is -raised. +If the items in the array are not 1, 2, 4, or 8 bytes in size, +RuntimeError is raised. [clinic start generated code]*/ static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=6a85591b950a0186]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=3005a63cc263b839]*/ { char *p; Py_ssize_t i; @@ -1817,14 +1817,14 @@ array.array.fromunicode Extends this array with data from the unicode string ustr. -The array must be a unicode type array; otherwise a ValueError is raised. -Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of -some other type. +The array must be a unicode type array; otherwise a ValueError is +raised. Use array.frombytes(ustr.encode(...)) to append Unicode +data to an array of some other type. [clinic start generated code]*/ static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=025db1fdade7a4ce]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=01fa592ec7b948b6]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -1876,14 +1876,15 @@ array.array.tounicode Extends this array with data from the unicode string ustr. -Convert the array to a unicode string. The array must be a unicode type array; -otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a -unicode string from an array of some other type. +Convert the array to a unicode string. The array must be a unicode +type array; otherwise a ValueError is raised. Use +array.tobytes().decode() to obtain a unicode string from an array of +some other type. [clinic start generated code]*/ static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=127242eebe70b66d]*/ +/*[clinic end generated code: output=08e442378336e1ef input=d4d5f398aa71a2be]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -2440,7 +2441,8 @@ static PyMethodDef array_methods[] = { ARRAY_ARRAY_TOBYTES_METHODDEF ARRAY_ARRAY_TOUNICODE_METHODDEF ARRAY_ARRAY___SIZEOF___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Arrays are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2821,8 +2823,10 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) len = 0; a = newarrayobject(type, len, descr); - if (a == NULL) + if (a == NULL) { + Py_XDECREF(it); return NULL; + } if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; @@ -2831,11 +2835,13 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PySequence_GetItem(initial, i); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } if (setarrayitem(a, i, v) != 0) { Py_DECREF(v); Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -2848,6 +2854,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) initial); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -2858,6 +2865,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -2878,6 +2886,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -2905,6 +2914,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return a; } } + Py_XDECREF(it); PyErr_SetString(PyExc_ValueError, "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f or d)"); return NULL; diff --git a/Modules/binascii.c b/Modules/binascii.c index 1030eb15f41..6b762b809b5 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -219,6 +219,14 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) assert(ascii_len >= 0); /* First byte: binary data length (in bytes) */ + if (ascii_len == 0) { + state = get_binascii_state(module); + if (state == NULL) { + return NULL; + } + PyErr_SetString(state->Error, "Missing length byte"); + return NULL; + } bin_len = (*ascii_data++ - ' ') & 077; ascii_len--; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b5639d5cf10..0fef4982a49 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -14,10 +14,11 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "\n" "Return an encoded string version of `input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" -"values are \'ignore\', \'replace\' and \'xmlcharrefreplace\' as well as any other name\n" -"registered with codecs.register_error that can handle UnicodeEncodeErrors."); +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +"and \'xmlcharrefreplace\' as well as any other name registered with\n" +"codecs.register_error that can handle UnicodeEncodeErrors."); #define _MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_multibytecodec_MultibyteCodec_encode), METH_FASTCALL|METH_KEYWORDS, _multibytecodec_MultibyteCodec_encode__doc__}, @@ -100,9 +101,10 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_decode__doc__, "\n" "Decodes \'input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeDecodeError. Other possible\n" -"values are \'ignore\' and \'replace\' as well as any other name registered with\n" +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeDecodeError. Other possible values are \'ignore\' and\n" +"\'replace\' as well as any other name registered with\n" "codecs.register_error that is able to handle UnicodeDecodeErrors.\""); #define _MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF \ @@ -682,4 +684,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=ee767a6d93c7108a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=de24dedc0342de51 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 1c671adb4ff..1274ba40940 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -576,17 +576,18 @@ _multibytecodec.MultibyteCodec.encode Return an encoded string version of `input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible -values are 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name -registered with codecs.register_error that can handle UnicodeEncodeErrors. +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeEncodeError. Other possible values are 'ignore', 'replace' +and 'xmlcharrefreplace' as well as any other name registered with +codecs.register_error that can handle UnicodeEncodeErrors. [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=606d0e128a577bae]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=e047a75a21752801]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -641,9 +642,10 @@ _multibytecodec.MultibyteCodec.decode Decodes 'input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeDecodeError. Other possible -values are 'ignore' and 'replace' as well as any other name registered with +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeDecodeError. Other possible values are 'ignore' and +'replace' as well as any other name registered with codecs.register_error that is able to handle UnicodeDecodeErrors." [clinic start generated code]*/ @@ -651,7 +653,7 @@ static PyObject * _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, Py_buffer *input, const char *errors) -/*[clinic end generated code: output=ff419f65bad6cc77 input=e0c78fc7ab190def]*/ +/*[clinic end generated code: output=ff419f65bad6cc77 input=dbf93d8bb98ca440]*/ { MultibyteCodec_State state; MultibyteDecodeBuffer buf; diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h index 04681fa2206..fa1c57dc26b 100644 --- a/Modules/clinic/_abc.c.h +++ b/Modules/clinic/_abc.c.h @@ -146,9 +146,9 @@ PyDoc_STRVAR(_abc_get_cache_token__doc__, "\n" "Returns the current ABC cache token.\n" "\n" -"The token is an opaque object (supporting equality testing) identifying the\n" -"current version of the ABC cache for virtual subclasses. The token changes\n" -"with every call to register() on any ABC."); +"The token is an opaque object (supporting equality testing) identifying\n" +"the current version of the ABC cache for virtual subclasses. The token\n" +"changes with every call to register() on any ABC."); #define _ABC_GET_CACHE_TOKEN_METHODDEF \ {"get_cache_token", (PyCFunction)_abc_get_cache_token, METH_NOARGS, _abc_get_cache_token__doc__}, @@ -161,4 +161,4 @@ _abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _abc_get_cache_token_impl(module); } -/*[clinic end generated code: output=1989b6716c950e17 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b05d599656aeb1e1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 6a9c8ff6d8f..0cbf1475d64 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -86,7 +86,8 @@ PyDoc_STRVAR(_asyncio_Future_result__doc__, "\n" "If the future has been cancelled, raises CancelledError. If the\n" "future\'s result isn\'t yet available, raises InvalidStateError. If\n" -"the future is done and has an exception set, this exception is raised."); +"the future is done and has an exception set, this exception is\n" +"raised."); #define _ASYNCIO_FUTURE_RESULT_METHODDEF \ {"result", (PyCFunction)_asyncio_Future_result, METH_NOARGS, _asyncio_Future_result__doc__}, @@ -227,8 +228,8 @@ PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__, "\n" "Add a callback to be run when the future becomes done.\n" "\n" -"The callback is called with a single argument - the future object. If\n" -"the future is already done when this is called, the callback is\n" +"The callback is called with a single argument - the future object.\n" +"If the future is already done when this is called, the callback is\n" "scheduled with call_soon."); #define _ASYNCIO_FUTURE_ADD_DONE_CALLBACK_METHODDEF \ @@ -340,9 +341,9 @@ PyDoc_STRVAR(_asyncio_Future_cancel__doc__, "\n" "Cancel the future and schedule callbacks.\n" "\n" -"If the future is already done or cancelled, return False. Otherwise,\n" -"change the future\'s state to cancelled, schedule the callbacks and\n" -"return True."); +"If the future is already done or cancelled, return False.\n" +"Otherwise, change the future\'s state to cancelled, schedule the\n" +"callbacks and return True."); #define _ASYNCIO_FUTURE_CANCEL_METHODDEF \ {"cancel", _PyCFunction_CAST(_asyncio_Future_cancel), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_cancel__doc__}, @@ -423,8 +424,8 @@ PyDoc_STRVAR(_asyncio_Future_done__doc__, "\n" "Return True if the future is done.\n" "\n" -"Done means either that a result / exception are available, or that the\n" -"future was cancelled."); +"Done means either that a result / exception are available, or that\n" +"the future was cancelled."); #define _ASYNCIO_FUTURE_DONE_METHODDEF \ {"done", (PyCFunction)_asyncio_Future_done, METH_NOARGS, _asyncio_Future_done__doc__}, @@ -1487,4 +1488,4 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=b26155080c82c472 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1842473913c9f884 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 528602c84a9..25ebb1727fd 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -16,8 +16,8 @@ PyDoc_STRVAR(_bisect_bisect_right__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e <= x, and all e in\n" -"a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just after the rightmost x already there.\n" +"a[i:] have e > x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just after the rightmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -229,8 +229,8 @@ PyDoc_STRVAR(_bisect_bisect_left__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e < x, and all e in\n" -"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just before the leftmost x already there.\n" +"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just before the leftmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -434,4 +434,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=0a8d5a32dd0a3f04 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=97bfe6ec4565161a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index de7b3993596..3c5f228a933 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -116,18 +116,19 @@ PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_bz2_BZ2Decompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _bz2_BZ2Decompressor_decompress__doc__}, @@ -234,4 +235,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=8daa62f47cc4853d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3f17a58daa209221 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 1c0f37442ab..6e7f26c95ec 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -14,9 +14,10 @@ PyDoc_STRVAR(_codecs_register__doc__, "\n" "Register a codec search function.\n" "\n" -"Search functions are expected to take one argument, the encoding name in\n" -"all lower case letters, and either return None, or a tuple of functions\n" -"(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object)."); +"Search functions are expected to take one argument, the encoding\n" +"name in all lower case letters, and either return None, or a tuple\n" +"of functions (encoder, decoder, stream_reader, stream_writer) (or\n" +"a CodecInfo object)."); #define _CODECS_REGISTER_METHODDEF \ {"register", (PyCFunction)_codecs_register, METH_O, _codecs_register__doc__}, @@ -76,10 +77,10 @@ PyDoc_STRVAR(_codecs_encode__doc__, "Encodes obj using the codec registered for encoding.\n" "\n" "The default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_codecs_encode), METH_FASTCALL|METH_KEYWORDS, _codecs_encode__doc__}, @@ -176,10 +177,10 @@ PyDoc_STRVAR(_codecs_decode__doc__, "Decodes obj using the codec registered for encoding.\n" "\n" "Default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(_codecs_decode), METH_FASTCALL|METH_KEYWORDS, _codecs_decode__doc__}, @@ -2643,8 +2644,9 @@ PyDoc_STRVAR(_codecs_register_error__doc__, "Register the specified error handler under the name errors.\n" "\n" "handler must be a callable object, that will be called with an exception\n" -"instance containing information about the location of the encoding/decoding\n" -"error and must return a (replacement, new position) tuple."); +"instance containing information about the location of the\n" +"encoding/decoding error and must return a (replacement, new position)\n" +"tuple."); #define _CODECS_REGISTER_ERROR_METHODDEF \ {"register_error", _PyCFunction_CAST(_codecs_register_error), METH_FASTCALL, _codecs_register_error__doc__}, @@ -2689,8 +2691,8 @@ PyDoc_STRVAR(_codecs_lookup_error__doc__, "\n" "lookup_error(errors) -> handler\n" "\n" -"Return the error handler for the specified error handling name or raise a\n" -"LookupError, if no handler exists under this name."); +"Return the error handler for the specified error handling name or raise\n" +"a LookupError, if no handler exists under this name."); #define _CODECS_LOOKUP_ERROR_METHODDEF \ {"lookup_error", (PyCFunction)_codecs_lookup_error, METH_O, _codecs_lookup_error__doc__}, @@ -2746,4 +2748,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e50d5fdf65bd45fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2badac642569e7a5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index c8788c461f7..520a0afb4c1 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -35,7 +35,8 @@ PyDoc_STRVAR(_curses_panel_panel_hide__doc__, "\n" "Hide the panel.\n" "\n" -"This does not delete the object, it just makes the window on screen invisible."); +"This does not delete the object, it just makes the window on screen\n" +"invisible."); #define _CURSES_PANEL_PANEL_HIDE_METHODDEF \ {"hide", _PyCFunction_CAST(_curses_panel_panel_hide), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_hide__doc__}, @@ -408,7 +409,8 @@ PyDoc_STRVAR(_curses_panel_update_panels__doc__, "\n" "Updates the virtual screen after changes in the panel stack.\n" "\n" -"This does not call curses.doupdate(), so you\'ll have to do this yourself."); +"This does not call curses.doupdate(), so you\'ll have to do this\n" +"yourself."); #define _CURSES_PANEL_UPDATE_PANELS_METHODDEF \ {"update_panels", (PyCFunction)_curses_panel_update_panels, METH_NOARGS, _curses_panel_update_panels__doc__}, @@ -421,4 +423,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=18dc5571174c7189 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=256acf1bc500a868 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index b4b3dca1f4c..16d70e4a1a5 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -422,10 +422,10 @@ PyDoc_STRVAR(_curses_window_border__doc__, " br\n" " Bottom-right corner.\n" "\n" -"Each parameter specifies the character to use for a specific part of the\n" -"border. The characters can be specified as integers or as one-character\n" -"strings. A 0 value for any parameter will cause the default character to be\n" -"used for that parameter."); +"Each parameter specifies the character to use for a specific part of\n" +"the border. The characters can be specified as integers or as\n" +"one-character strings. A 0 value for any parameter will cause the\n" +"default character to be used for that parameter."); #define _CURSES_WINDOW_BORDER_METHODDEF \ {"border", _PyCFunction_CAST(_curses_window_border), METH_FASTCALL, _curses_window_border__doc__}, @@ -500,8 +500,9 @@ PyDoc_STRVAR(_curses_window_box__doc__, " horch\n" " Top and bottom side.\n" "\n" -"Similar to border(), but both ls and rs are verch and both ts and bs are\n" -"horch. The default corner characters are always used by this function."); +"Similar to border(), but both ls and rs are verch and both ts and bs\n" +"are horch. The default corner characters are always used by this\n" +"function."); #define _CURSES_WINDOW_BOX_METHODDEF \ {"box", (PyCFunction)_curses_window_box, METH_VARARGS, _curses_window_box__doc__}, @@ -593,9 +594,9 @@ PyDoc_STRVAR(_curses_window_derwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"derwin() is the same as calling subwin(), except that begin_y and begin_x\n" -"are relative to the origin of the window, rather than relative to the entire\n" -"screen."); +"derwin() is the same as calling subwin(), except that begin_y and\n" +"begin_x are relative to the origin of the window, rather than\n" +"relative to the entire screen."); #define _CURSES_WINDOW_DERWIN_METHODDEF \ {"derwin", (PyCFunction)_curses_window_derwin, METH_VARARGS, _curses_window_derwin__doc__}, @@ -761,9 +762,10 @@ PyDoc_STRVAR(_curses_window_getch__doc__, " x\n" " X-coordinate.\n" "\n" -"The integer returned does not have to be in ASCII range: function keys,\n" -"keypad keys and so on return numbers higher than 256. In no-delay mode, -1\n" -"is returned if there is no input, else getch() waits until a key is pressed."); +"The integer returned does not have to be in ASCII range: function\n" +"keys, keypad keys and so on return numbers higher than 256. In\n" +"no-delay mode, -1 is returned if there is no input, else getch()\n" +"waits until a key is pressed."); #define _CURSES_WINDOW_GETCH_METHODDEF \ {"getch", (PyCFunction)_curses_window_getch, METH_VARARGS, _curses_window_getch__doc__}, @@ -808,9 +810,10 @@ PyDoc_STRVAR(_curses_window_getkey__doc__, " x\n" " X-coordinate.\n" "\n" -"Returning a string instead of an integer, as getch() does. Function keys,\n" -"keypad keys and other special keys return a multibyte string containing the\n" -"key name. In no-delay mode, an exception is raised if there is no input."); +"Returning a string instead of an integer, as getch() does. Function\n" +"keys, keypad keys and other special keys return a multibyte string\n" +"containing the key name. In no-delay mode, an exception is raised\n" +"if there is no input."); #define _CURSES_WINDOW_GETKEY_METHODDEF \ {"getkey", (PyCFunction)_curses_window_getkey, METH_VARARGS, _curses_window_getkey__doc__}, @@ -979,8 +982,8 @@ PyDoc_STRVAR(_curses_window_insch__doc__, " attr\n" " Attributes for the character.\n" "\n" -"All characters to the right of the cursor are shifted one position right, with\n" -"the rightmost characters on the line being lost."); +"All characters to the right of the cursor are shifted one position\n" +"right, with the rightmost characters on the line being lost."); #define _CURSES_WINDOW_INSCH_METHODDEF \ {"insch", (PyCFunction)_curses_window_insch, METH_VARARGS, _curses_window_insch__doc__}, @@ -1045,7 +1048,8 @@ PyDoc_STRVAR(_curses_window_inch__doc__, " x\n" " X-coordinate.\n" "\n" -"The bottom 8 bits are the character proper, and upper bits are the attributes."); +"The bottom 8 bits are the character proper, and upper bits are the\n" +"attributes."); #define _CURSES_WINDOW_INCH_METHODDEF \ {"inch", (PyCFunction)_curses_window_inch, METH_VARARGS, _curses_window_inch__doc__}, @@ -1099,11 +1103,11 @@ PyDoc_STRVAR(_curses_window_insstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor. All characters to the right of\n" -"the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x,\n" -"if specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor. All characters to the\n" +"right of the cursor are shifted right, with the rightmost characters\n" +"on the line being lost. The cursor position does not change (after\n" +"moving to y, x, if specified)."); #define _CURSES_WINDOW_INSSTR_METHODDEF \ {"insstr", (PyCFunction)_curses_window_insstr, METH_VARARGS, _curses_window_insstr__doc__}, @@ -1174,12 +1178,12 @@ PyDoc_STRVAR(_curses_window_insnstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor, up to n characters. If n is zero\n" -"or negative, the entire string is inserted. All characters to the right\n" -"of the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x, if\n" -"specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor, up to n characters. If\n" +"n is zero or negative, the entire string is inserted. All\n" +"characters to the right of the cursor are shifted right, with the\n" +"rightmost characters on the line being lost. The cursor position\n" +"does not change (after moving to y, x, if specified)."); #define _CURSES_WINDOW_INSNSTR_METHODDEF \ {"insnstr", (PyCFunction)_curses_window_insnstr, METH_VARARGS, _curses_window_insnstr__doc__}, @@ -1245,7 +1249,8 @@ PyDoc_STRVAR(_curses_window_is_linetouched__doc__, " line\n" " Line number.\n" "\n" -"Raise a curses.error exception if line is not valid for the given window."); +"Raise a curses.error exception if line is not valid for the given\n" +"window."); #define _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF \ {"is_linetouched", (PyCFunction)_curses_window_is_linetouched, METH_O, _curses_window_is_linetouched__doc__}, @@ -1275,9 +1280,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "noutrefresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol])\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_VARARGS, _curses_window_noutrefresh__doc__}, @@ -1329,9 +1334,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_NOARGS, _curses_window_noutrefresh__doc__}, @@ -1351,14 +1356,15 @@ PyDoc_STRVAR(_curses_window_overlay__doc__, "overlay(destwin, [sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol])\n" "Overlay the window on top of destwin.\n" "\n" -"The windows need not be the same size, only the overlapping region is copied.\n" -"This copy is non-destructive, which means that the current background\n" -"character does not overwrite the old contents of destwin.\n" +"The windows need not be the same size, only the overlapping region\n" +"is copied. This copy is non-destructive, which means that the\n" +"current background character does not overwrite the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overlay() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, and the other variables mark a rectangle in the\n" -"destination window."); +"To get fine-grained control over the copied region, the second form\n" +"of overlay() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, and the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERLAY_METHODDEF \ {"overlay", (PyCFunction)_curses_window_overlay, METH_VARARGS, _curses_window_overlay__doc__}, @@ -1409,14 +1415,15 @@ PyDoc_STRVAR(_curses_window_overwrite__doc__, " dmaxcol])\n" "Overwrite the window on top of destwin.\n" "\n" -"The windows need not be the same size, in which case only the overlapping\n" -"region is copied. This copy is destructive, which means that the current\n" -"background character overwrites the old contents of destwin.\n" +"The windows need not be the same size, in which case only the\n" +"overlapping region is copied. This copy is destructive, which means\n" +"that the current background character overwrites the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overwrite() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, the other variables mark a rectangle in the destination\n" -"window."); +"To get fine-grained control over the copied region, the second form\n" +"of overwrite() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERWRITE_METHODDEF \ {"overwrite", (PyCFunction)_curses_window_overwrite, METH_VARARGS, _curses_window_overwrite__doc__}, @@ -1522,16 +1529,17 @@ PyDoc_STRVAR(_curses_window_refresh__doc__, "Update the display immediately.\n" "\n" "Synchronize actual screen with previous drawing/deleting methods.\n" -"The 6 optional arguments can only be specified when the window is a pad\n" -"created with newpad(). The additional parameters are needed to indicate\n" -"what part of the pad and screen are involved. pminrow and pmincol specify\n" -"the upper left-hand corner of the rectangle to be displayed in the pad.\n" -"sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to\n" -"be displayed on the screen. The lower right-hand corner of the rectangle to\n" -"be displayed in the pad is calculated from the screen coordinates, since the\n" -"rectangles must be the same size. Both rectangles must be entirely contained\n" -"within their respective structures. Negative values of pminrow, pmincol,\n" -"sminrow, or smincol are treated as if they were zero."); +"The 6 optional arguments can only be specified when the window is\n" +"a pad created with newpad(). The additional parameters are needed\n" +"to indicate what part of the pad and screen are involved. pminrow\n" +"and pmincol specify the upper left-hand corner of the rectangle to\n" +"be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol\n" +"specify the edges of the rectangle to be displayed on the screen.\n" +"The lower right-hand corner of the rectangle to be displayed in the\n" +"pad is calculated from the screen coordinates, since the rectangles\n" +"must be the same size. Both rectangles must be entirely contained\n" +"within their respective structures. Negative values of pminrow,\n" +"pmincol, sminrow, or smincol are treated as if they were zero."); #define _CURSES_WINDOW_REFRESH_METHODDEF \ {"refresh", (PyCFunction)_curses_window_refresh, METH_VARARGS, _curses_window_refresh__doc__}, @@ -1629,8 +1637,8 @@ PyDoc_STRVAR(_curses_window_subwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the sub-window will extend from the specified position to the\n" -"lower right corner of the window."); +"By default, the sub-window will extend from the specified position\n" +"to the lower right corner of the window."); #define _CURSES_WINDOW_SUBWIN_METHODDEF \ {"subwin", (PyCFunction)_curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, @@ -1678,7 +1686,8 @@ PyDoc_STRVAR(_curses_window_scroll__doc__, " lines\n" " Number of lines to scroll.\n" "\n" -"Scroll upward if the argument is positive and downward if it is negative."); +"Scroll upward if the argument is positive and downward if it is\n" +"negative."); #define _CURSES_WINDOW_SCROLL_METHODDEF \ {"scroll", (PyCFunction)_curses_window_scroll, METH_VARARGS, _curses_window_scroll__doc__}, @@ -1717,8 +1726,9 @@ PyDoc_STRVAR(_curses_window_touchline__doc__, "touchline(start, count, [changed=True])\n" "Pretend count lines have been changed, starting with line start.\n" "\n" -"If changed is supplied, it specifies whether the affected lines are marked\n" -"as having been changed (changed=True) or unchanged (changed=False)."); +"If changed is supplied, it specifies whether the affected lines are\n" +"marked as having been changed (changed=True) or unchanged\n" +"(changed=False)."); #define _CURSES_WINDOW_TOUCHLINE_METHODDEF \ {"touchline", (PyCFunction)_curses_window_touchline, METH_VARARGS, _curses_window_touchline__doc__}, @@ -1912,11 +1922,12 @@ PyDoc_STRVAR(_curses_cbreak__doc__, " flag\n" " If false, the effect is the same as calling nocbreak().\n" "\n" -"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering is\n" -"turned off and characters are available to be read one by one. However,\n" -"unlike raw mode, special characters (interrupt, quit, suspend, and flow\n" -"control) retain their effects on the tty driver and calling program.\n" -"Calling first raw() then cbreak() leaves the terminal in cbreak mode."); +"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering\n" +"is turned off and characters are available to be read one by one.\n" +"However, unlike raw mode, special characters (interrupt, quit, suspend,\n" +"and flow control) retain their effects on the tty driver and calling\n" +"program. Calling first raw() then cbreak() leaves the terminal in\n" +"cbreak mode."); #define _CURSES_CBREAK_METHODDEF \ {"cbreak", _PyCFunction_CAST(_curses_cbreak), METH_FASTCALL, _curses_cbreak__doc__}, @@ -1956,8 +1967,9 @@ PyDoc_STRVAR(_curses_color_content__doc__, " color_number\n" " The number of the color (0 - (COLORS-1)).\n" "\n" -"A 3-tuple is returned, containing the R, G, B values for the given color,\n" -"which will be between 0 (no component) and 1000 (maximum amount of component)."); +"A 3-tuple is returned, containing the R, G, B values for the given\n" +"color, which will be between 0 (no component) and 1000 (maximum amount\n" +"of component)."); #define _CURSES_COLOR_CONTENT_METHODDEF \ {"color_content", (PyCFunction)_curses_color_content, METH_O, _curses_color_content__doc__}, @@ -1990,7 +2002,8 @@ PyDoc_STRVAR(_curses_color_pair__doc__, " The number of the color pair.\n" "\n" "This attribute value can be combined with A_STANDOUT, A_REVERSE, and the\n" -"other A_* attributes. pair_number() is the counterpart to this function."); +"other A_* attributes. pair_number() is the counterpart to this\n" +"function."); #define _CURSES_COLOR_PAIR_METHODDEF \ {"color_pair", (PyCFunction)_curses_color_pair, METH_O, _curses_color_pair__doc__}, @@ -2024,9 +2037,9 @@ PyDoc_STRVAR(_curses_curs_set__doc__, " 0 for invisible, 1 for normal visible, or 2 for very visible.\n" "\n" "If the terminal supports the visibility requested, the previous cursor\n" -"state is returned; otherwise, an exception is raised. On many terminals,\n" -"the \"visible\" mode is an underline cursor and the \"very visible\" mode is\n" -"a block cursor."); +"state is returned; otherwise, an exception is raised. On many\n" +"terminals, the \"visible\" mode is an underline cursor and the \"very\n" +"visible\" mode is a block cursor."); #define _CURSES_CURS_SET_METHODDEF \ {"curs_set", (PyCFunction)_curses_curs_set, METH_O, _curses_curs_set__doc__}, @@ -2078,7 +2091,8 @@ PyDoc_STRVAR(_curses_def_shell_mode__doc__, "\n" "Save the current terminal mode as the \"shell\" mode.\n" "\n" -"The \"shell\" mode is the mode when the running program is not using curses.\n" +"The \"shell\" mode is the mode when the running program is not using\n" +"curses.\n" "\n" "Subsequent calls to reset_shell_mode() will restore this mode."); @@ -2152,7 +2166,8 @@ PyDoc_STRVAR(_curses_echo__doc__, " flag\n" " If false, the effect is the same as calling noecho().\n" "\n" -"In echo mode, each character input is echoed to the screen as it is entered."); +"In echo mode, each character input is echoed to the screen as it is\n" +"entered."); #define _CURSES_ECHO_METHODDEF \ {"echo", _PyCFunction_CAST(_curses_echo), METH_FASTCALL, _curses_echo__doc__}, @@ -2225,7 +2240,8 @@ PyDoc_STRVAR(_curses_flash__doc__, "\n" "Flash the screen.\n" "\n" -"That is, change it to reverse-video and then change it back in a short interval."); +"That is, change it to reverse-video and then change it back in a short\n" +"interval."); #define _CURSES_FLASH_METHODDEF \ {"flash", (PyCFunction)_curses_flash, METH_NOARGS, _curses_flash__doc__}, @@ -2245,8 +2261,8 @@ PyDoc_STRVAR(_curses_flushinp__doc__, "\n" "Flush all input buffers.\n" "\n" -"This throws away any typeahead that has been typed by the user and has not\n" -"yet been processed by the program."); +"This throws away any typeahead that has been typed by the user and has\n" +"not yet been processed by the program."); #define _CURSES_FLUSHINP_METHODDEF \ {"flushinp", (PyCFunction)_curses_flushinp, METH_NOARGS, _curses_flushinp__doc__}, @@ -2601,8 +2617,9 @@ PyDoc_STRVAR(_curses_init_pair__doc__, " bg\n" " Background color number (-1 - (COLORS-1)).\n" "\n" -"If the color-pair was previously initialized, the screen is refreshed and\n" -"all occurrences of that color-pair are changed to the new definition."); +"If the color-pair was previously initialized, the screen is refreshed\n" +"and all occurrences of that color-pair are changed to the new\n" +"definition."); #define _CURSES_INIT_PAIR_METHODDEF \ {"init_pair", _PyCFunction_CAST(_curses_init_pair), METH_FASTCALL, _curses_init_pair__doc__}, @@ -2758,9 +2775,9 @@ PyDoc_STRVAR(_curses_get_escdelay__doc__, "\n" "Gets the curses ESCDELAY setting.\n" "\n" -"Gets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Gets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_GET_ESCDELAY_METHODDEF \ {"get_escdelay", (PyCFunction)_curses_get_escdelay, METH_NOARGS, _curses_get_escdelay__doc__}, @@ -2787,9 +2804,9 @@ PyDoc_STRVAR(_curses_set_escdelay__doc__, " ms\n" " length of the delay in milliseconds.\n" "\n" -"Sets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Sets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_SET_ESCDELAY_METHODDEF \ {"set_escdelay", (PyCFunction)_curses_set_escdelay, METH_O, _curses_set_escdelay__doc__}, @@ -2823,8 +2840,8 @@ PyDoc_STRVAR(_curses_get_tabsize__doc__, "\n" "Gets the curses TABSIZE setting.\n" "\n" -"Gets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Gets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_GET_TABSIZE_METHODDEF \ {"get_tabsize", (PyCFunction)_curses_get_tabsize, METH_NOARGS, _curses_get_tabsize__doc__}, @@ -2851,8 +2868,8 @@ PyDoc_STRVAR(_curses_set_tabsize__doc__, " size\n" " rendered cell width of a tab character.\n" "\n" -"Sets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Sets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_SET_TABSIZE_METHODDEF \ {"set_tabsize", (PyCFunction)_curses_set_tabsize, METH_O, _curses_set_tabsize__doc__}, @@ -3023,8 +3040,8 @@ PyDoc_STRVAR(_curses_longname__doc__, "\n" "Return the terminfo long name field describing the current terminal.\n" "\n" -"The maximum length of a verbose description is 128 characters. It is defined\n" -"only after the call to initscr()."); +"The maximum length of a verbose description is 128 characters. It is\n" +"defined only after the call to initscr()."); #define _CURSES_LONGNAME_METHODDEF \ {"longname", (PyCFunction)_curses_longname, METH_NOARGS, _curses_longname__doc__}, @@ -3081,8 +3098,8 @@ PyDoc_STRVAR(_curses_mouseinterval__doc__, " Time in milliseconds.\n" "\n" "Set the maximum time that can elapse between press and release events in\n" -"order for them to be recognized as a click, and return the previous interval\n" -"value."); +"order for them to be recognized as a click, and return the previous\n" +"interval value."); #define _CURSES_MOUSEINTERVAL_METHODDEF \ {"mouseinterval", (PyCFunction)_curses_mouseinterval, METH_O, _curses_mouseinterval__doc__}, @@ -3117,9 +3134,10 @@ PyDoc_STRVAR(_curses_mousemask__doc__, "Set the mouse events to be reported, and return a tuple (availmask, oldmask).\n" "\n" "Return a tuple (availmask, oldmask). availmask indicates which of the\n" -"specified mouse events can be reported; on complete failure it returns 0.\n" -"oldmask is the previous value of the given window\'s mouse event mask.\n" -"If this function is never called, no mouse events are ever reported."); +"specified mouse events can be reported; on complete failure it returns\n" +"0. oldmask is the previous value of the given window\'s mouse event\n" +"mask. If this function is never called, no mouse events are ever\n" +"reported."); #define _CURSES_MOUSEMASK_METHODDEF \ {"mousemask", (PyCFunction)_curses_mousemask, METH_O, _curses_mousemask__doc__}, @@ -3236,8 +3254,8 @@ PyDoc_STRVAR(_curses_newwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the window will extend from the specified position to the lower\n" -"right corner of the screen."); +"By default, the window will extend from the specified position to the\n" +"lower right corner of the screen."); #define _CURSES_NEWWIN_METHODDEF \ {"newwin", (PyCFunction)_curses_newwin, METH_VARARGS, _curses_newwin__doc__}, @@ -3287,8 +3305,9 @@ PyDoc_STRVAR(_curses_nl__doc__, " flag\n" " If false, the effect is the same as calling nonl().\n" "\n" -"This mode translates the return key into newline on input, and translates\n" -"newline into return and line-feed on output. Newline mode is initially on."); +"This mode translates the return key into newline on input, and\n" +"translates newline into return and line-feed on output. Newline mode\n" +"is initially on."); #define _CURSES_NL_METHODDEF \ {"nl", _PyCFunction_CAST(_curses_nl), METH_FASTCALL, _curses_nl__doc__}, @@ -3365,8 +3384,8 @@ PyDoc_STRVAR(_curses_nonl__doc__, "\n" "Leave newline mode.\n" "\n" -"Disable translation of return into newline on input, and disable low-level\n" -"translation of newline into newline/return on output."); +"Disable translation of return into newline on input, and disable\n" +"low-level translation of newline into newline/return on output."); #define _CURSES_NONL_METHODDEF \ {"nonl", (PyCFunction)_curses_nonl, METH_NOARGS, _curses_nonl__doc__}, @@ -3582,8 +3601,8 @@ PyDoc_STRVAR(_curses_raw__doc__, " If false, the effect is the same as calling noraw().\n" "\n" "In raw mode, normal line buffering and processing of interrupt, quit,\n" -"suspend, and flow control keys are turned off; characters are presented to\n" -"curses input functions one by one."); +"suspend, and flow control keys are turned off; characters are presented\n" +"to curses input functions one by one."); #define _CURSES_RAW_METHODDEF \ {"raw", _PyCFunction_CAST(_curses_raw), METH_FASTCALL, _curses_raw__doc__}, @@ -3681,8 +3700,8 @@ PyDoc_STRVAR(_curses_resizeterm__doc__, " ncols\n" " Width.\n" "\n" -"Adjusts other bookkeeping data used by the curses library that record the\n" -"window dimensions (in particular the SIGWINCH handler)."); +"Adjusts other bookkeeping data used by the curses library that record\n" +"the window dimensions (in particular the SIGWINCH handler)."); #define _CURSES_RESIZETERM_METHODDEF \ {"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__}, @@ -3760,10 +3779,11 @@ PyDoc_STRVAR(_curses_resize_term__doc__, " Width.\n" "\n" "When resizing the windows, resize_term() blank-fills the areas that are\n" -"extended. The calling application should fill in these areas with appropriate\n" -"data. The resize_term() function attempts to resize all windows. However,\n" -"due to the calling convention of pads, it is not possible to resize these\n" -"without additional interaction with the application."); +"extended. The calling application should fill in these areas with\n" +"appropriate data. The resize_term() function attempts to resize all\n" +"windows. However, due to the calling convention of pads, it is not\n" +"possible to resize these without additional interaction with the\n" +"application."); #define _CURSES_RESIZE_TERM_METHODDEF \ {"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__}, @@ -3898,12 +3918,12 @@ PyDoc_STRVAR(_curses_start_color__doc__, "\n" "Initializes eight basic colors and global variables COLORS and COLOR_PAIRS.\n" "\n" -"Must be called if the programmer wants to use colors, and before any other\n" -"color manipulation routine is called. It is good practice to call this\n" -"routine right after initscr().\n" +"Must be called if the programmer wants to use colors, and before any\n" +"other color manipulation routine is called. It is good practice to call\n" +"this routine right after initscr().\n" "\n" -"It also restores the colors on the terminal to the values they had when the\n" -"terminal was just turned on."); +"It also restores the colors on the terminal to the values they had when\n" +"the terminal was just turned on."); #define _CURSES_START_COLOR_METHODDEF \ {"start_color", (PyCFunction)_curses_start_color, METH_NOARGS, _curses_start_color__doc__}, @@ -4005,8 +4025,8 @@ PyDoc_STRVAR(_curses_tigetnum__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"The value -2 is returned if capname is not a numeric capability, or -1 if\n" -"it is canceled or absent from the terminal description."); +"The value -2 is returned if capname is not a numeric capability, or -1\n" +"if it is canceled or absent from the terminal description."); #define _CURSES_TIGETNUM_METHODDEF \ {"tigetnum", (PyCFunction)_curses_tigetnum, METH_O, _curses_tigetnum__doc__}, @@ -4048,8 +4068,8 @@ PyDoc_STRVAR(_curses_tigetstr__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"None is returned if capname is not a string capability, or is canceled or\n" -"absent from the terminal description."); +"None is returned if capname is not a string capability, or is canceled\n" +"or absent from the terminal description."); #define _CURSES_TIGETSTR_METHODDEF \ {"tigetstr", (PyCFunction)_curses_tigetstr, METH_O, _curses_tigetstr__doc__}, @@ -4203,14 +4223,14 @@ PyDoc_STRVAR(_curses_use_env__doc__, "\n" "Use environment variables LINES and COLUMNS.\n" "\n" -"If used, this function should be called before initscr() or newterm() are\n" -"called.\n" +"If used, this function should be called before initscr() or newterm()\n" +"are called.\n" "\n" -"When flag is False, the values of lines and columns specified in the terminfo\n" -"database will be used, even if environment variables LINES and COLUMNS (used\n" -"by default) are set, or if curses is running in a window (in which case\n" -"default behavior would be to use the window size if LINES and COLUMNS are\n" -"not set)."); +"When flag is False, the values of lines and columns specified in the\n" +"terminfo database will be used, even if environment variables LINES and\n" +"COLUMNS (used by default) are set, or if curses is running in a window\n" +"(in which case default behavior would be to use the window size if LINES\n" +"and COLUMNS are not set)."); #define _CURSES_USE_ENV_METHODDEF \ {"use_env", (PyCFunction)_curses_use_env, METH_O, _curses_use_env__doc__}, @@ -4373,4 +4393,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=265cd9f5affbad5e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=833ee704415ce160 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 48499e0aaf7..f8987d3eded 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "\n" "Create a date from a POSIX timestamp.\n" "\n" -"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" -"as local time."); +"The timestamp is a number, e.g. created via time.time(), that is\n" +"interpreted as local time."); #define DATETIME_DATE_FROMTIMESTAMP_METHODDEF \ {"fromtimestamp", (PyCFunction)datetime_date_fromtimestamp, METH_O|METH_CLASS, datetime_date_fromtimestamp__doc__}, @@ -496,4 +496,4 @@ datetime_datetime_replace(PyDateTime_DateTime *self, PyObject *const *args, Py_s exit: return return_value; } -/*[clinic end generated code: output=c7a04b865b1e0890 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1ba8c631dace0000 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 4379b433db3..cd0192a19d4 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -97,7 +97,8 @@ PyDoc_STRVAR(_dbm_dbm_setdefault__doc__, "\n" "Return the value for key if present, otherwise default.\n" "\n" -"If key is not in the database, it is inserted with default as the value."); +"If key is not in the database, it is inserted with default as the\n" +"value."); #define _DBM_DBM_SETDEFAULT_METHODDEF \ {"setdefault", _PyCFunction_CAST(_dbm_dbm_setdefault), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _dbm_dbm_setdefault__doc__}, @@ -221,4 +222,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f7d9a87d80a64278 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=43f744d870dc6036 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index bbf4365114c..b776b602cb3 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -122,9 +122,9 @@ PyDoc_STRVAR(_gdbm_gdbm_firstkey__doc__, "\n" "Return the starting key for the traversal.\n" "\n" -"It\'s possible to loop over every key in the database using this method\n" -"and the nextkey() method. The traversal is ordered by GDBM\'s internal\n" -"hash values, and won\'t be sorted by the key values."); +"It\'s possible to loop over every key in the database using this\n" +"method and the nextkey() method. The traversal is ordered by GDBM\'s\n" +"internal hash values, and won\'t be sorted by the key values."); #define _GDBM_GDBM_FIRSTKEY_METHODDEF \ {"firstkey", _PyCFunction_CAST(_gdbm_gdbm_firstkey), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_firstkey__doc__}, @@ -148,8 +148,8 @@ PyDoc_STRVAR(_gdbm_gdbm_nextkey__doc__, "\n" "Returns the key that follows key in the traversal.\n" "\n" -"The following code prints every key in the database db, without having\n" -"to create a list in memory that contains them all:\n" +"The following code prints every key in the database db, without\n" +"having to create a list in memory that contains them all:\n" "\n" " k = db.firstkey()\n" " while k is not None:\n" @@ -201,9 +201,9 @@ PyDoc_STRVAR(_gdbm_gdbm_reorganize__doc__, "\n" "If you have carried out a lot of deletions and would like to shrink\n" "the space used by the GDBM file, this routine will reorganize the\n" -"database. GDBM will not shorten the length of a database file except\n" -"by using this reorganization; otherwise, deleted file space will be\n" -"kept and reused as new (key,value) pairs are added."); +"database. GDBM will not shorten the length of a database file\n" +"except by using this reorganization; otherwise, deleted file space\n" +"will be kept and reused as new (key,value) pairs are added."); #define _GDBM_GDBM_REORGANIZE_METHODDEF \ {"reorganize", _PyCFunction_CAST(_gdbm_gdbm_reorganize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_reorganize__doc__}, @@ -343,4 +343,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=07bdeb4a8ecb328e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9c85138071dd473a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 621093e7e76..e323708f559 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -1812,8 +1812,8 @@ PyDoc_STRVAR(_hashlib_HMAC_hexdigest__doc__, "\n" "Return hexadecimal digest of the bytes passed to the update() method so far.\n" "\n" -"This may be used to exchange the value safely in email or other non-binary\n" -"environments."); +"This may be used to exchange the value safely in email or other\n" +"non-binary environments."); #define _HASHLIB_HMAC_HEXDIGEST_METHODDEF \ {"hexdigest", (PyCFunction)_hashlib_HMAC_hexdigest, METH_NOARGS, _hashlib_HMAC_hexdigest__doc__}, @@ -1834,8 +1834,8 @@ PyDoc_STRVAR(_hashlib_get_fips_mode__doc__, "Determine the OpenSSL FIPS mode of operation.\n" "\n" "For OpenSSL 3.0.0 and newer it returns the state of the default provider\n" -"in the default OSSL context. It\'s not quite the same as FIPS_mode() but good\n" -"enough for unittests.\n" +"in the default OSSL context. It\'s not quite the same as FIPS_mode() but\n" +"good enough for unittests.\n" "\n" "Effectively any non-zero return value indicates FIPS mode;\n" "values other than 1 may have additional significance."); @@ -1937,4 +1937,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=4c9222b02b194662 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b210ba56e3dfb58a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 51fab5eab3f..233d41bbf4a 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -74,18 +74,19 @@ PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_lzma_LZMADecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _lzma_LZMADecompressor_decompress__doc__}, @@ -327,4 +328,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=5e79c05ace76dc96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0edad6505009133f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index dd7644de6b7..3a0a1f738aa 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -14,15 +14,15 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, "\n" "Spawn a fresh new child process.\n" "\n" -"Fork a child process, close parent file descriptors as appropriate in the\n" -"child and duplicate the few that are needed before calling exec() in the\n" -"child process.\n" +"Fork a child process, close parent file descriptors as appropriate in\n" +"the child and duplicate the few that are needed before calling exec() in\n" +"the child process.\n" "\n" -"If close_fds is True, close file descriptors 3 and higher, except those listed\n" -"in the sorted tuple pass_fds.\n" +"If close_fds is True, close file descriptors 3 and higher, except those\n" +"listed in the sorted tuple pass_fds.\n" "\n" -"The preexec_fn, if supplied, will be called immediately before closing file\n" -"descriptors and exec.\n" +"The preexec_fn, if supplied, will be called immediately before closing\n" +"file descriptors and exec.\n" "\n" "WARNING: preexec_fn is NOT SAFE if your application uses threads.\n" " It may trigger infrequent, difficult to debug deadlocks.\n" @@ -155,4 +155,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=48555f5965a871be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47682cb62b0b08b2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 6f4c715c722..3f19520352b 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -44,8 +44,9 @@ PyDoc_STRVAR(_queue_SimpleQueue_put__doc__, "\n" "Put the item on the queue.\n" "\n" -"The optional \'block\' and \'timeout\' arguments are ignored, as this method\n" -"never blocks. They are provided for compatibility with the Queue class."); +"The optional \'block\' and \'timeout\' arguments are ignored, as this\n" +"method never blocks. They are provided for compatibility with the\n" +"Queue class."); #define _QUEUE_SIMPLEQUEUE_PUT_METHODDEF \ {"put", _PyCFunction_CAST(_queue_SimpleQueue_put), METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put__doc__}, @@ -182,10 +183,11 @@ PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, "\n" "Remove and return an item from the queue.\n" "\n" -"If optional args \'block\' is true and \'timeout\' is None (the default),\n" -"block if necessary until an item is available. If \'timeout\' is\n" -"a non-negative number, it blocks at most \'timeout\' seconds and raises\n" -"the Empty exception if no item was available within that time.\n" +"If optional args \'block\' is true and \'timeout\' is None (the\n" +"default), block if necessary until an item is available. If\n" +"\'timeout\' is a non-negative number, it blocks at most \'timeout\'\n" +"seconds and raises the Empty exception if no item was available\n" +"within that time.\n" "Otherwise (\'block\' is false), return an item if one is immediately\n" "available, else raise the Empty exception (\'timeout\' is ignored\n" "in that case)."); @@ -349,4 +351,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=44a718f40072018a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=25c2af38811bc536 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 628bd877fe1..2cb60b7dca8 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -264,9 +264,10 @@ _ssl__SSLSocket_compression(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, "This changes the context associated with the SSLSocket.\n" "\n" -"This is typically used from within a callback function set by the sni_callback\n" -"on the SSLContext to change the certificate information associated with the\n" -"SSLSocket before the cryptographic exchange handshake messages."); +"This is typically used from within a callback function set by the\n" +"sni_callback on the SSLContext to change the certificate information\n" +"associated with the SSLSocket before the cryptographic exchange\n" +"handshake messages."); #if defined(_ssl__SSLSocket_context_DOCSTR) # undef _ssl__SSLSocket_context_DOCSTR #endif @@ -583,8 +584,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_get_channel_binding__doc__, "\n" "Get channel binding data for current connection.\n" "\n" -"Raise ValueError if the requested `cb_type` is not supported. Return bytes\n" -"of the data or None if the data is not available (e.g. before the handshake).\n" +"Raise ValueError if the requested `cb_type` is not supported.\n" +"Return bytes of the data or None if the data is not available (e.g.\n" +"before the handshake).\n" "Only \'tls-unique\' channel binding data from RFC 5929 is supported."); #define _SSL__SSLSOCKET_GET_CHANNEL_BINDING_METHODDEF \ @@ -1796,8 +1798,9 @@ _ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n" "\n" -"If the argument is None then the callback is disabled. The method is called\n" -"with the SSLSocket, the server name as a string, and the SSLContext object.\n" +"If the argument is None then the callback is disabled. The method\n" +"is called with the SSLSocket, the server name as a string, and the\n" +"SSLContext object.\n" "\n" "See RFC 6066 for details of the SNI extension."); #if defined(_ssl__SSLContext_sni_callback_DOCSTR) @@ -1861,11 +1864,11 @@ PyDoc_STRVAR(_ssl__SSLContext_cert_store_stats__doc__, "\n" "Returns quantities of loaded X.509 certificates.\n" "\n" -"X.509 certificates with a CA extension and certificate revocation lists\n" -"inside the context\'s cert store.\n" +"X.509 certificates with a CA extension and certificate revocation\n" +"lists inside the context\'s cert store.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_CERT_STORE_STATS_METHODDEF \ {"cert_store_stats", (PyCFunction)_ssl__SSLContext_cert_store_stats, METH_NOARGS, _ssl__SSLContext_cert_store_stats__doc__}, @@ -1891,11 +1894,11 @@ PyDoc_STRVAR(_ssl__SSLContext_get_ca_certs__doc__, "\n" "Returns a list of dicts with information of loaded CA certs.\n" "\n" -"If the optional argument is True, returns a DER-encoded copy of the CA\n" -"certificate.\n" +"If the optional argument is True, returns a DER-encoded copy of the\n" +"CA certificate.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_GET_CA_CERTS_METHODDEF \ {"get_ca_certs", _PyCFunction_CAST(_ssl__SSLContext_get_ca_certs), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_get_ca_certs__doc__}, @@ -2547,8 +2550,8 @@ PyDoc_STRVAR(_ssl_RAND_status__doc__, "\n" "Returns True if the OpenSSL PRNG has been seeded with enough data and False if not.\n" "\n" -"It is necessary to seed the PRNG with RAND_add() on some platforms before\n" -"using the ssl() function."); +"It is necessary to seed the PRNG with RAND_add() on some platforms\n" +"before using the ssl() function."); #define _SSL_RAND_STATUS_METHODDEF \ {"RAND_status", (PyCFunction)_ssl_RAND_status, METH_NOARGS, _ssl_RAND_status__doc__}, @@ -2714,11 +2717,11 @@ PyDoc_STRVAR(_ssl_enum_certificates__doc__, "\n" "Retrieve certificates from Windows\' cert store.\n" "\n" -"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may provide\n" -"more cert storages, too. The function returns a list of (bytes,\n" -"encoding_type, trust) tuples. The encoding_type flag can be interpreted\n" -"with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either\n" -"a set of OIDs or the boolean True."); +"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may\n" +"provide more cert storages, too. The function returns a list of\n" +"(bytes, encoding_type, trust) tuples. The encoding_type flag can be\n" +"interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The\n" +"trust setting is either a set of OIDs or the boolean True."); #define _SSL_ENUM_CERTIFICATES_METHODDEF \ {"enum_certificates", _PyCFunction_CAST(_ssl_enum_certificates), METH_FASTCALL|METH_KEYWORDS, _ssl_enum_certificates__doc__}, @@ -2870,4 +2873,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=6849453c6b91db74 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d6407f7dbbc5d5b7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 452897b3fae..67a60c07fef 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_testmultiphase_StateAccessType_get_defining_module__doc__, "\n" "Return the module of the defining class.\n" "\n" -"Also tests that result of PyType_GetModuleByDef matches defining_class\'s\n" -"module."); +"Also tests that result of PyType_GetModuleByDef matches\n" +"defining_class\'s module."); #define _TESTMULTIPHASE_STATEACCESSTYPE_GET_DEFINING_MODULE_METHODDEF \ {"get_defining_module", _PyCFunction_CAST(_testmultiphase_StateAccessType_get_defining_module), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testmultiphase_StateAccessType_get_defining_module__doc__}, @@ -162,4 +162,4 @@ _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObj } return _testmultiphase_StateAccessType_get_count_impl(self, cls); } -/*[clinic end generated code: output=59cb50dae2d11dc1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb8568dff1205ece input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 2b1ac954b4d..6e2d8363506 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -829,7 +829,8 @@ PyDoc_STRVAR(_tkinter_setbusywaitinterval__doc__, "\n" "Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.\n" "\n" -"It should be set to a divisor of the maximum time between frames in an animation."); +"It should be set to a divisor of the maximum time between frames in\n" +"an animation."); #define _TKINTER_SETBUSYWAITINTERVAL_METHODDEF \ {"setbusywaitinterval", (PyCFunction)_tkinter_setbusywaitinterval, METH_O, _tkinter_setbusywaitinterval__doc__}, @@ -888,4 +889,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=d90c1a9850c63249 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4eca4b2509971772 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 1046355dc38..2494356aef7 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -252,8 +252,8 @@ PyDoc_STRVAR(array_array_buffer_info__doc__, "\n" "Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array\'s contents.\n" "\n" -"The length should be multiplied by the itemsize attribute to calculate\n" -"the buffer length in bytes."); +"The length should be multiplied by the itemsize attribute to\n" +"calculate the buffer length in bytes."); #define ARRAY_ARRAY_BUFFER_INFO_METHODDEF \ {"buffer_info", (PyCFunction)array_array_buffer_info, METH_NOARGS, array_array_buffer_info__doc__}, @@ -282,8 +282,8 @@ PyDoc_STRVAR(array_array_byteswap__doc__, "\n" "Byteswap all items of the array.\n" "\n" -"If the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is\n" -"raised."); +"If the items in the array are not 1, 2, 4, or 8 bytes in size,\n" +"RuntimeError is raised."); #define ARRAY_ARRAY_BYTESWAP_METHODDEF \ {"byteswap", (PyCFunction)array_array_byteswap, METH_NOARGS, array_array_byteswap__doc__}, @@ -498,9 +498,9 @@ PyDoc_STRVAR(array_array_fromunicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"The array must be a unicode type array; otherwise a ValueError is raised.\n" -"Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of\n" -"some other type."); +"The array must be a unicode type array; otherwise a ValueError is\n" +"raised. Use array.frombytes(ustr.encode(...)) to append Unicode\n" +"data to an array of some other type."); #define ARRAY_ARRAY_FROMUNICODE_METHODDEF \ {"fromunicode", (PyCFunction)array_array_fromunicode, METH_O, array_array_fromunicode__doc__}, @@ -531,9 +531,10 @@ PyDoc_STRVAR(array_array_tounicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"Convert the array to a unicode string. The array must be a unicode type array;\n" -"otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a\n" -"unicode string from an array of some other type."); +"Convert the array to a unicode string. The array must be a unicode\n" +"type array; otherwise a ValueError is raised. Use\n" +"array.tobytes().decode() to obtain a unicode string from an array of\n" +"some other type."); #define ARRAY_ARRAY_TOUNICODE_METHODDEF \ {"tounicode", (PyCFunction)array_array_tounicode, METH_NOARGS, array_array_tounicode__doc__}, @@ -688,4 +689,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=f675e0d433ae34b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fc424975be474308 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index 50745fd4f40..aa1ae087f16 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -644,7 +644,8 @@ PyDoc_STRVAR(cmath_log__doc__, "\n" "log(z[, base]) -> the logarithm of z to the given base.\n" "\n" -"If the base is not specified, returns the natural logarithm (base e) of z."); +"If the base is not specified, returns the natural logarithm (base e)\n" +"of z."); #define CMATH_LOG_METHODDEF \ {"log", _PyCFunction_CAST(cmath_log), METH_FASTCALL, cmath_log__doc__}, @@ -882,11 +883,12 @@ PyDoc_STRVAR(cmath_isclose__doc__, "\n" "Return True if a is close in value to b, and False otherwise.\n" "\n" -"For the values to be considered close, the difference between them must be\n" -"smaller than at least one of the tolerances.\n" +"For the values to be considered close, the difference between them must\n" +"be smaller than at least one of the tolerances.\n" "\n" -"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is\n" -"not close to anything, even itself. inf and -inf are only close to themselves."); +"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is,\n" +"NaN is not close to anything, even itself. inf and -inf are only close\n" +"to themselves."); #define CMATH_ISCLOSE_METHODDEF \ {"isclose", _PyCFunction_CAST(cmath_isclose), METH_FASTCALL|METH_KEYWORDS, cmath_isclose__doc__}, @@ -982,4 +984,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=454309b21cfa9bf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e01537e62454991 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 9fff4da616b..416b1acb044 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -381,8 +381,8 @@ PyDoc_STRVAR(gc_get_objects__doc__, " generation\n" " Generation to extract the objects from.\n" "\n" -"If generation is not None, return only the objects tracked by the collector\n" -"that are in that generation."); +"If generation is not None, return only the objects tracked by the\n" +"collector that are in that generation."); #define GC_GET_OBJECTS_METHODDEF \ {"get_objects", _PyCFunction_CAST(gc_get_objects), METH_FASTCALL|METH_KEYWORDS, gc_get_objects__doc__}, @@ -522,9 +522,10 @@ PyDoc_STRVAR(gc_freeze__doc__, "\n" "Freeze all current tracked objects and ignore them for future collections.\n" "\n" -"This can be used before a POSIX fork() call to make the gc copy-on-write friendly.\n" -"Note: collection before a POSIX fork() call may free pages for future allocation\n" -"which can cause copy-on-write."); +"This can be used before a POSIX fork() call to make the gc copy-on-write\n" +"friendly.\n" +"Note: collection before a POSIX fork() call may free pages for future\n" +"allocation which can cause copy-on-write."); #define GC_FREEZE_METHODDEF \ {"freeze", (PyCFunction)gc_freeze, METH_NOARGS, gc_freeze__doc__}, @@ -585,4 +586,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0041275fbac029b2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 050c21460d7..234e56012b7 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -768,8 +768,8 @@ PyDoc_STRVAR(itertools_compress__doc__, "\n" "Return data elements corresponding to true selector elements.\n" "\n" -"Forms a shorter iterator from selected data elements using the selectors to\n" -"choose the data elements."); +"Forms a shorter iterator from selected data elements using the selectors\n" +"to choose the data elements."); static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2); @@ -928,4 +928,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=7b13be3075f2d6d3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ed9db610dde299b5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 99534de0df6..473efd8ef05 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -76,8 +76,9 @@ PyDoc_STRVAR(math_frexp__doc__, "\n" "Return the mantissa and exponent of x, as pair (m, e).\n" "\n" -"m is a float and e is an int, such that x = m * 2.**e.\n" -"If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0."); +"If x is a finite nonzero number, then m is a float with\n" +"0.5 <= abs(m) < 1.0 and an integer e is such that\n" +"x == m * 2**e exactly. Else, return (x, 0)."); #define MATH_FREXP_METHODDEF \ {"frexp", (PyCFunction)math_frexp, METH_O, math_frexp__doc__}, @@ -892,8 +893,8 @@ PyDoc_STRVAR(math_nextafter__doc__, "\n" "If steps is not specified or is None, it defaults to 1.\n" "\n" -"Raises a TypeError, if x or y is not a double, or if steps is not an integer.\n" -"Raises ValueError if steps is negative."); +"Raises a TypeError, if x or y is not a double, or if steps is not\n" +"an integer. Raises ValueError if steps is negative."); #define MATH_NEXTAFTER_METHODDEF \ {"nextafter", _PyCFunction_CAST(math_nextafter), METH_FASTCALL|METH_KEYWORDS, math_nextafter__doc__}, @@ -1009,4 +1010,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=9fe02dc4af07c1e0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9ed980dea18198be input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 8b285e4a8f0..f884f03da66 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -526,8 +526,9 @@ PyDoc_STRVAR(_overlapped_Overlapped_getresult__doc__, "\n" "Retrieve result of operation.\n" "\n" -"If wait is true then it blocks until the operation is finished. If wait\n" -"is false and the operation is still pending then an error is raised."); +"If wait is true then it blocks until the operation is finished. If\n" +"wait is false and the operation is still pending then an error is\n" +"raised."); #define _OVERLAPPED_OVERLAPPED_GETRESULT_METHODDEF \ {"getresult", _PyCFunction_CAST(_overlapped_Overlapped_getresult), METH_FASTCALL, _overlapped_Overlapped_getresult__doc__}, @@ -1239,4 +1240,4 @@ _overlapped_Overlapped_WSARecvFromInto(OverlappedObject *self, PyObject *const * return return_value; } -/*[clinic end generated code: output=958cbddbcc355f47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=17f830e42fc6123d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 9f318133d87..758ba9c3368 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -216,8 +216,9 @@ PyDoc_STRVAR(pyexpat_xmlparser_GetInputContext__doc__, "\n" "Return the untranslated text of the input that caused the current event.\n" "\n" -"If the event was generated by a large amount of text (such as a start tag\n" -"for an element with many attributes), not all of the text may be available."); +"If the event was generated by a large amount of text (such as\n" +"a start tag for an element with many attributes), not all of the\n" +"text may be available."); #define PYEXPAT_XMLPARSER_GETINPUTCONTEXT_METHODDEF \ {"GetInputContext", (PyCFunction)pyexpat_xmlparser_GetInputContext, METH_NOARGS, pyexpat_xmlparser_GetInputContext__doc__}, @@ -354,9 +355,10 @@ PyDoc_STRVAR(pyexpat_xmlparser_UseForeignDTD__doc__, "\n" "Allows the application to provide an artificial external subset if one is not specified as part of the document instance.\n" "\n" -"This readily allows the use of a \'default\' document type controlled by the\n" -"application, while still getting the advantage of providing document type\n" -"information to the parser. \'flag\' defaults to True if not provided."); +"This readily allows the use of a \'default\' document type controlled\n" +"by the application, while still getting the advantage of providing\n" +"document type information to the parser. \'flag\' defaults to True if\n" +"not provided."); #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF \ {"UseForeignDTD", _PyCFunction_CAST(pyexpat_xmlparser_UseForeignDTD), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_UseForeignDTD__doc__}, @@ -405,6 +407,138 @@ pyexpat_xmlparser_UseForeignDTD(xmlparseobject *self, PyTypeObject *cls, PyObjec #endif /* (XML_COMBINED_VERSION >= 19505) */ +#if (XML_COMBINED_VERSION >= 20400) + +PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__, +"SetBillionLaughsAttackProtectionActivationThreshold($self, threshold, /)\n" +"--\n" +"\n" +"Sets the number of output bytes needed to activate protection against billion laughs attacks.\n" +"\n" +"The number of output bytes includes amplification from entity expansion\n" +"and reading DTD files.\n" +"\n" +"Parser objects usually have a protection activation threshold of 8 MiB,\n" +"but the actual default value depends on the underlying Expat library.\n" +"\n" +"Activation thresholds below 4 MiB are known to break support for DITA 1.3\n" +"payload and are hence not recommended."); + +#define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF \ + {"SetBillionLaughsAttackProtectionActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__}, + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold); + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold(xmlparseobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetBillionLaughsAttackProtectionActivationThreshold", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + unsigned long long threshold; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &threshold)) { + goto exit; + } + return_value = pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(self, cls, threshold); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20400) */ + +#if (XML_COMBINED_VERSION >= 20400) + +PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__, +"SetBillionLaughsAttackProtectionMaximumAmplification($self, max_factor,\n" +" /)\n" +"--\n" +"\n" +"Sets the maximum tolerated amplification factor for protection against billion laughs attacks.\n" +"\n" +"The amplification factor is calculated as \"(direct + indirect) / direct\"\n" +"while parsing, where \"direct\" is the number of bytes read from the primary\n" +"document in parsing and \"indirect\" is the number of bytes added by expanding\n" +"entities and reading external DTD files, combined.\n" +"\n" +"The \'max_factor\' value must be a non-NaN floating point value greater than\n" +"or equal to 1.0. Amplification factors greater than 30,000 can be observed\n" +"in the middle of parsing even with benign files in practice. In particular,\n" +"the activation threshold should be carefully chosen to avoid false positives.\n" +"\n" +"Parser objects usually have a maximum amplification factor of 100,\n" +"but the actual default value depends on the underlying Expat library."); + +#define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF \ + {"SetBillionLaughsAttackProtectionMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__}, + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor); + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification(xmlparseobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetBillionLaughsAttackProtectionMaximumAmplification", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + float max_factor; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + max_factor = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + max_factor = (float) PyFloat_AsDouble(args[0]); + if (max_factor == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(self, cls, max_factor); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20400) */ + #if (XML_COMBINED_VERSION >= 20702) PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__, @@ -413,7 +547,8 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__, "\n" "Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM.\n" "\n" -"By default, parser objects have an allocation activation threshold of 64 MiB."); +"Parser objects usually have an allocation activation threshold of 64 MiB,\n" +"but the actual default value depends on the underlying Expat library."); #define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF \ {"SetAllocTrackerActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__}, @@ -466,17 +601,19 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__, "\n" "Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated.\n" "\n" -"The amplification factor is calculated as \"allocated / direct\" while parsing,\n" -"where \"direct\" is the number of bytes read from the primary document in parsing\n" -"and \"allocated\" is the number of bytes of dynamic memory allocated in the parser\n" -"hierarchy.\n" +"The amplification factor is calculated as \"allocated / direct\" while\n" +"parsing, where \"direct\" is the number of bytes read from the primary\n" +"document in parsing and \"allocated\" is the number of bytes of\n" +"dynamic memory allocated in the parser hierarchy.\n" "\n" -"The \'max_factor\' value must be a non-NaN floating point value greater than\n" -"or equal to 1.0. Amplification factors greater than 100.0 can be observed\n" -"near the start of parsing even with benign files in practice. In particular,\n" -"the activation threshold should be carefully chosen to avoid false positives.\n" +"The \'max_factor\' value must be a non-NaN floating point value\n" +"greater than or equal to 1.0. Amplification factors greater than\n" +"100.0 can be observed near the start of parsing even with benign\n" +"files in practice. In particular, the activation threshold should\n" +"be carefully chosen to avoid false positives.\n" "\n" -"By default, parser objects have a maximum amplification factor of 100.0."); +"Parser objects usually have a maximum amplification factor of 100,\n" +"but the actual default value depends on the underlying Expat library."); #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF \ {"SetAllocTrackerMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__}, @@ -670,6 +807,14 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ +#ifndef PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF + #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF) */ + +#ifndef PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF + #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF) */ + #ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF #define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF) */ @@ -677,4 +822,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF) */ -/*[clinic end generated code: output=2e787fd99710eb5b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1994a20170f2325d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 49c0e48d2e0..2fea123785a 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(select_select__doc__, "\n" "Wait until one or more file descriptors are ready for some kind of I/O.\n" "\n" -"The first three arguments are iterables of file descriptors to be waited for:\n" +"The first three arguments are iterables of file descriptors to be waited\n" +"for:\n" "rlist -- wait until ready for reading\n" "wlist -- wait until ready for writing\n" "xlist -- wait for an \"exceptional condition\"\n" @@ -29,9 +30,9 @@ PyDoc_STRVAR(select_select__doc__, "a floating-point number to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" -"The return value is a tuple of three lists corresponding to the first three\n" -"arguments; each contains the subset of the corresponding file descriptors\n" -"that are ready.\n" +"The return value is a tuple of three lists corresponding to the first\n" +"three arguments; each contains the subset of the corresponding file\n" +"descriptors that are ready.\n" "\n" "*** IMPORTANT NOTICE ***\n" "On Windows, only sockets are supported; on Unix, all file\n" @@ -214,8 +215,8 @@ PyDoc_STRVAR(select_poll_poll__doc__, " The maximum time to wait in milliseconds, or else None (or a negative\n" " value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_POLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_poll_poll), METH_FASTCALL, select_poll_poll__doc__}, @@ -396,11 +397,11 @@ PyDoc_STRVAR(select_devpoll_poll__doc__, "Polls the set of registered file descriptors.\n" "\n" " timeout\n" -" The maximum time to wait in milliseconds, or else None (or a negative\n" -" value) to wait indefinitely.\n" +" The maximum time to wait in milliseconds, or else None (or\n" +" a negative value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_DEVPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_devpoll_poll), METH_FASTCALL, select_devpoll_poll__doc__}, @@ -498,8 +499,8 @@ PyDoc_STRVAR(select_poll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_POLL_METHODDEF \ {"poll", (PyCFunction)select_poll, METH_NOARGS, select_poll__doc__}, @@ -523,8 +524,8 @@ PyDoc_STRVAR(select_devpoll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_DEVPOLL_METHODDEF \ {"devpoll", (PyCFunction)select_devpoll, METH_NOARGS, select_devpoll__doc__}, @@ -942,8 +943,8 @@ PyDoc_STRVAR(select_epoll_poll__doc__, " maxevents\n" " the maximum number of events returned; -1 means no limit\n" "\n" -"Returns a list containing any descriptors that have events to report,\n" -"as a list of (fd, events) 2-tuples."); +"Returns a list containing any descriptors that have events to\n" +"report, as a list of (fd, events) 2-tuples."); #define SELECT_EPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_epoll_poll), METH_FASTCALL|METH_KEYWORDS, select_epoll_poll__doc__}, @@ -1360,4 +1361,4 @@ select_kqueue_control(kqueue_queue_Object *self, PyObject *const *args, Py_ssize #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=f99427b75cbe6d44 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6293bc1883273f7c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 986c0289f2b..dbb8e26b5be 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -138,11 +138,12 @@ PyDoc_STRVAR(signal_signal__doc__, "Set the action for the given signal.\n" "\n" "The action can be SIG_DFL, SIG_IGN, or a callable Python object.\n" -"The previous action is returned. See getsignal() for possible return values.\n" +"The previous action is returned. See getsignal() for possible return\n" +"values.\n" "\n" "*** IMPORTANT NOTICE ***\n" -"A signal handler function is called with two arguments:\n" -"the first is the signal number, the second is the interrupted stack frame."); +"A signal handler function is called with two arguments: the first is\n" +"the signal number, the second is the interrupted stack frame."); #define SIGNAL_SIGNAL_METHODDEF \ {"signal", _PyCFunction_CAST(signal_signal), METH_FASTCALL, signal_signal__doc__}, @@ -359,8 +360,8 @@ PyDoc_STRVAR(signal_setitimer__doc__, "\n" "Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF).\n" "\n" -"The timer will fire after value seconds and after that every interval seconds.\n" -"The itimer can be cleared by setting seconds to zero.\n" +"The timer will fire after value seconds and after that every interval\n" +"seconds. The itimer can be cleared by setting seconds to zero.\n" "\n" "Returns old values as a tuple: (delay, interval)."); @@ -505,8 +506,8 @@ PyDoc_STRVAR(signal_sigwait__doc__, "Wait for a signal.\n" "\n" "Suspend execution of the calling thread until the delivery of one of the\n" -"signals specified in the signal set sigset. The function accepts the signal\n" -"and returns the signal number."); +"signals specified in the signal set sigset. The function accepts the\n" +"signal and returns the signal number."); #define SIGNAL_SIGWAIT_METHODDEF \ {"sigwait", (PyCFunction)signal_sigwait, METH_O, signal_sigwait__doc__}, @@ -776,4 +777,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=c57b4b98fad6f4b8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=496d4bc3e9ed4528 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 83f5a4f6e9f..35522bef1dc 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -270,7 +270,8 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "Set the tty winsize for file descriptor fd.\n" "\n" "The winsize to be set is taken from the winsize argument, which\n" -"is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); +"is a two-item tuple (ws_row, ws_col) like the one returned by\n" +"tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, @@ -299,4 +300,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d2176c4d9043d3cc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 7ff3edf5a55..0dd395fb4df 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -485,8 +485,8 @@ PyDoc_STRVAR(zlib_Decompress_decompress__doc__, " Unconsumed input data will be stored in\n" " the unconsumed_tail attribute.\n" "\n" -"After calling this function, some of the input data may still be stored in\n" -"internal buffers for later processing.\n" +"After calling this function, some of the input data may still be\n" +"stored in internal buffers for later processing.\n" "Call the flush() method to clear these buffers."); #define ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF \ @@ -884,18 +884,19 @@ PyDoc_STRVAR(zlib_ZlibDecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define ZLIB_ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(zlib_ZlibDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, zlib_ZlibDecompressor_decompress__doc__}, @@ -1098,4 +1099,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=8bb840fb6af43dd4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=208bbc4d3ea84ac5 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index d901b350bc5..affb717f31b 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -910,12 +910,13 @@ cmath.log log(z[, base]) -> the logarithm of z to the given base. -If the base is not specified, returns the natural logarithm (base e) of z. +If the base is not specified, returns the natural logarithm (base e) +of z. [clinic start generated code]*/ static PyObject * cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj) -/*[clinic end generated code: output=4effdb7d258e0d94 input=e1f81d4fcfd26497]*/ +/*[clinic end generated code: output=4effdb7d258e0d94 input=eb25de0757baf4a0]*/ { Py_complex y; @@ -1132,17 +1133,18 @@ Determine whether two complex numbers are close in value. Return True if a is close in value to b, and False otherwise. -For the values to be considered close, the difference between them must be -smaller than at least one of the tolerances. +For the values to be considered close, the difference between them must +be smaller than at least one of the tolerances. --inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is -not close to anything, even itself. inf and -inf are only close to themselves. +-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, +NaN is not close to anything, even itself. inf and -inf are only close +to themselves. [clinic start generated code]*/ static int cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=8a2486cc6e0014d1 input=df9636d7de1d4ac3]*/ +/*[clinic end generated code: output=8a2486cc6e0014d1 input=301b56c90d9a79de]*/ { double diff; diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 18dbaebde29..ec3f58544cb 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -45,6 +45,7 @@ #ifndef Expat_INCLUDED # define Expat_INCLUDED 1 +# include <stdint.h> // for uint8_t # include <stdlib.h> # include "expat_external.h" @@ -917,10 +918,21 @@ XML_SetParamEntityParsing(XML_Parser parser, function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. Note: If parser == NULL, the function will do nothing and return 0. + DEPRECATED since Expat 2.8.0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); +/* Sets the hash salt to use for internal hash calculations. + Helps in preventing DoS attacks based on predicting hash function behavior. + This must be called before parsing is started. + Returns XML_TRUE if successful, XML_FALSE when called after parsing has + started or when parser is NULL. + Added in Expat 2.8.0. +*/ +XMLPARSEAPI(XML_Bool) +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]); + /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -1081,8 +1093,8 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); See https://semver.org */ # define XML_MAJOR_VERSION 2 -# define XML_MINOR_VERSION 7 -# define XML_MICRO_VERSION 5 +# define XML_MINOR_VERSION 8 +# define XML_MICRO_VERSION 1 # ifdef __cplusplus } diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index 09d3161dbc0..70df73c8e00 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -22,5 +22,10 @@ // bpo-30947: Python uses best available entropy sources to // call XML_SetHashSalt(), expat entropy sources are not needed #define XML_POOR_ENTROPY 1 +#undef HAVE_ARC4RANDOM +#undef HAVE_ARC4RANDOM_BUF +#undef HAVE_GETENTROPY +#undef HAVE_GETRANDOM +#undef HAVE_SYSCALL_GETRANDOM #endif /* EXPAT_CONFIG_H */ diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index cf4d445e68b..cc945c424e4 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -12,9 +12,10 @@ Copyright (c) 2001-2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2016 Cristian Rodríguez <crrodriguez@opensuse.org> - Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2025 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> Copyright (c) 2018 Yury Gribov <tetra2005@gmail.com> + Copyright (c) 2026 Matthew Fernandez <matthew.fernandez@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -48,7 +49,7 @@ /* Expat tries very hard to make the API boundary very specifically defined. There are two macros defined to control this boundary; each of these can be defined before including this header to - achieve some different behavior, but doing so it not recommended or + achieve some different behavior, but doing so is not recommended or tested frequently. XMLCALL - The calling convention to use for all calls across the diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 61266ebb772..420d4217a56 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,7 +28,7 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2003 Greg Stein <gstein@users.sourceforge.net> - Copyright (c) 2016-2025 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2018 Yury Gribov <tetra2005@gmail.com> Copyright (c) 2019 David Loffredo <loffredo@steptools.com> Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com> @@ -113,6 +113,7 @@ #if defined(_WIN32) \ && (! defined(__USE_MINGW_ANSI_STDIO) \ || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) +# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" # define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" # if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" @@ -122,6 +123,7 @@ # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" # endif #else +# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" # define EXPAT_FMT_ULL(midpart) "%" midpart "llu" # if ! defined(ULONG_MAX) # error Compiler did not define ULONG_MAX for us diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index f16eb89d11e..7f171290d1e 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_7_5" -expected_libexpat_version="2.7.5" -expected_libexpat_sha256="9931f9860d18e6cf72d183eb8f309bfb96196c00e1d40caa978e95bc9aa978b6" +expected_libexpat_tag="R_2_8_1" +expected_libexpat_version="2.8.1" +expected_libexpat_sha256="a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} @@ -57,6 +57,18 @@ rm libexpat.tar.gz # Step 3: Add the namespacing include to expat_external.h sed -i 's/# define Expat_External_INCLUDED 1/&\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h +# Step 4: Skip the Windows rand_s entropy path in xmlparse.c when +# XML_POOR_ENTROPY is set. +sed -z -i 's|#if defined(_WIN32)\n# include "random_rand_s\.h"\n#endif /\* defined(_WIN32) \*/|#if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n# include "random_rand_s.h"\n#endif /* defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY) */|' xmlparse.c +sed -z -i 's|# ifdef _WIN32\n if (writeRandomBytes_rand_s|# if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n if (writeRandomBytes_rand_s|' xmlparse.c + +if ! grep -q '#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY)' xmlparse.c; then + echo " +Error: rand_s gate not patched in xmlparse.c; +This may be due to source changes and will require updating this script" >&2 + exit 1 +fi + echo " Updated! next steps: - Verify all is okay: diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 0248b6651ff..95d34675856 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 93c1caa66e2b0310459482516af05505b57c5cb7b96df777105308fc585c85d1 (2.7.5+) +/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -41,10 +41,12 @@ Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com> Copyright (c) 2024-2025 Berkay Eren Ürün <berkay.ueruen@siemens.com> Copyright (c) 2024 Hanno Böck <hanno@gentoo.org> - Copyright (c) 2025 Matthew Fernandez <matthew.fernandez@gmail.com> + Copyright (c) 2025-2026 Matthew Fernandez <matthew.fernandez@gmail.com> Copyright (c) 2025 Atrem Borovik <polzovatellllk@gmail.com> Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com> Copyright (c) 2026 Rosen Penev <rosenp@gmail.com> + Copyright (c) 2026 Francesco Bertolaccini + Copyright (c) 2026 Christian Ng <christianrng@berkeley.edu> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -84,28 +86,16 @@ # error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) #endif -#if defined(HAVE_SYSCALL_GETRANDOM) -# if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ -# endif -#endif - -#ifdef _WIN32 -/* force stdlib to define rand_s() */ -# if ! defined(_CRT_RAND_S) -# define _CRT_RAND_S -# endif -#endif - #include <stdbool.h> #include <stddef.h> #include <string.h> /* memset(), memcpy() */ #include <assert.h> #include <limits.h> /* INT_MAX, UINT_MAX */ #include <stdio.h> /* fprintf */ -#include <stdlib.h> /* getenv, rand_s */ +#include <stdlib.h> /* getenv */ #include <stdint.h> /* SIZE_MAX, uintptr_t */ #include <math.h> /* isnan */ +#include <errno.h> #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -125,26 +115,34 @@ #include "expat.h" #include "siphash.h" +#if defined(HAVE_ARC4RANDOM) +# include "random_arc4random.h" +#endif /* defined(HAVE_ARC4RANDOM) */ + +#if defined(HAVE_ARC4RANDOM_BUF) +# include "random_arc4random_buf.h" +#endif // defined(HAVE_ARC4RANDOM_BUF) + +#if defined(XML_DEV_URANDOM) +# include "random_dev_urandom.h" +#endif /* defined(XML_DEV_URANDOM) */ + +#if defined(HAVE_GETENTROPY) +# include "random_getentropy.h" +#endif // defined(HAVE_GETENTROPY) + #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# if defined(HAVE_GETRANDOM) -# include <sys/random.h> /* getrandom */ -# else -# include <unistd.h> /* syscall */ -# include <sys/syscall.h> /* SYS_getrandom */ -# endif -# if ! defined(GRND_NONBLOCK) -# define GRND_NONBLOCK 0x0001 -# endif /* defined(GRND_NONBLOCK) */ -#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ +# include "random_getrandom.h" +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ -#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32) -# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 -#endif +#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) +# include "random_rand_s.h" +#endif /* defined(_WIN32) && ! defined(XML_POOR_ENTROPY) */ #if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM) \ && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) \ - && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32) \ - && ! defined(XML_POOR_ENTROPY) + && ! defined(HAVE_GETENTROPY) && ! defined(XML_DEV_URANDOM) \ + && ! defined(_WIN32) && ! defined(XML_POOR_ENTROPY) # error You do not have support for any sources of high quality entropy \ enabled. For end user security, that is probably not what you want. \ \ @@ -153,10 +151,11 @@ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ + * BSD / macOS >=10.12 / glibc >=2.25 (getentropy): HAVE_GETENTROPY, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ * Windows >=Vista (rand_s): _WIN32. \ \ - If insist on not using any of these, bypass this error by defining \ + If you insist on not using any of these, bypass this error by defining \ XML_POOR_ENTROPY; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ @@ -388,6 +387,7 @@ typedef struct { int nDefaultAtts; int allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; typedef struct { @@ -604,7 +604,7 @@ static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, static XML_Char *copyString(const XML_Char *s, XML_Parser parser); -static unsigned long generate_hash_secret_salt(XML_Parser parser); +static struct sipkey generate_hash_secret_salt(void); static XML_Bool startParsing(XML_Parser parser); static XML_Parser parserCreate(const XML_Char *encodingName, @@ -777,7 +777,8 @@ struct XML_ParserStruct { XML_Bool m_useForeignDTD; enum XML_ParamEntityParsing m_paramEntityParsing; #endif - unsigned long m_hash_secret_salt; + struct sipkey m_hash_secret_salt_128; + XML_Bool m_hash_secret_salt_set; #if XML_GE == 1 ACCOUNTING m_accounting; MALLOC_TRACKER m_alloc_tracker; @@ -1036,135 +1037,6 @@ static const XML_Char implicitContext[] ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; -/* To avoid warnings about unused functions: */ -#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) - -# if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - -/* Obtain entropy on Linux 3.17+ */ -static int -writeRandomBytes_getrandom_nonblock(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = GRND_NONBLOCK; - - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; - - assert(bytesToWrite <= INT_MAX); - - const int bytesWrittenMore = -# if defined(HAVE_GETRANDOM) - (int)getrandom(currentTarget, bytesToWrite, getrandomFlags); -# else - (int)syscall(SYS_getrandom, currentTarget, bytesToWrite, - getrandomFlags); -# endif - - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; - } - } while (! success && (errno == EINTR)); - - return success; -} - -# endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ - -# if ! defined(_WIN32) && defined(XML_DEV_URANDOM) - -/* Extract entropy from /dev/urandom */ -static int -writeRandomBytes_dev_urandom(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; - - const int fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) { - return 0; - } - - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; - - const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); - - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; - } - } while (! success && (errno == EINTR)); - - close(fd); - return success; -} - -# endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ - -#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ - -#if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) - -static void -writeRandomBytes_arc4random(void *target, size_t count) { - size_t bytesWrittenTotal = 0; - - while (bytesWrittenTotal < count) { - const uint32_t random32 = arc4random(); - size_t i = 0; - - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } - } -} - -#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */ - -#ifdef _WIN32 - -/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), - as it didn't declare it in its header prior to version 5.3.0 of its - runtime package (mingwrt, containing stdlib.h). The upstream fix - was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ -# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ - && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) -__declspec(dllimport) int rand_s(unsigned int *); -# endif - -/* Obtain entropy on Windows using the rand_s() function which - * generates cryptographically secure random numbers. Internally it - * uses RtlGenRandom API which is present in Windows XP and later. - */ -static int -writeRandomBytes_rand_s(void *target, size_t count) { - size_t bytesWrittenTotal = 0; - - while (bytesWrittenTotal < count) { - unsigned int random32 = 0; - size_t i = 0; - - if (rand_s(&random32)) - return 0; /* failure */ - - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } - } - return 1; /* success */ -} - -#endif /* _WIN32 */ - #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) static unsigned long @@ -1192,69 +1064,70 @@ gather_time_entropy(void) { #endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ -static unsigned long -ENTROPY_DEBUG(const char *label, unsigned long entropy) { +static struct sipkey +ENTROPY_DEBUG(const char *label, struct sipkey entropy_128) { if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { - fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, - (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); + fprintf(stderr, + "expat: Entropy: %s --> [0x" EXPAT_FMT_LLX( + "016") ", 0x" EXPAT_FMT_LLX("016") "] (16 bytes)\n", + label, (unsigned long long)entropy_128.k[0], + (unsigned long long)entropy_128.k[1]); } - return entropy; + return entropy_128; } -static unsigned long -generate_hash_secret_salt(XML_Parser parser) { - unsigned long entropy; - (void)parser; +static struct sipkey +generate_hash_secret_salt(void) { + struct sipkey entropy; /* "Failproof" high quality providers: */ #if defined(HAVE_ARC4RANDOM_BUF) - arc4random_buf(&entropy, sizeof(entropy)); + writeRandomBytes_arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); #elif defined(HAVE_ARC4RANDOM) - writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + writeRandomBytes_arc4random(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ -# ifdef _WIN32 - if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) { +# if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) + if (writeRandomBytes_rand_s(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("rand_s", entropy); } +# elif defined(HAVE_GETENTROPY) + if (writeRandomBytes_getentropy(&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getentropy", entropy); + } + errno = 0; # elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } # endif # if ! defined(_WIN32) && defined(XML_DEV_URANDOM) - if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_dev_urandom(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("/dev/urandom", entropy); } # endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ - entropy = gather_time_entropy(); + entropy.k[0] = 0; + entropy.k[1] = gather_time_entropy(); # if ! defined(__wasi__) /* Process ID is 0 bits entropy if attacker has local access */ - entropy ^= getpid(); + entropy.k[1] ^= getpid(); # endif /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); + entropy.k[1] *= 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy); } else { - return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951ULL); + entropy.k[1] *= 2305843009213693951ULL; + return ENTROPY_DEBUG("fallback(8)", entropy); } #endif } -static unsigned long -get_hash_secret_salt(XML_Parser parser) { - const XML_Parser rootParser = getRootParserOf(parser, NULL); - assert(! rootParser->m_parentParser); - - return rootParser->m_hash_secret_salt; -} - static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1323,8 +1196,10 @@ callProcessor(XML_Parser parser, const char *start, const char *end, static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ - if (parser->m_hash_secret_salt == 0) - parser->m_hash_secret_salt = generate_hash_secret_salt(parser); + if (parser->m_hash_secret_salt_set != XML_TRUE) { + parser->m_hash_secret_salt_128 = generate_hash_secret_salt(); + parser->m_hash_secret_salt_set = XML_TRUE; + } if (parser->m_ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -1612,7 +1487,9 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_useForeignDTD = XML_FALSE; parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif - parser->m_hash_secret_salt = 0; + parser->m_hash_secret_salt_128.k[0] = 0; + parser->m_hash_secret_salt_128.k[1] = 0; + parser->m_hash_secret_salt_set = XML_FALSE; #if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); @@ -1779,7 +1656,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt; + struct sipkey oldhash_secret_salt_128; + XML_Bool oldhash_secret_salt_set; XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ @@ -1825,7 +1703,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - oldhash_secret_salt = parser->m_hash_secret_salt; + oldhash_secret_salt_128 = parser->m_hash_secret_salt_128; + oldhash_secret_salt_set = parser->m_hash_secret_salt_set; oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD @@ -1880,7 +1759,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandlerArg = oldExternalEntityRefHandlerArg; parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; - parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_hash_secret_salt_128 = oldhash_secret_salt_128; + parser->m_hash_secret_salt_set = oldhash_secret_salt_set; parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD @@ -2324,6 +2204,7 @@ XML_SetParamEntityParsing(XML_Parser parser, #endif } +// DEPRECATED since Expat 2.8.0. int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) @@ -2335,10 +2216,46 @@ XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (parserBusy(rootParser)) return 0; - rootParser->m_hash_secret_salt = hash_salt; + + rootParser->m_hash_secret_salt_128.k[0] = 0; + rootParser->m_hash_secret_salt_128.k[1] = hash_salt; + + if (hash_salt != 0) { // to remain backwards compatible + rootParser->m_hash_secret_salt_set = XML_TRUE; + + if (sizeof(unsigned long) == 4) + ENTROPY_DEBUG("explicit(4)", rootParser->m_hash_secret_salt_128); + else + ENTROPY_DEBUG("explicit(8)", rootParser->m_hash_secret_salt_128); + } + return 1; } +XML_Bool XMLCALL +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { + if (parser == NULL) + return XML_FALSE; + + if (entropy == NULL) + return XML_FALSE; + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + /* block after XML_Parse()/XML_ParseBuffer() has been called */ + if (parserBusy(rootParser)) + return XML_FALSE; + + sip_tokey(&(rootParser->m_hash_secret_salt_128), entropy); + + rootParser->m_hash_secret_salt_set = XML_TRUE; + + ENTROPY_DEBUG("explicit(16)", rootParser->m_hash_secret_salt_128); + + return XML_TRUE; +} + enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { @@ -3853,6 +3770,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttsNames.parser) + hashTableInit(&(elementType->defaultAttsNames), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -7186,10 +7105,10 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } @@ -7236,6 +7155,12 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->isCdata = isCdata; if (! isCdata) attId->maybeTokenized = XML_TRUE; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + if (! nameAddedOrFound) + return 0; + type->nDefaultAtts += 1; return 1; } @@ -7561,6 +7486,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7602,6 +7528,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7695,6 +7622,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttsNames.parser) + hashTableInit(&(newE->defaultAttsNames), parser); + if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. * The preprocessor guard addresses the "always false" warning @@ -7719,8 +7650,9 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); for (i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7729,6 +7661,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + if (! nameAddedOrFound) { + return 0; + } } } @@ -7842,8 +7780,10 @@ keylen(KEY s) { static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { - key->k[0] = 0; - key->k[1] = get_hash_secret_salt(parser); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + *key = rootParser->m_hash_secret_salt_128; } static unsigned long FASTCALL @@ -8473,6 +8413,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttsNames.parser) + hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index b1dfb456e5d..d56bee82dd2 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -12,7 +12,7 @@ Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2005-2009 Steven Solie <steven@solie.ca> - Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2023 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> Copyright (c) 2019 David Loffredo <loffredo@steptools.com> Copyright (c) 2021 Donghee Na <donghee.na@python.org> diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index f6e5f742c92..32cd5f147e9 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -12,7 +12,7 @@ Copyright (c) 2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net> Copyright (c) 2005-2009 Steven Solie <steven@solie.ca> - Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2024 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2016 Pascal Cuoq <cuoq@trust-in-soft.com> Copyright (c) 2016 Don Lewis <truckman@apache.org> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index 1cd60de1e4f..810ca2c6d04 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -11,7 +11,7 @@ Copyright (c) 2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> - Copyright (c) 2017-2026 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2017-2021 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com> Licensed under the MIT license: diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 5a4efed9946..42521e40ba7 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1180,20 +1180,6 @@ faulthandler_stack_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif /* defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_SIGACTION) */ -static int -faulthandler_traverse(PyObject *module, visitproc visit, void *arg) -{ - Py_VISIT(thread.file); -#ifdef FAULTHANDLER_USER - if (user_signals != NULL) { - for (size_t signum=0; signum < Py_NSIG; signum++) - Py_VISIT(user_signals[signum].file); - } -#endif - Py_VISIT(fatal_error.file); - return 0; -} - #ifdef MS_WINDOWS static PyObject * faulthandler_raise_exception(PyObject *self, PyObject *args) @@ -1322,7 +1308,6 @@ static struct PyModuleDef module_def = { .m_name = "faulthandler", .m_doc = module_doc, .m_methods = module_methods, - .m_traverse = faulthandler_traverse, .m_slots = faulthandler_slots }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index ee24900ae88..f9a27d4b43e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -305,13 +305,13 @@ gc.get_objects Return a list of objects tracked by the collector (excluding the list returned). -If generation is not None, return only the objects tracked by the collector -that are in that generation. +If generation is not None, return only the objects tracked by the +collector that are in that generation. [clinic start generated code]*/ static PyObject * gc_get_objects_impl(PyObject *module, Py_ssize_t generation) -/*[clinic end generated code: output=48b35fea4ba6cb0e input=ef7da9df9806754c]*/ +/*[clinic end generated code: output=48b35fea4ba6cb0e input=3a819826fbde5eef]*/ { if (PySys_Audit("gc.get_objects", "n", generation) < 0) { return NULL; @@ -421,14 +421,15 @@ gc.freeze Freeze all current tracked objects and ignore them for future collections. -This can be used before a POSIX fork() call to make the gc copy-on-write friendly. -Note: collection before a POSIX fork() call may free pages for future allocation -which can cause copy-on-write. +This can be used before a POSIX fork() call to make the gc copy-on-write +friendly. +Note: collection before a POSIX fork() call may free pages for future +allocation which can cause copy-on-write. [clinic start generated code]*/ static PyObject * gc_freeze_impl(PyObject *module) -/*[clinic end generated code: output=502159d9cdc4c139 input=b602b16ac5febbe5]*/ +/*[clinic end generated code: output=502159d9cdc4c139 input=989012d0ba5a066f]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyGC_Freeze(interp); diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 5add3c216b3..96e8542def5 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -2229,6 +2229,10 @@ Return a chain object whose .__next__() method returns elements from the\n\ first iterable until it is exhausted, then elements from the next\n\ iterable, until all of the iterables are exhausted."); +PyDoc_STRVAR(chain_class_getitem_doc, +"chain is generic over the type of its contents.\n\ +This is the union of the types of the input iterable contents."); + static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF {"__reduce__", (PyCFunction)chain_reduce, METH_NOARGS, @@ -2236,7 +2240,7 @@ static PyMethodDef chain_methods[] = { {"__setstate__", (PyCFunction)chain_setstate, METH_O, setstate_doc}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, chain_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; @@ -3769,13 +3773,13 @@ itertools.compress.__new__ selectors as seq2: object Return data elements corresponding to true selector elements. -Forms a shorter iterator from selected data elements using the selectors to -choose the data elements. +Forms a shorter iterator from selected data elements using the selectors +to choose the data elements. [clinic start generated code]*/ static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2) -/*[clinic end generated code: output=7e67157212ed09e0 input=79596d7cd20c77e5]*/ +/*[clinic end generated code: output=7e67157212ed09e0 input=32ca4347dbc46749]*/ { PyObject *data=NULL, *selectors=NULL; compressobject *lz; diff --git a/Modules/main.c b/Modules/main.c index 581c87270ff..9d92ca0b2f9 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -511,6 +511,7 @@ pymain_run_interactive_hook(int *exitcode) } if (PySys_Audit("cpython.run_interactivehook", "O", hook) < 0) { + Py_DECREF(hook); goto error; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index aee1b17be9c..e0a1db0fea8 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2095,13 +2095,14 @@ math.frexp Return the mantissa and exponent of x, as pair (m, e). -m is a float and e is an int, such that x = m * 2.**e. -If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0. +If x is a finite nonzero number, then m is a float with +0.5 <= abs(m) < 1.0 and an integer e is such that +x == m * 2**e exactly. Else, return (x, 0). [clinic start generated code]*/ static PyObject * math_frexp_impl(PyObject *module, double x) -/*[clinic end generated code: output=03e30d252a15ad4a input=96251c9e208bc6e9]*/ +/*[clinic end generated code: output=03e30d252a15ad4a input=215cf8ea28a0959b]*/ { int i; /* deal with special cases directly, to sidestep platform @@ -3940,13 +3941,13 @@ Return the floating-point value the given number of steps after x towards y. If steps is not specified or is None, it defaults to 1. -Raises a TypeError, if x or y is not a double, or if steps is not an integer. -Raises ValueError if steps is negative. +Raises a TypeError, if x or y is not a double, or if steps is not +an integer. Raises ValueError if steps is negative. [clinic start generated code]*/ static PyObject * math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) -/*[clinic end generated code: output=cc6511f02afc099e input=7f2a5842112af2b4]*/ +/*[clinic end generated code: output=cc6511f02afc099e input=89764144d1a33160]*/ { #if defined(_AIX) if (x == y) { diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 567593e05c4..fdf43d615d1 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -875,13 +875,14 @@ _overlapped.Overlapped.getresult Retrieve result of operation. -If wait is true then it blocks until the operation is finished. If wait -is false and the operation is still pending then an error is raised. +If wait is true then it blocks until the operation is finished. If +wait is false and the operation is still pending then an error is +raised. [clinic start generated code]*/ static PyObject * _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) -/*[clinic end generated code: output=8c9bd04d08994f6c input=aa5b03e9897ca074]*/ +/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/ { DWORD transferred = 0; BOOL ret; @@ -1909,6 +1910,11 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, } #endif + if (bufobj->len < (Py_ssize_t)size) { + PyErr_SetString(PyExc_ValueError, "nbytes is greater than the length of the buffer"); + return NULL; + } + wsabuf.buf = bufobj->buf; wsabuf.len = size; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 78a8776c1ca..86187cbeb47 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -15889,7 +15889,7 @@ static PyMethodDef DirEntry_methods[] = { OS_DIRENTRY_INODE_METHODDEF OS_DIRENTRY___FSPATH___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("DirEntry is generic over the type of the path (str or bytes)")}, {NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 65802fca221..915d01e8bd0 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -994,13 +994,14 @@ pyexpat.xmlparser.GetInputContext Return the untranslated text of the input that caused the current event. -If the event was generated by a large amount of text (such as a start tag -for an element with many attributes), not all of the text may be available. +If the event was generated by a large amount of text (such as +a start tag for an element with many attributes), not all of the +text may be available. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) -/*[clinic end generated code: output=a88026d683fc22cc input=034df8712db68379]*/ +/*[clinic end generated code: output=a88026d683fc22cc input=3ff7cb00783c8f98]*/ { if (self->in_callback) { int offset, size; @@ -1133,15 +1134,16 @@ pyexpat.xmlparser.UseForeignDTD Allows the application to provide an artificial external subset if one is not specified as part of the document instance. -This readily allows the use of a 'default' document type controlled by the -application, while still getting the advantage of providing document type -information to the parser. 'flag' defaults to True if not provided. +This readily allows the use of a 'default' document type controlled +by the application, while still getting the advantage of providing +document type information to the parser. 'flag' defaults to True if +not provided. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, int flag) -/*[clinic end generated code: output=d7d98252bd25a20f input=23440ecb0573fb29]*/ +/*[clinic end generated code: output=d7d98252bd25a20f input=0387bce44fd53d99]*/ { pyexpat_state *state = PyType_GetModuleState(cls); enum XML_Error rc; @@ -1154,7 +1156,7 @@ pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, } #endif -#if XML_COMBINED_VERSION >= 20702 +#if XML_COMBINED_VERSION >= 20400 static PyObject * set_activation_threshold(xmlparseobject *self, PyTypeObject *cls, @@ -1198,6 +1200,76 @@ set_maximum_amplification(xmlparseobject *self, } #endif +#if XML_COMBINED_VERSION >= 20400 +/*[clinic input] +pyexpat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold + + cls: defining_class + threshold: unsigned_long_long + / + +Sets the number of output bytes needed to activate protection against billion laughs attacks. + +The number of output bytes includes amplification from entity expansion +and reading DTD files. + +Parser objects usually have a protection activation threshold of 8 MiB, +but the actual default value depends on the underlying Expat library. + +Activation thresholds below 4 MiB are known to break support for DITA 1.3 +payload and are hence not recommended. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold) +/*[clinic end generated code: output=0c082342f1c78114 input=5a51695a481def92]*/ +{ + return set_activation_threshold( + self, cls, threshold, + XML_SetBillionLaughsAttackProtectionActivationThreshold + ); +} +#endif + +#if XML_COMBINED_VERSION >= 20400 +/*[clinic input] +pyexpat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification + + cls: defining_class + max_factor: float + / + +Sets the maximum tolerated amplification factor for protection against billion laughs attacks. + +The amplification factor is calculated as "(direct + indirect) / direct" +while parsing, where "direct" is the number of bytes read from the primary +document in parsing and "indirect" is the number of bytes added by expanding +entities and reading external DTD files, combined. + +The 'max_factor' value must be a non-NaN floating point value greater than +or equal to 1.0. Amplification factors greater than 30,000 can be observed +in the middle of parsing even with benign files in practice. In particular, +the activation threshold should be carefully chosen to avoid false positives. + +Parser objects usually have a maximum amplification factor of 100, +but the actual default value depends on the underlying Expat library. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor) +/*[clinic end generated code: output=c590439eadf463fa input=5de7c6dd7169b3b0]*/ +{ + return set_maximum_amplification( + self, cls, max_factor, + XML_SetBillionLaughsAttackProtectionMaximumAmplification + ); +} +#endif + #if XML_COMBINED_VERSION >= 20702 /*[clinic input] pyexpat.xmlparser.SetAllocTrackerActivationThreshold @@ -1208,14 +1280,15 @@ pyexpat.xmlparser.SetAllocTrackerActivationThreshold Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM. -By default, parser objects have an allocation activation threshold of 64 MiB. +Parser objects usually have an allocation activation threshold of 64 MiB, +but the actual default value depends on the underlying Expat library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, PyTypeObject *cls, unsigned long long threshold) -/*[clinic end generated code: output=bed7e93207ba08c5 input=9c706b75c18e4ea1]*/ +/*[clinic end generated code: output=bed7e93207ba08c5 input=b74171709a77f2d9]*/ { return set_activation_threshold( self, cls, threshold, @@ -1234,24 +1307,26 @@ pyexpat.xmlparser.SetAllocTrackerMaximumAmplification Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated. -The amplification factor is calculated as "allocated / direct" while parsing, -where "direct" is the number of bytes read from the primary document in parsing -and "allocated" is the number of bytes of dynamic memory allocated in the parser -hierarchy. +The amplification factor is calculated as "allocated / direct" while +parsing, where "direct" is the number of bytes read from the primary +document in parsing and "allocated" is the number of bytes of +dynamic memory allocated in the parser hierarchy. -The 'max_factor' value must be a non-NaN floating point value greater than -or equal to 1.0. Amplification factors greater than 100.0 can be observed -near the start of parsing even with benign files in practice. In particular, -the activation threshold should be carefully chosen to avoid false positives. +The 'max_factor' value must be a non-NaN floating point value +greater than or equal to 1.0. Amplification factors greater than +100.0 can be observed near the start of parsing even with benign +files in practice. In particular, the activation threshold should +be carefully chosen to avoid false positives. -By default, parser objects have a maximum amplification factor of 100.0. +Parser objects usually have a maximum amplification factor of 100, +but the actual default value depends on the underlying Expat library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl(xmlparseobject *self, PyTypeObject *cls, float max_factor) -/*[clinic end generated code: output=6e44bd48c9b112a0 input=918b9266b490a722]*/ +/*[clinic end generated code: output=6e44bd48c9b112a0 input=db40271991462073]*/ { return set_maximum_amplification( self, cls, max_factor, @@ -1269,6 +1344,8 @@ static struct PyMethodDef xmlparse_methods[] = { PYEXPAT_XMLPARSER_EXTERNALENTITYPARSERCREATE_METHODDEF PYEXPAT_XMLPARSER_SETPARAMENTITYPARSING_METHODDEF PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF + PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF + PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF @@ -1384,7 +1461,10 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, Py_DECREF(self); return NULL; } -#if XML_COMBINED_VERSION >= 20100 +#if XML_COMBINED_VERSION >= 20800 + /* This feature was added upstream in libexpat 2.8.0. */ + XML_SetHashSalt16Bytes(self->itself, _Py_HashSecret.expat.hashsalt16); +#elif XML_COMBINED_VERSION >= 20100 /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.expat.hashsalt); @@ -2246,6 +2326,11 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20800 + capi->SetHashSalt16Bytes = XML_SetHashSalt16Bytes; +#else + capi->SetHashSalt16Bytes = NULL; +#endif #if XML_COMBINED_VERSION >= 20600 capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; #else @@ -2258,6 +2343,13 @@ pyexpat_exec(PyObject *mod) capi->SetAllocTrackerActivationThreshold = NULL; capi->SetAllocTrackerMaximumAmplification = NULL; #endif +#if XML_COMBINED_VERSION >= 20400 + capi->SetBillionLaughsAttackProtectionActivationThreshold = XML_SetBillionLaughsAttackProtectionActivationThreshold; + capi->SetBillionLaughsAttackProtectionMaximumAmplification = XML_SetBillionLaughsAttackProtectionMaximumAmplification; +#else + capi->SetBillionLaughsAttackProtectionActivationThreshold = NULL; + capi->SetBillionLaughsAttackProtectionMaximumAmplification = NULL; +#endif /* export using capsule */ PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, diff --git a/Modules/readline.c b/Modules/readline.c index f9362c312de..7a612deae43 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1318,6 +1318,10 @@ setup_readline(readlinestate *mod_state) completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ + + if (!completer_word_break_characters) { + goto error; + } #ifdef WITH_EDITLINE // libedit uses rl_basic_word_break_characters instead of // rl_completer_word_break_characters as complete delimiter @@ -1361,6 +1365,10 @@ setup_readline(readlinestate *mod_state) RESTORE_LOCALE(saved_locale) return 0; + +error: + RESTORE_LOCALE(saved_locale) + return -1; } /* Wrapper around GNU readline that handles signals differently. */ diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 5bd9b7732a4..f69df694d44 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -252,7 +252,8 @@ select.select Wait until one or more file descriptors are ready for some kind of I/O. -The first three arguments are iterables of file descriptors to be waited for: +The first three arguments are iterables of file descriptors to be waited +for: rlist -- wait until ready for reading wlist -- wait until ready for writing xlist -- wait for an "exceptional condition" @@ -265,9 +266,9 @@ The optional 4th argument specifies a timeout in seconds; it may be a floating-point number to specify fractions of seconds. If it is absent or None, the call will never time out. -The return value is a tuple of three lists corresponding to the first three -arguments; each contains the subset of the corresponding file descriptors -that are ready. +The return value is a tuple of three lists corresponding to the first +three arguments; each contains the subset of the corresponding file +descriptors that are ready. *** IMPORTANT NOTICE *** On Windows, only sockets are supported; on Unix, all file @@ -277,7 +278,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=1199d5e101abca4a]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=34a2c2075ca9830e]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -612,13 +613,13 @@ select.poll.poll Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=e0a9c0aa283de8c8]*/ { PyObject *result_list = NULL; int poll_result, i, j; @@ -936,19 +937,19 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) @critical_section select.devpoll.poll timeout as timeout_obj: object = None - The maximum time to wait in milliseconds, or else None (or a negative - value) to wait indefinitely. + The maximum time to wait in milliseconds, or else None (or + a negative value) to wait indefinitely. / Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=9e1672658d728539]*/ { struct dvpoll dvp; PyObject *result_list = NULL; @@ -1196,13 +1197,13 @@ select.poll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_poll_impl(PyObject *module) -/*[clinic end generated code: output=16a665a4e1d228c5 input=3f877909d5696bbf]*/ +/*[clinic end generated code: output=16a665a4e1d228c5 input=0aefd4527e99e0aa]*/ { return (PyObject *)newPollObject(module); } @@ -1214,13 +1215,13 @@ select.devpoll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_devpoll_impl(PyObject *module) -/*[clinic end generated code: output=ea9213cc87fd9581 input=53a1af94564f00a3]*/ +/*[clinic end generated code: output=ea9213cc87fd9581 input=4c2ac27d10248526]*/ { return (PyObject *)newDevPollObject(module); } @@ -1558,14 +1559,14 @@ select.epoll.poll Wait for events on the epoll file descriptor. -Returns a list containing any descriptors that have events to report, -as a list of (fd, events) 2-tuples. +Returns a list containing any descriptors that have events to +report, as a list of (fd, events) 2-tuples. [clinic start generated code]*/ static PyObject * select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int maxevents) -/*[clinic end generated code: output=e02d121a20246c6c input=33d34a5ea430fd5b]*/ +/*[clinic end generated code: output=e02d121a20246c6c input=5a49d65788c70c7a]*/ { int nfds, i; PyObject *elist = NULL, *etuple = NULL; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 6f53dcda824..ed811b4ecdc 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -467,16 +467,17 @@ signal.signal Set the action for the given signal. The action can be SIG_DFL, SIG_IGN, or a callable Python object. -The previous action is returned. See getsignal() for possible return values. +The previous action is returned. See getsignal() for possible return +values. *** IMPORTANT NOTICE *** -A signal handler function is called with two arguments: -the first is the signal number, the second is the interrupted stack frame. +A signal handler function is called with two arguments: the first is +the signal number, the second is the interrupted stack frame. [clinic start generated code]*/ static PyObject * signal_signal_impl(PyObject *module, int signalnum, PyObject *handler) -/*[clinic end generated code: output=b44cfda43780f3a1 input=deee84af5fa0432c]*/ +/*[clinic end generated code: output=b44cfda43780f3a1 input=99ce4035ec56ffc1]*/ { _signal_module_state *modstate = get_signal_state(module); PyObject *old_handler; @@ -854,8 +855,8 @@ signal.setitimer Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF). -The timer will fire after value seconds and after that every interval seconds. -The itimer can be cleared by setting seconds to zero. +The timer will fire after value seconds and after that every interval +seconds. The itimer can be cleared by setting seconds to zero. Returns old values as a tuple: (delay, interval). [clinic start generated code]*/ @@ -863,7 +864,7 @@ Returns old values as a tuple: (delay, interval). static PyObject * signal_setitimer_impl(PyObject *module, int which, PyObject *seconds, PyObject *interval) -/*[clinic end generated code: output=65f9dcbddc35527b input=de43daf194e6f66f]*/ +/*[clinic end generated code: output=65f9dcbddc35527b input=bd9f0d2ed8614193]*/ { _signal_module_state *modstate = get_signal_state(module); @@ -1024,13 +1025,13 @@ signal.sigwait Wait for a signal. Suspend execution of the calling thread until the delivery of one of the -signals specified in the signal set sigset. The function accepts the signal -and returns the signal number. +signals specified in the signal set sigset. The function accepts the +signal and returns the signal number. [clinic start generated code]*/ static PyObject * signal_sigwait_impl(PyObject *module, sigset_t sigset) -/*[clinic end generated code: output=f43770699d682f96 input=a6fbd47b1086d119]*/ +/*[clinic end generated code: output=f43770699d682f96 input=91773742dd416a3e]*/ { int err, signum; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index e3dba737c39..27d807f9af4 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8962,6 +8962,9 @@ socket_exec(PyObject *m) /* Initialize gethostbyname lock */ #if defined(USE_GETHOSTBYNAME_LOCK) netdb_lock = PyThread_allocate_lock(); + if (netdb_lock == NULL) { + goto error; + } #endif #ifdef MS_WINDOWS diff --git a/Modules/termios.c b/Modules/termios.c index f2c5a4bafa7..03c8e0889da 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -483,12 +483,13 @@ termios.tcsetwinsize Set the tty winsize for file descriptor fd. The winsize to be set is taken from the winsize argument, which -is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize(). +is a two-item tuple (ws_row, ws_col) like the one returned by +tcgetwinsize(). [clinic start generated code]*/ static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) -/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=4a06424465b24aee]*/ +/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=efc9beb16d06382a]*/ { if (!PySequence_Check(winsz) || PySequence_Size(winsz) != 2) { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 2d317bd6f71..6b191c1ec62 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -508,19 +508,80 @@ get_decomp_record(PyObject *self, Py_UCS4 code, (*index)++; } +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(int kind, void *data, + Py_ssize_t start, Py_ssize_t end) +{ + for (Py_ssize_t i = start + 1; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + Py_ssize_t j = i; + + while (j > start) { + Py_UCS4 previous = PyUnicode_READ(kind, data, j - 1); + if (_getrecord_ex(previous)->combining <= combining) { + break; + } + PyUnicode_WRITE(kind, data, j, previous); + j--; + } + if (j != i) { + PyUnicode_WRITE(kind, data, j, code); + } + } +} + +static void +canonical_ordering_sort_counting(int kind, void *data, + Py_ssize_t start, Py_ssize_t end, + Py_UCS4 *sortbuf) +{ + Py_ssize_t counts[256] = {0}; + Py_ssize_t run_length = end - start; + Py_ssize_t total = 0; + + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + counts[combining]++; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(counts); i++) { + Py_ssize_t count = counts[i]; + counts[i] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC. */ + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + sortbuf[counts[combining]++] = code; + } + for (Py_ssize_t i = 0; i < run_length; i++) { + PyUnicode_WRITE(kind, data, start + i, sortbuf[i]); + } +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { PyObject *result; Py_UCS4 *output; Py_ssize_t i, o, osize; - int kind; - const void *data; + int input_kind, result_kind; + const void *input_data; + void *result_data; /* Longest decomposition in Unicode 3.2: U+FDFA */ Py_UCS4 stack[20]; Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UCS4 *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_LENGTH(input); @@ -540,11 +601,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) return NULL; } i = o = 0; - kind = PyUnicode_KIND(input); - data = PyUnicode_DATA(input); + input_kind = PyUnicode_KIND(input); + input_data = PyUnicode_DATA(input); while (i < isize) { - stack[stackptr++] = PyUnicode_READ(kind, data, i++); + stack[stackptr++] = PyUnicode_READ(input_kind, input_data, i++); while(stackptr) { Py_UCS4 code = stack[--stackptr]; /* Hangul Decomposition adds three characters in @@ -611,35 +672,66 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) PyMem_Free(output); if (!result) return NULL; + /* result is guaranteed to be ready, as it is compact. */ - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); + result_kind = PyUnicode_KIND(result); + result_data = PyUnicode_DATA(result); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. */ i = 0; - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - for (i++; i < PyUnicode_GET_LENGTH(result); i++) { - cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < o) { + Py_ssize_t run_length, run_start; + int needs_sort = 0; + + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + prev = _getrecord_ex(ch)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1); - PyUnicode_WRITE(kind, data, o+1, - PyUnicode_READ(kind, data, o)); - PyUnicode_WRITE(kind, data, o, tmp); - o--; - if (o < 0) - break; - prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < o) { + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + cur = _getrecord_ex(ch)->combining; + if (cur == 0) { break; + } + if (prev > cur) { + needs_sort = 1; + } + prev = cur; + i++; } - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; + if (!needs_sort) { + continue; + } + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(result_kind, result_data, + run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UCS4 *new_sortbuf = PyMem_Resize(sortbuf, + Py_UCS4, + run_length); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + + canonical_ordering_sort_counting(result_kind, result_data, + run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; } @@ -1501,32 +1593,17 @@ capi_getcode(const char* name, int namelen, Py_UCS4* code, return _check_alias_and_seq(code, with_named_seq); } -static void -unicodedata_destroy_capi(PyObject *capsule) -{ - void *capi = PyCapsule_GetPointer(capsule, PyUnicodeData_CAPSULE_NAME); - PyMem_Free(capi); -} - static PyObject * unicodedata_create_capi(void) { - _PyUnicode_Name_CAPI *capi = PyMem_Malloc(sizeof(_PyUnicode_Name_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->getname = capi_getucname; - capi->getcode = capi_getcode; - - PyObject *capsule = PyCapsule_New(capi, - PyUnicodeData_CAPSULE_NAME, - unicodedata_destroy_capi); - if (capsule == NULL) { - PyMem_Free(capi); - } - return capsule; -}; + // Statically allocated so that any cached pointers stay valid after unicodedata + // is removed from sys.modules and the capsule is gc'd (gh-149449). + static _PyUnicode_Name_CAPI capi = { + .getname = capi_getucname, + .getcode = capi_getcode, + }; + return PyCapsule_New(&capi, PyUnicodeData_CAPSULE_NAME, NULL); +} /* -------------------------------------------------------------------- */ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index f59f9a76abd..820bace9420 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -875,15 +875,15 @@ zlib.Decompress.decompress Return a bytes object containing the decompressed version of the data. -After calling this function, some of the input data may still be stored in -internal buffers for later processing. +After calling this function, some of the input data may still be +stored in internal buffers for later processing. Call the flush() method to clear these buffers. [clinic start generated code]*/ static PyObject * zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=b024a93c2c922d57 input=bfb37b3864cfb606]*/ +/*[clinic end generated code: output=b024a93c2c922d57 input=b8c9e2d124fe4720]*/ { int err = Z_OK; Py_ssize_t ibuflen; @@ -1675,6 +1675,7 @@ decompress(ZlibDecompressor *self, uint8_t *data, return result; error: + self->zst.next_in = NULL; Py_XDECREF(result); return NULL; } @@ -1687,24 +1688,25 @@ zlib.ZlibDecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=990d32787b775f85 input=0b29d99715250b96]*/ +/*[clinic end generated code: output=990d32787b775f85 input=6fb56d60b48cd843]*/ { PyObject *result = NULL; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index feea0aef276..449010713db 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1395,14 +1395,15 @@ bytearray.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=b6a8f01c2a74e446 input=cfff956d4d127a9b]*/ +/*[clinic end generated code: output=b6a8f01c2a74e446 input=50a30905c8c8835c]*/ { char *input, *output; const char *table_chars; @@ -1500,15 +1501,15 @@ bytearray.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytearray_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=1df267d99f56b15e input=b10de38c85950a63]*/ +/*[clinic end generated code: output=1df267d99f56b15e input=b1e7b0acbbaeb48a]*/ { return _Py_bytes_maketrans(frm, to); } @@ -1545,8 +1546,8 @@ bytearray.split sep: object = None The delimiter according which to split the bytearray. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1557,7 +1558,7 @@ Return a list of the sections in the bytearray, using sep as the delimiter. static PyObject * bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=833e2cf385d9a04d input=24f82669f41bf523]*/ +/*[clinic end generated code: output=833e2cf385d9a04d input=14afd5161c17fb05]*/ { PyObject *list = NULL; @@ -1596,17 +1597,18 @@ bytearray.partition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray. If the separator is -found, returns a 3-tuple containing the part before the separator, the -separator itself, and the part after it as new bytearray objects. +This will search for the separator sep in the bytearray. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it as new +bytearray objects. -If the separator is not found, returns a 3-tuple containing the copy of the -original bytearray object and two empty bytearray objects. +If the separator is not found, returns a 3-tuple containing the copy +of the original bytearray object and two empty bytearray objects. [clinic start generated code]*/ static PyObject * bytearray_partition(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=45d2525ddd35f957 input=8f644749ee4fc83a]*/ +/*[clinic end generated code: output=45d2525ddd35f957 input=cba58c2f7620fd20]*/ { PyObject *bytesep, *result; @@ -1633,18 +1635,19 @@ bytearray.rpartition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray, starting at the end. -If the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it as new bytearray -objects. +This will search for the separator sep in the bytearray, starting at +the end. If the separator is found, returns a 3-tuple containing +the part before the separator, the separator itself, and the part +after it as new bytearray objects. -If the separator is not found, returns a 3-tuple containing two empty bytearray -objects and the copy of the original bytearray object. +If the separator is not found, returns a 3-tuple containing two +empty bytearray objects and the copy of the original bytearray +object. [clinic start generated code]*/ static PyObject * bytearray_rpartition(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=440de3c9426115e8 input=7e3df3e6cb8fa0ac]*/ +/*[clinic end generated code: output=440de3c9426115e8 input=6fb8e0f18e6bb0ec]*/ { PyObject *bytesep, *result; @@ -1668,13 +1671,14 @@ bytearray.rsplit = bytearray.split Return a list of the sections in the bytearray, using sep as the delimiter. -Splitting is done starting at the end of the bytearray and working to the front. +Splitting is done starting at the end of the bytearray and working +to the front. [clinic start generated code]*/ static PyObject * bytearray_rsplit_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=a55e0b5a03cb6190 input=a68286e4dd692ffe]*/ +/*[clinic end generated code: output=a55e0b5a03cb6190 input=3c6d2e7a19e14c85]*/ { PyObject *list = NULL; @@ -2045,12 +2049,13 @@ bytearray.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes) -/*[clinic end generated code: output=760412661a34ad5a input=ef7bb59b09c21d62]*/ +/*[clinic end generated code: output=760412661a34ad5a input=063ca44b1f107843]*/ { return bytearray_strip_impl_helper(self, bytes, BOTHSTRIP); } @@ -2097,11 +2102,11 @@ bytearray.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytearray. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytearray using the codec registered for encoding. [clinic start generated code]*/ @@ -2109,7 +2114,7 @@ Decode the bytearray using the codec registered for encoding. static PyObject * bytearray_decode_impl(PyByteArrayObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=f57d43f4a00b42c5 input=f28d8f903020257b]*/ +/*[clinic end generated code: output=f57d43f4a00b42c5 input=f2f539b09781a267]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -2135,14 +2140,15 @@ bytearray.join Concatenate any number of bytes/bytearray objects. -The bytearray whose method is called is inserted in between each pair. +The bytearray whose method is called is inserted in between each +pair. The result is returned as a new bytearray object. [clinic start generated code]*/ static PyObject * bytearray_join(PyByteArrayObject *self, PyObject *iterable_of_bytes) -/*[clinic end generated code: output=a8516370bf68ae08 input=aba6b1f9b30fcb8e]*/ +/*[clinic end generated code: output=a8516370bf68ae08 input=866170ddeab25355]*/ { self->ob_exports++; // this protects `self` from being cleared/resized if `iterable_of_bytes` is a custom iterator PyObject* ret = stringlib_bytes_join((PyObject*)self, iterable_of_bytes); @@ -2157,13 +2163,13 @@ bytearray.splitlines Return a list of the lines in the bytearray, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) -/*[clinic end generated code: output=4223c94b895f6ad9 input=66b2dcdea8d093bf]*/ +/*[clinic end generated code: output=4223c94b895f6ad9 input=f9fcab8a2a9b1432]*/ { return stringlib_splitlines( (PyObject*) self, PyByteArray_AS_STRING(self), @@ -2181,12 +2187,13 @@ bytearray.fromhex Create a bytearray object from a string of hexadecimal numbers. Spaces between two numbers are accepted. -Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') +Example: + bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') [clinic start generated code]*/ static PyObject * bytearray_fromhex_impl(PyTypeObject *type, PyObject *string) -/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=f033a16d1fb21f48]*/ +/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=45fdeb2c74fd76fa]*/ { PyObject *result = _PyBytes_FromHex(string, type == &PyByteArray_Type); if (type != &PyByteArray_Type && result != NULL) { @@ -2201,8 +2208,8 @@ bytearray.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: int = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytearray object. @@ -2220,7 +2227,7 @@ Create a string of hexadecimal numbers from a bytearray object. static PyObject * bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=29c4e5ef72c565a0 input=808667e49bcccb54]*/ +/*[clinic end generated code: output=29c4e5ef72c565a0 input=ad961bc88df6473a]*/ { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index 981aa571643..d4b33f20f34 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -328,8 +328,8 @@ _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len) PyDoc_STRVAR_shared(_Py_title__doc__, "B.title() -> copy of B\n\ \n\ -Return a titlecased version of B, i.e. ASCII words start with uppercase\n\ -characters, all remaining cased characters have lowercase."); +Return a titlecased version of B, i.e. ASCII words start with\n\ +uppercase characters, all remaining cased characters have lowercase."); void _Py_bytes_title(char *result, const char *s, Py_ssize_t len) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index d891121589f..af6ca7270b0 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1752,8 +1752,8 @@ bytes.split sep: object = None The delimiter according which to split the bytes. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1763,7 +1763,7 @@ Return a list of the sections in the bytes, using sep as the delimiter. static PyObject * bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=52126b5844c1d8ef input=8b809b39074abbfa]*/ +/*[clinic end generated code: output=52126b5844c1d8ef input=9c0faeffc870f672]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -1792,17 +1792,17 @@ bytes.partition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator sep in the bytes. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original bytes -object and two empty bytes objects. +If the separator is not found, returns a 3-tuple containing the +original bytes object and two empty bytes objects. [clinic start generated code]*/ static PyObject * bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=f532b392a17ff695 input=61cca95519406099]*/ +/*[clinic end generated code: output=f532b392a17ff695 input=2e6e551ea4f8b95a]*/ { return stringlib_partition( (PyObject*) self, @@ -1819,17 +1819,18 @@ bytes.rpartition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator sep in the bytes, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty bytes -objects and the original bytes object. +If the separator is not found, returns a 3-tuple containing two +empty bytes objects and the original bytes object. [clinic start generated code]*/ static PyObject * bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=191b114cbb028e50 input=d78db010c8cfdbe1]*/ +/*[clinic end generated code: output=191b114cbb028e50 input=f7d24f722a5470a4]*/ { return stringlib_rpartition( (PyObject*) self, @@ -1843,12 +1844,13 @@ bytes.rsplit = bytes.split Return a list of the sections in the bytes, using sep as the delimiter. -Splitting is done starting at the end of the bytes and working to the front. +Splitting is done starting at the end of the bytes and working to +the front. [clinic start generated code]*/ static PyObject * bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=ba698d9ea01e1c8f input=0f86c9f28f7d7b7b]*/ +/*[clinic end generated code: output=ba698d9ea01e1c8f input=7fd643d4337b6a9b]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -2063,12 +2065,13 @@ bytes.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytes_strip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=c7c228d3bd104a1b input=8a354640e4e0b3ef]*/ +/*[clinic end generated code: output=c7c228d3bd104a1b input=9ffea5f752032bd0]*/ { return do_argstrip(self, BOTHSTRIP, bytes); } @@ -2136,14 +2139,15 @@ bytes.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytes_translate_impl(PyBytesObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=43be3437f1956211 input=0ecdf159f654233c]*/ +/*[clinic end generated code: output=43be3437f1956211 input=4e4460a981d768c5]*/ { const char *input; char *output; @@ -2265,15 +2269,15 @@ bytes.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytes_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=a36f6399d4b77f6f input=a3bd00d430a0979f]*/ +/*[clinic end generated code: output=a36f6399d4b77f6f input=c88bcc17621f785d]*/ { return _Py_bytes_maketrans(frm, to); } @@ -2315,13 +2319,14 @@ bytes.removeprefix as bytes_removeprefix Return a bytes object with the given prefix string removed if present. -If the bytes starts with the prefix string, return bytes[len(prefix):]. -Otherwise, return a copy of the original bytes. +If the bytes starts with the prefix string, return +bytes[len(prefix):]. Otherwise, return a copy of the original +bytes. [clinic start generated code]*/ static PyObject * bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix) -/*[clinic end generated code: output=f006865331a06ab6 input=0c93bac817a8502c]*/ +/*[clinic end generated code: output=f006865331a06ab6 input=8f371f9421b8addd]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2351,14 +2356,14 @@ bytes.removesuffix as bytes_removesuffix Return a bytes object with the given suffix string removed if present. -If the bytes ends with the suffix string and that suffix is not empty, -return bytes[:-len(prefix)]. Otherwise, return a copy of the original -bytes. +If the bytes ends with the suffix string and that suffix is not +empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the +original bytes. [clinic start generated code]*/ static PyObject * bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) -/*[clinic end generated code: output=d887d308e3242eeb input=9f4e1da8c637bbf1]*/ +/*[clinic end generated code: output=d887d308e3242eeb input=35eada0260d1171b]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2436,11 +2441,11 @@ bytes.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytes. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytes using the codec registered for encoding. [clinic start generated code]*/ @@ -2448,7 +2453,7 @@ Decode the bytes using the codec registered for encoding. static PyObject * bytes_decode_impl(PyBytesObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=5649a53dde27b314 input=958174769d2a40ca]*/ +/*[clinic end generated code: output=5649a53dde27b314 input=94e9b8524f1d7f37]*/ { return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); } @@ -2461,13 +2466,13 @@ bytes.splitlines Return a list of the lines in the bytes, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends) -/*[clinic end generated code: output=3484149a5d880ffb input=5d7b898af2fe55c0]*/ +/*[clinic end generated code: output=3484149a5d880ffb input=8b7b6915be775bcf]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), @@ -2579,9 +2584,10 @@ bytes.hex sep: object = NULL An optional single character or byte to separate hex bytes. + bytes_per_sep: int = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytes object. @@ -2599,7 +2605,7 @@ Create a string of hexadecimal numbers from a bytes object. static PyObject * bytes_hex_impl(PyBytesObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=1f134da504064139 input=1a21282b1f1ae595]*/ +/*[clinic end generated code: output=1f134da504064139 input=67d7bb5f70f0d6f2]*/ { const char *argbuf = PyBytes_AS_STRING(self); Py_ssize_t arglen = PyBytes_GET_SIZE(self); diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index c748c53e1c0..c61cceb59bd 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -573,8 +573,9 @@ PyDoc_STRVAR(bytearray_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTEARRAY_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytearray_translate), METH_FASTCALL|METH_KEYWORDS, bytearray_translate__doc__}, @@ -639,8 +640,8 @@ PyDoc_STRVAR(bytearray_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -757,8 +758,8 @@ PyDoc_STRVAR(bytearray_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -842,12 +843,13 @@ PyDoc_STRVAR(bytearray_partition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray. If the separator is\n" -"found, returns a 3-tuple containing the part before the separator, the\n" -"separator itself, and the part after it as new bytearray objects.\n" +"This will search for the separator sep in the bytearray. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it as new\n" +"bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the copy of the\n" -"original bytearray object and two empty bytearray objects."); +"If the separator is not found, returns a 3-tuple containing the copy\n" +"of the original bytearray object and two empty bytearray objects."); #define BYTEARRAY_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytearray_partition, METH_O, bytearray_partition__doc__}, @@ -858,13 +860,14 @@ PyDoc_STRVAR(bytearray_rpartition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray, starting at the end.\n" -"If the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it as new bytearray\n" -"objects.\n" +"This will search for the separator sep in the bytearray, starting at\n" +"the end. If the separator is found, returns a 3-tuple containing\n" +"the part before the separator, the separator itself, and the part\n" +"after it as new bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytearray\n" -"objects and the copy of the original bytearray object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytearray objects and the copy of the original bytearray\n" +"object."); #define BYTEARRAY_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytearray_rpartition, METH_O, bytearray_rpartition__doc__}, @@ -877,13 +880,14 @@ PyDoc_STRVAR(bytearray_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytearray and working to the front."); +"Splitting is done starting at the end of the bytearray and working\n" +"to the front."); #define BYTEARRAY_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytearray_rsplit), METH_FASTCALL|METH_KEYWORDS, bytearray_rsplit__doc__}, @@ -1151,7 +1155,8 @@ PyDoc_STRVAR(bytearray_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTEARRAY_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytearray_strip), METH_FASTCALL, bytearray_strip__doc__}, @@ -1256,11 +1261,11 @@ PyDoc_STRVAR(bytearray_decode__doc__, " encoding\n" " The encoding with which to decode the bytearray.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTEARRAY_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytearray_decode), METH_FASTCALL|METH_KEYWORDS, bytearray_decode__doc__}, @@ -1354,7 +1359,8 @@ PyDoc_STRVAR(bytearray_join__doc__, "\n" "Concatenate any number of bytes/bytearray objects.\n" "\n" -"The bytearray whose method is called is inserted in between each pair.\n" +"The bytearray whose method is called is inserted in between each\n" +"pair.\n" "\n" "The result is returned as a new bytearray object."); @@ -1367,8 +1373,8 @@ PyDoc_STRVAR(bytearray_splitlines__doc__, "\n" "Return a list of the lines in the bytearray, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTEARRAY_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytearray_splitlines), METH_FASTCALL|METH_KEYWORDS, bytearray_splitlines__doc__}, @@ -1434,7 +1440,8 @@ PyDoc_STRVAR(bytearray_fromhex__doc__, "Create a bytearray object from a string of hexadecimal numbers.\n" "\n" "Spaces between two numbers are accepted.\n" -"Example: bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); +"Example:\n" +" bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); #define BYTEARRAY_FROMHEX_METHODDEF \ {"fromhex", (PyCFunction)bytearray_fromhex, METH_O|METH_CLASS, bytearray_fromhex__doc__}, @@ -1468,8 +1475,8 @@ PyDoc_STRVAR(bytearray_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = bytearray([0xb9, 0x01, 0xef])\n" @@ -1616,4 +1623,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=5f861b02e3fa278b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ba624d41653ce07 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 0b4b3750173..01dc77bbbf3 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -35,8 +35,8 @@ PyDoc_STRVAR(bytes_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -119,12 +119,12 @@ PyDoc_STRVAR(bytes_partition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator sep in the bytes. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original bytes\n" -"object and two empty bytes objects."); +"If the separator is not found, returns a 3-tuple containing the\n" +"original bytes object and two empty bytes objects."); #define BYTES_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytes_partition, METH_O, bytes_partition__doc__}, @@ -158,12 +158,13 @@ PyDoc_STRVAR(bytes_rpartition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator sep in the bytes, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytes\n" -"objects and the original bytes object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytes objects and the original bytes object."); #define BYTES_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytes_rpartition, METH_O, bytes_rpartition__doc__}, @@ -199,13 +200,14 @@ PyDoc_STRVAR(bytes_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytes and working to the front."); +"Splitting is done starting at the end of the bytes and working to\n" +"the front."); #define BYTES_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytes_rsplit), METH_FASTCALL|METH_KEYWORDS, bytes_rsplit__doc__}, @@ -504,7 +506,8 @@ PyDoc_STRVAR(bytes_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTES_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytes_strip), METH_FASTCALL, bytes_strip__doc__}, @@ -658,8 +661,9 @@ PyDoc_STRVAR(bytes_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTES_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytes_translate), METH_FASTCALL|METH_KEYWORDS, bytes_translate__doc__}, @@ -724,8 +728,8 @@ PyDoc_STRVAR(bytes_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -840,8 +844,9 @@ PyDoc_STRVAR(bytes_removeprefix__doc__, "\n" "Return a bytes object with the given prefix string removed if present.\n" "\n" -"If the bytes starts with the prefix string, return bytes[len(prefix):].\n" -"Otherwise, return a copy of the original bytes."); +"If the bytes starts with the prefix string, return\n" +"bytes[len(prefix):]. Otherwise, return a copy of the original\n" +"bytes."); #define BYTES_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)bytes_removeprefix, METH_O, bytes_removeprefix__doc__}, @@ -875,9 +880,9 @@ PyDoc_STRVAR(bytes_removesuffix__doc__, "\n" "Return a bytes object with the given suffix string removed if present.\n" "\n" -"If the bytes ends with the suffix string and that suffix is not empty,\n" -"return bytes[:-len(prefix)]. Otherwise, return a copy of the original\n" -"bytes."); +"If the bytes ends with the suffix string and that suffix is not\n" +"empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the\n" +"original bytes."); #define BYTES_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)bytes_removesuffix, METH_O, bytes_removesuffix__doc__}, @@ -1016,11 +1021,11 @@ PyDoc_STRVAR(bytes_decode__doc__, " encoding\n" " The encoding with which to decode the bytes.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTES_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytes_decode), METH_FASTCALL|METH_KEYWORDS, bytes_decode__doc__}, @@ -1114,8 +1119,8 @@ PyDoc_STRVAR(bytes_splitlines__doc__, "\n" "Return a list of the lines in the bytes, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTES_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytes_splitlines), METH_FASTCALL|METH_KEYWORDS, bytes_splitlines__doc__}, @@ -1215,8 +1220,8 @@ PyDoc_STRVAR(bytes_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = b\'\\xb9\\x01\\xef\'\n" @@ -1384,4 +1389,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=d6801c6001e57f91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d2bb8823b8a6a75b input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 68e2d7f528c..3883a726be4 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -411,7 +411,8 @@ PyDoc_STRVAR(code__varname_from_oparg__doc__, "\n" "(internal-only) Return the local variable name for the given oparg.\n" "\n" -"WARNING: this method is for internal use only and may change or go away."); +"WARNING: this method is for internal use only and may change or go\n" +"away."); #define CODE__VARNAME_FROM_OPARG_METHODDEF \ {"_varname_from_oparg", _PyCFunction_CAST(code__varname_from_oparg), METH_FASTCALL|METH_KEYWORDS, code__varname_from_oparg__doc__}, @@ -464,4 +465,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=d604263a3ca72a0f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ffe4f70404970c51 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index d104b071890..1e6b29d3054 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -255,9 +255,9 @@ PyDoc_STRVAR(float___getformat____doc__, "\n" "It exists mainly to be used in Python\'s test suite.\n" "\n" -"This function returns whichever of \'unknown\', \'IEEE, big-endian\' or \'IEEE,\n" -"little-endian\' best describes the format of floating-point numbers used by the\n" -"C type named by typestr."); +"This function returns whichever of \'unknown\', \'IEEE, big-endian\' or\n" +"\'IEEE, little-endian\' best describes the format of floating-point\n" +"numbers used by the C type named by typestr."); #define FLOAT___GETFORMAT___METHODDEF \ {"__getformat__", (PyCFunction)float___getformat__, METH_O|METH_CLASS, float___getformat____doc__}, @@ -318,4 +318,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=d8bbcd83977d516f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=87a117517a970173 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 588e021fb71..1b251fd9cc7 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -187,11 +187,11 @@ PyDoc_STRVAR(list_sort__doc__, "\n" "Sort the list in ascending order and return None.\n" "\n" -"The sort is in-place (i.e. the list itself is modified) and stable (i.e. the\n" -"order of two equal elements is maintained).\n" +"The sort is in-place (i.e. the list itself is modified) and stable\n" +"(i.e. the order of two equal elements is maintained).\n" "\n" -"If a key function is given, apply it once to each list item and sort them,\n" -"ascending or descending, according to their function values.\n" +"If a key function is given, apply it once to each list item and sort\n" +"them, ascending or descending, according to their function values.\n" "\n" "The reverse flag can be set to sort in descending order."); @@ -439,4 +439,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=854957a1d4a89bbd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=005658119324f9ae input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 56bc3864582..06d2b4538bc 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -259,19 +259,20 @@ PyDoc_STRVAR(int_to_bytes__doc__, "Return an array of bytes representing an integer.\n" "\n" " length\n" -" Length of bytes object to use. An OverflowError is raised if the\n" -" integer is not representable with the given number of bytes. Default\n" -" is length 1.\n" +" Length of bytes object to use. An OverflowError is raised if\n" +" the integer is not representable with the given number of bytes.\n" +" Default is length 1.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Determines whether two\'s complement is used to represent the integer.\n" -" If signed is False and a negative integer is given, an OverflowError\n" -" is raised."); +" Determines whether two\'s complement is used to represent the\n" +" integer. If signed is False and a negative integer is given,\n" +" an OverflowError is raised."); #define INT_TO_BYTES_METHODDEF \ {"to_bytes", _PyCFunction_CAST(int_to_bytes), METH_FASTCALL|METH_KEYWORDS, int_to_bytes__doc__}, @@ -372,17 +373,19 @@ PyDoc_STRVAR(int_from_bytes__doc__, "\n" " bytes\n" " Holds the array of bytes to convert. The argument must either\n" -" support the buffer protocol or be an iterable object producing bytes.\n" -" Bytes and bytearray are examples of built-in objects that support the\n" -" buffer protocol.\n" +" support the buffer protocol or be an iterable object producing\n" +" bytes. Bytes and bytearray are examples of built-in objects that\n" +" support the buffer protocol.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Indicates whether two\'s complement is used to represent the integer."); +" Indicates whether two\'s complement is used to represent the\n" +" integer."); #define INT_FROM_BYTES_METHODDEF \ {"from_bytes", _PyCFunction_CAST(int_from_bytes), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, int_from_bytes__doc__}, @@ -476,4 +479,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=2ba2d8dcda9b99da input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2ccde4b93141d1a1 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index f199434dacb..9f6541dfbc2 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -249,11 +249,12 @@ PyDoc_STRVAR(memoryview_tobytes__doc__, "\n" "Return the data in the buffer as a byte string.\n" "\n" -"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of the\n" -"original array is converted to C or Fortran order. For contiguous views,\n" -"\'A\' returns an exact copy of the physical memory. In particular, in-memory\n" -"Fortran order is preserved. For non-contiguous views, the data is converted\n" -"to C first. order=None is the same as order=\'C\'."); +"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of\n" +"the original array is converted to C or Fortran order. For\n" +"contiguous views, \'A\' returns an exact copy of the physical memory.\n" +"In particular, in-memory Fortran order is preserved. For\n" +"non-contiguous views, the data is converted to C first. order=None\n" +"is the same as order=\'C\'."); #define MEMORYVIEW_TOBYTES_METHODDEF \ {"tobytes", _PyCFunction_CAST(memoryview_tobytes), METH_FASTCALL|METH_KEYWORDS, memoryview_tobytes__doc__}, @@ -335,8 +336,8 @@ PyDoc_STRVAR(memoryview_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = memoryview(b\'\\xb9\\x01\\xef\')\n" @@ -413,4 +414,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=7e76a09106921ba2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e6eb7b47a172cf6 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index 5ef53806569..6ba432ccc0c 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -207,7 +207,8 @@ PyDoc_STRVAR(OrderedDict_popitem__doc__, "\n" "Remove and return a (key, value) pair from the dictionary.\n" "\n" -"Pairs are returned in LIFO order if last is true or FIFO order if false."); +"Pairs are returned in LIFO order if last is true or FIFO order if\n" +"false."); #define ORDEREDDICT_POPITEM_METHODDEF \ {"popitem", _PyCFunction_CAST(OrderedDict_popitem), METH_FASTCALL|METH_KEYWORDS, OrderedDict_popitem__doc__}, @@ -332,4 +333,4 @@ OrderedDict_move_to_end(PyODictObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=eff78d2a3f9379bd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1441f72b8624d808 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 1db304e7063..62192443a1c 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -33,8 +33,8 @@ PyDoc_STRVAR(unicode_title__doc__, "\n" "Return a version of the string where each word is titlecased.\n" "\n" -"More specifically, words start with uppercased characters and all remaining\n" -"cased characters have lower case."); +"More specifically, words start with uppercased characters and all\n" +"remaining cased characters have lower case."); #define UNICODE_TITLE_METHODDEF \ {"title", (PyCFunction)unicode_title, METH_NOARGS, unicode_title__doc__}, @@ -54,8 +54,8 @@ PyDoc_STRVAR(unicode_capitalize__doc__, "\n" "Return a capitalized version of the string.\n" "\n" -"More specifically, make the first character have upper case and the rest lower\n" -"case."); +"More specifically, make the first character have upper case and the\n" +"rest lower case."); #define UNICODE_CAPITALIZE_METHODDEF \ {"capitalize", (PyCFunction)unicode_capitalize, METH_NOARGS, unicode_capitalize__doc__}, @@ -93,7 +93,8 @@ PyDoc_STRVAR(unicode_center__doc__, "\n" "Return a centered string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_CENTER_METHODDEF \ {"center", _PyCFunction_CAST(unicode_center), METH_FASTCALL, unicode_center__doc__}, @@ -142,7 +143,8 @@ PyDoc_STRVAR(unicode_count__doc__, "\n" "Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation."); +"Optional arguments start and end are interpreted as in slice\n" +"notation."); #define UNICODE_COUNT_METHODDEF \ {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, @@ -202,8 +204,8 @@ PyDoc_STRVAR(unicode_encode__doc__, " errors\n" " The error handling scheme to use for encoding errors.\n" " The default is \'strict\' meaning that encoding errors raise a\n" -" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\' and\n" -" \'xmlcharrefreplace\' as well as any other name registered with\n" +" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +" and \'xmlcharrefreplace\' as well as any other name registered with\n" " codecs.register_error that can handle UnicodeEncodeErrors."); #define UNICODE_ENCODE_METHODDEF \ @@ -362,8 +364,8 @@ PyDoc_STRVAR(unicode_find__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_FIND_METHODDEF \ {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, @@ -418,8 +420,8 @@ PyDoc_STRVAR(unicode_index__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_INDEX_METHODDEF \ {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, @@ -495,8 +497,8 @@ PyDoc_STRVAR(unicode_islower__doc__, "\n" "Return True if the string is a lowercase string, False otherwise.\n" "\n" -"A string is lowercase if all cased characters in the string are lowercase and\n" -"there is at least one cased character in the string."); +"A string is lowercase if all cased characters in the string are\n" +"lowercase and there is at least one cased character in the string."); #define UNICODE_ISLOWER_METHODDEF \ {"islower", (PyCFunction)unicode_islower, METH_NOARGS, unicode_islower__doc__}, @@ -516,8 +518,8 @@ PyDoc_STRVAR(unicode_isupper__doc__, "\n" "Return True if the string is an uppercase string, False otherwise.\n" "\n" -"A string is uppercase if all cased characters in the string are uppercase and\n" -"there is at least one cased character in the string."); +"A string is uppercase if all cased characters in the string are\n" +"uppercase and there is at least one cased character in the string."); #define UNICODE_ISUPPER_METHODDEF \ {"isupper", (PyCFunction)unicode_isupper, METH_NOARGS, unicode_isupper__doc__}, @@ -558,8 +560,8 @@ PyDoc_STRVAR(unicode_isspace__doc__, "\n" "Return True if the string is a whitespace string, False otherwise.\n" "\n" -"A string is whitespace if all characters in the string are whitespace and there\n" -"is at least one character in the string."); +"A string is whitespace if all characters in the string are\n" +"whitespace and there is at least one character in the string."); #define UNICODE_ISSPACE_METHODDEF \ {"isspace", (PyCFunction)unicode_isspace, METH_NOARGS, unicode_isspace__doc__}, @@ -579,8 +581,8 @@ PyDoc_STRVAR(unicode_isalpha__doc__, "\n" "Return True if the string is an alphabetic string, False otherwise.\n" "\n" -"A string is alphabetic if all characters in the string are alphabetic and there\n" -"is at least one character in the string."); +"A string is alphabetic if all characters in the string are\n" +"alphabetic and there is at least one character in the string."); #define UNICODE_ISALPHA_METHODDEF \ {"isalpha", (PyCFunction)unicode_isalpha, METH_NOARGS, unicode_isalpha__doc__}, @@ -600,8 +602,8 @@ PyDoc_STRVAR(unicode_isalnum__doc__, "\n" "Return True if the string is an alpha-numeric string, False otherwise.\n" "\n" -"A string is alpha-numeric if all characters in the string are alpha-numeric and\n" -"there is at least one character in the string."); +"A string is alpha-numeric if all characters in the string are\n" +"alpha-numeric and there is at least one character in the string."); #define UNICODE_ISALNUM_METHODDEF \ {"isalnum", (PyCFunction)unicode_isalnum, METH_NOARGS, unicode_isalnum__doc__}, @@ -621,8 +623,8 @@ PyDoc_STRVAR(unicode_isdecimal__doc__, "\n" "Return True if the string is a decimal string, False otherwise.\n" "\n" -"A string is a decimal string if all characters in the string are decimal and\n" -"there is at least one character in the string."); +"A string is a decimal string if all characters in the string are\n" +"decimal and there is at least one character in the string."); #define UNICODE_ISDECIMAL_METHODDEF \ {"isdecimal", (PyCFunction)unicode_isdecimal, METH_NOARGS, unicode_isdecimal__doc__}, @@ -642,8 +644,8 @@ PyDoc_STRVAR(unicode_isdigit__doc__, "\n" "Return True if the string is a digit string, False otherwise.\n" "\n" -"A string is a digit string if all characters in the string are digits and there\n" -"is at least one character in the string."); +"A string is a digit string if all characters in the string are\n" +"digits and there is at least one character in the string."); #define UNICODE_ISDIGIT_METHODDEF \ {"isdigit", (PyCFunction)unicode_isdigit, METH_NOARGS, unicode_isdigit__doc__}, @@ -663,8 +665,8 @@ PyDoc_STRVAR(unicode_isnumeric__doc__, "\n" "Return True if the string is a numeric string, False otherwise.\n" "\n" -"A string is numeric if all characters in the string are numeric and there is at\n" -"least one character in the string."); +"A string is numeric if all characters in the string are numeric and\n" +"there is at least one character in the string."); #define UNICODE_ISNUMERIC_METHODDEF \ {"isnumeric", (PyCFunction)unicode_isnumeric, METH_NOARGS, unicode_isnumeric__doc__}, @@ -684,8 +686,8 @@ PyDoc_STRVAR(unicode_isidentifier__doc__, "\n" "Return True if the string is a valid Python identifier, False otherwise.\n" "\n" -"Call keyword.iskeyword(s) to test whether string s is a reserved identifier,\n" -"such as \"def\" or \"class\"."); +"Call keyword.iskeyword(s) to test whether string s is a reserved\n" +"identifier, such as \"def\" or \"class\"."); #define UNICODE_ISIDENTIFIER_METHODDEF \ {"isidentifier", (PyCFunction)unicode_isidentifier, METH_NOARGS, unicode_isidentifier__doc__}, @@ -725,8 +727,8 @@ PyDoc_STRVAR(unicode_join__doc__, "\n" "Concatenate any number of strings.\n" "\n" -"The string whose method is called is inserted in between each given string.\n" -"The result is returned as a new string.\n" +"The string whose method is called is inserted in between each given\n" +"string. The result is returned as a new string.\n" "\n" "Example: \'.\'.join([\'ab\', \'pq\', \'rs\']) -> \'ab.pq.rs\'"); @@ -739,7 +741,8 @@ PyDoc_STRVAR(unicode_ljust__doc__, "\n" "Return a left-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_LJUST_METHODDEF \ {"ljust", _PyCFunction_CAST(unicode_ljust), METH_FASTCALL, unicode_ljust__doc__}, @@ -999,8 +1002,9 @@ PyDoc_STRVAR(unicode_removeprefix__doc__, "\n" "Return a str with the given prefix string removed if present.\n" "\n" -"If the string starts with the prefix string, return string[len(prefix):].\n" -"Otherwise, return a copy of the original string."); +"If the string starts with the prefix string, return\n" +"string[len(prefix):]. Otherwise, return a copy of the original\n" +"string."); #define UNICODE_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)unicode_removeprefix, METH_O, unicode_removeprefix__doc__}, @@ -1031,9 +1035,9 @@ PyDoc_STRVAR(unicode_removesuffix__doc__, "\n" "Return a str with the given suffix string removed if present.\n" "\n" -"If the string ends with the suffix string and that suffix is not empty,\n" -"return string[:-len(suffix)]. Otherwise, return a copy of the original\n" -"string."); +"If the string ends with the suffix string and that suffix is not\n" +"empty, return string[:-len(suffix)]. Otherwise, return a copy of\n" +"the original string."); #define UNICODE_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)unicode_removesuffix, METH_O, unicode_removesuffix__doc__}, @@ -1064,8 +1068,8 @@ PyDoc_STRVAR(unicode_rfind__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_RFIND_METHODDEF \ {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, @@ -1120,8 +1124,8 @@ PyDoc_STRVAR(unicode_rindex__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_RINDEX_METHODDEF \ {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, @@ -1176,7 +1180,8 @@ PyDoc_STRVAR(unicode_rjust__doc__, "\n" "Return a right-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_RJUST_METHODDEF \ {"rjust", _PyCFunction_CAST(unicode_rjust), METH_FASTCALL, unicode_rjust__doc__}, @@ -1228,18 +1233,18 @@ PyDoc_STRVAR(unicode_split__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" "Splitting starts at the front of the string and works to the end.\n" "\n" -"Note, str.split() is mainly useful for data that has been intentionally\n" -"delimited. With natural text that includes punctuation, consider using\n" -"the regular expression module."); +"Note, str.split() is mainly useful for data that has been\n" +"intentionally delimited. With natural text that includes\n" +"punctuation, consider using the regular expression module."); #define UNICODE_SPLIT_METHODDEF \ {"split", _PyCFunction_CAST(unicode_split), METH_FASTCALL|METH_KEYWORDS, unicode_split__doc__}, @@ -1319,12 +1324,12 @@ PyDoc_STRVAR(unicode_partition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator in the string. If the separator\n" +"is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original string\n" -"and two empty strings."); +"If the separator is not found, returns a 3-tuple containing\n" +"the original string and two empty strings."); #define UNICODE_PARTITION_METHODDEF \ {"partition", (PyCFunction)unicode_partition, METH_O, unicode_partition__doc__}, @@ -1335,12 +1340,13 @@ PyDoc_STRVAR(unicode_rpartition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator in the string, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty strings\n" -"and the original string."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty strings and the original string."); #define UNICODE_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)unicode_rpartition, METH_O, unicode_rpartition__doc__}, @@ -1354,9 +1360,9 @@ PyDoc_STRVAR(unicode_rsplit__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" @@ -1441,8 +1447,8 @@ PyDoc_STRVAR(unicode_splitlines__doc__, "\n" "Return a list of the lines in the string, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define UNICODE_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(unicode_splitlines), METH_FASTCALL|METH_KEYWORDS, unicode_splitlines__doc__}, @@ -1525,13 +1531,14 @@ PyDoc_STRVAR(unicode_maketrans__doc__, "\n" "Return a translation table usable for str.translate().\n" "\n" -"If there is only one argument, it must be a dictionary mapping Unicode\n" -"ordinals (integers) or characters to Unicode ordinals, strings or None.\n" -"Character keys will be then converted to ordinals.\n" -"If there are two arguments, they must be strings of equal length, and\n" -"in the resulting dictionary, each character in x will be mapped to the\n" -"character at the same position in y. If there is a third argument, it\n" -"must be a string, whose characters will be mapped to None in the result."); +"If there is only one argument, it must be a dictionary mapping\n" +"Unicode ordinals (integers) or characters to Unicode ordinals,\n" +"strings or None. Character keys will be then converted to ordinals.\n" +"If there are two arguments, they must be strings of equal length,\n" +"and in the resulting dictionary, each character in x will be mapped\n" +"to the character at the same position in y. If there is a third\n" +"argument, it must be a string, whose characters will be mapped to\n" +"None in the result."); #define UNICODE_MAKETRANS_METHODDEF \ {"maketrans", _PyCFunction_CAST(unicode_maketrans), METH_FASTCALL|METH_STATIC, unicode_maketrans__doc__}, @@ -1581,12 +1588,13 @@ PyDoc_STRVAR(unicode_translate__doc__, "Replace each character in the string using the given translation table.\n" "\n" " table\n" -" Translation table, which must be a mapping of Unicode ordinals to\n" -" Unicode ordinals, strings, or None.\n" +" Translation table, which must be a mapping of Unicode ordinals\n" +" to Unicode ordinals, strings, or None.\n" "\n" -"The table must implement lookup/indexing via __getitem__, for instance a\n" -"dictionary or list. If this operation raises LookupError, the character is\n" -"left untouched. Characters mapped to None are deleted."); +"The table must implement lookup/indexing via __getitem__, for\n" +"instance a dictionary or list. If this operation raises\n" +"LookupError, the character is left untouched. Characters mapped to\n" +"None are deleted."); #define UNICODE_TRANSLATE_METHODDEF \ {"translate", (PyCFunction)unicode_translate, METH_O, unicode_translate__doc__}, @@ -1887,4 +1895,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b7d75c4898e8198d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=88f8309f41babf81 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index a361cb4957c..d775f0a54db 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2355,12 +2355,13 @@ code._varname_from_oparg (internal-only) Return the local variable name for the given oparg. -WARNING: this method is for internal use only and may change or go away. +WARNING: this method is for internal use only and may change or go +away. [clinic start generated code]*/ static PyObject * code__varname_from_oparg_impl(PyCodeObject *self, int oparg) -/*[clinic end generated code: output=1fd1130413184206 input=c5fa3ee9bac7d4ca]*/ +/*[clinic end generated code: output=1fd1130413184206 input=6ba7d6df0d566463]*/ { PyObject *name = PyTuple_GetItem(self->co_localsplusnames, oparg); if (name == NULL) { diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 0560d066851..7d3f0e40567 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1177,7 +1177,7 @@ static PyMethodDef mappingproxy_methods[] = { {"copy", mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("mappingproxy objects are generic over two types, signifying (respectively) the types of their keys and values")}, {"__reversed__", mappingproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator")}, {0} diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 843ece535be..12dd7bbdeec 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4695,7 +4695,8 @@ static PyMethodDef mapp_methods[] = { DICT_CLEAR_METHODDEF DICT_COPY_METHODDEF DICT___REVERSED___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("dicts are generic over two types, signifying (respectively) the types of their keys and values")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/enumobject.c b/Objects/enumobject.c index bffe7172a39..5f21bfa326f 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -283,7 +283,7 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef enum_methods[] = { {"__reduce__", (PyCFunction)enum_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("'enumerate' objects are generic over the type of their values")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 1eb2a3bbe24..81242e4405d 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1541,7 +1541,8 @@ static PyMemberDef BaseExceptionGroup_members[] = { static PyMethodDef BaseExceptionGroup_methods[] = { {"__class_getitem__", (PyCFunction)Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Exception groups are generic over the type of their contained exceptions")}, {"derive", (PyCFunction)BaseExceptionGroup_derive, METH_O}, {"split", (PyCFunction)BaseExceptionGroup_split, METH_O}, {"subgroup", (PyCFunction)BaseExceptionGroup_subgroup, METH_O}, @@ -3933,4 +3934,3 @@ _PyException_AddNote(PyObject *exc, PyObject *note) Py_XDECREF(r); return res; } - diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 986edd597d6..21c47a4bbf8 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1718,14 +1718,14 @@ You probably don't want to use this function. It exists mainly to be used in Python's test suite. -This function returns whichever of 'unknown', 'IEEE, big-endian' or 'IEEE, -little-endian' best describes the format of floating-point numbers used by the -C type named by typestr. +This function returns whichever of 'unknown', 'IEEE, big-endian' or +'IEEE, little-endian' best describes the format of floating-point +numbers used by the C type named by typestr. [clinic start generated code]*/ static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=90d5e246409a246e]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=933cc4bdcf8fa8d3]*/ { float_format_type r; diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 8e59b3884cf..bbb4f77d4a9 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -278,7 +278,6 @@ _Py_make_parameters(PyObject *args) len += needed; if (_PyTuple_Resize(&parameters, len) < 0) { Py_DECREF(subparams); - Py_DECREF(parameters); return NULL; } } @@ -444,6 +443,9 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje self); } item = _unpack_args(item); + if (item == NULL) { + return NULL; + } for (Py_ssize_t i = 0; i < nparams; i++) { PyObject *param = PyTuple_GET_ITEM(parameters, i); PyObject *prepare, *tmp; @@ -636,7 +638,7 @@ ga_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { gaobject *alias = (gaobject *) self; - PyObject *obj = PyVectorcall_Function(alias->origin)(alias->origin, args, nargsf, kwnames); + PyObject *obj = PyObject_Vectorcall(alias->origin, args, nargsf, kwnames); return set_orig_class(obj, self); } diff --git a/Objects/genobject.c b/Objects/genobject.c index b19ff252ddb..2d9f89dbc34 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -797,7 +797,8 @@ static PyMethodDef gen_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("generators are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -1150,7 +1151,8 @@ static PyMethodDef coro_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("coroutines are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -1564,7 +1566,7 @@ static PyMethodDef async_gen_methods[] = { {"aclose", (PyCFunction)async_gen_aclose, METH_NOARGS, async_aclose_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("async generators are generic over the types of their yield and send values")}, {NULL, NULL} /* Sentinel */ }; diff --git a/Objects/listobject.c b/Objects/listobject.c index f5acc5c1bef..591ea3a956e 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2875,18 +2875,18 @@ list.sort Sort the list in ascending order and return None. -The sort is in-place (i.e. the list itself is modified) and stable (i.e. the -order of two equal elements is maintained). +The sort is in-place (i.e. the list itself is modified) and stable +(i.e. the order of two equal elements is maintained). -If a key function is given, apply it once to each list item and sort them, -ascending or descending, according to their function values. +If a key function is given, apply it once to each list item and sort +them, ascending or descending, according to their function values. The reverse flag can be set to sort in descending order. [clinic start generated code]*/ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=c145526281e1fb9f]*/ { MergeState ms; Py_ssize_t nremaining; @@ -3532,7 +3532,8 @@ static PyMethodDef list_methods[] = { LIST_COUNT_METHODDEF LIST_REVERSE_METHODDEF LIST_SORT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("lists are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/longobject.c b/Objects/longobject.c index 4942c57f1ac..63de665c9fb 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6321,20 +6321,21 @@ int_as_integer_ratio_impl(PyObject *self) int.to_bytes length: Py_ssize_t = 1 - Length of bytes object to use. An OverflowError is raised if the - integer is not representable with the given number of bytes. Default - is length 1. + Length of bytes object to use. An OverflowError is raised if + the integer is not representable with the given number of bytes. + Default is length 1. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Determines whether two's complement is used to represent the integer. - If signed is False and a negative integer is given, an OverflowError - is raised. + Determines whether two's complement is used to represent the + integer. If signed is False and a negative integer is given, + an OverflowError is raised. Return an array of bytes representing an integer. [clinic start generated code]*/ @@ -6342,7 +6343,7 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=a0103d0e9ad85c2b]*/ +/*[clinic end generated code: output=89c801df114050a3 input=661d7e615a4f132d]*/ { int little_endian; PyObject *bytes; @@ -6385,18 +6386,20 @@ int.from_bytes bytes as bytes_obj: object Holds the array of bytes to convert. The argument must either - support the buffer protocol or be an iterable object producing bytes. - Bytes and bytearray are examples of built-in objects that support the - buffer protocol. + support the buffer protocol or be an iterable object producing + bytes. Bytes and bytearray are examples of built-in objects that + support the buffer protocol. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Indicates whether two's complement is used to represent the integer. + Indicates whether two's complement is used to represent the + integer. Return the integer represented by the given array of bytes. [clinic start generated code]*/ @@ -6404,7 +6407,7 @@ Return the integer represented by the given array of bytes. static PyObject * int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=efc5d68e31f9314f input=2ff527997fe7b0c5]*/ +/*[clinic end generated code: output=efc5d68e31f9314f input=95801e50b942e164]*/ { int little_endian; PyObject *long_obj, *bytes; @@ -6533,7 +6536,8 @@ If x is not a number or if base is given, then x must be a string,\n\ bytes, or bytearray instance representing an integer literal in the\n\ given base. The literal can be preceded by '+' or '-' and be surrounded\n\ by whitespace. The base defaults to 10. Valid bases are 0 and 2-36.\n\ -Base 0 means to interpret the base from the string as an integer literal.\n\ +Base 0 means to interpret the base from the string as an integer\n\ +iteral.\n\ >>> int('0b100', base=0)\n\ 4"); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 535e0b3c1dc..d7586a2a458 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1653,6 +1653,10 @@ fix_error_int(const char *fmt) return -1; } +// UNPACK_TO_BOOL: Return 0 if PTR represents "false", and 1 otherwise. +static const _Bool bool_false = 0; +#define UNPACK_TO_BOOL(PTR) (memcmp((PTR), &bool_false, sizeof(_Bool)) != 0) + /* Accept integer objects or objects with an __index__() method. */ static long pylong_as_ld(PyObject *item) @@ -1788,7 +1792,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) case 'l': UNPACK_SINGLE(ld, ptr, long); goto convert_ld; /* boolean */ - case '?': UNPACK_SINGLE(ld, ptr, _Bool); goto convert_bool; + case '?': ld = UNPACK_TO_BOOL(ptr); goto convert_bool; /* unsigned integers */ case 'H': UNPACK_SINGLE(lu, ptr, unsigned short); goto convert_lu; @@ -2252,16 +2256,17 @@ memoryview.tobytes Return the data in the buffer as a byte string. -Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of the -original array is converted to C or Fortran order. For contiguous views, -'A' returns an exact copy of the physical memory. In particular, in-memory -Fortran order is preserved. For non-contiguous views, the data is converted -to C first. order=None is the same as order='C'. +Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of +the original array is converted to C or Fortran order. For +contiguous views, 'A' returns an exact copy of the physical memory. +In particular, in-memory Fortran order is preserved. For +non-contiguous views, the data is converted to C first. order=None +is the same as order='C'. [clinic start generated code]*/ static PyObject * memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) -/*[clinic end generated code: output=1288b62560a32a23 input=0efa3ddaeda573a8]*/ +/*[clinic end generated code: output=1288b62560a32a23 input=119c70aa91791dc8]*/ { Py_buffer *src = VIEW_ADDR(self); char ord = 'C'; @@ -2301,8 +2306,8 @@ memoryview.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: int = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Return the data in the buffer as a str of hexadecimal numbers. @@ -2321,7 +2326,7 @@ Return the data in the buffer as a str of hexadecimal numbers. static PyObject * memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=430ca760f94f3ca7 input=539f6a3a5fb56946]*/ +/*[clinic end generated code: output=430ca760f94f3ca7 input=94c2495f886c786b]*/ { Py_buffer *src = VIEW_ADDR(self); PyObject *bytes; @@ -2835,7 +2840,7 @@ unpack_cmp(const char *p, const char *q, char fmt, case 'l': CMP_SINGLE(p, q, long); return equal; /* boolean */ - case '?': CMP_SINGLE(p, q, _Bool); return equal; + case '?': return UNPACK_TO_BOOL(p) == UNPACK_TO_BOOL(q); /* unsigned integers */ case 'H': CMP_SINGLE(p, q, unsigned short); return equal; diff --git a/Objects/odictobject.c b/Objects/odictobject.c index ca5f7c8d3a8..70fd9f5a0f6 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1128,12 +1128,13 @@ OrderedDict.popitem Remove and return a (key, value) pair from the dictionary. -Pairs are returned in LIFO order if last is true or FIFO order if false. +Pairs are returned in LIFO order if last is true or FIFO order if +false. [clinic start generated code]*/ static PyObject * OrderedDict_popitem_impl(PyODictObject *self, int last) -/*[clinic end generated code: output=98e7d986690d49eb input=d992ac5ee8305e1a]*/ +/*[clinic end generated code: output=98e7d986690d49eb input=9cc253963e351588]*/ { PyObject *key, *value, *item = NULL; _ODictNode *node; diff --git a/Objects/setobject.c b/Objects/setobject.c index 891987e3519..0743870cec8 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2416,7 +2416,8 @@ static PyMethodDef set_methods[] = { SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF SET_UNION_METHODDEF SET_UPDATE_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("sets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2519,7 +2520,8 @@ static PyMethodDef frozenset_methods[] = { SET___SIZEOF___METHODDEF SET_SYMMETRIC_DIFFERENCE_METHODDEF SET_UNION_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("frozensets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 245bea98d58..5b034597275 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -360,7 +360,9 @@ PyDoc_STRVAR(slice_doc, "slice(stop)\n\ slice(start, stop[, step])\n\ \n\ -Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); +Create a slice object.\n\ +\n\ +This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PySliceObject *r) diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 818814b663d..42c94c8b1e0 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -827,11 +827,17 @@ tuple___getnewargs___impl(PyTupleObject *self) return Py_BuildValue("(N)", tupleslice(self, 0, Py_SIZE(self))); } + +PyDoc_STRVAR(tuple_class_getitem_doc, +"Tuples are generic over the types of their contents.\n\n\ +For example, use ``tuple[int, str]`` for a pair whose first element is an int and second element is a string.\n\n\ +Tuples also support the form ``tuple[T, ...]`` to indicate an arbitrary length tuple of elements of type T."); + static PyMethodDef tuple_methods[] = { TUPLE___GETNEWARGS___METHODDEF TUPLE_INDEX_METHODDEF TUPLE_COUNT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, tuple_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 51d1ce573d1..8f44c99f6a7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5398,6 +5398,15 @@ void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) { BEGIN_TYPE_LOCK(); + /* Ideally, changing flags and invalidating the old version tag would + happen in one step. For this backport, keep it simple and invalidate + first while holding TYPE_LOCK. Immutable types are skipped because + set_flags_recursive() does not modify them. */ + if (!PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) && + (self->tp_flags & mask) != flags) + { + type_modified_unlocked(self); + } set_flags_recursive(self, mask, flags); END_TYPE_LOCK(); } @@ -11488,7 +11497,8 @@ PyDoc_STRVAR(super_doc, "super() -> same as super(__class__, <first argument>)\n" "super(type) -> unbound super object\n" "super(type, obj) -> bound super object; requires isinstance(obj, type)\n" -"super(type, type2) -> bound super object; requires issubclass(type2, type)\n" +"super(type, type2) -> bound super object; requires\n" +" issubclass(type2, type)\n" "Typical use to call a cooperative superclass method:\n" "class C(B):\n" " def meth(self, arg):\n" diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 696bb100c6b..4c987fa7dbc 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -10643,13 +10643,13 @@ str.title as unicode_title Return a version of the string where each word is titlecased. -More specifically, words start with uppercased characters and all remaining -cased characters have lower case. +More specifically, words start with uppercased characters and all +remaining cased characters have lower case. [clinic start generated code]*/ static PyObject * unicode_title_impl(PyObject *self) -/*[clinic end generated code: output=c75ae03809574902 input=fa945d669b26e683]*/ +/*[clinic end generated code: output=c75ae03809574902 input=2a07e2c7df94627a]*/ { return case_operation(self, do_title); } @@ -10659,13 +10659,13 @@ str.capitalize as unicode_capitalize Return a capitalized version of the string. -More specifically, make the first character have upper case and the rest lower -case. +More specifically, make the first character have upper case and the +rest lower case. [clinic start generated code]*/ static PyObject * unicode_capitalize_impl(PyObject *self) -/*[clinic end generated code: output=e49a4c333cdb7667 input=f4cbf1016938da6d]*/ +/*[clinic end generated code: output=e49a4c333cdb7667 input=e50e50ed45a654cf]*/ { if (PyUnicode_GET_LENGTH(self) == 0) return unicode_result_unchanged(self); @@ -10719,12 +10719,13 @@ str.center as unicode_center Return a centered string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_center_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=420c8859effc7c0c input=b42b247eb26e6519]*/ +/*[clinic end generated code: output=420c8859effc7c0c input=df91017dfd186a78]*/ { Py_ssize_t marg, left; @@ -11332,13 +11333,14 @@ str.count as unicode_count -> Py_ssize_t Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -Optional arguments start and end are interpreted as in slice notation. +Optional arguments start and end are interpreted as in slice +notation. [clinic start generated code]*/ static Py_ssize_t unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=8fcc3aef0b18edbf input=6f168ffd94be8785]*/ +/*[clinic end generated code: output=8fcc3aef0b18edbf input=0db5f2367599b50d]*/ { assert(PyUnicode_Check(str)); assert(PyUnicode_Check(substr)); @@ -11411,8 +11413,8 @@ str.encode as unicode_encode errors: str(c_default="NULL") = 'strict' The error handling scheme to use for encoding errors. The default is 'strict' meaning that encoding errors raise a - UnicodeEncodeError. Other possible values are 'ignore', 'replace' and - 'xmlcharrefreplace' as well as any other name registered with + UnicodeEncodeError. Other possible values are 'ignore', 'replace' + and 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle UnicodeEncodeErrors. Encode the string using the codec registered for encoding. @@ -11420,7 +11422,7 @@ Encode the string using the codec registered for encoding. static PyObject * unicode_encode_impl(PyObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=bf78b6e2a9470e3c input=f0a9eb293d08fe02]*/ +/*[clinic end generated code: output=bf78b6e2a9470e3c input=b85a9645cb33b729]*/ { return PyUnicode_AsEncodedString(self, encoding, errors); } @@ -11516,14 +11518,14 @@ str.find as unicode_find = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=51dbe6255712e278 input=4a89d2d68ef57256]*/ +/*[clinic end generated code: output=51dbe6255712e278 input=6be5f4af6fc545d8]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result < 0) { @@ -11579,14 +11581,14 @@ str.index as unicode_index = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=77558288837cdf40 input=d986aeac0be14a1c]*/ +/*[clinic end generated code: output=77558288837cdf40 input=39f29462607c38cf]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result == -1) { @@ -11619,13 +11621,13 @@ str.islower as unicode_islower Return True if the string is a lowercase string, False otherwise. -A string is lowercase if all cased characters in the string are lowercase and -there is at least one cased character in the string. +A string is lowercase if all cased characters in the string are +lowercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_islower_impl(PyObject *self) -/*[clinic end generated code: output=dbd41995bd005b81 input=acec65ac6821ae47]*/ +/*[clinic end generated code: output=dbd41995bd005b81 input=1879b48dfc628366]*/ { Py_ssize_t i, length; int kind; @@ -11662,13 +11664,13 @@ str.isupper as unicode_isupper Return True if the string is an uppercase string, False otherwise. -A string is uppercase if all cased characters in the string are uppercase and -there is at least one cased character in the string. +A string is uppercase if all cased characters in the string are +uppercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_isupper_impl(PyObject *self) -/*[clinic end generated code: output=049209c8e7f15f59 input=e9b1feda5d17f2d3]*/ +/*[clinic end generated code: output=049209c8e7f15f59 input=77d29904aef0e3a0]*/ { Py_ssize_t i, length; int kind; @@ -11761,13 +11763,13 @@ str.isspace as unicode_isspace Return True if the string is a whitespace string, False otherwise. -A string is whitespace if all characters in the string are whitespace and there -is at least one character in the string. +A string is whitespace if all characters in the string are +whitespace and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isspace_impl(PyObject *self) -/*[clinic end generated code: output=163a63bfa08ac2b9 input=fe462cb74f8437d8]*/ +/*[clinic end generated code: output=163a63bfa08ac2b9 input=29e09560fc23fbeb]*/ { Py_ssize_t i, length; int kind; @@ -11799,13 +11801,13 @@ str.isalpha as unicode_isalpha Return True if the string is an alphabetic string, False otherwise. -A string is alphabetic if all characters in the string are alphabetic and there -is at least one character in the string. +A string is alphabetic if all characters in the string are +alphabetic and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalpha_impl(PyObject *self) -/*[clinic end generated code: output=cc81b9ac3883ec4f input=d0fd18a96cbca5eb]*/ +/*[clinic end generated code: output=cc81b9ac3883ec4f input=9906a07f3e04892e]*/ { Py_ssize_t i, length; int kind; @@ -11836,13 +11838,13 @@ str.isalnum as unicode_isalnum Return True if the string is an alpha-numeric string, False otherwise. -A string is alpha-numeric if all characters in the string are alpha-numeric and -there is at least one character in the string. +A string is alpha-numeric if all characters in the string are +alpha-numeric and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalnum_impl(PyObject *self) -/*[clinic end generated code: output=a5a23490ffc3660c input=5c6579bf2e04758c]*/ +/*[clinic end generated code: output=a5a23490ffc3660c input=fd90c03fd83af0c7]*/ { int kind; const void *data; @@ -11875,13 +11877,13 @@ str.isdecimal as unicode_isdecimal Return True if the string is a decimal string, False otherwise. -A string is a decimal string if all characters in the string are decimal and -there is at least one character in the string. +A string is a decimal string if all characters in the string are +decimal and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdecimal_impl(PyObject *self) -/*[clinic end generated code: output=fb2dcdb62d3fc548 input=336bc97ab4c8268f]*/ +/*[clinic end generated code: output=fb2dcdb62d3fc548 input=63b0453c48cad0af]*/ { Py_ssize_t i, length; int kind; @@ -11912,13 +11914,13 @@ str.isdigit as unicode_isdigit Return True if the string is a digit string, False otherwise. -A string is a digit string if all characters in the string are digits and there -is at least one character in the string. +A string is a digit string if all characters in the string are +digits and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdigit_impl(PyObject *self) -/*[clinic end generated code: output=10a6985311da6858 input=901116c31deeea4c]*/ +/*[clinic end generated code: output=10a6985311da6858 input=353b03747b062e4b]*/ { Py_ssize_t i, length; int kind; @@ -11950,13 +11952,13 @@ str.isnumeric as unicode_isnumeric Return True if the string is a numeric string, False otherwise. -A string is numeric if all characters in the string are numeric and there is at -least one character in the string. +A string is numeric if all characters in the string are numeric and +there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isnumeric_impl(PyObject *self) -/*[clinic end generated code: output=9172a32d9013051a input=722507db976f826c]*/ +/*[clinic end generated code: output=9172a32d9013051a input=83b2a072ed7aff48]*/ { Py_ssize_t i, length; int kind; @@ -12030,13 +12032,13 @@ str.isidentifier as unicode_isidentifier Return True if the string is a valid Python identifier, False otherwise. -Call keyword.iskeyword(s) to test whether string s is a reserved identifier, -such as "def" or "class". +Call keyword.iskeyword(s) to test whether string s is a reserved +identifier, such as "def" or "class". [clinic start generated code]*/ static PyObject * unicode_isidentifier_impl(PyObject *self) -/*[clinic end generated code: output=fe585a9666572905 input=2d807a104f21c0c5]*/ +/*[clinic end generated code: output=fe585a9666572905 input=82e830f25b2a7945]*/ { return PyBool_FromLong(PyUnicode_IsIdentifier(self)); } @@ -12082,15 +12084,15 @@ str.join as unicode_join Concatenate any number of strings. -The string whose method is called is inserted in between each given string. -The result is returned as a new string. +The string whose method is called is inserted in between each given +string. The result is returned as a new string. Example: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs' [clinic start generated code]*/ static PyObject * unicode_join(PyObject *self, PyObject *iterable) -/*[clinic end generated code: output=6857e7cecfe7bf98 input=2f70422bfb8fa189]*/ +/*[clinic end generated code: output=6857e7cecfe7bf98 input=fd330a11ee845fb2]*/ { return PyUnicode_Join(self, iterable); } @@ -12110,12 +12112,13 @@ str.ljust as unicode_ljust Return a left-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_ljust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=3ab599e335e60a32]*/ +/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=8a55f06694c20ed6]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -12465,13 +12468,14 @@ str.removeprefix as unicode_removeprefix Return a str with the given prefix string removed if present. -If the string starts with the prefix string, return string[len(prefix):]. -Otherwise, return a copy of the original string. +If the string starts with the prefix string, return +string[len(prefix):]. Otherwise, return a copy of the original +string. [clinic start generated code]*/ static PyObject * unicode_removeprefix_impl(PyObject *self, PyObject *prefix) -/*[clinic end generated code: output=f1e5945e9763bcb9 input=27ec40b99a37eb88]*/ +/*[clinic end generated code: output=f1e5945e9763bcb9 input=90d162724944bfa7]*/ { int match = tailmatch(self, prefix, 0, PY_SSIZE_T_MAX, -1); if (match == -1) { @@ -12492,14 +12496,14 @@ str.removesuffix as unicode_removesuffix Return a str with the given suffix string removed if present. -If the string ends with the suffix string and that suffix is not empty, -return string[:-len(suffix)]. Otherwise, return a copy of the original -string. +If the string ends with the suffix string and that suffix is not +empty, return string[:-len(suffix)]. Otherwise, return a copy of +the original string. [clinic start generated code]*/ static PyObject * unicode_removesuffix_impl(PyObject *self, PyObject *suffix) -/*[clinic end generated code: output=d36629e227636822 input=12cc32561e769be4]*/ +/*[clinic end generated code: output=d36629e227636822 input=6efc96152d4bfcd5]*/ { int match = tailmatch(self, suffix, 0, PY_SSIZE_T_MAX, +1); if (match == -1) { @@ -12679,14 +12683,14 @@ str.rfind as unicode_rfind = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=880b29f01dd014c8 input=898361fb71f59294]*/ +/*[clinic end generated code: output=880b29f01dd014c8 input=130dfb5a94532dd6]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result < 0) { @@ -12700,14 +12704,14 @@ str.rindex as unicode_rindex = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=5f3aef124c867fe1 input=35943dead6c1ea9d]*/ +/*[clinic end generated code: output=5f3aef124c867fe1 input=97a766ba968a214e]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result == -1) { @@ -12728,12 +12732,13 @@ str.rjust as unicode_rjust Return a right-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_rjust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=804a1a57fbe8d5cf input=d05f550b5beb1f72]*/ +/*[clinic end generated code: output=804a1a57fbe8d5cf input=1256a8d659589907]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -12756,9 +12761,9 @@ str.split as unicode_split sep: object = None The separator used to split the string. - When set to None (the default value), will split on any whitespace - character (including \n \r \t \f and spaces) and will discard - empty strings from the result. + When set to None (the default value), will split on any + whitespace character (including \n \r \t \f and spaces) and + will discard empty strings from the result. maxsplit: Py_ssize_t = -1 Maximum number of splits. -1 (the default value) means no limit. @@ -12767,15 +12772,15 @@ Return a list of the substrings in the string, using sep as the separator string Splitting starts at the front of the string and works to the end. -Note, str.split() is mainly useful for data that has been intentionally -delimited. With natural text that includes punctuation, consider using -the regular expression module. +Note, str.split() is mainly useful for data that has been +intentionally delimited. With natural text that includes +punctuation, consider using the regular expression module. [clinic start generated code]*/ static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=a29bcc0c7a5af0eb]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=9dc157701983897d]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); @@ -12899,17 +12904,17 @@ str.partition as unicode_partition Partition the string into three parts using the given separator. -This will search for the separator in the string. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator in the string. If the separator +is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original string -and two empty strings. +If the separator is not found, returns a 3-tuple containing +the original string and two empty strings. [clinic start generated code]*/ static PyObject * unicode_partition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=e4ced7bd253ca3c4 input=f29b8d06c63e50be]*/ +/*[clinic end generated code: output=e4ced7bd253ca3c4 input=e45faa8c26270cb1]*/ { return PyUnicode_Partition(self, sep); } @@ -12919,17 +12924,18 @@ str.rpartition as unicode_rpartition = str.partition Partition the string into three parts using the given separator. -This will search for the separator in the string, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator in the string, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty strings -and the original string. +If the separator is not found, returns a 3-tuple containing two +empty strings and the original string. [clinic start generated code]*/ static PyObject * unicode_rpartition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=1aa13cf1156572aa input=c4b7db3ef5cf336a]*/ +/*[clinic end generated code: output=1aa13cf1156572aa input=53a7f8cb19975b7c]*/ { return PyUnicode_RPartition(self, sep); } @@ -12973,13 +12979,13 @@ str.splitlines as unicode_splitlines Return a list of the lines in the string, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=ba6ad05ee85d2b55]*/ +/*[clinic end generated code: output=f664dcdad153ec40 input=bf780246bee5462b]*/ { return PyUnicode_Splitlines(self, keepends); } @@ -13018,18 +13024,19 @@ str.maketrans as unicode_maketrans Return a translation table usable for str.translate(). -If there is only one argument, it must be a dictionary mapping Unicode -ordinals (integers) or characters to Unicode ordinals, strings or None. -Character keys will be then converted to ordinals. -If there are two arguments, they must be strings of equal length, and -in the resulting dictionary, each character in x will be mapped to the -character at the same position in y. If there is a third argument, it -must be a string, whose characters will be mapped to None in the result. +If there is only one argument, it must be a dictionary mapping +Unicode ordinals (integers) or characters to Unicode ordinals, +strings or None. Character keys will be then converted to ordinals. +If there are two arguments, they must be strings of equal length, +and in the resulting dictionary, each character in x will be mapped +to the character at the same position in y. If there is a third +argument, it must be a string, whose characters will be mapped to +None in the result. [clinic start generated code]*/ static PyObject * unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) -/*[clinic end generated code: output=a925c89452bd5881 input=7bfbf529a293c6c5]*/ +/*[clinic end generated code: output=a925c89452bd5881 input=66bc00a1b4258a6e]*/ { PyObject *new = NULL, *key, *value; Py_ssize_t i = 0; @@ -13137,20 +13144,21 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) str.translate as unicode_translate table: object - Translation table, which must be a mapping of Unicode ordinals to - Unicode ordinals, strings, or None. + Translation table, which must be a mapping of Unicode ordinals + to Unicode ordinals, strings, or None. / Replace each character in the string using the given translation table. -The table must implement lookup/indexing via __getitem__, for instance a -dictionary or list. If this operation raises LookupError, the character is -left untouched. Characters mapped to None are deleted. +The table must implement lookup/indexing via __getitem__, for +instance a dictionary or list. If this operation raises +LookupError, the character is left untouched. Characters mapped to +None are deleted. [clinic start generated code]*/ static PyObject * unicode_translate(PyObject *self, PyObject *table) -/*[clinic end generated code: output=3cb448ff2fd96bf3 input=6d38343db63d8eb0]*/ +/*[clinic end generated code: output=3cb448ff2fd96bf3 input=9874c06808f58900]*/ { return _PyUnicode_TranslateCharmap(self, table, "ignore"); } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 61f05514a48..f45522c29a1 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -486,7 +486,8 @@ static PyMemberDef weakref_members[] = { static PyMethodDef weakref_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Weakrefs are generic over the type of the referenced object.")}, {NULL} /* Sentinel */ }; diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 58cb9a8c463..251088253f6 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -230,13 +230,13 @@ _wmi.exec_query Runs a WMI query against the local machine. -This returns a single string with 'name=value' pairs in a flat array separated -by null characters. +This returns a single string with 'name=value' pairs in a flat array +separated by null characters. [clinic start generated code]*/ static PyObject * _wmi_exec_query_impl(PyObject *module, PyObject *query) -/*[clinic end generated code: output=a62303d5bb5e003f input=48d2d0a1e1a7e3c2]*/ +/*[clinic end generated code: output=a62303d5bb5e003f input=a8d5710acdfbf515]*/ /*[clinic end generated code]*/ { diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index fccf3d795d5..73d93ed9eaf 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_wmi_exec_query__doc__, "\n" "Runs a WMI query against the local machine.\n" "\n" -"This returns a single string with \'name=value\' pairs in a flat array separated\n" -"by null characters."); +"This returns a single string with \'name=value\' pairs in a flat array\n" +"separated by null characters."); #define _WMI_EXEC_QUERY_METHODDEF \ {"exec_query", _PyCFunction_CAST(_wmi_exec_query), METH_FASTCALL|METH_KEYWORDS, _wmi_exec_query__doc__}, @@ -69,4 +69,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=ba04920d127f3ceb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fefbbab59fa1584b input=a9049054013a1b77]*/ diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 44dbf234813..02d755f8d5c 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -126,6 +126,7 @@ <ClCompile Include="..\Modules\_testcapi\gc.c" /> <ClCompile Include="..\Modules\_testcapi\run.c" /> <ClCompile Include="..\Modules\_testcapi\monitoring.c" /> + <ClCompile Include="..\Modules\_testcapi\weakref.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index cae44bc955f..63025cecd6d 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -111,6 +111,9 @@ <ClCompile Include="..\Modules\_testcapi\monitoring.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_testcapi\weakref.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc"> diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index a5e0be93ab9..e020c05fb8d 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -113,6 +113,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\unicode.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\vectorcall_limited.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\file.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 4b3521afc06..73979d971ee 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -29,6 +29,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\unicode.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\vectorcall_limited.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\file.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" /> <ClCompile Include="..\Modules\_testlimitedcapi.c" /> </ItemGroup> <ItemGroup> diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 4fda1c5b28e..65f1e1b6c59 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.19 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.21 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.19 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.21 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/python.props b/PCbuild/python.props index c29caa61bbd..64e4cd08db7 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -76,8 +76,8 @@ <libffiOutDir Condition="$(libffiOutDir) == ''">$(libffiDir)$(ArchName)\</libffiOutDir> <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir> <mpdecimalDir Condition="$(mpdecimalDir) == ''">$(ExternalsDir)\mpdecimal-4.0.0\</mpdecimalDir> - <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.19\</opensslDir> - <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.19\$(ArchName)\</opensslOutDir> + <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.21\</opensslDir> + <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.21\$(ArchName)\</opensslOutDir> <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslOutDir)include</opensslIncludeDir> <nasmDir Condition="$(nasmDir) == ''">$(ExternalsDir)\nasm-2.11.06\</nasmDir> <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.3.1\</zlibDir> diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 1665debea30..7d3b9e9ae59 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void) struct tok_state *tok = (struct tok_state *)PyMem_Calloc( 1, sizeof(struct tok_state)); - if (tok == NULL) + if (tok == NULL) { + PyErr_NoMemory(); return NULL; + } + tok->buf = tok->cur = tok->inp = NULL; tok->fp_interactive = 0; tok->interactive_src_start = NULL; diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 2750527da48..3e711eb8792 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -355,6 +355,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index aa6da82319e..976d039f571 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -183,6 +183,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok) char* result = (char *)PyMem_Malloc(len + 1); if (!result) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } memcpy(result, s, len); @@ -211,6 +212,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf buf = PyMem_Malloc(needed_length); if (buf == NULL) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } for (current = buf; *s; s++, current++) { diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index a2637af9902..b325cbd6d1c 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index a14e68c0e81..2513cde4e76 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,3 +1,5 @@ +#include "asm_trampoline_aarch64.h" + .text .globl _Py_trampoline_func_start # The following assembly is equivalent to: @@ -21,10 +23,12 @@ _Py_trampoline_func_start: #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) // ARM64 little endian, 64bit ABI // generate with aarch64-linux-gnu-gcc 12.1 + SIGN_LR stp x29, x30, [sp, -16]! mov x29, sp blr x3 ldp x29, x30, [sp], 16 + VERIFY_LR ret #endif #ifdef __riscv diff --git a/Python/asm_trampoline_aarch64.h b/Python/asm_trampoline_aarch64.h new file mode 100644 index 00000000000..bc83aa460b6 --- /dev/null +++ b/Python/asm_trampoline_aarch64.h @@ -0,0 +1,56 @@ +#ifndef ASM_TRAMPOLINE_AARCH_64_H_ +#define ASM_TRAMPOLINE_AARCH_64_H_ + +/* + * References: + * - https://developer.arm.com/documentation/101028/0012/5--Feature-test-macros + * - https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + */ + +#if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 + #define BTI_J hint 36 /* bti j: for jumps, IE br instructions */ + #define BTI_C hint 34 /* bti c: for calls, IE bl instructions */ + #define GNU_PROPERTY_AARCH64_BTI 1 /* bit 0 GNU Notes is for BTI support */ +#else + #define BTI_J + #define BTI_C + #define GNU_PROPERTY_AARCH64_BTI 0 +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) + #if __ARM_FEATURE_PAC_DEFAULT & 1 + #define SIGN_LR hint 25 /* paciasp: sign with the A key */ + #define VERIFY_LR hint 29 /* autiasp: verify with the A key */ + #elif __ARM_FEATURE_PAC_DEFAULT & 2 + #define SIGN_LR hint 27 /* pacibsp: sign with the b key */ + #define VERIFY_LR hint 31 /* autibsp: verify with the b key */ + #endif + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 2 /* bit 1 GNU Notes is for PAC support */ +#else + #define SIGN_LR BTI_C + #define VERIFY_LR + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 +#endif + +#if defined(__ARM_FEATURE_GCS_DEFAULT) && __ARM_FEATURE_GCS_DEFAULT == 1 + #define GNU_PROPERTY_AARCH64_GCS 4 /* bit 2 GNU Notes is for GCS support */ +#else + #define GNU_PROPERTY_AARCH64_GCS 0 +#endif + +/* Add the BTI, PAC and GCS support to GNU Notes section */ +#if GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_GCS != 0 + .pushsection .note.gnu.property, "a"; /* Start a new allocatable section */ + .balign 8; /* align it on a byte boundry */ + .long 4; /* size of "GNU\0" */ + .long 0x10; /* size of descriptor */ + .long 0x5; /* NT_GNU_PROPERTY_TYPE_0 */ + .asciz "GNU"; + .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4; /* Four bytes of data */ + .long (GNU_PROPERTY_AARCH64_BTI|GNU_PROPERTY_AARCH64_POINTER_AUTH|GNU_PROPERTY_AARCH64_GCS); /* BTI, PAC or GCS is enabled */ + .long 0; /* padding for 8 byte alignment */ + .popsection; /* end the section */ +#endif + +#endif diff --git a/Python/assemble.c b/Python/assemble.c index 35453277dd8..288d786461b 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -80,9 +80,9 @@ assemble_init(struct assembler *a, int firstlineno) } return SUCCESS; error: - Py_XDECREF(a->a_bytecode); - Py_XDECREF(a->a_linetable); - Py_XDECREF(a->a_except_table); + Py_CLEAR(a->a_bytecode); + Py_CLEAR(a->a_linetable); + Py_CLEAR(a->a_except_table); return ERROR; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index bcf7e7355da..e107c997e71 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -264,15 +264,16 @@ should be a list of names to emulate ``from name import ...``, or an empty list to emulate ``import name``. When importing a module from a package, note that __import__('A.B', ...) returns package A when fromlist is empty, but its submodule B when -fromlist is not empty. The level argument is used to determine whether to -perform absolute or relative imports: 0 is absolute, while a positive number -is the number of parent directories to search relative to the current module. +fromlist is not empty. The level argument is used to determine whether +to perform absolute or relative imports: 0 is absolute, while a positive +number is the number of parent directories to search relative to the +current module. [clinic start generated code]*/ static PyObject * builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=4febeda88a0cd245 input=73f4b960ea5b9dd6]*/ +/*[clinic end generated code: output=4febeda88a0cd245 input=e3096a230383f72d]*/ { return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); @@ -622,8 +623,9 @@ PyDoc_STRVAR(filter_doc, "filter(function, iterable, /)\n\ --\n\ \n\ -Return an iterator yielding those items of iterable for which function(item)\n\ -is true. If function is None, return the items that are true."); +Return an iterator yielding those items of iterable for which\n\ +function(item) is true. If function is None, return the items that\n\ +are true."); PyTypeObject PyFilter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -743,23 +745,24 @@ compile as builtin_compile Compile source into a code object that can be executed by exec() or eval(). -The source code may represent a Python module, statement or expression. +The source code may represent a Python module, statement or +expression. The filename will be used for run-time error messages. The mode must be 'exec' to compile a module, 'single' to compile a single (interactive) statement, or 'eval' to compile an expression. -The flags argument, if present, controls which future statements influence -the compilation of the code. +The flags argument, if present, controls which future statements +influence the compilation of the code. The dont_inherit argument, if true, stops the compilation inheriting the effects of any future statements in effect in the code calling -compile; if absent or false these statements do influence the compilation, -in addition to any features explicitly specified. +compile; if absent or false these statements do influence the +compilation, in addition to any features explicitly specified. [clinic start generated code]*/ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=8f0069edbdac381b]*/ +/*[clinic end generated code: output=b0c09c84f116d3d7 input=d69ec2180ff24031]*/ { PyObject *source_copy; const char *str; @@ -903,10 +906,10 @@ PyDoc_STRVAR(dir_doc, "dir([object]) -> list of strings\n" "\n" "If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" +"Else, return an alphabetized list of names comprising (some of) the\n" +"attributes of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used;\n" +"otherwise the default dir() logic is used and returns:\n" " for a module object: the module's attributes.\n" " for a class object: its attributes, and recursively the attributes\n" " of its bases.\n" @@ -1209,9 +1212,11 @@ builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(getattr_doc, "getattr(object, name[, default]) -> value\n\ \n\ -Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ -When a default argument is given, it is returned when the attribute doesn't\n\ -exist; without it, an exception is raised in that case."); +Get a named attribute from an object.\n\ +\n\ +getattr(x, 'y') is equivalent to x.y.\n\ +When a default argument is given, it is returned when the attribute\n\ +doesn't exist; without it, an exception is raised in that case."); /*[clinic input] @@ -1219,13 +1224,13 @@ globals as builtin_globals Return the dictionary containing the current scope's global variables. -NOTE: Updates to this dictionary *will* affect name lookups in the current -global scope and vice-versa. +NOTE: Updates to this dictionary *will* affect name lookups in the +current global scope and vice-versa. [clinic start generated code]*/ static PyObject * builtin_globals_impl(PyObject *module) -/*[clinic end generated code: output=e5dd1527067b94d2 input=9327576f92bb48ba]*/ +/*[clinic end generated code: output=e5dd1527067b94d2 input=6d725a9b48d1eaeb]*/ { PyObject *d; @@ -1564,8 +1569,8 @@ builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(next_doc, "next(iterator[, default])\n\ \n\ -Return the next item from the iterator. If default is given and the iterator\n\ -is exhausted, it is returned instead of raising StopIteration."); +Return the next item from the iterator. If default is given and the\n\ +iterator is exhausted, it is returned instead of raising StopIteration."); /*[clinic input] @@ -1686,7 +1691,8 @@ iter(callable, sentinel) -> iterator\n\ \n\ Get an iterator from an object. In the first form, the argument must\n\ supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the sentinel."); +In the second form, the callable is called until it returns the\n\ +sentinel."); /*[clinic input] @@ -1780,14 +1786,15 @@ locals as builtin_locals Return a dictionary containing the current scope's local variables. -NOTE: Whether or not updates to this dictionary will affect name lookups in -the local scope and vice-versa is *implementation dependent* and not -covered by any backwards compatibility guarantees. +NOTE: Whether or not updates to this dictionary will affect name +lookups in the local scope and vice-versa is *implementation +dependent* and not covered by any backwards compatibility +guarantees. [clinic start generated code]*/ static PyObject * builtin_locals_impl(PyObject *module) -/*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ +/*[clinic end generated code: output=b46c94015ce11448 input=989cc75c22167c42]*/ { return _PyEval_GetFrameLocals(); } @@ -2033,14 +2040,14 @@ pow as builtin_pow Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments -Some types, such as ints, are able to use a more efficient algorithm when -invoked using the three argument form. +Some types, such as ints, are able to use a more efficient algorithm +when invoked using the three argument form. [clinic start generated code]*/ static PyObject * builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp, PyObject *mod) -/*[clinic end generated code: output=3ca1538221bbf15f input=435dbd48a12efb23]*/ +/*[clinic end generated code: output=3ca1538221bbf15f input=6133ded72c7db82e]*/ { return PyNumber_Power(base, exp, mod); } @@ -2160,13 +2167,14 @@ Read a string from standard input. The trailing newline is stripped. The prompt string, if given, is printed to standard output without a trailing newline before reading input. -If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError. +If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise +EOFError. On *nix systems, readline is used if available. [clinic start generated code]*/ static PyObject * builtin_input_impl(PyObject *module, PyObject *prompt) -/*[clinic end generated code: output=83db5a191e7a0d60 input=159c46d4ae40977e]*/ +/*[clinic end generated code: output=83db5a191e7a0d60 input=ebb939c954639427]*/ { PyObject *fin = NULL; PyObject *fout = NULL; @@ -2434,13 +2442,14 @@ round as builtin_round Round a number to a given precision in decimal digits. -The return value is an integer if ndigits is omitted or None. Otherwise -the return value has the same type as the number. ndigits may be negative. +The return value is an integer if ndigits is omitted or None. +Otherwise the return value has the same type as the number. ndigits +may be negative. [clinic start generated code]*/ static PyObject * builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) -/*[clinic end generated code: output=ff0d9dd176c02ede input=275678471d7aca15]*/ +/*[clinic end generated code: output=ff0d9dd176c02ede input=bdcb7c67bf4a4320]*/ { PyObject *round, *result; @@ -2475,8 +2484,8 @@ sorted as builtin_sorted Return a new list containing all items from the iterable in ascending order. -A custom key function can be supplied to customize the sort order, and the -reverse flag can be set to request the result in descending order. +A custom key function can be supplied to customize the sort order, and +the reverse flag can be set to request the result in descending order. [end disabled clinic input]*/ PyDoc_STRVAR(builtin_sorted__doc__, @@ -2561,13 +2570,13 @@ sum as builtin_sum Return the sum of a 'start' value (default: 0) plus an iterable of numbers When the iterable is empty, return the start value. -This function is intended specifically for use with numeric values and may -reject non-numeric types. +This function is intended specifically for use with numeric values and +may reject non-numeric types. [clinic start generated code]*/ static PyObject * builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) -/*[clinic end generated code: output=df758cec7d1d302f input=162b50765250d222]*/ +/*[clinic end generated code: output=df758cec7d1d302f input=83d1bc7fd7f7ad3b]*/ { PyObject *result = start; PyObject *temp, *item, *iter; @@ -2771,15 +2780,15 @@ isinstance as builtin_isinstance Return whether an object is an instance of a class or of a subclass thereof. -A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B) -or ...`` etc. +A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``isinstance(x, A) or +isinstance(x, B) or ...`` etc. [clinic start generated code]*/ static PyObject * builtin_isinstance_impl(PyObject *module, PyObject *obj, PyObject *class_or_tuple) -/*[clinic end generated code: output=6faf01472c13b003 input=ffa743db1daf7549]*/ +/*[clinic end generated code: output=6faf01472c13b003 input=477d04768621c24b]*/ { int retval; @@ -2799,15 +2808,15 @@ issubclass as builtin_issubclass Return whether 'cls' is derived from another class or is the same class. -A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B) -or ...``. +A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``issubclass(x, A) or +issubclass(x, B) or ...``. [clinic start generated code]*/ static PyObject * builtin_issubclass_impl(PyObject *module, PyObject *cls, PyObject *class_or_tuple) -/*[clinic end generated code: output=358412410cd7a250 input=a24b9f3d58c370d6]*/ +/*[clinic end generated code: output=358412410cd7a250 input=a91ce96345a6705d]*/ { int retval; @@ -3032,13 +3041,13 @@ PyDoc_STRVAR(zip_doc, "zip(*iterables, strict=False)\n\ --\n\ \n\ -The zip object yields n-length tuples, where n is the number of iterables\n\ -passed as positional arguments to zip(). The i-th element in every tuple\n\ -comes from the i-th iterable argument to zip(). This continues until the\n\ -shortest argument is exhausted.\n\ +The zip object yields n-length tuples, where n is the number of\n\ +iterables passed as positional arguments to zip(). The i-th element\n\ +in every tuple comes from the i-th iterable argument to zip(). This\n\ +continues until the shortest argument is exhausted.\n\ \n\ -If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError.\n\ +If strict is true and one of the arguments is exhausted before the\n\ +others, raise a ValueError.\n\ \n\ >>> list(zip('abcdefg', range(3), range(4)))\n\ [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]"); diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 370cf3c929b..9a956a3f238 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -25,9 +25,10 @@ PyDoc_STRVAR(builtin___import____doc__, "empty list to emulate ``import name``.\n" "When importing a module from a package, note that __import__(\'A.B\', ...)\n" "returns package A when fromlist is empty, but its submodule B when\n" -"fromlist is not empty. The level argument is used to determine whether to\n" -"perform absolute or relative imports: 0 is absolute, while a positive number\n" -"is the number of parent directories to search relative to the current module."); +"fromlist is not empty. The level argument is used to determine whether\n" +"to perform absolute or relative imports: 0 is absolute, while a positive\n" +"number is the number of parent directories to search relative to the\n" +"current module."); #define BUILTIN___IMPORT___METHODDEF \ {"__import__", _PyCFunction_CAST(builtin___import__), METH_FASTCALL|METH_KEYWORDS, builtin___import____doc__}, @@ -240,16 +241,17 @@ PyDoc_STRVAR(builtin_compile__doc__, "\n" "Compile source into a code object that can be executed by exec() or eval().\n" "\n" -"The source code may represent a Python module, statement or expression.\n" +"The source code may represent a Python module, statement or\n" +"expression.\n" "The filename will be used for run-time error messages.\n" "The mode must be \'exec\' to compile a module, \'single\' to compile a\n" "single (interactive) statement, or \'eval\' to compile an expression.\n" -"The flags argument, if present, controls which future statements influence\n" -"the compilation of the code.\n" +"The flags argument, if present, controls which future statements\n" +"influence the compilation of the code.\n" "The dont_inherit argument, if true, stops the compilation inheriting\n" "the effects of any future statements in effect in the code calling\n" -"compile; if absent or false these statements do influence the compilation,\n" -"in addition to any features explicitly specified."); +"compile; if absent or false these statements do influence the\n" +"compilation, in addition to any features explicitly specified."); #define BUILTIN_COMPILE_METHODDEF \ {"compile", _PyCFunction_CAST(builtin_compile), METH_FASTCALL|METH_KEYWORDS, builtin_compile__doc__}, @@ -568,8 +570,8 @@ PyDoc_STRVAR(builtin_globals__doc__, "\n" "Return the dictionary containing the current scope\'s global variables.\n" "\n" -"NOTE: Updates to this dictionary *will* affect name lookups in the current\n" -"global scope and vice-versa."); +"NOTE: Updates to this dictionary *will* affect name lookups in the\n" +"current global scope and vice-versa."); #define BUILTIN_GLOBALS_METHODDEF \ {"globals", (PyCFunction)builtin_globals, METH_NOARGS, builtin_globals__doc__}, @@ -780,9 +782,10 @@ PyDoc_STRVAR(builtin_locals__doc__, "\n" "Return a dictionary containing the current scope\'s local variables.\n" "\n" -"NOTE: Whether or not updates to this dictionary will affect name lookups in\n" -"the local scope and vice-versa is *implementation dependent* and not\n" -"covered by any backwards compatibility guarantees."); +"NOTE: Whether or not updates to this dictionary will affect name\n" +"lookups in the local scope and vice-versa is *implementation\n" +"dependent* and not covered by any backwards compatibility\n" +"guarantees."); #define BUILTIN_LOCALS_METHODDEF \ {"locals", (PyCFunction)builtin_locals, METH_NOARGS, builtin_locals__doc__}, @@ -829,8 +832,8 @@ PyDoc_STRVAR(builtin_pow__doc__, "\n" "Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n" "\n" -"Some types, such as ints, are able to use a more efficient algorithm when\n" -"invoked using the three argument form."); +"Some types, such as ints, are able to use a more efficient algorithm\n" +"when invoked using the three argument form."); #define BUILTIN_POW_METHODDEF \ {"pow", _PyCFunction_CAST(builtin_pow), METH_FASTCALL|METH_KEYWORDS, builtin_pow__doc__}, @@ -997,7 +1000,8 @@ PyDoc_STRVAR(builtin_input__doc__, "The prompt string, if given, is printed to standard output without a\n" "trailing newline before reading input.\n" "\n" -"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError.\n" +"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise\n" +"EOFError.\n" "On *nix systems, readline is used if available."); #define BUILTIN_INPUT_METHODDEF \ @@ -1043,8 +1047,9 @@ PyDoc_STRVAR(builtin_round__doc__, "\n" "Round a number to a given precision in decimal digits.\n" "\n" -"The return value is an integer if ndigits is omitted or None. Otherwise\n" -"the return value has the same type as the number. ndigits may be negative."); +"The return value is an integer if ndigits is omitted or None.\n" +"Otherwise the return value has the same type as the number. ndigits\n" +"may be negative."); #define BUILTIN_ROUND_METHODDEF \ {"round", _PyCFunction_CAST(builtin_round), METH_FASTCALL|METH_KEYWORDS, builtin_round__doc__}, @@ -1109,8 +1114,8 @@ PyDoc_STRVAR(builtin_sum__doc__, "Return the sum of a \'start\' value (default: 0) plus an iterable of numbers\n" "\n" "When the iterable is empty, return the start value.\n" -"This function is intended specifically for use with numeric values and may\n" -"reject non-numeric types."); +"This function is intended specifically for use with numeric values and\n" +"may reject non-numeric types."); #define BUILTIN_SUM_METHODDEF \ {"sum", _PyCFunction_CAST(builtin_sum), METH_FASTCALL|METH_KEYWORDS, builtin_sum__doc__}, @@ -1174,9 +1179,9 @@ PyDoc_STRVAR(builtin_isinstance__doc__, "\n" "Return whether an object is an instance of a class or of a subclass thereof.\n" "\n" -"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B)\n" -"or ...`` etc."); +"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``isinstance(x, A) or\n" +"isinstance(x, B) or ...`` etc."); #define BUILTIN_ISINSTANCE_METHODDEF \ {"isinstance", _PyCFunction_CAST(builtin_isinstance), METH_FASTCALL, builtin_isinstance__doc__}, @@ -1209,9 +1214,9 @@ PyDoc_STRVAR(builtin_issubclass__doc__, "\n" "Return whether \'cls\' is derived from another class or is the same class.\n" "\n" -"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B)\n" -"or ...``."); +"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``issubclass(x, A) or\n" +"issubclass(x, B) or ...``."); #define BUILTIN_ISSUBCLASS_METHODDEF \ {"issubclass", _PyCFunction_CAST(builtin_issubclass), METH_FASTCALL, builtin_issubclass__doc__}, @@ -1237,4 +1242,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=39173c70790d9cd0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8829159dfea1f8ad input=a9049054013a1b77]*/ diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h index 997ac6f6338..9d00b088026 100644 --- a/Python/clinic/context.c.h +++ b/Python/clinic/context.c.h @@ -10,8 +10,8 @@ PyDoc_STRVAR(_contextvars_Context_get__doc__, "\n" "Return the value for `key` if `key` has the value in the context object.\n" "\n" -"If `key` does not exist, return `default`. If `default` is not given,\n" -"return None."); +"If `key` does not exist, return `default`. If `default` is not\n" +"given, return None."); #define _CONTEXTVARS_CONTEXT_GET_METHODDEF \ {"get", _PyCFunction_CAST(_contextvars_Context_get), METH_FASTCALL, _contextvars_Context_get__doc__}, @@ -122,10 +122,12 @@ PyDoc_STRVAR(_contextvars_ContextVar_get__doc__, "\n" "Return a value for the context variable for the current context.\n" "\n" -"If there is no value for the variable in the current context, the method will:\n" -" * return the value of the default argument of the method, if provided; or\n" -" * return the default value for the context variable, if it was created\n" -" with one; or\n" +"If there is no value for the variable in the current context, the\n" +"method will:\n" +" * return the value of the default argument of the method, if\n" +" provided; or\n" +" * return the default value for the context variable, if it was\n" +" created with one; or\n" " * raise a LookupError."); #define _CONTEXTVARS_CONTEXTVAR_GET_METHODDEF \ @@ -160,10 +162,11 @@ PyDoc_STRVAR(_contextvars_ContextVar_set__doc__, "\n" "Call to set a new value for the context variable in the current context.\n" "\n" -"The required value argument is the new value for the context variable.\n" +"The required value argument is the new value for the context\n" +"variable.\n" "\n" -"Returns a Token object that can be used to restore the variable to its previous\n" -"value via the `ContextVar.reset()` method."); +"Returns a Token object that can be used to restore the variable to\n" +"its previous value via the `ContextVar.reset()` method."); #define _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF \ {"set", (PyCFunction)_contextvars_ContextVar_set, METH_O, _contextvars_ContextVar_set__doc__}, @@ -174,9 +177,9 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__, "\n" "Reset the context variable.\n" "\n" -"The variable is reset to the value it had before the `ContextVar.set()` that\n" -"created the token was used."); +"The variable is reset to the value it had before the\n" +"`ContextVar.set()` that created the token was used."); #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF \ {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__}, -/*[clinic end generated code: output=b667826178444c3f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ddcd15dcce797d4a input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 5edeaef656e..4dcaac46fc3 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -34,8 +34,9 @@ PyDoc_STRVAR(_imp_acquire_lock__doc__, "\n" "Acquires the interpreter\'s import lock for the current thread.\n" "\n" -"This lock should be used by import hooks to ensure thread-safety when importing\n" -"modules. On platforms without threads, this function does nothing."); +"This lock should be used by import hooks to ensure thread-safety when\n" +"importing modules. On platforms without threads, this function does\n" +"nothing."); #define _IMP_ACQUIRE_LOCK_METHODDEF \ {"acquire_lock", (PyCFunction)_imp_acquire_lock, METH_NOARGS, _imp_acquire_lock__doc__}, @@ -623,4 +624,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=dbd63707bd40b07c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e8d0a877136c038 input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index c19a3ed5050..b251d290830 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -189,8 +189,8 @@ PyDoc_STRVAR(marshal_dumps__doc__, " allow_code\n" " Allow to write code objects.\n" "\n" -"Raise a ValueError exception if value has (or contains an object that has) an\n" -"unsupported type."); +"Raise a ValueError exception if value has (or contains an object that\n" +"has) an unsupported type."); #define MARSHAL_DUMPS_METHODDEF \ {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL|METH_KEYWORDS, marshal_dumps__doc__}, @@ -271,8 +271,8 @@ PyDoc_STRVAR(marshal_loads__doc__, " allow_code\n" " Allow to load code objects.\n" "\n" -"If no valid value is found, raise EOFError, ValueError or TypeError. Extra\n" -"bytes in the input are ignored."); +"If no valid value is found, raise EOFError, ValueError or TypeError.\n" +"Extra bytes in the input are ignored."); #define MARSHAL_LOADS_METHODDEF \ {"loads", _PyCFunction_CAST(marshal_loads), METH_FASTCALL|METH_KEYWORDS, marshal_loads__doc__}, @@ -339,4 +339,4 @@ marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -/*[clinic end generated code: output=1575b9a3ae48ad3d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c26d748754801cfc input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 8277d286cf5..95e33ea5205 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1305,7 +1305,8 @@ PyDoc_STRVAR(sys__stats_dump__doc__, "\n" "Dump stats to file, and clears the stats.\n" "\n" -"Return False if no statistics were not dumped because stats gathering was off."); +"Return False if no statistics were not dumped because stats gathering\n" +"was off."); #define SYS__STATS_DUMP_METHODDEF \ {"_stats_dump", (PyCFunction)sys__stats_dump, METH_NOARGS, sys__stats_dump__doc__}, @@ -1614,4 +1615,4 @@ sys__is_gil_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=9cc9069aef1482bc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ea92bafc107d8521 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 8f8b6773440..4ab1280e91d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7844,6 +7844,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, { PyObject *res = NULL; PyObject *metadata = NULL; + PyObject *consts_list = NULL; if (!PyAST_Check(ast)) { PyErr_SetString(PyExc_TypeError, "expected an AST"); @@ -7889,7 +7890,6 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, SET_MATADATA_ITEM("name", umd->u_name); SET_MATADATA_ITEM("qualname", umd->u_qualname); - SET_MATADATA_ITEM("consts", umd->u_consts); SET_MATADATA_ITEM("names", umd->u_names); SET_MATADATA_ITEM("varnames", umd->u_varnames); SET_MATADATA_ITEM("cellvars", umd->u_cellvars); @@ -7915,12 +7915,21 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, } if (_PyInstructionSequence_ApplyLabelMap(INSTR_SEQUENCE(c)) < 0) { - return NULL; + goto finally; + } + /* After add_return_at_end: const indices match final instruction stream. */ + consts_list = consts_dict_keys_inorder(umd->u_consts); + if (consts_list == NULL) { + goto finally; + } + if (PyDict_SetItemString(metadata, "consts", consts_list) < 0) { + goto finally; } /* Allocate a copy of the instruction sequence on the heap */ res = PyTuple_Pack(2, INSTR_SEQUENCE(c), metadata); finally: + Py_XDECREF(consts_list); Py_XDECREF(metadata); compiler_exit_scope(c); compiler_free(c); @@ -7935,6 +7944,10 @@ _PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); return NULL; } + if (!PyList_Check(consts)) { + PyErr_SetString(PyExc_TypeError, "consts must be a list"); + return NULL; + } PyObject *const_cache = PyDict_New(); if (const_cache == NULL) { return NULL; diff --git a/Python/context.c b/Python/context.c index 1b2797b8af5..bc8b45cf6f1 100644 --- a/Python/context.c +++ b/Python/context.c @@ -556,14 +556,14 @@ _contextvars.Context.get Return the value for `key` if `key` has the value in the context object. -If `key` does not exist, return `default`. If `default` is not given, -return None. +If `key` does not exist, return `default`. If `default` is not +given, return None. [clinic start generated code]*/ static PyObject * _contextvars_Context_get_impl(PyContext *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=0c54aa7664268189 input=c8eeb81505023995]*/ +/*[clinic end generated code: output=0c54aa7664268189 input=d1be897231334ea9]*/ { if (context_check_key_type(key)) { return NULL; @@ -962,16 +962,18 @@ _contextvars.ContextVar.get Return a value for the context variable for the current context. -If there is no value for the variable in the current context, the method will: - * return the value of the default argument of the method, if provided; or - * return the default value for the context variable, if it was created - with one; or +If there is no value for the variable in the current context, the +method will: + * return the value of the default argument of the method, if + provided; or + * return the default value for the context variable, if it was + created with one; or * raise a LookupError. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) -/*[clinic end generated code: output=0746bd0aa2ced7bf input=30aa2ab9e433e401]*/ +/*[clinic end generated code: output=0746bd0aa2ced7bf input=83814c6aef4a9fe3]*/ { PyObject *val; if (PyContextVar_Get((PyObject *)self, default_value, &val) < 0) { @@ -993,15 +995,16 @@ _contextvars.ContextVar.set Call to set a new value for the context variable in the current context. -The required value argument is the new value for the context variable. +The required value argument is the new value for the context +variable. -Returns a Token object that can be used to restore the variable to its previous -value via the `ContextVar.reset()` method. +Returns a Token object that can be used to restore the variable to +its previous value via the `ContextVar.reset()` method. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_set(PyContextVar *self, PyObject *value) -/*[clinic end generated code: output=446ed5e820d6d60b input=c0a6887154227453]*/ +/*[clinic end generated code: output=446ed5e820d6d60b input=6ffee66796d67896]*/ { return PyContextVar_Set((PyObject *)self, value); } @@ -1013,13 +1016,13 @@ _contextvars.ContextVar.reset Reset the context variable. -The variable is reset to the value it had before the `ContextVar.set()` that -created the token was used. +The variable is reset to the value it had before the +`ContextVar.set()` that created the token was used. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_reset(PyContextVar *self, PyObject *token) -/*[clinic end generated code: output=d4ee34d0742d62ee input=ebe2881e5af4ffda]*/ +/*[clinic end generated code: output=d4ee34d0742d62ee input=dd33cfcb18c00e37]*/ { if (!PyContextToken_CheckExact(token)) { PyErr_Format(PyExc_TypeError, @@ -1045,7 +1048,8 @@ static PyMethodDef PyContextVar_methods[] = { _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("ContextVars are generic over the type of their contained values")}, {NULL, NULL} }; @@ -1184,7 +1188,8 @@ static PyGetSetDef PyContextTokenType_getsetlist[] = { static PyMethodDef PyContextTokenType_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Tokens are generic over the same type as the ContextVar which created them.")}, {NULL} }; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index ecf510842ea..baba1f75b9c 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1244,6 +1244,14 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) PyObject *constant = NULL; assert(OPCODE_HAS_CONST(opcode)); if (opcode == LOAD_CONST) { + assert(PyList_Check(co_consts)); + Py_ssize_t n = PyList_GET_SIZE(co_consts); + if (oparg < 0 || oparg >= n) { + PyErr_Format(PyExc_ValueError, + "LOAD_CONST index %d is out of range for consts (len=%zd)", + oparg, n); + return NULL; + } constant = PyList_GET_ITEM(co_consts, oparg); } @@ -2032,6 +2040,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) index_map = PyMem_Malloc(nconsts * sizeof(Py_ssize_t)); if (index_map == NULL) { + PyErr_NoMemory(); goto end; } for (Py_ssize_t i = 1; i < nconsts; i++) { @@ -2045,6 +2054,12 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) for (int i = 0; i < b->b_iused; i++) { if (OPCODE_HAS_CONST(b->b_instr[i].i_opcode)) { int index = b->b_instr[i].i_oparg; + if (index < 0 || index >= nconsts) { + PyErr_Format(PyExc_ValueError, + "LOAD_CONST index %d is out of range for consts (len=%zd)", + index, nconsts); + goto end; + } index_map[index] = index; } } @@ -2083,6 +2098,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) /* adjust const indices in the bytecode */ reverse_index_map = PyMem_Malloc(nconsts * sizeof(Py_ssize_t)); if (reverse_index_map == NULL) { + PyErr_NoMemory(); goto end; } for (Py_ssize_t i = 0; i < nconsts; i++) { diff --git a/Python/hamt.c b/Python/hamt.c index a9f811f4422..98c8cc3192a 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -718,6 +718,7 @@ hamt_node_bitmap_assoc(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *ret = hamt_node_bitmap_clone(self); if (ret == NULL) { + Py_DECREF(sub_node); return NULL; } Py_SETREF(ret->b_array[val_idx], (PyObject*)sub_node); @@ -1010,6 +1011,7 @@ hamt_node_bitmap_without(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *clone = hamt_node_bitmap_clone(self); if (clone == NULL) { + Py_DECREF(sub_node); return W_ERROR; } diff --git a/Python/import.c b/Python/import.c index f34e32c9582..44d0e5420cf 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2986,7 +2986,7 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) if (nameobj == NULL || nameobj == Py_None) { return FROZEN_BAD_NAME; } - const char *name = PyUnicode_AsUTF8(nameobj); + const char *name = _PyUnicode_AsUTF8NoNUL(nameobj); if (name == NULL) { // Note that this function previously used // _PyUnicode_EqualToASCIIString(). We clear the error here @@ -4310,13 +4310,14 @@ _imp.acquire_lock Acquires the interpreter's import lock for the current thread. -This lock should be used by import hooks to ensure thread-safety when importing -modules. On platforms without threads, this function does nothing. +This lock should be used by import hooks to ensure thread-safety when +importing modules. On platforms without threads, this function does +nothing. [clinic start generated code]*/ static PyObject * _imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ +/*[clinic end generated code: output=1aff58cb0ee1b026 input=60e9c1b4ab471ead]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyImport_AcquireLock(interp); diff --git a/Python/lock.c b/Python/lock.c index 24c404a1246..6def8567b11 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -210,7 +210,16 @@ _PyRawMutex_LockSlow(_PyRawMutex *m) // Wait for us to be woken up. Note that we still have to lock the // mutex ourselves: it is NOT handed off to us. - _PySemaphore_Wait(&waiter.sema, -1, /*detach=*/0); + // + // Loop until we observe an actual wakeup. A return of Py_PARK_INTR + // could otherwise let us exit _PySemaphore_Wait and destroy + // `waiter.sema` while _PyRawMutex_UnlockSlow's matching + // _PySemaphore_Wakeup is still pending, since the unlocker has + // already CAS-removed us from the waiter list without any handshake. + int res; + do { + res = _PySemaphore_Wait(&waiter.sema, -1, /*detach=*/0); + } while (res != Py_PARK_OK); } _PySemaphore_Destroy(&waiter.sema); diff --git a/Python/marshal.c b/Python/marshal.c index 65481341bdf..1bbce41d65f 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -310,7 +310,6 @@ static int w_ref(PyObject *v, char *flag, WFILE *p) { _Py_hashtable_entry_t *entry; - int w; if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ @@ -327,20 +326,28 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - w = (int)(uintptr_t)entry->value; + uintptr_t w = (uintptr_t)entry->value; + if (w & 0x80000000LU) { + PyErr_Format(PyExc_ValueError, "cannot marshal recursion %T objects", v); + goto err; + } /* we don't store "long" indices in the dict */ - assert(0 <= w && w <= 0x7fffffff); + assert(w <= 0x7fffffff); w_byte(TYPE_REF, p); - w_long(w, p); + w_long((int)w, p); return 1; } else { - size_t s = p->hashtable->nentries; + size_t w = p->hashtable->nentries; /* we don't support long indices */ - if (s >= 0x7fffffff) { + if (w >= 0x7fffffff) { PyErr_SetString(PyExc_ValueError, "too many objects"); goto err; } - w = (int)s; + // Corresponding code should call w_complete() after + // writing the object. + if (PyCode_Check(v)) { + w |= 0x80000000LU; + } if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); @@ -354,6 +361,27 @@ w_ref(PyObject *v, char *flag, WFILE *p) return 1; } +static void +w_complete(PyObject *v, WFILE *p) +{ + if (p->version < 3 || p->hashtable == NULL) { + return; + } + if (Py_REFCNT(v) == 1) { + return; + } + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(p->hashtable, v); + if (entry == NULL) { + return; + } + assert(entry != NULL); + uintptr_t w = (uintptr_t)entry->value; + assert(w & 0x80000000LU); + w &= ~0x80000000LU; + entry->value = (void *)(uintptr_t)w; +} + static void w_complex_object(PyObject *v, char flag, WFILE *p); @@ -603,6 +631,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_linetable, p); w_object(co->co_exceptiontable, p); Py_DECREF(co_code); + w_complete(v, p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -1458,7 +1487,7 @@ r_object(RFILE *p) goto code_error; firstlineno = (int)r_long(p); if (firstlineno == -1 && PyErr_Occurred()) - break; + goto code_error; linetable = r_object(p); if (linetable == NULL) goto code_error; @@ -1866,14 +1895,14 @@ marshal.dumps Return the bytes object that would be written to a file by dump(value, file). -Raise a ValueError exception if value has (or contains an object that has) an -unsupported type. +Raise a ValueError exception if value has (or contains an object that +has) an unsupported type. [clinic start generated code]*/ static PyObject * marshal_dumps_impl(PyObject *module, PyObject *value, int version, int allow_code) -/*[clinic end generated code: output=115f90da518d1d49 input=167eaecceb63f0a8]*/ +/*[clinic end generated code: output=115f90da518d1d49 input=d9609c4dee4507fb]*/ { return _PyMarshal_WriteObjectToString(value, version, allow_code); } @@ -1889,13 +1918,13 @@ marshal.loads Convert the bytes-like object to a value. -If no valid value is found, raise EOFError, ValueError or TypeError. Extra -bytes in the input are ignored. +If no valid value is found, raise EOFError, ValueError or TypeError. +Extra bytes in the input are ignored. [clinic start generated code]*/ static PyObject * marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code) -/*[clinic end generated code: output=62c0c538d3edc31f input=14de68965b45aaa7]*/ +/*[clinic end generated code: output=62c0c538d3edc31f input=286f1dbd6811d2ad]*/ { RFILE rf; char *s = bytes->buf; diff --git a/Python/parking_lot.c b/Python/parking_lot.c index a7e9760e35d..48577be4735 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -61,7 +61,9 @@ _PySemaphore_Init(_PySemaphore *sema) NULL // unnamed ); if (!sema->platform_sem) { - Py_FatalError("parking_lot: CreateSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: CreateSemaphore failed (error: %u)", + GetLastError()); } #elif defined(_Py_USE_SEMAPHORES) if (sem_init(&sema->platform_sem, /*pshared=*/0, /*value=*/0) < 0) { @@ -229,7 +231,9 @@ _PySemaphore_Wakeup(_PySemaphore *sema) { #if defined(MS_WINDOWS) if (!ReleaseSemaphore(sema->platform_sem, 1, NULL)) { - Py_FatalError("parking_lot: ReleaseSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: ReleaseSemaphore failed (error: %u, handle: %p)", + GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err = sem_post(&sema->platform_sem); diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index b03da66e92e..3901df7f17d 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -405,6 +405,9 @@ enum { DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction +#if defined(__aarch64__) + DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state +#endif DWRF_CFA_restore = 0xc0 // Restore register }; @@ -923,6 +926,13 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ +#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ + (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) +#endif +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here +#endif DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 DWRF_UV(16); // Stack pointer moved by 16 bytes @@ -931,6 +941,9 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed +#endif DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 0 (stack restored) diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 7ef9eca11f5..e296605c16a 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -211,9 +211,8 @@ enum perf_trampoline_type { static void free_code_arenas(void); static void -perf_trampoline_reset_state(void) +perf_trampoline_clear_code_watcher(void) { - free_code_arenas(); if (code_watcher_id >= 0) { PyCode_ClearWatcher(code_watcher_id); code_watcher_id = -1; @@ -221,6 +220,13 @@ perf_trampoline_reset_state(void) extra_code_index = -1; } +static void +perf_trampoline_reset_state(void) +{ + free_code_arenas(); + perf_trampoline_clear_code_watcher(); +} + static int perf_trampoline_code_watcher(PyCodeEvent event, PyCodeObject *co) { @@ -623,9 +629,10 @@ _PyPerfTrampoline_AfterFork_Child(void) // After fork, Fini may leave the old code watcher registered // if trampolined code objects from the parent still exist // (trampoline_refcount > 0). Clear it unconditionally before - // Init registers a new one, to prevent two watchers sharing - // the same globals and double-decrementing trampoline_refcount. - perf_trampoline_reset_state(); + // Init registers a new one, but keep the old arenas mapped: the + // child may still need to return through trampoline frames that + // were on the C stack at fork(). + perf_trampoline_clear_code_watcher(); _PyPerfTrampoline_Init(1); } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8ba9b2bd006..fbbc38bc8e1 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2976,7 +2976,9 @@ apple_log_write_impl(PyObject *self, PyObject *args) // Pass the user-provided text through explicit %s formatting // to avoid % literals being interpreted as a formatting directive. - os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + // Using {public} ensures "dynamic" string messages are visible + // in the log without special configuration. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%{public}s", text); Py_RETURN_NONE; } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 632d2c56a67..0acb92e96ef 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1150,6 +1150,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) if (print_exception_fn == NULL || !PyCallable_Check(print_exception_fn)) { Py_DECREF(traceback_module); + Py_XDECREF(print_exception_fn); goto fallback; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 8d47177340d..f3132f3a121 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1782,7 +1782,7 @@ sys_getwindowsversion_impl(PyObject *module) PyObject *realVersion = _sys_getwindowsversion_from_kernel32(); if (!realVersion) { if (!PyErr_ExceptionMatches(PyExc_WindowsError)) { - return NULL; + goto error; } PyErr_Clear(); @@ -2329,12 +2329,13 @@ sys._stats_dump -> bool Dump stats to file, and clears the stats. -Return False if no statistics were not dumped because stats gathering was off. +Return False if no statistics were not dumped because stats gathering +was off. [clinic start generated code]*/ static int sys__stats_dump_impl(PyObject *module) -/*[clinic end generated code: output=6e346b4ba0de4489 input=31a489e39418b2a5]*/ +/*[clinic end generated code: output=6e346b4ba0de4489 input=7f3b7758cb59d2ff]*/ { int res = _Py_PrintSpecializationStats(1); _Py_StatsClear(); diff --git a/README.rst b/README.rst index df4ec5ab666..480a559cb79 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.13.13 +This is Python version 3.13.14 ============================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 668db8df0bd..d6b0db0684c 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -416,7 +416,7 @@ def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] - return + return # type: ignore[unreachable] try: assert spec.origin is not None st = os.stat(spec.origin) diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 13d47b9a2e2..ed0950166f1 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -20,8 +20,6 @@ python_version = 3.10 # ...And be strict: strict = True -strict_bytes = True -local_partial_types = True extra_checks = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 06e58e92184..7da6120711d 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -349,7 +349,7 @@ Modules/_testclinic.c - TestClass - ################################## ## global non-objects to fix in builtin modules -# <none> +Objects/memoryobject.c - bool_false - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 5e271a5014c..681f68dfd8f 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -312,6 +312,7 @@ Modules/pyexpat.c - error_info_of - Modules/pyexpat.c - handler_info - Modules/termios.c - termios_constants - Modules/timemodule.c init_timezone YEAR - +Modules/unicodedata.c unicodedata_create_capi capi - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 606f280a14d..b68da1b6be1 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -9,7 +9,9 @@ import typing _LLVM_VERSION = 18 -_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") +_LLVM_VERSION_PATTERN = re.compile( + rf"(?<!Apple )(LLVM|clang) version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+" +) _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index c94ed54b1d0..dadd5c7ede7 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.155.0 +hypothesis==6.155.1 diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 8b0c1e04587..8c6c2be5873 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -46,14 +46,14 @@ "1.1.1w", "3.1.8", "3.2.6", + "3.3.7", ] OPENSSL_RECENT_VERSIONS = [ - "3.0.19", - "3.3.6", - "3.4.4", - "3.5.5", - "3.6.1", + "3.0.21", + "3.4.6", + "3.5.7", + "3.6.3", # See make_ssl_data.py for notes on adding a new version. ] diff --git a/config.guess b/config.guess index e81d3ae7c21..a9d01fde461 100755 --- a/config.guess +++ b/config.guess @@ -1,14 +1,14 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2021 Free Software Foundation, Inc. +# Copyright 1992-2025 Free Software Foundation, Inc. # shellcheck disable=SC2006,SC2268 # see below for rationale -timestamp='2021-06-03' +timestamp='2025-07-10' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -47,7 +47,7 @@ me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] -Output the configuration name of the system \`$me' is run on. +Output the configuration name of the system '$me' is run on. Options: -h, --help print this help, then exit @@ -60,13 +60,13 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2021 Free Software Foundation, Inc. +Copyright 1992-2025 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -102,8 +102,8 @@ GUESS= # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. +# Historically, 'CC_FOR_BUILD' used to be named 'HOST_CC'. We still +# use 'HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. @@ -123,7 +123,7 @@ set_cc_for_build() { dummy=$tmp/dummy case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in ,,) echo "int x;" > "$dummy.c" - for driver in cc gcc c89 c99 ; do + for driver in cc gcc c17 c99 c89 ; do if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then CC_FOR_BUILD=$driver break @@ -155,6 +155,9 @@ Linux|GNU|GNU/*) set_cc_for_build cat <<-EOF > "$dummy.c" + #if defined(__ANDROID__) + LIBC=android + #else #include <features.h> #if defined(__UCLIBC__) LIBC=uclibc @@ -162,6 +165,8 @@ Linux|GNU|GNU/*) LIBC=dietlibc #elif defined(__GLIBC__) LIBC=gnu + #elif defined(__LLVM_LIBC__) + LIBC=llvm #else #include <stdarg.h> /* First heuristic to detect musl libc. */ @@ -169,6 +174,7 @@ Linux|GNU|GNU/*) LIBC=musl #endif #endif + #endif EOF cc_set_libc=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` eval "$cc_set_libc" @@ -437,7 +443,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in # This test works for both compilers. if test "$CC_FOR_BUILD" != no_compiler_found; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -m64 -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then SUN_ARCH=x86_64 @@ -459,7 +465,7 @@ case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in UNAME_RELEASE=`uname -v` ;; esac - # Japanese Language versions have a version number like `4.1.3-JL'. + # Japanese Language versions have a version number like '4.1.3-JL'. SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/'` GUESS=sparc-sun-sunos$SUN_REL ;; @@ -628,7 +634,8 @@ EOF sed 's/^ //' << EOF > "$dummy.c" #include <sys/systemcfg.h> - main() + int + main () { if (!__power_pc()) exit(1); @@ -712,7 +719,8 @@ EOF #include <stdlib.h> #include <unistd.h> - int main () + int + main () { #if defined(_SC_KERNEL_BITS) long bits = sysconf(_SC_KERNEL_BITS); @@ -904,7 +912,7 @@ EOF fi ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` + UNAME_PROCESSOR=`uname -p` case $UNAME_PROCESSOR in amd64) UNAME_PROCESSOR=x86_64 ;; @@ -929,6 +937,9 @@ EOF i*:PW*:*) GUESS=$UNAME_MACHINE-pc-pw32 ;; + *:SerenityOS:*:*) + GUESS=$UNAME_MACHINE-pc-serenity + ;; *:Interix*:*) case $UNAME_MACHINE in x86) @@ -963,11 +974,37 @@ EOF GNU_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` GUESS=$UNAME_MACHINE-unknown-$GNU_SYS$GNU_REL-$LIBC ;; + x86_64:[Mm]anagarm:*:*|i?86:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-pc-managarm-mlibc" + ;; + *:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-unknown-managarm-mlibc" + ;; *:Minix:*:*) GUESS=$UNAME_MACHINE-unknown-minix ;; aarch64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + set_cc_for_build + CPU=$UNAME_MACHINE + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __ARM_EABI__ + #ifdef __ARM_PCS_VFP + ABI=eabihf + #else + ABI=eabi + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + eabi | eabihf) CPU=armv8l; LIBCABI=$LIBC$ABI ;; + esac + fi + GUESS=$CPU-unknown-linux-$LIBCABI ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be @@ -1033,7 +1070,16 @@ EOF k1om:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; - loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) + kvx:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + kvx:cos:*:*) + GUESS=$UNAME_MACHINE-unknown-cos + ;; + kvx:mbr:*:*) + GUESS=$UNAME_MACHINE-unknown-mbr + ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC ;; m32r*:Linux:*:*) @@ -1148,16 +1194,27 @@ EOF ;; x86_64:Linux:*:*) set_cc_for_build + CPU=$UNAME_MACHINE LIBCABI=$LIBC if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_X32 >/dev/null - then - LIBCABI=${LIBC}x32 - fi + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __i386__ + ABI=x86 + #else + #ifdef __ILP32__ + ABI=x32 + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + x86) CPU=i686 ;; + x32) LIBCABI=${LIBC}x32 ;; + esac fi - GUESS=$UNAME_MACHINE-pc-linux-$LIBCABI + GUESS=$CPU-pc-linux-$LIBCABI ;; xtensa*:Linux:*:*) GUESS=$UNAME_MACHINE-unknown-linux-$LIBC @@ -1177,7 +1234,7 @@ EOF GUESS=$UNAME_MACHINE-pc-sysv4.2uw$UNAME_VERSION ;; i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility + # If we were able to find 'uname', then EMX Unix compatibility # is probably installed. GUESS=$UNAME_MACHINE-pc-os2-emx ;; @@ -1318,7 +1375,7 @@ EOF GUESS=ns32k-sni-sysv fi ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + PENTIUM:*:4.0*:*) # Unisys 'ClearPath HMP IX 4000' SVR4/MP effort # says <Richard.M.Bartel@ccMail.Census.GOV> GUESS=i586-unisys-sysv4 ;; @@ -1364,8 +1421,11 @@ EOF BePC:Haiku:*:*) # Haiku running on Intel PC compatible. GUESS=i586-pc-haiku ;; - x86_64:Haiku:*:*) - GUESS=x86_64-unknown-haiku + ppc:Haiku:*:*) # Haiku running on Apple PowerPC + GUESS=powerpc-apple-haiku + ;; + *:Haiku:*:*) # Haiku modern gcc (not bound by BeOS compat) + GUESS=$UNAME_MACHINE-unknown-haiku ;; SX-4:SUPER-UX:*:*) GUESS=sx4-nec-superux$UNAME_RELEASE @@ -1522,6 +1582,9 @@ EOF i*86:rdos:*:*) GUESS=$UNAME_MACHINE-pc-rdos ;; + i*86:Fiwix:*:*) + GUESS=$UNAME_MACHINE-pc-fiwix + ;; *:AROS:*:*) GUESS=$UNAME_MACHINE-unknown-aros ;; @@ -1534,6 +1597,12 @@ EOF *:Unleashed:*:*) GUESS=$UNAME_MACHINE-unknown-unleashed$UNAME_RELEASE ;; + x86_64:[Ii]ronclad:*:*|i?86:[Ii]ronclad:*:*) + GUESS=$UNAME_MACHINE-pc-ironclad-mlibc + ;; + *:[Ii]ronclad:*:*) + GUESS=$UNAME_MACHINE-unknown-ironclad-mlibc + ;; esac # Do we have a guess based on uname results? @@ -1557,6 +1626,7 @@ cat > "$dummy.c" <<EOF #endif #endif #endif +int main () { #if defined (sony) @@ -1741,8 +1811,8 @@ fi exit 1 # Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp nil t) # time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-format: "%Y-%02m-%02d" # time-stamp-end: "'" # End: diff --git a/config.sub b/config.sub index 1bb6a05dc11..3d35cde174d 100755 --- a/config.sub +++ b/config.sub @@ -1,11 +1,10 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2024 Free Software Foundation, Inc. +# Copyright 1992-2025 Free Software Foundation, Inc. -# shellcheck disable=SC2006,SC2268 # see below for rationale +# shellcheck disable=SC2006,SC2268,SC2162 # see below for rationale -# Patched 2024-02-03 to include support for arm64_32 and iOS/tvOS/watchOS simulators -timestamp='2024-01-01' +timestamp='2025-07-10' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -77,7 +76,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright 1992-2024 Free Software Foundation, Inc. +Copyright 1992-2025 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -121,7 +120,6 @@ case $# in esac # Split fields of configuration type -# shellcheck disable=SC2162 saved_IFS=$IFS IFS="-" read field1 field2 field3 field4 <<EOF $1 @@ -143,10 +141,21 @@ case $1 in # parts maybe_os=$field2-$field3 case $maybe_os in - nto-qnx* | linux-* | uclinux-uclibc* \ - | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ - | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova* | managarm-* \ + cloudabi*-eabi* \ + | kfreebsd*-gnu* \ + | knetbsd*-gnu* \ + | kopensolaris*-gnu* \ + | ironclad-* \ + | linux-* \ + | managarm-* \ + | netbsd*-eabi* \ + | netbsd*-gnu* \ + | nto-qnx* \ + | os2-emx* \ + | rtmk-nova* \ + | storm-chaos* \ + | uclinux-gnu* \ + | uclinux-uclibc* \ | windows-* ) basic_machine=$field1 basic_os=$maybe_os @@ -162,8 +171,12 @@ case $1 in esac ;; *-*) - # A lone config we happen to match not fitting any pattern case $field1-$field2 in + # Shorthands that happen to contain a single dash + convex-c[12] | convex-c3[248]) + basic_machine=$field2-convex + basic_os= + ;; decstation-3100) basic_machine=mips-dec basic_os= @@ -171,28 +184,87 @@ case $1 in *-*) # Second component is usually, but not always the OS case $field2 in - # Prevent following clause from handling this valid os + # Do not treat sunos as a manufacturer sun*os*) basic_machine=$field1 basic_os=$field2 ;; - zephyr*) - basic_machine=$field1-unknown - basic_os=$field2 - ;; # Manufacturers - dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ - | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ - | unicom* | ibm* | next | hp | isi* | apollo | altos* \ - | convergent* | ncr* | news | 32* | 3600* | 3100* \ - | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ - | ultra | tti* | harris | dolphin | highlevel | gould \ - | cbm | ns | masscomp | apple | axis | knuth | cray \ - | microblaze* | sim | cisco \ - | oki | wec | wrs | winbond) + 3100* \ + | 32* \ + | 3300* \ + | 3600* \ + | 7300* \ + | acorn \ + | altos* \ + | apollo \ + | apple \ + | atari \ + | att* \ + | axis \ + | be \ + | bull \ + | cbm \ + | ccur \ + | cisco \ + | commodore \ + | convergent* \ + | convex* \ + | cray \ + | crds \ + | dec* \ + | delta* \ + | dg \ + | digital \ + | dolphin \ + | encore* \ + | gould \ + | harris \ + | highlevel \ + | hitachi* \ + | hp \ + | ibm* \ + | intergraph \ + | isi* \ + | knuth \ + | masscomp \ + | microblaze* \ + | mips* \ + | motorola* \ + | ncr* \ + | news \ + | next \ + | ns \ + | oki \ + | omron* \ + | pc533* \ + | rebel \ + | rom68k \ + | rombug \ + | semi \ + | sequent* \ + | sgi* \ + | siemens \ + | sim \ + | sni \ + | sony* \ + | stratus \ + | sun \ + | sun[234]* \ + | tektronix \ + | tti* \ + | ultra \ + | unicom* \ + | wec \ + | winbond \ + | wrs) basic_machine=$field1-$field2 basic_os= ;; + tock* | zephyr*) + basic_machine=$field1-unknown + basic_os=$field2 + ;; *) basic_machine=$field1 basic_os=$field2 @@ -273,26 +345,6 @@ case $1 in basic_machine=arm-unknown basic_os=cegcc ;; - convex-c1) - basic_machine=c1-convex - basic_os=bsd - ;; - convex-c2) - basic_machine=c2-convex - basic_os=bsd - ;; - convex-c32) - basic_machine=c32-convex - basic_os=bsd - ;; - convex-c34) - basic_machine=c34-convex - basic_os=bsd - ;; - convex-c38) - basic_machine=c38-convex - basic_os=bsd - ;; cray) basic_machine=j90-cray basic_os=unicos @@ -715,15 +767,26 @@ case $basic_machine in vendor=dec basic_os=tops20 ;; - delta | 3300 | motorola-3300 | motorola-delta \ - | 3300-motorola | delta-motorola) + delta | 3300 | delta-motorola | 3300-motorola | motorola-delta | motorola-3300) cpu=m68k vendor=motorola ;; - dpx2*) + # This used to be dpx2*, but that gets the RS6000-based + # DPX/20 and the x86-based DPX/2-100 wrong. See + # https://oldskool.silicium.org/stations/bull_dpx20.htm + # https://www.feb-patrimoine.com/english/bull_dpx2.htm + # https://www.feb-patrimoine.com/english/unix_and_bull.htm + dpx2 | dpx2[23]00 | dpx2[23]xx) cpu=m68k vendor=bull - basic_os=sysv3 + ;; + dpx2100 | dpx21xx) + cpu=i386 + vendor=bull + ;; + dpx20) + cpu=rs6000 + vendor=bull ;; encore | umax | mmax) cpu=ns32k @@ -838,18 +901,6 @@ case $basic_machine in next | m*-next) cpu=m68k vendor=next - case $basic_os in - openstep*) - ;; - nextstep*) - ;; - ns2*) - basic_os=nextstep2 - ;; - *) - basic_os=nextstep3 - ;; - esac ;; np1) cpu=np1 @@ -938,7 +989,6 @@ case $basic_machine in ;; *-*) - # shellcheck disable=SC2162 saved_IFS=$IFS IFS="-" read cpu vendor <<EOF $basic_machine @@ -973,15 +1023,19 @@ unset -v basic_machine # Decode basic machines in the full and proper CPU-Company form. case $cpu-$vendor in - # Here we handle the default manufacturer of certain CPU types in canonical form. It is in - # some cases the only manufacturer, in others, it is the most popular. + # Here we handle the default manufacturer of certain CPU types in canonical form. + # It is in some cases the only manufacturer, in others, it is the most popular. + c[12]-convex | c[12]-unknown | c3[248]-convex | c3[248]-unknown) + vendor=convex + basic_os=${basic_os:-bsd} + ;; craynv-unknown) vendor=cray basic_os=${basic_os:-unicosmp} ;; c90-unknown | c90-cray) vendor=cray - basic_os=${Basic_os:-unicos} + basic_os=${basic_os:-unicos} ;; fx80-unknown) vendor=alliant @@ -1027,11 +1081,29 @@ case $cpu-$vendor in vendor=alt basic_os=${basic_os:-linux-gnueabihf} ;; - dpx20-unknown | dpx20-bull) - cpu=rs6000 - vendor=bull + + # Normalized CPU+vendor pairs that imply an OS, if not otherwise specified + m68k-isi) + basic_os=${basic_os:-sysv} + ;; + m68k-sony) + basic_os=${basic_os:-newsos} + ;; + m68k-tektronix) + basic_os=${basic_os:-bsd} + ;; + m88k-harris) + basic_os=${basic_os:-sysv3} + ;; + i386-bull | m68k-bull) + basic_os=${basic_os:-sysv3} + ;; + rs6000-bull) basic_os=${basic_os:-bosx} ;; + mips-sni) + basic_os=${basic_os:-sysv4} + ;; # Here we normalize CPU types irrespective of the vendor amd64-*) @@ -1039,7 +1111,7 @@ case $cpu-$vendor in ;; blackfin-*) cpu=bfin - basic_os=linux + basic_os=${basic_os:-linux} ;; c54x-*) cpu=tic54x @@ -1062,7 +1134,7 @@ case $cpu-$vendor in ;; m68knommu-*) cpu=m68k - basic_os=linux + basic_os=${basic_os:-linux} ;; m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) cpu=s12z @@ -1072,7 +1144,7 @@ case $cpu-$vendor in ;; parisc-*) cpu=hppa - basic_os=linux + basic_os=${basic_os:-linux} ;; pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) cpu=i586 @@ -1086,9 +1158,6 @@ case $cpu-$vendor in pentium4-*) cpu=i786 ;; - pc98-*) - cpu=i386 - ;; ppc-* | ppcbe-*) cpu=powerpc ;; @@ -1122,9 +1191,6 @@ case $cpu-$vendor in tx39el-*) cpu=mipstx39el ;; - x64-*) - cpu=x86_64 - ;; xscale-* | xscalee[bl]-*) cpu=`echo "$cpu" | sed 's/^xscale/arm/'` ;; @@ -1180,90 +1246,228 @@ case $cpu-$vendor in # Recognize the canonical CPU types that are allowed with any # company name. case $cpu in - 1750a | 580 \ + 1750a \ + | 580 \ + | [cjt]90 \ | a29k \ - | aarch64 | aarch64_be | aarch64c | arm64ec \ + | aarch64 \ + | aarch64_be \ + | aarch64c \ | abacus \ - | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] \ - | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] \ - | alphapca5[67] | alpha64pca5[67] \ + | alpha \ + | alpha64 \ + | alpha64ev56 \ + | alpha64ev6[78] \ + | alpha64ev[4-8] \ + | alpha64pca5[67] \ + | alphaev56 \ + | alphaev6[78] \ + | alphaev[4-8] \ + | alphapca5[67] \ | am33_2.0 \ | amdgcn \ - | arc | arceb | arc32 | arc64 \ - | arm | arm[lb]e | arme[lb] | armv* \ - | avr | avr32 \ + | arc \ + | arc32 \ + | arc64 \ + | arceb \ + | arm \ + | arm64e \ + | arm64ec \ + | arm[lb]e \ + | arme[lb] \ + | armv* \ | asmjs \ + | avr \ + | avr32 \ | ba \ - | be32 | be64 \ - | bfin | bpf | bs2000 \ - | c[123]* | c30 | [cjt]90 | c4x \ - | c8051 | clipper | craynv | csky | cydra \ - | d10v | d30v | dlx | dsp16xx \ - | e2k | elxsi | epiphany \ - | f30[01] | f700 | fido | fr30 | frv | ft32 | fx80 \ - | javascript \ - | h8300 | h8500 \ - | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | be32 \ + | be64 \ + | bfin \ + | bpf \ + | bs2000 \ + | c30 \ + | c4x \ + | c8051 \ + | c[123]* \ + | clipper \ + | craynv \ + | csky \ + | cydra \ + | d10v \ + | d30v \ + | dlx \ + | dsp16xx \ + | e2k \ + | elxsi \ + | epiphany \ + | f30[01] \ + | f700 \ + | fido \ + | fr30 \ + | frv \ + | ft32 \ + | fx80 \ + | h8300 \ + | h8500 \ | hexagon \ - | i370 | i*86 | i860 | i960 | ia16 | ia64 \ - | ip2k | iq2000 \ + | hppa \ + | hppa1.[01] \ + | hppa2.0 \ + | hppa2.0[nw] \ + | hppa64 \ + | i*86 \ + | i370 \ + | i860 \ + | i960 \ + | ia16 \ + | ia64 \ + | intelgt \ + | ip2k \ + | iq2000 \ + | javascript \ | k1om \ | kvx \ - | le32 | le64 \ + | le32 \ + | le64 \ | lm32 \ - | loongarch32 | loongarch64 \ - | m32c | m32r | m32rle \ - | m5200 | m68000 | m680[012346]0 | m68360 | m683?2 | m68k \ - | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x \ - | m88110 | m88k | maxq | mb | mcore | mep | metag \ - | microblaze | microblazeel \ + | loongarch32 \ + | loongarch64 \ + | m32c \ + | m32r \ + | m32rle \ + | m5200 \ + | m68000 \ + | m680[012346]0 \ + | m6811 \ + | m6812 \ + | m68360 \ + | m683?2 \ + | m68hc11 \ + | m68hc12 \ + | m68hcs12x \ + | m68k \ + | m88110 \ + | m88k \ + | maxq \ + | mb \ + | mcore \ + | mep \ + | metag \ + | microblaze \ + | microblazeel \ | mips* \ | mmix \ - | mn10200 | mn10300 \ + | mn10200 \ + | mn10300 \ | moxie \ - | mt \ | msp430 \ + | mt \ | nanomips* \ - | nds32 | nds32le | nds32be \ + | nds32 \ + | nds32be \ + | nds32le \ | nfp \ - | nios | nios2 | nios2eb | nios2el \ - | none | np1 | ns16k | ns32k | nvptx \ + | nios \ + | nios2 \ + | nios2eb \ + | nios2el \ + | none \ + | np1 \ + | ns16k \ + | ns32k \ + | nvptx \ | open8 \ | or1k* \ | or32 \ | orion \ + | pdp10 \ + | pdp11 \ | picochip \ - | pdp10 | pdp11 | pj | pjl | pn | power \ - | powerpc | powerpc64 | powerpc64le | powerpcle | powerpcspe \ + | pj \ + | pjl \ + | pn \ + | power \ + | powerpc \ + | powerpc64 \ + | powerpc64le \ + | powerpcle \ + | powerpcspe \ | pru \ | pyramid \ - | riscv | riscv32 | riscv32be | riscv64 | riscv64be \ - | rl78 | romp | rs6000 | rx \ - | s390 | s390x \ + | riscv \ + | riscv32 \ + | riscv32be \ + | riscv64 \ + | riscv64be \ + | rl78 \ + | romp \ + | rs6000 \ + | rx \ + | s390 \ + | s390x \ | score \ - | sh | shl \ - | sh[1234] | sh[24]a | sh[24]ae[lb] | sh[23]e | she[lb] | sh[lb]e \ - | sh[1234]e[lb] | sh[12345][lb]e | sh[23]ele | sh64 | sh64le \ - | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet \ + | sh \ + | sh64 \ + | sh64le \ + | sh[12345][lb]e \ + | sh[1234] \ + | sh[1234]e[lb] \ + | sh[23]e \ + | sh[23]ele \ + | sh[24]a \ + | sh[24]ae[lb] \ + | sh[lb]e \ + | she[lb] \ + | shl \ + | sparc \ + | sparc64 \ + | sparc64b \ + | sparc64v \ + | sparc86x \ + | sparclet \ | sparclite \ - | sparcv8 | sparcv9 | sparcv9b | sparcv9v | sv1 | sx* \ + | sparcv8 \ + | sparcv9 \ + | sparcv9b \ + | sparcv9v \ | spu \ + | sv1 \ + | sx* \ | tahoe \ | thumbv7* \ - | tic30 | tic4x | tic54x | tic55x | tic6x | tic80 \ + | tic30 \ + | tic4x \ + | tic54x \ + | tic55x \ + | tic6x \ + | tic80 \ | tron \ | ubicom32 \ - | v70 | v850 | v850e | v850e1 | v850es | v850e2 | v850e2v3 \ + | v70 \ + | v810 \ + | v850 \ + | v850e \ + | v850e1 \ + | v850e2 \ + | v850e2v3 \ + | v850es \ | vax \ | vc4 \ | visium \ | w65 \ - | wasm32 | wasm64 \ + | wasm32 \ + | wasm64 \ | we32k \ - | x86 | x86_64 | xc16x | xgate | xps100 \ - | xstormy16 | xtensa* \ + | x86 \ + | x86_64 \ + | xc16x \ + | xgate \ + | xps100 \ + | xstormy16 \ + | xtensa* \ | ymp \ - | z8k | z80) + | z80 \ + | z8k) ;; *) @@ -1308,7 +1512,6 @@ case $basic_os in os=`echo "$basic_os" | sed -e 's|nto-qnx|qnx|'` ;; *-*) - # shellcheck disable=SC2162 saved_IFS=$IFS IFS="-" read kernel os <<EOF $basic_os @@ -1320,6 +1523,10 @@ EOF kernel=nto os=`echo "$basic_os" | sed -e 's|nto|qnx|'` ;; + ironclad*) + kernel=ironclad + os=`echo "$basic_os" | sed -e 's|ironclad|mlibc|'` + ;; linux*) kernel=linux os=`echo "$basic_os" | sed -e 's|linux|gnu|'` @@ -1355,6 +1562,23 @@ case $os in unixware*) os=sysv4.2uw ;; + # The marketing names for NeXT's operating systems were + # NeXTSTEP, NeXTSTEP 2, OpenSTEP 3, OpenSTEP 4. 'openstep' is + # mapped to 'openstep3', but 'openstep1' and 'openstep2' are + # mapped to 'nextstep' and 'nextstep2', consistent with the + # treatment of SunOS/Solaris. + ns | ns1 | nextstep | nextstep1 | openstep1) + os=nextstep + ;; + ns2 | nextstep2 | openstep2) + os=nextstep2 + ;; + ns3 | nextstep3 | openstep | openstep3) + os=openstep3 + ;; + ns4 | nextstep4 | openstep4) + os=openstep4 + ;; # es1800 is here to avoid being matched by es* (a different OS) es1800*) os=ose @@ -1425,6 +1649,7 @@ case $os in ;; utek*) os=bsd + vendor=`echo "$vendor" | sed -e 's|^unknown$|tektronix|'` ;; dynix*) os=bsd @@ -1441,21 +1666,25 @@ case $os in 386bsd) os=bsd ;; - ctix* | uts*) + ctix*) os=sysv + vendor=`echo "$vendor" | sed -e 's|^unknown$|convergent|'` ;; - nova*) - os=rtmk-nova + uts*) + os=sysv ;; - ns2) - os=nextstep2 + nova*) + kernel=rtmk + os=nova ;; # Preserve the version number of sinix5. sinix5.*) os=`echo "$os" | sed -e 's|sinix|sysv|'` + vendor=`echo "$vendor" | sed -e 's|^unknown$|sni|'` ;; sinix*) os=sysv4 + vendor=`echo "$vendor" | sed -e 's|^unknown$|sni|'` ;; tpf*) os=tpf @@ -1596,6 +1825,14 @@ case $cpu-$vendor in os= obj=elf ;; + # The -sgi and -siemens entries must be before the mips- entry + # or we get the wrong os. + *-sgi) + os=irix + ;; + *-siemens) + os=sysv4 + ;; mips*-cisco) os= obj=elf @@ -1608,7 +1845,8 @@ case $cpu-$vendor in os= obj=coff ;; - *-tti) # must be before sparc entry or we get the wrong os. + # This must be before the sparc-* entry or we get the wrong os. + *-tti) os=sysv3 ;; sparc-* | *-sun) @@ -1640,7 +1878,7 @@ case $cpu-$vendor in os=hpux ;; *-hitachi) - os=hiux + os=hiuxwe2 ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) os=sysv @@ -1684,12 +1922,6 @@ case $cpu-$vendor in *-encore) os=bsd ;; - *-sgi) - os=irix - ;; - *-siemens) - os=sysv4 - ;; *-masscomp) os=rtu ;; @@ -1736,40 +1968,194 @@ case $os in ghcjs) ;; # Now accept the basic system types. - # The portable systems comes first. # Each alternative MUST end in a * to match a version number. - gnu* | android* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ - | *vms* | esix* | aix* | cnk* | sunos | sunos[34]* \ - | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ - | sym* | plan9* | psp* | sim* | xray* | os68k* | v88r* \ - | hiux* | abug | nacl* | netware* | windows* \ - | os9* | macos* | osx* | ios* | tvos* | watchos* \ - | mpw* | magic* | mmixware* | mon960* | lnews* \ - | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ - | aos* | aros* | cloudabi* | sortix* | twizzler* \ - | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ - | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ - | mirbsd* | netbsd* | dicos* | openedition* | ose* \ - | bitrig* | openbsd* | secbsd* | solidbsd* | libertybsd* | os108* \ - | ekkobsd* | freebsd* | riscix* | lynxos* | os400* \ - | bosx* | nextstep* | cxux* | oabi* \ - | ptx* | ecoff* | winnt* | domain* | vsta* \ - | udi* | lites* | ieee* | go32* | aux* | hcos* \ - | chorusrdb* | cegcc* | glidix* | serenity* \ - | cygwin* | msys* | moss* | proelf* | rtems* \ - | midipix* | mingw32* | mingw64* | mint* \ - | uxpv* | beos* | mpeix* | udk* | moxiebox* \ - | interix* | uwin* | mks* | rhapsody* | darwin* \ - | openstep* | oskit* | conix* | pw32* | nonstopux* \ - | storm-chaos* | tops10* | tenex* | tops20* | its* \ - | os2* | vos* | palmos* | uclinux* | nucleus* | morphos* \ - | scout* | superux* | sysv* | rtmk* | tpf* | windiss* \ - | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ - | skyos* | haiku* | rdos* | toppers* | drops* | es* \ - | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ - | midnightbsd* | amdhsa* | unleashed* | emscripten* | wasi* \ - | nsk* | powerunix* | genode* | zvmoe* | qnx* | emx* | zephyr* \ - | fiwix* | mlibc* | cos* | mbr* | ironclad* ) + abug \ + | aix* \ + | amdhsa* \ + | amigados* \ + | amigaos* \ + | android* \ + | aof* \ + | aos* \ + | aros* \ + | atheos* \ + | auroraux* \ + | aux* \ + | banan_os* \ + | beos* \ + | bitrig* \ + | bme* \ + | bosx* \ + | bsd* \ + | cegcc* \ + | chorusos* \ + | chorusrdb* \ + | clix* \ + | cloudabi* \ + | cnk* \ + | conix* \ + | cos* \ + | cxux* \ + | cygwin* \ + | darwin* \ + | dgux* \ + | dicos* \ + | dnix* \ + | domain* \ + | dragonfly* \ + | drops* \ + | ebmon* \ + | ecoff* \ + | ekkobsd* \ + | emscripten* \ + | emx* \ + | es* \ + | fiwix* \ + | freebsd* \ + | fuchsia* \ + | genix* \ + | genode* \ + | glidix* \ + | gnu* \ + | go32* \ + | haiku* \ + | hcos* \ + | hiux* \ + | hms* \ + | hpux* \ + | ieee* \ + | interix* \ + | ios* \ + | iris* \ + | irix* \ + | isc* \ + | its* \ + | l4re* \ + | libertybsd* \ + | lites* \ + | lnews* \ + | luna* \ + | lynxos* \ + | mach* \ + | macos* \ + | magic* \ + | mbr* \ + | midipix* \ + | midnightbsd* \ + | mingw32* \ + | mingw64* \ + | minix* \ + | mint* \ + | mirbsd* \ + | mks* \ + | mlibc* \ + | mmixware* \ + | mon960* \ + | morphos* \ + | moss* \ + | moxiebox* \ + | mpeix* \ + | mpw* \ + | msdos* \ + | msys* \ + | mvs* \ + | nacl* \ + | netbsd* \ + | netware* \ + | newsos* \ + | nextstep* \ + | nindy* \ + | nonstopux* \ + | nova* \ + | nsk* \ + | nucleus* \ + | nx6 \ + | nx7 \ + | oabi* \ + | ohos* \ + | onefs* \ + | openbsd* \ + | openedition* \ + | openstep* \ + | os108* \ + | os2* \ + | os400* \ + | os68k* \ + | os9* \ + | ose* \ + | osf* \ + | oskit* \ + | osx* \ + | palmos* \ + | phoenix* \ + | plan9* \ + | powermax* \ + | powerunix* \ + | proelf* \ + | psos* \ + | psp* \ + | ptx* \ + | pw32* \ + | qnx* \ + | rdos* \ + | redox* \ + | rhapsody* \ + | riscix* \ + | riscos* \ + | rtems* \ + | rtmk* \ + | rtu* \ + | scout* \ + | secbsd* \ + | sei* \ + | serenity* \ + | sim* \ + | skyos* \ + | solaris* \ + | solidbsd* \ + | sortix* \ + | storm-chaos* \ + | sunos \ + | sunos[34]* \ + | superux* \ + | syllable* \ + | sym* \ + | sysv* \ + | tenex* \ + | tirtos* \ + | tock* \ + | toppers* \ + | tops10* \ + | tops20* \ + | tpf* \ + | tvos* \ + | twizzler* \ + | uclinux* \ + | udi* \ + | udk* \ + | ultrix* \ + | unicos* \ + | uniplus* \ + | unleashed* \ + | unos* \ + | uwin* \ + | uxpv* \ + | v88r* \ + |*vms* \ + | vos* \ + | vsta* \ + | vxsim* \ + | vxworks* \ + | wasi* \ + | watchos* \ + | wince* \ + | windiss* \ + | windows* \ + | winnt* \ + | xenix* \ + | xray* \ + | zephyr* \ + | zvmoe* ) ;; # This one is extra strict with allowed versions sco3.2v2 | sco3.2v[4-9]* | sco5v6*) @@ -1830,9 +2216,11 @@ esac case $kernel-$os-$obj in linux-gnu*- | linux-android*- | linux-dietlibc*- | linux-llvm*- \ | linux-mlibc*- | linux-musl*- | linux-newlib*- \ - | linux-relibc*- | linux-uclibc*- ) + | linux-relibc*- | linux-uclibc*- | linux-ohos*- ) ;; - uclinux-uclibc*- ) + uclinux-uclibc*- | uclinux-gnu*- ) + ;; + ironclad-mlibc*-) ;; managarm-mlibc*- | managarm-kernel*- ) ;; @@ -1857,7 +2245,7 @@ case $kernel-$os-$obj in echo "Invalid configuration '$1': '$os' needs 'windows'." 1>&2 exit 1 ;; - kfreebsd*-gnu*- | kopensolaris*-gnu*-) + kfreebsd*-gnu*- | knetbsd*-gnu*- | netbsd*-gnu*- | kopensolaris*-gnu*-) ;; vxworks-simlinux- | vxworks-simwindows- | vxworks-spe-) ;; @@ -1865,6 +2253,8 @@ case $kernel-$os-$obj in ;; os2-emx-) ;; + rtmk-nova-) + ;; *-eabi*- | *-gnueabi*-) ;; ios*-simulator- | tvos*-simulator- | watchos*-simulator- ) @@ -1893,7 +2283,7 @@ case $vendor in *-riscix*) vendor=acorn ;; - *-sunos*) + *-sunos* | *-solaris*) vendor=sun ;; *-cnk* | *-aix*) @@ -1967,8 +2357,8 @@ echo "$cpu-$vendor${kernel:+-$kernel}${os:+-$os}${obj:+-$obj}" exit # Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) +# eval: (add-hook 'before-save-hook 'time-stamp nil t) # time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-format: "%Y-%02m-%02d" # time-stamp-end: "'" # End: diff --git a/configure b/configure index c8eac507836..74988f49391 100755 --- a/configure +++ b/configure @@ -9000,7 +9000,55 @@ case "$CC_BASENAME" in fi ;; *) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fprofile-update=atomic" >&5 + # Check for 32-bit x86 ISA + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for i686" >&5 +printf %s "checking for i686... " >&6; } +if test ${ac_cv_i686+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #ifdef __i386__ + # error "i386" + #endif + +int +main (void) +{ + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_i686=no +else $as_nop + ac_cv_i686=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_i686" >&5 +printf "%s\n" "$ac_cv_i686" >&6; } + + PGO_PROF_GEN_FLAG="-fprofile-generate" + + # Use -fprofile-update=atomic to fix a random GCC internal error on PGO + # build (gh-145801) caused by corruption of profile data (.gcda files). + # + # gh-148535: On i686, using -fprofile-update=atomic makes the PGO build + # way slower (up to 47x slower). So far, the GCC internal error on PGO + # build was not seen on i686, so don't use this flag on i686. + if test "x$ac_cv_i686" = xno +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fprofile-update=atomic" >&5 printf %s "checking whether C compiler accepts -fprofile-update=atomic... " >&6; } if test ${ax_cv_check_cflags___fprofile_update_atomic+y} then : @@ -9033,9 +9081,12 @@ fi printf "%s\n" "$ax_cv_check_cflags___fprofile_update_atomic" >&6; } if test "x$ax_cv_check_cflags___fprofile_update_atomic" = xyes then : - PGO_PROF_GEN_FLAG="-fprofile-generate -fprofile-update=atomic" + PGO_PROF_GEN_FLAG="$PGO_PROF_GEN_FLAG -fprofile-update=atomic" else $as_nop - PGO_PROF_GEN_FLAG="-fprofile-generate" + : +fi + + fi PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" @@ -9731,7 +9782,7 @@ fi case $GCC in yes) - CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" + CFLAGS_NODIST="$CFLAGS_NODIST -std=c17" @@ -18239,12 +18290,6 @@ if test "x$ac_cv_func_dup" = xyes then : printf "%s\n" "#define HAVE_DUP 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" -if test "x$ac_cv_func_dup3" = xyes -then : - printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "execv" "ac_cv_func_execv" if test "x$ac_cv_func_execv" = xyes @@ -18719,12 +18764,6 @@ if test "x$ac_cv_func_pipe" = xyes then : printf "%s\n" "#define HAVE_PIPE 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" -if test "x$ac_cv_func_pipe2" = xyes -then : - printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "plock" "ac_cv_func_plock" if test "x$ac_cv_func_plock" = xyes @@ -19340,7 +19379,13 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" + ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" +if test "x$ac_cv_func_dup3" = xyes +then : + printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" if test "x$ac_cv_func_getentropy" = xyes then : printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h @@ -19351,6 +19396,12 @@ if test "x$ac_cv_func_getgroups" = xyes then : printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" +if test "x$ac_cv_func_pipe2" = xyes +then : + printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" if test "x$ac_cv_func_system" = xyes diff --git a/configure.ac b/configure.ac index 82c0d6e3359..fe7119e205f 100644 --- a/configure.ac +++ b/configure.ac @@ -2146,10 +2146,32 @@ case "$CC_BASENAME" in fi ;; *) - AX_CHECK_COMPILE_FLAG( - [-fprofile-update=atomic], - [PGO_PROF_GEN_FLAG="-fprofile-generate -fprofile-update=atomic"], - [PGO_PROF_GEN_FLAG="-fprofile-generate"]) + # Check for 32-bit x86 ISA + AC_CACHE_CHECK([for i686], [ac_cv_i686], [ + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([ + #ifdef __i386__ + # error "i386" + #endif + ], []) + ],[ac_cv_i686=no],[ac_cv_i686=yes]) + ]) + + PGO_PROF_GEN_FLAG="-fprofile-generate" + + # Use -fprofile-update=atomic to fix a random GCC internal error on PGO + # build (gh-145801) caused by corruption of profile data (.gcda files). + # + # gh-148535: On i686, using -fprofile-update=atomic makes the PGO build + # way slower (up to 47x slower). So far, the GCC internal error on PGO + # build was not seen on i686, so don't use this flag on i686. + AS_VAR_IF([ac_cv_i686], [no], [ + AX_CHECK_COMPILE_FLAG( + [-fprofile-update=atomic], + [PGO_PROF_GEN_FLAG="$PGO_PROF_GEN_FLAG -fprofile-update=atomic"], + []) + ]) + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" LLVM_PROF_MERGER="true" LLVM_PROF_FILE="" @@ -5229,7 +5251,7 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ - copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ + copy_file_range ctermid dup execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ @@ -5239,7 +5261,7 @@ AC_CHECK_FUNCS([ \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ + plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ @@ -5267,7 +5289,7 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - AC_CHECK_FUNCS([getentropy getgroups system]) + AC_CHECK_FUNCS([dup3 getentropy getgroups pipe2 system]) fi AC_CHECK_DECL([dirfd], diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py index 62a85a034eb..7fddc3c026d 100644 --- a/iOS/testbed/__main__.py +++ b/iOS/testbed/__main__.py @@ -17,7 +17,7 @@ LOG_PREFIX_REGEX = re.compile( r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ - r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID + r"\s+iOSTestbed\[\d+:\w+\] " # Process/thread ID )