Merge branch '3.14' into backport-97babb8-3.14

Author: hugovk

Android/android.py

@@ -34,7 +34,12 @@
 TESTBED_DIR = ANDROID_DIR / "testbed"
 CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
 
-HOSTS = ["aarch64-linux-android", "x86_64-linux-android"]
+HOSTS = [
+    "aarch64-linux-android",
+    "arm-linux-androideabi",
+    "i686-linux-android",
+    "x86_64-linux-android",
+]
 APP_ID = "org.python.testbed"
 DECODE_ARGS = ("UTF-8", "backslashreplace")
 
@@ -205,38 +210,48 @@ def make_build_python(context):
 #
 # If you're a member of the Python core team, and you'd like to be able to push
 # these tags yourself, please contact Malcolm Smith or Russell Keith-Magee.
-def unpack_deps(host, prefix_dir):
+def unpack_deps(host, prefix_dir, cache_dir):
     os.chdir(prefix_dir)
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
     for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.19-1",
-                     "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]:
+                     "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-2"]:
         filename = f"{name_ver}-{host}.tar.gz"
-        download(f"{deps_url}/{name_ver}/{filename}")
-        shutil.unpack_archive(filename)
-        os.remove(filename)
+        out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir)
+        shutil.unpack_archive(out_path)
 
 
-def download(url, target_dir="."):
-    out_path = f"{target_dir}/{basename(url)}"
-    run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url])
+def download(url, cache_dir):
+    out_path = cache_dir / basename(url)
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    if not out_path.is_file():
+        run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url])
+    else:
+        print(f"Using cached version of {basename(url)}")
     return out_path
 
 
-def configure_host_python(context):
+def configure_host_python(context, host=None):
+    if host is None:
+        host = context.host
     if context.clean:
-        clean(context.host)
+        clean(host)
 
-    host_dir = subdir(context.host, create=True)
+    host_dir = subdir(host, create=True)
     prefix_dir = host_dir / "prefix"
     if not prefix_dir.exists():
         prefix_dir.mkdir()
-        unpack_deps(context.host, prefix_dir)
+        cache_dir = (
+            Path(context.cache_dir).resolve()
+            if context.cache_dir
+            else CROSS_BUILD_DIR / "downloads"
+        )
+        unpack_deps(host, prefix_dir, cache_dir)
 
     os.chdir(host_dir)
     command = [
         # Basic cross-compiling configuration
         relpath(PYTHON_DIR / "configure"),
-        f"--host={context.host}",
+        f"--host={host}",
         f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}",
         f"--with-build-python={build_python_path()}",
         "--without-ensurepip",
@@ -252,14 +267,16 @@ def configure_host_python(context):
 
     if context.args:
         command.extend(context.args)
-    run(command, host=context.host)
+    run(command, host=host)
 
 
-def make_host_python(context):
+def make_host_python(context, host=None):
+    if host is None:
+        host = context.host
     # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so
     # delete any previous Python installation to prevent it being used during
     # the build.
-    host_dir = subdir(context.host)
+    host_dir = subdir(host)
     prefix_dir = host_dir / "prefix"
     for pattern in ("include/python*", "lib/libpython*", "lib/python*"):
         delete_glob(f"{prefix_dir}/{pattern}")
@@ -278,20 +295,28 @@ def make_host_python(context):
     )
 
 
-def build_all(context):
-    steps = [configure_build_python, make_build_python, configure_host_python,
-             make_host_python]
-    for step in steps:
-        step(context)
+def build_targets(context):
+    if context.target in {"all", "build"}:
+        configure_build_python(context)
+        make_build_python(context)
+
+    for host in HOSTS:
+        if context.target in {"all", "hosts", host}:
+            configure_host_python(context, host)
+            make_host_python(context, host)
 
 
 def clean(host):
     delete_glob(CROSS_BUILD_DIR / host)
 
 
-def clean_all(context):
-    for host in HOSTS + ["build"]:
-        clean(host)
+def clean_targets(context):
+    if context.target in {"all", "build"}:
+        clean("build")
+
+    for host in HOSTS:
+        if context.target in {"all", "hosts", host}:
+            clean(host)
 
 
 def setup_ci():
@@ -853,31 +878,85 @@ def add_parser(*args, **kwargs):
 
     # Subcommands
     build = add_parser(
-        "build", help="Run configure-build, make-build, configure-host and "
-        "make-host")
+        "build",
+        help="Run configure and make for the selected target"
+    )
     configure_build = add_parser(
         "configure-build", help="Run `configure` for the build Python")
-    add_parser(
+    make_build = add_parser(
         "make-build", help="Run `make` for the build Python")
     configure_host = add_parser(
         "configure-host", help="Run `configure` for Android")
     make_host = add_parser(
         "make-host", help="Run `make` for Android")
 
-    add_parser("clean", help="Delete all build directories")
+    clean = add_parser(
+        "clean",
+        help="Delete build directories for the selected target"
+    )
+
     add_parser("build-testbed", help="Build the testbed app")
     test = add_parser("test", help="Run the testbed app")
     package = add_parser("package", help="Make a release package")
     ci = add_parser("ci", help="Run build, package and test")
     env = add_parser("env", help="Print environment variables")
 
     # Common arguments
+    # --cross-build-dir argument
+    for cmd in [
+        clean,
+        configure_build,
+        make_build,
+        configure_host,
+        make_host,
+        build,
+        package,
+        test,
+        ci,
+    ]:
+        cmd.add_argument(
+            "--cross-build-dir",
+            action="store",
+            default=os.environ.get("CROSS_BUILD_DIR"),
+            dest="cross_build_dir",
+            type=Path,
+            help=(
+                "Path to the cross-build directory "
+                f"(default: {CROSS_BUILD_DIR}). Can also be set "
+                "with the CROSS_BUILD_DIR environment variable."
+            ),
+        )
+
+    # --cache-dir option
+    for cmd in [configure_host, build, ci]:
+        cmd.add_argument(
+            "--cache-dir",
+            default=os.environ.get("CACHE_DIR"),
+            help="The directory to store cached downloads.",
+        )
+
+    # --clean option
     for subcommand in [build, configure_build, configure_host, ci]:
         subcommand.add_argument(
             "--clean", action="store_true", default=False, dest="clean",
             help="Delete the relevant build directories first")
 
-    host_commands = [build, configure_host, make_host, package, ci]
+    # Allow "all", "build" and "hosts" targets for some commands
+    for subcommand in [clean, build]:
+        subcommand.add_argument(
+            "target",
+            nargs="?",
+            default="all",
+            choices=["all", "build", "hosts"] + HOSTS,
+            help=(
+                "The host triplet (e.g., aarch64-linux-android), "
+                "or 'build' for just the build platform, or 'hosts' for all "
+                "host platforms, or 'all' for the build platform and all "
+                "hosts. Defaults to 'all'"
+            ),
+        )
+
+    host_commands = [configure_host, make_host, package, ci]
     if in_source_tree:
         host_commands.append(env)
     for subcommand in host_commands:
@@ -939,13 +1018,19 @@ def main():
         stream.reconfigure(line_buffering=True)
 
     context = parse_args()
+
+    # Set the CROSS_BUILD_DIR if an argument was provided
+    if context.cross_build_dir:
+        global CROSS_BUILD_DIR
+        CROSS_BUILD_DIR = context.cross_build_dir.resolve()
+
     dispatch = {
         "configure-build": configure_build_python,
         "make-build": make_build_python,
         "configure-host": configure_host_python,
         "make-host": make_host_python,
-        "build": build_all,
-        "clean": clean_all,
+        "build": build_targets,
+        "clean": clean_targets,
         "build-testbed": build_testbed,
         "test": run_testbed,
         "package": package,

Android/testbed/app/build.gradle.kts

@@ -15,6 +15,8 @@ val inSourceTree = (
 
 val KNOWN_ABIS = mapOf(
     "aarch64-linux-android" to "arm64-v8a",
+    "arm-linux-androideabi" to "armeabi-v7a",
+    "i686-linux-android" to "x86",
     "x86_64-linux-android" to "x86_64",
 )
 

Doc/conf.py

@@ -570,6 +570,17 @@
 stable_abi_file = 'data/stable_abi.dat'
 threadsafety_file = 'data/threadsafety.dat'
 
+# Options for notfound.extension
+# -------------------------------
+
+if not os.getenv("READTHEDOCS"):
+    if language_code:
+        notfound_urls_prefix = (
+            f'/{language_code.replace("_", "-").lower()}/{version}/'
+        )
+    else:
+        notfound_urls_prefix = f'/{version}/'
+
 # Options for sphinxext-opengraph
 # -------------------------------
 

Doc/howto/remote_debugging.rst

@@ -624,3 +624,58 @@ To inject and execute a Python script in a remote process:
 6. Set ``_PY_EVAL_PLEASE_STOP_BIT`` in the ``eval_breaker`` field.
 7. Resume the process (if suspended). The script will execute at the next safe
    evaluation point.
+
+.. _remote-debugging-threat-model:
+
+Security and threat model
+=========================
+
+The remote debugging protocol relies on the same operating system primitives
+used by native debuggers such as GDB and LLDB.  Attaching to a process
+requires the **same privileges** that those debuggers require, for example
+``ptrace`` / Yama LSM on Linux, ``task_for_pid`` on macOS, and
+``SeDebugPrivilege`` on Windows.  Python does not introduce any new privilege
+escalation path; if an attacker already possesses the permissions needed to
+attach to a process, they could equally use GDB to read memory or inject
+code.
+
+The following principles define what is, and is not, considered a security
+vulnerability in this feature:
+
+Attaching requires OS-level privileges
+   On every supported platform the operating system gates cross-process
+   memory access behind privilege checks (``CAP_SYS_PTRACE``, root, or
+   administrator rights).  A report that demonstrates an issue only after
+   these privileges have already been obtained is **not** a vulnerability in
+   CPython, since the OS security boundary was already crossed.
+
+Crashes or memory errors when reading a compromised process are not vulnerabilities
+   A tool that reads internal interpreter state from a target process must
+   trust that memory to be well-formed.  If the target process has been
+   corrupted or is controlled by an attacker, the debugger or profiler may
+   crash, produce garbage output, or behave unpredictably.  This is the same
+   risk accepted by every ``ptrace``-based debugger.  Bugs in this category
+   (buffer overflows, segmentation faults, or undefined behaviour triggered
+   by reading corrupted state) are **not** treated as security issues, though
+   fixes that improve robustness are welcome.
+
+Vulnerabilities in the target process are not in scope
+   If the Python process being debugged has already been compromised, the
+   attacker already controls execution in that process.  Demonstrating further
+   impact from that starting point does not constitute a vulnerability in the
+   remote debugging protocol.
+
+When to use ``PYTHON_DISABLE_REMOTE_DEBUG``
+-------------------------------------------
+
+The environment variable :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` (and the
+equivalent :option:`-X disable_remote_debug` flag) allows operators to disable
+the in-process side of the protocol as a **defence-in-depth** measure.  This
+may be useful in hardened or sandboxed deployment environments where no
+debugging or profiling of the process is expected and reducing attack surface
+is a priority, even though the OS-level privilege checks already prevent
+unprivileged access.
+
+Setting this variable does **not** affect other OS-level debugging interfaces
+(``ptrace``, ``/proc``, ``task_for_pid``, etc.), which remain available
+according to their own permission models.

Doc/library/argparse.rst

@@ -1111,7 +1111,15 @@ User defined functions can be used as well:
 
 The :func:`bool` function is not recommended as a type converter.  All it does
 is convert empty strings to ``False`` and non-empty strings to ``True``.
-This is usually not what is desired.
+This is usually not what is desired::
+
+   >>> parser = argparse.ArgumentParser()
+   >>> _ = parser.add_argument('--verbose', type=bool)
+   >>> parser.parse_args(['--verbose', 'False'])
+   Namespace(verbose=True)
+
+See :class:`BooleanOptionalAction` or ``action='store_true'`` for common
+alternatives.
 
 In general, the ``type`` keyword is a convenience that should only be used for
 simple conversions that can only raise one of the three supported exceptions.

Doc/library/http.server.rst

@@ -287,6 +287,8 @@ instantiation, of which this module provides three different variants:
       specifying its value. Note that, after the send_header calls are done,
       :meth:`end_headers` MUST BE called in order to complete the operation.
 
+      This method does not reject input containing CRLF sequences.
+
       .. versionchanged:: 3.2
          Headers are stored in an internal buffer.
 
@@ -297,6 +299,8 @@ instantiation, of which this module provides three different variants:
       buffered and sent directly the output stream.If the *message* is not
       specified, the HTTP message corresponding the response *code*  is sent.
 
+      This method does not reject *message* containing CRLF sequences.
+
       .. versionadded:: 3.2
 
    .. method:: end_headers()
@@ -622,6 +626,11 @@ Security considerations
 requests, this makes it possible for files outside of the specified directory
 to be served.
 
+Methods :meth:`BaseHTTPRequestHandler.send_header` and
+:meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input
+and does not perform input validation such as checking for the presence of CRLF
+sequences. Untrusted input may result in HTTP Header injection attacks.
+
 Earlier versions of Python did not scrub control characters from the
 log messages emitted to stderr from ``python -m http.server`` or the
 default :class:`BaseHTTPRequestHandler` ``.log_message``

Doc/using/configure.rst

@@ -838,9 +838,11 @@ See also the :ref:`Python Development Mode <devmode>` and the
 :option:`--with-trace-refs` configure option.
 
 .. versionchanged:: 3.8
-   Release builds and debug builds are now ABI compatible: defining the
+   Release builds are now ABI compatible with debug builds: defining the
    ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the
-   :option:`--with-trace-refs` option).
+   :option:`--with-trace-refs` option). However, debug builds still expose
+   more symbols than release builds and code built against a debug build is not
+   necessarily compatible with a release build.
 
 
 Debug options