Merge branch 'main' into base64-base32-padded

Author: serhiy-storchaka

Doc/library/functions.rst

@@ -1754,7 +1754,7 @@ are always available.  They are listed here in alphabetical order.
             self.age = age
 
          def __repr__(self):
-            return f"Person('{self.name}', {self.age})"
+            return f"Person({self.name!r}, {self.age!r})"
 
 
 .. function:: reversed(object, /)

Include/internal/pycore_ceval.h

@@ -211,16 +211,16 @@ extern void _PyEval_DeactivateOpCache(void);
 
 /* --- _Py_EnterRecursiveCall() ----------------------------------------- */
 
-static inline int _Py_MakeRecCheck(PyThreadState *tstate)  {
+static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate)  {
     uintptr_t here_addr = _Py_get_machine_stack_pointer();
     _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
-    // Overflow if stack pointer is between soft limit and the base of the hardware stack.
-    // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing.
-    // We could have the wrong stack limits because of limited platform support, or user-space threads.
+    // Possible overflow if stack pointer is beyond the soft limit.
+    // _Py_CheckRecursiveCall will check for corner cases and
+    // report an error if there is an overflow.
 #if _Py_STACK_GROWS_DOWN
-    return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES;
+    return here_addr < _tstate->c_stack_soft_limit;
 #else
-    return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES;
+    return here_addr > _tstate->c_stack_soft_limit;
 #endif
 }
 
@@ -235,7 +235,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallPy(
 
 static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate,
                                                const char *where) {
-    return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where));
+    return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where));
 }
 
 static inline int _Py_EnterRecursiveCall(const char *where) {
@@ -249,8 +249,6 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) {
 
 PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate);
 
-PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate);
-
 // Export for test_peg_generator
 PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(
     PyThreadState *tstate,

Include/internal/pycore_long.h

@@ -238,11 +238,12 @@ _PyLong_IsSmallInt(const PyLongObject *op)
 {
     assert(PyLong_Check(op));
     bool is_small_int = (op->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0;
-    assert(PyLong_CheckExact(op) || (!is_small_int));
-    assert(_Py_IsImmortal(op) || (!is_small_int));
-    assert((_PyLong_IsCompact(op)
-            && _PY_IS_SMALL_INT(_PyLong_CompactValue(op)))
-           || (!is_small_int));
+    if (is_small_int) {
+        assert(PyLong_CheckExact(op));
+        assert(_Py_IsImmortal(op));
+        assert((_PyLong_IsCompact(op)
+                && _PY_IS_SMALL_INT(_PyLong_CompactValue(op))));
+    }
     return is_small_int;
 }
 
@@ -285,6 +286,14 @@ _PyLong_SameSign(const PyLongObject *a, const PyLongObject *b)
     return (a->long_value.lv_tag & SIGN_MASK) == (b->long_value.lv_tag & SIGN_MASK);
 }
 
+/* Initialize the tag of a freshly-allocated int. */
+static inline void
+_PyLong_InitTag(PyLongObject *op)
+{
+    assert(PyLong_Check(op));
+    op->long_value.lv_tag = SIGN_ZERO; /* non-immortal zero */
+}
+
 #define TAG_FROM_SIGN_AND_SIZE(sign, size) \
     ((uintptr_t)(1 - (sign)) | ((uintptr_t)(size) << NON_SIZE_BITS))
 
@@ -294,13 +303,15 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size)
     assert(size >= 0);
     assert(-1 <= sign && sign <= 1);
     assert(sign != 0 || size == 0);
+    assert(!_PyLong_IsSmallInt(op));
     op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, size);
 }
 
 static inline void
 _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size)
 {
     assert(size >= 0);
+    assert(!_PyLong_IsSmallInt(op));
     op->long_value.lv_tag = (((size_t)size) << NON_SIZE_BITS) | (op->long_value.lv_tag & SIGN_MASK);
 }
 

Include/internal/pycore_pystate.h

@@ -306,23 +306,23 @@ _Py_AssertHoldsTstateFunc(const char *func)
 #define _Py_AssertHoldsTstate()
 #endif
 
-#if !_Py__has_builtin(__builtin_frame_address) && !defined(__GNUC__) && !defined(_MSC_VER)
-static uintptr_t return_pointer_as_int(char* p) {
-    return (uintptr_t)p;
-}
-#endif
 
 static inline uintptr_t
 _Py_get_machine_stack_pointer(void) {
-#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__)
-    return (uintptr_t)__builtin_frame_address(0);
-#elif defined(_MSC_VER)
-    return (uintptr_t)_AddressOfReturnAddress();
+    uintptr_t result;
+#if !defined(_MSC_VER) && defined(_M_ARM64)
+    result = __getReg(31);
+#elif defined(_MSC_VER) && defined(_M_X64)
+    result = (uintptr_t)_AddressOfReturnAddress();
+#elif defined(__aarch64__)
+    __asm__ ("mov %0, sp" : "=r" (result));
+#elif defined(__x86_64__)
+    __asm__("{movq %%rsp, %0" : "=r" (result));
 #else
     char here;
-    /* Avoid compiler warning about returning stack address */
-    return return_pointer_as_int(&here);
+    result = (uintptr_t)&here;
 #endif
+    return result;
 }
 
 static inline intptr_t

Include/internal/pycore_pythonrun.h

@@ -46,7 +46,8 @@ extern PyObject * _Py_CompileStringObjectWithModule(
  * stack consumption of PyEval_EvalDefault */
 #if (defined(Py_DEBUG) \
      || defined(_Py_ADDRESS_SANITIZER) \
-     || defined(_Py_THREAD_SANITIZER))
+     || defined(_Py_THREAD_SANITIZER)) \
+     || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER)
 #  define _PyOS_LOG2_STACK_MARGIN 12
 #else
 #  define _PyOS_LOG2_STACK_MARGIN 11

Include/pyport.h

@@ -598,6 +598,11 @@ extern "C" {
 #      define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread))
 #    endif
 #  endif
+#  if __has_feature(undefined_behavior_sanitizer)
+#    if !defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER)
+#      define _Py_UNDEFINED_BEHAVIOR_SANITIZER
+#    endif
+#  endif
 #elif defined(__GNUC__)
 #  if defined(__SANITIZE_ADDRESS__)
 #    define _Py_ADDRESS_SANITIZER

Lib/test/test_importlib/test_discover.py

@@ -11,7 +11,7 @@ def __init__(self, discover=[]):
         self._discovered_values = discover
 
     def find_spec(self, fullname, path=None, target=None):
-        raise NotImplemented
+        raise NotImplementedError
 
     def discover(self, parent=None):
         yield from self._discovered_values

Lib/test/test_pyexpat.py

@@ -707,7 +707,7 @@ def test_trigger_leak(self):
     def test_deeply_nested_content_model(self):
         # This should raise a RecursionError and not crash.
         # See https://github.com/python/cpython/issues/145986.
-        N = 500_000
+        N = 800_000
         data = (
             b'<!DOCTYPE root [\n<!ELEMENT root '
             + b'(a, ' * N + b'a' + b')' * N

Objects/longobject.c

@@ -185,6 +185,7 @@ long_alloc(Py_ssize_t size)
             return NULL;
         }
         _PyObject_Init((PyObject*)result, &PyLong_Type);
+        _PyLong_InitTag(result);
     }
     _PyLong_SetSignAndDigitCount(result, size != 0, size);
     /* The digit has to be initialized explicitly to avoid
@@ -258,6 +259,7 @@ _PyLong_FromMedium(sdigit x)
             return NULL;
         }
         _PyObject_Init((PyObject*)v, &PyLong_Type);
+        _PyLong_InitTag(v);
     }
     digit abs_x = x < 0 ? -x : x;
     _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1);
@@ -337,6 +339,7 @@ medium_from_stwodigits(stwodigits x)
             return PyStackRef_NULL;
         }
         _PyObject_Init((PyObject*)v, &PyLong_Type);
+        _PyLong_InitTag(v);
     }
     digit abs_x = x < 0 ? (digit)(-x) : (digit)x;
     _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1);
@@ -6011,29 +6014,34 @@ static PyObject *
 long_subtype_new(PyTypeObject *type, PyObject *x, PyObject *obase)
 {
     PyLongObject *tmp, *newobj;
-    Py_ssize_t i, n;
+    Py_ssize_t size, ndigits;
+    int sign;
 
     assert(PyType_IsSubtype(type, &PyLong_Type));
     tmp = (PyLongObject *)long_new_impl(&PyLong_Type, x, obase);
     if (tmp == NULL)
         return NULL;
     assert(PyLong_Check(tmp));
-    n = _PyLong_DigitCount(tmp);
+    size = _PyLong_DigitCount(tmp);
     /* Fast operations for single digit integers (including zero)
      * assume that there is always at least one digit present. */
-    if (n == 0) {
-        n = 1;
-    }
-    newobj = (PyLongObject *)type->tp_alloc(type, n);
+    ndigits = size ? size : 1;
+    newobj = (PyLongObject *)type->tp_alloc(type, ndigits);
     if (newobj == NULL) {
         Py_DECREF(tmp);
         return NULL;
     }
     assert(PyLong_Check(newobj));
-    newobj->long_value.lv_tag = tmp->long_value.lv_tag & ~IMMORTALITY_BIT_MASK;
-    for (i = 0; i < n; i++) {
-        newobj->long_value.ob_digit[i] = tmp->long_value.ob_digit[i];
+    if (_PyLong_IsCompact(tmp)) {
+        sign = _PyLong_CompactSign(tmp);
+    }
+    else {
+        sign = _PyLong_NonCompactSign(tmp);
     }
+    _PyLong_InitTag(newobj);
+    _PyLong_SetSignAndDigitCount(newobj, sign, size);
+    memcpy(newobj->long_value.ob_digit, tmp->long_value.ob_digit,
+           ndigits * sizeof(digit));
     Py_DECREF(tmp);
     return (PyObject *)newobj;
 }

Python/ceval.c

@@ -49,20 +49,6 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count)
 #endif
 }
 
-void
-_Py_EnterRecursiveCallUnchecked(PyThreadState *tstate)
-{
-    uintptr_t here_addr = _Py_get_machine_stack_pointer();
-    _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
-#if _Py_STACK_GROWS_DOWN
-    if (here_addr < _tstate->c_stack_hard_limit) {
-#else
-    if (here_addr > _tstate->c_stack_hard_limit) {
-#endif
-        Py_FatalError("Unchecked stack overflow.");
-    }
-}
-
 #if defined(__s390x__)
 #  define Py_C_STACK_SIZE 320000
 #elif defined(_WIN32)
@@ -278,7 +264,7 @@ PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate)
 
 
 /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall()
-   if the stack pointer is between the stack base and c_stack_hard_limit. */
+   if the stack pointer is beyond c_stack_soft_limit. */
 int
 _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
 {
@@ -287,16 +273,21 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
     assert(_tstate->c_stack_soft_limit != 0);
     assert(_tstate->c_stack_hard_limit != 0);
 #if _Py_STACK_GROWS_DOWN
-    assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES);
     if (here_addr < _tstate->c_stack_hard_limit) {
-        /* Overflowing while handling an overflow. Give up. */
+        if (here_addr < _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES) {
+            // Far out of bounds -- Assume stack switching has occurred
+            return 0;
+        }
         int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024;
 #else
-    assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES);
     if (here_addr > _tstate->c_stack_hard_limit) {
-        /* Overflowing while handling an overflow. Give up. */
+        if (here_addr > _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES) {
+            // Far out of bounds -- Assume stack switching has occurred
+            return 0;
+        }
         int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024;
 #endif
+        /* Too much stack used to safely raise an exception. Give up. */
         char buffer[80];
         snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where);
         Py_FatalError(buffer);
@@ -1201,19 +1192,6 @@ _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *index_or_null, int yield_from
     return PyStackRef_FromPyObjectSteal(iter_o);
 }
 
-Py_NO_INLINE int
-_Py_ReachedRecursionLimit(PyThreadState *tstate)  {
-    uintptr_t here_addr = _Py_get_machine_stack_pointer();
-    _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate;
-    assert(_tstate->c_stack_hard_limit != 0);
-#if _Py_STACK_GROWS_DOWN
-    return here_addr <= _tstate->c_stack_soft_limit;
-#else
-    return here_addr >= _tstate->c_stack_soft_limit;
-#endif
-}
-
-
 #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__)
 /*
  * gh-129987: The SLP autovectorizer can cause poor code generation for