Merge branch 'python:main' into update-docs-importlib

Author: whyvineet

Lib/test/test_coroutines.py

@@ -2265,6 +2265,20 @@ def c():
         # before fixing, visible stack from throw would be shorter than from send.
         self.assertEqual(len_send, len_throw)
 
+    def test_call_generator_in_frame_clear(self):
+        # gh-143939: Running a generator while clearing the coroutine's frame
+        # should not be misinterpreted as a yield.
+        class CallGeneratorOnDealloc:
+            def __del__(self):
+                next(x for x in [1])
+
+        async def coro():
+            obj = CallGeneratorOnDealloc()
+            return 42
+
+        yielded, result = run_async(coro())
+        self.assertEqual(yielded, [])
+        self.assertEqual(result, 42)
 
 @unittest.skipIf(
     support.is_emscripten or support.is_wasi,

Misc/NEWS.d/next/Core_and_Builtins/2026-01-16-23-19-38.gh-issue-143939.w9TWch.rst

@@ -0,0 +1,3 @@
+Fix erroneous "cannot reuse already awaited coroutine" error that could
+occur when a generator was run during the process of clearing a coroutine's
+frame.

Objects/genobject.c

@@ -280,6 +280,9 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc)
 
     if (return_kind == GENERATOR_YIELD) {
         assert(result != NULL && !_PyErr_Occurred(tstate));
+#ifndef Py_GIL_DISABLED
+        assert(FRAME_STATE_SUSPENDED(gen->gi_frame_state));
+#endif
         *presult = result;
         return PYGEN_NEXT;
     }

Python/ceval.c

@@ -1914,14 +1914,16 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame)
     assert(frame->owner == FRAME_OWNED_BY_GENERATOR);
     PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame);
     FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_CLEARED);
-    ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_RETURN;
     assert(tstate->exc_info == &gen->gi_exc_state);
     tstate->exc_info = gen->gi_exc_state.previous_item;
     gen->gi_exc_state.previous_item = NULL;
     assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame);
     frame->previous = NULL;
     _PyFrame_ClearExceptCode(frame);
     _PyErr_ClearExcState(&gen->gi_exc_state);
+    // gh-143939: There must not be any escaping calls between setting
+    // the generator return kind and returning from _PyEval_EvalFrame.
+    ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_RETURN;
 }
 
 void

Python/gc_free_threading.c

@@ -308,18 +308,17 @@ disable_deferred_refcounting(PyObject *op)
         // should also be disabled when we turn off deferred refcounting.
         _PyObject_DisablePerThreadRefcounting(op);
     }
-    if (_PyObject_GC_IS_TRACKED(op)) {
-        // Generators and frame objects may contain deferred references to other
-        // objects. If the pointed-to objects are part of cyclic trash, we may
-        // have disabled deferred refcounting on them and need to ensure that we
-        // use strong references, in case the generator or frame object is
-        // resurrected by a finalizer.
-        if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) {
-            frame_disable_deferred_refcounting(&((PyGenObject *)op)->gi_iframe);
-        }
-        else if (PyFrame_Check(op)) {
-            frame_disable_deferred_refcounting(((PyFrameObject *)op)->f_frame);
-        }
+
+    // Generators and frame objects may contain deferred references to other
+    // objects. If the pointed-to objects are part of cyclic trash, we may
+    // have disabled deferred refcounting on them and need to ensure that we
+    // use strong references, in case the generator or frame object is
+    // resurrected by a finalizer.
+    if (PyGen_CheckExact(op) || PyCoro_CheckExact(op) || PyAsyncGen_CheckExact(op)) {
+        frame_disable_deferred_refcounting(&((PyGenObject *)op)->gi_iframe);
+    }
+    else if (PyFrame_Check(op)) {
+        frame_disable_deferred_refcounting(((PyFrameObject *)op)->f_frame);
     }
 }
 
@@ -507,6 +506,10 @@ gc_visit_thread_stacks(PyInterpreterState *interp, struct collection_state *stat
 static bool
 gc_maybe_untrack(PyObject *op)
 {
+    if (_PyObject_HasDeferredRefcount(op)) {
+        // deferred refcounting only works if the object is tracked
+        return false;
+    }
     // Currently we only check for tuples containing only non-GC objects.  In
     // theory we could check other immutable objects that contain references
     // to non-GC objects.
@@ -1019,7 +1022,7 @@ update_refs(const mi_heap_t *heap, const mi_heap_area_t *area,
     }
     _PyObject_ASSERT(op, refcount >= 0);
 
-    if (refcount > 0 && !_PyObject_HasDeferredRefcount(op)) {
+    if (refcount > 0) {
         if (gc_maybe_untrack(op)) {
             gc_restore_refs(op);
             return true;
@@ -1241,30 +1244,19 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area,
         return true;
     }
 
+    if (state->reason == _Py_GC_REASON_SHUTDOWN) {
+        // Disable deferred refcounting for reachable objects as well during
+        // interpreter shutdown. This ensures that these objects are collected
+        // immediately when their last reference is removed.
+        disable_deferred_refcounting(op);
+    }
+
     // object is reachable, restore `ob_tid`; we're done with these objects
     gc_restore_tid(op);
     gc_clear_alive(op);
     return true;
 }
 
-// Disable deferred refcounting for reachable objects during interpreter
-// shutdown. This ensures that these objects are collected immediately when
-// their last reference is removed. This needs to consider both tracked and
-// untracked GC objects, since either might have deferred refcounts enabled.
-static bool
-scan_heap_disable_deferred(const mi_heap_t *heap, const mi_heap_area_t *area,
-        void *block, size_t block_size, void *args)
-{
-    PyObject *op = op_from_block_all_gc(block, args);
-    if (op == NULL) {
-        return true;
-    }
-    if (!_Py_IsImmortal(op)) {
-        disable_deferred_refcounting(op);
-    }
-    return true;
-}
-
 static int
 move_legacy_finalizer_reachable(struct collection_state *state);
 
@@ -1499,10 +1491,6 @@ deduce_unreachable_heap(PyInterpreterState *interp,
     // Restores ob_tid for reachable objects.
     gc_visit_heaps(interp, &scan_heap_visitor, &state->base);
 
-    if (state->reason == _Py_GC_REASON_SHUTDOWN) {
-        gc_visit_heaps(interp, &scan_heap_disable_deferred, &state->base);
-    }
-
     if (state->legacy_finalizers.head) {
         // There may be objects reachable from legacy finalizers that are in
         // the unreachable set. We need to mark them as reachable.

Python/specialize.c

@@ -362,7 +362,7 @@ static uint32_t function_get_version(PyObject *o, int opcode);
 static void
 maybe_enable_deferred_ref_count(PyObject *op)
 {
-    if (!_Py_IsOwnedByCurrentThread(op)) {
+    if (!_Py_IsOwnedByCurrentThread(op) && _PyObject_GC_IS_TRACKED(op)) {
         // For module level variables that are heavily used from multiple
         // threads, deferred reference counting provides good scaling
         // benefits.  The downside is that the object will only be deallocated