diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4b1e289c6ff468..c538d446256525 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1582,6 +1582,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_interpreters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alphabet)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 6ee649b59a5c37..c4a9d7dc53ae48 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -305,6 +305,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_interpreters) STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(alphabet) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 778db946c2a3aa..9f8115ff5bd8a0 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1580,6 +1580,7 @@ extern "C" { INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_interpreters), \ INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(alphabet), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index bd8f50ff0ee732..c5563e8f96eac0 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1000,6 +1000,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_interpreters); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(all_threads); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Modules/Setup b/Modules/Setup index 7d816ead8432ef..33737c21cb4066 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -285,7 +285,7 @@ PYTHONPATH=$(COREPYTHONPATH) #*shared* #_ctypes_test _ctypes/_ctypes_test.c -#_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/threads.c _remote_debugging/asyncio.c +#_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/threads.c _remote_debugging/asyncio.c _remote_debugging/interpreters.c #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 0d520684c795d6..0305bf23cc3756 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -41,7 +41,7 @@ @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c -@MODULE__REMOTE_DEBUGGING_TRUE@_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/frame_cache.c _remote_debugging/threads.c _remote_debugging/asyncio.c _remote_debugging/binary_io_writer.c _remote_debugging/binary_io_reader.c _remote_debugging/subprocess.c +@MODULE__REMOTE_DEBUGGING_TRUE@_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/frame_cache.c _remote_debugging/threads.c _remote_debugging/asyncio.c _remote_debugging/binary_io_writer.c _remote_debugging/binary_io_reader.c _remote_debugging/subprocess.c _remote_debugging/interpreters.c @MODULE__STRUCT_TRUE@_struct _struct.c # build supports subinterpreters diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 3722273dfd2998..c7942ca72589dc 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -345,6 +345,12 @@ typedef struct { size_t count; } StackChunkList; +typedef struct { + proc_handle_t handle; + uintptr_t runtime_start_address; + struct _Py_DebugOffsets debug_offsets; +} RuntimeOffsets; + /* * Context for frame chain traversal operations. */ @@ -389,6 +395,14 @@ typedef int (*set_entry_processor_func)( void *context ); +typedef int (*interpreter_processor_func)( + RuntimeOffsets *offsets, + uintptr_t interpreter_state_addr, + unsigned long iid, + void *context +); + + /* ============================================================================ * STRUCTSEQ DESCRIPTORS (extern declarations) * ============================================================================ */ @@ -586,6 +600,17 @@ extern void _Py_RemoteDebug_InitThreadsState(RemoteUnwinderObject *unwinder, _Py extern int _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st); extern void _Py_RemoteDebug_ResumeAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st); +/* ============================================================================ + * INTERPRETER FUNCTION DECLARATIONS + * ============================================================================ */ + +extern int +iterate_interpreters( + RuntimeOffsets *offsets, + interpreter_processor_func processor, + void *context +); + /* ============================================================================ * ASYNCIO FUNCTION DECLARATIONS * ============================================================================ */ diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 15df48fabb56b2..0176bad5f9a34b 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -1296,4 +1296,101 @@ _remote_debugging_is_python_process(PyObject *module, PyObject *const *args, Py_ exit: return return_value; } -/*[clinic end generated code: output=34f50b18f317b9b6 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_remote_debugging_get_gc_stats__doc__, +"get_gc_stats($module, /, pid, *, all_interpreters=False)\n" +"--\n" +"\n" +"Get garbage collector statistics from external Python process.\n" +"\n" +" all_interpreters\n" +" If True, return GC statistics from all interpreters.\n" +" If False, return only from main interpreter.\n" +"\n" +"Returns:\n" +" List of dicts.\n" +" dict: A dictionary containing:\n" +" - gen:\n" +" - iid:\n" +" - ts_start:\n" +" - ts_stop:\n" +" - heap_size:\n" +" - work_to_do:\n" +" - collections:\n" +" - object_visits:\n" +" - collected:\n" +" - uncollectable:\n" +" - candidates:\n" +" - objects_transitively_reachable:\n" +" - objects_not_transitively_reachable:\n" +" - duration:\n" +"\n" +"Raises:\n" +" RuntimeError:"); + +#define _REMOTE_DEBUGGING_GET_GC_STATS_METHODDEF \ + {"get_gc_stats", _PyCFunction_CAST(_remote_debugging_get_gc_stats), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_get_gc_stats__doc__}, + +static PyObject * +_remote_debugging_get_gc_stats_impl(PyObject *module, int pid, + int all_interpreters); + +static PyObject * +_remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_interpreters), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pid", "all_interpreters", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "get_gc_stats", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + int pid; + int all_interpreters = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + pid = PyLong_AsInt(args[0]); + if (pid == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + all_interpreters = PyObject_IsTrue(args[1]); + if (all_interpreters < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _remote_debugging_get_gc_stats_impl(module, pid, all_interpreters); + +exit: + return return_value; +} +/*[clinic end generated code: output=bdd3092b9cbc4313 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/gc_stats.h b/Modules/_remote_debugging/gc_stats.h new file mode 100644 index 00000000000000..e2b0ff57904318 --- /dev/null +++ b/Modules/_remote_debugging/gc_stats.h @@ -0,0 +1,143 @@ +/****************************************************************************** + * Remote Debugging Module - GC Stats Functions + * + * This file contains function for read GC stats from interpreter state. + ******************************************************************************/ + +#ifndef Py_REMOTE_DEBUGGING_GC_STATS_H +#define Py_REMOTE_DEBUGGING_GC_STATS_H + + #ifdef __cplusplus +extern "C" { +#endif + +#include "_remote_debugging.h" + +static int +read_gc_stats(struct gc_stats *stats, unsigned long iid, PyObject *result) +{ +#define ADD_LOCAL_ULONG(name) do { \ + val = PyLong_FromUnsignedLong(name); \ + if (!val || PyDict_SetItemString(item, #name, val) < 0) { \ + goto error; \ + } \ + Py_DECREF(val); \ +} while(0) + +#define ADD_STATS_SSIZE(name) do { \ + val = PyLong_FromSsize_t(stats_item->name); \ + if (!val || PyDict_SetItemString(item, #name, val) < 0) { \ + goto error; \ + } \ + Py_DECREF(val); \ +} while(0) + +#define ADD_STATS_INT64(name) do { \ + val = PyLong_FromInt64(stats_item->name); \ + if (!val || PyDict_SetItemString(item, #name, val) < 0) { \ + goto error; \ + } \ + Py_DECREF(val); \ +} while(0) + +#define ADD_STATS_DOUBLE(name) do { \ + val = PyFloat_FromDouble(stats_item->name); \ + if (!val || PyDict_SetItemString(item, #name, val) < 0) { \ + goto error; \ + } \ + Py_DECREF(val); \ +} while(0) + + PyObject *item = NULL; + PyObject *val = NULL; + + for(unsigned long gen = 0; gen < NUM_GENERATIONS; gen++) { + struct gc_generation_stats *items; + int size; + if (gen == 0) { + items = (struct gc_generation_stats *)stats->young.items; + size = GC_YOUNG_STATS_SIZE; + } + else { + items = (struct gc_generation_stats *)stats->old[gen-1].items; + size = GC_OLD_STATS_SIZE; + } + for(int i = 0; i < size; i++, items++) { + struct gc_generation_stats *stats_item = items; + item = PyDict_New(); + if (item == NULL) { + goto error; + } + + ADD_LOCAL_ULONG(gen); + ADD_LOCAL_ULONG(iid); + + ADD_STATS_INT64(ts_start); + ADD_STATS_INT64(ts_stop); + ADD_STATS_SSIZE(heap_size); + ADD_STATS_SSIZE(work_to_do); + ADD_STATS_SSIZE(collections); + ADD_STATS_SSIZE(object_visits); + ADD_STATS_SSIZE(collected); + ADD_STATS_SSIZE(uncollectable); + ADD_STATS_SSIZE(candidates); + ADD_STATS_SSIZE(objects_transitively_reachable); + ADD_STATS_SSIZE(objects_not_transitively_reachable); + + ADD_STATS_DOUBLE(duration); + val = NULL; + + int rc = PyList_Append(result, item); + Py_CLEAR(item); + if (rc < 0) { + goto error; + } + } + } + +#undef ADD_LOCAL_ULONG +#undef ADD_STATS_SSIZE +#undef ADD_STATS_INT64 +#undef ADD_STATS_DOUBLE + + return 0; + +error: + Py_XDECREF(val); + Py_XDECREF(item); + + return -1; +} + +static int +get_gc_stats_from_interpreter_state(RuntimeOffsets *offsets, + uintptr_t interpreter_state_addr, + unsigned long iid, + void *context) +{ + struct gc_stats stats; + uintptr_t gc_stats_address = interpreter_state_addr + + offsets->debug_offsets.interpreter_state.gc + + offsets->debug_offsets.gc.generation_stats; + uint64_t gc_stats_size = offsets->debug_offsets.gc.generation_stats_size; + if (_Py_RemoteDebug_ReadRemoteMemory(&offsets->handle, + gc_stats_address, + gc_stats_size, + &stats) < 0) { + PyErr_SetString(PyExc_RuntimeError, "Failed to read GC state"); + return -1; + } + + PyObject *result = context; + if (read_gc_stats(&stats, iid, result) < 0) { + return -1; + } + + return 0; +} + +#ifdef __cplusplus +} +#endif + +#endif /* Py_REMOTE_DEBUGGING_GC_STATS_H */ diff --git a/Modules/_remote_debugging/interpreters.c b/Modules/_remote_debugging/interpreters.c new file mode 100644 index 00000000000000..f48d1870a61831 --- /dev/null +++ b/Modules/_remote_debugging/interpreters.c @@ -0,0 +1,82 @@ +/****************************************************************************** + * Remote Debugging Module - Interpreters Functions + * + * This file contains function for iterating interpreters. + ******************************************************************************/ + +#include "_remote_debugging.h" + +#ifndef MS_WINDOWS +#include +#endif + +#ifdef __linux__ +#include +#include +#include +#endif + +/* ============================================================================ + * INTERPRETERS ITERATION FUNCTION + * ============================================================================ */ + +int +iterate_interpreters( + RuntimeOffsets *offsets, + interpreter_processor_func processor, + void *context +) { + + uintptr_t interpreter_state_list_head = + (uintptr_t)offsets->debug_offsets.runtime_state.interpreters_head; + uintptr_t interpreter_state_offset = + offsets->runtime_start_address + interpreter_state_list_head; + uintptr_t interpreter_id_offset = + (uintptr_t)offsets->debug_offsets.interpreter_state.id; + uintptr_t interpreter_next_offset = + (uintptr_t)offsets->debug_offsets.interpreter_state.next; + + unsigned long iid = 0; + uintptr_t interpreter_state_addr; + if (_Py_RemoteDebug_ReadRemoteMemory(&offsets->handle, + interpreter_state_offset, + sizeof(void*), + &interpreter_state_addr) < 0) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to read interpreter state address"); + return -1; + } + + if (interpreter_state_addr == 0) { + _set_debug_exception_cause(PyExc_RuntimeError, "No interpreter state found"); + return -1; + } + + while (interpreter_state_addr != 0) { + + if (0 > _Py_RemoteDebug_ReadRemoteMemory( + &offsets->handle, + interpreter_state_addr + interpreter_id_offset, + sizeof(iid), + &iid)) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to read next interpreter state"); + return -1; + } + + + // Call the processor function for this interpreter + if (processor(offsets, interpreter_state_addr, iid, context) < 0) { + return -1; + } + + if (0 > _Py_RemoteDebug_ReadRemoteMemory( + &offsets->handle, + interpreter_state_addr + interpreter_next_offset, + sizeof(void*), + &interpreter_state_addr)) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to read next interpreter state"); + return -1; + } + } + + return 0; +} diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index f86bbf8ce5526e..d6005c6f5c9010 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -7,6 +7,7 @@ #include "_remote_debugging.h" #include "binary_io.h" +#include "gc_stats.h" /* Forward declarations for clinic-generated code */ typedef struct { @@ -1832,10 +1833,98 @@ _remote_debugging_is_python_process_impl(PyObject *module, int pid) Py_RETURN_TRUE; } +/*[clinic input] +_remote_debugging.get_gc_stats + + pid: int + * + all_interpreters: bool = False + If True, return GC statistics from all interpreters. + If False, return only from main interpreter. + +Get garbage collector statistics from external Python process. + +Returns: + List of dicts. + dict: A dictionary containing: + - gen: + - iid: + - ts_start: + - ts_stop: + - heap_size: + - work_to_do: + - collections: + - object_visits: + - collected: + - uncollectable: + - candidates: + - objects_transitively_reachable: + - objects_not_transitively_reachable: + - duration: + +Raises: + RuntimeError: +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_get_gc_stats_impl(PyObject *module, int pid, + int all_interpreters) +/*[clinic end generated code: output=d9dce5f7add149bb input=8f05aee4d4230428]*/ +{ + RuntimeOffsets offsets; + + PyObject *result = NULL; + + if (_Py_RemoteDebug_InitProcHandle(&offsets.handle, pid) < 0) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize process handle"); + return NULL; + } + + offsets.runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&offsets.handle); + if (offsets.runtime_start_address == 0) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to get Python runtime address"); + goto error; + } + + if (_Py_RemoteDebug_ReadDebugOffsets(&offsets.handle, + &offsets.runtime_start_address, + &offsets.debug_offsets) < 0) + { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to read debug offsets"); + goto error; + } + + // Validate that the debug offsets are valid + if (validate_debug_offsets(&offsets.debug_offsets) == -1) { + _set_debug_exception_cause(PyExc_RuntimeError, "Invalid debug offsets found"); + goto error; + } + + result = PyList_New(0); + if (result == NULL) { + goto error; + } + if (0 > iterate_interpreters(&offsets, get_gc_stats_from_interpreter_state, result)) { + goto error; + } + + goto done; + +error: + Py_CLEAR(result); + +done: + _Py_RemoteDebug_ClearCache(&offsets.handle); + _Py_RemoteDebug_CleanupProcHandle(&offsets.handle); + + return result; +} + static PyMethodDef remote_debugging_methods[] = { _REMOTE_DEBUGGING_ZSTD_AVAILABLE_METHODDEF _REMOTE_DEBUGGING_GET_CHILD_PIDS_METHODDEF _REMOTE_DEBUGGING_IS_PYTHON_PROCESS_METHODDEF + _REMOTE_DEBUGGING_GET_GC_STATS_METHODDEF {NULL, NULL, 0, NULL}, }; diff --git a/PCbuild/_remote_debugging.vcxproj b/PCbuild/_remote_debugging.vcxproj index 0e86ce9f4c918c..688ac44d83d9ed 100644 --- a/PCbuild/_remote_debugging.vcxproj +++ b/PCbuild/_remote_debugging.vcxproj @@ -108,10 +108,12 @@ + + diff --git a/PCbuild/_remote_debugging.vcxproj.filters b/PCbuild/_remote_debugging.vcxproj.filters index 59d4d5c5c335fb..e3252a4eadde07 100644 --- a/PCbuild/_remote_debugging.vcxproj.filters +++ b/PCbuild/_remote_debugging.vcxproj.filters @@ -42,6 +42,9 @@ Source Files + + Source Files + @@ -50,6 +53,9 @@ Header Files + + Header Files + diff --git a/Python/gc.c b/Python/gc.c index 7bca40f6e3f58e..32392ff12ce3bc 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1435,7 +1435,6 @@ add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) memcpy(cur_stats, prev_stats, sizeof(struct gc_generation_stats)); cur_stats->ts_start = stats->ts_start; - cur_stats->ts_stop = stats->ts_stop; cur_stats->heap_size = stats->heap_size; cur_stats->work_to_do = stats->work_to_do; @@ -1449,6 +1448,7 @@ add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) cur_stats->objects_not_transitively_reachable += stats->objects_not_transitively_reachable; cur_stats->duration += stats->duration; + cur_stats->ts_stop = stats->ts_stop; } static void