Add per-size freelists. Work in progress

Author: markshannon

Include/internal/pycore_ceval.h

@@ -342,7 +342,7 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit)
 void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
 void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
 
-PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value);
+PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(PyThreadState *ts, _PyStackRef left, _PyStackRef right, double value);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_floatobject.h

@@ -31,9 +31,7 @@ struct _Py_float_runtime_state {
 };
 
 
-
-
-PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op);
+PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyThreadState *ts, PyObject *op);
 
 
 extern void _PyFloat_DebugMallocStats(FILE* out);

Include/internal/pycore_freelist.h

@@ -30,11 +30,10 @@ _Py_freelists_GET(void)
 
 // Pushes `op` to the freelist, calls `freefunc` if the freelist is full
 #define _Py_FREELIST_FREE(NAME, op, freefunc) \
-    _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
-                     Py_ ## NAME ## _MAXFREELIST, freefunc)
+    _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), freefunc)
 // Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
-#define _Py_FREELIST_PUSH(NAME, op, limit) \
-    _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
+#define _Py_FREELIST_PUSH(NAME, op) \
+    _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op))
 
 // Pops a PyObject from the freelist, returns NULL if the freelist is empty.
 #define _Py_FREELIST_POP(TYPE, NAME) \
@@ -45,26 +44,36 @@ _Py_freelists_GET(void)
 #define _Py_FREELIST_POP_MEM(NAME) \
     _PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
 
-#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
+static inline uint32_t
+_PyFreeList_Size(struct _Py_freelist *fl)
+{
+    return fl->capacity - fl->available;
+}
+
+static inline void
+_PyFreeList_Init(struct _Py_freelist *fl, uint32_t capacity)
+{
+    fl->freelist = NULL;
+    fl->capacity = fl->available = 0; // capacity;
+}
 
 static inline int
-_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
+_PyFreeList_Push(struct _Py_freelist *fl, void *obj)
 {
-    if (fl->size < maxsize && fl->size >= 0) {
+    if (fl->available != 0) {
         FT_ATOMIC_STORE_PTR_RELAXED(*(void **)obj, fl->freelist);
         fl->freelist = obj;
-        fl->size++;
+        fl->available--;
         OBJECT_STAT_INC(to_freelist);
         return 1;
     }
     return 0;
 }
 
 static inline void
-_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
-                 freefunc dofree)
+_PyFreeList_Free(struct _Py_freelist *fl, void *obj, freefunc dofree)
 {
-    if (!_PyFreeList_Push(fl, obj, maxsize)) {
+    if (!_PyFreeList_Push(fl, obj)) {
         dofree(obj);
     }
 }
@@ -74,9 +83,9 @@ _PyFreeList_PopNoStats(struct _Py_freelist *fl)
 {
     void *obj = fl->freelist;
     if (obj != NULL) {
-        assert(fl->size > 0);
+        assert(fl->capacity > 0);
         fl->freelist = *(void **)obj;
-        fl->size--;
+        fl->available++;
     }
     return obj;
 }

Include/internal/pycore_freelist_state.h

@@ -32,28 +32,10 @@ struct _Py_freelist {
     // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
     // field.
     void *freelist;
-
-    // The number of items in the free list or -1 if the free list is disabled
-    Py_ssize_t size;
-};
-
-struct _Py_freelists {
-    struct _Py_freelist floats;
-    struct _Py_freelist ints;
-    struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
-    struct _Py_freelist lists;
-    struct _Py_freelist list_iters;
-    struct _Py_freelist tuple_iters;
-    struct _Py_freelist dicts;
-    struct _Py_freelist dictkeys;
-    struct _Py_freelist slices;
-    struct _Py_freelist contexts;
-    struct _Py_freelist async_gens;
-    struct _Py_freelist async_gen_asends;
-    struct _Py_freelist futureiters;
-    struct _Py_freelist object_stack_chunks;
-    struct _Py_freelist unicode_writers;
-    struct _Py_freelist pymethodobjects;
+    // The remaining space in this freelist;
+    uint32_t available;
+    // The maximum number of items this freelist is allowed to hold
+    uint32_t capacity;
 };
 
 #ifdef __cplusplus

Include/internal/pycore_object_alloc.h

@@ -4,6 +4,7 @@
 #include "pycore_object.h"      // _PyType_HasFeature()
 #include "pycore_pystate.h"     // _PyThreadState_GET()
 #include "pycore_tstate.h"      // _PyThreadStateImpl
+#include "pycore_freelist.h"     // _PyFreeList_Pop()
 
 #ifdef __cplusplus
 extern "C" {
@@ -65,6 +66,43 @@ _PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size)
     return mem;
 }
 
+static inline PyObject *
+_PyObject_NewTstate(PyThreadState *ts, PyTypeObject *tp, size_t size)
+{
+    assert(size > 0);
+    if (size <= SMALL_REQUEST_THRESHOLD) {
+        int size_cls = (size - 1) >> ALIGNMENT_SHIFT;
+        struct _Py_freelist *fl = &ts->interp->object_state.freelists.by_size[size_cls];
+        PyObject *op = (PyObject *)_PyFreeList_Pop(fl);
+        if (op != NULL) {
+            Py_SET_TYPE(op, tp);
+            op->ob_refcnt = 1;
+            return op;
+        }
+    }
+    PyObject *op = (PyObject *) PyObject_Malloc(size);
+    if (op == NULL) {
+        return PyErr_NoMemory();
+    }
+    _PyObject_Init(op, tp);
+    return op;
+}
+
+static inline void
+_PyObject_FreeTstate(PyThreadState *ts, PyObject *obj, size_t size)
+{
+    assert(size > 0);
+    if (size <= SMALL_REQUEST_THRESHOLD) {
+        int size_cls = (size - 1) >> ALIGNMENT_SHIFT;
+        struct _Py_freelist *fl = &ts->interp->object_state.freelists.by_size[size_cls];
+        if (_PyFreeList_Push(fl, obj)) {
+            return;
+        }
+    }
+    PyObject_Free(obj);
+}
+
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_object_state.h

@@ -10,6 +10,7 @@ extern "C" {
 
 #include "pycore_freelist_state.h"  // _Py_freelists
 #include "pycore_hashtable.h"       // _Py_hashtable_t
+#include "pycore_obmalloc.h"        // NB_SMALL_SIZE_CLASSES
 
 
 /* Reference tracer state */
@@ -26,6 +27,26 @@ struct _py_object_runtime_state {
     int _not_used;
 };
 
+struct _Py_freelists {
+    struct _Py_freelist by_size[NB_SMALL_SIZE_CLASSES];
+    struct _Py_freelist floats;
+    struct _Py_freelist ints;
+    struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
+    struct _Py_freelist lists;
+    struct _Py_freelist list_iters;
+    struct _Py_freelist tuple_iters;
+    struct _Py_freelist dicts;
+    struct _Py_freelist dictkeys;
+    struct _Py_freelist slices;
+    struct _Py_freelist contexts;
+    struct _Py_freelist async_gens;
+    struct _Py_freelist async_gen_asends;
+    struct _Py_freelist futureiters;
+    struct _Py_freelist object_stack_chunks;
+    struct _Py_freelist unicode_writers;
+    struct _Py_freelist pymethodobjects;
+};
+
 struct _py_object_state {
 #if !defined(Py_GIL_DISABLED)
     struct _Py_freelists freelists;

Include/internal/pycore_stackref.h

@@ -171,6 +171,7 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
 #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
 
 extern void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct);
+extern void PyStackRef_CLOSE_SPECIALIZED_TSTATE(PyThreadState *tstate, _PyStackRef ref, destructor_tstate destruct);
 
 static inline _PyStackRef
 PyStackRef_MakeHeapSafe(_PyStackRef ref)
@@ -316,6 +317,13 @@ PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
     PyStackRef_CLOSE(ref);
 }
 
+static inline void
+PyStackRef_CLOSE_SPECIALIZED_TSTATE(PyThreadState *ts, _PyStackRef ref, destructor_tstate destruct)
+{
+    (void)destruct;
+    PyStackRef_CLOSE(ref);
+}
+
 static inline _PyStackRef
 PyStackRef_DUP(_PyStackRef stackref)
 {
@@ -575,6 +583,15 @@ PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
     }
 }
 
+static inline void
+PyStackRef_CLOSE_SPECIALIZED_TSTATE(PyThreadState *tstate, _PyStackRef ref, destructor_tstate destruct)
+{
+    assert(!PyStackRef_IsNull(ref));
+    if (PyStackRef_RefcountOnObject(ref)) {
+        Py_DECREF_MORTAL_SPECIALIZED_TSTATE(tstate, BITS_TO_PTR(ref), destruct);
+    }
+}
+
 #ifdef _WIN32
 #define PyStackRef_XCLOSE PyStackRef_CLOSE
 #else

Include/internal/pycore_tstate.h

@@ -9,7 +9,6 @@ extern "C" {
 #endif
 
 #include "pycore_brc.h"             // struct _brc_thread_state
-#include "pycore_freelist_state.h"  // struct _Py_freelists
 #include "pycore_mimalloc.h"        // struct _mimalloc_thread_state
 #include "pycore_qsbr.h"            // struct qsbr
 

Include/object.h

@@ -351,6 +351,7 @@ typedef int (*traverseproc)(PyObject *, visitproc, void *);
 
 typedef void (*freefunc)(void *);
 typedef void (*destructor)(PyObject *);
+typedef void (*destructor_tstate)(struct _ts *, PyObject *);
 typedef PyObject *(*getattrfunc)(PyObject *, char *);
 typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
 typedef int (*setattrfunc)(PyObject *, char *, PyObject *);

Include/refcount.h

@@ -420,6 +420,23 @@ static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int linen
 }
 #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) _Py_DECREF_MORTAL_SPECIALIZED(__FILE__, __LINE__, op, destruct)
 
+
+static inline void _Py_DECREF_MORTAL_SPECIALIZED_TSTATE(const char *filename, int lineno, PyThreadState *tstate, PyObject *op, destructor_tstate destruct)
+{
+    if (op->ob_refcnt <= 0) {
+        _Py_NegativeRefcount(filename, lineno, op);
+    }
+    _Py_DECREF_STAT_INC();
+    assert(!_Py_IsStaticImmortal(op));
+    if (!_Py_IsImmortal(op)) {
+        _Py_DECREF_DecRefTotal();
+    }
+    if (--op->ob_refcnt == 0) {
+        destruct(tstate, op);
+    }
+}
+#define Py_DECREF_MORTAL_SPECIALIZED_TSTATE(ts, op, destruct) _Py_DECREF_MORTAL_SPECIALIZED_TSTATE(__FILE__, __LINE__, ts, op, destruct)
+
 static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
 {
 #if SIZEOF_VOID_P > 4
@@ -464,6 +481,16 @@ static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruc
 }
 #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct)
 
+static inline void Py_DECREF_MORTAL_SPECIALIZED_TSTATE(PyThreadState *tstate, PyObject *op, destructor_tstate destruct)
+{
+    assert(!_Py_IsStaticImmortal(op));
+    _Py_DECREF_STAT_INC();
+    if (--op->ob_refcnt == 0) {
+        destruct(tstate, op);
+    }
+}
+#define Py_DECREF_MORTAL_SPECIALIZED_TSTATE(op, destruct) Py_DECREF_MORTAL_SPECIALIZED_TSTATE(_PyObject_CAST(op), destruct)
+
 static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
 {
     // Non-limited C API and limited C API for Python 3.9 and older access