Nuitka
The Python compiler
Loading...
Searching...
No Matches
allocator.h
1// Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
2
3#ifndef __NUITKA_ALLOCATOR_H__
4#define __NUITKA_ALLOCATOR_H__
5
6/* This file is included from another C file, help IDEs to still parse it on its own. */
7#ifdef __IDE_ONLY__
8#include "nuitka/prelude.h"
9#endif
10
11// For Python2.6, these assertions cannot be done easily, just disable them with dummy code.
12#if PYTHON_VERSION < 0x270 && !defined(_PyObject_GC_IS_TRACKED)
13#define _PyObject_GC_IS_TRACKED(obj) (1)
14#endif
15
16// The full API is available for Python 3.5 only
17#if PYTHON_VERSION >= 0x350 && !defined(_NUITKA_EXPERIMENTAL_DISABLE_ALLOCATORS)
18extern void *(*python_obj_malloc)(void *ctx, size_t size);
19extern void *(*python_mem_malloc)(void *ctx, size_t size);
20extern void *(*python_mem_calloc)(void *ctx, size_t nelem, size_t elsize);
21#ifndef Py_GIL_DISABLED
22extern void *(*python_mem_realloc)(void *ctx, void *ptr, size_t new_size);
23#else
24extern void (*python_mem_free)(void *ctx, void *ptr);
25#endif
26
27#if defined(Py_DEBUG)
28extern void *python_obj_ctx;
29extern void *python_mem_ctx;
30#else
31#define python_obj_ctx (NULL)
32#define python_mem_ctx (NULL)
33#endif
34
35extern void initNuitkaAllocators(void);
36
37// Our version of "PyObject_Malloc".
38NUITKA_MAY_BE_UNUSED static void *NuitkaObject_Malloc(size_t size) { return python_obj_malloc(python_obj_ctx, size); }
39
40// Our version of "PyMem_Malloc".
41NUITKA_MAY_BE_UNUSED static void *NuitkaMem_Malloc(size_t size) { return python_mem_malloc(python_mem_ctx, size); }
42
43// Our version of "PyMem_Calloc".
44NUITKA_MAY_BE_UNUSED static void *NuitkaMem_Calloc(size_t nelem, size_t elsize) {
45 return python_mem_calloc(python_mem_ctx, nelem, elsize);
46}
47
48#ifndef Py_GIL_DISABLED
49// Our version of "PyMem_Realloc".
50NUITKA_MAY_BE_UNUSED static void *NuitkaMem_Realloc(void *ptr, size_t new_size) {
51 return python_mem_realloc(python_mem_ctx, ptr, new_size);
52}
53#else
54NUITKA_MAY_BE_UNUSED static void NuitkaMem_Free(void *ptr) { python_mem_free(python_mem_ctx, ptr); }
55#endif
56
57#else
58#define NuitkaObject_Malloc(size) PyObject_MALLOC(size)
59#define NuitkaMem_Malloc(size) PyMem_MALLOC(size)
60#define NuitkaMem_Calloc(elem, elsize) PyMem_Calloc(elem, elsize)
61#ifndef Py_GIL_DISABLED
62#if defined(_WIN32)
63// On Windows, mixing different runtime DLLs can cause issues at
64// release, so we need to go through the API to get the proper
65// DLL runtime.
66#define NuitkaMem_Realloc(ptr, new_size) PyMem_Realloc(ptr, new_size)
67#else
68#define NuitkaMem_Realloc(ptr, new_size) PyMem_REALLOC(ptr, new_size)
69#endif
70#else
71#define NuitkaMem_Free(ptr) PyMem_Free(ptr)
72#endif
73#endif
74
75#if PYTHON_VERSION >= 0x380 && PYTHON_VERSION < 0x3c0
76// Need to make Py_DECREF a macro again that doesn't call an API
77static inline void _Nuitka_Py_DECREF(PyObject *ob) {
78 assert(ob != NULL && ob->ob_refcnt >= 0);
79
80 // Non-limited C API and limited C API for Python 3.9 and older access
81 // directly PyObject.ob_refcnt.
82#ifdef Py_REF_DEBUG
83 _Py_RefTotal--;
84#endif
85 if (--ob->ob_refcnt == 0) {
86 destructor dealloc = Py_TYPE(ob)->tp_dealloc;
87#ifdef Py_TRACE_REFS
88 _Py_ForgetReference(ob);
89#endif
90 (*dealloc)(ob);
91 }
92}
93
94#undef Py_DECREF
95#define Py_DECREF(ob) _Nuitka_Py_DECREF((PyObject *)(ob))
96
97// Need to make Py_XDECREF a macro again that doesn't call an API
98static inline void _Nuitka_Py_XDECREF(PyObject *ob) {
99 if (ob != NULL) {
100 assert(ob->ob_refcnt >= 0);
101
102 // Non-limited C API and limited C API for Python 3.9 and older access
103 // directly PyObject.ob_refcnt.
104#ifdef Py_REF_DEBUG
105 _Py_RefTotal--;
106#endif
107 if (--ob->ob_refcnt == 0) {
108 destructor dealloc = Py_TYPE(ob)->tp_dealloc;
109#ifdef Py_TRACE_REFS
110 _Py_ForgetReference(ob);
111#endif
112 (*dealloc)(ob);
113 }
114 }
115}
116
117#undef Py_XDECREF
118#define Py_XDECREF(ob) _Nuitka_Py_XDECREF((PyObject *)(ob))
119
120// Need to make Py_XDECREF a macro again that uses our Py_DECREF
121#undef Py_CLEAR
122#define Py_CLEAR(op) \
123 do { \
124 PyObject *_py_tmp = (PyObject *)(op); \
125 if (_py_tmp != NULL) { \
126 (op) = NULL; \
127 Py_DECREF(_py_tmp); \
128 } \
129 } while (0)
130
131#elif PYTHON_VERSION >= 0x3c0 && defined(_WIN32) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && \
132 !defined(Py_GIL_DISABLED) && !defined(_NUITKA_EXPERIMENTAL_DISABLE_PY_DECREF_OVERRIDE)
133
134#undef Py_DECREF
135#define Py_DECREF(arg) \
136 do { \
137 PyObject *op = _PyObject_CAST(arg); \
138 if (_Py_IsImmortal(op)) { \
139 break; \
140 } \
141 _Py_DECREF_STAT_INC(); \
142 if (--op->ob_refcnt == 0) { \
143 destructor dealloc = Py_TYPE(op)->tp_dealloc; \
144 (*dealloc)(op); \
145 } \
146 } while (0)
147
148#undef Py_XDECREF
149#define Py_XDECREF(arg) \
150 do { \
151 PyObject *xop = _PyObject_CAST(arg); \
152 if (xop != NULL) { \
153 Py_DECREF(xop); \
154 } \
155 } while (0)
156
157#undef Py_IS_TYPE
158#define Py_IS_TYPE(ob, type) (_PyObject_CAST(ob)->ob_type == (type))
159
160#undef _Py_DECREF_SPECIALIZED
161#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
162 do { \
163 PyObject *op = _PyObject_CAST(arg); \
164 if (_Py_IsImmortal(op)) { \
165 break; \
166 } \
167 _Py_DECREF_STAT_INC(); \
168 if (--op->ob_refcnt == 0) { \
169 destructor d = (destructor)(dealloc); \
170 d(op); \
171 } \
172 } while (0)
173#endif
174
175// Note: The value should be used >= for checking immortality.
176#if PYTHON_VERSION < 0x3e0
177#define _Py_IMMORTAL_INITIAL_REFCNT _Py_IMMORTAL_REFCNT
178#endif
179
180// For Python3.12, avoid reference management if value is known to be immortal.
181#if PYTHON_VERSION < 0x3c0
182#define Py_INCREF_IMMORTAL(value) Py_INCREF(value)
183#define Py_DECREF_IMMORTAL(value) Py_DECREF(value)
184#elif defined(_NUITKA_DEBUG_DEBUG_IMMORTAL)
185// We strive for Nuitka to never harm these, the assertion is going to fail with
186// third party extension modules.
187#define Py_INCREF_IMMORTAL(value) assert(Py_REFCNT(value) == _Py_IMMORTAL_INITIAL_REFCNT)
188#define Py_DECREF_IMMORTAL(value) assert(Py_REFCNT(value) == _Py_IMMORTAL_INITIAL_REFCNT)
189#else
190#define Py_INCREF_IMMORTAL(value)
191#define Py_DECREF_IMMORTAL(value)
192#endif
193
194// Macro introduced with Python3.9 or higher, make it generally available.
195#ifndef Py_SET_TYPE
196static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; }
197#define Py_SET_TYPE(ob, type) _Py_SET_TYPE((PyObject *)(ob), type)
198#endif
199
200// After Python 3.9 this was moved into the DLL potentially, making
201// it expensive to call.
202#if PYTHON_VERSION >= 0x390
203static inline void Nuitka_Py_NewReferenceNoTotal(PyObject *op) { Py_SET_REFCNT(op, 1); }
204static inline void Nuitka_Py_NewReference(PyObject *op) {
205#ifdef Py_REF_DEBUG
206#if PYTHON_VERSION < 0x3c0
207 _Py_RefTotal++;
208#else
209 // Refcounts are now in the interpreter state, spell-checker: ignore reftotal
210 _PyInterpreterState_GET()->object_state.reftotal++;
211#endif
212#endif
213#if !defined(Py_GIL_DISABLED)
214 op->ob_refcnt = 1;
215#else
216 op->ob_tid = _Py_ThreadId();
217 op->_padding = 0;
218 op->ob_mutex = (PyMutex){0};
219 op->ob_gc_bits = 0;
220 op->ob_ref_local = 1;
221 op->ob_ref_shared = 0;
222#endif
223}
224#else
225#define Nuitka_Py_NewReference(op) _Py_NewReference(op)
226#endif
227
228static inline int Nuitka_PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
229 return ((type->tp_flags & feature) != 0);
230}
231
232#if PYTHON_VERSION >= 0x3b0
233
234static inline size_t Nuitka_PyType_PreHeaderSize(PyTypeObject *tp) {
235 return _PyType_IS_GC(tp) * sizeof(PyGC_Head) +
236 Nuitka_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * 2 * sizeof(PyObject *);
237}
238
239extern void Nuitka_PyObject_GC_Link(PyObject *op);
240
241static PyObject *Nuitka_PyType_AllocNoTrackVar(PyTypeObject *type, Py_ssize_t nitems) {
242 // There is always a sentinel now, therefore add one
243 const size_t size = _PyObject_VAR_SIZE(type, nitems + 1);
244
245 // TODO: This ought to be static for all our types, so remove it as a call.
246 const size_t pre_size = Nuitka_PyType_PreHeaderSize(type);
247 assert(pre_size == sizeof(PyGC_Head));
248
249 char *alloc = (char *)NuitkaObject_Malloc(size + pre_size);
250 assert(alloc);
251 PyObject *obj = (PyObject *)(alloc + pre_size);
252
253 assert(pre_size);
254 if (pre_size) {
255 ((PyObject **)alloc)[0] = NULL;
256 ((PyObject **)alloc)[1] = NULL;
257
258 Nuitka_PyObject_GC_Link(obj);
259 }
260
261 // We might be able to avoid this, but it's unclear what e.g. the sentinel
262 // is supposed to be.
263 memset(obj, 0, size);
264
265 // This is the "var" branch, we already know we are variable size here.
266 assert(type->tp_itemsize != 0);
267 Py_SET_SIZE((PyVarObject *)obj, nitems);
268
269 // Initialize the object references.
270 Py_SET_TYPE(obj, type);
271 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
272 Py_INCREF(type);
273 }
274
275 Nuitka_Py_NewReference(obj);
276
277 return obj;
278}
279
280static PyObject *Nuitka_PyType_AllocNoTrack(PyTypeObject *type) {
281 // TODO: This ought to be static for all our types, so remove it as a call.
282 const size_t pre_size = Nuitka_PyType_PreHeaderSize(type);
283
284 char *alloc = (char *)NuitkaObject_Malloc(_PyObject_SIZE(type) + pre_size);
285 assert(alloc);
286 PyObject *obj = (PyObject *)(alloc + pre_size);
287
288 assert(pre_size);
289 ((PyObject **)alloc)[0] = NULL;
290 ((PyObject **)alloc)[1] = NULL;
291
292 Nuitka_PyObject_GC_Link(obj);
293
294 // Initialize the object references.
295 Py_SET_TYPE(obj, type);
296
297 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
298 Py_INCREF(type);
299 }
300
301 Nuitka_Py_NewReference(obj);
302
303 return obj;
304}
305#endif
306
307NUITKA_MAY_BE_UNUSED static void *Nuitka_GC_NewVar(PyTypeObject *type, Py_ssize_t nitems) {
308 assert(nitems >= 0);
309
310#if PYTHON_VERSION < 0x3b0
311 size_t size = _PyObject_VAR_SIZE(type, nitems);
312 PyVarObject *op = (PyVarObject *)_PyObject_GC_Malloc(size);
313 assert(op != NULL);
314
315 Py_SIZE(op) = nitems;
316 Py_SET_TYPE(op, type);
317
318#if PYTHON_VERSION >= 0x380
319 // TODO: Might have two variants, or more sure this is also false for all of our types,
320 // we are just wasting time for compiled times here.
321 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
322 Py_INCREF(type);
323 }
324#endif
325
326 Nuitka_Py_NewReference((PyObject *)op);
327
328 return op;
329#else
330 // TODO: We ought to inline this probably too, no point as a separate function.
331 PyObject *op = Nuitka_PyType_AllocNoTrackVar(type, nitems);
332#endif
333 assert(Py_SIZE(op) == nitems);
334 return op;
335}
336
337NUITKA_MAY_BE_UNUSED static void *Nuitka_GC_New(PyTypeObject *type) {
338#if PYTHON_VERSION < 0x3b0
339 size_t size = _PyObject_SIZE(type);
340
341 PyVarObject *op = (PyVarObject *)_PyObject_GC_Malloc(size);
342 assert(op != NULL);
343
344 Py_SET_TYPE(op, type);
345
346#if PYTHON_VERSION >= 0x380
347 // TODO: Might have two variants, or more sure this is also false for all of our types,
348 // we are just wasting time for compiled times here.
349 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
350 Py_INCREF(type);
351 }
352#endif
353
354 Nuitka_Py_NewReference((PyObject *)op);
355#else
356 // TODO: We ought to inline this probably too, no point as a separate function.
357 PyObject *op = Nuitka_PyType_AllocNoTrack(type);
358#endif
359 return op;
360}
361
362static bool inline Nuitka_GC_IS_TRACKED_X(PyObject *object) {
363 return object == NULL || _PyObject_GC_IS_TRACKED(object);
364}
365
366// To allow us marking some of our own values as immortal.
367#if PYTHON_VERSION >= 0x3c0
368static void inline Py_SET_REFCNT_IMMORTAL(PyObject *object) {
369 assert(object != NULL);
370
371 // Normally done only with 3.13, but it makes sense to do this.
372 if (_PyObject_IS_GC(object) && _PyObject_GC_IS_TRACKED(object)) {
373 Nuitka_GC_UnTrack(object);
374 }
375
376#ifdef Py_GIL_DISABLED
377 object->ob_tid = _Py_UNOWNED_TID;
378 object->ob_ref_local = _Py_IMMORTAL_INITIAL_REFCNT;
379 object->ob_ref_shared = 0;
380#else
381 object->ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT;
382#endif
383}
384#else
385#define Py_SET_REFCNT_IMMORTAL(object)
386#endif
387
388// Have these defines from newer Python for all Python versions available
389#ifndef _Py_CAST
390#define _Py_CAST(type, expr) ((type)(expr))
391#endif
392
393#ifndef _PyObject_CAST
394#define _PyObject_CAST(op) _Py_CAST(PyObject *, (op))
395#endif
396
397#ifndef Py_SETREF
398#ifdef _Py_TYPEOF
399#define Py_SETREF(dst, src) \
400 do { \
401 _Py_TYPEOF(dst) *_tmp_dst_ptr = &(dst); \
402 _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
403 *_tmp_dst_ptr = (src); \
404 Py_DECREF(_tmp_old_dst); \
405 } while (0)
406#else
407#define Py_SETREF(dst, src) \
408 do { \
409 PyObject **_tmp_dst_ptr = _Py_CAST(PyObject **, &(dst)); \
410 PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
411 PyObject *_tmp_src = _PyObject_CAST(src); \
412 memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject *)); \
413 Py_DECREF(_tmp_old_dst); \
414 } while (0)
415#endif
416#endif
417
418#ifndef Py_XSETREF
419/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of
420 * Py_DECREF().
421 */
422#ifdef _Py_TYPEOF
423#define Py_XSETREF(dst, src) \
424 do { \
425 _Py_TYPEOF(dst) *_tmp_dst_ptr = &(dst); \
426 _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
427 *_tmp_dst_ptr = (src); \
428 Py_XDECREF(_tmp_old_dst); \
429 } while (0)
430#else
431#define Py_XSETREF(dst, src) \
432 do { \
433 PyObject **_tmp_dst_ptr = _Py_CAST(PyObject **, &(dst)); \
434 PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
435 PyObject *_tmp_src = _PyObject_CAST(src); \
436 memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject *)); \
437 Py_XDECREF(_tmp_old_dst); \
438 } while (0)
439#endif
440#endif
441
442#endif
443
444// Part of "Nuitka", an optimizing Python compiler that is compatible and
445// integrates with CPython, but also works on its own.
446//
447// Licensed under the Apache License, Version 2.0 (the "License");
448// you may not use this file except in compliance with the License.
449// You may obtain a copy of the License at
450//
451// http://www.apache.org/licenses/LICENSE-2.0
452//
453// Unless required by applicable law or agreed to in writing, software
454// distributed under the License is distributed on an "AS IS" BASIS,
455// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
456// See the License for the specific language governing permissions and
457// limitations under the License.