Nuitka
The Python compiler
Loading...
Searching...
No Matches
allocator.h
1// Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
2
3#ifndef __NUITKA_ALLOCATOR_H__
4#define __NUITKA_ALLOCATOR_H__
5
6/* This file is included from another C file, help IDEs to still parse it on its own. */
7#ifdef __IDE_ONLY__
8#include "nuitka/prelude.h"
9#endif
10
11// For Python2.6, these assertions cannot be done easily, just disable them with dummy code.
12#if PYTHON_VERSION < 0x270 && !defined(_PyObject_GC_IS_TRACKED)
13#define _PyObject_GC_IS_TRACKED(obj) (1)
14#endif
15
16// The full API is available for Python 3.5 only
17#if PYTHON_VERSION >= 0x350 && !defined(_NUITKA_EXPERIMENTAL_DISABLE_ALLOCATORS)
18extern void *(*python_obj_malloc)(void *ctx, size_t size);
19extern void *(*python_mem_malloc)(void *ctx, size_t size);
20extern void *(*python_mem_calloc)(void *ctx, size_t nelem, size_t elsize);
21#ifndef Py_GIL_DISABLED
22extern void *(*python_mem_realloc)(void *ctx, void *ptr, size_t new_size);
23#else
24extern void (*python_mem_free)(void *ctx, void *ptr);
25#endif
26
27#if defined(Py_DEBUG)
28extern void *python_obj_ctx;
29extern void *python_mem_ctx;
30#else
31#define python_obj_ctx (NULL)
32#define python_mem_ctx (NULL)
33#endif
34
35extern void initNuitkaAllocators(void);
36
37// Our version of "PyObject_Malloc".
38NUITKA_MAY_BE_UNUSED static void *NuitkaObject_Malloc(size_t size) { return python_obj_malloc(python_obj_ctx, size); }
39
40// Our version of "PyMem_Malloc".
41NUITKA_MAY_BE_UNUSED static void *NuitkaMem_Malloc(size_t size) { return python_mem_malloc(python_mem_ctx, size); }
42
43// Our version of "PyMem_Calloc".
44NUITKA_MAY_BE_UNUSED static void *NuitkaMem_Calloc(size_t nelem, size_t elsize) {
45 return python_mem_calloc(python_mem_ctx, nelem, elsize);
46}
47
48#ifndef Py_GIL_DISABLED
49// Our version of "PyMem_Realloc".
50NUITKA_MAY_BE_UNUSED static void *NuitkaMem_Realloc(void *ptr, size_t new_size) {
51 return python_mem_realloc(python_mem_ctx, ptr, new_size);
52}
53#else
54NUITKA_MAY_BE_UNUSED static void NuitkaMem_Free(void *ptr) { python_mem_free(python_mem_ctx, ptr); }
55#endif
56
57#else
58#define NuitkaObject_Malloc(size) PyObject_MALLOC(size)
59#define NuitkaMem_Malloc(size) PyMem_MALLOC(size)
60#define NuitkaMem_Calloc(elem, elsize) PyMem_Calloc(elem, elsize)
61#ifndef Py_GIL_DISABLED
62#if defined(_WIN32)
63// On Windows, mixing different runtime DLLs can cause issues at
64// release, so we need to go through the API to get the proper
65// DLL runtime.
66#define NuitkaMem_Realloc(ptr, new_size) PyMem_Realloc(ptr, new_size)
67#else
68#define NuitkaMem_Realloc(ptr, new_size) PyMem_REALLOC(ptr, new_size)
69#endif
70#else
71#define NuitkaMem_Free(ptr) PyMem_Free(ptr)
72#endif
73#endif
74
75#if PYTHON_VERSION >= 0x380 && PYTHON_VERSION < 0x3c0
76// Need to make Py_DECREF a macro again that doesn't call an API
77static inline void _Nuitka_Py_DECREF(PyObject *ob) {
78 assert(ob != NULL && ob->ob_refcnt >= 0);
79
80 // Non-limited C API and limited C API for Python 3.9 and older access
81 // directly PyObject.ob_refcnt.
82#ifdef Py_REF_DEBUG
83 _Py_RefTotal--;
84#endif
85 if (--ob->ob_refcnt == 0) {
86 destructor dealloc = Py_TYPE(ob)->tp_dealloc;
87#ifdef Py_TRACE_REFS
88 _Py_ForgetReference(ob);
89#endif
90 (*dealloc)(ob);
91 }
92}
93
94#undef Py_DECREF
95#define Py_DECREF(ob) _Nuitka_Py_DECREF((PyObject *)(ob))
96
97// Need to make Py_XDECREF a macro again that doesn't call an API
98static inline void _Nuitka_Py_XDECREF(PyObject *ob) {
99 if (ob != NULL) {
100 assert(ob->ob_refcnt >= 0);
101
102 // Non-limited C API and limited C API for Python 3.9 and older access
103 // directly PyObject.ob_refcnt.
104#ifdef Py_REF_DEBUG
105 _Py_RefTotal--;
106#endif
107 if (--ob->ob_refcnt == 0) {
108 destructor dealloc = Py_TYPE(ob)->tp_dealloc;
109#ifdef Py_TRACE_REFS
110 _Py_ForgetReference(ob);
111#endif
112 (*dealloc)(ob);
113 }
114 }
115}
116
117#undef Py_XDECREF
118#define Py_XDECREF(ob) _Nuitka_Py_XDECREF((PyObject *)(ob))
119
120// Need to make Py_XDECREF a macro again that uses our Py_DECREF
121#undef Py_CLEAR
122#define Py_CLEAR(op) \
123 do { \
124 PyObject *_py_tmp = (PyObject *)(op); \
125 if (_py_tmp != NULL) { \
126 (op) = NULL; \
127 Py_DECREF(_py_tmp); \
128 } \
129 } while (0)
130
131#elif PYTHON_VERSION >= 0x3c0 && defined(_WIN32) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && \
132 !defined(Py_GIL_DISABLED) && !defined(_NUITKA_EXPERIMENTAL_DISABLE_PY_DECREF_OVERRIDE)
133
134#undef Py_DECREF
135#define Py_DECREF(arg) \
136 do { \
137 PyObject *op = _PyObject_CAST(arg); \
138 if (_Py_IsImmortal(op)) { \
139 break; \
140 } \
141 _Py_DECREF_STAT_INC(); \
142 if (--op->ob_refcnt == 0) { \
143 destructor dealloc = Py_TYPE(op)->tp_dealloc; \
144 (*dealloc)(op); \
145 } \
146 } while (0)
147
148#undef Py_XDECREF
149#define Py_XDECREF(arg) \
150 do { \
151 PyObject *xop = _PyObject_CAST(arg); \
152 if (xop != NULL) { \
153 Py_DECREF(xop); \
154 } \
155 } while (0)
156
157#undef Py_IS_TYPE
158#define Py_IS_TYPE(ob, type) (_PyObject_CAST(ob)->ob_type == (type))
159
160#undef _Py_DECREF_SPECIALIZED
161#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
162 do { \
163 PyObject *op = _PyObject_CAST(arg); \
164 if (_Py_IsImmortal(op)) { \
165 break; \
166 } \
167 _Py_DECREF_STAT_INC(); \
168 if (--op->ob_refcnt == 0) { \
169 destructor d = (destructor)(dealloc); \
170 d(op); \
171 } \
172 } while (0)
173#endif
174
175// For Python3.12, avoid reference management if value is known to be immortal.
176#if PYTHON_VERSION < 0x3c0
177#define Py_INCREF_IMMORTAL(value) Py_INCREF(value)
178#define Py_DECREF_IMMORTAL(value) Py_DECREF(value)
179#elif defined(_NUITKA_DEBUG_DEBUG_IMMORTAL)
180#define Py_INCREF_IMMORTAL(value) assert(Py_REFCNT(value) == _Py_IMMORTAL_REFCNT)
181#define Py_DECREF_IMMORTAL(value) assert(Py_REFCNT(value) == _Py_IMMORTAL_REFCNT)
182#else
183#define Py_INCREF_IMMORTAL(value)
184#define Py_DECREF_IMMORTAL(value)
185#endif
186
187// Macro introduced with Python3.9 or higher, make it generally available.
188#ifndef Py_SET_TYPE
189static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; }
190#define Py_SET_TYPE(ob, type) _Py_SET_TYPE((PyObject *)(ob), type)
191#endif
192
193// After Python 3.9 this was moved into the DLL potentially, making
194// it expensive to call.
195#if PYTHON_VERSION >= 0x390
196static inline void Nuitka_Py_NewReferenceNoTotal(PyObject *op) { Py_SET_REFCNT(op, 1); }
197static inline void Nuitka_Py_NewReference(PyObject *op) {
198#ifdef Py_REF_DEBUG
199#if PYTHON_VERSION < 0x3c0
200 _Py_RefTotal++;
201#else
202 // Refcounts are now in the interpreter state, spell-checker: ignore reftotal
203 _PyInterpreterState_GET()->object_state.reftotal++;
204#endif
205#endif
206#if !defined(Py_GIL_DISABLED)
207 op->ob_refcnt = 1;
208#else
209 op->ob_tid = _Py_ThreadId();
210 op->_padding = 0;
211 op->ob_mutex = (PyMutex){0};
212 op->ob_gc_bits = 0;
213 op->ob_ref_local = 1;
214 op->ob_ref_shared = 0;
215#endif
216}
217#else
218#define Nuitka_Py_NewReference(op) _Py_NewReference(op)
219#endif
220
221static inline int Nuitka_PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
222 return ((type->tp_flags & feature) != 0);
223}
224
225#if PYTHON_VERSION >= 0x3b0
226
227static inline size_t Nuitka_PyType_PreHeaderSize(PyTypeObject *tp) {
228 return _PyType_IS_GC(tp) * sizeof(PyGC_Head) +
229 Nuitka_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * 2 * sizeof(PyObject *);
230}
231
232extern void Nuitka_PyObject_GC_Link(PyObject *op);
233
234static PyObject *Nuitka_PyType_AllocNoTrackVar(PyTypeObject *type, Py_ssize_t nitems) {
235 // There is always a sentinel now, therefore add one
236 const size_t size = _PyObject_VAR_SIZE(type, nitems + 1);
237
238 // TODO: This ought to be static for all our types, so remove it as a call.
239 const size_t pre_size = Nuitka_PyType_PreHeaderSize(type);
240 assert(pre_size == sizeof(PyGC_Head));
241
242 char *alloc = (char *)NuitkaObject_Malloc(size + pre_size);
243 assert(alloc);
244 PyObject *obj = (PyObject *)(alloc + pre_size);
245
246 assert(pre_size);
247 if (pre_size) {
248 ((PyObject **)alloc)[0] = NULL;
249 ((PyObject **)alloc)[1] = NULL;
250
251 Nuitka_PyObject_GC_Link(obj);
252 }
253
254 // We might be able to avoid this, but it's unclear what e.g. the sentinel
255 // is supposed to be.
256 memset(obj, 0, size);
257
258 // This is the "var" branch, we already know we are variable size here.
259 assert(type->tp_itemsize != 0);
260 Py_SET_SIZE((PyVarObject *)obj, nitems);
261
262 // Initialize the object references.
263 Py_SET_TYPE(obj, type);
264 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
265 Py_INCREF(type);
266 }
267
268 Nuitka_Py_NewReference(obj);
269
270 return obj;
271}
272
273static PyObject *Nuitka_PyType_AllocNoTrack(PyTypeObject *type) {
274 // TODO: This ought to be static for all our types, so remove it as a call.
275 const size_t pre_size = Nuitka_PyType_PreHeaderSize(type);
276
277 char *alloc = (char *)NuitkaObject_Malloc(_PyObject_SIZE(type) + pre_size);
278 assert(alloc);
279 PyObject *obj = (PyObject *)(alloc + pre_size);
280
281 assert(pre_size);
282 ((PyObject **)alloc)[0] = NULL;
283 ((PyObject **)alloc)[1] = NULL;
284
285 Nuitka_PyObject_GC_Link(obj);
286
287 // Initialize the object references.
288 Py_SET_TYPE(obj, type);
289
290 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
291 Py_INCREF(type);
292 }
293
294 Nuitka_Py_NewReference(obj);
295
296 return obj;
297}
298#endif
299
300NUITKA_MAY_BE_UNUSED static void *Nuitka_GC_NewVar(PyTypeObject *type, Py_ssize_t nitems) {
301 assert(nitems >= 0);
302
303#if PYTHON_VERSION < 0x3b0
304 size_t size = _PyObject_VAR_SIZE(type, nitems);
305 PyVarObject *op = (PyVarObject *)_PyObject_GC_Malloc(size);
306 assert(op != NULL);
307
308 Py_SIZE(op) = nitems;
309 Py_SET_TYPE(op, type);
310
311#if PYTHON_VERSION >= 0x380
312 // TODO: Might have two variants, or more sure this is also false for all of our types,
313 // we are just wasting time for compiled times here.
314 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
315 Py_INCREF(type);
316 }
317#endif
318
319 Nuitka_Py_NewReference((PyObject *)op);
320
321 return op;
322#else
323 // TODO: We ought to inline this probably too, no point as a separate function.
324 PyObject *op = Nuitka_PyType_AllocNoTrackVar(type, nitems);
325#endif
326 assert(Py_SIZE(op) == nitems);
327 return op;
328}
329
330NUITKA_MAY_BE_UNUSED static void *Nuitka_GC_New(PyTypeObject *type) {
331#if PYTHON_VERSION < 0x3b0
332 size_t size = _PyObject_SIZE(type);
333
334 PyVarObject *op = (PyVarObject *)_PyObject_GC_Malloc(size);
335 assert(op != NULL);
336
337 Py_SET_TYPE(op, type);
338
339#if PYTHON_VERSION >= 0x380
340 // TODO: Might have two variants, or more sure this is also false for all of our types,
341 // we are just wasting time for compiled times here.
342 if (Nuitka_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
343 Py_INCREF(type);
344 }
345#endif
346
347 Nuitka_Py_NewReference((PyObject *)op);
348#else
349 // TODO: We ought to inline this probably too, no point as a separate function.
350 PyObject *op = Nuitka_PyType_AllocNoTrack(type);
351#endif
352 return op;
353}
354
355static bool inline Nuitka_GC_IS_TRACKED_X(PyObject *object) {
356 return object == NULL || _PyObject_GC_IS_TRACKED(object);
357}
358
359// To allow us marking some of our own values as immortal.
360#if PYTHON_VERSION >= 0x3c0
361static void inline Py_SET_REFCNT_IMMORTAL(PyObject *object) {
362 assert(object != NULL);
363
364 // Normally done only with 3.13, but it makes sense to do this.
365 if (_PyObject_IS_GC(object) && _PyObject_GC_IS_TRACKED(object)) {
366 Nuitka_GC_UnTrack(object);
367 }
368
369#ifdef Py_GIL_DISABLED
370 object->ob_tid = _Py_UNOWNED_TID;
371 object->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL;
372 object->ob_ref_shared = 0;
373#else
374 object->ob_refcnt = _Py_IMMORTAL_REFCNT;
375#endif
376}
377#else
378#define Py_SET_REFCNT_IMMORTAL(object)
379#endif
380
381// Have these defines from newer Python for all Python versions available
382#ifndef _Py_CAST
383#define _Py_CAST(type, expr) ((type)(expr))
384#endif
385
386#ifndef _PyObject_CAST
387#define _PyObject_CAST(op) _Py_CAST(PyObject *, (op))
388#endif
389
390#ifndef Py_SETREF
391#ifdef _Py_TYPEOF
392#define Py_SETREF(dst, src) \
393 do { \
394 _Py_TYPEOF(dst) *_tmp_dst_ptr = &(dst); \
395 _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
396 *_tmp_dst_ptr = (src); \
397 Py_DECREF(_tmp_old_dst); \
398 } while (0)
399#else
400#define Py_SETREF(dst, src) \
401 do { \
402 PyObject **_tmp_dst_ptr = _Py_CAST(PyObject **, &(dst)); \
403 PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
404 PyObject *_tmp_src = _PyObject_CAST(src); \
405 memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject *)); \
406 Py_DECREF(_tmp_old_dst); \
407 } while (0)
408#endif
409#endif
410
411#ifndef Py_XSETREF
412/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of
413 * Py_DECREF().
414 */
415#ifdef _Py_TYPEOF
416#define Py_XSETREF(dst, src) \
417 do { \
418 _Py_TYPEOF(dst) *_tmp_dst_ptr = &(dst); \
419 _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
420 *_tmp_dst_ptr = (src); \
421 Py_XDECREF(_tmp_old_dst); \
422 } while (0)
423#else
424#define Py_XSETREF(dst, src) \
425 do { \
426 PyObject **_tmp_dst_ptr = _Py_CAST(PyObject **, &(dst)); \
427 PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
428 PyObject *_tmp_src = _PyObject_CAST(src); \
429 memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject *)); \
430 Py_XDECREF(_tmp_old_dst); \
431 } while (0)
432#endif
433#endif
434
435#endif
436
437// Part of "Nuitka", an optimizing Python compiler that is compatible and
438// integrates with CPython, but also works on its own.
439//
440// Licensed under the Apache License, Version 2.0 (the "License");
441// you may not use this file except in compliance with the License.
442// You may obtain a copy of the License at
443//
444// http://www.apache.org/licenses/LICENSE-2.0
445//
446// Unless required by applicable law or agreed to in writing, software
447// distributed under the License is distributed on an "AS IS" BASIS,
448// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
449// See the License for the specific language governing permissions and
450// limitations under the License.