Nuitka
The Python compiler
Loading...
Searching...
No Matches
MetaPathBasedLoader.c
1// Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
2
3// This implements the loading of C compiled modules and shared library
4// extension modules bundled for standalone mode.
5
6// This is achieved mainly by registered a "sys.meta_path" loader, that then
7// gets asked for module names, and responds if knows about one. It's fed by
8// a table created at compile time.
9
10// The nature and use of these 2 loaded module kinds is very different, but
11// having them as distinct loaders would only require to duplicate the search
12// and registering of stuff.
13
14// This file is included from another C file, help IDEs to still parse it on
15// its own.
16#ifdef __IDE_ONLY__
17#include "nuitka/prelude.h"
18#endif
19
20#include "nuitka/unfreezing.h"
21
22#ifdef _WIN32
23#include <windows.h>
24#endif
25
26extern PyTypeObject Nuitka_Loader_Type;
27
28struct Nuitka_LoaderObject {
29 /* Python object folklore: */
30 PyObject_HEAD
31
32 /* The loader entry, to know what was loaded exactly. */
33 struct Nuitka_MetaPathBasedLoaderEntry const *m_loader_entry;
34};
35
36#if _NUITKA_EXE_MODE
37static inline bool isVerbose(void) { return Py_VerboseFlag != 0; }
38#elif _NUITKA_SYSFLAG_VERBOSE
39static inline bool isVerbose(void) { return true; }
40#else
41static inline bool isVerbose(void) { return false; }
42#endif
43
44static struct Nuitka_MetaPathBasedLoaderEntry *loader_entries = NULL;
45
46static bool hasFrozenModule(char const *name) {
47 for (struct _frozen const *p = PyImport_FrozenModules; p != NULL; p++) {
48 if (p->name == NULL) {
49 return false;
50 }
51
52 if (strcmp(p->name, name) == 0) {
53 break;
54 }
55 }
56
57 return true;
58}
59
60static char *appendModuleNameAsPath(char *buffer, char const *module_name, size_t buffer_size) {
61 // Skip to the end
62 while (*buffer != 0) {
63 buffer++;
64 buffer_size -= 1;
65 }
66
67 while (*module_name) {
68 if (buffer_size < 1) {
69 abort();
70 }
71
72 if (*module_name == '.') {
73 *buffer++ = SEP;
74 module_name++;
75 } else {
76 *buffer++ = *module_name++;
77 }
78
79 buffer_size -= 1;
80 }
81
82 *buffer = 0;
83
84 return buffer;
85}
86
87#if defined(_WIN32) && _NUITKA_STANDALONE_MODE
88
89static void appendModuleNameAsPathW(wchar_t *buffer, PyObject *module_name, size_t buffer_size) {
90 Py_ssize_t size;
91 wchar_t const *module_name_wstr = Nuitka_UnicodeAsWideString(module_name, &size);
92
93 while (size > 0) {
94 wchar_t c = *module_name_wstr++;
95 size -= 1;
96
97 if (c == L'.') {
98 c = FILENAME_SEP_CHAR;
99 }
100
101 appendWCharSafeW(buffer, c, buffer_size);
102 }
103}
104#endif
105
106// TODO: This updates the wrong absolute path. We ought to change it to
107// the "module_path_name" at the time of writing it, then we save a few
108// bytes in the blob, and don't have to create that string here.
109#if _NUITKA_STANDALONE_MODE
110static void patchCodeObjectPaths(PyCodeObject *code_object, PyObject *module_path) {
111 code_object->co_filename = module_path;
112 Py_INCREF(module_path);
113
114 Py_ssize_t consts_count = PyTuple_GET_SIZE(code_object->co_consts);
115
116 for (Py_ssize_t i = 0; i < consts_count; i++) {
117 PyObject *constant = PyTuple_GET_ITEM(code_object->co_consts, i);
118
119 if (PyCode_Check(constant)) {
120 patchCodeObjectPaths((PyCodeObject *)constant, module_path);
121 }
122 }
123}
124#endif
125
126NUITKA_MAY_BE_UNUSED static PyObject *MAKE_RELATIVE_PATH_FROM_NAME(char const *name, bool is_package, bool dir_only) {
127 char buffer[MAXPATHLEN + 1] = {0};
128
129 appendModuleNameAsPath(buffer, name, sizeof(buffer));
130
131 if (dir_only == false) {
132 if (is_package) {
133 appendCharSafe(buffer, SEP, sizeof(buffer));
134 appendStringSafe(buffer, "__init__.py", sizeof(buffer));
135 } else {
136 appendStringSafe(buffer, ".py", sizeof(buffer));
137 }
138 } else {
139 if (is_package == false) {
140 char *sep = strrchr(buffer, SEP);
141 if (sep) {
142 *sep = 0;
143 } else {
144 buffer[0] = '.';
145 buffer[1] = 0;
146 }
147 }
148 }
149
150 PyObject *module_path_entry_base = Nuitka_String_FromString(buffer);
151
152 PyObject *result = MAKE_RELATIVE_PATH(module_path_entry_base);
153
154 Py_DECREF(module_path_entry_base);
155
156 return result;
157}
158
159static PyObject *_makeDunderPathObject(PyThreadState *tstate, PyObject *module_path_entry) {
160 CHECK_OBJECT(module_path_entry);
161
162 PyObject *path_list = MAKE_LIST_EMPTY(tstate, 1);
163 if (unlikely(path_list == NULL)) {
164 return NULL;
165 }
166
167 PyList_SET_ITEM0(path_list, 0, module_path_entry);
168
169 CHECK_OBJECT(path_list);
170 return path_list;
171}
172
173static PyObject *loadModuleFromCodeObject(PyThreadState *tstate, PyObject *module, PyCodeObject *code_object,
174 char const *name, bool is_package) {
175 assert(code_object != NULL);
176
177 {
178 NUITKA_MAY_BE_UNUSED bool b_res = Nuitka_SetModuleString(name, module);
179 assert(b_res != false);
180 }
181
182 char buffer[MAXPATHLEN + 1] = {0};
183
184 PyObject *module_path_entry = NULL;
185
186 if (is_package) {
187 appendModuleNameAsPath(buffer, name, sizeof(buffer));
188 PyObject *module_path_entry_base = Nuitka_String_FromString(buffer);
189
190 module_path_entry = MAKE_RELATIVE_PATH(module_path_entry_base);
191 Py_DECREF(module_path_entry_base);
192
193 appendCharSafe(buffer, SEP, sizeof(buffer));
194 appendStringSafe(buffer, "__init__.py", sizeof(buffer));
195 } else {
196 appendModuleNameAsPath(buffer, name, sizeof(buffer));
197 appendStringSafe(buffer, ".py", sizeof(buffer));
198 }
199
200 PyObject *module_path_name = Nuitka_String_FromString(buffer);
201
202 PyObject *module_path = MAKE_RELATIVE_PATH(module_path_name);
203 Py_DECREF(module_path_name);
204
205 if (is_package) {
206 /* Set __path__ properly, unlike frozen module importer does. */
207 PyObject *path_list = _makeDunderPathObject(tstate, module_path_entry);
208
209 int res = PyObject_SetAttr(module, const_str_plain___path__, path_list);
210 if (unlikely(res != 0)) {
211 return NULL;
212 }
213
214 Py_DECREF(path_list);
215
216 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___name__);
217 CHECK_OBJECT(module_name);
218
219 res = PyObject_SetAttr(module, const_str_plain___package__, module_name);
220
221 if (unlikely(res != 0)) {
222 return NULL;
223 }
224 }
225
226#if _NUITKA_STANDALONE_MODE
227 patchCodeObjectPaths(code_object, module_path);
228#endif
229
230 PGO_onModuleEntered(name);
231 module = PyImport_ExecCodeModuleEx((char *)name, (PyObject *)code_object, Nuitka_String_AsString(module_path));
232 PGO_onModuleExit(name, module == NULL);
233
234 Py_DECREF(module_path);
235
236 return module;
237}
238
239static struct Nuitka_MetaPathBasedLoaderEntry *findEntry(char const *name) {
240 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
241 assert(current);
242
243 while (current->name != NULL) {
244 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
245 current->name = UN_TRANSLATE(current->name);
246 current->flags -= NUITKA_TRANSLATED_FLAG;
247 }
248
249 if (strcmp(name, current->name) == 0) {
250 return current;
251 }
252
253 current++;
254 }
255
256 return NULL;
257}
258
259#if !_NUITKA_STANDALONE_MODE
260static struct Nuitka_MetaPathBasedLoaderEntry *findContainingPackageEntry(char const *name) {
261 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
262
263 // Consider the package name of the searched entry.
264 char const *package_name_end = strrchr(name, '.');
265 if (package_name_end == NULL) {
266 return NULL;
267 }
268
269 size_t length = package_name_end - name;
270
271 while (current->name != NULL) {
272 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
273 current->name = UN_TRANSLATE(current->name);
274 current->flags -= NUITKA_TRANSLATED_FLAG;
275 }
276
277 if ((current->flags & NUITKA_PACKAGE_FLAG) != 0) {
278 if (strlen(current->name) == length && strncmp(name, current->name, length) == 0) {
279 return current;
280 }
281 }
282
283 current++;
284 }
285
286 return NULL;
287}
288
289static PyObject *_getFileList(PyThreadState *tstate, PyObject *dirname) {
290 static PyObject *listdir_func = NULL;
291
292 // TODO: Use OS_LISTDIR instead.
293
294 if (listdir_func == NULL) {
295 listdir_func = PyObject_GetAttrString(IMPORT_HARD_OS(), "listdir");
296 }
297
298 if (unlikely(listdir_func == NULL)) {
299 return NULL;
300 }
301
302 PyObject *result = CALL_FUNCTION_WITH_SINGLE_ARG(tstate, listdir_func, dirname);
303
304 return result;
305}
306
307#if PYTHON_VERSION < 0x300
308static PyObject *_getImportingSuffixesByPriority(PyThreadState *tstate, int kind) {
309 static PyObject *result = NULL;
310
311 if (result == NULL) {
312 result = MAKE_LIST_EMPTY(tstate, 0);
313
314 PyObject *imp_module = PyImport_ImportModule("imp");
315 PyObject *get_suffixes_func = PyObject_GetAttrString(imp_module, "get_suffixes");
316
317 PyObject *suffix_list = CALL_FUNCTION_NO_ARGS(tstate, get_suffixes_func);
318
319 for (int i = 0; i < PyList_GET_SIZE(suffix_list); i++) {
320 PyObject *module_kind = PyTuple_GET_ITEM(PyList_GET_ITEM(suffix_list, i), 2);
321
322 if (PyInt_AsLong(module_kind) == kind) {
323 LIST_APPEND0(result, PyTuple_GET_ITEM(PyList_GET_ITEM(suffix_list, i), 0));
324 }
325 }
326
327 Py_DECREF(suffix_list);
328 }
329
330 return result;
331}
332#endif
333
334static PyObject *getExtensionModuleSuffixesByPriority(PyThreadState *tstate) {
335 static PyObject *result = NULL;
336
337 if (result == NULL) {
338#if PYTHON_VERSION < 0x300
339 result = _getImportingSuffixesByPriority(tstate, 3);
340#else
341 static PyObject *machinery_module = NULL;
342
343 if (machinery_module == NULL) {
344 machinery_module = PyImport_ImportModule("importlib.machinery");
345 }
346
347 result = PyObject_GetAttrString(machinery_module, "EXTENSION_SUFFIXES");
348#endif
349 }
350
351 CHECK_OBJECT(result);
352 return result;
353}
354
355static PyObject *installed_extension_modules = NULL;
356
357static bool scanModuleInPackagePath(PyThreadState *tstate, PyObject *module_name, char const *parent_module_name) {
358 PyObject *sys_modules = Nuitka_GetSysModules();
359
360 PyObject *parent_module = PyDict_GetItemString(sys_modules, parent_module_name);
361 CHECK_OBJECT(parent_module);
362
363 PyObject *parent_path = PyObject_GetAttr(parent_module, const_str_plain___path__);
364
365 // Accept that it might be deleted.
366 if (parent_path == NULL || !PyList_Check(parent_path)) {
367 return false;
368 }
369
370 PyObject *candidates = MAKE_LIST_EMPTY(tstate, 0);
371
372 // Search only relative to the parent name of course.
373 char const *module_relative_name_str = Nuitka_String_AsString(module_name) + strlen(parent_module_name) + 1;
374
375 Py_ssize_t parent_path_size = PyList_GET_SIZE(parent_path);
376
377 for (Py_ssize_t i = 0; i < parent_path_size; i += 1) {
378 PyObject *path_element = PyList_GET_ITEM(parent_path, i);
379
380 PyObject *filenames_list = _getFileList(tstate, path_element);
381
382 if (filenames_list == NULL) {
383 CLEAR_ERROR_OCCURRED(tstate);
384 continue;
385 }
386
387 Py_ssize_t filenames_list_size = PyList_GET_SIZE(filenames_list);
388
389 for (Py_ssize_t j = 0; j < filenames_list_size; j += 1) {
390 PyObject *filename = PyList_GET_ITEM(filenames_list, j);
391
392 if (Nuitka_String_CheckExact(filename)) {
393 char const *filename_str = Nuitka_String_AsString(filename);
394
395 if (strncmp(filename_str, module_relative_name_str, strlen(module_relative_name_str)) == 0 &&
396 filename_str[strlen(module_relative_name_str)] == '.') {
397 LIST_APPEND1(candidates, MAKE_TUPLE2(tstate, path_element, filename));
398 }
399 }
400 }
401 }
402
403#if 0
404 PRINT_STRING("CANDIDATES:");
405 PRINT_STRING(Nuitka_String_AsString(module_name));
406 PRINT_STRING(module_relative_name_str);
407 PRINT_ITEM(candidates);
408 PRINT_NEW_LINE();
409#endif
410
411 // Look up C-extension suffixes, these are used with highest priority.
412 PyObject *suffix_list = getExtensionModuleSuffixesByPriority(tstate);
413
414 bool result = false;
415
416 for (Py_ssize_t i = 0; i < PyList_GET_SIZE(suffix_list); i += 1) {
417 PyObject *suffix = PyList_GET_ITEM(suffix_list, i);
418
419 char const *suffix_str = Nuitka_String_AsString(suffix);
420
421 for (Py_ssize_t j = 0; j < PyList_GET_SIZE(candidates); j += 1) {
422 PyObject *entry = PyList_GET_ITEM(candidates, j);
423
424 PyObject *directory = PyTuple_GET_ITEM(entry, 0);
425 PyObject *candidate = PyTuple_GET_ITEM(entry, 1);
426
427 char const *candidate_str = Nuitka_String_AsString(candidate);
428
429 if (strcmp(suffix_str, candidate_str + strlen(module_relative_name_str)) == 0) {
430 PyObject *fullpath = JOIN_PATH2(directory, candidate);
431
432 if (installed_extension_modules == NULL) {
433 installed_extension_modules = MAKE_DICT_EMPTY(tstate);
434 }
435
436// Force path to unicode, to have easier consumption, as we need a wchar_t or char *
437// from it later, and we don't want to test there.
438#if PYTHON_VERSION < 0x300 && defined(_WIN32)
439 PyObject *tmp = PyUnicode_FromObject(fullpath);
440 Py_DECREF(fullpath);
441 fullpath = tmp;
442#endif
443
444 DICT_SET_ITEM(installed_extension_modules, module_name, fullpath);
445
446 result = true;
447 break;
448 }
449 }
450 }
451
452 Py_DECREF(candidates);
453
454 return result;
455}
456
457static PyObject *callIntoExtensionModule(PyThreadState *tstate, char const *full_name, const filename_char_t *filename,
458 bool is_package);
459
460static PyObject *callIntoInstalledExtensionModule(PyThreadState *tstate, PyObject *module_name,
461 PyObject *extension_module_filename) {
462#if _WIN32
463 // We can rely on unicode object to be there in case of Windows, to have an easier time to
464 // create the string needed.
465 assert(PyUnicode_CheckExact(extension_module_filename));
466
467 wchar_t const *extension_module_filename_str = Nuitka_UnicodeAsWideString(extension_module_filename, NULL);
468#else
469 char const *extension_module_filename_str = Nuitka_String_AsString(extension_module_filename);
470#endif
471
472 // TODO: The value of "is_package" is guessed, maybe infer from filename being
473 // a "__init__.so" and the like.
474 return callIntoExtensionModule(tstate, Nuitka_String_AsString(module_name), extension_module_filename_str, false);
475}
476
477#endif
478
479static char const *getEntryModeString(struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
480 char const *mode = "compiled";
481
482 if ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0) {
483 mode = "extension";
484 } else if ((entry->flags & NUITKA_BYTECODE_FLAG) != 0) {
485 mode = "bytecode";
486 }
487
488 return mode;
489}
490
491static char *_kw_list_find_module[] = {(char *)"fullname", (char *)"unused", NULL};
492
493static PyObject *_nuitka_loader_find_module(PyObject *self, PyObject *args, PyObject *kwds) {
494 PyObject *module_name;
495 PyObject *unused;
496
497 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O|O:find_module", (char **)_kw_list_find_module, &module_name,
498 &unused);
499
500 if (unlikely(res == 0)) {
501 return NULL;
502 }
503
504 char const *name = Nuitka_String_AsString(module_name);
505
506 if (isVerbose()) {
507 PySys_WriteStderr("import %s # considering responsibility (find_module)\n", name);
508 }
509
510 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
511
512 if (entry) {
513 if (isVerbose()) {
514 PySys_WriteStderr("import %s # claimed responsibility (%s)\n", name, getEntryModeString(entry));
515 }
516
517 PyObject *metapath_based_loader = (PyObject *)&Nuitka_Loader_Type;
518
519 Py_INCREF(metapath_based_loader);
520 return metapath_based_loader;
521 }
522
523 if (hasFrozenModule(name)) {
524 if (isVerbose()) {
525 PySys_WriteStderr("import %s # claimed responsibility (frozen)\n", name);
526 }
527
528 PyObject *metapath_based_loader = (PyObject *)&Nuitka_Loader_Type;
529
530 Py_INCREF(metapath_based_loader);
531 return metapath_based_loader;
532 }
533
534#if !_NUITKA_STANDALONE_MODE
535 entry = findContainingPackageEntry(name);
536
537 if (entry != NULL) {
538 PyThreadState *tstate = PyThreadState_GET();
539
540 bool result = scanModuleInPackagePath(tstate, module_name, entry->name);
541
542 if (result) {
543 PyObject *metapath_based_loader = (PyObject *)&Nuitka_Loader_Type;
544
545 Py_INCREF(metapath_based_loader);
546 return metapath_based_loader;
547 }
548 }
549#endif
550
551 if (isVerbose()) {
552 PySys_WriteStderr("import %s # denied responsibility\n", name);
553 }
554
555 Py_INCREF_IMMORTAL(Py_None);
556 return Py_None;
557}
558
559static char const *_kw_list_get_data[] = {"filename", NULL};
560
561static PyObject *_nuitka_loader_get_data(PyObject *self, PyObject *args, PyObject *kwds) {
562 PyObject *filename;
563
564 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:get_data", (char **)_kw_list_get_data, &filename);
565
566 if (unlikely(res == 0)) {
567 return NULL;
568 }
569
570 PyThreadState *tstate = PyThreadState_GET();
571
572 return GET_FILE_BYTES(tstate, filename);
573}
574
575static void setModuleFileValue(PyThreadState *tstate, PyObject *module, filename_char_t const *filename) {
576 CHECK_OBJECT(module);
577 assert(filename != NULL);
578
579 assert(PyModule_Check(module));
580
581 PyObject *dict = PyModule_GetDict(module);
582
583 // TODO: We should have DICT_SET_ITEM0/1 for these things.
584 PyObject *new_file_value = Nuitka_String_FromFilename(filename);
585 DICT_SET_ITEM(dict, const_str_plain___file__, new_file_value);
586 Py_DECREF(new_file_value);
587}
588
589#if PYTHON_VERSION < 0x300
590typedef void (*entrypoint_t)(void);
591#else
592typedef PyObject *(*entrypoint_t)(void);
593#endif
594
595#ifndef _WIN32
596// Shared libraries loading.
597#include <dlfcn.h>
598#endif
599
600#if PYTHON_VERSION >= 0x350
601static PyObject *createModuleSpec(PyThreadState *tstate, PyObject *module_name, PyObject *origin, bool is_package);
602#endif
603
604static void _fillExtensionModuleDllEntryFunctionName(PyThreadState *tstate, char *buffer, size_t buffer_size,
605 char const *name) {
606
607#if PYTHON_VERSION >= 0x350
608 PyObject *name_bytes_obj = PyBytes_FromString(name);
609 PyObject *name_obj = BYTES_DECODE2(tstate, name_bytes_obj, Nuitka_String_FromString("utf8"));
610 Py_DECREF(name_bytes_obj);
611
612 PyObject *name_ascii = UNICODE_ENCODE2(tstate, name_obj, const_str_plain_ascii);
613
614 if (name_ascii == NULL) {
615 DROP_ERROR_OCCURRED(tstate);
616
617 PyObject *name_punycode = UNICODE_ENCODE2(tstate, name_obj, const_str_plain_punycode);
618
619 CHECK_OBJECT(name_punycode);
620
621 snprintf(buffer, buffer_size, "PyInitU_%s", PyBytes_AsString(name_punycode));
622
623 Py_DECREF(name_punycode);
624 } else {
625 Py_DECREF(name_ascii);
626
627 snprintf(buffer, buffer_size, "PyInit_%s", name);
628 }
629 Py_DECREF(name_obj);
630#else
631
632 snprintf(buffer, buffer_size,
633#if PYTHON_VERSION < 0x300
634 "init%s",
635#else
636 "PyInit_%s",
637#endif
638 name);
639#endif
640}
641
642#if _NUITKA_STANDALONE_MODE
643// Create the filename from full path module name with dots, and translate these
644// into directory separators.
645static void _makeModuleCFilenameValue(filename_char_t *filename, size_t filename_size, char const *module_name_cstr,
646 PyObject *module_name, bool is_package) {
647#ifdef _WIN32
648#if _NUITKA_ONEFILE_DLL_MODE
649 filename_char_t const *base_directory = getDllDirectory();
650#else
651 filename_char_t const *base_directory = getBinaryDirectoryWideChars(true);
652#endif
653 copyStringSafeW(filename, base_directory, filename_size);
654 appendWCharSafeW(filename, FILENAME_SEP_CHAR, filename_size);
655 appendModuleNameAsPathW(filename, module_name, filename_size);
656 if (is_package) {
657 appendWCharSafeW(filename, FILENAME_SEP_CHAR, filename_size);
658 appendStringSafeW(filename, "__init__", filename_size);
659 }
660 appendStringSafeW(filename, ".pyd", filename_size);
661#else
662#if _NUITKA_ONEFILE_DLL_MODE
663 filename_char_t const *base_directory = getDllDirectory();
664#else
665 filename_char_t const *base_directory = getBinaryDirectoryHostEncoded(true);
666#endif
667 copyStringSafe(filename, base_directory, filename_size);
668 appendCharSafe(filename, FILENAME_SEP_CHAR, filename_size);
669 appendModuleNameAsPath(filename, module_name_cstr, filename_size);
670 if (is_package) {
671 appendCharSafe(filename, FILENAME_SEP_CHAR, filename_size);
672 appendStringSafe(filename, "__init__", filename_size);
673 }
674 appendStringSafe(filename, ".so", filename_size);
675#endif
676}
677#endif
678
679#if PYTHON_VERSION >= 0x3c0 && defined(_NUITKA_USE_UNEXPOSED_API)
680extern _Thread_local const char *pkgcontext;
681#endif
682
683static const char *NuitkaImport_SwapPackageContext(const char *new_context) {
684// TODO: The locking APIs for 3.13 give errors here that are not explained
685// yet.
686#if PYTHON_VERSION >= 0x3c0
687 // spell-checker: ignore pkgcontext
688 const char *old_context = _PyRuntime.imports.pkgcontext;
689 _PyRuntime.imports.pkgcontext = new_context;
690#if PYTHON_VERSION >= 0x3c0 && defined(_NUITKA_USE_UNEXPOSED_API)
691 pkgcontext = new_context;
692#endif
693 return old_context;
694#elif PYTHON_VERSION >= 0x370
695 char const *old_context = _Py_PackageContext;
696 _Py_PackageContext = (char *)new_context;
697 return old_context;
698#else
699 char *old_context = _Py_PackageContext;
700 _Py_PackageContext = (char *)new_context;
701 return (char const *)old_context;
702#endif
703}
704
705static PyObject *callIntoExtensionModule(PyThreadState *tstate, char const *full_name, const filename_char_t *filename,
706 bool is_package) {
707 // Determine the package name and basename of the module to load.
708 char const *dot = strrchr(full_name, '.');
709 char const *name;
710 char const *package;
711
712 if (dot == NULL) {
713 package = NULL;
714 name = full_name;
715 } else {
716 // The extension modules do expect it to be full name in context.
717 package = (char *)full_name;
718 name = dot + 1;
719 }
720
721 char entry_function_name[1024];
722 _fillExtensionModuleDllEntryFunctionName(tstate, entry_function_name, sizeof(entry_function_name), name);
723
724#ifdef _WIN32
725 if (isVerbose()) {
726 PySys_WriteStderr("import %s # LoadLibraryExW(\"%S\");\n", full_name, filename);
727 }
728
729#ifndef _NUITKA_EXPERIMENTAL_DEBUG_STANDALONE
730 // Disable all but critical errors, prevents dialogs from showing.
731 // spell-checker: ignore SEM_FAILCRITICALERRORS
732 unsigned int old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
733#endif
734
735 HINSTANCE hDLL;
736#if PYTHON_VERSION >= 0x380
737 Py_BEGIN_ALLOW_THREADS;
738 hDLL = LoadLibraryExW(filename, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
739 Py_END_ALLOW_THREADS;
740#else
741 hDLL = LoadLibraryExW(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
742#endif
743
744#ifndef _NUITKA_EXPERIMENTAL_DEBUG_STANDALONE
745 SetErrorMode(old_mode);
746#endif
747
748 if (unlikely(hDLL == NULL)) {
749 char buffer[1024];
750
751 char error_message[1024];
752 int size;
753
754 unsigned int error_code = GetLastError();
755
756 size = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, error_code, 0,
757 (LPTSTR)error_message, sizeof(error_message), NULL);
758
759 // Report either way even if failed to get error message.
760 if (size == 0) {
761 int ret = PyOS_snprintf(buffer, sizeof(buffer), "LoadLibraryExW '%S' failed with error code %d", filename,
762 error_code);
763
764 assert(ret >= 0);
765 } else {
766 // Strip trailing newline.
767 if (size >= 2 && error_message[size - 2] == '\r' && error_message[size - 1] == '\n') {
768 size -= 2;
769 error_message[size] = '\0';
770 }
771 int ret = PyOS_snprintf(buffer, sizeof(buffer), "LoadLibraryExW '%S' failed: %s", filename, error_message);
772 assert(ret >= 0);
773 }
774
775 SET_CURRENT_EXCEPTION_TYPE0_STR(tstate, PyExc_ImportError, buffer);
776 return NULL;
777 }
778
779 entrypoint_t entrypoint = (entrypoint_t)GetProcAddress(hDLL, entry_function_name);
780#else
781 // This code would work for all versions, we are avoiding access to interpreter
782 // structure internals of 3.8 or higher.
783 // spell-checker: ignore getdlopenflags,dlopenflags
784
785#ifdef __wasi__
786 const char *error = "dynamic libraries are not implemented in wasi";
787 SET_CURRENT_EXCEPTION_TYPE0_STR(tstate, PyExc_ImportError, error);
788 return NULL;
789
790 entrypoint_t entrypoint = NULL;
791#else
792 static PyObject *dlopenflags_object = NULL;
793 if (dlopenflags_object == NULL) {
794 dlopenflags_object = CALL_FUNCTION_NO_ARGS(tstate, Nuitka_SysGetObject("getdlopenflags"));
795 }
796 int dlopenflags = PyInt_AsLong(dlopenflags_object);
797
798 if (isVerbose()) {
799 PySys_WriteStderr("import %s # dlopen(\"%s\", %x);\n", full_name, filename, dlopenflags);
800 }
801
802 void *handle = dlopen(filename, dlopenflags);
803
804 if (unlikely(handle == NULL)) {
805 const char *error = dlerror();
806
807 if (unlikely(error == NULL)) {
808 error = "unknown dlopen() error";
809 }
810
811 SET_CURRENT_EXCEPTION_TYPE0_STR(tstate, PyExc_ImportError, error);
812 return NULL;
813 }
814
815 entrypoint_t entrypoint = (entrypoint_t)dlsym(handle, entry_function_name);
816#endif // __wasi__
817#endif
818 assert(entrypoint);
819
820 char const *old_context = NuitkaImport_SwapPackageContext(package);
821
822#if PYTHON_VERSION >= 0x3c0 && !defined(_NUITKA_USE_UNEXPOSED_API)
823 char const *base_name = strrchr(full_name, '.');
824 PyObject *base_name_obj = NULL;
825 PyObject *prefix_name_obj = NULL;
826 PyObject *preserved_basename_module = NULL;
827 PyObject *preserved_sub_modules = NULL;
828
829 if (base_name != NULL) {
830 base_name_obj = Nuitka_String_FromString(base_name + 1);
831 preserved_basename_module = Nuitka_GetModule(tstate, base_name_obj);
832 Py_XINCREF(preserved_basename_module);
833
834 prefix_name_obj = Nuitka_String_FromStringAndSize(full_name, base_name - full_name + 1);
835
836 if (preserved_basename_module != NULL) {
837 preserved_sub_modules = MAKE_DICT_EMPTY(tstate);
838
839 PyObject *modules_dict = Nuitka_GetSysModules();
840
841 Py_ssize_t pos = 0;
842 PyObject *key, *value;
843
844 PyObject *base_name_prefix = BINARY_OPERATION_ADD_OBJECT_UNICODE_UNICODE(base_name_obj, const_str_dot);
845#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
846 PRINT_STRING("Scanning for sub-modules needing protection:");
847 PRINT_ITEM_LINE(base_name_prefix);
848#endif
849 while (Nuitka_DictNext(modules_dict, &pos, &key, &value)) {
850 // TODO: Should have nuitka_bool return values for these as well maybe.
851 PyObject *starts_with_result = UNICODE_STARTSWITH2(tstate, key, base_name_prefix);
852
853 if (CHECK_IF_TRUE(starts_with_result) == 1) {
854 DICT_SET_ITEM(preserved_sub_modules, key, value);
855 }
856
857 Py_DECREF(starts_with_result);
858 }
859
860#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
861 PRINT_ITEM_LINE(preserved_sub_modules);
862#endif
863 }
864 }
865#endif
866
867 // Finally call into the DLL.
868 PGO_onModuleEntered(full_name);
869
870 if (isVerbose()) {
871 PySys_WriteStderr("import %s # calling entrypoint\n", full_name);
872 }
873
874 Nuitka_DelModuleString(tstate, full_name);
875
876#if PYTHON_VERSION < 0x300
877 (*entrypoint)();
878#else
879 PyObject *module = (*entrypoint)();
880#endif
881
882 if (isVerbose()) {
883 PySys_WriteStderr("import %s # return from entrypoint\n", full_name);
884 }
885
886#if 0
887 PRINT_FORMAT("import %s # create module as ", full_name);
888 PRINT_ITEM(module);
889 PRINT_NEW_LINE();
890#endif
891
892 NuitkaImport_SwapPackageContext(old_context);
893
894#if PYTHON_VERSION < 0x300
895 PyObject *module = Nuitka_GetModuleString(tstate, full_name);
896#endif
897
898 PGO_onModuleExit(name, module == NULL);
899
900 if (unlikely(module == NULL)) {
901 if (unlikely(!HAS_ERROR_OCCURRED(tstate))) {
902 PyErr_Format(PyExc_SystemError, "dynamic module '%s' not initialized properly", full_name);
903 }
904
905 return NULL;
906 }
907
908#if PYTHON_VERSION >= 0x300
909#if PYTHON_VERSION >= 0x350
910 PyModuleDef *def;
911
912 if (Py_TYPE(module) == &PyModuleDef_Type) {
913 if (isVerbose()) {
914 PySys_WriteStderr("import %s # entrypoint returned module def\n", full_name);
915 }
916
917 def = (PyModuleDef *)module;
918
919 PyObject *full_name_obj = Nuitka_String_FromString(full_name);
920
921 PyObject *origin = Nuitka_String_FromFilename(filename);
922
923 PyObject *spec_value = createModuleSpec(tstate, full_name_obj, origin, is_package);
924 CHECK_OBJECT(spec_value);
925
926 module = PyModule_FromDefAndSpec(def, spec_value);
927
928 if (unlikely(module == NULL)) {
929 Py_DECREF(spec_value);
930
931 PyErr_Format(PyExc_SystemError, "dynamic module '%s' not initialized properly from def", full_name);
932
933 return NULL;
934 }
935
936 SET_ATTRIBUTE(tstate, module, const_str_plain___spec__, spec_value);
937
938 setModuleFileValue(tstate, module, filename);
939
940 /* Set __path__ properly, unlike frozen module importer does. */
941 PyObject *path_list = _makeDunderPathObject(tstate, origin);
942
943 int res = PyObject_SetAttr(module, const_str_plain___path__, path_list);
944 if (unlikely(res != 0)) {
945 return NULL;
946 }
947
948 Py_DECREF(path_list);
949
950 Nuitka_SetModule(full_name_obj, module);
951 Py_DECREF(full_name_obj);
952
953#if _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
954 SET_ATTRIBUTE(tstate, spec_value, const_str_plain__initializing, Py_True);
955 res = PyModule_ExecDef(module, def);
956 SET_ATTRIBUTE(tstate, spec_value, const_str_plain__initializing, Py_False);
957
958 Py_DECREF(spec_value);
959 CHECK_OBJECT(spec_value);
960
961 if (unlikely(res == -1)) {
962 return NULL;
963 }
964
965 if (isVerbose()) {
966 PySys_WriteStderr("import %s # executed module def\n", full_name);
967 }
968#endif
969
970 CHECK_OBJECT(module);
971
972 return module;
973 } else {
974 if (isVerbose()) {
975 PySys_WriteStderr("import %s # entrypoint returned module\n", full_name);
976 }
977
978 def = PyModule_GetDef(module);
979 assert(def != NULL);
980
981 def->m_base.m_init = entrypoint;
982
983#if PYTHON_VERSION >= 0x3d0
984 if (PyState_AddModule(module, def) == -1) {
985 return NULL;
986 }
987#endif
988
989 // Set "__spec__" and "__file__" after load.
990 setModuleFileValue(tstate, module, filename);
991 PyObject *full_name_obj = Nuitka_String_FromString(full_name);
992 SET_ATTRIBUTE(tstate, module, const_str_plain___name__, full_name_obj);
993 PyObject *spec_value =
994 createModuleSpec(tstate, full_name_obj, LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___file__), false);
995
996 SET_ATTRIBUTE(tstate, module, const_str_plain___spec__, spec_value);
997
998 // Fixup "__package__" after load. It seems some modules ignore _Py_PackageContext value.
999 // so we patch it up here if it's None, but a package was specified.
1000 if (package != NULL) {
1001 PyObject *package_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___package__);
1002
1003 if (package_name == Py_None) {
1004 char package2[2048];
1005 copyStringSafeN(package2, full_name, dot - full_name, sizeof(package2));
1006
1007 PyObject *package_name_obj = Nuitka_String_FromString(package2);
1008 SET_ATTRIBUTE(tstate, module, const_str_plain___package__, package_name_obj);
1009 Py_DECREF(package_name_obj);
1010 }
1011
1012 Py_DECREF(package_name);
1013 }
1014 }
1015
1016 if (likely(def != NULL)) {
1017 def->m_base.m_init = entrypoint;
1018 }
1019
1020#else
1021 PyModuleDef *def = PyModule_GetDef(module);
1022
1023 if (unlikely(def == NULL)) {
1024 PyErr_Format(PyExc_SystemError, "initialization of %s did not return an extension module", filename);
1025
1026 return NULL;
1027 }
1028
1029 def->m_base.m_init = entrypoint;
1030#endif
1031
1032#endif
1033
1034 // Set filename attribute if not already set, in some branches we don't
1035 // do it, esp. not for older Python.
1036 setModuleFileValue(tstate, module, filename);
1037
1038 // Call the standard import fix-ups for extension modules. Their interface
1039 // changed over releases.
1040#if PYTHON_VERSION < 0x300
1041 PyObject *res2 = _PyImport_FixupExtension((char *)full_name, (char *)filename);
1042
1043 if (unlikely(res2 == NULL)) {
1044 return NULL;
1045 }
1046#else
1047 PyObject *full_name_obj = PyUnicode_FromString(full_name);
1048 CHECK_OBJECT(full_name_obj);
1049 PyObject *filename_obj = Nuitka_String_FromFilename(filename);
1050
1051 CHECK_OBJECT(filename_obj);
1052
1053// See above, we need to correct modules imported if we don't successfully swap
1054// the package context.
1055#if PYTHON_VERSION >= 0x3c0 && !defined(_NUITKA_USE_UNEXPOSED_API)
1056 if (preserved_basename_module != NULL) {
1057#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1058 PRINT_STRING("Handling for preservation: ");
1059 PRINT_ITEM_LINE(full_name_obj);
1060 PRINT_STRING("Restoring preserved module: ");
1061 PRINT_ITEM(base_name_obj);
1062 if (Nuitka_HasModule(tstate, base_name_obj)) {
1063 PRINT_STRING(" changes ");
1064 PRINT_ITEM(Nuitka_GetModule(tstate, base_name_obj));
1065 }
1066 PRINT_STRING(" -> ");
1067 PRINT_ITEM_LINE(preserved_basename_module);
1068#endif
1069 Nuitka_SetModule(base_name_obj, preserved_basename_module);
1070 Py_DECREF(preserved_basename_module);
1071 }
1072
1073 if (base_name_obj != NULL) {
1074 PyObject *need_correction = MAKE_LIST_EMPTY(tstate, 0);
1075
1076 {
1077 PyObject *modules_dict = Nuitka_GetSysModules();
1078
1079 Py_ssize_t pos = 0;
1080 PyObject *key, *value;
1081
1082 PyObject *base_name_prefix = BINARY_OPERATION_ADD_OBJECT_UNICODE_UNICODE(base_name_obj, const_str_dot);
1083#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1084 PRINT_STRING("Scanning for modules needing correction: ");
1085 PRINT_ITEM_LINE(base_name_prefix);
1086#endif
1087 while (Nuitka_DictNext(modules_dict, &pos, &key, &value)) {
1088 // TODO: Should have nuitka_bool return values for these as well maybe.
1089 if (preserved_sub_modules != NULL && DICT_HAS_ITEM(tstate, preserved_sub_modules, key) == 1) {
1090 continue;
1091 }
1092
1093 PyObject *starts_with_result = UNICODE_STARTSWITH2(tstate, key, base_name_prefix);
1094
1095 if (CHECK_IF_TRUE(starts_with_result) == 1) {
1096 LIST_APPEND0(need_correction, key);
1097 }
1098
1099 Py_DECREF(starts_with_result);
1100 }
1101
1102 Py_XDECREF(preserved_sub_modules);
1103 }
1104
1105#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1106 PRINT_STRING("Correction needed:");
1107 PRINT_ITEM_LINE(need_correction);
1108#endif
1109
1110 Py_ssize_t n = PyList_GET_SIZE(need_correction);
1111
1112 for (Py_ssize_t i = 0; i < n; i++) {
1113 PyObject *module_to_correct_name = PyList_GET_ITEM(need_correction, i);
1114
1115 PyObject *correct_module_name =
1116 BINARY_OPERATION_ADD_OBJECT_UNICODE_UNICODE(prefix_name_obj, module_to_correct_name);
1117
1118 PyObject *module_to_correct = Nuitka_GetModule(tstate, module_to_correct_name);
1119
1120#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1121 PRINT_ITEM(module_to_correct_name);
1122 PRINT_STRING(" -> ");
1123 PRINT_ITEM(correct_module_name);
1124 PRINT_STRING(" changes ");
1125 PRINT_ITEM_LINE(module_to_correct);
1126#endif
1127 Nuitka_SetModule(correct_module_name, module_to_correct);
1128
1129 Nuitka_DelModule(tstate, module_to_correct_name);
1130 }
1131
1132 Py_DECREF(base_name_obj);
1133 Py_DECREF(prefix_name_obj);
1134 }
1135#endif
1136
1137#if PYTHON_VERSION < 0x3d0
1138 int res = _PyImport_FixupExtensionObject(module, full_name_obj, filename_obj
1139#if PYTHON_VERSION >= 0x370
1140 ,
1141 Nuitka_GetSysModules()
1142#endif
1143 );
1144#endif
1145
1146#if PYTHON_VERSION >= 0x3d0
1147 Nuitka_SetModuleString(full_name, module);
1148#endif
1149
1150 Py_DECREF(full_name_obj);
1151 Py_DECREF(filename_obj);
1152
1153#if PYTHON_VERSION < 0x3d0
1154 if (unlikely(res == -1)) {
1155 return NULL;
1156 }
1157#endif
1158#endif
1159
1160 return module;
1161}
1162
1163static void loadTriggeredModule(PyThreadState *tstate, char const *name, char const *trigger_name) {
1164 char trigger_module_name[2048];
1165
1166 copyStringSafe(trigger_module_name, name, sizeof(trigger_module_name));
1167 appendStringSafe(trigger_module_name, trigger_name, sizeof(trigger_module_name));
1168
1169 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(trigger_module_name);
1170
1171 if (entry != NULL) {
1172 if (isVerbose()) {
1173 PySys_WriteStderr("Loading %s\n", trigger_module_name);
1174 }
1175
1176 IMPORT_EMBEDDED_MODULE(tstate, trigger_module_name);
1177
1178 if (unlikely(HAS_ERROR_OCCURRED(tstate))) {
1179 if ((entry->flags & NUITKA_ABORT_MODULE_FLAG) != 0) {
1180 printf("Critical error loading %s.\n", trigger_module_name);
1181 abort();
1182 } else {
1183 PyObject *trigger_module_name_str = Nuitka_String_FromString(trigger_module_name);
1184 PyErr_WriteUnraisable(trigger_module_name_str);
1185 Py_DECREF(trigger_module_name_str);
1186 }
1187 }
1188 }
1189}
1190
1191#if PYTHON_VERSION >= 0x300
1192static void _fixupSpecAttribute(PyThreadState *tstate, PyObject *module) {
1193 PyObject *spec_value = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___spec__);
1194
1195 if (spec_value && spec_value != Py_None) {
1196 if (HAS_ATTR_BOOL(tstate, spec_value, const_str_plain__initializing)) {
1197 SET_ATTRIBUTE(tstate, spec_value, const_str_plain__initializing, Py_False);
1198 }
1199 }
1200}
1201#endif
1202
1203// Pointers to bytecode data.
1204static char **_bytecode_data = NULL;
1205
1206static PyObject *loadModule(PyThreadState *tstate, PyObject *module, PyObject *module_name,
1207 struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
1208#if _NUITKA_STANDALONE_MODE
1209 if ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0) {
1210 bool is_package = (entry->flags & NUITKA_PACKAGE_FLAG) != 0;
1211
1212 filename_char_t filename[MAXPATHLEN + 1];
1213 _makeModuleCFilenameValue(filename, sizeof(filename) / sizeof(filename_char_t), entry->name, module_name,
1214 is_package);
1215
1216 callIntoExtensionModule(tstate, entry->name, filename, is_package);
1217 } else
1218#endif
1219 if ((entry->flags & NUITKA_BYTECODE_FLAG) != 0) {
1220 // TODO: Do node use marshal, but our own stuff, once we
1221 // can do code objects too.
1222
1223 PyCodeObject *code_object =
1224 (PyCodeObject *)PyMarshal_ReadObjectFromString(_bytecode_data[entry->bytecode_index], entry->bytecode_size);
1225
1226 // TODO: Probably a bit harsh reaction.
1227 if (unlikely(code_object == NULL)) {
1228 PyErr_Print();
1229 abort();
1230 }
1231
1232 return loadModuleFromCodeObject(tstate, module, code_object, entry->name,
1233 (entry->flags & NUITKA_PACKAGE_FLAG) != 0);
1234 } else {
1235 assert((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) == 0);
1236 assert(entry->python_init_func);
1237
1238 {
1239 NUITKA_MAY_BE_UNUSED bool res = Nuitka_SetModule(module_name, module);
1240 assert(res != false);
1241 }
1242
1243 // Run the compiled module code, we get the module returned.
1244#if PYTHON_VERSION < 0x300
1245 NUITKA_MAY_BE_UNUSED
1246#endif
1247 PyObject *result = entry->python_init_func(tstate, module, entry);
1248 CHECK_OBJECT_X(result);
1249
1250#if PYTHON_VERSION >= 0x300
1251 if (likely(result != NULL)) {
1252 _fixupSpecAttribute(tstate, result);
1253 }
1254#endif
1255 }
1256
1257 if (unlikely(HAS_ERROR_OCCURRED(tstate))) {
1258 return NULL;
1259 }
1260
1261 if (isVerbose()) {
1262 PySys_WriteStderr("Loaded %s\n", entry->name);
1263 }
1264
1265 return Nuitka_GetModule(tstate, module_name);
1266}
1267
1268static PyObject *_EXECUTE_EMBEDDED_MODULE(PyThreadState *tstate, PyObject *module, PyObject *module_name,
1269 char const *name) {
1270 // In case of extension modules, that is fine to be NULL.
1271 CHECK_OBJECT_X(module);
1272 CHECK_OBJECT(module_name);
1273
1274 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1275 bool frozen_import = entry == NULL && hasFrozenModule(name);
1276
1277 if (entry != NULL || frozen_import) {
1278 // Execute the "preLoad" code produced for the module potentially. This
1279 // is from plugins typically, that want to modify things for the the
1280 // module before loading, to e.g. set a plug-in path, or do some monkey
1281 // patching in order to make things compatible.
1282 loadTriggeredModule(tstate, name, "-preLoad");
1283 }
1284
1285 PyObject *result = NULL;
1286
1287 if (entry != NULL) {
1288#ifdef _NUITKA_EXPERIMENTAL_FORCE_GC_COLLECT_ON_IMPORT
1289 PyGC_Collect();
1290#endif
1291
1292 result = loadModule(tstate, module, module_name, entry);
1293
1294#if !defined(_NUITKA_DEPLOYMENT_MODE) && !defined(_NUITKA_NO_DEPLOYMENT_PERFECT_SUPPORT)
1295 if (unlikely(HAS_ERROR_OCCURRED(tstate))) {
1296 if ((entry->flags & NUITKA_PERFECT_SUPPORTED_FLAG) != 0) {
1297 struct Nuitka_ExceptionPreservationItem saved_exception;
1298 FETCH_ERROR_OCCURRED_STATE(tstate, &saved_exception);
1299
1300 PyObject *exception_arg = PyUnicode_FromFormat("Nuitka: import of module '%s' failed unexpectedly "
1301 "despite intended perfect support, please raise a "
1302 "Nuitka issue and compile with an older version of "
1303 "the module in the meantime",
1304 name);
1305
1306 raiseReplacementRuntimeError(tstate, &saved_exception, exception_arg);
1307 }
1308 }
1309#endif
1310
1311#ifdef _NUITKA_EXPERIMENTAL_FORCE_GC_COLLECT_ON_IMPORT
1312 PyGC_Collect();
1313#endif
1314
1315 if (unlikely(result == NULL)) {
1316 return NULL;
1317 }
1318 }
1319
1320 if (frozen_import) {
1321 PGO_onModuleEntered(name);
1322 int res = PyImport_ImportFrozenModule((char *)name);
1323 PGO_onModuleExit(name, res == -1);
1324
1325 if (unlikely(res == -1)) {
1326 return NULL;
1327 }
1328
1329 if (res == 1) {
1330 result = Nuitka_GetModule(tstate, module_name);
1331 }
1332 }
1333
1334 if (result != NULL) {
1335 // Execute the "postLoad" code produced for the module potentially. This
1336 // is from plugins typically, that want to modify the module immediately
1337 // after loading, to e.g. set a plug-in path, or do some monkey patching
1338 // in order to make things compatible.
1339 loadTriggeredModule(tstate, name, "-postLoad");
1340
1341 return result;
1342 }
1343
1344 Py_INCREF_IMMORTAL(Py_None);
1345 return Py_None;
1346}
1347
1348// Note: This may become an entry point for hard coded imports of compiled
1349// stuff.
1350PyObject *IMPORT_EMBEDDED_MODULE(PyThreadState *tstate, char const *name) {
1351 PyObject *module_name = Nuitka_String_FromString(name);
1352
1353 // Check if it's already loaded, and don't do it again otherwise.
1354 PyObject *module = Nuitka_GetModule(tstate, module_name);
1355
1356 if (module != NULL) {
1357 Py_DECREF(module_name);
1358 return module;
1359 }
1360
1361#if PYTHON_VERSION < 0x300
1362 module = PyModule_New(name);
1363#else
1364 module = PyModule_NewObject(module_name);
1365#endif
1366
1367 PyObject *result = _EXECUTE_EMBEDDED_MODULE(tstate, module, module_name, name);
1368
1369#if PYTHON_VERSION < 0x350
1370 if (unlikely(result == NULL)) {
1371 Nuitka_DelModule(tstate, module_name);
1372 }
1373#endif
1374
1375 Py_DECREF(module_name);
1376
1377 return result;
1378}
1379
1380PyObject *EXECUTE_EMBEDDED_MODULE(PyThreadState *tstate, PyObject *module) {
1381 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___name__);
1382 assert(module_name);
1383
1384 char const *name = Nuitka_String_AsString(module_name);
1385
1386 return _EXECUTE_EMBEDDED_MODULE(tstate, module, module_name, name);
1387}
1388
1389static PyObject *_nuitka_loader_load_module(PyObject *self, PyObject *args, PyObject *kwds) {
1390 PyObject *module_name;
1391 PyObject *unused;
1392
1393 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O|O:load_module", (char **)_kw_list_find_module, &module_name,
1394 &unused);
1395
1396 if (unlikely(res == 0)) {
1397 return NULL;
1398 }
1399
1400 assert(module_name);
1401 assert(Nuitka_String_Check(module_name));
1402
1403 char const *name = Nuitka_String_AsString(module_name);
1404
1405 if (isVerbose()) {
1406 PySys_WriteStderr("Loading %s\n", name);
1407 }
1408
1409 PyThreadState *tstate = PyThreadState_GET();
1410
1411#if !_NUITKA_STANDALONE_MODE
1412 if (installed_extension_modules != NULL) {
1413 PyObject *extension_module_filename = DICT_GET_ITEM0(tstate, installed_extension_modules, module_name);
1414
1415 if (extension_module_filename != NULL) {
1416 // TODO: Should we not set __file__ for the module here, but there is no object.
1417 return callIntoInstalledExtensionModule(tstate, module_name, extension_module_filename);
1418 }
1419 }
1420#endif
1421
1422 return IMPORT_EMBEDDED_MODULE(tstate, name);
1423}
1424
1425static char const *_kw_list_is_package[] = {"fullname", NULL};
1426
1427static PyObject *_nuitka_loader_is_package(PyObject *self, PyObject *args, PyObject *kwds) {
1428 PyObject *module_name;
1429
1430 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:is_package", (char **)_kw_list_is_package, &module_name);
1431
1432 if (unlikely(res == 0)) {
1433 return NULL;
1434 }
1435
1436 assert(module_name);
1437 assert(Nuitka_String_Check(module_name));
1438
1439 char const *name = Nuitka_String_AsString(module_name);
1440
1441 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1442
1443 PyObject *result;
1444
1445 if (entry) {
1446 result = BOOL_FROM((entry->flags & NUITKA_PACKAGE_FLAG) != 0);
1447 } else {
1448 // TODO: Maybe needs to be an exception.
1449 result = Py_None;
1450 }
1451
1452 Py_INCREF_IMMORTAL(result);
1453 return result;
1454}
1455
1456static char const *_kw_list_iter_modules[] = {"package", NULL};
1457
1458static PyObject *_nuitka_loader_iter_modules(struct Nuitka_LoaderObject *self, PyObject *args, PyObject *kwds) {
1459 PyObject *prefix;
1460
1461 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:iter_modules", (char **)_kw_list_iter_modules, &prefix);
1462
1463 if (unlikely(res == 0)) {
1464 return NULL;
1465 }
1466
1467 NUITKA_MAY_BE_UNUSED PyThreadState *tstate = PyThreadState_GET();
1468
1469 PyObject *result = MAKE_LIST_EMPTY(tstate, 0);
1470
1471 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
1472 assert(current);
1473
1474 char const *s;
1475
1476 if (self->m_loader_entry) {
1477 s = self->m_loader_entry->name;
1478 } else {
1479 s = "";
1480 }
1481
1482 while (current->name != NULL) {
1483 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
1484 current->name = UN_TRANSLATE(current->name);
1485 current->flags -= NUITKA_TRANSLATED_FLAG;
1486 }
1487
1488 int c = strncmp(s, current->name, strlen(s));
1489
1490 if (c != 0) {
1491 current++;
1492 continue;
1493 }
1494
1495 if (strcmp(current->name, "__main__") == 0) {
1496 current++;
1497 continue;
1498 }
1499
1500 if (current->name[strlen(s)] != '.') {
1501 current++;
1502 continue;
1503 }
1504
1505 char const *sub = strchr(current->name + strlen(s) + 1, '.');
1506
1507 if (sub != NULL) {
1508 current++;
1509 continue;
1510 }
1511
1512 PyObject *name;
1513 if (self->m_loader_entry) {
1514 name = Nuitka_String_FromString(current->name + strlen(s) + 1);
1515 } else {
1516 name = Nuitka_String_FromString(current->name);
1517 }
1518
1519 if (CHECK_IF_TRUE(prefix)) {
1520 PyObject *old = name;
1521 name = PyUnicode_Concat(prefix, name);
1522 Py_DECREF(old);
1523 }
1524
1525 PyObject *r = MAKE_TUPLE_EMPTY(tstate, 2);
1526 PyTuple_SET_ITEM(r, 0, name);
1527 PyTuple_SET_ITEM_IMMORTAL(r, 1, BOOL_FROM((current->flags & NUITKA_PACKAGE_FLAG) != 0));
1528
1529 LIST_APPEND1(result, r);
1530
1531 current++;
1532 }
1533
1534 return result;
1535}
1536
1537static PyObject *getModuleDirectory(PyThreadState *tstate, struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
1538#if defined(_NUITKA_FREEZER_HAS_FILE_PATH)
1539#if defined(_WIN32)
1540 wchar_t buffer[1024];
1541 buffer[0] = 0;
1542
1543 appendWStringSafeW(buffer, entry->file_path, sizeof(buffer));
1544 stripFilenameW(buffer);
1545 PyObject *dir_name = NuitkaUnicode_FromWideChar(buffer, -1);
1546#else
1547 char buffer[1024];
1548 copyStringSafe(buffer, entry->file_path, sizeof(buffer));
1549
1550 PyObject *dir_name = Nuitka_String_FromString(dirname(buffer));
1551#endif
1552#else
1553 PyObject *module_name;
1554 if ((entry->flags & NUITKA_PACKAGE_FLAG) != 0) {
1555 module_name = Nuitka_String_FromString(entry->name);
1556 } else {
1557 char buffer[1024];
1558 copyStringSafe(buffer, entry->name, sizeof(buffer));
1559
1560 char *dot = strrchr(buffer, '.');
1561 if (dot != NULL) {
1562 *dot = 0;
1563 }
1564
1565 module_name = Nuitka_String_FromString(buffer);
1566 }
1567
1568#if PYTHON_VERSION < 0x300
1569 PyObject *module_path = STR_REPLACE3(tstate, module_name, const_str_dot, getPathSeparatorStringObject());
1570#else
1571 PyObject *module_path = UNICODE_REPLACE3(tstate, module_name, const_str_dot, getPathSeparatorStringObject());
1572#endif
1573 Py_DECREF(module_name);
1574
1575 if (unlikely(module_path == NULL)) {
1576 return NULL;
1577 }
1578
1579 PyObject *dir_name = MAKE_RELATIVE_PATH(module_path);
1580 Py_DECREF(module_path);
1581#endif
1582
1583 return dir_name;
1584}
1585
1586#if PYTHON_VERSION >= 0x300
1587// Used in module template too, therefore exported.
1588PyObject *getImportLibBootstrapModule(void) {
1589 static PyObject *importlib = NULL;
1590 if (importlib == NULL) {
1591 importlib = PyImport_ImportModule("importlib._bootstrap");
1592 }
1593
1594 return importlib;
1595}
1596#endif
1597
1598static PyObject *getModuleFileValue(PyThreadState *tstate, struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
1599 PyObject *dir_name = getModuleDirectory(tstate, entry);
1600
1601 char filename_buffer[1024];
1602
1603 if ((entry->flags & NUITKA_PACKAGE_FLAG) != 0) {
1604 copyStringSafe(filename_buffer, "__init__", sizeof(filename_buffer));
1605 } else {
1606 char const *basename = strrchr(entry->name, '.');
1607
1608 if (basename == NULL) {
1609 basename = entry->name;
1610 } else {
1611 basename += 1;
1612 }
1613
1614 copyStringSafe(filename_buffer, basename, sizeof(filename_buffer));
1615 }
1616
1617 if ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0) {
1618#if defined(_WIN32)
1619 appendStringSafe(filename_buffer, ".pyd", sizeof(filename_buffer));
1620#else
1621 appendStringSafe(filename_buffer, ".so", sizeof(filename_buffer));
1622#endif
1623 } else {
1624 appendStringSafe(filename_buffer, ".py", sizeof(filename_buffer));
1625 }
1626
1627 PyObject *module_filename = Nuitka_String_FromString(filename_buffer);
1628
1629 PyObject *result = JOIN_PATH2(dir_name, module_filename);
1630
1631 Py_DECREF(module_filename);
1632
1633 return result;
1634}
1635
1636#if PYTHON_VERSION >= 0x300
1637
1638static PyObject *_nuitka_loader_repr_module(PyObject *self, PyObject *args, PyObject *kwds) {
1639 PyObject *module;
1640 PyObject *unused;
1641
1642 int res =
1643 PyArg_ParseTupleAndKeywords(args, kwds, "O|O:module_repr", (char **)_kw_list_find_module, &module, &unused);
1644
1645 if (unlikely(res == 0)) {
1646 return NULL;
1647 }
1648
1649 PyThreadState *tstate = PyThreadState_GET();
1650
1651 return PyUnicode_FromFormat("<module '%s' from %R>", PyModule_GetName(module),
1652 Nuitka_GetFilenameObject(tstate, module));
1653}
1654
1655static PyObject *getModuleSpecClass(PyObject *importlib_module) {
1656 static PyObject *module_spec_class = NULL;
1657
1658 if (module_spec_class == NULL) {
1659 module_spec_class = PyObject_GetAttrString(importlib_module, "ModuleSpec");
1660 }
1661
1662 return module_spec_class;
1663}
1664
1665static PyObject *createModuleSpec(PyThreadState *tstate, PyObject *module_name, PyObject *origin, bool is_package) {
1666 CHECK_OBJECT(module_name);
1667 assert(Nuitka_String_Check(module_name));
1668 CHECK_OBJECT_X(origin);
1669
1670 PyObject *importlib_module = getImportLibBootstrapModule();
1671
1672 if (unlikely(importlib_module == NULL)) {
1673 return NULL;
1674 }
1675
1676 PyObject *module_spec_class = getModuleSpecClass(importlib_module);
1677
1678 if (unlikely(module_spec_class == NULL)) {
1679 return NULL;
1680 }
1681
1682 PyObject *args = MAKE_TUPLE2(tstate, module_name, (PyObject *)&Nuitka_Loader_Type);
1683
1684 PyObject *kw_values[] = {is_package ? Py_True : Py_False, origin};
1685
1686 char const *kw_keys[] = {"is_package", "origin"};
1687
1688 PyObject *kw_args = MAKE_DICT_X_CSTR(kw_keys, kw_values, sizeof(kw_values) / sizeof(PyObject *));
1689
1690 PyObject *result = CALL_FUNCTION(tstate, module_spec_class, args, kw_args);
1691
1692 Py_DECREF(args);
1693 Py_DECREF(kw_args);
1694
1695 return result;
1696}
1697
1698#if !_NUITKA_STANDALONE_MODE
1699// We might have to load stuff from installed modules in our package namespaces.
1700static PyObject *createModuleSpecViaPathFinder(PyThreadState *tstate, PyObject *module_name,
1701 char const *parent_module_name) {
1702 if (scanModuleInPackagePath(tstate, module_name, parent_module_name)) {
1703 return createModuleSpec(tstate, module_name, NULL, false);
1704 } else {
1705 // Without error this means we didn't make it.
1706 return NULL;
1707 }
1708}
1709#endif
1710
1711static char const *_kw_list_find_spec[] = {"fullname", "is_package", "path", NULL};
1712
1713static PyObject *_nuitka_loader_find_spec(PyObject *self, PyObject *args, PyObject *kwds) {
1714 PyObject *module_name;
1715 PyObject *unused1; // We ignore "is_package"
1716 PyObject *unused2; // We ignore "path"
1717
1718 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O|OO:find_spec", (char **)_kw_list_find_spec, &module_name,
1719 &unused1, &unused2);
1720
1721 if (unlikely(res == 0)) {
1722 return NULL;
1723 }
1724
1725 char const *full_name = Nuitka_String_AsString(module_name);
1726
1727 if (isVerbose()) {
1728 PySys_WriteStderr("import %s # considering responsibility (find_spec)\n", full_name);
1729 }
1730
1731 struct Nuitka_MetaPathBasedLoaderEntry const *entry = findEntry(full_name);
1732
1733 PyThreadState *tstate = PyThreadState_GET();
1734
1735#if !_NUITKA_STANDALONE_MODE
1736 // We need to deal with things located in compiled packages, that were not included,
1737 // e.g. extension modules, but also other files, that were asked to not be included
1738 // or added later.
1739 if (entry == NULL) {
1740 entry = findContainingPackageEntry(full_name);
1741
1742 if (entry != NULL) {
1743 PyObject *result = createModuleSpecViaPathFinder(tstate, module_name, entry->name);
1744
1745 if (result != NULL) {
1746 if (isVerbose()) {
1747 PySys_WriteStderr("import %s # claimed responsibility (%s, contained in compiled package %s)\n",
1748 full_name, getEntryModeString(entry), entry->name);
1749 }
1750
1751 return result;
1752 }
1753
1754 if (HAS_ERROR_OCCURRED(tstate)) {
1755 return NULL;
1756 }
1757
1758 entry = NULL;
1759 }
1760 }
1761#endif
1762
1763 if (entry == NULL) {
1764 if (isVerbose()) {
1765 PySys_WriteStderr("import %s # denied responsibility\n", full_name);
1766 }
1767
1768 Py_INCREF_IMMORTAL(Py_None);
1769 return Py_None;
1770 }
1771
1772 if (isVerbose()) {
1773 PySys_WriteStderr("import %s # claimed responsibility (%s)\n", Nuitka_String_AsString(module_name),
1774 getEntryModeString(entry));
1775 }
1776
1777 return createModuleSpec(tstate, module_name, getModuleFileValue(tstate, entry),
1778 (entry->flags & NUITKA_PACKAGE_FLAG) != 0);
1779}
1780
1781#if PYTHON_VERSION >= 0x350
1782static char const *_kw_list_create_module[] = {"spec", NULL};
1783
1784static PyObject *_nuitka_loader_create_module(PyObject *self, PyObject *args, PyObject *kwds) {
1785 PyObject *spec;
1786
1787 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:create_module", (char **)_kw_list_create_module, &spec);
1788
1789 if (unlikely(res == 0)) {
1790 return NULL;
1791 }
1792
1793 PyThreadState *tstate = PyThreadState_GET();
1794
1795 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, spec, const_str_plain_name);
1796
1797 if (unlikely(module_name == NULL)) {
1798 return NULL;
1799 }
1800
1801 // Check if module name is a proper string.
1802 char const *name = Nuitka_String_AsString(module_name);
1803 if (unlikely(name == NULL)) {
1804 Py_DECREF(module_name);
1805 return NULL;
1806 }
1807
1808 if (isVerbose()) {
1809 PySys_WriteStderr("import %s # create module\n", name);
1810 }
1811
1812 // During spec creation, we have populated the dictionary with a filename to load from
1813 // for extension modules that were found installed in the system and below our package.
1814#if !_NUITKA_STANDALONE_MODE && !_NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1815 if (installed_extension_modules != NULL) {
1816 PyObject *extension_module_filename = DICT_GET_ITEM0(tstate, installed_extension_modules, module_name);
1817
1818 if (extension_module_filename != NULL) {
1819 Py_DECREF(module_name);
1820 return callIntoInstalledExtensionModule(tstate, module_name, extension_module_filename);
1821 }
1822 }
1823#endif
1824
1825#if _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1826 struct Nuitka_MetaPathBasedLoaderEntry *entry = NULL;
1827#else
1828 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1829#endif
1830
1831 PyObject *result;
1832
1833 if ((entry != NULL) && ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0)) {
1834 result = _EXECUTE_EMBEDDED_MODULE(tstate, NULL, module_name, name);
1835 } else {
1836 result = PyModule_NewObject(module_name);
1837 }
1838
1839 Py_DECREF(module_name);
1840
1841 return result;
1842}
1843
1844static char const *_kw_list_exec_module[] = {"module", NULL};
1845
1846static PyObject *_nuitka_loader_exec_module(PyObject *self, PyObject *args, PyObject *kwds) {
1847 PyObject *module;
1848
1849 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:exec_module", (char **)_kw_list_exec_module, &module);
1850
1851 if (unlikely(res == 0)) {
1852 return NULL;
1853 }
1854
1855 PyThreadState *tstate = PyThreadState_GET();
1856
1857 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___name__);
1858
1859 if (unlikely(module_name == NULL)) {
1860 return NULL;
1861 }
1862
1863 // Check if module name is a proper string.
1864 char const *name = Nuitka_String_AsString(module_name);
1865 if (unlikely(name == NULL)) {
1866 Py_DECREF(module_name);
1867 return NULL;
1868 }
1869
1870 if (isVerbose()) {
1871 PySys_WriteStderr("import %s # execute module\n", name);
1872 }
1873
1874 // During spec creation, we have populated the dictionary with a filename to load from
1875 // for extension modules that were found installed in the system and below our package.
1876#if !_NUITKA_STANDALONE_MODE && _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1877 if (installed_extension_modules != NULL) {
1878 PyObject *extension_module_filename = DICT_GET_ITEM0(tstate, installed_extension_modules, module_name);
1879
1880 if (extension_module_filename != NULL) {
1881 // Set filename attribute
1882 bool b_res = SET_ATTRIBUTE(tstate, module, const_str_plain___file__, extension_module_filename);
1883
1884 if (unlikely(b_res == false)) {
1885 // Might be refuted, which wouldn't be harmful.
1886 CLEAR_ERROR_OCCURRED(tstate);
1887 }
1888
1889 return callIntoInstalledExtensionModule(tstate, module_name, extension_module_filename);
1890 }
1891 }
1892#endif
1893
1894#if _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1895 struct Nuitka_MetaPathBasedLoaderEntry *entry = NULL;
1896#else
1897 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1898#endif
1899
1900 if ((entry != NULL) && ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0)) {
1901 Py_INCREF(module);
1902
1903 if (unlikely(!PyModule_Check(module))) {
1904 return module;
1905 }
1906
1907 PyModuleDef *def = PyModule_GetDef(module);
1908 if (def == NULL) {
1909 return module;
1910 }
1911
1912 void *state = PyModule_GetState(module);
1913 if (state != NULL) {
1914 return module;
1915 }
1916
1917 res = PyModule_ExecDef(module, def);
1918
1919 if (unlikely(res == -1)) {
1920 Py_DECREF(module);
1921
1922 return NULL;
1923 }
1924
1925 CHECK_OBJECT(module);
1926
1927 return module;
1928 }
1929
1930 Py_DECREF(module_name);
1931
1932 return EXECUTE_EMBEDDED_MODULE(tstate, module);
1933}
1934
1935#if PYTHON_VERSION >= 0x370
1936
1937// The resource reader class is implemented in a separate file.
1938#include "MetaPathBasedLoaderResourceReader.c"
1939
1940static PyObject *_nuitka_loader_get_resource_reader(PyObject *self, PyObject *args, PyObject *kwds) {
1941 PyObject *module_name;
1942
1943 int res =
1944 PyArg_ParseTupleAndKeywords(args, kwds, "O:get_resource_reader", (char **)_kw_list_exec_module, &module_name);
1945
1946 if (unlikely(res == 0)) {
1947 return NULL;
1948 }
1949
1950 char const *name = Nuitka_String_AsString(module_name);
1951
1952 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1953
1954 if (entry) {
1955 if (isVerbose()) {
1956 PySys_WriteStderr("import %s # get_resource_reader (%s)\n", name, getEntryModeString(entry));
1957 }
1958
1959 return Nuitka_ResourceReader_New(entry);
1960 }
1961
1962 PyErr_Format(PyExc_RuntimeError, "Requested resource reader for unhandled module %s", module_name);
1963 return NULL;
1964}
1965
1966#endif
1967
1968#endif
1969
1970#endif
1971
1972static char const *_kw_list_find_distributions[] = {"context", NULL};
1973
1974static PyObject *_nuitka_loader_find_distributions(PyObject *self, PyObject *args, PyObject *kwds) {
1975 PyObject *context;
1976
1977 int res =
1978 PyArg_ParseTupleAndKeywords(args, kwds, "O:find_distributions", (char **)_kw_list_find_distributions, &context);
1979
1980 if (unlikely(res == 0)) {
1981 return NULL;
1982 }
1983
1984 PyObject *name = PyObject_GetAttr(context, const_str_plain_name);
1985
1986 if (unlikely(name == NULL)) {
1987 return NULL;
1988 }
1989
1990 PyThreadState *tstate = PyThreadState_GET();
1991
1992 PyObject *temp = MAKE_LIST_EMPTY(tstate, 0);
1993
1994 Py_ssize_t pos = 0;
1995 PyObject *distribution_name;
1996
1997 while (Nuitka_DistributionNext(&pos, &distribution_name)) {
1998 bool include = false;
1999 if (name == Py_None) {
2000 include = true;
2001 } else {
2002 nuitka_bool cmp_res = RICH_COMPARE_EQ_NBOOL_OBJECT_OBJECT(name, distribution_name);
2003
2004 if (unlikely(cmp_res == NUITKA_BOOL_EXCEPTION)) {
2005 Py_DECREF(name);
2006 return NULL;
2007 }
2008
2009 include = cmp_res == NUITKA_BOOL_TRUE;
2010 }
2011
2012 if (include) {
2013 // Create a distribution object from our data.
2014 PyObject *distribution = Nuitka_Distribution_New(tstate, distribution_name);
2015
2016 if (distribution == NULL) {
2017 Py_DECREF(temp);
2018 return NULL;
2019 }
2020
2021 LIST_APPEND1(temp, distribution);
2022 }
2023 }
2024
2025 // We are expected to return an iterator.
2026 PyObject *result = MAKE_ITERATOR_INFALLIBLE(temp);
2027
2028 Py_DECREF(temp);
2029 return result;
2030}
2031
2032static char const *_kw_list_sys_path_hook[] = {"path", NULL};
2033
2034static PyObject *_nuitka_loader_sys_path_hook(PyObject *self, PyObject *args, PyObject *kwds) {
2035 PyObject *path;
2036
2037 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:sys_path_hook", (char **)_kw_list_sys_path_hook, &path);
2038
2039 if (unlikely(res == 0)) {
2040 return NULL;
2041 }
2042
2043#if 0
2044 PRINT_STRING("CONSIDER PATH:");
2045 PRINT_ITEM(path);
2046 PRINT_NEW_LINE();
2047#endif
2048
2049 struct Nuitka_MetaPathBasedLoaderEntry *entry = loader_entries;
2050 assert(entry);
2051
2052 PyThreadState *tstate = PyThreadState_GET();
2053
2054 while (entry->name != NULL) {
2055 if ((entry->flags & NUITKA_TRANSLATED_FLAG) != 0) {
2056 entry->name = UN_TRANSLATE(entry->name);
2057 entry->flags -= NUITKA_TRANSLATED_FLAG;
2058 }
2059
2060 if ((entry->flags & NUITKA_PACKAGE_FLAG) != 0) {
2061 PyObject *module_directory = getModuleDirectory(tstate, entry);
2062
2063#if 0
2064 PRINT_STRING(entry->name);
2065 PRINT_STRING(" ");
2066 PRINT_ITEM(module_directory);
2067 PRINT_NEW_LINE();
2068#endif
2069
2070 nuitka_bool cmp_res = compareFilePaths(tstate, module_directory, path);
2071
2072 if (unlikely(cmp_res == NUITKA_BOOL_EXCEPTION)) {
2073 return NULL;
2074 }
2075
2076 // Create the loader for the module.
2077 if (cmp_res == NUITKA_BOOL_TRUE) {
2078 return Nuitka_Loader_New(entry);
2079 }
2080 }
2081
2082 entry++;
2083 }
2084
2085 SET_CURRENT_EXCEPTION_TYPE0(tstate, PyExc_ImportError);
2086 return NULL;
2087}
2088
2089static PyMethodDef Nuitka_Loader_methods[] = {
2090 {"iter_modules", (PyCFunction)_nuitka_loader_iter_modules, METH_VARARGS | METH_KEYWORDS, NULL},
2091 {"get_data", (PyCFunction)_nuitka_loader_get_data, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2092 {"find_module", (PyCFunction)_nuitka_loader_find_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2093 {"load_module", (PyCFunction)_nuitka_loader_load_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2094 {"is_package", (PyCFunction)_nuitka_loader_is_package, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2095#if PYTHON_VERSION >= 0x300
2096 {"module_repr", (PyCFunction)_nuitka_loader_repr_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2097 {"find_spec", (PyCFunction)_nuitka_loader_find_spec, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2098#endif
2099#if PYTHON_VERSION >= 0x350
2100 {"create_module", (PyCFunction)_nuitka_loader_create_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2101 {"exec_module", (PyCFunction)_nuitka_loader_exec_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2102#endif
2103#if PYTHON_VERSION >= 0x370
2104 {"get_resource_reader", (PyCFunction)_nuitka_loader_get_resource_reader, METH_STATIC | METH_VARARGS | METH_KEYWORDS,
2105 NULL},
2106#endif
2107
2108 {"find_distributions", (PyCFunction)_nuitka_loader_find_distributions, METH_STATIC | METH_VARARGS | METH_KEYWORDS,
2109 NULL},
2110
2111 {"sys_path_hook", (PyCFunction)_nuitka_loader_sys_path_hook, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2112
2113 {NULL, NULL} // terminator
2114};
2115
2116static PyObject *Nuitka_Loader_tp_repr(struct Nuitka_LoaderObject *loader) {
2117 if (loader->m_loader_entry == NULL) {
2118 // TODO: Indicate in module mode, which one it is for.
2119 return Nuitka_String_FromString("<nuitka_module_loader>");
2120 } else {
2121 return Nuitka_String_FromFormat("<nuitka_module_loader for '%s'>", loader->m_loader_entry->name);
2122 }
2123}
2124
2125#include "nuitka/freelists.h"
2126
2127// TODO: A free list is not the right thing for those, they are probably living forever, but it's
2128// no big harm too, but make it small, maybe be allowing a toggle that makes a specific macro not
2129// use the free list mechanism at all.
2130
2131// Freelist setup
2132#define MAX_LOADER_FREE_LIST_COUNT 10
2133static struct Nuitka_LoaderObject *free_list_loaders = NULL;
2134static int free_list_loaders_count = 0;
2135
2136static void Nuitka_Loader_tp_dealloc(struct Nuitka_LoaderObject *loader) {
2137 Nuitka_GC_UnTrack(loader);
2138
2139 releaseToFreeList(free_list_loaders, loader, MAX_LOADER_FREE_LIST_COUNT);
2140}
2141
2142static int Nuitka_Loader_tp_traverse(struct Nuitka_LoaderObject *loader, visitproc visit, void *arg) { return 0; }
2143
2144static PyObject *Nuitka_Loader_get_name(struct Nuitka_LoaderObject *loader, void *closure) {
2145 PyObject *result = Nuitka_String_FromString(loader->m_loader_entry->name);
2146
2147 return result;
2148}
2149static PyObject *Nuitka_Loader_get_path(struct Nuitka_LoaderObject *loader, void *closure) {
2150 PyThreadState *tstate = PyThreadState_GET();
2151 PyObject *result = getModuleFileValue(tstate, loader->m_loader_entry);
2152
2153 return result;
2154}
2155
2156static PyObject *Nuitka_Loader_get__module__(struct Nuitka_LoaderObject *loader, void *closure) {
2157 PyObject *result = const_str_plain___nuitka__;
2158
2159 Py_INCREF_IMMORTAL(result);
2160 return result;
2161}
2162
2163static PyGetSetDef Nuitka_Loader_tp_getset[] = {{(char *)"__module__", (getter)Nuitka_Loader_get__module__, NULL, NULL},
2164 {(char *)"name", (getter)Nuitka_Loader_get_name, NULL, NULL},
2165 {(char *)"path", (getter)Nuitka_Loader_get_path, NULL, NULL},
2166 {NULL}};
2167
2168PyTypeObject Nuitka_Loader_Type = {
2169 PyVarObject_HEAD_INIT(NULL, 0) "nuitka_module_loader",
2170 sizeof(struct Nuitka_LoaderObject), // tp_basicsize
2171 0, // tp_itemsize
2172 (destructor)Nuitka_Loader_tp_dealloc, // tp_dealloc
2173 0, // tp_print
2174 0, // tp_getattr
2175 0, // tp_setattr
2176 0, // tp_reserved
2177 (reprfunc)Nuitka_Loader_tp_repr, // tp_repr
2178 0, // tp_as_number
2179 0, // tp_as_sequence
2180 0, // tp_as_mapping
2181 0, // tp_hash
2182 0, // tp_call
2183 0, // tp_str
2184 0, // tp_getattro (PyObject_GenericGetAttr)
2185 0, // tp_setattro
2186 0, // tp_as_buffer
2187 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags
2188 0, // tp_doc
2189 (traverseproc)Nuitka_Loader_tp_traverse, // tp_traverse
2190 0, // tp_clear
2191 0, // tp_richcompare
2192 0, // tp_weaklistoffset
2193 0, // tp_iter
2194 0, // tp_iternext
2195 Nuitka_Loader_methods, // tp_methods
2196 0, // tp_members
2197 Nuitka_Loader_tp_getset, // tp_getset
2198};
2199
2200/* Used by modules to register child loaders for packages. */
2201PyObject *Nuitka_Loader_New(struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
2202 struct Nuitka_LoaderObject *result;
2203
2204 allocateFromFreeListFixed(free_list_loaders, struct Nuitka_LoaderObject, Nuitka_Loader_Type);
2205 Nuitka_GC_Track(result);
2206
2207 result->m_loader_entry = entry;
2208
2209 return (PyObject *)result;
2210}
2211
2212#if _NUITKA_MODULE_MODE
2213void updateMetaPathBasedLoaderModuleRoot(char const *module_root_name) {
2214 assert(module_root_name != NULL);
2215 char const *last_dot = strrchr(module_root_name, '.');
2216
2217 if (last_dot != NULL) {
2218 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
2219 assert(current);
2220
2221 while (current->name != NULL) {
2222 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
2223 current->name = UN_TRANSLATE(current->name);
2224 current->flags -= NUITKA_TRANSLATED_FLAG;
2225 }
2226
2227 char name[2048];
2228
2229 if (strcmp(last_dot + 1, current->name) == 0) {
2230 copyStringSafeN(name, module_root_name, last_dot - module_root_name + 1, sizeof(name));
2231 appendStringSafe(name, current->name, sizeof(name));
2232
2233 current->name = strdup(name);
2234 } else if (strncmp(last_dot + 1, current->name, strlen(last_dot + 1)) == 0 &&
2235 current->name[strlen(last_dot + 1)] == '.') {
2236 copyStringSafeN(name, module_root_name, last_dot - module_root_name + 1, sizeof(name));
2237 appendStringSafe(name, current->name, sizeof(name));
2238
2239 current->name = strdup(name);
2240 }
2241
2242 current++;
2243 }
2244 }
2245}
2246#endif
2247
2248void registerMetaPathBasedLoader(struct Nuitka_MetaPathBasedLoaderEntry *_loader_entries,
2249 unsigned char **bytecode_data) {
2250 // Do it only once.
2251 if (loader_entries) {
2252 assert(_loader_entries == loader_entries);
2253
2254 return;
2255 }
2256
2257 _bytecode_data = (char **)bytecode_data;
2258
2259 if (isVerbose()) {
2260 PySys_WriteStderr("Setup nuitka compiled module/bytecode/extension importer.\n");
2261 }
2262
2263 loader_entries = _loader_entries;
2264
2265#if _NUITKA_MODULE_MODE && PYTHON_VERSION < 0x3c0
2266 if (_Py_PackageContext != NULL) {
2267 updateMetaPathBasedLoaderModuleRoot(_Py_PackageContext);
2268 }
2269#endif
2270
2271 Nuitka_PyType_Ready(&Nuitka_Loader_Type, NULL, true, false, false, false, false);
2272
2273#if PYTHON_VERSION >= 0x370
2274 Nuitka_PyType_Ready(&Nuitka_ResourceReader_Type, NULL, true, false, false, false, false);
2275#endif
2276
2277 // Register it as a meta path loader.
2278 PyObject *global_loader = Nuitka_Loader_New(NULL);
2279
2280 LIST_INSERT_CONST(Nuitka_SysGetObject("meta_path"),
2281#if PYTHON_VERSION < 0x300
2282 0,
2283#else
2284 2,
2285#endif
2286
2287 global_loader);
2288
2289 // Our "sys.path_hooks" entry uses "os.path" to compare filenames, so we need
2290 // to load it without.
2291 PyThreadState *tstate = PyThreadState_GET();
2292 IMPORT_HARD_OS_PATH(tstate);
2293
2294 // Register our loader as a "sys.path_hook" so it can provide for the paths.
2295#if _NUITKA_STANDALONE_MODE
2296 LIST_INSERT_CONST(Nuitka_SysGetObject("path_hooks"), 0, PyObject_GetAttrString(global_loader, "sys_path_hook"));
2297#else
2298 // TODO: For accelerated mode the loader doesn't have the ability
2299 // to handle "site-packages" on its own yet, and therefore we
2300 // cannot have it take precedence.
2301 LIST_APPEND1(Nuitka_SysGetObject("path_hooks"), PyObject_GetAttrString(global_loader, "sys_path_hook"));
2302#endif
2303}
2304
2305#if _NUITKA_STANDALONE_MODE
2306// This is called for the technical module imported early on during interpreter
2307// into, to still get compatible "__file__" attributes.
2308void setEarlyFrozenModulesFileAttribute(PyThreadState *tstate) {
2309 PyObject *sys_modules = Nuitka_GetSysModules();
2310 Py_ssize_t pos = 0;
2311 PyObject *key, *value;
2312
2313 PyObject *builtin_module_names = Nuitka_SysGetObject("builtin_module_names");
2314
2315 while (Nuitka_DictNext(sys_modules, &pos, &key, &value)) {
2316 if (key != NULL && value != NULL && PyModule_Check(value)) {
2317 bool is_package = HAS_ATTR_BOOL(tstate, value, const_str_plain___path__);
2318
2319 if (is_package || HAS_ATTR_BOOL(tstate, value, const_str_plain___file__) ||
2320 PySequence_Contains(builtin_module_names, key) == 0) {
2321 PyObject *file_value = MAKE_RELATIVE_PATH_FROM_NAME(Nuitka_String_AsString(key), is_package, false);
2322 PyObject_SetAttr(value, const_str_plain___file__, file_value);
2323 Py_DECREF(file_value);
2324 CHECK_OBJECT(file_value);
2325 }
2326 }
2327 }
2328
2329 assert(!HAS_ERROR_OCCURRED(tstate));
2330}
2331
2332#endif
2333
2334// The importlib distribution class is implemented in a separate file.
2335#include "MetaPathBasedLoaderImportlibMetadataDistribution.c"
2336
2337// Part of "Nuitka", an optimizing Python compiler that is compatible and
2338// integrates with CPython, but also works on its own.
2339//
2340// Licensed under the Apache License, Version 2.0 (the "License");
2341// you may not use this file except in compliance with the License.
2342// You may obtain a copy of the License at
2343//
2344// http://www.apache.org/licenses/LICENSE-2.0
2345//
2346// Unless required by applicable law or agreed to in writing, software
2347// distributed under the License is distributed on an "AS IS" BASIS,
2348// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2349// See the License for the specific language governing permissions and
2350// limitations under the License.
Nuitka_ExceptionPreservationItem
Definition exceptions.h:712
Nuitka_LoaderObject
Definition MetaPathBasedLoader.c:28
Nuitka_MetaPathBasedLoaderEntry
Definition unfreezing.h:31