Nuitka
The Python compiler
Loading...
Searching...
No Matches
MetaPathBasedLoader.c
1// Copyright 2025, Kay Hayen, mailto:kay.hayen@gmail.com find license text at end of file
2
3// This implements the loading of C compiled modules and shared library
4// extension modules bundled for standalone mode.
5
6// This is achieved mainly by registered a "sys.meta_path" loader, that then
7// gets asked for module names, and responds if knows about one. It's fed by
8// a table created at compile time.
9
10// The nature and use of these 2 loaded module kinds is very different, but
11// having them as distinct loaders would only require to duplicate the search
12// and registering of stuff.
13
14// This file is included from another C file, help IDEs to still parse it on
15// its own.
16#ifdef __IDE_ONLY__
17#include "nuitka/prelude.h"
18#endif
19
20#include "nuitka/unfreezing.h"
21
22#ifdef _WIN32
23#include <windows.h>
24#endif
25
26extern PyTypeObject Nuitka_Loader_Type;
27
28struct Nuitka_LoaderObject {
29 /* Python object folklore: */
30 PyObject_HEAD
31
32 /* The loader entry, to know what was loaded exactly. */
33 struct Nuitka_MetaPathBasedLoaderEntry const *m_loader_entry;
34};
35
36#if _NUITKA_EXE_MODE
37static inline bool isVerbose(void) { return Py_VerboseFlag != 0; }
38#elif _NUITKA_SYSFLAG_VERBOSE
39static inline bool isVerbose(void) { return true; }
40#else
41static inline bool isVerbose(void) { return false; }
42#endif
43
44static struct Nuitka_MetaPathBasedLoaderEntry *loader_entries = NULL;
45
46static bool hasFrozenModule(char const *name) {
47 for (struct _frozen const *p = PyImport_FrozenModules; p != NULL; p++) {
48 if (p->name == NULL) {
49 return false;
50 }
51
52 if (strcmp(p->name, name) == 0) {
53 break;
54 }
55 }
56
57 return true;
58}
59
60static char *appendModuleNameAsPath(char *buffer, char const *module_name, size_t buffer_size) {
61 // Skip to the end
62 while (*buffer != 0) {
63 buffer++;
64 buffer_size -= 1;
65 }
66
67 while (*module_name) {
68 if (buffer_size < 1) {
69 abort();
70 }
71
72 if (*module_name == '.') {
73 *buffer++ = SEP;
74 module_name++;
75 } else {
76 *buffer++ = *module_name++;
77 }
78
79 buffer_size -= 1;
80 }
81
82 *buffer = 0;
83
84 return buffer;
85}
86
87#if defined(_WIN32) && _NUITKA_STANDALONE_MODE
88
89static void appendModuleNameAsPathW(wchar_t *buffer, PyObject *module_name, size_t buffer_size) {
90 Py_ssize_t size;
91 wchar_t const *module_name_wstr = Nuitka_UnicodeAsWideString(module_name, &size);
92
93 while (size > 0) {
94 wchar_t c = *module_name_wstr++;
95 size -= 1;
96
97 if (c == L'.') {
98 c = FILENAME_SEP_CHAR;
99 }
100
101 appendWCharSafeW(buffer, c, buffer_size);
102 }
103}
104#endif
105
106// TODO: This updates the wrong absolute path. We ought to change it to
107// the "module_path_name" at the time of writing it, then we save a few
108// bytes in the blob, and don't have to create that string here.
109#if _NUITKA_STANDALONE_MODE
110static void patchCodeObjectPaths(PyCodeObject *code_object, PyObject *module_path) {
111 code_object->co_filename = module_path;
112 Py_INCREF(module_path);
113
114 Py_ssize_t consts_count = PyTuple_GET_SIZE(code_object->co_consts);
115
116 for (Py_ssize_t i = 0; i < consts_count; i++) {
117 PyObject *constant = PyTuple_GET_ITEM(code_object->co_consts, i);
118
119 if (PyCode_Check(constant)) {
120 patchCodeObjectPaths((PyCodeObject *)constant, module_path);
121 }
122 }
123}
124#endif
125
126NUITKA_MAY_BE_UNUSED static PyObject *MAKE_RELATIVE_PATH_FROM_NAME(char const *name, bool is_package, bool dir_only) {
127 char buffer[MAXPATHLEN + 1] = {0};
128
129 appendModuleNameAsPath(buffer, name, sizeof(buffer));
130
131 if (dir_only == false) {
132 if (is_package) {
133 appendCharSafe(buffer, SEP, sizeof(buffer));
134 appendStringSafe(buffer, "__init__.py", sizeof(buffer));
135 } else {
136 appendStringSafe(buffer, ".py", sizeof(buffer));
137 }
138 } else {
139 if (is_package == false) {
140 char *sep = strrchr(buffer, SEP);
141 if (sep) {
142 *sep = 0;
143 } else {
144 buffer[0] = '.';
145 buffer[1] = 0;
146 }
147 }
148 }
149
150 PyObject *module_path_entry_base = Nuitka_String_FromString(buffer);
151
152 PyObject *result = MAKE_RELATIVE_PATH(module_path_entry_base);
153
154 Py_DECREF(module_path_entry_base);
155
156 return result;
157}
158
159static PyObject *_makeDunderPathObject(PyThreadState *tstate, PyObject *module_path_entry) {
160 CHECK_OBJECT(module_path_entry);
161
162 PyObject *path_list = MAKE_LIST_EMPTY(tstate, 1);
163 if (unlikely(path_list == NULL)) {
164 return NULL;
165 }
166
167 PyList_SET_ITEM0(path_list, 0, module_path_entry);
168
169 CHECK_OBJECT(path_list);
170 return path_list;
171}
172
173static PyObject *loadModuleFromCodeObject(PyThreadState *tstate, PyObject *module, PyCodeObject *code_object,
174 char const *name, bool is_package) {
175 assert(code_object != NULL);
176
177 {
178 NUITKA_MAY_BE_UNUSED bool b_res = Nuitka_SetModuleString(name, module);
179 assert(b_res != false);
180 }
181
182 char buffer[MAXPATHLEN + 1] = {0};
183
184 PyObject *module_path_entry = NULL;
185
186 if (is_package) {
187 appendModuleNameAsPath(buffer, name, sizeof(buffer));
188 PyObject *module_path_entry_base = Nuitka_String_FromString(buffer);
189
190 module_path_entry = MAKE_RELATIVE_PATH(module_path_entry_base);
191 Py_DECREF(module_path_entry_base);
192
193 appendCharSafe(buffer, SEP, sizeof(buffer));
194 appendStringSafe(buffer, "__init__.py", sizeof(buffer));
195 } else {
196 appendModuleNameAsPath(buffer, name, sizeof(buffer));
197 appendStringSafe(buffer, ".py", sizeof(buffer));
198 }
199
200 PyObject *module_path_name = Nuitka_String_FromString(buffer);
201
202 PyObject *module_path = MAKE_RELATIVE_PATH(module_path_name);
203 Py_DECREF(module_path_name);
204
205 if (is_package) {
206 /* Set __path__ properly, unlike frozen module importer does. */
207 PyObject *path_list = _makeDunderPathObject(tstate, module_path_entry);
208
209 int res = PyObject_SetAttr(module, const_str_plain___path__, path_list);
210 if (unlikely(res != 0)) {
211 return NULL;
212 }
213
214 Py_DECREF(path_list);
215
216 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___name__);
217 CHECK_OBJECT(module_name);
218
219 res = PyObject_SetAttr(module, const_str_plain___package__, module_name);
220
221 if (unlikely(res != 0)) {
222 return NULL;
223 }
224 }
225
226#if _NUITKA_STANDALONE_MODE
227 patchCodeObjectPaths(code_object, module_path);
228#endif
229
230 PGO_onModuleEntered(name);
231 module = PyImport_ExecCodeModuleEx((char *)name, (PyObject *)code_object, Nuitka_String_AsString(module_path));
232 PGO_onModuleExit(name, module == NULL);
233
234 Py_DECREF(module_path);
235
236 return module;
237}
238
239static struct Nuitka_MetaPathBasedLoaderEntry *findEntry(char const *name) {
240 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
241 assert(current);
242
243 while (current->name != NULL) {
244 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
245 current->name = UN_TRANSLATE(current->name);
246 current->flags -= NUITKA_TRANSLATED_FLAG;
247 }
248
249 if (strcmp(name, current->name) == 0) {
250 return current;
251 }
252
253 current++;
254 }
255
256 return NULL;
257}
258
259#if !_NUITKA_STANDALONE_MODE
260static struct Nuitka_MetaPathBasedLoaderEntry *findContainingPackageEntry(char const *name) {
261 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
262
263 // Consider the package name of the searched entry.
264 char const *package_name_end = strrchr(name, '.');
265 if (package_name_end == NULL) {
266 return NULL;
267 }
268
269 size_t length = package_name_end - name;
270
271 while (current->name != NULL) {
272 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
273 current->name = UN_TRANSLATE(current->name);
274 current->flags -= NUITKA_TRANSLATED_FLAG;
275 }
276
277 if ((current->flags & NUITKA_PACKAGE_FLAG) != 0) {
278 if (strlen(current->name) == length && strncmp(name, current->name, length) == 0) {
279 return current;
280 }
281 }
282
283 current++;
284 }
285
286 return NULL;
287}
288
289static PyObject *_getFileList(PyThreadState *tstate, PyObject *dirname) {
290 static PyObject *listdir_func = NULL;
291
292 // TODO: Use OS_LISTDIR instead.
293
294 if (listdir_func == NULL) {
295 listdir_func = PyObject_GetAttrString(IMPORT_HARD_OS(), "listdir");
296 }
297
298 if (unlikely(listdir_func == NULL)) {
299 return NULL;
300 }
301
302 PyObject *result = CALL_FUNCTION_WITH_SINGLE_ARG(tstate, listdir_func, dirname);
303
304 return result;
305}
306
307#if PYTHON_VERSION < 0x300
308static PyObject *_getImportingSuffixesByPriority(PyThreadState *tstate, int kind) {
309 static PyObject *result = NULL;
310
311 if (result == NULL) {
312 result = MAKE_LIST_EMPTY(tstate, 0);
313
314 PyObject *imp_module = PyImport_ImportModule("imp");
315 PyObject *get_suffixes_func = PyObject_GetAttrString(imp_module, "get_suffixes");
316
317 PyObject *suffix_list = CALL_FUNCTION_NO_ARGS(tstate, get_suffixes_func);
318
319 for (int i = 0; i < PyList_GET_SIZE(suffix_list); i++) {
320 PyObject *module_kind = PyTuple_GET_ITEM(PyList_GET_ITEM(suffix_list, i), 2);
321
322 if (PyInt_AsLong(module_kind) == kind) {
323 LIST_APPEND0(result, PyTuple_GET_ITEM(PyList_GET_ITEM(suffix_list, i), 0));
324 }
325 }
326
327 Py_DECREF(suffix_list);
328 }
329
330 return result;
331}
332#endif
333
334static PyObject *getExtensionModuleSuffixesByPriority(PyThreadState *tstate) {
335 static PyObject *result = NULL;
336
337 if (result == NULL) {
338#if PYTHON_VERSION < 0x300
339 result = _getImportingSuffixesByPriority(tstate, 3);
340#else
341 static PyObject *machinery_module = NULL;
342
343 if (machinery_module == NULL) {
344 machinery_module = PyImport_ImportModule("importlib.machinery");
345 }
346
347 result = PyObject_GetAttrString(machinery_module, "EXTENSION_SUFFIXES");
348#endif
349 }
350
351 CHECK_OBJECT(result);
352 return result;
353}
354
355static PyObject *installed_extension_modules = NULL;
356
357static bool scanModuleInPackagePath(PyThreadState *tstate, PyObject *module_name, char const *parent_module_name) {
358 PyObject *sys_modules = Nuitka_GetSysModules();
359
360 PyObject *parent_module = PyDict_GetItemString(sys_modules, parent_module_name);
361 CHECK_OBJECT(parent_module);
362
363 PyObject *parent_path = PyObject_GetAttr(parent_module, const_str_plain___path__);
364
365 // Accept that it might be deleted.
366 if (parent_path == NULL || !PyList_Check(parent_path)) {
367 return false;
368 }
369
370 PyObject *candidates = MAKE_LIST_EMPTY(tstate, 0);
371
372 // Search only relative to the parent name of course.
373 char const *module_relative_name_str = Nuitka_String_AsString(module_name) + strlen(parent_module_name) + 1;
374
375 Py_ssize_t parent_path_size = PyList_GET_SIZE(parent_path);
376
377 for (Py_ssize_t i = 0; i < parent_path_size; i += 1) {
378 PyObject *path_element = PyList_GET_ITEM(parent_path, i);
379
380 PyObject *filenames_list = _getFileList(tstate, path_element);
381
382 if (filenames_list == NULL) {
383 CLEAR_ERROR_OCCURRED(tstate);
384 continue;
385 }
386
387 Py_ssize_t filenames_list_size = PyList_GET_SIZE(filenames_list);
388
389 for (Py_ssize_t j = 0; j < filenames_list_size; j += 1) {
390 PyObject *filename = PyList_GET_ITEM(filenames_list, j);
391
392 if (Nuitka_String_CheckExact(filename)) {
393 char const *filename_str = Nuitka_String_AsString(filename);
394
395 if (strncmp(filename_str, module_relative_name_str, strlen(module_relative_name_str)) == 0 &&
396 filename_str[strlen(module_relative_name_str)] == '.') {
397 LIST_APPEND1(candidates, MAKE_TUPLE2(tstate, path_element, filename));
398 }
399 }
400 }
401 }
402
403#if 0
404 PRINT_STRING("CANDIDATES:");
405 PRINT_STRING(Nuitka_String_AsString(module_name));
406 PRINT_STRING(module_relative_name_str);
407 PRINT_ITEM(candidates);
408 PRINT_NEW_LINE();
409#endif
410
411 // Look up C-extension suffixes, these are used with highest priority.
412 PyObject *suffix_list = getExtensionModuleSuffixesByPriority(tstate);
413
414 bool result = false;
415
416 for (Py_ssize_t i = 0; i < PyList_GET_SIZE(suffix_list); i += 1) {
417 PyObject *suffix = PyList_GET_ITEM(suffix_list, i);
418
419 char const *suffix_str = Nuitka_String_AsString(suffix);
420
421 for (Py_ssize_t j = 0; j < PyList_GET_SIZE(candidates); j += 1) {
422 PyObject *entry = PyList_GET_ITEM(candidates, j);
423
424 PyObject *directory = PyTuple_GET_ITEM(entry, 0);
425 PyObject *candidate = PyTuple_GET_ITEM(entry, 1);
426
427 char const *candidate_str = Nuitka_String_AsString(candidate);
428
429 if (strcmp(suffix_str, candidate_str + strlen(module_relative_name_str)) == 0) {
430 PyObject *fullpath = JOIN_PATH2(directory, candidate);
431
432 if (installed_extension_modules == NULL) {
433 installed_extension_modules = MAKE_DICT_EMPTY(tstate);
434 }
435
436// Force path to unicode, to have easier consumption, as we need a wchar_t or char *
437// from it later, and we don't want to test there.
438#if PYTHON_VERSION < 0x300 && defined(_WIN32)
439 PyObject *tmp = PyUnicode_FromObject(fullpath);
440 Py_DECREF(fullpath);
441 fullpath = tmp;
442#endif
443
444 DICT_SET_ITEM(installed_extension_modules, module_name, fullpath);
445
446 result = true;
447 break;
448 }
449 }
450 }
451
452 Py_DECREF(candidates);
453
454 return result;
455}
456
457static PyObject *callIntoExtensionModule(PyThreadState *tstate, char const *full_name, const filename_char_t *filename,
458 bool is_package);
459
460static PyObject *callIntoInstalledExtensionModule(PyThreadState *tstate, PyObject *module_name,
461 PyObject *extension_module_filename) {
462#if _WIN32
463 // We can rely on unicode object to be there in case of Windows, to have an easier time to
464 // create the string needed.
465 assert(PyUnicode_CheckExact(extension_module_filename));
466
467 wchar_t const *extension_module_filename_str = Nuitka_UnicodeAsWideString(extension_module_filename, NULL);
468#else
469 char const *extension_module_filename_str = Nuitka_String_AsString(extension_module_filename);
470#endif
471
472 // TODO: The value of "is_package" is guessed, maybe infer from filename being
473 // a "__init__.so" and the like.
474 return callIntoExtensionModule(tstate, Nuitka_String_AsString(module_name), extension_module_filename_str, false);
475}
476
477#endif
478
479static char const *getEntryModeString(struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
480 char const *mode = "compiled";
481
482 if ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0) {
483 mode = "extension";
484 } else if ((entry->flags & NUITKA_BYTECODE_FLAG) != 0) {
485 mode = "bytecode";
486 }
487
488 return mode;
489}
490
491static char *_kw_list_find_module[] = {(char *)"fullname", (char *)"unused", NULL};
492
493static PyObject *_nuitka_loader_find_module(PyObject *self, PyObject *args, PyObject *kwds) {
494 PyObject *module_name;
495 PyObject *unused;
496
497 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O|O:find_module", (char **)_kw_list_find_module, &module_name,
498 &unused);
499
500 if (unlikely(res == 0)) {
501 return NULL;
502 }
503
504 char const *name = Nuitka_String_AsString(module_name);
505
506 if (isVerbose()) {
507 PySys_WriteStderr("import %s # considering responsibility (find_module)\n", name);
508 }
509
510 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
511
512 if (entry) {
513 if (isVerbose()) {
514 PySys_WriteStderr("import %s # claimed responsibility (%s)\n", name, getEntryModeString(entry));
515 }
516
517 PyObject *metapath_based_loader = (PyObject *)&Nuitka_Loader_Type;
518
519 Py_INCREF(metapath_based_loader);
520 return metapath_based_loader;
521 }
522
523 if (hasFrozenModule(name)) {
524 if (isVerbose()) {
525 PySys_WriteStderr("import %s # claimed responsibility (frozen)\n", name);
526 }
527
528 PyObject *metapath_based_loader = (PyObject *)&Nuitka_Loader_Type;
529
530 Py_INCREF(metapath_based_loader);
531 return metapath_based_loader;
532 }
533
534#if !_NUITKA_STANDALONE_MODE
535 entry = findContainingPackageEntry(name);
536
537 if (entry != NULL) {
538 PyThreadState *tstate = PyThreadState_GET();
539
540 bool result = scanModuleInPackagePath(tstate, module_name, entry->name);
541
542 if (result) {
543 PyObject *metapath_based_loader = (PyObject *)&Nuitka_Loader_Type;
544
545 Py_INCREF(metapath_based_loader);
546 return metapath_based_loader;
547 }
548 }
549#endif
550
551 if (isVerbose()) {
552 PySys_WriteStderr("import %s # denied responsibility\n", name);
553 }
554
555 Py_INCREF_IMMORTAL(Py_None);
556 return Py_None;
557}
558
559static char const *_kw_list_get_data[] = {"filename", NULL};
560
561static PyObject *_nuitka_loader_get_data(PyObject *self, PyObject *args, PyObject *kwds) {
562 PyObject *filename;
563
564 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:get_data", (char **)_kw_list_get_data, &filename);
565
566 if (unlikely(res == 0)) {
567 return NULL;
568 }
569
570 PyThreadState *tstate = PyThreadState_GET();
571
572 return GET_FILE_BYTES(tstate, filename);
573}
574
575static void setModuleFileValue(PyThreadState *tstate, PyObject *module, filename_char_t const *filename) {
576 CHECK_OBJECT(module);
577 assert(filename != NULL);
578
579 assert(PyModule_Check(module));
580
581 PyObject *dict = PyModule_GetDict(module);
582
583 // TODO: We should have DICT_SET_ITEM0/1 for these things.
584 PyObject *new_file_value = Nuitka_String_FromFilename(filename);
585 DICT_SET_ITEM(dict, const_str_plain___file__, new_file_value);
586 Py_DECREF(new_file_value);
587}
588
589#if PYTHON_VERSION < 0x300
590typedef void (*entrypoint_t)(void);
591#else
592typedef PyObject *(*entrypoint_t)(void);
593#endif
594
595#ifndef _WIN32
596// Shared libraries loading.
597#include <dlfcn.h>
598#endif
599
600#if PYTHON_VERSION >= 0x350
601static PyObject *createModuleSpec(PyThreadState *tstate, PyObject *module_name, PyObject *origin, bool is_package);
602#endif
603
604static void _fillExtensionModuleDllEntryFunctionName(PyThreadState *tstate, char *buffer, size_t buffer_size,
605 char const *name) {
606
607#if PYTHON_VERSION >= 0x350
608 PyObject *name_bytes_obj = PyBytes_FromString(name);
609 PyObject *name_obj = BYTES_DECODE2(tstate, name_bytes_obj, Nuitka_String_FromString("utf8"));
610 Py_DECREF(name_bytes_obj);
611
612 PyObject *name_ascii = UNICODE_ENCODE2(tstate, name_obj, const_str_plain_ascii);
613
614 if (name_ascii == NULL) {
615 DROP_ERROR_OCCURRED(tstate);
616
617 PyObject *name_punycode = UNICODE_ENCODE2(tstate, name_obj, const_str_plain_punycode);
618
619 CHECK_OBJECT(name_punycode);
620
621 snprintf(buffer, buffer_size, "PyInitU_%s", PyBytes_AsString(name_punycode));
622
623 Py_DECREF(name_punycode);
624 } else {
625 Py_DECREF(name_ascii);
626
627 snprintf(buffer, buffer_size, "PyInit_%s", name);
628 }
629 Py_DECREF(name_obj);
630#else
631
632 snprintf(buffer, buffer_size,
633#if PYTHON_VERSION < 0x300
634 "init%s",
635#else
636 "PyInit_%s",
637#endif
638 name);
639#endif
640}
641
642#if _NUITKA_STANDALONE_MODE
643// Create the filename from full path module name with dots, and translate these
644// into directory separators.
645static void _makeModuleCFilenameValue(filename_char_t *filename, size_t filename_size, char const *module_name_cstr,
646 PyObject *module_name, bool is_package) {
647#ifdef _WIN32
648#if _NUITKA_ONEFILE_DLL_MODE
649 filename_char_t const *base_directory = getDllDirectory();
650#else
651 filename_char_t const *base_directory = getBinaryDirectoryWideChars(true);
652#endif
653 copyStringSafeW(filename, base_directory, filename_size);
654 appendWCharSafeW(filename, FILENAME_SEP_CHAR, filename_size);
655 appendModuleNameAsPathW(filename, module_name, filename_size);
656 if (is_package) {
657 appendWCharSafeW(filename, FILENAME_SEP_CHAR, filename_size);
658 appendStringSafeW(filename, "__init__", filename_size);
659 }
660 appendStringSafeW(filename, ".pyd", filename_size);
661#else
662#if _NUITKA_ONEFILE_DLL_MODE
663 filename_char_t const *base_directory = getDllDirectory();
664#else
665 filename_char_t const *base_directory = getBinaryDirectoryHostEncoded(true);
666#endif
667 copyStringSafe(filename, base_directory, filename_size);
668 appendCharSafe(filename, FILENAME_SEP_CHAR, filename_size);
669 appendModuleNameAsPath(filename, module_name_cstr, filename_size);
670 if (is_package) {
671 appendCharSafe(filename, FILENAME_SEP_CHAR, filename_size);
672 appendStringSafe(filename, "__init__", filename_size);
673 }
674 appendStringSafe(filename, ".so", filename_size);
675#endif
676}
677#endif
678
679#if PYTHON_VERSION >= 0x3c0 && defined(_NUITKA_USE_UNEXPOSED_API)
680extern _Thread_local const char *pkgcontext;
681#endif
682
683static const char *NuitkaImport_SwapPackageContext(const char *new_context) {
684// TODO: The locking APIs for 3.13 give errors here that are not explained
685// yet.
686#if PYTHON_VERSION >= 0x3c0
687 // spell-checker: ignore pkgcontext
688 const char *old_context = _PyRuntime.imports.pkgcontext;
689 _PyRuntime.imports.pkgcontext = new_context;
690#if PYTHON_VERSION >= 0x3c0 && defined(_NUITKA_USE_UNEXPOSED_API)
691 pkgcontext = new_context;
692#endif
693 return old_context;
694#elif PYTHON_VERSION >= 0x370
695 char const *old_context = _Py_PackageContext;
696 _Py_PackageContext = (char *)new_context;
697 return old_context;
698#else
699 char *old_context = _Py_PackageContext;
700 _Py_PackageContext = (char *)new_context;
701 return (char const *)old_context;
702#endif
703}
704
705static PyObject *callIntoExtensionModule(PyThreadState *tstate, char const *full_name, const filename_char_t *filename,
706 bool is_package) {
707 // Determine the package name and basename of the module to load.
708 char const *dot = strrchr(full_name, '.');
709 char const *name;
710 char const *package;
711
712 if (dot == NULL) {
713 package = NULL;
714 name = full_name;
715 } else {
716 // The extension modules do expect it to be full name in context.
717 package = (char *)full_name;
718 name = dot + 1;
719 }
720
721 char entry_function_name[1024];
722 _fillExtensionModuleDllEntryFunctionName(tstate, entry_function_name, sizeof(entry_function_name), name);
723
724#ifdef _WIN32
725 if (isVerbose()) {
726 PySys_WriteStderr("import %s # LoadLibraryExW(\"%S\");\n", full_name, filename);
727 }
728
729#ifndef _NUITKA_EXPERIMENTAL_DEBUG_STANDALONE
730 // Disable all but critical errors, prevents dialogs from showing.
731 // spell-checker: ignore SEM_FAILCRITICALERRORS
732 unsigned int old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
733#endif
734
735 HINSTANCE hDLL;
736#if PYTHON_VERSION >= 0x380
737 Py_BEGIN_ALLOW_THREADS;
738 hDLL = LoadLibraryExW(filename, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
739 Py_END_ALLOW_THREADS;
740#else
741 hDLL = LoadLibraryExW(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
742#endif
743
744#ifndef _NUITKA_EXPERIMENTAL_DEBUG_STANDALONE
745 SetErrorMode(old_mode);
746#endif
747
748 if (unlikely(hDLL == NULL)) {
749 char buffer[1024];
750
751 char error_message[1024];
752 int size;
753
754 unsigned int error_code = GetLastError();
755
756 size = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, error_code, 0,
757 (LPTSTR)error_message, sizeof(error_message), NULL);
758
759 // Report either way even if failed to get error message.
760 if (size == 0) {
761 int ret = PyOS_snprintf(buffer, sizeof(buffer), "LoadLibraryExW '%S' failed with error code %d", filename,
762 error_code);
763
764 assert(ret >= 0);
765 } else {
766 // Strip trailing newline.
767 if (size >= 2 && error_message[size - 2] == '\r' && error_message[size - 1] == '\n') {
768 size -= 2;
769 error_message[size] = '\0';
770 }
771 int ret = PyOS_snprintf(buffer, sizeof(buffer), "LoadLibraryExW '%S' failed: %s", filename, error_message);
772 assert(ret >= 0);
773 }
774
775 SET_CURRENT_EXCEPTION_TYPE0_STR(tstate, PyExc_ImportError, buffer);
776 return NULL;
777 }
778
779 entrypoint_t entrypoint = (entrypoint_t)GetProcAddress(hDLL, entry_function_name);
780#else
781 // This code would work for all versions, we are avoiding access to interpreter
782 // structure internals of 3.8 or higher.
783 // spell-checker: ignore getdlopenflags,dlopenflags
784
785#ifdef __wasi__
786 const char *error = "dynamic libraries are not implemented in wasi";
787 SET_CURRENT_EXCEPTION_TYPE0_STR(tstate, PyExc_ImportError, error);
788 return NULL;
789
790 entrypoint_t entrypoint = NULL;
791#else
792 static PyObject *dlopenflags_object = NULL;
793 if (dlopenflags_object == NULL) {
794 dlopenflags_object = CALL_FUNCTION_NO_ARGS(tstate, Nuitka_SysGetObject("getdlopenflags"));
795 }
796 int dlopenflags = PyInt_AsLong(dlopenflags_object);
797
798 if (isVerbose()) {
799 PySys_WriteStderr("import %s # dlopen(\"%s\", %x);\n", full_name, filename, dlopenflags);
800 }
801
802 void *handle = dlopen(filename, dlopenflags);
803
804 if (unlikely(handle == NULL)) {
805 const char *error = dlerror();
806
807 if (unlikely(error == NULL)) {
808 error = "unknown dlopen() error";
809 }
810
811 SET_CURRENT_EXCEPTION_TYPE0_STR(tstate, PyExc_ImportError, error);
812 return NULL;
813 }
814
815 entrypoint_t entrypoint = (entrypoint_t)dlsym(handle, entry_function_name);
816#endif // __wasi__
817#endif
818 assert(entrypoint);
819
820 char const *old_context = NuitkaImport_SwapPackageContext(package);
821
822#if PYTHON_VERSION >= 0x3c0 && !defined(_NUITKA_USE_UNEXPOSED_API)
823 char const *base_name = strrchr(full_name, '.');
824 PyObject *base_name_obj = NULL;
825 PyObject *prefix_name_obj = NULL;
826 PyObject *preserved_basename_module = NULL;
827
828 if (base_name != NULL) {
829 base_name_obj = Nuitka_String_FromString(base_name + 1);
830 preserved_basename_module = Nuitka_GetModule(tstate, base_name_obj);
831 Py_XINCREF(preserved_basename_module);
832
833 prefix_name_obj = Nuitka_String_FromStringAndSize(full_name, base_name - full_name + 1);
834 }
835#endif
836
837 // Finally call into the DLL.
838 PGO_onModuleEntered(full_name);
839
840 if (isVerbose()) {
841 PySys_WriteStderr("import %s # calling entrypoint\n", full_name);
842 }
843
844 Nuitka_DelModuleString(tstate, full_name);
845
846#if PYTHON_VERSION < 0x300
847 (*entrypoint)();
848#else
849 PyObject *module = (*entrypoint)();
850#endif
851
852 if (isVerbose()) {
853 PySys_WriteStderr("import %s # return from entrypoint\n", full_name);
854 }
855
856#if 0
857 PRINT_FORMAT("import %s # create module as ", full_name);
858 PRINT_ITEM(module);
859 PRINT_NEW_LINE();
860#endif
861
862 NuitkaImport_SwapPackageContext(old_context);
863
864#if PYTHON_VERSION < 0x300
865 PyObject *module = Nuitka_GetModuleString(tstate, full_name);
866#endif
867
868 PGO_onModuleExit(name, module == NULL);
869
870 if (unlikely(module == NULL)) {
871 if (unlikely(!HAS_ERROR_OCCURRED(tstate))) {
872 PyErr_Format(PyExc_SystemError, "dynamic module '%s' not initialized properly", full_name);
873 }
874
875 return NULL;
876 }
877
878#if PYTHON_VERSION >= 0x300
879#if PYTHON_VERSION >= 0x350
880 PyModuleDef *def;
881
882 if (Py_TYPE(module) == &PyModuleDef_Type) {
883 if (isVerbose()) {
884 PySys_WriteStderr("import %s # entrypoint returned module def\n", full_name);
885 }
886
887 def = (PyModuleDef *)module;
888
889 PyObject *full_name_obj = Nuitka_String_FromString(full_name);
890
891 PyObject *origin = Nuitka_String_FromFilename(filename);
892
893 PyObject *spec_value = createModuleSpec(tstate, full_name_obj, origin, is_package);
894 CHECK_OBJECT(spec_value);
895
896 module = PyModule_FromDefAndSpec(def, spec_value);
897
898 if (unlikely(module == NULL)) {
899 Py_DECREF(spec_value);
900
901 PyErr_Format(PyExc_SystemError, "dynamic module '%s' not initialized properly from def", full_name);
902
903 return NULL;
904 }
905
906 SET_ATTRIBUTE(tstate, module, const_str_plain___spec__, spec_value);
907
908 setModuleFileValue(tstate, module, filename);
909
910 /* Set __path__ properly, unlike frozen module importer does. */
911 PyObject *path_list = _makeDunderPathObject(tstate, origin);
912
913 int res = PyObject_SetAttr(module, const_str_plain___path__, path_list);
914 if (unlikely(res != 0)) {
915 return NULL;
916 }
917
918 Py_DECREF(path_list);
919
920 Nuitka_SetModule(full_name_obj, module);
921 Py_DECREF(full_name_obj);
922
923#if _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
924 SET_ATTRIBUTE(tstate, spec_value, const_str_plain__initializing, Py_True);
925 res = PyModule_ExecDef(module, def);
926 SET_ATTRIBUTE(tstate, spec_value, const_str_plain__initializing, Py_False);
927
928 Py_DECREF(spec_value);
929 CHECK_OBJECT(spec_value);
930
931 if (unlikely(res == -1)) {
932 return NULL;
933 }
934
935 if (isVerbose()) {
936 PySys_WriteStderr("import %s # executed module def\n", full_name);
937 }
938#endif
939
940 CHECK_OBJECT(module);
941
942 return module;
943 } else {
944 if (isVerbose()) {
945 PySys_WriteStderr("import %s # entrypoint returned module\n", full_name);
946 }
947
948 def = PyModule_GetDef(module);
949
950 def->m_base.m_init = entrypoint;
951
952 // Set "__spec__" and "__file__" after load.
953 setModuleFileValue(tstate, module, filename);
954 PyObject *full_name_obj = Nuitka_String_FromString(full_name);
955 SET_ATTRIBUTE(tstate, module, const_str_plain___name__, full_name_obj);
956 PyObject *spec_value =
957 createModuleSpec(tstate, full_name_obj, LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___file__), false);
958
959 SET_ATTRIBUTE(tstate, module, const_str_plain___spec__, spec_value);
960
961 // Fixup "__package__" after load. It seems some modules ignore _Py_PackageContext value.
962 // so we patch it up here if it's None, but a package was specified.
963 if (package != NULL) {
964 PyObject *package_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___package__);
965
966 if (package_name == Py_None) {
967 char package2[2048];
968 copyStringSafeN(package2, full_name, dot - full_name, sizeof(package2));
969
970 PyObject *package_name_obj = Nuitka_String_FromString(package2);
971 SET_ATTRIBUTE(tstate, module, const_str_plain___package__, package_name_obj);
972 Py_DECREF(package_name_obj);
973 }
974
975 Py_DECREF(package_name);
976 }
977 }
978
979 if (likely(def != NULL)) {
980 def->m_base.m_init = entrypoint;
981 }
982
983#else
984 PyModuleDef *def = PyModule_GetDef(module);
985
986 if (unlikely(def == NULL)) {
987 PyErr_Format(PyExc_SystemError, "initialization of %s did not return an extension module", filename);
988
989 return NULL;
990 }
991
992 def->m_base.m_init = entrypoint;
993#endif
994
995#endif
996
997 // Set filename attribute if not already set, in some branches we don't
998 // do it, esp. not for older Python.
999 setModuleFileValue(tstate, module, filename);
1000
1001 // Call the standard import fix-ups for extension modules. Their interface
1002 // changed over releases.
1003#if PYTHON_VERSION < 0x300
1004 PyObject *res2 = _PyImport_FixupExtension((char *)full_name, (char *)filename);
1005
1006 if (unlikely(res2 == NULL)) {
1007 return NULL;
1008 }
1009#else
1010 PyObject *full_name_obj = PyUnicode_FromString(full_name);
1011 CHECK_OBJECT(full_name_obj);
1012 PyObject *filename_obj = Nuitka_String_FromFilename(filename);
1013
1014 CHECK_OBJECT(filename_obj);
1015
1016// See above, we need to correct modules imported if we don't successfully swap
1017// the package context.
1018#if PYTHON_VERSION >= 0x3c0 && !defined(_NUITKA_USE_UNEXPOSED_API)
1019 if (preserved_basename_module != NULL) {
1020#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1021 PRINT_STRING("Handling for preservation: ");
1022 PRINT_ITEM_LINE(full_name_obj);
1023 PRINT_STRING("Restoring preserved module: ");
1024 PRINT_ITEM(base_name_obj);
1025 if (Nuitka_HasModule(tstate, base_name_obj)) {
1026 PRINT_STRING(" changes ");
1027 PRINT_ITEM(Nuitka_GetModule(tstate, base_name_obj));
1028 }
1029 PRINT_STRING(" -> ");
1030 PRINT_ITEM_LINE(preserved_basename_module);
1031#endif
1032 Nuitka_SetModule(base_name_obj, preserved_basename_module);
1033 Py_DECREF(preserved_basename_module);
1034 }
1035
1036 if (base_name_obj != NULL) {
1037 PyObject *need_correction = MAKE_LIST_EMPTY(tstate, 0);
1038
1039 {
1040 PyObject *modules_dict = Nuitka_GetSysModules();
1041
1042 Py_ssize_t pos = 0;
1043 PyObject *key, *value;
1044
1045 PyObject *base_name_prefix = BINARY_OPERATION_ADD_OBJECT_UNICODE_UNICODE(base_name_obj, const_str_dot);
1046#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1047 PRINT_STRING("Scanning for modules needing correction: ");
1048 PRINT_ITEM_LINE(base_name_prefix);
1049#endif
1050 while (Nuitka_DictNext(modules_dict, &pos, &key, &value)) {
1051 // TODO: Should have nuitka_bool return values for these as well maybe.
1052 PyObject *starts_with_result = UNICODE_STARTSWITH2(tstate, key, base_name_prefix);
1053
1054 if (CHECK_IF_TRUE(starts_with_result) == 1) {
1055 LIST_APPEND0(need_correction, key);
1056 }
1057
1058 Py_DECREF(starts_with_result);
1059 }
1060 }
1061
1062#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1063 PRINT_STRING("Correction needed:");
1064 PRINT_ITEM_LINE(need_correction);
1065#endif
1066
1067 Py_ssize_t n = PyList_GET_SIZE(need_correction);
1068
1069 for (Py_ssize_t i = 0; i < n; i++) {
1070 PyObject *module_to_correct_name = PyList_GET_ITEM(need_correction, i);
1071
1072 PyObject *correct_module_name =
1073 BINARY_OPERATION_ADD_OBJECT_UNICODE_UNICODE(prefix_name_obj, module_to_correct_name);
1074
1075 PyObject *module_to_correct = Nuitka_GetModule(tstate, module_to_correct_name);
1076
1077#if _NUITKA_EXPERIMENTAL_DEBUG_EXTENSION_MODULE_PRESERVATION_HACK
1078 PRINT_ITEM(module_to_correct_name);
1079 PRINT_STRING(" -> ");
1080 PRINT_ITEM(correct_module_name);
1081 PRINT_STRING(" changes ");
1082 PRINT_ITEM_LINE(module_to_correct);
1083#endif
1084 Nuitka_SetModule(correct_module_name, module_to_correct);
1085
1086 Nuitka_DelModule(tstate, module_to_correct_name);
1087 }
1088
1089 Py_DECREF(base_name_obj);
1090 Py_DECREF(prefix_name_obj);
1091 }
1092#endif
1093
1094#if PYTHON_VERSION < 0x3d0
1095 int res = _PyImport_FixupExtensionObject(module, full_name_obj, filename_obj
1096#if PYTHON_VERSION >= 0x370
1097 ,
1098 Nuitka_GetSysModules()
1099#endif
1100 );
1101#endif
1102
1103#if PYTHON_VERSION >= 0x3d0
1104 Nuitka_SetModuleString(full_name, module);
1105#endif
1106
1107 Py_DECREF(full_name_obj);
1108 Py_DECREF(filename_obj);
1109
1110#if PYTHON_VERSION < 0x3d0
1111 if (unlikely(res == -1)) {
1112 return NULL;
1113 }
1114#endif
1115#endif
1116
1117 return module;
1118}
1119
1120static void loadTriggeredModule(PyThreadState *tstate, char const *name, char const *trigger_name) {
1121 char trigger_module_name[2048];
1122
1123 copyStringSafe(trigger_module_name, name, sizeof(trigger_module_name));
1124 appendStringSafe(trigger_module_name, trigger_name, sizeof(trigger_module_name));
1125
1126 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(trigger_module_name);
1127
1128 if (entry != NULL) {
1129 if (isVerbose()) {
1130 PySys_WriteStderr("Loading %s\n", trigger_module_name);
1131 }
1132
1133 IMPORT_EMBEDDED_MODULE(tstate, trigger_module_name);
1134
1135 if (unlikely(HAS_ERROR_OCCURRED(tstate))) {
1136 if ((entry->flags & NUITKA_ABORT_MODULE_FLAG) != 0) {
1137 printf("Critical error loading %s.\n", trigger_module_name);
1138 abort();
1139 } else {
1140 PyObject *trigger_module_name_str = Nuitka_String_FromString(trigger_module_name);
1141 PyErr_WriteUnraisable(trigger_module_name_str);
1142 Py_DECREF(trigger_module_name_str);
1143 }
1144 }
1145 }
1146}
1147
1148#if PYTHON_VERSION >= 0x300
1149static void _fixupSpecAttribute(PyThreadState *tstate, PyObject *module) {
1150 PyObject *spec_value = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___spec__);
1151
1152 if (spec_value && spec_value != Py_None) {
1153 if (HAS_ATTR_BOOL(tstate, spec_value, const_str_plain__initializing)) {
1154 SET_ATTRIBUTE(tstate, spec_value, const_str_plain__initializing, Py_False);
1155 }
1156 }
1157}
1158#endif
1159
1160// Pointers to bytecode data.
1161static char **_bytecode_data = NULL;
1162
1163static PyObject *loadModule(PyThreadState *tstate, PyObject *module, PyObject *module_name,
1164 struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
1165#if _NUITKA_STANDALONE_MODE
1166 if ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0) {
1167 bool is_package = (entry->flags & NUITKA_PACKAGE_FLAG) != 0;
1168
1169 filename_char_t filename[MAXPATHLEN + 1];
1170 _makeModuleCFilenameValue(filename, sizeof(filename) / sizeof(filename_char_t), entry->name, module_name,
1171 is_package);
1172
1173 callIntoExtensionModule(tstate, entry->name, filename, is_package);
1174 } else
1175#endif
1176 if ((entry->flags & NUITKA_BYTECODE_FLAG) != 0) {
1177 // TODO: Do node use marshal, but our own stuff, once we
1178 // can do code objects too.
1179
1180 PyCodeObject *code_object =
1181 (PyCodeObject *)PyMarshal_ReadObjectFromString(_bytecode_data[entry->bytecode_index], entry->bytecode_size);
1182
1183 // TODO: Probably a bit harsh reaction.
1184 if (unlikely(code_object == NULL)) {
1185 PyErr_Print();
1186 abort();
1187 }
1188
1189 return loadModuleFromCodeObject(tstate, module, code_object, entry->name,
1190 (entry->flags & NUITKA_PACKAGE_FLAG) != 0);
1191 } else {
1192 assert((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) == 0);
1193 assert(entry->python_init_func);
1194
1195 {
1196 NUITKA_MAY_BE_UNUSED bool res = Nuitka_SetModule(module_name, module);
1197 assert(res != false);
1198 }
1199
1200 // Run the compiled module code, we get the module returned.
1201#if PYTHON_VERSION < 0x300
1202 NUITKA_MAY_BE_UNUSED
1203#endif
1204 PyObject *result = entry->python_init_func(tstate, module, entry);
1205 CHECK_OBJECT_X(result);
1206
1207#if PYTHON_VERSION >= 0x300
1208 if (likely(result != NULL)) {
1209 _fixupSpecAttribute(tstate, result);
1210 }
1211#endif
1212 }
1213
1214 if (unlikely(HAS_ERROR_OCCURRED(tstate))) {
1215 return NULL;
1216 }
1217
1218 if (isVerbose()) {
1219 PySys_WriteStderr("Loaded %s\n", entry->name);
1220 }
1221
1222 return Nuitka_GetModule(tstate, module_name);
1223}
1224
1225static PyObject *_EXECUTE_EMBEDDED_MODULE(PyThreadState *tstate, PyObject *module, PyObject *module_name,
1226 char const *name) {
1227 // In case of extension modules, that is fine to be NULL.
1228 CHECK_OBJECT_X(module);
1229 CHECK_OBJECT(module_name);
1230
1231 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1232 bool frozen_import = entry == NULL && hasFrozenModule(name);
1233
1234 if (entry != NULL || frozen_import) {
1235 // Execute the "preLoad" code produced for the module potentially. This
1236 // is from plugins typically, that want to modify things for the the
1237 // module before loading, to e.g. set a plug-in path, or do some monkey
1238 // patching in order to make things compatible.
1239 loadTriggeredModule(tstate, name, "-preLoad");
1240 }
1241
1242 PyObject *result = NULL;
1243
1244 if (entry != NULL) {
1245#ifdef _NUITKA_EXPERIMENTAL_FORCE_GC_COLLECT_ON_IMPORT
1246 PyGC_Collect();
1247#endif
1248
1249 result = loadModule(tstate, module, module_name, entry);
1250
1251#if !defined(_NUITKA_DEPLOYMENT_MODE) && !defined(_NUITKA_NO_DEPLOYMENT_PERFECT_SUPPORT)
1252 if (unlikely(HAS_ERROR_OCCURRED(tstate))) {
1253 if ((entry->flags & NUITKA_PERFECT_SUPPORTED_FLAG) != 0) {
1254 PyObject *exception_arg =
1255 PyUnicode_FromFormat("import of module '%s' failed unexpectedly despite intended perfect "
1256 "support, please raise a Nuitka issue and compile "
1257 "with an older version of the module in the meantime",
1258 name);
1259
1260 struct Nuitka_ExceptionPreservationItem saved_exception;
1261 FETCH_ERROR_OCCURRED_STATE(tstate, &saved_exception);
1262
1263#if PYTHON_VERSION < 0x3c0
1264 NORMALIZE_EXCEPTION_STATE(tstate, &saved_exception);
1265#endif
1266
1267 struct Nuitka_ExceptionPreservationItem new_exception_state;
1268 SET_EXCEPTION_PRESERVATION_STATE_FROM_ARGS(tstate, &new_exception_state, PyExc_RuntimeError,
1269 exception_arg, NULL);
1270 Py_INCREF_IMMORTAL(PyExc_RuntimeError);
1271
1272 Nuitka_Exception_SetContext(new_exception_state.exception_value, saved_exception.exception_value);
1273
1274 RESTORE_ERROR_OCCURRED_STATE(tstate, &new_exception_state);
1275 }
1276 }
1277#endif
1278
1279#ifdef _NUITKA_EXPERIMENTAL_FORCE_GC_COLLECT_ON_IMPORT
1280 PyGC_Collect();
1281#endif
1282
1283 if (unlikely(result == NULL)) {
1284 return NULL;
1285 }
1286 }
1287
1288 if (frozen_import) {
1289 PGO_onModuleEntered(name);
1290 int res = PyImport_ImportFrozenModule((char *)name);
1291 PGO_onModuleExit(name, res == -1);
1292
1293 if (unlikely(res == -1)) {
1294 return NULL;
1295 }
1296
1297 if (res == 1) {
1298 result = Nuitka_GetModule(tstate, module_name);
1299 }
1300 }
1301
1302 if (result != NULL) {
1303 // Execute the "postLoad" code produced for the module potentially. This
1304 // is from plugins typically, that want to modify the module immediately
1305 // after loading, to e.g. set a plug-in path, or do some monkey patching
1306 // in order to make things compatible.
1307 loadTriggeredModule(tstate, name, "-postLoad");
1308
1309 return result;
1310 }
1311
1312 Py_INCREF_IMMORTAL(Py_None);
1313 return Py_None;
1314}
1315
1316// Note: This may become an entry point for hard coded imports of compiled
1317// stuff.
1318PyObject *IMPORT_EMBEDDED_MODULE(PyThreadState *tstate, char const *name) {
1319 PyObject *module_name = Nuitka_String_FromString(name);
1320
1321 // Check if it's already loaded, and don't do it again otherwise.
1322 PyObject *module = Nuitka_GetModule(tstate, module_name);
1323
1324 if (module != NULL) {
1325 Py_DECREF(module_name);
1326 return module;
1327 }
1328
1329#if PYTHON_VERSION < 0x300
1330 module = PyModule_New(name);
1331#else
1332 module = PyModule_NewObject(module_name);
1333#endif
1334
1335 PyObject *result = _EXECUTE_EMBEDDED_MODULE(tstate, module, module_name, name);
1336
1337#if PYTHON_VERSION < 0x350
1338 if (unlikely(result == NULL)) {
1339 Nuitka_DelModule(tstate, module_name);
1340 }
1341#endif
1342
1343 Py_DECREF(module_name);
1344
1345 return result;
1346}
1347
1348PyObject *EXECUTE_EMBEDDED_MODULE(PyThreadState *tstate, PyObject *module) {
1349 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___name__);
1350 assert(module_name);
1351
1352 char const *name = Nuitka_String_AsString(module_name);
1353
1354 return _EXECUTE_EMBEDDED_MODULE(tstate, module, module_name, name);
1355}
1356
1357static PyObject *_nuitka_loader_load_module(PyObject *self, PyObject *args, PyObject *kwds) {
1358 PyObject *module_name;
1359 PyObject *unused;
1360
1361 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O|O:load_module", (char **)_kw_list_find_module, &module_name,
1362 &unused);
1363
1364 if (unlikely(res == 0)) {
1365 return NULL;
1366 }
1367
1368 assert(module_name);
1369 assert(Nuitka_String_Check(module_name));
1370
1371 char const *name = Nuitka_String_AsString(module_name);
1372
1373 if (isVerbose()) {
1374 PySys_WriteStderr("Loading %s\n", name);
1375 }
1376
1377 PyThreadState *tstate = PyThreadState_GET();
1378
1379#if !_NUITKA_STANDALONE_MODE
1380 if (installed_extension_modules != NULL) {
1381 PyObject *extension_module_filename = DICT_GET_ITEM0(tstate, installed_extension_modules, module_name);
1382
1383 if (extension_module_filename != NULL) {
1384 // TODO: Should we not set __file__ for the module here, but there is no object.
1385 return callIntoInstalledExtensionModule(tstate, module_name, extension_module_filename);
1386 }
1387 }
1388#endif
1389
1390 return IMPORT_EMBEDDED_MODULE(tstate, name);
1391}
1392
1393static char const *_kw_list_is_package[] = {"fullname", NULL};
1394
1395static PyObject *_nuitka_loader_is_package(PyObject *self, PyObject *args, PyObject *kwds) {
1396 PyObject *module_name;
1397
1398 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:is_package", (char **)_kw_list_is_package, &module_name);
1399
1400 if (unlikely(res == 0)) {
1401 return NULL;
1402 }
1403
1404 assert(module_name);
1405 assert(Nuitka_String_Check(module_name));
1406
1407 char const *name = Nuitka_String_AsString(module_name);
1408
1409 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1410
1411 PyObject *result;
1412
1413 if (entry) {
1414 result = BOOL_FROM((entry->flags & NUITKA_PACKAGE_FLAG) != 0);
1415 } else {
1416 // TODO: Maybe needs to be an exception.
1417 result = Py_None;
1418 }
1419
1420 Py_INCREF_IMMORTAL(result);
1421 return result;
1422}
1423
1424static char const *_kw_list_iter_modules[] = {"package", NULL};
1425
1426static PyObject *_nuitka_loader_iter_modules(struct Nuitka_LoaderObject *self, PyObject *args, PyObject *kwds) {
1427 PyObject *prefix;
1428
1429 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:iter_modules", (char **)_kw_list_iter_modules, &prefix);
1430
1431 if (unlikely(res == 0)) {
1432 return NULL;
1433 }
1434
1435 NUITKA_MAY_BE_UNUSED PyThreadState *tstate = PyThreadState_GET();
1436
1437 PyObject *result = MAKE_LIST_EMPTY(tstate, 0);
1438
1439 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
1440 assert(current);
1441
1442 char const *s;
1443
1444 if (self->m_loader_entry) {
1445 s = self->m_loader_entry->name;
1446 } else {
1447 s = "";
1448 }
1449
1450 while (current->name != NULL) {
1451 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
1452 current->name = UN_TRANSLATE(current->name);
1453 current->flags -= NUITKA_TRANSLATED_FLAG;
1454 }
1455
1456 int c = strncmp(s, current->name, strlen(s));
1457
1458 if (c != 0) {
1459 current++;
1460 continue;
1461 }
1462
1463 if (strcmp(current->name, "__main__") == 0) {
1464 current++;
1465 continue;
1466 }
1467
1468 if (current->name[strlen(s)] == 0) {
1469 current++;
1470 continue;
1471 }
1472
1473 char const *sub = strchr(current->name + strlen(s) + 1, '.');
1474
1475 if (sub != NULL) {
1476 current++;
1477 continue;
1478 }
1479
1480 PyObject *name;
1481 if (self->m_loader_entry) {
1482 name = Nuitka_String_FromString(current->name + strlen(s) + 1);
1483 } else {
1484 name = Nuitka_String_FromString(current->name);
1485 }
1486
1487 if (CHECK_IF_TRUE(prefix)) {
1488 PyObject *old = name;
1489 name = PyUnicode_Concat(prefix, name);
1490 Py_DECREF(old);
1491 }
1492
1493 PyObject *r = MAKE_TUPLE_EMPTY(tstate, 2);
1494 PyTuple_SET_ITEM(r, 0, name);
1495 PyTuple_SET_ITEM_IMMORTAL(r, 1, BOOL_FROM((current->flags & NUITKA_PACKAGE_FLAG) != 0));
1496
1497 LIST_APPEND1(result, r);
1498
1499 current++;
1500 }
1501
1502 return result;
1503}
1504
1505static PyObject *getModuleDirectory(PyThreadState *tstate, struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
1506#if defined(_NUITKA_FREEZER_HAS_FILE_PATH)
1507#if defined(_WIN32)
1508 wchar_t buffer[1024];
1509 buffer[0] = 0;
1510
1511 appendWStringSafeW(buffer, entry->file_path, sizeof(buffer));
1512 stripFilenameW(buffer);
1513 PyObject *dir_name = NuitkaUnicode_FromWideChar(buffer, -1);
1514#else
1515 char buffer[1024];
1516 copyStringSafe(buffer, entry->file_path, sizeof(buffer));
1517
1518 PyObject *dir_name = Nuitka_String_FromString(dirname(buffer));
1519#endif
1520#else
1521 PyObject *module_name;
1522 if ((entry->flags & NUITKA_PACKAGE_FLAG) != 0) {
1523 module_name = Nuitka_String_FromString(entry->name);
1524 } else {
1525 char buffer[1024];
1526 copyStringSafe(buffer, entry->name, sizeof(buffer));
1527
1528 char *dot = strrchr(buffer, '.');
1529 if (dot != NULL) {
1530 *dot = 0;
1531 }
1532
1533 module_name = Nuitka_String_FromString(buffer);
1534 }
1535
1536#if PYTHON_VERSION < 0x300
1537 PyObject *module_path = STR_REPLACE3(tstate, module_name, const_str_dot, getPathSeparatorStringObject());
1538#else
1539 PyObject *module_path = UNICODE_REPLACE3(tstate, module_name, const_str_dot, getPathSeparatorStringObject());
1540#endif
1541 Py_DECREF(module_name);
1542
1543 if (unlikely(module_path == NULL)) {
1544 return NULL;
1545 }
1546
1547 PyObject *dir_name = MAKE_RELATIVE_PATH(module_path);
1548 Py_DECREF(module_path);
1549#endif
1550
1551 return dir_name;
1552}
1553
1554#if PYTHON_VERSION >= 0x300
1555// Used in module template too, therefore exported.
1556PyObject *getImportLibBootstrapModule(void) {
1557 static PyObject *importlib = NULL;
1558 if (importlib == NULL) {
1559 importlib = PyImport_ImportModule("importlib._bootstrap");
1560 }
1561
1562 return importlib;
1563}
1564#endif
1565
1566static PyObject *getModuleFileValue(PyThreadState *tstate, struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
1567 PyObject *dir_name = getModuleDirectory(tstate, entry);
1568
1569 char filename_buffer[1024];
1570
1571 if ((entry->flags & NUITKA_PACKAGE_FLAG) != 0) {
1572 copyStringSafe(filename_buffer, "__init__", sizeof(filename_buffer));
1573 } else {
1574 char const *basename = strrchr(entry->name, '.');
1575
1576 if (basename == NULL) {
1577 basename = entry->name;
1578 } else {
1579 basename += 1;
1580 }
1581
1582 copyStringSafe(filename_buffer, basename, sizeof(filename_buffer));
1583 }
1584
1585 if ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0) {
1586#if defined(_WIN32)
1587 appendStringSafe(filename_buffer, ".pyd", sizeof(filename_buffer));
1588#else
1589 appendStringSafe(filename_buffer, ".so", sizeof(filename_buffer));
1590#endif
1591 } else {
1592 appendStringSafe(filename_buffer, ".py", sizeof(filename_buffer));
1593 }
1594
1595 PyObject *module_filename = Nuitka_String_FromString(filename_buffer);
1596
1597 PyObject *result = JOIN_PATH2(dir_name, module_filename);
1598
1599 Py_DECREF(module_filename);
1600
1601 return result;
1602}
1603
1604#if PYTHON_VERSION >= 0x300
1605
1606static PyObject *_nuitka_loader_repr_module(PyObject *self, PyObject *args, PyObject *kwds) {
1607 PyObject *module;
1608 PyObject *unused;
1609
1610 int res =
1611 PyArg_ParseTupleAndKeywords(args, kwds, "O|O:module_repr", (char **)_kw_list_find_module, &module, &unused);
1612
1613 if (unlikely(res == 0)) {
1614 return NULL;
1615 }
1616
1617 PyThreadState *tstate = PyThreadState_GET();
1618
1619 return PyUnicode_FromFormat("<module '%s' from %R>", PyModule_GetName(module),
1620 Nuitka_GetFilenameObject(tstate, module));
1621}
1622
1623static PyObject *getModuleSpecClass(PyObject *importlib_module) {
1624 static PyObject *module_spec_class = NULL;
1625
1626 if (module_spec_class == NULL) {
1627 module_spec_class = PyObject_GetAttrString(importlib_module, "ModuleSpec");
1628 }
1629
1630 return module_spec_class;
1631}
1632
1633static PyObject *createModuleSpec(PyThreadState *tstate, PyObject *module_name, PyObject *origin, bool is_package) {
1634 CHECK_OBJECT(module_name);
1635 assert(Nuitka_String_Check(module_name));
1636 CHECK_OBJECT_X(origin);
1637
1638 PyObject *importlib_module = getImportLibBootstrapModule();
1639
1640 if (unlikely(importlib_module == NULL)) {
1641 return NULL;
1642 }
1643
1644 PyObject *module_spec_class = getModuleSpecClass(importlib_module);
1645
1646 if (unlikely(module_spec_class == NULL)) {
1647 return NULL;
1648 }
1649
1650 PyObject *args = MAKE_TUPLE2(tstate, module_name, (PyObject *)&Nuitka_Loader_Type);
1651
1652 PyObject *kw_values[] = {is_package ? Py_True : Py_False, origin};
1653
1654 char const *kw_keys[] = {"is_package", "origin"};
1655
1656 PyObject *kw_args = MAKE_DICT_X_CSTR(kw_keys, kw_values, sizeof(kw_values) / sizeof(PyObject *));
1657
1658 PyObject *result = CALL_FUNCTION(tstate, module_spec_class, args, kw_args);
1659
1660 Py_DECREF(args);
1661 Py_DECREF(kw_args);
1662
1663 return result;
1664}
1665
1666#if !_NUITKA_STANDALONE_MODE
1667// We might have to load stuff from installed modules in our package namespaces.
1668static PyObject *createModuleSpecViaPathFinder(PyThreadState *tstate, PyObject *module_name,
1669 char const *parent_module_name) {
1670 if (scanModuleInPackagePath(tstate, module_name, parent_module_name)) {
1671 return createModuleSpec(tstate, module_name, NULL, false);
1672 } else {
1673 // Without error this means we didn't make it.
1674 return NULL;
1675 }
1676}
1677#endif
1678
1679static char const *_kw_list_find_spec[] = {"fullname", "is_package", "path", NULL};
1680
1681static PyObject *_nuitka_loader_find_spec(PyObject *self, PyObject *args, PyObject *kwds) {
1682 PyObject *module_name;
1683 PyObject *unused1; // We ignore "is_package"
1684 PyObject *unused2; // We ignore "path"
1685
1686 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O|OO:find_spec", (char **)_kw_list_find_spec, &module_name,
1687 &unused1, &unused2);
1688
1689 if (unlikely(res == 0)) {
1690 return NULL;
1691 }
1692
1693 char const *full_name = Nuitka_String_AsString(module_name);
1694
1695 if (isVerbose()) {
1696 PySys_WriteStderr("import %s # considering responsibility (find_spec)\n", full_name);
1697 }
1698
1699 struct Nuitka_MetaPathBasedLoaderEntry const *entry = findEntry(full_name);
1700
1701 PyThreadState *tstate = PyThreadState_GET();
1702
1703#if !_NUITKA_STANDALONE_MODE
1704 // We need to deal with things located in compiled packages, that were not included,
1705 // e.g. extension modules, but also other files, that were asked to not be included
1706 // or added later.
1707 if (entry == NULL) {
1708 entry = findContainingPackageEntry(full_name);
1709
1710 if (entry != NULL) {
1711 PyObject *result = createModuleSpecViaPathFinder(tstate, module_name, entry->name);
1712
1713 if (result != NULL) {
1714 if (isVerbose()) {
1715 PySys_WriteStderr("import %s # claimed responsibility (%s, contained in compiled package %s)\n",
1716 full_name, getEntryModeString(entry), entry->name);
1717 }
1718
1719 return result;
1720 }
1721
1722 if (HAS_ERROR_OCCURRED(tstate)) {
1723 return NULL;
1724 }
1725
1726 entry = NULL;
1727 }
1728 }
1729#endif
1730
1731 if (entry == NULL) {
1732 if (isVerbose()) {
1733 PySys_WriteStderr("import %s # denied responsibility\n", full_name);
1734 }
1735
1736 Py_INCREF_IMMORTAL(Py_None);
1737 return Py_None;
1738 }
1739
1740 if (isVerbose()) {
1741 PySys_WriteStderr("import %s # claimed responsibility (%s)\n", Nuitka_String_AsString(module_name),
1742 getEntryModeString(entry));
1743 }
1744
1745 return createModuleSpec(tstate, module_name, getModuleFileValue(tstate, entry),
1746 (entry->flags & NUITKA_PACKAGE_FLAG) != 0);
1747}
1748
1749#if PYTHON_VERSION >= 0x350
1750static char const *_kw_list_create_module[] = {"spec", NULL};
1751
1752static PyObject *_nuitka_loader_create_module(PyObject *self, PyObject *args, PyObject *kwds) {
1753 PyObject *spec;
1754
1755 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:create_module", (char **)_kw_list_create_module, &spec);
1756
1757 if (unlikely(res == 0)) {
1758 return NULL;
1759 }
1760
1761 PyThreadState *tstate = PyThreadState_GET();
1762
1763 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, spec, const_str_plain_name);
1764
1765 if (unlikely(module_name == NULL)) {
1766 return NULL;
1767 }
1768
1769 // Check if module name is a proper string.
1770 char const *name = Nuitka_String_AsString(module_name);
1771 if (unlikely(name == NULL)) {
1772 Py_DECREF(module_name);
1773 return NULL;
1774 }
1775
1776 if (isVerbose()) {
1777 PySys_WriteStderr("import %s # create module\n", name);
1778 }
1779
1780 // During spec creation, we have populated the dictionary with a filename to load from
1781 // for extension modules that were found installed in the system and below our package.
1782#if !_NUITKA_STANDALONE_MODE && !_NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1783 if (installed_extension_modules != NULL) {
1784 PyObject *extension_module_filename = DICT_GET_ITEM0(tstate, installed_extension_modules, module_name);
1785
1786 if (extension_module_filename != NULL) {
1787 Py_DECREF(module_name);
1788 return callIntoInstalledExtensionModule(tstate, module_name, extension_module_filename);
1789 }
1790 }
1791#endif
1792
1793#if _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1794 struct Nuitka_MetaPathBasedLoaderEntry *entry = NULL;
1795#else
1796 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1797#endif
1798
1799 PyObject *result;
1800
1801 if ((entry != NULL) && ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0)) {
1802 result = _EXECUTE_EMBEDDED_MODULE(tstate, NULL, module_name, name);
1803 } else {
1804 result = PyModule_NewObject(module_name);
1805 }
1806
1807 Py_DECREF(module_name);
1808
1809 return result;
1810}
1811
1812static char const *_kw_list_exec_module[] = {"module", NULL};
1813
1814static PyObject *_nuitka_loader_exec_module(PyObject *self, PyObject *args, PyObject *kwds) {
1815 PyObject *module;
1816
1817 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:exec_module", (char **)_kw_list_exec_module, &module);
1818
1819 if (unlikely(res == 0)) {
1820 return NULL;
1821 }
1822
1823 PyThreadState *tstate = PyThreadState_GET();
1824
1825 PyObject *module_name = LOOKUP_ATTRIBUTE(tstate, module, const_str_plain___name__);
1826
1827 if (unlikely(module_name == NULL)) {
1828 return NULL;
1829 }
1830
1831 // Check if module name is a proper string.
1832 char const *name = Nuitka_String_AsString(module_name);
1833 if (unlikely(name == NULL)) {
1834 Py_DECREF(module_name);
1835 return NULL;
1836 }
1837
1838 if (isVerbose()) {
1839 PySys_WriteStderr("import %s # execute module\n", name);
1840 }
1841
1842 // During spec creation, we have populated the dictionary with a filename to load from
1843 // for extension modules that were found installed in the system and below our package.
1844#if !_NUITKA_STANDALONE_MODE && _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1845 if (installed_extension_modules != NULL) {
1846 PyObject *extension_module_filename = DICT_GET_ITEM0(tstate, installed_extension_modules, module_name);
1847
1848 if (extension_module_filename != NULL) {
1849 // Set filename attribute
1850 bool b_res = SET_ATTRIBUTE(tstate, module, const_str_plain___file__, extension_module_filename);
1851
1852 if (unlikely(b_res == false)) {
1853 // Might be refuted, which wouldn't be harmful.
1854 CLEAR_ERROR_OCCURRED(tstate);
1855 }
1856
1857 return callIntoInstalledExtensionModule(tstate, module_name, extension_module_filename);
1858 }
1859 }
1860#endif
1861
1862#if _NUITKA_EXPERIMENTAL_OLD_EXTENSION_MODULE_LOADING
1863 struct Nuitka_MetaPathBasedLoaderEntry *entry = NULL;
1864#else
1865 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1866#endif
1867
1868 if ((entry != NULL) && ((entry->flags & NUITKA_EXTENSION_MODULE_FLAG) != 0)) {
1869 Py_INCREF(module);
1870
1871 if (unlikely(!PyModule_Check(module))) {
1872 return module;
1873 }
1874
1875 PyModuleDef *def = PyModule_GetDef(module);
1876 if (def == NULL) {
1877 return module;
1878 }
1879
1880 void *state = PyModule_GetState(module);
1881 if (state != NULL) {
1882 return module;
1883 }
1884
1885 res = PyModule_ExecDef(module, def);
1886
1887 if (unlikely(res == -1)) {
1888 Py_DECREF(module);
1889
1890 return NULL;
1891 }
1892
1893 CHECK_OBJECT(module);
1894
1895 return module;
1896 }
1897
1898 Py_DECREF(module_name);
1899
1900 return EXECUTE_EMBEDDED_MODULE(tstate, module);
1901}
1902
1903#if PYTHON_VERSION >= 0x370
1904
1905// The resource reader class is implemented in a separate file.
1906#include "MetaPathBasedLoaderResourceReader.c"
1907
1908static PyObject *_nuitka_loader_get_resource_reader(PyObject *self, PyObject *args, PyObject *kwds) {
1909 PyObject *module_name;
1910
1911 int res =
1912 PyArg_ParseTupleAndKeywords(args, kwds, "O:get_resource_reader", (char **)_kw_list_exec_module, &module_name);
1913
1914 if (unlikely(res == 0)) {
1915 return NULL;
1916 }
1917
1918 char const *name = Nuitka_String_AsString(module_name);
1919
1920 struct Nuitka_MetaPathBasedLoaderEntry *entry = findEntry(name);
1921
1922 if (entry) {
1923 if (isVerbose()) {
1924 PySys_WriteStderr("import %s # get_resource_reader (%s)\n", name, getEntryModeString(entry));
1925 }
1926
1927 return Nuitka_ResourceReader_New(entry);
1928 }
1929
1930 PyErr_Format(PyExc_RuntimeError, "Requested resource reader for unhandled module %s", module_name);
1931 return NULL;
1932}
1933
1934#endif
1935
1936#endif
1937
1938#endif
1939
1940static char const *_kw_list_find_distributions[] = {"context", NULL};
1941
1942static PyObject *_nuitka_loader_find_distributions(PyObject *self, PyObject *args, PyObject *kwds) {
1943 PyObject *context;
1944
1945 int res =
1946 PyArg_ParseTupleAndKeywords(args, kwds, "O:find_distributions", (char **)_kw_list_find_distributions, &context);
1947
1948 if (unlikely(res == 0)) {
1949 return NULL;
1950 }
1951
1952 PyObject *name = PyObject_GetAttr(context, const_str_plain_name);
1953
1954 if (unlikely(name == NULL)) {
1955 return NULL;
1956 }
1957
1958 PyThreadState *tstate = PyThreadState_GET();
1959
1960 PyObject *temp = MAKE_LIST_EMPTY(tstate, 0);
1961
1962 Py_ssize_t pos = 0;
1963 PyObject *distribution_name;
1964
1965 while (Nuitka_DistributionNext(&pos, &distribution_name)) {
1966 bool include = false;
1967 if (name == Py_None) {
1968 include = true;
1969 } else {
1970 nuitka_bool cmp_res = RICH_COMPARE_EQ_NBOOL_OBJECT_OBJECT(name, distribution_name);
1971
1972 if (unlikely(cmp_res == NUITKA_BOOL_EXCEPTION)) {
1973 Py_DECREF(name);
1974 return NULL;
1975 }
1976
1977 include = cmp_res == NUITKA_BOOL_TRUE;
1978 }
1979
1980 if (include) {
1981 // Create a distribution object from our data.
1982 PyObject *distribution = Nuitka_Distribution_New(tstate, distribution_name);
1983
1984 if (distribution == NULL) {
1985 Py_DECREF(temp);
1986 return NULL;
1987 }
1988
1989 LIST_APPEND1(temp, distribution);
1990 }
1991 }
1992
1993 // We are expected to return an iterator.
1994 PyObject *result = MAKE_ITERATOR_INFALLIBLE(temp);
1995
1996 Py_DECREF(temp);
1997 return result;
1998}
1999
2000static char const *_kw_list_sys_path_hook[] = {"path", NULL};
2001
2002static PyObject *_nuitka_loader_sys_path_hook(PyObject *self, PyObject *args, PyObject *kwds) {
2003 PyObject *path;
2004
2005 int res = PyArg_ParseTupleAndKeywords(args, kwds, "O:sys_path_hook", (char **)_kw_list_sys_path_hook, &path);
2006
2007 if (unlikely(res == 0)) {
2008 return NULL;
2009 }
2010
2011#if 0
2012 PRINT_STRING("CONSIDER PATH:");
2013 PRINT_ITEM(path);
2014 PRINT_NEW_LINE();
2015#endif
2016
2017 struct Nuitka_MetaPathBasedLoaderEntry *entry = loader_entries;
2018 assert(entry);
2019
2020 PyThreadState *tstate = PyThreadState_GET();
2021
2022 while (entry->name != NULL) {
2023 if ((entry->flags & NUITKA_TRANSLATED_FLAG) != 0) {
2024 entry->name = UN_TRANSLATE(entry->name);
2025 entry->flags -= NUITKA_TRANSLATED_FLAG;
2026 }
2027
2028 if ((entry->flags & NUITKA_PACKAGE_FLAG) != 0) {
2029 PyObject *module_directory = getModuleDirectory(tstate, entry);
2030
2031#if 0
2032 PRINT_STRING(entry->name);
2033 PRINT_STRING(" ");
2034 PRINT_ITEM(module_directory);
2035 PRINT_NEW_LINE();
2036#endif
2037
2038 nuitka_bool cmp_res = compareFilePaths(tstate, module_directory, path);
2039
2040 if (unlikely(cmp_res == NUITKA_BOOL_EXCEPTION)) {
2041 return NULL;
2042 }
2043
2044 // Create the loader for the module.
2045 if (cmp_res == NUITKA_BOOL_TRUE) {
2046 return Nuitka_Loader_New(entry);
2047 }
2048 }
2049
2050 entry++;
2051 }
2052
2053 SET_CURRENT_EXCEPTION_TYPE0(tstate, PyExc_ImportError);
2054 return NULL;
2055}
2056
2057static PyMethodDef Nuitka_Loader_methods[] = {
2058 {"iter_modules", (PyCFunction)_nuitka_loader_iter_modules, METH_VARARGS | METH_KEYWORDS, NULL},
2059 {"get_data", (PyCFunction)_nuitka_loader_get_data, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2060 {"find_module", (PyCFunction)_nuitka_loader_find_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2061 {"load_module", (PyCFunction)_nuitka_loader_load_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2062 {"is_package", (PyCFunction)_nuitka_loader_is_package, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2063#if PYTHON_VERSION >= 0x300
2064 {"module_repr", (PyCFunction)_nuitka_loader_repr_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2065 {"find_spec", (PyCFunction)_nuitka_loader_find_spec, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2066#endif
2067#if PYTHON_VERSION >= 0x350
2068 {"create_module", (PyCFunction)_nuitka_loader_create_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2069 {"exec_module", (PyCFunction)_nuitka_loader_exec_module, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2070#endif
2071#if PYTHON_VERSION >= 0x370
2072 {"get_resource_reader", (PyCFunction)_nuitka_loader_get_resource_reader, METH_STATIC | METH_VARARGS | METH_KEYWORDS,
2073 NULL},
2074#endif
2075
2076 {"find_distributions", (PyCFunction)_nuitka_loader_find_distributions, METH_STATIC | METH_VARARGS | METH_KEYWORDS,
2077 NULL},
2078
2079 {"sys_path_hook", (PyCFunction)_nuitka_loader_sys_path_hook, METH_STATIC | METH_VARARGS | METH_KEYWORDS, NULL},
2080
2081 {NULL, NULL} // terminator
2082};
2083
2084static PyObject *Nuitka_Loader_tp_repr(struct Nuitka_LoaderObject *loader) {
2085 if (loader->m_loader_entry == NULL) {
2086 // TODO: Indicate in module mode, which one it is for.
2087 return Nuitka_String_FromString("<nuitka_module_loader>");
2088 } else {
2089 return Nuitka_String_FromFormat("<nuitka_module_loader for '%s'>", loader->m_loader_entry->name);
2090 }
2091}
2092
2093#include "nuitka/freelists.h"
2094
2095// TODO: A free list is not the right thing for those, they are probably living forever, but it's
2096// no big harm too, but make it small, maybe be allowing a toggle that makes a specific macro not
2097// use the free list mechanism at all.
2098
2099// Freelist setup
2100#define MAX_LOADER_FREE_LIST_COUNT 10
2101static struct Nuitka_LoaderObject *free_list_loaders = NULL;
2102static int free_list_loaders_count = 0;
2103
2104static void Nuitka_Loader_tp_dealloc(struct Nuitka_LoaderObject *loader) {
2105 Nuitka_GC_UnTrack(loader);
2106
2107 releaseToFreeList(free_list_loaders, loader, MAX_LOADER_FREE_LIST_COUNT);
2108}
2109
2110static int Nuitka_Loader_tp_traverse(struct Nuitka_LoaderObject *loader, visitproc visit, void *arg) { return 0; }
2111
2112static PyObject *Nuitka_Loader_get_name(struct Nuitka_LoaderObject *loader, void *closure) {
2113 PyObject *result = Nuitka_String_FromString(loader->m_loader_entry->name);
2114
2115 return result;
2116}
2117static PyObject *Nuitka_Loader_get_path(struct Nuitka_LoaderObject *loader, void *closure) {
2118 PyThreadState *tstate = PyThreadState_GET();
2119 PyObject *result = getModuleFileValue(tstate, loader->m_loader_entry);
2120
2121 return result;
2122}
2123
2124static PyObject *Nuitka_Loader_get__module__(struct Nuitka_LoaderObject *loader, void *closure) {
2125 PyObject *result = const_str_plain___nuitka__;
2126
2127 Py_INCREF_IMMORTAL(result);
2128 return result;
2129}
2130
2131static PyGetSetDef Nuitka_Loader_tp_getset[] = {{(char *)"__module__", (getter)Nuitka_Loader_get__module__, NULL, NULL},
2132 {(char *)"name", (getter)Nuitka_Loader_get_name, NULL, NULL},
2133 {(char *)"path", (getter)Nuitka_Loader_get_path, NULL, NULL},
2134 {NULL}};
2135
2136PyTypeObject Nuitka_Loader_Type = {
2137 PyVarObject_HEAD_INIT(NULL, 0) "nuitka_module_loader",
2138 sizeof(struct Nuitka_LoaderObject), // tp_basicsize
2139 0, // tp_itemsize
2140 (destructor)Nuitka_Loader_tp_dealloc, // tp_dealloc
2141 0, // tp_print
2142 0, // tp_getattr
2143 0, // tp_setattr
2144 0, // tp_reserved
2145 (reprfunc)Nuitka_Loader_tp_repr, // tp_repr
2146 0, // tp_as_number
2147 0, // tp_as_sequence
2148 0, // tp_as_mapping
2149 0, // tp_hash
2150 0, // tp_call
2151 0, // tp_str
2152 0, // tp_getattro (PyObject_GenericGetAttr)
2153 0, // tp_setattro
2154 0, // tp_as_buffer
2155 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags
2156 0, // tp_doc
2157 (traverseproc)Nuitka_Loader_tp_traverse, // tp_traverse
2158 0, // tp_clear
2159 0, // tp_richcompare
2160 0, // tp_weaklistoffset
2161 0, // tp_iter
2162 0, // tp_iternext
2163 Nuitka_Loader_methods, // tp_methods
2164 0, // tp_members
2165 Nuitka_Loader_tp_getset, // tp_getset
2166};
2167
2168/* Used by modules to register child loaders for packages. */
2169PyObject *Nuitka_Loader_New(struct Nuitka_MetaPathBasedLoaderEntry const *entry) {
2170 struct Nuitka_LoaderObject *result;
2171
2172 allocateFromFreeListFixed(free_list_loaders, struct Nuitka_LoaderObject, Nuitka_Loader_Type);
2173 Nuitka_GC_Track(result);
2174
2175 result->m_loader_entry = entry;
2176
2177 return (PyObject *)result;
2178}
2179
2180#if _NUITKA_MODULE_MODE
2181void updateMetaPathBasedLoaderModuleRoot(char const *module_root_name) {
2182 assert(module_root_name != NULL);
2183 char const *last_dot = strrchr(module_root_name, '.');
2184
2185 if (last_dot != NULL) {
2186 struct Nuitka_MetaPathBasedLoaderEntry *current = loader_entries;
2187 assert(current);
2188
2189 while (current->name != NULL) {
2190 if ((current->flags & NUITKA_TRANSLATED_FLAG) != 0) {
2191 current->name = UN_TRANSLATE(current->name);
2192 current->flags -= NUITKA_TRANSLATED_FLAG;
2193 }
2194
2195 char name[2048];
2196
2197 if (strcmp(last_dot + 1, current->name) == 0) {
2198 copyStringSafeN(name, module_root_name, last_dot - module_root_name + 1, sizeof(name));
2199 appendStringSafe(name, current->name, sizeof(name));
2200
2201 current->name = strdup(name);
2202 } else if (strncmp(last_dot + 1, current->name, strlen(last_dot + 1)) == 0 &&
2203 current->name[strlen(last_dot + 1)] == '.') {
2204 copyStringSafeN(name, module_root_name, last_dot - module_root_name + 1, sizeof(name));
2205 appendStringSafe(name, current->name, sizeof(name));
2206
2207 current->name = strdup(name);
2208 }
2209
2210 current++;
2211 }
2212 }
2213}
2214#endif
2215
2216void registerMetaPathBasedLoader(struct Nuitka_MetaPathBasedLoaderEntry *_loader_entries,
2217 unsigned char **bytecode_data) {
2218 // Do it only once.
2219 if (loader_entries) {
2220 assert(_loader_entries == loader_entries);
2221
2222 return;
2223 }
2224
2225 _bytecode_data = (char **)bytecode_data;
2226
2227 if (isVerbose()) {
2228 PySys_WriteStderr("Setup nuitka compiled module/bytecode/extension importer.\n");
2229 }
2230
2231 loader_entries = _loader_entries;
2232
2233#if _NUITKA_MODULE_MODE && PYTHON_VERSION < 0x3c0
2234 if (_Py_PackageContext != NULL) {
2235 updateMetaPathBasedLoaderModuleRoot(_Py_PackageContext);
2236 }
2237#endif
2238
2239 Nuitka_PyType_Ready(&Nuitka_Loader_Type, NULL, true, false, false, false, false);
2240
2241#if PYTHON_VERSION >= 0x370
2242 Nuitka_PyType_Ready(&Nuitka_ResourceReader_Type, NULL, true, false, false, false, false);
2243#endif
2244
2245 // Register it as a meta path loader.
2246 PyObject *global_loader = Nuitka_Loader_New(NULL);
2247
2248 LIST_INSERT_CONST(Nuitka_SysGetObject("meta_path"),
2249#if PYTHON_VERSION < 0x300
2250 0,
2251#else
2252 2,
2253#endif
2254
2255 global_loader);
2256
2257 // Our "sys.path_hooks" entry uses "os.path" to compare filenames, so we need
2258 // to load it without.
2259 PyThreadState *tstate = PyThreadState_GET();
2260 IMPORT_HARD_OS_PATH(tstate);
2261
2262 // Register our loader as a "sys.path_hook" so it can provide for the paths.
2263#if _NUITKA_STANDALONE_MODE
2264 LIST_INSERT_CONST(Nuitka_SysGetObject("path_hooks"), 0, PyObject_GetAttrString(global_loader, "sys_path_hook"));
2265#else
2266 // TODO: For accelerated mode the loader doesn't have the ability
2267 // to handle "site-packages" on its own yet, and therefore we
2268 // cannot have it take precedence.
2269 LIST_APPEND1(Nuitka_SysGetObject("path_hooks"), PyObject_GetAttrString(global_loader, "sys_path_hook"));
2270#endif
2271}
2272
2273#if _NUITKA_STANDALONE_MODE
2274// This is called for the technical module imported early on during interpreter
2275// into, to still get compatible "__file__" attributes.
2276void setEarlyFrozenModulesFileAttribute(PyThreadState *tstate) {
2277 PyObject *sys_modules = Nuitka_GetSysModules();
2278 Py_ssize_t pos = 0;
2279 PyObject *key, *value;
2280
2281 PyObject *builtin_module_names = Nuitka_SysGetObject("builtin_module_names");
2282
2283 while (Nuitka_DictNext(sys_modules, &pos, &key, &value)) {
2284 if (key != NULL && value != NULL && PyModule_Check(value)) {
2285 bool is_package = HAS_ATTR_BOOL(tstate, value, const_str_plain___path__);
2286
2287 if (is_package || HAS_ATTR_BOOL(tstate, value, const_str_plain___file__) ||
2288 PySequence_Contains(builtin_module_names, key) == 0) {
2289 PyObject *file_value = MAKE_RELATIVE_PATH_FROM_NAME(Nuitka_String_AsString(key), is_package, false);
2290 PyObject_SetAttr(value, const_str_plain___file__, file_value);
2291 Py_DECREF(file_value);
2292 CHECK_OBJECT(file_value);
2293 }
2294 }
2295 }
2296
2297 assert(!HAS_ERROR_OCCURRED(tstate));
2298}
2299
2300#endif
2301
2302// The importlib distribution class is implemented in a separate file.
2303#include "MetaPathBasedLoaderImportlibMetadataDistribution.c"
2304
2305// Part of "Nuitka", an optimizing Python compiler that is compatible and
2306// integrates with CPython, but also works on its own.
2307//
2308// Licensed under the Apache License, Version 2.0 (the "License");
2309// you may not use this file except in compliance with the License.
2310// You may obtain a copy of the License at
2311//
2312// http://www.apache.org/licenses/LICENSE-2.0
2313//
2314// Unless required by applicable law or agreed to in writing, software
2315// distributed under the License is distributed on an "AS IS" BASIS,
2316// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2317// See the License for the specific language governing permissions and
2318// limitations under the License.
Nuitka_ExceptionPreservationItem
Definition exceptions.h:712
Nuitka_LoaderObject
Definition MetaPathBasedLoader.c:28
Nuitka_MetaPathBasedLoaderEntry
Definition unfreezing.h:31