From 62000a394b890daead97943672a5d64117c15b8f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 13:44:46 +0300 Subject: [PATCH 01/15] [mypyc] WIP free threading when compiling multiple modules --- mypyc/build.py | 10 ++++-- mypyc/lib-rt/module_shim_multiphase.tmpl | 43 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 mypyc/lib-rt/module_shim_multiphase.tmpl diff --git a/mypyc/build.py b/mypyc/build.py index b7d3e1b25366..44432dd9d087 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -36,7 +36,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import RUNTIME_C_FILES, shared_lib_name +from mypyc.common import RUNTIME_C_FILES, shared_lib_name, IS_FREE_THREADED from mypyc.errors import Errors from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name @@ -176,9 +176,15 @@ def generate_c_extension_shim( cname = "%s.c" % full_module_name.replace(".", os.sep) cpath = os.path.join(dir_name, cname) + if IS_FREE_THREADED: + # We use multi-phase init in free-threaded builds to enable free threading. + shim_name = "module_shim_multiphase.tmpl" + else: + shim_name = "module_shim.tmpl" + # We load the C extension shim template from a file. # (So that the file could be reused as a bazel template also.) - with open(os.path.join(include_dir(), "module_shim.tmpl")) as f: + with open(os.path.join(include_dir(), shim_name)) as f: shim_template = f.read() write_file( diff --git a/mypyc/lib-rt/module_shim_multiphase.tmpl b/mypyc/lib-rt/module_shim_multiphase.tmpl new file mode 100644 index 000000000000..2f9fb21e9880 --- /dev/null +++ b/mypyc/lib-rt/module_shim_multiphase.tmpl @@ -0,0 +1,43 @@ +#include + +static int {modname}_exec(PyObject *module) +{{ + PyObject *tmp; + if (!(tmp = PyImport_ImportModule("{libname}"))) return -1; + PyObject *capsule = PyObject_GetAttrString(tmp, "init_{full_modname}"); + Py_DECREF(tmp); + if (capsule == NULL) return -1; + void *init_func = PyCapsule_GetPointer(capsule, "{libname}.init_{full_modname}"); + Py_DECREF(capsule); + if (!init_func) {{ + return -1; + }} + if (((PyObject *(*)(void))init_func)() == NULL) return -1; + return 0; +}} + +static PyModuleDef_Slot {modname}_slots[] = {{ + {{Py_mod_exec, {modname}_exec}}, + {{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}}, + {{Py_mod_gil, Py_MOD_GIL_NOT_USED}}, + {{0, NULL}}, +}}; + +static struct PyModuleDef {modname}_module = {{ + PyModuleDef_HEAD_INIT, + .m_name = "{modname}", + .m_doc = NULL, + .m_methods = NULL, + .m_size = 0, + .m_slots = {modname}_slots, +}}; + +PyMODINIT_FUNC +PyInit_{modname}(void) +{{ + return PyModuleDef_Init(&{modname}_module); +}} + +// distutils sometimes spuriously tells cl to export CPyInit___init__, +// so provide that so it chills out +PyMODINIT_FUNC PyInit___init__(void) {{ return PyInit_{modname}(); }} From 5109f1fa00ca2b763060993e31fe23f2ea79e74b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 15:19:32 +0300 Subject: [PATCH 02/15] WIP use multi-phase init for the shared library --- mypyc/codegen/emitmodule.py | 48 +++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 047309ec71e3..98e6e04e6c04 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -784,28 +784,20 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: assert self.group_name is not None emitter.emit_line() + + short_name = "g" + shared_lib_name(self.group_name).split(".")[-1] # TODO + emitter.emit_lines( - "PyMODINIT_FUNC PyInit_{}(void)".format( - shared_lib_name(self.group_name).split(".")[-1] - ), + f"static int {short_name}_exec(PyObject *module_arg)", "{", - ( - 'static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'.format( - shared_lib_name(self.group_name) - ) - ), "int res;", "PyObject *capsule;", "PyObject *tmp;", "static PyObject *module;", "if (module) {", - "Py_INCREF(module);", - "return module;", - "}", - "module = PyModule_Create(&def);", - "if (!module) {", - "goto fail;", + "return 0;", "}", + "module = module_arg;", "", ) @@ -861,7 +853,33 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: "", ) - emitter.emit_lines("return module;", "fail:", "Py_XDECREF(module);", "return NULL;", "}") + emitter.emit_lines("return 0;", "fail:", "return -1;", "}") + + emitter.emit_lines( + f"static PyModuleDef_Slot {short_name}_slots[] = {{", + f"{{Py_mod_exec, {short_name}_exec}},", + "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},", + "{Py_mod_gil, Py_MOD_GIL_NOT_USED},", + "{0, NULL},", + "};", + ) + + emitter.emit_lines( + f"static PyModuleDef {short_name}_module_def = {{", + "PyModuleDef_HEAD_INIT,", + f'.m_name = "{shared_lib_name(self.group_name)}",', + ".m_doc = NULL,", + ".m_size = 0,", # -1 originally + ".m_methods = NULL,", + f".m_slots = {short_name}_slots," + "};", + ) + + emitter.emit_lines( + f"PyMODINIT_FUNC PyInit_{short_name[1:]}(void) {{", + f"return PyModuleDef_Init(&{short_name}_module_def);", + "}", + ) def generate_globals_init(self, emitter: Emitter) -> None: emitter.emit_lines( From c7c681a3dfff262726f165905e1b14e741fa09bb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 15:57:56 +0300 Subject: [PATCH 03/15] WIP run module top levels (but method table is still not applied) --- mypyc/codegen/emitmodule.py | 16 ++++++++-------- mypyc/lib-rt/module_shim_multiphase.tmpl | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 98e6e04e6c04..b4ff4a3317a8 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -820,14 +820,14 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: for mod in self.modules: name = exported_name(mod) emitter.emit_lines( - f"extern PyObject *CPyInit_{name}(void);", - 'capsule = PyCapsule_New((void *)CPyInit_{}, "{}.init_{}", NULL);'.format( + f"extern int CPyExec_{name}(PyObject *);", + 'capsule = PyCapsule_New((void *)CPyExec_{}, "{}.exec_{}", NULL);'.format( name, shared_lib_name(self.group_name), name ), "if (!capsule) {", "goto fail;", "}", - f'res = PyObject_SetAttrString(module, "init_{name}", capsule);', + f'res = PyObject_SetAttrString(module, "exec_{name}", capsule);', "Py_DECREF(capsule);", "if (res < 0) {", "goto fail;", @@ -907,14 +907,14 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module module_prefix = emitter.names.private_name(module_name) self.emit_module_exec_func(emitter, module_name, module_prefix, module) if self.multi_phase_init: - self.emit_module_def_slots(emitter, module_prefix) + self.emit_module_def_slots(emitter, module_prefix, module_name) self.emit_module_methods(emitter, module_name, module_prefix, module) self.emit_module_def_struct(emitter, module_name, module_prefix) self.emit_module_init_func(emitter, module_name, module_prefix) - def emit_module_def_slots(self, emitter: Emitter, module_prefix: str) -> None: + def emit_module_def_slots(self, emitter: Emitter, module_prefix: str, module_name: str) -> None: name = f"{module_prefix}_slots" - exec_name = f"{module_prefix}_exec" + exec_name = f"CPyExec_{exported_name(module_name)}" emitter.emit_line(f"static PyModuleDef_Slot {name}[] = {{") emitter.emit_line(f"{{Py_mod_exec, {exec_name}}},") @@ -988,7 +988,7 @@ def emit_module_exec_func( library, and in this case we use an internal module initialized function that will be called by the shim. """ - declaration = f"static int {module_prefix}_exec(PyObject *module)" + declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)" module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(declaration, "{") emitter.emit_line("PyObject* modname = NULL;") @@ -1056,7 +1056,7 @@ def emit_module_init_func( emitter.emit_line("}") return - exec_func = f"{module_prefix}_exec" + exec_func = f"CPyExec_{exported_name(module_name)}" # Store the module reference in a static and return it when necessary. # This is separate from the *global* reference to the module that will diff --git a/mypyc/lib-rt/module_shim_multiphase.tmpl b/mypyc/lib-rt/module_shim_multiphase.tmpl index 2f9fb21e9880..272e4a2b7d80 100644 --- a/mypyc/lib-rt/module_shim_multiphase.tmpl +++ b/mypyc/lib-rt/module_shim_multiphase.tmpl @@ -4,15 +4,15 @@ static int {modname}_exec(PyObject *module) {{ PyObject *tmp; if (!(tmp = PyImport_ImportModule("{libname}"))) return -1; - PyObject *capsule = PyObject_GetAttrString(tmp, "init_{full_modname}"); + PyObject *capsule = PyObject_GetAttrString(tmp, "exec_{full_modname}"); Py_DECREF(tmp); if (capsule == NULL) return -1; - void *init_func = PyCapsule_GetPointer(capsule, "{libname}.init_{full_modname}"); + void *exec_func = PyCapsule_GetPointer(capsule, "{libname}.exec_{full_modname}"); Py_DECREF(capsule); - if (!init_func) {{ + if (!exec_func) {{ return -1; }} - if (((PyObject *(*)(void))init_func)() == NULL) return -1; + if (((int (*)(PyObject *))exec_func)(module) != 0) return -1; return 0; }} From 63d2f56bd1dc81fcaf7ae14086ef87baf82e444a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 16:26:14 +0300 Subject: [PATCH 04/15] Actually populate module functions --- mypyc/codegen/emitmodule.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index b4ff4a3317a8..26e3d26aa573 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -905,10 +905,10 @@ def generate_globals_init(self, emitter: Emitter) -> None: def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None: """Emit the PyModuleDef struct for a module and the module init function.""" module_prefix = emitter.names.private_name(module_name) + self.emit_module_methods(emitter, module_name, module_prefix, module) self.emit_module_exec_func(emitter, module_name, module_prefix, module) if self.multi_phase_init: self.emit_module_def_slots(emitter, module_prefix, module_name) - self.emit_module_methods(emitter, module_name, module_prefix, module) self.emit_module_def_struct(emitter, module_name, module_prefix) self.emit_module_init_func(emitter, module_name, module_prefix) @@ -1005,6 +1005,12 @@ def emit_module_exec_func( " goto fail;", ) + if self.multi_phase_init: + emitter.emit_lines( + f"if (PyModule_AddFunctions(module, {module_prefix}module_methods) < 0)", + " goto fail;", + ) + # HACK: Manually instantiate generated classes here type_structs: list[str] = [] for cl in module.classes: From e5579080b586f411095dbab728fdcbe66c9b03e0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 16:26:41 +0300 Subject: [PATCH 05/15] Update docstring --- mypyc/codegen/emitmodule.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 26e3d26aa573..937c6f180f2a 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -980,13 +980,14 @@ def emit_module_def_struct( def emit_module_exec_func( self, emitter: Emitter, module_name: str, module_prefix: str, module: ModuleIR ) -> None: - """Emit the module init function. + """Emit the module exec function. - If we are compiling just one module, this will be the C API init - function. If we are compiling 2+ modules, we generate a shared + If we are compiling just one module, this will be the normal C API + exec function. If we are compiling 2+ modules, we generate a shared library for the modules and shims that call into the shared - library, and in this case we use an internal module initialized - function that will be called by the shim. + library, and in this case the shared module defines an internal + exec function for each module and these will be called by the shims + via Capsules. """ declaration = f"int CPyExec_{exported_name(module_name)}(PyObject *module)" module_static = self.module_internal_static_name(module_name, emitter) From ce60137d7548b4151ef58dab89ced6abbb1b7f42 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 16:32:45 +0300 Subject: [PATCH 06/15] Don't generate redundant slots tables --- mypyc/codegen/emitmodule.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 937c6f180f2a..36e1ff3a989d 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -907,8 +907,13 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module module_prefix = emitter.names.private_name(module_name) self.emit_module_methods(emitter, module_name, module_prefix, module) self.emit_module_exec_func(emitter, module_name, module_prefix, module) - if self.multi_phase_init: + + # The slots table is only created here when using multi-phase init but + # not a shared library. If using a shared library, the slots tables will + # be included in shim modules instead. + if self.multi_phase_init and not self.use_shared_lib: self.emit_module_def_slots(emitter, module_prefix, module_name) + self.emit_module_def_struct(emitter, module_name, module_prefix) self.emit_module_init_func(emitter, module_name, module_prefix) @@ -969,7 +974,7 @@ def emit_module_def_struct( "0, /* size of per-interpreter state of the module */", f"{module_prefix}module_methods,", ) - if self.multi_phase_init: + if self.multi_phase_init and not self.use_shared_lib: slots_name = f"{module_prefix}_slots" emitter.emit_line(f"{slots_name}, /* m_slots */") else: From 0225e3430eb27baeb0e7570b2667ac9f17dd0eaa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 16:36:39 +0300 Subject: [PATCH 07/15] Skip more steps if using multi-phase init and shared lib --- mypyc/codegen/emitmodule.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 36e1ff3a989d..13a4362fec2e 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -908,14 +908,13 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module self.emit_module_methods(emitter, module_name, module_prefix, module) self.emit_module_exec_func(emitter, module_name, module_prefix, module) - # The slots table is only created here when using multi-phase init but - # not a shared library. If using a shared library, the slots tables will - # be included in shim modules instead. - if self.multi_phase_init and not self.use_shared_lib: - self.emit_module_def_slots(emitter, module_prefix, module_name) - - self.emit_module_def_struct(emitter, module_name, module_prefix) - self.emit_module_init_func(emitter, module_name, module_prefix) + # If using multi-phase init and a shared lib, parts of module definition + # will happen in the shim modules, so we skip some steps here. + if not (self.multi_phase_init and self.use_shared_lib): + if self.multi_phase_init: + self.emit_module_def_slots(emitter, module_prefix, module_name) + self.emit_module_def_struct(emitter, module_name, module_prefix) + self.emit_module_init_func(emitter, module_name, module_prefix) def emit_module_def_slots(self, emitter: Emitter, module_prefix: str, module_name: str) -> None: name = f"{module_prefix}_slots" From 462314d86de8b40f8c6a21f60b9bad7063323d41 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 16:53:15 +0300 Subject: [PATCH 08/15] Fix non-multi-phase init shared lib --- mypyc/codegen/emitmodule.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 13a4362fec2e..dd00afb22089 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -819,15 +819,22 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: for mod in self.modules: name = exported_name(mod) + if self.multi_phase_init: + capsule_func_prefix = "CPyExec_" + capsule_name_prefix = "exec_" + emitter.emit_line(f"extern int CPyExec_{name}(PyObject *);") + else: + capsule_func_prefix = "CPyInit_" + capsule_name_prefix = "init_" + emitter.emit_line(f"extern PyObject *CPyInit_{name}(void);") emitter.emit_lines( - f"extern int CPyExec_{name}(PyObject *);", - 'capsule = PyCapsule_New((void *)CPyExec_{}, "{}.exec_{}", NULL);'.format( - name, shared_lib_name(self.group_name), name + 'capsule = PyCapsule_New((void *){}{}, "{}.{}{}", NULL);'.format( + capsule_func_prefix, name, shared_lib_name(self.group_name), capsule_name_prefix, name ), "if (!capsule) {", "goto fail;", "}", - f'res = PyObject_SetAttrString(module, "exec_{name}", capsule);', + f'res = PyObject_SetAttrString(module, "{capsule_name_prefix}{name}", capsule);', "Py_DECREF(capsule);", "if (res < 0) {", "goto fail;", From 2d1f1bf691352359b9acb5dfd3b475b6f98fce52 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 23 Jul 2025 17:41:54 +0300 Subject: [PATCH 09/15] Lint --- mypyc/build.py | 2 +- mypyc/codegen/emitmodule.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mypyc/build.py b/mypyc/build.py index 44432dd9d087..2856c7f91422 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -36,7 +36,7 @@ from mypy.util import write_junit_xml from mypyc.annotate import generate_annotated_html from mypyc.codegen import emitmodule -from mypyc.common import RUNTIME_C_FILES, shared_lib_name, IS_FREE_THREADED +from mypyc.common import IS_FREE_THREADED, RUNTIME_C_FILES, shared_lib_name from mypyc.errors import Errors from mypyc.ir.pprint import format_modules from mypyc.namegen import exported_name diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index dd00afb22089..27510bd28137 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -829,7 +829,11 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: emitter.emit_line(f"extern PyObject *CPyInit_{name}(void);") emitter.emit_lines( 'capsule = PyCapsule_New((void *){}{}, "{}.{}{}", NULL);'.format( - capsule_func_prefix, name, shared_lib_name(self.group_name), capsule_name_prefix, name + capsule_func_prefix, + name, + shared_lib_name(self.group_name), + capsule_name_prefix, + name, ), "if (!capsule) {", "goto fail;", @@ -878,7 +882,7 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: ".m_doc = NULL,", ".m_size = 0,", # -1 originally ".m_methods = NULL,", - f".m_slots = {short_name}_slots," + f".m_slots = {short_name}_slots,", "};", ) @@ -923,7 +927,9 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module self.emit_module_def_struct(emitter, module_name, module_prefix) self.emit_module_init_func(emitter, module_name, module_prefix) - def emit_module_def_slots(self, emitter: Emitter, module_prefix: str, module_name: str) -> None: + def emit_module_def_slots( + self, emitter: Emitter, module_prefix: str, module_name: str + ) -> None: name = f"{module_prefix}_slots" exec_name = f"CPyExec_{exported_name(module_name)}" From a3d23c98abd194115dbad402ca127fb6a88e790e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 13:41:58 +0300 Subject: [PATCH 10/15] Fix running on older Python versions --- mypyc/codegen/emitmodule.py | 50 ++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 27510bd28137..81478707b59a 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -866,14 +866,15 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: emitter.emit_lines("return 0;", "fail:", "return -1;", "}") - emitter.emit_lines( - f"static PyModuleDef_Slot {short_name}_slots[] = {{", - f"{{Py_mod_exec, {short_name}_exec}},", - "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},", - "{Py_mod_gil, Py_MOD_GIL_NOT_USED},", - "{0, NULL},", - "};", - ) + if self.multi_phase_init: + emitter.emit_lines( + f"static PyModuleDef_Slot {short_name}_slots[] = {{", + f"{{Py_mod_exec, {short_name}_exec}},", + "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},", + "{Py_mod_gil, Py_MOD_GIL_NOT_USED},", + "{0, NULL},", + "};", + ) emitter.emit_lines( f"static PyModuleDef {short_name}_module_def = {{", @@ -882,15 +883,34 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: ".m_doc = NULL,", ".m_size = 0,", # -1 originally ".m_methods = NULL,", - f".m_slots = {short_name}_slots,", - "};", ) + if self.multi_phase_init: + emitter.emit_line(f".m_slots = {short_name}_slots,") + emitter.emit_line("};") - emitter.emit_lines( - f"PyMODINIT_FUNC PyInit_{short_name[1:]}(void) {{", - f"return PyModuleDef_Init(&{short_name}_module_def);", - "}", - ) + if self.multi_phase_init: + emitter.emit_lines( + f"PyMODINIT_FUNC PyInit_{short_name[1:]}(void) {{", + f"return PyModuleDef_Init(&{short_name}_module_def);", + "}", + ) + else: + emitter.emit_lines( + f"PyMODINIT_FUNC PyInit_{short_name[1:]}(void) {{", + "static PyObject *module = NULL;", + "if (module) {", + "Py_INCREF(module);", + "return module;", + "}", + f"module = PyModule_Create(&{short_name}_module_def);", + "if (!module) {", + "return NULL;}", + f"if ({short_name}_exec(module) < 0) {{", + "Py_DECREF(module);", + "return NULL;}", + "return module;", + "}", + ) def generate_globals_init(self, emitter: Emitter) -> None: emitter.emit_lines( From 70b0c7de404b109f9b597ec939667b118bc015d2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 15:30:36 +0300 Subject: [PATCH 11/15] Simplify exec function --- mypyc/codegen/emitmodule.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 81478707b59a..419016de585a 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -788,16 +788,11 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: short_name = "g" + shared_lib_name(self.group_name).split(".")[-1] # TODO emitter.emit_lines( - f"static int {short_name}_exec(PyObject *module_arg)", + f"static int {short_name}_exec(PyObject *module)", "{", "int res;", "PyObject *capsule;", "PyObject *tmp;", - "static PyObject *module;", - "if (module) {", - "return 0;", - "}", - "module = module_arg;", "", ) From ce4252a69a45a513070b781bcb5fb8a63d2efa59 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 15:40:28 +0300 Subject: [PATCH 12/15] Rename shim template file --- mypyc/build.py | 2 +- ..._shim_multiphase.tmpl => module_shim_no_gil_multiphase.tmpl} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename mypyc/lib-rt/{module_shim_multiphase.tmpl => module_shim_no_gil_multiphase.tmpl} (100%) diff --git a/mypyc/build.py b/mypyc/build.py index 2856c7f91422..4a2d703b9f10 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -178,7 +178,7 @@ def generate_c_extension_shim( if IS_FREE_THREADED: # We use multi-phase init in free-threaded builds to enable free threading. - shim_name = "module_shim_multiphase.tmpl" + shim_name = "module_shim_no_gil_multiphase.tmpl" else: shim_name = "module_shim.tmpl" diff --git a/mypyc/lib-rt/module_shim_multiphase.tmpl b/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl similarity index 100% rename from mypyc/lib-rt/module_shim_multiphase.tmpl rename to mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl From 7c672179861e87803080048948eb9c4b959fea5c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 15:45:28 +0300 Subject: [PATCH 13/15] Clean up --- mypyc/codegen/emitmodule.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 419016de585a..ef19b945b60b 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -785,7 +785,7 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: emitter.emit_line() - short_name = "g" + shared_lib_name(self.group_name).split(".")[-1] # TODO + short_name = shared_lib_name(self.group_name).split(".")[-1] emitter.emit_lines( f"static int {short_name}_exec(PyObject *module)", @@ -885,13 +885,13 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: if self.multi_phase_init: emitter.emit_lines( - f"PyMODINIT_FUNC PyInit_{short_name[1:]}(void) {{", + f"PyMODINIT_FUNC PyInit_{short_name}(void) {{", f"return PyModuleDef_Init(&{short_name}_module_def);", "}", ) else: emitter.emit_lines( - f"PyMODINIT_FUNC PyInit_{short_name[1:]}(void) {{", + f"PyMODINIT_FUNC PyInit_{short_name}(void) {{", "static PyObject *module = NULL;", "if (module) {", "Py_INCREF(module);", From d54c0983f5b4c03a61417db007462833eccdd5b7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 16:07:38 +0300 Subject: [PATCH 14/15] Minor tweak and formatting changes --- mypyc/codegen/emitmodule.py | 11 +++++++---- mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl | 4 +--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index ef19b945b60b..4958328804a2 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -871,12 +871,13 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: "};", ) + size = 0 if self.multi_phase_init else -1 emitter.emit_lines( f"static PyModuleDef {short_name}_module_def = {{", "PyModuleDef_HEAD_INIT,", f'.m_name = "{shared_lib_name(self.group_name)}",', - ".m_doc = NULL,", - ".m_size = 0,", # -1 originally + ".m_doc = NULL," + f".m_size = {size},", ".m_methods = NULL,", ) if self.multi_phase_init: @@ -899,10 +900,12 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: "}", f"module = PyModule_Create(&{short_name}_module_def);", "if (!module) {", - "return NULL;}", + "return NULL;", + "}", f"if ({short_name}_exec(module) < 0) {{", "Py_DECREF(module);", - "return NULL;}", + "return NULL;", + "}", "return module;", "}", ) diff --git a/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl b/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl index 272e4a2b7d80..b9bfe9c91962 100644 --- a/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl +++ b/mypyc/lib-rt/module_shim_no_gil_multiphase.tmpl @@ -9,9 +9,7 @@ static int {modname}_exec(PyObject *module) if (capsule == NULL) return -1; void *exec_func = PyCapsule_GetPointer(capsule, "{libname}.exec_{full_modname}"); Py_DECREF(capsule); - if (!exec_func) {{ - return -1; - }} + if (!exec_func) return -1; if (((int (*)(PyObject *))exec_func)(module) != 0) return -1; return 0; }} From d14265b95c5e825d675e63bb99ee6c34530f6d77 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 30 Jul 2025 16:15:25 +0300 Subject: [PATCH 15/15] Fix bad generated names --- mypyc/codegen/emitmodule.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 4958328804a2..de34ed9fc7da 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -788,7 +788,7 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: short_name = shared_lib_name(self.group_name).split(".")[-1] emitter.emit_lines( - f"static int {short_name}_exec(PyObject *module)", + f"static int exec_{short_name}(PyObject *module)", "{", "int res;", "PyObject *capsule;", @@ -863,8 +863,8 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: if self.multi_phase_init: emitter.emit_lines( - f"static PyModuleDef_Slot {short_name}_slots[] = {{", - f"{{Py_mod_exec, {short_name}_exec}},", + f"static PyModuleDef_Slot slots_{short_name}[] = {{", + f"{{Py_mod_exec, exec_{short_name}}},", "{Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED},", "{Py_mod_gil, Py_MOD_GIL_NOT_USED},", "{0, NULL},", @@ -873,21 +873,21 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: size = 0 if self.multi_phase_init else -1 emitter.emit_lines( - f"static PyModuleDef {short_name}_module_def = {{", + f"static PyModuleDef module_def_{short_name} = {{", "PyModuleDef_HEAD_INIT,", f'.m_name = "{shared_lib_name(self.group_name)}",', - ".m_doc = NULL," + ".m_doc = NULL,", f".m_size = {size},", ".m_methods = NULL,", ) if self.multi_phase_init: - emitter.emit_line(f".m_slots = {short_name}_slots,") + emitter.emit_line(f".m_slots = slots_{short_name},") emitter.emit_line("};") if self.multi_phase_init: emitter.emit_lines( f"PyMODINIT_FUNC PyInit_{short_name}(void) {{", - f"return PyModuleDef_Init(&{short_name}_module_def);", + f"return PyModuleDef_Init(&module_def_{short_name});", "}", ) else: @@ -898,11 +898,11 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: "Py_INCREF(module);", "return module;", "}", - f"module = PyModule_Create(&{short_name}_module_def);", + f"module = PyModule_Create(&module_def_{short_name});", "if (!module) {", "return NULL;", "}", - f"if ({short_name}_exec(module) < 0) {{", + f"if (exec_{short_name}(module) < 0) {{", "Py_DECREF(module);", "return NULL;", "}",