WxPython: PyInstaller fails with No module named _core_

Question

I am converting my wxpython (3.0.2.0) application to binaries using PyInstaller. The binaries work fine when built and executed on Ubuntu 12.04. However if I build on Ubuntu

Answer 1

If PyInstaller development version is not desired for some reason, here goes some fix.

Instance ofBuiltinImporter, FrozenImporter and CExtensionImporter from PyInstaller.loader.pyi_importers are appended to sys.meta_path. And find_module method of which are called in order until one of them succeeds when a module is imported.

CExtensionImporter chooses only one of the many suffixes for the C extension to load, f.e. wx._core_.i386-linux-gnu.so. That's why it fails to load the C extension wx._core_.so.

Buggy code;

class CExtensionImporter(object):
    def __init__(self):
        # Find the platform specific suffix. On Windows it is .pyd, on Linux/Unix .so.
        for ext, mode, typ in imp.get_suffixes():
            if typ == imp.C_EXTENSION:
                self._c_ext_tuple = (ext, mode, typ)
                self._suffix = ext  # Just string like .pyd  or  .so
                break

Fix;

1. Runtime hooks
It's possible to fix the problem without code change using runtime hooks. This is a quick fix which fixes 'WxPython' problems.
This runtime hook changes some private attributes of instance of CExtensionImporter. To use this hook, give --runtime-hook=wx-run-hook.py to pyinstaller.

wx-run-hook.py

import sys
import imp

sys.meta_path[-1]._c_ext_tuple = imp.get_suffixes()[1]
sys.meta_path[-1]._suffix = sys.meta_path[-1]._c_ext_tuple[0]

This second runtime hook completely replaces object in sys.meta_path[-1]. So it should work in most situations. Use as pyinstaller --runtime-hook=pyinstaller-run-hook.py application.py.

pyinstaller-run-hook.py

import sys
import imp

from PyInstaller.loader import pyi_os_path

class CExtensionImporter(object):
    """
    PEP-302 hook for sys.meta_path to load Python C extension modules.

    C extension modules are present on the sys.prefix as filenames:

        full.module.name.pyd
        full.module.name.so
    """
    def __init__(self):
        # TODO cache directory content for faster module lookup without file system access.
        # Find the platform specific suffix. On Windows it is .pyd, on Linux/Unix .so.
        self._c_ext_tuples = [(ext, mode, typ) for ext, mode, typ in imp.get_suffixes() if typ == imp.C_EXTENSION]

        # Create hashmap of directory content for better performance.
        files = pyi_os_path.os_listdir(sys.prefix)
        self._file_cache = set(files)

    def find_module(self, fullname, path=None):
        imp.acquire_lock()
        module_loader = None  # None means - no module found by this importer.

        # Look in the file list of sys.prefix path (alias PYTHONHOME).
        for ext, mode, typ in self._c_ext_tuples:
            if fullname + ext in self._file_cache:
                module_loader = self
                self._suffix = ext
                self._c_ext_tuple = (ext, mode, typ)
                break

        imp.release_lock()
        return module_loader

    def load_module(self, fullname, path=None):
        imp.acquire_lock()

        try:
            # PEP302 If there is an existing module object named 'fullname'
            # in sys.modules, the loader must use that existing module.
            module = sys.modules.get(fullname)

            if module is None:
                filename = pyi_os_path.os_path_join(sys.prefix, fullname + self._suffix)
                fp = open(filename, 'rb')
                module = imp.load_module(fullname, fp, filename, self._c_ext_tuple)
                # Set __file__ attribute.
                if hasattr(module, '__setattr__'):
                    module.__file__ = filename
                else:
                    # Some modules (eg: Python for .NET) have no __setattr__
                    # and dict entry have to be set.
                    module.__dict__['__file__'] = filename

        except Exception:
            # Remove 'fullname' from sys.modules if it was appended there.
            if fullname in sys.modules:
                sys.modules.pop(fullname)
            # Release the interpreter's import lock.
            imp.release_lock()
            raise  # Raise the same exception again.

        # Release the interpreter's import lock.
        imp.release_lock()

        return module

    ### Optional Extensions to the PEP302 Importer Protocol

    def is_package(self, fullname):
        """
        Return always False since C extension modules are never packages.
        """
        return False

    def get_code(self, fullname):
        """
        Return None for a C extension module.
        """
        if fullname + self._suffix in self._file_cache:
            return None
        else:
            # ImportError should be raised if module not found.
            raise ImportError('No module named ' + fullname)

    def get_source(self, fullname):
        """
        Return None for a C extension module.
        """
        if fullname + self._suffix in self._file_cache:
            return None
        else:
            # ImportError should be raised if module not found.
            raise ImportError('No module named ' + fullname)

    def get_data(self, path):
        """
        This returns the data as a string, or raise IOError if the "file"
        wasn't found. The data is always returned as if "binary" mode was used.

        The 'path' argument is a path that can be constructed by munging
        module.__file__ (or pkg.__path__ items)
        """
        # Since __file__ attribute works properly just try to open and read it.
        fp = open(path, 'rb')
        content = fp.read()
        fp.close()
        return content

    # TODO Do we really need to implement this method?
    def get_filename(self, fullname):
        """
        This method should return the value that __file__ would be set to
        if the named module was loaded. If the module is not found, then
        ImportError should be raised.
        """
        if fullname + self._suffix in self._file_cache:
            return pyi_os_path.os_path_join(sys.prefix, fullname + self._suffix)
        else:
            # ImportError should be raised if module not found.
            raise ImportError('No module named ' + fullname)

#This may overwrite some other object
#sys.meta_path[-1] = CExtensionImporter()

#isinstance(object, CExtensionImporter)
#type(object) == CExtensioImporter
#the above two doesn't work here

#grab the index of instance of CExtensionImporter

for i, obj in enumerate(sys.meta_path):
    if obj.__class__.__name__ == CExtensionImporter.__name__:
        sys.meta_path[i] = CExtensionImporter()
        break

2. Code change

class CExtensionImporter(object):
    def __init__(self):
        # Find the platform specific suffix. On Windows it is .pyd, on Linux/Unix .so.
        self._c_ext_tuples = [(ext, mode, typ) for ext, mode, typ in imp.get_suffixes() if typ == imp.C_EXTENSION]

        files = pyi_os_path.os_listdir(sys.prefix)
        self._file_cache = set(files)

Because imp.get_suffixes returns more than one suffixes for type imp.C_EXTENSION and the right one can't be known in advance until a module is found, I store all of them in a list self._c_ext_tuples. The right suffix is set in self._suffix, which is used along with self._c_ext_tuple by load_module method, from find_module method if the module is found.

def find_module(self, fullname, path=None):
    imp.acquire_lock()
    module_loader = None  # None means - no module found by this importer.

    # Look in the file list of sys.prefix path (alias PYTHONHOME).
    for ext, mode, typ in self._c_ext_tuples:
        if fullname + ext in self._file_cache:
            module_loader = self
            self._suffix = ext
            self._c_ext_tuple = (ext, mode, typ)
            break

    imp.release_lock()
    return module_loader