Packing Python files into a single .py script

Question

Does anybody know if there is any tool for packing a Python project that uses several files and modules into a single script?

Answer 1

Save this as python_header.py:

#!/bin/env/python
# -*- coding: ascii -*-
import os
import sys
import imp
import tarfile
import tempfile


RUN_MODULE = "__run__"
SENTINEL = 'RzlBTXhya3ljIzl6PFFkQiRKLntEdHF+c2hvWid0IX5NVlxWd' \
           'FxcJ0NWQ2xKVUI0TVEuNl0rWUtnKiRr'.decode('base64')


class FileOffset(object):
    def __init__(self, fileobj, offset=0):
        self._fileobj = fileobj
        self._offset = offset
        self._fileobj.seek(offset)

    def tell(self):
        return self._fileobj.tell() - self._offset

    def seek(self, position, whence=os.SEEK_SET):
        if whence == os.SEEK_SET:
            if position < 0: raise IOErrror("Negative seek")
            self._fileobj.seek(position + self._offset)
        else:
            oldposition = self._fileobj.tell()
            self._fileobj.seek(position, whence)
            if self._fileobj.tell() < self._offset:
                self._fileobj.seek(oldposition, os.SEEK_SET)
                raise IOError("Negative seek")

    def __getattr__(self, attrname):
        return getattr(self._fileobj, attrname)

    def __enter__(self, *args):
        return self._fileobj.__enter__(*args)

    def __exit__(self, *args):
        return self._fileobj.__exit__(*args)


class TarImport(object):
    def __init__(self, tarobj, tarname=None):
        if tarname is None:
            tarname = ''
        self._tarname = tarname
        self._tarobj = tarobj

    def find_module(self, name, path=None):
        module_path = os.path.join(*name.split('.'))
        package_path = os.path.join(module_path, '__init__')

        for path in [module_path, package_path]:
            for suffix, mode, module_type in imp.get_suffixes():
                if module_type != imp.PY_SOURCE:
                    continue
                member = os.path.join(path) + suffix
                try:
                    modulefileobj = self._tarobj.extractfile(member)
                except KeyError:
                    pass
                else:
                    return Loader(name, modulefileobj,
                                  "%s/%s" % (self._tarname, member),
                                  (suffix, mode, module_type))


class Loader(object):
    def __init__(self, name, fileobj, filename, description):
        self._name = name
        self._fileobj = fileobj
        self._filename = filename
        self._description = description

    def load_module(self, name):
        imp.acquire_lock()
        try:
            module = sys.modules.get(name)
            if module is None:
                module = imp.new_module(name)

            module_script = self._fileobj.read()
            module.__file__ = self._filename
            module.__path__ = []
            sys.modules[name] = module
            exec(module_script, module.__dict__, module.__dict__)
        finally:
            imp.release_lock()

        return module


def find_offset(fileobj, sentinel):
    read_bytes = 0
    for line in fileobj:
        try:
            offset = line.index(sentinel)
        except ValueError:
            read_bytes += len(line)
        else:
            return read_bytes + offset + len(sentinel)
    raise ValueError("sentinel not found in %r" % (fileobj, ))


if __name__ == "__main__":
    sys.argv[:] = sys.argv[1:]
    archive_path = os.path.abspath(sys.argv[0])
    archive_offset = find_offset(open(archive_path), SENTINEL)

    archive = FileOffset(open(archive_path), archive_offset)

    tarobj = tarfile.TarFile(fileobj=archive)
    importer = TarImport(tarobj, archive_path)

    sys.meta_path.insert(0, importer)

    importer.find_module(RUN_MODULE).load_module(RUN_MODULE)

Save this as sh_header.sh:

#!/bin/sh

head -n @@TO@@ "$0" | tail -n +@@FROM@@ | python - "$0"

exit $?

Save this as create_tarred_program.py:

#!/usr/bin/env python
# -*- coding: latin-1 -*-

import sys
import imp
import shutil

sh_filename, runner_filename, tar_archive, dst_filename = sys.argv[1:]

runner = imp.load_module("tarfile_runner",
                        open(runner_filename, 'U'),
                        runner_filename,
                        ('.py', 'U', imp.PY_SOURCE))



sh_lines = open(sh_filename, 'r').readlines()
runner_lines = open(runner_filename, 'r').readlines()

sh_block = ''.join(sh_lines)
runner_block = ''.join(runner_lines)

if runner.SENTINEL in runner_block or runner.SENTINEL in sh_block:
    raise ValueError("Can't have the sentinel inside the runner module")
if not runner_block.endswith('\n') or not sh_block.endswith('\n'):
    raise ValueError("Trailing newline required in both headers")

to_pos = len(sh_lines) + len(runner_lines)
from_pos = len(sh_lines) + 1

sh_block = sh_block.replace("@@TO@@", str(to_pos))
sh_block = sh_block.replace("@@FROM@@", str(from_pos))


dst = open(dst_filename, 'wb')

dst.write(sh_block)
dst.write(runner_block)
dst.write(runner.SENTINEL)

shutil.copyfileobj(open(tar_archive, 'rb'), dst)

dst.flush()
dst.close()    

Create a tar archive with your packages named packages.tar. The main module should be called __run__.py, you should never import __main__. Run:

create_tarred_program.py sh_header.sh python_header.py packages.tar program.sh

Distrubute program.sh.

It's possible to avoid dependency on /bin/sh by an extended first line, but it still won't work on anything but *nix, so there's no point in it.