diff --git a/Build.py b/Build.py index 1549626..c40a257 100755 --- a/Build.py +++ b/Build.py @@ -316,10 +316,13 @@ class Analysis(Target): elif isinstance(mod, mf.PkgInZipModule): zipfiles.append((os.path.basename(str(mod.owner)), str(mod.owner), 'ZIPFILE')) - elif modnm == '__main__': - pass else: - pure.append((modnm, fnm, 'PYMODULE')) + # mf.PyModule instances expose a list of binary + # dependencies, most probably shared libraries accessed + # via ctypes. Add them to the overall required binaries. + binaries.extend(mod.binaries) + if modnm != '__main__': + pure.append((modnm, fnm, 'PYMODULE')) binaries.extend(bindepend.Dependencies(binaries, platform=target_platform)) self.fixMissingPythonLib(binaries) diff --git a/buildtests/ctypes/testctypes.c b/buildtests/ctypes/testctypes.c new file mode 100644 index 0000000..3bf8b84 --- /dev/null +++ b/buildtests/ctypes/testctypes.c @@ -0,0 +1,4 @@ +int dummy(int arg) +{ + return arg; +} diff --git a/buildtests/runtests.py b/buildtests/runtests.py index 30f9161..6a138f7 100755 --- a/buildtests/runtests.py +++ b/buildtests/runtests.py @@ -114,9 +114,18 @@ def runtests(alltests, filters=None, configfile=None, run_executable=1): # Run the test in a clean environment to make sure they're # really self-contained del os.environ["PATH"] - prog = os.path.join('dist', test, test) - if not os.path.exists(prog): - prog = os.path.join(prog + '.exe') + + of_prog = os.path.join('dist', test) # one-file deploy filename + od_prog = os.path.join('dist', test, test) # one-dir deploy filename + + if os.path.isfile(of_prog): + prog = of_prog + else: + if os.path.isfile(od_prog): + prog = od_prog + else: + prog = od_prog + ".exe" + print "RUNNING:", prog res = os.system(prog) os.environ["PATH"] = path diff --git a/buildtests/test15.py b/buildtests/test15.py new file mode 100644 index 0000000..886ba54 --- /dev/null +++ b/buildtests/test15.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +try: + import ctypes +except ImportError: + # ctypes unavailable, testing ctypes support is pointless. + sys.exit(0) + +import test15a +assert test15a.dummy(42) == 42 diff --git a/buildtests/test15.spec b/buildtests/test15.spec new file mode 100644 index 0000000..e36a787 --- /dev/null +++ b/buildtests/test15.spec @@ -0,0 +1,36 @@ +# -*- mode: python -*- + +import sys +if not sys.platform.startswith("darwin"): + raise RuntimeError("please port test15 under linux2 and win32") + +import os + +# If the required dylib does not reside in the current directory, the Analysis +# class machinery, based on ctypes.util.find_library, will not find it. This was +# done on purpose for this test, to show how to give Analysis class a clue. +os.environ["DYLD_LIBRARY_PATH"] = "ctypes/" + +# Check for presence of testctypes shared library, build it if not present +if not os.path.exists("ctypes/testctypes.dylib"): + os.chdir("ctypes") + os.system("gcc -Wall -dynamiclib testctypes.c -o testctypes.dylib -headerpad_max_install_names") + id_dylib = os.path.abspath("testctypes.dylib") + os.system("install_name_tool -id %s testctypes.dylib" % (id_dylib,)) + os.chdir("..") + +__testname__ = 'test15' + +a = Analysis(['../support/_mountzlib.py', + '../support/useUnicode.py', + 'test15.py'], + pathex=[]) +pyz = PYZ(a.pure) +exe = EXE(pyz, + a.scripts, + a.binaries, + name=os.path.join('dist', __testname__), + debug=False, + strip=False, + upx=False, + console=1 ) diff --git a/buildtests/test15a.py b/buildtests/test15a.py new file mode 100644 index 0000000..c8a0e97 --- /dev/null +++ b/buildtests/test15a.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +from ctypes import * + +def dummy(arg): + tct = CDLL("testctypes.dylib") + return tct.dummy(arg) diff --git a/mf.py b/mf.py index 7c27bf1..aa724c2 100644 --- a/mf.py +++ b/mf.py @@ -25,6 +25,13 @@ try: except ImportError: zipimport = None +try: + # if ctypes is present, we can enable specific dependency discovery + import ctypes + from ctypes.util import find_library +except ImportError: + ctypes = None + import suffixes try: @@ -78,7 +85,7 @@ class DirOwner(Owner): def _getsuffixes(self): return suffixes.get_suffixes(self.target_platform) - + def getmod(self, nm, getsuffixes=None, loadco=marshal.loads): if getsuffixes is None: getsuffixes = self._getsuffixes @@ -508,6 +515,41 @@ class ImportTracker: def ispackage(self, nm): return self.modules[nm].ispackage() + def _resolveCtypesImports(self, mod): + """Completes ctypes BINARY entries for modules with their full path. + """ + if sys.platform.startswith("linux"): + envvar = "LD_LIBRARY_PATH" + elif sys.platform.startswith("darwin"): + envvar = "DYLD_LIBRARY_PATH" + else: + envvar = "PATH" + + def _savePaths(): + old = os.environ.get(envvar, None) + os.environ[envvar] = os.pathsep.join(self.path) + if old is not None: + os.environ[envvar] = os.pathsep.join([os.environ[envvar], old]) + return old + + def _restorePaths(old): + del os.environ[envvar] + if old is not None: + os.environ[envvar] = old + + cbinaries = list(mod.binaries) + mod.binaries = [] + # Executes find_library prepending ImportTracker's local paths to + # library search paths, then replaces original values. + old = _savePaths() + for cbin in cbinaries: + cpath = find_library(os.path.splitext(cbin)[0]) + if cpath is None: + print "W: library %s required via ctypes not found" % (cbin,) + else: + mod.binaries.append((cbin, cpath, "BINARY")) + _restorePaths(old) + def doimport(self, nm, ctx, fqname): # Not that nm is NEVER a dotted name at this point assert ("." not in nm), nm @@ -536,6 +578,8 @@ class ImportTracker: # or # mod = director.getmod(nm) if mod: + if ctypes and isinstance(mod, PyModule): + self._resolveCtypesImports(mod) mod.__name__ = fqname self.modules[fqname] = mod # now look for hooks @@ -602,6 +646,7 @@ class Module: self._all = [] self.imports = [] self.warnings = [] + self.binaries = [] self._xref = {} def ispackage(self): @@ -614,7 +659,7 @@ class Module: self._xref[nm] = 1 def __str__(self): - return "" % (self.__name__, self.__file__, self.imports) + return "" % (self.__name__, self.__file__, self.imports, self.binaries) class BuiltinModule(Module): typ = 'BUILTIN' @@ -641,7 +686,7 @@ class PyModule(Module): self.scancode() def scancode(self): - self.imports, self.warnings, allnms = scan_code(self.co) + self.imports, self.warnings, self.binaries, allnms = scan_code(self.co) if allnms: self._all = allnms @@ -715,6 +760,8 @@ STORE_NAME = dis.opname.index('STORE_NAME') STORE_FAST = dis.opname.index('STORE_FAST') STORE_GLOBAL = dis.opname.index('STORE_GLOBAL') LOAD_GLOBAL = dis.opname.index('LOAD_GLOBAL') +LOAD_ATTR = dis.opname.index('LOAD_ATTR') +LOAD_NAME = dis.opname.index('LOAD_NAME') EXEC_STMT = dis.opname.index('EXEC_STMT') try: SET_LINENO = dis.opname.index('SET_LINENO') @@ -767,12 +814,14 @@ def pass1(code): instrs.append((op, oparg, incondition, curline)) return instrs -def scan_code(co, m=None, w=None, nested=0): +def scan_code(co, m=None, w=None, b=None, nested=0): instrs = pass1(co.co_code) if m is None: m = [] if w is None: w = [] + if b is None: + b = [] all = None lastname = None level = -1 # import-level, same behaviour as up to Python 2.4 @@ -833,7 +882,99 @@ def scan_code(co, m=None, w=None, nested=0): w.append("W: %s %s exec statement detected at line %s" % (lvl, cndtl, curline)) else: lastname = None + + if ctypes: + # ctypes scanning requires a scope wider than one bytecode instruction, + # so the code resides in a separate function for clarity. + ctypesb, ctypesw = scan_code_for_ctypes(co, instrs, i) + b.extend(ctypesb) + w.extend(ctypesw) + for c in co.co_consts: if isinstance(c, type(co)): - scan_code(c, m, w, 1) - return m, w, all + # FIXME: "all" was not updated here nor returned. Was it the desired + # behaviour? + _, _, _, all_nested = scan_code(c, m, w, b, 1) + if all_nested: + all.extend(all_nested) + return m, w, b, all + +def scan_code_for_ctypes(co, instrs, i): + """Detects ctypes dependencies, using reasonable heuristics that should + cover most common ctypes usages; returns a tuple of two lists, one + containing names of binaries detected as dependencies, the other containing + warnings. + """ + + def _libFromConst(i): + """Extracts library name from an expected LOAD_CONST instruction and + appends it to local binaries list. + """ + op, oparg, conditional, curline = instrs[i] + if op == LOAD_CONST: + soname = co.co_consts[oparg] + b.append(soname) + + b = [] + + op, oparg, conditional, curline = instrs[i] + + if op in (LOAD_GLOBAL, LOAD_NAME): + name = co.co_names[oparg] + + if name in ("CDLL", "WinDLL"): + # Guesses ctypes imports of this type: CDLL("library.so") + + # LOAD_GLOBAL 0 (CDLL) <--- we "are" here right now + # LOAD_CONST 1 ('library.so') + + _libFromConst(i+1) + + elif name == "ctypes": + # Guesses ctypes imports of this type: ctypes.DLL("library.so") + + # LOAD_GLOBAL 0 (ctypes) <--- we "are" here right now + # LOAD_ATTR 1 (CDLL) + # LOAD_CONST 1 ('library.so') + + op2, oparg2, conditional2, curline2 = instrs[i+1] + if op2 == LOAD_ATTR: + if co.co_names[oparg2] in ("CDLL", "WinDLL"): + # Fetch next, and finally get the library name + _libFromConst(i+2) + + elif name == ("cdll", "windll"): + # Guesses ctypes imports of these types: + + # * cdll.library (only valid on Windows) + + # LOAD_GLOBAL 0 (cdll) <--- we "are" here right now + # LOAD_ATTR 1 (library) + + # * cdll.LoadLibrary("library.so") + + # LOAD_GLOBAL 0 (cdll) <--- we "are" here right now + # LOAD_ATTR 1 (LoadLibrary) + # LOAD_CONST 1 ('library.so') + + op2, oparg2, conditional2, curline2 = instrs[i+1] + if op2 == LOAD_ATTR: + if co.co_names[oparg2] != "LoadLibrary": + # First type + soname = co.co_names[oparg2] + ".dll" + b.append(soname) + else: + # Second type, needs to fetch one more instruction + _libFromConst(i+2) + + # If any of the libraries has been requested with anything different from + # the bare filename, drop that entry and warn the user - pyinstaller would + # need to patch the compiled pyc file to make it work correctly! + + w = [] + for bin in list(b): + if bin != os.path.basename(bin): + b.remove(bin) + w.append("W: ignoring %s - ctypes imports only supported using bare filenames" % (bin,)) + + return b, w