Ticket #47: add a pragmatic but effective ctypes dependency support; a simple testcase is included (tested and working on Mac OS X only).

git-svn-id: http://svn.pyinstaller.org/trunk@599 8dd32b29-ccff-0310-8a9a-9233e24343b1

Build.py

@@ -316,10 +316,13 @@ class Analysis(Target):
                     elif isinstance(mod, mf.PkgInZipModule):
                         zipfiles.append((os.path.basename(str(mod.owner)),
                                          str(mod.owner), 'ZIPFILE'))
-                    elif modnm == '__main__':
-                        pass
                     else:
-                        pure.append((modnm, fnm, 'PYMODULE'))
+                        # mf.PyModule instances expose a list of binary
+                        # dependencies, most probably shared libraries accessed
+                        # via ctypes. Add them to the overall required binaries.
+                        binaries.extend(mod.binaries)
+                        if modnm != '__main__':
+                            pure.append((modnm, fnm, 'PYMODULE'))
         binaries.extend(bindepend.Dependencies(binaries,
                                                platform=target_platform))
         self.fixMissingPythonLib(binaries)

buildtests/ctypes/testctypes.c

@@ -0,0 +1,4 @@
+int dummy(int arg)
+{
+    return arg;
+}

buildtests/runtests.py

@@ -114,9 +114,18 @@ def runtests(alltests, filters=None, configfile=None, run_executable=1):
             # Run the test in a clean environment to make sure they're
             # really self-contained
             del os.environ["PATH"]
-            prog = os.path.join('dist', test, test)
-            if not os.path.exists(prog):
-                prog = os.path.join(prog + '.exe')
+
+            of_prog = os.path.join('dist', test) # one-file deploy filename
+            od_prog = os.path.join('dist', test, test) # one-dir deploy filename
+
+            if os.path.isfile(of_prog):
+                prog = of_prog
+            else:
+                if os.path.isfile(od_prog):
+                    prog = od_prog
+                else:
+                    prog = od_prog + ".exe"
+
             print "RUNNING:", prog
             res = os.system(prog)
             os.environ["PATH"] = path

buildtests/test15.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+try:
+    import ctypes
+except ImportError:
+    # ctypes unavailable, testing ctypes support is pointless.
+    sys.exit(0)
+
+import test15a
+assert test15a.dummy(42) == 42

buildtests/test15.spec

@@ -0,0 +1,36 @@
+# -*- mode: python -*-
+
+import sys
+if not sys.platform.startswith("darwin"):
+    raise RuntimeError("please port test15 under linux2 and win32")
+
+import os
+
+# If the required dylib does not reside in the current directory, the Analysis
+# class machinery, based on ctypes.util.find_library, will not find it. This was
+# done on purpose for this test, to show how to give Analysis class a clue.
+os.environ["DYLD_LIBRARY_PATH"] = "ctypes/"
+
+# Check for presence of testctypes shared library, build it if not present
+if not os.path.exists("ctypes/testctypes.dylib"):
+    os.chdir("ctypes")
+    os.system("gcc -Wall -dynamiclib testctypes.c -o testctypes.dylib -headerpad_max_install_names")
+    id_dylib = os.path.abspath("testctypes.dylib")
+    os.system("install_name_tool -id %s testctypes.dylib" % (id_dylib,))
+    os.chdir("..")
+
+__testname__ = 'test15'
+
+a = Analysis(['../support/_mountzlib.py',
+              '../support/useUnicode.py',
+              'test15.py'],
+             pathex=[])
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          name=os.path.join('dist', __testname__),
+          debug=False,
+          strip=False,
+          upx=False,
+          console=1 )

buildtests/test15a.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+from ctypes import *
+
+def dummy(arg):
+    tct = CDLL("testctypes.dylib")
+    return tct.dummy(arg)

mf.py

@@ -25,6 +25,13 @@ try:
 except ImportError:
     zipimport = None
 
+try:
+    # if ctypes is present, we can enable specific dependency discovery
+    import ctypes
+    from ctypes.util import find_library
+except ImportError:
+    ctypes = None
+
 import suffixes
 
 try:
@@ -78,7 +85,7 @@ class DirOwner(Owner):
 
     def _getsuffixes(self):
         return suffixes.get_suffixes(self.target_platform)
-        
+
     def getmod(self, nm, getsuffixes=None, loadco=marshal.loads):
         if getsuffixes is None:
             getsuffixes = self._getsuffixes
@@ -508,6 +515,41 @@ class ImportTracker:
     def ispackage(self, nm):
         return self.modules[nm].ispackage()
 
+    def _resolveCtypesImports(self, mod):
+        """Completes ctypes BINARY entries for modules with their full path.
+        """
+        if sys.platform.startswith("linux"):
+            envvar = "LD_LIBRARY_PATH"
+        elif sys.platform.startswith("darwin"):
+            envvar = "DYLD_LIBRARY_PATH"
+        else:
+            envvar = "PATH"
+
+        def _savePaths():
+            old = os.environ.get(envvar, None)
+            os.environ[envvar] = os.pathsep.join(self.path)
+            if old is not None:
+                os.environ[envvar] = os.pathsep.join([os.environ[envvar], old])
+            return old
+
+        def _restorePaths(old):
+            del os.environ[envvar]
+            if old is not None:
+                os.environ[envvar] = old
+
+        cbinaries = list(mod.binaries)
+        mod.binaries = []
+        # Executes find_library prepending ImportTracker's local paths to
+        # library search paths, then replaces original values.
+        old = _savePaths()
+        for cbin in cbinaries:
+            cpath = find_library(os.path.splitext(cbin)[0])
+            if cpath is None:
+                print "W: library %s required via ctypes not found" % (cbin,)
+            else:
+                mod.binaries.append((cbin, cpath, "BINARY"))
+        _restorePaths(old)
+
     def doimport(self, nm, ctx, fqname):
         # Not that nm is NEVER a dotted name at this point
         assert ("." not in nm), nm
@@ -536,6 +578,8 @@ class ImportTracker:
         # or
         #   mod = director.getmod(nm)
         if mod:
+            if ctypes and isinstance(mod, PyModule):
+                self._resolveCtypesImports(mod)
             mod.__name__ = fqname
             self.modules[fqname] = mod
             # now look for hooks
@@ -602,6 +646,7 @@ class Module:
         self._all = []
         self.imports = []
         self.warnings = []
+        self.binaries = []
         self._xref = {}
 
     def ispackage(self):
@@ -614,7 +659,7 @@ class Module:
         self._xref[nm] = 1
 
     def __str__(self):
-        return "<Module %s %s %s>" % (self.__name__, self.__file__, self.imports)
+        return "<Module %s %s %s %s>" % (self.__name__, self.__file__, self.imports, self.binaries)
 
 class BuiltinModule(Module):
     typ = 'BUILTIN'
@@ -641,7 +686,7 @@ class PyModule(Module):
         self.scancode()
 
     def scancode(self):
-        self.imports, self.warnings, allnms = scan_code(self.co)
+        self.imports, self.warnings, self.binaries, allnms = scan_code(self.co)
         if allnms:
             self._all = allnms
 
@@ -715,6 +760,8 @@ STORE_NAME = dis.opname.index('STORE_NAME')
 STORE_FAST = dis.opname.index('STORE_FAST')
 STORE_GLOBAL = dis.opname.index('STORE_GLOBAL')
 LOAD_GLOBAL = dis.opname.index('LOAD_GLOBAL')
+LOAD_ATTR = dis.opname.index('LOAD_ATTR')
+LOAD_NAME = dis.opname.index('LOAD_NAME')
 EXEC_STMT = dis.opname.index('EXEC_STMT')
 try:
     SET_LINENO = dis.opname.index('SET_LINENO')
@@ -767,12 +814,14 @@ def pass1(code):
             instrs.append((op, oparg, incondition, curline))
     return instrs
 
-def scan_code(co, m=None, w=None, nested=0):
+def scan_code(co, m=None, w=None, b=None, nested=0):
     instrs = pass1(co.co_code)
     if m is None:
         m = []
     if w is None:
         w = []
+    if b is None:
+        b = []
     all = None
     lastname = None
     level = -1 # import-level, same behaviour as up to Python 2.4
@@ -833,7 +882,99 @@ def scan_code(co, m=None, w=None, nested=0):
             w.append("W: %s %s exec statement detected at line %s"  % (lvl, cndtl, curline))
         else:
             lastname = None
+
+        if ctypes:
+            # ctypes scanning requires a scope wider than one bytecode instruction,
+            # so the code resides in a separate function for clarity.
+            ctypesb, ctypesw = scan_code_for_ctypes(co, instrs, i)
+            b.extend(ctypesb)
+            w.extend(ctypesw)
+
     for c in co.co_consts:
         if isinstance(c, type(co)):
-            scan_code(c, m, w, 1)
-    return m, w, all
+            # FIXME: "all" was not updated here nor returned. Was it the desired
+            # behaviour?
+            _, _, _, all_nested = scan_code(c, m, w, b, 1)
+            if all_nested:
+                all.extend(all_nested)
+    return m, w, b, all
+
+def scan_code_for_ctypes(co, instrs, i):
+    """Detects ctypes dependencies, using reasonable heuristics that should
+    cover most common ctypes usages; returns a tuple of two lists, one
+    containing names of binaries detected as dependencies, the other containing
+    warnings.
+    """
+
+    def _libFromConst(i):
+        """Extracts library name from an expected LOAD_CONST instruction and
+        appends it to local binaries list.
+        """
+        op, oparg, conditional, curline = instrs[i]
+        if op == LOAD_CONST:
+            soname = co.co_consts[oparg]
+            b.append(soname)
+
+    b = []
+
+    op, oparg, conditional, curline = instrs[i]
+
+    if op in (LOAD_GLOBAL, LOAD_NAME):
+        name = co.co_names[oparg]
+
+        if name in ("CDLL", "WinDLL"):
+            # Guesses ctypes imports of this type: CDLL("library.so")
+
+            # LOAD_GLOBAL 0 (CDLL) <--- we "are" here right now
+            # LOAD_CONST 1 ('library.so')
+
+            _libFromConst(i+1)
+
+        elif name == "ctypes":
+            # Guesses ctypes imports of this type: ctypes.DLL("library.so")
+
+            # LOAD_GLOBAL 0 (ctypes) <--- we "are" here right now
+            # LOAD_ATTR 1 (CDLL)
+            # LOAD_CONST 1 ('library.so')
+
+            op2, oparg2, conditional2, curline2 = instrs[i+1]
+            if op2 == LOAD_ATTR:
+                if co.co_names[oparg2] in ("CDLL", "WinDLL"):
+                    # Fetch next, and finally get the library name
+                    _libFromConst(i+2)
+
+        elif name == ("cdll", "windll"):
+            # Guesses ctypes imports of these types:
+
+            #  * cdll.library (only valid on Windows)
+
+            #     LOAD_GLOBAL 0 (cdll) <--- we "are" here right now
+            #     LOAD_ATTR 1 (library)
+
+            #  * cdll.LoadLibrary("library.so")
+
+            #     LOAD_GLOBAL              0 (cdll) <--- we "are" here right now
+            #     LOAD_ATTR                1 (LoadLibrary)
+            #     LOAD_CONST               1 ('library.so')
+
+            op2, oparg2, conditional2, curline2 = instrs[i+1]
+            if op2 == LOAD_ATTR:
+                if co.co_names[oparg2] != "LoadLibrary":
+                    # First type
+                    soname = co.co_names[oparg2] + ".dll"
+                    b.append(soname)
+                else:
+                    # Second type, needs to fetch one more instruction
+                    _libFromConst(i+2)
+
+    # If any of the libraries has been requested with anything different from
+    # the bare filename, drop that entry and warn the user - pyinstaller would
+    # need to patch the compiled pyc file to make it work correctly!
+
+    w = []
+    for bin in list(b):
+        if bin != os.path.basename(bin):
+            b.remove(bin)
+            w.append("W: ignoring %s - ctypes imports only supported using bare filenames" % (bin,))
+
+    return b, w