Add support for datafiles in import hooks.

Now import hooks can specify data files (even full trees) and get
them automatically collected (in --onedir mode) or bundled within
the executabled (in --onefile mode).


git-svn-id: http://svn.pyinstaller.org/trunk@613 8dd32b29-ccff-0310-8a9a-9233e24343b1

Build.py

@@ -230,6 +230,7 @@ class Analysis(Target):
         self.pure = TOC()
         self.binaries = TOC()
         self.zipfiles = TOC()
+        self.datas = TOC()
         self.__postinit__()
 
     GUTS = (('inputs',    _check_guts_eq),
@@ -241,6 +242,7 @@ class Analysis(Target):
                                               args, {'pyc': 1 }   )),
             ('binaries',  _check_guts_toc_mtime),
             ('zipfiles',  _check_guts_toc_mtime),
+            ('datas',     _check_guts_toc_mtime),
             )
 
     def check_guts(self, last_build):
@@ -255,11 +257,12 @@ class Analysis(Target):
         data = Target.get_guts(self, last_build)
         if not data:
             return True
-        scripts, pure, binaries, zipfiles = data[-4:]
+        scripts, pure, binaries, zipfiles, datas = data[-5:]
         self.scripts = TOC(scripts)
         self.pure = TOC(pure)
         self.binaries = TOC(binaries)
         self.zipfiles = TOC(zipfiles)
+        self.datas = TOC(datas)
         return False
 
     def assemble(self):
@@ -300,6 +303,7 @@ class Analysis(Target):
         pure = []     # pure python modules
         binaries = [] # binaries to bundle
         zipfiles = [] # zipfiles to bundle
+        datas = []    # datafiles to bundle
         rthooks = []  # rthooks if needed
         for modnm, mod in analyzer.modules.items():
             # FIXME: why can we have a mod == None here?
@@ -307,6 +311,7 @@ class Analysis(Target):
                 hooks = findRTHook(modnm)  #XXX
                 if hooks:
                     rthooks.extend(hooks)
+                datas.extend(mod.datas)
                 if isinstance(mod, mf.BuiltinModule):
                     pass
                 else:
@@ -331,6 +336,7 @@ class Analysis(Target):
         self.pure = TOC(pure)
         self.binaries = TOC(binaries)
         self.zipfiles = TOC(zipfiles)
+        self.datas = TOC(datas)
         try: # read .toc
             oldstuff = _load_data(self.out)
         except:
@@ -339,7 +345,7 @@ class Analysis(Target):
         self.pure = TOC(compile_pycos(self.pure))
 
         newstuff = (self.inputs, self.pathex, self.hookspath, self.excludes,
-                    self.scripts, self.pure, self.binaries, self.zipfiles)
+                    self.scripts, self.pure, self.binaries, self.zipfiles, self.datas)
         if oldstuff != newstuff:
             _save_data(self.out, newstuff)
             wf = open(WARNFILE, 'w')

Makespec.py

@@ -36,6 +36,7 @@ exe = EXE(%(tkpkg)s pyz,
           a.scripts,
           a.binaries,
           a.zipfiles,
+          a.datas,
           name=os.path.join('%(distdir)s', '%(exename)s'),
           debug=%(debug)s,
           strip=%(strip)s,
@@ -58,6 +59,7 @@ exe = EXE(pyz,
 coll = COLLECT(%(tktree)s exe,
                a.binaries,
                a.zipfiles,
+               a.datas,
                strip=%(strip)s,
                upx=%(upx)s,
                name=os.path.join('%(distdir)s', '%(name)s'))
@@ -83,6 +85,7 @@ dll = DLL(pyz,
 coll = COLLECT(exe, dll,
                a.binaries,
                a.zipfiles,
+               a.datas,
                strip=%(strip)s,
                upx=%(upx)s,
                name=os.path.join('%(distdir)s', '%(name)s'))

mf.py

@@ -18,7 +18,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 #
 
-import sys, string, os, imp, marshal, dircache
+import sys, string, os, imp, marshal, dircache, glob
 try:
     # zipimport is supported starting with Python 2.3
     import zipimport
@@ -588,8 +588,7 @@ class ImportTracker:
                 hookmodnm = 'hook-'+fqname
                 hooks = __import__('hooks', globals(), locals(), [hookmodnm])
                 hook = getattr(hooks, hookmodnm)
-                #print `hook`
-            except (ImportError, AttributeError):
+            except AttributeError:
                 pass
             else:
                 # rearranged so that hook() has a chance to mess with hiddenimports & attrs
@@ -601,7 +600,19 @@ class ImportTracker:
                 if hasattr(hook, 'attrs'):
                     for attr, val in hook.attrs:
                         setattr(mod, attr, val)
-
+                if hasattr(hook, 'datas'):
+                    # hook.datas is a list of globs of files or directories to bundle
+                    # as datafiles. For each glob, a destination directory is specified.
+                    for g,dest_dir in hook.datas:
+                        for fn in glob.glob(g):
+                            if os.path.isfile(fn):
+                                mod.datas.append((dest_dir + "/" + os.path.basename(fn), fn, 'DATA'))
+                            else:
+                                def visit((base,dest_dir,datas), dirname, names):
+                                    for fn in names:
+                                        fn = os.path.join(dirname, fn)
+                                        datas.append(dest_dir + "/" + (fn[len(base)+1:], fn, 'DATA'))
+                                os.path.walk(fn, visit, (os.path.dirname(fn),dest_dir,mod.datas))
                 if fqname != mod.__name__:
                     print "W: %s is changing it's name to %s" % (fqname, mod.__name__)
                     self.modules[mod.__name__] = mod
@@ -647,6 +658,7 @@ class Module:
         self.imports = []
         self.warnings = []
         self.binaries = []
+        self.datas = []
         self._xref = {}
 
     def ispackage(self):
@@ -659,7 +671,8 @@ class Module:
         self._xref[nm] = 1
 
     def __str__(self):
-        return "<Module %s %s %s %s>" % (self.__name__, self.__file__, self.imports, self.binaries)
+        return "<Module %s %s imports=%s binaries=%s datas=%s>" % \
+            (self.__name__, self.__file__, self.imports, self.binaries, self.datas)
 
 class BuiltinModule(Module):
     typ = 'BUILTIN'