buildtools/wafsamba/samba_dist.py

   1 # customised version of 'waf dist' for Samba tools
   2 # uses git ls-files to get file lists
   3
   4 import os, sys, tarfile
   5 from waflib import Utils, Scripting, Logs, Options
   6 from waflib.Configure import conf
   7 from samba_utils import get_string
   8 from waflib import Context
   9
  10 dist_dirs = None
  11 dist_files = None
  12 dist_blacklist = ""
  13 dist_archive = None
  14
  15 class Dist(Context.Context):
  16     # TODO remove
  17     cmd = 'dist'
  18     fun = 'dist'
  19     def execute(self):
  20         Context.g_module.dist()
  21
  22 class DistCheck(Scripting.DistCheck):
  23     fun = 'distcheck'
  24     cmd = 'distcheck'
  25     def execute(self):
  26         Options.options.distcheck_args = ''
  27         if Context.g_module.distcheck is Scripting.distcheck:
  28             # default
  29             Context.g_module.distcheck(self)
  30         else:
  31             Context.g_module.distcheck()
  32         Context.g_module.dist()
  33         self.check()
  34     def get_arch_name(self):
  35         global dist_archive
  36         return dist_archive
  37     def make_distcheck_cmd(self, tmpdir):
  38         waf = os.path.abspath(sys.argv[0])
  39         return [sys.executable, waf, 'configure', 'build', 'install', 'uninstall', '--destdir=' + tmpdir]
  40
  41 def add_symlink(tar, fname, abspath, basedir):
  42     '''handle symlinks to directories that may move during packaging'''
  43     if not os.path.islink(abspath):
  44         return False
  45     tinfo = tar.gettarinfo(name=abspath, arcname=fname)
  46     tgt = os.readlink(abspath)
  47
  48     if dist_dirs:
  49         # we need to find the target relative to the main directory
  50         # this is here to cope with symlinks into the buildtools
  51         # directory from within the standalone libraries in Samba. For example,
  52         # a symlink to ../../builtools/scripts/autogen-waf.sh needs
  53         # to be rewritten as a symlink to buildtools/scripts/autogen-waf.sh
  54         # when the tarball for talloc is built
  55
  56         # the filename without the appname-version
  57         rel_fname = '/'.join(fname.split('/')[1:])
  58
  59         # join this with the symlink target
  60         tgt_full = os.path.join(os.path.dirname(rel_fname), tgt)
  61
  62         # join with the base directory
  63         tgt_base = os.path.normpath(os.path.join(basedir, tgt_full))
  64
  65         # see if this is inside one of our dist_dirs
  66         for dir in dist_dirs.split():
  67             if dir.find(':') != -1:
  68                 destdir=dir.split(':')[1]
  69                 dir=dir.split(':')[0]
  70             else:
  71                 destdir = '.'
  72             if dir == basedir:
  73                 # internal links don't get rewritten
  74                 continue
  75             if dir == tgt_base[0:len(dir)] and tgt_base[len(dir)] == '/':
  76                 new_tgt = destdir + tgt_base[len(dir):]
  77                 tinfo.linkname = new_tgt
  78                 break
  79
  80     tinfo.uid   = 0
  81     tinfo.gid   = 0
  82     tinfo.uname = 'root'
  83     tinfo.gname = 'root'
  84     tar.addfile(tinfo)
  85     return True
  86
  87 def add_tarfile(tar, fname, abspath, basedir):
  88     '''add a file to the tarball'''
  89     if add_symlink(tar, fname, abspath, basedir):
  90         return
  91     try:
  92         tinfo = tar.gettarinfo(name=abspath, arcname=fname)
  93     except OSError:
  94         Logs.error('Unable to find file %s - missing from git checkout?' % abspath)
  95         sys.exit(1)
  96     tinfo.uid   = 0
  97     tinfo.gid   = 0
  98     tinfo.uname = 'root'
  99     tinfo.gname = 'root'
 100     fh = open(abspath, "rb")
 101     tar.addfile(tinfo, fileobj=fh)
 102     fh.close()
 103
 104
 105 def vcs_dir_contents(path):
 106     """Return the versioned files under a path.
 107
 108     :return: List of paths relative to path
 109     """
 110     repo = path
 111     while repo != "/":
 112         if os.path.exists(os.path.join(repo, ".git")):
 113             ls_files_cmd = [ 'git', 'ls-files', '--full-name',
 114                              os.path.relpath(path, repo) ]
 115             cwd = None
 116             env = dict(os.environ)
 117             env["GIT_DIR"] = os.path.join(repo, ".git")
 118             break
 119         repo = os.path.dirname(repo)
 120     if repo == "/":
 121         raise Exception("unsupported or no vcs for %s" % path)
 122     return get_string(Utils.cmd_output(ls_files_cmd, cwd=cwd, env=env)).split('\n')
 123
 124
 125 def dist(appname='', version=''):
 126
 127     def add_files_to_tarball(tar, srcdir, srcsubdir, dstdir, dstsubdir, blacklist, files):
 128         if blacklist is None:
 129             blacklist = []
 130         for f in files:
 131             abspath = os.path.join(srcdir, f)
 132
 133             if srcsubdir != '.':
 134                 f = f[len(srcsubdir)+1:]
 135
 136             # Remove files in the blacklist
 137             if f in blacklist:
 138                 continue
 139             blacklisted = False
 140             # Remove directories in the blacklist
 141             for d in blacklist:
 142                 if f.startswith(d):
 143                     blacklisted = True
 144             if blacklisted:
 145                 continue
 146             if os.path.isdir(abspath) and not os.path.islink(abspath):
 147                 continue
 148             if dstsubdir != '.':
 149                 f = dstsubdir + '/' + f
 150             fname = dstdir + '/' + f
 151             add_tarfile(tar, fname, abspath, srcsubdir)
 152
 153
 154     def list_directory_files(path):
 155         curdir = os.getcwd()
 156         os.chdir(srcdir)
 157         out_files = []
 158         for root, dirs, files in os.walk(path):
 159             for f in files:
 160                 out_files.append(os.path.join(root, f))
 161         os.chdir(curdir)
 162         return out_files
 163
 164
 165     if not isinstance(appname, str) or not appname:
 166         # this copes with a mismatch in the calling arguments for dist()
 167         appname = Context.g_module.APPNAME
 168         version = Context.g_module.VERSION
 169     if not version:
 170         version = Context.g_module.VERSION
 171
 172     srcdir = os.path.normpath(
 173                 os.path.join(os.path.dirname(Context.g_module.root_path),
 174                     Context.g_module.top))
 175
 176     if not dist_dirs:
 177         Logs.error('You must use samba_dist.DIST_DIRS() to set which directories to package')
 178         sys.exit(1)
 179
 180     dist_base = '%s-%s' % (appname, version)
 181
 182     if Options.options.SIGN_RELEASE:
 183         dist_name = '%s.tar' % (dist_base)
 184         tar = tarfile.open(dist_name, 'w')
 185     else:
 186         dist_name = '%s.tar.gz' % (dist_base)
 187         tar = tarfile.open(dist_name, 'w:gz')
 188
 189     blacklist = dist_blacklist.split()
 190
 191     for dir in dist_dirs.split():
 192         if dir.find(':') != -1:
 193             destdir=dir.split(':')[1]
 194             dir=dir.split(':')[0]
 195         else:
 196             destdir = '.'
 197         absdir = os.path.join(srcdir, dir)
 198         try:
 199             files = vcs_dir_contents(absdir)
 200         except Exception as e:
 201             Logs.error('unable to get contents of %s: %s' % (absdir, e))
 202             sys.exit(1)
 203         add_files_to_tarball(tar, srcdir, dir, dist_base, destdir, blacklist, files)
 204
 205     if dist_files:
 206         for file in dist_files.split():
 207             if file.find(':') != -1:
 208                 destfile = file.split(':')[1]
 209                 file = file.split(':')[0]
 210             else:
 211                 destfile = file
 212
 213             absfile = os.path.join(srcdir, file)
 214
 215             if os.path.isdir(absfile) and not os.path.islink(absfile):
 216                 destdir = destfile
 217                 dir = file
 218                 files = list_directory_files(dir)
 219                 add_files_to_tarball(tar, srcdir, dir, dist_base, destdir, blacklist, files)
 220             else:
 221                 fname = dist_base + '/' + destfile
 222                 add_tarfile(tar, fname, absfile, destfile)
 223
 224     tar.close()
 225
 226     if Options.options.SIGN_RELEASE:
 227         import gzip
 228         try:
 229             os.unlink(dist_name + '.asc')
 230         except OSError:
 231             pass
 232
 233         cmd = "gpg --detach-sign --armor " + dist_name
 234         os.system(cmd)
 235         uncompressed_tar = open(dist_name, 'rb')
 236         compressed_tar = gzip.open(dist_name + '.gz', 'wb')
 237         while 1:
 238             buffer = uncompressed_tar.read(1048576)
 239             if buffer:
 240                 compressed_tar.write(buffer)
 241             else:
 242                 break
 243         uncompressed_tar.close()
 244         compressed_tar.close()
 245         os.unlink(dist_name)
 246         Logs.info('Created %s.gz %s.asc' % (dist_name, dist_name))
 247         dist_name = dist_name + '.gz'
 248     else:
 249         Logs.info('Created %s' % dist_name)
 250
 251     # TODO use the ctx object instead
 252     global dist_archive
 253     dist_archive = dist_name
 254     return dist_name
 255
 256
 257 @conf
 258 def DIST_DIRS(dirs):
 259     '''set the directories to package, relative to top srcdir'''
 260     global dist_dirs
 261     if not dist_dirs:
 262         dist_dirs = dirs
 263
 264 @conf
 265 def DIST_FILES(files, extend=False):
 266     '''set additional files for packaging, relative to top srcdir'''
 267     global dist_files
 268     if not dist_files:
 269         dist_files = files
 270     elif extend:
 271         dist_files = dist_files + " " + files
 272
 273 @conf
 274 def DIST_BLACKLIST(blacklist):
 275     '''set the files to exclude from packaging, relative to top srcdir'''
 276     global dist_blacklist
 277     if not dist_blacklist:
 278         dist_blacklist = blacklist
 279
 280 Scripting.dist = dist