buildtools/wafsamba/samba_dist.py

   1 # customised version of 'waf dist' for Samba tools
   2 # uses git ls-files to get file lists
   3
   4 import Utils, os, sys, tarfile, stat, Scripting, Logs, Options
   5 from samba_utils import *
   6
   7 dist_dirs = None
   8 dist_files = None
   9 dist_blacklist = ""
  10
  11 def add_symlink(tar, fname, abspath, basedir):
  12     '''handle symlinks to directories that may move during packaging'''
  13     if not os.path.islink(abspath):
  14         return False
  15     tinfo = tar.gettarinfo(name=abspath, arcname=fname)
  16     tgt = os.readlink(abspath)
  17
  18     if dist_dirs:
  19         # we need to find the target relative to the main directory
  20         # this is here to cope with symlinks into the buildtools
  21         # directory from within the standalone libraries in Samba. For example,
  22         # a symlink to ../../builtools/scripts/autogen-waf.sh needs
  23         # to be rewritten as a symlink to buildtools/scripts/autogen-waf.sh
  24         # when the tarball for talloc is built
  25
  26         # the filename without the appname-version
  27         rel_fname = '/'.join(fname.split('/')[1:])
  28
  29         # join this with the symlink target
  30         tgt_full = os.path.join(os.path.dirname(rel_fname), tgt)
  31
  32         # join with the base directory
  33         tgt_base = os.path.normpath(os.path.join(basedir, tgt_full))
  34
  35         # see if this is inside one of our dist_dirs
  36         for dir in dist_dirs.split():
  37             if dir.find(':') != -1:
  38                 destdir=dir.split(':')[1]
  39                 dir=dir.split(':')[0]
  40             else:
  41                 destdir = '.'
  42             if dir == basedir:
  43                 # internal links don't get rewritten
  44                 continue
  45             if dir == tgt_base[0:len(dir)] and tgt_base[len(dir)] == '/':
  46                 new_tgt = destdir + tgt_base[len(dir):]
  47                 tinfo.linkname = new_tgt
  48                 break
  49
  50     tinfo.uid   = 0
  51     tinfo.gid   = 0
  52     tinfo.uname = 'root'
  53     tinfo.gname = 'root'
  54     tar.addfile(tinfo)
  55     return True
  56
  57 def add_tarfile(tar, fname, abspath, basedir):
  58     '''add a file to the tarball'''
  59     if add_symlink(tar, fname, abspath, basedir):
  60         return
  61     try:
  62         tinfo = tar.gettarinfo(name=abspath, arcname=fname)
  63     except OSError:
  64         Logs.error('Unable to find file %s - missing from git checkout?' % abspath)
  65         sys.exit(1)
  66     tinfo.uid   = 0
  67     tinfo.gid   = 0
  68     tinfo.uname = 'root'
  69     tinfo.gname = 'root'
  70     fh = open(abspath)
  71     tar.addfile(tinfo, fileobj=fh)
  72     fh.close()
  73
  74
  75 def vcs_dir_contents(path):
  76     """Return the versioned files under a path.
  77
  78     :return: List of paths relative to path
  79     """
  80     repo = path
  81     while repo != "/":
  82         if os.path.isdir(os.path.join(repo, ".git")):
  83             ls_files_cmd = [ 'git', 'ls-files', '--full-name',
  84                              os_path_relpath(path, repo) ]
  85             cwd = None
  86             env = dict(os.environ)
  87             env["GIT_DIR"] = os.path.join(repo, ".git")
  88             break
  89         elif os.path.isdir(os.path.join(repo, ".bzr")):
  90             ls_files_cmd = [ 'bzr', 'ls', '--recursive', '--versioned',
  91                              os_path_relpath(path, repo)]
  92             cwd = repo
  93             env = None
  94             break
  95         repo = os.path.dirname(repo)
  96     if repo == "/":
  97         raise Exception("unsupported or no vcs for %s" % path)
  98     return Utils.cmd_output(ls_files_cmd, cwd=cwd, env=env).split()
  99
 100
 101 def dist(appname='',version=''):
 102
 103     def add_files_to_tarball(tar, srcdir, srcsubdir, dstdir, dstsubdir, blacklist, files):
 104         if blacklist == None:
 105             blacklist = []
 106         for f in files:
 107             abspath = os.path.join(srcdir, f)
 108
 109             if srcsubdir != '.':
 110                 f = f[len(srcsubdir)+1:]
 111
 112             # Remove files in the blacklist
 113             if f in blacklist:
 114                 continue
 115             blacklisted = False
 116             # Remove directories in the blacklist
 117             for d in blacklist:
 118                 if f.startswith(d):
 119                     blacklisted = True
 120             if blacklisted:
 121                 continue
 122             if os.path.isdir(abspath):
 123                 continue
 124             if dstsubdir != '.':
 125                 f = dstsubdir + '/' + f
 126             fname = dstdir + '/' + f
 127             add_tarfile(tar, fname, abspath, srcsubdir)
 128
 129
 130     if not isinstance(appname, str) or not appname:
 131         # this copes with a mismatch in the calling arguments for dist()
 132         appname = Utils.g_module.APPNAME
 133         version = Utils.g_module.VERSION
 134     if not version:
 135         version = Utils.g_module.VERSION
 136
 137     srcdir = os.path.normpath(os.path.join(os.path.dirname(Utils.g_module.root_path), Utils.g_module.srcdir))
 138
 139     if not dist_dirs:
 140         Logs.error('You must use samba_dist.DIST_DIRS() to set which directories to package')
 141         sys.exit(1)
 142
 143     dist_base = '%s-%s' % (appname, version)
 144
 145     if Options.options.SIGN_RELEASE:
 146         dist_name = '%s.tar' % (dist_base)
 147         tar = tarfile.open(dist_name, 'w')
 148     else:
 149         dist_name = '%s.tar.gz' % (dist_base)
 150         tar = tarfile.open(dist_name, 'w:gz')
 151
 152     blacklist = dist_blacklist.split()
 153
 154     for dir in dist_dirs.split():
 155         if dir.find(':') != -1:
 156             destdir=dir.split(':')[1]
 157             dir=dir.split(':')[0]
 158         else:
 159             destdir = '.'
 160         absdir = os.path.join(srcdir, dir)
 161         try:
 162             files = vcs_dir_contents(absdir)
 163         except Exception, e:
 164             Logs.error('unable to get contents of %s: %s' % (absdir, e))
 165             sys.exit(1)
 166         add_files_to_tarball(tar, srcdir, dir, dist_base, destdir, blacklist, files)
 167
 168     if dist_files:
 169         for file in dist_files.split():
 170             if file.find(':') != -1:
 171                 destfile = file.split(':')[1]
 172                 file = file.split(':')[0]
 173             else:
 174                 destfile = file
 175
 176             absfile = os.path.join(srcdir, file)
 177
 178             if destfile != file:
 179                 file = destfile
 180
 181             fname = dist_base + '/' + file
 182             add_tarfile(tar, fname, absfile, file)
 183
 184     tar.close()
 185
 186     if Options.options.SIGN_RELEASE:
 187         import gzip
 188         try:
 189             os.unlink(dist_name + '.asc')
 190         except OSError:
 191             pass
 192
 193         cmd = "gpg --detach-sign --armor " + dist_name
 194         os.system(cmd)
 195         uncompressed_tar = open(dist_name, 'rb')
 196         compressed_tar = gzip.open(dist_name + '.gz', 'wb')
 197         while 1:
 198             buffer = uncompressed_tar.read(1048576)
 199             if buffer:
 200                 compressed_tar.write(buffer)
 201             else:
 202                 break
 203         uncompressed_tar.close()
 204         compressed_tar.close()
 205         os.unlink(dist_name)
 206         Logs.info('Created %s.gz %s.asc' % (dist_name, dist_name))
 207         dist_name = dist_name + '.gz'
 208     else:
 209         Logs.info('Created %s' % dist_name)
 210
 211     return dist_name
 212
 213
 214 @conf
 215 def DIST_DIRS(dirs):
 216     '''set the directories to package, relative to top srcdir'''
 217     global dist_dirs
 218     if not dist_dirs:
 219         dist_dirs = dirs
 220
 221 @conf
 222 def DIST_FILES(files, extend=False):
 223     '''set additional files for packaging, relative to top srcdir'''
 224     global dist_files
 225     if not dist_files:
 226         dist_files = files
 227     elif extend:
 228         dist_files = dist_files + " " + files
 229
 230 @conf
 231 def DIST_BLACKLIST(blacklist):
 232     '''set the files to exclude from packaging, relative to top srcdir'''
 233     global dist_blacklist
 234     if not dist_blacklist:
 235         dist_blacklist = blacklist
 236
 237 Scripting.dist = dist