buildtools/wafsamba/samba_dist.py

   1 # customised version of 'waf dist' for Samba tools
   2 # uses git ls-files to get file lists
   3
   4 import Utils, os, sys, tarfile, stat, Scripting, Logs, Options
   5 from samba_utils import *
   6
   7 dist_dirs = None
   8 dist_files = None
   9 dist_blacklist = ""
  10
  11 def add_symlink(tar, fname, abspath, basedir):
  12     '''handle symlinks to directories that may move during packaging'''
  13     if not os.path.islink(abspath):
  14         return False
  15     tinfo = tar.gettarinfo(name=abspath, arcname=fname)
  16     tgt = os.readlink(abspath)
  17
  18     if dist_dirs:
  19         # we need to find the target relative to the main directory
  20         # this is here to cope with symlinks into the buildtools
  21         # directory from within the standalone libraries in Samba. For example,
  22         # a symlink to ../../builtools/scripts/autogen-waf.sh needs
  23         # to be rewritten as a symlink to buildtools/scripts/autogen-waf.sh
  24         # when the tarball for talloc is built
  25
  26         # the filename without the appname-version
  27         rel_fname = '/'.join(fname.split('/')[1:])
  28
  29         # join this with the symlink target
  30         tgt_full = os.path.join(os.path.dirname(rel_fname), tgt)
  31
  32         # join with the base directory
  33         tgt_base = os.path.normpath(os.path.join(basedir, tgt_full))
  34
  35         # see if this is inside one of our dist_dirs
  36         for dir in dist_dirs.split():
  37             if dir.find(':') != -1:
  38                 destdir=dir.split(':')[1]
  39                 dir=dir.split(':')[0]
  40             else:
  41                 destdir = '.'
  42             if dir == basedir:
  43                 # internal links don't get rewritten
  44                 continue
  45             if dir == tgt_base[0:len(dir)] and tgt_base[len(dir)] == '/':
  46                 new_tgt = destdir + tgt_base[len(dir):]
  47                 tinfo.linkname = new_tgt
  48                 break
  49
  50     tinfo.uid   = 0
  51     tinfo.gid   = 0
  52     tinfo.uname = 'root'
  53     tinfo.gname = 'root'
  54     tar.addfile(tinfo)
  55     return True
  56
  57 def add_tarfile(tar, fname, abspath, basedir):
  58     '''add a file to the tarball'''
  59     if add_symlink(tar, fname, abspath, basedir):
  60         return
  61     try:
  62         tinfo = tar.gettarinfo(name=abspath, arcname=fname)
  63     except OSError:
  64         Logs.error('Unable to find file %s - missing from git checkout?' % abspath)
  65         sys.exit(1)
  66     tinfo.uid   = 0
  67     tinfo.gid   = 0
  68     tinfo.uname = 'root'
  69     tinfo.gname = 'root'
  70     fh = open(abspath)
  71     tar.addfile(tinfo, fileobj=fh)
  72     fh.close()
  73
  74
  75 def vcs_dir_contents(path):
  76     """Return the versioned files under a path.
  77
  78     :return: List of paths relative to path
  79     """
  80     repo = path
  81     while repo != "/":
  82         if os.path.isdir(os.path.join(repo, ".git")):
  83             ls_files_cmd = [ 'git', 'ls-files', '--full-name',
  84                              os_path_relpath(path, repo) ]
  85             cwd = None
  86             env = dict(os.environ)
  87             env["GIT_DIR"] = os.path.join(repo, ".git")
  88             break
  89         repo = os.path.dirname(repo)
  90     if repo == "/":
  91         raise Exception("unsupported or no vcs for %s" % path)
  92     return Utils.cmd_output(ls_files_cmd, cwd=cwd, env=env).split()
  93
  94
  95 def dist(appname='', version=''):
  96
  97     def add_files_to_tarball(tar, srcdir, srcsubdir, dstdir, dstsubdir, blacklist, files):
  98         if blacklist is None:
  99             blacklist = []
 100         for f in files:
 101             abspath = os.path.join(srcdir, f)
 102
 103             if srcsubdir != '.':
 104                 f = f[len(srcsubdir)+1:]
 105
 106             # Remove files in the blacklist
 107             if f in blacklist:
 108                 continue
 109             blacklisted = False
 110             # Remove directories in the blacklist
 111             for d in blacklist:
 112                 if f.startswith(d):
 113                     blacklisted = True
 114             if blacklisted:
 115                 continue
 116             if os.path.isdir(abspath):
 117                 continue
 118             if dstsubdir != '.':
 119                 f = dstsubdir + '/' + f
 120             fname = dstdir + '/' + f
 121             add_tarfile(tar, fname, abspath, srcsubdir)
 122
 123
 124     def list_directory_files(path):
 125         curdir = os.getcwd()
 126         os.chdir(srcdir)
 127         out_files = []
 128         for root, dirs, files in os.walk(path):
 129             for f in files:
 130                 out_files.append(os.path.join(root, f))
 131         os.chdir(curdir)
 132         return out_files
 133
 134
 135     if not isinstance(appname, str) or not appname:
 136         # this copes with a mismatch in the calling arguments for dist()
 137         appname = Utils.g_module.APPNAME
 138         version = Utils.g_module.VERSION
 139     if not version:
 140         version = Utils.g_module.VERSION
 141
 142     srcdir = os.path.normpath(os.path.join(os.path.dirname(Utils.g_module.root_path), Utils.g_module.srcdir))
 143
 144     if not dist_dirs:
 145         Logs.error('You must use samba_dist.DIST_DIRS() to set which directories to package')
 146         sys.exit(1)
 147
 148     dist_base = '%s-%s' % (appname, version)
 149
 150     if Options.options.SIGN_RELEASE:
 151         dist_name = '%s.tar' % (dist_base)
 152         tar = tarfile.open(dist_name, 'w')
 153     else:
 154         dist_name = '%s.tar.gz' % (dist_base)
 155         tar = tarfile.open(dist_name, 'w:gz')
 156
 157     blacklist = dist_blacklist.split()
 158
 159     for dir in dist_dirs.split():
 160         if dir.find(':') != -1:
 161             destdir=dir.split(':')[1]
 162             dir=dir.split(':')[0]
 163         else:
 164             destdir = '.'
 165         absdir = os.path.join(srcdir, dir)
 166         try:
 167             files = vcs_dir_contents(absdir)
 168         except Exception, e:
 169             Logs.error('unable to get contents of %s: %s' % (absdir, e))
 170             sys.exit(1)
 171         add_files_to_tarball(tar, srcdir, dir, dist_base, destdir, blacklist, files)
 172
 173     if dist_files:
 174         for file in dist_files.split():
 175             if file.find(':') != -1:
 176                 destfile = file.split(':')[1]
 177                 file = file.split(':')[0]
 178             else:
 179                 destfile = file
 180
 181             absfile = os.path.join(srcdir, file)
 182
 183             if os.path.isdir(absfile):
 184                 destdir = destfile
 185                 dir = file
 186                 files = list_directory_files(dir)
 187                 add_files_to_tarball(tar, srcdir, dir, dist_base, destdir, blacklist, files)
 188             else:
 189                 fname = dist_base + '/' + destfile
 190                 add_tarfile(tar, fname, absfile, destfile)
 191
 192     tar.close()
 193
 194     if Options.options.SIGN_RELEASE:
 195         import gzip
 196         try:
 197             os.unlink(dist_name + '.asc')
 198         except OSError:
 199             pass
 200
 201         cmd = "gpg --detach-sign --armor " + dist_name
 202         os.system(cmd)
 203         uncompressed_tar = open(dist_name, 'rb')
 204         compressed_tar = gzip.open(dist_name + '.gz', 'wb')
 205         while 1:
 206             buffer = uncompressed_tar.read(1048576)
 207             if buffer:
 208                 compressed_tar.write(buffer)
 209             else:
 210                 break
 211         uncompressed_tar.close()
 212         compressed_tar.close()
 213         os.unlink(dist_name)
 214         Logs.info('Created %s.gz %s.asc' % (dist_name, dist_name))
 215         dist_name = dist_name + '.gz'
 216     else:
 217         Logs.info('Created %s' % dist_name)
 218
 219     return dist_name
 220
 221
 222 @conf
 223 def DIST_DIRS(dirs):
 224     '''set the directories to package, relative to top srcdir'''
 225     global dist_dirs
 226     if not dist_dirs:
 227         dist_dirs = dirs
 228
 229 @conf
 230 def DIST_FILES(files, extend=False):
 231     '''set additional files for packaging, relative to top srcdir'''
 232     global dist_files
 233     if not dist_files:
 234         dist_files = files
 235     elif extend:
 236         dist_files = dist_files + " " + files
 237
 238 @conf
 239 def DIST_BLACKLIST(blacklist):
 240     '''set the files to exclude from packaging, relative to top srcdir'''
 241     global dist_blacklist
 242     if not dist_blacklist:
 243         dist_blacklist = blacklist
 244
 245 Scripting.dist = dist