wscript

   1 #! /usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2005-2015
   4
   5 """
   6 to make a custom waf file use the option --tools
   7
   8 To add a tool that does not exist in the folder compat15, pass an absolute path:
   9 ./waf-light  --tools=compat15,/comp/waf/aba.py --prelude=$'\tfrom waflib.extras import aba\n\taba.foo()'
  10 """
  11
  12
  13 VERSION="1.8.8"
  14 APPNAME='waf'
  15 REVISION=''
  16
  17 top = '.'
  18 out = 'build'
  19
  20 zip_types = ['bz2', 'gz', 'xz']
  21
  22 PRELUDE = ''
  23
  24 #from tokenize import *
  25 import tokenize
  26
  27 import os, sys, re, io, optparse
  28
  29 from waflib import Utils, Options, Logs
  30 from hashlib import md5
  31
  32 from waflib import Configure
  33 Configure.autoconfig = 1
  34
  35 def sub_file(fname, lst):
  36
  37         f = open(fname, 'rU')
  38         try:
  39                 txt = f.read()
  40         finally:
  41                 f.close()
  42
  43         for (key, val) in lst:
  44                 re_pat = re.compile(key, re.M)
  45                 txt = re_pat.sub(val, txt)
  46
  47         f = open(fname, 'w')
  48         try:
  49                 f.write(txt)
  50         finally:
  51                 f.close()
  52
  53 def to_bytes(x):
  54         if sys.hexversion>0x300000f:
  55                 return x.encode()
  56         return x
  57
  58 print("------> Executing code from the top-level wscript <-----")
  59 def init(ctx):
  60         if Options.options.setver: # maintainer only (ita)
  61                 ver = Options.options.setver
  62                 hexver = Utils.num2ver(ver)
  63                 hexver = '0x%x'%hexver
  64                 sub_file('wscript', (('^VERSION=(.*)', 'VERSION="%s"' % ver), ))
  65                 sub_file('waf-light', (('^VERSION=(.*)', 'VERSION="%s"' % ver), ))
  66
  67                 pats = []
  68                 pats.append(('^WAFVERSION=(.*)', 'WAFVERSION="%s"' % ver))
  69                 pats.append(('^HEXVERSION(.*)', 'HEXVERSION=%s' % hexver))
  70
  71                 try:
  72                         rev = ctx.cmd_and_log("git rev-parse HEAD").strip()
  73                         pats.append(('^WAFREVISION(.*)', 'WAFREVISION="%s"' % rev))
  74                 except Exception:
  75                         rev = ''
  76
  77                 sub_file('waflib/Context.py', pats)
  78
  79                 sys.exit(0)
  80
  81 def check(ctx):
  82         Logs.warn('Nothing to do')
  83
  84 # this function is called before any other for parsing the command-line
  85 def options(opt):
  86
  87         # generate waf
  88         opt.add_option('--make-waf', action='store_true', default=True,
  89                 help='creates the waf script', dest='waf')
  90
  91         opt.add_option('--sign', action='store_true', default=False, help='make a signed file', dest='signed')
  92
  93         opt.add_option('--zip-type', action='store', default='bz2',
  94                 help='specify the zip type [Allowed values: %s]' % ' '.join(zip_types), dest='zip')
  95
  96         opt.add_option('--make-batch', action='store_true', default=False,
  97                 help='creates a convenience waf.bat file (done automatically on win32 systems)',
  98                 dest='make_batch')
  99
 100         opt.add_option('--yes', action='store_true', default=False,
 101                 help=optparse.SUPPRESS_HELP,
 102                 dest='yes')
 103
 104         # those ones are not too interesting
 105         opt.add_option('--set-version', default='',
 106                 help='sets the version number for waf releases (for the maintainer)', dest='setver')
 107
 108         opt.add_option('--strip', action='store_true', default=True,
 109                 help='shrinks waf (strip docstrings, saves 33kb)',
 110                 dest='strip_comments')
 111         opt.add_option('--nostrip', action='store_false', help='no shrinking',
 112                 dest='strip_comments')
 113         opt.add_option('--tools', action='store', help='Comma-separated 3rd party tools to add, eg: "compat,ocaml" [Default: "compat15"]',
 114                 dest='add3rdparty', default='compat15')
 115         opt.add_option('--coretools', action='store', help='Comma-separated core tools to add, eg: "vala,tex" [Default: all of them]',
 116                 dest='coretools', default='default')
 117         opt.add_option('--prelude', action='store', help='Code to execute before calling waf', dest='prelude', default=PRELUDE)
 118         opt.load('python')
 119
 120 def process_tokens(tokens):
 121         accu = []
 122         prev = tokenize.NEWLINE
 123
 124         indent = 0
 125         line_buf = []
 126
 127         for (type, token, start, end, line) in tokens:
 128                 token = token.replace('\r\n', '\n')
 129                 if type == tokenize.NEWLINE:
 130                         if line_buf:
 131                                 accu.append(indent * '\t')
 132                                 ln = "".join(line_buf)
 133                                 if ln == 'if __name__=="__main__":': break
 134                                 #ln = ln.replace('\n', '')
 135                                 accu.append(ln)
 136                                 accu.append('\n')
 137                                 line_buf = []
 138                                 prev = tokenize.NEWLINE
 139                 elif type == tokenize.INDENT:
 140                         indent += 1
 141                 elif type == tokenize.DEDENT:
 142                         indent -= 1
 143                 elif type == tokenize.NAME:
 144                         if prev == tokenize.NAME or prev == tokenize.NUMBER: line_buf.append(' ')
 145                         line_buf.append(token)
 146                 elif type == tokenize.NUMBER:
 147                         if prev == tokenize.NAME or prev == tokenize.NUMBER: line_buf.append(' ')
 148                         line_buf.append(token)
 149                 elif type == tokenize.STRING:
 150                         if not line_buf and token.startswith('"'): pass
 151                         else: line_buf.append(token)
 152                 elif type == tokenize.COMMENT:
 153                         pass
 154                 elif type == tokenize.OP:
 155                         line_buf.append(token)
 156                 else:
 157                         if token != "\n": line_buf.append(token)
 158
 159                 if token != '\n':
 160                         prev = type
 161
 162         body = "".join(accu)
 163         return body
 164
 165 deco_re = re.compile('(def|class)\\s+(\w+)\\(.*')
 166 def process_decorators(body):
 167         lst = body.splitlines()
 168         accu = []
 169         all_deco = []
 170         buf = [] # put the decorator lines
 171         for line in lst:
 172                 if line.startswith('@'):
 173                         buf.append(line[1:])
 174                 elif buf:
 175                         name = deco_re.sub('\\2', line)
 176                         if not name:
 177                                 raise IOError("decorator not followed by a function!" + line)
 178                         for x in buf:
 179                                 all_deco.append("%s(%s)" % (x, name))
 180                         accu.append(line)
 181                         buf = []
 182                 else:
 183                         accu.append(line)
 184         return "\n".join(accu+all_deco)
 185
 186 def sfilter(path):
 187
 188         if path.endswith('.py') :
 189                 if Options.options.strip_comments:
 190                         if sys.version_info[0] >= 3:
 191                                 f = open(path, "rb")
 192                                 try:
 193                                         tk = tokenize.tokenize(f.readline)
 194                                         next(tk) # the first one is always tokenize.ENCODING for Python 3, ignore it
 195                                         cnt = process_tokens(tk)
 196                                 finally:
 197                                         f.close()
 198                         else:
 199                                 f = open(path, "r")
 200                                 try:
 201                                         cnt = process_tokens(tokenize.generate_tokens(f.readline))
 202                                 finally:
 203                                         f.close()
 204                 else:
 205                         f = open(path, "r")
 206                         try:
 207                                 cnt = f.read()
 208                         finally:
 209                                 f.close()
 210                 # WARNING: since we now require python 2.4, we do not process the decorators anymore
 211                 # if you need such a thing, uncomment the code below:
 212                 #cnt = process_decorators(cnt)
 213                 #if cnt.find('set(') > -1:
 214                 #       cnt = 'import sys\nif sys.hexversion < 0x020400f0: from sets import Set as set\n' + cnt
 215                 cnt = '#! /usr/bin/env python\n# encoding: utf-8\n# WARNING! Do not edit! http://waf.googlecode.com/git/docs/wafbook/single.html#_obtaining_the_waf_file\n\n' + cnt
 216
 217         else:
 218                 f = open(path, "r")
 219                 try:
 220                         cnt = f.read()
 221                 finally:
 222                         f.close()
 223
 224         if sys.hexversion > 0x030000f0:
 225                 return (io.BytesIO(cnt.encode('utf-8')), len(cnt.encode('utf-8')), cnt)
 226         return (io.BytesIO(cnt), len(cnt), cnt)
 227
 228 def create_waf(self, *k, **kw):
 229         mw = 'tmp-waf-'+VERSION
 230         print("-> preparing %r" % mw)
 231
 232         import tarfile, re, zipfile
 233
 234         zipType = Options.options.zip.strip().lower()
 235         if zipType not in zip_types:
 236                 zipType = zip_types[0]
 237
 238
 239         files = []
 240         add3rdparty = []
 241         for x in Options.options.add3rdparty.split(','):
 242                 if os.path.isabs(x):
 243                         files.append(x)
 244                 else:
 245                         add3rdparty.append(x + '.py')
 246
 247         coretools = []
 248         for x in Options.options.coretools.split(','):
 249                 coretools.append(x + '.py')
 250
 251         for d in '. Tools extras'.split():
 252                 dd = os.path.join('waflib', d)
 253                 for k in os.listdir(dd):
 254                         if k == '__init__.py':
 255                                 files.append(os.path.join(dd, k))
 256                                 continue
 257                         if d == 'Tools' and Options.options.coretools != 'default':
 258                                 if not k in coretools:
 259                                         continue
 260                         if d == 'extras':
 261                                 if not k in add3rdparty:
 262                                         continue
 263                         if k.endswith('.py'):
 264                                 files.append(os.path.join(dd, k))
 265
 266         #open a file as tar.[extension] for writing
 267         tar = tarfile.open('%s.tar.%s' % (mw, zipType), "w:%s" % zipType)
 268         z = zipfile.ZipFile("zip/waflib.zip", "w", compression=zipfile.ZIP_DEFLATED)
 269         for x in files:
 270                 tarinfo = tar.gettarinfo(x, x)
 271                 tarinfo.uid   = tarinfo.gid   = 0
 272                 tarinfo.uname = tarinfo.gname = 'root'
 273                 (code, size, cnt) = sfilter(x)
 274                 tarinfo.size = size
 275
 276                 if os.path.isabs(x):
 277                         tarinfo.name = 'waflib/extras/' + os.path.split(x)[1]
 278
 279                 print("   adding %s as %s" % (x, tarinfo.name))
 280
 281                 def dest(x):
 282                         if os.path.isabs(x):
 283                                 return os.path.join("extras", os.path.basename(x))
 284                         else:
 285                                 return os.path.normpath(os.path.relpath(x, "."))
 286
 287                 z.write(x, dest(x))
 288                 tar.addfile(tarinfo, code)
 289         tar.close()
 290         z.close()
 291
 292         f = open('waf-light', 'rU')
 293         try:
 294                 code1 = f.read()
 295         finally:
 296                 f.close()
 297
 298         # now store the revision unique number in waf
 299         code1 = code1.replace("if sys.hexversion<0x206000f:\n\traise ImportError('Python >= 2.6 is required to create the waf file')\n", '')
 300         code1 = code1.replace('\t#import waflib.extras.compat15#PRELUDE', Options.options.prelude)
 301
 302         # when possible, set the git revision in the waf file
 303         bld = self.generator.bld
 304         try:
 305                 rev = bld.cmd_and_log("git rev-parse HEAD", quiet=0).strip()
 306         except Exception:
 307                 rev = ''
 308         else:
 309                 reg = re.compile('^GIT(.*)', re.M)
 310                 code1 = reg.sub('GIT="%s"' % rev, code1)
 311
 312         # if the waf file is installed somewhere... but do not do that
 313         prefix = ''
 314         reg = re.compile('^INSTALL=(.*)', re.M)
 315         code1 = reg.sub(r'INSTALL=%r' % prefix, code1)
 316         #change the tarfile extension in the waf script
 317         reg = re.compile('bz2', re.M)
 318         code1 = reg.sub(zipType, code1)
 319         if zipType == 'gz':
 320                 code1 = code1.replace('bunzip2', 'gzip -d')
 321         elif zipType == 'xz':
 322                 code1 = code1.replace('bunzip2', 'xz -d')
 323
 324         f = open('%s.tar.%s' % (mw, zipType), 'rb')
 325         try:
 326                 cnt = f.read()
 327         finally:
 328                 f.close()
 329
 330         # the REVISION value is the md5 sum of the binary blob (facilitate audits)
 331         m = md5()
 332         m.update(cnt)
 333         REVISION = m.hexdigest()
 334         reg = re.compile('^REVISION=(.*)', re.M)
 335         code1 = reg.sub(r'REVISION="%s"' % REVISION, code1)
 336
 337         def find_unused(kd, ch):
 338                 for i in range(35, 125):
 339                         for j in range(35, 125):
 340                                 if i==j: continue
 341                                 if i == 39 or j == 39: continue
 342                                 if i == 92 or j == 92: continue
 343                                 s = chr(i) + chr(j)
 344                                 if -1 == kd.find(s.encode()):
 345                                         return (kd.replace(ch.encode(), s.encode()), s)
 346                 raise
 347
 348         # The reverse order prevents collisions
 349         (cnt, C3) = find_unused(cnt, '\x00')
 350         (cnt, C2) = find_unused(cnt, '\r')
 351         (cnt, C1) = find_unused(cnt, '\n')
 352         ccc = code1.replace("C1='x'", "C1='%s'" % C1).replace("C2='x'", "C2='%s'" % C2).replace("C3='x'", "C3='%s'" % C3)
 353
 354         f = open('waf', 'wb')
 355         try:
 356                 f.write(ccc.encode())
 357                 f.write(to_bytes('#==>\n#'))
 358                 f.write(cnt)
 359                 f.write(to_bytes('\n#<==\n'))
 360
 361                 if Options.options.signed:
 362                         f.flush()
 363                         try:
 364                                 os.remove('waf.asc')
 365                         except OSError:
 366                                 pass
 367                         ret = Utils.subprocess.Popen('gpg -bass waf', shell=True).wait()
 368                         if ret:
 369                                 raise ValueError('Could not sign the waf file!')
 370
 371                         sig = Utils.readf('waf.asc')
 372                         sig = sig.replace('\r', '').replace('\n', '\\n')
 373                         f.write('#')
 374                         f.write(sig)
 375                         f.write('\n')
 376         finally:
 377                 f.close()
 378
 379
 380         if sys.platform == 'win32' or Options.options.make_batch:
 381                 f = open('waf.bat', 'w')
 382                 try:
 383                         f.write('@python -x "%~dp0waf" %*\n@exit /b %ERRORLEVEL%\n')
 384                 finally:
 385                         f.close()
 386
 387         if sys.platform != 'win32':
 388                 os.chmod('waf', Utils.O755)
 389         os.remove('%s.tar.%s' % (mw, zipType))
 390
 391 def make_copy(inf, outf):
 392         (a, b, cnt) = sfilter(inf)
 393         f = open(outf, "wb")
 394         try:
 395                 f.write(cnt)
 396         finally:
 397                 f.close()
 398
 399 def configure(conf):
 400         conf.load('python')
 401         conf.check_python_version((2,4))
 402
 403 def build(bld):
 404         waf = bld.path.make_node('waf') # create the node right here
 405         bld(name='create_waf', rule=create_waf, target=waf, always=True, color='PINK', update_outputs=True)
 406