waflib/Utils.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2005-2010 (ita)
   4
   5 """
   6 Utilities and platform-specific fixes
   7
   8 The portability fixes try to provide a consistent behavior of the Waf API
   9 through Python versions 2.3 to 3.X and across different platforms (win32, linux, etc)
  10 """
  11
  12 import os, sys, errno, traceback, inspect, re, shutil, datetime, gc, platform
  13 import subprocess # <- leave this!
  14
  15 from collections import deque, defaultdict
  16
  17 try:
  18         import _winreg as winreg
  19 except ImportError:
  20         try:
  21                 import winreg
  22         except ImportError:
  23                 winreg = None
  24
  25 from waflib import Errors
  26
  27 try:
  28         from collections import UserDict
  29 except ImportError:
  30         from UserDict import UserDict
  31
  32 try:
  33         from hashlib import md5
  34 except ImportError:
  35         try:
  36                 from md5 import md5
  37         except ImportError:
  38                 # never fail to enable fixes from another module
  39                 pass
  40
  41 try:
  42         import threading
  43 except ImportError:
  44         if not 'JOBS' in os.environ:
  45                 # no threading :-(
  46                 os.environ['JOBS'] = '1'
  47
  48         class threading(object):
  49                 """
  50                         A fake threading class for platforms lacking the threading module.
  51                         Use ``waf -j1`` on those platforms
  52                 """
  53                 pass
  54         class Lock(object):
  55                 """Fake Lock class"""
  56                 def acquire(self):
  57                         pass
  58                 def release(self):
  59                         pass
  60         threading.Lock = threading.Thread = Lock
  61 else:
  62         run_old = threading.Thread.run
  63         def run(*args, **kwargs):
  64                 try:
  65                         run_old(*args, **kwargs)
  66                 except (KeyboardInterrupt, SystemExit):
  67                         raise
  68                 except Exception:
  69                         sys.excepthook(*sys.exc_info())
  70         threading.Thread.run = run
  71
  72 SIG_NIL = 'iluvcuteoverload'.encode()
  73 """Arbitrary null value for a md5 hash. This value must be changed when the hash value is replaced (size)"""
  74
  75 O644 = 420
  76 """Constant representing the permissions for regular files (0644 raises a syntax error on python 3)"""
  77
  78 O755 = 493
  79 """Constant representing the permissions for executable files (0755 raises a syntax error on python 3)"""
  80
  81 rot_chr = ['\\', '|', '/', '-']
  82 "List of characters to use when displaying the throbber (progress bar)"
  83
  84 rot_idx = 0
  85 "Index of the current throbber character (progress bar)"
  86
  87 try:
  88         from collections import OrderedDict as ordered_iter_dict
  89 except ImportError:
  90         class ordered_iter_dict(dict):
  91                 def __init__(self, *k, **kw):
  92                         self.lst = []
  93                         dict.__init__(self, *k, **kw)
  94                 def clear(self):
  95                         dict.clear(self)
  96                         self.lst = []
  97                 def __setitem__(self, key, value):
  98                         dict.__setitem__(self, key, value)
  99                         try:
 100                                 self.lst.remove(key)
 101                         except ValueError:
 102                                 pass
 103                         self.lst.append(key)
 104                 def __delitem__(self, key):
 105                         dict.__delitem__(self, key)
 106                         try:
 107                                 self.lst.remove(key)
 108                         except ValueError:
 109                                 pass
 110                 def __iter__(self):
 111                         for x in self.lst:
 112                                 yield x
 113                 def keys(self):
 114                         return self.lst
 115
 116 is_win32 = os.sep == '\\' or sys.platform == 'win32' # msys2
 117
 118 def readf(fname, m='r', encoding='ISO8859-1'):
 119         """
 120         Read an entire file into a string, use this function instead of os.open() whenever possible.
 121
 122         In practice the wrapper node.read(..) should be preferred to this function::
 123
 124                 def build(ctx):
 125                         from waflib import Utils
 126                         txt = Utils.readf(self.path.find_node('wscript').abspath())
 127                         txt = ctx.path.find_node('wscript').read()
 128
 129         :type  fname: string
 130         :param fname: Path to file
 131         :type  m: string
 132         :param m: Open mode
 133         :type encoding: string
 134         :param encoding: encoding value, only used for python 3
 135         :rtype: string
 136         :return: Content of the file
 137         """
 138
 139         if sys.hexversion > 0x3000000 and not 'b' in m:
 140                 m += 'b'
 141                 f = open(fname, m)
 142                 try:
 143                         txt = f.read()
 144                 finally:
 145                         f.close()
 146                 if encoding:
 147                         txt = txt.decode(encoding)
 148                 else:
 149                         txt = txt.decode()
 150         else:
 151                 f = open(fname, m)
 152                 try:
 153                         txt = f.read()
 154                 finally:
 155                         f.close()
 156         return txt
 157
 158 def writef(fname, data, m='w', encoding='ISO8859-1'):
 159         """
 160         Write an entire file from a string, use this function instead of os.open() whenever possible.
 161
 162         In practice the wrapper node.write(..) should be preferred to this function::
 163
 164                 def build(ctx):
 165                         from waflib import Utils
 166                         txt = Utils.writef(self.path.make_node('i_like_kittens').abspath(), 'some data')
 167                         self.path.make_node('i_like_kittens').write('some data')
 168
 169         :type  fname: string
 170         :param fname: Path to file
 171         :type   data: string
 172         :param  data: The contents to write to the file
 173         :type  m: string
 174         :param m: Open mode
 175         :type encoding: string
 176         :param encoding: encoding value, only used for python 3
 177         """
 178         if sys.hexversion > 0x3000000 and not 'b' in m:
 179                 data = data.encode(encoding)
 180                 m += 'b'
 181         f = open(fname, m)
 182         try:
 183                 f.write(data)
 184         finally:
 185                 f.close()
 186
 187 def h_file(fname):
 188         """
 189         Compute a hash value for a file by using md5. This method may be replaced by
 190         a faster version if necessary. The following uses the file size and the timestamp value::
 191
 192                 import stat
 193                 from waflib import Utils
 194                 def h_file(fname):
 195                         st = os.stat(fname)
 196                         if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
 197                         m = Utils.md5()
 198                         m.update(str(st.st_mtime))
 199                         m.update(str(st.st_size))
 200                         m.update(fname)
 201                         return m.digest()
 202                 Utils.h_file = h_file
 203
 204         :type fname: string
 205         :param fname: path to the file to hash
 206         :return: hash of the file contents
 207         """
 208         f = open(fname, 'rb')
 209         m = md5()
 210         try:
 211                 while fname:
 212                         fname = f.read(200000)
 213                         m.update(fname)
 214         finally:
 215                 f.close()
 216         return m.digest()
 217
 218 def readf_win32(f, m='r', encoding='ISO8859-1'):
 219         flags = os.O_NOINHERIT | os.O_RDONLY
 220         if 'b' in m:
 221                 flags |= os.O_BINARY
 222         if '+' in m:
 223                 flags |= os.O_RDWR
 224         try:
 225                 fd = os.open(f, flags)
 226         except OSError:
 227                 raise IOError('Cannot read from %r' % f)
 228
 229         if sys.hexversion > 0x3000000 and not 'b' in m:
 230                 m += 'b'
 231                 f = os.fdopen(fd, m)
 232                 try:
 233                         txt = f.read()
 234                 finally:
 235                         f.close()
 236                 if encoding:
 237                         txt = txt.decode(encoding)
 238                 else:
 239                         txt = txt.decode()
 240         else:
 241                 f = os.fdopen(fd, m)
 242                 try:
 243                         txt = f.read()
 244                 finally:
 245                         f.close()
 246         return txt
 247
 248 def writef_win32(f, data, m='w', encoding='ISO8859-1'):
 249         if sys.hexversion > 0x3000000 and not 'b' in m:
 250                 data = data.encode(encoding)
 251                 m += 'b'
 252         flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | os.O_NOINHERIT
 253         if 'b' in m:
 254                 flags |= os.O_BINARY
 255         if '+' in m:
 256                 flags |= os.O_RDWR
 257         try:
 258                 fd = os.open(f, flags)
 259         except OSError:
 260                 raise IOError('Cannot write to %r' % f)
 261         f = os.fdopen(fd, m)
 262         try:
 263                 f.write(data)
 264         finally:
 265                 f.close()
 266
 267 def h_file_win32(fname):
 268         try:
 269                 fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT)
 270         except OSError:
 271                 raise IOError('Cannot read from %r' % fname)
 272         f = os.fdopen(fd, 'rb')
 273         m = md5()
 274         try:
 275                 while fname:
 276                         fname = f.read(200000)
 277                         m.update(fname)
 278         finally:
 279                 f.close()
 280         return m.digest()
 281
 282 # always save these
 283 readf_unix = readf
 284 writef_unix = writef
 285 h_file_unix = h_file
 286 if hasattr(os, 'O_NOINHERIT') and sys.hexversion < 0x3040000:
 287         # replace the default functions
 288         readf = readf_win32
 289         writef = writef_win32
 290         h_file = h_file_win32
 291
 292 try:
 293         x = ''.encode('hex')
 294 except LookupError:
 295         import binascii
 296         def to_hex(s):
 297                 ret = binascii.hexlify(s)
 298                 if not isinstance(ret, str):
 299                         ret = ret.decode('utf-8')
 300                 return ret
 301 else:
 302         def to_hex(s):
 303                 return s.encode('hex')
 304
 305 to_hex.__doc__ = """
 306 Return the hexadecimal representation of a string
 307
 308 :param s: string to convert
 309 :type s: string
 310 """
 311
 312 def listdir_win32(s):
 313         """
 314         List the contents of a folder in a portable manner.
 315         On Win32, return the list of drive letters: ['C:', 'X:', 'Z:']
 316
 317         :type s: string
 318         :param s: a string, which can be empty on Windows
 319         """
 320         if not s:
 321                 try:
 322                         import ctypes
 323                 except ImportError:
 324                         # there is nothing much we can do
 325                         return [x + ':\\' for x in list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')]
 326                 else:
 327                         dlen = 4 # length of "?:\\x00"
 328                         maxdrives = 26
 329                         buf = ctypes.create_string_buffer(maxdrives * dlen)
 330                         ndrives = ctypes.windll.kernel32.GetLogicalDriveStringsA(maxdrives*dlen, ctypes.byref(buf))
 331                         return [ str(buf.raw[4*i:4*i+2].decode('ascii')) for i in range(int(ndrives/dlen)) ]
 332
 333         if len(s) == 2 and s[1] == ":":
 334                 s += os.sep
 335
 336         if not os.path.isdir(s):
 337                 e = OSError('%s is not a directory' % s)
 338                 e.errno = errno.ENOENT
 339                 raise e
 340         return os.listdir(s)
 341
 342 listdir = os.listdir
 343 if is_win32:
 344         listdir = listdir_win32
 345
 346 def num2ver(ver):
 347         """
 348         Convert a string, tuple or version number into an integer. The number is supposed to have at most 4 digits::
 349
 350                 from waflib.Utils import num2ver
 351                 num2ver('1.3.2') == num2ver((1,3,2)) == num2ver((1,3,2,0))
 352
 353         :type ver: string or tuple of numbers
 354         :param ver: a version number
 355         """
 356         if isinstance(ver, str):
 357                 ver = tuple(ver.split('.'))
 358         if isinstance(ver, tuple):
 359                 ret = 0
 360                 for i in range(4):
 361                         if i < len(ver):
 362                                 ret += 256**(3 - i) * int(ver[i])
 363                 return ret
 364         return ver
 365
 366 def ex_stack():
 367         """
 368         Extract the stack to display exceptions
 369
 370         :return: a string represening the last exception
 371         """
 372         exc_type, exc_value, tb = sys.exc_info()
 373         exc_lines = traceback.format_exception(exc_type, exc_value, tb)
 374         return ''.join(exc_lines)
 375
 376 def to_list(sth):
 377         """
 378         Convert a string argument to a list by splitting on spaces, and pass
 379         through a list argument unchanged::
 380
 381                 from waflib.Utils import to_list
 382                 lst = to_list("a b c d")
 383
 384         :param sth: List or a string of items separated by spaces
 385         :rtype: list
 386         :return: Argument converted to list
 387
 388         """
 389         if isinstance(sth, str):
 390                 return sth.split()
 391         else:
 392                 return sth
 393
 394 def split_path_unix(path):
 395         return path.split('/')
 396
 397 def split_path_cygwin(path):
 398         if path.startswith('//'):
 399                 ret = path.split('/')[2:]
 400                 ret[0] = '/' + ret[0]
 401                 return ret
 402         return path.split('/')
 403
 404 re_sp = re.compile('[/\\\\]')
 405 def split_path_win32(path):
 406         if path.startswith('\\\\'):
 407                 ret = re.split(re_sp, path)[2:]
 408                 ret[0] = '\\' + ret[0]
 409                 return ret
 410         return re.split(re_sp, path)
 411
 412 msysroot = None
 413 def split_path_msys(path):
 414         if (path.startswith('/') or path.startswith('\\')) and not path.startswith('//') and not path.startswith('\\\\'):
 415                 # msys paths can be in the form /usr/bin
 416                 global msysroot
 417                 if not msysroot:
 418                         # msys has python 2.7 or 3, so we can use this
 419                         msysroot = subprocess.check_output(['cygpath', '-w', '/']).decode(sys.stdout.encoding or 'iso8859-1')
 420                         msysroot = msysroot.strip()
 421                 path = os.path.normpath(msysroot + os.sep + path)
 422         return split_path_win32(path)
 423
 424 if sys.platform == 'cygwin':
 425         split_path = split_path_cygwin
 426 elif is_win32:
 427         if os.environ.get('MSYSTEM', None):
 428                 split_path = split_path_msys
 429         else:
 430                 split_path = split_path_win32
 431 else:
 432         split_path = split_path_unix
 433
 434 split_path.__doc__ = """
 435 Split a path by / or \\. This function is not like os.path.split
 436
 437 :type  path: string
 438 :param path: path to split
 439 :return:     list of strings
 440 """
 441
 442 def check_dir(path):
 443         """
 444         Ensure that a directory exists (similar to ``mkdir -p``).
 445
 446         :type  path: string
 447         :param path: Path to directory
 448         """
 449         if not os.path.isdir(path):
 450                 try:
 451                         os.makedirs(path)
 452                 except OSError as e:
 453                         if not os.path.isdir(path):
 454                                 raise Errors.WafError('Cannot create the folder %r' % path, ex=e)
 455
 456 def check_exe(name, env=None):
 457         """
 458         Ensure that a program exists
 459
 460         :type name: string
 461         :param name: name or path to program
 462         :return: path of the program or None
 463         """
 464         if not name:
 465                 raise ValueError('Cannot execute an empty string!')
 466         def is_exe(fpath):
 467                 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
 468
 469         fpath, fname = os.path.split(name)
 470         if fpath and is_exe(name):
 471                 return os.path.abspath(name)
 472         else:
 473                 env = env or os.environ
 474                 for path in env["PATH"].split(os.pathsep):
 475                         path = path.strip('"')
 476                         exe_file = os.path.join(path, name)
 477                         if is_exe(exe_file):
 478                                 return os.path.abspath(exe_file)
 479         return None
 480
 481 def def_attrs(cls, **kw):
 482         """
 483         Set default attributes on a class instance
 484
 485         :type cls: class
 486         :param cls: the class to update the given attributes in.
 487         :type kw: dict
 488         :param kw: dictionary of attributes names and values.
 489         """
 490         for k, v in kw.items():
 491                 if not hasattr(cls, k):
 492                         setattr(cls, k, v)
 493
 494 def quote_define_name(s):
 495         """
 496         Convert a string to an identifier suitable for C defines.
 497
 498         :type  s: string
 499         :param s: String to convert
 500         :rtype: string
 501         :return: Identifier suitable for C defines
 502         """
 503         fu = re.sub('[^a-zA-Z0-9]', '_', s)
 504         fu = re.sub('_+', '_', fu)
 505         fu = fu.upper()
 506         return fu
 507
 508 def h_list(lst):
 509         """
 510         Hash lists. For tuples, using hash(tup) is much more efficient,
 511         except on python >= 3.3 where hash randomization assumes everybody is running a web application.
 512
 513         :param lst: list to hash
 514         :type lst: list of strings
 515         :return: hash of the list
 516         """
 517         m = md5()
 518         m.update(str(lst).encode())
 519         return m.digest()
 520
 521 def h_fun(fun):
 522         """
 523         Hash functions
 524
 525         :param fun: function to hash
 526         :type  fun: function
 527         :return: hash of the function
 528         """
 529         try:
 530                 return fun.code
 531         except AttributeError:
 532                 try:
 533                         h = inspect.getsource(fun)
 534                 except IOError:
 535                         h = "nocode"
 536                 try:
 537                         fun.code = h
 538                 except AttributeError:
 539                         pass
 540                 return h
 541
 542 def h_cmd(ins):
 543         """
 544         Task command hashes are calculated by calling this function. The inputs can be
 545         strings, functions, tuples/lists containing strings/functions
 546         """
 547         # this function is not meant to be particularly fast
 548         if isinstance(ins, str):
 549                 # a command is either a string
 550                 ret = ins
 551         elif isinstance(ins, list) or isinstance(ins, tuple):
 552                 # or a list of functions/strings
 553                 ret = str([h_cmd(x) for x in ins])
 554         else:
 555                 # or just a python function
 556                 ret = str(h_fun(ins))
 557         if sys.hexversion > 0x3000000:
 558                 ret = ret.encode('iso8859-1', 'xmlcharrefreplace')
 559         return ret
 560
 561 reg_subst = re.compile(r"(\\\\)|(\$\$)|\$\{([^}]+)\}")
 562 def subst_vars(expr, params):
 563         """
 564         Replace ${VAR} with the value of VAR taken from a dict or a config set::
 565
 566                 from waflib import Utils
 567                 s = Utils.subst_vars('${PREFIX}/bin', env)
 568
 569         :type  expr: string
 570         :param expr: String to perform substitution on
 571         :param params: Dictionary or config set to look up variable values.
 572         """
 573         def repl_var(m):
 574                 if m.group(1):
 575                         return '\\'
 576                 if m.group(2):
 577                         return '$'
 578                 try:
 579                         # ConfigSet instances may contain lists
 580                         return params.get_flat(m.group(3))
 581                 except AttributeError:
 582                         return params[m.group(3)]
 583                 # if you get a TypeError, it means that 'expr' is not a string...
 584                 # Utils.subst_vars(None, env)  will not work
 585         return reg_subst.sub(repl_var, expr)
 586
 587 def destos_to_binfmt(key):
 588         """
 589         Return the binary format based on the unversioned platform name.
 590
 591         :param key: platform name
 592         :type  key: string
 593         :return: string representing the binary format
 594         """
 595         if key == 'darwin':
 596                 return 'mac-o'
 597         elif key in ('win32', 'cygwin', 'uwin', 'msys'):
 598                 return 'pe'
 599         return 'elf'
 600
 601 def unversioned_sys_platform():
 602         """
 603         Return the unversioned platform name.
 604         Some Python platform names contain versions, that depend on
 605         the build environment, e.g. linux2, freebsd6, etc.
 606         This returns the name without the version number. Exceptions are
 607         os2 and win32, which are returned verbatim.
 608
 609         :rtype: string
 610         :return: Unversioned platform name
 611         """
 612         s = sys.platform
 613         if s.startswith('java'):
 614                 # The real OS is hidden under the JVM.
 615                 from java.lang import System
 616                 s = System.getProperty('os.name')
 617                 # see http://lopica.sourceforge.net/os.html for a list of possible values
 618                 if s == 'Mac OS X':
 619                         return 'darwin'
 620                 elif s.startswith('Windows '):
 621                         return 'win32'
 622                 elif s == 'OS/2':
 623                         return 'os2'
 624                 elif s == 'HP-UX':
 625                         return 'hp-ux'
 626                 elif s in ('SunOS', 'Solaris'):
 627                         return 'sunos'
 628                 else: s = s.lower()
 629
 630         # powerpc == darwin for our purposes
 631         if s == 'powerpc':
 632                 return 'darwin'
 633         if s == 'win32' or s == 'os2':
 634                 return s
 635         if s == 'cli' and os.name == 'nt':
 636                 # ironpython is only on windows as far as we know
 637                 return 'win32'
 638         return re.split('\d+$', s)[0]
 639
 640 def nada(*k, **kw):
 641         """
 642         A function that does nothing
 643
 644         :return: None
 645         """
 646         pass
 647
 648 class Timer(object):
 649         """
 650         Simple object for timing the execution of commands.
 651         Its string representation is the current time::
 652
 653                 from waflib.Utils import Timer
 654                 timer = Timer()
 655                 a_few_operations()
 656                 s = str(timer)
 657         """
 658         def __init__(self):
 659                 self.start_time = datetime.datetime.utcnow()
 660
 661         def __str__(self):
 662                 delta = datetime.datetime.utcnow() - self.start_time
 663                 days = delta.days
 664                 hours, rem = divmod(delta.seconds, 3600)
 665                 minutes, seconds = divmod(rem, 60)
 666                 seconds += delta.microseconds * 1e-6
 667                 result = ''
 668                 if days:
 669                         result += '%dd' % days
 670                 if days or hours:
 671                         result += '%dh' % hours
 672                 if days or hours or minutes:
 673                         result += '%dm' % minutes
 674                 return '%s%.3fs' % (result, seconds)
 675
 676 if is_win32:
 677         old = shutil.copy2
 678         def copy2(src, dst):
 679                 """
 680                 shutil.copy2 does not copy the file attributes on windows, so we
 681                 hack into the shutil module to fix the problem
 682                 """
 683                 old(src, dst)
 684                 shutil.copystat(src, dst)
 685         setattr(shutil, 'copy2', copy2)
 686
 687 if os.name == 'java':
 688         # Jython cannot disable the gc but they can enable it ... wtf?
 689         try:
 690                 gc.disable()
 691                 gc.enable()
 692         except NotImplementedError:
 693                 gc.disable = gc.enable
 694
 695 def read_la_file(path):
 696         """
 697         Read property files, used by msvc.py
 698
 699         :param path: file to read
 700         :type path: string
 701         """
 702         sp = re.compile(r'^([^=]+)=\'(.*)\'$')
 703         dc = {}
 704         for line in readf(path).splitlines():
 705                 try:
 706                         _, left, right, _ = sp.split(line.strip())
 707                         dc[left] = right
 708                 except ValueError:
 709                         pass
 710         return dc
 711
 712 def nogc(fun):
 713         """
 714         Decorator: let a function disable the garbage collector during its execution.
 715         It is used in the build context when storing/loading the build cache file (pickle)
 716
 717         :param fun: function to execute
 718         :type fun: function
 719         :return: the return value of the function executed
 720         """
 721         def f(*k, **kw):
 722                 try:
 723                         gc.disable()
 724                         ret = fun(*k, **kw)
 725                 finally:
 726                         gc.enable()
 727                 return ret
 728         f.__doc__ = fun.__doc__
 729         return f
 730
 731 def run_once(fun):
 732         """
 733         Decorator: let a function cache its results, use like this::
 734
 735                 @run_once
 736                 def foo(k):
 737                         return 345*2343
 738
 739         :param fun: function to execute
 740         :type fun: function
 741         :return: the return value of the function executed
 742         """
 743         cache = {}
 744         def wrap(k):
 745                 try:
 746                         return cache[k]
 747                 except KeyError:
 748                         ret = fun(k)
 749                         cache[k] = ret
 750                         return ret
 751         wrap.__cache__ = cache
 752         wrap.__name__ = fun.__name__
 753         return wrap
 754
 755 def get_registry_app_path(key, filename):
 756         if not winreg:
 757                 return None
 758         try:
 759                 result = winreg.QueryValue(key, "Software\\Microsoft\\Windows\\CurrentVersion\\App Paths\\%s.exe" % filename[0])
 760         except WindowsError:
 761                 pass
 762         else:
 763                 if os.path.isfile(result):
 764                         return result
 765
 766 def lib64():
 767         # default settings for /usr/lib
 768         if os.sep == '/':
 769                 if platform.architecture()[0] == '64bit':
 770                         if os.path.exists('/usr/lib64') and not os.path.exists('/usr/lib32'):
 771                                 return '64'
 772         return ''
 773
 774 def sane_path(p):
 775         # private function for the time being!
 776         return os.path.abspath(os.path.expanduser(p))
 777