Fix bug #9586 - smbd[29175]: disk_free: sys_popen() failed" message logged in /var...
[Samba.git] / buildtools / wafadmin / Utils.py
blob91ded93b6e7d0ef194b2166e3d773b3f9fe7e8e5
1 #!/usr/bin/env python
2 # encoding: utf-8
3 # Thomas Nagy, 2005 (ita)
5 """
6 Utilities, the stable ones are the following:
8 * h_file: compute a unique value for a file (hash), it uses
9 the module fnv if it is installed (see waf/utils/fnv & http://code.google.com/p/waf/wiki/FAQ)
10 else, md5 (see the python docs)
12 For large projects (projects with more than 15000 files) or slow hard disks and filesystems (HFS)
13 it is possible to use a hashing based on the path and the size (may give broken cache results)
14 The method h_file MUST raise an OSError if the file is a folder
16 import stat
17 def h_file(filename):
18 st = os.lstat(filename)
19 if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
20 m = Utils.md5()
21 m.update(str(st.st_mtime))
22 m.update(str(st.st_size))
23 m.update(filename)
24 return m.digest()
26 To replace the function in your project, use something like this:
27 import Utils
28 Utils.h_file = h_file
30 * h_list
31 * h_fun
32 * get_term_cols
33 * ordered_dict
35 """
37 import os, sys, imp, string, errno, traceback, inspect, re, shutil, datetime, gc
39 # In python 3.0 we can get rid of all this
40 try: from UserDict import UserDict
41 except ImportError: from collections import UserDict
42 if sys.hexversion >= 0x2060000 or os.name == 'java':
43 import subprocess as pproc
44 else:
45 import pproc
46 import Logs
47 from Constants import *
49 try:
50 from collections import deque
51 except ImportError:
52 class deque(list):
53 def popleft(self):
54 return self.pop(0)
56 is_win32 = sys.platform == 'win32'
58 try:
59 # defaultdict in python 2.5
60 from collections import defaultdict as DefaultDict
61 except ImportError:
62 class DefaultDict(dict):
63 def __init__(self, default_factory):
64 super(DefaultDict, self).__init__()
65 self.default_factory = default_factory
66 def __getitem__(self, key):
67 try:
68 return super(DefaultDict, self).__getitem__(key)
69 except KeyError:
70 value = self.default_factory()
71 self[key] = value
72 return value
74 class WafError(Exception):
75 def __init__(self, *args):
76 self.args = args
77 try:
78 self.stack = traceback.extract_stack()
79 except:
80 pass
81 Exception.__init__(self, *args)
82 def __str__(self):
83 return str(len(self.args) == 1 and self.args[0] or self.args)
85 class WscriptError(WafError):
86 def __init__(self, message, wscript_file=None):
87 if wscript_file:
88 self.wscript_file = wscript_file
89 self.wscript_line = None
90 else:
91 try:
92 (self.wscript_file, self.wscript_line) = self.locate_error()
93 except:
94 (self.wscript_file, self.wscript_line) = (None, None)
96 msg_file_line = ''
97 if self.wscript_file:
98 msg_file_line = "%s:" % self.wscript_file
99 if self.wscript_line:
100 msg_file_line += "%s:" % self.wscript_line
101 err_message = "%s error: %s" % (msg_file_line, message)
102 WafError.__init__(self, err_message)
104 def locate_error(self):
105 stack = traceback.extract_stack()
106 stack.reverse()
107 for frame in stack:
108 file_name = os.path.basename(frame[0])
109 is_wscript = (file_name == WSCRIPT_FILE or file_name == WSCRIPT_BUILD_FILE)
110 if is_wscript:
111 return (frame[0], frame[1])
112 return (None, None)
114 indicator = is_win32 and '\x1b[A\x1b[K%s%s%s\r' or '\x1b[K%s%s%s\r'
116 try:
117 from fnv import new as md5
118 import Constants
119 Constants.SIG_NIL = 'signofnv'
121 def h_file(filename):
122 m = md5()
123 try:
124 m.hfile(filename)
125 x = m.digest()
126 if x is None: raise OSError("not a file")
127 return x
128 except SystemError:
129 raise OSError("not a file" + filename)
131 except ImportError:
132 try:
133 try:
134 from hashlib import md5
135 except ImportError:
136 from md5 import md5
138 def h_file(filename):
139 f = open(filename, 'rb')
140 m = md5()
141 while (filename):
142 filename = f.read(100000)
143 m.update(filename)
144 f.close()
145 return m.digest()
146 except ImportError:
147 # portability fixes may be added elsewhere (although, md5 should be everywhere by now)
148 md5 = None
150 class ordered_dict(UserDict):
151 def __init__(self, dict = None):
152 self.allkeys = []
153 UserDict.__init__(self, dict)
155 def __delitem__(self, key):
156 self.allkeys.remove(key)
157 UserDict.__delitem__(self, key)
159 def __setitem__(self, key, item):
160 if key not in self.allkeys: self.allkeys.append(key)
161 UserDict.__setitem__(self, key, item)
163 def exec_command(s, **kw):
164 if 'log' in kw:
165 kw['stdout'] = kw['stderr'] = kw['log']
166 del(kw['log'])
167 kw['shell'] = isinstance(s, str)
169 try:
170 proc = pproc.Popen(s, **kw)
171 return proc.wait()
172 except OSError:
173 return -1
175 if is_win32:
176 def exec_command(s, **kw):
177 if 'log' in kw:
178 kw['stdout'] = kw['stderr'] = kw['log']
179 del(kw['log'])
180 kw['shell'] = isinstance(s, str)
182 if len(s) > 2000:
183 startupinfo = pproc.STARTUPINFO()
184 startupinfo.dwFlags |= pproc.STARTF_USESHOWWINDOW
185 kw['startupinfo'] = startupinfo
187 try:
188 if 'stdout' not in kw:
189 kw['stdout'] = pproc.PIPE
190 kw['stderr'] = pproc.PIPE
191 kw['universal_newlines'] = True
192 proc = pproc.Popen(s,**kw)
193 (stdout, stderr) = proc.communicate()
194 Logs.info(stdout)
195 if stderr:
196 Logs.error(stderr)
197 return proc.returncode
198 else:
199 proc = pproc.Popen(s,**kw)
200 return proc.wait()
201 except OSError:
202 return -1
204 listdir = os.listdir
205 if is_win32:
206 def listdir_win32(s):
207 if re.match('^[A-Za-z]:$', s):
208 # os.path.isdir fails if s contains only the drive name... (x:)
209 s += os.sep
210 if not os.path.isdir(s):
211 e = OSError()
212 e.errno = errno.ENOENT
213 raise e
214 return os.listdir(s)
215 listdir = listdir_win32
217 def waf_version(mini = 0x010000, maxi = 0x100000):
218 "Halts if the waf version is wrong"
219 ver = HEXVERSION
220 try: min_val = mini + 0
221 except TypeError: min_val = int(mini.replace('.', '0'), 16)
223 if min_val > ver:
224 Logs.error("waf version should be at least %s (%s found)" % (mini, ver))
225 sys.exit(1)
227 try: max_val = maxi + 0
228 except TypeError: max_val = int(maxi.replace('.', '0'), 16)
230 if max_val < ver:
231 Logs.error("waf version should be at most %s (%s found)" % (maxi, ver))
232 sys.exit(1)
234 def python_24_guard():
235 if sys.hexversion < 0x20400f0 or sys.hexversion >= 0x3000000:
236 raise ImportError("Waf requires Python >= 2.3 but the raw source requires Python 2.4, 2.5 or 2.6")
238 def ex_stack():
239 exc_type, exc_value, tb = sys.exc_info()
240 if Logs.verbose > 1:
241 exc_lines = traceback.format_exception(exc_type, exc_value, tb)
242 return ''.join(exc_lines)
243 return str(exc_value)
245 def to_list(sth):
246 if isinstance(sth, str):
247 return sth.split()
248 else:
249 return sth
251 g_loaded_modules = {}
252 "index modules by absolute path"
254 g_module=None
255 "the main module is special"
257 def load_module(file_path, name=WSCRIPT_FILE):
258 "this function requires an absolute path"
259 try:
260 return g_loaded_modules[file_path]
261 except KeyError:
262 pass
264 module = imp.new_module(name)
266 try:
267 code = readf(file_path, m='rU')
268 except (IOError, OSError):
269 raise WscriptError('Could not read the file %r' % file_path)
271 module.waf_hash_val = code
273 dt = os.path.dirname(file_path)
274 sys.path.insert(0, dt)
275 try:
276 exec(compile(code, file_path, 'exec'), module.__dict__)
277 except Exception:
278 exc_type, exc_value, tb = sys.exc_info()
279 raise WscriptError("".join(traceback.format_exception(exc_type, exc_value, tb)), file_path)
280 sys.path.remove(dt)
282 g_loaded_modules[file_path] = module
284 return module
286 def set_main_module(file_path):
287 "Load custom options, if defined"
288 global g_module
289 g_module = load_module(file_path, 'wscript_main')
290 g_module.root_path = file_path
292 try:
293 g_module.APPNAME
294 except:
295 g_module.APPNAME = 'noname'
296 try:
297 g_module.VERSION
298 except:
299 g_module.VERSION = '1.0'
301 # note: to register the module globally, use the following:
302 # sys.modules['wscript_main'] = g_module
304 def to_hashtable(s):
305 "used for importing env files"
306 tbl = {}
307 lst = s.split('\n')
308 for line in lst:
309 if not line: continue
310 mems = line.split('=')
311 tbl[mems[0]] = mems[1]
312 return tbl
314 def get_term_cols():
315 "console width"
316 return 80
317 try:
318 import struct, fcntl, termios
319 except ImportError:
320 pass
321 else:
322 if Logs.got_tty:
323 def myfun():
324 dummy_lines, cols = struct.unpack("HHHH", \
325 fcntl.ioctl(sys.stderr.fileno(),termios.TIOCGWINSZ , \
326 struct.pack("HHHH", 0, 0, 0, 0)))[:2]
327 return cols
328 # we actually try the function once to see if it is suitable
329 try:
330 myfun()
331 except:
332 pass
333 else:
334 get_term_cols = myfun
336 rot_idx = 0
337 rot_chr = ['\\', '|', '/', '-']
338 "the rotation character in the progress bar"
341 def split_path(path):
342 return path.split('/')
344 def split_path_cygwin(path):
345 if path.startswith('//'):
346 ret = path.split('/')[2:]
347 ret[0] = '/' + ret[0]
348 return ret
349 return path.split('/')
351 re_sp = re.compile('[/\\\\]')
352 def split_path_win32(path):
353 if path.startswith('\\\\'):
354 ret = re.split(re_sp, path)[2:]
355 ret[0] = '\\' + ret[0]
356 return ret
357 return re.split(re_sp, path)
359 if sys.platform == 'cygwin':
360 split_path = split_path_cygwin
361 elif is_win32:
362 split_path = split_path_win32
364 def copy_attrs(orig, dest, names, only_if_set=False):
365 for a in to_list(names):
366 u = getattr(orig, a, ())
367 if u or not only_if_set:
368 setattr(dest, a, u)
370 def def_attrs(cls, **kw):
372 set attributes for class.
373 @param cls [any class]: the class to update the given attributes in.
374 @param kw [dictionary]: dictionary of attributes names and values.
376 if the given class hasn't one (or more) of these attributes, add the attribute with its value to the class.
378 for k, v in kw.iteritems():
379 if not hasattr(cls, k):
380 setattr(cls, k, v)
382 def quote_define_name(path):
383 fu = re.compile("[^a-zA-Z0-9]").sub("_", path)
384 fu = fu.upper()
385 return fu
387 def quote_whitespace(path):
388 return (path.strip().find(' ') > 0 and '"%s"' % path or path).replace('""', '"')
390 def trimquotes(s):
391 if not s: return ''
392 s = s.rstrip()
393 if s[0] == "'" and s[-1] == "'": return s[1:-1]
394 return s
396 def h_list(lst):
397 m = md5()
398 m.update(str(lst))
399 return m.digest()
401 def h_fun(fun):
402 try:
403 return fun.code
404 except AttributeError:
405 try:
406 h = inspect.getsource(fun)
407 except IOError:
408 h = "nocode"
409 try:
410 fun.code = h
411 except AttributeError:
412 pass
413 return h
415 def pprint(col, str, label='', sep='\n'):
416 "print messages in color"
417 sys.stderr.write("%s%s%s %s%s" % (Logs.colors(col), str, Logs.colors.NORMAL, label, sep))
419 def check_dir(dir):
420 """If a folder doesn't exists, create it."""
421 try:
422 os.lstat(dir)
423 except OSError:
424 try:
425 os.makedirs(dir)
426 except OSError, e:
427 raise WafError("Cannot create folder '%s' (original error: %s)" % (dir, e))
429 def cmd_output(cmd, **kw):
431 silent = False
432 if 'silent' in kw:
433 silent = kw['silent']
434 del(kw['silent'])
436 if 'e' in kw:
437 tmp = kw['e']
438 del(kw['e'])
439 kw['env'] = tmp
441 kw['shell'] = isinstance(cmd, str)
442 kw['stdout'] = pproc.PIPE
443 if silent:
444 kw['stderr'] = pproc.PIPE
446 try:
447 p = pproc.Popen(cmd, **kw)
448 output = p.communicate()[0]
449 except OSError, e:
450 raise ValueError(str(e))
452 if p.returncode:
453 if not silent:
454 msg = "command execution failed: %s -> %r" % (cmd, str(output))
455 raise ValueError(msg)
456 output = ''
457 return output
459 reg_subst = re.compile(r"(\\\\)|(\$\$)|\$\{([^}]+)\}")
460 def subst_vars(expr, params):
461 "substitute ${PREFIX}/bin in /usr/local/bin"
462 def repl_var(m):
463 if m.group(1):
464 return '\\'
465 if m.group(2):
466 return '$'
467 try:
468 # environments may contain lists
469 return params.get_flat(m.group(3))
470 except AttributeError:
471 return params[m.group(3)]
472 return reg_subst.sub(repl_var, expr)
474 def unversioned_sys_platform_to_binary_format(unversioned_sys_platform):
475 "infers the binary format from the unversioned_sys_platform name."
477 if unversioned_sys_platform in ('linux', 'freebsd', 'netbsd', 'openbsd', 'sunos', 'gnu'):
478 return 'elf'
479 elif unversioned_sys_platform == 'darwin':
480 return 'mac-o'
481 elif unversioned_sys_platform in ('win32', 'cygwin', 'uwin', 'msys'):
482 return 'pe'
483 # TODO we assume all other operating systems are elf, which is not true.
484 # we may set this to 'unknown' and have ccroot and other tools handle the case "gracefully" (whatever that means).
485 return 'elf'
487 def unversioned_sys_platform():
488 """returns an unversioned name from sys.platform.
489 sys.plaform is not very well defined and depends directly on the python source tree.
490 The version appended to the names is unreliable as it's taken from the build environment at the time python was built,
491 i.e., it's possible to get freebsd7 on a freebsd8 system.
492 So we remove the version from the name, except for special cases where the os has a stupid name like os2 or win32.
493 Some possible values of sys.platform are, amongst others:
494 aix3 aix4 atheos beos5 darwin freebsd2 freebsd3 freebsd4 freebsd5 freebsd6 freebsd7
495 generic gnu0 irix5 irix6 linux2 mac netbsd1 next3 os2emx riscos sunos5 unixware7
496 Investigating the python source tree may reveal more values.
498 s = sys.platform
499 if s == 'java':
500 # The real OS is hidden under the JVM.
501 from java.lang import System
502 s = System.getProperty('os.name')
503 # see http://lopica.sourceforge.net/os.html for a list of possible values
504 if s == 'Mac OS X':
505 return 'darwin'
506 elif s.startswith('Windows '):
507 return 'win32'
508 elif s == 'OS/2':
509 return 'os2'
510 elif s == 'HP-UX':
511 return 'hpux'
512 elif s in ('SunOS', 'Solaris'):
513 return 'sunos'
514 else: s = s.lower()
515 if s == 'win32' or s.endswith('os2') and s != 'sunos2': return s
516 return re.split('\d+$', s)[0]
518 #@deprecated('use unversioned_sys_platform instead')
519 def detect_platform():
520 """this function has been in the Utils module for some time.
521 It's hard to guess what people have used it for.
522 It seems its goal is to return an unversionned sys.platform, but it's not handling all platforms.
523 For example, the version is not removed on freebsd and netbsd, amongst others.
525 s = sys.platform
527 # known POSIX
528 for x in 'cygwin linux irix sunos hpux aix darwin gnu'.split():
529 # sys.platform may be linux2
530 if s.find(x) >= 0:
531 return x
533 # unknown POSIX
534 if os.name in 'posix java os2'.split():
535 return os.name
537 return s
539 def load_tool(tool, tooldir=None):
541 load_tool: import a Python module, optionally using several directories.
542 @param tool [string]: name of tool to import.
543 @param tooldir [list]: directories to look for the tool.
544 @return: the loaded module.
546 Warning: this function is not thread-safe: plays with sys.path,
547 so must run in sequence.
549 if tooldir:
550 assert isinstance(tooldir, list)
551 sys.path = tooldir + sys.path
552 else:
553 tooldir = []
554 try:
555 return __import__(tool)
556 finally:
557 for dt in tooldir:
558 sys.path.remove(dt)
560 def readf(fname, m='r'):
561 "get the contents of a file, it is not used anywhere for the moment"
562 f = open(fname, m)
563 try:
564 txt = f.read()
565 finally:
566 f.close()
567 return txt
569 def nada(*k, **kw):
570 """A function that does nothing"""
571 pass
573 def diff_path(top, subdir):
574 """difference between two absolute paths"""
575 top = os.path.normpath(top).replace('\\', '/').split('/')
576 subdir = os.path.normpath(subdir).replace('\\', '/').split('/')
577 if len(top) == len(subdir): return ''
578 diff = subdir[len(top) - len(subdir):]
579 return os.path.join(*diff)
581 class Context(object):
582 """A base class for commands to be executed from Waf scripts"""
584 def set_curdir(self, dir):
585 self.curdir_ = dir
587 def get_curdir(self):
588 try:
589 return self.curdir_
590 except AttributeError:
591 self.curdir_ = os.getcwd()
592 return self.get_curdir()
594 curdir = property(get_curdir, set_curdir)
596 def recurse(self, dirs, name=''):
597 """The function for calling scripts from folders, it tries to call wscript + function_name
598 and if that file does not exist, it will call the method 'function_name' from a file named wscript
599 the dirs can be a list of folders or a string containing space-separated folder paths
601 if not name:
602 name = inspect.stack()[1][3]
604 if isinstance(dirs, str):
605 dirs = to_list(dirs)
607 for x in dirs:
608 if os.path.isabs(x):
609 nexdir = x
610 else:
611 nexdir = os.path.join(self.curdir, x)
613 base = os.path.join(nexdir, WSCRIPT_FILE)
614 file_path = base + '_' + name
616 try:
617 txt = readf(file_path, m='rU')
618 except (OSError, IOError):
619 try:
620 module = load_module(base)
621 except OSError:
622 raise WscriptError('No such script %s' % base)
624 try:
625 f = module.__dict__[name]
626 except KeyError:
627 raise WscriptError('No function %s defined in %s' % (name, base))
629 if getattr(self.__class__, 'pre_recurse', None):
630 self.pre_recurse(f, base, nexdir)
631 old = self.curdir
632 self.curdir = nexdir
633 try:
634 f(self)
635 finally:
636 self.curdir = old
637 if getattr(self.__class__, 'post_recurse', None):
638 self.post_recurse(module, base, nexdir)
639 else:
640 dc = {'ctx': self}
641 if getattr(self.__class__, 'pre_recurse', None):
642 dc = self.pre_recurse(txt, file_path, nexdir)
643 old = self.curdir
644 self.curdir = nexdir
645 try:
646 try:
647 exec(compile(txt, file_path, 'exec'), dc)
648 except Exception:
649 exc_type, exc_value, tb = sys.exc_info()
650 raise WscriptError("".join(traceback.format_exception(exc_type, exc_value, tb)), base)
651 finally:
652 self.curdir = old
653 if getattr(self.__class__, 'post_recurse', None):
654 self.post_recurse(txt, file_path, nexdir)
656 if is_win32:
657 old = shutil.copy2
658 def copy2(src, dst):
659 old(src, dst)
660 shutil.copystat(src, src)
661 setattr(shutil, 'copy2', copy2)
663 def zip_folder(dir, zip_file_name, prefix):
665 prefix represents the app to add in the archive
667 import zipfile
668 zip = zipfile.ZipFile(zip_file_name, 'w', compression=zipfile.ZIP_DEFLATED)
669 base = os.path.abspath(dir)
671 if prefix:
672 if prefix[-1] != os.sep:
673 prefix += os.sep
675 n = len(base)
676 for root, dirs, files in os.walk(base):
677 for f in files:
678 archive_name = prefix + root[n:] + os.sep + f
679 zip.write(root + os.sep + f, archive_name, zipfile.ZIP_DEFLATED)
680 zip.close()
682 def get_elapsed_time(start):
683 "Format a time delta (datetime.timedelta) using the format DdHhMmS.MSs"
684 delta = datetime.datetime.now() - start
685 # cast to int necessary for python 3.0
686 days = int(delta.days)
687 hours = int(delta.seconds / 3600)
688 minutes = int((delta.seconds - hours * 3600) / 60)
689 seconds = delta.seconds - hours * 3600 - minutes * 60 \
690 + float(delta.microseconds) / 1000 / 1000
691 result = ''
692 if days:
693 result += '%dd' % days
694 if days or hours:
695 result += '%dh' % hours
696 if days or hours or minutes:
697 result += '%dm' % minutes
698 return '%s%.3fs' % (result, seconds)
700 if os.name == 'java':
701 # For Jython (they should really fix the inconsistency)
702 try:
703 gc.disable()
704 gc.enable()
705 except NotImplementedError:
706 gc.disable = gc.enable
708 def run_once(fun):
710 decorator, make a function cache its results, use like this:
712 @run_once
713 def foo(k):
714 return 345*2343
716 cache = {}
717 def wrap(k):
718 try:
719 return cache[k]
720 except KeyError:
721 ret = fun(k)
722 cache[k] = ret
723 return ret
724 wrap.__cache__ = cache
725 return wrap