torture: Fix a buffer overrun
[Samba.git] / buildtools / wafadmin / Node.py
blob236dd0d2b3f5ee044bb4a31eda439dd8840a4846
1 #!/usr/bin/env python
2 # encoding: utf-8
3 # Thomas Nagy, 2005 (ita)
5 """
6 Node: filesystem structure, contains lists of nodes
8 IMPORTANT:
9 1. Each file/folder is represented by exactly one node.
11 2. Most would-be class properties are stored in Build: nodes to depend on, signature, flags, ..
12 unused class members increase the .wafpickle file size sensibly with lots of objects.
14 3. The build is launched from the top of the build dir (for example, in _build_/).
16 4. Node should not be instantiated directly.
17 Each instance of Build.BuildContext has a Node subclass.
18 (aka: 'Nodu', see BuildContext initializer)
19 The BuildContext is referenced here as self.__class__.bld
20 Its Node class is referenced here as self.__class__
22 The public and advertised apis are the following:
23 ${TGT} -> dir/to/file.ext
24 ${TGT[0].base()} -> dir/to/file
25 ${TGT[0].dir(env)} -> dir/to
26 ${TGT[0].file()} -> file.ext
27 ${TGT[0].file_base()} -> file
28 ${TGT[0].suffix()} -> .ext
29 ${TGT[0].abspath(env)} -> /path/to/dir/to/file.ext
31 """
33 import os, sys, fnmatch, re, stat
34 import Utils, Constants
36 UNDEFINED = 0
37 DIR = 1
38 FILE = 2
39 BUILD = 3
41 type_to_string = {UNDEFINED: "unk", DIR: "dir", FILE: "src", BUILD: "bld"}
43 # These fnmatch expressions are used by default to prune the directory tree
44 # while doing the recursive traversal in the find_iter method of the Node class.
45 prune_pats = '.git .bzr .hg .svn _MTN _darcs CVS SCCS'.split()
47 # These fnmatch expressions are used by default to exclude files and dirs
48 # while doing the recursive traversal in the find_iter method of the Node class.
49 exclude_pats = prune_pats + '*~ #*# .#* %*% ._* .gitignore .cvsignore vssver.scc .DS_Store'.split()
51 # These Utils.jar_regexp expressions are used by default to exclude files and dirs and also prune the directory tree
52 # while doing the recursive traversal in the ant_glob method of the Node class.
53 exclude_regs = '''
54 **/*~
55 **/#*#
56 **/.#*
57 **/%*%
58 **/._*
59 **/CVS
60 **/CVS/**
61 **/.cvsignore
62 **/SCCS
63 **/SCCS/**
64 **/vssver.scc
65 **/.svn
66 **/.svn/**
67 **/.git
68 **/.git/**
69 **/.gitignore
70 **/.bzr
71 **/.bzr/**
72 **/.hg
73 **/.hg/**
74 **/_MTN
75 **/_MTN/**
76 **/_darcs
77 **/_darcs/**
78 **/.DS_Store'''
80 class Node(object):
81 __slots__ = ("name", "parent", "id", "childs")
82 def __init__(self, name, parent, node_type = UNDEFINED):
83 self.name = name
84 self.parent = parent
86 # assumption: one build object at a time
87 self.__class__.bld.id_nodes += 4
88 self.id = self.__class__.bld.id_nodes + node_type
90 if node_type == DIR: self.childs = {}
92 # We do not want to add another type attribute (memory)
93 # use the id to find out: type = id & 3
94 # for setting: new type = type + x - type & 3
96 if parent and name in parent.childs:
97 raise Utils.WafError('node %s exists in the parent files %r already' % (name, parent))
99 if parent: parent.childs[name] = self
101 def __setstate__(self, data):
102 if len(data) == 4:
103 (self.parent, self.name, self.id, self.childs) = data
104 else:
105 (self.parent, self.name, self.id) = data
107 def __getstate__(self):
108 if getattr(self, 'childs', None) is None:
109 return (self.parent, self.name, self.id)
110 else:
111 return (self.parent, self.name, self.id, self.childs)
113 def __str__(self):
114 if not self.parent: return ''
115 return "%s://%s" % (type_to_string[self.id & 3], self.abspath())
117 def __repr__(self):
118 return self.__str__()
120 def __hash__(self):
121 "expensive, make certain it is not used"
122 raise Utils.WafError('nodes, you are doing it wrong')
124 def __copy__(self):
125 "nodes are not supposed to be copied"
126 raise Utils.WafError('nodes are not supposed to be cloned')
128 def get_type(self):
129 return self.id & 3
131 def set_type(self, t):
132 "dangerous, you are not supposed to use this"
133 self.id = self.id + t - self.id & 3
135 def dirs(self):
136 return [x for x in self.childs.values() if x.id & 3 == DIR]
138 def files(self):
139 return [x for x in self.childs.values() if x.id & 3 == FILE]
141 def get_dir(self, name, default=None):
142 node = self.childs.get(name, None)
143 if not node or node.id & 3 != DIR: return default
144 return node
146 def get_file(self, name, default=None):
147 node = self.childs.get(name, None)
148 if not node or node.id & 3 != FILE: return default
149 return node
151 def get_build(self, name, default=None):
152 node = self.childs.get(name, None)
153 if not node or node.id & 3 != BUILD: return default
154 return node
156 def find_resource(self, lst):
157 "Find an existing input file: either a build node declared previously or a source node"
158 if isinstance(lst, str):
159 lst = Utils.split_path(lst)
161 if len(lst) == 1:
162 parent = self
163 else:
164 parent = self.find_dir(lst[:-1])
165 if not parent: return None
166 self.__class__.bld.rescan(parent)
168 name = lst[-1]
169 node = parent.childs.get(name, None)
170 if node:
171 tp = node.id & 3
172 if tp == FILE or tp == BUILD:
173 return node
174 else:
175 return None
177 tree = self.__class__.bld
178 if not name in tree.cache_dir_contents[parent.id]:
179 return None
181 path = parent.abspath() + os.sep + name
182 try:
183 st = Utils.h_file(path)
184 except IOError:
185 return None
187 child = self.__class__(name, parent, FILE)
188 tree.node_sigs[0][child.id] = st
189 return child
191 def find_or_declare(self, lst):
192 "Used for declaring a build node representing a file being built"
193 if isinstance(lst, str):
194 lst = Utils.split_path(lst)
196 if len(lst) == 1:
197 parent = self
198 else:
199 parent = self.find_dir(lst[:-1])
200 if not parent: return None
201 self.__class__.bld.rescan(parent)
203 name = lst[-1]
204 node = parent.childs.get(name, None)
205 if node:
206 tp = node.id & 3
207 if tp != BUILD:
208 raise Utils.WafError('find_or_declare found a source file where a build file was expected %r' % '/'.join(lst))
209 return node
210 node = self.__class__(name, parent, BUILD)
211 return node
213 def find_dir(self, lst):
214 "search a folder in the filesystem"
216 if isinstance(lst, str):
217 lst = Utils.split_path(lst)
219 current = self
220 for name in lst:
221 self.__class__.bld.rescan(current)
222 prev = current
224 if not current.parent and name == current.name:
225 continue
226 elif not name:
227 continue
228 elif name == '.':
229 continue
230 elif name == '..':
231 current = current.parent or current
232 else:
233 current = prev.childs.get(name, None)
234 if current is None:
235 dir_cont = self.__class__.bld.cache_dir_contents
236 if prev.id in dir_cont and name in dir_cont[prev.id]:
237 if not prev.name:
238 if os.sep == '/':
239 # cygwin //machine/share
240 dirname = os.sep + name
241 else:
242 # windows c:
243 dirname = name
244 else:
245 # regular path
246 dirname = prev.abspath() + os.sep + name
247 if not os.path.isdir(dirname):
248 return None
249 current = self.__class__(name, prev, DIR)
250 elif (not prev.name and len(name) == 2 and name[1] == ':') or name.startswith('\\\\'):
251 # drive letter or \\ path for windows
252 current = self.__class__(name, prev, DIR)
253 else:
254 return None
255 else:
256 if current.id & 3 != DIR:
257 return None
258 return current
260 def ensure_dir_node_from_path(self, lst):
261 "used very rarely, force the construction of a branch of node instance for representing folders"
263 if isinstance(lst, str):
264 lst = Utils.split_path(lst)
266 current = self
267 for name in lst:
268 if not name:
269 continue
270 elif name == '.':
271 continue
272 elif name == '..':
273 current = current.parent or current
274 else:
275 prev = current
276 current = prev.childs.get(name, None)
277 if current is None:
278 current = self.__class__(name, prev, DIR)
279 return current
281 def exclusive_build_node(self, path):
283 create a hierarchy in the build dir (no source folders) for ill-behaving compilers
284 the node is not hashed, so you must do it manually
286 after declaring such a node, find_dir and find_resource should work as expected
288 lst = Utils.split_path(path)
289 name = lst[-1]
290 if len(lst) > 1:
291 parent = None
292 try:
293 parent = self.find_dir(lst[:-1])
294 except OSError:
295 pass
296 if not parent:
297 parent = self.ensure_dir_node_from_path(lst[:-1])
298 self.__class__.bld.rescan(parent)
299 else:
300 try:
301 self.__class__.bld.rescan(parent)
302 except OSError:
303 pass
304 else:
305 parent = self
307 node = parent.childs.get(name, None)
308 if not node:
309 node = self.__class__(name, parent, BUILD)
311 return node
313 def path_to_parent(self, parent):
314 "path relative to a direct ancestor, as string"
315 lst = []
316 p = self
317 h1 = parent.height()
318 h2 = p.height()
319 while h2 > h1:
320 h2 -= 1
321 lst.append(p.name)
322 p = p.parent
323 if lst:
324 lst.reverse()
325 ret = os.path.join(*lst)
326 else:
327 ret = ''
328 return ret
330 def find_ancestor(self, node):
331 "find a common ancestor for two nodes - for the shortest path in hierarchy"
332 dist = self.height() - node.height()
333 if dist < 0: return node.find_ancestor(self)
334 # now the real code
335 cand = self
336 while dist > 0:
337 cand = cand.parent
338 dist -= 1
339 if cand == node: return cand
340 cursor = node
341 while cand.parent:
342 cand = cand.parent
343 cursor = cursor.parent
344 if cand == cursor: return cand
346 def relpath_gen(self, from_node):
347 "string representing a relative path between self to another node"
349 if self == from_node: return '.'
350 if from_node.parent == self: return '..'
352 # up_path is '../../../' and down_path is 'dir/subdir/subdir/file'
353 ancestor = self.find_ancestor(from_node)
354 lst = []
355 cand = self
356 while not cand.id == ancestor.id:
357 lst.append(cand.name)
358 cand = cand.parent
359 cand = from_node
360 while not cand.id == ancestor.id:
361 lst.append('..')
362 cand = cand.parent
363 lst.reverse()
364 return os.sep.join(lst)
366 def nice_path(self, env=None):
367 "printed in the console, open files easily from the launch directory"
368 tree = self.__class__.bld
369 ln = tree.launch_node()
371 if self.id & 3 == FILE: return self.relpath_gen(ln)
372 else: return os.path.join(tree.bldnode.relpath_gen(ln), env.variant(), self.relpath_gen(tree.srcnode))
374 def is_child_of(self, node):
375 "does this node belong to the subtree node"
376 p = self
377 diff = self.height() - node.height()
378 while diff > 0:
379 diff -= 1
380 p = p.parent
381 return p.id == node.id
383 def variant(self, env):
384 "variant, or output directory for this node, a source has for variant 0"
385 if not env: return 0
386 elif self.id & 3 == FILE: return 0
387 else: return env.variant()
389 def height(self):
390 "amount of parents"
391 # README a cache can be added here if necessary
392 d = self
393 val = -1
394 while d:
395 d = d.parent
396 val += 1
397 return val
399 # helpers for building things
401 def abspath(self, env=None):
403 absolute path
404 @param env [Environment]:
405 * obligatory for build nodes: build/variant/src/dir/bar.o
406 * optional for dirs: get either src/dir or build/variant/src/dir
407 * excluded for source nodes: src/dir/bar.c
409 Instead of computing the absolute path each time again,
410 store the already-computed absolute paths in one of (variants+1) dictionaries:
411 bld.cache_node_abspath[0] holds absolute paths for source nodes.
412 bld.cache_node_abspath[variant] holds the absolute path for the build nodes
413 which reside in the variant given by env.
415 ## absolute path - hot zone, so do not touch
417 # less expensive
418 variant = (env and (self.id & 3 != FILE) and env.variant()) or 0
420 ret = self.__class__.bld.cache_node_abspath[variant].get(self.id, None)
421 if ret: return ret
423 if not variant:
424 # source directory
425 if not self.parent:
426 val = os.sep == '/' and os.sep or ''
427 elif not self.parent.name: # root
428 val = (os.sep == '/' and os.sep or '') + self.name
429 else:
430 val = self.parent.abspath() + os.sep + self.name
431 else:
432 # build directory
433 val = os.sep.join((self.__class__.bld.bldnode.abspath(), variant, self.path_to_parent(self.__class__.bld.srcnode)))
434 self.__class__.bld.cache_node_abspath[variant][self.id] = val
435 return val
437 def change_ext(self, ext):
438 "node of the same path, but with a different extension - hot zone so do not touch"
439 name = self.name
440 k = name.rfind('.')
441 if k >= 0:
442 name = name[:k] + ext
443 else:
444 name = name + ext
446 return self.parent.find_or_declare([name])
448 def src_dir(self, env):
449 "src path without the file name"
450 return self.parent.srcpath(env)
452 def bld_dir(self, env):
453 "build path without the file name"
454 return self.parent.bldpath(env)
456 def bld_base(self, env):
457 "build path without the extension: src/dir/foo(.cpp)"
458 s = os.path.splitext(self.name)[0]
459 return os.path.join(self.bld_dir(env), s)
461 def bldpath(self, env=None):
462 "path seen from the build dir default/src/foo.cpp"
463 if self.id & 3 == FILE:
464 return self.relpath_gen(self.__class__.bld.bldnode)
465 p = self.path_to_parent(self.__class__.bld.srcnode)
466 if p is not '':
467 return env.variant() + os.sep + p
468 return env.variant()
470 def srcpath(self, env=None):
471 "path in the srcdir from the build dir ../src/foo.cpp"
472 if self.id & 3 == BUILD:
473 return self.bldpath(env)
474 return self.relpath_gen(self.__class__.bld.bldnode)
476 def read(self, env):
477 "get the contents of a file, it is not used anywhere for the moment"
478 return Utils.readf(self.abspath(env))
480 def dir(self, env):
481 "scons-like"
482 return self.parent.abspath(env)
484 def file(self):
485 "scons-like"
486 return self.name
488 def file_base(self):
489 "scons-like"
490 return os.path.splitext(self.name)[0]
492 def suffix(self):
493 "scons-like - hot zone so do not touch"
494 k = max(0, self.name.rfind('.'))
495 return self.name[k:]
497 def find_iter_impl(self, src=True, bld=True, dir=True, accept_name=None, is_prune=None, maxdepth=25):
498 """find nodes in the filesystem hierarchy, try to instanciate the nodes passively; same gotcha as ant_glob"""
499 bld_ctx = self.__class__.bld
500 bld_ctx.rescan(self)
501 for name in bld_ctx.cache_dir_contents[self.id]:
502 if accept_name(self, name):
503 node = self.find_resource(name)
504 if node:
505 if src and node.id & 3 == FILE:
506 yield node
507 else:
508 node = self.find_dir(name)
509 if node and node.id != bld_ctx.bldnode.id:
510 if dir:
511 yield node
512 if not is_prune(self, name):
513 if maxdepth:
514 for k in node.find_iter_impl(src, bld, dir, accept_name, is_prune, maxdepth=maxdepth - 1):
515 yield k
516 else:
517 if not is_prune(self, name):
518 node = self.find_resource(name)
519 if not node:
520 # not a file, it is a dir
521 node = self.find_dir(name)
522 if node and node.id != bld_ctx.bldnode.id:
523 if maxdepth:
524 for k in node.find_iter_impl(src, bld, dir, accept_name, is_prune, maxdepth=maxdepth - 1):
525 yield k
527 if bld:
528 for node in self.childs.values():
529 if node.id == bld_ctx.bldnode.id:
530 continue
531 if node.id & 3 == BUILD:
532 if accept_name(self, node.name):
533 yield node
534 raise StopIteration
536 def find_iter(self, in_pat=['*'], ex_pat=exclude_pats, prune_pat=prune_pats, src=True, bld=True, dir=False, maxdepth=25, flat=False):
537 """find nodes recursively, this returns everything but folders by default; same gotcha as ant_glob"""
539 if not (src or bld or dir):
540 raise StopIteration
542 if self.id & 3 != DIR:
543 raise StopIteration
545 in_pat = Utils.to_list(in_pat)
546 ex_pat = Utils.to_list(ex_pat)
547 prune_pat = Utils.to_list(prune_pat)
549 def accept_name(node, name):
550 for pat in ex_pat:
551 if fnmatch.fnmatchcase(name, pat):
552 return False
553 for pat in in_pat:
554 if fnmatch.fnmatchcase(name, pat):
555 return True
556 return False
558 def is_prune(node, name):
559 for pat in prune_pat:
560 if fnmatch.fnmatchcase(name, pat):
561 return True
562 return False
564 ret = self.find_iter_impl(src, bld, dir, accept_name, is_prune, maxdepth=maxdepth)
565 if flat:
566 return " ".join([x.relpath_gen(self) for x in ret])
568 return ret
570 def ant_glob(self, *k, **kw):
572 known gotcha: will enumerate the files, but only if the folder exists in the source directory
575 src=kw.get('src', 1)
576 bld=kw.get('bld', 0)
577 dir=kw.get('dir', 0)
578 excl = kw.get('excl', exclude_regs)
579 incl = k and k[0] or kw.get('incl', '**')
581 def to_pat(s):
582 lst = Utils.to_list(s)
583 ret = []
584 for x in lst:
585 x = x.replace('//', '/')
586 if x.endswith('/'):
587 x += '**'
588 lst2 = x.split('/')
589 accu = []
590 for k in lst2:
591 if k == '**':
592 accu.append(k)
593 else:
594 k = k.replace('.', '[.]').replace('*', '.*').replace('?', '.')
595 k = '^%s$' % k
596 #print "pattern", k
597 accu.append(re.compile(k))
598 ret.append(accu)
599 return ret
601 def filtre(name, nn):
602 ret = []
603 for lst in nn:
604 if not lst:
605 pass
606 elif lst[0] == '**':
607 ret.append(lst)
608 if len(lst) > 1:
609 if lst[1].match(name):
610 ret.append(lst[2:])
611 else:
612 ret.append([])
613 elif lst[0].match(name):
614 ret.append(lst[1:])
615 return ret
617 def accept(name, pats):
618 nacc = filtre(name, pats[0])
619 nrej = filtre(name, pats[1])
620 if [] in nrej:
621 nacc = []
622 return [nacc, nrej]
624 def ant_iter(nodi, maxdepth=25, pats=[]):
625 nodi.__class__.bld.rescan(nodi)
626 tmp = list(nodi.__class__.bld.cache_dir_contents[nodi.id])
627 tmp.sort()
628 for name in tmp:
629 npats = accept(name, pats)
630 if npats and npats[0]:
631 accepted = [] in npats[0]
632 #print accepted, nodi, name
634 node = nodi.find_resource(name)
635 if node and accepted:
636 if src and node.id & 3 == FILE:
637 yield node
638 else:
639 node = nodi.find_dir(name)
640 if node and node.id != nodi.__class__.bld.bldnode.id:
641 if accepted and dir:
642 yield node
643 if maxdepth:
644 for k in ant_iter(node, maxdepth=maxdepth - 1, pats=npats):
645 yield k
646 if bld:
647 for node in nodi.childs.values():
648 if node.id == nodi.__class__.bld.bldnode.id:
649 continue
650 if node.id & 3 == BUILD:
651 npats = accept(node.name, pats)
652 if npats and npats[0] and [] in npats[0]:
653 yield node
654 raise StopIteration
656 ret = [x for x in ant_iter(self, pats=[to_pat(incl), to_pat(excl)])]
658 if kw.get('flat', True):
659 return " ".join([x.relpath_gen(self) for x in ret])
661 return ret
663 def update_build_dir(self, env=None):
665 if not env:
666 for env in bld.all_envs:
667 self.update_build_dir(env)
668 return
670 path = self.abspath(env)
672 lst = Utils.listdir(path)
673 try:
674 self.__class__.bld.cache_dir_contents[self.id].update(lst)
675 except KeyError:
676 self.__class__.bld.cache_dir_contents[self.id] = set(lst)
677 self.__class__.bld.cache_scanned_folders[self.id] = True
679 for k in lst:
680 npath = path + os.sep + k
681 st = os.stat(npath)
682 if stat.S_ISREG(st[stat.ST_MODE]):
683 ick = self.find_or_declare(k)
684 if not (ick.id in self.__class__.bld.node_sigs[env.variant()]):
685 self.__class__.bld.node_sigs[env.variant()][ick.id] = Constants.SIG_NIL
686 elif stat.S_ISDIR(st[stat.ST_MODE]):
687 child = self.find_dir(k)
688 if not child:
689 child = self.ensure_dir_node_from_path(k)
690 child.update_build_dir(env)
693 class Nodu(Node):
694 pass