s3:lib: remove function connections_forall()
[Samba/gebeck_regimport.git] / buildtools / wafadmin / TaskGen.py
blobae1834a10a4212c4e533ab5e32045c1519e82734
1 #!/usr/bin/env python
2 # encoding: utf-8
3 # Thomas Nagy, 2005-2008 (ita)
5 """
6 The class task_gen encapsulates the creation of task objects (low-level code)
7 The instances can have various parameters, but the creation of task nodes (Task.py)
8 is delayed. To achieve this, various methods are called from the method "apply"
10 The class task_gen contains lots of methods, and a configuration table:
11 * the methods to call (self.meths) can be specified dynamically (removing, adding, ..)
12 * the order of the methods (self.prec or by default task_gen.prec) is configurable
13 * new methods can be inserted dynamically without pasting old code
15 Additionally, task_gen provides the method apply_core
16 * file extensions are mapped to methods: def meth(self, name_or_node)
17 * if a mapping is not found in self.mappings, it is searched in task_gen.mappings
18 * when called, the functions may modify self.allnodes to re-add source to process
19 * the mappings can map an extension or a filename (see the code below)
21 WARNING: subclasses must reimplement the clone method
22 """
24 import os, traceback, copy
25 import Build, Task, Utils, Logs, Options
26 from Logs import debug, error, warn
27 from Constants import *
29 typos = {
30 'sources':'source',
31 'targets':'target',
32 'include':'includes',
33 'define':'defines',
34 'importpath':'importpaths',
35 'install_var':'install_path',
36 'install_subdir':'install_path',
37 'inst_var':'install_path',
38 'inst_dir':'install_path',
39 'feature':'features',
42 class register_obj(type):
43 """no decorators for classes, so we use a metaclass
44 we store into task_gen.classes the classes that inherit task_gen
45 and whose names end in '_taskgen'
46 """
47 def __init__(cls, name, bases, dict):
48 super(register_obj, cls).__init__(name, bases, dict)
49 name = cls.__name__
50 suffix = '_taskgen'
51 if name.endswith(suffix):
52 task_gen.classes[name.replace(suffix, '')] = cls
54 class task_gen(object):
55 """
56 Most methods are of the form 'def meth(self):' without any parameters
57 there are many of them, and they do many different things:
58 * task creation
59 * task results installation
60 * environment modification
61 * attribute addition/removal
63 The inheritance approach is complicated
64 * mixing several languages at once
65 * subclassing is needed even for small changes
66 * inserting new methods is complicated
68 This new class uses a configuration table:
69 * adding new methods easily
70 * obtaining the order in which to call the methods
71 * postponing the method calls (post() -> apply)
73 Additionally, a 'traits' static attribute is provided:
74 * this list contains methods
75 * the methods can remove or add methods from self.meths
76 Example1: the attribute 'staticlib' is set on an instance
77 a method set in the list of traits is executed when the
78 instance is posted, it finds that flag and adds another method for execution
79 Example2: a method set in the list of traits finds the msvc
80 compiler (from self.env['MSVC']==1); more methods are added to self.meths
81 """
83 __metaclass__ = register_obj
84 mappings = {}
85 mapped = {}
86 prec = Utils.DefaultDict(list)
87 traits = Utils.DefaultDict(set)
88 classes = {}
90 def __init__(self, *kw, **kwargs):
91 self.prec = Utils.DefaultDict(list)
92 "map precedence of function names to call"
93 # so we will have to play with directed acyclic graphs
94 # detect cycles, etc
96 self.source = ''
97 self.target = ''
99 # list of methods to execute - does not touch it by hand unless you know
100 self.meths = []
102 # list of mappings extension -> function
103 self.mappings = {}
105 # list of features (see the documentation on traits)
106 self.features = list(kw)
108 # not always a good idea
109 self.tasks = []
111 self.default_chmod = O644
112 self.default_install_path = None
114 # kind of private, beware of what you put in it, also, the contents are consumed
115 self.allnodes = []
117 self.bld = kwargs.get('bld', Build.bld)
118 self.env = self.bld.env.copy()
120 self.path = self.bld.path # emulate chdir when reading scripts
121 self.name = '' # give a name to the target (static+shlib with the same targetname ambiguity)
123 # provide a unique id
124 self.idx = self.bld.idx[self.path.id] = self.bld.idx.get(self.path.id, 0) + 1
126 for key, val in kwargs.iteritems():
127 setattr(self, key, val)
129 self.bld.task_manager.add_task_gen(self)
130 self.bld.all_task_gen.append(self)
132 def __str__(self):
133 return ("<task_gen '%s' of type %s defined in %s>"
134 % (self.name or self.target, self.__class__.__name__, str(self.path)))
136 def __setattr__(self, name, attr):
137 real = typos.get(name, name)
138 if real != name:
139 warn('typo %s -> %s' % (name, real))
140 if Logs.verbose > 0:
141 traceback.print_stack()
142 object.__setattr__(self, real, attr)
144 def to_list(self, value):
145 "helper: returns a list"
146 if isinstance(value, str): return value.split()
147 else: return value
149 def apply(self):
150 "order the methods to execute using self.prec or task_gen.prec"
151 keys = set(self.meths)
153 # add the methods listed in the features
154 self.features = Utils.to_list(self.features)
155 for x in self.features + ['*']:
156 st = task_gen.traits[x]
157 if not st:
158 warn('feature %r does not exist - bind at least one method to it' % x)
159 keys.update(st)
161 # copy the precedence table
162 prec = {}
163 prec_tbl = self.prec or task_gen.prec
164 for x in prec_tbl:
165 if x in keys:
166 prec[x] = prec_tbl[x]
168 # elements disconnected
169 tmp = []
170 for a in keys:
171 for x in prec.values():
172 if a in x: break
173 else:
174 tmp.append(a)
176 # topological sort
177 out = []
178 while tmp:
179 e = tmp.pop()
180 if e in keys: out.append(e)
181 try:
182 nlst = prec[e]
183 except KeyError:
184 pass
185 else:
186 del prec[e]
187 for x in nlst:
188 for y in prec:
189 if x in prec[y]:
190 break
191 else:
192 tmp.append(x)
194 if prec: raise Utils.WafError("graph has a cycle %s" % str(prec))
195 out.reverse()
196 self.meths = out
198 # then we run the methods in order
199 debug('task_gen: posting %s %d', self, id(self))
200 for x in out:
201 try:
202 v = getattr(self, x)
203 except AttributeError:
204 raise Utils.WafError("tried to retrieve %s which is not a valid method" % x)
205 debug('task_gen: -> %s (%d)', x, id(self))
208 def post(self):
209 "runs the code to create the tasks, do not subclass"
210 if not self.name:
211 if isinstance(self.target, list):
212 self.name = ' '.join(self.target)
213 else:
214 self.name = self.target
216 if getattr(self, 'posted', None):
217 #error("OBJECT ALREADY POSTED" + str( self))
218 return
220 self.apply()
221 self.posted = True
222 debug('task_gen: posted %s', self.name)
224 def get_hook(self, ext):
225 try: return self.mappings[ext]
226 except KeyError:
227 try: return task_gen.mappings[ext]
228 except KeyError: return None
230 # TODO waf 1.6: always set the environment
231 # TODO waf 1.6: create_task(self, name, inputs, outputs)
232 def create_task(self, name, src=None, tgt=None, env=None):
233 env = env or self.env
234 task = Task.TaskBase.classes[name](env.copy(), generator=self)
235 if src:
236 task.set_inputs(src)
237 if tgt:
238 task.set_outputs(tgt)
239 self.tasks.append(task)
240 return task
242 def name_to_obj(self, name):
243 return self.bld.name_to_obj(name, self.env)
245 def find_sources_in_dirs(self, dirnames, excludes=[], exts=[]):
247 The attributes "excludes" and "exts" must be lists to avoid the confusion
248 find_sources_in_dirs('a', 'b', 'c') <-> find_sources_in_dirs('a b c')
250 do not use absolute paths
251 do not use paths outside of the source tree
252 the files or folder beginning by . are not returned
254 # TODO: remove in Waf 1.6
257 err_msg = "'%s' attribute must be a list"
258 if not isinstance(excludes, list):
259 raise Utils.WscriptError(err_msg % 'excludes')
260 if not isinstance(exts, list):
261 raise Utils.WscriptError(err_msg % 'exts')
263 lst = []
265 #make sure dirnames is a list helps with dirnames with spaces
266 dirnames = self.to_list(dirnames)
268 ext_lst = exts or list(self.mappings.keys()) + list(task_gen.mappings.keys())
270 for name in dirnames:
271 anode = self.path.find_dir(name)
273 if not anode or not anode.is_child_of(self.bld.srcnode):
274 raise Utils.WscriptError("Unable to use '%s' - either because it's not a relative path" \
275 ", or it's not child of '%s'." % (name, self.bld.srcnode))
277 self.bld.rescan(anode)
278 for name in self.bld.cache_dir_contents[anode.id]:
280 # ignore hidden files
281 if name.startswith('.'):
282 continue
284 (base, ext) = os.path.splitext(name)
285 if ext in ext_lst and not name in lst and not name in excludes:
286 lst.append((anode.relpath_gen(self.path) or '.') + os.path.sep + name)
288 lst.sort()
289 self.source = self.to_list(self.source)
290 if not self.source: self.source = lst
291 else: self.source += lst
293 def clone(self, env):
294 """when creating a clone in a task generator method,
295 make sure to set posted=False on the clone
296 else the other task generator will not create its tasks"""
297 newobj = task_gen(bld=self.bld)
298 for x in self.__dict__:
299 if x in ['env', 'bld']:
300 continue
301 elif x in ["path", "features"]:
302 setattr(newobj, x, getattr(self, x))
303 else:
304 setattr(newobj, x, copy.copy(getattr(self, x)))
306 newobj.__class__ = self.__class__
307 if isinstance(env, str):
308 newobj.env = self.bld.all_envs[env].copy()
309 else:
310 newobj.env = env.copy()
312 return newobj
314 def get_inst_path(self):
315 return getattr(self, '_install_path', getattr(self, 'default_install_path', ''))
317 def set_inst_path(self, val):
318 self._install_path = val
320 install_path = property(get_inst_path, set_inst_path)
323 def get_chmod(self):
324 return getattr(self, '_chmod', getattr(self, 'default_chmod', O644))
326 def set_chmod(self, val):
327 self._chmod = val
329 chmod = property(get_chmod, set_chmod)
331 def declare_extension(var, func):
332 try:
333 for x in Utils.to_list(var):
334 task_gen.mappings[x] = func
335 except:
336 raise Utils.WscriptError('declare_extension takes either a list or a string %r' % var)
337 task_gen.mapped[func.__name__] = func
339 def declare_order(*k):
340 assert(len(k) > 1)
341 n = len(k) - 1
342 for i in xrange(n):
343 f1 = k[i]
344 f2 = k[i+1]
345 if not f1 in task_gen.prec[f2]:
346 task_gen.prec[f2].append(f1)
348 def declare_chain(name='', action='', ext_in='', ext_out='', reentrant=True, color='BLUE',
349 install=0, before=[], after=[], decider=None, rule=None, scan=None):
351 see Tools/flex.py for an example
352 while i do not like such wrappers, some people really do
355 action = action or rule
356 if isinstance(action, str):
357 act = Task.simple_task_type(name, action, color=color)
358 else:
359 act = Task.task_type_from_func(name, action, color=color)
360 act.ext_in = tuple(Utils.to_list(ext_in))
361 act.ext_out = tuple(Utils.to_list(ext_out))
362 act.before = Utils.to_list(before)
363 act.after = Utils.to_list(after)
364 act.scan = scan
366 def x_file(self, node):
367 if decider:
368 ext = decider(self, node)
369 else:
370 ext = ext_out
372 if isinstance(ext, str):
373 out_source = node.change_ext(ext)
374 if reentrant:
375 self.allnodes.append(out_source)
376 elif isinstance(ext, list):
377 out_source = [node.change_ext(x) for x in ext]
378 if reentrant:
379 for i in xrange((reentrant is True) and len(out_source) or reentrant):
380 self.allnodes.append(out_source[i])
381 else:
382 # XXX: useless: it will fail on Utils.to_list above...
383 raise Utils.WafError("do not know how to process %s" % str(ext))
385 tsk = self.create_task(name, node, out_source)
387 if node.__class__.bld.is_install:
388 tsk.install = install
390 declare_extension(act.ext_in, x_file)
391 return x_file
393 def bind_feature(name, methods):
394 lst = Utils.to_list(methods)
395 task_gen.traits[name].update(lst)
398 All the following decorators are registration decorators, i.e add an attribute to current class
399 (task_gen and its derivatives), with same name as func, which points to func itself.
400 For example:
401 @taskgen
402 def sayHi(self):
403 print("hi")
404 Now taskgen.sayHi() may be called
406 If python were really smart, it could infer itself the order of methods by looking at the
407 attributes. A prerequisite for execution is to have the attribute set before.
408 Intelligent compilers binding aspect-oriented programming and parallelization, what a nice topic for studies.
410 def taskgen(func):
412 register a method as a task generator method
414 setattr(task_gen, func.__name__, func)
415 return func
417 def feature(*k):
419 declare a task generator method that will be executed when the
420 object attribute 'feature' contains the corresponding key(s)
422 def deco(func):
423 setattr(task_gen, func.__name__, func)
424 for name in k:
425 task_gen.traits[name].update([func.__name__])
426 return func
427 return deco
429 def before(*k):
431 declare a task generator method which will be executed
432 before the functions of given name(s)
434 def deco(func):
435 setattr(task_gen, func.__name__, func)
436 for fun_name in k:
437 if not func.__name__ in task_gen.prec[fun_name]:
438 task_gen.prec[fun_name].append(func.__name__)
439 return func
440 return deco
442 def after(*k):
444 declare a task generator method which will be executed
445 after the functions of given name(s)
447 def deco(func):
448 setattr(task_gen, func.__name__, func)
449 for fun_name in k:
450 if not fun_name in task_gen.prec[func.__name__]:
451 task_gen.prec[func.__name__].append(fun_name)
452 return func
453 return deco
455 def extension(var):
457 declare a task generator method which will be invoked during
458 the processing of source files for the extension given
460 def deco(func):
461 setattr(task_gen, func.__name__, func)
462 try:
463 for x in Utils.to_list(var):
464 task_gen.mappings[x] = func
465 except:
466 raise Utils.WafError('extension takes either a list or a string %r' % var)
467 task_gen.mapped[func.__name__] = func
468 return func
469 return deco
471 # TODO make certain the decorators may be used here
473 def apply_core(self):
474 """Process the attribute source
475 transform the names into file nodes
476 try to process the files by name first, later by extension"""
477 # get the list of folders to use by the scanners
478 # all our objects share the same include paths anyway
479 find_resource = self.path.find_resource
481 for filename in self.to_list(self.source):
482 # if self.mappings or task_gen.mappings contains a file of the same name
483 x = self.get_hook(filename)
484 if x:
485 x(self, filename)
486 else:
487 node = find_resource(filename)
488 if not node: raise Utils.WafError("source not found: '%s' in '%s'" % (filename, str(self.path)))
489 self.allnodes.append(node)
491 for node in self.allnodes:
492 # self.mappings or task_gen.mappings map the file extension to a function
493 x = self.get_hook(node.suffix())
495 if not x:
496 raise Utils.WafError("Cannot guess how to process %s (got mappings %r in %r) -> try conf.check_tool(..)?" % \
497 (str(node), self.__class__.mappings.keys(), self.__class__))
498 x(self, node)
499 feature('*')(apply_core)
501 def exec_rule(self):
502 """Process the attribute rule, when provided the method apply_core will be disabled
504 if not getattr(self, 'rule', None):
505 return
507 # someone may have removed it already
508 try:
509 self.meths.remove('apply_core')
510 except ValueError:
511 pass
513 # get the function and the variables
514 func = self.rule
516 vars2 = []
517 if isinstance(func, str):
518 # use the shell by default for user-defined commands
519 (func, vars2) = Task.compile_fun('', self.rule, shell=getattr(self, 'shell', True))
520 func.code = self.rule
522 # create the task class
523 name = getattr(self, 'name', None) or self.target or self.rule
524 if not isinstance(name, str):
525 name = str(self.idx)
526 cls = Task.task_type_from_func(name, func, getattr(self, 'vars', vars2))
527 cls.color = getattr(self, 'color', 'BLUE')
529 # now create one instance
530 tsk = self.create_task(name)
532 dep_vars = getattr(self, 'dep_vars', ['ruledeps'])
533 if dep_vars:
534 tsk.dep_vars = dep_vars
535 if isinstance(self.rule, str):
536 tsk.env.ruledeps = self.rule
537 else:
538 # only works if the function is in a global module such as a waf tool
539 tsk.env.ruledeps = Utils.h_fun(self.rule)
541 # we assume that the user knows that without inputs or outputs
542 #if not getattr(self, 'target', None) and not getattr(self, 'source', None):
543 # cls.quiet = True
545 if getattr(self, 'target', None):
546 cls.quiet = True
547 tsk.outputs = [self.path.find_or_declare(x) for x in self.to_list(self.target)]
549 if getattr(self, 'source', None):
550 cls.quiet = True
551 tsk.inputs = []
552 for x in self.to_list(self.source):
553 y = self.path.find_resource(x)
554 if not y:
555 raise Utils.WafError('input file %r could not be found (%r)' % (x, self.path.abspath()))
556 tsk.inputs.append(y)
558 if self.allnodes:
559 tsk.inputs.extend(self.allnodes)
561 if getattr(self, 'scan', None):
562 cls.scan = self.scan
564 if getattr(self, 'install_path', None):
565 tsk.install_path = self.install_path
567 if getattr(self, 'cwd', None):
568 tsk.cwd = self.cwd
570 if getattr(self, 'on_results', None):
571 Task.update_outputs(cls)
573 if getattr(self, 'always', None):
574 Task.always_run(cls)
576 for x in ['after', 'before', 'ext_in', 'ext_out']:
577 setattr(cls, x, getattr(self, x, []))
578 feature('*')(exec_rule)
579 before('apply_core')(exec_rule)
581 def sequence_order(self):
583 add a strict sequential constraint between the tasks generated by task generators
584 it uses the fact that task generators are posted in order
585 it will not post objects which belong to other folders
586 there is also an awesome trick for executing the method in last position
588 to use:
589 bld(features='javac seq')
590 bld(features='jar seq')
592 to start a new sequence, set the attribute seq_start, for example:
593 obj.seq_start = True
595 if self.meths and self.meths[-1] != 'sequence_order':
596 self.meths.append('sequence_order')
597 return
599 if getattr(self, 'seq_start', None):
600 return
602 # all the tasks previously declared must be run before these
603 if getattr(self.bld, 'prev', None):
604 self.bld.prev.post()
605 for x in self.bld.prev.tasks:
606 for y in self.tasks:
607 y.set_run_after(x)
609 self.bld.prev = self
611 feature('seq')(sequence_order)