buildtools/wafadmin/TaskGen.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2005-2008 (ita)
   4
   5 """
   6 The class task_gen encapsulates the creation of task objects (low-level code)
   7 The instances can have various parameters, but the creation of task nodes (Task.py)
   8 is delayed. To achieve this, various methods are called from the method "apply"
   9
  10 The class task_gen contains lots of methods, and a configuration table:
  11 * the methods to call (self.meths) can be specified dynamically (removing, adding, ..)
  12 * the order of the methods (self.prec or by default task_gen.prec) is configurable
  13 * new methods can be inserted dynamically without pasting old code
  14
  15 Additionally, task_gen provides the method apply_core
  16 * file extensions are mapped to methods: def meth(self, name_or_node)
  17 * if a mapping is not found in self.mappings, it is searched in task_gen.mappings
  18 * when called, the functions may modify self.allnodes to re-add source to process
  19 * the mappings can map an extension or a filename (see the code below)
  20
  21 WARNING: subclasses must reimplement the clone method
  22 """
  23
  24 import os, traceback, copy
  25 import Build, Task, Utils, Logs, Options
  26 from Logs import debug, error, warn
  27 from Constants import *
  28
  29 typos = {
  30 'sources':'source',
  31 'targets':'target',
  32 'include':'includes',
  33 'define':'defines',
  34 'importpath':'importpaths',
  35 'install_var':'install_path',
  36 'install_subdir':'install_path',
  37 'inst_var':'install_path',
  38 'inst_dir':'install_path',
  39 'feature':'features',
  40 }
  41
  42 class register_obj(type):
  43         """no decorators for classes, so we use a metaclass
  44         we store into task_gen.classes the classes that inherit task_gen
  45         and whose names end in '_taskgen'
  46         """
  47         def __init__(cls, name, bases, dict):
  48                 super(register_obj, cls).__init__(name, bases, dict)
  49                 name = cls.__name__
  50                 suffix = '_taskgen'
  51                 if name.endswith(suffix):
  52                         task_gen.classes[name.replace(suffix, '')] = cls
  53
  54 class task_gen(object):
  55         """
  56         Most methods are of the form 'def meth(self):' without any parameters
  57         there are many of them, and they do many different things:
  58         * task creation
  59         * task results installation
  60         * environment modification
  61         * attribute addition/removal
  62
  63         The inheritance approach is complicated
  64         * mixing several languages at once
  65         * subclassing is needed even for small changes
  66         * inserting new methods is complicated
  67
  68         This new class uses a configuration table:
  69         * adding new methods easily
  70         * obtaining the order in which to call the methods
  71         * postponing the method calls (post() -> apply)
  72
  73         Additionally, a 'traits' static attribute is provided:
  74         * this list contains methods
  75         * the methods can remove or add methods from self.meths
  76         Example1: the attribute 'staticlib' is set on an instance
  77         a method set in the list of traits is executed when the
  78         instance is posted, it finds that flag and adds another method for execution
  79         Example2: a method set in the list of traits finds the msvc
  80         compiler (from self.env['MSVC']==1); more methods are added to self.meths
  81         """
  82
  83         __metaclass__ = register_obj
  84         mappings = {}
  85         mapped = {}
  86         prec = Utils.DefaultDict(list)
  87         traits = Utils.DefaultDict(set)
  88         classes = {}
  89
  90         def __init__(self, *kw, **kwargs):
  91                 self.prec = Utils.DefaultDict(list)
  92                 "map precedence of function names to call"
  93                 # so we will have to play with directed acyclic graphs
  94                 # detect cycles, etc
  95
  96                 self.source = ''
  97                 self.target = ''
  98
  99                 # list of methods to execute - does not touch it by hand unless you know
 100                 self.meths = []
 101
 102                 # list of mappings extension -> function
 103                 self.mappings = {}
 104
 105                 # list of features (see the documentation on traits)
 106                 self.features = list(kw)
 107
 108                 # not always a good idea
 109                 self.tasks = []
 110
 111                 self.default_chmod = O644
 112                 self.default_install_path = None
 113
 114                 # kind of private, beware of what you put in it, also, the contents are consumed
 115                 self.allnodes = []
 116
 117                 self.bld = kwargs.get('bld', Build.bld)
 118                 self.env = self.bld.env.copy()
 119
 120                 self.path = self.bld.path # emulate chdir when reading scripts
 121                 self.name = '' # give a name to the target (static+shlib with the same targetname ambiguity)
 122
 123                 # provide a unique id
 124                 self.idx = self.bld.idx[self.path.id] = self.bld.idx.get(self.path.id, 0) + 1
 125
 126                 for key, val in kwargs.iteritems():
 127                         setattr(self, key, val)
 128
 129                 self.bld.task_manager.add_task_gen(self)
 130                 self.bld.all_task_gen.append(self)
 131
 132         def __str__(self):
 133                 return ("<task_gen '%s' of type %s defined in %s>"
 134                         % (self.name or self.target, self.__class__.__name__, str(self.path)))
 135
 136         def __setattr__(self, name, attr):
 137                 real = typos.get(name, name)
 138                 if real != name:
 139                         warn('typo %s -> %s' % (name, real))
 140                         if Logs.verbose > 0:
 141                                 traceback.print_stack()
 142                 object.__setattr__(self, real, attr)
 143
 144         def to_list(self, value):
 145                 "helper: returns a list"
 146                 if isinstance(value, str): return value.split()
 147                 else: return value
 148
 149         def apply(self):
 150                 "order the methods to execute using self.prec or task_gen.prec"
 151                 keys = set(self.meths)
 152
 153                 # add the methods listed in the features
 154                 self.features = Utils.to_list(self.features)
 155                 for x in self.features + ['*']:
 156                         st = task_gen.traits[x]
 157                         if not st:
 158                                 warn('feature %r does not exist - bind at least one method to it' % x)
 159                         keys.update(st)
 160
 161                 # copy the precedence table
 162                 prec = {}
 163                 prec_tbl = self.prec or task_gen.prec
 164                 for x in prec_tbl:
 165                         if x in keys:
 166                                 prec[x] = prec_tbl[x]
 167
 168                 # elements disconnected
 169                 tmp = []
 170                 for a in keys:
 171                         for x in prec.values():
 172                                 if a in x: break
 173                         else:
 174                                 tmp.append(a)
 175
 176                 # topological sort
 177                 out = []
 178                 while tmp:
 179                         e = tmp.pop()
 180                         if e in keys: out.append(e)
 181                         try:
 182                                 nlst = prec[e]
 183                         except KeyError:
 184                                 pass
 185                         else:
 186                                 del prec[e]
 187                                 for x in nlst:
 188                                         for y in prec:
 189                                                 if x in prec[y]:
 190                                                         break
 191                                         else:
 192                                                 tmp.append(x)
 193
 194                 if prec: raise Utils.WafError("graph has a cycle %s" % str(prec))
 195                 out.reverse()
 196                 self.meths = out
 197
 198                 # then we run the methods in order
 199                 debug('task_gen: posting %s %d', self, id(self))
 200                 for x in out:
 201                         try:
 202                                 v = getattr(self, x)
 203                         except AttributeError:
 204                                 raise Utils.WafError("tried to retrieve %s which is not a valid method" % x)
 205                         debug('task_gen: -> %s (%d)', x, id(self))
 206                         v()
 207
 208         def post(self):
 209                 "runs the code to create the tasks, do not subclass"
 210                 if not self.name:
 211                         if isinstance(self.target, list):
 212                                 self.name = ' '.join(self.target)
 213                         else:
 214                                 self.name = self.target
 215
 216                 if getattr(self, 'posted', None):
 217                         #error("OBJECT ALREADY POSTED" + str( self))
 218                         return
 219
 220                 self.apply()
 221                 self.posted = True
 222                 debug('task_gen: posted %s', self.name)
 223
 224         def get_hook(self, ext):
 225                 try: return self.mappings[ext]
 226                 except KeyError:
 227                         try: return task_gen.mappings[ext]
 228                         except KeyError: return None
 229
 230         # TODO waf 1.6: always set the environment
 231         # TODO waf 1.6: create_task(self, name, inputs, outputs)
 232         def create_task(self, name, src=None, tgt=None, env=None):
 233                 env = env or self.env
 234                 task = Task.TaskBase.classes[name](env.copy(), generator=self)
 235                 if src:
 236                         task.set_inputs(src)
 237                 if tgt:
 238                         task.set_outputs(tgt)
 239                 self.tasks.append(task)
 240                 return task
 241
 242         def name_to_obj(self, name):
 243                 return self.bld.name_to_obj(name, self.env)
 244
 245         def find_sources_in_dirs(self, dirnames, excludes=[], exts=[]):
 246                 """
 247                 The attributes "excludes" and "exts" must be lists to avoid the confusion
 248                 find_sources_in_dirs('a', 'b', 'c') <-> find_sources_in_dirs('a b c')
 249
 250                 do not use absolute paths
 251                 do not use paths outside of the source tree
 252                 the files or folder beginning by . are not returned
 253
 254                 # TODO: remove in Waf 1.6
 255                 """
 256
 257                 err_msg = "'%s' attribute must be a list"
 258                 if not isinstance(excludes, list):
 259                         raise Utils.WscriptError(err_msg % 'excludes')
 260                 if not isinstance(exts, list):
 261                         raise Utils.WscriptError(err_msg % 'exts')
 262
 263                 lst = []
 264
 265                 #make sure dirnames is a list helps with dirnames with spaces
 266                 dirnames = self.to_list(dirnames)
 267
 268                 ext_lst = exts or list(self.mappings.keys()) + list(task_gen.mappings.keys())
 269
 270                 for name in dirnames:
 271                         anode = self.path.find_dir(name)
 272
 273                         if not anode or not anode.is_child_of(self.bld.srcnode):
 274                                 raise Utils.WscriptError("Unable to use '%s' - either because it's not a relative path" \
 275                                          ", or it's not child of '%s'." % (name, self.bld.srcnode))
 276
 277                         self.bld.rescan(anode)
 278                         for name in self.bld.cache_dir_contents[anode.id]:
 279
 280                                 # ignore hidden files
 281                                 if name.startswith('.'):
 282                                         continue
 283
 284                                 (base, ext) = os.path.splitext(name)
 285                                 if ext in ext_lst and not name in lst and not name in excludes:
 286                                         lst.append((anode.relpath_gen(self.path) or '.') + os.path.sep + name)
 287
 288                 lst.sort()
 289                 self.source = self.to_list(self.source)
 290                 if not self.source: self.source = lst
 291                 else: self.source += lst
 292
 293         def clone(self, env):
 294                 """when creating a clone in a task generator method,
 295                 make sure to set posted=False on the clone
 296                 else the other task generator will not create its tasks"""
 297                 newobj = task_gen(bld=self.bld)
 298                 for x in self.__dict__:
 299                         if x in ['env', 'bld']:
 300                                 continue
 301                         elif x in ["path", "features"]:
 302                                 setattr(newobj, x, getattr(self, x))
 303                         else:
 304                                 setattr(newobj, x, copy.copy(getattr(self, x)))
 305
 306                 newobj.__class__ = self.__class__
 307                 if isinstance(env, str):
 308                         newobj.env = self.bld.all_envs[env].copy()
 309                 else:
 310                         newobj.env = env.copy()
 311
 312                 return newobj
 313
 314         def get_inst_path(self):
 315                 return getattr(self, '_install_path', getattr(self, 'default_install_path', ''))
 316
 317         def set_inst_path(self, val):
 318                 self._install_path = val
 319
 320         install_path = property(get_inst_path, set_inst_path)
 321
 322
 323         def get_chmod(self):
 324                 return getattr(self, '_chmod', getattr(self, 'default_chmod', O644))
 325
 326         def set_chmod(self, val):
 327                 self._chmod = val
 328
 329         chmod = property(get_chmod, set_chmod)
 330
 331 def declare_extension(var, func):
 332         try:
 333                 for x in Utils.to_list(var):
 334                         task_gen.mappings[x] = func
 335         except:
 336                 raise Utils.WscriptError('declare_extension takes either a list or a string %r' % var)
 337         task_gen.mapped[func.__name__] = func
 338
 339 def declare_order(*k):
 340         assert(len(k) > 1)
 341         n = len(k) - 1
 342         for i in xrange(n):
 343                 f1 = k[i]
 344                 f2 = k[i+1]
 345                 if not f1 in task_gen.prec[f2]:
 346                         task_gen.prec[f2].append(f1)
 347
 348 def declare_chain(name='', action='', ext_in='', ext_out='', reentrant=True, color='BLUE',
 349         install=0, before=[], after=[], decider=None, rule=None, scan=None):
 350         """
 351         see Tools/flex.py for an example
 352         while i do not like such wrappers, some people really do
 353         """
 354
 355         action = action or rule
 356         if isinstance(action, str):
 357                 act = Task.simple_task_type(name, action, color=color)
 358         else:
 359                 act = Task.task_type_from_func(name, action, color=color)
 360         act.ext_in = tuple(Utils.to_list(ext_in))
 361         act.ext_out = tuple(Utils.to_list(ext_out))
 362         act.before = Utils.to_list(before)
 363         act.after = Utils.to_list(after)
 364         act.scan = scan
 365
 366         def x_file(self, node):
 367                 if decider:
 368                         ext = decider(self, node)
 369                 else:
 370                         ext = ext_out
 371
 372                 if isinstance(ext, str):
 373                         out_source = node.change_ext(ext)
 374                         if reentrant:
 375                                 self.allnodes.append(out_source)
 376                 elif isinstance(ext, list):
 377                         out_source = [node.change_ext(x) for x in ext]
 378                         if reentrant:
 379                                 for i in xrange((reentrant is True) and len(out_source) or reentrant):
 380                                         self.allnodes.append(out_source[i])
 381                 else:
 382                         # XXX: useless: it will fail on Utils.to_list above...
 383                         raise Utils.WafError("do not know how to process %s" % str(ext))
 384
 385                 tsk = self.create_task(name, node, out_source)
 386
 387                 if node.__class__.bld.is_install:
 388                         tsk.install = install
 389
 390         declare_extension(act.ext_in, x_file)
 391         return x_file
 392
 393 def bind_feature(name, methods):
 394         lst = Utils.to_list(methods)
 395         task_gen.traits[name].update(lst)
 396
 397 """
 398 All the following decorators are registration decorators, i.e add an attribute to current class
 399  (task_gen and its derivatives), with same name as func, which points to func itself.
 400 For example:
 401    @taskgen
 402    def sayHi(self):
 403         print("hi")
 404 Now taskgen.sayHi() may be called
 405
 406 If python were really smart, it could infer itself the order of methods by looking at the
 407 attributes. A prerequisite for execution is to have the attribute set before.
 408 Intelligent compilers binding aspect-oriented programming and parallelization, what a nice topic for studies.
 409 """
 410 def taskgen(func):
 411         """
 412         register a method as a task generator method
 413         """
 414         setattr(task_gen, func.__name__, func)
 415         return func
 416
 417 def feature(*k):
 418         """
 419         declare a task generator method that will be executed when the
 420         object attribute 'feature' contains the corresponding key(s)
 421         """
 422         def deco(func):
 423                 setattr(task_gen, func.__name__, func)
 424                 for name in k:
 425                         task_gen.traits[name].update([func.__name__])
 426                 return func
 427         return deco
 428
 429 def before(*k):
 430         """
 431         declare a task generator method which will be executed
 432         before the functions of given name(s)
 433         """
 434         def deco(func):
 435                 setattr(task_gen, func.__name__, func)
 436                 for fun_name in k:
 437                         if not func.__name__ in task_gen.prec[fun_name]:
 438                                 task_gen.prec[fun_name].append(func.__name__)
 439                 return func
 440         return deco
 441
 442 def after(*k):
 443         """
 444         declare a task generator method which will be executed
 445         after the functions of given name(s)
 446         """
 447         def deco(func):
 448                 setattr(task_gen, func.__name__, func)
 449                 for fun_name in k:
 450                         if not fun_name in task_gen.prec[func.__name__]:
 451                                 task_gen.prec[func.__name__].append(fun_name)
 452                 return func
 453         return deco
 454
 455 def extension(var):
 456         """
 457         declare a task generator method which will be invoked during
 458         the processing of source files for the extension given
 459         """
 460         def deco(func):
 461                 setattr(task_gen, func.__name__, func)
 462                 try:
 463                         for x in Utils.to_list(var):
 464                                 task_gen.mappings[x] = func
 465                 except:
 466                         raise Utils.WafError('extension takes either a list or a string %r' % var)
 467                 task_gen.mapped[func.__name__] = func
 468                 return func
 469         return deco
 470
 471 # TODO make certain the decorators may be used here
 472
 473 def apply_core(self):
 474         """Process the attribute source
 475         transform the names into file nodes
 476         try to process the files by name first, later by extension"""
 477         # get the list of folders to use by the scanners
 478         # all our objects share the same include paths anyway
 479         find_resource = self.path.find_resource
 480
 481         for filename in self.to_list(self.source):
 482                 # if self.mappings or task_gen.mappings contains a file of the same name
 483                 x = self.get_hook(filename)
 484                 if x:
 485                         x(self, filename)
 486                 else:
 487                         node = find_resource(filename)
 488                         if not node: raise Utils.WafError("source not found: '%s' in '%s'" % (filename, str(self.path)))
 489                         self.allnodes.append(node)
 490
 491         for node in self.allnodes:
 492                 # self.mappings or task_gen.mappings map the file extension to a function
 493                 x = self.get_hook(node.suffix())
 494
 495                 if not x:
 496                         raise Utils.WafError("Cannot guess how to process %s (got mappings %r in %r) -> try conf.check_tool(..)?" % \
 497                                 (str(node), self.__class__.mappings.keys(), self.__class__))
 498                 x(self, node)
 499 feature('*')(apply_core)
 500
 501 def exec_rule(self):
 502         """Process the attribute rule, when provided the method apply_core will be disabled
 503         """
 504         if not getattr(self, 'rule', None):
 505                 return
 506
 507         # someone may have removed it already
 508         try:
 509                 self.meths.remove('apply_core')
 510         except ValueError:
 511                 pass
 512
 513         # get the function and the variables
 514         func = self.rule
 515
 516         vars2 = []
 517         if isinstance(func, str):
 518                 # use the shell by default for user-defined commands
 519                 (func, vars2) = Task.compile_fun('', self.rule, shell=getattr(self, 'shell', True))
 520                 func.code = self.rule
 521
 522         # create the task class
 523         name = getattr(self, 'name', None) or self.target or self.rule
 524         if not isinstance(name, str):
 525                 name = str(self.idx)
 526         cls = Task.task_type_from_func(name, func, getattr(self, 'vars', vars2))
 527         cls.color = getattr(self, 'color', 'BLUE')
 528
 529         # now create one instance
 530         tsk = self.create_task(name)
 531
 532         dep_vars = getattr(self, 'dep_vars', ['ruledeps'])
 533         if dep_vars:
 534                 tsk.dep_vars = dep_vars
 535         if isinstance(self.rule, str):
 536                 tsk.env.ruledeps = self.rule
 537         else:
 538                 # only works if the function is in a global module such as a waf tool
 539                 tsk.env.ruledeps = Utils.h_fun(self.rule)
 540
 541         # we assume that the user knows that without inputs or outputs
 542         #if not getattr(self, 'target', None) and not getattr(self, 'source', None):
 543         #       cls.quiet = True
 544
 545         if getattr(self, 'target', None):
 546                 cls.quiet = True
 547                 tsk.outputs = [self.path.find_or_declare(x) for x in self.to_list(self.target)]
 548
 549         if getattr(self, 'source', None):
 550                 cls.quiet = True
 551                 tsk.inputs = []
 552                 for x in self.to_list(self.source):
 553                         y = self.path.find_resource(x)
 554                         if not y:
 555                                 raise Utils.WafError('input file %r could not be found (%r)' % (x, self.path.abspath()))
 556                         tsk.inputs.append(y)
 557
 558         if self.allnodes:
 559                 tsk.inputs.extend(self.allnodes)
 560
 561         if getattr(self, 'scan', None):
 562                 cls.scan = self.scan
 563
 564         if getattr(self, 'install_path', None):
 565                 tsk.install_path = self.install_path
 566
 567         if getattr(self, 'cwd', None):
 568                 tsk.cwd = self.cwd
 569
 570         if getattr(self, 'on_results', None):
 571                 Task.update_outputs(cls)
 572
 573         if getattr(self, 'always', None):
 574                 Task.always_run(cls)
 575
 576         for x in ['after', 'before', 'ext_in', 'ext_out']:
 577                 setattr(cls, x, getattr(self, x, []))
 578 feature('*')(exec_rule)
 579 before('apply_core')(exec_rule)
 580
 581 def sequence_order(self):
 582         """
 583         add a strict sequential constraint between the tasks generated by task generators
 584         it uses the fact that task generators are posted in order
 585         it will not post objects which belong to other folders
 586         there is also an awesome trick for executing the method in last position
 587
 588         to use:
 589         bld(features='javac seq')
 590         bld(features='jar seq')
 591
 592         to start a new sequence, set the attribute seq_start, for example:
 593         obj.seq_start = True
 594         """
 595         if self.meths and self.meths[-1] != 'sequence_order':
 596                 self.meths.append('sequence_order')
 597                 return
 598
 599         if getattr(self, 'seq_start', None):
 600                 return
 601
 602         # all the tasks previously declared must be run before these
 603         if getattr(self.bld, 'prev', None):
 604                 self.bld.prev.post()
 605                 for x in self.bld.prev.tasks:
 606                         for y in self.tasks:
 607                                 y.set_run_after(x)
 608
 609         self.bld.prev = self
 610
 611 feature('seq')(sequence_order)
 612