third_party/waf/wafadmin/TaskGen.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2005-2008 (ita)
   4
   5 """
   6 The class task_gen encapsulates the creation of task objects (low-level code)
   7 The instances can have various parameters, but the creation of task nodes (Task.py)
   8 is delayed. To achieve this, various methods are called from the method "apply"
   9
  10 The class task_gen contains lots of methods, and a configuration table:
  11 * the methods to call (self.meths) can be specified dynamically (removing, adding, ..)
  12 * the order of the methods (self.prec or by default task_gen.prec) is configurable
  13 * new methods can be inserted dynamically without pasting old code
  14
  15 Additionally, task_gen provides the method apply_core
  16 * file extensions are mapped to methods: def meth(self, name_or_node)
  17 * if a mapping is not found in self.mappings, it is searched in task_gen.mappings
  18 * when called, the functions may modify self.allnodes to re-add source to process
  19 * the mappings can map an extension or a filename (see the code below)
  20
  21 WARNING: subclasses must reimplement the clone method
  22 """
  23
  24 import os, traceback, copy
  25 import Build, Task, Utils, Logs, Options
  26 from Logs import debug, error, warn
  27 from Constants import *
  28
  29 typos = {
  30 'sources':'source',
  31 'targets':'target',
  32 'include':'includes',
  33 'define':'defines',
  34 'importpath':'importpaths',
  35 'install_var':'install_path',
  36 'install_subdir':'install_path',
  37 'inst_var':'install_path',
  38 'inst_dir':'install_path',
  39 'feature':'features',
  40 }
  41
  42 class register_obj(type):
  43         """no decorators for classes, so we use a metaclass
  44         we store into task_gen.classes the classes that inherit task_gen
  45         and whose names end in '_taskgen'
  46         """
  47         def __init__(cls, name, bases, dict):
  48                 super(register_obj, cls).__init__(name, bases, dict)
  49                 name = cls.__name__
  50                 suffix = '_taskgen'
  51                 if name.endswith(suffix):
  52                         task_gen.classes[name.replace(suffix, '')] = cls
  53
  54 class task_gen(object):
  55         """
  56         Most methods are of the form 'def meth(self):' without any parameters
  57         there are many of them, and they do many different things:
  58         * task creation
  59         * task results installation
  60         * environment modification
  61         * attribute addition/removal
  62
  63         The inheritance approach is complicated
  64         * mixing several languages at once
  65         * subclassing is needed even for small changes
  66         * inserting new methods is complicated
  67
  68         This new class uses a configuration table:
  69         * adding new methods easily
  70         * obtaining the order in which to call the methods
  71         * postponing the method calls (post() -> apply)
  72
  73         Additionally, a 'traits' static attribute is provided:
  74         * this list contains methods
  75         * the methods can remove or add methods from self.meths
  76         Example1: the attribute 'staticlib' is set on an instance
  77         a method set in the list of traits is executed when the
  78         instance is posted, it finds that flag and adds another method for execution
  79         Example2: a method set in the list of traits finds the msvc
  80         compiler (from self.env['MSVC']==1); more methods are added to self.meths
  81         """
  82
  83         __metaclass__ = register_obj
  84         mappings = {}
  85         mapped = {}
  86         prec = Utils.DefaultDict(list)
  87         traits = Utils.DefaultDict(set)
  88         classes = {}
  89
  90         def __init__(self, *kw, **kwargs):
  91                 self.prec = Utils.DefaultDict(list)
  92                 "map precedence of function names to call"
  93                 # so we will have to play with directed acyclic graphs
  94                 # detect cycles, etc
  95
  96                 self.source = ''
  97                 self.target = ''
  98
  99                 # list of methods to execute - does not touch it by hand unless you know
 100                 self.meths = []
 101
 102                 # list of mappings extension -> function
 103                 self.mappings = {}
 104
 105                 # list of features (see the documentation on traits)
 106                 self.features = list(kw)
 107
 108                 # not always a good idea
 109                 self.tasks = []
 110
 111                 self.default_chmod = O644
 112                 self.default_install_path = None
 113
 114                 # kind of private, beware of what you put in it, also, the contents are consumed
 115                 self.allnodes = []
 116
 117                 self.bld = kwargs.get('bld', Build.bld)
 118                 self.env = self.bld.env.copy()
 119
 120                 self.path = self.bld.path # emulate chdir when reading scripts
 121                 self.name = '' # give a name to the target (static+shlib with the same targetname ambiguity)
 122
 123                 # provide a unique id
 124                 self.idx = self.bld.idx[self.path.id] = self.bld.idx.get(self.path.id, 0) + 1
 125
 126                 for key, val in kwargs.iteritems():
 127                         setattr(self, key, val)
 128
 129                 self.bld.task_manager.add_task_gen(self)
 130                 self.bld.all_task_gen.append(self)
 131
 132         def __str__(self):
 133                 return ("<task_gen '%s' of type %s defined in %s>"
 134                         % (self.name or self.target, self.__class__.__name__, str(self.path)))
 135
 136         def __setattr__(self, name, attr):
 137                 real = typos.get(name, name)
 138                 if real != name:
 139                         warn('typo %s -> %s' % (name, real))
 140                         if Logs.verbose > 0:
 141                                 traceback.print_stack()
 142                 object.__setattr__(self, real, attr)
 143
 144         def to_list(self, value):
 145                 "helper: returns a list"
 146                 if isinstance(value, str): return value.split()
 147                 else: return value
 148
 149         def apply(self):
 150                 "order the methods to execute using self.prec or task_gen.prec"
 151                 keys = set(self.meths)
 152
 153                 # add the methods listed in the features
 154                 self.features = Utils.to_list(self.features)
 155                 for x in self.features + ['*']:
 156                         st = task_gen.traits[x]
 157                         if not st:
 158                                 warn('feature %r does not exist - bind at least one method to it' % x)
 159                         keys.update(st)
 160
 161                 # copy the precedence table
 162                 prec = {}
 163                 prec_tbl = self.prec or task_gen.prec
 164                 for x in prec_tbl:
 165                         if x in keys:
 166                                 prec[x] = prec_tbl[x]
 167
 168                 # elements disconnected
 169                 tmp = []
 170                 for a in keys:
 171                         for x in prec.values():
 172                                 if a in x: break
 173                         else:
 174                                 tmp.append(a)
 175
 176                 # topological sort
 177                 out = []
 178                 while tmp:
 179                         e = tmp.pop()
 180                         if e in keys: out.append(e)
 181                         try:
 182                                 nlst = prec[e]
 183                         except KeyError:
 184                                 pass
 185                         else:
 186                                 del prec[e]
 187                                 for x in nlst:
 188                                         for y in prec:
 189                                                 if x in prec[y]:
 190                                                         break
 191                                         else:
 192                                                 tmp.append(x)
 193
 194                 if prec: raise Utils.WafError("graph has a cycle %s" % str(prec))
 195                 out.reverse()
 196                 self.meths = out
 197
 198                 # then we run the methods in order
 199                 debug('task_gen: posting %s %d', self, id(self))
 200                 for x in out:
 201                         try:
 202                                 v = getattr(self, x)
 203                         except AttributeError:
 204                                 raise Utils.WafError("tried to retrieve %s which is not a valid method" % x)
 205                         debug('task_gen: -> %s (%d)', x, id(self))
 206                         v()
 207
 208         def post(self):
 209                 "runs the code to create the tasks, do not subclass"
 210                 if not self.name:
 211                         if isinstance(self.target, list):
 212                                 self.name = ' '.join(self.target)
 213                         else:
 214                                 self.name = self.target
 215
 216                 if getattr(self, 'posted', None):
 217                         #error("OBJECT ALREADY POSTED" + str( self))
 218                         return
 219
 220                 self.apply()
 221                 self.posted = True
 222                 debug('task_gen: posted %s', self.name)
 223
 224         def get_hook(self, ext):
 225                 try: return self.mappings[ext]
 226                 except KeyError:
 227                         try: return task_gen.mappings[ext]
 228                         except KeyError: return None
 229
 230         # TODO waf 1.6: always set the environment
 231         # TODO waf 1.6: create_task(self, name, inputs, outputs)
 232         def create_task(self, name, src=None, tgt=None, env=None):
 233                 env = env or self.env
 234                 task = Task.TaskBase.classes[name](env.copy(), generator=self)
 235                 if src:
 236                         task.set_inputs(src)
 237                 if tgt:
 238                         task.set_outputs(tgt)
 239                 self.tasks.append(task)
 240                 return task
 241
 242         def name_to_obj(self, name):
 243                 return self.bld.name_to_obj(name, self.env)
 244
 245         def get_tgen_by_name(self, name):
 246                 return self.bld.get_tgen_by_name(name)
 247
 248         def find_sources_in_dirs(self, dirnames, excludes=[], exts=[]):
 249                 """
 250                 The attributes "excludes" and "exts" must be lists to avoid the confusion
 251                 find_sources_in_dirs('a', 'b', 'c') <-> find_sources_in_dirs('a b c')
 252
 253                 do not use absolute paths
 254                 do not use paths outside of the source tree
 255                 the files or folder beginning by . are not returned
 256
 257                 # TODO: remove in Waf 1.6
 258                 """
 259
 260                 err_msg = "'%s' attribute must be a list"
 261                 if not isinstance(excludes, list):
 262                         raise Utils.WscriptError(err_msg % 'excludes')
 263                 if not isinstance(exts, list):
 264                         raise Utils.WscriptError(err_msg % 'exts')
 265
 266                 lst = []
 267
 268                 #make sure dirnames is a list helps with dirnames with spaces
 269                 dirnames = self.to_list(dirnames)
 270
 271                 ext_lst = exts or list(self.mappings.keys()) + list(task_gen.mappings.keys())
 272
 273                 for name in dirnames:
 274                         anode = self.path.find_dir(name)
 275
 276                         if not anode or not anode.is_child_of(self.bld.srcnode):
 277                                 raise Utils.WscriptError("Unable to use '%s' - either because it's not a relative path" \
 278                                          ", or it's not child of '%s'." % (name, self.bld.srcnode))
 279
 280                         self.bld.rescan(anode)
 281                         for name in self.bld.cache_dir_contents[anode.id]:
 282
 283                                 # ignore hidden files
 284                                 if name.startswith('.'):
 285                                         continue
 286
 287                                 (base, ext) = os.path.splitext(name)
 288                                 if ext in ext_lst and not name in lst and not name in excludes:
 289                                         lst.append((anode.relpath_gen(self.path) or '.') + os.path.sep + name)
 290
 291                 lst.sort()
 292                 self.source = self.to_list(self.source)
 293                 if not self.source: self.source = lst
 294                 else: self.source += lst
 295
 296         def clone(self, env):
 297                 """when creating a clone in a task generator method,
 298                 make sure to set posted=False on the clone
 299                 else the other task generator will not create its tasks"""
 300                 newobj = task_gen(bld=self.bld)
 301                 for x in self.__dict__:
 302                         if x in ['env', 'bld']:
 303                                 continue
 304                         elif x in ["path", "features"]:
 305                                 setattr(newobj, x, getattr(self, x))
 306                         else:
 307                                 setattr(newobj, x, copy.copy(getattr(self, x)))
 308
 309                 newobj.__class__ = self.__class__
 310                 if isinstance(env, str):
 311                         newobj.env = self.bld.all_envs[env].copy()
 312                 else:
 313                         newobj.env = env.copy()
 314
 315                 return newobj
 316
 317         def get_inst_path(self):
 318                 return getattr(self, '_install_path', getattr(self, 'default_install_path', ''))
 319
 320         def set_inst_path(self, val):
 321                 self._install_path = val
 322
 323         install_path = property(get_inst_path, set_inst_path)
 324
 325
 326         def get_chmod(self):
 327                 return getattr(self, '_chmod', getattr(self, 'default_chmod', O644))
 328
 329         def set_chmod(self, val):
 330                 self._chmod = val
 331
 332         chmod = property(get_chmod, set_chmod)
 333
 334 def declare_extension(var, func):
 335         try:
 336                 for x in Utils.to_list(var):
 337                         task_gen.mappings[x] = func
 338         except:
 339                 raise Utils.WscriptError('declare_extension takes either a list or a string %r' % var)
 340         task_gen.mapped[func.__name__] = func
 341
 342 def declare_order(*k):
 343         assert(len(k) > 1)
 344         n = len(k) - 1
 345         for i in xrange(n):
 346                 f1 = k[i]
 347                 f2 = k[i+1]
 348                 if not f1 in task_gen.prec[f2]:
 349                         task_gen.prec[f2].append(f1)
 350
 351 def declare_chain(name='', action='', ext_in='', ext_out='', reentrant=True, color='BLUE',
 352         install=0, before=[], after=[], decider=None, rule=None, scan=None):
 353         """
 354         see Tools/flex.py for an example
 355         while i do not like such wrappers, some people really do
 356         """
 357
 358         action = action or rule
 359         if isinstance(action, str):
 360                 act = Task.simple_task_type(name, action, color=color)
 361         else:
 362                 act = Task.task_type_from_func(name, action, color=color)
 363         act.ext_in = tuple(Utils.to_list(ext_in))
 364         act.ext_out = tuple(Utils.to_list(ext_out))
 365         act.before = Utils.to_list(before)
 366         act.after = Utils.to_list(after)
 367         act.scan = scan
 368
 369         def x_file(self, node):
 370                 if decider:
 371                         ext = decider(self, node)
 372                 else:
 373                         ext = ext_out
 374
 375                 if isinstance(ext, str):
 376                         out_source = node.change_ext(ext)
 377                         if reentrant:
 378                                 self.allnodes.append(out_source)
 379                 elif isinstance(ext, list):
 380                         out_source = [node.change_ext(x) for x in ext]
 381                         if reentrant:
 382                                 for i in xrange((reentrant is True) and len(out_source) or reentrant):
 383                                         self.allnodes.append(out_source[i])
 384                 else:
 385                         # XXX: useless: it will fail on Utils.to_list above...
 386                         raise Utils.WafError("do not know how to process %s" % str(ext))
 387
 388                 tsk = self.create_task(name, node, out_source)
 389
 390                 if node.__class__.bld.is_install:
 391                         tsk.install = install
 392
 393         declare_extension(act.ext_in, x_file)
 394         return x_file
 395
 396 def bind_feature(name, methods):
 397         lst = Utils.to_list(methods)
 398         task_gen.traits[name].update(lst)
 399
 400 """
 401 All the following decorators are registration decorators, i.e add an attribute to current class
 402  (task_gen and its derivatives), with same name as func, which points to func itself.
 403 For example:
 404    @taskgen
 405    def sayHi(self):
 406         print("hi")
 407 Now taskgen.sayHi() may be called
 408
 409 If python were really smart, it could infer itself the order of methods by looking at the
 410 attributes. A prerequisite for execution is to have the attribute set before.
 411 Intelligent compilers binding aspect-oriented programming and parallelization, what a nice topic for studies.
 412 """
 413 def taskgen(func):
 414         """
 415         register a method as a task generator method
 416         """
 417         setattr(task_gen, func.__name__, func)
 418         return func
 419
 420 def feature(*k):
 421         """
 422         declare a task generator method that will be executed when the
 423         object attribute 'feature' contains the corresponding key(s)
 424         """
 425         def deco(func):
 426                 setattr(task_gen, func.__name__, func)
 427                 for name in k:
 428                         task_gen.traits[name].update([func.__name__])
 429                 return func
 430         return deco
 431
 432 def before(*k):
 433         """
 434         declare a task generator method which will be executed
 435         before the functions of given name(s)
 436         """
 437         def deco(func):
 438                 setattr(task_gen, func.__name__, func)
 439                 for fun_name in k:
 440                         if not func.__name__ in task_gen.prec[fun_name]:
 441                                 task_gen.prec[fun_name].append(func.__name__)
 442                 return func
 443         return deco
 444
 445 def after(*k):
 446         """
 447         declare a task generator method which will be executed
 448         after the functions of given name(s)
 449         """
 450         def deco(func):
 451                 setattr(task_gen, func.__name__, func)
 452                 for fun_name in k:
 453                         if not fun_name in task_gen.prec[func.__name__]:
 454                                 task_gen.prec[func.__name__].append(fun_name)
 455                 return func
 456         return deco
 457
 458 def extension(var):
 459         """
 460         declare a task generator method which will be invoked during
 461         the processing of source files for the extension given
 462         """
 463         def deco(func):
 464                 setattr(task_gen, func.__name__, func)
 465                 try:
 466                         for x in Utils.to_list(var):
 467                                 task_gen.mappings[x] = func
 468                 except:
 469                         raise Utils.WafError('extension takes either a list or a string %r' % var)
 470                 task_gen.mapped[func.__name__] = func
 471                 return func
 472         return deco
 473
 474 # TODO make certain the decorators may be used here
 475
 476 def apply_core(self):
 477         """Process the attribute source
 478         transform the names into file nodes
 479         try to process the files by name first, later by extension"""
 480         # get the list of folders to use by the scanners
 481         # all our objects share the same include paths anyway
 482         find_resource = self.path.find_resource
 483
 484         for filename in self.to_list(self.source):
 485                 # if self.mappings or task_gen.mappings contains a file of the same name
 486                 x = self.get_hook(filename)
 487                 if x:
 488                         x(self, filename)
 489                 else:
 490                         node = find_resource(filename)
 491                         if not node: raise Utils.WafError("source not found: '%s' in '%s'" % (filename, str(self.path)))
 492                         self.allnodes.append(node)
 493
 494         for node in self.allnodes:
 495                 # self.mappings or task_gen.mappings map the file extension to a function
 496                 x = self.get_hook(node.suffix())
 497
 498                 if not x:
 499                         raise Utils.WafError("Cannot guess how to process %s (got mappings %r in %r) -> try conf.check_tool(..)?" % \
 500                                 (str(node), self.__class__.mappings.keys(), self.__class__))
 501                 x(self, node)
 502 feature('*')(apply_core)
 503
 504 def exec_rule(self):
 505         """Process the attribute rule, when provided the method apply_core will be disabled
 506         """
 507         if not getattr(self, 'rule', None):
 508                 return
 509
 510         # someone may have removed it already
 511         try:
 512                 self.meths.remove('apply_core')
 513         except ValueError:
 514                 pass
 515
 516         # get the function and the variables
 517         func = self.rule
 518
 519         vars2 = []
 520         if isinstance(func, str):
 521                 # use the shell by default for user-defined commands
 522                 (func, vars2) = Task.compile_fun('', self.rule, shell=getattr(self, 'shell', True))
 523                 func.code = self.rule
 524
 525         # create the task class
 526         name = getattr(self, 'name', None) or self.target or self.rule
 527         if not isinstance(name, str):
 528                 name = str(self.idx)
 529         cls = Task.task_type_from_func(name, func, getattr(self, 'vars', vars2))
 530         cls.color = getattr(self, 'color', 'BLUE')
 531
 532         # now create one instance
 533         tsk = self.create_task(name)
 534
 535         dep_vars = getattr(self, 'dep_vars', ['ruledeps'])
 536         if dep_vars:
 537                 tsk.dep_vars = dep_vars
 538         if isinstance(self.rule, str):
 539                 tsk.env.ruledeps = self.rule
 540         else:
 541                 # only works if the function is in a global module such as a waf tool
 542                 tsk.env.ruledeps = Utils.h_fun(self.rule)
 543
 544         # we assume that the user knows that without inputs or outputs
 545         #if not getattr(self, 'target', None) and not getattr(self, 'source', None):
 546         #       cls.quiet = True
 547
 548         if getattr(self, 'target', None):
 549                 cls.quiet = True
 550                 tsk.outputs = [self.path.find_or_declare(x) for x in self.to_list(self.target)]
 551
 552         if getattr(self, 'source', None):
 553                 cls.quiet = True
 554                 tsk.inputs = []
 555                 for x in self.to_list(self.source):
 556                         y = self.path.find_resource(x)
 557                         if not y:
 558                                 raise Utils.WafError('input file %r could not be found (%r)' % (x, self.path.abspath()))
 559                         tsk.inputs.append(y)
 560
 561         if self.allnodes:
 562                 tsk.inputs.extend(self.allnodes)
 563
 564         if getattr(self, 'scan', None):
 565                 cls.scan = self.scan
 566
 567         if getattr(self, 'install_path', None):
 568                 tsk.install_path = self.install_path
 569
 570         if getattr(self, 'cwd', None):
 571                 tsk.cwd = self.cwd
 572
 573         if getattr(self, 'on_results', None) or getattr(self, 'update_outputs', None):
 574                 Task.update_outputs(cls)
 575
 576         if getattr(self, 'always', None):
 577                 Task.always_run(cls)
 578
 579         for x in ['after', 'before', 'ext_in', 'ext_out']:
 580                 setattr(cls, x, getattr(self, x, []))
 581 feature('*')(exec_rule)
 582 before('apply_core')(exec_rule)
 583
 584 def sequence_order(self):
 585         """
 586         add a strict sequential constraint between the tasks generated by task generators
 587         it uses the fact that task generators are posted in order
 588         it will not post objects which belong to other folders
 589         there is also an awesome trick for executing the method in last position
 590
 591         to use:
 592         bld(features='javac seq')
 593         bld(features='jar seq')
 594
 595         to start a new sequence, set the attribute seq_start, for example:
 596         obj.seq_start = True
 597         """
 598         if self.meths and self.meths[-1] != 'sequence_order':
 599                 self.meths.append('sequence_order')
 600                 return
 601
 602         if getattr(self, 'seq_start', None):
 603                 return
 604
 605         # all the tasks previously declared must be run before these
 606         if getattr(self.bld, 'prev', None):
 607                 self.bld.prev.post()
 608                 for x in self.bld.prev.tasks:
 609                         for y in self.tasks:
 610                                 y.set_run_after(x)
 611
 612         self.bld.prev = self
 613
 614 feature('seq')(sequence_order)