waflib/Task.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2005-2010 (ita)
   4
   5 """
   6 Tasks represent atomic operations such as processes.
   7 """
   8
   9 import os, re, sys
  10 from waflib import Utils, Logs, Errors
  11
  12 # task states
  13 NOT_RUN = 0
  14 """The task was not executed yet"""
  15
  16 MISSING = 1
  17 """The task has been executed but the files have not been created"""
  18
  19 CRASHED = 2
  20 """The task execution returned a non-zero exit status"""
  21
  22 EXCEPTION = 3
  23 """An exception occured in the task execution"""
  24
  25 SKIPPED = 8
  26 """The task did not have to be executed"""
  27
  28 SUCCESS = 9
  29 """The task was successfully executed"""
  30
  31 ASK_LATER = -1
  32 """The task is not ready to be executed"""
  33
  34 SKIP_ME = -2
  35 """The task does not need to be executed"""
  36
  37 RUN_ME = -3
  38 """The task must be executed"""
  39
  40 # To save some memory during the build, consider discarding tsk.last_cmd in the two templates below
  41
  42 COMPILE_TEMPLATE_SHELL = '''
  43 def f(tsk):
  44         env = tsk.env
  45         gen = tsk.generator
  46         bld = gen.bld
  47         cwdx = getattr(bld, 'cwdx', bld.bldnode) # TODO single cwd value in waf 1.9
  48         wd = getattr(tsk, 'cwd', None)
  49         p = env.get_flat
  50         tsk.last_cmd = cmd = \'\'\' %s \'\'\' % s
  51         return tsk.exec_command(cmd, cwd=wd, env=env.env or None)
  52 '''
  53
  54 COMPILE_TEMPLATE_NOSHELL = '''
  55 def f(tsk):
  56         env = tsk.env
  57         gen = tsk.generator
  58         bld = gen.bld
  59         cwdx = getattr(bld, 'cwdx', bld.bldnode) # TODO single cwd value in waf 1.9
  60         wd = getattr(tsk, 'cwd', None)
  61         def to_list(xx):
  62                 if isinstance(xx, str): return [xx]
  63                 return xx
  64         tsk.last_cmd = lst = []
  65         %s
  66         lst = [x for x in lst if x]
  67         return tsk.exec_command(lst, cwd=wd, env=env.env or None)
  68 '''
  69
  70 classes = {}
  71 "Class tasks created by user scripts or Waf tools (maps names to class objects). Task classes defined in Waf tools are registered here through the metaclass :py:class:`waflib.Task.store_task_type`."
  72
  73 class store_task_type(type):
  74         """
  75         Metaclass: store the task classes into :py:const:`waflib.Task.classes`, or to the dict pointed
  76         by the class attribute 'register'.
  77         The attribute 'run_str' will be processed to compute a method 'run' on the task class
  78         The decorator :py:func:`waflib.Task.cache_outputs` is also applied to the class
  79         """
  80         def __init__(cls, name, bases, dict):
  81                 super(store_task_type, cls).__init__(name, bases, dict)
  82                 name = cls.__name__
  83
  84                 if name.endswith('_task'):
  85                         name = name.replace('_task', '')
  86                 if name != 'evil' and name != 'TaskBase':
  87                         global classes
  88                         if getattr(cls, 'run_str', None):
  89                                 # if a string is provided, convert it to a method
  90                                 (f, dvars) = compile_fun(cls.run_str, cls.shell)
  91                                 cls.hcode = Utils.h_cmd(cls.run_str)
  92                                 cls.orig_run_str = cls.run_str
  93                                 # change the name of run_str or it is impossible to subclass with a function
  94                                 cls.run_str = None
  95                                 cls.run = f
  96                                 cls.vars = list(set(cls.vars + dvars))
  97                                 cls.vars.sort()
  98                         elif getattr(cls, 'run', None) and not 'hcode' in cls.__dict__:
  99                                 # getattr(cls, 'hcode') would look in the upper classes
 100                                 cls.hcode = Utils.h_cmd(cls.run)
 101
 102                         # be creative
 103                         getattr(cls, 'register', classes)[name] = cls
 104
 105 evil = store_task_type('evil', (object,), {})
 106 "Base class provided to avoid writing a metaclass, so the code can run in python 2.6 and 3.x unmodified"
 107
 108 class TaskBase(evil):
 109         """
 110         Base class for all Waf tasks, which should be seen as an interface.
 111         For illustration purposes, instances of this class will execute the attribute
 112         'fun' in :py:meth:`waflib.Task.TaskBase.run`. When in doubt, create
 113         subclasses of :py:class:`waflib.Task.Task` instead.
 114
 115         Subclasses should override these methods:
 116
 117         #. __str__: string to display to the user
 118         #. runnable_status: ask the task if it should be run, skipped, or if we have to ask later
 119         #. run: let threads execute the task
 120         #. post_run: let threads update the data regarding the task (cache)
 121
 122         .. warning:: For backward compatibility reasons, the suffix "_task" is truncated in derived class names. This limitation will be removed in Waf 1.9.
 123         """
 124
 125         color = 'GREEN'
 126         """Color for the console display, see :py:const:`waflib.Logs.colors_lst`"""
 127
 128         ext_in = []
 129         """File extensions that objects of this task class might use"""
 130
 131         ext_out = []
 132         """File extensions that objects of this task class might create"""
 133
 134         before = []
 135         """List of task class names to execute before instances of this class"""
 136
 137         after = []
 138         """List of task class names to execute after instances of this class"""
 139
 140         hcode = ''
 141         """String representing an additional hash for the class representation"""
 142
 143         def __init__(self, *k, **kw):
 144                 """
 145                 The base task class requires a task generator, which will be itself if missing
 146                 """
 147                 self.hasrun = NOT_RUN
 148                 try:
 149                         self.generator = kw['generator']
 150                 except KeyError:
 151                         self.generator = self
 152
 153         def __repr__(self):
 154                 "for debugging purposes"
 155                 return '\n\t{task %r: %s %s}' % (self.__class__.__name__, id(self), str(getattr(self, 'fun', '')))
 156
 157         def __str__(self):
 158                 "string to display to the user"
 159                 if hasattr(self, 'fun'):
 160                         return self.fun.__name__
 161                 return self.__class__.__name__
 162
 163         def __hash__(self):
 164                 "Very fast hashing scheme but not persistent (replace/implement in subclasses and see :py:meth:`waflib.Task.Task.uid`)"
 165                 return id(self)
 166
 167         def keyword(self):
 168                 if hasattr(self, 'fun'):
 169                         return 'Function'
 170                 return 'Processing'
 171
 172         def exec_command(self, cmd, **kw):
 173                 """
 174                 Wrapper for :py:meth:`waflib.Context.Context.exec_command` which sets a current working directory to ``build.variant_dir``
 175
 176                 :return: the return code
 177                 :rtype: int
 178                 """
 179                 bld = self.generator.bld
 180                 try:
 181                         if not kw.get('cwd', None):
 182                                 kw['cwd'] = bld.cwd
 183                 except AttributeError:
 184                         bld.cwd = kw['cwd'] = bld.variant_dir
 185                 return bld.exec_command(cmd, **kw)
 186
 187         def runnable_status(self):
 188                 """
 189                 State of the task
 190
 191                 :return: a task state in :py:const:`waflib.Task.RUN_ME`, :py:const:`waflib.Task.SKIP_ME` or :py:const:`waflib.Task.ASK_LATER`.
 192                 :rtype: int
 193                 """
 194                 return RUN_ME
 195
 196         def process(self):
 197                 """
 198                 Assume that the task has had a new attribute ``master`` which is an instance of :py:class:`waflib.Runner.Parallel`.
 199                 Execute the task and then put it back in the queue :py:attr:`waflib.Runner.Parallel.out` (may be replaced by subclassing).
 200                 """
 201                 m = self.master
 202                 if m.stop:
 203                         m.out.put(self)
 204                         return
 205
 206                 # remove the task signature immediately before it is executed
 207                 # in case of failure the task will be executed again
 208                 try:
 209                         # TODO waf 1.9 - this breaks encapsulation
 210                         del self.generator.bld.task_sigs[self.uid()]
 211                 except KeyError:
 212                         pass
 213
 214                 try:
 215                         self.generator.bld.returned_tasks.append(self)
 216                         self.log_display(self.generator.bld)
 217                         ret = self.run()
 218                 except Exception:
 219                         self.err_msg = Utils.ex_stack()
 220                         self.hasrun = EXCEPTION
 221
 222                         # TODO cleanup
 223                         m.error_handler(self)
 224                         m.out.put(self)
 225                         return
 226
 227                 if ret:
 228                         self.err_code = ret
 229                         self.hasrun = CRASHED
 230                 else:
 231                         try:
 232                                 self.post_run()
 233                         except Errors.WafError:
 234                                 pass
 235                         except Exception:
 236                                 self.err_msg = Utils.ex_stack()
 237                                 self.hasrun = EXCEPTION
 238                         else:
 239                                 self.hasrun = SUCCESS
 240                 if self.hasrun != SUCCESS:
 241                         m.error_handler(self)
 242
 243                 m.out.put(self)
 244
 245         def run(self):
 246                 """
 247                 Called by threads to execute the tasks. The default is empty and meant to be overridden in subclasses.
 248                 It is a bad idea to create nodes in this method (so, no node.ant_glob)
 249
 250                 :rtype: int
 251                 """
 252                 if hasattr(self, 'fun'):
 253                         return self.fun(self)
 254                 return 0
 255
 256         def post_run(self):
 257                 "Update the cache files (executed by threads). Override in subclasses."
 258                 pass
 259
 260         def log_display(self, bld):
 261                 "Write the execution status on the context logger"
 262                 if self.generator.bld.progress_bar == 3:
 263                         return
 264
 265                 s = self.display()
 266                 if s:
 267                         if bld.logger:
 268                                 logger = bld.logger
 269                         else:
 270                                 logger = Logs
 271
 272                         if self.generator.bld.progress_bar == 1:
 273                                 c1 = Logs.colors.cursor_off
 274                                 c2 = Logs.colors.cursor_on
 275                                 logger.info(s, extra={'stream': sys.stderr, 'terminator':'', 'c1': c1, 'c2' : c2})
 276                         else:
 277                                 logger.info(s, extra={'terminator':'', 'c1': '', 'c2' : ''})
 278
 279         def display(self):
 280                 """
 281                 Return an execution status for the console, the progress bar, or the IDE output.
 282
 283                 :rtype: string
 284                 """
 285                 col1 = Logs.colors(self.color)
 286                 col2 = Logs.colors.NORMAL
 287                 master = self.master
 288
 289                 def cur():
 290                         # the current task position, computed as late as possible
 291                         tmp = -1
 292                         if hasattr(master, 'ready'):
 293                                 tmp -= master.ready.qsize()
 294                         return master.processed + tmp
 295
 296                 if self.generator.bld.progress_bar == 1:
 297                         return self.generator.bld.progress_line(cur(), master.total, col1, col2)
 298
 299                 if self.generator.bld.progress_bar == 2:
 300                         ela = str(self.generator.bld.timer)
 301                         try:
 302                                 ins  = ','.join([n.name for n in self.inputs])
 303                         except AttributeError:
 304                                 ins = ''
 305                         try:
 306                                 outs = ','.join([n.name for n in self.outputs])
 307                         except AttributeError:
 308                                 outs = ''
 309                         return '|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n' % (master.total, cur(), ins, outs, ela)
 310
 311                 s = str(self)
 312                 if not s:
 313                         return None
 314
 315                 total = master.total
 316                 n = len(str(total))
 317                 fs = '[%%%dd/%%%dd] %%s%%s%%s%%s\n' % (n, n)
 318                 kw = self.keyword()
 319                 if kw:
 320                         kw += ' '
 321                 return fs % (cur(), total, kw, col1, s, col2)
 322
 323         def attr(self, att, default=None):
 324                 """
 325                 Retrieve an attribute from the instance or from the class.
 326
 327                 :param att: variable name
 328                 :type att: string
 329                 :param default: default value
 330                 """
 331                 ret = getattr(self, att, self)
 332                 if ret is self: return getattr(self.__class__, att, default)
 333                 return ret
 334
 335         def hash_constraints(self):
 336                 """
 337                 Identify a task type for all the constraints relevant for the scheduler: precedence, file production
 338
 339                 :return: a hash value
 340                 :rtype: string
 341                 """
 342                 cls = self.__class__
 343                 tup = (str(cls.before), str(cls.after), str(cls.ext_in), str(cls.ext_out), cls.__name__, cls.hcode)
 344                 h = hash(tup)
 345                 return h
 346
 347         def format_error(self):
 348                 """
 349                 Error message to display to the user when a build fails
 350
 351                 :rtype: string
 352                 """
 353                 msg = getattr(self, 'last_cmd', '')
 354                 name = getattr(self.generator, 'name', '')
 355                 if getattr(self, "err_msg", None):
 356                         return self.err_msg
 357                 elif not self.hasrun:
 358                         return 'task in %r was not executed for some reason: %r' % (name, self)
 359                 elif self.hasrun == CRASHED:
 360                         try:
 361                                 return ' -> task in %r failed (exit status %r): %r\n%r' % (name, self.err_code, self, msg)
 362                         except AttributeError:
 363                                 return ' -> task in %r failed: %r\n%r' % (name, self, msg)
 364                 elif self.hasrun == MISSING:
 365                         return ' -> missing files in %r: %r\n%r' % (name, self, msg)
 366                 else:
 367                         return 'invalid status for task in %r: %r' % (name, self.hasrun)
 368
 369         def colon(self, var1, var2):
 370                 """
 371                 Support code for scriptlet expressions such as ${FOO_ST:FOO}
 372                 If the first variable (FOO_ST) is empty, then an empty list is returned
 373
 374                 The results will be slightly different if FOO_ST is a list, for example::
 375
 376                         env.FOO_ST = ['-a', '-b']
 377                         env.FOO_ST = '-I%s'
 378                         # ${FOO_ST:FOO} returns
 379                         ['-Ip1', '-Ip2']
 380
 381                         env.FOO    = ['p1', 'p2']
 382                         # ${FOO_ST:FOO} returns
 383                         ['-a', '-b', 'p1', '-a', '-b', 'p2']
 384                 """
 385                 tmp = self.env[var1]
 386                 if not tmp:
 387                         return []
 388
 389                 if isinstance(var2, str):
 390                         it = self.env[var2]
 391                 else:
 392                         it = var2
 393                 if isinstance(tmp, str):
 394                         return [tmp % x for x in it]
 395                 else:
 396                         lst = []
 397                         for y in it:
 398                                 lst.extend(tmp)
 399                                 lst.append(y)
 400                         return lst
 401
 402 class Task(TaskBase):
 403         """
 404         This class deals with the filesystem (:py:class:`waflib.Node.Node`). The method :py:class:`waflib.Task.Task.runnable_status`
 405         uses a hash value (from :py:class:`waflib.Task.Task.signature`) which is persistent from build to build. When the value changes,
 406         the task has to be executed. The method :py:class:`waflib.Task.Task.post_run` will assign the task signature to the output
 407         nodes (if present).
 408
 409         .. warning:: For backward compatibility reasons, the suffix "_task" is truncated in derived class names. This limitation will be removed in Waf 1.9.
 410         """
 411         vars = []
 412         """Variables to depend on (class attribute used for :py:meth:`waflib.Task.Task.sig_vars`)"""
 413
 414         shell = False
 415         """Execute the command with the shell (class attribute)"""
 416
 417         def __init__(self, *k, **kw):
 418                 TaskBase.__init__(self, *k, **kw)
 419
 420                 self.env = kw['env']
 421                 """ConfigSet object (make sure to provide one)"""
 422
 423                 self.inputs  = []
 424                 """List of input nodes, which represent the files used by the task instance"""
 425
 426                 self.outputs = []
 427                 """List of output nodes, which represent the files created by the task instance"""
 428
 429                 self.dep_nodes = []
 430                 """List of additional nodes to depend on"""
 431
 432                 self.run_after = set([])
 433                 """Set of tasks that must be executed before this one"""
 434
 435                 # Additionally, you may define the following
 436                 #self.dep_vars  = 'PREFIX DATADIR'
 437
 438         def __str__(self):
 439                 "string to display to the user"
 440                 name = self.__class__.__name__
 441                 if self.outputs:
 442                         if (name.endswith('lib') or name.endswith('program')) or not self.inputs:
 443                                 node = self.outputs[0]
 444                                 return node.path_from(node.ctx.launch_node())
 445                 if not (self.inputs or self.outputs):
 446                         return self.__class__.__name__
 447                 if len(self.inputs) == 1:
 448                         node = self.inputs[0]
 449                         return node.path_from(node.ctx.launch_node())
 450
 451                 src_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.inputs])
 452                 tgt_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.outputs])
 453                 if self.outputs: sep = ' -> '
 454                 else: sep = ''
 455                 return '%s: %s%s%s' % (self.__class__.__name__.replace('_task', ''), src_str, sep, tgt_str)
 456
 457         def keyword(self):
 458                 name = self.__class__.__name__
 459                 if name.endswith('lib') or name.endswith('program'):
 460                         return 'Linking'
 461                 if len(self.inputs) == 1 and len(self.outputs) == 1:
 462                         return 'Compiling'
 463                 if not self.inputs:
 464                         if self.outputs:
 465                                 return 'Creating'
 466                         else:
 467                                 return 'Running'
 468                 return 'Processing'
 469
 470         def __repr__(self):
 471                 "for debugging purposes"
 472                 try:
 473                         ins = ",".join([x.name for x in self.inputs])
 474                         outs = ",".join([x.name for x in self.outputs])
 475                 except AttributeError:
 476                         ins = ",".join([str(x) for x in self.inputs])
 477                         outs = ",".join([str(x) for x in self.outputs])
 478                 return "".join(['\n\t{task %r: ' % id(self), self.__class__.__name__, " ", ins, " -> ", outs, '}'])
 479
 480         def uid(self):
 481                 """
 482                 Return an identifier used to determine if tasks are up-to-date. Since the
 483                 identifier will be stored between executions, it must be:
 484
 485                         - unique: no two tasks return the same value (for a given build context)
 486                         - the same for a given task instance
 487
 488                 By default, the node paths, the class name, and the function are used
 489                 as inputs to compute a hash.
 490
 491                 The pointer to the object (python built-in 'id') will change between build executions,
 492                 and must be avoided in such hashes.
 493
 494                 :return: hash value
 495                 :rtype: string
 496                 """
 497                 try:
 498                         return self.uid_
 499                 except AttributeError:
 500                         m = Utils.md5()
 501                         up = m.update
 502                         up(self.__class__.__name__)
 503                         for x in self.inputs + self.outputs:
 504                                 up(x.abspath())
 505                         self.uid_ = m.digest()
 506                         return self.uid_
 507
 508
 509         def set_inputs(self, inp):
 510                 """
 511                 Append the nodes to the *inputs*
 512
 513                 :param inp: input nodes
 514                 :type inp: node or list of nodes
 515                 """
 516                 if isinstance(inp, list): self.inputs += inp
 517                 else: self.inputs.append(inp)
 518
 519         def set_outputs(self, out):
 520                 """
 521                 Append the nodes to the *outputs*
 522
 523                 :param out: output nodes
 524                 :type out: node or list of nodes
 525                 """
 526                 if isinstance(out, list): self.outputs += out
 527                 else: self.outputs.append(out)
 528
 529         def set_run_after(self, task):
 530                 """
 531                 Run this task only after *task*. Affect :py:meth:`waflib.Task.runnable_status`
 532                 You probably want to use tsk.run_after.add(task) directly
 533
 534                 :param task: task
 535                 :type task: :py:class:`waflib.Task.Task`
 536                 """
 537                 assert isinstance(task, TaskBase)
 538                 self.run_after.add(task)
 539
 540         def signature(self):
 541                 """
 542                 Task signatures are stored between build executions, they are use to track the changes
 543                 made to the input nodes (not to the outputs!). The signature hashes data from various sources:
 544
 545                 * explicit dependencies: files listed in the inputs (list of node objects) :py:meth:`waflib.Task.Task.sig_explicit_deps`
 546                 * implicit dependencies: list of nodes returned by scanner methods (when present) :py:meth:`waflib.Task.Task.sig_implicit_deps`
 547                 * hashed data: variables/values read from task.__class__.vars/task.env :py:meth:`waflib.Task.Task.sig_vars`
 548
 549                 If the signature is expected to give a different result, clear the cache kept in ``self.cache_sig``::
 550
 551                         from waflib import Task
 552                         class cls(Task.Task):
 553                                 def signature(self):
 554                                         sig = super(Task.Task, self).signature()
 555                                         delattr(self, 'cache_sig')
 556                                         return super(Task.Task, self).signature()
 557                 """
 558                 try: return self.cache_sig
 559                 except AttributeError: pass
 560
 561                 self.m = Utils.md5()
 562                 self.m.update(self.hcode)
 563
 564                 # explicit deps
 565                 self.sig_explicit_deps()
 566
 567                 # env vars
 568                 self.sig_vars()
 569
 570                 # implicit deps / scanner results
 571                 if self.scan:
 572                         try:
 573                                 self.sig_implicit_deps()
 574                         except Errors.TaskRescan:
 575                                 return self.signature()
 576
 577                 ret = self.cache_sig = self.m.digest()
 578                 return ret
 579
 580         def runnable_status(self):
 581                 """
 582                 Override :py:meth:`waflib.Task.TaskBase.runnable_status` to determine if the task is ready
 583                 to be run (:py:attr:`waflib.Task.Task.run_after`)
 584                 """
 585                 #return 0 # benchmarking
 586
 587                 for t in self.run_after:
 588                         if not t.hasrun:
 589                                 return ASK_LATER
 590
 591                 bld = self.generator.bld
 592
 593                 # first compute the signature
 594                 try:
 595                         new_sig = self.signature()
 596                 except Errors.TaskNotReady:
 597                         return ASK_LATER
 598
 599                 # compare the signature to a signature computed previously
 600                 key = self.uid()
 601                 try:
 602                         prev_sig = bld.task_sigs[key]
 603                 except KeyError:
 604                         Logs.debug("task: task %r must run as it was never run before or the task code changed" % self)
 605                         return RUN_ME
 606
 607                 # compare the signatures of the outputs
 608                 for node in self.outputs:
 609                         try:
 610                                 if node.sig != new_sig:
 611                                         return RUN_ME
 612                         except AttributeError:
 613                                 Logs.debug("task: task %r must run as the output nodes do not exist" % self)
 614                                 return RUN_ME
 615
 616                 if new_sig != prev_sig:
 617                         return RUN_ME
 618                 return SKIP_ME
 619
 620         def post_run(self):
 621                 """
 622                 Called after successful execution to update the cache data :py:class:`waflib.Node.Node` sigs
 623                 and :py:attr:`waflib.Build.BuildContext.task_sigs`.
 624
 625                 The node signature is obtained from the task signature, but the output nodes may also get the signature
 626                 of their contents. See the class decorator :py:func:`waflib.Task.update_outputs` if you need this behaviour.
 627                 """
 628                 bld = self.generator.bld
 629                 sig = self.signature()
 630
 631                 for node in self.outputs:
 632                         # check if the node exists ..
 633                         try:
 634                                 os.stat(node.abspath())
 635                         except OSError:
 636                                 self.hasrun = MISSING
 637                                 self.err_msg = '-> missing file: %r' % node.abspath()
 638                                 raise Errors.WafError(self.err_msg)
 639
 640                         # important, store the signature for the next run
 641                         node.sig = node.cache_sig = sig
 642
 643                 bld.task_sigs[self.uid()] = self.cache_sig
 644
 645         def sig_explicit_deps(self):
 646                 """
 647                 Used by :py:meth:`waflib.Task.Task.signature`, hash :py:attr:`waflib.Task.Task.inputs`
 648                 and :py:attr:`waflib.Task.Task.dep_nodes` signatures.
 649
 650                 :rtype: hash value
 651                 """
 652                 bld = self.generator.bld
 653                 upd = self.m.update
 654
 655                 # the inputs
 656                 for x in self.inputs + self.dep_nodes:
 657                         try:
 658                                 upd(x.get_bld_sig())
 659                         except (AttributeError, TypeError):
 660                                 raise Errors.WafError('Missing node signature for %r (required by %r)' % (x, self))
 661
 662                 # manual dependencies, they can slow down the builds
 663                 if bld.deps_man:
 664                         additional_deps = bld.deps_man
 665                         for x in self.inputs + self.outputs:
 666                                 try:
 667                                         d = additional_deps[id(x)]
 668                                 except KeyError:
 669                                         continue
 670
 671                                 for v in d:
 672                                         if isinstance(v, bld.root.__class__):
 673                                                 try:
 674                                                         v = v.get_bld_sig()
 675                                                 except AttributeError:
 676                                                         raise Errors.WafError('Missing node signature for %r (required by %r)' % (v, self))
 677                                         elif hasattr(v, '__call__'):
 678                                                 v = v() # dependency is a function, call it
 679                                         upd(v)
 680
 681                 return self.m.digest()
 682
 683         def sig_vars(self):
 684                 """
 685                 Used by :py:meth:`waflib.Task.Task.signature`, hash :py:attr:`waflib.Task.Task.env` variables/values
 686
 687                 :rtype: hash value
 688                 """
 689                 bld = self.generator.bld
 690                 env = self.env
 691                 upd = self.m.update
 692
 693                 # dependencies on the environment vars
 694                 act_sig = bld.hash_env_vars(env, self.__class__.vars)
 695                 upd(act_sig)
 696
 697                 # additional variable dependencies, if provided
 698                 dep_vars = getattr(self, 'dep_vars', None)
 699                 if dep_vars:
 700                         upd(bld.hash_env_vars(env, dep_vars))
 701
 702                 return self.m.digest()
 703
 704         scan = None
 705         """
 706         This method, when provided, returns a tuple containing:
 707
 708         * a list of nodes corresponding to real files
 709         * a list of names for files not found in path_lst
 710
 711         For example::
 712
 713                 from waflib.Task import Task
 714                 class mytask(Task):
 715                         def scan(self, node):
 716                                 return ((), ())
 717
 718         The first and second lists are stored in :py:attr:`waflib.Build.BuildContext.node_deps` and
 719         :py:attr:`waflib.Build.BuildContext.raw_deps` respectively.
 720         """
 721
 722         def sig_implicit_deps(self):
 723                 """
 724                 Used by :py:meth:`waflib.Task.Task.signature` hashes node signatures obtained by scanning for dependencies (:py:meth:`waflib.Task.Task.scan`).
 725
 726                 The exception :py:class:`waflib.Errors.TaskRescan` is thrown
 727                 when a file has changed. When this occurs, :py:meth:`waflib.Task.Task.signature` is called
 728                 once again, and this method will be executed once again, this time calling :py:meth:`waflib.Task.Task.scan`
 729                 for searching the dependencies.
 730
 731                 :rtype: hash value
 732                 """
 733
 734                 bld = self.generator.bld
 735
 736                 # get the task signatures from previous runs
 737                 key = self.uid()
 738                 prev = bld.task_sigs.get((key, 'imp'), [])
 739
 740                 # for issue #379
 741                 if prev:
 742                         try:
 743                                 if prev == self.compute_sig_implicit_deps():
 744                                         return prev
 745                         except Errors.TaskNotReady:
 746                                 raise
 747                         except EnvironmentError:
 748                                 # when a file was renamed (IOError usually), remove the stale nodes (headers in folders without source files)
 749                                 # this will break the order calculation for headers created during the build in the source directory (should be uncommon)
 750                                 # the behaviour will differ when top != out
 751                                 for x in bld.node_deps.get(self.uid(), []):
 752                                         if not x.is_bld():
 753                                                 try:
 754                                                         os.stat(x.abspath())
 755                                                 except OSError:
 756                                                         try:
 757                                                                 del x.parent.children[x.name]
 758                                                         except KeyError:
 759                                                                 pass
 760                         del bld.task_sigs[(key, 'imp')]
 761                         raise Errors.TaskRescan('rescan')
 762
 763                 # no previous run or the signature of the dependencies has changed, rescan the dependencies
 764                 (nodes, names) = self.scan()
 765                 if Logs.verbose:
 766                         Logs.debug('deps: scanner for %s returned %s %s' % (str(self), str(nodes), str(names)))
 767
 768                 # store the dependencies in the cache
 769                 bld.node_deps[key] = nodes
 770                 bld.raw_deps[key] = names
 771
 772                 # might happen
 773                 self.are_implicit_nodes_ready()
 774
 775                 # recompute the signature and return it
 776                 try:
 777                         bld.task_sigs[(key, 'imp')] = sig = self.compute_sig_implicit_deps()
 778                 except Exception:
 779                         if Logs.verbose:
 780                                 for k in bld.node_deps.get(self.uid(), []):
 781                                         try:
 782                                                 k.get_bld_sig()
 783                                         except Exception:
 784                                                 Logs.warn('Missing signature for node %r (may cause rebuilds)' % k)
 785                 else:
 786                         return sig
 787
 788         def compute_sig_implicit_deps(self):
 789                 """
 790                 Used by :py:meth:`waflib.Task.Task.sig_implicit_deps` for computing the actual hash of the
 791                 :py:class:`waflib.Node.Node` returned by the scanner.
 792
 793                 :return: hash value
 794                 :rtype: string
 795                 """
 796
 797                 upd = self.m.update
 798
 799                 bld = self.generator.bld
 800
 801                 self.are_implicit_nodes_ready()
 802
 803                 # scanner returns a node that does not have a signature
 804                 # just *ignore* the error and let them figure out from the compiler output
 805                 # waf -k behaviour
 806                 for k in bld.node_deps.get(self.uid(), []):
 807                         upd(k.get_bld_sig())
 808                 return self.m.digest()
 809
 810         def are_implicit_nodes_ready(self):
 811                 """
 812                 For each node returned by the scanner, see if there is a task behind it, and force the build order
 813
 814                 The performance impact on null builds is nearly invisible (1.66s->1.86s), but this is due to
 815                 agressive caching (1.86s->28s)
 816                 """
 817                 bld = self.generator.bld
 818                 try:
 819                         cache = bld.dct_implicit_nodes
 820                 except AttributeError:
 821                         bld.dct_implicit_nodes = cache = {}
 822
 823                 try:
 824                         dct = cache[bld.cur]
 825                 except KeyError:
 826                         dct = cache[bld.cur] = {}
 827                         for tsk in bld.cur_tasks:
 828                                 for x in tsk.outputs:
 829                                         dct[x] = tsk
 830
 831                 modified = False
 832                 for x in bld.node_deps.get(self.uid(), []):
 833                         if x in dct:
 834                                 self.run_after.add(dct[x])
 835                                 modified = True
 836
 837                 if modified:
 838                         for tsk in self.run_after:
 839                                 if not tsk.hasrun:
 840                                         #print "task is not ready..."
 841                                         raise Errors.TaskNotReady('not ready')
 842 if sys.hexversion > 0x3000000:
 843         def uid(self):
 844                 try:
 845                         return self.uid_
 846                 except AttributeError:
 847                         m = Utils.md5()
 848                         up = m.update
 849                         up(self.__class__.__name__.encode('iso8859-1', 'xmlcharrefreplace'))
 850                         for x in self.inputs + self.outputs:
 851                                 up(x.abspath().encode('iso8859-1', 'xmlcharrefreplace'))
 852                         self.uid_ = m.digest()
 853                         return self.uid_
 854         uid.__doc__ = Task.uid.__doc__
 855         Task.uid = uid
 856
 857 def is_before(t1, t2):
 858         """
 859         Return a non-zero value if task t1 is to be executed before task t2::
 860
 861                 t1.ext_out = '.h'
 862                 t2.ext_in = '.h'
 863                 t2.after = ['t1']
 864                 t1.before = ['t2']
 865                 waflib.Task.is_before(t1, t2) # True
 866
 867         :param t1: task
 868         :type t1: :py:class:`waflib.Task.TaskBase`
 869         :param t2: task
 870         :type t2: :py:class:`waflib.Task.TaskBase`
 871         """
 872         to_list = Utils.to_list
 873         for k in to_list(t2.ext_in):
 874                 if k in to_list(t1.ext_out):
 875                         return 1
 876
 877         if t1.__class__.__name__ in to_list(t2.after):
 878                 return 1
 879
 880         if t2.__class__.__name__ in to_list(t1.before):
 881                 return 1
 882
 883         return 0
 884
 885 def set_file_constraints(tasks):
 886         """
 887         Adds tasks to the task 'run_after' attribute based on the task inputs and outputs
 888
 889         :param tasks: tasks
 890         :type tasks: list of :py:class:`waflib.Task.TaskBase`
 891         """
 892         ins = Utils.defaultdict(set)
 893         outs = Utils.defaultdict(set)
 894         for x in tasks:
 895                 for a in getattr(x, 'inputs', []) + getattr(x, 'dep_nodes', []):
 896                         ins[id(a)].add(x)
 897                 for a in getattr(x, 'outputs', []):
 898                         outs[id(a)].add(x)
 899
 900         links = set(ins.keys()).intersection(outs.keys())
 901         for k in links:
 902                 for a in ins[k]:
 903                         a.run_after.update(outs[k])
 904
 905 def set_precedence_constraints(tasks):
 906         """
 907         Add tasks to the task 'run_after' attribute based on the after/before/ext_out/ext_in attributes
 908
 909         :param tasks: tasks
 910         :type tasks: list of :py:class:`waflib.Task.TaskBase`
 911         """
 912         cstr_groups = Utils.defaultdict(list)
 913         for x in tasks:
 914                 h = x.hash_constraints()
 915                 cstr_groups[h].append(x)
 916
 917         keys = list(cstr_groups.keys())
 918         maxi = len(keys)
 919
 920         # this list should be short
 921         for i in range(maxi):
 922                 t1 = cstr_groups[keys[i]][0]
 923                 for j in range(i + 1, maxi):
 924                         t2 = cstr_groups[keys[j]][0]
 925
 926                         # add the constraints based on the comparisons
 927                         if is_before(t1, t2):
 928                                 a = i
 929                                 b = j
 930                         elif is_before(t2, t1):
 931                                 a = j
 932                                 b = i
 933                         else:
 934                                 continue
 935
 936                         aval = set(cstr_groups[keys[a]])
 937                         for x in cstr_groups[keys[b]]:
 938                                 x.run_after.update(aval)
 939
 940 def funex(c):
 941         """
 942         Compile a function by 'exec'
 943
 944         :param c: function to compile
 945         :type c: string
 946         :return: the function 'f' declared in the input string
 947         :rtype: function
 948         """
 949         dc = {}
 950         exec(c, dc)
 951         return dc['f']
 952
 953 reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})", re.M)
 954 def compile_fun_shell(line):
 955         """
 956         Create a compiled function to execute a process with the shell
 957         WARNING: this method may disappear anytime, so use compile_fun instead
 958         """
 959
 960         extr = []
 961         def repl(match):
 962                 g = match.group
 963                 if g('dollar'): return "$"
 964                 elif g('backslash'): return '\\\\'
 965                 elif g('subst'): extr.append((g('var'), g('code'))); return "%s"
 966                 return None
 967
 968         line = reg_act.sub(repl, line) or line
 969
 970         parm = []
 971         dvars = []
 972         app = parm.append
 973         for (var, meth) in extr:
 974                 if var == 'SRC':
 975                         if meth: app('tsk.inputs%s' % meth)
 976                         else: app('" ".join([a.path_from(cwdx) for a in tsk.inputs])')
 977                 elif var == 'TGT':
 978                         if meth: app('tsk.outputs%s' % meth)
 979                         else: app('" ".join([a.path_from(cwdx) for a in tsk.outputs])')
 980                 elif meth:
 981                         if meth.startswith(':'):
 982                                 m = meth[1:]
 983                                 if m == 'SRC':
 984                                         m = '[a.path_from(cwdx) for a in tsk.inputs]'
 985                                 elif m == 'TGT':
 986                                         m = '[a.path_from(cwdx) for a in tsk.outputs]'
 987                                 elif m[:3] not in ('tsk', 'gen', 'bld'):
 988                                         dvars.extend([var, meth[1:]])
 989                                         m = '%r' % m
 990                                 app('" ".join(tsk.colon(%r, %s))' % (var, m))
 991                         else:
 992                                 app('%s%s' % (var, meth))
 993                 else:
 994                         if not var in dvars: dvars.append(var)
 995                         app("p('%s')" % var)
 996         if parm: parm = "%% (%s) " % (',\n\t\t'.join(parm))
 997         else: parm = ''
 998
 999         c = COMPILE_TEMPLATE_SHELL % (line, parm)
1000
1001         Logs.debug('action: %s' % c.strip().splitlines())
1002         return (funex(c), dvars)
1003
1004 def compile_fun_noshell(line):
1005         """
1006         Create a compiled function to execute a process without the shell
1007         WARNING: this method may disappear anytime, so use compile_fun instead
1008         """
1009         extr = []
1010         def repl(match):
1011                 g = match.group
1012                 if g('dollar'): return "$"
1013                 elif g('backslash'): return '\\'
1014                 elif g('subst'): extr.append((g('var'), g('code'))); return "<<|@|>>"
1015                 return None
1016
1017         line2 = reg_act.sub(repl, line)
1018         params = line2.split('<<|@|>>')
1019         assert(extr)
1020
1021         buf = []
1022         dvars = []
1023         app = buf.append
1024         for x in range(len(extr)):
1025                 params[x] = params[x].strip()
1026                 if params[x]:
1027                         app("lst.extend(%r)" % params[x].split())
1028                 (var, meth) = extr[x]
1029                 if var == 'SRC':
1030                         if meth: app('lst.append(tsk.inputs%s)' % meth)
1031                         else: app("lst.extend([a.path_from(cwdx) for a in tsk.inputs])")
1032                 elif var == 'TGT':
1033                         if meth: app('lst.append(tsk.outputs%s)' % meth)
1034                         else: app("lst.extend([a.path_from(cwdx) for a in tsk.outputs])")
1035                 elif meth:
1036                         if meth.startswith(':'):
1037                                 m = meth[1:]
1038                                 if m == 'SRC':
1039                                         m = '[a.path_from(cwdx) for a in tsk.inputs]'
1040                                 elif m == 'TGT':
1041                                         m = '[a.path_from(cwdx) for a in tsk.outputs]'
1042                                 elif m[:3] not in ('tsk', 'gen', 'bld'):
1043                                         dvars.extend([var, m])
1044                                         m = '%r' % m
1045                                 app('lst.extend(tsk.colon(%r, %s))' % (var, m))
1046                         else:
1047                                 app('lst.extend(gen.to_list(%s%s))' % (var, meth))
1048                 else:
1049                         app('lst.extend(to_list(env[%r]))' % var)
1050                         if not var in dvars: dvars.append(var)
1051
1052         if extr:
1053                 if params[-1]:
1054                         app("lst.extend(%r)" % params[-1].split())
1055         fun = COMPILE_TEMPLATE_NOSHELL % "\n\t".join(buf)
1056         Logs.debug('action: %s' % fun.strip().splitlines())
1057         return (funex(fun), dvars)
1058
1059 def compile_fun(line, shell=False):
1060         """
1061         Parse a string expression such as "${CC} ${SRC} -o ${TGT}" and return a pair containing:
1062
1063         * the function created (compiled) for use as :py:meth:`waflib.Task.TaskBase.run`
1064         * the list of variables that imply a dependency from self.env
1065
1066         for example::
1067
1068                 from waflib.Task import compile_fun
1069                 compile_fun('cxx', '${CXX} -o ${TGT[0]} ${SRC} -I ${SRC[0].parent.bldpath()}')
1070
1071                 def build(bld):
1072                         bld(source='wscript', rule='echo "foo\\${SRC[0].name}\\bar"')
1073
1074         The env variables (CXX, ..) on the task must not hold dicts (order)
1075         The reserved keywords *TGT* and *SRC* represent the task input and output nodes
1076
1077         """
1078         if isinstance(line, str):
1079                 if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0:
1080                         shell = True
1081         else:
1082                 dvars_lst = []
1083                 funs_lst = []
1084                 for x in line:
1085                         if isinstance(x, str):
1086                                 fun, dvars = compile_fun(x, shell)
1087                                 dvars_lst += dvars
1088                                 funs_lst.append(fun)
1089                         else:
1090                                 # assume a function to let through
1091                                 funs_lst.append(x)
1092                 def composed_fun(task):
1093                         for x in funs_lst:
1094                                 ret = x(task)
1095                                 if ret:
1096                                         return ret
1097                         return None
1098                 return composed_fun, dvars
1099         if shell:
1100                 return compile_fun_shell(line)
1101         else:
1102                 return compile_fun_noshell(line)
1103
1104 def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[], before=[], after=[], shell=False, scan=None):
1105         """
1106         Returns a new task subclass with the function ``run`` compiled from the line given.
1107
1108         :param func: method run
1109         :type func: string or function
1110         :param vars: list of variables to hash
1111         :type vars: list of string
1112         :param color: color to use
1113         :type color: string
1114         :param shell: when *func* is a string, enable/disable the use of the shell
1115         :type shell: bool
1116         :param scan: method scan
1117         :type scan: function
1118         :rtype: :py:class:`waflib.Task.Task`
1119         """
1120
1121         params = {
1122                 'vars': vars or [], # function arguments are static, and this one may be modified by the class
1123                 'color': color,
1124                 'name': name,
1125                 'ext_in': Utils.to_list(ext_in),
1126                 'ext_out': Utils.to_list(ext_out),
1127                 'before': Utils.to_list(before),
1128                 'after': Utils.to_list(after),
1129                 'shell': shell,
1130                 'scan': scan,
1131         }
1132
1133         if isinstance(func, str) or isinstance(func, tuple):
1134                 params['run_str'] = func
1135         else:
1136                 params['run'] = func
1137
1138         cls = type(Task)(name, (Task,), params)
1139         global classes
1140         classes[name] = cls
1141         return cls
1142
1143
1144 def always_run(cls):
1145         """
1146         Task class decorator
1147
1148         Set all task instances of this class to be executed whenever a build is started
1149         The task signature is calculated, but the result of the comparation between
1150         task signatures is bypassed
1151         """
1152         old = cls.runnable_status
1153         def always(self):
1154                 ret = old(self)
1155                 if ret == SKIP_ME:
1156                         ret = RUN_ME
1157                 return ret
1158         cls.runnable_status = always
1159         return cls
1160
1161 def update_outputs(cls):
1162         """
1163         Task class decorator
1164
1165         If you want to create files in the source directory. For example, to keep *foo.txt* in the source
1166         directory, create it first and declare::
1167
1168                 def build(bld):
1169                         bld(rule='cp ${SRC} ${TGT}', source='wscript', target='foo.txt', update_outputs=True)
1170         """
1171         old_post_run = cls.post_run
1172         def post_run(self):
1173                 old_post_run(self)
1174                 for node in self.outputs:
1175                         node.sig = node.cache_sig = Utils.h_file(node.abspath())
1176                         self.generator.bld.task_sigs[node.abspath()] = self.uid() # issue #1017
1177         cls.post_run = post_run
1178
1179
1180         old_runnable_status = cls.runnable_status
1181         def runnable_status(self):
1182                 status = old_runnable_status(self)
1183                 if status != RUN_ME:
1184                         return status
1185
1186                 try:
1187                         # by default, we check that the output nodes have the signature of the task
1188                         # perform a second check, returning 'SKIP_ME' as we are expecting that
1189                         # the signatures do not match
1190                         bld = self.generator.bld
1191                         prev_sig = bld.task_sigs[self.uid()]
1192                         if prev_sig == self.signature():
1193                                 for x in self.outputs:
1194                                         if not x.is_child_of(bld.bldnode):
1195                                                 # special case of files created in the source directory
1196                                                 # hash them here for convenience -_-
1197                                                 x.sig = Utils.h_file(x.abspath())
1198                                         if not x.sig or bld.task_sigs[x.abspath()] != self.uid():
1199                                                 return RUN_ME
1200                                 return SKIP_ME
1201                 except OSError:
1202                         pass
1203                 except IOError:
1204                         pass
1205                 except KeyError:
1206                         pass
1207                 except IndexError:
1208                         pass
1209                 except AttributeError:
1210                         pass
1211                 return RUN_ME
1212         cls.runnable_status = runnable_status
1213
1214         return cls
1215