Fix yet another merge conflict
[jack2.git] / waflib / Task.py
blob6aebc6074d1f7f760656aebe7c92cf8b572d9d2d
1 #!/usr/bin/env python
2 # encoding: utf-8
3 # Thomas Nagy, 2005-2018 (ita)
5 """
6 Tasks represent atomic operations such as processes.
7 """
9 import os, re, sys, tempfile, traceback
10 from waflib import Utils, Logs, Errors
12 # task states
13 NOT_RUN = 0
14 """The task was not executed yet"""
16 MISSING = 1
17 """The task has been executed but the files have not been created"""
19 CRASHED = 2
20 """The task execution returned a non-zero exit status"""
22 EXCEPTION = 3
23 """An exception occurred in the task execution"""
25 CANCELED = 4
26 """A dependency for the task is missing so it was cancelled"""
28 SKIPPED = 8
29 """The task did not have to be executed"""
31 SUCCESS = 9
32 """The task was successfully executed"""
34 ASK_LATER = -1
35 """The task is not ready to be executed"""
37 SKIP_ME = -2
38 """The task does not need to be executed"""
40 RUN_ME = -3
41 """The task must be executed"""
43 CANCEL_ME = -4
44 """The task cannot be executed because of a dependency problem"""
46 COMPILE_TEMPLATE_SHELL = '''
47 def f(tsk):
48 env = tsk.env
49 gen = tsk.generator
50 bld = gen.bld
51 cwdx = tsk.get_cwd()
52 p = env.get_flat
53 def to_list(xx):
54 if isinstance(xx, str): return [xx]
55 return xx
56 tsk.last_cmd = cmd = \'\'\' %s \'\'\' % s
57 return tsk.exec_command(cmd, cwd=cwdx, env=env.env or None)
58 '''
60 COMPILE_TEMPLATE_NOSHELL = '''
61 def f(tsk):
62 env = tsk.env
63 gen = tsk.generator
64 bld = gen.bld
65 cwdx = tsk.get_cwd()
66 def to_list(xx):
67 if isinstance(xx, str): return [xx]
68 return xx
69 def merge(lst1, lst2):
70 if lst1 and lst2:
71 return lst1[:-1] + [lst1[-1] + lst2[0]] + lst2[1:]
72 return lst1 + lst2
73 lst = []
75 if '' in lst:
76 lst = [x for x in lst if x]
77 tsk.last_cmd = lst
78 return tsk.exec_command(lst, cwd=cwdx, env=env.env or None)
79 '''
81 COMPILE_TEMPLATE_SIG_VARS = '''
82 def f(tsk):
83 sig = tsk.generator.bld.hash_env_vars(tsk.env, tsk.vars)
84 tsk.m.update(sig)
85 env = tsk.env
86 gen = tsk.generator
87 bld = gen.bld
88 cwdx = tsk.get_cwd()
89 p = env.get_flat
90 buf = []
92 tsk.m.update(repr(buf).encode())
93 '''
95 classes = {}
96 """
97 The metaclass :py:class:`waflib.Task.store_task_type` stores all class tasks
98 created by user scripts or Waf tools to this dict. It maps class names to class objects.
99 """
101 class store_task_type(type):
103 Metaclass: store the task classes into the dict pointed by the
104 class attribute 'register' which defaults to :py:const:`waflib.Task.classes`,
106 The attribute 'run_str' is compiled into a method 'run' bound to the task class.
108 def __init__(cls, name, bases, dict):
109 super(store_task_type, cls).__init__(name, bases, dict)
110 name = cls.__name__
112 if name != 'evil' and name != 'Task':
113 if getattr(cls, 'run_str', None):
114 # if a string is provided, convert it to a method
115 (f, dvars) = compile_fun(cls.run_str, cls.shell)
116 cls.hcode = Utils.h_cmd(cls.run_str)
117 cls.orig_run_str = cls.run_str
118 # change the name of run_str or it is impossible to subclass with a function
119 cls.run_str = None
120 cls.run = f
121 # process variables
122 cls.vars = list(set(cls.vars + dvars))
123 cls.vars.sort()
124 if cls.vars:
125 fun = compile_sig_vars(cls.vars)
126 if fun:
127 cls.sig_vars = fun
128 elif getattr(cls, 'run', None) and not 'hcode' in cls.__dict__:
129 # getattr(cls, 'hcode') would look in the upper classes
130 cls.hcode = Utils.h_cmd(cls.run)
132 # be creative
133 getattr(cls, 'register', classes)[name] = cls
135 evil = store_task_type('evil', (object,), {})
136 "Base class provided to avoid writing a metaclass, so the code can run in python 2.6 and 3.x unmodified"
138 class Task(evil):
140 Task objects represents actions to perform such as commands to execute by calling the `run` method.
142 Detecting when to execute a task occurs in the method :py:meth:`waflib.Task.Task.runnable_status`.
144 Detecting which tasks to execute is performed through a hash value returned by
145 :py:meth:`waflib.Task.Task.signature`. The task signature is persistent from build to build.
147 vars = []
148 """ConfigSet variables that should trigger a rebuild (class attribute used for :py:meth:`waflib.Task.Task.sig_vars`)"""
150 always_run = False
151 """Specify whether task instances must always be executed or not (class attribute)"""
153 shell = False
154 """Execute the command with the shell (class attribute)"""
156 color = 'GREEN'
157 """Color for the console display, see :py:const:`waflib.Logs.colors_lst`"""
159 ext_in = []
160 """File extensions that objects of this task class may use"""
162 ext_out = []
163 """File extensions that objects of this task class may create"""
165 before = []
166 """List of task class names to execute before instances of this class"""
168 after = []
169 """List of task class names to execute after instances of this class"""
171 hcode = Utils.SIG_NIL
172 """String representing an additional hash for the class representation"""
174 keep_last_cmd = False
175 """Whether to keep the last command executed on the instance after execution.
176 This may be useful for certain extensions but it can a lot of memory.
179 weight = 0
180 """Optional weight to tune the priority for task instances.
181 The higher, the earlier. The weight only applies to single task objects."""
183 tree_weight = 0
184 """Optional weight to tune the priority of task instances and whole subtrees.
185 The higher, the earlier."""
187 prio_order = 0
188 """Priority order set by the scheduler on instances during the build phase.
189 You most likely do not need to set it.
192 __slots__ = ('hasrun', 'generator', 'env', 'inputs', 'outputs', 'dep_nodes', 'run_after')
194 def __init__(self, *k, **kw):
195 self.hasrun = NOT_RUN
196 try:
197 self.generator = kw['generator']
198 except KeyError:
199 self.generator = self
201 self.env = kw['env']
202 """:py:class:`waflib.ConfigSet.ConfigSet` object (make sure to provide one)"""
204 self.inputs = []
205 """List of input nodes, which represent the files used by the task instance"""
207 self.outputs = []
208 """List of output nodes, which represent the files created by the task instance"""
210 self.dep_nodes = []
211 """List of additional nodes to depend on"""
213 self.run_after = set()
214 """Set of tasks that must be executed before this one"""
216 def __lt__(self, other):
217 return self.priority() > other.priority()
218 def __le__(self, other):
219 return self.priority() >= other.priority()
220 def __gt__(self, other):
221 return self.priority() < other.priority()
222 def __ge__(self, other):
223 return self.priority() <= other.priority()
225 def get_cwd(self):
227 :return: current working directory
228 :rtype: :py:class:`waflib.Node.Node`
230 bld = self.generator.bld
231 ret = getattr(self, 'cwd', None) or getattr(bld, 'cwd', bld.bldnode)
232 if isinstance(ret, str):
233 if os.path.isabs(ret):
234 ret = bld.root.make_node(ret)
235 else:
236 ret = self.generator.path.make_node(ret)
237 return ret
239 def quote_flag(self, x):
241 Surround a process argument by quotes so that a list of arguments can be written to a file
243 :param x: flag
244 :type x: string
245 :return: quoted flag
246 :rtype: string
248 old = x
249 if '\\' in x:
250 x = x.replace('\\', '\\\\')
251 if '"' in x:
252 x = x.replace('"', '\\"')
253 if old != x or ' ' in x or '\t' in x or "'" in x:
254 x = '"%s"' % x
255 return x
257 def priority(self):
259 Priority of execution; the higher, the earlier
261 :return: the priority value
262 :rtype: a tuple of numeric values
264 return (self.weight + self.prio_order, - getattr(self.generator, 'tg_idx_count', 0))
266 def split_argfile(self, cmd):
268 Splits a list of process commands into the executable part and its list of arguments
270 :return: a tuple containing the executable first and then the rest of arguments
271 :rtype: tuple
273 return ([cmd[0]], [self.quote_flag(x) for x in cmd[1:]])
275 def exec_command(self, cmd, **kw):
277 Wrapper for :py:meth:`waflib.Context.Context.exec_command`.
278 This version set the current working directory (``build.variant_dir``),
279 applies PATH settings (if self.env.PATH is provided), and can run long
280 commands through a temporary ``@argfile``.
282 :param cmd: process command to execute
283 :type cmd: list of string (best) or string (process will use a shell)
284 :return: the return code
285 :rtype: int
287 Optional parameters:
289 #. cwd: current working directory (Node or string)
290 #. stdout: set to None to prevent waf from capturing the process standard output
291 #. stderr: set to None to prevent waf from capturing the process standard error
292 #. timeout: timeout value (Python 3)
294 if not 'cwd' in kw:
295 kw['cwd'] = self.get_cwd()
297 if hasattr(self, 'timeout'):
298 kw['timeout'] = self.timeout
300 if self.env.PATH:
301 env = kw['env'] = dict(kw.get('env') or self.env.env or os.environ)
302 env['PATH'] = self.env.PATH if isinstance(self.env.PATH, str) else os.pathsep.join(self.env.PATH)
304 if hasattr(self, 'stdout'):
305 kw['stdout'] = self.stdout
306 if hasattr(self, 'stderr'):
307 kw['stderr'] = self.stderr
309 # workaround for command line length limit:
310 # http://support.microsoft.com/kb/830473
311 if not isinstance(cmd, str) and (len(repr(cmd)) >= 8192 if Utils.is_win32 else len(cmd) > 200000):
312 cmd, args = self.split_argfile(cmd)
313 try:
314 (fd, tmp) = tempfile.mkstemp()
315 os.write(fd, '\r\n'.join(args).encode())
316 os.close(fd)
317 if Logs.verbose:
318 Logs.debug('argfile: @%r -> %r', tmp, args)
319 return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw)
320 finally:
321 try:
322 os.remove(tmp)
323 except OSError:
324 # anti-virus and indexers can keep files open -_-
325 pass
326 else:
327 return self.generator.bld.exec_command(cmd, **kw)
329 def process(self):
331 Runs the task and handles errors
333 :return: 0 or None if everything is fine
334 :rtype: integer
336 # remove the task signature immediately before it is executed
337 # so that the task will be executed again in case of failure
338 try:
339 del self.generator.bld.task_sigs[self.uid()]
340 except KeyError:
341 pass
343 try:
344 ret = self.run()
345 except Exception:
346 self.err_msg = traceback.format_exc()
347 self.hasrun = EXCEPTION
348 else:
349 if ret:
350 self.err_code = ret
351 self.hasrun = CRASHED
352 else:
353 try:
354 self.post_run()
355 except Errors.WafError:
356 pass
357 except Exception:
358 self.err_msg = traceback.format_exc()
359 self.hasrun = EXCEPTION
360 else:
361 self.hasrun = SUCCESS
363 if self.hasrun != SUCCESS and self.scan:
364 # rescan dependencies on next run
365 try:
366 del self.generator.bld.imp_sigs[self.uid()]
367 except KeyError:
368 pass
370 def log_display(self, bld):
371 "Writes the execution status on the context logger"
372 if self.generator.bld.progress_bar == 3:
373 return
375 s = self.display()
376 if s:
377 if bld.logger:
378 logger = bld.logger
379 else:
380 logger = Logs
382 if self.generator.bld.progress_bar == 1:
383 c1 = Logs.colors.cursor_off
384 c2 = Logs.colors.cursor_on
385 logger.info(s, extra={'stream': sys.stderr, 'terminator':'', 'c1': c1, 'c2' : c2})
386 else:
387 logger.info(s, extra={'terminator':'', 'c1': '', 'c2' : ''})
389 def display(self):
391 Returns an execution status for the console, the progress bar, or the IDE output.
393 :rtype: string
395 col1 = Logs.colors(self.color)
396 col2 = Logs.colors.NORMAL
397 master = self.generator.bld.producer
399 def cur():
400 # the current task position, computed as late as possible
401 return master.processed - master.ready.qsize()
403 if self.generator.bld.progress_bar == 1:
404 return self.generator.bld.progress_line(cur(), master.total, col1, col2)
406 if self.generator.bld.progress_bar == 2:
407 ela = str(self.generator.bld.timer)
408 try:
409 ins = ','.join([n.name for n in self.inputs])
410 except AttributeError:
411 ins = ''
412 try:
413 outs = ','.join([n.name for n in self.outputs])
414 except AttributeError:
415 outs = ''
416 return '|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n' % (master.total, cur(), ins, outs, ela)
418 s = str(self)
419 if not s:
420 return None
422 total = master.total
423 n = len(str(total))
424 fs = '[%%%dd/%%%dd] %%s%%s%%s%%s\n' % (n, n)
425 kw = self.keyword()
426 if kw:
427 kw += ' '
428 return fs % (cur(), total, kw, col1, s, col2)
430 def hash_constraints(self):
432 Identifies a task type for all the constraints relevant for the scheduler: precedence, file production
434 :return: a hash value
435 :rtype: string
437 return (tuple(self.before), tuple(self.after), tuple(self.ext_in), tuple(self.ext_out), self.__class__.__name__, self.hcode)
439 def format_error(self):
441 Returns an error message to display the build failure reasons
443 :rtype: string
445 if Logs.verbose:
446 msg = ': %r\n%r' % (self, getattr(self, 'last_cmd', ''))
447 else:
448 msg = ' (run with -v to display more information)'
449 name = getattr(self.generator, 'name', '')
450 if getattr(self, "err_msg", None):
451 return self.err_msg
452 elif not self.hasrun:
453 return 'task in %r was not executed for some reason: %r' % (name, self)
454 elif self.hasrun == CRASHED:
455 try:
456 return ' -> task in %r failed with exit status %r%s' % (name, self.err_code, msg)
457 except AttributeError:
458 return ' -> task in %r failed%s' % (name, msg)
459 elif self.hasrun == MISSING:
460 return ' -> missing files in %r%s' % (name, msg)
461 elif self.hasrun == CANCELED:
462 return ' -> %r canceled because of missing dependencies' % name
463 else:
464 return 'invalid status for task in %r: %r' % (name, self.hasrun)
466 def colon(self, var1, var2):
468 Enable scriptlet expressions of the form ${FOO_ST:FOO}
469 If the first variable (FOO_ST) is empty, then an empty list is returned
471 The results will be slightly different if FOO_ST is a list, for example::
473 env.FOO = ['p1', 'p2']
474 env.FOO_ST = '-I%s'
475 # ${FOO_ST:FOO} returns
476 ['-Ip1', '-Ip2']
478 env.FOO_ST = ['-a', '-b']
479 # ${FOO_ST:FOO} returns
480 ['-a', '-b', 'p1', '-a', '-b', 'p2']
482 tmp = self.env[var1]
483 if not tmp:
484 return []
486 if isinstance(var2, str):
487 it = self.env[var2]
488 else:
489 it = var2
490 if isinstance(tmp, str):
491 return [tmp % x for x in it]
492 else:
493 lst = []
494 for y in it:
495 lst.extend(tmp)
496 lst.append(y)
497 return lst
499 def __str__(self):
500 "string to display to the user"
501 name = self.__class__.__name__
502 if self.outputs:
503 if name.endswith(('lib', 'program')) or not self.inputs:
504 node = self.outputs[0]
505 return node.path_from(node.ctx.launch_node())
506 if not (self.inputs or self.outputs):
507 return self.__class__.__name__
508 if len(self.inputs) == 1:
509 node = self.inputs[0]
510 return node.path_from(node.ctx.launch_node())
512 src_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.inputs])
513 tgt_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.outputs])
514 if self.outputs:
515 sep = ' -> '
516 else:
517 sep = ''
518 return '%s: %s%s%s' % (self.__class__.__name__, src_str, sep, tgt_str)
520 def keyword(self):
521 "Display keyword used to prettify the console outputs"
522 name = self.__class__.__name__
523 if name.endswith(('lib', 'program')):
524 return 'Linking'
525 if len(self.inputs) == 1 and len(self.outputs) == 1:
526 return 'Compiling'
527 if not self.inputs:
528 if self.outputs:
529 return 'Creating'
530 else:
531 return 'Running'
532 return 'Processing'
534 def __repr__(self):
535 "for debugging purposes"
536 try:
537 ins = ",".join([x.name for x in self.inputs])
538 outs = ",".join([x.name for x in self.outputs])
539 except AttributeError:
540 ins = ",".join([str(x) for x in self.inputs])
541 outs = ",".join([str(x) for x in self.outputs])
542 return "".join(['\n\t{task %r: ' % id(self), self.__class__.__name__, " ", ins, " -> ", outs, '}'])
544 def uid(self):
546 Returns an identifier used to determine if tasks are up-to-date. Since the
547 identifier will be stored between executions, it must be:
549 - unique for a task: no two tasks return the same value (for a given build context)
550 - the same for a given task instance
552 By default, the node paths, the class name, and the function are used
553 as inputs to compute a hash.
555 The pointer to the object (python built-in 'id') will change between build executions,
556 and must be avoided in such hashes.
558 :return: hash value
559 :rtype: string
561 try:
562 return self.uid_
563 except AttributeError:
564 m = Utils.md5(self.__class__.__name__)
565 up = m.update
566 for x in self.inputs + self.outputs:
567 up(x.abspath())
568 self.uid_ = m.digest()
569 return self.uid_
571 def set_inputs(self, inp):
573 Appends the nodes to the *inputs* list
575 :param inp: input nodes
576 :type inp: node or list of nodes
578 if isinstance(inp, list):
579 self.inputs += inp
580 else:
581 self.inputs.append(inp)
583 def set_outputs(self, out):
585 Appends the nodes to the *outputs* list
587 :param out: output nodes
588 :type out: node or list of nodes
590 if isinstance(out, list):
591 self.outputs += out
592 else:
593 self.outputs.append(out)
595 def set_run_after(self, task):
597 Run this task only after the given *task*.
599 Calling this method from :py:meth:`waflib.Task.Task.runnable_status` may cause
600 build deadlocks; see :py:meth:`waflib.Tools.fc.fc.runnable_status` for details.
602 :param task: task
603 :type task: :py:class:`waflib.Task.Task`
605 assert isinstance(task, Task)
606 self.run_after.add(task)
608 def signature(self):
610 Task signatures are stored between build executions, they are use to track the changes
611 made to the input nodes (not to the outputs!). The signature hashes data from various sources:
613 * explicit dependencies: files listed in the inputs (list of node objects) :py:meth:`waflib.Task.Task.sig_explicit_deps`
614 * implicit dependencies: list of nodes returned by scanner methods (when present) :py:meth:`waflib.Task.Task.sig_implicit_deps`
615 * hashed data: variables/values read from task.vars/task.env :py:meth:`waflib.Task.Task.sig_vars`
617 If the signature is expected to give a different result, clear the cache kept in ``self.cache_sig``::
619 from waflib import Task
620 class cls(Task.Task):
621 def signature(self):
622 sig = super(Task.Task, self).signature()
623 delattr(self, 'cache_sig')
624 return super(Task.Task, self).signature()
626 :return: the signature value
627 :rtype: string or bytes
629 try:
630 return self.cache_sig
631 except AttributeError:
632 pass
634 self.m = Utils.md5(self.hcode)
636 # explicit deps
637 self.sig_explicit_deps()
639 # env vars
640 self.sig_vars()
642 # implicit deps / scanner results
643 if self.scan:
644 try:
645 self.sig_implicit_deps()
646 except Errors.TaskRescan:
647 return self.signature()
649 ret = self.cache_sig = self.m.digest()
650 return ret
652 def runnable_status(self):
654 Returns the Task status
656 :return: a task state in :py:const:`waflib.Task.RUN_ME`,
657 :py:const:`waflib.Task.SKIP_ME`, :py:const:`waflib.Task.CANCEL_ME` or :py:const:`waflib.Task.ASK_LATER`.
658 :rtype: int
660 bld = self.generator.bld
661 if bld.is_install < 0:
662 return SKIP_ME
664 for t in self.run_after:
665 if not t.hasrun:
666 return ASK_LATER
667 elif t.hasrun < SKIPPED:
668 # a dependency has an error
669 return CANCEL_ME
671 # first compute the signature
672 try:
673 new_sig = self.signature()
674 except Errors.TaskNotReady:
675 return ASK_LATER
677 # compare the signature to a signature computed previously
678 key = self.uid()
679 try:
680 prev_sig = bld.task_sigs[key]
681 except KeyError:
682 Logs.debug('task: task %r must run: it was never run before or the task code changed', self)
683 return RUN_ME
685 if new_sig != prev_sig:
686 Logs.debug('task: task %r must run: the task signature changed', self)
687 return RUN_ME
689 # compare the signatures of the outputs
690 for node in self.outputs:
691 sig = bld.node_sigs.get(node)
692 if not sig:
693 Logs.debug('task: task %r must run: an output node has no signature', self)
694 return RUN_ME
695 if sig != key:
696 Logs.debug('task: task %r must run: an output node was produced by another task', self)
697 return RUN_ME
698 if not node.exists():
699 Logs.debug('task: task %r must run: an output node does not exist', self)
700 return RUN_ME
702 return (self.always_run and RUN_ME) or SKIP_ME
704 def post_run(self):
706 Called after successful execution to record that the task has run by
707 updating the entry in :py:attr:`waflib.Build.BuildContext.task_sigs`.
709 bld = self.generator.bld
710 for node in self.outputs:
711 if not node.exists():
712 self.hasrun = MISSING
713 self.err_msg = '-> missing file: %r' % node.abspath()
714 raise Errors.WafError(self.err_msg)
715 bld.node_sigs[node] = self.uid() # make sure this task produced the files in question
716 bld.task_sigs[self.uid()] = self.signature()
717 if not self.keep_last_cmd:
718 try:
719 del self.last_cmd
720 except AttributeError:
721 pass
723 def sig_explicit_deps(self):
725 Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.inputs`
726 and :py:attr:`waflib.Task.Task.dep_nodes` signatures.
728 bld = self.generator.bld
729 upd = self.m.update
731 # the inputs
732 for x in self.inputs + self.dep_nodes:
733 upd(x.get_bld_sig())
735 # manual dependencies, they can slow down the builds
736 if bld.deps_man:
737 additional_deps = bld.deps_man
738 for x in self.inputs + self.outputs:
739 try:
740 d = additional_deps[x]
741 except KeyError:
742 continue
744 for v in d:
745 try:
746 v = v.get_bld_sig()
747 except AttributeError:
748 if hasattr(v, '__call__'):
749 v = v() # dependency is a function, call it
750 upd(v)
752 def sig_deep_inputs(self):
754 Enable rebuilds on input files task signatures. Not used by default.
756 Example: hashes of output programs can be unchanged after being re-linked,
757 despite the libraries being different. This method can thus prevent stale unit test
758 results (waf_unit_test.py).
760 Hashing input file timestamps is another possibility for the implementation.
761 This may cause unnecessary rebuilds when input tasks are frequently executed.
762 Here is an implementation example::
764 lst = []
765 for node in self.inputs + self.dep_nodes:
766 st = os.stat(node.abspath())
767 lst.append(st.st_mtime)
768 lst.append(st.st_size)
769 self.m.update(Utils.h_list(lst))
771 The downside of the implementation is that it absolutely requires all build directory
772 files to be declared within the current build.
774 bld = self.generator.bld
775 lst = [bld.task_sigs[bld.node_sigs[node]] for node in (self.inputs + self.dep_nodes) if node.is_bld()]
776 self.m.update(Utils.h_list(lst))
778 def sig_vars(self):
780 Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.env` variables/values
781 When overriding this method, and if scriptlet expressions are used, make sure to follow
782 the code in :py:meth:`waflib.Task.Task.compile_sig_vars` to enable dependencies on scriptlet results.
784 This method may be replaced on subclasses by the metaclass to force dependencies on scriptlet code.
786 sig = self.generator.bld.hash_env_vars(self.env, self.vars)
787 self.m.update(sig)
789 scan = None
791 This method, when provided, returns a tuple containing:
793 * a list of nodes corresponding to real files
794 * a list of names for files not found in path_lst
796 For example::
798 from waflib.Task import Task
799 class mytask(Task):
800 def scan(self, node):
801 return ([], [])
803 The first and second lists in the tuple are stored in :py:attr:`waflib.Build.BuildContext.node_deps` and
804 :py:attr:`waflib.Build.BuildContext.raw_deps` respectively.
807 def sig_implicit_deps(self):
809 Used by :py:meth:`waflib.Task.Task.signature`; it hashes node signatures
810 obtained by scanning for dependencies (:py:meth:`waflib.Task.Task.scan`).
812 The exception :py:class:`waflib.Errors.TaskRescan` is thrown
813 when a file has changed. In this case, the method :py:meth:`waflib.Task.Task.signature` is called
814 once again, and return here to call :py:meth:`waflib.Task.Task.scan` and searching for dependencies.
816 bld = self.generator.bld
818 # get the task signatures from previous runs
819 key = self.uid()
820 prev = bld.imp_sigs.get(key, [])
822 # for issue #379
823 if prev:
824 try:
825 if prev == self.compute_sig_implicit_deps():
826 return prev
827 except Errors.TaskNotReady:
828 raise
829 except EnvironmentError:
830 # when a file was renamed, remove the stale nodes (headers in folders without source files)
831 # this will break the order calculation for headers created during the build in the source directory (should be uncommon)
832 # the behaviour will differ when top != out
833 for x in bld.node_deps.get(self.uid(), []):
834 if not x.is_bld() and not x.exists():
835 try:
836 del x.parent.children[x.name]
837 except KeyError:
838 pass
839 del bld.imp_sigs[key]
840 raise Errors.TaskRescan('rescan')
842 # no previous run or the signature of the dependencies has changed, rescan the dependencies
843 (bld.node_deps[key], bld.raw_deps[key]) = self.scan()
844 if Logs.verbose:
845 Logs.debug('deps: scanner for %s: %r; unresolved: %r', self, bld.node_deps[key], bld.raw_deps[key])
847 # recompute the signature and return it
848 try:
849 bld.imp_sigs[key] = self.compute_sig_implicit_deps()
850 except EnvironmentError:
851 for k in bld.node_deps.get(self.uid(), []):
852 if not k.exists():
853 Logs.warn('Dependency %r for %r is missing: check the task declaration and the build order!', k, self)
854 raise
856 def compute_sig_implicit_deps(self):
858 Used by :py:meth:`waflib.Task.Task.sig_implicit_deps` for computing the actual hash of the
859 :py:class:`waflib.Node.Node` returned by the scanner.
861 :return: a hash value for the implicit dependencies
862 :rtype: string or bytes
864 upd = self.m.update
865 self.are_implicit_nodes_ready()
867 # scanner returns a node that does not have a signature
868 # just *ignore* the error and let them figure out from the compiler output
869 # waf -k behaviour
870 for k in self.generator.bld.node_deps.get(self.uid(), []):
871 upd(k.get_bld_sig())
872 return self.m.digest()
874 def are_implicit_nodes_ready(self):
876 For each node returned by the scanner, see if there is a task that creates it,
877 and infer the build order
879 This has a low performance impact on null builds (1.86s->1.66s) thanks to caching (28s->1.86s)
881 bld = self.generator.bld
882 try:
883 cache = bld.dct_implicit_nodes
884 except AttributeError:
885 bld.dct_implicit_nodes = cache = {}
887 # one cache per build group
888 try:
889 dct = cache[bld.current_group]
890 except KeyError:
891 dct = cache[bld.current_group] = {}
892 for tsk in bld.cur_tasks:
893 for x in tsk.outputs:
894 dct[x] = tsk
896 modified = False
897 for x in bld.node_deps.get(self.uid(), []):
898 if x in dct:
899 self.run_after.add(dct[x])
900 modified = True
902 if modified:
903 for tsk in self.run_after:
904 if not tsk.hasrun:
905 #print "task is not ready..."
906 raise Errors.TaskNotReady('not ready')
907 if sys.hexversion > 0x3000000:
908 def uid(self):
909 try:
910 return self.uid_
911 except AttributeError:
912 m = Utils.md5(self.__class__.__name__.encode('latin-1', 'xmlcharrefreplace'))
913 up = m.update
914 for x in self.inputs + self.outputs:
915 up(x.abspath().encode('latin-1', 'xmlcharrefreplace'))
916 self.uid_ = m.digest()
917 return self.uid_
918 uid.__doc__ = Task.uid.__doc__
919 Task.uid = uid
921 def is_before(t1, t2):
923 Returns a non-zero value if task t1 is to be executed before task t2::
925 t1.ext_out = '.h'
926 t2.ext_in = '.h'
927 t2.after = ['t1']
928 t1.before = ['t2']
929 waflib.Task.is_before(t1, t2) # True
931 :param t1: Task object
932 :type t1: :py:class:`waflib.Task.Task`
933 :param t2: Task object
934 :type t2: :py:class:`waflib.Task.Task`
936 to_list = Utils.to_list
937 for k in to_list(t2.ext_in):
938 if k in to_list(t1.ext_out):
939 return 1
941 if t1.__class__.__name__ in to_list(t2.after):
942 return 1
944 if t2.__class__.__name__ in to_list(t1.before):
945 return 1
947 return 0
949 def set_file_constraints(tasks):
951 Updates the ``run_after`` attribute of all tasks based on the task inputs and outputs
953 :param tasks: tasks
954 :type tasks: list of :py:class:`waflib.Task.Task`
956 ins = Utils.defaultdict(set)
957 outs = Utils.defaultdict(set)
958 for x in tasks:
959 for a in x.inputs:
960 ins[a].add(x)
961 for a in x.dep_nodes:
962 ins[a].add(x)
963 for a in x.outputs:
964 outs[a].add(x)
966 links = set(ins.keys()).intersection(outs.keys())
967 for k in links:
968 for a in ins[k]:
969 a.run_after.update(outs[k])
972 class TaskGroup(object):
974 Wrap nxm task order constraints into a single object
975 to prevent the creation of large list/set objects
977 This is an optimization
979 def __init__(self, prev, next):
980 self.prev = prev
981 self.next = next
982 self.done = False
984 def get_hasrun(self):
985 for k in self.prev:
986 if not k.hasrun:
987 return NOT_RUN
988 return SUCCESS
990 hasrun = property(get_hasrun, None)
992 def set_precedence_constraints(tasks):
994 Updates the ``run_after`` attribute of all tasks based on the after/before/ext_out/ext_in attributes
996 :param tasks: tasks
997 :type tasks: list of :py:class:`waflib.Task.Task`
999 cstr_groups = Utils.defaultdict(list)
1000 for x in tasks:
1001 h = x.hash_constraints()
1002 cstr_groups[h].append(x)
1004 keys = list(cstr_groups.keys())
1005 maxi = len(keys)
1007 # this list should be short
1008 for i in range(maxi):
1009 t1 = cstr_groups[keys[i]][0]
1010 for j in range(i + 1, maxi):
1011 t2 = cstr_groups[keys[j]][0]
1013 # add the constraints based on the comparisons
1014 if is_before(t1, t2):
1015 a = i
1016 b = j
1017 elif is_before(t2, t1):
1018 a = j
1019 b = i
1020 else:
1021 continue
1023 a = cstr_groups[keys[a]]
1024 b = cstr_groups[keys[b]]
1026 if len(a) < 2 or len(b) < 2:
1027 for x in b:
1028 x.run_after.update(a)
1029 else:
1030 group = TaskGroup(set(a), set(b))
1031 for x in b:
1032 x.run_after.add(group)
1034 def funex(c):
1036 Compiles a scriptlet expression into a Python function
1038 :param c: function to compile
1039 :type c: string
1040 :return: the function 'f' declared in the input string
1041 :rtype: function
1043 dc = {}
1044 exec(c, dc)
1045 return dc['f']
1047 re_cond = re.compile('(?P<var>\w+)|(?P<or>\|)|(?P<and>&)')
1048 re_novar = re.compile(r'^(SRC|TGT)\W+.*?$')
1049 reg_act = re.compile(r'(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})', re.M)
1050 def compile_fun_shell(line):
1052 Creates a compiled function to execute a process through a sub-shell
1054 extr = []
1055 def repl(match):
1056 g = match.group
1057 if g('dollar'):
1058 return "$"
1059 elif g('backslash'):
1060 return '\\\\'
1061 elif g('subst'):
1062 extr.append((g('var'), g('code')))
1063 return "%s"
1064 return None
1065 line = reg_act.sub(repl, line) or line
1066 dvars = []
1067 def add_dvar(x):
1068 if x not in dvars:
1069 dvars.append(x)
1071 def replc(m):
1072 # performs substitutions and populates dvars
1073 if m.group('and'):
1074 return ' and '
1075 elif m.group('or'):
1076 return ' or '
1077 else:
1078 x = m.group('var')
1079 add_dvar(x)
1080 return 'env[%r]' % x
1082 parm = []
1083 app = parm.append
1084 for (var, meth) in extr:
1085 if var == 'SRC':
1086 if meth:
1087 app('tsk.inputs%s' % meth)
1088 else:
1089 app('" ".join([a.path_from(cwdx) for a in tsk.inputs])')
1090 elif var == 'TGT':
1091 if meth:
1092 app('tsk.outputs%s' % meth)
1093 else:
1094 app('" ".join([a.path_from(cwdx) for a in tsk.outputs])')
1095 elif meth:
1096 if meth.startswith(':'):
1097 add_dvar(var)
1098 m = meth[1:]
1099 if m == 'SRC':
1100 m = '[a.path_from(cwdx) for a in tsk.inputs]'
1101 elif m == 'TGT':
1102 m = '[a.path_from(cwdx) for a in tsk.outputs]'
1103 elif re_novar.match(m):
1104 m = '[tsk.inputs%s]' % m[3:]
1105 elif re_novar.match(m):
1106 m = '[tsk.outputs%s]' % m[3:]
1107 else:
1108 add_dvar(m)
1109 if m[:3] not in ('tsk', 'gen', 'bld'):
1110 m = '%r' % m
1111 app('" ".join(tsk.colon(%r, %s))' % (var, m))
1112 elif meth.startswith('?'):
1113 # In A?B|C output env.A if one of env.B or env.C is non-empty
1114 expr = re_cond.sub(replc, meth[1:])
1115 app('p(%r) if (%s) else ""' % (var, expr))
1116 else:
1117 call = '%s%s' % (var, meth)
1118 add_dvar(call)
1119 app(call)
1120 else:
1121 add_dvar(var)
1122 app("p('%s')" % var)
1123 if parm:
1124 parm = "%% (%s) " % (',\n\t\t'.join(parm))
1125 else:
1126 parm = ''
1128 c = COMPILE_TEMPLATE_SHELL % (line, parm)
1129 Logs.debug('action: %s', c.strip().splitlines())
1130 return (funex(c), dvars)
1132 reg_act_noshell = re.compile(r"(?P<space>\s+)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})|(?P<text>([^$ \t\n\r\f\v]|\$\$)+)", re.M)
1133 def compile_fun_noshell(line):
1135 Creates a compiled function to execute a process without a sub-shell
1137 buf = []
1138 dvars = []
1139 merge = False
1140 app = buf.append
1142 def add_dvar(x):
1143 if x not in dvars:
1144 dvars.append(x)
1146 def replc(m):
1147 # performs substitutions and populates dvars
1148 if m.group('and'):
1149 return ' and '
1150 elif m.group('or'):
1151 return ' or '
1152 else:
1153 x = m.group('var')
1154 add_dvar(x)
1155 return 'env[%r]' % x
1157 for m in reg_act_noshell.finditer(line):
1158 if m.group('space'):
1159 merge = False
1160 continue
1161 elif m.group('text'):
1162 app('[%r]' % m.group('text').replace('$$', '$'))
1163 elif m.group('subst'):
1164 var = m.group('var')
1165 code = m.group('code')
1166 if var == 'SRC':
1167 if code:
1168 app('[tsk.inputs%s]' % code)
1169 else:
1170 app('[a.path_from(cwdx) for a in tsk.inputs]')
1171 elif var == 'TGT':
1172 if code:
1173 app('[tsk.outputs%s]' % code)
1174 else:
1175 app('[a.path_from(cwdx) for a in tsk.outputs]')
1176 elif code:
1177 if code.startswith(':'):
1178 # a composed variable ${FOO:OUT}
1179 add_dvar(var)
1180 m = code[1:]
1181 if m == 'SRC':
1182 m = '[a.path_from(cwdx) for a in tsk.inputs]'
1183 elif m == 'TGT':
1184 m = '[a.path_from(cwdx) for a in tsk.outputs]'
1185 elif re_novar.match(m):
1186 m = '[tsk.inputs%s]' % m[3:]
1187 elif re_novar.match(m):
1188 m = '[tsk.outputs%s]' % m[3:]
1189 else:
1190 add_dvar(m)
1191 if m[:3] not in ('tsk', 'gen', 'bld'):
1192 m = '%r' % m
1193 app('tsk.colon(%r, %s)' % (var, m))
1194 elif code.startswith('?'):
1195 # In A?B|C output env.A if one of env.B or env.C is non-empty
1196 expr = re_cond.sub(replc, code[1:])
1197 app('to_list(env[%r] if (%s) else [])' % (var, expr))
1198 else:
1199 # plain code such as ${tsk.inputs[0].abspath()}
1200 call = '%s%s' % (var, code)
1201 add_dvar(call)
1202 app('to_list(%s)' % call)
1203 else:
1204 # a plain variable such as # a plain variable like ${AR}
1205 app('to_list(env[%r])' % var)
1206 add_dvar(var)
1207 if merge:
1208 tmp = 'merge(%s, %s)' % (buf[-2], buf[-1])
1209 del buf[-1]
1210 buf[-1] = tmp
1211 merge = True # next turn
1213 buf = ['lst.extend(%s)' % x for x in buf]
1214 fun = COMPILE_TEMPLATE_NOSHELL % "\n\t".join(buf)
1215 Logs.debug('action: %s', fun.strip().splitlines())
1216 return (funex(fun), dvars)
1218 def compile_fun(line, shell=False):
1220 Parses a string expression such as '${CC} ${SRC} -o ${TGT}' and returns a pair containing:
1222 * The function created (compiled) for use as :py:meth:`waflib.Task.Task.run`
1223 * The list of variables that must cause rebuilds when *env* data is modified
1225 for example::
1227 from waflib.Task import compile_fun
1228 compile_fun('cxx', '${CXX} -o ${TGT[0]} ${SRC} -I ${SRC[0].parent.bldpath()}')
1230 def build(bld):
1231 bld(source='wscript', rule='echo "foo\\${SRC[0].name}\\bar"')
1233 The env variables (CXX, ..) on the task must not hold dicts so as to preserve a consistent order.
1234 The reserved keywords ``TGT`` and ``SRC`` represent the task input and output nodes
1237 if isinstance(line, str):
1238 if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0:
1239 shell = True
1240 else:
1241 dvars_lst = []
1242 funs_lst = []
1243 for x in line:
1244 if isinstance(x, str):
1245 fun, dvars = compile_fun(x, shell)
1246 dvars_lst += dvars
1247 funs_lst.append(fun)
1248 else:
1249 # assume a function to let through
1250 funs_lst.append(x)
1251 def composed_fun(task):
1252 for x in funs_lst:
1253 ret = x(task)
1254 if ret:
1255 return ret
1256 return None
1257 return composed_fun, dvars_lst
1258 if shell:
1259 return compile_fun_shell(line)
1260 else:
1261 return compile_fun_noshell(line)
1263 def compile_sig_vars(vars):
1265 This method produces a sig_vars method suitable for subclasses that provide
1266 scriptlet code in their run_str code.
1267 If no such method can be created, this method returns None.
1269 The purpose of the sig_vars method returned is to ensures
1270 that rebuilds occur whenever the contents of the expression changes.
1271 This is the case B below::
1273 import time
1274 # case A: regular variables
1275 tg = bld(rule='echo ${FOO}')
1276 tg.env.FOO = '%s' % time.time()
1277 # case B
1278 bld(rule='echo ${gen.foo}', foo='%s' % time.time())
1280 :param vars: env variables such as CXXFLAGS or gen.foo
1281 :type vars: list of string
1282 :return: A sig_vars method relevant for dependencies if adequate, else None
1283 :rtype: A function, or None in most cases
1285 buf = []
1286 for x in sorted(vars):
1287 if x[:3] in ('tsk', 'gen', 'bld'):
1288 buf.append('buf.append(%s)' % x)
1289 if buf:
1290 return funex(COMPILE_TEMPLATE_SIG_VARS % '\n\t'.join(buf))
1291 return None
1293 def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[], before=[], after=[], shell=False, scan=None):
1295 Returns a new task subclass with the function ``run`` compiled from the line given.
1297 :param func: method run
1298 :type func: string or function
1299 :param vars: list of variables to hash
1300 :type vars: list of string
1301 :param color: color to use
1302 :type color: string
1303 :param shell: when *func* is a string, enable/disable the use of the shell
1304 :type shell: bool
1305 :param scan: method scan
1306 :type scan: function
1307 :rtype: :py:class:`waflib.Task.Task`
1310 params = {
1311 'vars': vars or [], # function arguments are static, and this one may be modified by the class
1312 'color': color,
1313 'name': name,
1314 'shell': shell,
1315 'scan': scan,
1318 if isinstance(func, str) or isinstance(func, tuple):
1319 params['run_str'] = func
1320 else:
1321 params['run'] = func
1323 cls = type(Task)(name, (Task,), params)
1324 classes[name] = cls
1326 if ext_in:
1327 cls.ext_in = Utils.to_list(ext_in)
1328 if ext_out:
1329 cls.ext_out = Utils.to_list(ext_out)
1330 if before:
1331 cls.before = Utils.to_list(before)
1332 if after:
1333 cls.after = Utils.to_list(after)
1335 return cls
1337 def deep_inputs(cls):
1339 Task class decorator to enable rebuilds on input files task signatures
1341 def sig_explicit_deps(self):
1342 Task.sig_explicit_deps(self)
1343 Task.sig_deep_inputs(self)
1344 cls.sig_explicit_deps = sig_explicit_deps
1345 return cls
1347 TaskBase = Task
1348 "Provided for compatibility reasons, TaskBase should not be used"
1350 class TaskSemaphore(object):
1352 Task semaphores provide a simple and efficient way of throttling the amount of
1353 a particular task to run concurrently. The throttling value is capped
1354 by the amount of maximum jobs, so for example, a `TaskSemaphore(10)`
1355 has no effect in a `-j2` build.
1357 Task semaphores are typically specified on the task class level::
1359 class compile(waflib.Task.Task):
1360 semaphore = waflib.Task.TaskSemaphore(2)
1361 run_str = 'touch ${TGT}'
1363 Task semaphores are meant to be used by the build scheduler in the main
1364 thread, so there are no guarantees of thread safety.
1366 def __init__(self, num):
1368 :param num: maximum value of concurrent tasks
1369 :type num: int
1371 self.num = num
1372 self.locking = set()
1373 self.waiting = set()
1375 def is_locked(self):
1376 """Returns True if this semaphore cannot be acquired by more tasks"""
1377 return len(self.locking) >= self.num
1379 def acquire(self, tsk):
1381 Mark the semaphore as used by the given task (not re-entrant).
1383 :param tsk: task object
1384 :type tsk: :py:class:`waflib.Task.Task`
1385 :raises: :py:class:`IndexError` in case the resource is already acquired
1387 if self.is_locked():
1388 raise IndexError('Cannot lock more %r' % self.locking)
1389 self.locking.add(tsk)
1391 def release(self, tsk):
1393 Mark the semaphore as unused by the given task.
1395 :param tsk: task object
1396 :type tsk: :py:class:`waflib.Task.Task`
1397 :raises: :py:class:`KeyError` in case the resource is not acquired by the task
1399 self.locking.remove(tsk)