debug: Set close-on-exec for the main log file FD

[Samba.git] / buildtools / wafadmin / Task.py

#!/usr/bin/env python
# encoding: utf-8
# Thomas Nagy, 2005-2008 (ita)

"""
Running tasks in parallel is a simple problem, but in practice it is more complicated:
* dependencies discovered during the build (dynamic task creation)
* dependencies discovered after files are compiled
* the amount of tasks and dependencies (graph size) can be huge

This is why the dependency management is split on three different levels:
1. groups of tasks that run all after another group of tasks
2. groups of tasks that can be run in parallel
3. tasks that can run in parallel, but with possible unknown ad-hoc dependencies

The point #1 represents a strict sequential order between groups of tasks, for example a compiler is produced
and used to compile the rest, whereas #2 and #3 represent partial order constraints where #2 applies to the kind of task
and #3 applies to the task instances.

#1 is held by the task manager: ordered list of TaskGroups (see bld.add_group)
#2 is held by the task groups and the task types: precedence after/before (topological sort),
   and the constraints extracted from file extensions
#3 is held by the tasks individually (attribute run_after),
   and the scheduler (Runner.py) use Task::runnable_status to reorder the tasks

--

To try, use something like this in your code:
import Constants, Task
Task.algotype = Constants.MAXPARALLEL

--

There are two concepts with the tasks (individual units of change):
* dependency (if 1 is recompiled, recompile 2)
* order (run 2 after 1)

example 1: if t1 depends on t2 and t2 depends on t3 it is not necessary to make t1 depend on t3 (dependency is transitive)
example 2: if t1 depends on a node produced by t2, it is not immediately obvious that t1 must run after t2 (order is not obvious)

The role of the Task Manager is to give the tasks in order (groups of task that may be run in parallel one after the other)

"""

import os, shutil, sys, re, random, datetime, tempfile, shlex
from Utils import md5
import Build, Runner, Utils, Node, Logs, Options
from Logs import debug, warn, error
from Constants import *

algotype = NORMAL
#algotype = JOBCONTROL
#algotype = MAXPARALLEL

COMPILE_TEMPLATE_SHELL = '''
def f(task):
        env = task.env
        wd = getattr(task, 'cwd', None)
        p = env.get_flat
        cmd = \'\'\' %s \'\'\' % s
        return task.exec_command(cmd, cwd=wd)
'''

COMPILE_TEMPLATE_NOSHELL = '''
def f(task):
        env = task.env
        wd = getattr(task, 'cwd', None)
        def to_list(xx):
                if isinstance(xx, str): return [xx]
                return xx
        lst = []
        %s
        lst = [x for x in lst if x]
        return task.exec_command(lst, cwd=wd)
'''


"""
Enable different kind of dependency algorithms:
1 make groups: first compile all cpps and then compile all links (NORMAL)
2 parallelize all (each link task run after its dependencies) (MAXPARALLEL)
3 like 1 but provide additional constraints for the parallelization (MAXJOBS)

In theory 1. will be faster than 2 for waf, but might be slower for builds
The scheme 2 will not allow for running tasks one by one so it can cause disk thrashing on huge builds
"""

file_deps = Utils.nada
"""
Additional dependency pre-check may be added by replacing the function file_deps.
e.g. extract_outputs, extract_deps below.
"""

class TaskManager(object):
        """The manager is attached to the build object, it holds a list of TaskGroup"""
        def __init__(self):
                self.groups = []
                self.tasks_done = []
                self.current_group = 0
                self.groups_names = {}

        def group_name(self, g):
                """name for the group g (utility)"""
                if not isinstance(g, TaskGroup):
                        g = self.groups[g]
                for x in self.groups_names:
                        if id(self.groups_names[x]) == id(g):
                                return x
                return ''

        def group_idx(self, tg):
                """group the task generator tg is in"""
                se = id(tg)
                for i in range(len(self.groups)):
                        g = self.groups[i]
                        for t in g.tasks_gen:
                                if id(t) == se:
                                        return i
                return None

        def get_next_set(self):
                """return the next set of tasks to execute
                the first parameter is the maximum amount of parallelization that may occur"""
                ret = None
                while not ret and self.current_group < len(self.groups):
                        ret = self.groups[self.current_group].get_next_set()
                        if ret: return ret
                        else:
                                self.groups[self.current_group].process_install()
                                self.current_group += 1
                return (None, None)

        def add_group(self, name=None, set=True):
                #if self.groups and not self.groups[0].tasks:
                #       error('add_group: an empty group is already present')
                g = TaskGroup()

                if name and name in self.groups_names:
                        error('add_group: name %s already present' % name)
                self.groups_names[name] = g
                self.groups.append(g)
                if set:
                        self.current_group = len(self.groups) - 1

        def set_group(self, idx):
                if isinstance(idx, str):
                        g = self.groups_names[idx]
                        for x in xrange(len(self.groups)):
                                if id(g) == id(self.groups[x]):
                                        self.current_group = x
                else:
                        self.current_group = idx

        def add_task_gen(self, tgen):
                if not self.groups: self.add_group()
                self.groups[self.current_group].tasks_gen.append(tgen)

        def add_task(self, task):
                if not self.groups: self.add_group()
                self.groups[self.current_group].tasks.append(task)

        def total(self):
                total = 0
                if not self.groups: return 0
                for group in self.groups:
                        total += len(group.tasks)
                return total

        def add_finished(self, tsk):
                self.tasks_done.append(tsk)
                bld = tsk.generator.bld
                if bld.is_install:
                        f = None
                        if 'install' in tsk.__dict__:
                                f = tsk.__dict__['install']
                                # install=0 to prevent installation
                                if f: f(tsk)
                        else:
                                tsk.install()

class TaskGroup(object):
        "the compilation of one group does not begin until the previous group has finished (in the manager)"
        def __init__(self):
                self.tasks = [] # this list will be consumed
                self.tasks_gen = []

                self.cstr_groups = Utils.DefaultDict(list) # tasks having equivalent constraints
                self.cstr_order = Utils.DefaultDict(set) # partial order between the cstr groups
                self.temp_tasks = [] # tasks put on hold
                self.ready = 0
                self.post_funs = []

        def reset(self):
                "clears the state of the object (put back the tasks into self.tasks)"
                for x in self.cstr_groups:
                        self.tasks += self.cstr_groups[x]
                self.tasks = self.temp_tasks + self.tasks
                self.temp_tasks = []
                self.cstr_groups = Utils.DefaultDict(list)
                self.cstr_order = Utils.DefaultDict(set)
                self.ready = 0

        def process_install(self):
                for (f, k, kw) in self.post_funs:
                        f(*k, **kw)

        def prepare(self):
                "prepare the scheduling"
                self.ready = 1
                file_deps(self.tasks)
                self.make_cstr_groups()
                self.extract_constraints()

        def get_next_set(self):
                "next list of tasks to execute using max job settings, returns (maxjobs, task_list)"
                global algotype
                if algotype == NORMAL:
                        tasks = self.tasks_in_parallel()
                        maxj = MAXJOBS
                elif algotype == JOBCONTROL:
                        (maxj, tasks) = self.tasks_by_max_jobs()
                elif algotype == MAXPARALLEL:
                        tasks = self.tasks_with_inner_constraints()
                        maxj = MAXJOBS
                else:
                        raise Utils.WafError("unknown algorithm type %s" % (algotype))

                if not tasks: return ()
                return (maxj, tasks)

        def make_cstr_groups(self):
                "unite the tasks that have similar constraints"
                self.cstr_groups = Utils.DefaultDict(list)
                for x in self.tasks:
                        h = x.hash_constraints()
                        self.cstr_groups[h].append(x)

        def set_order(self, a, b):
                self.cstr_order[a].add(b)

        def compare_exts(self, t1, t2):
                "extension production"
                x = "ext_in"
                y = "ext_out"
                in_ = t1.attr(x, ())
                out_ = t2.attr(y, ())
                for k in in_:
                        if k in out_:
                                return -1
                in_ = t2.attr(x, ())
                out_ = t1.attr(y, ())
                for k in in_:
                        if k in out_:
                                return 1
                return 0

        def compare_partial(self, t1, t2):
                "partial relations after/before"
                m = "after"
                n = "before"
                name = t2.__class__.__name__
                if name in Utils.to_list(t1.attr(m, ())): return -1
                elif name in Utils.to_list(t1.attr(n, ())): return 1
                name = t1.__class__.__name__
                if name in Utils.to_list(t2.attr(m, ())): return 1
                elif name in Utils.to_list(t2.attr(n, ())): return -1
                return 0

        def extract_constraints(self):
                "extract the parallelization constraints from the tasks with different constraints"
                keys = self.cstr_groups.keys()
                max = len(keys)
                # hopefully the length of this list is short
                for i in xrange(max):
                        t1 = self.cstr_groups[keys[i]][0]
                        for j in xrange(i + 1, max):
                                t2 = self.cstr_groups[keys[j]][0]

                                # add the constraints based on the comparisons
                                val = (self.compare_exts(t1, t2)
                                        or self.compare_partial(t1, t2)
                                        )
                                if val > 0:
                                        self.set_order(keys[i], keys[j])
                                elif val < 0:
                                        self.set_order(keys[j], keys[i])

        def tasks_in_parallel(self):
                "(NORMAL) next list of tasks that may be executed in parallel"

                if not self.ready: self.prepare()

                keys = self.cstr_groups.keys()

                unconnected = []
                remainder = []

                for u in keys:
                        for k in self.cstr_order.values():
                                if u in k:
                                        remainder.append(u)
                                        break
                        else:
                                unconnected.append(u)

                toreturn = []
                for y in unconnected:
                        toreturn.extend(self.cstr_groups[y])

                # remove stuff only after
                for y in unconnected:
                                try: self.cstr_order.__delitem__(y)
                                except KeyError: pass
                                self.cstr_groups.__delitem__(y)

                if not toreturn and remainder:
                        raise Utils.WafError("circular order constraint detected %r" % remainder)

                return toreturn

        def tasks_by_max_jobs(self):
                "(JOBCONTROL) returns the tasks that can run in parallel with the max amount of jobs"
                if not self.ready: self.prepare()
                if not self.temp_tasks: self.temp_tasks = self.tasks_in_parallel()
                if not self.temp_tasks: return (None, None)

                maxjobs = MAXJOBS
                ret = []
                remaining = []
                for t in self.temp_tasks:
                        m = getattr(t, "maxjobs", getattr(self.__class__, "maxjobs", MAXJOBS))
                        if m > maxjobs:
                                remaining.append(t)
                        elif m < maxjobs:
                                remaining += ret
                                ret = [t]
                                maxjobs = m
                        else:
                                ret.append(t)
                self.temp_tasks = remaining
                return (maxjobs, ret)

        def tasks_with_inner_constraints(self):
                """(MAXPARALLEL) returns all tasks in this group, but add the constraints on each task instance
                as an optimization, it might be desirable to discard the tasks which do not have to run"""
                if not self.ready: self.prepare()

                if getattr(self, "done", None): return None

                for p in self.cstr_order:
                        for v in self.cstr_order[p]:
                                for m in self.cstr_groups[p]:
                                        for n in self.cstr_groups[v]:
                                                n.set_run_after(m)
                self.cstr_order = Utils.DefaultDict(set)
                self.cstr_groups = Utils.DefaultDict(list)
                self.done = 1
                return self.tasks[:] # make a copy

class store_task_type(type):
        "store the task types that have a name ending in _task into a map (remember the existing task types)"
        def __init__(cls, name, bases, dict):
                super(store_task_type, cls).__init__(name, bases, dict)
                name = cls.__name__

                if name.endswith('_task'):
                        name = name.replace('_task', '')
                if name != 'TaskBase':
                        TaskBase.classes[name] = cls

class TaskBase(object):
        """Base class for all Waf tasks

        The most important methods are (by usual order of call):
        1 runnable_status: ask the task if it should be run, skipped, or if we have to ask later
        2 __str__: string to display to the user
        3 run: execute the task
        4 post_run: after the task is run, update the cache about the task

        This class should be seen as an interface, it provides the very minimum necessary for the scheduler
        so it does not do much.

        For illustration purposes, TaskBase instances try to execute self.fun (if provided)
        """

        __metaclass__ = store_task_type

        color = "GREEN"
        maxjobs = MAXJOBS
        classes = {}
        stat = None

        def __init__(self, *k, **kw):
                self.hasrun = NOT_RUN

                try:
                        self.generator = kw['generator']
                except KeyError:
                        self.generator = self
                        self.bld = Build.bld

                if kw.get('normal', 1):
                        self.generator.bld.task_manager.add_task(self)

        def __repr__(self):
                "used for debugging"
                return '\n\t{task: %s %s}' % (self.__class__.__name__, str(getattr(self, "fun", "")))

        def __str__(self):
                "string to display to the user"
                if hasattr(self, 'fun'):
                        return 'executing: %s\n' % self.fun.__name__
                return self.__class__.__name__ + '\n'

        def exec_command(self, *k, **kw):
                "use this for executing commands from tasks"
                # TODO in waf 1.6, eliminate bld.exec_command, and move the cwd processing to here
                if self.env['env']:
                        kw['env'] = self.env['env']
                return self.generator.bld.exec_command(*k, **kw)

        def runnable_status(self):
                "RUN_ME SKIP_ME or ASK_LATER"
                return RUN_ME

        def can_retrieve_cache(self):
                return False

        def call_run(self):
                if self.can_retrieve_cache():
                        return 0
                return self.run()

        def run(self):
                "called if the task must run"
                if hasattr(self, 'fun'):
                        return self.fun(self)
                return 0

        def post_run(self):
                "update the dependency tree (node stats)"
                pass

        def display(self):
                "print either the description (using __str__) or the progress bar or the ide output"
                col1 = Logs.colors(self.color)
                col2 = Logs.colors.NORMAL

                if Options.options.progress_bar == 1:
                        return self.generator.bld.progress_line(self.position[0], self.position[1], col1, col2)

                if Options.options.progress_bar == 2:
                        ela = Utils.get_elapsed_time(self.generator.bld.ini)
                        try:
                                ins  = ','.join([n.name for n in self.inputs])
                        except AttributeError:
                                ins = ''
                        try:
                                outs = ','.join([n.name for n in self.outputs])
                        except AttributeError:
                                outs = ''
                        return '|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n' % (self.position[1], self.position[0], ins, outs, ela)

                total = self.position[1]
                n = len(str(total))
                fs = '[%%%dd/%%%dd] %%s%%s%%s' % (n, n)
                return fs % (self.position[0], self.position[1], col1, str(self), col2)

        def attr(self, att, default=None):
                "retrieve an attribute from the instance or from the class (microoptimization here)"
                ret = getattr(self, att, self)
                if ret is self: return getattr(self.__class__, att, default)
                return ret

        def hash_constraints(self):
                "identify a task type for all the constraints relevant for the scheduler: precedence, file production"
                a = self.attr
                sum = hash((self.__class__.__name__,
                        str(a('before', '')),
                        str(a('after', '')),
                        str(a('ext_in', '')),
                        str(a('ext_out', '')),
                        self.__class__.maxjobs))
                return sum

        def format_error(self):
                "error message to display to the user (when a build fails)"
                if getattr(self, "err_msg", None):
                        return self.err_msg
                elif self.hasrun == CRASHED:
                        try:
                                return " -> task failed (err #%d): %r" % (self.err_code, self)
                        except AttributeError:
                                return " -> task failed: %r" % self
                elif self.hasrun == MISSING:
                        return " -> missing files: %r" % self
                else:
                        return ''

        def install(self):
                """
                installation is performed by looking at the task attributes:
                * install_path: installation path like "${PREFIX}/bin"
                * filename: install the first node in the outputs as a file with a particular name, be certain to give os.sep
                * chmod: permissions
                """
                bld = self.generator.bld
                d = self.attr('install')

                if self.attr('install_path'):
                        lst = [a.relpath_gen(bld.srcnode) for a in self.outputs]
                        perm = self.attr('chmod', O644)
                        if self.attr('src'):
                                # if src is given, install the sources too
                                lst += [a.relpath_gen(bld.srcnode) for a in self.inputs]
                        if self.attr('filename'):
                                dir = self.install_path.rstrip(os.sep) + os.sep + self.attr('filename')
                                bld.install_as(dir, lst[0], self.env, perm)
                        else:
                                bld.install_files(self.install_path, lst, self.env, perm)

class Task(TaskBase):
        """The parent class is quite limited, in this version:
        * file system interaction: input and output nodes
        * persistence: do not re-execute tasks that have already run
        * caching: same files can be saved and retrieved from a cache directory
        * dependencies:
                implicit, like .c files depending on .h files
                explicit, like the input nodes or the dep_nodes
                environment variables, like the CXXFLAGS in self.env
        """
        vars = []
        def __init__(self, env, **kw):
                TaskBase.__init__(self, **kw)
                self.env = env

                # inputs and outputs are nodes
                # use setters when possible
                self.inputs  = []
                self.outputs = []

                self.dep_nodes = []
                self.run_after = []

                # Additionally, you may define the following
                #self.dep_vars  = 'PREFIX DATADIR'

        def __str__(self):
                "string to display to the user"
                env = self.env
                src_str = ' '.join([a.nice_path(env) for a in self.inputs])
                tgt_str = ' '.join([a.nice_path(env) for a in self.outputs])
                if self.outputs: sep = ' -> '
                else: sep = ''
                return '%s: %s%s%s\n' % (self.__class__.__name__.replace('_task', ''), src_str, sep, tgt_str)

        def __repr__(self):
                return "".join(['\n\t{task: ', self.__class__.__name__, " ", ",".join([x.name for x in self.inputs]), " -> ", ",".join([x.name for x in self.outputs]), '}'])

        def unique_id(self):
                "get a unique id: hash the node paths, the variant, the class, the function"
                try:
                        return self.uid
                except AttributeError:
                        "this is not a real hot zone, but we want to avoid surprizes here"
                        m = md5()
                        up = m.update
                        up(self.__class__.__name__)
                        up(self.env.variant())
                        p = None
                        for x in self.inputs + self.outputs:
                                if p != x.parent.id:
                                        p = x.parent.id
                                        up(x.parent.abspath())
                                up(x.name)
                        self.uid = m.digest()
                        return self.uid

        def set_inputs(self, inp):
                if isinstance(inp, list): self.inputs += inp
                else: self.inputs.append(inp)

        def set_outputs(self, out):
                if isinstance(out, list): self.outputs += out
                else: self.outputs.append(out)

        def set_run_after(self, task):
                "set (scheduler) order on another task"
                # TODO: handle list or object
                assert isinstance(task, TaskBase)
                self.run_after.append(task)

        def add_file_dependency(self, filename):
                "TODO user-provided file dependencies"
                node = self.generator.bld.path.find_resource(filename)
                self.dep_nodes.append(node)

        def signature(self):
                # compute the result one time, and suppose the scan_signature will give the good result
                try: return self.cache_sig[0]
                except AttributeError: pass

                self.m = md5()

                # explicit deps
                exp_sig = self.sig_explicit_deps()

                # env vars
                var_sig = self.sig_vars()

                # implicit deps

                imp_sig = SIG_NIL
                if self.scan:
                        try:
                                imp_sig = self.sig_implicit_deps()
                        except ValueError:
                                return self.signature()

                # we now have the signature (first element) and the details (for debugging)
                ret = self.m.digest()
                self.cache_sig = (ret, exp_sig, imp_sig, var_sig)
                return ret

        def runnable_status(self):
                "SKIP_ME RUN_ME or ASK_LATER"
                #return 0 # benchmarking

                if self.inputs and (not self.outputs):
                        if not getattr(self.__class__, 'quiet', None):
                                warn("invalid task (no inputs OR outputs): override in a Task subclass or set the attribute 'quiet' %r" % self)

                for t in self.run_after:
                        if not t.hasrun:
                                return ASK_LATER

                env = self.env
                bld = self.generator.bld

                # first compute the signature
                new_sig = self.signature()

                # compare the signature to a signature computed previously
                key = self.unique_id()
                try:
                        prev_sig = bld.task_sigs[key][0]
                except KeyError:
                        debug("task: task %r must run as it was never run before or the task code changed", self)
                        return RUN_ME

                # compare the signatures of the outputs
                for node in self.outputs:
                        variant = node.variant(env)
                        try:
                                if bld.node_sigs[variant][node.id] != new_sig:
                                        return RUN_ME
                        except KeyError:
                                debug("task: task %r must run as the output nodes do not exist", self)
                                return RUN_ME

                # debug if asked to
                if Logs.verbose: self.debug_why(bld.task_sigs[key])

                if new_sig != prev_sig:
                        return RUN_ME
                return SKIP_ME

        def post_run(self):
                "called after a successful task run"
                bld = self.generator.bld
                env = self.env
                sig = self.signature()
                ssig = sig.encode('hex')

                variant = env.variant()
                for node in self.outputs:
                        # check if the node exists ..
                        try:
                                os.stat(node.abspath(env))
                        except OSError:
                                self.hasrun = MISSING
                                self.err_msg = '-> missing file: %r' % node.abspath(env)
                                raise Utils.WafError

                        # important, store the signature for the next run
                        bld.node_sigs[variant][node.id] = sig
                bld.task_sigs[self.unique_id()] = self.cache_sig

                # file caching, if possible
                # try to avoid data corruption as much as possible
                if not Options.cache_global or Options.options.nocache or not self.outputs:
                        return None

                if getattr(self, 'cached', None):
                        return None

                dname = os.path.join(Options.cache_global, ssig)
                tmpdir = tempfile.mkdtemp(prefix=Options.cache_global + os.sep + 'waf')

                try:
                        shutil.rmtree(dname)
                except:
                        pass

                try:
                        i = 0
                        for node in self.outputs:
                                variant = node.variant(env)
                                dest = os.path.join(tmpdir, str(i) + node.name)
                                shutil.copy2(node.abspath(env), dest)
                                i += 1
                except (OSError, IOError):
                        try:
                                shutil.rmtree(tmpdir)
                        except:
                                pass
                else:
                        try:
                                os.rename(tmpdir, dname)
                        except OSError:
                                try:
                                        shutil.rmtree(tmpdir)
                                except:
                                        pass
                        else:
                                try:
                                        os.chmod(dname, O755)
                                except:
                                        pass

        def can_retrieve_cache(self):
                """
                Retrieve build nodes from the cache
                update the file timestamps to help cleaning the least used entries from the cache
                additionally, set an attribute 'cached' to avoid re-creating the same cache files

                suppose there are files in cache/dir1/file1 and cache/dir2/file2
                first, read the timestamp of dir1
                then try to copy the files
                then look at the timestamp again, if it has changed, the data may have been corrupt (cache update by another process)
                should an exception occur, ignore the data
                """
                if not Options.cache_global or Options.options.nocache or not self.outputs:
                        return None

                env = self.env
                sig = self.signature()
                ssig = sig.encode('hex')

                # first try to access the cache folder for the task
                dname = os.path.join(Options.cache_global, ssig)
                try:
                        t1 = os.stat(dname).st_mtime
                except OSError:
                        return None

                i = 0
                for node in self.outputs:
                        variant = node.variant(env)

                        orig = os.path.join(dname, str(i) + node.name)
                        try:
                                shutil.copy2(orig, node.abspath(env))
                                # mark the cache file as used recently (modified)
                                os.utime(orig, None)
                        except (OSError, IOError):
                                debug('task: failed retrieving file')
                                return None
                        i += 1

                # is it the same folder?
                try:
                        t2 = os.stat(dname).st_mtime
                except OSError:
                        return None

                if t1 != t2:
                        return None

                for node in self.outputs:
                        self.generator.bld.node_sigs[variant][node.id] = sig
                        if Options.options.progress_bar < 1:
                                self.generator.bld.printout('restoring from cache %r\n' % node.bldpath(env))

                self.cached = True
                return 1

        def debug_why(self, old_sigs):
                "explains why a task is run"

                new_sigs = self.cache_sig
                def v(x):
                        return x.encode('hex')

                debug("Task %r", self)
                msgs = ['Task must run', '* Source file or manual dependency', '* Implicit dependency', '* Environment variable']
                tmp = 'task: -> %s: %s %s'
                for x in xrange(len(msgs)):
                        if (new_sigs[x] != old_sigs[x]):
                                debug(tmp, msgs[x], v(old_sigs[x]), v(new_sigs[x]))

        def sig_explicit_deps(self):
                bld = self.generator.bld
                up = self.m.update

                # the inputs
                for x in self.inputs + getattr(self, 'dep_nodes', []):
                        if not x.parent.id in bld.cache_scanned_folders:
                                bld.rescan(x.parent)

                        variant = x.variant(self.env)
                        try:
                                up(bld.node_sigs[variant][x.id])
                        except KeyError:
                                raise Utils.WafError('Missing node signature for %r (required by %r)' % (x, self))

                # manual dependencies, they can slow down the builds
                if bld.deps_man:
                        additional_deps = bld.deps_man
                        for x in self.inputs + self.outputs:
                                try:
                                        d = additional_deps[x.id]
                                except KeyError:
                                        continue

                                for v in d:
                                        if isinstance(v, Node.Node):
                                                bld.rescan(v.parent)
                                                variant = v.variant(self.env)
                                                try:
                                                        v = bld.node_sigs[variant][v.id]
                                                except KeyError:
                                                        raise Utils.WafError('Missing node signature for %r (required by %r)' % (v, self))
                                        elif hasattr(v, '__call__'):
                                                v = v() # dependency is a function, call it
                                        up(v)

                for x in self.dep_nodes:
                        v = bld.node_sigs[x.variant(self.env)][x.id]
                        up(v)

                return self.m.digest()

        def sig_vars(self):
                bld = self.generator.bld
                env = self.env

                # dependencies on the environment vars
                act_sig = bld.hash_env_vars(env, self.__class__.vars)
                self.m.update(act_sig)

                # additional variable dependencies, if provided
                dep_vars = getattr(self, 'dep_vars', None)
                if dep_vars:
                        self.m.update(bld.hash_env_vars(env, dep_vars))

                return self.m.digest()

        #def scan(self, node):
        #       """this method returns a tuple containing:
        #       * a list of nodes corresponding to real files
        #       * a list of names for files not found in path_lst
        #       the input parameters may have more parameters that the ones used below
        #       """
        #       return ((), ())
        scan = None

        # compute the signature, recompute it if there is no match in the cache
        def sig_implicit_deps(self):
                "the signature obtained may not be the one if the files have changed, we do it in two steps"

                bld = self.generator.bld

                # get the task signatures from previous runs
                key = self.unique_id()
                prev_sigs = bld.task_sigs.get(key, ())
                if prev_sigs:
                        try:
                                # for issue #379
                                if prev_sigs[2] == self.compute_sig_implicit_deps():
                                        return prev_sigs[2]
                        except (KeyError, OSError):
                                pass
                        del bld.task_sigs[key]
                        raise ValueError('rescan')

                # no previous run or the signature of the dependencies has changed, rescan the dependencies
                (nodes, names) = self.scan()
                if Logs.verbose:
                        debug('deps: scanner for %s returned %s %s', str(self), str(nodes), str(names))

                # store the dependencies in the cache
                bld.node_deps[key] = nodes
                bld.raw_deps[key] = names

                # recompute the signature and return it
                try:
                        sig = self.compute_sig_implicit_deps()
                except KeyError:
                        try:
                                nodes = []
                                for k in bld.node_deps.get(self.unique_id(), []):
                                        if k.id & 3 == 2: # Node.FILE:
                                                if not k.id in bld.node_sigs[0]:
                                                        nodes.append(k)
                                        else:
                                                if not k.id in bld.node_sigs[self.env.variant()]:
                                                        nodes.append(k)
                        except:
                                nodes = '?'
                        raise Utils.WafError('Missing node signature for %r (for implicit dependencies %r)' % (nodes, self))

                return sig

        def compute_sig_implicit_deps(self):
                """it is intended for .cpp and inferred .h files
                there is a single list (no tree traversal)
                this is the hot spot so ... do not touch"""
                upd = self.m.update

                bld = self.generator.bld
                tstamp = bld.node_sigs
                env = self.env

                for k in bld.node_deps.get(self.unique_id(), []):
                        # unlikely but necessary if it happens
                        if not k.parent.id in bld.cache_scanned_folders:
                                # if the parent folder is removed, an OSError may be thrown
                                bld.rescan(k.parent)

                        # if the parent folder is removed, a KeyError will be thrown
                        if k.id & 3 == 2: # Node.FILE:
                                upd(tstamp[0][k.id])
                        else:
                                upd(tstamp[env.variant()][k.id])

                return self.m.digest()

def funex(c):
        dc = {}
        exec(c, dc)
        return dc['f']

reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})", re.M)
def compile_fun_shell(name, line):
        """Compiles a string (once) into a function, eg:
        simple_task_type('c++', '${CXX} -o ${TGT[0]} ${SRC} -I ${SRC[0].parent.bldpath()}')

        The env variables (CXX, ..) on the task must not hold dicts (order)
        The reserved keywords TGT and SRC represent the task input and output nodes

        quick test:
        bld(source='wscript', rule='echo "foo\\${SRC[0].name}\\bar"')
        """

        extr = []
        def repl(match):
                g = match.group
                if g('dollar'): return "$"
                elif g('backslash'): return '\\\\'
                elif g('subst'): extr.append((g('var'), g('code'))); return "%s"
                return None

        line = reg_act.sub(repl, line) or line

        parm = []
        dvars = []
        app = parm.append
        for (var, meth) in extr:
                if var == 'SRC':
                        if meth: app('task.inputs%s' % meth)
                        else: app('" ".join([a.srcpath(env) for a in task.inputs])')
                elif var == 'TGT':
                        if meth: app('task.outputs%s' % meth)
                        else: app('" ".join([a.bldpath(env) for a in task.outputs])')
                else:
                        if not var in dvars: dvars.append(var)
                        app("p('%s')" % var)
        if parm: parm = "%% (%s) " % (',\n\t\t'.join(parm))
        else: parm = ''

        c = COMPILE_TEMPLATE_SHELL % (line, parm)

        debug('action: %s', c)
        return (funex(c), dvars)

def compile_fun_noshell(name, line):

        extr = []
        def repl(match):
                g = match.group
                if g('dollar'): return "$"
                elif g('subst'): extr.append((g('var'), g('code'))); return "<<|@|>>"
                return None

        line2 = reg_act.sub(repl, line)
        params = line2.split('<<|@|>>')

        buf = []
        dvars = []
        app = buf.append
        for x in xrange(len(extr)):
                params[x] = params[x].strip()
                if params[x]:
                        app("lst.extend(%r)" % params[x].split())
                (var, meth) = extr[x]
                if var == 'SRC':
                        if meth: app('lst.append(task.inputs%s)' % meth)
                        else: app("lst.extend([a.srcpath(env) for a in task.inputs])")
                elif var == 'TGT':
                        if meth: app('lst.append(task.outputs%s)' % meth)
                        else: app("lst.extend([a.bldpath(env) for a in task.outputs])")
                else:
                        app('lst.extend(to_list(env[%r]))' % var)
                        if not var in dvars: dvars.append(var)

        if params[-1]:
                app("lst.extend(%r)" % shlex.split(params[-1]))

        fun = COMPILE_TEMPLATE_NOSHELL % "\n\t".join(buf)
        debug('action: %s', fun)
        return (funex(fun), dvars)

def compile_fun(name, line, shell=None):
        "commands can be launched by the shell or not"
        if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0:
                shell = True
        #else:
        #       shell = False

        if shell is None:
                if sys.platform == 'win32':
                        shell = False
                else:
                        shell = True

        if shell:
                return compile_fun_shell(name, line)
        else:
                return compile_fun_noshell(name, line)

def simple_task_type(name, line, color='GREEN', vars=[], ext_in=[], ext_out=[], before=[], after=[], shell=None):
        """return a new Task subclass with the function run compiled from the line given"""
        (fun, dvars) = compile_fun(name, line, shell)
        fun.code = line
        return task_type_from_func(name, fun, vars or dvars, color, ext_in, ext_out, before, after)

def task_type_from_func(name, func, vars=[], color='GREEN', ext_in=[], ext_out=[], before=[], after=[]):
        """return a new Task subclass with the function run compiled from the line given"""
        params = {
                'run': func,
                'vars': vars,
                'color': color,
                'name': name,
                'ext_in': Utils.to_list(ext_in),
                'ext_out': Utils.to_list(ext_out),
                'before': Utils.to_list(before),
                'after': Utils.to_list(after),
        }

        cls = type(Task)(name, (Task,), params)
        TaskBase.classes[name] = cls
        return cls

def always_run(cls):
        """Set all task instances of this class to be executed whenever a build is started
        The task signature is calculated, but the result of the comparation between
        task signatures is bypassed
        """
        old = cls.runnable_status
        def always(self):
                ret = old(self)
                if ret == SKIP_ME:
                        return RUN_ME
                return ret
        cls.runnable_status = always

def update_outputs(cls):
        """When a command is always run, it is possible that the output only change
        sometimes. By default the build node have as a hash the signature of the task
        which may not change. With this, the output nodes (produced) are hashed,
        and the hashes are set to the build nodes

        This may avoid unnecessary recompilations, but it uses more resources
        (hashing the output files) so it is not used by default
        """
        old_post_run = cls.post_run
        def post_run(self):
                old_post_run(self)
                bld = self.generator.bld
                for output in self.outputs:
                        bld.node_sigs[self.env.variant()][output.id] = Utils.h_file(output.abspath(self.env))
                        bld.task_sigs[output.id] = self.unique_id()
        cls.post_run = post_run

        old_runnable_status = cls.runnable_status
        def runnable_status(self):
                status = old_runnable_status(self)
                if status != RUN_ME:
                        return status

                uid = self.unique_id()
                try:
                        bld = self.outputs[0].__class__.bld
                        new_sig  = self.signature()
                        prev_sig = bld.task_sigs[uid][0]
                        if prev_sig == new_sig:
                                for x in self.outputs:
                                        if not x.id in bld.node_sigs[self.env.variant()]:
                                                return RUN_ME
                                        if bld.task_sigs[x.id] != uid: # ensure the outputs are associated with *this* task
                                                return RUN_ME
                                return SKIP_ME
                except KeyError:
                        pass
                except IndexError:
                        pass
                return RUN_ME
        cls.runnable_status = runnable_status

def extract_outputs(tasks):
        """file_deps: Infer additional dependencies from task input and output nodes
        """
        v = {}
        for x in tasks:
                try:
                        (ins, outs) = v[x.env.variant()]
                except KeyError:
                        ins = {}
                        outs = {}
                        v[x.env.variant()] = (ins, outs)

                for a in getattr(x, 'inputs', []):
                        try: ins[a.id].append(x)
                        except KeyError: ins[a.id] = [x]
                for a in getattr(x, 'outputs', []):
                        try: outs[a.id].append(x)
                        except KeyError: outs[a.id] = [x]

        for (ins, outs) in v.values():
                links = set(ins.iterkeys()).intersection(outs.iterkeys())
                for k in links:
                        for a in ins[k]:
                                for b in outs[k]:
                                        a.set_run_after(b)

def extract_deps(tasks):
        """file_deps: Infer additional dependencies from task input and output nodes and from implicit dependencies
        returned by the scanners - that will only work if all tasks are created

        this is aimed at people who have pathological builds and who do not care enough
        to implement the build dependencies properly

        with two loops over the list of tasks, do not expect this to be really fast
        """

        # first reuse the function above
        extract_outputs(tasks)

        # map the output nodes to the tasks producing them
        out_to_task = {}
        for x in tasks:
                v = x.env.variant()
                try:
                        lst = x.outputs
                except AttributeError:
                        pass
                else:
                        for node in lst:
                                out_to_task[(v, node.id)] = x

        # map the dependencies found to the tasks compiled
        dep_to_task = {}
        for x in tasks:
                try:
                        x.signature()
                except: # this is on purpose
                        pass

                v = x.env.variant()
                key = x.unique_id()
                for k in x.generator.bld.node_deps.get(x.unique_id(), []):
                        try: dep_to_task[(v, k.id)].append(x)
                        except KeyError: dep_to_task[(v, k.id)] = [x]

        # now get the intersection
        deps = set(dep_to_task.keys()).intersection(set(out_to_task.keys()))

        # and add the dependencies from task to task
        for idx in deps:
                for k in dep_to_task[idx]:
                        k.set_run_after(out_to_task[idx])

        # cleanup, remove the signatures
        for x in tasks:
                try:
                        delattr(x, 'cache_sig')
                except AttributeError:
                        pass