waflib/extras/batched_cc.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2006-2015 (ita)
   4
   5 """
   6 Instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
   7 cc -c ../file1.c ../file2.c ../file3.c
   8
   9 Files are output on the directory where the compiler is called, and dependencies are more difficult
  10 to track (do not run the command on all source files if only one file changes)
  11 As such, we do as if the files were compiled one by one, but no command is actually run:
  12 replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the
  13 signatures from each slave and finds out the command-line to run.
  14
  15 Just import this module to start using it:
  16 def build(bld):
  17         bld.load('batched_cc')
  18
  19 Note that this is provided as an example, unity builds are recommended
  20 for best performance results (fewer tasks and fewer jobs to execute).
  21 See waflib/extras/unity.py.
  22 """
  23
  24 from waflib import Task, Utils
  25 from waflib.TaskGen import extension, feature, after_method
  26 from waflib.Tools import c, cxx
  27
  28 MAX_BATCH = 50
  29
  30 c_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
  31 c_fun, _ = Task.compile_fun_noshell(c_str)
  32
  33 cxx_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
  34 cxx_fun, _ = Task.compile_fun_noshell(cxx_str)
  35
  36 count = 70000
  37 class batch(Task.Task):
  38         color = 'PINK'
  39
  40         after = ['c', 'cxx']
  41         before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib']
  42
  43         def uid(self):
  44                 return Utils.h_list([Task.Task.uid(self), self.generator.idx, self.generator.path.abspath(), self.generator.target])
  45
  46         def __str__(self):
  47                 return 'Batch compilation for %d slaves' % len(self.slaves)
  48
  49         def __init__(self, *k, **kw):
  50                 Task.Task.__init__(self, *k, **kw)
  51                 self.slaves = []
  52                 self.inputs = []
  53                 self.hasrun = 0
  54
  55                 global count
  56                 count += 1
  57                 self.idx = count
  58
  59         def add_slave(self, slave):
  60                 self.slaves.append(slave)
  61                 self.set_run_after(slave)
  62
  63         def runnable_status(self):
  64                 for t in self.run_after:
  65                         if not t.hasrun:
  66                                 return Task.ASK_LATER
  67
  68                 for t in self.slaves:
  69                         #if t.executed:
  70                         if t.hasrun != Task.SKIPPED:
  71                                 return Task.RUN_ME
  72
  73                 return Task.SKIP_ME
  74
  75         def get_cwd(self):
  76                 return self.slaves[0].outputs[0].parent
  77
  78         def batch_incpaths(self):
  79                 st = self.env.CPPPATH_ST
  80                 return [st % node.abspath() for node in self.generator.includes_nodes]
  81
  82         def run(self):
  83                 self.outputs = []
  84
  85                 srclst = []
  86                 slaves = []
  87                 for t in self.slaves:
  88                         if t.hasrun != Task.SKIPPED:
  89                                 slaves.append(t)
  90                                 srclst.append(t.inputs[0].abspath())
  91
  92                 self.env.SRCLST = srclst
  93
  94                 if self.slaves[0].__class__.__name__ == 'c':
  95                         ret = c_fun(self)
  96                 else:
  97                         ret = cxx_fun(self)
  98
  99                 if ret:
 100                         return ret
 101
 102                 for t in slaves:
 103                         t.old_post_run()
 104
 105 def hook(cls_type):
 106         def n_hook(self, node):
 107
 108                 ext = '.obj' if self.env.CC_NAME == 'msvc' else '.o'
 109                 name = node.name
 110                 k = name.rfind('.')
 111                 if k >= 0:
 112                         basename = name[:k] + ext
 113                 else:
 114                         basename = name + ext
 115
 116                 outdir = node.parent.get_bld().make_node('%d' % self.idx)
 117                 outdir.mkdir()
 118                 out = outdir.find_or_declare(basename)
 119
 120                 task = self.create_task(cls_type, node, out)
 121
 122                 try:
 123                         self.compiled_tasks.append(task)
 124                 except AttributeError:
 125                         self.compiled_tasks = [task]
 126
 127                 if not getattr(self, 'masters', None):
 128                         self.masters = {}
 129                         self.allmasters = []
 130
 131                 def fix_path(tsk):
 132                         if self.env.CC_NAME == 'msvc':
 133                                 tsk.env.append_unique('CXX_TGT_F_BATCHED', '/Fo%s\\' % outdir.abspath())
 134
 135                 if not node.parent in self.masters:
 136                         m = self.masters[node.parent] = self.master = self.create_task('batch')
 137                         fix_path(m)
 138                         self.allmasters.append(m)
 139                 else:
 140                         m = self.masters[node.parent]
 141                         if len(m.slaves) > MAX_BATCH:
 142                                 m = self.masters[node.parent] = self.master = self.create_task('batch')
 143                                 fix_path(m)
 144                                 self.allmasters.append(m)
 145                 m.add_slave(task)
 146                 return task
 147         return n_hook
 148
 149 extension('.c')(hook('c'))
 150 extension('.cpp','.cc','.cxx','.C','.c++')(hook('cxx'))
 151
 152 @feature('cprogram', 'cshlib', 'cstaticlib', 'cxxprogram', 'cxxshlib', 'cxxstlib')
 153 @after_method('apply_link')
 154 def link_after_masters(self):
 155         if getattr(self, 'allmasters', None):
 156                 for m in self.allmasters:
 157                         self.link_task.set_run_after(m)
 158
 159 # Modify the c and cxx task classes - in theory it would be best to
 160 # create subclasses and to re-map the c/c++ extensions
 161 for x in ('c', 'cxx'):
 162         t = Task.classes[x]
 163         def run(self):
 164                 pass
 165
 166         def post_run(self):
 167                 pass
 168
 169         setattr(t, 'oldrun', getattr(t, 'run', None))
 170         setattr(t, 'run', run)
 171         setattr(t, 'old_post_run', t.post_run)
 172         setattr(t, 'post_run', post_run)
 173