buildtools/wafadmin/3rdparty/batched_cc.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy, 2006 (ita)
   4
   5 """
   6 Batched builds - compile faster
   7 instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
   8 cc -c ../file1.c ../file2.c ../file3.c
   9
  10 Files are output on the directory where the compiler is called, and dependencies are more difficult
  11 to track (do not run the command on all source files if only one file changes)
  12
  13 As such, we do as if the files were compiled one by one, but no command is actually run:
  14 replace each cc/cpp Task by a TaskSlave
  15 A new task called TaskMaster collects the signatures from each slave and finds out the command-line
  16 to run.
  17
  18 To set this up, the method ccroot::create_task is replaced by a new version, to enable batched builds
  19 it is only necessary to import this module in the configuration (no other change required)
  20 """
  21
  22 MAX_BATCH = 50
  23 MAXPARALLEL = False
  24
  25 EXT_C = ['.c', '.cc', '.cpp', '.cxx']
  26
  27 import os, threading
  28 import TaskGen, Task, ccroot, Build, Logs
  29 from TaskGen import extension, feature, before
  30 from Constants import *
  31
  32 cc_str = '${CC} ${CCFLAGS} ${CPPFLAGS} ${_CCINCFLAGS} ${_CCDEFFLAGS} -c ${SRCLST}'
  33 cc_fun = Task.compile_fun_noshell('batched_cc', cc_str)[0]
  34
  35 cxx_str = '${CXX} ${CXXFLAGS} ${CPPFLAGS} ${_CXXINCFLAGS} ${_CXXDEFFLAGS} -c ${SRCLST}'
  36 cxx_fun = Task.compile_fun_noshell('batched_cxx', cxx_str)[0]
  37
  38 count = 70000
  39 class batch_task(Task.Task):
  40         color = 'RED'
  41
  42         after = 'cc cxx'
  43         before = 'cc_link cxx_link static_link'
  44
  45         def __str__(self):
  46                 return '(batch compilation for %d slaves)\n' % len(self.slaves)
  47
  48         def __init__(self, *k, **kw):
  49                 Task.Task.__init__(self, *k, **kw)
  50                 self.slaves = []
  51                 self.inputs = []
  52                 self.hasrun = 0
  53
  54                 global count
  55                 count += 1
  56                 self.idx = count
  57
  58         def add_slave(self, slave):
  59                 self.slaves.append(slave)
  60                 self.set_run_after(slave)
  61
  62         def runnable_status(self):
  63                 for t in self.run_after:
  64                         if not t.hasrun:
  65                                 return ASK_LATER
  66
  67                 for t in self.slaves:
  68                         #if t.executed:
  69                         if t.hasrun != SKIPPED:
  70                                 return RUN_ME
  71
  72                 return SKIP_ME
  73
  74         def run(self):
  75                 outputs = []
  76                 self.outputs = []
  77
  78                 srclst = []
  79                 slaves = []
  80                 for t in self.slaves:
  81                         if t.hasrun != SKIPPED:
  82                                 slaves.append(t)
  83                                 srclst.append(t.inputs[0].abspath(self.env))
  84
  85                 self.env.SRCLST = srclst
  86                 self.cwd = slaves[0].inputs[0].parent.abspath(self.env)
  87
  88                 env = self.env
  89                 app = env.append_unique
  90                 cpppath_st = env['CPPPATH_ST']
  91                 env._CCINCFLAGS = env.CXXINCFLAGS = []
  92
  93                 # local flags come first
  94                 # set the user-defined includes paths
  95                 for i in env['INC_PATHS']:
  96                         app('_CCINCFLAGS', cpppath_st % i.abspath())
  97                         app('_CXXINCFLAGS', cpppath_st % i.abspath())
  98                         app('_CCINCFLAGS', cpppath_st % i.abspath(env))
  99                         app('_CXXINCFLAGS', cpppath_st % i.abspath(env))
 100
 101                 # set the library include paths
 102                 for i in env['CPPPATH']:
 103                         app('_CCINCFLAGS', cpppath_st % i)
 104                         app('_CXXINCFLAGS', cpppath_st % i)
 105
 106                 if self.slaves[0].__class__.__name__ == 'cc':
 107                         ret = cc_fun(self)
 108                 else:
 109                         ret = cxx_fun(self)
 110
 111                 if ret:
 112                         return ret
 113
 114                 for t in slaves:
 115                         t.old_post_run()
 116
 117 from TaskGen import extension, feature, after
 118
 119 import cc, cxx
 120 def wrap(fun):
 121         def foo(self, node):
 122                 # we cannot control the extension, this sucks
 123                 self.obj_ext = '.o'
 124
 125                 task = fun(self, node)
 126                 if not getattr(self, 'masters', None):
 127                         self.masters = {}
 128                         self.allmasters = []
 129
 130                 if not node.parent.id in self.masters:
 131                         m = self.masters[node.parent.id] = self.master = self.create_task('batch')
 132                         self.allmasters.append(m)
 133                 else:
 134                         m = self.masters[node.parent.id]
 135                         if len(m.slaves) > MAX_BATCH:
 136                                 m = self.masters[node.parent.id] = self.master = self.create_task('batch')
 137                                 self.allmasters.append(m)
 138
 139                 m.add_slave(task)
 140                 return task
 141         return foo
 142
 143 c_hook = wrap(cc.c_hook)
 144 extension(cc.EXT_CC)(c_hook)
 145
 146 cxx_hook = wrap(cxx.cxx_hook)
 147 extension(cxx.EXT_CXX)(cxx_hook)
 148
 149
 150 @feature('cprogram', 'cshlib', 'cstaticlib')
 151 @after('apply_link')
 152 def link_after_masters(self):
 153         if getattr(self, 'allmasters', None):
 154                 for m in self.allmasters:
 155                         self.link_task.set_run_after(m)
 156
 157 for c in ['cc', 'cxx']:
 158         t = Task.TaskBase.classes[c]
 159         def run(self):
 160                 pass
 161
 162         def post_run(self):
 163                 #self.executed=1
 164                 pass
 165
 166         def can_retrieve_cache(self):
 167                 if self.old_can_retrieve_cache():
 168                         for m in self.generator.allmasters:
 169                                 try:
 170                                         m.slaves.remove(self)
 171                                 except ValueError:
 172                                         pass    #this task wasn't included in that master
 173                         return 1
 174                 else:
 175                         return None
 176
 177         setattr(t, 'oldrun', t.__dict__['run'])
 178         setattr(t, 'run', run)
 179         setattr(t, 'old_post_run', t.post_run)
 180         setattr(t, 'post_run', post_run)
 181         setattr(t, 'old_can_retrieve_cache', t.can_retrieve_cache)
 182         setattr(t, 'can_retrieve_cache', can_retrieve_cache)
 183