buildtools/wafadmin/3rdparty/lru_cache.py

   1 #! /usr/bin/env python
   2 # encoding: utf-8
   3 # Thomas Nagy 2011
   4
   5 import os, shutil, re
   6 import Options, Build, Logs
   7
   8 """
   9 Apply a least recently used policy to the Waf cache.
  10
  11 For performance reasons, it is called after the build is complete.
  12
  13 We assume that the the folders are written atomically
  14
  15 Do export WAFCACHE=/tmp/foo-xyz where xyz represents the cache size in megabytes
  16 If missing, the default cache size will be set to 10GB
  17 """
  18
  19 re_num = re.compile('[a-zA-Z_]+(\d+)')
  20
  21 CACHESIZE = 10*1024*1024*1024 # in bytes
  22 CLEANRATIO = 0.8
  23 DIRSIZE = 4096
  24
  25 def compile(self):
  26         if Options.cache_global and not Options.options.nocache:
  27                 try:
  28                         os.makedirs(Options.cache_global)
  29                 except:
  30                         pass
  31
  32         try:
  33                 self.raw_compile()
  34         finally:
  35                 if Options.cache_global and not Options.options.nocache:
  36                         self.sweep()
  37
  38 def sweep(self):
  39         global CACHESIZE
  40         CACHEDIR = Options.cache_global
  41
  42         # get the cache max size from the WAFCACHE filename
  43         re_num = re.compile('[a-zA-Z_]+(\d+)')
  44         val = re_num.sub('\\1', os.path.basename(Options.cache_global))
  45         try:
  46                 CACHESIZE = int(val)
  47         except:
  48                 pass
  49
  50         # map folder names to timestamps
  51         flist = {}
  52         for x in os.listdir(CACHEDIR):
  53                 j = os.path.join(CACHEDIR, x)
  54                 if os.path.isdir(j) and len(x) == 32: # dir names are md5 hexdigests
  55                         flist[x] = [os.stat(j).st_mtime, 0]
  56
  57         for (x, v) in flist.items():
  58                 cnt = DIRSIZE # each entry takes 4kB
  59                 d = os.path.join(CACHEDIR, x)
  60                 for k in os.listdir(d):
  61                         cnt += os.stat(os.path.join(d, k)).st_size
  62                 flist[x][1] = cnt
  63
  64         total = sum([x[1] for x in flist.values()])
  65         Logs.debug('lru: Cache size is %r' % total)
  66
  67         if total >= CACHESIZE:
  68                 Logs.debug('lru: Trimming the cache since %r > %r' % (total, CACHESIZE))
  69
  70                 # make a list to sort the folders by timestamp
  71                 lst = [(p, v[0], v[1]) for (p, v) in flist.items()]
  72                 lst.sort(key=lambda x: x[1]) # sort by timestamp
  73                 lst.reverse()
  74
  75                 while total >= CACHESIZE * CLEANRATIO:
  76                         (k, t, s) = lst.pop()
  77                         p = os.path.join(CACHEDIR, k)
  78                         v = p + '.del'
  79                         try:
  80                                 os.rename(p, v)
  81                         except:
  82                                 # someone already did it
  83                                 pass
  84                         else:
  85                                 try:
  86                                         shutil.rmtree(v)
  87                                 except:
  88                                         # this should not happen, but who knows?
  89                                         Logs.warn('If you ever see this message, report it (%r)' % v)
  90                         total -= s
  91                         del flist[k]
  92         Logs.debug('lru: Total at the end %r' % total)
  93
  94 Build.BuildContext.raw_compile = Build.BuildContext.compile
  95 Build.BuildContext.compile = compile
  96 Build.BuildContext.sweep = sweep
  97