build/toolchain/get_concurrent_links.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 # This script computs the number of concurrent links we want to run in the build
   6 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
   7
   8 import os
   9 import re
  10 import subprocess
  11 import sys
  12
  13 def GetDefaultConcurrentLinks():
  14   # Inherit the legacy environment variable for people that have set it in GYP.
  15   pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
  16   if pool_size:
  17     return pool_size
  18
  19   if sys.platform in ('win32', 'cygwin'):
  20     import ctypes
  21
  22     class MEMORYSTATUSEX(ctypes.Structure):
  23       _fields_ = [
  24         ("dwLength", ctypes.c_ulong),
  25         ("dwMemoryLoad", ctypes.c_ulong),
  26         ("ullTotalPhys", ctypes.c_ulonglong),
  27         ("ullAvailPhys", ctypes.c_ulonglong),
  28         ("ullTotalPageFile", ctypes.c_ulonglong),
  29         ("ullAvailPageFile", ctypes.c_ulonglong),
  30         ("ullTotalVirtual", ctypes.c_ulonglong),
  31         ("ullAvailVirtual", ctypes.c_ulonglong),
  32         ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
  33       ]
  34
  35     stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
  36     ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
  37
  38     # VS 2015 uses 20% more working set than VS 2013 and can consume all RAM
  39     # on a 64 GB machine.
  40     mem_limit = max(1, stat.ullTotalPhys / (5 * (2 ** 30)))  # total / 5GB
  41     hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
  42     return min(mem_limit, hard_cap)
  43   elif sys.platform.startswith('linux'):
  44     if os.path.exists("/proc/meminfo"):
  45       with open("/proc/meminfo") as meminfo:
  46         memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
  47         for line in meminfo:
  48           match = memtotal_re.match(line)
  49           if not match:
  50             continue
  51           # Allow 8Gb per link on Linux because Gold is quite memory hungry
  52           return max(1, int(match.group(1)) / (8 * (2 ** 20)))
  53     return 1
  54   elif sys.platform == 'darwin':
  55     try:
  56       avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
  57       # A static library debug build of Chromium's unit_tests takes ~2.7GB, so
  58       # 4GB per ld process allows for some more bloat.
  59       return max(1, avail_bytes / (4 * (2 ** 30)))  # total / 4GB
  60     except Exception:
  61       return 1
  62   else:
  63     # TODO(scottmg): Implement this for other platforms.
  64     return 1
  65
  66 print GetDefaultConcurrentLinks()