build/pgo/profileserver.py

   1 #!/usr/bin/python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import json
   8 import os
   9 import sys
  10 import glob
  11 import subprocess
  12
  13 import mozcrash
  14 from mozbuild.base import MozbuildObject, BinaryNotFoundException
  15 from mozfile import TemporaryDirectory
  16 from mozhttpd import MozHttpd
  17 from mozprofile import FirefoxProfile, Preferences
  18 from mozprofile.permissions import ServerLocations
  19 from mozrunner import FirefoxRunner, CLI
  20 from six import string_types
  21
  22 PORT = 8888
  23
  24 PATH_MAPPINGS = {
  25     '/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests',
  26     # It is tempting to map to `testing/talos/talos/tests` instead, to avoid
  27     # writing `tests/` in every path, but we can't do that because some files
  28     # refer to scripts located in `../..`.
  29     '/talos': 'testing/talos/talos',
  30 }
  31
  32
  33 def get_crashreports(directory, name=None):
  34     rc = 0
  35     upload_path = os.environ.get('UPLOAD_PATH')
  36     if upload_path:
  37         # For automation, log the minidumps with stackwalk and get them moved to
  38         # the artifacts directory.
  39         fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
  40         if not fetches_dir:
  41             raise Exception("Unable to process minidump in automation because "
  42                             "$MOZ_FETCHES_DIR is not set in the environment")
  43         stackwalk_binary = os.path.join(fetches_dir, 'minidump_stackwalk', 'minidump_stackwalk')
  44         if sys.platform == 'win32':
  45             stackwalk_binary += '.exe'
  46         minidump_path = os.path.join(directory, "minidumps")
  47         rc = mozcrash.check_for_crashes(
  48             minidump_path,
  49             symbols_path=fetches_dir,
  50             stackwalk_binary=stackwalk_binary,
  51             dump_save_path=upload_path,
  52             test_name=name,
  53         )
  54     return rc
  55
  56
  57 if __name__ == '__main__':
  58     cli = CLI()
  59     debug_args, interactive = cli.debugger_arguments()
  60     runner_args = cli.runner_args()
  61
  62     build = MozbuildObject.from_environment()
  63
  64     binary = runner_args.get('binary')
  65     if not binary:
  66         try:
  67             binary = build.get_binary_path(where="staged-package")
  68         except BinaryNotFoundException as e:
  69             print('{}\n\n{}\n'.format(e, e.help()))
  70             sys.exit(1)
  71     binary = os.path.normpath(os.path.abspath(binary))
  72
  73     path_mappings = {
  74         k: os.path.join(build.topsrcdir, v)
  75         for k, v in PATH_MAPPINGS.items()
  76     }
  77     httpd = MozHttpd(port=PORT,
  78                      docroot=os.path.join(build.topsrcdir, "build", "pgo"),
  79                      path_mappings=path_mappings)
  80     httpd.start(block=False)
  81
  82     locations = ServerLocations()
  83     locations.add_host(host='127.0.0.1',
  84                        port=PORT,
  85                        options='primary,privileged')
  86
  87     old_profraw_files = glob.glob('*.profraw')
  88     for f in old_profraw_files:
  89         os.remove(f)
  90
  91     with TemporaryDirectory() as profilePath:
  92         # TODO: refactor this into mozprofile
  93         profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
  94         with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
  95             base_profiles = json.load(fh)['profileserver']
  96
  97         prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
  98                      for profile in base_profiles]
  99
 100         prefs = {}
 101         for path in prefpaths:
 102             prefs.update(Preferences.read_prefs(path))
 103
 104         interpolation = {"server": "%s:%d" % httpd.httpd.server_address}
 105         for k, v in prefs.items():
 106             if isinstance(v, string_types):
 107                 v = v.format(**interpolation)
 108             prefs[k] = Preferences.cast(v)
 109
 110         # Enforce e10s. This isn't in one of the user.js files because those
 111         # are shared with android, which doesn't want this on. We can't
 112         # interpolate because the formatting code only works for strings,
 113         # and this is a bool pref.
 114         prefs["browser.tabs.remote.autostart"] = True
 115
 116         profile = FirefoxProfile(profile=profilePath,
 117                                  preferences=prefs,
 118                                  addons=[os.path.join(
 119                                      build.topsrcdir, 'tools', 'quitter',
 120                                      'quitter@mozilla.org.xpi')],
 121                                  locations=locations)
 122
 123         env = os.environ.copy()
 124         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 125         env["XPCOM_DEBUG_BREAK"] = "warn"
 126         # We disable sandboxing to make writing profiling data actually work
 127         # Bug 1553850 considers fixing this.
 128         env["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
 129         env["MOZ_DISABLE_RDD_SANDBOX"] = "1"
 130         env["MOZ_DISABLE_SOCKET_PROCESS_SANDBOX"] = "1"
 131         env["MOZ_DISABLE_GPU_SANDBOX"] = "1"
 132         env["MOZ_DISABLE_GMP_SANDBOX"] = "1"
 133         env["MOZ_DISABLE_NPAPI_SANDBOX"] = "1"
 134         env["MOZ_DISABLE_VR_SANDBOX"] = "1"
 135
 136         # Ensure different pids write to different files
 137         env["LLVM_PROFILE_FILE"] = "default_%p_random_%m.profraw"
 138
 139         # Write to an output file if we're running in automation
 140         process_args = {'universal_newlines': True}
 141         if 'UPLOAD_PATH' in env:
 142             process_args['logfile'] = os.path.join(env['UPLOAD_PATH'], 'profile-run-1.log')
 143
 144         # Run Firefox a first time to initialize its profile
 145         runner = FirefoxRunner(profile=profile,
 146                                binary=binary,
 147                                cmdargs=['data:text/html,<script>Quitter.quit()</script>'],
 148                                env=env,
 149                                process_args=process_args)
 150         runner.start()
 151         ret = runner.wait()
 152         if ret:
 153             print("Firefox exited with code %d during profile initialization"
 154                   % ret)
 155             logfile = process_args.get('logfile')
 156             if logfile:
 157                 print("Firefox output (%s):" % logfile)
 158                 with open(logfile) as f:
 159                     print(f.read())
 160             httpd.stop()
 161             get_crashreports(profilePath, name='Profile initialization')
 162             sys.exit(ret)
 163
 164         jarlog = os.getenv("JARLOG_FILE")
 165         if jarlog:
 166             env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
 167             print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
 168             if os.path.exists(jarlog):
 169                 os.remove(jarlog)
 170
 171         if 'UPLOAD_PATH' in env:
 172             process_args['logfile'] = os.path.join(env['UPLOAD_PATH'], 'profile-run-2.log')
 173         cmdargs = ["http://localhost:%d/index.html" % PORT]
 174         runner = FirefoxRunner(profile=profile,
 175                                binary=binary,
 176                                cmdargs=cmdargs,
 177                                env=env,
 178                                process_args=process_args)
 179         runner.start(debug_args=debug_args, interactive=interactive)
 180         ret = runner.wait()
 181         httpd.stop()
 182         if ret:
 183             print("Firefox exited with code %d during profiling" % ret)
 184             logfile = process_args.get('logfile')
 185             if logfile:
 186                 print("Firefox output (%s):" % logfile)
 187                 with open(logfile) as f:
 188                     print(f.read())
 189             get_crashreports(profilePath, name='Profiling run')
 190             sys.exit(ret)
 191
 192         # Try to move the crash reports to the artifacts even if Firefox appears
 193         # to exit successfully, in case there's a crash that doesn't set the
 194         # return code to non-zero for some reason.
 195         if get_crashreports(profilePath, name='Firefox exited successfully?') != 0:
 196             print("Firefox exited successfully, but produced a crashreport")
 197             sys.exit(1)
 198
 199         llvm_profdata = env.get('LLVM_PROFDATA')
 200         if llvm_profdata:
 201             profraw_files = glob.glob('*.profraw')
 202             if not profraw_files:
 203                 print('Could not find profraw files in the current directory: %s' % os.getcwd())
 204                 sys.exit(1)
 205             merge_cmd = [
 206                 llvm_profdata,
 207                 'merge',
 208                 '-o',
 209                 'merged.profdata',
 210             ] + profraw_files
 211             rc = subprocess.call(merge_cmd)
 212             if rc != 0:
 213                 print('INFRA-ERROR: Failed to merge profile data. Corrupt profile?')
 214                 # exit with TBPL_RETRY
 215                 sys.exit(4)