build/pgo/profileserver.py

   1 #!/usr/bin/python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import json
   8 import os
   9 import sys
  10 import glob
  11 import subprocess
  12
  13 import mozcrash
  14 from mozbuild.base import MozbuildObject, BinaryNotFoundException
  15 from mozfile import TemporaryDirectory
  16 from mozhttpd import MozHttpd
  17 from mozprofile import FirefoxProfile, Preferences
  18 from mozprofile.permissions import ServerLocations
  19 from mozrunner import FirefoxRunner, CLI
  20 from six import string_types
  21
  22 PORT = 8888
  23
  24 PATH_MAPPINGS = {
  25     '/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests',
  26     # It is tempting to map to `testing/talos/talos/tests` instead, to avoid
  27     # writing `tests/` in every path, but we can't do that because some files
  28     # refer to scripts located in `../..`.
  29     '/talos': 'testing/talos/talos',
  30 }
  31
  32
  33 def get_crashreports(directory, name=None):
  34     rc = 0
  35     upload_path = os.environ.get('UPLOAD_PATH')
  36     if upload_path:
  37         # For automation, log the minidumps with stackwalk and get them moved to
  38         # the artifacts directory.
  39         fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
  40         if not fetches_dir:
  41             raise Exception("Unable to process minidump in automation because "
  42                             "$MOZ_FETCHES_DIR is not set in the environment")
  43         stackwalk_binary = os.path.join(fetches_dir, 'minidump_stackwalk', 'minidump_stackwalk')
  44         if sys.platform == 'win32':
  45             stackwalk_binary += '.exe'
  46         minidump_path = os.path.join(directory, "minidumps")
  47         rc = mozcrash.check_for_crashes(
  48             minidump_path,
  49             symbols_path=fetches_dir,
  50             stackwalk_binary=stackwalk_binary,
  51             dump_save_path=upload_path,
  52             test_name=name,
  53         )
  54     return rc
  55
  56
  57 if __name__ == '__main__':
  58     cli = CLI()
  59     debug_args, interactive = cli.debugger_arguments()
  60     runner_args = cli.runner_args()
  61
  62     build = MozbuildObject.from_environment()
  63
  64     binary = runner_args.get('binary')
  65     if not binary:
  66         try:
  67             binary = build.get_binary_path(where="staged-package")
  68         except BinaryNotFoundException as e:
  69             print('{}\n\n{}\n'.format(e, e.help()))
  70             sys.exit(1)
  71     binary = os.path.normpath(os.path.abspath(binary))
  72
  73     path_mappings = {
  74         k: os.path.join(build.topsrcdir, v)
  75         for k, v in PATH_MAPPINGS.items()
  76     }
  77     httpd = MozHttpd(port=PORT,
  78                      docroot=os.path.join(build.topsrcdir, "build", "pgo"),
  79                      path_mappings=path_mappings)
  80     httpd.start(block=False)
  81
  82     locations = ServerLocations()
  83     locations.add_host(host='127.0.0.1',
  84                        port=PORT,
  85                        options='primary,privileged')
  86
  87     old_profraw_files = glob.glob('*.profraw')
  88     for f in old_profraw_files:
  89         os.remove(f)
  90
  91     with TemporaryDirectory() as profilePath:
  92         # TODO: refactor this into mozprofile
  93         profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
  94         with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh:
  95             base_profiles = json.load(fh)['profileserver']
  96
  97         prefpaths = [os.path.join(profile_data_dir, profile, 'user.js')
  98                      for profile in base_profiles]
  99
 100         prefs = {}
 101         for path in prefpaths:
 102             prefs.update(Preferences.read_prefs(path))
 103
 104         interpolation = {"server": "%s:%d" % httpd.httpd.server_address}
 105         for k, v in prefs.items():
 106             if isinstance(v, string_types):
 107                 v = v.format(**interpolation)
 108             prefs[k] = Preferences.cast(v)
 109
 110         # Enforce e10s. This isn't in one of the user.js files because those
 111         # are shared with android, which doesn't want this on. We can't
 112         # interpolate because the formatting code only works for strings,
 113         # and this is a bool pref.
 114         prefs["browser.tabs.remote.autostart"] = True
 115
 116         profile = FirefoxProfile(profile=profilePath,
 117                                  preferences=prefs,
 118                                  addons=[os.path.join(
 119                                      build.topsrcdir, 'tools', 'quitter',
 120                                      'quitter@mozilla.org.xpi')],
 121                                  locations=locations)
 122
 123         env = os.environ.copy()
 124         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 125         env["XPCOM_DEBUG_BREAK"] = "warn"
 126         # We disable sandboxing to make writing profiling data actually work
 127         # Bug 1553850 considers fixing this.
 128         env["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
 129         env["MOZ_DISABLE_RDD_SANDBOX"] = "1"
 130
 131         # Ensure different pids write to different files
 132         env["LLVM_PROFILE_FILE"] = "default_%p_random_%m.profraw"
 133
 134         # Write to an output file if we're running in automation
 135         process_args = {'universal_newlines': True}
 136         if 'UPLOAD_PATH' in env:
 137             process_args['logfile'] = os.path.join(env['UPLOAD_PATH'], 'profile-run-1.log')
 138
 139         # Run Firefox a first time to initialize its profile
 140         runner = FirefoxRunner(profile=profile,
 141                                binary=binary,
 142                                cmdargs=['data:text/html,<script>Quitter.quit()</script>'],
 143                                env=env,
 144                                process_args=process_args)
 145         runner.start()
 146         ret = runner.wait()
 147         if ret:
 148             print("Firefox exited with code %d during profile initialization"
 149                   % ret)
 150             logfile = process_args.get('logfile')
 151             if logfile:
 152                 print("Firefox output (%s):" % logfile)
 153                 with open(logfile) as f:
 154                     print(f.read())
 155             httpd.stop()
 156             get_crashreports(profilePath, name='Profile initialization')
 157             sys.exit(ret)
 158
 159         jarlog = os.getenv("JARLOG_FILE")
 160         if jarlog:
 161             env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
 162             print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
 163             if os.path.exists(jarlog):
 164                 os.remove(jarlog)
 165
 166         if 'UPLOAD_PATH' in env:
 167             process_args['logfile'] = os.path.join(env['UPLOAD_PATH'], 'profile-run-2.log')
 168         cmdargs = ["http://localhost:%d/index.html" % PORT]
 169         runner = FirefoxRunner(profile=profile,
 170                                binary=binary,
 171                                cmdargs=cmdargs,
 172                                env=env,
 173                                process_args=process_args)
 174         runner.start(debug_args=debug_args, interactive=interactive)
 175         ret = runner.wait()
 176         httpd.stop()
 177         if ret:
 178             print("Firefox exited with code %d during profiling" % ret)
 179             logfile = process_args.get('logfile')
 180             if logfile:
 181                 print("Firefox output (%s):" % logfile)
 182                 with open(logfile) as f:
 183                     print(f.read())
 184             get_crashreports(profilePath, name='Profiling run')
 185             sys.exit(ret)
 186
 187         # Try to move the crash reports to the artifacts even if Firefox appears
 188         # to exit successfully, in case there's a crash that doesn't set the
 189         # return code to non-zero for some reason.
 190         if get_crashreports(profilePath, name='Firefox exited successfully?') != 0:
 191             print("Firefox exited successfully, but produced a crashreport")
 192             sys.exit(1)
 193
 194         llvm_profdata = env.get('LLVM_PROFDATA')
 195         if llvm_profdata:
 196             profraw_files = glob.glob('*.profraw')
 197             if not profraw_files:
 198                 print('Could not find profraw files in the current directory: %s' % os.getcwd())
 199                 sys.exit(1)
 200             merge_cmd = [
 201                 llvm_profdata,
 202                 'merge',
 203                 '-o',
 204                 'merged.profdata',
 205             ] + profraw_files
 206             rc = subprocess.call(merge_cmd)
 207             if rc != 0:
 208                 print('INFRA-ERROR: Failed to merge profile data. Corrupt profile?')
 209                 # exit with TBPL_RETRY
 210                 sys.exit(4)