build/pgo/profileserver.py

   1 #!/usr/bin/python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import glob
   8 import json
   9 import os
  10 import subprocess
  11 import sys
  12
  13 import mozcrash
  14 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  15 from mozfile import TemporaryDirectory
  16 from mozhttpd import MozHttpd
  17 from mozprofile import FirefoxProfile, Preferences
  18 from mozprofile.permissions import ServerLocations
  19 from mozrunner import CLI, FirefoxRunner
  20 from six import string_types
  21
  22 PORT = 8888
  23
  24 PATH_MAPPINGS = {
  25     "/webkit/PerformanceTests": "third_party/webkit/PerformanceTests",
  26     # It is tempting to map to `testing/talos/talos/tests` instead, to avoid
  27     # writing `tests/` in every path, but we can't do that because some files
  28     # refer to scripts located in `../..`.
  29     "/talos": "testing/talos/talos",
  30 }
  31
  32
  33 def get_crashreports(directory, name=None):
  34     rc = 0
  35     upload_path = os.environ.get("UPLOAD_PATH")
  36     if upload_path:
  37         # For automation, log the minidumps with stackwalk and get them moved to
  38         # the artifacts directory.
  39         fetches_dir = os.environ.get("MOZ_FETCHES_DIR")
  40         if not fetches_dir:
  41             raise Exception(
  42                 "Unable to process minidump in automation because "
  43                 "$MOZ_FETCHES_DIR is not set in the environment"
  44             )
  45         stackwalk_binary = os.path.join(
  46             fetches_dir, "minidump-stackwalk", "minidump-stackwalk"
  47         )
  48         if sys.platform == "win32":
  49             stackwalk_binary += ".exe"
  50         minidump_path = os.path.join(directory, "minidumps")
  51         rc = mozcrash.check_for_crashes(
  52             minidump_path,
  53             symbols_path=fetches_dir,
  54             stackwalk_binary=stackwalk_binary,
  55             dump_save_path=upload_path,
  56             test_name=name,
  57         )
  58     return rc
  59
  60
  61 if __name__ == "__main__":
  62     cli = CLI()
  63     debug_args, interactive = cli.debugger_arguments()
  64     runner_args = cli.runner_args()
  65
  66     build = MozbuildObject.from_environment()
  67
  68     binary = runner_args.get("binary")
  69     if not binary:
  70         try:
  71             binary = build.get_binary_path(where="staged-package")
  72         except BinaryNotFoundException as e:
  73             print("{}\n\n{}\n".format(e, e.help()))
  74             sys.exit(1)
  75     binary = os.path.normpath(os.path.abspath(binary))
  76
  77     path_mappings = {
  78         k: os.path.join(build.topsrcdir, v) for k, v in PATH_MAPPINGS.items()
  79     }
  80     httpd = MozHttpd(
  81         port=PORT,
  82         docroot=os.path.join(build.topsrcdir, "build", "pgo"),
  83         path_mappings=path_mappings,
  84     )
  85     httpd.start(block=False)
  86
  87     locations = ServerLocations()
  88     locations.add_host(host="127.0.0.1", port=PORT, options="primary,privileged")
  89
  90     old_profraw_files = glob.glob("*.profraw")
  91     for f in old_profraw_files:
  92         os.remove(f)
  93
  94     with TemporaryDirectory() as profilePath:
  95         # TODO: refactor this into mozprofile
  96         profile_data_dir = os.path.join(build.topsrcdir, "testing", "profiles")
  97         with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
  98             base_profiles = json.load(fh)["profileserver"]
  99
 100         prefpaths = [
 101             os.path.join(profile_data_dir, profile, "user.js")
 102             for profile in base_profiles
 103         ]
 104
 105         prefs = {}
 106         for path in prefpaths:
 107             prefs.update(Preferences.read_prefs(path))
 108
 109         interpolation = {"server": "%s:%d" % httpd.httpd.server_address}
 110         for k, v in prefs.items():
 111             if isinstance(v, string_types):
 112                 v = v.format(**interpolation)
 113             prefs[k] = Preferences.cast(v)
 114
 115         # Enforce e10s. This isn't in one of the user.js files because those
 116         # are shared with android, which doesn't want this on. We can't
 117         # interpolate because the formatting code only works for strings,
 118         # and this is a bool pref.
 119         prefs["browser.tabs.remote.autostart"] = True
 120
 121         profile = FirefoxProfile(
 122             profile=profilePath,
 123             preferences=prefs,
 124             addons=[
 125                 os.path.join(
 126                     build.topsrcdir, "tools", "quitter", "quitter@mozilla.org.xpi"
 127                 )
 128             ],
 129             locations=locations,
 130         )
 131
 132         env = os.environ.copy()
 133         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 134         env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
 135         env["XPCOM_DEBUG_BREAK"] = "warn"
 136         # We disable sandboxing to make writing profiling data actually work
 137         # Bug 1553850 considers fixing this.
 138         env["MOZ_DISABLE_CONTENT_SANDBOX"] = "1"
 139         env["MOZ_DISABLE_RDD_SANDBOX"] = "1"
 140         env["MOZ_DISABLE_SOCKET_PROCESS_SANDBOX"] = "1"
 141         env["MOZ_DISABLE_GPU_SANDBOX"] = "1"
 142         env["MOZ_DISABLE_GMP_SANDBOX"] = "1"
 143         env["MOZ_DISABLE_NPAPI_SANDBOX"] = "1"
 144         env["MOZ_DISABLE_VR_SANDBOX"] = "1"
 145
 146         # Ensure different pids write to different files
 147         env["LLVM_PROFILE_FILE"] = "default_%p_random_%m.profraw"
 148
 149         # Write to an output file if we're running in automation
 150         process_args = {"universal_newlines": True}
 151         if "UPLOAD_PATH" in env:
 152             process_args["logfile"] = os.path.join(
 153                 env["UPLOAD_PATH"], "profile-run-1.log"
 154             )
 155
 156         # Run Firefox a first time to initialize its profile
 157         runner = FirefoxRunner(
 158             profile=profile,
 159             binary=binary,
 160             cmdargs=["data:text/html,<script>Quitter.quit()</script>"],
 161             env=env,
 162             process_args=process_args,
 163         )
 164         runner.start()
 165         ret = runner.wait()
 166         if ret:
 167             print("Firefox exited with code %d during profile initialization" % ret)
 168             logfile = process_args.get("logfile")
 169             if logfile:
 170                 print("Firefox output (%s):" % logfile)
 171                 with open(logfile) as f:
 172                     print(f.read())
 173             httpd.stop()
 174             get_crashreports(profilePath, name="Profile initialization")
 175             sys.exit(ret)
 176
 177         jarlog = os.getenv("JARLOG_FILE")
 178         if jarlog:
 179             env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
 180             print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
 181             if os.path.exists(jarlog):
 182                 os.remove(jarlog)
 183
 184         if "UPLOAD_PATH" in env:
 185             process_args["logfile"] = os.path.join(
 186                 env["UPLOAD_PATH"], "profile-run-2.log"
 187             )
 188         cmdargs = ["http://localhost:%d/index.html" % PORT]
 189         runner = FirefoxRunner(
 190             profile=profile,
 191             binary=binary,
 192             cmdargs=cmdargs,
 193             env=env,
 194             process_args=process_args,
 195         )
 196         runner.start(debug_args=debug_args, interactive=interactive)
 197         ret = runner.wait()
 198         httpd.stop()
 199         if ret:
 200             print("Firefox exited with code %d during profiling" % ret)
 201             logfile = process_args.get("logfile")
 202             if logfile:
 203                 print("Firefox output (%s):" % logfile)
 204                 with open(logfile) as f:
 205                     print(f.read())
 206             get_crashreports(profilePath, name="Profiling run")
 207             sys.exit(ret)
 208
 209         # Try to move the crash reports to the artifacts even if Firefox appears
 210         # to exit successfully, in case there's a crash that doesn't set the
 211         # return code to non-zero for some reason.
 212         if get_crashreports(profilePath, name="Firefox exited successfully?") != 0:
 213             print("Firefox exited successfully, but produced a crashreport")
 214             sys.exit(1)
 215
 216         llvm_profdata = env.get("LLVM_PROFDATA")
 217         if llvm_profdata:
 218             profraw_files = glob.glob("*.profraw")
 219             if not profraw_files:
 220                 print(
 221                     "Could not find profraw files in the current directory: %s"
 222                     % os.getcwd()
 223                 )
 224                 sys.exit(1)
 225             merge_cmd = [
 226                 llvm_profdata,
 227                 "merge",
 228                 "-o",
 229                 "merged.profdata",
 230             ] + profraw_files
 231             rc = subprocess.call(merge_cmd)
 232             if rc != 0:
 233                 print("INFRA-ERROR: Failed to merge profile data. Corrupt profile?")
 234                 # exit with TBPL_RETRY
 235                 sys.exit(4)