tools/tryselect/push.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5
   6 import json
   7 import os
   8 import sys
   9 import traceback
  10
  11 import six
  12 from mach.util import get_state_dir
  13 from mozbuild.base import MozbuildObject
  14 from mozversioncontrol import get_repository_object, MissingVCSExtension
  15 from .util.manage_estimates import (
  16     download_task_history_data,
  17     make_trimmed_taskgraph_cache,
  18 )
  19 from .util.estimates import duration_summary
  20
  21 GIT_CINNABAR_NOT_FOUND = """
  22 Could not detect `git-cinnabar`.
  23
  24 The `mach try` command requires git-cinnabar to be installed when
  25 pushing from git. Please install it by running:
  26
  27     $ ./mach vcs-setup
  28 """.lstrip()
  29
  30 HG_PUSH_TO_TRY_NOT_FOUND = """
  31 Could not detect `push-to-try`.
  32
  33 The `mach try` command requires the push-to-try extension enabled
  34 when pushing from hg. Please install it by running:
  35
  36     $ ./mach vcs-setup
  37 """.lstrip()
  38
  39 VCS_NOT_FOUND = """
  40 Could not detect version control. Only `hg` or `git` are supported.
  41 """.strip()
  42
  43 UNCOMMITTED_CHANGES = """
  44 ERROR please commit changes before continuing
  45 """.strip()
  46
  47 MAX_HISTORY = 10
  48
  49 here = os.path.abspath(os.path.dirname(__file__))
  50 build = MozbuildObject.from_environment(cwd=here)
  51 vcs = get_repository_object(build.topsrcdir)
  52
  53 history_path = os.path.join(
  54     get_state_dir(specific_to_topsrcdir=True), "history", "try_task_configs.json"
  55 )
  56
  57
  58 def write_task_config(try_task_config):
  59     config_path = os.path.join(vcs.path, "try_task_config.json")
  60     with open(config_path, "w") as fh:
  61         json.dump(try_task_config, fh, indent=4, separators=(",", ": "), sort_keys=True)
  62         fh.write("\n")
  63     return config_path
  64
  65
  66 def write_task_config_history(msg, try_task_config):
  67     if not os.path.isfile(history_path):
  68         if not os.path.isdir(os.path.dirname(history_path)):
  69             os.makedirs(os.path.dirname(history_path))
  70         history = []
  71     else:
  72         with open(history_path) as fh:
  73             history = fh.read().strip().splitlines()
  74
  75     history.insert(0, json.dumps([msg, try_task_config]))
  76     history = history[:MAX_HISTORY]
  77     with open(history_path, "w") as fh:
  78         fh.write("\n".join(history))
  79
  80
  81 def check_working_directory(push=True):
  82     if not push:
  83         return
  84
  85     if not vcs.working_directory_clean():
  86         print(UNCOMMITTED_CHANGES)
  87         sys.exit(1)
  88
  89
  90 def generate_try_task_config(method, labels, try_config=None, routes=None):
  91     try_task_config = try_config or {}
  92     try_task_config.setdefault("env", {})["TRY_SELECTOR"] = method
  93     try_task_config.update(
  94         {
  95             "version": 1,
  96             "tasks": sorted(labels),
  97         }
  98     )
  99     if routes:
 100         try_task_config["routes"] = routes
 101
 102     return try_task_config
 103
 104
 105 def task_labels_from_try_config(try_task_config):
 106     if try_task_config["version"] == 2:
 107         parameters = try_task_config.get("parameters", {})
 108         if parameters.get("try_mode") == "try_task_config":
 109             return parameters["try_task_config"]["tasks"]
 110         else:
 111             return None
 112     elif try_task_config["version"] == 1:
 113         return try_task_config.get("tasks", list())
 114     else:
 115         return None
 116
 117
 118 def display_push_estimates(try_task_config):
 119     task_labels = task_labels_from_try_config(try_task_config)
 120     if task_labels is None:
 121         return
 122
 123     cache_dir = os.path.join(
 124         get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph"
 125     )
 126
 127     graph_cache = None
 128     dep_cache = None
 129     target_file = None
 130     for graph_cache_file in ["target_task_graph", "full_task_graph"]:
 131         graph_cache = os.path.join(cache_dir, graph_cache_file)
 132         if os.path.isfile(graph_cache):
 133             dep_cache = graph_cache.replace("task_graph", "task_dependencies")
 134             target_file = graph_cache.replace("task_graph", "task_set")
 135             break
 136
 137     if not dep_cache:
 138         return
 139
 140     download_task_history_data(cache_dir=cache_dir)
 141     make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_file)
 142
 143     durations = duration_summary(dep_cache, task_labels, cache_dir)
 144
 145     print(
 146         "estimates: Runs {} tasks ({} selected, {} dependencies)".format(
 147             durations["dependency_count"] + durations["selected_count"],
 148             durations["selected_count"],
 149             durations["dependency_count"],
 150         )
 151     )
 152     print(
 153         "estimates: Total task duration {}".format(
 154             durations["dependency_duration"] + durations["selected_duration"]
 155         )
 156     )
 157     if "percentile" in durations:
 158         print(
 159             "estimates: In the top {}% of durations".format(
 160                 100 - durations["percentile"]
 161             )
 162         )
 163     print(
 164         "estimates: Should take about {} (Finished around {})".format(
 165             durations["wall_duration_seconds"],
 166             durations["eta_datetime"].strftime("%Y-%m-%d %H:%M"),
 167         )
 168     )
 169
 170
 171 def push_to_try(
 172     method,
 173     msg,
 174     try_task_config=None,
 175     stage_changes=False,
 176     dry_run=False,
 177     closed_tree=False,
 178     files_to_change=None,
 179 ):
 180     push = not stage_changes and not dry_run
 181     check_working_directory(push)
 182
 183     if try_task_config and method not in ("auto", "empty"):
 184         try:
 185             display_push_estimates(try_task_config)
 186         except Exception:
 187             traceback.print_exc()
 188             print("warning: unable to display push estimates")
 189
 190     # Format the commit message
 191     closed_tree_string = " ON A CLOSED TREE" if closed_tree else ""
 192     commit_message = "{}{}\n\nPushed via `mach try {}`".format(
 193         msg,
 194         closed_tree_string,
 195         method,
 196     )
 197
 198     config_path = None
 199     changed_files = []
 200     if try_task_config:
 201         if push and method not in ("again", "auto", "empty"):
 202             write_task_config_history(msg, try_task_config)
 203         config_path = write_task_config(try_task_config)
 204         changed_files.append(config_path)
 205
 206     if (push or stage_changes) and files_to_change:
 207         for path, content in files_to_change.items():
 208             path = os.path.join(vcs.path, path)
 209             with open(path, "wb") as fh:
 210                 fh.write(six.ensure_binary(content))
 211             changed_files.append(path)
 212
 213     try:
 214         if not push:
 215             print("Commit message:")
 216             print(commit_message)
 217             if config_path:
 218                 print("Calculated try_task_config.json:")
 219                 with open(config_path) as fh:
 220                     print(fh.read())
 221             return
 222
 223         vcs.add_remove_files(*changed_files)
 224
 225         try:
 226             vcs.push_to_try(commit_message)
 227         except MissingVCSExtension as e:
 228             if e.ext == "push-to-try":
 229                 print(HG_PUSH_TO_TRY_NOT_FOUND)
 230             elif e.ext == "cinnabar":
 231                 print(GIT_CINNABAR_NOT_FOUND)
 232             else:
 233                 raise
 234             sys.exit(1)
 235     finally:
 236         if config_path and os.path.isfile(config_path):
 237             os.remove(config_path)