tools/tryselect/push.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5
   6 import json
   7 import os
   8 import sys
   9 import traceback
  10
  11 import six
  12 from mach.util import get_state_dir
  13 from mozbuild.base import MozbuildObject
  14 from mozversioncontrol import MissingVCSExtension, get_repository_object
  15
  16 from .lando import push_to_lando_try
  17 from .util.estimates import duration_summary
  18 from .util.manage_estimates import (
  19     download_task_history_data,
  20     make_trimmed_taskgraph_cache,
  21 )
  22
  23 GIT_CINNABAR_NOT_FOUND = """
  24 Could not detect `git-cinnabar`.
  25
  26 The `mach try` command requires git-cinnabar to be installed when
  27 pushing from git. Please install it by running:
  28
  29     $ ./mach vcs-setup
  30 """.lstrip()
  31
  32 HG_PUSH_TO_TRY_NOT_FOUND = """
  33 Could not detect `push-to-try`.
  34
  35 The `mach try` command requires the push-to-try extension enabled
  36 when pushing from hg. Please install it by running:
  37
  38     $ ./mach vcs-setup
  39 """.lstrip()
  40
  41 VCS_NOT_FOUND = """
  42 Could not detect version control. Only `hg` or `git` are supported.
  43 """.strip()
  44
  45 UNCOMMITTED_CHANGES = """
  46 ERROR please commit changes before continuing
  47 """.strip()
  48
  49 MAX_HISTORY = 10
  50
  51 here = os.path.abspath(os.path.dirname(__file__))
  52 build = MozbuildObject.from_environment(cwd=here)
  53 vcs = get_repository_object(build.topsrcdir)
  54
  55 history_path = os.path.join(
  56     get_state_dir(specific_to_topsrcdir=True), "history", "try_task_configs.json"
  57 )
  58
  59
  60 def write_task_config(try_task_config):
  61     config_path = os.path.join(vcs.path, "try_task_config.json")
  62     with open(config_path, "w") as fh:
  63         json.dump(try_task_config, fh, indent=4, separators=(",", ": "), sort_keys=True)
  64         fh.write("\n")
  65     return config_path
  66
  67
  68 def write_task_config_history(msg, try_task_config):
  69     if not os.path.isfile(history_path):
  70         if not os.path.isdir(os.path.dirname(history_path)):
  71             os.makedirs(os.path.dirname(history_path))
  72         history = []
  73     else:
  74         with open(history_path) as fh:
  75             history = fh.read().strip().splitlines()
  76
  77     history.insert(0, json.dumps([msg, try_task_config]))
  78     history = history[:MAX_HISTORY]
  79     with open(history_path, "w") as fh:
  80         fh.write("\n".join(history))
  81
  82
  83 def check_working_directory(push=True):
  84     if not push:
  85         return
  86
  87     if not vcs.working_directory_clean():
  88         print(UNCOMMITTED_CHANGES)
  89         sys.exit(1)
  90
  91
  92 def generate_try_task_config(method, labels, params=None, routes=None):
  93     params = params or {}
  94
  95     # The user has explicitly requested a set of jobs, so run them all
  96     # regardless of optimization (unless the selector explicitly sets this to
  97     # True). Their dependencies can be optimized though.
  98     params.setdefault("optimize_target_tasks", False)
  99
 100     # Remove selected labels from 'existing_tasks' parameter if present
 101     if "existing_tasks" in params:
 102         params["existing_tasks"] = {
 103             label: tid
 104             for label, tid in params["existing_tasks"].items()
 105             if label not in labels
 106         }
 107
 108     try_config = params.setdefault("try_task_config", {})
 109     try_config.setdefault("env", {})["TRY_SELECTOR"] = method
 110
 111     try_config["tasks"] = sorted(labels)
 112
 113     if routes:
 114         try_config["routes"] = routes
 115
 116     try_task_config = {"version": 2, "parameters": params}
 117     return try_task_config
 118
 119
 120 def task_labels_from_try_config(try_task_config):
 121     if try_task_config["version"] == 2:
 122         parameters = try_task_config.get("parameters", {})
 123         if "try_task_config" in parameters:
 124             return parameters["try_task_config"]["tasks"]
 125         else:
 126             return None
 127     elif try_task_config["version"] == 1:
 128         return try_task_config.get("tasks", list())
 129     else:
 130         return None
 131
 132
 133 def display_push_estimates(try_task_config):
 134     task_labels = task_labels_from_try_config(try_task_config)
 135     if task_labels is None:
 136         return
 137
 138     cache_dir = os.path.join(
 139         get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph"
 140     )
 141
 142     graph_cache = None
 143     dep_cache = None
 144     target_file = None
 145     for graph_cache_file in ["target_task_graph", "full_task_graph"]:
 146         graph_cache = os.path.join(cache_dir, graph_cache_file)
 147         if os.path.isfile(graph_cache):
 148             dep_cache = graph_cache.replace("task_graph", "task_dependencies")
 149             target_file = graph_cache.replace("task_graph", "task_set")
 150             break
 151
 152     if not dep_cache:
 153         return
 154
 155     download_task_history_data(cache_dir=cache_dir)
 156     make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_file)
 157
 158     durations = duration_summary(dep_cache, task_labels, cache_dir)
 159
 160     print(
 161         "estimates: Runs {} tasks ({} selected, {} dependencies)".format(
 162             durations["dependency_count"] + durations["selected_count"],
 163             durations["selected_count"],
 164             durations["dependency_count"],
 165         )
 166     )
 167     print(
 168         "estimates: Total task duration {}".format(
 169             durations["dependency_duration"] + durations["selected_duration"]
 170         )
 171     )
 172     if "percentile" in durations:
 173         percentile = durations["percentile"]
 174         if percentile > 50:
 175             print("estimates: In the longest {}% of durations".format(100 - percentile))
 176         else:
 177             print("estimates: In the shortest {}% of durations".format(percentile))
 178     print(
 179         "estimates: Should take about {} (Finished around {})".format(
 180             durations["wall_duration_seconds"],
 181             durations["eta_datetime"].strftime("%Y-%m-%d %H:%M"),
 182         )
 183     )
 184
 185
 186 def push_to_try(
 187     method,
 188     msg,
 189     try_task_config=None,
 190     stage_changes=False,
 191     dry_run=False,
 192     closed_tree=False,
 193     files_to_change=None,
 194     allow_log_capture=False,
 195     push_to_lando=False,
 196 ):
 197     push = not stage_changes and not dry_run
 198     check_working_directory(push)
 199
 200     if try_task_config and method not in ("auto", "empty"):
 201         try:
 202             display_push_estimates(try_task_config)
 203         except Exception:
 204             traceback.print_exc()
 205             print("warning: unable to display push estimates")
 206
 207     # Format the commit message
 208     closed_tree_string = " ON A CLOSED TREE" if closed_tree else ""
 209     commit_message = "{}{}\n\nPushed via `mach try {}`".format(
 210         msg,
 211         closed_tree_string,
 212         method,
 213     )
 214
 215     config_path = None
 216     changed_files = []
 217     if try_task_config:
 218         if push and method not in ("again", "auto", "empty"):
 219             write_task_config_history(msg, try_task_config)
 220         config_path = write_task_config(try_task_config)
 221         changed_files.append(config_path)
 222
 223     if (push or stage_changes) and files_to_change:
 224         for path, content in files_to_change.items():
 225             path = os.path.join(vcs.path, path)
 226             with open(path, "wb") as fh:
 227                 fh.write(six.ensure_binary(content))
 228             changed_files.append(path)
 229
 230     try:
 231         if not push:
 232             print("Commit message:")
 233             print(commit_message)
 234             if config_path:
 235                 print("Calculated try_task_config.json:")
 236                 with open(config_path) as fh:
 237                     print(fh.read())
 238             return
 239
 240         vcs.add_remove_files(*changed_files)
 241
 242         try:
 243             if push_to_lando:
 244                 push_to_lando_try(vcs, commit_message)
 245             else:
 246                 vcs.push_to_try(commit_message, allow_log_capture=allow_log_capture)
 247         except MissingVCSExtension as e:
 248             if e.ext == "push-to-try":
 249                 print(HG_PUSH_TO_TRY_NOT_FOUND)
 250             elif e.ext == "cinnabar":
 251                 print(GIT_CINNABAR_NOT_FOUND)
 252             else:
 253                 raise
 254             sys.exit(1)
 255     finally:
 256         if config_path and os.path.isfile(config_path):
 257             os.remove(config_path)