tools/tryselect/push.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5
   6 import json
   7 import os
   8 import sys
   9 import traceback
  10
  11 import six
  12 from mach.util import get_state_dir
  13 from mozbuild.base import MozbuildObject
  14 from mozversioncontrol import MissingVCSExtension, get_repository_object
  15
  16 from .util.estimates import duration_summary
  17 from .util.manage_estimates import (
  18     download_task_history_data,
  19     make_trimmed_taskgraph_cache,
  20 )
  21
  22 GIT_CINNABAR_NOT_FOUND = """
  23 Could not detect `git-cinnabar`.
  24
  25 The `mach try` command requires git-cinnabar to be installed when
  26 pushing from git. Please install it by running:
  27
  28     $ ./mach vcs-setup
  29 """.lstrip()
  30
  31 HG_PUSH_TO_TRY_NOT_FOUND = """
  32 Could not detect `push-to-try`.
  33
  34 The `mach try` command requires the push-to-try extension enabled
  35 when pushing from hg. Please install it by running:
  36
  37     $ ./mach vcs-setup
  38 """.lstrip()
  39
  40 VCS_NOT_FOUND = """
  41 Could not detect version control. Only `hg` or `git` are supported.
  42 """.strip()
  43
  44 UNCOMMITTED_CHANGES = """
  45 ERROR please commit changes before continuing
  46 """.strip()
  47
  48 MAX_HISTORY = 10
  49
  50 here = os.path.abspath(os.path.dirname(__file__))
  51 build = MozbuildObject.from_environment(cwd=here)
  52 vcs = get_repository_object(build.topsrcdir)
  53
  54 history_path = os.path.join(
  55     get_state_dir(specific_to_topsrcdir=True), "history", "try_task_configs.json"
  56 )
  57
  58
  59 def write_task_config(try_task_config):
  60     config_path = os.path.join(vcs.path, "try_task_config.json")
  61     with open(config_path, "w") as fh:
  62         json.dump(try_task_config, fh, indent=4, separators=(",", ": "), sort_keys=True)
  63         fh.write("\n")
  64     return config_path
  65
  66
  67 def write_task_config_history(msg, try_task_config):
  68     if not os.path.isfile(history_path):
  69         if not os.path.isdir(os.path.dirname(history_path)):
  70             os.makedirs(os.path.dirname(history_path))
  71         history = []
  72     else:
  73         with open(history_path) as fh:
  74             history = fh.read().strip().splitlines()
  75
  76     history.insert(0, json.dumps([msg, try_task_config]))
  77     history = history[:MAX_HISTORY]
  78     with open(history_path, "w") as fh:
  79         fh.write("\n".join(history))
  80
  81
  82 def check_working_directory(push=True):
  83     if not push:
  84         return
  85
  86     if not vcs.working_directory_clean():
  87         print(UNCOMMITTED_CHANGES)
  88         sys.exit(1)
  89
  90
  91 def generate_try_task_config(method, labels, try_config=None, routes=None):
  92     try_task_config = try_config or {}
  93     try_task_config.setdefault("env", {})["TRY_SELECTOR"] = method
  94     try_task_config.update(
  95         {
  96             "version": 1,
  97             "tasks": sorted(labels),
  98         }
  99     )
 100     if routes:
 101         try_task_config["routes"] = routes
 102
 103     return try_task_config
 104
 105
 106 def task_labels_from_try_config(try_task_config):
 107     if try_task_config["version"] == 2:
 108         parameters = try_task_config.get("parameters", {})
 109         if parameters.get("try_mode") == "try_task_config":
 110             return parameters["try_task_config"]["tasks"]
 111         else:
 112             return None
 113     elif try_task_config["version"] == 1:
 114         return try_task_config.get("tasks", list())
 115     else:
 116         return None
 117
 118
 119 def display_push_estimates(try_task_config):
 120     task_labels = task_labels_from_try_config(try_task_config)
 121     if task_labels is None:
 122         return
 123
 124     cache_dir = os.path.join(
 125         get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph"
 126     )
 127
 128     graph_cache = None
 129     dep_cache = None
 130     target_file = None
 131     for graph_cache_file in ["target_task_graph", "full_task_graph"]:
 132         graph_cache = os.path.join(cache_dir, graph_cache_file)
 133         if os.path.isfile(graph_cache):
 134             dep_cache = graph_cache.replace("task_graph", "task_dependencies")
 135             target_file = graph_cache.replace("task_graph", "task_set")
 136             break
 137
 138     if not dep_cache:
 139         return
 140
 141     download_task_history_data(cache_dir=cache_dir)
 142     make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_file)
 143
 144     durations = duration_summary(dep_cache, task_labels, cache_dir)
 145
 146     print(
 147         "estimates: Runs {} tasks ({} selected, {} dependencies)".format(
 148             durations["dependency_count"] + durations["selected_count"],
 149             durations["selected_count"],
 150             durations["dependency_count"],
 151         )
 152     )
 153     print(
 154         "estimates: Total task duration {}".format(
 155             durations["dependency_duration"] + durations["selected_duration"]
 156         )
 157     )
 158     if "percentile" in durations:
 159         print(
 160             "estimates: In the top {}% of durations".format(
 161                 100 - durations["percentile"]
 162             )
 163         )
 164     print(
 165         "estimates: Should take about {} (Finished around {})".format(
 166             durations["wall_duration_seconds"],
 167             durations["eta_datetime"].strftime("%Y-%m-%d %H:%M"),
 168         )
 169     )
 170
 171
 172 def push_to_try(
 173     method,
 174     msg,
 175     try_task_config=None,
 176     stage_changes=False,
 177     dry_run=False,
 178     closed_tree=False,
 179     files_to_change=None,
 180     allow_log_capture=False,
 181 ):
 182     push = not stage_changes and not dry_run
 183     check_working_directory(push)
 184
 185     if try_task_config and method not in ("auto", "empty"):
 186         try:
 187             display_push_estimates(try_task_config)
 188         except Exception:
 189             traceback.print_exc()
 190             print("warning: unable to display push estimates")
 191
 192     # Format the commit message
 193     closed_tree_string = " ON A CLOSED TREE" if closed_tree else ""
 194     commit_message = "{}{}\n\nPushed via `mach try {}`".format(
 195         msg,
 196         closed_tree_string,
 197         method,
 198     )
 199
 200     config_path = None
 201     changed_files = []
 202     if try_task_config:
 203         if push and method not in ("again", "auto", "empty"):
 204             write_task_config_history(msg, try_task_config)
 205         config_path = write_task_config(try_task_config)
 206         changed_files.append(config_path)
 207
 208     if (push or stage_changes) and files_to_change:
 209         for path, content in files_to_change.items():
 210             path = os.path.join(vcs.path, path)
 211             with open(path, "wb") as fh:
 212                 fh.write(six.ensure_binary(content))
 213             changed_files.append(path)
 214
 215     try:
 216         if not push:
 217             print("Commit message:")
 218             print(commit_message)
 219             if config_path:
 220                 print("Calculated try_task_config.json:")
 221                 with open(config_path) as fh:
 222                     print(fh.read())
 223             return
 224
 225         vcs.add_remove_files(*changed_files)
 226
 227         try:
 228             vcs.push_to_try(commit_message, allow_log_capture=allow_log_capture)
 229         except MissingVCSExtension as e:
 230             if e.ext == "push-to-try":
 231                 print(HG_PUSH_TO_TRY_NOT_FOUND)
 232             elif e.ext == "cinnabar":
 233                 print(GIT_CINNABAR_NOT_FOUND)
 234             else:
 235                 raise
 236             sys.exit(1)
 237     finally:
 238         if config_path and os.path.isfile(config_path):
 239             os.remove(config_path)