tools/tryselect/push.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function
   6
   7 import json
   8 import os
   9 import sys
  10
  11 import six
  12 from mozboot.util import get_state_dir
  13 from mozbuild.base import MozbuildObject
  14 from mozversioncontrol import get_repository_object, MissingVCSExtension
  15 from .util.manage_estimates import (
  16     download_task_history_data,
  17     make_trimmed_taskgraph_cache
  18 )
  19 from .util.estimates import duration_summary
  20
  21 GIT_CINNABAR_NOT_FOUND = """
  22 Could not detect `git-cinnabar`.
  23
  24 The `mach try` command requires git-cinnabar to be installed when
  25 pushing from git. Please install it by running:
  26
  27     $ ./mach vcs-setup
  28 """.lstrip()
  29
  30 HG_PUSH_TO_TRY_NOT_FOUND = """
  31 Could not detect `push-to-try`.
  32
  33 The `mach try` command requires the push-to-try extension enabled
  34 when pushing from hg. Please install it by running:
  35
  36     $ ./mach vcs-setup
  37 """.lstrip()
  38
  39 VCS_NOT_FOUND = """
  40 Could not detect version control. Only `hg` or `git` are supported.
  41 """.strip()
  42
  43 UNCOMMITTED_CHANGES = """
  44 ERROR please commit changes before continuing
  45 """.strip()
  46
  47 MAX_HISTORY = 10
  48
  49 here = os.path.abspath(os.path.dirname(__file__))
  50 build = MozbuildObject.from_environment(cwd=here)
  51 vcs = get_repository_object(build.topsrcdir)
  52
  53 history_path = os.path.join(get_state_dir(srcdir=True), 'history', 'try_task_configs.json')
  54
  55
  56 def write_task_config(try_task_config):
  57     config_path = os.path.join(vcs.path, 'try_task_config.json')
  58     with open(config_path, 'w') as fh:
  59         json.dump(try_task_config, fh, indent=4, separators=(',', ': '), sort_keys=True)
  60         fh.write('\n')
  61     return config_path
  62
  63
  64 def write_task_config_history(msg, try_task_config):
  65     if not os.path.isfile(history_path):
  66         if not os.path.isdir(os.path.dirname(history_path)):
  67             os.makedirs(os.path.dirname(history_path))
  68         history = []
  69     else:
  70         with open(history_path, 'r') as fh:
  71             history = fh.read().strip().splitlines()
  72
  73     history.insert(0, json.dumps([msg, try_task_config]))
  74     history = history[:MAX_HISTORY]
  75     with open(history_path, 'w') as fh:
  76         fh.write('\n'.join(history))
  77
  78
  79 def check_working_directory(push=True):
  80     if not push:
  81         return
  82
  83     if not vcs.working_directory_clean():
  84         print(UNCOMMITTED_CHANGES)
  85         sys.exit(1)
  86
  87
  88 def generate_try_task_config(method, labels, try_config=None, routes=None):
  89     try_task_config = try_config or {}
  90     try_task_config.setdefault('env', {})['TRY_SELECTOR'] = method
  91     try_task_config.update({
  92         'version': 1,
  93         'tasks': sorted(labels),
  94     })
  95     if routes:
  96         try_task_config["routes"] = routes
  97
  98     return try_task_config
  99
 100
 101 def task_labels_from_try_config(try_task_config):
 102     if try_task_config['version'] == 2:
 103         parameters = try_task_config.get('parameters', {})
 104         if parameters.get('try_mode') == 'try_task_config':
 105             return parameters['try_task_config']['tasks']
 106         else:
 107             return None
 108     elif try_task_config['version'] == 1:
 109         return try_task_config.get("tasks", list())
 110     else:
 111         return None
 112
 113
 114 def display_push_estimates(try_task_config):
 115     task_labels = task_labels_from_try_config(try_task_config)
 116     if task_labels is None:
 117         return
 118
 119     cache_dir = os.path.join(get_state_dir(srcdir=True), 'cache', 'taskgraph')
 120
 121     graph_cache = None
 122     dep_cache = None
 123     target_file = None
 124     for graph_cache_file in ["target_task_graph", "full_task_graph"]:
 125         graph_cache = os.path.join(cache_dir, graph_cache_file)
 126         if os.path.isfile(graph_cache):
 127             dep_cache = graph_cache.replace("task_graph", "task_dependencies")
 128             target_file = graph_cache.replace("task_graph", "task_set")
 129             break
 130
 131     if not dep_cache:
 132         return
 133
 134     download_task_history_data(cache_dir=cache_dir)
 135     make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_file)
 136
 137     durations = duration_summary(
 138         dep_cache, task_labels, cache_dir)
 139
 140     print("estimates: Runs {} tasks ({} selected, {} dependencies)".format(
 141         durations["dependency_count"] + durations["selected_count"],
 142         durations["selected_count"],
 143         durations["dependency_count"])
 144     )
 145     print("estimates: Total task duration {}".format(
 146         durations["dependency_duration"] + durations["selected_duration"]
 147     ))
 148     print("estimates: In the {}% percentile".format(durations["quantile"]))
 149     print("estimates: Should take about {} (Finished around {})".format(
 150         durations["wall_duration_seconds"],
 151         durations["eta_datetime"].strftime("%Y-%m-%d %H:%M"))
 152     )
 153
 154
 155 def push_to_try(method, msg, try_task_config=None,
 156                 push=True, closed_tree=False, files_to_change=None):
 157     check_working_directory(push)
 158
 159     if try_task_config and method not in ('auto', 'empty'):
 160         display_push_estimates(try_task_config)
 161
 162     # Format the commit message
 163     closed_tree_string = " ON A CLOSED TREE" if closed_tree else ""
 164     commit_message = ('%s%s\n\nPushed via `mach try %s`' %
 165                       (msg, closed_tree_string, method))
 166
 167     config_path = None
 168     changed_files = []
 169     if try_task_config:
 170         if push and method not in ('again', 'auto', 'empty'):
 171             write_task_config_history(msg, try_task_config)
 172         config_path = write_task_config(try_task_config)
 173         changed_files.append(config_path)
 174
 175     if files_to_change:
 176         for path, content in files_to_change.items():
 177             path = os.path.join(vcs.path, path)
 178             with open(path, 'wb') as fh:
 179                 fh.write(six.ensure_binary(content))
 180             changed_files.append(path)
 181
 182     try:
 183         if not push:
 184             print("Commit message:")
 185             print(commit_message)
 186             if config_path:
 187                 print("Calculated try_task_config.json:")
 188                 with open(config_path) as fh:
 189                     print(fh.read())
 190             return
 191
 192         vcs.add_remove_files(*changed_files)
 193
 194         try:
 195             vcs.push_to_try(commit_message)
 196         except MissingVCSExtension as e:
 197             if e.ext == 'push-to-try':
 198                 print(HG_PUSH_TO_TRY_NOT_FOUND)
 199             elif e.ext == 'cinnabar':
 200                 print(GIT_CINNABAR_NOT_FOUND)
 201             else:
 202                 raise
 203             sys.exit(1)
 204     finally:
 205         if config_path and os.path.isfile(config_path):
 206             os.remove(config_path)