taskcluster/gecko_taskgraph/decision.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5
   6 import json
   7 import logging
   8 import os
   9 import shutil
  10 import sys
  11 import time
  12 from collections import defaultdict
  13
  14 import yaml
  15 from redo import retry
  16 from taskgraph import create
  17 from taskgraph.create import create_tasks
  18
  19 # TODO: Let standalone taskgraph generate parameters instead of calling internals
  20 from taskgraph.decision import (
  21     _determine_more_accurate_base_ref,
  22     _determine_more_accurate_base_rev,
  23     _get_env_prefix,
  24 )
  25 from taskgraph.generator import TaskGraphGenerator
  26 from taskgraph.parameters import Parameters
  27 from taskgraph.taskgraph import TaskGraph
  28 from taskgraph.util.python_path import find_object
  29 from taskgraph.util.schema import Schema, validate_schema
  30 from taskgraph.util.taskcluster import get_artifact
  31 from taskgraph.util.vcs import get_repository
  32 from taskgraph.util.yaml import load_yaml
  33 from voluptuous import Any, Optional, Required
  34
  35 from . import GECKO
  36 from .actions import render_actions_json
  37 from .parameters import get_app_version, get_version
  38 from .try_option_syntax import parse_message
  39 from .util.backstop import BACKSTOP_INDEX, is_backstop
  40 from .util.bugbug import push_schedules
  41 from .util.chunking import resolver
  42 from .util.hg import get_hg_commit_message, get_hg_revision_branch
  43 from .util.partials import populate_release_history
  44 from .util.taskcluster import insert_index
  45 from .util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds
  46
  47 logger = logging.getLogger(__name__)
  48
  49 ARTIFACTS_DIR = "artifacts"
  50
  51 # For each project, this gives a set of parameters specific to the project.
  52 # See `taskcluster/docs/parameters.rst` for information on parameters.
  53 PER_PROJECT_PARAMETERS = {
  54     "try": {
  55         "enable_always_target": True,
  56         "target_tasks_method": "try_tasks",
  57     },
  58     "kaios-try": {
  59         "target_tasks_method": "try_tasks",
  60     },
  61     "ash": {
  62         "target_tasks_method": "default",
  63     },
  64     "cedar": {
  65         "target_tasks_method": "default",
  66     },
  67     "holly": {
  68         "enable_always_target": True,
  69         "target_tasks_method": "holly_tasks",
  70     },
  71     "oak": {
  72         "target_tasks_method": "default",
  73         "release_type": "nightly-oak",
  74     },
  75     "graphics": {
  76         "target_tasks_method": "graphics_tasks",
  77     },
  78     "autoland": {
  79         "optimize_strategies": "gecko_taskgraph.optimize:project.autoland",
  80         "target_tasks_method": "autoland_tasks",
  81         "test_manifest_loader": "bugbug",  # Remove this line to disable "manifest scheduling".
  82     },
  83     "mozilla-central": {
  84         "target_tasks_method": "mozilla_central_tasks",
  85         "release_type": "nightly",
  86     },
  87     "mozilla-beta": {
  88         "target_tasks_method": "mozilla_beta_tasks",
  89         "release_type": "beta",
  90     },
  91     "mozilla-release": {
  92         "target_tasks_method": "mozilla_release_tasks",
  93         "release_type": "release",
  94     },
  95     "mozilla-esr102": {
  96         "target_tasks_method": "mozilla_esr102_tasks",
  97         "release_type": "esr102",
  98     },
  99     "mozilla-esr115": {
 100         "target_tasks_method": "mozilla_esr115_tasks",
 101         "release_type": "esr115",
 102     },
 103     "pine": {
 104         "target_tasks_method": "pine_tasks",
 105     },
 106     "kaios": {
 107         "target_tasks_method": "kaios_tasks",
 108     },
 109     "toolchains": {
 110         "target_tasks_method": "mozilla_central_tasks",
 111     },
 112     # the default parameters are used for projects that do not match above.
 113     "default": {
 114         "target_tasks_method": "default",
 115     },
 116 }
 117
 118 try_task_config_schema = Schema(
 119     {
 120         Required("tasks"): [str],
 121         Optional("browsertime"): bool,
 122         Optional("chemspill-prio"): bool,
 123         Optional("disable-pgo"): bool,
 124         Optional("env"): {str: str},
 125         Optional("gecko-profile"): bool,
 126         Optional("gecko-profile-interval"): float,
 127         Optional("gecko-profile-entries"): int,
 128         Optional("gecko-profile-features"): str,
 129         Optional("gecko-profile-threads"): str,
 130         Optional(
 131             "perftest-options",
 132             description="Options passed from `mach perftest` to try.",
 133         ): object,
 134         Optional(
 135             "optimize-strategies",
 136             description="Alternative optimization strategies to use instead of the default. "
 137             "A module path pointing to a dict to be use as the `strategy_override` "
 138             "argument in `taskgraph.optimize.base.optimize_task_graph`.",
 139         ): str,
 140         Optional("rebuild"): int,
 141         Optional("tasks-regex"): {
 142             "include": Any(None, [str]),
 143             "exclude": Any(None, [str]),
 144         },
 145         Optional("use-artifact-builds"): bool,
 146         Optional(
 147             "worker-overrides",
 148             description="Mapping of worker alias to worker pools to use for those aliases.",
 149         ): {str: str},
 150         Optional("routes"): [str],
 151     }
 152 )
 153 """
 154 Schema for try_task_config.json files.
 155 """
 156
 157 try_task_config_schema_v2 = Schema(
 158     {
 159         Optional("parameters"): {str: object},
 160     }
 161 )
 162
 163
 164 def full_task_graph_to_runnable_jobs(full_task_json):
 165     runnable_jobs = {}
 166     for label, node in full_task_json.items():
 167         if not ("extra" in node["task"] and "treeherder" in node["task"]["extra"]):
 168             continue
 169
 170         th = node["task"]["extra"]["treeherder"]
 171         runnable_jobs[label] = {"symbol": th["symbol"]}
 172
 173         for i in ("groupName", "groupSymbol", "collection"):
 174             if i in th:
 175                 runnable_jobs[label][i] = th[i]
 176         if th.get("machine", {}).get("platform"):
 177             runnable_jobs[label]["platform"] = th["machine"]["platform"]
 178     return runnable_jobs
 179
 180
 181 def full_task_graph_to_manifests_by_task(full_task_json):
 182     manifests_by_task = defaultdict(list)
 183     for label, node in full_task_json.items():
 184         manifests = node["attributes"].get("test_manifests")
 185         if not manifests:
 186             continue
 187
 188         manifests_by_task[label].extend(manifests)
 189     return manifests_by_task
 190
 191
 192 def try_syntax_from_message(message):
 193     """
 194     Parse the try syntax out of a commit message, returning '' if none is
 195     found.
 196     """
 197     try_idx = message.find("try:")
 198     if try_idx == -1:
 199         return ""
 200     return message[try_idx:].split("\n", 1)[0]
 201
 202
 203 def taskgraph_decision(options, parameters=None):
 204     """
 205     Run the decision task.  This function implements `mach taskgraph decision`,
 206     and is responsible for
 207
 208      * processing decision task command-line options into parameters
 209      * running task-graph generation exactly the same way the other `mach
 210        taskgraph` commands do
 211      * generating a set of artifacts to memorialize the graph
 212      * calling TaskCluster APIs to create the graph
 213     """
 214
 215     parameters = parameters or (
 216         lambda graph_config: get_decision_parameters(graph_config, options)
 217     )
 218
 219     decision_task_id = os.environ["TASK_ID"]
 220
 221     # create a TaskGraphGenerator instance
 222     tgg = TaskGraphGenerator(
 223         root_dir=options.get("root"),
 224         parameters=parameters,
 225         decision_task_id=decision_task_id,
 226         write_artifacts=True,
 227     )
 228
 229     if not create.testing:
 230         # set additional index paths for the decision task
 231         set_decision_indexes(decision_task_id, tgg.parameters, tgg.graph_config)
 232
 233     # write out the parameters used to generate this graph
 234     write_artifact("parameters.yml", dict(**tgg.parameters))
 235
 236     # write out the public/actions.json file
 237     write_artifact(
 238         "actions.json",
 239         render_actions_json(tgg.parameters, tgg.graph_config, decision_task_id),
 240     )
 241
 242     # write out the full graph for reference
 243     full_task_json = tgg.full_task_graph.to_json()
 244     write_artifact("full-task-graph.json", full_task_json)
 245
 246     # write out the public/runnable-jobs.json file
 247     write_artifact(
 248         "runnable-jobs.json", full_task_graph_to_runnable_jobs(full_task_json)
 249     )
 250
 251     # write out the public/manifests-by-task.json file
 252     write_artifact(
 253         "manifests-by-task.json.gz",
 254         full_task_graph_to_manifests_by_task(full_task_json),
 255     )
 256
 257     # write out the public/tests-by-manifest.json file
 258     write_artifact("tests-by-manifest.json.gz", resolver.tests_by_manifest)
 259
 260     # this is just a test to check whether the from_json() function is working
 261     _, _ = TaskGraph.from_json(full_task_json)
 262
 263     # write out the target task set to allow reproducing this as input
 264     write_artifact("target-tasks.json", list(tgg.target_task_set.tasks.keys()))
 265
 266     # write out the optimized task graph to describe what will actually happen,
 267     # and the map of labels to taskids
 268     write_artifact("task-graph.json", tgg.morphed_task_graph.to_json())
 269     write_artifact("label-to-taskid.json", tgg.label_to_taskid)
 270
 271     # write bugbug scheduling information if it was invoked
 272     if len(push_schedules) > 0:
 273         write_artifact("bugbug-push-schedules.json", push_schedules.popitem()[1])
 274
 275     # cache run-task & misc/fetch-content
 276     scripts_root_dir = os.path.join(
 277         "/builds/worker/checkouts/gecko/taskcluster/scripts"
 278     )
 279     run_task_file_path = os.path.join(scripts_root_dir, "run-task")
 280     fetch_content_file_path = os.path.join(scripts_root_dir, "misc/fetch-content")
 281     shutil.copy2(run_task_file_path, ARTIFACTS_DIR)
 282     shutil.copy2(fetch_content_file_path, ARTIFACTS_DIR)
 283
 284     # actually create the graph
 285     create_tasks(
 286         tgg.graph_config,
 287         tgg.morphed_task_graph,
 288         tgg.label_to_taskid,
 289         tgg.parameters,
 290         decision_task_id=decision_task_id,
 291     )
 292
 293
 294 def get_decision_parameters(graph_config, options):
 295     """
 296     Load parameters from the command-line options for 'taskgraph decision'.
 297     This also applies per-project parameters, based on the given project.
 298
 299     """
 300     product_dir = graph_config["product-dir"]
 301
 302     parameters = {
 303         n: options[n]
 304         for n in [
 305             "base_repository",
 306             "base_ref",
 307             "base_rev",
 308             "head_repository",
 309             "head_rev",
 310             "head_ref",
 311             "head_tag",
 312             "project",
 313             "pushlog_id",
 314             "pushdate",
 315             "owner",
 316             "level",
 317             "repository_type",
 318             "target_tasks_method",
 319             "tasks_for",
 320         ]
 321         if n in options
 322     }
 323
 324     commit_message = get_hg_commit_message(os.path.join(GECKO, product_dir))
 325
 326     repo_path = os.getcwd()
 327     repo = get_repository(repo_path)
 328     parameters["base_ref"] = _determine_more_accurate_base_ref(
 329         repo,
 330         candidate_base_ref=options.get("base_ref"),
 331         head_ref=options.get("head_ref"),
 332         base_rev=options.get("base_rev"),
 333     )
 334
 335     parameters["base_rev"] = _determine_more_accurate_base_rev(
 336         repo,
 337         base_ref=parameters["base_ref"],
 338         candidate_base_rev=options.get("base_rev"),
 339         head_rev=options.get("head_rev"),
 340         env_prefix=_get_env_prefix(graph_config),
 341     )
 342
 343     # Define default filter list, as most configurations shouldn't need
 344     # custom filters.
 345     parameters["filters"] = [
 346         "target_tasks_method",
 347     ]
 348     parameters["enable_always_target"] = False
 349     parameters["existing_tasks"] = {}
 350     parameters["do_not_optimize"] = []
 351     parameters["build_number"] = 1
 352     parameters["version"] = get_version(product_dir)
 353     parameters["app_version"] = get_app_version(product_dir)
 354     parameters["message"] = try_syntax_from_message(commit_message)
 355     parameters["hg_branch"] = get_hg_revision_branch(
 356         GECKO, revision=parameters["head_rev"]
 357     )
 358     parameters["next_version"] = None
 359     parameters["optimize_strategies"] = None
 360     parameters["optimize_target_tasks"] = True
 361     parameters["phabricator_diff"] = None
 362     parameters["release_type"] = ""
 363     parameters["release_eta"] = ""
 364     parameters["release_enable_partner_repack"] = False
 365     parameters["release_enable_partner_attribution"] = False
 366     parameters["release_partners"] = []
 367     parameters["release_partner_config"] = {}
 368     parameters["release_partner_build_number"] = 1
 369     parameters["release_enable_emefree"] = False
 370     parameters["release_product"] = None
 371     parameters["required_signoffs"] = []
 372     parameters["signoff_urls"] = {}
 373     parameters["test_manifest_loader"] = "default"
 374     parameters["try_mode"] = None
 375     parameters["try_task_config"] = {}
 376     parameters["try_options"] = None
 377
 378     # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
 379     # case, fake it
 380     if "@" not in parameters["owner"]:
 381         parameters["owner"] += "@noreply.mozilla.org"
 382
 383     # use the pushdate as build_date if given, else use current time
 384     parameters["build_date"] = parameters["pushdate"] or int(time.time())
 385     # moz_build_date is the build identifier based on build_date
 386     parameters["moz_build_date"] = time.strftime(
 387         "%Y%m%d%H%M%S", time.gmtime(parameters["build_date"])
 388     )
 389
 390     project = parameters["project"]
 391     try:
 392         parameters.update(PER_PROJECT_PARAMETERS[project])
 393     except KeyError:
 394         logger.warning(
 395             "using default project parameters; add {} to "
 396             "PER_PROJECT_PARAMETERS in {} to customize behavior "
 397             "for this project".format(project, __file__)
 398         )
 399         parameters.update(PER_PROJECT_PARAMETERS["default"])
 400
 401     # `target_tasks_method` has higher precedence than `project` parameters
 402     if options.get("target_tasks_method"):
 403         parameters["target_tasks_method"] = options["target_tasks_method"]
 404
 405     # ..but can be overridden by the commit message: if it contains the special
 406     # string "DONTBUILD" and this is an on-push decision task, then use the
 407     # special 'nothing' target task method.
 408     if "DONTBUILD" in commit_message and options["tasks_for"] == "hg-push":
 409         parameters["target_tasks_method"] = "nothing"
 410
 411     if options.get("include_push_tasks"):
 412         get_existing_tasks(options.get("rebuild_kinds", []), parameters, graph_config)
 413
 414     # If the target method is nightly, we should build partials. This means
 415     # knowing what has been released previously.
 416     # An empty release_history is fine, it just means no partials will be built
 417     parameters.setdefault("release_history", dict())
 418     if "nightly" in parameters.get("target_tasks_method", ""):
 419         parameters["release_history"] = populate_release_history("Firefox", project)
 420
 421     if options.get("try_task_config_file"):
 422         task_config_file = os.path.abspath(options.get("try_task_config_file"))
 423     else:
 424         # if try_task_config.json is present, load it
 425         task_config_file = os.path.join(os.getcwd(), "try_task_config.json")
 426
 427     # load try settings
 428     if "try" in project and options["tasks_for"] == "hg-push":
 429         set_try_config(parameters, task_config_file)
 430
 431     if options.get("optimize_target_tasks") is not None:
 432         parameters["optimize_target_tasks"] = options["optimize_target_tasks"]
 433
 434     # Determine if this should be a backstop push.
 435     parameters["backstop"] = is_backstop(parameters)
 436
 437     if "decision-parameters" in graph_config["taskgraph"]:
 438         find_object(graph_config["taskgraph"]["decision-parameters"])(
 439             graph_config, parameters
 440         )
 441
 442     result = Parameters(**parameters)
 443     result.check()
 444     return result
 445
 446
 447 def get_existing_tasks(rebuild_kinds, parameters, graph_config):
 448     """
 449     Find the decision task corresponding to the on-push graph, and return
 450     a mapping of labels to task-ids from it. This will skip the kinds specificed
 451     by `rebuild_kinds`.
 452     """
 453     try:
 454         decision_task = retry(
 455             find_decision_task,
 456             args=(parameters, graph_config),
 457             attempts=4,
 458             sleeptime=5 * 60,
 459         )
 460     except Exception:
 461         logger.exception("Didn't find existing push task.")
 462         sys.exit(1)
 463     _, task_graph = TaskGraph.from_json(
 464         get_artifact(decision_task, "public/full-task-graph.json")
 465     )
 466     parameters["existing_tasks"] = find_existing_tasks_from_previous_kinds(
 467         task_graph, [decision_task], rebuild_kinds
 468     )
 469
 470
 471 def set_try_config(parameters, task_config_file):
 472     if os.path.isfile(task_config_file):
 473         logger.info(f"using try tasks from {task_config_file}")
 474         with open(task_config_file) as fh:
 475             task_config = json.load(fh)
 476         task_config_version = task_config.pop("version", 1)
 477         if task_config_version == 1:
 478             validate_schema(
 479                 try_task_config_schema,
 480                 task_config,
 481                 "Invalid v1 `try_task_config.json`.",
 482             )
 483             parameters["try_mode"] = "try_task_config"
 484             parameters["try_task_config"] = task_config
 485         elif task_config_version == 2:
 486             validate_schema(
 487                 try_task_config_schema_v2,
 488                 task_config,
 489                 "Invalid v2 `try_task_config.json`.",
 490             )
 491             parameters.update(task_config["parameters"])
 492             return
 493         else:
 494             raise Exception(
 495                 f"Unknown `try_task_config.json` version: {task_config_version}"
 496             )
 497
 498     if "try:" in parameters["message"]:
 499         parameters["try_mode"] = "try_option_syntax"
 500         parameters.update(parse_message(parameters["message"]))
 501     else:
 502         parameters["try_options"] = None
 503
 504     if parameters["try_mode"] == "try_task_config":
 505         # The user has explicitly requested a set of jobs, so run them all
 506         # regardless of optimization.  Their dependencies can be optimized,
 507         # though.
 508         parameters["optimize_target_tasks"] = False
 509     else:
 510         # For a try push with no task selection, apply the default optimization
 511         # process to all of the tasks.
 512         parameters["optimize_target_tasks"] = True
 513
 514
 515 def set_decision_indexes(decision_task_id, params, graph_config):
 516     index_paths = []
 517     if params["backstop"]:
 518         index_paths.append(BACKSTOP_INDEX)
 519
 520     subs = params.copy()
 521     subs["trust-domain"] = graph_config["trust-domain"]
 522
 523     index_paths = [i.format(**subs) for i in index_paths]
 524     for index_path in index_paths:
 525         insert_index(index_path, decision_task_id, use_proxy=True)
 526
 527
 528 def write_artifact(filename, data):
 529     logger.info(f"writing artifact file `{filename}`")
 530     if not os.path.isdir(ARTIFACTS_DIR):
 531         os.mkdir(ARTIFACTS_DIR)
 532     path = os.path.join(ARTIFACTS_DIR, filename)
 533     if filename.endswith(".yml"):
 534         with open(path, "w") as f:
 535             yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False)
 536     elif filename.endswith(".json"):
 537         with open(path, "w") as f:
 538             json.dump(data, f, sort_keys=True, indent=2, separators=(",", ": "))
 539     elif filename.endswith(".json.gz"):
 540         import gzip
 541
 542         with gzip.open(path, "wb") as f:
 543             f.write(json.dumps(data).encode("utf-8"))
 544     else:
 545         raise TypeError(f"Don't know how to write to {filename}")
 546
 547
 548 def read_artifact(filename):
 549     path = os.path.join(ARTIFACTS_DIR, filename)
 550     if filename.endswith(".yml"):
 551         return load_yaml(path, filename)
 552     if filename.endswith(".json"):
 553         with open(path) as f:
 554             return json.load(f)
 555     if filename.endswith(".json.gz"):
 556         import gzip
 557
 558         with gzip.open(path, "rb") as f:
 559             return json.load(f.decode("utf-8"))
 560     else:
 561         raise TypeError(f"Don't know how to read {filename}")
 562
 563
 564 def rename_artifact(src, dest):
 565     os.rename(os.path.join(ARTIFACTS_DIR, src), os.path.join(ARTIFACTS_DIR, dest))