build/moz.configure/bootstrap.configure

   1 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
   2 # vim: set filetype=python:
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 option(
   8     env="MOZ_FETCHES_DIR",
   9     nargs=1,
  10     when="MOZ_AUTOMATION",
  11     help="Directory containing fetched artifacts",
  12 )
  13
  14
  15 @depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION")
  16 def moz_fetches_dir(value):
  17     if value:
  18         return value[0]
  19
  20
  21 @depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION")
  22 def bootstrap_default(vcs_checkout_type, is_nightly, automation):
  23     if automation:
  24         return False
  25     # We only enable if building off a VCS checkout of central.
  26     if is_nightly and vcs_checkout_type:
  27         return True
  28
  29
  30 option(
  31     "--enable-bootstrap",
  32     nargs="*",
  33     default=bootstrap_default,
  34     help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}",
  35 )
  36
  37
  38 @depends_if("--enable-bootstrap")
  39 def enable_bootstrap(bootstrap):
  40     include = set()
  41     exclude = set()
  42     for item in bootstrap:
  43         if item.startswith("-"):
  44             exclude.add(item.lstrip("-"))
  45         else:
  46             include.add(item)
  47
  48     def match(name):
  49         if name in exclude:
  50             return False
  51         if include and name in include:
  52             return True
  53         return not bool(include)
  54
  55     return match
  56
  57
  58 @depends(developer_options, "--enable-bootstrap", moz_fetches_dir)
  59 def bootstrap_search_path_order(developer_options, bootstrap, moz_fetches_dir):
  60     if moz_fetches_dir:
  61         log.debug("Prioritizing MOZ_FETCHES_DIR in toolchain path.")
  62         return "prepend"
  63
  64     if bootstrap:
  65         log.debug(
  66             "Prioritizing mozbuild state dir in toolchain paths because "
  67             "bootstrap mode is enabled."
  68         )
  69         return "maybe-prepend"
  70
  71     if developer_options:
  72         log.debug(
  73             "Prioritizing mozbuild state dir in toolchain paths because "
  74             "you are not building in release mode."
  75         )
  76         return "prepend"
  77
  78     log.debug(
  79         "Prioritizing system over mozbuild state dir in "
  80         "toolchain paths because you are building in "
  81         "release mode."
  82     )
  83     return "append"
  84
  85
  86 toolchains_base_dir = moz_fetches_dir | mozbuild_state_path
  87
  88
  89 @dependable
  90 @imports("os")
  91 @imports(_from="os", _import="environ")
  92 def original_path():
  93     return environ["PATH"].split(os.pathsep)
  94
  95
  96 @depends(host, when="--enable-bootstrap")
  97 @imports("os")
  98 @imports("traceback")
  99 @imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
 100 @imports(_from="__builtin__", _import="Exception")
 101 def bootstrap_toolchain_tasks(host):
 102     prefix = {
 103         ("x86_64", "GNU", "Linux"): "linux64",
 104         ("x86_64", "OSX", "Darwin"): "macosx64",
 105         ("aarch64", "OSX", "Darwin"): "macosx64-aarch64",
 106         ("x86_64", "WINNT", "WINNT"): "win64",
 107         ("aarch64", "WINNT", "WINNT"): "win64-aarch64",
 108     }.get((host.cpu, host.os, host.kernel))
 109     try:
 110         tasks = toolchain_task_definitions()
 111     except Exception as e:
 112         message = traceback.format_exc()
 113         log.warning(str(e))
 114         log.debug(message)
 115         return None
 116
 117     def task_data(t):
 118         result = {
 119             "index": t.optimization["index-search"],
 120             "artifact": t.attributes["toolchain-artifact"],
 121         }
 122         command = t.attributes.get("toolchain-command")
 123         if command:
 124             result["command"] = command
 125         return result
 126
 127     # We only want to use toolchains annotated with "local-toolchain". We also limit the
 128     # amount of data to what we use, so that trace logs can be more useful.
 129     tasks = {
 130         k: task_data(t)
 131         for k, t in tasks.items()
 132         if t.attributes.get("local-toolchain") and "index-search" in t.optimization
 133     }
 134
 135     return namespace(prefix=prefix, tasks=tasks)
 136
 137
 138 @template
 139 def bootstrap_path(path, **kwargs):
 140     when = kwargs.pop("when", None)
 141     allow_failure = kwargs.pop("allow_failure", None)
 142     if kwargs:
 143         configure_error(
 144             "bootstrap_path only takes `when` and `allow_failure` as a keyword argument"
 145         )
 146
 147     @depends(
 148         enable_bootstrap,
 149         toolchains_base_dir,
 150         moz_fetches_dir,
 151         bootstrap_toolchain_tasks,
 152         build_environment,
 153         dependable(path),
 154         dependable(allow_failure),
 155         when=when,
 156     )
 157     @imports("os")
 158     @imports("subprocess")
 159     @imports("sys")
 160     @imports(_from="mozbuild.util", _import="ensureParentDir")
 161     @imports(_from="importlib", _import="import_module")
 162     @imports(_from="shutil", _import="rmtree")
 163     @imports(_from="__builtin__", _import="open")
 164     @imports(_from="__builtin__", _import="Exception")
 165     def bootstrap_path(
 166         bootstrap,
 167         toolchains_base_dir,
 168         moz_fetches_dir,
 169         tasks,
 170         build_env,
 171         path,
 172         allow_failure,
 173     ):
 174         if not path:
 175             return
 176         path_parts = path.split("/")
 177         path_prefix = ""
 178         # Small hack until clang-tidy stops being a separate toolchain in a
 179         # weird location.
 180         if path_parts[0] == "clang-tools":
 181             path_prefix = path_parts.pop(0)
 182
 183         def try_bootstrap(exists):
 184             if not tasks:
 185                 return False
 186             prefixes = [""]
 187             if tasks.prefix:
 188                 prefixes.insert(0, "{}-".format(tasks.prefix))
 189             for prefix in prefixes:
 190                 label = "toolchain-{}{}".format(prefix, path_parts[0])
 191                 task = tasks.tasks.get(label)
 192                 if task:
 193                     break
 194             log.debug("Trying to bootstrap %s", label)
 195             if not task:
 196                 return False
 197             task_index = task["index"]
 198             log.debug("Resolved %s to %s", label, task_index[0])
 199             task_index = task_index[0].split(".")[-1]
 200             artifact = task["artifact"]
 201             # `mach artifact toolchain` doesn't support authentication for
 202             # private artifacts. Some toolchains may provide a command that can be
 203             # used for local production of the artifact.
 204             command = None
 205             if not artifact.startswith("public/"):
 206                 command = task.get("command")
 207                 if not command:
 208                     log.debug("Cannot bootstrap %s: not a public artifact", label)
 209                     return False
 210             index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
 211             try:
 212                 with open(index_file) as fh:
 213                     index = fh.read().strip()
 214             except Exception:
 215                 # On automation, if there's an artifact in MOZ_FETCHES_DIR, we assume it's
 216                 # up-to-date.
 217                 index = task_index if moz_fetches_dir else None
 218             if index == task_index and exists:
 219                 log.debug("%s is up-to-date", label)
 220                 return True
 221             # Manually import with import_module so that we can gracefully disable bootstrap
 222             # when e.g. building from a js standalone tarball, that doesn't contain the
 223             # taskgraph code. In those cases, `mach artifact toolchain --from-build` would
 224             # also fail.
 225             task_id = None
 226             if not command:
 227                 try:
 228                     IndexSearch = import_module(
 229                         "taskgraph.optimize.strategies"
 230                     ).IndexSearch
 231                 except Exception:
 232                     log.debug("Cannot bootstrap %s: missing taskgraph module", label)
 233                     return False
 234                 task_id = IndexSearch().should_replace_task(
 235                     task, {}, None, task["index"]
 236                 )
 237             if task_id:
 238                 # If we found the task in the index, use the `mach artifact toolchain`
 239                 # fast path.
 240                 command = [
 241                     "artifact",
 242                     "toolchain",
 243                     "--from-task",
 244                     f"{task_id}:{artifact}",
 245                 ]
 246             elif command:
 247                 # For private local toolchains, run the associated command.
 248                 command = (
 249                     [
 250                         "python",
 251                         "--virtualenv",
 252                         "build",
 253                         os.path.join(
 254                             build_env.topsrcdir,
 255                             "taskcluster/scripts/misc",
 256                             command["script"],
 257                         ),
 258                     ]
 259                     + command["arguments"]
 260                     + [path_parts[0]]
 261                 )
 262                 # Clean up anything that was bootstrapped previously before going
 263                 # forward. In other cases, that's taken care of by mach artifact toolchain.
 264                 rmtree(
 265                     os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
 266                     ignore_errors=True,
 267                 )
 268             else:
 269                 # Otherwise, use the slower path, which will print a better error than
 270                 # we would be able to.
 271                 command = ["artifact", "toolchain", "--from-build", label]
 272
 273             log.info(
 274                 "%s bootstrapped toolchain in %s",
 275                 "Updating" if exists else "Installing",
 276                 os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
 277             )
 278             os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)
 279             proc = subprocess.run(
 280                 [
 281                     sys.executable,
 282                     os.path.join(build_env.topsrcdir, "mach"),
 283                     "--log-no-times",
 284                 ]
 285                 + command,
 286                 cwd=os.path.join(toolchains_base_dir, path_prefix),
 287                 check=not allow_failure,
 288             )
 289             if proc.returncode != 0 and allow_failure:
 290                 return False
 291             ensureParentDir(index_file)
 292             with open(index_file, "w") as fh:
 293                 fh.write(task_index)
 294             return True
 295
 296         path = os.path.join(toolchains_base_dir, path_prefix, *path_parts)
 297         if bootstrap and bootstrap(path_parts[0]):
 298             try:
 299                 if not try_bootstrap(os.path.exists(path)):
 300                     # If there aren't toolchain artifacts to use for this build,
 301                     # don't return a path.
 302                     return None
 303             except Exception as e:
 304                 log.error("%s", e)
 305                 die("If you can't fix the above, retry with --disable-bootstrap.")
 306         # We re-test whether the path exists because it may have been created by
 307         # try_bootstrap. Automation will not have gone through the bootstrap
 308         # process, but we want to return the path if it exists.
 309         if os.path.exists(path):
 310             return path
 311
 312     return bootstrap_path
 313
 314
 315 @template
 316 def bootstrap_search_path(path, paths=original_path, **kwargs):
 317     @depends(
 318         enable_bootstrap,
 319         bootstrap_path(path, **kwargs),
 320         bootstrap_search_path_order,
 321         paths,
 322         original_path,
 323     )
 324     def bootstrap_search_path(bootstrap, path, order, paths, original_path):
 325         if paths is None:
 326             paths = original_path
 327         if not path:
 328             return paths
 329         if order == "maybe-prepend":
 330             if bootstrap(path.split("/")[0]):
 331                 order = "prepend"
 332             else:
 333                 order = "append"
 334         if order == "prepend":
 335             return [path] + paths
 336         return paths + [path]
 337
 338     return bootstrap_search_path