build/moz.configure/bootstrap.configure

   1 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
   2 # vim: set filetype=python:
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 option(
   8     env="MOZ_FETCHES_DIR",
   9     nargs=1,
  10     when="MOZ_AUTOMATION",
  11     help="Directory containing fetched artifacts",
  12 )
  13
  14
  15 @depends("MOZ_FETCHES_DIR", when="MOZ_AUTOMATION")
  16 def moz_fetches_dir(value):
  17     if value:
  18         return value[0]
  19
  20
  21 @depends(vcs_checkout_type, milestone.is_nightly, "MOZ_AUTOMATION")
  22 def bootstrap_default(vcs_checkout_type, is_nightly, automation):
  23     if automation:
  24         return False
  25     # We only enable if building off a VCS checkout of central.
  26     if is_nightly and vcs_checkout_type:
  27         return True
  28
  29
  30 option(
  31     "--enable-bootstrap",
  32     nargs="*",
  33     default=bootstrap_default,
  34     help="{Automatically bootstrap or update some toolchains|Disable bootstrap or update of toolchains}",
  35 )
  36
  37
  38 @depends_if("--enable-bootstrap")
  39 def want_bootstrap(bootstrap):
  40     include = set()
  41     exclude = set()
  42     for item in bootstrap:
  43         if item == "no-update":
  44             continue
  45         if item.startswith("-"):
  46             exclude.add(item.lstrip("-"))
  47         else:
  48             include.add(item)
  49
  50     def match(name):
  51         if name in exclude:
  52             return False
  53         if include and name in include:
  54             return True
  55         return not bool(include)
  56
  57     return match
  58
  59
  60 toolchains_base_dir = moz_fetches_dir | mozbuild_state_path
  61
  62
  63 @dependable
  64 @imports("os")
  65 @imports(_from="os", _import="environ")
  66 def original_path():
  67     return environ["PATH"].split(os.pathsep)
  68
  69
  70 @depends(host, when="--enable-bootstrap")
  71 @imports("os")
  72 @imports("traceback")
  73 @imports(_from="mozbuild.toolchains", _import="toolchain_task_definitions")
  74 @imports(_from="__builtin__", _import="Exception")
  75 def bootstrap_toolchain_tasks(host):
  76     prefix = {
  77         ("x86_64", "GNU", "Linux"): "linux64",
  78         ("x86_64", "OSX", "Darwin"): "macosx64",
  79         ("aarch64", "OSX", "Darwin"): "macosx64-aarch64",
  80         ("x86_64", "WINNT", "WINNT"): "win64",
  81         ("aarch64", "WINNT", "WINNT"): "win64-aarch64",
  82     }.get((host.cpu, host.os, host.kernel))
  83     try:
  84         tasks = toolchain_task_definitions()
  85     except Exception as e:
  86         message = traceback.format_exc()
  87         log.warning(str(e))
  88         log.debug(message)
  89         return None
  90
  91     def task_data(t):
  92         result = {
  93             "index": t.optimization["index-search"],
  94             "artifact": t.attributes["toolchain-artifact"],
  95         }
  96         command = t.attributes.get("toolchain-command")
  97         if command:
  98             result["command"] = command
  99         return result
 100
 101     # We only want to use toolchains annotated with "local-toolchain". We also limit the
 102     # amount of data to what we use, so that trace logs can be more useful.
 103     tasks = {
 104         k: task_data(t)
 105         for k, t in tasks.items()
 106         if t.attributes.get("local-toolchain") and "index-search" in t.optimization
 107     }
 108
 109     return namespace(prefix=prefix, tasks=tasks)
 110
 111
 112 @template
 113 def bootstrap_path(path, **kwargs):
 114     when = kwargs.pop("when", None)
 115     allow_failure = kwargs.pop("allow_failure", None)
 116     if kwargs:
 117         configure_error(
 118             "bootstrap_path only takes `when` and `allow_failure` as a keyword argument"
 119         )
 120
 121     @depends(
 122         "--enable-bootstrap",
 123         want_bootstrap,
 124         toolchains_base_dir,
 125         moz_fetches_dir,
 126         bootstrap_toolchain_tasks,
 127         build_environment,
 128         dependable(path),
 129         dependable(allow_failure),
 130         when=when,
 131     )
 132     @imports("os")
 133     @imports("subprocess")
 134     @imports("sys")
 135     @imports(_from="mozbuild.util", _import="ensureParentDir")
 136     @imports(_from="importlib", _import="import_module")
 137     @imports(_from="shutil", _import="rmtree")
 138     @imports(_from="__builtin__", _import="open")
 139     @imports(_from="__builtin__", _import="Exception")
 140     def bootstrap_path(
 141         enable_bootstrap,
 142         want_bootstrap,
 143         toolchains_base_dir,
 144         moz_fetches_dir,
 145         tasks,
 146         build_env,
 147         path,
 148         allow_failure,
 149     ):
 150         if not path:
 151             return
 152         path_parts = path.split("/")
 153         path_prefix = ""
 154         # Small hack until clang-tidy stops being a separate toolchain in a
 155         # weird location.
 156         if path_parts[0] == "clang-tools":
 157             path_prefix = path_parts.pop(0)
 158
 159         def try_bootstrap(exists):
 160             if not tasks:
 161                 return False
 162             prefixes = [""]
 163             if tasks.prefix:
 164                 prefixes.insert(0, "{}-".format(tasks.prefix))
 165             for prefix in prefixes:
 166                 label = "toolchain-{}{}".format(prefix, path_parts[0])
 167                 task = tasks.tasks.get(label)
 168                 if task:
 169                     break
 170             log.debug("Trying to bootstrap %s", label)
 171             if not task:
 172                 return False
 173             task_index = task["index"]
 174             log.debug("Resolved %s to %s", label, task_index[0])
 175             task_index = task_index[0].split(".")[-1]
 176             artifact = task["artifact"]
 177             # `mach artifact toolchain` doesn't support authentication for
 178             # private artifacts. Some toolchains may provide a command that can be
 179             # used for local production of the artifact.
 180             command = None
 181             if not artifact.startswith("public/"):
 182                 command = task.get("command")
 183                 if not command:
 184                     log.debug("Cannot bootstrap %s: not a public artifact", label)
 185                     return False
 186             index_file = os.path.join(toolchains_base_dir, "indices", path_parts[0])
 187             try:
 188                 with open(index_file) as fh:
 189                     index = fh.read().strip()
 190             except Exception:
 191                 # On automation, if there's an artifact in MOZ_FETCHES_DIR, we assume it's
 192                 # up-to-date.
 193                 index = task_index if moz_fetches_dir else None
 194             if index == task_index and exists:
 195                 log.debug("%s is up-to-date", label)
 196                 return True
 197             # Manually import with import_module so that we can gracefully disable bootstrap
 198             # when e.g. building from a js standalone tarball, that doesn't contain the
 199             # taskgraph code. In those cases, `mach artifact toolchain --from-build` would
 200             # also fail.
 201             task_id = None
 202             if not command:
 203                 try:
 204                     IndexSearch = import_module(
 205                         "taskgraph.optimize.strategies"
 206                     ).IndexSearch
 207                 except Exception:
 208                     log.debug("Cannot bootstrap %s: missing taskgraph module", label)
 209                     return False
 210                 task_id = IndexSearch().should_replace_task(
 211                     task, {}, None, task["index"]
 212                 )
 213             if task_id:
 214                 # If we found the task in the index, use the `mach artifact toolchain`
 215                 # fast path.
 216                 command = [
 217                     "artifact",
 218                     "toolchain",
 219                     "--from-task",
 220                     f"{task_id}:{artifact}",
 221                 ]
 222             elif command:
 223                 # For private local toolchains, run the associated command.
 224                 command = (
 225                     [
 226                         "python",
 227                         "--virtualenv",
 228                         "build",
 229                         os.path.join(
 230                             build_env.topsrcdir,
 231                             "taskcluster/scripts/misc",
 232                             command["script"],
 233                         ),
 234                     ]
 235                     + command["arguments"]
 236                     + [path_parts[0]]
 237                 )
 238                 # Clean up anything that was bootstrapped previously before going
 239                 # forward. In other cases, that's taken care of by mach artifact toolchain.
 240                 rmtree(
 241                     os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
 242                     ignore_errors=True,
 243                 )
 244             else:
 245                 # Otherwise, use the slower path, which will print a better error than
 246                 # we would be able to.
 247                 command = ["artifact", "toolchain", "--from-build", label]
 248
 249             log.info(
 250                 "%s bootstrapped toolchain in %s",
 251                 "Updating" if exists else "Installing",
 252                 os.path.join(toolchains_base_dir, path_prefix, path_parts[0]),
 253             )
 254             os.makedirs(os.path.join(toolchains_base_dir, path_prefix), exist_ok=True)
 255             proc = subprocess.run(
 256                 [
 257                     sys.executable,
 258                     os.path.join(build_env.topsrcdir, "mach"),
 259                     "--log-no-times",
 260                 ]
 261                 + command,
 262                 cwd=os.path.join(toolchains_base_dir, path_prefix),
 263                 check=not allow_failure,
 264             )
 265             if proc.returncode != 0 and allow_failure:
 266                 return False
 267             ensureParentDir(index_file)
 268             with open(index_file, "w") as fh:
 269                 fh.write(task_index)
 270             return True
 271
 272         path = os.path.join(toolchains_base_dir, path_prefix, *path_parts)
 273         if enable_bootstrap and want_bootstrap(path_parts[0]):
 274             exists = os.path.exists(path)
 275             try:
 276                 # With --enable-bootstrap=no-update, we don't `try_bootstrap`, except
 277                 # when the toolchain can't be found.
 278                 if (
 279                     "no-update" not in enable_bootstrap or not exists
 280                 ) and not try_bootstrap(exists):
 281                     # If there aren't toolchain artifacts to use for this build,
 282                     # don't return a path.
 283                     return None
 284             except Exception as e:
 285                 log.error("%s", e)
 286                 die("If you can't fix the above, retry with --disable-bootstrap.")
 287         if enable_bootstrap or enable_bootstrap.origin == "default":
 288             # We re-test whether the path exists because it may have been created by
 289             # try_bootstrap. Automation will not have gone through the bootstrap
 290             # process, but we want to return the path if it exists.
 291             if os.path.exists(path):
 292                 return path
 293
 294     return bootstrap_path
 295
 296
 297 @template
 298 def bootstrap_search_path(path, paths=original_path, **kwargs):
 299     @depends(
 300         bootstrap_path(path, **kwargs),
 301         paths,
 302         original_path,
 303     )
 304     def bootstrap_search_path(path, paths, original_path):
 305         if paths is None:
 306             paths = original_path
 307         if not path:
 308             return paths
 309         return [path] + paths
 310
 311     return bootstrap_search_path