third-party/watchman/src/build/fbcode_builder/getdeps/load.py

   1 # Copyright (c) Meta Platforms, Inc. and affiliates.
   2 #
   3 # This source code is licensed under the MIT license found in the
   4 # LICENSE file in the root directory of this source tree.
   5
   6 import base64
   7 import copy
   8 import hashlib
   9 import os
  10
  11 from . import fetcher
  12 from .envfuncs import path_search
  13 from .errors import ManifestNotFound
  14 from .manifest import ManifestParser
  15
  16
  17 class Loader(object):
  18     """The loader allows our tests to patch the load operation"""
  19
  20     def _list_manifests(self, build_opts):
  21         """Returns a generator that iterates all the available manifests"""
  22         for (path, _, files) in os.walk(build_opts.manifests_dir):
  23             for name in files:
  24                 # skip hidden files
  25                 if name.startswith("."):
  26                     continue
  27
  28                 yield os.path.join(path, name)
  29
  30     def _load_manifest(self, path):
  31         return ManifestParser(path)
  32
  33     def load_project(self, build_opts, project_name):
  34         if "/" in project_name or "\\" in project_name:
  35             # Assume this is a path already
  36             return ManifestParser(project_name)
  37
  38         for manifest in self._list_manifests(build_opts):
  39             if os.path.basename(manifest) == project_name:
  40                 return ManifestParser(manifest)
  41
  42         raise ManifestNotFound(project_name)
  43
  44     def load_all(self, build_opts):
  45         manifests_by_name = {}
  46
  47         for manifest in self._list_manifests(build_opts):
  48             m = self._load_manifest(manifest)
  49
  50             if m.name in manifests_by_name:
  51                 raise Exception("found duplicate manifest '%s'" % m.name)
  52
  53             manifests_by_name[m.name] = m
  54
  55         return manifests_by_name
  56
  57
  58 class ResourceLoader(Loader):
  59     def __init__(self, namespace, manifests_dir) -> None:
  60         self.namespace = namespace
  61         self.manifests_dir = manifests_dir
  62
  63     def _list_manifests(self, _build_opts):
  64         import pkg_resources
  65
  66         dirs = [self.manifests_dir]
  67
  68         while dirs:
  69             current = dirs.pop(0)
  70             for name in pkg_resources.resource_listdir(self.namespace, current):
  71                 path = "%s/%s" % (current, name)
  72
  73                 if pkg_resources.resource_isdir(self.namespace, path):
  74                     dirs.append(path)
  75                 else:
  76                     yield "%s/%s" % (current, name)
  77
  78     def _find_manifest(self, project_name):
  79         for name in self._list_manifests():
  80             if name.endswith("/%s" % project_name):
  81                 return name
  82
  83         raise ManifestNotFound(project_name)
  84
  85     def _load_manifest(self, path: str):
  86         import pkg_resources
  87
  88         contents = pkg_resources.resource_string(self.namespace, path).decode("utf8")
  89         return ManifestParser(file_name=path, fp=contents)
  90
  91     def load_project(self, build_opts, project_name):
  92         project_name = self._find_manifest(project_name)
  93         return self._load_resource_manifest(project_name)
  94
  95
  96 LOADER = Loader()
  97
  98
  99 def patch_loader(namespace, manifests_dir: str = "manifests") -> None:
 100     global LOADER
 101     LOADER = ResourceLoader(namespace, manifests_dir)
 102
 103
 104 def load_project(build_opts, project_name):
 105     """given the name of a project or a path to a manifest file,
 106     load up the ManifestParser instance for it and return it"""
 107     return LOADER.load_project(build_opts, project_name)
 108
 109
 110 def load_all_manifests(build_opts):
 111     return LOADER.load_all(build_opts)
 112
 113
 114 class ManifestLoader(object):
 115     """ManifestLoader stores information about project manifest relationships for a
 116     given set of (build options + platform) configuration.
 117
 118     The ManifestLoader class primarily serves as a location to cache project dependency
 119     relationships and project hash values for this build configuration.
 120     """
 121
 122     def __init__(self, build_opts, ctx_gen=None) -> None:
 123         self._loader = LOADER
 124         self.build_opts = build_opts
 125         if ctx_gen is None:
 126             self.ctx_gen = self.build_opts.get_context_generator()
 127         else:
 128             self.ctx_gen = ctx_gen
 129
 130         self.manifests_by_name = {}
 131         self._loaded_all = False
 132         self._project_hashes = {}
 133         self._fetcher_overrides = {}
 134         self._build_dir_overrides = {}
 135         self._install_dir_overrides = {}
 136         self._install_prefix_overrides = {}
 137
 138     def load_manifest(self, name):
 139         manifest = self.manifests_by_name.get(name)
 140         if manifest is None:
 141             manifest = self._loader.load_project(self.build_opts, name)
 142             self.manifests_by_name[name] = manifest
 143         return manifest
 144
 145     def load_all_manifests(self):
 146         if not self._loaded_all:
 147             all_manifests_by_name = self._loader.load_all(self.build_opts)
 148             if self.manifests_by_name:
 149                 # To help ensure that we only ever have a single manifest object for a
 150                 # given project, and that it can't change once we have loaded it,
 151                 # only update our mapping for projects that weren't already loaded.
 152                 for name, manifest in all_manifests_by_name.items():
 153                     self.manifests_by_name.setdefault(name, manifest)
 154             else:
 155                 self.manifests_by_name = all_manifests_by_name
 156             self._loaded_all = True
 157
 158         return self.manifests_by_name
 159
 160     def manifests_in_dependency_order(self, manifest=None):
 161         """Compute all dependencies of the specified project.  Returns a list of the
 162         dependencies plus the project itself, in topologically sorted order.
 163
 164         Each entry in the returned list only depends on projects that appear before it
 165         in the list.
 166
 167         If the input manifest is None, the dependencies for all currently loaded
 168         projects will be computed.  i.e., if you call load_all_manifests() followed by
 169         manifests_in_dependency_order() this will return a global dependency ordering of
 170         all projects."""
 171         # The list of deps that have been fully processed
 172         seen = set()
 173         # The list of deps which have yet to be evaluated.  This
 174         # can potentially contain duplicates.
 175         if manifest is None:
 176             deps = list(self.manifests_by_name.values())
 177         else:
 178             assert manifest.name in self.manifests_by_name
 179             deps = [manifest]
 180         # The list of manifests in dependency order
 181         dep_order = []
 182         system_packages = {}
 183
 184         while len(deps) > 0:
 185             m = deps.pop(0)
 186             if m.name in seen:
 187                 continue
 188
 189             # Consider its deps, if any.
 190             # We sort them for increased determinism; we'll produce
 191             # a correct order even if they aren't sorted, but we prefer
 192             # to produce the same order regardless of how they are listed
 193             # in the project manifest files.
 194             ctx = self.ctx_gen.get_context(m.name)
 195             dep_list = m.get_dependencies(ctx)
 196
 197             dep_count = 0
 198             for dep_name in dep_list:
 199                 # If we're not sure whether it is done, queue it up
 200                 if dep_name not in seen:
 201                     dep = self.manifests_by_name.get(dep_name)
 202                     if dep is None:
 203                         dep = self._loader.load_project(self.build_opts, dep_name)
 204                         self.manifests_by_name[dep.name] = dep
 205
 206                     deps.append(dep)
 207                     dep_count += 1
 208
 209             if dep_count > 0:
 210                 # If we queued anything, re-queue this item, as it depends
 211                 # those new item(s) and their transitive deps.
 212                 deps.append(m)
 213                 continue
 214
 215             # Its deps are done, so we can emit it
 216             seen.add(m.name)
 217             # Capture system packages as we may need to set PATHs to then later
 218             if (
 219                 self.build_opts.allow_system_packages
 220                 and self.build_opts.host_type.get_package_manager()
 221             ):
 222                 packages = m.get_required_system_packages(ctx)
 223                 for pkg_type, v in packages.items():
 224                     merged = system_packages.get(pkg_type, [])
 225                     if v not in merged:
 226                         merged += v
 227                     system_packages[pkg_type] = merged
 228                 # A manifest depends on all system packages in it dependencies as well
 229                 m.resolved_system_packages = copy.copy(system_packages)
 230             dep_order.append(m)
 231
 232         return dep_order
 233
 234     def set_project_src_dir(self, project_name, path) -> None:
 235         self._fetcher_overrides[project_name] = fetcher.LocalDirFetcher(path)
 236
 237     def set_project_build_dir(self, project_name, path) -> None:
 238         self._build_dir_overrides[project_name] = path
 239
 240     def set_project_install_dir(self, project_name, path) -> None:
 241         self._install_dir_overrides[project_name] = path
 242
 243     def set_project_install_prefix(self, project_name, path) -> None:
 244         self._install_prefix_overrides[project_name] = path
 245
 246     def create_fetcher(self, manifest):
 247         override = self._fetcher_overrides.get(manifest.name)
 248         if override is not None:
 249             return override
 250
 251         ctx = self.ctx_gen.get_context(manifest.name)
 252         return manifest.create_fetcher(self.build_opts, ctx)
 253
 254     def get_project_hash(self, manifest):
 255         h = self._project_hashes.get(manifest.name)
 256         if h is None:
 257             h = self._compute_project_hash(manifest)
 258             self._project_hashes[manifest.name] = h
 259         return h
 260
 261     def _compute_project_hash(self, manifest) -> str:
 262         """This recursive function computes a hash for a given manifest.
 263         The hash takes into account some environmental factors on the
 264         host machine and includes the hashes of its dependencies.
 265         No caching of the computation is performed, which is theoretically
 266         wasteful but the computation is fast enough that it is not required
 267         to cache across multiple invocations."""
 268         ctx = self.ctx_gen.get_context(manifest.name)
 269
 270         hasher = hashlib.sha256()
 271         # Some environmental and configuration things matter
 272         env = {}
 273         env["install_dir"] = self.build_opts.install_dir
 274         env["scratch_dir"] = self.build_opts.scratch_dir
 275         env["vcvars_path"] = self.build_opts.vcvars_path
 276         env["os"] = self.build_opts.host_type.ostype
 277         env["distro"] = self.build_opts.host_type.distro
 278         env["distro_vers"] = self.build_opts.host_type.distrovers
 279         env["shared_libs"] = str(self.build_opts.shared_libs)
 280         for name in [
 281             "CXXFLAGS",
 282             "CPPFLAGS",
 283             "LDFLAGS",
 284             "CXX",
 285             "CC",
 286             "GETDEPS_CMAKE_DEFINES",
 287         ]:
 288             env[name] = os.environ.get(name)
 289         for tool in ["cc", "c++", "gcc", "g++", "clang", "clang++"]:
 290             env["tool-%s" % tool] = path_search(os.environ, tool)
 291         for name in manifest.get_section_as_args("depends.environment", ctx):
 292             env[name] = os.environ.get(name)
 293
 294         fetcher = self.create_fetcher(manifest)
 295         env["fetcher.hash"] = fetcher.hash()
 296
 297         for name in sorted(env.keys()):
 298             hasher.update(name.encode("utf-8"))
 299             value = env.get(name)
 300             if value is not None:
 301                 try:
 302                     hasher.update(value.encode("utf-8"))
 303                 except AttributeError as exc:
 304                     raise AttributeError("name=%r, value=%r: %s" % (name, value, exc))
 305
 306         manifest.update_hash(hasher, ctx)
 307
 308         dep_list = manifest.get_dependencies(ctx)
 309         for dep in dep_list:
 310             dep_manifest = self.load_manifest(dep)
 311             dep_hash = self.get_project_hash(dep_manifest)
 312             hasher.update(dep_hash.encode("utf-8"))
 313
 314         # Use base64 to represent the hash, rather than the simple hex digest,
 315         # so that the string is shorter.  Use the URL-safe encoding so that
 316         # the hash can also be safely used as a filename component.
 317         h = base64.urlsafe_b64encode(hasher.digest()).decode("ascii")
 318         # ... and because cmd.exe is troublesome with `=` signs, nerf those.
 319         # They tend to be padding characters at the end anyway, so we can
 320         # safely discard them.
 321         h = h.replace("=", "")
 322
 323         return h
 324
 325     def _get_project_dir_name(self, manifest):
 326         if manifest.is_first_party_project():
 327             return manifest.name
 328         else:
 329             project_hash = self.get_project_hash(manifest)
 330             return "%s-%s" % (manifest.name, project_hash)
 331
 332     def get_project_install_dir(self, manifest):
 333         override = self._install_dir_overrides.get(manifest.name)
 334         if override:
 335             return override
 336
 337         project_dir_name = self._get_project_dir_name(manifest)
 338         return os.path.join(self.build_opts.install_dir, project_dir_name)
 339
 340     def get_project_build_dir(self, manifest):
 341         override = self._build_dir_overrides.get(manifest.name)
 342         if override:
 343             return override
 344
 345         project_dir_name = self._get_project_dir_name(manifest)
 346         return os.path.join(self.build_opts.scratch_dir, "build", project_dir_name)
 347
 348     def get_project_install_prefix(self, manifest):
 349         return self._install_prefix_overrides.get(manifest.name)
 350
 351     def get_project_install_dir_respecting_install_prefix(self, manifest):
 352         inst_dir = self.get_project_install_dir(manifest)
 353         prefix = self.get_project_install_prefix(manifest)
 354         if prefix:
 355             return inst_dir + prefix
 356         return inst_dir