third-party/watchman/src/build/fbcode_builder/getdeps/manifest.py

   1 # Copyright (c) Meta Platforms, Inc. and affiliates.
   2 #
   3 # This source code is licensed under the MIT license found in the
   4 # LICENSE file in the root directory of this source tree.
   5
   6 import configparser
   7 import io
   8 import os
   9 from typing import List
  10
  11 from .builder import (
  12     AutoconfBuilder,
  13     Boost,
  14     CMakeBootStrapBuilder,
  15     CMakeBuilder,
  16     Iproute2Builder,
  17     MakeBuilder,
  18     NinjaBootstrap,
  19     NopBuilder,
  20     OpenNSABuilder,
  21     OpenSSLBuilder,
  22     SqliteBuilder,
  23 )
  24 from .cargo import CargoBuilder
  25 from .expr import parse_expr
  26 from .fetcher import (
  27     ArchiveFetcher,
  28     GitFetcher,
  29     PreinstalledNopFetcher,
  30     ShipitTransformerFetcher,
  31     SimpleShipitTransformerFetcher,
  32     SystemPackageFetcher,
  33 )
  34 from .py_wheel_builder import PythonWheelBuilder
  35
  36
  37 REQUIRED = "REQUIRED"
  38 OPTIONAL = "OPTIONAL"
  39
  40 SCHEMA = {
  41     "manifest": {
  42         "optional_section": False,
  43         "fields": {
  44             "name": REQUIRED,
  45             "fbsource_path": OPTIONAL,
  46             "shipit_project": OPTIONAL,
  47             "shipit_fbcode_builder": OPTIONAL,
  48         },
  49     },
  50     "dependencies": {"optional_section": True, "allow_values": False},
  51     "depends.environment": {"optional_section": True},
  52     "git": {
  53         "optional_section": True,
  54         "fields": {"repo_url": REQUIRED, "rev": OPTIONAL, "depth": OPTIONAL},
  55     },
  56     "download": {
  57         "optional_section": True,
  58         "fields": {"url": REQUIRED, "sha256": REQUIRED},
  59     },
  60     "build": {
  61         "optional_section": True,
  62         "fields": {
  63             "builder": REQUIRED,
  64             "subdir": OPTIONAL,
  65             "make_binary": OPTIONAL,
  66             "build_in_src_dir": OPTIONAL,
  67             "job_weight_mib": OPTIONAL,
  68             "patchfile": OPTIONAL,
  69             "patchfile_opts": OPTIONAL,
  70         },
  71     },
  72     "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}},
  73     "cargo": {
  74         "optional_section": True,
  75         "fields": {
  76             "build_doc": OPTIONAL,
  77             "workspace_dir": OPTIONAL,
  78             "manifests_to_build": OPTIONAL,
  79             # Where to write cargo config (defaults to build_dir/.cargo/config)
  80             "cargo_config_file": OPTIONAL,
  81         },
  82     },
  83     "github.actions": {
  84         "optional_section": True,
  85         "fields": {
  86             "run_tests": OPTIONAL,
  87         },
  88     },
  89     "crate.pathmap": {"optional_section": True},
  90     "cmake.defines": {"optional_section": True},
  91     "autoconf.args": {"optional_section": True},
  92     "autoconf.envcmd.LDFLAGS": {"optional_section": True},
  93     "rpms": {"optional_section": True},
  94     "debs": {"optional_section": True},
  95     "homebrew": {"optional_section": True},
  96     "preinstalled.env": {"optional_section": True},
  97     "bootstrap.args": {"optional_section": True},
  98     "b2.args": {"optional_section": True},
  99     "make.build_args": {"optional_section": True},
 100     "make.install_args": {"optional_section": True},
 101     "make.test_args": {"optional_section": True},
 102     "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
 103     "shipit.pathmap": {"optional_section": True},
 104     "shipit.strip": {"optional_section": True},
 105     "install.files": {"optional_section": True},
 106     # fb-only
 107     "sandcastle": {"optional_section": True, "fields": {"run_tests": OPTIONAL}},
 108 }
 109
 110 # These sections are allowed to vary for different platforms
 111 # using the expression syntax to enable/disable sections
 112 ALLOWED_EXPR_SECTIONS = [
 113     "autoconf.args",
 114     "autoconf.envcmd.LDFLAGS",
 115     "build",
 116     "cmake.defines",
 117     "dependencies",
 118     "make.build_args",
 119     "make.install_args",
 120     "bootstrap.args",
 121     "b2.args",
 122     "download",
 123     "git",
 124     "install.files",
 125     "rpms",
 126     "debs",
 127     "shipit.pathmap",
 128     "shipit.strip",
 129     "homebrew",
 130     "github.actions",
 131 ]
 132
 133
 134 def parse_conditional_section_name(name, section_def):
 135     expr = name[len(section_def) + 1 :]
 136     return parse_expr(expr, ManifestContext.ALLOWED_VARIABLES)
 137
 138
 139 def validate_allowed_fields(file_name, section, config, allowed_fields):
 140     for field in config.options(section):
 141         if not allowed_fields.get(field):
 142             raise Exception(
 143                 ("manifest file %s section '%s' contains " "unknown field '%s'")
 144                 % (file_name, section, field)
 145             )
 146
 147     for field in allowed_fields:
 148         if allowed_fields[field] == REQUIRED and not config.has_option(section, field):
 149             raise Exception(
 150                 ("manifest file %s section '%s' is missing " "required field '%s'")
 151                 % (file_name, section, field)
 152             )
 153
 154
 155 def validate_allow_values(file_name, section, config):
 156     for field in config.options(section):
 157         value = config.get(section, field)
 158         if value is not None:
 159             raise Exception(
 160                 (
 161                     "manifest file %s section '%s' has '%s = %s' but "
 162                     "this section doesn't allow specifying values "
 163                     "for its entries"
 164                 )
 165                 % (file_name, section, field, value)
 166             )
 167
 168
 169 def validate_section(file_name, section, config):
 170     section_def = SCHEMA.get(section)
 171     if not section_def:
 172         for name in ALLOWED_EXPR_SECTIONS:
 173             if section.startswith(name + "."):
 174                 # Verify that the conditional parses, but discard it
 175                 try:
 176                     parse_conditional_section_name(section, name)
 177                 except Exception as exc:
 178                     raise Exception(
 179                         ("manifest file %s section '%s' has invalid " "conditional: %s")
 180                         % (file_name, section, str(exc))
 181                     )
 182                 section_def = SCHEMA.get(name)
 183                 canonical_section_name = name
 184                 break
 185         if not section_def:
 186             raise Exception(
 187                 "manifest file %s contains unknown section '%s'" % (file_name, section)
 188             )
 189     else:
 190         canonical_section_name = section
 191
 192     allowed_fields = section_def.get("fields")
 193     if allowed_fields:
 194         validate_allowed_fields(file_name, section, config, allowed_fields)
 195     elif not section_def.get("allow_values", True):
 196         validate_allow_values(file_name, section, config)
 197     return canonical_section_name
 198
 199
 200 class ManifestParser(object):
 201     def __init__(self, file_name, fp=None):
 202         # allow_no_value enables listing parameters in the
 203         # autoconf.args section one per line
 204         config = configparser.RawConfigParser(allow_no_value=True)
 205         config.optionxform = str  # make it case sensitive
 206
 207         if fp is None:
 208             with open(file_name, "r") as fp:
 209                 config.read_file(fp)
 210         elif isinstance(fp, type("")):
 211             # For testing purposes, parse from a string (str
 212             # or unicode)
 213             config.read_file(io.StringIO(fp))
 214         else:
 215             config.read_file(fp)
 216
 217         # validate against the schema
 218         seen_sections = set()
 219
 220         for section in config.sections():
 221             seen_sections.add(validate_section(file_name, section, config))
 222
 223         for section in SCHEMA.keys():
 224             section_def = SCHEMA[section]
 225             if (
 226                 not section_def.get("optional_section", False)
 227                 and section not in seen_sections
 228             ):
 229                 raise Exception(
 230                     "manifest file %s is missing required section %s"
 231                     % (file_name, section)
 232                 )
 233
 234         self._config = config
 235         self.name = config.get("manifest", "name")
 236         self.fbsource_path = self.get("manifest", "fbsource_path")
 237         self.shipit_project = self.get("manifest", "shipit_project")
 238         self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder")
 239         self.resolved_system_packages = {}
 240
 241         if self.name != os.path.basename(file_name):
 242             raise Exception(
 243                 "filename of the manifest '%s' does not match the manifest name '%s'"
 244                 % (file_name, self.name)
 245             )
 246
 247     def get(self, section, key, defval=None, ctx=None):
 248         ctx = ctx or {}
 249
 250         for s in self._config.sections():
 251             if s == section:
 252                 if self._config.has_option(s, key):
 253                     return self._config.get(s, key)
 254                 return defval
 255
 256             if s.startswith(section + "."):
 257                 expr = parse_conditional_section_name(s, section)
 258                 if not expr.eval(ctx):
 259                     continue
 260
 261                 if self._config.has_option(s, key):
 262                     return self._config.get(s, key)
 263
 264         return defval
 265
 266     def get_dependencies(self, ctx):
 267         dep_list = list(self.get_section_as_dict("dependencies", ctx).keys())
 268         dep_list.sort()
 269         builder = self.get("build", "builder", ctx=ctx)
 270         if builder in ("cmake", "python-wheel"):
 271             dep_list.insert(0, "cmake")
 272         elif builder == "autoconf" and self.name not in (
 273             "autoconf",
 274             "libtool",
 275             "automake",
 276         ):
 277             # they need libtool and its deps (automake, autoconf) so add
 278             # those as deps (but obviously not if we're building those
 279             # projects themselves)
 280             dep_list.insert(0, "libtool")
 281
 282         return dep_list
 283
 284     def get_section_as_args(self, section, ctx=None) -> List[str]:
 285         """Intended for use with the make.[build_args/install_args] and
 286         autoconf.args sections, this method collects the entries and returns an
 287         array of strings.
 288         If the manifest contains conditional sections, ctx is used to
 289         evaluate the condition and merge in the values.
 290         """
 291         args = []
 292         ctx = ctx or {}
 293
 294         for s in self._config.sections():
 295             if s != section:
 296                 if not s.startswith(section + "."):
 297                     continue
 298                 expr = parse_conditional_section_name(s, section)
 299                 if not expr.eval(ctx):
 300                     continue
 301             for field in self._config.options(s):
 302                 value = self._config.get(s, field)
 303                 if value is None:
 304                     args.append(field)
 305                 else:
 306                     args.append("%s=%s" % (field, value))
 307         return args
 308
 309     def get_section_as_ordered_pairs(self, section, ctx=None):
 310         """Used for eg: shipit.pathmap which has strong
 311         ordering requirements"""
 312         res = []
 313         ctx = ctx or {}
 314
 315         for s in self._config.sections():
 316             if s != section:
 317                 if not s.startswith(section + "."):
 318                     continue
 319                 expr = parse_conditional_section_name(s, section)
 320                 if not expr.eval(ctx):
 321                     continue
 322
 323             for key in self._config.options(s):
 324                 value = self._config.get(s, key)
 325                 res.append((key, value))
 326         return res
 327
 328     def get_section_as_dict(self, section, ctx):
 329         d = {}
 330
 331         for s in self._config.sections():
 332             if s != section:
 333                 if not s.startswith(section + "."):
 334                     continue
 335                 expr = parse_conditional_section_name(s, section)
 336                 if not expr.eval(ctx):
 337                     continue
 338             for field in self._config.options(s):
 339                 value = self._config.get(s, field)
 340                 d[field] = value
 341         return d
 342
 343     def update_hash(self, hasher, ctx):
 344         """Compute a hash over the configuration for the given
 345         context.  The goal is for the hash to change if the config
 346         for that context changes, but not if a change is made to
 347         the config only for a different platform than that expressed
 348         by ctx.  The hash is intended to be used to help invalidate
 349         a future cache for the third party build products.
 350         The hasher argument is a hash object returned from hashlib."""
 351         for section in sorted(SCHEMA.keys()):
 352             hasher.update(section.encode("utf-8"))
 353
 354             # Note: at the time of writing, nothing in the implementation
 355             # relies on keys in any config section being ordered.
 356             # In theory we could have conflicting flags in different
 357             # config sections and later flags override earlier flags.
 358             # For the purposes of computing a hash we're not super
 359             # concerned about this: manifest changes should be rare
 360             # enough and we'd rather that this trigger an invalidation
 361             # than strive for a cache hit at this time.
 362             pairs = self.get_section_as_ordered_pairs(section, ctx)
 363             pairs.sort(key=lambda pair: pair[0])
 364             for key, value in pairs:
 365                 hasher.update(key.encode("utf-8"))
 366                 if value is not None:
 367                     hasher.update(value.encode("utf-8"))
 368
 369     def is_first_party_project(self):
 370         """returns true if this is an FB first-party project"""
 371         return self.shipit_project is not None
 372
 373     def get_required_system_packages(self, ctx):
 374         """Returns dictionary of packager system -> list of packages"""
 375         return {
 376             "rpm": self.get_section_as_args("rpms", ctx),
 377             "deb": self.get_section_as_args("debs", ctx),
 378             "homebrew": self.get_section_as_args("homebrew", ctx),
 379         }
 380
 381     def _is_satisfied_by_preinstalled_environment(self, ctx):
 382         envs = self.get_section_as_args("preinstalled.env", ctx)
 383         if not envs:
 384             return False
 385         for key in envs:
 386             val = os.environ.get(key, None)
 387             print(f"Testing ENV[{key}]: {repr(val)}")
 388             if val is None:
 389                 return False
 390             if len(val) == 0:
 391                 return False
 392
 393         return True
 394
 395     def get_repo_url(self, ctx):
 396         return self.get("git", "repo_url", ctx=ctx)
 397
 398     def create_fetcher(self, build_options, ctx):
 399         use_real_shipit = (
 400             ShipitTransformerFetcher.available() and build_options.use_shipit
 401         )
 402         if (
 403             not use_real_shipit
 404             and self.fbsource_path
 405             and build_options.fbsource_dir
 406             and self.shipit_project
 407         ):
 408             return SimpleShipitTransformerFetcher(build_options, self, ctx)
 409
 410         if (
 411             self.fbsource_path
 412             and build_options.fbsource_dir
 413             and self.shipit_project
 414             and ShipitTransformerFetcher.available()
 415         ):
 416             # We can use the code from fbsource
 417             return ShipitTransformerFetcher(build_options, self.shipit_project)
 418
 419         # Can we satisfy this dep with system packages?
 420         if build_options.allow_system_packages:
 421             if self._is_satisfied_by_preinstalled_environment(ctx):
 422                 return PreinstalledNopFetcher()
 423
 424             packages = self.get_required_system_packages(ctx)
 425             package_fetcher = SystemPackageFetcher(build_options, packages)
 426             if package_fetcher.packages_are_installed():
 427                 return package_fetcher
 428
 429         repo_url = self.get_repo_url(ctx)
 430         if repo_url:
 431             rev = self.get("git", "rev")
 432             depth = self.get("git", "depth")
 433             return GitFetcher(build_options, self, repo_url, rev, depth)
 434
 435         url = self.get("download", "url", ctx=ctx)
 436         if url:
 437             # We need to defer this import until now to avoid triggering
 438             # a cycle when the facebook/__init__.py is loaded.
 439             try:
 440                 from .facebook.lfs import LFSCachingArchiveFetcher
 441
 442                 return LFSCachingArchiveFetcher(
 443                     build_options, self, url, self.get("download", "sha256", ctx=ctx)
 444                 )
 445             except ImportError:
 446                 # This FB internal module isn't shippped to github,
 447                 # so just use its base class
 448                 return ArchiveFetcher(
 449                     build_options, self, url, self.get("download", "sha256", ctx=ctx)
 450                 )
 451
 452         raise KeyError(
 453             "project %s has no fetcher configuration matching %s" % (self.name, ctx)
 454         )
 455
 456     def get_builder_name(self, ctx):
 457         builder = self.get("build", "builder", ctx=ctx)
 458         if not builder:
 459             raise Exception("project %s has no builder for %r" % (self.name, ctx))
 460         return builder
 461
 462     def create_builder(  # noqa:C901
 463         self,
 464         build_options,
 465         src_dir,
 466         build_dir,
 467         inst_dir,
 468         ctx,
 469         loader,
 470         final_install_prefix=None,
 471         extra_cmake_defines=None,
 472         extra_b2_args=None,
 473     ):
 474         builder = self.get_builder_name(ctx)
 475         build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx)
 476         if build_in_src_dir == "true":
 477             # Some scripts don't work when they are configured and build in
 478             # a different directory than source (or when the build directory
 479             # is not a subdir of source).
 480             build_dir = src_dir
 481             subdir = self.get("build", "subdir", None, ctx=ctx)
 482             if subdir is not None:
 483                 build_dir = os.path.join(build_dir, subdir)
 484             print("build_dir is %s" % build_dir)  # just to quiet lint
 485
 486         if builder == "make" or builder == "cmakebootstrap":
 487             build_args = self.get_section_as_args("make.build_args", ctx)
 488             install_args = self.get_section_as_args("make.install_args", ctx)
 489             test_args = self.get_section_as_args("make.test_args", ctx)
 490             if builder == "cmakebootstrap":
 491                 return CMakeBootStrapBuilder(
 492                     build_options,
 493                     ctx,
 494                     self,
 495                     src_dir,
 496                     None,
 497                     inst_dir,
 498                     build_args,
 499                     install_args,
 500                     test_args,
 501                 )
 502             else:
 503                 return MakeBuilder(
 504                     build_options,
 505                     ctx,
 506                     self,
 507                     src_dir,
 508                     None,
 509                     inst_dir,
 510                     build_args,
 511                     install_args,
 512                     test_args,
 513                 )
 514
 515         if builder == "autoconf":
 516             args = self.get_section_as_args("autoconf.args", ctx)
 517             conf_env_args = {}
 518             ldflags_cmd = self.get_section_as_args("autoconf.envcmd.LDFLAGS", ctx)
 519             if ldflags_cmd:
 520                 conf_env_args["LDFLAGS"] = ldflags_cmd
 521             return AutoconfBuilder(
 522                 build_options,
 523                 ctx,
 524                 self,
 525                 src_dir,
 526                 build_dir,
 527                 inst_dir,
 528                 args,
 529                 conf_env_args,
 530             )
 531
 532         if builder == "boost":
 533             args = self.get_section_as_args("b2.args", ctx)
 534             if extra_b2_args is not None:
 535                 args += extra_b2_args
 536             return Boost(build_options, ctx, self, src_dir, build_dir, inst_dir, args)
 537
 538         if builder == "cmake":
 539             defines = self.get_section_as_dict("cmake.defines", ctx)
 540             return CMakeBuilder(
 541                 build_options,
 542                 ctx,
 543                 self,
 544                 src_dir,
 545                 build_dir,
 546                 inst_dir,
 547                 defines,
 548                 loader,
 549                 final_install_prefix,
 550                 extra_cmake_defines,
 551             )
 552
 553         if builder == "python-wheel":
 554             return PythonWheelBuilder(
 555                 build_options, ctx, self, src_dir, build_dir, inst_dir
 556             )
 557
 558         if builder == "sqlite":
 559             return SqliteBuilder(build_options, ctx, self, src_dir, build_dir, inst_dir)
 560
 561         if builder == "ninja_bootstrap":
 562             return NinjaBootstrap(
 563                 build_options, ctx, self, build_dir, src_dir, inst_dir
 564             )
 565
 566         if builder == "nop":
 567             return NopBuilder(build_options, ctx, self, src_dir, inst_dir)
 568
 569         if builder == "openssl":
 570             return OpenSSLBuilder(
 571                 build_options, ctx, self, build_dir, src_dir, inst_dir
 572             )
 573
 574         if builder == "iproute2":
 575             return Iproute2Builder(
 576                 build_options, ctx, self, src_dir, build_dir, inst_dir
 577             )
 578
 579         if builder == "cargo":
 580             return self.create_cargo_builder(
 581                 build_options, ctx, src_dir, build_dir, inst_dir, loader
 582             )
 583
 584         if builder == "OpenNSA":
 585             return OpenNSABuilder(build_options, ctx, self, src_dir, inst_dir)
 586
 587         raise KeyError("project %s has no known builder" % (self.name))
 588
 589     def create_prepare_builders(
 590         self, build_options, ctx, src_dir, build_dir, inst_dir, loader
 591     ):
 592         """Create builders that have a prepare step run, e.g. to write config files"""
 593         prepare_builders = []
 594         builder = self.get_builder_name(ctx)
 595         cargo = self.get_section_as_dict("cargo", ctx)
 596         if not builder == "cargo" and cargo:
 597             cargo_builder = self.create_cargo_builder(
 598                 build_options, ctx, src_dir, build_dir, inst_dir, loader
 599             )
 600             prepare_builders.append(cargo_builder)
 601         return prepare_builders
 602
 603     def create_cargo_builder(
 604         self, build_options, ctx, src_dir, build_dir, inst_dir, loader
 605     ):
 606         build_doc = self.get("cargo", "build_doc", False, ctx)
 607         workspace_dir = self.get("cargo", "workspace_dir", None, ctx)
 608         manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx)
 609         cargo_config_file = self.get("cargo", "cargo_config_file", None, ctx)
 610         return CargoBuilder(
 611             build_options,
 612             ctx,
 613             self,
 614             src_dir,
 615             build_dir,
 616             inst_dir,
 617             build_doc,
 618             workspace_dir,
 619             manifests_to_build,
 620             loader,
 621             cargo_config_file,
 622         )
 623
 624
 625 class ManifestContext(object):
 626     """ProjectContext contains a dictionary of values to use when evaluating boolean
 627     expressions in a project manifest.
 628
 629     This object should be passed as the `ctx` parameter in ManifestParser.get() calls.
 630     """
 631
 632     ALLOWED_VARIABLES = {
 633         "os",
 634         "distro",
 635         "distro_vers",
 636         "fb",
 637         "fbsource",
 638         "test",
 639         "shared_libs",
 640     }
 641
 642     def __init__(self, ctx_dict):
 643         assert set(ctx_dict.keys()) == self.ALLOWED_VARIABLES
 644         self.ctx_dict = ctx_dict
 645
 646     def get(self, key):
 647         return self.ctx_dict[key]
 648
 649     def set(self, key, value):
 650         assert key in self.ALLOWED_VARIABLES
 651         self.ctx_dict[key] = value
 652
 653     def copy(self):
 654         return ManifestContext(dict(self.ctx_dict))
 655
 656     def __str__(self):
 657         s = ", ".join(
 658             "%s=%s" % (key, value) for key, value in sorted(self.ctx_dict.items())
 659         )
 660         return "{" + s + "}"
 661
 662
 663 class ContextGenerator(object):
 664     """ContextGenerator allows creating ManifestContext objects on a per-project basis.
 665     This allows us to evaluate different projects with slightly different contexts.
 666
 667     For instance, this can be used to only enable tests for some projects."""
 668
 669     def __init__(self, default_ctx):
 670         self.default_ctx = ManifestContext(default_ctx)
 671         self.ctx_by_project = {}
 672
 673     def set_value_for_project(self, project_name, key, value):
 674         project_ctx = self.ctx_by_project.get(project_name)
 675         if project_ctx is None:
 676             project_ctx = self.default_ctx.copy()
 677             self.ctx_by_project[project_name] = project_ctx
 678         project_ctx.set(key, value)
 679
 680     def set_value_for_all_projects(self, key, value):
 681         self.default_ctx.set(key, value)
 682         for ctx in self.ctx_by_project.values():
 683             ctx.set(key, value)
 684
 685     def get_context(self, project_name):
 686         return self.ctx_by_project.get(project_name, self.default_ctx)