build/fbcode_builder/getdeps/builder.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) Meta Platforms, Inc. and affiliates.
   3 #
   4 # This source code is licensed under the MIT license found in the
   5 # LICENSE file in the root directory of this source tree.
   6
   7 import glob
   8 import json
   9 import os
  10 import pathlib
  11 import shutil
  12 import stat
  13 import subprocess
  14 import sys
  15 import typing
  16 from typing import Optional
  17
  18 from .dyndeps import create_dyn_dep_munger
  19 from .envfuncs import add_path_entry, Env, path_search
  20 from .fetcher import copy_if_different
  21 from .runcmd import run_cmd
  22
  23 if typing.TYPE_CHECKING:
  24     from .buildopts import BuildOptions
  25
  26
  27 class BuilderBase(object):
  28     def __init__(
  29         self,
  30         build_opts: "BuildOptions",
  31         ctx,
  32         manifest,
  33         src_dir,
  34         build_dir,
  35         inst_dir,
  36         env=None,
  37         final_install_prefix=None,
  38     ) -> None:
  39         self.env = Env()
  40         if env:
  41             self.env.update(env)
  42
  43         subdir = manifest.get("build", "subdir", ctx=ctx)
  44         if subdir:
  45             src_dir = os.path.join(src_dir, subdir)
  46
  47         self.patchfile = manifest.get("build", "patchfile", ctx=ctx)
  48         self.patchfile_opts = manifest.get("build", "patchfile_opts", ctx=ctx) or ""
  49         self.ctx = ctx
  50         self.src_dir = src_dir
  51         self.build_dir = build_dir or src_dir
  52         self.inst_dir = inst_dir
  53         self.build_opts = build_opts
  54         self.manifest = manifest
  55         self.final_install_prefix = final_install_prefix
  56
  57     def _get_cmd_prefix(self):
  58         if self.build_opts.is_windows():
  59             vcvarsall = self.build_opts.get_vcvars_path()
  60             if vcvarsall is not None:
  61                 # Since it sets rather a large number of variables we mildly abuse
  62                 # the cmd quoting rules to assemble a command that calls the script
  63                 # to prep the environment and then triggers the actual command that
  64                 # we wanted to run.
  65                 return [vcvarsall, "amd64", "&&"]
  66         return []
  67
  68     def _run_cmd(
  69         self,
  70         cmd,
  71         cwd=None,
  72         env=None,
  73         use_cmd_prefix: bool = True,
  74         allow_fail: bool = False,
  75     ) -> int:
  76         if env:
  77             e = self.env.copy()
  78             e.update(env)
  79             env = e
  80         else:
  81             env = self.env
  82
  83         if use_cmd_prefix:
  84             cmd_prefix = self._get_cmd_prefix()
  85             if cmd_prefix:
  86                 cmd = cmd_prefix + cmd
  87
  88         log_file = os.path.join(self.build_dir, "getdeps_build.log")
  89         return run_cmd(
  90             cmd=cmd,
  91             env=env,
  92             cwd=cwd or self.build_dir,
  93             log_file=log_file,
  94             allow_fail=allow_fail,
  95         )
  96
  97     def _reconfigure(self, reconfigure: bool) -> bool:
  98         if self.build_dir is not None:
  99             if not os.path.isdir(self.build_dir):
 100                 os.makedirs(self.build_dir)
 101                 reconfigure = True
 102         return reconfigure
 103
 104     def _apply_patchfile(self) -> None:
 105         if self.patchfile is None:
 106             return
 107         patched_sentinel_file = pathlib.Path(self.src_dir + "/.getdeps_patched")
 108         if patched_sentinel_file.exists():
 109             return
 110         old_wd = os.getcwd()
 111         os.chdir(self.src_dir)
 112         print(f"Patching {self.manifest.name} with {self.patchfile} in {self.src_dir}")
 113         patchfile = os.path.join(
 114             self.build_opts.fbcode_builder_dir, "patches", self.patchfile
 115         )
 116         patchcmd = ["git", "apply"]
 117         if self.patchfile_opts:
 118             patchcmd.append(self.patchfile_opts)
 119         try:
 120             subprocess.check_call(patchcmd + [patchfile])
 121         except subprocess.CalledProcessError:
 122             raise ValueError(f"Failed to apply patch to {self.manifest.name}")
 123         os.chdir(old_wd)
 124         patched_sentinel_file.touch()
 125
 126     def prepare(self, install_dirs, reconfigure: bool) -> None:
 127         print("Preparing %s..." % self.manifest.name)
 128         reconfigure = self._reconfigure(reconfigure)
 129         self._apply_patchfile()
 130         self._prepare(install_dirs=install_dirs, reconfigure=reconfigure)
 131
 132     def build(self, install_dirs, reconfigure: bool) -> None:
 133         print("Building %s..." % self.manifest.name)
 134         reconfigure = self._reconfigure(reconfigure)
 135         self._apply_patchfile()
 136         self._prepare(install_dirs=install_dirs, reconfigure=reconfigure)
 137         self._build(install_dirs=install_dirs, reconfigure=reconfigure)
 138
 139         # On Windows, emit a wrapper script that can be used to run build artifacts
 140         # directly from the build directory, without installing them.  On Windows $PATH
 141         # needs to be updated to include all of the directories containing the runtime
 142         # library dependencies in order to run the binaries.
 143         if self.build_opts.is_windows():
 144             script_path = self.get_dev_run_script_path()
 145             dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs)
 146             dep_dirs = self.get_dev_run_extra_path_dirs(install_dirs, dep_munger)
 147             # pyre-fixme[16]: Optional type has no attribute `emit_dev_run_script`.
 148             dep_munger.emit_dev_run_script(script_path, dep_dirs)
 149
 150     @property
 151     def num_jobs(self) -> int:
 152         # This is a hack, but we don't have a "defaults manifest" that we can
 153         # customize per platform.
 154         # TODO: Introduce some sort of defaults config that can select by
 155         # platform, just like manifest contexts.
 156         if sys.platform.startswith("freebsd"):
 157             # clang on FreeBSD is quite memory-efficient.
 158             default_job_weight = 512
 159         else:
 160             # 1.5 GiB is a lot to assume, but it's typical of Facebook-style C++.
 161             # Some manifests are even heavier and should override.
 162             default_job_weight = 1536
 163         return self.build_opts.get_num_jobs(
 164             int(
 165                 self.manifest.get(
 166                     "build", "job_weight_mib", default_job_weight, ctx=self.ctx
 167                 )
 168             )
 169         )
 170
 171     def run_tests(
 172         self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
 173     ) -> None:
 174         """Execute any tests that we know how to run.  If they fail,
 175         raise an exception."""
 176         pass
 177
 178     def _prepare(self, install_dirs, reconfigure) -> None:
 179         """Prepare the build. Useful when need to generate config,
 180         but builder is not the primary build system.
 181         e.g. cargo when called from cmake"""
 182         pass
 183
 184     def _build(self, install_dirs, reconfigure) -> None:
 185         """Perform the build.
 186         install_dirs contains the list of installation directories for
 187         the dependencies of this project.
 188         reconfigure will be set to true if the fetcher determined
 189         that the sources have changed in such a way that the build
 190         system needs to regenerate its rules."""
 191         pass
 192
 193     def _compute_env(self, install_dirs):
 194         # CMAKE_PREFIX_PATH is only respected when passed through the
 195         # environment, so we construct an appropriate path to pass down
 196         return self.build_opts.compute_env_for_install_dirs(
 197             install_dirs, env=self.env, manifest=self.manifest
 198         )
 199
 200     def get_dev_run_script_path(self):
 201         assert self.build_opts.is_windows()
 202         return os.path.join(self.build_dir, "run.ps1")
 203
 204     def get_dev_run_extra_path_dirs(self, install_dirs, dep_munger=None):
 205         assert self.build_opts.is_windows()
 206         if dep_munger is None:
 207             dep_munger = create_dyn_dep_munger(self.build_opts, install_dirs)
 208         return dep_munger.compute_dependency_paths(self.build_dir)
 209
 210
 211 class MakeBuilder(BuilderBase):
 212     def __init__(
 213         self,
 214         build_opts,
 215         ctx,
 216         manifest,
 217         src_dir,
 218         build_dir,
 219         inst_dir,
 220         build_args,
 221         install_args,
 222         test_args,
 223     ) -> None:
 224         super(MakeBuilder, self).__init__(
 225             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
 226         )
 227         self.build_args = build_args or []
 228         self.install_args = install_args or []
 229         self.test_args = test_args
 230
 231     @property
 232     def _make_binary(self):
 233         return self.manifest.get("build", "make_binary", "make", ctx=self.ctx)
 234
 235     def _get_prefix(self):
 236         return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
 237
 238     def _build(self, install_dirs, reconfigure) -> None:
 239
 240         env = self._compute_env(install_dirs)
 241
 242         # Need to ensure that PREFIX is set prior to install because
 243         # libbpf uses it when generating its pkg-config file.
 244         # The lowercase prefix is used by some projects.
 245         cmd = (
 246             [self._make_binary, "-j%s" % self.num_jobs]
 247             + self.build_args
 248             + self._get_prefix()
 249         )
 250         self._run_cmd(cmd, env=env)
 251
 252         install_cmd = [self._make_binary] + self.install_args + self._get_prefix()
 253         self._run_cmd(install_cmd, env=env)
 254
 255         # bz2's Makefile doesn't install its .so properly
 256         if self.manifest and self.manifest.name == "bz2":
 257             libdir = os.path.join(self.inst_dir, "lib")
 258             srcpattern = os.path.join(self.src_dir, "lib*.so.*")
 259             print(f"copying to {libdir} from {srcpattern}")
 260             for file in glob.glob(srcpattern):
 261                 shutil.copy(file, libdir)
 262
 263     def run_tests(
 264         self, install_dirs, schedule_type, owner, test_filter, retry, no_testpilot
 265     ) -> None:
 266         if not self.test_args:
 267             return
 268
 269         env = self._compute_env(install_dirs)
 270
 271         cmd = [self._make_binary] + self.test_args + self._get_prefix()
 272         self._run_cmd(cmd, env=env)
 273
 274
 275 class CMakeBootStrapBuilder(MakeBuilder):
 276     def _build(self, install_dirs, reconfigure) -> None:
 277         self._run_cmd(
 278             [
 279                 "./bootstrap",
 280                 "--prefix=" + self.inst_dir,
 281                 f"--parallel={self.num_jobs}",
 282             ]
 283         )
 284         super(CMakeBootStrapBuilder, self)._build(install_dirs, reconfigure)
 285
 286
 287 class AutoconfBuilder(BuilderBase):
 288     def __init__(
 289         self,
 290         build_opts,
 291         ctx,
 292         manifest,
 293         src_dir,
 294         build_dir,
 295         inst_dir,
 296         args,
 297         conf_env_args,
 298     ) -> None:
 299         super(AutoconfBuilder, self).__init__(
 300             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
 301         )
 302         self.args = args or []
 303         self.conf_env_args = conf_env_args or {}
 304
 305     @property
 306     def _make_binary(self):
 307         return self.manifest.get("build", "make_binary", "make", ctx=self.ctx)
 308
 309     def _build(self, install_dirs, reconfigure) -> None:
 310         configure_path = os.path.join(self.src_dir, "configure")
 311         autogen_path = os.path.join(self.src_dir, "autogen.sh")
 312
 313         env = self._compute_env(install_dirs)
 314
 315         # Some configure scripts need additional env values passed derived from cmds
 316         for (k, cmd_args) in self.conf_env_args.items():
 317             out = (
 318                 subprocess.check_output(cmd_args, env=dict(env.items()))
 319                 .decode("utf-8")
 320                 .strip()
 321             )
 322             if out:
 323                 env.set(k, out)
 324
 325         if not os.path.exists(configure_path):
 326             print("%s doesn't exist, so reconfiguring" % configure_path)
 327             # This libtoolize call is a bit gross; the issue is that
 328             # `autoreconf` as invoked by libsodium's `autogen.sh` doesn't
 329             # seem to realize that it should invoke libtoolize and then
 330             # error out when the configure script references a libtool
 331             # related symbol.
 332             self._run_cmd(["libtoolize"], cwd=self.src_dir, env=env)
 333
 334             # We generally prefer to call the `autogen.sh` script provided
 335             # by the project on the basis that it may know more than plain
 336             # autoreconf does.
 337             if os.path.exists(autogen_path):
 338                 self._run_cmd(["bash", autogen_path], cwd=self.src_dir, env=env)
 339             else:
 340                 self._run_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env)
 341         configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args
 342         self._run_cmd(configure_cmd, env=env)
 343         self._run_cmd([self._make_binary, "-j%s" % self.num_jobs], env=env)
 344         self._run_cmd([self._make_binary, "install"], env=env)
 345
 346
 347 class Iproute2Builder(BuilderBase):
 348     # ./configure --prefix does not work for iproute2.
 349     # Thus, explicitly copy sources from src_dir to build_dir, bulid,
 350     # and then install to inst_dir using DESTDIR
 351     # lastly, also copy include from build_dir to inst_dir
 352     def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir) -> None:
 353         super(Iproute2Builder, self).__init__(
 354             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
 355         )
 356
 357     def _patch(self) -> None:
 358         # FBOSS build currently depends on an old version of iproute2 (commit
 359         # 7ca63aef7d1b0c808da0040c6b366ef7a61f38c1). This is missing a commit
 360         # (ae717baf15fb4d30749ada3948d9445892bac239) needed to build iproute2
 361         # successfully. Apply it viz.: include stdint.h
 362         # Reference: https://fburl.com/ilx9g5xm
 363         with open(self.build_dir + "/tc/tc_core.c", "r") as f:
 364             data = f.read()
 365
 366         with open(self.build_dir + "/tc/tc_core.c", "w") as f:
 367             f.write("#include <stdint.h>\n")
 368             f.write(data)
 369
 370     def _build(self, install_dirs, reconfigure) -> None:
 371         configure_path = os.path.join(self.src_dir, "configure")
 372
 373         env = self.env.copy()
 374         self._run_cmd([configure_path], env=env)
 375         shutil.rmtree(self.build_dir)
 376         shutil.copytree(self.src_dir, self.build_dir)
 377         self._patch()
 378         self._run_cmd(["make", "-j%s" % self.num_jobs], env=env)
 379         install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir]
 380
 381         for d in ["include", "lib"]:
 382             if not os.path.isdir(os.path.join(self.inst_dir, d)):
 383                 shutil.copytree(
 384                     os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d)
 385                 )
 386
 387         self._run_cmd(install_cmd, env=env)
 388
 389
 390 class CMakeBuilder(BuilderBase):
 391     MANUAL_BUILD_SCRIPT = """\
 392 #!{sys.executable}
 393
 394
 395 import argparse
 396 import subprocess
 397 import sys
 398
 399 CMAKE = {cmake!r}
 400 CTEST = {ctest!r}
 401 SRC_DIR = {src_dir!r}
 402 BUILD_DIR = {build_dir!r}
 403 INSTALL_DIR = {install_dir!r}
 404 CMD_PREFIX = {cmd_prefix!r}
 405 CMAKE_ENV = {env_str}
 406 CMAKE_DEFINE_ARGS = {define_args_str}
 407
 408
 409 def get_jobs_argument(num_jobs_arg: int) -> str:
 410     if num_jobs_arg > 0:
 411         return "-j" + str(num_jobs_arg)
 412
 413     import multiprocessing
 414     num_jobs = multiprocessing.cpu_count() // 2
 415     return "-j" + str(num_jobs)
 416
 417
 418 def main():
 419     ap = argparse.ArgumentParser()
 420     ap.add_argument(
 421       "cmake_args",
 422       nargs=argparse.REMAINDER,
 423       help='Any extra arguments after an "--" argument will be passed '
 424       "directly to CMake."
 425     )
 426     ap.add_argument(
 427       "--mode",
 428       choices=["configure", "build", "install", "test"],
 429       default="configure",
 430       help="The mode to run: configure, build, or install.  "
 431       "Defaults to configure",
 432     )
 433     ap.add_argument(
 434       "--build",
 435       action="store_const",
 436       const="build",
 437       dest="mode",
 438       help="An alias for --mode=build",
 439     )
 440     ap.add_argument(
 441       "-j",
 442       "--num-jobs",
 443       action="store",
 444       type=int,
 445       default=0,
 446       help="Run the build or tests with the specified number of parallel jobs",
 447     )
 448     ap.add_argument(
 449       "--install",
 450       action="store_const",
 451       const="install",
 452       dest="mode",
 453       help="An alias for --mode=install",
 454     )
 455     ap.add_argument(
 456       "--test",
 457       action="store_const",
 458       const="test",
 459       dest="mode",
 460       help="An alias for --mode=test",
 461     )
 462     args = ap.parse_args()
 463
 464     # Strip off a leading "--" from the additional CMake arguments
 465     if args.cmake_args and args.cmake_args[0] == "--":
 466         args.cmake_args = args.cmake_args[1:]
 467
 468     env = CMAKE_ENV
 469
 470     if args.mode == "configure":
 471         full_cmd = CMD_PREFIX + [CMAKE, SRC_DIR] + CMAKE_DEFINE_ARGS + args.cmake_args
 472     elif args.mode in ("build", "install"):
 473         target = "all" if args.mode == "build" else "install"
 474         full_cmd = CMD_PREFIX + [
 475                 CMAKE,
 476                 "--build",
 477                 BUILD_DIR,
 478                 "--target",
 479                 target,
 480                 "--config",
 481                 "Release",
 482                 get_jobs_argument(args.num_jobs),
 483         ] + args.cmake_args
 484     elif args.mode == "test":
 485         full_cmd = CMD_PREFIX + [
 486             {dev_run_script}CTEST,
 487             "--output-on-failure",
 488             get_jobs_argument(args.num_jobs),
 489         ] + args.cmake_args
 490     else:
 491         ap.error("unknown invocation mode: %s" % (args.mode,))
 492
 493     cmd_str = " ".join(full_cmd)
 494     print("Running: %r" % (cmd_str,))
 495     proc = subprocess.run(full_cmd, env=env, cwd=BUILD_DIR)
 496     sys.exit(proc.returncode)
 497
 498
 499 if __name__ == "__main__":
 500     main()
 501 """
 502
 503     def __init__(
 504         self,
 505         build_opts,
 506         ctx,
 507         manifest,
 508         src_dir,
 509         build_dir,
 510         inst_dir,
 511         defines,
 512         loader=None,
 513         final_install_prefix=None,
 514         extra_cmake_defines=None,
 515     ) -> None:
 516         super(CMakeBuilder, self).__init__(
 517             build_opts,
 518             ctx,
 519             manifest,
 520             src_dir,
 521             build_dir,
 522             inst_dir,
 523             final_install_prefix=final_install_prefix,
 524         )
 525         self.defines = defines or {}
 526         if extra_cmake_defines:
 527             self.defines.update(extra_cmake_defines)
 528
 529         try:
 530             from .facebook.vcvarsall import extra_vc_cmake_defines
 531         except ImportError:
 532             pass
 533         else:
 534             self.defines.update(extra_vc_cmake_defines)
 535
 536         self.loader = loader
 537         if build_opts.shared_libs:
 538             self.defines["BUILD_SHARED_LIBS"] = "ON"
 539
 540     def _invalidate_cache(self) -> None:
 541         for name in [
 542             "CMakeCache.txt",
 543             "CMakeFiles/CMakeError.log",
 544             "CMakeFiles/CMakeOutput.log",
 545         ]:
 546             name = os.path.join(self.build_dir, name)
 547             if os.path.isdir(name):
 548                 shutil.rmtree(name)
 549             elif os.path.exists(name):
 550                 os.unlink(name)
 551
 552     def _needs_reconfigure(self) -> bool:
 553         for name in ["CMakeCache.txt", "build.ninja"]:
 554             name = os.path.join(self.build_dir, name)
 555             if not os.path.exists(name):
 556                 return True
 557         return False
 558
 559     def _write_build_script(self, **kwargs) -> None:
 560         env_lines = ["    {!r}: {!r},".format(k, v) for k, v in kwargs["env"].items()]
 561         kwargs["env_str"] = "\n".join(["{"] + env_lines + ["}"])
 562
 563         if self.build_opts.is_windows():
 564             kwargs["dev_run_script"] = '"powershell.exe", {!r}, '.format(
 565                 self.get_dev_run_script_path()
 566             )
 567         else:
 568             kwargs["dev_run_script"] = ""
 569
 570         define_arg_lines = ["["]
 571         for arg in kwargs["define_args"]:
 572             # Replace the CMAKE_INSTALL_PREFIX argument to use the INSTALL_DIR
 573             # variable that we define in the MANUAL_BUILD_SCRIPT code.
 574             if arg.startswith("-DCMAKE_INSTALL_PREFIX="):
 575                 value = "    {!r}.format(INSTALL_DIR),".format(
 576                     "-DCMAKE_INSTALL_PREFIX={}"
 577                 )
 578             else:
 579                 value = "    {!r},".format(arg)
 580             define_arg_lines.append(value)
 581         define_arg_lines.append("]")
 582         kwargs["define_args_str"] = "\n".join(define_arg_lines)
 583
 584         # In order to make it easier for developers to manually run builds for
 585         # CMake-based projects, write out some build scripts that can be used to invoke
 586         # CMake manually.
 587         build_script_path = os.path.join(self.build_dir, "run_cmake.py")
 588         script_contents = self.MANUAL_BUILD_SCRIPT.format(**kwargs)
 589         with open(build_script_path, "wb") as f:
 590             f.write(script_contents.encode())
 591         os.chmod(build_script_path, 0o755)
 592
 593     def _compute_cmake_define_args(self, env):
 594         defines = {
 595             "CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir,
 596             "BUILD_SHARED_LIBS": "OFF",
 597             # Some of the deps (rsocket) default to UBSAN enabled if left
 598             # unspecified.  Some of the deps fail to compile in release mode
 599             # due to warning->error promotion.  RelWithDebInfo is the happy
 600             # medium.
 601             "CMAKE_BUILD_TYPE": "RelWithDebInfo",
 602         }
 603         if "SANDCASTLE" not in os.environ:
 604             # We sometimes see intermittent ccache related breakages on some
 605             # of the FB internal CI hosts, so we prefer to disable ccache
 606             # when running in that environment.
 607             ccache = path_search(env, "ccache")
 608             if ccache:
 609                 defines["CMAKE_CXX_COMPILER_LAUNCHER"] = ccache
 610         else:
 611             # rocksdb does its own probing for ccache.
 612             # Ensure that it is disabled on sandcastle
 613             env["CCACHE_DISABLE"] = "1"
 614             # Some sandcastle hosts have broken ccache related dirs, and
 615             # even though we've asked for it to be disabled ccache is
 616             # still invoked by rocksdb's cmake.
 617             # Redirect its config directory to somewhere that is guaranteed
 618             # fresh to us, and that won't have any ccache data inside.
 619             env["CCACHE_DIR"] = f"{self.build_opts.scratch_dir}/ccache"
 620
 621         if "GITHUB_ACTIONS" in os.environ and self.build_opts.is_windows():
 622             # GitHub actions: the host has both gcc and msvc installed, and
 623             # the default behavior of cmake is to prefer gcc.
 624             # Instruct cmake that we want it to use cl.exe; this is important
 625             # because Boost prefers cl.exe and the mismatch results in cmake
 626             # with gcc not being able to find boost built with cl.exe.
 627             defines["CMAKE_C_COMPILER"] = "cl.exe"
 628             defines["CMAKE_CXX_COMPILER"] = "cl.exe"
 629
 630         if self.build_opts.is_darwin():
 631             # Try to persuade cmake to set the rpath to match the lib
 632             # dirs of the dependencies.  This isn't automatic, and to
 633             # make things more interesting, cmake uses `;` as the path
 634             # separator, so translate the runtime path to something
 635             # that cmake will parse
 636             defines["CMAKE_INSTALL_RPATH"] = ";".join(
 637                 env.get("DYLD_LIBRARY_PATH", "").split(":")
 638             )
 639             # Tell cmake that we want to set the rpath in the tree
 640             # at build time.  Without this the rpath is only set
 641             # at the moment that the binaries are installed.  That
 642             # default is problematic for example when using the
 643             # gtest integration in cmake which runs the built test
 644             # executables during the build to discover the set of
 645             # tests.
 646             defines["CMAKE_BUILD_WITH_INSTALL_RPATH"] = "ON"
 647
 648         boost_169_is_required = False
 649         if self.loader:
 650             for m in self.loader.manifests_in_dependency_order():
 651                 preinstalled = m.get_section_as_dict("preinstalled.env", self.ctx)
 652                 boost_169_is_required = "BOOST_ROOT_1_69_0" in preinstalled.keys()
 653                 if boost_169_is_required:
 654                     break
 655
 656         if (
 657             boost_169_is_required
 658             and self.build_opts.allow_system_packages
 659             and self.build_opts.host_type.get_package_manager()
 660             and self.build_opts.host_type.get_package_manager() == "rpm"
 661         ):
 662             # Boost 1.69 rpms don't install cmake config to the system, so to point to them explicitly
 663             defines["BOOST_INCLUDEDIR"] = "/usr/include/boost169"
 664             defines["BOOST_LIBRARYDIR"] = "/usr/lib64/boost169"
 665
 666         defines.update(self.defines)
 667         define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
 668
 669         # if self.build_opts.is_windows():
 670         #    define_args += ["-G", "Visual Studio 15 2017 Win64"]
 671         define_args += ["-G", "Ninja"]
 672
 673         return define_args
 674
 675     def _build(self, install_dirs, reconfigure: bool) -> None:
 676         reconfigure = reconfigure or self._needs_reconfigure()
 677
 678         env = self._compute_env(install_dirs)
 679         if not self.build_opts.is_windows() and self.final_install_prefix:
 680             env["DESTDIR"] = self.inst_dir
 681
 682         # Resolve the cmake that we installed
 683         cmake = path_search(env, "cmake")
 684         if cmake is None:
 685             raise Exception("Failed to find CMake")
 686
 687         if reconfigure:
 688             define_args = self._compute_cmake_define_args(env)
 689             self._write_build_script(
 690                 cmd_prefix=self._get_cmd_prefix(),
 691                 cmake=cmake,
 692                 ctest=path_search(env, "ctest"),
 693                 env=env,
 694                 define_args=define_args,
 695                 src_dir=self.src_dir,
 696                 build_dir=self.build_dir,
 697                 install_dir=self.inst_dir,
 698                 sys=sys,
 699             )
 700
 701             self._invalidate_cache()
 702             self._run_cmd([cmake, self.src_dir] + define_args, env=env)
 703
 704         self._run_cmd(
 705             [
 706                 cmake,
 707                 "--build",
 708                 self.build_dir,
 709                 "--target",
 710                 "install",
 711                 "--config",
 712                 "Release",
 713                 "-j",
 714                 str(self.num_jobs),
 715             ],
 716             env=env,
 717         )
 718
 719     def run_tests(
 720         self, install_dirs, schedule_type, owner, test_filter, retry: int, no_testpilot
 721     ) -> None:
 722         env = self._compute_env(install_dirs)
 723         ctest = path_search(env, "ctest")
 724         cmake = path_search(env, "cmake")
 725
 726         def require_command(path: Optional[str], name: str) -> str:
 727             if path is None:
 728                 raise RuntimeError("unable to find command `{}`".format(name))
 729             return path
 730
 731         # On Windows, we also need to update $PATH to include the directories that
 732         # contain runtime library dependencies.  This is not needed on other platforms
 733         # since CMake will emit RPATH properly in the binary so they can find these
 734         # dependencies.
 735         if self.build_opts.is_windows():
 736             path_entries = self.get_dev_run_extra_path_dirs(install_dirs)
 737             path = env.get("PATH")
 738             if path:
 739                 path_entries.insert(0, path)
 740             env["PATH"] = ";".join(path_entries)
 741
 742         # Don't use the cmd_prefix when running tests.  This is vcvarsall.bat on
 743         # Windows.  vcvarsall.bat is only needed for the build, not tests.  It
 744         # unfortunately fails if invoked with a long PATH environment variable when
 745         # running the tests.
 746         use_cmd_prefix = False
 747
 748         def get_property(test, propname, defval=None):
 749             """extracts a named property from a cmake test info json blob.
 750             The properties look like:
 751             [{"name": "WORKING_DIRECTORY"},
 752              {"value": "something"}]
 753             We assume that it is invalid for the same named property to be
 754             listed more than once.
 755             """
 756             props = test.get("properties", [])
 757             for p in props:
 758                 if p.get("name", None) == propname:
 759                     return p.get("value", defval)
 760             return defval
 761
 762         def list_tests():
 763             output = subprocess.check_output(
 764                 [require_command(ctest, "ctest"), "--show-only=json-v1"],
 765                 env=env,
 766                 cwd=self.build_dir,
 767             )
 768             try:
 769                 data = json.loads(output.decode("utf-8"))
 770             except ValueError as exc:
 771                 raise Exception(
 772                     "Failed to decode cmake test info using %s: %s.  Output was: %r"
 773                     % (ctest, str(exc), output)
 774                 )
 775
 776             tests = []
 777             machine_suffix = self.build_opts.host_type.as_tuple_string()
 778             for test in data["tests"]:
 779                 working_dir = get_property(test, "WORKING_DIRECTORY")
 780                 labels = []
 781                 machine_suffix = self.build_opts.host_type.as_tuple_string()
 782                 labels.append("tpx-fb-test-type=3")
 783                 labels.append("tpx_test_config::buildsystem=getdeps")
 784                 labels.append("tpx_test_config::platform={}".format(machine_suffix))
 785
 786                 if get_property(test, "DISABLED"):
 787                     labels.append("disabled")
 788                 command = test["command"]
 789                 if working_dir:
 790                     command = [
 791                         require_command(cmake, "cmake"),
 792                         "-E",
 793                         "chdir",
 794                         working_dir,
 795                     ] + command
 796
 797                 import os
 798
 799                 tests.append(
 800                     {
 801                         "type": "custom",
 802                         "target": "%s-%s-getdeps-%s"
 803                         % (self.manifest.name, test["name"], machine_suffix),
 804                         "command": command,
 805                         "labels": labels,
 806                         "env": {},
 807                         "required_paths": [],
 808                         "contacts": [],
 809                         "cwd": os.getcwd(),
 810                     }
 811                 )
 812             return tests
 813
 814         if schedule_type == "continuous" or schedule_type == "testwarden":
 815             # for continuous and testwarden runs, disabling retry can give up
 816             # better signals for flaky tests.
 817             retry = 0
 818
 819         tpx = path_search(env, "tpx")
 820         if tpx and not no_testpilot:
 821             buck_test_info = list_tests()
 822             import os
 823
 824             from .facebook.testinfra import start_run
 825
 826             buck_test_info_name = os.path.join(self.build_dir, ".buck-test-info.json")
 827             with open(buck_test_info_name, "w") as f:
 828                 json.dump(buck_test_info, f)
 829
 830             env.set("http_proxy", "")
 831             env.set("https_proxy", "")
 832             runs = []
 833             from sys import platform
 834
 835             with start_run(env["FBSOURCE_HASH"]) as run_id:
 836                 testpilot_args = [
 837                     tpx,
 838                     "--force-local-execution",
 839                     "--buck-test-info",
 840                     buck_test_info_name,
 841                     "--retry=%d" % retry,
 842                     "-j=%s" % str(self.num_jobs),
 843                     "--print-long-results",
 844                 ]
 845
 846                 if owner:
 847                     testpilot_args += ["--contacts", owner]
 848
 849                 if env:
 850                     testpilot_args.append("--env")
 851                     testpilot_args.extend(f"{key}={val}" for key, val in env.items())
 852
 853                 if run_id is not None:
 854                     testpilot_args += ["--run-id", run_id]
 855
 856                 if test_filter:
 857                     testpilot_args += ["--", test_filter]
 858
 859                 if schedule_type == "diff":
 860                     runs.append(["--collection", "oss-diff", "--purpose", "diff"])
 861                 elif schedule_type == "continuous":
 862                     runs.append(
 863                         [
 864                             "--tag-new-tests",
 865                             "--collection",
 866                             "oss-continuous",
 867                             "--purpose",
 868                             "continuous",
 869                         ]
 870                     )
 871                 elif schedule_type == "testwarden":
 872                     # One run to assess new tests
 873                     runs.append(
 874                         [
 875                             "--tag-new-tests",
 876                             "--collection",
 877                             "oss-new-test-stress",
 878                             "--stress-runs",
 879                             "10",
 880                             "--purpose",
 881                             "stress-run-new-test",
 882                         ]
 883                     )
 884                     # And another for existing tests
 885                     runs.append(
 886                         [
 887                             "--tag-new-tests",
 888                             "--collection",
 889                             "oss-existing-test-stress",
 890                             "--stress-runs",
 891                             "10",
 892                             "--purpose",
 893                             "stress-run",
 894                         ]
 895                     )
 896                 else:
 897                     runs.append([])
 898
 899                 for run in runs:
 900                     self._run_cmd(
 901                         testpilot_args + run,
 902                         cwd=self.build_opts.fbcode_builder_dir,
 903                         env=env,
 904                         use_cmd_prefix=use_cmd_prefix,
 905                     )
 906         else:
 907             args = [
 908                 require_command(ctest, "ctest"),
 909                 "--output-on-failure",
 910                 "-j",
 911                 str(self.num_jobs),
 912             ]
 913             if test_filter:
 914                 args += ["-R", test_filter]
 915
 916             count = 0
 917             while count <= retry:
 918                 retcode = self._run_cmd(
 919                     args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True
 920                 )
 921
 922                 if retcode == 0:
 923                     break
 924                 if count == 0:
 925                     # Only add this option in the second run.
 926                     args += ["--rerun-failed"]
 927                 count += 1
 928             # pyre-fixme[61]: `retcode` is undefined, or not always defined.
 929             if retcode != 0:
 930                 # Allow except clause in getdeps.main to catch and exit gracefully
 931                 # This allows non-testpilot runs to fail through the same logic as failed testpilot runs, which may become handy in case if post test processing is needed in the future
 932                 # pyre-fixme[61]: `retcode` is undefined, or not always defined.
 933                 raise subprocess.CalledProcessError(retcode, args)
 934
 935
 936 class NinjaBootstrap(BuilderBase):
 937     def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir) -> None:
 938         super(NinjaBootstrap, self).__init__(
 939             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
 940         )
 941
 942     def _build(self, install_dirs, reconfigure) -> None:
 943         self._run_cmd([sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir)
 944         src_ninja = os.path.join(self.src_dir, "ninja")
 945         dest_ninja = os.path.join(self.inst_dir, "bin/ninja")
 946         bin_dir = os.path.dirname(dest_ninja)
 947         if not os.path.exists(bin_dir):
 948             os.makedirs(bin_dir)
 949         shutil.copyfile(src_ninja, dest_ninja)
 950         shutil.copymode(src_ninja, dest_ninja)
 951
 952
 953 class OpenSSLBuilder(BuilderBase):
 954     def __init__(self, build_opts, ctx, manifest, build_dir, src_dir, inst_dir) -> None:
 955         super(OpenSSLBuilder, self).__init__(
 956             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
 957         )
 958
 959     def _build(self, install_dirs, reconfigure) -> None:
 960         configure = os.path.join(self.src_dir, "Configure")
 961
 962         # prefer to resolve the perl that we installed from
 963         # our manifest on windows, but fall back to the system
 964         # path on eg: darwin
 965         env = self.env.copy()
 966         for d in install_dirs:
 967             bindir = os.path.join(d, "bin")
 968             add_path_entry(env, "PATH", bindir, append=False)
 969
 970         perl = typing.cast(str, path_search(env, "perl", "perl"))
 971
 972         make_j_args = []
 973         if self.build_opts.is_windows():
 974             make = "nmake.exe"
 975             args = ["VC-WIN64A-masm", "-utf-8"]
 976         elif self.build_opts.is_darwin():
 977             make = "make"
 978             make_j_args = ["-j%s" % self.num_jobs]
 979             args = (
 980                 ["darwin64-x86_64-cc"]
 981                 if not self.build_opts.is_arm()
 982                 else ["darwin64-arm64-cc"]
 983             )
 984         elif self.build_opts.is_linux():
 985             make = "make"
 986             make_j_args = ["-j%s" % self.num_jobs]
 987             args = (
 988                 ["linux-x86_64"] if not self.build_opts.is_arm() else ["linux-aarch64"]
 989             )
 990         else:
 991             raise Exception("don't know how to build openssl for %r" % self.ctx)
 992
 993         self._run_cmd(
 994             [
 995                 perl,
 996                 configure,
 997                 "--prefix=%s" % self.inst_dir,
 998                 "--openssldir=%s" % self.inst_dir,
 999             ]
1000             + args
1001             + [
1002                 "enable-static-engine",
1003                 "enable-capieng",
1004                 "no-makedepend",
1005                 "no-unit-test",
1006                 "no-tests",
1007             ]
1008         )
1009         make_build = [make] + make_j_args
1010         self._run_cmd(make_build)
1011         make_install = [make, "install_sw", "install_ssldirs"]
1012         self._run_cmd(make_install)
1013
1014
1015 class Boost(BuilderBase):
1016     def __init__(
1017         self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir, b2_args
1018     ) -> None:
1019         children = os.listdir(src_dir)
1020         assert len(children) == 1, "expected a single directory entry: %r" % (children,)
1021         boost_src = children[0]
1022         assert boost_src.startswith("boost")
1023         src_dir = os.path.join(src_dir, children[0])
1024         super(Boost, self).__init__(
1025             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
1026         )
1027         self.b2_args = b2_args
1028
1029     def _build(self, install_dirs, reconfigure) -> None:
1030         env = self._compute_env(install_dirs)
1031         linkage = ["static"]
1032         if self.build_opts.is_windows() or self.build_opts.shared_libs:
1033             linkage.append("shared")
1034
1035         args = []
1036         if self.build_opts.is_darwin():
1037             clang = subprocess.check_output(["xcrun", "--find", "clang"])
1038             user_config = os.path.join(self.build_dir, "project-config.jam")
1039             with open(user_config, "w") as jamfile:
1040                 jamfile.write("using clang : : %s ;\n" % clang.decode().strip())
1041             args.append("--user-config=%s" % user_config)
1042
1043         for link in linkage:
1044             bootstrap_args = self.manifest.get_section_as_args(
1045                 "bootstrap.args", self.ctx
1046             )
1047             if self.build_opts.is_windows():
1048                 bootstrap = os.path.join(self.src_dir, "bootstrap.bat")
1049                 self._run_cmd([bootstrap] + bootstrap_args, cwd=self.src_dir, env=env)
1050                 args += ["address-model=64"]
1051             else:
1052                 bootstrap = os.path.join(self.src_dir, "bootstrap.sh")
1053                 self._run_cmd(
1054                     [bootstrap, "--prefix=%s" % self.inst_dir] + bootstrap_args,
1055                     cwd=self.src_dir,
1056                     env=env,
1057                 )
1058
1059             b2 = os.path.join(self.src_dir, "b2")
1060             self._run_cmd(
1061                 [
1062                     b2,
1063                     "-j%s" % self.num_jobs,
1064                     "--prefix=%s" % self.inst_dir,
1065                     "--builddir=%s" % self.build_dir,
1066                 ]
1067                 + args
1068                 + self.b2_args
1069                 + [
1070                     "link=%s" % link,
1071                     "runtime-link=shared",
1072                     "variant=release",
1073                     "threading=multi",
1074                     "debug-symbols=on",
1075                     "visibility=global",
1076                     "-d2",
1077                     "install",
1078                 ],
1079                 cwd=self.src_dir,
1080                 env=env,
1081             )
1082
1083
1084 class NopBuilder(BuilderBase):
1085     def __init__(self, build_opts, ctx, manifest, src_dir, inst_dir) -> None:
1086         super(NopBuilder, self).__init__(
1087             build_opts, ctx, manifest, src_dir, None, inst_dir
1088         )
1089
1090     def build(self, install_dirs, reconfigure: bool) -> None:
1091         print("Installing %s -> %s" % (self.src_dir, self.inst_dir))
1092         parent = os.path.dirname(self.inst_dir)
1093         if not os.path.exists(parent):
1094             os.makedirs(parent)
1095
1096         install_files = self.manifest.get_section_as_ordered_pairs(
1097             "install.files", self.ctx
1098         )
1099         if install_files:
1100             for src_name, dest_name in self.manifest.get_section_as_ordered_pairs(
1101                 "install.files", self.ctx
1102             ):
1103                 full_dest = os.path.join(self.inst_dir, dest_name)
1104                 full_src = os.path.join(self.src_dir, src_name)
1105
1106                 dest_parent = os.path.dirname(full_dest)
1107                 if not os.path.exists(dest_parent):
1108                     os.makedirs(dest_parent)
1109                 if os.path.isdir(full_src):
1110                     if not os.path.exists(full_dest):
1111                         shutil.copytree(full_src, full_dest)
1112                 else:
1113                     shutil.copyfile(full_src, full_dest)
1114                     shutil.copymode(full_src, full_dest)
1115                     # This is a bit gross, but the mac ninja.zip doesn't
1116                     # give ninja execute permissions, so force them on
1117                     # for things that look like they live in a bin dir
1118                     if os.path.dirname(dest_name) == "bin":
1119                         st = os.lstat(full_dest)
1120                         os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
1121         else:
1122             if not os.path.exists(self.inst_dir):
1123                 shutil.copytree(self.src_dir, self.inst_dir)
1124
1125
1126 class OpenNSABuilder(NopBuilder):
1127     # OpenNSA libraries are stored with git LFS. As a result, fetcher fetches
1128     # LFS pointers and not the contents. Use git-lfs to pull the real contents
1129     # before copying to install dir using NoopBuilder.
1130     # In future, if more builders require git-lfs, we would consider installing
1131     # git-lfs as part of the sandcastle infra as against repeating similar
1132     # logic for each builder that requires git-lfs.
1133     def __init__(self, build_opts, ctx, manifest, src_dir, inst_dir) -> None:
1134         super(OpenNSABuilder, self).__init__(
1135             build_opts, ctx, manifest, src_dir, inst_dir
1136         )
1137
1138     def build(self, install_dirs, reconfigure: bool) -> None:
1139         env = self._compute_env(install_dirs)
1140         self._run_cmd(["git", "lfs", "install", "--local"], cwd=self.src_dir, env=env)
1141         self._run_cmd(["git", "lfs", "pull"], cwd=self.src_dir, env=env)
1142
1143         super(OpenNSABuilder, self).build(install_dirs, reconfigure)
1144
1145
1146 class SqliteBuilder(BuilderBase):
1147     def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir) -> None:
1148         super(SqliteBuilder, self).__init__(
1149             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
1150         )
1151
1152     def _build(self, install_dirs, reconfigure) -> None:
1153         for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]:
1154             src = os.path.join(self.src_dir, f)
1155             dest = os.path.join(self.build_dir, f)
1156             copy_if_different(src, dest)
1157
1158         cmake_lists = """
1159 cmake_minimum_required(VERSION 3.1.3 FATAL_ERROR)
1160 project(sqlite3 C)
1161 add_library(sqlite3 STATIC sqlite3.c)
1162 # These options are taken from the defaults in Makefile.msc in
1163 # the sqlite distribution
1164 target_compile_definitions(sqlite3 PRIVATE
1165     -DSQLITE_ENABLE_COLUMN_METADATA=1
1166     -DSQLITE_ENABLE_FTS3=1
1167     -DSQLITE_ENABLE_RTREE=1
1168     -DSQLITE_ENABLE_GEOPOLY=1
1169     -DSQLITE_ENABLE_JSON1=1
1170     -DSQLITE_ENABLE_STMTVTAB=1
1171     -DSQLITE_ENABLE_DBPAGE_VTAB=1
1172     -DSQLITE_ENABLE_DBSTAT_VTAB=1
1173     -DSQLITE_INTROSPECTION_PRAGMAS=1
1174     -DSQLITE_ENABLE_DESERIALIZE=1
1175 )
1176 install(TARGETS sqlite3)
1177 install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
1178             """
1179
1180         with open(os.path.join(self.build_dir, "CMakeLists.txt"), "w") as f:
1181             f.write(cmake_lists)
1182
1183         defines = {
1184             "CMAKE_INSTALL_PREFIX": self.inst_dir,
1185             "BUILD_SHARED_LIBS": "ON" if self.build_opts.shared_libs else "OFF",
1186             "CMAKE_BUILD_TYPE": "RelWithDebInfo",
1187         }
1188         define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
1189         define_args += ["-G", "Ninja"]
1190
1191         env = self._compute_env(install_dirs)
1192
1193         # Resolve the cmake that we installed
1194         cmake = path_search(env, "cmake")
1195
1196         self._run_cmd([cmake, self.build_dir] + define_args, env=env)
1197         self._run_cmd(
1198             [
1199                 cmake,
1200                 "--build",
1201                 self.build_dir,
1202                 "--target",
1203                 "install",
1204                 "--config",
1205                 "Release",
1206                 "-j",
1207                 str(self.num_jobs),
1208             ],
1209             env=env,
1210         )