js/src/make-source-package.py

   1 #!/usr/bin/env -S python3 -B
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4 # You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import argparse
   7 import enum
   8 import logging
   9 import os
  10 import shutil
  11 import subprocess
  12 import sys
  13 from pathlib import Path
  14
  15 logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
  16
  17
  18 def find_command(names):
  19     """Search for command in `names`, and returns the first one that exists."""
  20
  21     for name in names:
  22         path = shutil.which(name)
  23         if path is not None:
  24             return name
  25
  26     return None
  27
  28
  29 def assert_command(env_var, name):
  30     """Assert that the command is not empty
  31     The command name comes from either environment variable or find_command.
  32     """
  33     if not name:
  34         logging.error("{} command not found".format(env_var))
  35         sys.exit(1)
  36
  37
  38 def parse_version(topsrc_dir):
  39     """Parse milestone.txt and return the entire milestone and major version."""
  40     milestone_file = topsrc_dir / "config" / "milestone.txt"
  41     if not milestone_file.is_file():
  42         return ("", "", "")
  43
  44     with milestone_file.open("r") as f:
  45         for line in f:
  46             line = line.strip()
  47             if not line:
  48                 continue
  49             if line.startswith("#"):
  50                 continue
  51
  52             v = line.split(".")
  53             return tuple((v + ["", ""])[:3])
  54
  55     return ("", "", "")
  56
  57
  58 tmp_dir = Path("/tmp")
  59
  60 tar = os.environ.get("TAR", find_command(["tar"]))
  61 assert_command("TAR", tar)
  62
  63 rsync = os.environ.get("RSYNC", find_command(["rsync"]))
  64 assert_command("RSYNC", rsync)
  65
  66 m4 = os.environ.get("M4", find_command(["m4"]))
  67 assert_command("M4", m4)
  68
  69 awk = os.environ.get("AWK", find_command(["awk"]))
  70 assert_command("AWK", awk)
  71
  72 src_dir = Path(os.environ.get("SRC_DIR", Path(__file__).parent.absolute()))
  73 mozjs_name = os.environ.get("MOZJS_NAME", "mozjs")
  74 staging_dir = Path(os.environ.get("STAGING", tmp_dir / "mozjs-src-pkg"))
  75 dist_dir = Path(os.environ.get("DIST", tmp_dir))
  76 topsrc_dir = src_dir.parent.parent.absolute()
  77
  78 parsed_major_version, parsed_minor_version, parsed_patch_version = parse_version(
  79     topsrc_dir
  80 )
  81
  82 major_version = os.environ.get("MOZJS_MAJOR_VERSION", parsed_major_version)
  83 minor_version = os.environ.get("MOZJS_MINOR_VERSION", parsed_minor_version)
  84 patch_version = os.environ.get("MOZJS_PATCH_VERSION", parsed_patch_version)
  85 alpha = os.environ.get("MOZJS_ALPHA", "")
  86
  87 version = "{}-{}.{}.{}".format(
  88     mozjs_name, major_version, minor_version, patch_version or alpha or "0"
  89 )
  90 target_dir = staging_dir / version
  91 package_name = "{}.tar.xz".format(version)
  92 package_file = dist_dir / package_name
  93 tar_opts = ["-Jcf"]
  94
  95 # Given there might be some external program that reads the following output,
  96 # use raw `print`, instead of logging.
  97 print("Environment:")
  98 print("    TAR = {}".format(tar))
  99 print("    RSYNC = {}".format(rsync))
 100 print("    M4 = {}".format(m4))
 101 print("    AWK = {}".format(awk))
 102 print("    STAGING = {}".format(staging_dir))
 103 print("    DIST = {}".format(dist_dir))
 104 print("    SRC_DIR = {}".format(src_dir))
 105 print("    MOZJS_NAME = {}".format(mozjs_name))
 106 print("    MOZJS_MAJOR_VERSION = {}".format(major_version))
 107 print("    MOZJS_MINOR_VERSION = {}".format(minor_version))
 108 print("    MOZJS_PATCH_VERSION = {}".format(patch_version))
 109 print("    MOZJS_ALPHA = {}".format(alpha))
 110 print("")
 111
 112 rsync_filter_list = """
 113 # Top-level config and build files
 114
 115 + /aclocal.m4
 116 + /client.mk
 117 + /configure.py
 118 + /LICENSE
 119 + /mach
 120 + /Makefile.in
 121 + /moz.build
 122 + /moz.configure
 123 + /test.mozbuild
 124 + /.babel-eslint.rc.js
 125 + /.eslintignore
 126 + /.eslintrc.js
 127 + /.flake8
 128 + /.gitignore
 129 + /.hgignore
 130 + /.lldbinit
 131 + /.prettierignore
 132 + /.prettierrc
 133 + /.ycm_extra_conf.py
 134
 135 # Additional libraries (optionally) used by SpiderMonkey
 136
 137 + /mfbt/**
 138 + /nsprpub/**
 139
 140 - /intl/icu/source/data
 141 - /intl/icu/source/test
 142 - /intl/icu/source/tools
 143 + /intl/icu/**
 144
 145 + /intl/icu_testdata/**
 146
 147 - /intl/components/gtest
 148 + /intl/components/**
 149
 150 + /memory/replace/dmd/dmd.py
 151 + /memory/build/**
 152 + /memory/moz.build
 153 + /memory/mozalloc/**
 154
 155 + /modules/fdlibm/**
 156 + /modules/zlib/**
 157
 158 + /mozglue/baseprofiler/**
 159 + /mozglue/build/**
 160 + /mozglue/interposers/**
 161 + /mozglue/misc/**
 162 + /mozglue/moz.build
 163 + /mozglue/static/**
 164
 165 + /tools/rb/fix_stacks.py
 166 + /tools/fuzzing/moz.build
 167 + /tools/fuzzing/interface/**
 168 + /tools/fuzzing/registry/**
 169 + /tools/fuzzing/libfuzzer/**
 170 + /tools/fuzzing/*.mozbuild
 171
 172 # Build system and dependencies
 173
 174 + /Cargo.lock
 175 + /build/**
 176 + /config/**
 177 + /python/**
 178
 179 + /.cargo/config.in
 180
 181 + /third_party/function2/**
 182 - /third_party/python/gyp
 183 + /third_party/python/**
 184 + /third_party/rust/**
 185 + /third_party/gemmology/**
 186 + /third_party/xsimd/**
 187 + /layout/tools/reftest/reftest/**
 188
 189 + /testing/mach_commands.py
 190 + /testing/moz.build
 191 + /testing/mozbase/**
 192 + /testing/performance/**
 193 + /testing/web-platform/*.ini
 194 + /testing/web-platform/*.py
 195 + /testing/web-platform/meta/streams/**
 196 + /testing/web-platform/mozilla/**
 197 + /testing/web-platform/tests/resources/**
 198 + /testing/web-platform/tests/streams/**
 199 + /testing/web-platform/tests/tools/**
 200
 201 + /toolkit/crashreporter/tools/symbolstore.py
 202 + /toolkit/mozapps/installer/package-name.mk
 203
 204 + /xpcom/geckoprocesstypes_generator/**
 205
 206 # SpiderMonkey itself
 207
 208 + /js/src/**
 209 + /js/app.mozbuild
 210 + /js/*.configure
 211 + /js/examples/**
 212 + /js/public/**
 213
 214 + */
 215 - /**
 216 """
 217
 218 INSTALL_CONTENT = """\
 219 Documentation for SpiderMonkey is available at:
 220
 221   https://spidermonkey.dev/
 222
 223 In particular, it points to build documentation at
 224
 225   https://firefox-source-docs.mozilla.org/js/build.html
 226
 227 Note that the libraries produced by the build system include symbols,
 228 causing the binaries to be extremely large. It is highly suggested that `strip`
 229 be run over the binaries before deploying them.
 230
 231 Building with default options may be performed as follows:
 232
 233   ./mach build
 234
 235 This will produce a debug build (much more suitable for developing against the
 236 SpiderMonkey JSAPI). To produce an optimized build:
 237
 238   export MOZCONFIG=$(pwd)/mozconfig.opt
 239   ./mach build
 240
 241 You may edit the mozconfig and mozconfig.opt files to configure your own build
 242 appropriately.
 243 """
 244
 245 MOZCONFIG_DEBUG_CONTENT = """\
 246 # Much slower when running, but adds assertions that are much better for
 247 # developing against the JSAPI.
 248 ac_add_options --enable-debug
 249
 250 # Much faster when running, worse for debugging.
 251 ac_add_options --enable-optimize
 252
 253 mk_add_options MOZ_OBJDIR=obj-debug
 254 """
 255
 256 MOZCONFIG_OPT_CONTENT = """\
 257 # Much faster when running, but very error-prone to develop against because
 258 # this will skip all the assertions critical to using the JSAPI properly.
 259 ac_add_options --disable-debug
 260
 261 # Much faster when running, worse for debugging.
 262 ac_add_options --enable-optimize
 263
 264 mk_add_options MOZ_OBJDIR=obj-opt
 265 """
 266
 267 README_CONTENT = """\
 268 This directory contains SpiderMonkey {major_version}.
 269
 270 This release is based on a revision of Mozilla {major_version}:
 271   https://hg.mozilla.org/releases/
 272 The changes in the patches/ directory were applied.
 273
 274 See https://spidermonkey.dev/ for documentation, examples, and release notes.
 275 """.format(
 276     major_version=major_version
 277 )
 278
 279
 280 def is_mozjs_cargo_member(line):
 281     """Checks if the line in workspace.members is mozjs-related"""
 282
 283     return '"js/' in line
 284
 285
 286 def is_mozjs_crates_io_local_patch(line):
 287     """Checks if the line in patch.crates-io is mozjs-related"""
 288
 289     return any(
 290         f'path = "{p}' in line for p in ("js", "build", "third_party/rust", "intl")
 291     )
 292
 293
 294 def clean():
 295     """Remove temporary directory and package file."""
 296     logging.info("Cleaning {} and {} ...".format(package_file, target_dir))
 297     if package_file.exists():
 298         package_file.unlink()
 299     if target_dir.exists():
 300         shutil.rmtree(str(target_dir))
 301
 302
 303 def assert_clean():
 304     """Assert that target directory does not contain generated files."""
 305     makefile_file = target_dir / "js" / "src" / "Makefile"
 306     if makefile_file.exists():
 307         logging.error("found js/src/Makefile. Please clean before packaging.")
 308         sys.exit(1)
 309
 310
 311 def create_target_dir():
 312     if target_dir.exists():
 313         logging.warning("dist tree {} already exists!".format(target_dir))
 314     else:
 315         target_dir.mkdir(parents=True)
 316
 317
 318 def sync_files():
 319     # Output of the command should directly go to stdout/stderr.
 320     p = subprocess.Popen(
 321         [
 322             str(rsync),
 323             "--delete-excluded",
 324             "--prune-empty-dirs",
 325             "--quiet",
 326             "--recursive",
 327             "{}/".format(topsrc_dir),
 328             "{}/".format(target_dir),
 329             "--filter=. -",
 330         ],
 331         stdin=subprocess.PIPE,
 332     )
 333
 334     p.communicate(rsync_filter_list.encode())
 335
 336     if p.returncode != 0:
 337         sys.exit(p.returncode)
 338
 339
 340 def copy_cargo_toml():
 341     cargo_toml_file = topsrc_dir / "Cargo.toml"
 342     target_cargo_toml_file = target_dir / "Cargo.toml"
 343
 344     with cargo_toml_file.open("r") as f:
 345
 346         class State(enum.Enum):
 347             BEFORE_MEMBER = 1
 348             INSIDE_MEMBER = 2
 349             AFTER_MEMBER = 3
 350             INSIDE_PATCH = 4
 351             AFTER_PATCH = 5
 352
 353         content = ""
 354         state = State.BEFORE_MEMBER
 355         for line in f:
 356             if state == State.BEFORE_MEMBER:
 357                 if line.strip() == "members = [":
 358                     state = State.INSIDE_MEMBER
 359             elif state == State.INSIDE_MEMBER:
 360                 if line.strip() == "]":
 361                     state = State.AFTER_MEMBER
 362                 elif not is_mozjs_cargo_member(line):
 363                     continue
 364             elif state == State.AFTER_MEMBER:
 365                 if line.strip() == "[patch.crates-io]":
 366                     state = State.INSIDE_PATCH
 367             elif state == State.INSIDE_PATCH:
 368                 if line.startswith("["):
 369                     state = State.AFTER_PATCH
 370                 if "path = " in line:
 371                     if not is_mozjs_crates_io_local_patch(line):
 372                         continue
 373
 374             content += line
 375
 376     with target_cargo_toml_file.open("w") as f:
 377         f.write(content)
 378
 379
 380 def generate_configure():
 381     """Generate configure files to avoid build dependency on autoconf-2.13"""
 382
 383     src_old_configure_in_file = topsrc_dir / "js" / "src" / "old-configure.in"
 384     dest_old_configure_file = target_dir / "js" / "src" / "old-configure"
 385
 386     js_src_dir = topsrc_dir / "js" / "src"
 387
 388     env = os.environ.copy()
 389     env["M4"] = m4
 390     env["AWK"] = awk
 391     env["AC_MACRODIR"] = topsrc_dir / "build" / "autoconf"
 392
 393     with dest_old_configure_file.open("w") as f:
 394         subprocess.run(
 395             [
 396                 "sh",
 397                 str(topsrc_dir / "build" / "autoconf" / "autoconf.sh"),
 398                 "--localdir={}".format(js_src_dir),
 399                 str(src_old_configure_in_file),
 400             ],
 401             stdout=f,
 402             check=True,
 403             env=env,
 404         )
 405
 406
 407 def copy_file(filename, content):
 408     """Copy an existing file from the staging area, or create a new file
 409     with the given contents if it does not exist."""
 410
 411     staging_file = staging_dir / filename
 412     target_file = target_dir / filename
 413
 414     if staging_file.exists():
 415         shutil.copy2(str(staging_file), str(target_file))
 416     else:
 417         with target_file.open("w") as f:
 418             f.write(content)
 419
 420
 421 def copy_patches():
 422     """Copy patches dir, if it exists."""
 423
 424     staging_patches_dir = staging_dir / "patches"
 425     top_patches_dir = topsrc_dir / "patches"
 426     target_patches_dir = target_dir / "patches"
 427
 428     if staging_patches_dir.is_dir():
 429         shutil.copytree(str(staging_patches_dir), str(target_patches_dir))
 430     elif top_patches_dir.is_dir():
 431         shutil.copytree(str(top_patches_dir), str(target_patches_dir))
 432
 433
 434 def remove_python_cache():
 435     """Remove *.pyc and *.pyo files if any."""
 436     for f in target_dir.glob("**/*.pyc"):
 437         f.unlink()
 438     for f in target_dir.glob("**/*.pyo"):
 439         f.unlink()
 440
 441
 442 def stage():
 443     """Stage source tarball content."""
 444     logging.info("Staging source tarball in {}...".format(target_dir))
 445
 446     create_target_dir()
 447     sync_files()
 448     copy_cargo_toml()
 449     generate_configure()
 450     copy_file("INSTALL", INSTALL_CONTENT)
 451     copy_file("README", README_CONTENT)
 452     copy_file("mozconfig", MOZCONFIG_DEBUG_CONTENT)
 453     copy_file("mozconfig.opt", MOZCONFIG_OPT_CONTENT)
 454     copy_patches()
 455     remove_python_cache()
 456
 457
 458 def create_tar():
 459     """Roll the tarball."""
 460
 461     logging.info("Packaging source tarball at {}...".format(package_file))
 462
 463     subprocess.run(
 464         [str(tar)] + tar_opts + [str(package_file), "-C", str(staging_dir), version],
 465         check=True,
 466     )
 467
 468
 469 def build():
 470     assert_clean()
 471     stage()
 472     create_tar()
 473
 474
 475 parser = argparse.ArgumentParser(description="Make SpiderMonkey source package")
 476 subparsers = parser.add_subparsers(dest="COMMAND")
 477 subparser_update = subparsers.add_parser("clean", help="")
 478 subparser_update = subparsers.add_parser("build", help="")
 479 args = parser.parse_args()
 480
 481 if args.COMMAND == "clean":
 482     clean()
 483 elif not args.COMMAND or args.COMMAND == "build":
 484     build()