js/src/make-source-package.py

   1 #!/usr/bin/env -S python3 -B
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4 # You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import argparse
   7 import enum
   8 import logging
   9 import os
  10 import shutil
  11 import subprocess
  12 import sys
  13 from pathlib import Path
  14
  15 logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
  16
  17
  18 def find_command(names):
  19     """Search for command in `names`, and returns the first one that exists."""
  20
  21     for name in names:
  22         path = shutil.which(name)
  23         if path is not None:
  24             return name
  25
  26     return None
  27
  28
  29 def assert_command(env_var, name):
  30     """Assert that the command is not empty
  31     The command name comes from either environment variable or find_command.
  32     """
  33     if not name:
  34         logging.error("{} command not found".format(env_var))
  35         sys.exit(1)
  36
  37
  38 def parse_version(topsrc_dir):
  39     """Parse milestone.txt and return the entire milestone and major version."""
  40     milestone_file = topsrc_dir / "config" / "milestone.txt"
  41     if not milestone_file.is_file():
  42         return ("", "", "")
  43
  44     with milestone_file.open("r") as f:
  45         for line in f:
  46             line = line.strip()
  47             if not line:
  48                 continue
  49             if line.startswith("#"):
  50                 continue
  51
  52             v = line.split(".")
  53             return tuple((v + ["", ""])[:3])
  54
  55     return ("", "", "")
  56
  57
  58 tmp_dir = Path("/tmp")
  59
  60 tar = os.environ.get("TAR", find_command(["tar"]))
  61 assert_command("TAR", tar)
  62
  63 rsync = os.environ.get("RSYNC", find_command(["rsync"]))
  64 assert_command("RSYNC", rsync)
  65
  66 m4 = os.environ.get("M4", find_command(["m4"]))
  67 assert_command("M4", m4)
  68
  69 awk = os.environ.get("AWK", find_command(["awk"]))
  70 assert_command("AWK", awk)
  71
  72 src_dir = Path(os.environ.get("SRC_DIR", Path(__file__).parent.absolute()))
  73 mozjs_name = os.environ.get("MOZJS_NAME", "mozjs")
  74 staging_dir = Path(os.environ.get("STAGING", tmp_dir / "mozjs-src-pkg"))
  75 dist_dir = Path(os.environ.get("DIST", tmp_dir))
  76 topsrc_dir = src_dir.parent.parent.absolute()
  77
  78 parsed_major_version, parsed_minor_version, parsed_patch_version = parse_version(
  79     topsrc_dir
  80 )
  81
  82 major_version = os.environ.get("MOZJS_MAJOR_VERSION", parsed_major_version)
  83 minor_version = os.environ.get("MOZJS_MINOR_VERSION", parsed_minor_version)
  84 patch_version = os.environ.get("MOZJS_PATCH_VERSION", parsed_patch_version)
  85 alpha = os.environ.get("MOZJS_ALPHA", "")
  86
  87 version = "{}-{}.{}.{}".format(
  88     mozjs_name, major_version, minor_version, patch_version or alpha or "0"
  89 )
  90 target_dir = staging_dir / version
  91 package_name = "{}.tar.xz".format(version)
  92 package_file = dist_dir / package_name
  93 tar_opts = ["-Jcf"]
  94
  95 # Given there might be some external program that reads the following output,
  96 # use raw `print`, instead of logging.
  97 print("Environment:")
  98 print("    TAR = {}".format(tar))
  99 print("    RSYNC = {}".format(rsync))
 100 print("    M4 = {}".format(m4))
 101 print("    AWK = {}".format(awk))
 102 print("    STAGING = {}".format(staging_dir))
 103 print("    DIST = {}".format(dist_dir))
 104 print("    SRC_DIR = {}".format(src_dir))
 105 print("    MOZJS_NAME = {}".format(mozjs_name))
 106 print("    MOZJS_MAJOR_VERSION = {}".format(major_version))
 107 print("    MOZJS_MINOR_VERSION = {}".format(minor_version))
 108 print("    MOZJS_PATCH_VERSION = {}".format(patch_version))
 109 print("    MOZJS_ALPHA = {}".format(alpha))
 110 print("")
 111
 112 rsync_filter_list = """
 113 # Top-level config and build files
 114
 115 + /aclocal.m4
 116 + /client.mk
 117 + /configure.py
 118 + /LICENSE
 119 + /mach
 120 + /Makefile.in
 121 + /moz.build
 122 + /moz.configure
 123 + /test.mozbuild
 124 + /.babel-eslint.rc.js
 125 + /.eslintignore
 126 + /.eslintrc.js
 127 + /.flake8
 128 + /.gitignore
 129 + /.hgignore
 130 + /.lldbinit
 131 + /.prettierignore
 132 + /.prettierrc
 133 + /.ycm_extra_conf.py
 134
 135 # Additional libraries (optionally) used by SpiderMonkey
 136
 137 + /mfbt/**
 138 + /nsprpub/**
 139
 140 + /intl/bidi/**
 141
 142 - /intl/icu/source/data
 143 - /intl/icu/source/test
 144 - /intl/icu/source/tools
 145 + /intl/icu/**
 146
 147 + /intl/icu_capi/**
 148 + /intl/icu_segmenter_data/**
 149
 150 - /intl/components/gtest
 151 + /intl/components/**
 152
 153 + /memory/replace/dmd/dmd.py
 154 + /memory/build/**
 155 + /memory/moz.build
 156 + /memory/mozalloc/**
 157
 158 + /modules/fdlibm/**
 159 + /modules/zlib/**
 160
 161 + /mozglue/baseprofiler/**
 162 + /mozglue/build/**
 163 + /mozglue/interposers/**
 164 + /mozglue/misc/**
 165 + /mozglue/moz.build
 166 + /mozglue/static/**
 167
 168 + /tools/rb/fix_stacks.py
 169 + /tools/fuzzing/moz.build
 170 + /tools/fuzzing/interface/**
 171 + /tools/fuzzing/registry/**
 172 + /tools/fuzzing/libfuzzer/**
 173 + /tools/fuzzing/*.mozbuild
 174
 175 # Build system and dependencies
 176
 177 + /Cargo.lock
 178 + /build/**
 179 + /config/**
 180 + /python/**
 181
 182 + /.cargo/config.toml.in
 183
 184 + /third_party/function2/**
 185 - /third_party/python/gyp
 186 + /third_party/python/**
 187 + /third_party/rust/**
 188 + /third_party/gemmology/**
 189 + /third_party/xsimd/**
 190 + /layout/tools/reftest/reftest/**
 191
 192 + /testing/mach_commands.py
 193 + /testing/moz.build
 194 + /testing/mozbase/**
 195 + /testing/performance/**
 196 + /testing/test/**
 197 + /testing/web-platform/*.ini
 198 + /testing/web-platform/*.py
 199 + /testing/web-platform/meta/streams/**
 200 + /testing/web-platform/mozilla/**
 201 + /testing/web-platform/tests/resources/**
 202 + /testing/web-platform/tests/streams/**
 203 + /testing/web-platform/tests/tools/**
 204
 205 + /toolkit/crashreporter/tools/symbolstore.py
 206 + /toolkit/mozapps/installer/package-name.mk
 207
 208 + /xpcom/geckoprocesstypes_generator/**
 209
 210 # List of prefs.
 211 + /modules/libpref/init/StaticPrefList.yaml
 212
 213 # SpiderMonkey itself
 214
 215 + /js/src/**
 216 + /js/app.mozbuild
 217 + /js/*.configure
 218 + /js/examples/**
 219 + /js/public/**
 220
 221 + */
 222 - /**
 223 """
 224
 225 INSTALL_CONTENT = """\
 226 Documentation for SpiderMonkey is available at:
 227
 228   https://spidermonkey.dev/
 229
 230 In particular, it points to build documentation at
 231
 232   https://firefox-source-docs.mozilla.org/js/build.html
 233
 234 Note that the libraries produced by the build system include symbols,
 235 causing the binaries to be extremely large. It is highly suggested that `strip`
 236 be run over the binaries before deploying them.
 237
 238 Building with default options may be performed as follows:
 239
 240   ./mach build
 241
 242 This will produce a debug build (much more suitable for developing against the
 243 SpiderMonkey JSAPI). To produce an optimized build:
 244
 245   export MOZCONFIG=$(pwd)/mozconfig.opt
 246   ./mach build
 247
 248 You may edit the mozconfig and mozconfig.opt files to configure your own build
 249 appropriately.
 250 """
 251
 252 MOZCONFIG_DEBUG_CONTENT = """\
 253 # Much slower when running, but adds assertions that are much better for
 254 # developing against the JSAPI.
 255 ac_add_options --enable-debug
 256
 257 # Much faster when running, worse for debugging.
 258 ac_add_options --enable-optimize
 259
 260 mk_add_options MOZ_OBJDIR=obj-debug
 261 """
 262
 263 MOZCONFIG_OPT_CONTENT = """\
 264 # Much faster when running, but very error-prone to develop against because
 265 # this will skip all the assertions critical to using the JSAPI properly.
 266 ac_add_options --disable-debug
 267
 268 # Much faster when running, worse for debugging.
 269 ac_add_options --enable-optimize
 270
 271 mk_add_options MOZ_OBJDIR=obj-opt
 272 """
 273
 274 README_CONTENT = """\
 275 This directory contains SpiderMonkey {major_version}.
 276
 277 This release is based on a revision of Mozilla {major_version}:
 278   https://hg.mozilla.org/releases/
 279 The changes in the patches/ directory were applied.
 280
 281 See https://spidermonkey.dev/ for documentation, examples, and release notes.
 282 """.format(
 283     major_version=major_version
 284 )
 285
 286
 287 def is_mozjs_cargo_member(line):
 288     """Checks if the line in workspace.members is mozjs-related"""
 289
 290     return '"js/' in line
 291
 292
 293 def is_mozjs_crates_io_local_patch(line):
 294     """Checks if the line in patch.crates-io is mozjs-related"""
 295
 296     return any(
 297         f'path = "{p}' in line for p in ("js", "build", "third_party/rust", "intl")
 298     )
 299
 300
 301 def clean():
 302     """Remove temporary directory and package file."""
 303     logging.info("Cleaning {} and {} ...".format(package_file, target_dir))
 304     if package_file.exists():
 305         package_file.unlink()
 306     if target_dir.exists():
 307         shutil.rmtree(str(target_dir))
 308
 309
 310 def assert_clean():
 311     """Assert that target directory does not contain generated files."""
 312     makefile_file = target_dir / "js" / "src" / "Makefile"
 313     if makefile_file.exists():
 314         logging.error("found js/src/Makefile. Please clean before packaging.")
 315         sys.exit(1)
 316
 317
 318 def create_target_dir():
 319     if target_dir.exists():
 320         logging.warning("dist tree {} already exists!".format(target_dir))
 321     else:
 322         target_dir.mkdir(parents=True)
 323
 324
 325 def sync_files():
 326     # Output of the command should directly go to stdout/stderr.
 327     p = subprocess.Popen(
 328         [
 329             str(rsync),
 330             "--delete-excluded",
 331             "--prune-empty-dirs",
 332             "--quiet",
 333             "--recursive",
 334             "{}/".format(topsrc_dir),
 335             "{}/".format(target_dir),
 336             "--filter=. -",
 337         ],
 338         stdin=subprocess.PIPE,
 339     )
 340
 341     p.communicate(rsync_filter_list.encode())
 342
 343     if p.returncode != 0:
 344         sys.exit(p.returncode)
 345
 346
 347 def copy_cargo_toml():
 348     cargo_toml_file = topsrc_dir / "Cargo.toml"
 349     target_cargo_toml_file = target_dir / "Cargo.toml"
 350
 351     with cargo_toml_file.open("r") as f:
 352
 353         class State(enum.Enum):
 354             BEFORE_MEMBER = 1
 355             INSIDE_MEMBER = 2
 356             AFTER_MEMBER = 3
 357             INSIDE_PATCH = 4
 358             AFTER_PATCH = 5
 359
 360         content = ""
 361         state = State.BEFORE_MEMBER
 362         for line in f:
 363             if state == State.BEFORE_MEMBER:
 364                 if line.strip() == "members = [":
 365                     state = State.INSIDE_MEMBER
 366             elif state == State.INSIDE_MEMBER:
 367                 if line.strip() == "]":
 368                     state = State.AFTER_MEMBER
 369                 elif not is_mozjs_cargo_member(line):
 370                     continue
 371             elif state == State.AFTER_MEMBER:
 372                 if line.strip() == "[patch.crates-io]":
 373                     state = State.INSIDE_PATCH
 374             elif state == State.INSIDE_PATCH:
 375                 if line.startswith("["):
 376                     state = State.AFTER_PATCH
 377                 if "path = " in line:
 378                     if not is_mozjs_crates_io_local_patch(line):
 379                         continue
 380
 381             content += line
 382
 383     with target_cargo_toml_file.open("w") as f:
 384         f.write(content)
 385
 386
 387 def generate_configure():
 388     """Generate configure files to avoid build dependency on autoconf-2.13"""
 389
 390     src_old_configure_in_file = topsrc_dir / "js" / "src" / "old-configure.in"
 391     dest_old_configure_file = target_dir / "js" / "src" / "old-configure"
 392
 393     js_src_dir = topsrc_dir / "js" / "src"
 394
 395     env = os.environ.copy()
 396     env["M4"] = m4
 397     env["AWK"] = awk
 398     env["AC_MACRODIR"] = topsrc_dir / "build" / "autoconf"
 399
 400     with dest_old_configure_file.open("w") as f:
 401         subprocess.run(
 402             [
 403                 "sh",
 404                 str(topsrc_dir / "build" / "autoconf" / "autoconf.sh"),
 405                 "--localdir={}".format(js_src_dir),
 406                 str(src_old_configure_in_file),
 407             ],
 408             stdout=f,
 409             check=True,
 410             env=env,
 411         )
 412
 413
 414 def copy_file(filename, content):
 415     """Copy an existing file from the staging area, or create a new file
 416     with the given contents if it does not exist."""
 417
 418     staging_file = staging_dir / filename
 419     target_file = target_dir / filename
 420
 421     if staging_file.exists():
 422         shutil.copy2(str(staging_file), str(target_file))
 423     else:
 424         with target_file.open("w") as f:
 425             f.write(content)
 426
 427
 428 def copy_patches():
 429     """Copy patches dir, if it exists."""
 430
 431     staging_patches_dir = staging_dir / "patches"
 432     top_patches_dir = topsrc_dir / "patches"
 433     target_patches_dir = target_dir / "patches"
 434
 435     if staging_patches_dir.is_dir():
 436         shutil.copytree(str(staging_patches_dir), str(target_patches_dir))
 437     elif top_patches_dir.is_dir():
 438         shutil.copytree(str(top_patches_dir), str(target_patches_dir))
 439
 440
 441 def remove_python_cache():
 442     """Remove *.pyc and *.pyo files if any."""
 443     for f in target_dir.glob("**/*.pyc"):
 444         f.unlink()
 445     for f in target_dir.glob("**/*.pyo"):
 446         f.unlink()
 447
 448
 449 def stage():
 450     """Stage source tarball content."""
 451     logging.info("Staging source tarball in {}...".format(target_dir))
 452
 453     create_target_dir()
 454     sync_files()
 455     copy_cargo_toml()
 456     generate_configure()
 457     copy_file("INSTALL", INSTALL_CONTENT)
 458     copy_file("README", README_CONTENT)
 459     copy_file("mozconfig", MOZCONFIG_DEBUG_CONTENT)
 460     copy_file("mozconfig.opt", MOZCONFIG_OPT_CONTENT)
 461     copy_patches()
 462     remove_python_cache()
 463
 464
 465 def create_tar():
 466     """Roll the tarball."""
 467
 468     logging.info("Packaging source tarball at {}...".format(package_file))
 469
 470     subprocess.run(
 471         [str(tar)] + tar_opts + [str(package_file), "-C", str(staging_dir), version],
 472         check=True,
 473     )
 474
 475
 476 def build():
 477     assert_clean()
 478     stage()
 479     create_tar()
 480
 481
 482 parser = argparse.ArgumentParser(description="Make SpiderMonkey source package")
 483 subparsers = parser.add_subparsers(dest="COMMAND")
 484 subparser_update = subparsers.add_parser("clean", help="")
 485 subparser_update = subparsers.add_parser("build", help="")
 486 args = parser.parse_args()
 487
 488 if args.COMMAND == "clean":
 489     clean()
 490 elif not args.COMMAND or args.COMMAND == "build":
 491     build()