tools/esmify/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import json
   6 import logging
   7 import os
   8 import pathlib
   9 import re
  10 import subprocess
  11 import sys
  12
  13 from mach.decorators import Command, CommandArgument
  14
  15
  16 def path_sep_to_native(path_str):
  17     """Make separators in the path OS native."""
  18     return pathlib.os.sep.join(path_str.split("/"))
  19
  20
  21 def path_sep_from_native(path):
  22     """Make separators in the path OS native."""
  23     return "/".join(str(path).split(pathlib.os.sep))
  24
  25
  26 excluded_from_convert_prefix = list(
  27     map(
  28         path_sep_to_native,
  29         [
  30             # Testcases for actors.
  31             "toolkit/actors/TestProcessActorChild.jsm",
  32             "toolkit/actors/TestProcessActorParent.jsm",
  33             "toolkit/actors/TestWindowChild.jsm",
  34             "toolkit/actors/TestWindowParent.jsm",
  35             "js/xpconnect/tests/unit/",
  36             # Testcase for build system.
  37             "python/mozbuild/mozbuild/test/",
  38         ],
  39     )
  40 )
  41
  42
  43 def is_excluded_from_convert(path):
  44     """Returns true if the JSM file shouldn't be converted to ESM."""
  45     path_str = str(path)
  46     for prefix in excluded_from_convert_prefix:
  47         if path_str.startswith(prefix):
  48             return True
  49
  50     return False
  51
  52
  53 excluded_from_imports_prefix = list(
  54     map(
  55         path_sep_to_native,
  56         [
  57             # Vendored or auto-generated files.
  58             "browser/components/pocket/content/panels/js/vendor.bundle.js",
  59             "devtools/client/debugger/dist/parser-worker.js",
  60             "devtools/client/debugger/test/mochitest/examples/react/build/main.js",
  61             "devtools/client/debugger/test/mochitest/examples/sourcemapped/polyfill-bundle.js",
  62             "devtools/client/inspector/markup/test/shadowdom_open_debugger.min.js",
  63             "devtools/client/shared/source-map-loader/test/browser/fixtures/bundle.js",
  64             "layout/style/test/property_database.js",
  65             "services/fxaccounts/FxAccountsPairingChannel.js",
  66             "testing/talos/talos/tests/devtools/addon/content/pages/custom/debugger/static/js/main.js",  # noqa E501
  67             "testing/web-platform/",
  68             # Unrelated testcases that has edge case syntax.
  69             "browser/components/sessionstore/test/unit/data/",
  70             "devtools/client/debugger/src/workers/parser/tests/fixtures/",
  71             "devtools/client/debugger/test/mochitest/examples/sourcemapped/fixtures/",
  72             "devtools/client/webconsole/test/browser/test-syntaxerror-worklet.js",
  73             "devtools/server/tests/xpcshell/test_framebindings-03.js",
  74             "devtools/server/tests/xpcshell/test_framebindings-04.js",
  75             "devtools/shared/tests/xpcshell/test_eventemitter_basic.js",
  76             "devtools/shared/tests/xpcshell/test_eventemitter_static.js",
  77             "dom/base/crashtests/module-with-syntax-error.js",
  78             "dom/base/test/file_bug687859-16.js",
  79             "dom/base/test/file_bug687859-16.js",
  80             "dom/base/test/file_js_cache_syntax_error.js",
  81             "dom/base/test/jsmodules/module_badSyntax.js",
  82             "dom/canvas/test/reftest/webgl-utils.js",
  83             "dom/encoding/test/file_utf16_be_bom.js",
  84             "dom/encoding/test/file_utf16_le_bom.js",
  85             "dom/html/test/bug649134/file_bug649134-1.sjs",
  86             "dom/html/test/bug649134/file_bug649134-2.sjs",
  87             "dom/media/webrtc/tests/mochitests/identity/idp-bad.js",
  88             "dom/serviceworkers/test/file_js_cache_syntax_error.js",
  89             "dom/serviceworkers/test/parse_error_worker.js",
  90             "dom/workers/test/importScripts_worker_imported3.js",
  91             "dom/workers/test/invalid.js",
  92             "dom/workers/test/threadErrors_worker1.js",
  93             "dom/xhr/tests/browser_blobFromFile.js",
  94             "image/test/browser/browser_image.js",
  95             "js/xpconnect/tests/chrome/test_bug732665_meta.js",
  96             "js/xpconnect/tests/mochitest/class_static_worker.js",
  97             "js/xpconnect/tests/unit/bug451678_subscript.js",
  98             "js/xpconnect/tests/unit/error_other.sys.mjs",
  99             "js/xpconnect/tests/unit/es6module_parse_error.js",
 100             "js/xpconnect/tests/unit/recursive_importA.jsm",
 101             "js/xpconnect/tests/unit/recursive_importB.jsm",
 102             "js/xpconnect/tests/unit/syntax_error.jsm",
 103             "js/xpconnect/tests/unit/test_defineModuleGetter.js",
 104             "js/xpconnect/tests/unit/test_import.js",
 105             "js/xpconnect/tests/unit/test_import_shim.js",
 106             "js/xpconnect/tests/unit/test_recursive_import.js",
 107             "js/xpconnect/tests/unit/test_unload.js",
 108             "modules/libpref/test/unit/data/testParser.js",
 109             "python/mozbuild/mozbuild/test/",
 110             "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.sys.mjs",
 111             "testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js",
 112             "testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js",
 113             "toolkit/components/reader/Readerable.sys.mjs",
 114             "toolkit/components/workerloader/tests/moduleF-syntax-error.js",
 115             "tools/lint/test/",
 116             "tools/update-packaging/test/",
 117             # SpiderMonkey internals.
 118             "js/examples/",
 119             "js/src/",
 120             # Files has macro.
 121             "browser/app/profile/firefox.js",
 122             "browser/branding/official/pref/firefox-branding.js",
 123             "browser/components/enterprisepolicies/schemas/schema.sys.mjs",
 124             "browser/locales/en-US/firefox-l10n.js",
 125             "mobile/android/app/geckoview-prefs.js",
 126             "mobile/android/app/mobile.js",
 127             "mobile/android/locales/en-US/mobile-l10n.js",
 128             "modules/libpref/greprefs.js",
 129             "modules/libpref/init/all.js",
 130             "testing/condprofile/condprof/tests/profile/user.js",
 131             "testing/mozbase/mozprofile/tests/files/prefs_with_comments.js",
 132             "toolkit/modules/AppConstants.sys.mjs",
 133             "toolkit/mozapps/update/tests/data/xpcshellConstantsPP.js",
 134             # Uniffi templates
 135             "toolkit/components/uniffi-bindgen-gecko-js/src/templates/js/",
 136         ],
 137     )
 138 )
 139
 140 EXCLUSION_FILES = [
 141     os.path.join("tools", "rewriting", "Generated.txt"),
 142     os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"),
 143 ]
 144
 145
 146 def load_exclusion_files():
 147     for path in EXCLUSION_FILES:
 148         with open(path, "r") as f:
 149             for line in f:
 150                 p = path_sep_to_native(re.sub("\*$", "", line.strip()))
 151                 excluded_from_imports_prefix.append(p)
 152
 153
 154 def is_excluded_from_imports(path):
 155     """Returns true if the JS file content shouldn't be handled by
 156     jscodeshift.
 157
 158     This filter is necessary because jscodeshift cannot handle some
 159     syntax edge cases and results in unexpected rewrite."""
 160     path_str = str(path)
 161     for prefix in excluded_from_imports_prefix:
 162         if path_str.startswith(prefix):
 163             return True
 164
 165     return False
 166
 167
 168 # Wrapper for hg/git operations
 169 class VCSUtils:
 170     def run(self, cmd):
 171         # Do not pass check=True because the pattern can match no file.
 172         lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode()
 173         return filter(lambda x: x != "", lines.split("\n"))
 174
 175
 176 class HgUtils(VCSUtils):
 177     def is_available():
 178         return pathlib.Path(".hg").exists()
 179
 180     def rename(self, before, after):
 181         cmd = ["hg", "rename", before, after]
 182         subprocess.run(cmd, check=True)
 183
 184     def find_jsms(self, path):
 185         jsms = []
 186
 187         # NOTE: `set:glob:` syntax does not accept backslash on windows.
 188         path = path_sep_from_native(path)
 189
 190         cmd = ["hg", "files", f'set:glob:"{path}/**/*.jsm"']
 191         for line in self.run(cmd):
 192             jsm = pathlib.Path(line)
 193             if is_excluded_from_convert(jsm):
 194                 continue
 195             jsms.append(jsm)
 196
 197         cmd = [
 198             "hg",
 199             "files",
 200             f"set:grep('EXPORTED_SYMBOLS = \[') and glob:\"{path}/**/*.js\"",
 201         ]
 202         for line in self.run(cmd):
 203             jsm = pathlib.Path(line)
 204             if is_excluded_from_convert(jsm):
 205                 continue
 206             jsms.append(jsm)
 207
 208         return jsms
 209
 210     def find_all_jss(self, path):
 211         jss = []
 212
 213         # NOTE: `set:glob:` syntax does not accept backslash on windows.
 214         path = path_sep_from_native(path)
 215
 216         cmd = [
 217             "hg",
 218             "files",
 219             f'set:glob:"{path}/**/*.jsm" or glob:"{path}/**/*.js" or '
 220             + f'glob:"{path}/**/*.mjs" or glob:"{path}/**/*.sjs"',
 221         ]
 222         for line in self.run(cmd):
 223             js = pathlib.Path(line)
 224             if is_excluded_from_imports(js):
 225                 continue
 226             jss.append(js)
 227
 228         return jss
 229
 230
 231 class GitUtils(VCSUtils):
 232     def is_available():
 233         return pathlib.Path(".git").exists()
 234
 235     def rename(self, before, after):
 236         cmd = ["git", "mv", before, after]
 237         subprocess.run(cmd, check=True)
 238
 239     def find_jsms(self, path):
 240         jsms = []
 241
 242         cmd = ["git", "ls-files", f"{path}/*.jsm"]
 243         for line in self.run(cmd):
 244             jsm = pathlib.Path(line)
 245             if is_excluded_from_convert(jsm):
 246                 continue
 247             jsms.append(jsm)
 248
 249         handled = {}
 250         cmd = ["git", "grep", "EXPORTED_SYMBOLS = \[", f"{path}/*.js"]
 251         for line in self.run(cmd):
 252             m = re.search("^([^:]+):", line)
 253             if not m:
 254                 continue
 255             filename = m.group(1)
 256             if filename in handled:
 257                 continue
 258             handled[filename] = True
 259             jsm = pathlib.Path(filename)
 260             if is_excluded_from_convert(jsm):
 261                 continue
 262             jsms.append(jsm)
 263
 264         return jsms
 265
 266     def find_all_jss(self, path):
 267         jss = []
 268
 269         cmd = [
 270             "git",
 271             "ls-files",
 272             f"{path}/*.jsm",
 273             f"{path}/*.js",
 274             f"{path}/*.mjs",
 275             f"{path}/*.sjs",
 276         ]
 277         for line in self.run(cmd):
 278             js = pathlib.Path(line)
 279             if is_excluded_from_imports(js):
 280                 continue
 281             jss.append(js)
 282
 283         return jss
 284
 285
 286 class Summary:
 287     def __init__(self):
 288         self.convert_errors = []
 289         self.import_errors = []
 290         self.rename_errors = []
 291         self.no_refs = []
 292
 293
 294 @Command(
 295     "esmify",
 296     category="misc",
 297     description="ESMify JSM files.",
 298 )
 299 @CommandArgument(
 300     "path",
 301     nargs=1,
 302     help="Path to the JSM file to ESMify, or the directory that contains "
 303     "JSM files and/or JS files that imports ESM-ified JSM.",
 304 )
 305 @CommandArgument(
 306     "--convert",
 307     action="store_true",
 308     help="Only perform the step 1 = convert part",
 309 )
 310 @CommandArgument(
 311     "--imports",
 312     action="store_true",
 313     help="Only perform the step 2 = import calls part",
 314 )
 315 @CommandArgument(
 316     "--prefix",
 317     default="",
 318     help="Restrict the target of import in the step 2 to ESM-ified JSM, by the "
 319     "prefix match for the JSM file's path.  e.g. 'browser/'.",
 320 )
 321 def esmify(command_context, path=None, convert=False, imports=False, prefix=""):
 322     """
 323     This command does the following 2 steps:
 324       1. Convert the JSM file specified by `path` to ESM file, or the JSM files
 325          inside the directory specified by `path` to ESM files, and also
 326          fix references in build files and test definitions
 327       2. Convert import calls inside file(s) specified by `path` for ESM-ified
 328          files to use new APIs
 329
 330     Example 1:
 331       # Convert all JSM files inside `browser/components/pagedata` directory,
 332       # and replace all references for ESM-ified files in the entire tree to use
 333       # new APIs
 334
 335       $ ./mach esmify --convert browser/components/pagedata
 336       $ ./mach esmify --imports . --prefix=browser/components/pagedata
 337
 338     Example 2:
 339       # Convert all JSM files inside `browser` directory, and replace all
 340       # references for the JSM files inside `browser` directory to use
 341       # new APIs
 342
 343       $ ./mach esmify browser
 344     """
 345
 346     def error(text):
 347         command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}")
 348
 349     def warn(text):
 350         command_context.log(logging.WARN, "esmify", {}, f"[WARN] {text}")
 351
 352     def info(text):
 353         command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
 354
 355     # If no options is specified, perform both.
 356     if not convert and not imports:
 357         convert = True
 358         imports = True
 359
 360     path = pathlib.Path(path[0])
 361
 362     if not verify_path(command_context, path):
 363         return 1
 364
 365     if HgUtils.is_available():
 366         vcs_utils = HgUtils()
 367     elif GitUtils.is_available():
 368         vcs_utils = GitUtils()
 369     else:
 370         error(
 371             "This script needs to be run inside mozilla-central "
 372             "checkout of either mercurial or git."
 373         )
 374         return 1
 375
 376     load_exclusion_files()
 377
 378     info("Setting up jscodeshift...")
 379     setup_jscodeshift()
 380
 381     is_single_file = path.is_file()
 382
 383     modified_files = []
 384     summary = Summary()
 385
 386     if convert:
 387         info("Searching files to convert to ESM...")
 388         if is_single_file:
 389             jsms = [path]
 390         else:
 391             jsms = vcs_utils.find_jsms(path)
 392
 393         info(f"Found {len(jsms)} file(s) to convert to ESM.")
 394
 395         info("Converting to ESM...")
 396         jsms = convert_module(jsms, summary)
 397         if jsms is None:
 398             error("Failed to rewrite exports.")
 399             return 1
 400
 401         info("Renaming...")
 402         esms = rename_jsms(command_context, vcs_utils, jsms, summary)
 403
 404         modified_files += esms
 405
 406     if imports:
 407         info("Searching files to rewrite imports...")
 408
 409         if is_single_file:
 410             if convert:
 411                 # Already converted above
 412                 jss = esms
 413             else:
 414                 jss = [path]
 415         else:
 416             jss = vcs_utils.find_all_jss(path)
 417
 418         info(f"Checking {len(jss)} JS file(s). Rewriting any matching imports...")
 419
 420         result = rewrite_imports(jss, prefix, summary)
 421         if result is None:
 422             return 1
 423
 424         info(f"Rewritten {len(result)} file(s).")
 425
 426         # Only modified files needs eslint fix
 427         modified_files += result
 428
 429     modified_files = list(set(modified_files))
 430
 431     info(f"Applying eslint --fix for {len(modified_files)} file(s)...")
 432     eslint_fix(command_context, modified_files)
 433
 434     def print_files(f, errors):
 435         for [path, message] in errors:
 436             f(f"  * {path}")
 437             if message:
 438                 f(f"    {message}")
 439
 440     if len(summary.convert_errors):
 441         error("========")
 442         error("Following files are not converted into ESM due to error:")
 443         print_files(error, summary.convert_errors)
 444
 445     if len(summary.import_errors):
 446         warn("========")
 447         warn("Following files are not rewritten to import ESMs due to error:")
 448         warn(
 449             "(NOTE: Errors related to 'private names' are mostly due to "
 450             " preprocessor macros in the file):"
 451         )
 452         print_files(warn, summary.import_errors)
 453
 454     if len(summary.rename_errors):
 455         error("========")
 456         error("Following files are not renamed due to error:")
 457         print_files(error, summary.rename_errors)
 458
 459     if len(summary.no_refs):
 460         warn("========")
 461         warn("Following files are not found in any build files.")
 462         warn("Please update references to those files manually:")
 463         print_files(warn, summary.rename_errors)
 464
 465     return 0
 466
 467
 468 def verify_path(command_context, path):
 469     """Check if the path passed to the command is valid relative path."""
 470
 471     def error(text):
 472         command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}")
 473
 474     if not path.exists():
 475         error(f"{path} does not exist.")
 476         return False
 477
 478     if path.is_absolute():
 479         error("Path must be a relative path from mozilla-central checkout.")
 480         return False
 481
 482     return True
 483
 484
 485 def find_file(path, target):
 486     """Find `target` file in ancestor of path."""
 487     target_path = path.parent / target
 488     if not target_path.exists():
 489         if path.parent == path:
 490             return None
 491
 492         return find_file(path.parent, target)
 493
 494     return target_path
 495
 496
 497 def try_rename_in(command_context, path, target, jsm_name, esm_name, jsm_path):
 498     """Replace the occurrences of `jsm_name` with `esm_name` in `target`
 499     file."""
 500
 501     def info(text):
 502         command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
 503
 504     if type(target) is str:
 505         # Target is specified by filename, that may exist somewhere in
 506         # the jsm's directory or ancestor directories.
 507         target_path = find_file(path, target)
 508         if not target_path:
 509             return False
 510
 511         # JSM should be specified with relative path in the file.
 512         #
 513         # Single moz.build or jar.mn can contain multiple files with same name.
 514         # Search for relative path.
 515         jsm_relative_path = jsm_path.relative_to(target_path.parent)
 516         jsm_path_str = path_sep_from_native(str(jsm_relative_path))
 517     else:
 518         # Target is specified by full path.
 519         target_path = target
 520
 521         # JSM should be specified with full path in the file.
 522         jsm_path_str = path_sep_from_native(str(jsm_path))
 523
 524     jsm_path_re = re.compile(r"\b" + jsm_path_str.replace(".", r"\.") + r"\b")
 525     jsm_name_re = re.compile(r"\b" + jsm_name.replace(".", r"\.") + r"\b")
 526
 527     modified = False
 528     content = ""
 529     with open(target_path, "r") as f:
 530         for line in f:
 531             if jsm_path_re.search(line):
 532                 modified = True
 533                 line = jsm_name_re.sub(esm_name, line)
 534
 535             content += line
 536
 537     if modified:
 538         info(f"  {str(target_path)}")
 539         info(f"    {jsm_name} => {esm_name}")
 540         with open(target_path, "w", newline="\n") as f:
 541             f.write(content)
 542
 543     return True
 544
 545
 546 def try_rename_uri_in(command_context, target, jsm_name, esm_name, jsm_uri, esm_uri):
 547     """Replace the occurrences of `jsm_uri` with `esm_uri` in `target` file."""
 548
 549     def info(text):
 550         command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
 551
 552     modified = False
 553     content = ""
 554     with open(target, "r") as f:
 555         for line in f:
 556             if jsm_uri in line:
 557                 modified = True
 558                 line = line.replace(jsm_uri, esm_uri)
 559
 560             content += line
 561
 562     if modified:
 563         info(f"  {str(target)}")
 564         info(f"    {jsm_name} => {esm_name}")
 565         with open(target, "w", newline="\n") as f:
 566             f.write(content)
 567
 568     return True
 569
 570
 571 def try_rename_components_conf(command_context, path, jsm_name, esm_name):
 572     """Replace the occurrences of `jsm_name` with `esm_name` in components.conf
 573     file."""
 574
 575     def info(text):
 576         command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
 577
 578     target_path = find_file(path, "components.conf")
 579     if not target_path:
 580         return False
 581
 582     # Unlike try_rename_in, components.conf contains the URL instead of
 583     # relative path, and also there are no known files with same name.
 584     # Simply replace the filename.
 585
 586     with open(target_path, "r") as f:
 587         content = f.read()
 588
 589     prop_re = re.compile(
 590         "[\"']jsm[\"']:(.*)" + r"\b" + jsm_name.replace(".", r"\.") + r"\b"
 591     )
 592
 593     if not prop_re.search(content):
 594         return False
 595
 596     info(f"  {str(target_path)}")
 597     info(f"    {jsm_name} => {esm_name}")
 598
 599     content = prop_re.sub(r"'esModule':\1" + esm_name, content)
 600     with open(target_path, "w", newline="\n") as f:
 601         f.write(content)
 602
 603     return True
 604
 605
 606 def esmify_name(name):
 607     return re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", name)
 608
 609
 610 def esmify_path(jsm_path):
 611     jsm_name = jsm_path.name
 612     esm_name = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_name)
 613     esm_path = jsm_path.parent / esm_name
 614     return esm_path
 615
 616
 617 path_to_uri_map = None
 618
 619
 620 def load_path_to_uri_map():
 621     global path_to_uri_map
 622
 623     if path_to_uri_map:
 624         return
 625
 626     if "ESMIFY_MAP_JSON" in os.environ:
 627         json_map = pathlib.Path(os.environ["ESMIFY_MAP_JSON"])
 628     else:
 629         json_map = pathlib.Path(__file__).parent / "map.json"
 630
 631     with open(json_map, "r") as f:
 632         uri_to_path_map = json.loads(f.read())
 633
 634     path_to_uri_map = dict()
 635
 636     for uri, paths in uri_to_path_map.items():
 637         if type(paths) is str:
 638             paths = [paths]
 639
 640         for path in paths:
 641             path_to_uri_map[path] = uri
 642
 643
 644 def find_jsm_uri(jsm_path):
 645     load_path_to_uri_map()
 646
 647     path = path_sep_from_native(jsm_path)
 648
 649     if path in path_to_uri_map:
 650         return path_to_uri_map[path]
 651
 652     return None
 653
 654
 655 def rename_single_file(command_context, vcs_utils, jsm_path, summary):
 656     """Rename `jsm_path` to .sys.mjs, and fix references to the file in build
 657     and test definitions."""
 658
 659     def info(text):
 660         command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
 661
 662     esm_path = esmify_path(jsm_path)
 663
 664     jsm_name = jsm_path.name
 665     esm_name = esm_path.name
 666
 667     target_files = [
 668         ".eslintignore",
 669         "moz.build",
 670         "jar.mn",
 671         "browser.ini",
 672         "browser-common.ini",
 673         "chrome.ini",
 674         "mochitest.ini",
 675         "xpcshell.ini",
 676         "xpcshell-child-process.ini",
 677         "xpcshell-common.ini",
 678         "xpcshell-parent-process.ini",
 679         pathlib.Path("tools", "lint", "eslint.yml"),
 680         pathlib.Path("tools", "lint", "rejected-words.yml"),
 681     ]
 682
 683     info(f"{jsm_path} => {esm_path}")
 684
 685     renamed = False
 686     for target in target_files:
 687         if try_rename_in(
 688             command_context, jsm_path, target, jsm_name, esm_name, jsm_path
 689         ):
 690             renamed = True
 691
 692     if try_rename_components_conf(command_context, jsm_path, jsm_name, esm_name):
 693         renamed = True
 694
 695     uri_target_files = [
 696         pathlib.Path(
 697             "browser", "base", "content", "test", "performance", "browser_startup.js"
 698         ),
 699         pathlib.Path(
 700             "browser",
 701             "base",
 702             "content",
 703             "test",
 704             "performance",
 705             "browser_startup_content.js",
 706         ),
 707         pathlib.Path(
 708             "browser",
 709             "base",
 710             "content",
 711             "test",
 712             "performance",
 713             "browser_startup_content_subframe.js",
 714         ),
 715         pathlib.Path(
 716             "toolkit",
 717             "components",
 718             "backgroundtasks",
 719             "tests",
 720             "browser",
 721             "browser_xpcom_graph_wait.js",
 722         ),
 723     ]
 724
 725     jsm_uri = find_jsm_uri(jsm_path)
 726     if jsm_uri:
 727         esm_uri = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_uri)
 728
 729         for target in uri_target_files:
 730             if try_rename_uri_in(
 731                 command_context, target, jsm_uri, esm_uri, jsm_name, esm_name
 732             ):
 733                 renamed = True
 734
 735     if not renamed:
 736         summary.no_refs.append([jsm_path, None])
 737
 738     if not esm_path.exists():
 739         vcs_utils.rename(jsm_path, esm_path)
 740     else:
 741         summary.rename_errors.append([jsm_path, f"{esm_path} already exists"])
 742
 743     return esm_path
 744
 745
 746 def rename_jsms(command_context, vcs_utils, jsms, summary):
 747     esms = []
 748     for jsm in jsms:
 749         esm = rename_single_file(command_context, vcs_utils, jsm, summary)
 750         esms.append(esm)
 751
 752     return esms
 753
 754
 755 npm_prefix = pathlib.Path("tools") / "esmify"
 756 path_from_npm_prefix = pathlib.Path("..") / ".."
 757
 758
 759 def setup_jscodeshift():
 760     """Install jscodeshift."""
 761     cmd = [
 762         sys.executable,
 763         "./mach",
 764         "npm",
 765         "install",
 766         "jscodeshift",
 767         "--save-dev",
 768         "--prefix",
 769         str(npm_prefix),
 770     ]
 771     subprocess.run(cmd, check=True)
 772
 773
 774 def run_npm_command(args, env, stdin):
 775     cmd = [
 776         sys.executable,
 777         "./mach",
 778         "npm",
 779         "run",
 780     ] + args
 781     p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 782     p.stdin.write(stdin)
 783     p.stdin.close()
 784
 785     ok_files = []
 786     errors = []
 787     while True:
 788         line = p.stdout.readline()
 789         if not line:
 790             break
 791         line = line.rstrip().decode()
 792
 793         if line.startswith(" NOC "):
 794             continue
 795
 796         print(line)
 797
 798         m = re.search(r"^ (OKK|ERR) ([^ ]+)(?: (.+))?", line)
 799         if not m:
 800             continue
 801
 802         result = m.group(1)
 803         # NOTE: path is written from `tools/esmify`.
 804         path = pathlib.Path(m.group(2)).relative_to(path_from_npm_prefix)
 805         error = m.group(3)
 806
 807         if result == "OKK":
 808             ok_files.append(path)
 809
 810         if result == "ERR":
 811             errors.append([path, error])
 812
 813     if p.wait() != 0:
 814         return [None, None]
 815
 816     return ok_files, errors
 817
 818
 819 def convert_module(jsms, summary):
 820     """Replace EXPORTED_SYMBOLS with export declarations, and replace
 821     ChromeUtils.importESModule with static import as much as possible,
 822     and return the list of successfully rewritten files."""
 823
 824     if len(jsms) == 0:
 825         return []
 826
 827     env = os.environ.copy()
 828
 829     stdin = "\n".join(map(str, paths_from_npm_prefix(jsms))).encode()
 830
 831     ok_files, errors = run_npm_command(
 832         [
 833             "convert_module",
 834             "--prefix",
 835             str(npm_prefix),
 836         ],
 837         env=env,
 838         stdin=stdin,
 839     )
 840
 841     if ok_files is None and errors is None:
 842         return None
 843
 844     summary.convert_errors.extend(errors)
 845
 846     return ok_files
 847
 848
 849 def rewrite_imports(jss, prefix, summary):
 850     """Replace import calls for JSM with import calls for ESM or static import
 851     for ESM."""
 852
 853     if len(jss) == 0:
 854         return []
 855
 856     env = os.environ.copy()
 857     env["ESMIFY_TARGET_PREFIX"] = prefix
 858
 859     stdin = "\n".join(map(str, paths_from_npm_prefix(jss))).encode()
 860
 861     ok_files, errors = run_npm_command(
 862         [
 863             "rewrite_imports",
 864             "--prefix",
 865             str(npm_prefix),
 866         ],
 867         env=env,
 868         stdin=stdin,
 869     )
 870
 871     if ok_files is None and errors is None:
 872         return None
 873
 874     summary.import_errors.extend(errors)
 875
 876     return ok_files
 877
 878
 879 def paths_from_npm_prefix(paths):
 880     """Convert relative path from mozilla-central to relative path from
 881     tools/esmify."""
 882     return list(map(lambda path: path_from_npm_prefix / path, paths))
 883
 884
 885 def eslint_fix(command_context, files):
 886     """Auto format files."""
 887
 888     def info(text):
 889         command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}")
 890
 891     if len(files) == 0:
 892         return
 893
 894     remaining = files[0:]
 895
 896     # There can be too many files for single command line, perform by chunk.
 897     max_files = 16
 898     while len(remaining) > max_files:
 899         info(f"{len(remaining)} files remaining")
 900
 901         chunk = remaining[0:max_files]
 902         remaining = remaining[max_files:]
 903
 904         cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk
 905         subprocess.run(cmd, check=True)
 906
 907     info(f"{len(remaining)} files remaining")
 908     chunk = remaining
 909     cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk
 910     subprocess.run(cmd, check=True)