1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
13 from mach
.decorators
import Command
, CommandArgument
16 def path_sep_to_native(path_str
):
17 """Make separators in the path OS native."""
18 return pathlib
.os
.sep
.join(path_str
.split("/"))
21 def path_sep_from_native(path
):
22 """Make separators in the path OS native."""
23 return "/".join(str(path
).split(pathlib
.os
.sep
))
26 excluded_from_convert_prefix
= list(
30 # Testcases for actors.
31 "toolkit/actors/TestProcessActorChild.jsm",
32 "toolkit/actors/TestProcessActorParent.jsm",
33 "toolkit/actors/TestWindowChild.jsm",
34 "toolkit/actors/TestWindowParent.jsm",
35 "js/xpconnect/tests/unit/",
36 # Testcase for build system.
37 "python/mozbuild/mozbuild/test/",
43 def is_excluded_from_convert(path
):
44 """Returns true if the JSM file shouldn't be converted to ESM."""
46 for prefix
in excluded_from_convert_prefix
:
47 if path_str
.startswith(prefix
):
53 excluded_from_imports_prefix
= list(
57 # Vendored or auto-generated files.
58 "browser/components/pocket/content/panels/js/vendor.bundle.js",
59 "devtools/client/debugger/dist/parser-worker.js",
60 "devtools/client/debugger/test/mochitest/examples/react/build/main.js",
61 "devtools/client/debugger/test/mochitest/examples/sourcemapped/polyfill-bundle.js",
62 "devtools/client/inspector/markup/test/shadowdom_open_debugger.min.js",
63 "devtools/client/shared/source-map-loader/test/browser/fixtures/bundle.js",
64 "layout/style/test/property_database.js",
65 "services/fxaccounts/FxAccountsPairingChannel.js",
66 "testing/talos/talos/tests/devtools/addon/content/pages/custom/debugger/static/js/main.js", # noqa E501
67 "testing/web-platform/",
68 # Unrelated testcases that has edge case syntax.
69 "browser/components/sessionstore/test/unit/data/",
70 "devtools/client/debugger/src/workers/parser/tests/fixtures/",
71 "devtools/client/debugger/test/mochitest/examples/sourcemapped/fixtures/",
72 "devtools/client/webconsole/test/browser/test-syntaxerror-worklet.js",
73 "devtools/server/tests/xpcshell/test_framebindings-03.js",
74 "devtools/server/tests/xpcshell/test_framebindings-04.js",
75 "devtools/shared/tests/xpcshell/test_eventemitter_basic.js",
76 "devtools/shared/tests/xpcshell/test_eventemitter_static.js",
77 "dom/base/crashtests/module-with-syntax-error.js",
78 "dom/base/test/file_bug687859-16.js",
79 "dom/base/test/file_bug687859-16.js",
80 "dom/base/test/file_js_cache_syntax_error.js",
81 "dom/base/test/jsmodules/module_badSyntax.js",
82 "dom/canvas/test/reftest/webgl-utils.js",
83 "dom/encoding/test/file_utf16_be_bom.js",
84 "dom/encoding/test/file_utf16_le_bom.js",
85 "dom/html/test/bug649134/file_bug649134-1.sjs",
86 "dom/html/test/bug649134/file_bug649134-2.sjs",
87 "dom/media/webrtc/tests/mochitests/identity/idp-bad.js",
88 "dom/serviceworkers/test/file_js_cache_syntax_error.js",
89 "dom/serviceworkers/test/parse_error_worker.js",
90 "dom/workers/test/importScripts_worker_imported3.js",
91 "dom/workers/test/invalid.js",
92 "dom/workers/test/threadErrors_worker1.js",
93 "dom/xhr/tests/browser_blobFromFile.js",
94 "image/test/browser/browser_image.js",
95 "js/xpconnect/tests/chrome/test_bug732665_meta.js",
96 "js/xpconnect/tests/mochitest/class_static_worker.js",
97 "js/xpconnect/tests/unit/bug451678_subscript.js",
98 "js/xpconnect/tests/unit/error_other.sys.mjs",
99 "js/xpconnect/tests/unit/es6module_parse_error.js",
100 "js/xpconnect/tests/unit/recursive_importA.jsm",
101 "js/xpconnect/tests/unit/recursive_importB.jsm",
102 "js/xpconnect/tests/unit/syntax_error.jsm",
103 "js/xpconnect/tests/unit/test_defineModuleGetter.js",
104 "js/xpconnect/tests/unit/test_import.js",
105 "js/xpconnect/tests/unit/test_import_shim.js",
106 "js/xpconnect/tests/unit/test_recursive_import.js",
107 "js/xpconnect/tests/unit/test_unload.js",
108 "modules/libpref/test/unit/data/testParser.js",
109 "python/mozbuild/mozbuild/test/",
110 "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.sys.mjs",
111 "testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js",
112 "testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js",
113 "toolkit/components/reader/Readerable.sys.mjs",
114 "toolkit/components/workerloader/tests/moduleF-syntax-error.js",
116 "tools/update-packaging/test/",
117 # SpiderMonkey internals.
121 "browser/app/profile/firefox.js",
122 "browser/branding/official/pref/firefox-branding.js",
123 "browser/components/enterprisepolicies/schemas/schema.sys.mjs",
124 "browser/locales/en-US/firefox-l10n.js",
125 "mobile/android/app/geckoview-prefs.js",
126 "mobile/android/app/mobile.js",
127 "mobile/android/locales/en-US/mobile-l10n.js",
128 "modules/libpref/greprefs.js",
129 "modules/libpref/init/all.js",
130 "testing/condprofile/condprof/tests/profile/user.js",
131 "testing/mozbase/mozprofile/tests/files/prefs_with_comments.js",
132 "toolkit/modules/AppConstants.sys.mjs",
133 "toolkit/mozapps/update/tests/data/xpcshellConstantsPP.js",
135 "toolkit/components/uniffi-bindgen-gecko-js/src/templates/js/",
141 os
.path
.join("tools", "rewriting", "Generated.txt"),
142 os
.path
.join("tools", "rewriting", "ThirdPartyPaths.txt"),
146 def load_exclusion_files():
147 for path
in EXCLUSION_FILES
:
148 with
open(path
, "r") as f
:
150 p
= path_sep_to_native(re
.sub("\*$", "", line
.strip()))
151 excluded_from_imports_prefix
.append(p
)
154 def is_excluded_from_imports(path
):
155 """Returns true if the JS file content shouldn't be handled by
158 This filter is necessary because jscodeshift cannot handle some
159 syntax edge cases and results in unexpected rewrite."""
161 for prefix
in excluded_from_imports_prefix
:
162 if path_str
.startswith(prefix
):
168 # Wrapper for hg/git operations
171 # Do not pass check=True because the pattern can match no file.
172 lines
= subprocess
.run(cmd
, stdout
=subprocess
.PIPE
).stdout
.decode()
173 return filter(lambda x
: x
!= "", lines
.split("\n"))
176 class HgUtils(VCSUtils
):
178 return pathlib
.Path(".hg").exists()
180 def rename(self
, before
, after
):
181 cmd
= ["hg", "rename", before
, after
]
182 subprocess
.run(cmd
, check
=True)
184 def find_jsms(self
, path
):
187 # NOTE: `set:glob:` syntax does not accept backslash on windows.
188 path
= path_sep_from_native(path
)
190 cmd
= ["hg", "files", f
'set:glob:"{path}/**/*.jsm"']
191 for line
in self
.run(cmd
):
192 jsm
= pathlib
.Path(line
)
193 if is_excluded_from_convert(jsm
):
200 f
"set:grep('EXPORTED_SYMBOLS = \[') and glob:\"{path}/**/*.js\"",
202 for line
in self
.run(cmd
):
203 jsm
= pathlib
.Path(line
)
204 if is_excluded_from_convert(jsm
):
210 def find_all_jss(self
, path
):
213 # NOTE: `set:glob:` syntax does not accept backslash on windows.
214 path
= path_sep_from_native(path
)
219 f
'set:glob:"{path}/**/*.jsm" or glob:"{path}/**/*.js" or '
220 + f
'glob:"{path}/**/*.mjs" or glob:"{path}/**/*.sjs"',
222 for line
in self
.run(cmd
):
223 js
= pathlib
.Path(line
)
224 if is_excluded_from_imports(js
):
231 class GitUtils(VCSUtils
):
233 return pathlib
.Path(".git").exists()
235 def rename(self
, before
, after
):
236 cmd
= ["git", "mv", before
, after
]
237 subprocess
.run(cmd
, check
=True)
239 def find_jsms(self
, path
):
242 cmd
= ["git", "ls-files", f
"{path}/*.jsm"]
243 for line
in self
.run(cmd
):
244 jsm
= pathlib
.Path(line
)
245 if is_excluded_from_convert(jsm
):
250 cmd
= ["git", "grep", "EXPORTED_SYMBOLS = \[", f
"{path}/*.js"]
251 for line
in self
.run(cmd
):
252 m
= re
.search("^([^:]+):", line
)
255 filename
= m
.group(1)
256 if filename
in handled
:
258 handled
[filename
] = True
259 jsm
= pathlib
.Path(filename
)
260 if is_excluded_from_convert(jsm
):
266 def find_all_jss(self
, path
):
277 for line
in self
.run(cmd
):
278 js
= pathlib
.Path(line
)
279 if is_excluded_from_imports(js
):
288 self
.convert_errors
= []
289 self
.import_errors
= []
290 self
.rename_errors
= []
297 description
="ESMify JSM files.",
302 help="Path to the JSM file to ESMify, or the directory that contains "
303 "JSM files and/or JS files that imports ESM-ified JSM.",
308 help="Only perform the step 1 = convert part",
313 help="Only perform the step 2 = import calls part",
318 help="Restrict the target of import in the step 2 to ESM-ified JSM, by the "
319 "prefix match for the JSM file's path. e.g. 'browser/'.",
321 def esmify(command_context
, path
=None, convert
=False, imports
=False, prefix
=""):
323 This command does the following 2 steps:
324 1. Convert the JSM file specified by `path` to ESM file, or the JSM files
325 inside the directory specified by `path` to ESM files, and also
326 fix references in build files and test definitions
327 2. Convert import calls inside file(s) specified by `path` for ESM-ified
328 files to use new APIs
331 # Convert all JSM files inside `browser/components/pagedata` directory,
332 # and replace all references for ESM-ified files in the entire tree to use
335 $ ./mach esmify --convert browser/components/pagedata
336 $ ./mach esmify --imports . --prefix=browser/components/pagedata
339 # Convert all JSM files inside `browser` directory, and replace all
340 # references for the JSM files inside `browser` directory to use
343 $ ./mach esmify browser
347 command_context
.log(logging
.ERROR
, "esmify", {}, f
"[ERROR] {text}")
350 command_context
.log(logging
.WARN
, "esmify", {}, f
"[WARN] {text}")
353 command_context
.log(logging
.INFO
, "esmify", {}, f
"[INFO] {text}")
355 # If no options is specified, perform both.
356 if not convert
and not imports
:
360 path
= pathlib
.Path(path
[0])
362 if not verify_path(command_context
, path
):
365 if HgUtils
.is_available():
366 vcs_utils
= HgUtils()
367 elif GitUtils
.is_available():
368 vcs_utils
= GitUtils()
371 "This script needs to be run inside mozilla-central "
372 "checkout of either mercurial or git."
376 load_exclusion_files()
378 info("Setting up jscodeshift...")
381 is_single_file
= path
.is_file()
387 info("Searching files to convert to ESM...")
391 jsms
= vcs_utils
.find_jsms(path
)
393 info(f
"Found {len(jsms)} file(s) to convert to ESM.")
395 info("Converting to ESM...")
396 jsms
= convert_module(jsms
, summary
)
398 error("Failed to rewrite exports.")
402 esms
= rename_jsms(command_context
, vcs_utils
, jsms
, summary
)
404 modified_files
+= esms
407 info("Searching files to rewrite imports...")
411 # Already converted above
416 jss
= vcs_utils
.find_all_jss(path
)
418 info(f
"Checking {len(jss)} JS file(s). Rewriting any matching imports...")
420 result
= rewrite_imports(jss
, prefix
, summary
)
424 info(f
"Rewritten {len(result)} file(s).")
426 # Only modified files needs eslint fix
427 modified_files
+= result
429 modified_files
= list(set(modified_files
))
431 info(f
"Applying eslint --fix for {len(modified_files)} file(s)...")
432 eslint_fix(command_context
, modified_files
)
434 def print_files(f
, errors
):
435 for [path
, message
] in errors
:
440 if len(summary
.convert_errors
):
442 error("Following files are not converted into ESM due to error:")
443 print_files(error
, summary
.convert_errors
)
445 if len(summary
.import_errors
):
447 warn("Following files are not rewritten to import ESMs due to error:")
449 "(NOTE: Errors related to 'private names' are mostly due to "
450 " preprocessor macros in the file):"
452 print_files(warn
, summary
.import_errors
)
454 if len(summary
.rename_errors
):
456 error("Following files are not renamed due to error:")
457 print_files(error
, summary
.rename_errors
)
459 if len(summary
.no_refs
):
461 warn("Following files are not found in any build files.")
462 warn("Please update references to those files manually:")
463 print_files(warn
, summary
.rename_errors
)
468 def verify_path(command_context
, path
):
469 """Check if the path passed to the command is valid relative path."""
472 command_context
.log(logging
.ERROR
, "esmify", {}, f
"[ERROR] {text}")
474 if not path
.exists():
475 error(f
"{path} does not exist.")
478 if path
.is_absolute():
479 error("Path must be a relative path from mozilla-central checkout.")
485 def find_file(path
, target
):
486 """Find `target` file in ancestor of path."""
487 target_path
= path
.parent
/ target
488 if not target_path
.exists():
489 if path
.parent
== path
:
492 return find_file(path
.parent
, target
)
497 def try_rename_in(command_context
, path
, target
, jsm_name
, esm_name
, jsm_path
):
498 """Replace the occurrences of `jsm_name` with `esm_name` in `target`
502 command_context
.log(logging
.INFO
, "esmify", {}, f
"[INFO] {text}")
504 if type(target
) is str:
505 # Target is specified by filename, that may exist somewhere in
506 # the jsm's directory or ancestor directories.
507 target_path
= find_file(path
, target
)
511 # JSM should be specified with relative path in the file.
513 # Single moz.build or jar.mn can contain multiple files with same name.
514 # Search for relative path.
515 jsm_relative_path
= jsm_path
.relative_to(target_path
.parent
)
516 jsm_path_str
= path_sep_from_native(str(jsm_relative_path
))
518 # Target is specified by full path.
521 # JSM should be specified with full path in the file.
522 jsm_path_str
= path_sep_from_native(str(jsm_path
))
524 jsm_path_re
= re
.compile(r
"\b" + jsm_path_str
.replace(".", r
"\.") + r
"\b")
525 jsm_name_re
= re
.compile(r
"\b" + jsm_name
.replace(".", r
"\.") + r
"\b")
529 with
open(target_path
, "r") as f
:
531 if jsm_path_re
.search(line
):
533 line
= jsm_name_re
.sub(esm_name
, line
)
538 info(f
" {str(target_path)}")
539 info(f
" {jsm_name} => {esm_name}")
540 with
open(target_path
, "w", newline
="\n") as f
:
546 def try_rename_uri_in(command_context
, target
, jsm_name
, esm_name
, jsm_uri
, esm_uri
):
547 """Replace the occurrences of `jsm_uri` with `esm_uri` in `target` file."""
550 command_context
.log(logging
.INFO
, "esmify", {}, f
"[INFO] {text}")
554 with
open(target
, "r") as f
:
558 line
= line
.replace(jsm_uri
, esm_uri
)
563 info(f
" {str(target)}")
564 info(f
" {jsm_name} => {esm_name}")
565 with
open(target
, "w", newline
="\n") as f
:
571 def try_rename_components_conf(command_context
, path
, jsm_name
, esm_name
):
572 """Replace the occurrences of `jsm_name` with `esm_name` in components.conf
576 command_context
.log(logging
.INFO
, "esmify", {}, f
"[INFO] {text}")
578 target_path
= find_file(path
, "components.conf")
582 # Unlike try_rename_in, components.conf contains the URL instead of
583 # relative path, and also there are no known files with same name.
584 # Simply replace the filename.
586 with
open(target_path
, "r") as f
:
589 prop_re
= re
.compile(
590 "[\"']jsm[\"']:(.*)" + r
"\b" + jsm_name
.replace(".", r
"\.") + r
"\b"
593 if not prop_re
.search(content
):
596 info(f
" {str(target_path)}")
597 info(f
" {jsm_name} => {esm_name}")
599 content
= prop_re
.sub(r
"'esModule':\1" + esm_name
, content
)
600 with
open(target_path
, "w", newline
="\n") as f
:
606 def esmify_name(name
):
607 return re
.sub(r
"\.(jsm|js|jsm\.js)$", ".sys.mjs", name
)
610 def esmify_path(jsm_path
):
611 jsm_name
= jsm_path
.name
612 esm_name
= re
.sub(r
"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_name
)
613 esm_path
= jsm_path
.parent
/ esm_name
617 path_to_uri_map
= None
620 def load_path_to_uri_map():
621 global path_to_uri_map
626 if "ESMIFY_MAP_JSON" in os
.environ
:
627 json_map
= pathlib
.Path(os
.environ
["ESMIFY_MAP_JSON"])
629 json_map
= pathlib
.Path(__file__
).parent
/ "map.json"
631 with
open(json_map
, "r") as f
:
632 uri_to_path_map
= json
.loads(f
.read())
634 path_to_uri_map
= dict()
636 for uri
, paths
in uri_to_path_map
.items():
637 if type(paths
) is str:
641 path_to_uri_map
[path
] = uri
644 def find_jsm_uri(jsm_path
):
645 load_path_to_uri_map()
647 path
= path_sep_from_native(jsm_path
)
649 if path
in path_to_uri_map
:
650 return path_to_uri_map
[path
]
655 def rename_single_file(command_context
, vcs_utils
, jsm_path
, summary
):
656 """Rename `jsm_path` to .sys.mjs, and fix references to the file in build
657 and test definitions."""
660 command_context
.log(logging
.INFO
, "esmify", {}, f
"[INFO] {text}")
662 esm_path
= esmify_path(jsm_path
)
664 jsm_name
= jsm_path
.name
665 esm_name
= esm_path
.name
672 "browser-common.ini",
676 "xpcshell-child-process.ini",
677 "xpcshell-common.ini",
678 "xpcshell-parent-process.ini",
679 pathlib
.Path("tools", "lint", "eslint.yml"),
680 pathlib
.Path("tools", "lint", "rejected-words.yml"),
683 info(f
"{jsm_path} => {esm_path}")
686 for target
in target_files
:
688 command_context
, jsm_path
, target
, jsm_name
, esm_name
, jsm_path
692 if try_rename_components_conf(command_context
, jsm_path
, jsm_name
, esm_name
):
697 "browser", "base", "content", "test", "performance", "browser_startup.js"
705 "browser_startup_content.js",
713 "browser_startup_content_subframe.js",
721 "browser_xpcom_graph_wait.js",
725 jsm_uri
= find_jsm_uri(jsm_path
)
727 esm_uri
= re
.sub(r
"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_uri
)
729 for target
in uri_target_files
:
730 if try_rename_uri_in(
731 command_context
, target
, jsm_uri
, esm_uri
, jsm_name
, esm_name
736 summary
.no_refs
.append([jsm_path
, None])
738 if not esm_path
.exists():
739 vcs_utils
.rename(jsm_path
, esm_path
)
741 summary
.rename_errors
.append([jsm_path
, f
"{esm_path} already exists"])
746 def rename_jsms(command_context
, vcs_utils
, jsms
, summary
):
749 esm
= rename_single_file(command_context
, vcs_utils
, jsm
, summary
)
755 npm_prefix
= pathlib
.Path("tools") / "esmify"
756 path_from_npm_prefix
= pathlib
.Path("..") / ".."
759 def setup_jscodeshift():
760 """Install jscodeshift."""
771 subprocess
.run(cmd
, check
=True)
774 def run_npm_command(args
, env
, stdin
):
781 p
= subprocess
.Popen(cmd
, env
=env
, stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
)
788 line
= p
.stdout
.readline()
791 line
= line
.rstrip().decode()
793 if line
.startswith(" NOC "):
798 m
= re
.search(r
"^ (OKK|ERR) ([^ ]+)(?: (.+))?", line
)
803 # NOTE: path is written from `tools/esmify`.
804 path
= pathlib
.Path(m
.group(2)).relative_to(path_from_npm_prefix
)
808 ok_files
.append(path
)
811 errors
.append([path
, error
])
816 return ok_files
, errors
819 def convert_module(jsms
, summary
):
820 """Replace EXPORTED_SYMBOLS with export declarations, and replace
821 ChromeUtils.importESModule with static import as much as possible,
822 and return the list of successfully rewritten files."""
827 env
= os
.environ
.copy()
829 stdin
= "\n".join(map(str, paths_from_npm_prefix(jsms
))).encode()
831 ok_files
, errors
= run_npm_command(
841 if ok_files
is None and errors
is None:
844 summary
.convert_errors
.extend(errors
)
849 def rewrite_imports(jss
, prefix
, summary
):
850 """Replace import calls for JSM with import calls for ESM or static import
856 env
= os
.environ
.copy()
857 env
["ESMIFY_TARGET_PREFIX"] = prefix
859 stdin
= "\n".join(map(str, paths_from_npm_prefix(jss
))).encode()
861 ok_files
, errors
= run_npm_command(
871 if ok_files
is None and errors
is None:
874 summary
.import_errors
.extend(errors
)
879 def paths_from_npm_prefix(paths
):
880 """Convert relative path from mozilla-central to relative path from
882 return list(map(lambda path
: path_from_npm_prefix
/ path
, paths
))
885 def eslint_fix(command_context
, files
):
886 """Auto format files."""
889 command_context
.log(logging
.INFO
, "esmify", {}, f
"[INFO] {text}")
894 remaining
= files
[0:]
896 # There can be too many files for single command line, perform by chunk.
898 while len(remaining
) > max_files
:
899 info(f
"{len(remaining)} files remaining")
901 chunk
= remaining
[0:max_files
]
902 remaining
= remaining
[max_files
:]
904 cmd
= [sys
.executable
, "./mach", "eslint", "--fix"] + chunk
905 subprocess
.run(cmd
, check
=True)
907 info(f
"{len(remaining)} files remaining")
909 cmd
= [sys
.executable
, "./mach", "eslint", "--fix"] + chunk
910 subprocess
.run(cmd
, check
=True)