No Bug, mozilla-central repo-update HSTS HPKP remote-settings mobile-experiments...
[gecko.git] / mobile / android / mach_commands.py
blob3e0d22ff8c3bc27aec6c4f086abda805f5aca24b
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 import argparse
6 import logging
7 import os
8 import sys
9 import tarfile
10 import time
12 import mozpack.path as mozpath
13 from mach.decorators import Command, CommandArgument, SubCommand
14 from mozbuild.base import MachCommandConditions as conditions
15 from mozbuild.shellutil import split as shell_split
17 # Mach's conditions facility doesn't support subcommands. Print a
18 # deprecation message ourselves instead.
19 LINT_DEPRECATION_MESSAGE = """
20 Android lints are now integrated with mozlint. Instead of
21 `mach android {api-lint,checkstyle,lint,test}`, run
22 `mach lint --linter android-{api-lint,checkstyle,lint,test}`.
23 Or run `mach lint`.
24 """
27 # NOTE python/mach/mach/commands/commandinfo.py references this function
28 # by name. If this function is renamed or removed, that file should
29 # be updated accordingly as well.
30 def REMOVED(cls):
31 """Command no longer exists! Use the Gradle configuration rooted in the top source directory
32 instead.
34 See https://developer.mozilla.org/en-US/docs/Simple_Firefox_for_Android_build#Developing_Firefox_for_Android_in_Android_Studio_or_IDEA_IntelliJ. # NOQA: E501
35 """
36 return False
39 @Command(
40 "android",
41 category="devenv",
42 description="Run Android-specific commands.",
43 conditions=[conditions.is_android],
45 def android(command_context):
46 pass
49 @SubCommand(
50 "android",
51 "assemble-app",
52 """Assemble Firefox for Android.
53 See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""", # NOQA: E501
55 @CommandArgument("args", nargs=argparse.REMAINDER)
56 def android_assemble_app(command_context, args):
57 ret = gradle(
58 command_context,
59 command_context.substs["GRADLE_ANDROID_APP_TASKS"] + ["-x", "lint"] + args,
60 verbose=True,
63 return ret
66 @SubCommand(
67 "android",
68 "generate-sdk-bindings",
69 """Generate SDK bindings used when building GeckoView.""",
71 @CommandArgument(
72 "inputs",
73 nargs="+",
74 help="config files, like [/path/to/ClassName-classes.txt]+",
76 @CommandArgument("args", nargs=argparse.REMAINDER)
77 def android_generate_sdk_bindings(command_context, inputs, args):
78 import itertools
80 def stem(input):
81 # Turn "/path/to/ClassName-classes.txt" into "ClassName".
82 return os.path.basename(input).rsplit("-classes.txt", 1)[0]
84 bindings_inputs = list(itertools.chain(*((input, stem(input)) for input in inputs)))
85 bindings_args = "-Pgenerate_sdk_bindings_args={}".format(";".join(bindings_inputs))
87 ret = gradle(
88 command_context,
89 command_context.substs["GRADLE_ANDROID_GENERATE_SDK_BINDINGS_TASKS"]
90 + [bindings_args]
91 + args,
92 verbose=True,
95 return ret
98 @SubCommand(
99 "android",
100 "generate-generated-jni-wrappers",
101 """Generate GeckoView JNI wrappers used when building GeckoView.""",
103 @CommandArgument("args", nargs=argparse.REMAINDER)
104 def android_generate_generated_jni_wrappers(command_context, args):
105 ret = gradle(
106 command_context,
107 command_context.substs["GRADLE_ANDROID_GENERATE_GENERATED_JNI_WRAPPERS_TASKS"]
108 + args,
109 verbose=True,
112 return ret
115 @SubCommand(
116 "android",
117 "api-lint",
118 """Run Android api-lint.
119 REMOVED/DEPRECATED: Use 'mach lint --linter android-api-lint'.""",
121 def android_apilint_REMOVED(command_context):
122 print(LINT_DEPRECATION_MESSAGE)
123 return 1
126 @SubCommand(
127 "android",
128 "test",
129 """Run Android test.
130 REMOVED/DEPRECATED: Use 'mach lint --linter android-test'.""",
132 def android_test_REMOVED(command_context):
133 print(LINT_DEPRECATION_MESSAGE)
134 return 1
137 @SubCommand(
138 "android",
139 "lint",
140 """Run Android lint.
141 REMOVED/DEPRECATED: Use 'mach lint --linter android-lint'.""",
143 def android_lint_REMOVED(command_context):
144 print(LINT_DEPRECATION_MESSAGE)
145 return 1
148 @SubCommand(
149 "android",
150 "checkstyle",
151 """Run Android checkstyle.
152 REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
154 def android_checkstyle_REMOVED(command_context):
155 print(LINT_DEPRECATION_MESSAGE)
156 return 1
159 @SubCommand(
160 "android",
161 "gradle-dependencies",
162 """Collect Android Gradle dependencies.
163 See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""", # NOQA: E501
165 @CommandArgument("args", nargs=argparse.REMAINDER)
166 def android_gradle_dependencies(command_context, args):
167 # We don't want to gate producing dependency archives on clean
168 # lint or checkstyle, particularly because toolchain versions
169 # can change the outputs for those processes.
170 gradle(
171 command_context,
172 command_context.substs["GRADLE_ANDROID_DEPENDENCIES_TASKS"]
173 + ["--continue"]
174 + args,
175 verbose=True,
178 return 0
181 def get_maven_archive_paths(maven_folder):
182 for subdir, _, files in os.walk(maven_folder):
183 if "-SNAPSHOT" in subdir:
184 continue
185 for file in files:
186 yield os.path.join(subdir, file)
189 def create_maven_archive(topobjdir):
190 gradle_folder = os.path.join(topobjdir, "gradle")
191 maven_folder = os.path.join(gradle_folder, "maven")
193 # Create the archive, with no compression: The archive contents are large
194 # files which cannot be significantly compressed; attempting to compress
195 # the archive is usually expensive in time and results in minimal
196 # reduction in size.
197 # Even though the archive is not compressed, use the .xz file extension
198 # so that the taskcluster worker also skips compression.
199 with tarfile.open(os.path.join(gradle_folder, "target.maven.tar.xz"), "w") as tar:
200 for abs_path in get_maven_archive_paths(maven_folder):
201 tar.add(
202 abs_path,
203 arcname=os.path.join(
204 "geckoview", os.path.relpath(abs_path, maven_folder)
209 @SubCommand(
210 "android",
211 "archive-geckoview",
212 """Create GeckoView archives.
213 See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""", # NOQA: E501
215 @CommandArgument("args", nargs=argparse.REMAINDER)
216 def android_archive_geckoview(command_context, args):
217 tasks = command_context.substs["GRADLE_ANDROID_ARCHIVE_GECKOVIEW_TASKS"]
218 subproject = command_context.substs.get("MOZ_ANDROID_SUBPROJECT")
219 if subproject in (None, "geckoview_example"):
220 tasks += command_context.substs[
221 "GRADLE_ANDROID_ARCHIVE_GECKOVIEW_SUBPROJECT_TASKS"
223 ret = gradle(
224 command_context,
225 tasks + args,
226 verbose=True,
229 if ret != 0:
230 return ret
231 if "MOZ_AUTOMATION" in os.environ:
232 create_maven_archive(command_context.topobjdir)
234 return 0
237 @SubCommand("android", "build-geckoview_example", """Build geckoview_example """)
238 @CommandArgument("args", nargs=argparse.REMAINDER)
239 def android_build_geckoview_example(command_context, args):
240 gradle(
241 command_context,
242 command_context.substs["GRADLE_ANDROID_BUILD_GECKOVIEW_EXAMPLE_TASKS"] + args,
243 verbose=True,
246 print(
247 "Execute `mach android install-geckoview_example` "
248 "to push the geckoview_example and test APKs to a device."
251 return 0
254 @SubCommand("android", "compile-all", """Build all source files""")
255 @CommandArgument("args", nargs=argparse.REMAINDER)
256 def android_compile_all(command_context, args):
257 ret = gradle(
258 command_context,
259 command_context.substs["GRADLE_ANDROID_COMPILE_ALL_TASKS"] + args,
260 verbose=True,
263 return ret
266 def install_app_bundle(command_context, bundle):
267 from mozdevice import ADBDeviceFactory
269 bundletool = mozpath.join(command_context._mach_context.state_dir, "bundletool.jar")
270 device = ADBDeviceFactory(verbose=True)
271 bundle_path = mozpath.join(command_context.topobjdir, bundle)
272 java_home = java_home = os.path.dirname(
273 os.path.dirname(command_context.substs["JAVA"])
275 device.install_app_bundle(bundletool, bundle_path, java_home, timeout=120)
278 @SubCommand("android", "install-geckoview_example", """Install geckoview_example """)
279 @CommandArgument("args", nargs=argparse.REMAINDER)
280 def android_install_geckoview_example(command_context, args):
281 gradle(
282 command_context,
283 command_context.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_EXAMPLE_TASKS"] + args,
284 verbose=True,
287 print(
288 "Execute `mach android build-geckoview_example` "
289 "to just build the geckoview_example and test APKs."
292 return 0
295 @SubCommand("android", "install-fenix", """Install fenix """)
296 @CommandArgument("args", nargs=argparse.REMAINDER)
297 def android_install_fenix(command_context, args):
298 gradle(
299 command_context,
300 ["fenix:installFenixDebug"] + args,
301 verbose=True,
303 return 0
306 @SubCommand("android", "install-focus", """Install focus """)
307 @CommandArgument("args", nargs=argparse.REMAINDER)
308 def android_install_focus(command_context, args):
309 gradle(
310 command_context,
311 ["focus-android:installFocusDebug"] + args,
312 verbose=True,
314 return 0
317 @SubCommand(
318 "android", "install-geckoview-test_runner", """Install geckoview.test_runner """
320 @CommandArgument("args", nargs=argparse.REMAINDER)
321 def android_install_geckoview_test_runner(command_context, args):
322 gradle(
323 command_context,
324 command_context.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_TEST_RUNNER_TASKS"]
325 + args,
326 verbose=True,
328 return 0
331 @SubCommand("android", "installFenixRelease", """Install fenix Release""")
332 @CommandArgument("args", nargs=argparse.REMAINDER)
333 def android_install_fenix_release(command_context, args):
334 gradle(
335 command_context,
336 ["installFenixRelease"] + args,
337 verbose=True,
338 gradle_path=mozpath.join(
339 command_context.topsrcdir, "mobile", "android", "fenix", "gradlew"
341 topsrcdir=mozpath.join(command_context.topsrcdir, "mobile", "android", "fenix"),
343 return 0
346 @SubCommand(
347 "android",
348 "install-geckoview-test_runner-aab",
349 """Install geckoview.test_runner with AAB""",
351 @CommandArgument("args", nargs=argparse.REMAINDER)
352 def android_install_geckoview_test_runner_aab(command_context, args):
353 install_app_bundle(
354 command_context,
355 command_context.substs["GRADLE_ANDROID_GECKOVIEW_TEST_RUNNER_BUNDLE"],
357 return 0
360 @SubCommand(
361 "android",
362 "install-geckoview_example-aab",
363 """Install geckoview_example with AAB""",
365 @CommandArgument("args", nargs=argparse.REMAINDER)
366 def android_install_geckoview_example_aab(command_context, args):
367 install_app_bundle(
368 command_context,
369 command_context.substs["GRADLE_ANDROID_GECKOVIEW_EXAMPLE_BUNDLE"],
371 return 0
374 @SubCommand("android", "install-geckoview-test", """Install geckoview.test """)
375 @CommandArgument("args", nargs=argparse.REMAINDER)
376 def android_install_geckoview_test(command_context, args):
377 gradle(
378 command_context,
379 command_context.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_TEST_TASKS"] + args,
380 verbose=True,
382 return 0
385 @SubCommand(
386 "android",
387 "geckoview-docs",
388 """Create GeckoView javadoc and optionally upload to Github""",
390 @CommandArgument("--archive", action="store_true", help="Generate a javadoc archive.")
391 @CommandArgument(
392 "--upload",
393 metavar="USER/REPO",
394 help="Upload geckoview documentation to Github, using the specified USER/REPO.",
396 @CommandArgument(
397 "--upload-branch",
398 metavar="BRANCH[/PATH]",
399 default="gh-pages",
400 help="Use the specified branch/path for documentation commits.",
402 @CommandArgument(
403 "--javadoc-path",
404 metavar="/PATH",
405 default="javadoc",
406 help="Use the specified path for javadoc commits.",
408 @CommandArgument(
409 "--upload-message",
410 metavar="MSG",
411 default="GeckoView docs upload",
412 help="Use the specified message for commits.",
414 def android_geckoview_docs(
415 command_context,
416 archive,
417 upload,
418 upload_branch,
419 javadoc_path,
420 upload_message,
422 tasks = (
423 command_context.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_ARCHIVE_TASKS"]
424 if archive or upload
425 else command_context.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"]
428 ret = gradle(command_context, tasks, verbose=True)
429 if ret or not upload:
430 return ret
432 # Upload to Github.
433 fmt = {
434 "level": os.environ.get("MOZ_SCM_LEVEL", "0"),
435 "project": os.environ.get("MH_BRANCH", "unknown"),
436 "revision": os.environ.get("GECKO_HEAD_REV", "tip"),
438 env = {}
440 # In order to push to GitHub from TaskCluster, we store a private key
441 # in the TaskCluster secrets store in the format {"content": "<KEY>"},
442 # and the corresponding public key as a writable deploy key for the
443 # destination repo on GitHub.
444 secret = os.environ.get("GECKOVIEW_DOCS_UPLOAD_SECRET", "").format(**fmt)
445 if secret:
446 # Set up a private key from the secrets store if applicable.
447 import requests
449 req = requests.get("http://taskcluster/secrets/v1/secret/" + secret)
450 req.raise_for_status()
452 keyfile = mozpath.abspath("gv-docs-upload-key")
453 with open(keyfile, "w") as f:
454 os.chmod(keyfile, 0o600)
455 f.write(req.json()["secret"]["content"])
457 # Turn off strict host key checking so ssh does not complain about
458 # unknown github.com host. We're not pushing anything sensitive, so
459 # it's okay to not check GitHub's host keys.
460 env["GIT_SSH_COMMAND"] = 'ssh -i "%s" -o StrictHostKeyChecking=no' % keyfile
462 # Clone remote repo.
463 branch = upload_branch.format(**fmt)
464 repo_url = "git@github.com:%s.git" % upload
465 repo_path = mozpath.abspath("gv-docs-repo")
466 command_context.run_process(
468 "git",
469 "clone",
470 "--branch",
471 upload_branch,
472 "--depth",
473 "1",
474 repo_url,
475 repo_path,
477 append_env=env,
478 pass_thru=True,
480 env["GIT_DIR"] = mozpath.join(repo_path, ".git")
481 env["GIT_WORK_TREE"] = repo_path
482 env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "GeckoView Docs Bot"
483 env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "nobody@mozilla.com"
485 # Copy over user documentation.
486 import mozfile
488 # Extract new javadoc to specified directory inside repo.
489 src_tar = mozpath.join(
490 command_context.topobjdir,
491 "gradle",
492 "build",
493 "mobile",
494 "android",
495 "geckoview",
496 "libs",
497 "geckoview-javadoc.jar",
499 dst_path = mozpath.join(repo_path, javadoc_path.format(**fmt))
500 mozfile.remove(dst_path)
501 mozfile.extract_zip(src_tar, dst_path)
503 # Commit and push.
504 command_context.run_process(["git", "add", "--all"], append_env=env, pass_thru=True)
505 if (
506 command_context.run_process(
507 ["git", "diff", "--cached", "--quiet"],
508 append_env=env,
509 pass_thru=True,
510 ensure_exit_code=False,
512 != 0
514 # We have something to commit.
515 command_context.run_process(
516 ["git", "commit", "--message", upload_message.format(**fmt)],
517 append_env=env,
518 pass_thru=True,
520 command_context.run_process(
521 ["git", "push", "origin", branch], append_env=env, pass_thru=True
524 mozfile.remove(repo_path)
525 if secret:
526 mozfile.remove(keyfile)
527 return 0
530 @Command(
531 "gradle",
532 category="devenv",
533 description="Run gradle.",
534 conditions=[conditions.is_android],
536 @CommandArgument(
537 "-v",
538 "--verbose",
539 action="store_true",
540 help="Verbose output for what commands the build is running.",
542 @CommandArgument("args", nargs=argparse.REMAINDER)
543 def gradle(command_context, args, verbose=False, gradle_path=None, topsrcdir=None):
544 if not verbose:
545 # Avoid logging the command
546 command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
548 if not gradle_path:
549 gradle_path = command_context.substs["GRADLE"]
551 if not topsrcdir:
552 topsrcdir = mozpath.join(command_context.topsrcdir)
554 # In automation, JAVA_HOME is set via mozconfig, which needs
555 # to be specially handled in each mach command. This turns
556 # $JAVA_HOME/bin/java into $JAVA_HOME.
557 java_home = os.path.dirname(os.path.dirname(command_context.substs["JAVA"]))
559 gradle_flags = command_context.substs.get("GRADLE_FLAGS", "") or os.environ.get(
560 "GRADLE_FLAGS", ""
562 gradle_flags = shell_split(gradle_flags)
564 # We force the Gradle JVM to run with the UTF-8 encoding, since we
565 # filter strings.xml, which is really UTF-8; the ellipsis character is
566 # replaced with ??? in some encodings (including ASCII). It's not yet
567 # possible to filter with encodings in Gradle
568 # (https://github.com/gradle/gradle/pull/520) and it's challenging to
569 # do our filtering with Gradle's Ant support. Moreover, all of the
570 # Android tools expect UTF-8: see
571 # http://tools.android.com/knownissues/encoding. See
572 # http://stackoverflow.com/a/21267635 for discussion of this approach.
574 # It's not even enough to set the encoding just for Gradle; it
575 # needs to be for JVMs spawned by Gradle as well. This
576 # happens during the maven deployment generating the GeckoView
577 # documents; this works around "error: unmappable character
578 # for encoding ASCII" in exoplayer2. See
579 # https://discuss.gradle.org/t/unmappable-character-for-encoding-ascii-when-building-a-utf-8-project/10692/11 # NOQA: E501
580 # and especially https://stackoverflow.com/a/21755671.
582 if command_context.substs.get("MOZ_AUTOMATION"):
583 gradle_flags += ["--console=plain"]
585 env = os.environ.copy()
587 env.update(
589 "GRADLE_OPTS": "-Dfile.encoding=utf-8",
590 "JAVA_HOME": java_home,
591 "JAVA_TOOL_OPTIONS": "-Dfile.encoding=utf-8",
592 # Let Gradle get the right Python path on Windows
593 "GRADLE_MACH_PYTHON": sys.executable,
596 # Set ANDROID_SDK_ROOT if --with-android-sdk was set.
597 # See https://bugzilla.mozilla.org/show_bug.cgi?id=1576471
598 android_sdk_root = command_context.substs.get("ANDROID_SDK_ROOT", "")
599 if android_sdk_root:
600 env["ANDROID_SDK_ROOT"] = android_sdk_root
602 should_print_status = env.get("MACH") and not env.get("NO_BUILDSTATUS_MESSAGES")
603 if should_print_status:
604 print("BUILDSTATUS " + str(time.time()) + " START_Gradle " + args[0])
605 rv = command_context.run_process(
606 [gradle_path] + gradle_flags + args,
607 explicit_env=env,
608 pass_thru=True, # Allow user to run gradle interactively.
609 ensure_exit_code=False, # Don't throw on non-zero exit code.
610 cwd=topsrcdir,
612 if should_print_status:
613 print("BUILDSTATUS " + str(time.time()) + " END_Gradle " + args[0])
614 return rv
617 @Command("gradle-install", category="devenv", conditions=[REMOVED])
618 def gradle_install_REMOVED(command_context):
619 pass
622 @Command(
623 "android-emulator",
624 category="devenv",
625 conditions=[],
626 description="Run the Android emulator with an AVD from test automation. "
627 "Environment variable MOZ_EMULATOR_COMMAND_ARGS, if present, will "
628 "over-ride the command line arguments used to launch the emulator.",
630 @CommandArgument(
631 "--version",
632 metavar="VERSION",
633 choices=["arm", "arm64", "x86_64"],
634 help="Specify which AVD to run in emulator. "
635 'One of "arm" (Android supporting armv7 binaries), '
636 '"arm64" (for Apple Silicon), or '
637 '"x86_64" (Android supporting x86 or x86_64 binaries, '
638 "recommended for most applications). "
639 "By default, the value will match the current build environment.",
641 @CommandArgument("--wait", action="store_true", help="Wait for emulator to be closed.")
642 @CommandArgument("--gpu", help="Over-ride the emulator -gpu argument.")
643 @CommandArgument(
644 "--verbose", action="store_true", help="Log informative status messages."
646 def emulator(
647 command_context,
648 version,
649 wait=False,
650 gpu=None,
651 verbose=False,
654 Run the Android emulator with one of the AVDs used in the Mozilla
655 automated test environment. If necessary, the AVD is fetched from
656 the taskcluster server and installed.
658 from mozrunner.devices.android_device import AndroidEmulator
660 emulator = AndroidEmulator(
661 version,
662 verbose,
663 substs=command_context.substs,
664 device_serial="emulator-5554",
666 if emulator.is_running():
667 # It is possible to run multiple emulators simultaneously, but:
668 # - if more than one emulator is using the same avd, errors may
669 # occur due to locked resources;
670 # - additional parameters must be specified when running tests,
671 # to select a specific device.
672 # To avoid these complications, allow just one emulator at a time.
673 command_context.log(
674 logging.ERROR,
675 "emulator",
677 "An Android emulator is already running.\n"
678 "Close the existing emulator and re-run this command.",
680 return 1
682 if not emulator.check_avd():
683 command_context.log(
684 logging.WARN,
685 "emulator",
687 "AVD not found. Please run |mach bootstrap|.",
689 return 2
691 if not emulator.is_available():
692 command_context.log(
693 logging.WARN,
694 "emulator",
696 "Emulator binary not found.\n"
697 "Install the Android SDK and make sure 'emulator' is in your PATH.",
699 return 2
701 command_context.log(
702 logging.INFO,
703 "emulator",
705 "Starting Android emulator running %s..." % emulator.get_avd_description(),
707 emulator.start(gpu)
708 if emulator.wait_for_start():
709 command_context.log(
710 logging.INFO, "emulator", {}, "Android emulator is running."
712 else:
713 # This is unusual but the emulator may still function.
714 command_context.log(
715 logging.WARN,
716 "emulator",
718 "Unable to verify that emulator is running.",
721 if conditions.is_android(command_context):
722 command_context.log(
723 logging.INFO,
724 "emulator",
726 "Use 'mach install' to install or update Firefox on your emulator.",
728 else:
729 command_context.log(
730 logging.WARN,
731 "emulator",
733 "No Firefox for Android build detected.\n"
734 "Switch to a Firefox for Android build context or use 'mach bootstrap'\n"
735 "to setup an Android build environment.",
738 if wait:
739 command_context.log(
740 logging.INFO, "emulator", {}, "Waiting for Android emulator to close..."
742 rc = emulator.wait()
743 if rc is not None:
744 command_context.log(
745 logging.INFO,
746 "emulator",
748 "Android emulator completed with return code %d." % rc,
750 else:
751 command_context.log(
752 logging.WARN,
753 "emulator",
755 "Unable to retrieve Android emulator return code.",
757 return 0