mobile/android/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import logging
   7 import os
   8
   9 import mozpack.path as mozpath
  10 from mach.decorators import Command, CommandArgument, SubCommand
  11 from mozbuild.base import MachCommandConditions as conditions
  12 from mozbuild.shellutil import split as shell_split
  13
  14 # Mach's conditions facility doesn't support subcommands.  Print a
  15 # deprecation message ourselves instead.
  16 LINT_DEPRECATION_MESSAGE = """
  17 Android lints are now integrated with mozlint.  Instead of
  18 `mach android {api-lint,checkstyle,lint,test}`, run
  19 `mach lint --linter android-{api-lint,checkstyle,lint,test}`.
  20 Or run `mach lint`.
  21 """
  22
  23
  24 # NOTE python/mach/mach/commands/commandinfo.py references this function
  25 #      by name. If this function is renamed or removed, that file should
  26 #      be updated accordingly as well.
  27 def REMOVED(cls):
  28     """Command no longer exists! Use the Gradle configuration rooted in the top source directory
  29     instead.
  30
  31     See https://developer.mozilla.org/en-US/docs/Simple_Firefox_for_Android_build#Developing_Firefox_for_Android_in_Android_Studio_or_IDEA_IntelliJ.  # NOQA: E501
  32     """
  33     return False
  34
  35
  36 @Command(
  37     "android",
  38     category="devenv",
  39     description="Run Android-specific commands.",
  40     conditions=[conditions.is_android],
  41 )
  42 def android(command_context):
  43     pass
  44
  45
  46 @SubCommand(
  47     "android",
  48     "assemble-app",
  49     """Assemble Firefox for Android.
  50     See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""",  # NOQA: E501
  51 )
  52 @CommandArgument("args", nargs=argparse.REMAINDER)
  53 def android_assemble_app(command_context, args):
  54     ret = gradle(
  55         command_context,
  56         command_context.substs["GRADLE_ANDROID_APP_TASKS"] + ["-x", "lint"] + args,
  57         verbose=True,
  58     )
  59
  60     return ret
  61
  62
  63 @SubCommand(
  64     "android",
  65     "generate-sdk-bindings",
  66     """Generate SDK bindings used when building GeckoView.""",
  67 )
  68 @CommandArgument(
  69     "inputs",
  70     nargs="+",
  71     help="config files, like [/path/to/ClassName-classes.txt]+",
  72 )
  73 @CommandArgument("args", nargs=argparse.REMAINDER)
  74 def android_generate_sdk_bindings(command_context, inputs, args):
  75     import itertools
  76
  77     def stem(input):
  78         # Turn "/path/to/ClassName-classes.txt" into "ClassName".
  79         return os.path.basename(input).rsplit("-classes.txt", 1)[0]
  80
  81     bindings_inputs = list(itertools.chain(*((input, stem(input)) for input in inputs)))
  82     bindings_args = "-Pgenerate_sdk_bindings_args={}".format(";".join(bindings_inputs))
  83
  84     ret = gradle(
  85         command_context,
  86         command_context.substs["GRADLE_ANDROID_GENERATE_SDK_BINDINGS_TASKS"]
  87         + [bindings_args]
  88         + args,
  89         verbose=True,
  90     )
  91
  92     return ret
  93
  94
  95 @SubCommand(
  96     "android",
  97     "generate-generated-jni-wrappers",
  98     """Generate GeckoView JNI wrappers used when building GeckoView.""",
  99 )
 100 @CommandArgument("args", nargs=argparse.REMAINDER)
 101 def android_generate_generated_jni_wrappers(command_context, args):
 102     ret = gradle(
 103         command_context,
 104         command_context.substs["GRADLE_ANDROID_GENERATE_GENERATED_JNI_WRAPPERS_TASKS"]
 105         + args,
 106         verbose=True,
 107     )
 108
 109     return ret
 110
 111
 112 @SubCommand(
 113     "android",
 114     "api-lint",
 115     """Run Android api-lint.
 116 REMOVED/DEPRECATED: Use 'mach lint --linter android-api-lint'.""",
 117 )
 118 def android_apilint_REMOVED(command_context):
 119     print(LINT_DEPRECATION_MESSAGE)
 120     return 1
 121
 122
 123 @SubCommand(
 124     "android",
 125     "test",
 126     """Run Android test.
 127 REMOVED/DEPRECATED: Use 'mach lint --linter android-test'.""",
 128 )
 129 def android_test_REMOVED(command_context):
 130     print(LINT_DEPRECATION_MESSAGE)
 131     return 1
 132
 133
 134 @SubCommand(
 135     "android",
 136     "lint",
 137     """Run Android lint.
 138 REMOVED/DEPRECATED: Use 'mach lint --linter android-lint'.""",
 139 )
 140 def android_lint_REMOVED(command_context):
 141     print(LINT_DEPRECATION_MESSAGE)
 142     return 1
 143
 144
 145 @SubCommand(
 146     "android",
 147     "checkstyle",
 148     """Run Android checkstyle.
 149 REMOVED/DEPRECATED: Use 'mach lint --linter android-checkstyle'.""",
 150 )
 151 def android_checkstyle_REMOVED(command_context):
 152     print(LINT_DEPRECATION_MESSAGE)
 153     return 1
 154
 155
 156 @SubCommand(
 157     "android",
 158     "gradle-dependencies",
 159     """Collect Android Gradle dependencies.
 160     See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""",  # NOQA: E501
 161 )
 162 @CommandArgument("args", nargs=argparse.REMAINDER)
 163 def android_gradle_dependencies(command_context, args):
 164     # We don't want to gate producing dependency archives on clean
 165     # lint or checkstyle, particularly because toolchain versions
 166     # can change the outputs for those processes.
 167     gradle(
 168         command_context,
 169         command_context.substs["GRADLE_ANDROID_DEPENDENCIES_TASKS"]
 170         + ["--continue"]
 171         + args,
 172         verbose=True,
 173     )
 174
 175     return 0
 176
 177
 178 @SubCommand(
 179     "android",
 180     "archive-geckoview",
 181     """Create GeckoView archives.
 182     See http://firefox-source-docs.mozilla.org/build/buildsystem/toolchains.html#firefox-for-android-with-gradle""",  # NOQA: E501
 183 )
 184 @CommandArgument("args", nargs=argparse.REMAINDER)
 185 def android_archive_geckoview(command_context, args):
 186     ret = gradle(
 187         command_context,
 188         command_context.substs["GRADLE_ANDROID_ARCHIVE_GECKOVIEW_TASKS"] + args,
 189         verbose=True,
 190     )
 191
 192     return ret
 193
 194
 195 @SubCommand("android", "build-geckoview_example", """Build geckoview_example """)
 196 @CommandArgument("args", nargs=argparse.REMAINDER)
 197 def android_build_geckoview_example(command_context, args):
 198     gradle(
 199         command_context,
 200         command_context.substs["GRADLE_ANDROID_BUILD_GECKOVIEW_EXAMPLE_TASKS"] + args,
 201         verbose=True,
 202     )
 203
 204     print(
 205         "Execute `mach android install-geckoview_example` "
 206         "to push the geckoview_example and test APKs to a device."
 207     )
 208
 209     return 0
 210
 211
 212 def install_app_bundle(command_context, bundle):
 213     from mozdevice import ADBDeviceFactory
 214
 215     bundletool = mozpath.join(command_context._mach_context.state_dir, "bundletool.jar")
 216     device = ADBDeviceFactory(verbose=True)
 217     bundle_path = mozpath.join(command_context.topobjdir, bundle)
 218     java_home = java_home = os.path.dirname(
 219         os.path.dirname(command_context.substs["JAVA"])
 220     )
 221     device.install_app_bundle(bundletool, bundle_path, java_home, timeout=120)
 222
 223
 224 @SubCommand("android", "install-geckoview_example", """Install geckoview_example """)
 225 @CommandArgument("args", nargs=argparse.REMAINDER)
 226 def android_install_geckoview_example(command_context, args):
 227     gradle(
 228         command_context,
 229         command_context.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_EXAMPLE_TASKS"] + args,
 230         verbose=True,
 231     )
 232
 233     print(
 234         "Execute `mach android build-geckoview_example` "
 235         "to just build the geckoview_example and test APKs."
 236     )
 237
 238     return 0
 239
 240
 241 @SubCommand(
 242     "android", "install-geckoview-test_runner", """Install geckoview.test_runner """
 243 )
 244 @CommandArgument("args", nargs=argparse.REMAINDER)
 245 def android_install_geckoview_test_runner(command_context, args):
 246     gradle(
 247         command_context,
 248         command_context.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_TEST_RUNNER_TASKS"]
 249         + args,
 250         verbose=True,
 251     )
 252     return 0
 253
 254
 255 @SubCommand(
 256     "android",
 257     "install-geckoview-test_runner-aab",
 258     """Install geckoview.test_runner with AAB""",
 259 )
 260 @CommandArgument("args", nargs=argparse.REMAINDER)
 261 def android_install_geckoview_test_runner_aab(command_context, args):
 262     install_app_bundle(
 263         command_context,
 264         command_context.substs["GRADLE_ANDROID_GECKOVIEW_TEST_RUNNER_BUNDLE"],
 265     )
 266     return 0
 267
 268
 269 @SubCommand(
 270     "android",
 271     "install-geckoview_example-aab",
 272     """Install geckoview_example with AAB""",
 273 )
 274 @CommandArgument("args", nargs=argparse.REMAINDER)
 275 def android_install_geckoview_example_aab(command_context, args):
 276     install_app_bundle(
 277         command_context,
 278         command_context.substs["GRADLE_ANDROID_GECKOVIEW_EXAMPLE_BUNDLE"],
 279     )
 280     return 0
 281
 282
 283 @SubCommand("android", "install-geckoview-test", """Install geckoview.test """)
 284 @CommandArgument("args", nargs=argparse.REMAINDER)
 285 def android_install_geckoview_test(command_context, args):
 286     gradle(
 287         command_context,
 288         command_context.substs["GRADLE_ANDROID_INSTALL_GECKOVIEW_TEST_TASKS"] + args,
 289         verbose=True,
 290     )
 291     return 0
 292
 293
 294 @SubCommand(
 295     "android",
 296     "geckoview-docs",
 297     """Create GeckoView javadoc and optionally upload to Github""",
 298 )
 299 @CommandArgument("--archive", action="store_true", help="Generate a javadoc archive.")
 300 @CommandArgument(
 301     "--upload",
 302     metavar="USER/REPO",
 303     help="Upload geckoview documentation to Github, using the specified USER/REPO.",
 304 )
 305 @CommandArgument(
 306     "--upload-branch",
 307     metavar="BRANCH[/PATH]",
 308     default="gh-pages",
 309     help="Use the specified branch/path for documentation commits.",
 310 )
 311 @CommandArgument(
 312     "--javadoc-path",
 313     metavar="/PATH",
 314     default="javadoc",
 315     help="Use the specified path for javadoc commits.",
 316 )
 317 @CommandArgument(
 318     "--upload-message",
 319     metavar="MSG",
 320     default="GeckoView docs upload",
 321     help="Use the specified message for commits.",
 322 )
 323 def android_geckoview_docs(
 324     command_context,
 325     archive,
 326     upload,
 327     upload_branch,
 328     javadoc_path,
 329     upload_message,
 330 ):
 331
 332     tasks = (
 333         command_context.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_ARCHIVE_TASKS"]
 334         if archive or upload
 335         else command_context.substs["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"]
 336     )
 337
 338     ret = gradle(command_context, tasks, verbose=True)
 339     if ret or not upload:
 340         return ret
 341
 342     # Upload to Github.
 343     fmt = {
 344         "level": os.environ.get("MOZ_SCM_LEVEL", "0"),
 345         "project": os.environ.get("MH_BRANCH", "unknown"),
 346         "revision": os.environ.get("GECKO_HEAD_REV", "tip"),
 347     }
 348     env = {}
 349
 350     # In order to push to GitHub from TaskCluster, we store a private key
 351     # in the TaskCluster secrets store in the format {"content": "<KEY>"},
 352     # and the corresponding public key as a writable deploy key for the
 353     # destination repo on GitHub.
 354     secret = os.environ.get("GECKOVIEW_DOCS_UPLOAD_SECRET", "").format(**fmt)
 355     if secret:
 356         # Set up a private key from the secrets store if applicable.
 357         import requests
 358
 359         req = requests.get("http://taskcluster/secrets/v1/secret/" + secret)
 360         req.raise_for_status()
 361
 362         keyfile = mozpath.abspath("gv-docs-upload-key")
 363         with open(keyfile, "w") as f:
 364             os.chmod(keyfile, 0o600)
 365             f.write(req.json()["secret"]["content"])
 366
 367         # Turn off strict host key checking so ssh does not complain about
 368         # unknown github.com host. We're not pushing anything sensitive, so
 369         # it's okay to not check GitHub's host keys.
 370         env["GIT_SSH_COMMAND"] = 'ssh -i "%s" -o StrictHostKeyChecking=no' % keyfile
 371
 372     # Clone remote repo.
 373     branch = upload_branch.format(**fmt)
 374     repo_url = "git@github.com:%s.git" % upload
 375     repo_path = mozpath.abspath("gv-docs-repo")
 376     command_context.run_process(
 377         [
 378             "git",
 379             "clone",
 380             "--branch",
 381             upload_branch,
 382             "--depth",
 383             "1",
 384             repo_url,
 385             repo_path,
 386         ],
 387         append_env=env,
 388         pass_thru=True,
 389     )
 390     env["GIT_DIR"] = mozpath.join(repo_path, ".git")
 391     env["GIT_WORK_TREE"] = repo_path
 392     env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "GeckoView Docs Bot"
 393     env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "nobody@mozilla.com"
 394
 395     # Copy over user documentation.
 396     import mozfile
 397
 398     # Extract new javadoc to specified directory inside repo.
 399     src_tar = mozpath.join(
 400         command_context.topobjdir,
 401         "gradle",
 402         "build",
 403         "mobile",
 404         "android",
 405         "geckoview",
 406         "libs",
 407         "geckoview-javadoc.jar",
 408     )
 409     dst_path = mozpath.join(repo_path, javadoc_path.format(**fmt))
 410     mozfile.remove(dst_path)
 411     mozfile.extract_zip(src_tar, dst_path)
 412
 413     # Commit and push.
 414     command_context.run_process(["git", "add", "--all"], append_env=env, pass_thru=True)
 415     if (
 416         command_context.run_process(
 417             ["git", "diff", "--cached", "--quiet"],
 418             append_env=env,
 419             pass_thru=True,
 420             ensure_exit_code=False,
 421         )
 422         != 0
 423     ):
 424         # We have something to commit.
 425         command_context.run_process(
 426             ["git", "commit", "--message", upload_message.format(**fmt)],
 427             append_env=env,
 428             pass_thru=True,
 429         )
 430         command_context.run_process(
 431             ["git", "push", "origin", branch], append_env=env, pass_thru=True
 432         )
 433
 434     mozfile.remove(repo_path)
 435     if secret:
 436         mozfile.remove(keyfile)
 437     return 0
 438
 439
 440 @Command(
 441     "gradle",
 442     category="devenv",
 443     description="Run gradle.",
 444     conditions=[conditions.is_android],
 445 )
 446 @CommandArgument(
 447     "-v",
 448     "--verbose",
 449     action="store_true",
 450     help="Verbose output for what commands the build is running.",
 451 )
 452 @CommandArgument("args", nargs=argparse.REMAINDER)
 453 def gradle(command_context, args, verbose=False):
 454     if not verbose:
 455         # Avoid logging the command
 456         command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
 457
 458     # In automation, JAVA_HOME is set via mozconfig, which needs
 459     # to be specially handled in each mach command. This turns
 460     # $JAVA_HOME/bin/java into $JAVA_HOME.
 461     java_home = os.path.dirname(os.path.dirname(command_context.substs["JAVA"]))
 462
 463     gradle_flags = command_context.substs.get("GRADLE_FLAGS", "") or os.environ.get(
 464         "GRADLE_FLAGS", ""
 465     )
 466     gradle_flags = shell_split(gradle_flags)
 467
 468     # We force the Gradle JVM to run with the UTF-8 encoding, since we
 469     # filter strings.xml, which is really UTF-8; the ellipsis character is
 470     # replaced with ??? in some encodings (including ASCII).  It's not yet
 471     # possible to filter with encodings in Gradle
 472     # (https://github.com/gradle/gradle/pull/520) and it's challenging to
 473     # do our filtering with Gradle's Ant support.  Moreover, all of the
 474     # Android tools expect UTF-8: see
 475     # http://tools.android.com/knownissues/encoding.  See
 476     # http://stackoverflow.com/a/21267635 for discussion of this approach.
 477     #
 478     # It's not even enough to set the encoding just for Gradle; it
 479     # needs to be for JVMs spawned by Gradle as well.  This
 480     # happens during the maven deployment generating the GeckoView
 481     # documents; this works around "error: unmappable character
 482     # for encoding ASCII" in exoplayer2.  See
 483     # https://discuss.gradle.org/t/unmappable-character-for-encoding-ascii-when-building-a-utf-8-project/10692/11  # NOQA: E501
 484     # and especially https://stackoverflow.com/a/21755671.
 485
 486     if command_context.substs.get("MOZ_AUTOMATION"):
 487         gradle_flags += ["--console=plain"]
 488
 489     env = os.environ.copy()
 490     env.update(
 491         {
 492             "GRADLE_OPTS": "-Dfile.encoding=utf-8",
 493             "JAVA_HOME": java_home,
 494             "JAVA_TOOL_OPTIONS": "-Dfile.encoding=utf-8",
 495         }
 496     )
 497     # Set ANDROID_SDK_ROOT if --with-android-sdk was set.
 498     # See https://bugzilla.mozilla.org/show_bug.cgi?id=1576471
 499     android_sdk_root = command_context.substs.get("ANDROID_SDK_ROOT", "")
 500     if android_sdk_root:
 501         env["ANDROID_SDK_ROOT"] = android_sdk_root
 502
 503     return command_context.run_process(
 504         [command_context.substs["GRADLE"]] + gradle_flags + args,
 505         explicit_env=env,
 506         pass_thru=True,  # Allow user to run gradle interactively.
 507         ensure_exit_code=False,  # Don't throw on non-zero exit code.
 508         cwd=mozpath.join(command_context.topsrcdir),
 509     )
 510
 511
 512 @Command("gradle-install", category="devenv", conditions=[REMOVED])
 513 def gradle_install_REMOVED(command_context):
 514     pass
 515
 516
 517 @Command(
 518     "android-emulator",
 519     category="devenv",
 520     conditions=[],
 521     description="Run the Android emulator with an AVD from test automation. "
 522     "Environment variable MOZ_EMULATOR_COMMAND_ARGS, if present, will "
 523     "over-ride the command line arguments used to launch the emulator.",
 524 )
 525 @CommandArgument(
 526     "--version",
 527     metavar="VERSION",
 528     choices=["arm", "arm64", "x86_64"],
 529     help="Specify which AVD to run in emulator. "
 530     'One of "arm" (Android supporting armv7 binaries), '
 531     '"arm64" (for Apple Silicon), or '
 532     '"x86_64" (Android supporting x86 or x86_64 binaries, '
 533     "recommended for most applications). "
 534     "By default, the value will match the current build environment.",
 535 )
 536 @CommandArgument("--wait", action="store_true", help="Wait for emulator to be closed.")
 537 @CommandArgument("--gpu", help="Over-ride the emulator -gpu argument.")
 538 @CommandArgument(
 539     "--verbose", action="store_true", help="Log informative status messages."
 540 )
 541 def emulator(
 542     command_context,
 543     version,
 544     wait=False,
 545     gpu=None,
 546     verbose=False,
 547 ):
 548     """
 549     Run the Android emulator with one of the AVDs used in the Mozilla
 550     automated test environment. If necessary, the AVD is fetched from
 551     the taskcluster server and installed.
 552     """
 553     from mozrunner.devices.android_device import AndroidEmulator
 554
 555     emulator = AndroidEmulator(
 556         version,
 557         verbose,
 558         substs=command_context.substs,
 559         device_serial="emulator-5554",
 560     )
 561     if emulator.is_running():
 562         # It is possible to run multiple emulators simultaneously, but:
 563         #  - if more than one emulator is using the same avd, errors may
 564         #    occur due to locked resources;
 565         #  - additional parameters must be specified when running tests,
 566         #    to select a specific device.
 567         # To avoid these complications, allow just one emulator at a time.
 568         command_context.log(
 569             logging.ERROR,
 570             "emulator",
 571             {},
 572             "An Android emulator is already running.\n"
 573             "Close the existing emulator and re-run this command.",
 574         )
 575         return 1
 576
 577     if not emulator.check_avd():
 578         command_context.log(
 579             logging.WARN,
 580             "emulator",
 581             {},
 582             "AVD not found. Please run |mach bootstrap|.",
 583         )
 584         return 2
 585
 586     if not emulator.is_available():
 587         command_context.log(
 588             logging.WARN,
 589             "emulator",
 590             {},
 591             "Emulator binary not found.\n"
 592             "Install the Android SDK and make sure 'emulator' is in your PATH.",
 593         )
 594         return 2
 595
 596     command_context.log(
 597         logging.INFO,
 598         "emulator",
 599         {},
 600         "Starting Android emulator running %s..." % emulator.get_avd_description(),
 601     )
 602     emulator.start(gpu)
 603     if emulator.wait_for_start():
 604         command_context.log(
 605             logging.INFO, "emulator", {}, "Android emulator is running."
 606         )
 607     else:
 608         # This is unusual but the emulator may still function.
 609         command_context.log(
 610             logging.WARN,
 611             "emulator",
 612             {},
 613             "Unable to verify that emulator is running.",
 614         )
 615
 616     if conditions.is_android(command_context):
 617         command_context.log(
 618             logging.INFO,
 619             "emulator",
 620             {},
 621             "Use 'mach install' to install or update Firefox on your emulator.",
 622         )
 623     else:
 624         command_context.log(
 625             logging.WARN,
 626             "emulator",
 627             {},
 628             "No Firefox for Android build detected.\n"
 629             "Switch to a Firefox for Android build context or use 'mach bootstrap'\n"
 630             "to setup an Android build environment.",
 631         )
 632
 633     if wait:
 634         command_context.log(
 635             logging.INFO, "emulator", {}, "Waiting for Android emulator to close..."
 636         )
 637         rc = emulator.wait()
 638         if rc is not None:
 639             command_context.log(
 640                 logging.INFO,
 641                 "emulator",
 642                 {},
 643                 "Android emulator completed with return code %d." % rc,
 644             )
 645         else:
 646             command_context.log(
 647                 logging.WARN,
 648                 "emulator",
 649                 {},
 650                 "Unable to retrieve Android emulator return code.",
 651             )
 652     return 0