tools/compare-locales/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this,
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 from appdirs import user_config_dir
   8 from hglib.error import CommandError
   9 from mach.decorators import (
  10     CommandArgument,
  11     Command,
  12 )
  13 from mach.base import FailedCommandError
  14 from mozrelease.scriptworker_canary import get_secret
  15 from pathlib import Path
  16 from redo import retry
  17 import argparse
  18 import logging
  19 import os
  20 import tempfile
  21
  22
  23 @Command(
  24     "compare-locales",
  25     category="build",
  26     description="Run source checks on a localization.",
  27 )
  28 @CommandArgument(
  29     "config_paths",
  30     metavar="l10n.toml",
  31     nargs="+",
  32     help="TOML or INI file for the project",
  33 )
  34 @CommandArgument(
  35     "l10n_base_dir",
  36     metavar="l10n-base-dir",
  37     help="Parent directory of localizations",
  38 )
  39 @CommandArgument(
  40     "locales",
  41     nargs="*",
  42     metavar="locale-code",
  43     help="Locale code and top-level directory of each localization",
  44 )
  45 @CommandArgument(
  46     "-q",
  47     "--quiet",
  48     action="count",
  49     default=0,
  50     help="""Show less data.
  51 Specified once, don't show obsolete entities. Specified twice, also hide
  52 missing entities. Specify thrice to exclude warnings and four times to
  53 just show stats""",
  54 )
  55 @CommandArgument("-m", "--merge", help="""Use this directory to stage merged files""")
  56 @CommandArgument(
  57     "--validate", action="store_true", help="Run compare-locales against reference"
  58 )
  59 @CommandArgument(
  60     "--json",
  61     help="""Serialize to JSON. Value is the name of
  62 the output file, pass "-" to serialize to stdout and hide the default output.
  63 """,
  64 )
  65 @CommandArgument(
  66     "-D",
  67     action="append",
  68     metavar="var=value",
  69     default=[],
  70     dest="defines",
  71     help="Overwrite variables in TOML files",
  72 )
  73 @CommandArgument(
  74     "--full", action="store_true", help="Compare projects that are disabled"
  75 )
  76 @CommandArgument(
  77     "--return-zero", action="store_true", help="Return 0 regardless of l10n status"
  78 )
  79 def compare(command_context, **kwargs):
  80     """Run compare-locales."""
  81     from compare_locales.commands import CompareLocales
  82
  83     class ErrorHelper(object):
  84         """Dummy ArgumentParser to marshall compare-locales
  85         commandline errors to mach exceptions.
  86         """
  87
  88         def error(self, msg):
  89             raise FailedCommandError(msg)
  90
  91         def exit(self, message=None, status=0):
  92             raise FailedCommandError(message, exit_code=status)
  93
  94     cmd = CompareLocales()
  95     cmd.parser = ErrorHelper()
  96     return cmd.handle(**kwargs)
  97
  98
  99 # https://stackoverflow.com/a/14117511
 100 def _positive_int(value):
 101     value = int(value)
 102     if value <= 0:
 103         raise argparse.ArgumentTypeError(f"{value} must be a positive integer.")
 104     return value
 105
 106
 107 class RetryError(Exception):
 108     ...
 109
 110
 111 VCT_PATH = Path(".").resolve() / "vct"
 112 VCT_URL = "https://hg.mozilla.org/hgcustom/version-control-tools/"
 113 FXTREE_PATH = VCT_PATH / "hgext" / "firefoxtree"
 114 HGRC_PATH = Path(user_config_dir("hg")).joinpath("hgrc")
 115
 116
 117 @Command(
 118     "l10n-cross-channel",
 119     category="misc",
 120     description="Create cross-channel content.",
 121 )
 122 @CommandArgument(
 123     "--strings-path",
 124     "-s",
 125     metavar="en-US",
 126     type=Path,
 127     default=Path("en-US"),
 128     help="Path to mercurial repository for gecko-strings-quarantine",
 129 )
 130 @CommandArgument(
 131     "--outgoing-path",
 132     "-o",
 133     type=Path,
 134     help="create an outgoing() patch if there are changes",
 135 )
 136 @CommandArgument(
 137     "--attempts",
 138     type=_positive_int,
 139     default=1,
 140     help="Number of times to try (for automation)",
 141 )
 142 @CommandArgument(
 143     "--ssh-secret",
 144     action="store",
 145     help="Taskcluster secret to use to push (for automation)",
 146 )
 147 @CommandArgument(
 148     "actions",
 149     choices=("prep", "create", "push", "clean"),
 150     nargs="+",
 151     # This help block will be poorly formatted until we fix bug 1714239
 152     help="""
 153     "prep": clone repos and pull heads.
 154     "create": create the en-US strings commit an optionally create an
 155               outgoing() patch.
 156     "push": push the en-US strings to the quarantine repo.
 157     "clean": clean up any sub-repos.
 158     """,
 159 )
 160 def cross_channel(
 161     command_context,
 162     strings_path,
 163     outgoing_path,
 164     actions,
 165     attempts,
 166     ssh_secret,
 167     **kwargs,
 168 ):
 169     """Run l10n cross-channel content generation."""
 170     # This can be any path, as long as the name of the directory is en-US.
 171     # Not entirely sure where this is a requirement; perhaps in l10n
 172     # string manipulation logic?
 173     if strings_path.name != "en-US":
 174         raise FailedCommandError("strings_path needs to be named `en-US`")
 175     command_context.activate_virtualenv()
 176     # XXX pin python requirements
 177     command_context.virtualenv_manager.install_pip_requirements(
 178         Path(os.path.dirname(__file__)) / "requirements.in"
 179     )
 180     strings_path = strings_path.resolve()  # abspath
 181     if outgoing_path:
 182         outgoing_path = outgoing_path.resolve()  # abspath
 183     get_config = kwargs.get("get_config", None)
 184     try:
 185         with tempfile.TemporaryDirectory() as ssh_key_dir:
 186             retry(
 187                 _do_create_content,
 188                 attempts=attempts,
 189                 retry_exceptions=(RetryError,),
 190                 args=(
 191                     command_context,
 192                     strings_path,
 193                     outgoing_path,
 194                     ssh_secret,
 195                     Path(ssh_key_dir),
 196                     actions,
 197                     get_config,
 198                 ),
 199             )
 200     except RetryError as exc:
 201         raise FailedCommandError(exc) from exc
 202
 203
 204 def _do_create_content(
 205     command_context,
 206     strings_path,
 207     outgoing_path,
 208     ssh_secret,
 209     ssh_key_dir,
 210     actions,
 211     get_config,
 212 ):
 213     from mozxchannel import CrossChannelCreator, get_default_config
 214
 215     get_config = get_config or get_default_config
 216
 217     config = get_config(Path(command_context.topsrcdir), strings_path)
 218     ccc = CrossChannelCreator(config)
 219     status = 0
 220     changes = False
 221     ssh_key_secret = None
 222     ssh_key_file = None
 223
 224     if "prep" in actions:
 225         if ssh_secret:
 226             if not os.environ.get("MOZ_AUTOMATION"):
 227                 raise CommandError(
 228                     "I don't know how to fetch the ssh secret outside of automation!"
 229                 )
 230             ssh_key_secret = get_secret(ssh_secret)
 231             ssh_key_file = ssh_key_dir.joinpath("id_rsa")
 232             ssh_key_file.write_text(ssh_key_secret["ssh_privkey"])
 233             ssh_key_file.chmod(0o600)
 234         # Set up firefoxtree for comm per bug 1659691 comment 22
 235         if os.environ.get("MOZ_AUTOMATION") and not HGRC_PATH.exists():
 236             _clone_hg_repo(command_context, VCT_URL, VCT_PATH)
 237             hgrc_content = [
 238                 "[extensions]",
 239                 f"firefoxtree = {FXTREE_PATH}",
 240                 "",
 241                 "[ui]",
 242                 "username = trybld",
 243             ]
 244             if ssh_key_file:
 245                 hgrc_content.extend(
 246                     [
 247                         f"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}",
 248                     ]
 249                 )
 250             HGRC_PATH.write_text("\n".join(hgrc_content))
 251         if strings_path.exists() and _check_outgoing(command_context, strings_path):
 252             _strip_outgoing(command_context, strings_path)
 253         # Clone strings + source repos, pull heads
 254         for repo_config in (config["strings"], *config["source"].values()):
 255             if not repo_config["path"].exists():
 256                 _clone_hg_repo(
 257                     command_context, repo_config["url"], str(repo_config["path"])
 258                 )
 259             for head in repo_config["heads"].keys():
 260                 command = ["hg", "--cwd", str(repo_config["path"]), "pull"]
 261                 command.append(head)
 262                 status = _retry_run_process(
 263                     command_context, command, ensure_exit_code=False
 264                 )
 265                 if status not in (0, 255):  # 255 on pull with no changes
 266                     raise RetryError(f"Failure on pull: status {status}!")
 267                 if repo_config.get("update_on_pull"):
 268                     command = [
 269                         "hg",
 270                         "--cwd",
 271                         str(repo_config["path"]),
 272                         "up",
 273                         "-C",
 274                         "-r",
 275                         head,
 276                     ]
 277                     status = _retry_run_process(
 278                         command_context, command, ensure_exit_code=False
 279                     )
 280                     if status not in (0, 255):  # 255 on pull with no changes
 281                         raise RetryError(f"Failure on update: status {status}!")
 282             _check_hg_repo(
 283                 command_context,
 284                 repo_config["path"],
 285                 heads=repo_config.get("heads", {}).keys(),
 286             )
 287     else:
 288         _check_hg_repo(command_context, strings_path)
 289         for repo_config in config.get("source", {}).values():
 290             _check_hg_repo(
 291                 command_context,
 292                 repo_config["path"],
 293                 heads=repo_config.get("heads", {}).keys(),
 294             )
 295         if _check_outgoing(command_context, strings_path):
 296             raise RetryError(f"check: Outgoing changes in {strings_path}!")
 297
 298     if "create" in actions:
 299         try:
 300             status = ccc.create_content()
 301             changes = True
 302             _create_outgoing_patch(command_context, outgoing_path, strings_path)
 303         except CommandError as exc:
 304             if exc.ret != 1:
 305                 raise RetryError(exc) from exc
 306             command_context.log(logging.INFO, "create", {}, "No new strings.")
 307
 308     if "push" in actions:
 309         if changes:
 310             _retry_run_process(
 311                 command_context,
 312                 [
 313                     "hg",
 314                     "--cwd",
 315                     str(strings_path),
 316                     "push",
 317                     "-r",
 318                     ".",
 319                     config["strings"]["push_url"],
 320                 ],
 321                 line_handler=print,
 322             )
 323         else:
 324             command_context.log(logging.INFO, "push", {}, "Skipping empty push.")
 325
 326     if "clean" in actions:
 327         for repo_config in config.get("source", {}).values():
 328             if repo_config.get("post-clobber", False):
 329                 _nuke_hg_repo(command_context, str(repo_config["path"]))
 330
 331     return status
 332
 333
 334 def _check_outgoing(command_context, strings_path):
 335     status = _retry_run_process(
 336         command_context,
 337         ["hg", "--cwd", str(strings_path), "out", "-r", "."],
 338         ensure_exit_code=False,
 339     )
 340     if status == 0:
 341         return True
 342     if status == 1:
 343         return False
 344     raise RetryError(f"Outgoing check in {strings_path} returned unexpected {status}!")
 345
 346
 347 def _strip_outgoing(command_context, strings_path):
 348     _retry_run_process(
 349         command_context,
 350         [
 351             "hg",
 352             "--config",
 353             "extensions.strip=",
 354             "--cwd",
 355             str(strings_path),
 356             "strip",
 357             "--no-backup",
 358             "outgoing()",
 359         ],
 360     )
 361
 362
 363 def _create_outgoing_patch(command_context, path, strings_path):
 364     if not path:
 365         return
 366     if not path.parent.exists():
 367         os.makedirs(path.parent)
 368     with open(path, "w") as fh:
 369
 370         def writeln(line):
 371             fh.write(f"{line}\n")
 372
 373         _retry_run_process(
 374             command_context,
 375             [
 376                 "hg",
 377                 "--cwd",
 378                 str(strings_path),
 379                 "log",
 380                 "--patch",
 381                 "--verbose",
 382                 "-r",
 383                 "outgoing()",
 384             ],
 385             line_handler=writeln,
 386         )
 387
 388
 389 def _retry_run_process(command_context, *args, error_msg=None, **kwargs):
 390     try:
 391         return command_context.run_process(*args, **kwargs)
 392     except Exception as exc:
 393         raise RetryError(error_msg or str(exc)) from exc
 394
 395
 396 def _check_hg_repo(command_context, path, heads=None):
 397     if not (path.is_dir() and (path / ".hg").is_dir()):
 398         raise RetryError(f"{path} is not a Mercurial repository")
 399     if heads:
 400         for head in heads:
 401             _retry_run_process(
 402                 command_context,
 403                 ["hg", "--cwd", str(path), "log", "-r", head],
 404                 error_msg=f"check: {path} has no head {head}!",
 405             )
 406
 407
 408 def _clone_hg_repo(command_context, url, path):
 409     _retry_run_process(command_context, ["hg", "clone", url, str(path)])
 410
 411
 412 def _nuke_hg_repo(command_context, path):
 413     _retry_run_process(command_context, ["rm", "-rf", str(path)])