tools/compare-locales/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this,
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import logging
   7 import os
   8 import tempfile
   9 from pathlib import Path
  10
  11 from appdirs import user_config_dir
  12 from hglib.error import CommandError
  13 from mach.base import FailedCommandError
  14 from mach.decorators import Command, CommandArgument
  15 from mozrelease.scriptworker_canary import get_secret
  16 from redo import retry
  17
  18
  19 @Command(
  20     "compare-locales",
  21     category="build",
  22     description="Run source checks on a localization.",
  23 )
  24 @CommandArgument(
  25     "config_paths",
  26     metavar="l10n.toml",
  27     nargs="+",
  28     help="TOML or INI file for the project",
  29 )
  30 @CommandArgument(
  31     "l10n_base_dir",
  32     metavar="l10n-base-dir",
  33     help="Parent directory of localizations",
  34 )
  35 @CommandArgument(
  36     "locales",
  37     nargs="*",
  38     metavar="locale-code",
  39     help="Locale code and top-level directory of each localization",
  40 )
  41 @CommandArgument(
  42     "-q",
  43     "--quiet",
  44     action="count",
  45     default=0,
  46     help="""Show less data.
  47 Specified once, don't show obsolete entities. Specified twice, also hide
  48 missing entities. Specify thrice to exclude warnings and four times to
  49 just show stats""",
  50 )
  51 @CommandArgument("-m", "--merge", help="""Use this directory to stage merged files""")
  52 @CommandArgument(
  53     "--validate", action="store_true", help="Run compare-locales against reference"
  54 )
  55 @CommandArgument(
  56     "--json",
  57     help="""Serialize to JSON. Value is the name of
  58 the output file, pass "-" to serialize to stdout and hide the default output.
  59 """,
  60 )
  61 @CommandArgument(
  62     "-D",
  63     action="append",
  64     metavar="var=value",
  65     default=[],
  66     dest="defines",
  67     help="Overwrite variables in TOML files",
  68 )
  69 @CommandArgument(
  70     "--full", action="store_true", help="Compare projects that are disabled"
  71 )
  72 @CommandArgument(
  73     "--return-zero", action="store_true", help="Return 0 regardless of l10n status"
  74 )
  75 def compare(command_context, **kwargs):
  76     """Run compare-locales."""
  77     from compare_locales.commands import CompareLocales
  78
  79     class ErrorHelper(object):
  80         """Dummy ArgumentParser to marshall compare-locales
  81         commandline errors to mach exceptions.
  82         """
  83
  84         def error(self, msg):
  85             raise FailedCommandError(msg)
  86
  87         def exit(self, message=None, status=0):
  88             raise FailedCommandError(message, exit_code=status)
  89
  90     cmd = CompareLocales()
  91     cmd.parser = ErrorHelper()
  92     return cmd.handle(**kwargs)
  93
  94
  95 # https://stackoverflow.com/a/14117511
  96 def _positive_int(value):
  97     value = int(value)
  98     if value <= 0:
  99         raise argparse.ArgumentTypeError(f"{value} must be a positive integer.")
 100     return value
 101
 102
 103 class RetryError(Exception):
 104     ...
 105
 106
 107 VCT_PATH = Path(".").resolve() / "vct"
 108 VCT_URL = "https://hg.mozilla.org/hgcustom/version-control-tools/"
 109 FXTREE_PATH = VCT_PATH / "hgext" / "firefoxtree"
 110 HGRC_PATH = Path(user_config_dir("hg")).joinpath("hgrc")
 111
 112
 113 @Command(
 114     "l10n-cross-channel",
 115     category="misc",
 116     description="Create cross-channel content.",
 117 )
 118 @CommandArgument(
 119     "--strings-path",
 120     "-s",
 121     metavar="en-US",
 122     type=Path,
 123     default=Path("en-US"),
 124     help="Path to mercurial repository for gecko-strings-quarantine",
 125 )
 126 @CommandArgument(
 127     "--outgoing-path",
 128     "-o",
 129     type=Path,
 130     help="create an outgoing() patch if there are changes",
 131 )
 132 @CommandArgument(
 133     "--attempts",
 134     type=_positive_int,
 135     default=1,
 136     help="Number of times to try (for automation)",
 137 )
 138 @CommandArgument(
 139     "--ssh-secret",
 140     action="store",
 141     help="Taskcluster secret to use to push (for automation)",
 142 )
 143 @CommandArgument(
 144     "actions",
 145     choices=("prep", "create", "push", "clean"),
 146     nargs="+",
 147     # This help block will be poorly formatted until we fix bug 1714239
 148     help="""
 149     "prep": clone repos and pull heads.
 150     "create": create the en-US strings commit an optionally create an
 151               outgoing() patch.
 152     "push": push the en-US strings to the quarantine repo.
 153     "clean": clean up any sub-repos.
 154     """,
 155 )
 156 def cross_channel(
 157     command_context,
 158     strings_path,
 159     outgoing_path,
 160     actions,
 161     attempts,
 162     ssh_secret,
 163     **kwargs,
 164 ):
 165     """Run l10n cross-channel content generation."""
 166     # This can be any path, as long as the name of the directory is en-US.
 167     # Not entirely sure where this is a requirement; perhaps in l10n
 168     # string manipulation logic?
 169     if strings_path.name != "en-US":
 170         raise FailedCommandError("strings_path needs to be named `en-US`")
 171     command_context.activate_virtualenv()
 172     # XXX pin python requirements
 173     command_context.virtualenv_manager.install_pip_requirements(
 174         Path(os.path.dirname(__file__)) / "requirements.in"
 175     )
 176     strings_path = strings_path.resolve()  # abspath
 177     if outgoing_path:
 178         outgoing_path = outgoing_path.resolve()  # abspath
 179     get_config = kwargs.get("get_config", None)
 180     try:
 181         with tempfile.TemporaryDirectory() as ssh_key_dir:
 182             retry(
 183                 _do_create_content,
 184                 attempts=attempts,
 185                 retry_exceptions=(RetryError,),
 186                 args=(
 187                     command_context,
 188                     strings_path,
 189                     outgoing_path,
 190                     ssh_secret,
 191                     Path(ssh_key_dir),
 192                     actions,
 193                     get_config,
 194                 ),
 195             )
 196     except RetryError as exc:
 197         raise FailedCommandError(exc) from exc
 198
 199
 200 def _do_create_content(
 201     command_context,
 202     strings_path,
 203     outgoing_path,
 204     ssh_secret,
 205     ssh_key_dir,
 206     actions,
 207     get_config,
 208 ):
 209     from mozxchannel import CrossChannelCreator, get_default_config
 210
 211     get_config = get_config or get_default_config
 212
 213     config = get_config(Path(command_context.topsrcdir), strings_path)
 214     ccc = CrossChannelCreator(config)
 215     status = 0
 216     changes = False
 217     ssh_key_secret = None
 218     ssh_key_file = None
 219
 220     if "prep" in actions:
 221         if ssh_secret:
 222             if not os.environ.get("MOZ_AUTOMATION"):
 223                 raise CommandError(
 224                     "I don't know how to fetch the ssh secret outside of automation!"
 225                 )
 226             ssh_key_secret = get_secret(ssh_secret)
 227             ssh_key_file = ssh_key_dir.joinpath("id_rsa")
 228             ssh_key_file.write_text(ssh_key_secret["ssh_privkey"])
 229             ssh_key_file.chmod(0o600)
 230         # Set up firefoxtree for comm per bug 1659691 comment 22
 231         if os.environ.get("MOZ_AUTOMATION") and not HGRC_PATH.exists():
 232             _clone_hg_repo(command_context, VCT_URL, VCT_PATH)
 233             hgrc_content = [
 234                 "[extensions]",
 235                 f"firefoxtree = {FXTREE_PATH}",
 236                 "",
 237                 "[ui]",
 238                 "username = trybld",
 239             ]
 240             if ssh_key_file:
 241                 hgrc_content.extend(
 242                     [
 243                         f"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}",
 244                     ]
 245                 )
 246             HGRC_PATH.write_text("\n".join(hgrc_content))
 247         if strings_path.exists() and _check_outgoing(command_context, strings_path):
 248             _strip_outgoing(command_context, strings_path)
 249         # Clone strings + source repos, pull heads
 250         for repo_config in (config["strings"], *config["source"].values()):
 251             if not repo_config["path"].exists():
 252                 _clone_hg_repo(
 253                     command_context, repo_config["url"], str(repo_config["path"])
 254                 )
 255             for head in repo_config["heads"].keys():
 256                 command = ["hg", "--cwd", str(repo_config["path"]), "pull"]
 257                 command.append(head)
 258                 status = _retry_run_process(
 259                     command_context, command, ensure_exit_code=False
 260                 )
 261                 if status not in (0, 255):  # 255 on pull with no changes
 262                     raise RetryError(f"Failure on pull: status {status}!")
 263                 if repo_config.get("update_on_pull"):
 264                     command = [
 265                         "hg",
 266                         "--cwd",
 267                         str(repo_config["path"]),
 268                         "up",
 269                         "-C",
 270                         "-r",
 271                         head,
 272                     ]
 273                     status = _retry_run_process(
 274                         command_context, command, ensure_exit_code=False
 275                     )
 276                     if status not in (0, 255):  # 255 on pull with no changes
 277                         raise RetryError(f"Failure on update: status {status}!")
 278             _check_hg_repo(
 279                 command_context,
 280                 repo_config["path"],
 281                 heads=repo_config.get("heads", {}).keys(),
 282             )
 283     else:
 284         _check_hg_repo(command_context, strings_path)
 285         for repo_config in config.get("source", {}).values():
 286             _check_hg_repo(
 287                 command_context,
 288                 repo_config["path"],
 289                 heads=repo_config.get("heads", {}).keys(),
 290             )
 291         if _check_outgoing(command_context, strings_path):
 292             raise RetryError(f"check: Outgoing changes in {strings_path}!")
 293
 294     if "create" in actions:
 295         try:
 296             status = ccc.create_content()
 297             changes = True
 298             _create_outgoing_patch(command_context, outgoing_path, strings_path)
 299         except CommandError as exc:
 300             if exc.ret != 1:
 301                 raise RetryError(exc) from exc
 302             command_context.log(logging.INFO, "create", {}, "No new strings.")
 303
 304     if "push" in actions:
 305         if changes:
 306             _retry_run_process(
 307                 command_context,
 308                 [
 309                     "hg",
 310                     "--cwd",
 311                     str(strings_path),
 312                     "push",
 313                     "-r",
 314                     ".",
 315                     config["strings"]["push_url"],
 316                 ],
 317                 line_handler=print,
 318             )
 319         else:
 320             command_context.log(logging.INFO, "push", {}, "Skipping empty push.")
 321
 322     if "clean" in actions:
 323         for repo_config in config.get("source", {}).values():
 324             if repo_config.get("post-clobber", False):
 325                 _nuke_hg_repo(command_context, str(repo_config["path"]))
 326
 327     return status
 328
 329
 330 def _check_outgoing(command_context, strings_path):
 331     status = _retry_run_process(
 332         command_context,
 333         ["hg", "--cwd", str(strings_path), "out", "-r", "."],
 334         ensure_exit_code=False,
 335     )
 336     if status == 0:
 337         return True
 338     if status == 1:
 339         return False
 340     raise RetryError(f"Outgoing check in {strings_path} returned unexpected {status}!")
 341
 342
 343 def _strip_outgoing(command_context, strings_path):
 344     _retry_run_process(
 345         command_context,
 346         [
 347             "hg",
 348             "--config",
 349             "extensions.strip=",
 350             "--cwd",
 351             str(strings_path),
 352             "strip",
 353             "--no-backup",
 354             "outgoing()",
 355         ],
 356     )
 357
 358
 359 def _create_outgoing_patch(command_context, path, strings_path):
 360     if not path:
 361         return
 362     if not path.parent.exists():
 363         os.makedirs(path.parent)
 364     with open(path, "w") as fh:
 365
 366         def writeln(line):
 367             fh.write(f"{line}\n")
 368
 369         _retry_run_process(
 370             command_context,
 371             [
 372                 "hg",
 373                 "--cwd",
 374                 str(strings_path),
 375                 "log",
 376                 "--patch",
 377                 "--verbose",
 378                 "-r",
 379                 "outgoing()",
 380             ],
 381             line_handler=writeln,
 382         )
 383
 384
 385 def _retry_run_process(command_context, *args, error_msg=None, **kwargs):
 386     try:
 387         return command_context.run_process(*args, **kwargs)
 388     except Exception as exc:
 389         raise RetryError(error_msg or str(exc)) from exc
 390
 391
 392 def _check_hg_repo(command_context, path, heads=None):
 393     if not (path.is_dir() and (path / ".hg").is_dir()):
 394         raise RetryError(f"{path} is not a Mercurial repository")
 395     if heads:
 396         for head in heads:
 397             _retry_run_process(
 398                 command_context,
 399                 ["hg", "--cwd", str(path), "log", "-r", head],
 400                 error_msg=f"check: {path} has no head {head}!",
 401             )
 402
 403
 404 def _clone_hg_repo(command_context, url, path):
 405     _retry_run_process(command_context, ["hg", "clone", url, str(path)])
 406
 407
 408 def _nuke_hg_repo(command_context, path):
 409     _retry_run_process(command_context, ["rm", "-rf", str(path)])