python/l10n/test_fluent_migrations/fmt.py

   1 import codecs
   2 import logging
   3 import os
   4 import re
   5 import shutil
   6 import sys
   7 from datetime import datetime, timedelta
   8 from difflib import unified_diff
   9 from typing import Iterable
  10
  11 import hglib
  12 from compare_locales.merge import merge_channels
  13 from compare_locales.paths.configparser import TOMLParser
  14 from compare_locales.paths.files import ProjectFiles
  15 from fluent.migrate.repo_client import RepoClient, git
  16 from fluent.migrate.validator import Validator
  17 from fluent.syntax import FluentParser, FluentSerializer
  18 from mach.util import get_state_dir
  19 from mozpack.path import join, normpath
  20 from mozversioncontrol.repoupdate import update_git_repo, update_mercurial_repo
  21
  22 L10N_SOURCE_NAME = "l10n-source"
  23 L10N_SOURCE_REPO = "https://github.com/mozilla-l10n/firefox-l10n-source.git"
  24
  25 STRINGS_NAME = "gecko-strings"
  26 STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings"
  27
  28 PULL_AFTER = timedelta(days=2)
  29
  30
  31 def inspect_migration(path):
  32     """Validate recipe and extract some metadata."""
  33     return Validator.validate(path)
  34
  35
  36 def prepare_directories(cmd, use_git=False):
  37     """
  38     Ensure object dir exists,
  39     and that repo dir has a relatively up-to-date clone of l10n-source or gecko-strings.
  40
  41     We run this once per mach invocation, for all tested migrations.
  42     """
  43     obj_dir = join(cmd.topobjdir, "python", "l10n")
  44     if not os.path.exists(obj_dir):
  45         os.makedirs(obj_dir)
  46
  47     if use_git:
  48         repo_dir = join(get_state_dir(), L10N_SOURCE_NAME)
  49         marker = join(repo_dir, ".git", "l10n_pull_marker")
  50     else:
  51         repo_dir = join(get_state_dir(), STRINGS_NAME)
  52         marker = join(repo_dir, ".hg", "l10n_pull_marker")
  53
  54     try:
  55         last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
  56         skip_clone = datetime.now() < last_pull + PULL_AFTER
  57     except OSError:
  58         skip_clone = False
  59     if not skip_clone:
  60         if use_git:
  61             update_git_repo(L10N_SOURCE_REPO, repo_dir)
  62         else:
  63             update_mercurial_repo(STRINGS_REPO, repo_dir)
  64         with open(marker, "w") as fh:
  65             fh.flush()
  66
  67     return obj_dir, repo_dir
  68
  69
  70 def diff_resources(left_path, right_path):
  71     parser = FluentParser(with_spans=False)
  72     serializer = FluentSerializer(with_junk=True)
  73     lines = []
  74     for p in (left_path, right_path):
  75         with codecs.open(p, encoding="utf-8") as fh:
  76             res = parser.parse(fh.read())
  77             lines.append(serializer.serialize(res).splitlines(True))
  78     sys.stdout.writelines(
  79         chunk for chunk in unified_diff(lines[0], lines[1], left_path, right_path)
  80     )
  81
  82
  83 def test_migration(
  84     cmd,
  85     obj_dir: str,
  86     repo_dir: str,
  87     use_git: bool,
  88     to_test: list[str],
  89     references: Iterable[str],
  90 ):
  91     """Test the given recipe.
  92
  93     This creates a workdir by l10n-merging gecko-strings and the m-c source,
  94     to mimic gecko-strings after the patch to test landed.
  95     It then runs the recipe with a gecko-strings clone as localization, both
  96     dry and wet.
  97     It inspects the generated commits, and shows a diff between the merged
  98     reference and the generated content.
  99     The diff is intended to be visually inspected. Some changes might be
 100     expected, in particular when formatting of the en-US strings is different.
 101     """
 102     rv = 0
 103     migration_name = os.path.splitext(os.path.split(to_test)[1])[0]
 104     work_dir = join(obj_dir, migration_name)
 105
 106     paths = os.path.normpath(to_test).split(os.sep)
 107     # Migration modules should be in a sub-folder of l10n.
 108     migration_module = (
 109         ".".join(paths[paths.index("l10n") + 1 : -1]) + "." + migration_name
 110     )
 111
 112     if os.path.exists(work_dir):
 113         shutil.rmtree(work_dir)
 114     os.makedirs(join(work_dir, "reference"))
 115     l10n_toml = join(cmd.topsrcdir, cmd.substs["MOZ_BUILD_APP"], "locales", "l10n.toml")
 116     pc = TOMLParser().parse(l10n_toml, env={"l10n_base": work_dir})
 117     pc.set_locales(["reference"])
 118     files = ProjectFiles("reference", [pc])
 119     ref_root = join(work_dir, "reference")
 120     for ref in references:
 121         if ref != normpath(ref):
 122             cmd.log(
 123                 logging.ERROR,
 124                 "fluent-migration-test",
 125                 {"file": to_test, "ref": ref},
 126                 'Reference path "{ref}" needs to be normalized for {file}',
 127             )
 128             rv = 1
 129             continue
 130         full_ref = join(ref_root, ref)
 131         m = files.match(full_ref)
 132         if m is None:
 133             raise ValueError("Bad reference path: " + ref)
 134         m_c_path = m[1]
 135         g_s_path = join(work_dir, L10N_SOURCE_NAME if use_git else STRINGS_NAME, ref)
 136         resources = [
 137             b"" if not os.path.exists(f) else open(f, "rb").read()
 138             for f in (g_s_path, m_c_path)
 139         ]
 140         ref_dir = os.path.dirname(full_ref)
 141         if not os.path.exists(ref_dir):
 142             os.makedirs(ref_dir)
 143         open(full_ref, "wb").write(merge_channels(ref, resources))
 144     l10n_root = join(work_dir, "en-US")
 145     if use_git:
 146         git(work_dir, "clone", repo_dir, l10n_root)
 147     else:
 148         hglib.clone(source=repo_dir, dest=l10n_root)
 149     client = RepoClient(l10n_root)
 150     old_tip = client.head()
 151     run_migration = [
 152         cmd._virtualenv_manager.python_path,
 153         "-m",
 154         "fluent.migrate.tool",
 155         "--lang",
 156         "en-US",
 157         "--reference-dir",
 158         ref_root,
 159         "--localization-dir",
 160         l10n_root,
 161         "--dry-run",
 162         migration_module,
 163     ]
 164     cmd.run_process(run_migration, cwd=work_dir, line_handler=print)
 165     # drop --dry-run
 166     run_migration.pop(-2)
 167     cmd.run_process(run_migration, cwd=work_dir, line_handler=print)
 168     tip = client.head()
 169     if old_tip == tip:
 170         cmd.log(
 171             logging.WARN,
 172             "fluent-migration-test",
 173             {"file": to_test},
 174             "No migration applied for {file}",
 175         )
 176         return rv
 177     for ref in references:
 178         diff_resources(join(ref_root, ref), join(l10n_root, ref))
 179     messages = client.log(old_tip, tip)
 180     bug = re.search("[0-9]{5,}", migration_name)
 181     # Just check first message for bug number, they're all following the same pattern
 182     if bug is None or bug.group() not in messages[0]:
 183         rv = 1
 184         cmd.log(
 185             logging.ERROR,
 186             "fluent-migration-test",
 187             {"file": to_test},
 188             "Missing or wrong bug number for {file}",
 189         )
 190     if any("part {}".format(n + 1) not in msg for n, msg in enumerate(messages)):
 191         rv = 1
 192         cmd.log(
 193             logging.ERROR,
 194             "fluent-migration-test",
 195             {"file": to_test},
 196             'Commit messages should have "part {{index}}" for {file}',
 197         )
 198     return rv