tools/lint/python/l10n_lint.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 from datetime import datetime, timedelta
   7
   8 import mozversioncontrol.repoupdate
   9 from compare_locales import parser
  10 from compare_locales.lint.linter import L10nLinter
  11 from compare_locales.lint.util import l10n_base_reference_and_tests
  12 from compare_locales.paths import ProjectFiles, TOMLParser
  13 from mach import util as mach_util
  14 from mozlint import pathutils, result
  15 from mozpack import path as mozpath
  16
  17 LOCALE = "gecko-strings"
  18 STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings"
  19
  20 PULL_AFTER = timedelta(days=2)
  21
  22 # Wrapper to call lint_strings with mozilla-central configuration
  23 # comm-central defines its own wrapper since comm-central strings are
  24 # in separate repositories
  25 def lint(paths, lintconfig, **lintargs):
  26     return lint_strings(LOCALE, paths, lintconfig, **lintargs)
  27
  28
  29 def lint_strings(locale, paths, lintconfig, **lintargs):
  30     l10n_base = mach_util.get_state_dir()
  31     root = lintargs["root"]
  32     exclude = lintconfig.get("exclude")
  33     extensions = lintconfig.get("extensions")
  34
  35     # Load l10n.toml configs
  36     l10nconfigs = load_configs(lintconfig, root, l10n_base, locale)
  37
  38     # Check include paths in l10n.yml if it's in our given paths
  39     # Only the l10n.yml will show up here, but if the l10n.toml files
  40     # change, we also get the l10n.yml as the toml files are listed as
  41     # support files.
  42     if lintconfig["path"] in paths:
  43         results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
  44         paths.remove(lintconfig["path"])
  45     else:
  46         results = []
  47
  48     all_files = []
  49     for p in paths:
  50         fp = pathutils.FilterPath(p)
  51         if fp.isdir:
  52             for _, fileobj in fp.finder:
  53                 all_files.append(fileobj.path)
  54         if fp.isfile:
  55             all_files.append(p)
  56     # Filter again, our directories might have picked up files the
  57     # explicitly excluded in the l10n.yml configuration.
  58     # `browser/locales/en-US/firefox-l10n.js` is a good example.
  59     all_files, _ = pathutils.filterpaths(
  60         lintargs["root"],
  61         all_files,
  62         lintconfig["include"],
  63         exclude=exclude,
  64         extensions=extensions,
  65     )
  66     # These should be excluded in l10n.yml
  67     skips = {p for p in all_files if not parser.hasParser(p)}
  68     results.extend(
  69         result.from_config(
  70             lintconfig,
  71             level="warning",
  72             path=path,
  73             message="file format not supported in compare-locales",
  74         )
  75         for path in skips
  76     )
  77     all_files = [p for p in all_files if p not in skips]
  78     files = ProjectFiles(locale, l10nconfigs)
  79
  80     get_reference_and_tests = l10n_base_reference_and_tests(files)
  81
  82     linter = MozL10nLinter(lintconfig)
  83     results += linter.lint(all_files, get_reference_and_tests)
  84     return results
  85
  86
  87 # Similar to the lint/lint_strings wrapper setup, for comm-central support.
  88 def gecko_strings_setup(**lint_args):
  89     return strings_repo_setup(STRINGS_REPO, LOCALE)
  90
  91
  92 def strings_repo_setup(repo, locale):
  93     gs = mozpath.join(mach_util.get_state_dir(), locale)
  94     marker = mozpath.join(gs, ".hg", "l10n_pull_marker")
  95     try:
  96         last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
  97         skip_clone = datetime.now() < last_pull + PULL_AFTER
  98     except OSError:
  99         skip_clone = False
 100     if skip_clone:
 101         return
 102     try:
 103         hg = mozversioncontrol.get_tool_path("hg")
 104     except mozversioncontrol.MissingVCSTool:
 105         if os.environ.get("MOZ_AUTOMATION"):
 106             raise
 107         print("warning: l10n linter requires Mercurial but was unable to find 'hg'")
 108         return 1
 109     mozversioncontrol.repoupdate.update_mercurial_repo(hg, repo, gs)
 110     with open(marker, "w") as fh:
 111         fh.flush()
 112
 113
 114 def load_configs(lintconfig, root, l10n_base, locale):
 115     """Load l10n configuration files specified in the linter configuration."""
 116     configs = []
 117     env = {"l10n_base": l10n_base}
 118     for toml in lintconfig["l10n_configs"]:
 119         cfg = TOMLParser().parse(
 120             mozpath.join(root, toml), env=env, ignore_missing_includes=True
 121         )
 122         cfg.set_locales([locale], deep=True)
 123         configs.append(cfg)
 124     return configs
 125
 126
 127 def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
 128     """Check l10n.yml config against l10n.toml configs."""
 129     reference_paths = set(
 130         mozpath.relpath(p["reference"].prefix, lintargs["root"])
 131         for project in l10nconfigs
 132         for config in project.configs
 133         for p in config.paths
 134     )
 135     # Just check for directories
 136     reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
 137     missing_in_yml = [
 138         refd for refd in reference_dirs if refd not in lintconfig["include"]
 139     ]
 140     # These might be subdirectories in the config, though
 141     missing_in_yml = [
 142         d
 143         for d in missing_in_yml
 144         if not any(d.startswith(parent + "/") for parent in lintconfig["include"])
 145     ]
 146     if missing_in_yml:
 147         dirs = ", ".join(missing_in_yml)
 148         return [
 149             result.from_config(
 150                 lintconfig,
 151                 path=lintconfig["path"],
 152                 message="l10n.yml out of sync with l10n.toml, add: " + dirs,
 153             )
 154         ]
 155     return []
 156
 157
 158 class MozL10nLinter(L10nLinter):
 159     """Subclass linter to generate the right result type."""
 160
 161     def __init__(self, lintconfig):
 162         super(MozL10nLinter, self).__init__()
 163         self.lintconfig = lintconfig
 164
 165     def lint(self, files, get_reference_and_tests):
 166         return [
 167             result.from_config(self.lintconfig, **result_data)
 168             for result_data in super(MozL10nLinter, self).lint(
 169                 files, get_reference_and_tests
 170             )
 171         ]