tools/lint/python/l10n_lint.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 from datetime import datetime, timedelta
   7 from subprocess import check_call
   8
   9 from compare_locales import parser
  10 from compare_locales.lint.linter import L10nLinter
  11 from compare_locales.lint.util import l10n_base_reference_and_tests
  12 from compare_locales.paths import ProjectFiles, TOMLParser
  13 from mach import util as mach_util
  14 from mozfile import which
  15 from mozlint import pathutils, result
  16 from mozpack import path as mozpath
  17 from mozversioncontrol import MissingVCSTool
  18
  19 L10N_SOURCE_NAME = "l10n-source"
  20 L10N_SOURCE_REPO = "https://github.com/mozilla-l10n/firefox-l10n-source.git"
  21
  22 PULL_AFTER = timedelta(days=2)
  23
  24
  25 # Wrapper to call lint_strings with mozilla-central configuration
  26 # comm-central defines its own wrapper since comm-central strings are
  27 # in separate repositories
  28 def lint(paths, lintconfig, **lintargs):
  29     return lint_strings(L10N_SOURCE_NAME, paths, lintconfig, **lintargs)
  30
  31
  32 def lint_strings(name, paths, lintconfig, **lintargs):
  33     l10n_base = mach_util.get_state_dir()
  34     root = lintargs["root"]
  35     exclude = lintconfig.get("exclude")
  36     extensions = lintconfig.get("extensions")
  37
  38     # Load l10n.toml configs
  39     l10nconfigs = load_configs(lintconfig["l10n_configs"], root, l10n_base, name)
  40
  41     # If l10n.yml is included in the provided paths, validate it against the
  42     # TOML files, then remove it to avoid parsing it as a localizable resource.
  43     if lintconfig["path"] in paths:
  44         results = validate_linter_includes(lintconfig, l10nconfigs, lintargs)
  45         paths.remove(lintconfig["path"])
  46         lintconfig["include"].remove(mozpath.relpath(lintconfig["path"], root))
  47     else:
  48         results = []
  49
  50     all_files = []
  51     for p in paths:
  52         fp = pathutils.FilterPath(p)
  53         if fp.isdir:
  54             for _, fileobj in fp.finder:
  55                 all_files.append(fileobj.path)
  56         if fp.isfile:
  57             all_files.append(p)
  58     # Filter out files explicitly excluded in the l10n.yml configuration.
  59     # `browser/locales/en-US/firefox-l10n.js` is a good example.
  60     all_files, _ = pathutils.filterpaths(
  61         lintargs["root"],
  62         all_files,
  63         lintconfig["include"],
  64         exclude=exclude,
  65         extensions=extensions,
  66     )
  67     # Filter again, our directories might have picked up files that should be
  68     # excluded in l10n.yml
  69     skips = {p for p in all_files if not parser.hasParser(p)}
  70     results.extend(
  71         result.from_config(
  72             lintconfig,
  73             level="warning",
  74             path=path,
  75             message="file format not supported in compare-locales",
  76         )
  77         for path in skips
  78     )
  79     all_files = [p for p in all_files if p not in skips]
  80     files = ProjectFiles(name, l10nconfigs)
  81
  82     get_reference_and_tests = l10n_base_reference_and_tests(files)
  83
  84     linter = MozL10nLinter(lintconfig)
  85     results += linter.lint(all_files, get_reference_and_tests)
  86     return results
  87
  88
  89 # Similar to the lint/lint_strings wrapper setup, for comm-central support.
  90 def source_repo_setup(**lint_args):
  91     gs = mozpath.join(mach_util.get_state_dir(), L10N_SOURCE_NAME)
  92     marker = mozpath.join(gs, ".git", "l10n_pull_marker")
  93     try:
  94         last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime)
  95         skip_clone = datetime.now() < last_pull + PULL_AFTER
  96     except OSError:
  97         skip_clone = False
  98     if skip_clone:
  99         return
 100     git = which("git")
 101     if not git:
 102         if os.environ.get("MOZ_AUTOMATION"):
 103             raise MissingVCSTool("Unable to obtain git path.")
 104         print("warning: l10n linter requires Git but was unable to find 'git'")
 105         return 1
 106
 107     # If this is called from a source hook on a git repo, there might be a index
 108     # file listed in the environment as a git operation is ongoing. This seems
 109     # to confuse the git call here into thinking that it is actually operating
 110     # on the main repository, rather than the l10n-source repo. Therefore,
 111     # we remove this environment flag.
 112     if "GIT_INDEX_FILE" in os.environ:
 113         os.environ.pop("GIT_INDEX_FILE")
 114
 115     if os.path.exists(gs):
 116         check_call([git, "pull", L10N_SOURCE_REPO], cwd=gs)
 117     else:
 118         check_call([git, "clone", L10N_SOURCE_REPO, gs])
 119     with open(marker, "w") as fh:
 120         fh.flush()
 121
 122
 123 def load_configs(l10n_configs, root, l10n_base, locale):
 124     """Load l10n configuration files specified in the linter configuration."""
 125     configs = []
 126     env = {"l10n_base": l10n_base}
 127     for toml in l10n_configs:
 128         cfg = TOMLParser().parse(
 129             mozpath.join(root, toml), env=env, ignore_missing_includes=True
 130         )
 131         cfg.set_locales([locale], deep=True)
 132         configs.append(cfg)
 133     return configs
 134
 135
 136 def validate_linter_includes(lintconfig, l10nconfigs, lintargs):
 137     """Check l10n.yml config against l10n.toml configs."""
 138     reference_paths = set(
 139         mozpath.relpath(p["reference"].prefix, lintargs["root"])
 140         for project in l10nconfigs
 141         for config in project.configs
 142         for p in config.paths
 143     )
 144     # Just check for directories
 145     reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p))
 146     missing_in_yml = [
 147         refd for refd in reference_dirs if refd not in lintconfig["include"]
 148     ]
 149     # These might be subdirectories in the config, though
 150     missing_in_yml = [
 151         d
 152         for d in missing_in_yml
 153         if not any(d.startswith(parent + "/") for parent in lintconfig["include"])
 154     ]
 155     if missing_in_yml:
 156         dirs = ", ".join(missing_in_yml)
 157         return [
 158             result.from_config(
 159                 lintconfig,
 160                 path=lintconfig["path"],
 161                 message="l10n.yml out of sync with l10n.toml, add: " + dirs,
 162             )
 163         ]
 164     return []
 165
 166
 167 class MozL10nLinter(L10nLinter):
 168     """Subclass linter to generate the right result type."""
 169
 170     def __init__(self, lintconfig):
 171         super(MozL10nLinter, self).__init__()
 172         self.lintconfig = lintconfig
 173
 174     def lint(self, files, get_reference_and_tests):
 175         return [
 176             result.from_config(self.lintconfig, **result_data)
 177             for result_data in super(MozL10nLinter, self).lint(
 178                 files, get_reference_and_tests
 179             )
 180         ]