1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this,
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 from pathlib
import Path
11 from appdirs
import user_config_dir
12 from hglib
.error
import CommandError
13 from mach
.base
import FailedCommandError
14 from mach
.decorators
import Command
, CommandArgument
15 from mozrelease
.scriptworker_canary
import get_secret
16 from redo
import retry
22 description
="Run source checks on a localization.",
28 help="TOML or INI file for the project",
32 metavar
="l10n-base-dir",
33 help="Parent directory of localizations",
38 metavar
="locale-code",
39 help="Locale code and top-level directory of each localization",
46 help="""Show less data.
47 Specified once, don't show obsolete entities. Specified twice, also hide
48 missing entities. Specify thrice to exclude warnings and four times to
51 @CommandArgument("-m", "--merge", help="""Use this directory to stage merged files""")
53 "--validate", action
="store_true", help="Run compare-locales against reference"
57 help="""Serialize to JSON. Value is the name of
58 the output file, pass "-" to serialize to stdout and hide the default output.
67 help="Overwrite variables in TOML files",
70 "--full", action
="store_true", help="Compare projects that are disabled"
73 "--return-zero", action
="store_true", help="Return 0 regardless of l10n status"
75 def compare(command_context
, **kwargs
):
76 """Run compare-locales."""
77 from compare_locales
.commands
import CompareLocales
79 class ErrorHelper(object):
80 """Dummy ArgumentParser to marshall compare-locales
81 commandline errors to mach exceptions.
85 raise FailedCommandError(msg
)
87 def exit(self
, message
=None, status
=0):
88 raise FailedCommandError(message
, exit_code
=status
)
90 cmd
= CompareLocales()
91 cmd
.parser
= ErrorHelper()
92 return cmd
.handle(**kwargs
)
95 # https://stackoverflow.com/a/14117511
96 def _positive_int(value
):
99 raise argparse
.ArgumentTypeError(f
"{value} must be a positive integer.")
103 class RetryError(Exception):
107 VCT_PATH
= Path(".").resolve() / "vct"
108 VCT_URL
= "https://hg.mozilla.org/hgcustom/version-control-tools/"
109 FXTREE_PATH
= VCT_PATH
/ "hgext" / "firefoxtree"
110 HGRC_PATH
= Path(user_config_dir("hg")).joinpath("hgrc")
114 "l10n-cross-channel",
116 description
="Create cross-channel content.",
123 default
=Path("en-US"),
124 help="Path to mercurial repository for gecko-strings-quarantine",
130 help="create an outgoing() patch if there are changes",
136 help="Number of times to try (for automation)",
141 help="Taskcluster secret to use to push (for automation)",
145 choices
=("prep", "create", "push", "clean"),
147 # This help block will be poorly formatted until we fix bug 1714239
149 "prep": clone repos and pull heads.
150 "create": create the en-US strings commit an optionally create an
152 "push": push the en-US strings to the quarantine repo.
153 "clean": clean up any sub-repos.
165 """Run l10n cross-channel content generation."""
166 # This can be any path, as long as the name of the directory is en-US.
167 # Not entirely sure where this is a requirement; perhaps in l10n
168 # string manipulation logic?
169 if strings_path
.name
!= "en-US":
170 raise FailedCommandError("strings_path needs to be named `en-US`")
171 command_context
.activate_virtualenv()
172 # XXX pin python requirements
173 command_context
.virtualenv_manager
.install_pip_requirements(
174 Path(os
.path
.dirname(__file__
)) / "requirements.in"
176 strings_path
= strings_path
.resolve() # abspath
178 outgoing_path
= outgoing_path
.resolve() # abspath
179 get_config
= kwargs
.get("get_config", None)
181 with tempfile
.TemporaryDirectory() as ssh_key_dir
:
185 retry_exceptions
=(RetryError
,),
196 except RetryError
as exc
:
197 raise FailedCommandError(exc
) from exc
200 def _do_create_content(
209 from mozxchannel
import CrossChannelCreator
, get_default_config
211 get_config
= get_config
or get_default_config
213 config
= get_config(Path(command_context
.topsrcdir
), strings_path
)
214 ccc
= CrossChannelCreator(config
)
217 ssh_key_secret
= None
220 if "prep" in actions
:
222 if not os
.environ
.get("MOZ_AUTOMATION"):
224 "I don't know how to fetch the ssh secret outside of automation!"
226 ssh_key_secret
= get_secret(ssh_secret
)
227 ssh_key_file
= ssh_key_dir
.joinpath("id_rsa")
228 ssh_key_file
.write_text(ssh_key_secret
["ssh_privkey"])
229 ssh_key_file
.chmod(0o600)
230 # Set up firefoxtree for comm per bug 1659691 comment 22
231 if os
.environ
.get("MOZ_AUTOMATION") and not HGRC_PATH
.exists():
232 _clone_hg_repo(command_context
, VCT_URL
, VCT_PATH
)
235 f
"firefoxtree = {FXTREE_PATH}",
243 f
"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}",
246 HGRC_PATH
.write_text("\n".join(hgrc_content
))
247 if strings_path
.exists() and _check_outgoing(command_context
, strings_path
):
248 _strip_outgoing(command_context
, strings_path
)
249 # Clone strings + source repos, pull heads
250 for repo_config
in (config
["strings"], *config
["source"].values()):
251 if not repo_config
["path"].exists():
253 command_context
, repo_config
["url"], str(repo_config
["path"])
255 for head
in repo_config
["heads"].keys():
256 command
= ["hg", "--cwd", str(repo_config
["path"]), "pull"]
258 status
= _retry_run_process(
259 command_context
, command
, ensure_exit_code
=False
261 if status
not in (0, 255): # 255 on pull with no changes
262 raise RetryError(f
"Failure on pull: status {status}!")
263 if repo_config
.get("update_on_pull"):
267 str(repo_config
["path"]),
273 status
= _retry_run_process(
274 command_context
, command
, ensure_exit_code
=False
276 if status
not in (0, 255): # 255 on pull with no changes
277 raise RetryError(f
"Failure on update: status {status}!")
281 heads
=repo_config
.get("heads", {}).keys(),
284 _check_hg_repo(command_context
, strings_path
)
285 for repo_config
in config
.get("source", {}).values():
289 heads
=repo_config
.get("heads", {}).keys(),
291 if _check_outgoing(command_context
, strings_path
):
292 raise RetryError(f
"check: Outgoing changes in {strings_path}!")
294 if "create" in actions
:
296 status
= ccc
.create_content()
298 _create_outgoing_patch(command_context
, outgoing_path
, strings_path
)
299 except CommandError
as exc
:
301 raise RetryError(exc
) from exc
302 command_context
.log(logging
.INFO
, "create", {}, "No new strings.")
304 if "push" in actions
:
315 config
["strings"]["push_url"],
320 command_context
.log(logging
.INFO
, "push", {}, "Skipping empty push.")
322 if "clean" in actions
:
323 for repo_config
in config
.get("source", {}).values():
324 if repo_config
.get("post-clobber", False):
325 _nuke_hg_repo(command_context
, str(repo_config
["path"]))
330 def _check_outgoing(command_context
, strings_path
):
331 status
= _retry_run_process(
333 ["hg", "--cwd", str(strings_path
), "out", "-r", "."],
334 ensure_exit_code
=False,
340 raise RetryError(f
"Outgoing check in {strings_path} returned unexpected {status}!")
343 def _strip_outgoing(command_context
, strings_path
):
359 def _create_outgoing_patch(command_context
, path
, strings_path
):
362 if not path
.parent
.exists():
363 os
.makedirs(path
.parent
)
364 with
open(path
, "w") as fh
:
367 fh
.write(f
"{line}\n")
381 line_handler
=writeln
,
385 def _retry_run_process(command_context
, *args
, error_msg
=None, **kwargs
):
387 return command_context
.run_process(*args
, **kwargs
)
388 except Exception as exc
:
389 raise RetryError(error_msg
or str(exc
)) from exc
392 def _check_hg_repo(command_context
, path
, heads
=None):
393 if not (path
.is_dir() and (path
/ ".hg").is_dir()):
394 raise RetryError(f
"{path} is not a Mercurial repository")
399 ["hg", "--cwd", str(path
), "log", "-r", head
],
400 error_msg
=f
"check: {path} has no head {head}!",
404 def _clone_hg_repo(command_context
, url
, path
):
405 _retry_run_process(command_context
, ["hg", "clone", url
, str(path
)])
408 def _nuke_hg_repo(command_context
, path
):
409 _retry_run_process(command_context
, ["rm", "-rf", str(path
)])