testing/mozharness/scripts/desktop_l10n.py

   1 #!/usr/bin/env python
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5 # You can obtain one at http://mozilla.org/MPL/2.0/.
   6 # ***** END LICENSE BLOCK *****
   7 """desktop_l10n.py
   8
   9 This script manages Desktop repacks for nightly builds.
  10 """
  11 import glob
  12 import os
  13 import shlex
  14 import sys
  15
  16 # load modules from parent dir
  17 sys.path.insert(1, os.path.dirname(sys.path[0]))  # noqa
  18
  19 from mozharness.base.errors import MakefileErrorList
  20 from mozharness.base.script import BaseScript
  21 from mozharness.base.vcs.vcsbase import VCSMixin
  22 from mozharness.mozilla.automation import AutomationMixin
  23 from mozharness.mozilla.building.buildbase import (
  24     MakeUploadOutputParser,
  25     get_mozconfig_path,
  26 )
  27 from mozharness.mozilla.l10n.locales import LocalesMixin
  28
  29 try:
  30     import simplejson as json
  31
  32     assert json
  33 except ImportError:
  34     import json
  35
  36
  37 # needed by _map
  38 SUCCESS = 0
  39 FAILURE = 1
  40
  41 SUCCESS_STR = "Success"
  42 FAILURE_STR = "Failed"
  43
  44
  45 # DesktopSingleLocale {{{1
  46 class DesktopSingleLocale(LocalesMixin, AutomationMixin, VCSMixin, BaseScript):
  47     """Manages desktop repacks"""
  48
  49     config_options = [
  50         [
  51             [
  52                 "--locale",
  53             ],
  54             {
  55                 "action": "extend",
  56                 "dest": "locales",
  57                 "type": "string",
  58                 "help": "Specify the locale(s) to sign and update. Optionally pass"
  59                 " revision separated by colon, en-GB:default.",
  60             },
  61         ],
  62         [
  63             [
  64                 "--tag-override",
  65             ],
  66             {
  67                 "action": "store",
  68                 "dest": "tag_override",
  69                 "type": "string",
  70                 "help": "Override the tags set for all repos",
  71             },
  72         ],
  73         [
  74             [
  75                 "--en-us-installer-url",
  76             ],
  77             {
  78                 "action": "store",
  79                 "dest": "en_us_installer_url",
  80                 "type": "string",
  81                 "help": "Specify the url of the en-us binary",
  82             },
  83         ],
  84     ]
  85
  86     def __init__(self, require_config_file=True):
  87         # fxbuild style:
  88         buildscript_kwargs = {
  89             "all_actions": [
  90                 "clone-locales",
  91                 "list-locales",
  92                 "setup",
  93                 "repack",
  94                 "summary",
  95             ],
  96             "config": {
  97                 "ignore_locales": ["en-US"],
  98                 "locales_dir": "browser/locales",
  99                 "log_name": "single_locale",
 100                 "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
 101             },
 102         }
 103
 104         LocalesMixin.__init__(self)
 105         BaseScript.__init__(
 106             self,
 107             config_options=self.config_options,
 108             require_config_file=require_config_file,
 109             **buildscript_kwargs
 110         )
 111
 112         self.bootstrap_env = None
 113         self.upload_env = None
 114         self.upload_urls = {}
 115         self.pushdate = None
 116         # upload_files is a dictionary of files to upload, keyed by locale.
 117         self.upload_files = {}
 118
 119     # Helper methods {{{2
 120     def query_bootstrap_env(self):
 121         """returns the env for repacks"""
 122         if self.bootstrap_env:
 123             return self.bootstrap_env
 124         config = self.config
 125         abs_dirs = self.query_abs_dirs()
 126
 127         bootstrap_env = self.query_env(
 128             partial_env=config.get("bootstrap_env"), replace_dict=abs_dirs
 129         )
 130
 131         bootstrap_env["L10NBASEDIR"] = abs_dirs["abs_l10n_dir"]
 132         if self.query_is_nightly():
 133             # we might set update_channel explicitly
 134             if config.get("update_channel"):
 135                 update_channel = config["update_channel"]
 136             else:  # Let's just give the generic channel based on branch.
 137                 update_channel = "nightly-%s" % (config["branch"],)
 138             if not isinstance(update_channel, bytes):
 139                 update_channel = update_channel.encode("utf-8")
 140             bootstrap_env["MOZ_UPDATE_CHANNEL"] = update_channel
 141             self.info(
 142                 "Update channel set to: {}".format(bootstrap_env["MOZ_UPDATE_CHANNEL"])
 143             )
 144         self.bootstrap_env = bootstrap_env
 145         return self.bootstrap_env
 146
 147     def _query_upload_env(self):
 148         """returns the environment used for the upload step"""
 149         if self.upload_env:
 150             return self.upload_env
 151         config = self.config
 152
 153         upload_env = self.query_env(partial_env=config.get("upload_env"))
 154         # check if there are any extra option from the platform configuration
 155         # and append them to the env
 156
 157         if "upload_env_extra" in config:
 158             for extra in config["upload_env_extra"]:
 159                 upload_env[extra] = config["upload_env_extra"][extra]
 160
 161         self.upload_env = upload_env
 162         return self.upload_env
 163
 164     def query_l10n_env(self):
 165         l10n_env = self._query_upload_env().copy()
 166         l10n_env.update(self.query_bootstrap_env())
 167         return l10n_env
 168
 169     def _query_make_variable(self, variable, make_args=None):
 170         """returns the value of make echo-variable-<variable>
 171         it accepts extra make arguements (make_args)
 172         """
 173         dirs = self.query_abs_dirs()
 174         make_args = make_args or []
 175         target = ["echo-variable-%s" % variable] + make_args
 176         cwd = dirs["abs_locales_dir"]
 177         raw_output = self._get_output_from_make(
 178             target, cwd=cwd, env=self.query_bootstrap_env()
 179         )
 180         # we want to log all the messages from make
 181         output = []
 182         for line in raw_output.split("\n"):
 183             output.append(line.strip())
 184         output = " ".join(output).strip()
 185         self.info("echo-variable-%s: %s" % (variable, output))
 186         return output
 187
 188     def _map(self, func, items):
 189         """runs func for any item in items, calls the add_failure() for each
 190         error. It assumes that function returns 0 when successful.
 191         returns a two element tuple with (success_count, total_count)"""
 192         success_count = 0
 193         total_count = len(items)
 194         name = func.__name__
 195         for item in items:
 196             result = func(item)
 197             if result == SUCCESS:
 198                 #  success!
 199                 success_count += 1
 200             else:
 201                 #  func failed...
 202                 message = "failure: %s(%s)" % (name, item)
 203                 self.add_failure(item, message)
 204         return (success_count, total_count)
 205
 206     # Actions {{{2
 207     def clone_locales(self):
 208         self.pull_locale_source()
 209
 210     def setup(self):
 211         """setup step"""
 212         self._run_tooltool()
 213         self._copy_mozconfig()
 214         self._mach_configure()
 215         self._run_make_in_config_dir()
 216         self.make_wget_en_US()
 217         self.make_unpack_en_US()
 218
 219     def _run_make_in_config_dir(self):
 220         """this step creates nsinstall, needed my make_wget_en_US()"""
 221         dirs = self.query_abs_dirs()
 222         config_dir = os.path.join(dirs["abs_obj_dir"], "config")
 223         env = self.query_bootstrap_env()
 224         return self._make(target=["export"], cwd=config_dir, env=env)
 225
 226     def _copy_mozconfig(self):
 227         """copies the mozconfig file into abs_src_dir/.mozconfig
 228         and logs the content
 229         """
 230         config = self.config
 231         dirs = self.query_abs_dirs()
 232         src = get_mozconfig_path(self, config, dirs)
 233         dst = os.path.join(dirs["abs_src_dir"], ".mozconfig")
 234         self.copyfile(src, dst)
 235         self.read_from_file(dst, verbose=True)
 236
 237     def _mach(self, target, env, halt_on_failure=True, output_parser=None):
 238         dirs = self.query_abs_dirs()
 239         mach = self._get_mach_executable()
 240         return self.run_command(
 241             mach + target,
 242             halt_on_failure=True,
 243             env=env,
 244             cwd=dirs["abs_src_dir"],
 245             output_parser=None,
 246         )
 247
 248     def _mach_configure(self):
 249         """calls mach configure"""
 250         env = self.query_bootstrap_env()
 251         target = ["configure"]
 252         return self._mach(target=target, env=env)
 253
 254     def _get_mach_executable(self):
 255         return [sys.executable, "mach"]
 256
 257     def _make(
 258         self,
 259         target,
 260         cwd,
 261         env,
 262         error_list=MakefileErrorList,
 263         halt_on_failure=True,
 264         output_parser=None,
 265     ):
 266         """Runs make. Returns the exit code"""
 267         make = ["make"]
 268         if target:
 269             make = make + target
 270         return self.run_command(
 271             make,
 272             cwd=cwd,
 273             env=env,
 274             error_list=error_list,
 275             halt_on_failure=halt_on_failure,
 276             output_parser=output_parser,
 277         )
 278
 279     def _get_output_from_make(
 280         self, target, cwd, env, halt_on_failure=True, ignore_errors=False
 281     ):
 282         """runs make and returns the output of the command"""
 283         return self.get_output_from_command(
 284             ["make"] + target,
 285             cwd=cwd,
 286             env=env,
 287             silent=True,
 288             halt_on_failure=halt_on_failure,
 289             ignore_errors=ignore_errors,
 290         )
 291
 292     def make_unpack_en_US(self):
 293         """wrapper for make unpack"""
 294         config = self.config
 295         dirs = self.query_abs_dirs()
 296         env = self.query_bootstrap_env()
 297         cwd = os.path.join(dirs["abs_obj_dir"], config["locales_dir"])
 298         return self._make(target=["unpack"], cwd=cwd, env=env)
 299
 300     def make_wget_en_US(self):
 301         """wrapper for make wget-en-US"""
 302         env = self.query_bootstrap_env()
 303         dirs = self.query_abs_dirs()
 304         cwd = dirs["abs_locales_dir"]
 305         return self._make(target=["wget-en-US"], cwd=cwd, env=env)
 306
 307     def make_upload(self, locale):
 308         """wrapper for make upload command"""
 309         env = self.query_l10n_env()
 310         dirs = self.query_abs_dirs()
 311         target = ["upload", "AB_CD=%s" % (locale)]
 312         cwd = dirs["abs_locales_dir"]
 313         parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj)
 314         retval = self._make(
 315             target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser
 316         )
 317         if retval == SUCCESS:
 318             self.info("Upload successful (%s)" % locale)
 319             ret = SUCCESS
 320         else:
 321             self.error("failed to upload %s" % locale)
 322             ret = FAILURE
 323
 324         if ret == FAILURE:
 325             # If we failed above, we shouldn't even attempt a SIMPLE_NAME move
 326             # even if we are configured to do so
 327             return ret
 328
 329         # XXX Move the files to a SIMPLE_NAME format until we can enable
 330         #     Simple names in the build system
 331         if self.config.get("simple_name_move"):
 332             # Assume an UPLOAD PATH
 333             upload_target = self.config["upload_env"]["UPLOAD_PATH"]
 334             target_path = os.path.join(upload_target, locale)
 335             self.mkdir_p(target_path)
 336             glob_name = "*.%s.*" % locale
 337             matches = (
 338                 glob.glob(os.path.join(upload_target, glob_name))
 339                 + glob.glob(os.path.join(upload_target, "update", glob_name))
 340                 + glob.glob(os.path.join(upload_target, "*", "xpi", glob_name))
 341                 + glob.glob(os.path.join(upload_target, "install", "sea", glob_name))
 342                 + glob.glob(os.path.join(upload_target, "setup.exe"))
 343                 + glob.glob(os.path.join(upload_target, "setup-stub.exe"))
 344             )
 345             targets_exts = [
 346                 "tar.bz2",
 347                 "dmg",
 348                 "langpack.xpi",
 349                 "checksums",
 350                 "zip",
 351                 "installer.exe",
 352                 "installer-stub.exe",
 353             ]
 354             targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts]
 355             targets.extend([(f, f) for f in ("setup.exe", "setup-stub.exe")])
 356             for f in matches:
 357                 possible_targets = [
 358                     (tail, target_file)
 359                     for (tail, target_file) in targets
 360                     if f.endswith(tail)
 361                 ]
 362                 if len(possible_targets) == 1:
 363                     _, target_file = possible_targets[0]
 364                     # Remove from list of available options for this locale
 365                     targets.remove(possible_targets[0])
 366                 else:
 367                     # wasn't valid (or already matched)
 368                     raise RuntimeError(
 369                         "Unexpected matching file name encountered: %s" % f
 370                     )
 371                 self.move(os.path.join(f), os.path.join(target_path, target_file))
 372             self.log("Converted uploads for %s to simple names" % locale)
 373         return ret
 374
 375     def set_upload_files(self, locale):
 376         # The tree doesn't have a good way of exporting the list of files
 377         # created during locale generation, but we can grab them by echoing the
 378         # UPLOAD_FILES variable for each locale.
 379         env = self.query_l10n_env()
 380         target = [
 381             "echo-variable-UPLOAD_FILES",
 382             "echo-variable-CHECKSUM_FILES",
 383             "AB_CD=%s" % locale,
 384         ]
 385         dirs = self.query_abs_dirs()
 386         cwd = dirs["abs_locales_dir"]
 387         # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
 388         #    we should ignore stderr as unfortunately it's expected when parsing for values
 389         output = self._get_output_from_make(
 390             target=target, cwd=cwd, env=env, ignore_errors=True
 391         )
 392         self.info('UPLOAD_FILES is "%s"' % output)
 393         files = shlex.split(output)
 394         if not files:
 395             self.error("failed to get upload file list for locale %s" % locale)
 396             return FAILURE
 397
 398         self.upload_files[locale] = [
 399             os.path.abspath(os.path.join(cwd, f)) for f in files
 400         ]
 401         return SUCCESS
 402
 403     def make_installers(self, locale):
 404         """wrapper for make installers-(locale)"""
 405         env = self.query_l10n_env()
 406         env["PYTHONIOENCODING"] = "utf-8"
 407         self._copy_mozconfig()
 408         dirs = self.query_abs_dirs()
 409         cwd = os.path.join(dirs["abs_locales_dir"])
 410         target = [
 411             "installers-%s" % locale,
 412         ]
 413         return self._make(target=target, cwd=cwd, env=env, halt_on_failure=False)
 414
 415     def repack_locale(self, locale):
 416         """wraps the logic for make installers and generating
 417         complete updates."""
 418
 419         # run make installers
 420         if self.make_installers(locale) != SUCCESS:
 421             self.error("make installers-%s failed" % (locale))
 422             return FAILURE
 423
 424         # now try to upload the artifacts
 425         if self.make_upload(locale):
 426             self.error("make upload for locale %s failed!" % (locale))
 427             return FAILURE
 428
 429         # set_upload_files() should be called after make upload, to make sure
 430         # we have all files in place (checksums, etc)
 431         if self.set_upload_files(locale):
 432             self.error("failed to get list of files to upload for locale %s" % locale)
 433             return FAILURE
 434
 435         return SUCCESS
 436
 437     def repack(self):
 438         """creates the repacks and udpates"""
 439         self._map(self.repack_locale, self.query_locales())
 440
 441     def _run_tooltool(self):
 442         env = self.query_bootstrap_env()
 443         config = self.config
 444         dirs = self.query_abs_dirs()
 445         manifest_src = os.environ.get("TOOLTOOL_MANIFEST")
 446         if not manifest_src:
 447             manifest_src = config.get("tooltool_manifest_src")
 448         if not manifest_src:
 449             return
 450         python = sys.executable
 451
 452         cmd = [
 453             python,
 454             "-u",
 455             os.path.join(dirs["abs_src_dir"], "mach"),
 456             "artifact",
 457             "toolchain",
 458             "-v",
 459             "--retry",
 460             "4",
 461             "--artifact-manifest",
 462             os.path.join(dirs["abs_src_dir"], "toolchains.json"),
 463         ]
 464         if manifest_src:
 465             cmd.extend(
 466                 [
 467                     "--tooltool-manifest",
 468                     os.path.join(dirs["abs_src_dir"], manifest_src),
 469                 ]
 470             )
 471         cache = config["bootstrap_env"].get("TOOLTOOL_CACHE")
 472         if cache:
 473             cmd.extend(["--cache-dir", cache])
 474         self.info(str(cmd))
 475         self.run_command(cmd, cwd=dirs["abs_src_dir"], halt_on_failure=True, env=env)
 476
 477
 478 # main {{{
 479 if __name__ == "__main__":
 480     single_locale = DesktopSingleLocale()
 481     single_locale.run_and_exit()