testing/mozharness/scripts/bouncer_submitter.py

   1 #!/usr/bin/env python
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import os
   7 import sys
   8
   9 sys.path.insert(1, os.path.dirname(sys.path[0]))
  10
  11 from mozharness.base.script import BaseScript
  12 from mozharness.mozilla.bouncer.submitter import BouncerSubmitterMixin
  13 from mozharness.mozilla.buildbot import BuildbotMixin
  14 from mozharness.mozilla.purge import PurgeMixin
  15
  16
  17 class BouncerSubmitter(BaseScript, PurgeMixin, BouncerSubmitterMixin, BuildbotMixin):
  18     config_options = [
  19         [["--repo"], {
  20             "dest": "repo",
  21             "help": "Specify source repo, e.g. releases/mozilla-beta",
  22         }],
  23         [["--revision"], {
  24             "dest": "revision",
  25             "help": "Source revision/tag used to fetch shipped-locales",
  26         }],
  27         [["--version"], {
  28             "dest": "version",
  29             "help": "Current version",
  30         }],
  31         [["--previous-version"], {
  32             "dest": "prev_versions",
  33             "action": "extend",
  34             "help": "Previous version(s)",
  35         }],
  36         [["--build-number"], {
  37             "dest": "build_number",
  38             "help": "Build number of version",
  39         }],
  40         [["--bouncer-api-prefix"], {
  41             "dest": "bouncer-api-prefix",
  42             "help": "Bouncer admin API URL prefix",
  43         }],
  44         [["--credentials-file"], {
  45             "dest": "credentials_file",
  46             "help": "File containing Bouncer credentials",
  47         }],
  48     ]
  49
  50     def __init__(self, require_config_file=True):
  51         BaseScript.__init__(self,
  52                             config_options=self.config_options,
  53                             require_config_file=require_config_file,
  54                             # other stuff
  55                             all_actions=[
  56                                 'clobber',
  57                                 'download-shipped-locales',
  58                                 'submit',
  59                             ],
  60                             default_actions=[
  61                                 'clobber',
  62                                 'download-shipped-locales',
  63                                 'submit',
  64                             ],
  65                             config={
  66                                  'buildbot_json_path' : 'buildprops.json'
  67                             }
  68                             )
  69         self.locales = None
  70         self.credentials = None
  71
  72     def _pre_config_lock(self, rw_config):
  73         super(BouncerSubmitter, self)._pre_config_lock(rw_config)
  74
  75         #override properties from buildbot properties here as defined by taskcluster properties
  76         self.read_buildbot_config()
  77
  78         #check if release promotion is true first before overwriting these properties
  79         if self.buildbot_config["properties"].get("release_promotion"):
  80             for prop in ['product', 'version', 'build_number', 'revision', 'bouncer_submitter_config', ]:
  81                 if self.buildbot_config["properties"].get(prop):
  82                     self.info("Overriding %s with %s" % (prop,  self.buildbot_config["properties"].get(prop)))
  83                     self.config[prop] = self.buildbot_config["properties"].get(prop)
  84             if self.buildbot_config["properties"].get("partial_versions"):
  85                 self.config["prev_versions"] = self.buildbot_config["properties"].get("partial_versions").split(", ")
  86
  87         for opt in ["version", "credentials_file", "bouncer-api-prefix"]:
  88             if opt not in self.config:
  89                 self.fatal("%s must be specified" % opt)
  90         if self.need_shipped_locales():
  91             for opt in ["shipped-locales-url", "repo", "revision"]:
  92                 if opt not in self.config:
  93                     self.fatal("%s must be specified" % opt)
  94
  95     def need_shipped_locales(self):
  96         return any(e.get("add-locales") for e in
  97                    self.config["products"].values())
  98
  99     def query_shipped_locales_path(self):
 100         dirs = self.query_abs_dirs()
 101         return os.path.join(dirs["abs_work_dir"], "shipped-locales")
 102
 103     def download_shipped_locales(self):
 104         if not self.need_shipped_locales():
 105             self.info("No need to download shipped-locales")
 106             return
 107
 108         replace_dict = {"revision": self.config["revision"],
 109                         "repo": self.config["repo"]}
 110         url = self.config["shipped-locales-url"] % replace_dict
 111         dirs = self.query_abs_dirs()
 112         self.mkdir_p(dirs["abs_work_dir"])
 113         if not self.download_file(url=url,
 114                                   file_name=self.query_shipped_locales_path()):
 115             self.fatal("Unable to fetch shipped-locales from %s" % url)
 116         # populate the list
 117         self.load_shipped_locales()
 118
 119     def load_shipped_locales(self):
 120         if self.locales:
 121             return self.locales
 122         content = self.read_from_file(self.query_shipped_locales_path())
 123         locales = []
 124         for line in content.splitlines():
 125             locale = line.split()[0]
 126             if locale:
 127                 locales.append(locale)
 128         self.locales = locales
 129         return self.locales
 130
 131     def submit(self):
 132         subs = {
 133             "version": self.config["version"]
 134         }
 135         if self.config.get("build_number"):
 136             subs["build_number"] = self.config["build_number"]
 137
 138         for product, pr_config in sorted(self.config["products"].items()):
 139             product_name = pr_config["product-name"] % subs
 140             if self.product_exists(product_name):
 141                 self.warning("Product %s already exists. Skipping..." %
 142                              product_name)
 143                 continue
 144             self.info("Adding %s..." % product)
 145             self.api_add_product(
 146                 product_name=product_name,
 147                 add_locales=pr_config.get("add-locales"),
 148                 ssl_only=pr_config.get("ssl-only"))
 149             self.info("Adding paths...")
 150             for platform, pl_config in sorted(pr_config["paths"].items()):
 151                 bouncer_platform = pl_config["bouncer-platform"]
 152                 path = pl_config["path"] % subs
 153                 self.info("%s (%s): %s" % (platform, bouncer_platform, path))
 154                 self.api_add_location(product_name, bouncer_platform, path)
 155
 156         # Add partial updates
 157         if "partials" in self.config and self.config.get("prev_versions"):
 158             self.submit_partials()
 159
 160     def submit_partials(self):
 161         subs = {
 162             "version": self.config["version"]
 163         }
 164         if self.config.get("build_number"):
 165             subs["build_number"] = self.config["build_number"]
 166         prev_versions = self.config.get("prev_versions")
 167         for product, part_config in sorted(self.config["partials"].items()):
 168             product_name_tmpl = part_config["product-name"]
 169             for prev_version in prev_versions:
 170                 prev_version, prev_build_number = prev_version.split("build")
 171                 subs["prev_version"] = prev_version
 172                 subs["prev_build_number"] = prev_build_number
 173                 product_name = product_name_tmpl % subs
 174                 if self.product_exists(product_name):
 175                     self.warning("Product %s already exists. Skipping..." %
 176                                  product_name)
 177                     continue
 178                 self.info("Adding partial updates for %s" % product_name)
 179                 self.api_add_product(
 180                     product_name=product_name,
 181                     add_locales=part_config.get("add-locales"),
 182                     ssl_only=part_config.get("ssl-only"))
 183                 for platform, pl_config in sorted(part_config["paths"].items()):
 184                     bouncer_platform = pl_config["bouncer-platform"]
 185                     path = pl_config["path"] % subs
 186                     self.info("%s (%s): %s" % (platform, bouncer_platform, path))
 187                     self.api_add_location(product_name, bouncer_platform, path)
 188
 189
 190 if __name__ == '__main__':
 191     myScript = BouncerSubmitter()
 192     myScript.run_and_exit()