testing/mozharness/scripts/desktop_partner_repacks.py

   1 #!/usr/bin/env python
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   5 # You can obtain one at http://mozilla.org/MPL/2.0/.
   6 # ***** END LICENSE BLOCK *****
   7 """desktop_partner_repacks.py
   8
   9 This script manages Desktop partner repacks for beta/release builds.
  10 """
  11 import os
  12 import sys
  13
  14 # load modules from parent dir
  15 sys.path.insert(1, os.path.dirname(sys.path[0]))
  16
  17 from mozharness.base.script import BaseScript
  18 from mozharness.mozilla.buildbot import BuildbotMixin
  19 from mozharness.mozilla.purge import PurgeMixin
  20 from mozharness.mozilla.release import ReleaseMixin
  21 from mozharness.base.python import VirtualenvMixin
  22 from mozharness.base.log import FATAL
  23
  24
  25 # DesktopPartnerRepacks {{{1
  26 class DesktopPartnerRepacks(ReleaseMixin, BuildbotMixin, PurgeMixin,
  27                             BaseScript, VirtualenvMixin):
  28     """Manages desktop partner repacks"""
  29     actions = [
  30                 "clobber",
  31                 "create-virtualenv",
  32                 "activate-virtualenv",
  33                 "setup",
  34                 "repack",
  35                 "summary",
  36               ]
  37     config_options = [
  38         [["--version", "-v"], {
  39           "dest": "version",
  40           "help": "Version of Firefox to repack",
  41           }],
  42         [["--build-number", "-n"], {
  43           "dest": "build_number",
  44           "help": "Build number of Firefox to repack",
  45           }],
  46         [["--platform"], {
  47           "dest": "platform",
  48           "help": "Platform to repack (e.g. linux64, macosx64, ...)",
  49           }],
  50         [["--partner", "-p"], {
  51           "dest": "partner",
  52           "help": "Limit repackaging to partners matching this string",
  53           }],
  54         [["--s3cfg"], {
  55           "dest": "s3cfg",
  56           "help": "Configuration file for uploading to S3 using s3cfg",
  57           }],
  58         [["--hgroot"], {
  59           "dest": "hgroot",
  60           "help": "Use a different hg server for retrieving files",
  61           }],
  62         [["--hgrepo"], {
  63           "dest": "hgrepo",
  64           "help": "Use a different base repo for retrieving files",
  65           }],
  66         [["--require-buildprops"], {
  67             "action": "store_true",
  68             "dest": "require_buildprops",
  69             "default": False,
  70             "help": "Read in config options (like partner) from the buildbot properties file."
  71           }],
  72     ]
  73
  74     def __init__(self):
  75         # fxbuild style:
  76         buildscript_kwargs = {
  77             'all_actions': DesktopPartnerRepacks.actions,
  78             'default_actions': DesktopPartnerRepacks.actions,
  79             'config': {
  80                 'buildbot_json_path': 'buildprops.json',
  81                 "log_name": "partner-repacks",
  82                 "hashType": "sha512",
  83                 'virtualenv_modules': [
  84                     'requests==2.2.1',
  85                     'PyHawk-with-a-single-extra-commit==0.1.5',
  86                     'taskcluster==0.0.15',
  87                     's3cmd==1.6.0',
  88                 ],
  89                 'virtualenv_path': 'venv',
  90                 'workdir': 'partner-repacks',
  91             },
  92         }
  93         #
  94
  95         BaseScript.__init__(
  96             self,
  97             config_options=self.config_options,
  98             **buildscript_kwargs
  99         )
 100
 101
 102     def _pre_config_lock(self, rw_config):
 103         self.read_buildbot_config()
 104         if not self.buildbot_config:
 105             self.warning("Skipping buildbot properties overrides")
 106         else:
 107             if self.config.get('require_buildprops', False) is True:
 108                 if not self.buildbot_config:
 109                     self.fatal("Unable to load properties from file: %s" % self.config.get('buildbot_json_path'))
 110             props = self.buildbot_config["properties"]
 111             for prop in ['version', 'build_number', 'revision', 'repo_file', 'repack_manifests_url', 'partner']:
 112                 if props.get(prop):
 113                     self.info("Overriding %s with %s" % (prop, props[prop]))
 114                     self.config[prop] = props.get(prop)
 115
 116         if 'version' not in self.config:
 117             self.fatal("Version (-v) not supplied.")
 118         if 'build_number' not in self.config:
 119             self.fatal("Build number (-n) not supplied.")
 120         if 'repo_file' not in self.config:
 121             self.fatal("repo_file not supplied.")
 122         if 'repack_manifests_url' not in self.config:
 123             self.fatal("repack_manifests_url not supplied.")
 124
 125     def query_abs_dirs(self):
 126         if self.abs_dirs:
 127             return self.abs_dirs
 128         abs_dirs = super(DesktopPartnerRepacks, self).query_abs_dirs()
 129         for directory in abs_dirs:
 130             value = abs_dirs[directory]
 131             abs_dirs[directory] = value
 132         dirs = {}
 133         dirs['abs_repo_dir'] = os.path.join(abs_dirs['abs_work_dir'], '.repo')
 134         dirs['abs_partners_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'partners')
 135         dirs['abs_scripts_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'scripts')
 136         for key in dirs.keys():
 137             if key not in abs_dirs:
 138                 abs_dirs[key] = dirs[key]
 139         self.abs_dirs = abs_dirs
 140         return self.abs_dirs
 141
 142     # Actions {{{
 143     def _repo_cleanup(self):
 144         self.rmtree(self.query_abs_dirs()['abs_repo_dir'])
 145         self.rmtree(self.query_abs_dirs()['abs_partners_dir'])
 146         self.rmtree(self.query_abs_dirs()['abs_scripts_dir'])
 147
 148     def _repo_init(self, repo):
 149         status = self.run_command([repo, "init", "--no-repo-verify",
 150                                    "-u", self.config['repack_manifests_url']],
 151                                   cwd=self.query_abs_dirs()['abs_work_dir'])
 152         if status:
 153             return status
 154         return self.run_command([repo, "sync"],
 155                                 cwd=self.query_abs_dirs()['abs_work_dir'])
 156
 157     def setup(self):
 158         """setup step"""
 159         repo = self.download_file(self.config['repo_file'],
 160                                   file_name='repo',
 161                                   parent_dir=self.query_abs_dirs()['abs_work_dir'],
 162                                   error_level=FATAL)
 163         if not os.path.exists(repo):
 164             self.fatal("Unable to download repo tool.")
 165         self.chmod(repo, 0755)
 166         self.retry(self._repo_init,
 167                    args=(repo,),
 168                    error_level=FATAL,
 169                    cleanup=self._repo_cleanup(),
 170                    good_statuses=[0],
 171                    sleeptime=5)
 172
 173     def repack(self):
 174         """creates the repacks"""
 175         repack_cmd = [sys.executable, "partner-repacks.py",
 176                       "-v", self.config['version'],
 177                       "-n", str(self.config['build_number'])]
 178         if self.config.get('platform'):
 179             repack_cmd.extend(["--platform", self.config['platform']])
 180         if self.config.get('partner'):
 181             repack_cmd.extend(["--partner", self.config['partner']])
 182         if self.config.get('s3cfg'):
 183             repack_cmd.extend(["--s3cfg", self.config['s3cfg']])
 184         if self.config.get('hgroot'):
 185             repack_cmd.extend(["--hgroot", self.config['hgroot']])
 186         if self.config.get('hgrepo'):
 187             repack_cmd.extend(["--repo", self.config['hgrepo']])
 188         if self.config.get('revision'):
 189             repack_cmd.extend(["--tag", self.config["revision"]])
 190
 191         return self.run_command(repack_cmd,
 192                                 cwd=self.query_abs_dirs()['abs_scripts_dir'])
 193
 194 # main {{{
 195 if __name__ == '__main__':
 196     partner_repacks = DesktopPartnerRepacks()
 197     partner_repacks.run_and_exit()