testing/mozharness/scripts/repackage.py

   1 import os
   2 import sys
   3
   4 sys.path.insert(1, os.path.dirname(sys.path[0]))  # noqa - don't warn about imports
   5
   6 from mozharness.base.log import FATAL
   7 from mozharness.base.script import BaseScript
   8 from mozharness.mozilla.mock import ERROR_MSGS
   9
  10
  11 class Repackage(BaseScript):
  12
  13     def __init__(self, require_config_file=False):
  14         script_kwargs = {
  15             'all_actions': [
  16                 "download_input",
  17                 "setup",
  18                 "repackage",
  19             ],
  20         }
  21         BaseScript.__init__(
  22             self,
  23             require_config_file=require_config_file,
  24             **script_kwargs
  25         )
  26
  27     def download_input(self):
  28         config = self.config
  29         dirs = self.query_abs_dirs()
  30
  31         input_home = config['input_home'].format(**dirs)
  32
  33         for path, url in config["download_config"].items():
  34             status = self.download_file(url=url,
  35                                         file_name=path,
  36                                         parent_dir=input_home)
  37             if not status:
  38                 self.fatal("Unable to fetch signed input from %s" % url)
  39
  40             if 'mar' in path:
  41                 # Ensure mar is executable
  42                 self.chmod(os.path.join(input_home, path), 0755)
  43
  44     def setup(self):
  45         self._run_tooltool()
  46         if self.config.get("run_configure", True):
  47             self._get_mozconfig()
  48             self._run_configure()
  49
  50     def query_abs_dirs(self):
  51         if self.abs_dirs:
  52             return self.abs_dirs
  53         abs_dirs = super(Repackage, self).query_abs_dirs()
  54         config = self.config
  55         for directory in abs_dirs:
  56             value = abs_dirs[directory]
  57             abs_dirs[directory] = value
  58
  59         dirs = {}
  60         dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
  61         dirs['abs_mozilla_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'src')
  62         locale_dir = ''
  63         if config.get('locale'):
  64             locale_dir = "{}{}".format(os.path.sep, config['locale'])
  65         dirs['output_home'] = config['output_home'].format(locale=locale_dir, **abs_dirs)
  66         for key in dirs.keys():
  67             if key not in abs_dirs:
  68                 abs_dirs[key] = dirs[key]
  69         self.abs_dirs = abs_dirs
  70         return self.abs_dirs
  71
  72     def repackage(self):
  73         config = self.config
  74         dirs = self.query_abs_dirs()
  75
  76         # Make sure the upload dir is around.
  77         self.mkdir_p(dirs['output_home'])
  78
  79         for repack_config in config["repackage_config"]:
  80             command = [sys.executable, 'mach', '--log-no-times', 'repackage'] + \
  81                 [arg.format(**dirs)
  82                     for arg in list(repack_config)]
  83             self.run_command(
  84                 command=command,
  85                 cwd=dirs['abs_mozilla_dir'],
  86                 halt_on_failure=True,
  87             )
  88
  89     def _run_tooltool(self):
  90         config = self.config
  91         dirs = self.query_abs_dirs()
  92         manifest_src = os.environ.get('TOOLTOOL_MANIFEST')
  93         if not manifest_src:
  94             manifest_src = config.get('tooltool_manifest_src')
  95         if not manifest_src:
  96             return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
  97
  98         tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
  99                                               manifest_src)
 100         cmd = [
 101             sys.executable, '-u',
 102             os.path.join(dirs['abs_mozilla_dir'], 'mach'),
 103             'artifact',
 104             'toolchain',
 105             '-v',
 106             '--retry', '4',
 107             '--tooltool-manifest',
 108             tooltool_manifest_path,
 109             '--artifact-manifest',
 110             os.path.join(dirs['abs_mozilla_dir'], 'toolchains.json'),
 111             '--tooltool-url',
 112             config['tooltool_url'],
 113         ]
 114         auth_file = self._get_tooltool_auth_file()
 115         if auth_file:
 116             cmd.extend(['--authentication-file', auth_file])
 117         cache = config.get('tooltool_cache')
 118         if cache:
 119             cmd.extend(['--cache-dir', cache])
 120         self.info(str(cmd))
 121         self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
 122
 123     def _get_tooltool_auth_file(self):
 124         # set the default authentication file based on platform; this
 125         # corresponds to where puppet puts the token
 126         if 'tooltool_authentication_file' in self.config:
 127             fn = self.config['tooltool_authentication_file']
 128         elif self._is_windows():
 129             fn = r'c:\builds\relengapi.tok'
 130         else:
 131             fn = '/builds/relengapi.tok'
 132
 133         # if the file doesn't exist, don't pass it to tooltool (it will just
 134         # fail).  In taskcluster, this will work OK as the relengapi-proxy will
 135         # take care of auth.  Everywhere else, we'll get auth failures if
 136         # necessary.
 137         if os.path.exists(fn):
 138             return fn
 139
 140     def _get_mozconfig(self):
 141         """assign mozconfig."""
 142         c = self.config
 143         dirs = self.query_abs_dirs()
 144         abs_mozconfig_path = ''
 145
 146         # first determine the mozconfig path
 147         if c.get('src_mozconfig'):
 148             self.info('Using in-tree mozconfig')
 149             abs_mozconfig_path = os.path.join(dirs['abs_mozilla_dir'], c['src_mozconfig'])
 150         else:
 151             self.fatal("'src_mozconfig' must be in the config "
 152                        "in order to determine the mozconfig.")
 153
 154         # print its contents
 155         self.read_from_file(abs_mozconfig_path, error_level=FATAL)
 156
 157         # finally, copy the mozconfig to a path that 'mach build' expects it to be
 158         self.copyfile(abs_mozconfig_path, os.path.join(dirs['abs_mozilla_dir'], '.mozconfig'))
 159
 160     def _run_configure(self):
 161         dirs = self.query_abs_dirs()
 162         command = [sys.executable, 'mach', '--log-no-times', 'configure']
 163         return self.run_command(
 164             command=command,
 165             cwd=dirs['abs_mozilla_dir'],
 166             output_timeout=60*3,
 167             halt_on_failure=True,
 168         )
 169
 170
 171 if __name__ == '__main__':
 172     repack = Repackage()
 173     repack.run_and_exit()