testing/mozharness/scripts/spidermonkey_build.py

   1 #!/usr/bin/env python
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import os
   7 import sys
   8 import copy
   9 from datetime import datetime
  10 from functools import wraps
  11
  12 sys.path.insert(1, os.path.dirname(sys.path[0]))
  13
  14 from mozharness.base.errors import MakefileErrorList
  15 from mozharness.base.script import BaseScript
  16 from mozharness.base.transfer import TransferMixin
  17 from mozharness.base.vcs.vcsbase import VCSMixin
  18 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
  19 from mozharness.mozilla.buildbot import BuildbotMixin
  20 from mozharness.mozilla.building.hazards import HazardError, HazardAnalysis
  21 from mozharness.mozilla.purge import PurgeMixin
  22 from mozharness.mozilla.mock import MockMixin
  23 from mozharness.mozilla.tooltool import TooltoolMixin
  24
  25 SUCCESS, WARNINGS, FAILURE, EXCEPTION, RETRY = xrange(5)
  26
  27
  28 def requires(*queries):
  29     """Wrapper for detecting problems where some bit of information
  30     required by the wrapped step is unavailable. Use it put prepending
  31     @requires("foo"), which will check whether self.query_foo() returns
  32     something useful."""
  33     def make_wrapper(f):
  34         @wraps(f)
  35         def wrapper(self, *args, **kwargs):
  36             for query in queries:
  37                 val = query(self)
  38                 goodval = not (val is None or "None" in str(val))
  39                 assert goodval, f.__name__ + " requires " + query.__name__ + " to return a value"
  40             return f(self, *args, **kwargs)
  41         return wrapper
  42     return make_wrapper
  43
  44
  45 nuisance_env_vars = ['TERMCAP', 'LS_COLORS', 'PWD', '_']
  46
  47
  48 class SpidermonkeyBuild(MockMixin,
  49                         PurgeMixin, BaseScript,
  50                         VCSMixin, BuildbotMixin, TooltoolMixin, TransferMixin, BlobUploadMixin):
  51     config_options = [
  52         [["--repo"], {
  53             "dest": "repo",
  54             "help": "which gecko repo to get spidermonkey from",
  55         }],
  56         [["--source"], {
  57             "dest": "source",
  58             "help": "directory containing gecko source tree (instead of --repo)",
  59         }],
  60         [["--revision"], {
  61             "dest": "revision",
  62         }],
  63         [["--branch"], {
  64             "dest": "branch",
  65         }],
  66         [["--vcs-share-base"], {
  67             "dest": "vcs_share_base",
  68             "help": "base directory for shared repositories",
  69         }],
  70         [["-j"], {
  71             "dest": "concurrency",
  72             "type": int,
  73             "default": 4,
  74             "help": "number of simultaneous jobs used while building the shell " +
  75                     "(currently ignored for the analyzed build",
  76         }] + copy.deepcopy(blobupload_config_options)
  77     ]
  78
  79     def __init__(self):
  80         super(SpidermonkeyBuild, self).__init__(
  81             config_options=self.config_options,
  82             # other stuff
  83             all_actions=[
  84                 'purge',
  85                 'checkout-tools',
  86
  87                 # First, build an optimized JS shell for running the analysis
  88                 'checkout-source',
  89                 'get-blobs',
  90                 'clobber-shell',
  91                 'configure-shell',
  92                 'build-shell',
  93
  94                 # Next, build a tree with the analysis plugin active. Note that
  95                 # we are using the same checkout for the JS shell build and the
  96                 # build of the source to be analyzed, which is a little
  97                 # unnecessary (no need to rebuild the JS shell all the time).
  98                 # (Different objdir, though.)
  99
 100                 'clobber-analysis',
 101                 'setup-analysis',
 102                 'run-analysis',
 103                 'collect-analysis-output',
 104                 'upload-analysis',
 105                 'check-expectations',
 106             ],
 107             default_actions=[
 108                 'purge',
 109                 'checkout-tools',
 110                 'checkout-source',
 111                 'get-blobs',
 112                 'clobber-shell',
 113                 'configure-shell',
 114                 'build-shell',
 115                 'clobber-analysis',
 116                 'setup-analysis',
 117                 'run-analysis',
 118                 'collect-analysis-output',
 119                 # Temporarily disabled, see bug 1211402
 120                 # 'upload-analysis',
 121                 'check-expectations',
 122             ],
 123             config={
 124                 'default_vcs': 'hg',
 125                 'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
 126                 'ccache': True,
 127                 'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
 128                 'tools_repo': 'https://hg.mozilla.org/build/tools',
 129
 130                 'upload_ssh_server': None,
 131                 'upload_remote_basepath': None,
 132                 'enable_try_uploads': True,
 133                 'source': None,
 134                 'stage_product': 'firefox',
 135             },
 136         )
 137
 138         self.buildid = None
 139         self.create_virtualenv()
 140         self.analysis = HazardAnalysis()
 141
 142     def _pre_config_lock(self, rw_config):
 143         if self.config['source']:
 144             self.config['srcdir'] = self.config['source']
 145         super(SpidermonkeyBuild, self)._pre_config_lock(rw_config)
 146
 147         if self.buildbot_config is None:
 148             self.info("Reading buildbot build properties...")
 149             self.read_buildbot_config()
 150
 151         if self.buildbot_config:
 152             bb_props = [('mock_target', 'mock_target', None),
 153                         ('hgurl', 'hgurl', None),
 154                         ('clobberer_url', 'clobberer_url', 'https://api.pub.build.mozilla.org/clobberer/lastclobber'),
 155                         ('force_clobber', 'force_clobber', None),
 156                         ('branch', 'blob_upload_branch', None),
 157                         ]
 158             buildbot_props = self.buildbot_config.get('properties', {})
 159             for bb_prop, cfg_prop, default in bb_props:
 160                 if not self.config.get(cfg_prop) and buildbot_props.get(bb_prop, default):
 161                     self.config[cfg_prop] = buildbot_props.get(bb_prop, default)
 162             self.config['is_automation'] = True
 163         else:
 164             self.config['is_automation'] = False
 165             self.config.setdefault('blob_upload_branch', 'devel')
 166
 167         dirs = self.query_abs_dirs()
 168         replacements = self.config['env_replacements'].copy()
 169         for k,v in replacements.items():
 170             replacements[k] = v % dirs
 171
 172         self.env = self.query_env(replace_dict=replacements,
 173                                   partial_env=self.config['partial_env'],
 174                                   purge_env=nuisance_env_vars)
 175         self.env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
 176         self.env['TOOLTOOL_DIR'] = dirs['abs_work_dir']
 177
 178     def query_abs_dirs(self):
 179         if self.abs_dirs:
 180             return self.abs_dirs
 181         abs_dirs = BaseScript.query_abs_dirs(self)
 182
 183         abs_work_dir = abs_dirs['abs_work_dir']
 184         dirs = {
 185             'shell_objdir':
 186                 os.path.join(abs_work_dir, self.config['shell-objdir']),
 187             'mozharness_scriptdir':
 188                 os.path.abspath(os.path.dirname(__file__)),
 189             'abs_analysis_dir':
 190                 os.path.join(abs_work_dir, self.config['analysis-dir']),
 191             'abs_analyzed_objdir':
 192                 os.path.join(abs_work_dir, self.config['srcdir'], self.config['analysis-objdir']),
 193             'analysis_scriptdir':
 194                 os.path.join(self.config['srcdir'], self.config['analysis-scriptdir']),
 195             'abs_tools_dir':
 196                 os.path.join(abs_dirs['base_work_dir'], 'tools'),
 197             'gecko_src':
 198                 os.path.join(abs_work_dir, self.config['srcdir']),
 199             'abs_blob_upload_dir':
 200                 os.path.join(abs_work_dir, 'blobber_upload_dir'),
 201         }
 202
 203         abs_dirs.update(dirs)
 204         self.abs_dirs = abs_dirs
 205
 206         return self.abs_dirs
 207
 208     def query_repo(self):
 209         if self.config.get('repo'):
 210             return self.config['repo']
 211         elif self.buildbot_config and 'properties' in self.buildbot_config:
 212             return self.config['hgurl'] + self.buildbot_config['properties']['repo_path']
 213         else:
 214             return None
 215
 216     def query_revision(self):
 217         if 'revision' in self.buildbot_properties:
 218             revision = self.buildbot_properties['revision']
 219         elif self.buildbot_config and 'sourcestamp' in self.buildbot_config:
 220             revision = self.buildbot_config['sourcestamp']['revision']
 221         else:
 222             # Useful for local testing. In actual use, this would always be
 223             # None.
 224             revision = self.config.get('revision')
 225
 226         return revision
 227
 228     def query_branch(self):
 229         if self.buildbot_config and 'properties' in self.buildbot_config:
 230             return self.buildbot_config['properties']['branch']
 231         elif 'branch' in self.config:
 232             # Used for locally testing try vs non-try
 233             return self.config['branch']
 234         else:
 235             return os.path.basename(self.query_repo())
 236
 237     def query_compiler_manifest(self):
 238         dirs = self.query_abs_dirs()
 239         manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['compiler_manifest'])
 240         if os.path.exists(manifest):
 241             return manifest
 242         return os.path.join(dirs['abs_work_dir'], self.config['compiler_manifest'])
 243
 244     def query_sixgill_manifest(self):
 245         dirs = self.query_abs_dirs()
 246         manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['sixgill_manifest'])
 247         if os.path.exists(manifest):
 248             return manifest
 249         return os.path.join(dirs['abs_work_dir'], self.config['sixgill_manifest'])
 250
 251     def query_buildid(self):
 252         if self.buildid:
 253             return self.buildid
 254         if self.buildbot_config and 'properties' in self.buildbot_config:
 255             self.buildid = self.buildbot_config['properties'].get('buildid')
 256         if not self.buildid:
 257             self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
 258         return self.buildid
 259
 260     def query_upload_ssh_server(self):
 261         if self.buildbot_config and 'properties' in self.buildbot_config:
 262             return self.buildbot_config['properties']['upload_ssh_server']
 263         else:
 264             return self.config['upload_ssh_server']
 265
 266     def query_upload_ssh_key(self):
 267         if self.buildbot_config and 'properties' in self.buildbot_config:
 268             key = self.buildbot_config['properties']['upload_ssh_key']
 269         else:
 270             key = self.config['upload_ssh_key']
 271         if self.mock_enabled and not key.startswith("/"):
 272             key = "/home/mock_mozilla/.ssh/" + key
 273         return key
 274
 275     def query_upload_ssh_user(self):
 276         if self.buildbot_config and 'properties' in self.buildbot_config:
 277             return self.buildbot_config['properties']['upload_ssh_user']
 278         else:
 279             return self.config['upload_ssh_user']
 280
 281     def query_product(self):
 282         if self.buildbot_config and 'properties' in self.buildbot_config:
 283             return self.buildbot_config['properties']['product']
 284         else:
 285             return self.config['product']
 286
 287     def query_upload_remote_basepath(self):
 288         if self.config.get('upload_remote_basepath'):
 289             return self.config['upload_remote_basepath']
 290         else:
 291             return "/pub/mozilla.org/{product}".format(
 292                 product=self.query_product(),
 293             )
 294
 295     def query_upload_remote_baseuri(self):
 296         baseuri = self.config.get('upload_remote_baseuri')
 297         if self.buildbot_config and 'properties' in self.buildbot_config:
 298             buildprops = self.buildbot_config['properties']
 299             if 'upload_remote_baseuri' in buildprops:
 300                 baseuri = buildprops['upload_remote_baseuri']
 301         return baseuri.strip("/") if baseuri else None
 302
 303     def query_target(self):
 304         if self.buildbot_config and 'properties' in self.buildbot_config:
 305             return self.buildbot_config['properties']['platform']
 306         else:
 307             return self.config.get('target')
 308
 309     def query_upload_path(self):
 310         branch = self.query_branch()
 311
 312         common = {
 313             'basepath': self.query_upload_remote_basepath(),
 314             'branch': branch,
 315             'target': self.query_target(),
 316         }
 317
 318         if branch == 'try':
 319             if not self.config['enable_try_uploads']:
 320                 return None
 321             try:
 322                 user = self.buildbot_config['sourcestamp']['changes'][0]['who']
 323             except (KeyError, TypeError):
 324                 user = "unknown"
 325             return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
 326                 user=user,
 327                 rev=self.query_revision(),
 328                 **common
 329             )
 330         else:
 331             return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
 332                 buildid=self.query_buildid(),
 333                 **common
 334             )
 335
 336     def query_do_upload(self):
 337         if self.query_branch() == 'try':
 338             return self.config.get('enable_try_uploads')
 339         return True
 340
 341     # Actions {{{2
 342     def purge(self):
 343         dirs = self.query_abs_dirs()
 344         self.info("purging, abs_upload_dir=" + dirs['abs_upload_dir'])
 345         PurgeMixin.clobber(
 346             self,
 347             always_clobber_dirs=[
 348                 dirs['abs_upload_dir'],
 349             ],
 350         )
 351
 352     def checkout_tools(self):
 353         dirs = self.query_abs_dirs()
 354
 355         # If running from within a directory also passed as the --source dir,
 356         # this has the danger of clobbering <source>/tools/
 357         if self.config['source']:
 358             srcdir = self.config['source']
 359             if os.path.samefile(srcdir, os.path.dirname(dirs['abs_tools_dir'])):
 360                 raise Exception("Cannot run from source checkout to avoid overwriting subdirs")
 361
 362         rev = self.vcs_checkout(
 363             vcs='hg',
 364             branch="default",
 365             repo=self.config['tools_repo'],
 366             clean=False,
 367             dest=dirs['abs_tools_dir'],
 368         )
 369         self.set_buildbot_property("tools_revision", rev, write_to_file=True)
 370
 371     def do_checkout_source(self):
 372         # --source option means to use an existing source directory instead of checking one out.
 373         if self.config['source']:
 374             return
 375
 376         dirs = self.query_abs_dirs()
 377         dest = dirs['gecko_src']
 378
 379         # Pre-create the directory to appease the share extension
 380         if not os.path.exists(dest):
 381             self.mkdir_p(dest)
 382
 383         rev = self.vcs_checkout(
 384             repo=self.query_repo(),
 385             dest=dest,
 386             revision=self.query_revision(),
 387             branch=self.config.get('branch'),
 388             clean=True,
 389         )
 390         self.set_buildbot_property('source_revision', rev, write_to_file=True)
 391
 392     def checkout_source(self):
 393         try:
 394             self.do_checkout_source()
 395         except Exception as e:
 396             self.fatal("checkout failed: " + str(e), exit_code=RETRY)
 397
 398     def get_blobs(self):
 399         work_dir = self.query_abs_dirs()['abs_work_dir']
 400         if not os.path.exists(work_dir):
 401             self.mkdir_p(work_dir)
 402         self.tooltool_fetch(self.query_compiler_manifest(), output_dir=work_dir)
 403         self.tooltool_fetch(self.query_sixgill_manifest(), output_dir=work_dir)
 404
 405     def clobber_shell(self):
 406         self.analysis.clobber_shell(self)
 407
 408     def configure_shell(self):
 409         self.enable_mock()
 410
 411         try:
 412             self.analysis.configure_shell(self)
 413         except HazardError as e:
 414             self.fatal(e, exit_code=FAILURE)
 415
 416         self.disable_mock()
 417
 418     def build_shell(self):
 419         self.enable_mock()
 420
 421         try:
 422             self.analysis.build_shell(self)
 423         except HazardError as e:
 424             self.fatal(e, exit_code=FAILURE)
 425
 426         self.disable_mock()
 427
 428     def clobber_analysis(self):
 429         self.analysis.clobber(self)
 430
 431     def setup_analysis(self):
 432         self.analysis.setup(self)
 433
 434     def run_analysis(self):
 435         self.enable_mock()
 436
 437         upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
 438         if not os.path.exists(upload_dir):
 439             self.mkdir_p(upload_dir)
 440
 441         env = self.env.copy()
 442         env['MOZ_UPLOAD_DIR'] = upload_dir
 443
 444         try:
 445             self.analysis.run(self, env=env, error_list=MakefileErrorList)
 446         except HazardError as e:
 447             self.fatal(e, exit_code=FAILURE)
 448
 449         self.disable_mock()
 450
 451     def collect_analysis_output(self):
 452         self.analysis.collect_output(self)
 453
 454     def upload_analysis(self):
 455         if not self.config['is_automation']:
 456             return
 457
 458         if not self.query_do_upload():
 459             self.info("Uploads disabled for this build. Skipping...")
 460             return
 461
 462         self.enable_mock()
 463
 464         try:
 465             self.analysis.upload_results(self)
 466         except HazardError as e:
 467             self.error(e)
 468             self.return_code = WARNINGS
 469
 470         self.disable_mock()
 471
 472     def check_expectations(self):
 473         try:
 474             self.analysis.check_expectations(self)
 475         except HazardError as e:
 476             self.fatal(e, exit_code=FAILURE)
 477
 478
 479 # main {{{1
 480 if __name__ == '__main__':
 481     myScript = SpidermonkeyBuild()
 482     myScript.run_and_exit()