taskcluster/taskgraph/parameters.py

   1 # -*- coding: utf-8 -*-
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 from __future__ import absolute_import, print_function, unicode_literals
   8
   9 import os.path
  10 import json
  11 from datetime import datetime
  12
  13 from mozbuild.util import ReadOnlyDict, memoize
  14 from mozversioncontrol import get_repository_object
  15 from taskgraph.util.schema import validate_schema
  16 from voluptuous import (
  17     ALLOW_EXTRA,
  18     Any,
  19     Inclusive,
  20     PREVENT_EXTRA,
  21     Required,
  22     Schema,
  23 )
  24
  25 from . import GECKO
  26 from .util.attributes import release_level
  27
  28
  29 class ParameterMismatch(Exception):
  30     """Raised when a parameters.yml has extra or missing parameters."""
  31
  32
  33 @memoize
  34 def get_head_ref():
  35     return get_repository_object(GECKO).head_ref
  36
  37
  38 def get_contents(path):
  39     with open(path, "r") as fh:
  40         contents = fh.readline().rstrip()
  41     return contents
  42
  43
  44 def get_version(product_dir='browser'):
  45     version_path = os.path.join(GECKO, product_dir, 'config',
  46                                 'version_display.txt')
  47     return get_contents(version_path)
  48
  49
  50 def get_app_version(product_dir='browser'):
  51     app_version_path = os.path.join(GECKO, product_dir, 'config',
  52                                     'version.txt')
  53     return get_contents(app_version_path)
  54
  55
  56 base_schema = {
  57     Required('app_version'): basestring,
  58     Required('base_repository'): basestring,
  59     Required('build_date'): int,
  60     Required('build_number'): int,
  61     Inclusive('comm_base_repository', 'comm'): basestring,
  62     Inclusive('comm_head_ref', 'comm'): basestring,
  63     Inclusive('comm_head_repository', 'comm'): basestring,
  64     Inclusive('comm_head_rev', 'comm'): basestring,
  65     Required('do_not_optimize'): [basestring],
  66     Required('existing_tasks'): {basestring: basestring},
  67     Required('filters'): [basestring],
  68     Required('head_ref'): basestring,
  69     Required('head_repository'): basestring,
  70     Required('head_rev'): basestring,
  71     Required('hg_branch'): basestring,
  72     Required('level'): basestring,
  73     Required('message'): basestring,
  74     Required('moz_build_date'): basestring,
  75     Required('next_version'): Any(None, basestring),
  76     Required('optimize_target_tasks'): bool,
  77     Required('owner'): basestring,
  78     Required('phabricator_diff'): Any(None, basestring),
  79     Required('project'): basestring,
  80     Required('pushdate'): int,
  81     Required('pushlog_id'): basestring,
  82     Required('release_enable_emefree'): bool,
  83     Required('release_enable_partners'): bool,
  84     Required('release_eta'): Any(None, basestring),
  85     Required('release_history'): {basestring: dict},
  86     Required('release_partners'): Any(None, [basestring]),
  87     Required('release_partner_config'): Any(None, dict),
  88     Required('release_partner_build_number'): int,
  89     Required('release_type'): basestring,
  90     Required('release_product'): Any(None, basestring),
  91     Required('required_signoffs'): [basestring],
  92     Required('signoff_urls'): dict,
  93     Required('target_tasks_method'): basestring,
  94     Required('tasks_for'): basestring,
  95     Required('try_mode'): Any(None, basestring),
  96     Required('try_options'): Any(None, dict),
  97     Required('try_task_config'): dict,
  98     Required('version'): basestring,
  99 }
 100
 101
 102 COMM_PARAMETERS = [
 103     'comm_base_repository',
 104     'comm_head_ref',
 105     'comm_head_repository',
 106     'comm_head_rev',
 107 ]
 108
 109
 110 class Parameters(ReadOnlyDict):
 111     """An immutable dictionary with nicer KeyError messages on failure"""
 112
 113     def __init__(self, strict=True, **kwargs):
 114         self.strict = strict
 115
 116         if not self.strict:
 117             # apply defaults to missing parameters
 118             kwargs = Parameters._fill_defaults(**kwargs)
 119
 120         ReadOnlyDict.__init__(self, **kwargs)
 121
 122     @staticmethod
 123     def _fill_defaults(**kwargs):
 124         now = datetime.utcnow()
 125         epoch = datetime.utcfromtimestamp(0)
 126         seconds_from_epoch = int((now - epoch).total_seconds())
 127
 128         defaults = {
 129             'app_version': get_app_version(),
 130             'base_repository': 'https://hg.mozilla.org/mozilla-unified',
 131             'build_date': seconds_from_epoch,
 132             'build_number': 1,
 133             'do_not_optimize': [],
 134             'existing_tasks': {},
 135             'filters': ['target_tasks_method'],
 136             'head_ref': get_head_ref(),
 137             'head_repository': 'https://hg.mozilla.org/mozilla-central',
 138             'head_rev': get_head_ref(),
 139             'hg_branch': 'default',
 140             'level': '3',
 141             'message': '',
 142             'moz_build_date': now.strftime("%Y%m%d%H%M%S"),
 143             'next_version': None,
 144             'optimize_target_tasks': True,
 145             'owner': 'nobody@mozilla.com',
 146             'phabricator_diff': None,
 147             'project': 'mozilla-central',
 148             'pushdate': seconds_from_epoch,
 149             'pushlog_id': '0',
 150             'release_enable_emefree': False,
 151             'release_enable_partners': False,
 152             'release_eta': '',
 153             'release_history': {},
 154             'release_partners': [],
 155             'release_partner_config': None,
 156             'release_partner_build_number': 1,
 157             'release_product': None,
 158             'release_type': 'nightly',
 159             'required_signoffs': [],
 160             'signoff_urls': {},
 161             'target_tasks_method': 'default',
 162             'tasks_for': 'hg-push',
 163             'try_mode': None,
 164             'try_options': None,
 165             'try_task_config': {},
 166             'version': get_version(),
 167         }
 168
 169         if set(COMM_PARAMETERS) & set(kwargs):
 170             defaults.update({
 171                 'comm_base_repository': 'https://hg.mozilla.org/comm-central',
 172                 'comm_head_repository': 'https://hg.mozilla.org/comm-central',
 173             })
 174
 175         for name, default in defaults.items():
 176             if name not in kwargs:
 177                 kwargs[name] = default
 178
 179         return kwargs
 180
 181     def check(self):
 182         schema = Schema(base_schema, extra=PREVENT_EXTRA if self.strict else ALLOW_EXTRA)
 183         validate_schema(schema, self.copy(), 'Invalid parameters:')
 184
 185     def __getitem__(self, k):
 186         try:
 187             return super(Parameters, self).__getitem__(k)
 188         except KeyError:
 189             raise KeyError("taskgraph parameter {!r} not found".format(k))
 190
 191     def is_try(self):
 192         """
 193         Determine whether this graph is being built on a try project or for
 194         `mach try fuzzy`.
 195         """
 196         return 'try' in self['project'] or self['try_mode'] == 'try_select'
 197
 198     def file_url(self, path, pretty=False):
 199         """
 200         Determine the VCS URL for viewing a file in the tree, suitable for
 201         viewing by a human.
 202
 203         :param basestring path: The path, relative to the root of the repository.
 204         :param bool pretty: Whether to return a link to a formatted version of the
 205             file, or the raw file version.
 206         :return basestring: The URL displaying the given path.
 207         """
 208         if path.startswith('comm/'):
 209             path = path[len('comm/'):]
 210             repo = self['comm_head_repository']
 211             rev = self['comm_head_rev']
 212         else:
 213             repo = self['head_repository']
 214             rev = self['head_rev']
 215
 216         endpoint = 'file' if pretty else 'raw-file'
 217         return '{}/{}/{}/{}'.format(repo, endpoint, rev, path)
 218
 219     def release_level(self):
 220         """
 221         Whether this is a staging release or not.
 222
 223         :return six.text_type: One of "production" or "staging".
 224         """
 225         return release_level(self['project'])
 226
 227
 228 def load_parameters_file(filename, strict=True, overrides=None, trust_domain=None):
 229     """
 230     Load parameters from a path, url, decision task-id or project.
 231
 232     Examples:
 233         task-id=fdtgsD5DQUmAQZEaGMvQ4Q
 234         project=mozilla-central
 235     """
 236     import urllib
 237     from taskgraph.util.taskcluster import get_artifact_url, find_task_id
 238     from taskgraph.util import yaml
 239
 240     if overrides is None:
 241         overrides = {}
 242
 243     if not filename:
 244         return Parameters(strict=strict, **overrides)
 245
 246     try:
 247         # reading parameters from a local parameters.yml file
 248         f = open(filename)
 249     except IOError:
 250         # fetching parameters.yml using task task-id, project or supplied url
 251         task_id = None
 252         if filename.startswith("task-id="):
 253             task_id = filename.split("=")[1]
 254         elif filename.startswith("project="):
 255             if trust_domain is None:
 256                 raise ValueError(
 257                     "Can't specify parameters by project "
 258                     "if trust domain isn't supplied.",
 259                 )
 260             index = "{trust_domain}.v2.{project}.latest.taskgraph.decision".format(
 261                 trust_domain=trust_domain,
 262                 project=filename.split("=")[1],
 263             )
 264             task_id = find_task_id(index)
 265
 266         if task_id:
 267             filename = get_artifact_url(task_id, 'public/parameters.yml')
 268         f = urllib.urlopen(filename)
 269
 270     if filename.endswith('.yml'):
 271         kwargs = yaml.load_stream(f)
 272     elif filename.endswith('.json'):
 273         kwargs = json.load(f)
 274     else:
 275         raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
 276
 277     kwargs.update(overrides)
 278
 279     return Parameters(strict=strict, **kwargs)
 280
 281
 282 def parameters_loader(filename, strict=True, overrides=None):
 283     def get_parameters(graph_config):
 284         parameters = load_parameters_file(
 285             filename,
 286             strict=strict,
 287             overrides=overrides,
 288             trust_domain=graph_config["trust-domain"],
 289         )
 290         parameters.check()
 291         return parameters
 292     return get_parameters