tools/telemetry/catapult_base/dependency_manager/dependency_manager.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import stat
   8
   9 from catapult_base import cloud_storage
  10 from catapult_base import support_binaries
  11 from catapult_base.dependency_manager import base_config
  12 from catapult_base.dependency_manager import exceptions
  13
  14
  15 DEFAULT_TYPE = 'default'
  16
  17
  18 class DependencyManager(object):
  19   def __init__(self, configs, supported_config_types=None):
  20     """Manages file dependencies found locally or in cloud_storage.
  21
  22     Args:
  23         configs: A list of instances of BaseConfig or it's subclasses, passed
  24             in decreasing order of precedence.
  25         supported_config_types: A list of whitelisted config_types.
  26             No restrictions if None is specified.
  27
  28     Raises:
  29         ValueError: If |configs| is not a list of instances of BaseConfig or
  30             its subclasses.
  31         UnsupportedConfigFormatError: If supported_config_types is specified and
  32             configs contains a config not in the supported config_types.
  33
  34     Example: DependencyManager([config1, config2, config3])
  35         No requirements on the type of Config, and any dependencies that have
  36         local files for the same platform will first look in those from
  37         config1, then those from config2, and finally those from config3.
  38     """
  39     if configs is None or type(configs) != list:
  40       raise ValueError(
  41           'Must supply a list of config files to DependencyManager')
  42     # self._lookup_dict is a dictionary with the following format:
  43     # { dependency1: {platform1: dependency_info1,
  44     #                 platform2: dependency_info2}
  45     #   dependency2: {platform1: dependency_info3,
  46     #                  ...}
  47     #   ...}
  48     #
  49     # Where the dependencies and platforms are strings, and the
  50     # dependency_info's are DependencyInfo instances.
  51     self._lookup_dict = {}
  52     self.supported_configs = supported_config_types or []
  53     for config in configs:
  54       self._UpdateDependencies(config)
  55
  56   def FetchPath(self, dependency, platform, try_support_binaries=False):
  57     """Get a path to an executable for |dependency|, downloading as needed.
  58
  59     A path to a default executable may be returned if a platform specific
  60     version is not specified in the config(s).
  61
  62     Args:
  63         dependency: Name of the desired dependency, as given in the config(s)
  64             used in this DependencyManager.
  65         platform: Name of the platform the dependency will run on. Often of the
  66             form 'os_architecture'. Must match those specified in the config(s)
  67             used in this DependencyManager.
  68         try_support_binaries: True if support_binaries should be queried if the
  69             dependency_manager was not initialized with data for |dependency|.
  70
  71     Returns:
  72         A path to an executable of |dependency| that will run on |platform|,
  73         downloading from cloud storage if needed.
  74
  75     Raises:
  76         NoPathFoundError: If a local copy of the executable cannot be found and
  77             a remote path could not be downloaded from cloud_storage.
  78         CredentialsError: If cloud_storage credentials aren't configured.
  79         PermissionError: If cloud_storage credentials are configured, but not
  80             with an account that has permission to download the remote file.
  81         NotFoundError: If the remote file does not exist where expected in
  82             cloud_storage.
  83         ServerError: If an internal server error is hit while downloading the
  84             remote file.
  85         CloudStorageError: If another error occured while downloading the remote
  86             path.
  87         FileNotFoundError: If an attempted download was otherwise unsuccessful.
  88
  89     """
  90     dependency_info = self._GetDependencyInfo(dependency, platform)
  91     if not dependency_info:
  92       logging.error(
  93           'The dependency_manager was not initialized with the dependency.')
  94       if not try_support_binaries:
  95         raise exceptions.NoPathFoundError(dependency, platform)
  96       # TODO(aiolos): Remove the support_binaries call and always raise
  97       # NoPathFound once the binary dependencies are moved over to the new
  98       # system.
  99
 100       # platform should be of the form '%s_%s' % (os_name, arch_name) when
 101       # called from the binary_manager.
 102       platform_parts = platform.split('_', 1)
 103       assert len(platform_parts) == 2
 104       platform_os, platform_arch = platform_parts
 105       logging.info('Calling into support_binaries with dependency %s, platform '
 106                    '%s and arch %s.' % (dependency, platform_os,
 107                                         platform_arch))
 108       return support_binaries.FindPath(dependency, platform_arch,
 109                                        platform_os)
 110     path = self._LocalPath(dependency_info)
 111     if not path or not os.path.exists(path):
 112       path = self._CloudStoragePath(dependency_info)
 113       if not path or not os.path.exists(path):
 114         raise exceptions.NoPathFoundError(dependency, platform)
 115     return path
 116
 117   def LocalPath(self, dependency, platform, try_support_binaries=False):
 118     """Get a path to a locally stored executable for |dependency|.
 119
 120     A path to a default executable may be returned if a platform specific
 121     version is not specified in the config(s).
 122     Will not download the executable.
 123
 124     Args:
 125         dependency: Name of the desired dependency, as given in the config(s)
 126             used in this DependencyManager.
 127         platform: Name of the platform the dependency will run on. Often of the
 128             form 'os_architecture'. Must match those specified in the config(s)
 129             used in this DependencyManager.
 130         try_support_binaries: True if support_binaries should be queried if the
 131             dependency_manager was not initialized with data for |dependency|.
 132
 133     Returns:
 134         A path to an executable for |dependency| that will run on |platform|.
 135
 136     Raises:
 137         NoPathFoundError: If a local copy of the executable cannot be found.
 138     """
 139     # TODO(aiolos): Remove the support_binaries call and always raise
 140     # NoPathFound once the binary dependencies are moved over to the new
 141     # system.
 142     dependency_info = self._GetDependencyInfo(dependency, platform)
 143     if not dependency_info:
 144       logging.error(
 145           'The dependency_manager was not initialized with the dependency.')
 146       if not try_support_binaries:
 147         raise exceptions.NoPathFoundError(dependency, platform)
 148       return support_binaries.FindLocallyBuiltPath(dependency)
 149     local_path = self._LocalPath(dependency_info)
 150     if not local_path or not os.path.exists(local_path):
 151       raise exceptions.NoPathFoundError(dependency, platform)
 152     return local_path
 153
 154   def _UpdateDependencies(self, config):
 155     """Add the dependency information stored in |config| to this instance.
 156
 157     Args:
 158         config: An instances of BaseConfig or a subclasses.
 159
 160     Raises:
 161         UnsupportedConfigFormatError: If supported_config_types was specified
 162         and config is not in the supported config_types.
 163     """
 164     if not isinstance(config, base_config.BaseConfig):
 165       raise ValueError('Must use a BaseConfig or subclass instance with the '
 166                        'DependencyManager.')
 167     if (self.supported_configs and
 168         config.GetConfigType() not in self.supported_configs):
 169       raise exceptions.UnsupportedConfigFormatError(config.GetConfigType(),
 170                                                     config.config_path)
 171     for dep_info in config.IterDependencyInfo():
 172       dependency = dep_info.dependency
 173       platform = dep_info.platform
 174       if dependency not in self._lookup_dict:
 175         self._lookup_dict[dependency] = {}
 176       if platform not in self._lookup_dict[dependency]:
 177         self._lookup_dict[dependency][platform] = dep_info
 178       else:
 179         self._lookup_dict[dependency][platform].Update(dep_info)
 180
 181
 182   def _GetDependencyInfo(self, dependency, platform):
 183     """Get information for |dependency| on |platform|, or a default if needed.
 184
 185     Args:
 186         dependency: Name of the desired dependency, as given in the config(s)
 187             used in this DependencyManager.
 188         platform: Name of the platform the dependency will run on. Often of the
 189             form 'os_architecture'. Must match those specified in the config(s)
 190             used in this DependencyManager.
 191
 192     Returns: The dependency_info for |dependency| on |platform| if it exists.
 193         Or the default version of |dependency| if it exists, or None if neither
 194         exist.
 195     """
 196     if not self._lookup_dict or dependency not in self._lookup_dict:
 197       return None
 198     dependency_dict = self._lookup_dict[dependency]
 199     device_type = platform
 200     if not device_type in dependency_dict:
 201       device_type = DEFAULT_TYPE
 202     return dependency_dict.get(device_type)
 203
 204   @staticmethod
 205   def _LocalPath(dependency_info):
 206     """Return a path to a locally stored file for |dependency_info|.
 207
 208     Will not download the file.
 209
 210     Args:
 211         dependency_info: A DependencyInfo instance for the dependency to be
 212             found and the platform it should run on.
 213
 214     Returns: A path to a local file, or None if not found.
 215     """
 216     if dependency_info:
 217       paths = dependency_info.local_paths
 218       for local_path in paths:
 219         if os.path.exists(local_path):
 220           return local_path
 221     return None
 222
 223   @staticmethod
 224   def _CloudStoragePath(dependency_info):
 225     """Return a path to a downloaded file for |dependency_info|.
 226
 227     May not download the file if it has already been downloaded.
 228
 229     Args:
 230         dependency_info: A DependencyInfo instance for the dependency to be
 231             found and the platform it should run on.
 232
 233     Returns: A path to an executable that was stored in cloud_storage, or None
 234        if not found.
 235
 236     Raises:
 237         CredentialsError: If cloud_storage credentials aren't configured.
 238         PermissionError: If cloud_storage credentials are configured, but not
 239             with an account that has permission to download the needed file.
 240         NotFoundError: If the needed file does not exist where expected in
 241             cloud_storage.
 242         ServerError: If an internal server error is hit while downloading the
 243             needed file.
 244         CloudStorageError: If another error occured while downloading the remote
 245             path.
 246         FileNotFoundError: If the download was otherwise unsuccessful.
 247     """
 248     if not dependency_info:
 249       return None
 250     cs_path = dependency_info.cs_remote_path
 251     cs_hash = dependency_info.cs_hash
 252     cs_bucket = dependency_info.cs_bucket
 253     download_path = dependency_info.download_path
 254     if not cs_path or not cs_bucket or not cs_hash or not download_path:
 255       return None
 256
 257     download_dir = os.path.dirname(download_path)
 258     if not os.path.exists(download_dir):
 259       os.makedirs(download_dir)
 260
 261     cloud_storage.GetIfHashChanged(cs_path, download_path, cs_bucket, cs_hash)
 262     if not os.path.exists(download_path):
 263       raise exceptions.FileNotFoundError(download_path)
 264     #TODO(aiolos): Add support for unzipping files.
 265     os.chmod(download_path,
 266              stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP)
 267     return os.path.abspath(download_path)
 268