tools/findit/chromium_deps.py

   1 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import base64
   6 import json
   7 import os
   8 import re
   9 import time
  10 import urllib2
  11
  12 from common import utils
  13
  14
  15 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  16 CONFIG = json.loads(open(os.path.join(_THIS_DIR,
  17                                       'deps_config.json'), 'r').read())
  18 OLD_GIT_URL_PATTERN = re.compile(r'https?://git.chromium.org/(.*)')
  19
  20
  21 class _VarImpl(object):
  22
  23   def __init__(self, local_scope):
  24     self._local_scope = local_scope
  25
  26   def Lookup(self, var_name):
  27     if var_name in self._local_scope.get('vars', {}):
  28       return self._local_scope['vars'][var_name]
  29     raise Exception('Var is not defined: %s' % var_name)
  30
  31
  32 def _ParseDEPS(content):
  33   """Parse the DEPS file of chromium."""
  34   local_scope = {}
  35   var = _VarImpl(local_scope)
  36   global_scope = {
  37       'Var': var.Lookup,
  38       'deps': {},
  39       'deps_os': {},
  40       'include_rules': [],
  41       'skip_child_includes': [],
  42       'hooks': [],
  43   }
  44   exec(content, global_scope, local_scope)
  45
  46   local_scope.setdefault('deps', {})
  47   local_scope.setdefault('deps_os', {})
  48
  49   return (local_scope['deps'], local_scope['deps_os'])
  50
  51
  52 def _GetComponentName(path, host_dirs):
  53   """Return the component name of a path."""
  54   components_renamed = {
  55       'webkit': 'blink',
  56   }
  57
  58   for host_dir in host_dirs:
  59     if path.startswith(host_dir):
  60       path = path[len(host_dir):]
  61       name = path.split('/')[0].lower()
  62       if name in components_renamed:
  63         return components_renamed[name].lower()
  64       else:
  65         return name.lower()
  66
  67   # Unknown path, return the whole path as component name.
  68   return '_'.join(path.split('/'))
  69
  70
  71 def _GetContentOfDEPS(revision):
  72   chromium_git_file_url_template = CONFIG['chromium_git_file_url']
  73
  74   # Try .DEPS.git first, because before migration from SVN to GIT, the .DEPS.git
  75   # has the dependency in GIT repo while DEPS has dependency in SVN repo.
  76   url = chromium_git_file_url_template % (revision, '.DEPS.git')
  77   http_status_code, content = utils.GetHttpClient().Get(
  78       url, retries=5, retry_if_not=404)
  79
  80   # If .DEPS.git is not found, use DEPS, assuming it is a commit after migration
  81   # from SVN to GIT.
  82   if http_status_code == 404:
  83     url = chromium_git_file_url_template % (revision, 'DEPS')
  84     http_status_code, content = utils.GetHttpClient().Get(url, retries=5)
  85
  86   if http_status_code == 200:
  87     return base64.b64decode(content)
  88   else:
  89     return ''
  90
  91
  92 def GetChromiumComponents(chromium_revision,
  93                           os_platform='unix',
  94                           deps_file_downloader=_GetContentOfDEPS):
  95   """Return a list of components used by Chrome of the given revision.
  96
  97   Args:
  98     chromium_revision: Revision of the Chrome build: svn revision, or git hash.
  99     os_platform: The target platform of the Chrome build, eg. win, mac, etc.
 100     deps_file_downloader: A function that takes the chromium_revision as input,
 101                           and returns the content of the DEPS file. The returned
 102                           content is assumed to be trusted input and will be
 103                           evaluated as python code.
 104
 105   Returns:
 106     A map from component path to parsed component name, repository URL,
 107     repository type and revision.
 108     Return None if an error occurs.
 109   """
 110   if os_platform.lower() == 'linux':
 111     os_platform = 'unix'
 112
 113   chromium_git_base_url = CONFIG['chromium_git_base_url']
 114
 115   if not utils.IsGitHash(chromium_revision):
 116     # Convert svn revision or commit position to Git hash.
 117     cr_rev_url_template = CONFIG['cr_rev_url']
 118     url = cr_rev_url_template % chromium_revision
 119     status_code, content = utils.GetHttpClient().Get(
 120         url, timeout=120, retries=5, retry_if_not=404)
 121     if status_code != 200 or not content:
 122       if status_code == 404:
 123         print 'Chromium commit position %s is not found.' % chromium_revision
 124       return None
 125
 126     cr_rev_data = json.loads(content)
 127     if 'git_sha' not in cr_rev_data:
 128       return None
 129
 130     if 'repo' not in cr_rev_data or cr_rev_data['repo'] != 'chromium/src':
 131       print ('%s seems like a commit position of "%s", but not "chromium/src".'
 132              % (chromium_revision, cr_rev_data['repo']))
 133       return None
 134
 135     chromium_revision = cr_rev_data.get('git_sha')
 136     if not chromium_revision:
 137       return None
 138
 139   # Download the content of DEPS file in chromium.
 140   deps_content = deps_file_downloader(chromium_revision)
 141   if not deps_content:
 142     return None
 143
 144   all_deps = {}
 145
 146   # Parse the content of DEPS file.
 147   deps, deps_os = _ParseDEPS(deps_content)
 148   all_deps.update(deps)
 149   if os_platform is not None:
 150     all_deps.update(deps_os.get(os_platform, {}))
 151
 152   # Figure out components based on the dependencies.
 153   components = {}
 154   host_dirs = CONFIG['host_directories']
 155   for component_path, component_repo_url in all_deps.iteritems():
 156     if component_repo_url is None:
 157       # For some platform like iso, some component is ignored.
 158       continue
 159
 160     name = _GetComponentName(component_path, host_dirs)
 161     repository, revision = component_repo_url.split('@')
 162     match = OLD_GIT_URL_PATTERN.match(repository)
 163     if match:
 164       repository = 'https://chromium.googlesource.com/%s' % match.group(1)
 165     is_git_hash = utils.IsGitHash(revision)
 166     if is_git_hash:
 167       repository_type = 'git'
 168     else:
 169       repository_type = 'svn'
 170     if not component_path.endswith('/'):
 171       component_path += '/'
 172     components[component_path] = {
 173         'path': component_path,
 174         'name': name,
 175         'repository': repository,
 176         'repository_type': repository_type,
 177         'revision': revision
 178     }
 179
 180   # Add chromium as a component.
 181   components['src/'] = {
 182       'path': 'src/',
 183       'name': 'chromium',
 184       'repository': chromium_git_base_url,
 185       'repository_type': 'git',
 186       'revision': chromium_revision
 187   }
 188
 189   return components
 190
 191
 192 def GetChromiumComponentRange(old_revision,
 193                               new_revision,
 194                               os_platform='unix',
 195                               deps_file_downloader=_GetContentOfDEPS):
 196   """Return a list of components with their revision ranges.
 197
 198   Args:
 199     old_revision: The old revision of a Chrome build.
 200     new_revision: The new revision of a Chrome build.
 201     os_platform: The target platform of the Chrome build, eg. win, mac, etc.
 202     deps_file_downloader: A function that takes the chromium_revision as input,
 203                           and returns the content of the DEPS file. The returned
 204                           content is assumed to be trusted input and will be
 205                           evaluated as python code.
 206
 207   Returns:
 208     A map from component path to its parsed regression and other information.
 209     Return None if an error occurs.
 210   """
 211   old_components = GetChromiumComponents(old_revision, os_platform,
 212                                          deps_file_downloader)
 213   if not old_components:
 214     return None
 215
 216   new_components = GetChromiumComponents(new_revision, os_platform,
 217                                          deps_file_downloader)
 218   if not new_components:
 219     return None
 220
 221   components = {}
 222   for path in new_components:
 223     new_component = new_components[path]
 224     old_revision = None
 225
 226     if path in old_components:
 227       old_component = old_components[path]
 228       old_revision = old_component['revision']
 229
 230     components[path] = {
 231         'path': path,
 232         'rolled': new_component['revision'] != old_revision,
 233         'name': new_component['name'],
 234         'old_revision': old_revision,
 235         'new_revision': new_component['revision'],
 236         'repository': new_component['repository'],
 237         'repository_type': new_component['repository_type']
 238     }
 239
 240   return components