android_webview/buildbot/deps_whitelist.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5 #
   6 """Logic to generate lists of DEPS used by various parts of
   7 the android_webview continuous integration (buildbot) infrastructure.
   8
   9 Note: The root Chromium project (which is not explicitly listed here)
  10 has a couple of third_party libraries checked in directly into it. This means
  11 that the list of third parties present in this file is not a comprehensive
  12 list of third party android_webview dependencies.
  13 """
  14
  15 import argparse
  16 import json
  17 import logging
  18 import os
  19 import sys
  20
  21
  22 class DepsWhitelist(object):
  23   def __init__(self):
  24     # If a new DEPS entry is needed for the AOSP bot to compile please add it
  25     # here first.
  26     # This is a staging area for deps that are accepted by the android_webview
  27     # team and are in the process of having the required branches being created
  28     # in the Android tree.
  29     self._compile_but_not_snapshot_dependencies = [
  30       'third_party/mesa/src',
  31     ]
  32
  33     # Dependencies that need to be merged into the Android tree.
  34     self._snapshot_into_android_dependencies = [
  35       'googleurl',
  36       'sdch/open-vcdiff',
  37       'testing/gtest',
  38       'third_party/WebKit',
  39       'third_party/angle_dx11',
  40       ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
  41        'braille'),
  42       'third_party/freetype',
  43       'third_party/icu',
  44       'third_party/leveldatabase/src',
  45       'third_party/libjingle/source/talk',
  46       'third_party/libphonenumber/src/phonenumbers',
  47       'third_party/libphonenumber/src/resources',
  48       'third_party/openssl',
  49       'third_party/opus/src',
  50       'third_party/ots',
  51       'third_party/skia/gyp',
  52       'third_party/skia/include',
  53       'third_party/skia/src',
  54       'third_party/smhasher/src',
  55       'third_party/v8-i18n',
  56       'third_party/yasm/source/patched-yasm',
  57       'tools/grit',
  58       'tools/gyp',
  59       'v8',
  60     ]
  61
  62     # Dependencies required to build android_webview.
  63     self._compile_dependencies = (self._snapshot_into_android_dependencies +
  64                                   self._compile_but_not_snapshot_dependencies)
  65
  66     # Dependencies required to run android_webview tests but not required to
  67     # compile.
  68     self._test_data_dependencies = [
  69       'chrome/test/data/perf/third_party/octane',
  70     ]
  71
  72   @staticmethod
  73   def _read_deps_file(deps_file_path):
  74     class FileImplStub(object):
  75       """Stub for the File syntax."""
  76       def __init__(self, file_location):
  77         pass
  78
  79       @staticmethod
  80       def GetPath():
  81         return ''
  82
  83       @staticmethod
  84       def GetFilename():
  85         return ''
  86
  87       @staticmethod
  88       def GetRevision():
  89         return None
  90
  91     def from_stub(__, _=None):
  92       """Stub for the From syntax."""
  93       return ''
  94
  95     class VarImpl(object):
  96       def __init__(self, custom_vars, local_scope):
  97         self._custom_vars = custom_vars
  98         self._local_scope = local_scope
  99
 100       def Lookup(self, var_name):
 101         """Implements the Var syntax."""
 102         if var_name in self._custom_vars:
 103           return self._custom_vars[var_name]
 104         elif var_name in self._local_scope.get("vars", {}):
 105           return self._local_scope["vars"][var_name]
 106         raise Exception("Var is not defined: %s" % var_name)
 107
 108     local_scope = {}
 109     var = VarImpl({}, local_scope)
 110     global_scope = {
 111         'File': FileImplStub,
 112         'From': from_stub,
 113         'Var': var.Lookup,
 114         'deps_os': {},
 115     }
 116     execfile(deps_file_path, global_scope, local_scope)
 117     deps = local_scope.get('deps', {})
 118     deps_os = local_scope.get('deps_os', {})
 119     for os_specific_deps in deps_os.itervalues():
 120       deps.update(os_specific_deps)
 121     return deps.keys()
 122
 123   def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
 124     """Calculates the list of deps that need to be excluded from the deps_file
 125     so that the only deps left are the one in the whitelist."""
 126     all_deps = self._read_deps_file(deps_file_path)
 127     # The list of deps read from the DEPS file are prefixed with the source
 128     # tree root, which is 'src' for Chromium.
 129     def prepend_root(path):
 130       return os.path.join('src', path)
 131     whitelisted_deps = map(prepend_root, whitelisted_deps)
 132     deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
 133     return dict(map(lambda(x): (x, None), deps_blacklist))
 134
 135   def get_deps_for_android_build(self, deps_file_path):
 136     """This is used to calculate the custom_deps list for the Android bot.
 137     """
 138     if not deps_file_path:
 139       raise Exception('You need to specify a DEPS file path.')
 140     return self._make_gclient_blacklist(deps_file_path,
 141                                         self._compile_dependencies)
 142
 143   def get_deps_for_android_build_and_test(self, deps_file_path):
 144     """This is used to calculate the custom_deps list for the Android perf bot.
 145     """
 146     if not deps_file_path:
 147       raise Exception('You need to specify a DEPS file path.')
 148     return self._make_gclient_blacklist(deps_file_path,
 149                                         self._compile_dependencies +
 150                                         self._test_data_dependencies)
 151
 152   def get_deps_for_android_merge(self, _):
 153     """Calculates the list of deps that need to be merged into the Android tree
 154     in order to build the C++ and Java android_webview code."""
 155     return self._snapshot_into_android_dependencies
 156
 157   def get_deps_for_license_check(self, _):
 158     """Calculates the list of deps that need to be checked for Android license
 159     compatibility"""
 160     return self._compile_dependencies
 161
 162   def execute_method(self, method_name, deps_file_path):
 163     methods = {
 164       'android_build': self.get_deps_for_android_build,
 165       'android_build_and_test':
 166         self.get_deps_for_android_build_and_test,
 167       'android_merge': self.get_deps_for_android_merge,
 168       'license_check': self.get_deps_for_license_check
 169     }
 170     if not method_name in methods:
 171       raise Exception('Method name %s is not valid. Valid choices are %s' %
 172                       (method_name, methods.keys()))
 173     return methods[method_name](deps_file_path)
 174
 175 def main():
 176   parser = argparse.ArgumentParser()
 177   parser.add_argument('--method', help='Method to use to fetch from whitelist.',
 178                       required=True)
 179   parser.add_argument('--path-to-deps', help='Path to DEPS file.')
 180   parser.add_argument('--output-json', help='Name of file to write output to.')
 181   parser.add_argument('verbose', action='store_true', default=False)
 182   opts = parser.parse_args()
 183
 184   logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)
 185
 186   deps_whitelist = DepsWhitelist()
 187   blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)
 188
 189   if (opts.output_json):
 190     output_dict = {
 191         'blacklist' : blacklist
 192     }
 193     with open(opts.output_json, 'w') as output_json_file:
 194       json.dump(output_dict, output_json_file)
 195   else:
 196     print blacklist
 197
 198   return 0
 199
 200
 201 if __name__ == '__main__':
 202   sys.exit(main())