android_webview/buildbot/deps_whitelist.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5 #
   6 """Logic to generate lists of DEPS used by various parts of
   7 the android_webview continuous integration (buildbot) infrastructure.
   8
   9 Note: The root Chromium project (which is not explicitly listed here)
  10 has a couple of third_party libraries checked in directly into it. This means
  11 that the list of third parties present in this file is not a comprehensive
  12 list of third party android_webview dependencies.
  13 """
  14
  15 import argparse
  16 import json
  17 import logging
  18 import os
  19 import sys
  20
  21 # Add android_webview/tools to path to get at known_issues.
  22 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'tools'))
  23 import known_issues
  24
  25
  26 class DepsWhitelist(object):
  27   def __init__(self):
  28     # If a new DEPS entry is needed for the AOSP bot to compile please add it
  29     # here first.
  30     # This is a staging area for deps that are accepted by the android_webview
  31     # team and are in the process of having the required branches being created
  32     # in the Android tree.
  33     self._compile_but_not_snapshot_dependencies = [
  34     ]
  35
  36     # Dependencies that need to be merged into the Android tree.
  37     self._snapshot_into_android_dependencies = [
  38       'sdch/open-vcdiff',
  39       'testing/gtest',
  40       'third_party/WebKit',
  41       'third_party/angle',
  42       'third_party/boringssl/src',
  43       'third_party/brotli/src',
  44       ('third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/'
  45        'braille'),
  46       'third_party/freetype',
  47       'third_party/icu',
  48       'third_party/leveldatabase/src',
  49       'third_party/libaddressinput/src',
  50       'third_party/libjingle/source/talk',
  51       'third_party/libjpeg_turbo',
  52       'third_party/libphonenumber/src/phonenumbers',
  53       'third_party/libphonenumber/src/resources',
  54       'third_party/libsrtp',
  55       'third_party/libvpx',
  56       'third_party/libyuv',
  57       'third_party/mesa/src',
  58       'third_party/openmax_dl',
  59       'third_party/opus/src',
  60       'third_party/ots',
  61       'third_party/sfntly/cpp/src',
  62       'third_party/skia',
  63       'third_party/smhasher/src',
  64       'third_party/snappy',
  65       'third_party/usrsctp/usrsctplib',
  66       'third_party/webrtc',
  67       'third_party/yasm/source/patched-yasm',
  68       'tools/grit',
  69       'tools/gyp',
  70       'v8',
  71     ]
  72
  73     # We can save some time by not rsyncing code we don't use.
  74     self._prune_from_rsync_build = [
  75         'third_party/WebKit/LayoutTests',
  76     ]
  77
  78     # Dependencies required to build android_webview.
  79     self._compile_dependencies = (self._snapshot_into_android_dependencies +
  80                                   self._compile_but_not_snapshot_dependencies)
  81
  82     # Dependencies required to run android_webview tests but not required to
  83     # compile.
  84     self._test_data_dependencies = [
  85       'chrome/test/data/perf/third_party/octane',
  86     ]
  87
  88   @staticmethod
  89   def _read_deps_file(deps_file_path):
  90     class FileImplStub(object):
  91       """Stub for the File syntax."""
  92       def __init__(self, file_location):
  93         pass
  94
  95       @staticmethod
  96       def GetPath():
  97         return ''
  98
  99       @staticmethod
 100       def GetFilename():
 101         return ''
 102
 103       @staticmethod
 104       def GetRevision():
 105         return None
 106
 107     def from_stub(__, _=None):
 108       """Stub for the From syntax."""
 109       return ''
 110
 111     class VarImpl(object):
 112       def __init__(self, custom_vars, local_scope):
 113         self._custom_vars = custom_vars
 114         self._local_scope = local_scope
 115
 116       def Lookup(self, var_name):
 117         """Implements the Var syntax."""
 118         if var_name in self._custom_vars:
 119           return self._custom_vars[var_name]
 120         elif var_name in self._local_scope.get("vars", {}):
 121           return self._local_scope["vars"][var_name]
 122         raise Exception("Var is not defined: %s" % var_name)
 123
 124     local_scope = {}
 125     var = VarImpl({}, local_scope)
 126     global_scope = {
 127         'File': FileImplStub,
 128         'From': from_stub,
 129         'Var': var.Lookup,
 130         'deps_os': {},
 131     }
 132     execfile(deps_file_path, global_scope, local_scope)
 133     deps = local_scope.get('deps', {})
 134     deps_os = local_scope.get('deps_os', {})
 135     for os_specific_deps in deps_os.itervalues():
 136       deps.update(os_specific_deps)
 137     return deps.keys()
 138
 139   def _get_known_issues(self):
 140     issues = []
 141     for root, paths in known_issues.KNOWN_INCOMPATIBLE.items():
 142       for path in paths:
 143         issues.append(os.path.normpath(os.path.join(root, path)))
 144     return issues
 145
 146   def _make_gclient_blacklist(self, deps_file_path, whitelisted_deps):
 147     """Calculates the list of deps that need to be excluded from the deps_file
 148     so that the only deps left are the one in the whitelist."""
 149     all_deps = self._read_deps_file(deps_file_path)
 150     # The list of deps read from the DEPS file are prefixed with the source
 151     # tree root, which is 'src' for Chromium.
 152     def prepend_root(path):
 153       return os.path.join('src', path)
 154     whitelisted_deps = map(prepend_root, whitelisted_deps)
 155     deps_blacklist = set(all_deps).difference(set(whitelisted_deps))
 156     return dict(map(lambda(x): (x, None), deps_blacklist))
 157
 158   def _make_blacklist(self, deps_file_path, whitelisted_deps):
 159     """Calculates the list of paths we should exclude """
 160     all_deps = self._read_deps_file(deps_file_path)
 161     def remove_src_prefix(path):
 162       return path.replace('src/', '', 1)
 163     all_deps = map(remove_src_prefix, all_deps)
 164     # Ignore all deps except those whitelisted.
 165     blacklist = set(all_deps).difference(whitelisted_deps)
 166     # Ignore the 'known issues'. Typically these are the licence incompatible
 167     # things checked directly into Chromium.
 168     blacklist = blacklist.union(self._get_known_issues())
 169     # Ignore any other non-deps, non-licence paths we don't like.
 170     blacklist = blacklist.union(self._prune_from_rsync_build)
 171     return list(blacklist)
 172
 173   def get_deps_for_android_build(self, deps_file_path):
 174     """This is used to calculate the custom_deps list for the Android bot.
 175     """
 176     if not deps_file_path:
 177       raise Exception('You need to specify a DEPS file path.')
 178     return self._make_gclient_blacklist(deps_file_path,
 179                                         self._compile_dependencies)
 180
 181   def get_deps_for_android_build_and_test(self, deps_file_path):
 182     """This is used to calculate the custom_deps list for the Android perf bot.
 183     """
 184     if not deps_file_path:
 185       raise Exception('You need to specify a DEPS file path.')
 186     return self._make_gclient_blacklist(deps_file_path,
 187                                         self._compile_dependencies +
 188                                         self._test_data_dependencies)
 189
 190   def get_blacklist_for_android_rsync_build(self, deps_file_path):
 191     """Calculates the list of paths we should exclude when building Android
 192     either because of license compatibility or because they are large and
 193     uneeded.
 194     """
 195     if not deps_file_path:
 196       raise Exception('You need to specify a DEPS file path.')
 197     return self._make_blacklist(deps_file_path, self._compile_dependencies)
 198
 199   def get_deps_for_android_merge(self, _):
 200     """Calculates the list of deps that need to be merged into the Android tree
 201     in order to build the C++ and Java android_webview code."""
 202     return self._snapshot_into_android_dependencies
 203
 204   def get_deps_for_license_check(self, _):
 205     """Calculates the list of deps that need to be checked for Android license
 206     compatibility"""
 207     return self._compile_dependencies
 208
 209
 210   def execute_method(self, method_name, deps_file_path):
 211     methods = {
 212       'android_build': self.get_deps_for_android_build,
 213       'android_build_and_test':
 214         self.get_deps_for_android_build_and_test,
 215       'android_merge': self.get_deps_for_android_merge,
 216       'license_check': self.get_deps_for_license_check,
 217       'android_rsync_build': self.get_blacklist_for_android_rsync_build,
 218     }
 219     if not method_name in methods:
 220       raise Exception('Method name %s is not valid. Valid choices are %s' %
 221                       (method_name, methods.keys()))
 222     return methods[method_name](deps_file_path)
 223
 224 def main():
 225   parser = argparse.ArgumentParser()
 226   parser.add_argument('--method', help='Method to use to fetch from whitelist.',
 227                       required=True)
 228   parser.add_argument('--path-to-deps', help='Path to DEPS file.')
 229   parser.add_argument('--output-json', help='Name of file to write output to.')
 230   parser.add_argument('verbose', action='store_true', default=False)
 231   opts = parser.parse_args()
 232
 233   logging.getLogger().setLevel(logging.DEBUG if opts.verbose else logging.WARN)
 234
 235   deps_whitelist = DepsWhitelist()
 236   blacklist = deps_whitelist.execute_method(opts.method, opts.path_to_deps)
 237
 238   if (opts.output_json):
 239     output_dict = {
 240         'blacklist' : blacklist
 241     }
 242     with open(opts.output_json, 'w') as output_json_file:
 243       json.dump(output_dict, output_json_file)
 244   else:
 245     print blacklist
 246
 247   return 0
 248
 249
 250 if __name__ == '__main__':
 251   sys.exit(main())