android_webview/tools/webview_licenses.py

   1 #!/usr/bin/python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Checks third-party licenses for the purposes of the Android WebView build.
   7
   8 The Android tree includes a snapshot of Chromium in order to power the system
   9 WebView.  This tool checks that all code uses open-source licenses compatible
  10 with Android, and that we meet the requirements of those licenses. It can also
  11 be used to generate an Android NOTICE file for the third-party code.
  12
  13 It makes use of src/tools/licenses.py and the README.chromium files on which
  14 it depends. It also makes use of a data file, third_party_files_whitelist.txt,
  15 which whitelists indicidual files which contain third-party code but which
  16 aren't in a third-party directory with a README.chromium file.
  17 """
  18
  19 import glob
  20 import imp
  21 import multiprocessing
  22 import optparse
  23 import os
  24 import re
  25 import sys
  26 import textwrap
  27
  28
  29 REPOSITORY_ROOT = os.path.abspath(os.path.join(
  30     os.path.dirname(__file__), '..', '..'))
  31
  32 # Import third_party/PRESUBMIT.py via imp to avoid importing a random
  33 # PRESUBMIT.py from $PATH, also make sure we don't generate a .pyc file.
  34 sys.dont_write_bytecode = True
  35 third_party = \
  36   imp.load_source('PRESUBMIT', \
  37                   os.path.join(REPOSITORY_ROOT, 'third_party', 'PRESUBMIT.py'))
  38
  39 sys.path.append(os.path.join(REPOSITORY_ROOT, 'third_party'))
  40 import jinja2
  41 sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
  42 import licenses
  43
  44 import copyright_scanner
  45 import known_issues
  46
  47 class InputApi(object):
  48   def __init__(self):
  49     self.os_path = os.path
  50     self.os_walk = os.walk
  51     self.re = re
  52     self.ReadFile = _ReadFile
  53     self.change = InputApiChange()
  54
  55 class InputApiChange(object):
  56   def __init__(self):
  57     self.RepositoryRoot = lambda: REPOSITORY_ROOT
  58
  59
  60 def GetIncompatibleDirectories():
  61   """Gets a list of third-party directories which use licenses incompatible
  62   with Android. This is used by the snapshot tool.
  63   Returns:
  64     A list of directories.
  65   """
  66
  67   result = []
  68   for directory in _FindThirdPartyDirs():
  69     if directory in known_issues.KNOWN_ISSUES:
  70       result.append(directory)
  71       continue
  72     try:
  73       metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
  74                                    require_license_file=False,
  75                                    optional_keys=['License Android Compatible'])
  76     except licenses.LicenseError as e:
  77       print 'Got LicenseError while scanning ' + directory
  78       raise
  79     if metadata.get('License Android Compatible', 'no').upper() == 'YES':
  80       continue
  81     license = re.split(' [Ll]icenses?$', metadata['License'])[0]
  82     if not third_party.LicenseIsCompatibleWithAndroid(InputApi(), license):
  83       result.append(directory)
  84   return result
  85
  86 def GetUnknownIncompatibleDirectories():
  87   """Gets a list of third-party directories which use licenses incompatible
  88   with Android which are not present in the known_issues.py file.
  89   This is used by the AOSP bot.
  90   Returns:
  91     A list of directories.
  92   """
  93   incompatible_directories = frozenset(GetIncompatibleDirectories())
  94   known_incompatible = []
  95   input_api = InputApi()
  96   for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
  97     path = copyright_scanner.ForwardSlashesToOsPathSeps(input_api, path)
  98     for exclude in exclude_list:
  99       exclude = copyright_scanner.ForwardSlashesToOsPathSeps(input_api, exclude)
 100       if glob.has_magic(exclude):
 101         exclude_dirname = os.path.dirname(exclude)
 102         if glob.has_magic(exclude_dirname):
 103           print ('Exclude path %s contains an unexpected glob expression,' \
 104                  ' skipping.' % exclude)
 105         exclude = exclude_dirname
 106       known_incompatible.append(os.path.normpath(os.path.join(path, exclude)))
 107   known_incompatible = frozenset(known_incompatible)
 108   return incompatible_directories.difference(known_incompatible)
 109
 110
 111 class ScanResult(object):
 112   Ok, Warnings, Errors = range(3)
 113
 114 # Needs to be a top-level function for multiprocessing
 115 def _FindCopyrightViolations(files_to_scan_as_string):
 116   return copyright_scanner.FindCopyrightViolations(
 117     InputApi(), REPOSITORY_ROOT, files_to_scan_as_string)
 118
 119 def _ShardList(l, shard_len):
 120   return [l[i:i + shard_len] for i in range(0, len(l), shard_len)]
 121
 122 def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files):
 123   """Checks that all files which are not in a listed third-party directory,
 124   and which do not use the standard Chromium license, are whitelisted.
 125   Args:
 126     excluded_dirs_list: The list of directories to exclude from scanning.
 127     whitelisted_files: The whitelist of files.
 128   Returns:
 129     ScanResult.Ok if all files with non-standard license headers are whitelisted
 130     and the whitelist contains no stale entries;
 131     ScanResult.Warnings if there are stale entries;
 132     ScanResult.Errors if new non-whitelisted entries found.
 133   """
 134   input_api = InputApi()
 135   files_to_scan = copyright_scanner.FindFiles(
 136     input_api, REPOSITORY_ROOT, ['.'], excluded_dirs_list)
 137   sharded_files_to_scan = _ShardList(files_to_scan, 2000)
 138   pool = multiprocessing.Pool()
 139   offending_files_chunks = pool.map_async(
 140       _FindCopyrightViolations, sharded_files_to_scan).get(999999)
 141   pool.close()
 142   pool.join()
 143   # Flatten out the result
 144   offending_files = \
 145     [item for sublist in offending_files_chunks for item in sublist]
 146
 147   (unknown, missing, stale) = copyright_scanner.AnalyzeScanResults(
 148     input_api, whitelisted_files, offending_files)
 149
 150   if unknown:
 151     print 'The following files contain a third-party license but are not in ' \
 152           'a listed third-party directory and are not whitelisted. You must ' \
 153           'add the following files to the whitelist.\n%s' % \
 154           '\n'.join(sorted(unknown))
 155   if missing:
 156     print 'The following files are whitelisted, but do not exist.\n%s' % \
 157         '\n'.join(sorted(missing))
 158   if stale:
 159     print 'The following files are whitelisted unnecessarily. You must ' \
 160           'remove the following files from the whitelist.\n%s' % \
 161           '\n'.join(sorted(stale))
 162
 163   if unknown:
 164     return ScanResult.Errors
 165   elif stale or missing:
 166     return ScanResult.Warnings
 167   else:
 168     return ScanResult.Ok
 169
 170
 171 def _ReadFile(full_path, mode='rU'):
 172   """Reads a file from disk. This emulates presubmit InputApi.ReadFile func.
 173   Args:
 174     full_path: The path of the file to read.
 175   Returns:
 176     The contents of the file as a string.
 177   """
 178
 179   with open(full_path, mode) as f:
 180     return f.read()
 181
 182
 183 def _ReadLocalFile(path, mode='rb'):
 184   """Reads a file from disk.
 185   Args:
 186     path: The path of the file to read, relative to the root of the repository.
 187   Returns:
 188     The contents of the file as a string.
 189   """
 190
 191   return _ReadFile(os.path.join(REPOSITORY_ROOT, path), mode)
 192
 193
 194 def _FindThirdPartyDirs():
 195   """Gets the list of third-party directories.
 196   Returns:
 197     The list of third-party directories.
 198   """
 199
 200   # Please don't add here paths that have problems with license files,
 201   # as they will end up included in Android WebView snapshot.
 202   # Instead, add them into known_issues.py.
 203   prune_paths = [
 204     # Temporary until we figure out how not to check out quickoffice on the
 205     # Android license check bot. Tracked in crbug.com/350472.
 206     os.path.join('chrome', 'browser', 'resources', 'chromeos', 'quickoffice'),
 207     # Placeholder directory, no third-party code.
 208     os.path.join('third_party', 'adobe'),
 209     # Apache 2.0 license. See
 210     # https://code.google.com/p/chromium/issues/detail?id=140478.
 211     os.path.join('third_party', 'bidichecker'),
 212     # Isn't checked out on clients
 213     os.path.join('third_party', 'gles2_conform'),
 214     # The llvm-build doesn't exist for non-clang builder
 215     os.path.join('third_party', 'llvm-build'),
 216     # Binaries doesn't apply to android
 217     os.path.join('third_party', 'widevine'),
 218     # third_party directories in this tree aren't actually third party, but
 219     # provide a way to shadow experimental buildfiles into those directories.
 220     os.path.join('build', 'secondary'),
 221     # Not shipped, Chromium code
 222     os.path.join('tools', 'swarming_client'),
 223     # Not shipped, only relates to Chrome for Android, but not to WebView
 224     os.path.join('clank'),
 225     # Bots only, is not a part of the build
 226     os.path.join('isolate_deps_dir'),
 227   ]
 228   third_party_dirs = licenses.FindThirdPartyDirs(prune_paths, REPOSITORY_ROOT)
 229   return licenses.FilterDirsWithFiles(third_party_dirs, REPOSITORY_ROOT)
 230
 231
 232 def _Scan():
 233   """Checks that license meta-data is present for all third-party code and
 234      that all non third-party code doesn't contain external copyrighted code.
 235   Returns:
 236     ScanResult.Ok if everything is in order;
 237     ScanResult.Warnings if there are non-fatal problems (e.g. stale whitelist
 238       entries)
 239     ScanResult.Errors otherwise.
 240   """
 241
 242   third_party_dirs = _FindThirdPartyDirs()
 243
 244   # First, check designated third-party directories using src/tools/licenses.py.
 245   all_licenses_valid = True
 246   for path in sorted(third_party_dirs):
 247     try:
 248       licenses.ParseDir(path, REPOSITORY_ROOT)
 249     except licenses.LicenseError, e:
 250       if not (path in known_issues.KNOWN_ISSUES):
 251         print 'Got LicenseError "%s" while scanning %s' % (e, path)
 252         all_licenses_valid = False
 253
 254   # Second, check for non-standard license text.
 255   whitelisted_files = copyright_scanner.LoadWhitelistedFilesList(InputApi())
 256   licenses_check = _CheckLicenseHeaders(third_party_dirs, whitelisted_files)
 257
 258   return licenses_check if all_licenses_valid else ScanResult.Errors
 259
 260
 261 class TemplateEntryGenerator(object):
 262   def __init__(self):
 263     self._generate_licenses_file_list_only = False
 264     self._toc_index = 0
 265
 266   def SetGenerateLicensesFileListOnly(self, generate_licenses_file_list_only):
 267     self._generate_licenses_file_list_only = generate_licenses_file_list_only
 268
 269   def _ReadFileGuessEncoding(self, name):
 270     if self._generate_licenses_file_list_only:
 271       return ''
 272     contents = ''
 273     with open(name, 'rb') as input_file:
 274       contents = input_file.read()
 275     try:
 276       return contents.decode('utf8')
 277     except UnicodeDecodeError:
 278       pass
 279     # If it's not UTF-8, it must be CP-1252. Fail otherwise.
 280     return contents.decode('cp1252')
 281
 282   def MetadataToTemplateEntry(self, metadata):
 283     self._toc_index += 1
 284     return {
 285       'name': metadata['Name'],
 286       'url': metadata['URL'],
 287       'license_file': metadata['License File'],
 288       'license': self._ReadFileGuessEncoding(metadata['License File']),
 289       'toc_href': 'entry' + str(self._toc_index),
 290     }
 291
 292
 293 def GenerateNoticeFile(generate_licenses_file_list_only=False):
 294   """Generates the contents of an Android NOTICE file for the third-party code.
 295   This is used by the snapshot tool.
 296   Returns:
 297     The contents of the NOTICE file.
 298   """
 299
 300   generator = TemplateEntryGenerator()
 301   generator.SetGenerateLicensesFileListOnly(generate_licenses_file_list_only)
 302   # Start from Chromium's LICENSE file
 303   entries = [generator.MetadataToTemplateEntry({
 304     'Name': 'The Chromium Project',
 305     'URL': 'http://www.chromium.org',
 306     'License File': os.path.join(REPOSITORY_ROOT, 'LICENSE') })
 307   ]
 308
 309   third_party_dirs = _FindThirdPartyDirs()
 310   # We provide attribution for all third-party directories.
 311   # TODO(mnaganov): Limit this to only code used by the WebView binary.
 312   for directory in sorted(third_party_dirs):
 313     try:
 314       metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
 315                                    require_license_file=False)
 316     except licenses.LicenseError:
 317       # Since this code is called during project files generation,
 318       # we don't want to break the it. But we assume that release
 319       # WebView apks are built using checkouts that pass
 320       # 'webview_licenses.py scan' check, thus they don't contain
 321       # projects with non-compatible licenses.
 322       continue
 323     license_file = metadata['License File']
 324     if license_file and license_file != licenses.NOT_SHIPPED:
 325       entries.append(generator.MetadataToTemplateEntry(metadata))
 326
 327   if generate_licenses_file_list_only:
 328     return [entry['license_file'] for entry in entries]
 329   else:
 330     env = jinja2.Environment(
 331       loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
 332       extensions=['jinja2.ext.autoescape'])
 333     template = env.get_template('licenses_notice.tmpl')
 334     return template.render({ 'entries': entries }).encode('utf8')
 335
 336
 337 def _ProcessIncompatibleResult(incompatible_directories):
 338   if incompatible_directories:
 339     print ("Incompatibly licensed directories found:\n" +
 340            "\n".join(sorted(incompatible_directories)))
 341     return ScanResult.Errors
 342   return ScanResult.Ok
 343
 344 def main():
 345   class FormatterWithNewLines(optparse.IndentedHelpFormatter):
 346     def format_description(self, description):
 347       paras = description.split('\n')
 348       formatted_paras = [textwrap.fill(para, self.width) for para in paras]
 349       return '\n'.join(formatted_paras) + '\n'
 350
 351   parser = optparse.OptionParser(formatter=FormatterWithNewLines(),
 352                                  usage='%prog [options]')
 353   parser.description = (__doc__ +
 354                         '\nCommands:\n'
 355                         '  scan Check licenses.\n'
 356                         '  notice_deps Generate the list of dependencies for '
 357                         'Android NOTICE file.\n'
 358                         '  notice [file] Generate Android NOTICE file on '
 359                         'stdout or into |file|.\n'
 360                         '  incompatible_directories Scan for incompatibly'
 361                         ' licensed directories.\n'
 362                         '  all_incompatible_directories Scan for incompatibly'
 363                         ' licensed directories (even those in'
 364                         ' known_issues.py).\n'
 365                         '  display_copyrights Display autorship on the files'
 366                         ' using names provided via stdin.\n')
 367   (_, args) = parser.parse_args()
 368   if len(args) < 1:
 369     parser.print_help()
 370     return ScanResult.Errors
 371
 372   if args[0] == 'scan':
 373     scan_result = _Scan()
 374     if scan_result == ScanResult.Ok:
 375       print 'OK!'
 376     return scan_result
 377   elif args[0] == 'notice_deps':
 378     # 'set' is used to eliminate duplicate references to the same license file.
 379     print ' '.join(
 380       sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True))))
 381     return ScanResult.Ok
 382   elif args[0] == 'notice':
 383     notice_file_contents = GenerateNoticeFile()
 384     if len(args) == 1:
 385       print notice_file_contents
 386     else:
 387       with open(args[1], 'w') as output_file:
 388         output_file.write(notice_file_contents)
 389     return ScanResult.Ok
 390   elif args[0] == 'incompatible_directories':
 391     return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories())
 392   elif args[0] == 'all_incompatible_directories':
 393     return _ProcessIncompatibleResult(GetIncompatibleDirectories())
 394   elif args[0] == 'display_copyrights':
 395     files = sys.stdin.read().splitlines()
 396     for f, c in \
 397         zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)):
 398       print f, '\t', ' / '.join(sorted(c))
 399     return ScanResult.Ok
 400   parser.print_help()
 401   return ScanResult.Errors
 402
 403 if __name__ == '__main__':
 404   sys.exit(main())