tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 116         'UNKNOWN',
 117     ],
 118     # This contains files copied from elsewhere from the tree. Since the copied
 119     # directories might have suppressions below (like simplejson), whitelist the
 120     # whole directory. This is also not shipped code.
 121     'chrome/common/extensions/docs/server2/third_party': [
 122         'UNKNOWN',
 123     ],
 124     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 125         'UNKNOWN',
 126     ],
 127     'mojo/public/third_party/ply/__init__.py': [
 128         'UNKNOWN',
 129     ],
 130     'native_client': [  # http://crbug.com/98099
 131         'UNKNOWN',
 132     ],
 133     'native_client/toolchain': [
 134         'BSD GPL (v2 or later)',
 135         'BSD (2 clause) GPL (v2 or later)',
 136         'BSD (3 clause) GPL (v2 or later)',
 137         'BSL (v1.0) GPL',
 138         'BSL (v1.0) GPL (v3.1)',
 139         'GPL',
 140         'GPL (unversioned/unknown version)',
 141         'GPL (v2)',
 142         'GPL (v2 or later)',
 143         'GPL (v3.1)',
 144         'GPL (v3 or later)',
 145     ],
 146     'third_party/WebKit': [
 147         'UNKNOWN',
 148     ],
 149
 150     # http://code.google.com/p/angleproject/issues/detail?id=217
 151     'third_party/angle': [
 152         'UNKNOWN',
 153     ],
 154
 155     # http://crbug.com/222828
 156     # http://bugs.python.org/issue17514
 157     'third_party/chromite/third_party/argparse.py': [
 158         'UNKNOWN',
 159     ],
 160
 161     # http://crbug.com/326117
 162     # https://bitbucket.org/chrisatlee/poster/issue/21
 163     'third_party/chromite/third_party/poster': [
 164         'UNKNOWN',
 165     ],
 166
 167     # http://crbug.com/333508
 168     'third_party/clang_format/script': [
 169         'UNKNOWN',
 170     ],
 171
 172     # http://crbug.com/333508
 173     'buildtools/clang_format/script': [
 174         'UNKNOWN',
 175     ],
 176
 177     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 178     'third_party/cython': [
 179         'UNKNOWN',
 180     ],
 181
 182     'third_party/devscripts': [
 183         'GPL (v2 or later)',
 184     ],
 185     'third_party/expat/files/lib': [  # http://crbug.com/98121
 186         'UNKNOWN',
 187     ],
 188     'third_party/ffmpeg': [
 189         'GPL',
 190         'GPL (v2)',
 191         'GPL (v2 or later)',
 192         'GPL (v3 or later)',
 193         'UNKNOWN',  # http://crbug.com/98123
 194     ],
 195     'third_party/fontconfig': [
 196         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 197         'UNKNOWN',
 198     ],
 199     'third_party/freetype2': [ # http://crbug.com/177319
 200         'UNKNOWN',
 201     ],
 202     'third_party/hunspell': [  # http://crbug.com/98134
 203         'UNKNOWN',
 204     ],
 205     'third_party/iccjpeg': [  # http://crbug.com/98137
 206         'UNKNOWN',
 207     ],
 208     'third_party/icu': [  # http://crbug.com/98301
 209         'UNKNOWN',
 210     ],
 211     'third_party/lcov': [  # http://crbug.com/98304
 212         'UNKNOWN',
 213     ],
 214     'third_party/lcov/contrib/galaxy/genflat.pl': [
 215         'GPL (v2 or later)',
 216     ],
 217     'third_party/libc++/trunk/test': [
 218         # http://llvm.org/bugs/show_bug.cgi?id=18291
 219         'UNKNOWN',
 220     ],
 221     'third_party/libevent': [  # http://crbug.com/98309
 222         'UNKNOWN',
 223     ],
 224     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 225         'UNKNOWN',
 226     ],
 227     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 228         'UNKNOWN',
 229     ],
 230
 231     # Many liblouis files are mirrored but not used in the NaCl module.
 232     # They are not excluded from the mirror because of lack of infrastructure
 233     # support.  Getting license headers added to the files where missing is
 234     # tracked in https://github.com/liblouis/liblouis/issues/22.
 235     'third_party/liblouis/src': [
 236         'GPL (v3 or later)',
 237         'UNKNOWN',
 238     ],
 239
 240     'third_party/libpng': [  # http://crbug.com/98318
 241         'UNKNOWN',
 242     ],
 243
 244     # The following files lack license headers, but are trivial.
 245     'third_party/libusb/src/libusb/os/poll_posix.h': [
 246         'UNKNOWN',
 247     ],
 248
 249     'third_party/libvpx/source': [  # http://crbug.com/98319
 250         'UNKNOWN',
 251     ],
 252     'third_party/libxml': [
 253         'UNKNOWN',
 254     ],
 255     'third_party/libxslt': [
 256         'UNKNOWN',
 257     ],
 258     'third_party/lzma_sdk': [
 259         'UNKNOWN',
 260     ],
 261     'third_party/mesa/src': [
 262         'GPL (v2)',
 263         'GPL (v3 or later)',
 264         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 265         'UNKNOWN',  # http://crbug.com/98450
 266     ],
 267     'third_party/modp_b64': [
 268         'UNKNOWN',
 269     ],
 270     'third_party/openmax_dl/dl' : [
 271         'Khronos Group',
 272     ],
 273     'third_party/openssl': [  # http://crbug.com/98451
 274         'UNKNOWN',
 275     ],
 276     'third_party/boringssl': [
 277         # There are some files in BoringSSL which came from OpenSSL and have no
 278         # license in them. We don't wish to add the license header ourselves
 279         # thus we don't expect to pass license checks.
 280         'UNKNOWN',
 281     ],
 282     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 283         'UNKNOWN',
 284     ],
 285     'third_party/molokocacao': [  # http://crbug.com/98453
 286         'UNKNOWN',
 287     ],
 288     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 289         'UNKNOWN',
 290     ],
 291     'third_party/ply/__init__.py': [
 292         'UNKNOWN',
 293     ],
 294     'third_party/protobuf': [  # http://crbug.com/98455
 295         'UNKNOWN',
 296     ],
 297
 298     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 299     # BSD 2-clause license.
 300     'third_party/pycoverage': [
 301         'UNKNOWN',
 302     ],
 303
 304     'third_party/pyelftools': [ # http://crbug.com/222831
 305         'UNKNOWN',
 306     ],
 307     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 308         'UNKNOWN',
 309     ],
 310     'third_party/simplejson': [
 311         'UNKNOWN',
 312     ],
 313     'third_party/skia': [  # http://crbug.com/98463
 314         'UNKNOWN',
 315     ],
 316     'third_party/snappy/src': [  # http://crbug.com/98464
 317         'UNKNOWN',
 318     ],
 319     'third_party/smhasher/src': [  # http://crbug.com/98465
 320         'UNKNOWN',
 321     ],
 322     'third_party/speech-dispatcher/libspeechd.h': [
 323         'GPL (v2 or later)',
 324     ],
 325     'third_party/sqlite': [
 326         'UNKNOWN',
 327     ],
 328
 329     # http://crbug.com/334668
 330     # MIT license.
 331     'tools/swarming_client/third_party/httplib2': [
 332         'UNKNOWN',
 333     ],
 334
 335     # http://crbug.com/334668
 336     # Apache v2.0.
 337     'tools/swarming_client/third_party/oauth2client': [
 338         'UNKNOWN',
 339     ],
 340
 341     # https://github.com/kennethreitz/requests/issues/1610
 342     'tools/swarming_client/third_party/requests': [
 343         'UNKNOWN',
 344     ],
 345
 346     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 347         'UNKNOWN',
 348     ],
 349     'third_party/talloc': [
 350         'GPL (v3 or later)',
 351         'UNKNOWN',  # http://crbug.com/98588
 352     ],
 353     'third_party/tcmalloc': [
 354         'UNKNOWN',  # http://crbug.com/98589
 355     ],
 356     'third_party/tlslite': [
 357         'UNKNOWN',
 358     ],
 359     'third_party/webdriver': [  # http://crbug.com/98590
 360         'UNKNOWN',
 361     ],
 362
 363     # https://github.com/html5lib/html5lib-python/issues/125
 364     # https://github.com/KhronosGroup/WebGL/issues/435
 365     'third_party/webgl/src': [
 366         'UNKNOWN',
 367     ],
 368
 369     'third_party/webrtc': [  # http://crbug.com/98592
 370         'UNKNOWN',
 371     ],
 372     'third_party/xdg-utils': [  # http://crbug.com/98593
 373         'UNKNOWN',
 374     ],
 375     'third_party/yasm/source': [  # http://crbug.com/98594
 376         'UNKNOWN',
 377     ],
 378     'third_party/zlib/contrib/minizip': [
 379         'UNKNOWN',
 380     ],
 381     'third_party/zlib/trees.h': [
 382         'UNKNOWN',
 383     ],
 384     'tools/emacs': [  # http://crbug.com/98595
 385         'UNKNOWN',
 386     ],
 387     'tools/gyp/test': [
 388         'UNKNOWN',
 389     ],
 390     'tools/python/google/__init__.py': [
 391         'UNKNOWN',
 392     ],
 393     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 394         'UNKNOWN',
 395     ],
 396     'tools/symsrc/pefile.py': [
 397         'UNKNOWN',
 398     ],
 399     # Not shipped, downloaded on trybots sometimes.
 400     'tools/telemetry/third_party/gsutil': [
 401         'BSD MIT/X11 (BSD like)',
 402         'UNKNOWN',
 403     ],
 404     'tools/telemetry/third_party/pyserial': [
 405         # https://sourceforge.net/p/pyserial/feature-requests/35/
 406         'UNKNOWN',
 407     ],
 408     'v8/test/cctest': [  # http://crbug.com/98597
 409         'UNKNOWN',
 410     ],
 411     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 412         'UNKNOWN',
 413     ],
 414 }
 415
 416
 417 def check_licenses(options, args):
 418   # Figure out which directory we have to check.
 419   if len(args) == 0:
 420     # No directory to check specified, use the repository root.
 421     start_dir = options.base_directory
 422   elif len(args) == 1:
 423     # Directory specified. Start here. It's supposed to be relative to the
 424     # base directory.
 425     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 426   else:
 427     # More than one argument, we don't handle this.
 428     PrintUsage()
 429     return 1
 430
 431   print "Using base directory:", options.base_directory
 432   print "Checking:", start_dir
 433   print
 434
 435   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 436                                                    'third_party',
 437                                                    'devscripts',
 438                                                    'licensecheck.pl'))
 439
 440   licensecheck = subprocess.Popen([licensecheck_path,
 441                                    '-l', '100',
 442                                    '-r', start_dir],
 443                                   stdout=subprocess.PIPE,
 444                                   stderr=subprocess.PIPE)
 445   stdout, stderr = licensecheck.communicate()
 446   if options.verbose:
 447     print '----------- licensecheck stdout -----------'
 448     print stdout
 449     print '--------- end licensecheck stdout ---------'
 450   if licensecheck.returncode != 0 or stderr:
 451     print '----------- licensecheck stderr -----------'
 452     print stderr
 453     print '--------- end licensecheck stderr ---------'
 454     print "\nFAILED\n"
 455     return 1
 456
 457   used_suppressions = set()
 458   errors = []
 459
 460   for line in stdout.splitlines():
 461     filename, license = line.split(':', 1)
 462     filename = os.path.relpath(filename.strip(), options.base_directory)
 463
 464     # All files in the build output directory are generated one way or another.
 465     # There's no need to check them.
 466     if filename.startswith('out/'):
 467       continue
 468
 469     # For now we're just interested in the license.
 470     license = license.replace('*No copyright*', '').strip()
 471
 472     # Skip generated files.
 473     if 'GENERATED FILE' in license:
 474       continue
 475
 476     if license in WHITELISTED_LICENSES:
 477       continue
 478
 479     if not options.ignore_suppressions:
 480       matched_prefixes = [
 481           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 482           if filename.startswith(prefix) and
 483           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 484       if matched_prefixes:
 485         used_suppressions.update(set(matched_prefixes))
 486         continue
 487
 488     errors.append({'filename': filename, 'license': license})
 489
 490   if options.json:
 491     with open(options.json, 'w') as f:
 492       json.dump(errors, f)
 493
 494   if errors:
 495     for error in errors:
 496       print "'%s' has non-whitelisted license '%s'" % (
 497           error['filename'], error['license'])
 498     print "\nFAILED\n"
 499     print "Please read",
 500     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 501     print "for more info how to handle the failure."
 502     print
 503     print "Please respect OWNERS of checklicenses.py. Changes violating"
 504     print "this requirement may be reverted."
 505
 506     # Do not print unused suppressions so that above message is clearly
 507     # visible and gets proper attention. Too much unrelated output
 508     # would be distracting and make the important points easier to miss.
 509
 510     return 1
 511
 512   print "\nSUCCESS\n"
 513
 514   if not len(args):
 515     unused_suppressions = set(
 516         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 517             used_suppressions)
 518     if unused_suppressions:
 519       print "\nNOTE: unused suppressions detected:\n"
 520       print '\n'.join(unused_suppressions)
 521
 522   return 0
 523
 524
 525 def main():
 526   default_root = os.path.abspath(
 527       os.path.join(os.path.dirname(__file__), '..', '..'))
 528   option_parser = optparse.OptionParser()
 529   option_parser.add_option('--root', default=default_root,
 530                            dest='base_directory',
 531                            help='Specifies the repository root. This defaults '
 532                            'to "../.." relative to the script file, which '
 533                            'will normally be the repository root.')
 534   option_parser.add_option('-v', '--verbose', action='store_true',
 535                            default=False, help='Print debug logging')
 536   option_parser.add_option('--ignore-suppressions',
 537                            action='store_true',
 538                            default=False,
 539                            help='Ignore path-specific license whitelist.')
 540   option_parser.add_option('--json', help='Path to JSON output file')
 541   options, args = option_parser.parse_args()
 542   return check_licenses(options, args)
 543
 544
 545 if '__main__' == __name__:
 546   sys.exit(main())