tools/licenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Utility for checking and processing licensing information in third_party
   7 directories.
   8
   9 Usage: licenses.py <command>
  10
  11 Commands:
  12   scan     scan third_party directories, verifying that we have licensing info
  13   credits  generate about:credits on stdout
  14
  15 (You can also import this as a module.)
  16 """
  17
  18 import cgi
  19 import os
  20 import sys
  21
  22 # Paths from the root of the tree to directories to skip.
  23 PRUNE_PATHS = set([
  24     # Same module occurs in crypto/third_party/nss and net/third_party/nss, so
  25     # skip this one.
  26     os.path.join('third_party','nss'),
  27
  28     # Placeholder directory only, not third-party code.
  29     os.path.join('third_party','adobe'),
  30
  31     # Build files only, not third-party code.
  32     os.path.join('third_party','widevine'),
  33
  34     # Only binaries, used during development.
  35     os.path.join('third_party','valgrind'),
  36
  37     # Used for development and test, not in the shipping product.
  38     os.path.join('third_party','bidichecker'),
  39     os.path.join('third_party','cygwin'),
  40     os.path.join('third_party','gold'),
  41     os.path.join('third_party','lighttpd'),
  42     os.path.join('third_party','mingw-w64'),
  43     os.path.join('third_party','pefile'),
  44     os.path.join('third_party','python_26'),
  45     os.path.join('third_party','pywebsocket'),
  46
  47     # Stuff pulled in from chrome-internal for official builds/tools.
  48     os.path.join('third_party', 'clear_cache'),
  49     os.path.join('third_party', 'gnu'),
  50     os.path.join('third_party', 'googlemac'),
  51     os.path.join('third_party', 'pcre'),
  52     os.path.join('third_party', 'psutils'),
  53     os.path.join('third_party', 'sawbuck'),
  54
  55     # Redistribution does not require attribution in documentation.
  56     os.path.join('third_party','directxsdk'),
  57     os.path.join('third_party','platformsdk_win2008_6_1'),
  58     os.path.join('third_party','platformsdk_win7'),
  59 ])
  60
  61 # Directories we don't scan through.
  62 PRUNE_DIRS = ('.svn', '.git',             # VCS metadata
  63               'out', 'Debug', 'Release',  # build files
  64               'layout_tests')             # lots of subdirs
  65
  66 ADDITIONAL_PATHS = (
  67     os.path.join('breakpad'),
  68     os.path.join('chrome', 'common', 'extensions', 'docs', 'examples'),
  69     os.path.join('chrome', 'test', 'chromeos', 'autotest'),
  70     os.path.join('chrome', 'test', 'data'),
  71     os.path.join('googleurl'),
  72     os.path.join('native_client'),
  73     os.path.join('native_client_sdk'),
  74     os.path.join('net', 'tools', 'spdyshark'),
  75     os.path.join('ppapi'),
  76     os.path.join('sandbox', 'linux', 'seccomp-legacy'),
  77     os.path.join('sdch', 'open-vcdiff'),
  78     os.path.join('testing', 'gmock'),
  79     os.path.join('testing', 'gtest'),
  80     # The directory with the word list for Chinese and Japanese segmentation
  81     # with different license terms than ICU.
  82     os.path.join('third_party','icu','source','data','brkitr'),
  83     os.path.join('tools', 'grit'),
  84     os.path.join('tools', 'gyp'),
  85     os.path.join('tools', 'page_cycler', 'acid3'),
  86     os.path.join('v8'),
  87     # Fake directory so we can include the strongtalk license.
  88     os.path.join('v8', 'strongtalk'),
  89 )
  90
  91
  92 # Directories where we check out directly from upstream, and therefore
  93 # can't provide a README.chromium.  Please prefer a README.chromium
  94 # wherever possible.
  95 SPECIAL_CASES = {
  96     os.path.join('googleurl'): {
  97         "Name": "google-url",
  98         "URL": "http://code.google.com/p/google-url/",
  99         "License": "BSD and MPL 1.1/GPL 2.0/LGPL 2.1",
 100         "License File": "LICENSE.txt",
 101     },
 102     os.path.join('native_client'): {
 103         "Name": "native client",
 104         "URL": "http://code.google.com/p/nativeclient",
 105         "License": "BSD",
 106     },
 107     os.path.join('sandbox', 'linux', 'seccomp-legacy'): {
 108         "Name": "seccompsandbox",
 109         "URL": "http://code.google.com/p/seccompsandbox",
 110         "License": "BSD",
 111     },
 112     os.path.join('sdch', 'open-vcdiff'): {
 113         "Name": "open-vcdiff",
 114         "URL": "http://code.google.com/p/open-vcdiff",
 115         "License": "Apache 2.0, MIT, GPL v2 and custom licenses",
 116         "License Android Compatible": "yes",
 117     },
 118     os.path.join('testing', 'gmock'): {
 119         "Name": "gmock",
 120         "URL": "http://code.google.com/p/googlemock",
 121         "License": "BSD",
 122     },
 123     os.path.join('testing', 'gtest'): {
 124         "Name": "gtest",
 125         "URL": "http://code.google.com/p/googletest",
 126         "License": "BSD",
 127     },
 128     os.path.join('third_party', 'angle'): {
 129         "Name": "Almost Native Graphics Layer Engine",
 130         "URL": "http://code.google.com/p/angleproject/",
 131         "License": "BSD",
 132     },
 133     os.path.join('third_party', 'cros_system_api'): {
 134         "Name": "Chromium OS system API",
 135         "URL": "http://www.chromium.org/chromium-os",
 136         "License": "BSD",
 137         # Absolute path here is resolved as relative to the source root.
 138         "License File": "/LICENSE.chromium_os",
 139     },
 140     os.path.join('third_party', 'GTM'): {
 141         "Name": "Google Toolbox for Mac",
 142         "URL": "http://code.google.com/p/google-toolbox-for-mac/",
 143         "License": "Apache 2.0",
 144         "License File": "COPYING",
 145     },
 146     os.path.join('third_party', 'lss'): {
 147         "Name": "linux-syscall-support",
 148         "URL": "http://code.google.com/p/lss/",
 149         "License": "BSD",
 150         "License File": "/LICENSE",
 151     },
 152     os.path.join('third_party', 'ots'): {
 153         "Name": "OTS (OpenType Sanitizer)",
 154         "URL": "http://code.google.com/p/ots/",
 155         "License": "BSD",
 156     },
 157     os.path.join('third_party', 'pdfsqueeze'): {
 158         "Name": "pdfsqueeze",
 159         "URL": "http://code.google.com/p/pdfsqueeze/",
 160         "License": "Apache 2.0",
 161         "License File": "COPYING",
 162     },
 163     os.path.join('third_party', 'ppapi'): {
 164         "Name": "ppapi",
 165         "URL": "http://code.google.com/p/ppapi/",
 166     },
 167     os.path.join('third_party', 'scons-2.0.1'): {
 168         "Name": "scons-2.0.1",
 169         "URL": "http://www.scons.org",
 170         "License": "MIT",
 171     },
 172     os.path.join('third_party', 'trace-viewer'): {
 173         "Name": "trace-viewer",
 174         "URL": "http://code.google.com/p/trace-viewer",
 175         "License": "BSD",
 176     },
 177     os.path.join('third_party', 'v8-i18n'): {
 178         "Name": "Internationalization Library for v8",
 179         "URL": "http://code.google.com/p/v8-i18n/",
 180         "License": "Apache 2.0, BSD and others",
 181     },
 182     os.path.join('third_party', 'WebKit'): {
 183         "Name": "WebKit",
 184         "URL": "http://webkit.org/",
 185         "License": "BSD and GPL v2",
 186         # Absolute path here is resolved as relative to the source root.
 187         "License File": "/webkit/LICENSE",
 188     },
 189     os.path.join('third_party', 'webpagereplay'): {
 190         "Name": "webpagereplay",
 191         "URL": "http://code.google.com/p/web-page-replay",
 192         "License": "Apache 2.0",
 193     },
 194     os.path.join('tools', 'grit'): {
 195         "Name": "grit",
 196         "URL": "http://code.google.com/p/grit-i18n",
 197         "License": "BSD",
 198     },
 199     os.path.join('tools', 'gyp'): {
 200         "Name": "gyp",
 201         "URL": "http://code.google.com/p/gyp",
 202         "License": "BSD",
 203     },
 204     os.path.join('v8'): {
 205         "Name": "V8 JavaScript Engine",
 206         "URL": "http://code.google.com/p/v8",
 207         "License": "BSD",
 208     },
 209     os.path.join('v8', 'strongtalk'): {
 210         "Name": "Strongtalk",
 211         "URL": "http://www.strongtalk.org/",
 212         "License": "BSD",
 213         # Absolute path here is resolved as relative to the source root.
 214         "License File": "/v8/LICENSE.strongtalk",
 215     },
 216 }
 217
 218 # Special value for 'License File' field used to indicate that the license file
 219 # should not be used in about:credits.
 220 NOT_SHIPPED = "NOT_SHIPPED"
 221
 222
 223 class LicenseError(Exception):
 224     """We raise this exception when a directory's licensing info isn't
 225     fully filled out."""
 226     pass
 227
 228 def AbsolutePath(path, filename, root):
 229     """Convert a path in README.chromium to be absolute based on the source
 230     root."""
 231     if filename.startswith('/'):
 232         # Absolute-looking paths are relative to the source root
 233         # (which is the directory we're run from).
 234         absolute_path = os.path.join(root, filename[1:])
 235     else:
 236         absolute_path = os.path.join(root, path, filename)
 237     if os.path.exists(absolute_path):
 238         return absolute_path
 239     return None
 240
 241 def ParseDir(path, root, require_license_file=True):
 242     """Examine a third_party/foo component and extract its metadata."""
 243
 244     # Parse metadata fields out of README.chromium.
 245     # We examine "LICENSE" for the license file by default.
 246     metadata = {
 247         "License File": "LICENSE",  # Relative path to license text.
 248         "Name": None,               # Short name (for header on about:credits).
 249         "URL": None,                # Project home page.
 250         "License": None,            # Software license.
 251         }
 252
 253     # Relative path to a file containing some html we're required to place in
 254     # about:credits.
 255     optional_keys = ["Required Text", "License Android Compatible"]
 256
 257     if path in SPECIAL_CASES:
 258         metadata.update(SPECIAL_CASES[path])
 259     else:
 260         # Try to find README.chromium.
 261         readme_path = os.path.join(root, path, 'README.chromium')
 262         if not os.path.exists(readme_path):
 263             raise LicenseError("missing README.chromium or licenses.py "
 264                                "SPECIAL_CASES entry")
 265
 266         for line in open(readme_path):
 267             line = line.strip()
 268             if not line:
 269                 break
 270             for key in metadata.keys() + optional_keys:
 271                 field = key + ": "
 272                 if line.startswith(field):
 273                     metadata[key] = line[len(field):]
 274
 275     # Check that all expected metadata is present.
 276     for key, value in metadata.iteritems():
 277         if not value:
 278             raise LicenseError("couldn't find '" + key + "' line "
 279                                "in README.chromium or licences.py "
 280                                "SPECIAL_CASES")
 281
 282     # Special-case modules that aren't in the shipping product, so don't need
 283     # their license in about:credits.
 284     if metadata["License File"] != NOT_SHIPPED:
 285         # Check that the license file exists.
 286         for filename in (metadata["License File"], "COPYING"):
 287             license_path = AbsolutePath(path, filename, root)
 288             if license_path is not None:
 289                 break
 290
 291         if require_license_file and not license_path:
 292             raise LicenseError("License file not found. "
 293                                "Either add a file named LICENSE, "
 294                                "import upstream's COPYING if available, "
 295                                "or add a 'License File:' line to "
 296                                "README.chromium with the appropriate path.")
 297         metadata["License File"] = license_path
 298
 299     if "Required Text" in metadata:
 300         required_path = AbsolutePath(path, metadata["Required Text"], root)
 301         if required_path is not None:
 302             metadata["Required Text"] = required_path
 303         else:
 304             raise LicenseError("Required text file listed but not found.")
 305
 306     return metadata
 307
 308
 309 def ContainsFiles(path, root):
 310     """Determines whether any files exist in a directory or in any of its
 311     subdirectories."""
 312     for _, _, files in os.walk(os.path.join(root, path)):
 313         if files:
 314             return True
 315     return False
 316
 317
 318 def FindThirdPartyDirs(prune_paths, root):
 319     """Find all third_party directories underneath the source root."""
 320     third_party_dirs = []
 321     for path, dirs, files in os.walk(root):
 322         path = path[len(root)+1:]  # Pretty up the path.
 323
 324         if path in prune_paths:
 325             dirs[:] = []
 326             continue
 327
 328         # Prune out directories we want to skip.
 329         # (Note that we loop over PRUNE_DIRS so we're not iterating over a
 330         # list that we're simultaneously mutating.)
 331         for skip in PRUNE_DIRS:
 332             if skip in dirs:
 333                 dirs.remove(skip)
 334
 335         if os.path.basename(path) == 'third_party':
 336             # Add all subdirectories that are not marked for skipping.
 337             for dir in dirs:
 338                 dirpath = os.path.join(path, dir)
 339                 if dirpath not in prune_paths:
 340                     third_party_dirs.append(dirpath)
 341
 342             # Don't recurse into any subdirs from here.
 343             dirs[:] = []
 344             continue
 345
 346         # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular
 347         # third_party/foo paths.
 348         if path in ADDITIONAL_PATHS:
 349             dirs[:] = []
 350
 351     for dir in ADDITIONAL_PATHS:
 352         third_party_dirs.append(dir)
 353
 354     # If a directory contains no files, assume it's a DEPS directory for a
 355     # project not used by our current configuration and skip it.
 356     return [x for x in third_party_dirs if ContainsFiles(x, root)]
 357
 358
 359 def ScanThirdPartyDirs(root=None):
 360     """Scan a list of directories and report on any problems we find."""
 361     if root is None:
 362       root = os.getcwd()
 363     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
 364
 365     errors = []
 366     for path in sorted(third_party_dirs):
 367         try:
 368             metadata = ParseDir(path, root)
 369         except LicenseError, e:
 370             errors.append((path, e.args[0]))
 371             continue
 372
 373     for path, error in sorted(errors):
 374         print path + ": " + error
 375
 376     return len(errors) == 0
 377
 378
 379 def GenerateCredits(root=None):
 380     """Generate about:credits, dumping the result to stdout."""
 381
 382     def EvaluateTemplate(template, env, escape=True):
 383         """Expand a template with variables like {{foo}} using a
 384         dictionary of expansions."""
 385         for key, val in env.items():
 386             if escape and not key.endswith("_unescaped"):
 387                 val = cgi.escape(val)
 388             template = template.replace('{{%s}}' % key, val)
 389         return template
 390
 391     if root is None:
 392       root = os.getcwd()
 393     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
 394
 395     entry_template = open(os.path.join(root, 'chrome', 'browser', 'resources',
 396                                        'about_credits_entry.tmpl'), 'rb').read()
 397     entries = []
 398     for path in sorted(third_party_dirs):
 399         try:
 400             metadata = ParseDir(path, root)
 401         except LicenseError:
 402             print >>sys.stderr, ("WARNING: licensing info for " + path +
 403                                  " is incomplete, skipping.")
 404             continue
 405         if metadata['License File'] == NOT_SHIPPED:
 406             print >>sys.stderr, ("Path " + path + " marked as " + NOT_SHIPPED +
 407                                  ", skipping.")
 408             continue
 409         env = {
 410             'name': metadata['Name'],
 411             'url': metadata['URL'],
 412             'license': open(metadata['License File'], 'rb').read(),
 413             'license_unescaped': '',
 414         }
 415         if 'Required Text' in metadata:
 416             required_text = open(metadata['Required Text'], 'rb').read()
 417             env["license_unescaped"] = required_text
 418         entries.append(EvaluateTemplate(entry_template, env))
 419
 420     file_template = open(os.path.join(root, 'chrome', 'browser', 'resources',
 421                                       'about_credits.tmpl'), 'rb').read()
 422     print "<!-- Generated by licenses.py; do not edit. -->"
 423     print EvaluateTemplate(file_template, {'entries': '\n'.join(entries)},
 424                            escape=False)
 425
 426
 427 def main():
 428     command = 'help'
 429     if len(sys.argv) > 1:
 430         command = sys.argv[1]
 431
 432     if command == 'scan':
 433         if not ScanThirdPartyDirs():
 434             return 1
 435     elif command == 'credits':
 436         if not GenerateCredits():
 437             return 1
 438     else:
 439         print __doc__
 440         return 1
 441
 442
 443 if __name__ == '__main__':
 444   sys.exit(main())