2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Utility for checking and processing licensing information in third_party
9 Usage: licenses.py <command>
12 scan scan third_party directories, verifying that we have licensing info
13 credits generate about:credits on stdout
15 (You can also import this as a module.)
22 # Paths from the root of the tree to directories to skip.
24 # Same module occurs in crypto/third_party/nss and net/third_party/nss, so
26 os
.path
.join('third_party','nss'),
28 # Placeholder directory only, not third-party code.
29 os
.path
.join('third_party','adobe'),
31 # Build files only, not third-party code.
32 os
.path
.join('third_party','widevine'),
34 # Only binaries, used during development.
35 os
.path
.join('third_party','valgrind'),
37 # Used for development and test, not in the shipping product.
38 os
.path
.join('third_party','bison'),
39 os
.path
.join('third_party','cygwin'),
40 os
.path
.join('third_party','gnu_binutils'),
41 os
.path
.join('third_party','gold'),
42 os
.path
.join('third_party','gperf'),
43 os
.path
.join('third_party','lighttpd'),
44 os
.path
.join('third_party','llvm'),
45 os
.path
.join('third_party','llvm-build'),
46 os
.path
.join('third_party','mingw-w64'),
47 os
.path
.join('third_party','nacl_sdk_binaries'),
48 os
.path
.join('third_party','pefile'),
49 os
.path
.join('third_party','perl'),
50 os
.path
.join('third_party','psyco_win32'),
51 os
.path
.join('third_party','pylib'),
52 os
.path
.join('third_party','python_26'),
53 os
.path
.join('third_party','pywebsocket'),
54 os
.path
.join('third_party','syzygy'),
55 os
.path
.join('tools','gn'),
57 # Chromium code in third_party.
58 os
.path
.join('third_party','fuzzymatch'),
59 os
.path
.join('tools', 'swarming_client'),
61 # Stuff pulled in from chrome-internal for official builds/tools.
62 os
.path
.join('third_party', 'clear_cache'),
63 os
.path
.join('third_party', 'gnu'),
64 os
.path
.join('third_party', 'googlemac'),
65 os
.path
.join('third_party', 'pcre'),
66 os
.path
.join('third_party', 'psutils'),
67 os
.path
.join('third_party', 'sawbuck'),
69 # Redistribution does not require attribution in documentation.
70 os
.path
.join('third_party','directxsdk'),
71 os
.path
.join('third_party','platformsdk_win2008_6_1'),
72 os
.path
.join('third_party','platformsdk_win7'),
75 # Directories we don't scan through.
76 VCS_METADATA_DIRS
= ('.svn', '.git')
77 PRUNE_DIRS
= (VCS_METADATA_DIRS
+
78 ('out', 'Debug', 'Release', # build files
79 'layout_tests')) # lots of subdirs
82 os
.path
.join('breakpad'),
83 os
.path
.join('chrome', 'common', 'extensions', 'docs', 'examples'),
84 os
.path
.join('chrome', 'test', 'chromeos', 'autotest'),
85 os
.path
.join('chrome', 'test', 'data'),
86 os
.path
.join('native_client'),
87 os
.path
.join('net', 'tools', 'spdyshark'),
88 os
.path
.join('sdch', 'open-vcdiff'),
89 os
.path
.join('testing', 'gmock'),
90 os
.path
.join('testing', 'gtest'),
91 # The directory with the word list for Chinese and Japanese segmentation
92 # with different license terms than ICU.
93 os
.path
.join('third_party','icu','source','data','brkitr'),
94 os
.path
.join('tools', 'grit'),
95 os
.path
.join('tools', 'gyp'),
96 os
.path
.join('tools', 'page_cycler', 'acid3'),
97 os
.path
.join('url', 'third_party', 'mozilla'),
99 # Fake directory so we can include the strongtalk license.
100 os
.path
.join('v8', 'strongtalk'),
104 # Directories where we check out directly from upstream, and therefore
105 # can't provide a README.chromium. Please prefer a README.chromium
108 os
.path
.join('native_client'): {
109 "Name": "native client",
110 "URL": "http://code.google.com/p/nativeclient",
113 os
.path
.join('sdch', 'open-vcdiff'): {
114 "Name": "open-vcdiff",
115 "URL": "http://code.google.com/p/open-vcdiff",
116 "License": "Apache 2.0, MIT, GPL v2 and custom licenses",
117 "License Android Compatible": "yes",
119 os
.path
.join('testing', 'gmock'): {
121 "URL": "http://code.google.com/p/googlemock",
123 "License File": "NOT_SHIPPED",
125 os
.path
.join('testing', 'gtest'): {
127 "URL": "http://code.google.com/p/googletest",
129 "License File": "NOT_SHIPPED",
131 os
.path
.join('third_party', 'angle'): {
132 "Name": "Almost Native Graphics Layer Engine",
133 "URL": "http://code.google.com/p/angleproject/",
136 os
.path
.join('third_party', 'cros_system_api'): {
137 "Name": "Chromium OS system API",
138 "URL": "http://www.chromium.org/chromium-os",
140 # Absolute path here is resolved as relative to the source root.
141 "License File": "/LICENSE.chromium_os",
143 os
.path
.join('third_party', 'lss'): {
144 "Name": "linux-syscall-support",
145 "URL": "http://code.google.com/p/linux-syscall-support/",
147 "License File": "/LICENSE",
149 os
.path
.join('third_party', 'ots'): {
150 "Name": "OTS (OpenType Sanitizer)",
151 "URL": "http://code.google.com/p/ots/",
154 os
.path
.join('third_party', 'pdfsqueeze'): {
155 "Name": "pdfsqueeze",
156 "URL": "http://code.google.com/p/pdfsqueeze/",
157 "License": "Apache 2.0",
158 "License File": "COPYING",
160 os
.path
.join('third_party', 'ppapi'): {
162 "URL": "http://code.google.com/p/ppapi/",
164 os
.path
.join('third_party', 'scons-2.0.1'): {
165 "Name": "scons-2.0.1",
166 "URL": "http://www.scons.org",
168 "License File": "NOT_SHIPPED",
170 os
.path
.join('third_party', 'trace-viewer'): {
171 "Name": "trace-viewer",
172 "URL": "http://code.google.com/p/trace-viewer",
174 "License File": "NOT_SHIPPED",
176 os
.path
.join('third_party', 'v8-i18n'): {
177 "Name": "Internationalization Library for v8",
178 "URL": "http://code.google.com/p/v8-i18n/",
179 "License": "Apache 2.0",
181 os
.path
.join('third_party', 'WebKit'): {
183 "URL": "http://webkit.org/",
184 "License": "BSD and GPL v2",
185 # Absolute path here is resolved as relative to the source root.
186 "License File": "/webkit/LICENSE",
188 os
.path
.join('third_party', 'webpagereplay'): {
189 "Name": "webpagereplay",
190 "URL": "http://code.google.com/p/web-page-replay",
191 "License": "Apache 2.0",
192 "License File": "NOT_SHIPPED",
194 os
.path
.join('tools', 'grit'): {
196 "URL": "http://code.google.com/p/grit-i18n",
198 "License File": "NOT_SHIPPED",
200 os
.path
.join('tools', 'gyp'): {
202 "URL": "http://code.google.com/p/gyp",
204 "License File": "NOT_SHIPPED",
206 os
.path
.join('v8'): {
207 "Name": "V8 JavaScript Engine",
208 "URL": "http://code.google.com/p/v8",
211 os
.path
.join('v8', 'strongtalk'): {
212 "Name": "Strongtalk",
213 "URL": "http://www.strongtalk.org/",
215 # Absolute path here is resolved as relative to the source root.
216 "License File": "/v8/LICENSE.strongtalk",
220 # Special value for 'License File' field used to indicate that the license file
221 # should not be used in about:credits.
222 NOT_SHIPPED
= "NOT_SHIPPED"
225 class LicenseError(Exception):
226 """We raise this exception when a directory's licensing info isn't
230 def AbsolutePath(path
, filename
, root
):
231 """Convert a path in README.chromium to be absolute based on the source
233 if filename
.startswith('/'):
234 # Absolute-looking paths are relative to the source root
235 # (which is the directory we're run from).
236 absolute_path
= os
.path
.join(root
, filename
[1:])
238 absolute_path
= os
.path
.join(root
, path
, filename
)
239 if os
.path
.exists(absolute_path
):
243 def ParseDir(path
, root
, require_license_file
=True):
244 """Examine a third_party/foo component and extract its metadata."""
246 # Parse metadata fields out of README.chromium.
247 # We examine "LICENSE" for the license file by default.
249 "License File": "LICENSE", # Relative path to license text.
250 "Name": None, # Short name (for header on about:credits).
251 "URL": None, # Project home page.
252 "License": None, # Software license.
255 # Relative path to a file containing some html we're required to place in
257 optional_keys
= ["Required Text", "License Android Compatible"]
259 if path
in SPECIAL_CASES
:
260 metadata
.update(SPECIAL_CASES
[path
])
262 # Try to find README.chromium.
263 readme_path
= os
.path
.join(root
, path
, 'README.chromium')
264 if not os
.path
.exists(readme_path
):
265 raise LicenseError("missing README.chromium or licenses.py "
266 "SPECIAL_CASES entry")
268 for line
in open(readme_path
):
272 for key
in metadata
.keys() + optional_keys
:
274 if line
.startswith(field
):
275 metadata
[key
] = line
[len(field
):]
277 # Check that all expected metadata is present.
278 for key
, value
in metadata
.iteritems():
280 raise LicenseError("couldn't find '" + key
+ "' line "
281 "in README.chromium or licences.py "
284 # Special-case modules that aren't in the shipping product, so don't need
285 # their license in about:credits.
286 if metadata
["License File"] != NOT_SHIPPED
:
287 # Check that the license file exists.
288 for filename
in (metadata
["License File"], "COPYING"):
289 license_path
= AbsolutePath(path
, filename
, root
)
290 if license_path
is not None:
293 if require_license_file
and not license_path
:
294 raise LicenseError("License file not found. "
295 "Either add a file named LICENSE, "
296 "import upstream's COPYING if available, "
297 "or add a 'License File:' line to "
298 "README.chromium with the appropriate path.")
299 metadata
["License File"] = license_path
301 if "Required Text" in metadata
:
302 required_path
= AbsolutePath(path
, metadata
["Required Text"], root
)
303 if required_path
is not None:
304 metadata
["Required Text"] = required_path
306 raise LicenseError("Required text file listed but not found.")
311 def ContainsFiles(path
, root
):
312 """Determines whether any files exist in a directory or in any of its
314 for _
, dirs
, files
in os
.walk(os
.path
.join(root
, path
)):
317 for vcs_metadata
in VCS_METADATA_DIRS
:
318 if vcs_metadata
in dirs
:
319 dirs
.remove(vcs_metadata
)
323 def FilterDirsWithFiles(dirs_list
, root
):
324 # If a directory contains no files, assume it's a DEPS directory for a
325 # project not used by our current configuration and skip it.
326 return [x
for x
in dirs_list
if ContainsFiles(x
, root
)]
329 def FindThirdPartyDirs(prune_paths
, root
):
330 """Find all third_party directories underneath the source root."""
331 third_party_dirs
= set()
332 for path
, dirs
, files
in os
.walk(root
):
333 path
= path
[len(root
)+1:] # Pretty up the path.
335 if path
in prune_paths
:
339 # Prune out directories we want to skip.
340 # (Note that we loop over PRUNE_DIRS so we're not iterating over a
341 # list that we're simultaneously mutating.)
342 for skip
in PRUNE_DIRS
:
346 if os
.path
.basename(path
) == 'third_party':
347 # Add all subdirectories that are not marked for skipping.
349 dirpath
= os
.path
.join(path
, dir)
350 if dirpath
not in prune_paths
:
351 third_party_dirs
.add(dirpath
)
353 # Don't recurse into any subdirs from here.
357 # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular
358 # third_party/foo paths.
359 if path
in ADDITIONAL_PATHS
:
362 for dir in ADDITIONAL_PATHS
:
363 if dir not in prune_paths
:
364 third_party_dirs
.add(dir)
366 return third_party_dirs
369 def ScanThirdPartyDirs(root
=None):
370 """Scan a list of directories and report on any problems we find."""
373 third_party_dirs
= FindThirdPartyDirs(PRUNE_PATHS
, root
)
374 third_party_dirs
= FilterDirsWithFiles(third_party_dirs
, root
)
377 for path
in sorted(third_party_dirs
):
379 metadata
= ParseDir(path
, root
)
380 except LicenseError
, e
:
381 errors
.append((path
, e
.args
[0]))
384 for path
, error
in sorted(errors
):
385 print path
+ ": " + error
387 return len(errors
) == 0
390 def GenerateCredits():
391 """Generate about:credits."""
393 if len(sys
.argv
) not in (2, 3):
394 print 'usage: licenses.py credits [output_file]'
397 def EvaluateTemplate(template
, env
, escape
=True):
398 """Expand a template with variables like {{foo}} using a
399 dictionary of expansions."""
400 for key
, val
in env
.items():
401 if escape
and not key
.endswith("_unescaped"):
402 val
= cgi
.escape(val
)
403 template
= template
.replace('{{%s}}' % key
, val
)
406 root
= os
.path
.join(os
.path
.dirname(__file__
), '..')
407 third_party_dirs
= FindThirdPartyDirs(PRUNE_PATHS
, root
)
409 entry_template
= open(os
.path
.join(root
, 'chrome', 'browser', 'resources',
410 'about_credits_entry.tmpl'), 'rb').read()
412 for path
in sorted(third_party_dirs
):
414 metadata
= ParseDir(path
, root
)
416 # TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240).
418 if metadata
['License File'] == NOT_SHIPPED
:
421 'name': metadata
['Name'],
422 'url': metadata
['URL'],
423 'license': open(metadata
['License File'], 'rb').read(),
424 'license_unescaped': '',
426 if 'Required Text' in metadata
:
427 required_text
= open(metadata
['Required Text'], 'rb').read()
428 env
["license_unescaped"] = required_text
429 entries
.append(EvaluateTemplate(entry_template
, env
))
431 file_template
= open(os
.path
.join(root
, 'chrome', 'browser', 'resources',
432 'about_credits.tmpl'), 'rb').read()
433 template_contents
= "<!-- Generated by licenses.py; do not edit. -->"
434 template_contents
+= EvaluateTemplate(file_template
,
435 {'entries': '\n'.join(entries
)},
438 if len(sys
.argv
) == 3:
439 with
open(sys
.argv
[2], 'w') as output_file
:
440 output_file
.write(template_contents
)
441 elif len(sys
.argv
) == 2:
442 print template_contents
449 if len(sys
.argv
) > 1:
450 command
= sys
.argv
[1]
452 if command
== 'scan':
453 if not ScanThirdPartyDirs():
455 elif command
== 'credits':
456 if not GenerateCredits():
463 if __name__
== '__main__':