build/windows_toolchain.py

   1 #!/usr/bin/env python2.7
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # This script is used to create and manipulate archives containing
   7 # files necessary to build Firefox on Windows (referred to as the
   8 # "Windows toolchain").
   9 #
  10 # When updating behavior of this script, remember to update the docs
  11 # in ``build/docs/toolchains.rst``.
  12
  13 from __future__ import absolute_import, unicode_literals
  14
  15 import hashlib
  16 import os
  17 import sys
  18
  19 from mozpack.files import (
  20     FileFinder,
  21 )
  22 from mozpack.mozjar import (
  23     JarWriter,
  24 )
  25 import mozpack.path as mozpath
  26
  27
  28 # mozpack.match patterns for files under "Microsoft Visual Studio 14.0".
  29 VS_PATTERNS = [
  30     {
  31         'pattern': 'DIA SDK/bin/**',
  32         'ignore': (
  33             'DIA SDK/bin/arm/**',
  34         ),
  35     },
  36     {
  37         'pattern': 'DIA SDK/idl/**',
  38     },
  39     {
  40         'pattern': 'DIA SDK/include/**',
  41     },
  42     {
  43         'pattern': 'DIA SDK/lib/**',
  44         'ignore': (
  45             'DIA SDK/lib/arm/**',
  46         ),
  47     },
  48     # ATL is needed by Breakpad.
  49     {
  50         'pattern': 'VC/atlmfc/include/**',
  51     },
  52     {
  53         'pattern': 'VC/atlmfc/lib/atls.*',
  54     },
  55     {
  56         'pattern': 'VC/atlmfc/lib/amd64/atls.*',
  57     },
  58     {
  59         'pattern': 'VC/bin/**',
  60         # We only care about compiling on amd64 for amd64 or x86 targets.
  61         'ignore': (
  62             'VC/bin/amd64_arm/**',
  63             'VC/bin/arm/**',
  64             'VC/bin/x86_arm/**',
  65             'VC/bin/x86_amd64/**',
  66         ),
  67     },
  68     {
  69         'pattern': 'VC/include/**',
  70     },
  71     {
  72         'pattern': 'VC/lib/**',
  73         'ignore': (
  74             'VC/lib/arm/**',
  75             'VC/lib/onecore/**',
  76             'VC/lib/store/**',
  77         ),
  78     },
  79     {
  80         'pattern': 'VC/redist/x64/Microsoft.VC140.CRT/**',
  81     },
  82     {
  83         'pattern': 'VC/redist/x86/Microsoft.VC140.CRT/**',
  84     },
  85 ]
  86
  87 SDK_RELEASE = '10.0.14393.0'
  88
  89 # Files from the Windows 10 SDK to install.
  90 SDK_PATTERNS = [
  91     {
  92         'pattern': 'bin/x64/**',
  93     },
  94     {
  95         'pattern': 'Include/%s/**' % SDK_RELEASE,
  96     },
  97     {
  98         'pattern': 'Lib/%s/ucrt/x64/**' % SDK_RELEASE,
  99     },
 100     {
 101         'pattern': 'Lib/%s/ucrt/x86/**' % SDK_RELEASE,
 102     },
 103     {
 104         'pattern': 'Lib/%s/um/x64/**' % SDK_RELEASE,
 105     },
 106     {
 107         'pattern': 'Lib/%s/um/x86/**' % SDK_RELEASE,
 108     },
 109     {
 110         'pattern': 'Redist/D3D/**',
 111     },
 112     {
 113         'pattern': 'Redist/ucrt/DLLs/x64/**',
 114     },
 115     {
 116         'pattern': 'Redist/ucrt/DLLs/x86/**',
 117     },
 118 ]
 119
 120
 121 def find_vs_paths():
 122     """Resolve source locations of files.
 123
 124     Returns a 2-tuple of (Visual Studio Path, SDK Path).
 125     """
 126     pf = os.environ.get('ProgramFiles(x86)')
 127     if not pf:
 128         raise Exception('No "ProgramFiles(x86)" environment variable. '
 129                         'Not running on 64-bit Windows?')
 130
 131     vs_path = os.path.join(pf, 'Microsoft Visual Studio 14.0')
 132     if not os.path.exists(vs_path):
 133         raise Exception('%s does not exist; Visual Studio 2015 not installed?' %
 134                         vs_path)
 135
 136     sdk_path = os.path.join(pf, 'Windows Kits', '10')
 137     if not os.path.exists(sdk_path):
 138         raise Exception('%s does not exist; Windows 10 SDK not installed?' %
 139                         sdk_path)
 140
 141     return vs_path, sdk_path
 142
 143
 144 def resolve_files():
 145     """Resolve the files that constitute a standalone toolchain.
 146
 147     This is a generator of (dest path, file) where the destination
 148     path is relative and the file instance is a BaseFile from mozpack.
 149     """
 150     vs_path, sdk_path = find_vs_paths()
 151
 152     for entry in VS_PATTERNS:
 153         finder = FileFinder(vs_path, ignore=entry.get('ignore', []))
 154         for p, f in finder.find(entry['pattern']):
 155             assert p.startswith(('VC/', 'DIA SDK/'))
 156
 157             yield p.encode('utf-8'), f
 158
 159     for entry in SDK_PATTERNS:
 160         finder = FileFinder(sdk_path, ignore=entry.get('ignore', []))
 161         for p, f in finder.find(entry['pattern']):
 162             relpath = 'SDK/%s' % p
 163
 164             yield relpath.encode('utf-8'), f
 165
 166
 167 def resolve_files_and_hash(manifest):
 168     """Resolve files and hash their data.
 169
 170     This is a generator of 3-tuples of (relpath, data, mode).
 171
 172     As data is read, the manifest is populated with metadata.
 173     Keys are set to the relative file path. Values are 2-tuples
 174     of (data length, sha-256).
 175     """
 176     assert manifest == {}
 177     for p, f in resolve_files():
 178         data = f.read()
 179
 180         sha256 = hashlib.sha256()
 181         sha256.update(data)
 182         manifest[p] = (len(data), sha256.hexdigest())
 183
 184         yield p, data, f.mode
 185
 186
 187 def format_manifest(manifest):
 188     """Return formatted SHA-256 manifests as a byte strings."""
 189     sha256_lines = []
 190     for path, (length, sha256) in sorted(manifest.items()):
 191         sha256_lines.append(b'%s\t%d\t%s' % (sha256, length, path))
 192
 193     # Trailing newline.
 194     sha256_lines.append(b'')
 195
 196     return b'\n'.join(sha256_lines)
 197
 198
 199 def write_zip(zip_path, prefix=None):
 200     """Write toolchain data to a zip file."""
 201     if isinstance(prefix, unicode):
 202         prefix = prefix.encode('utf-8')
 203
 204     with JarWriter(file=zip_path, optimize=False, compress=5) as zip:
 205         manifest = {}
 206         for p, data, mode in resolve_files_and_hash(manifest):
 207             print(p)
 208             if prefix:
 209                 p = mozpath.join(prefix, p)
 210
 211             zip.add(p, data, mode=mode)
 212
 213         sha256_manifest = format_manifest(manifest)
 214
 215         sdk_path = b'SDK_VERSION'
 216         sha256_path = b'MANIFEST.SHA256'
 217         if prefix:
 218             sdk_path = mozpath.join(prefix, sdk_path)
 219             sha256_path = mozpath.join(prefix, sha256_path)
 220
 221         zip.add(sdk_path, SDK_RELEASE.encode('utf-8'))
 222         zip.add(sha256_path, sha256_manifest)
 223
 224
 225 if __name__ == '__main__':
 226     if len(sys.argv) != 3:
 227         print('usage: %s create-zip <path-prefix>' % sys.argv[0])
 228         sys.exit(1)
 229
 230     assert sys.argv[1] == 'create-zip'
 231     prefix = os.path.basename(sys.argv[2])
 232     destzip = '%s.zip' % sys.argv[2]
 233     write_zip(destzip, prefix=prefix)
 234
 235     sha1 = hashlib.sha1()
 236     sha256 = hashlib.sha256()
 237     sha512 = hashlib.sha512()
 238
 239     with open(destzip, 'rb') as fh:
 240         data = fh.read()
 241         sha1.update(data)
 242         sha256.update(data)
 243         sha512.update(data)
 244
 245     print('Hashes of %s (size=%d)' % (destzip, len(data)))
 246     print('SHA-1:   %s' % sha1.hexdigest())
 247     print('SHA-256: %s' % sha256.hexdigest())
 248     print('SHA-512: %s' % sha512.hexdigest())