build/windows_toolchain.py

   1 #!/usr/bin/env python2.7
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # This script is used to create and manipulate archives containing
   7 # files necessary to build Firefox on Windows (referred to as the
   8 # "Windows toolchain").
   9 #
  10 # When updating behavior of this script, remember to update the docs
  11 # in ``build/docs/toolchains.rst``.
  12
  13 from __future__ import absolute_import, unicode_literals
  14
  15 import hashlib
  16 import os
  17 import sys
  18
  19 from mozpack.files import (
  20     FileFinder,
  21 )
  22 from mozpack.mozjar import (
  23     JarWriter,
  24 )
  25 import mozpack.path as mozpath
  26
  27 SDK_RELEASE = '10.0.17134.0'
  28
  29 PATTERNS = [
  30     {
  31         'srcdir': '%(vs_path)s/DIA SDK',
  32         'dstdir': 'DIA SDK',
  33         'files': [
  34             {
  35                 'pattern': 'bin/**',
  36                 'ignore': (
  37                     'bin/arm/**',
  38                 ),
  39             },
  40             {
  41                 'pattern': 'idl/**',
  42             },
  43             {
  44                 'pattern': 'include/**',
  45             },
  46             {
  47                 'pattern': 'lib/**',
  48                 'ignore': (
  49                     'lib/arm/**',
  50                 ),
  51             },
  52         ],
  53     },
  54     {
  55         'srcdir': '%(vs_path)s/VC/Tools/MSVC/14.16.27023',
  56         'dstdir': 'VC',
  57         'files': [
  58             # ATL is needed by Breakpad.
  59             {
  60                 'pattern': 'atlmfc/include/**',
  61             },
  62             {
  63                 'pattern': 'atlmfc/lib/arm64/atls.*',
  64             },
  65             {
  66                 'pattern': 'atlmfc/lib/x64/atls.*',
  67             },
  68             {
  69                 'pattern': 'atlmfc/lib/x86/atls.*',
  70             },
  71             # ARM64 PGO-instrumented builds require ARM64 pgort140.dll.
  72             {
  73                 'pattern': 'bin/arm64/pgort140.dll',
  74             },
  75             {
  76                 'pattern': 'bin/Hostx64/**',
  77             },
  78             # 32-bit PGO-instrumented builds require 32-bit pgort140.dll.
  79             {
  80                 'pattern': 'bin/Hostx86/x86/pgort140.dll',
  81             },
  82             {
  83                 'pattern': 'include/**',
  84             },
  85             {
  86                 'pattern': 'lib/**',
  87                 'ignore': (
  88                     'lib/arm64/store/**',
  89                     'lib/onecore/**',
  90                     'lib/x64/store/**',
  91                     'lib/x86/store/**',
  92                 ),
  93             },
  94         ],
  95     },
  96     {
  97         'srcdir': '%(vs_path)s/VC/Redist/MSVC/14.16.27012',
  98         'dstdir': 'VC/redist',
  99         'files': [
 100             {
 101                 'pattern': 'arm64/Microsoft.VC141.CRT/**',
 102             },
 103             {
 104                 'pattern': 'x64/Microsoft.VC141.CRT/**',
 105             },
 106             {
 107                 'pattern': 'x86/Microsoft.VC141.CRT/**',
 108             },
 109         ],
 110     },
 111     {
 112         'srcdir': '%(sdk_path)s',
 113         'dstdir': 'SDK',
 114         'files': [
 115             {
 116                 'pattern': 'bin/%s/x64/**' % SDK_RELEASE,
 117             },
 118             {
 119                 'pattern': 'Include/%s/**' % SDK_RELEASE,
 120             },
 121             {
 122                 'pattern': 'Lib/%s/ucrt/arm64/**' % SDK_RELEASE,
 123             },
 124             {
 125                 'pattern': 'Lib/%s/ucrt/x64/**' % SDK_RELEASE,
 126             },
 127             {
 128                 'pattern': 'Lib/%s/ucrt/x86/**' % SDK_RELEASE,
 129             },
 130             {
 131                 'pattern': 'Lib/%s/um/arm64/**' % SDK_RELEASE,
 132             },
 133             {
 134                 'pattern': 'Lib/%s/um/x64/**' % SDK_RELEASE,
 135             },
 136             {
 137                 'pattern': 'Lib/%s/um/x86/**' % SDK_RELEASE,
 138             },
 139             {
 140                 'pattern': 'Redist/D3D/**',
 141             },
 142             {
 143                 'pattern': 'Redist/ucrt/DLLs/x64/**',
 144             },
 145             {
 146                 'pattern': 'Redist/ucrt/DLLs/x86/**',
 147             },
 148         ],
 149     },
 150 ]
 151
 152
 153 def find_vs_paths():
 154     """Resolve source locations of files.
 155
 156     Returns a 2-tuple of (Visual Studio Path, SDK Path).
 157     """
 158     pf = os.environ.get('ProgramFiles(x86)')
 159     if not pf:
 160         raise Exception('No "ProgramFiles(x86)" environment variable. '
 161                         'Not running on 64-bit Windows?')
 162
 163     vs_path = os.path.join(pf, 'Microsoft Visual Studio', '2017', 'Community')
 164     if not os.path.exists(vs_path):
 165         raise Exception('%s does not exist; Visual Studio 2017 not installed?' %
 166                         vs_path)
 167
 168     sdk_path = os.path.join(pf, 'Windows Kits', '10')
 169     if not os.path.exists(sdk_path):
 170         raise Exception('%s does not exist; Windows 10 SDK not installed?' %
 171                         sdk_path)
 172
 173     sdk_fullver_path = os.path.join(sdk_path, 'Include', SDK_RELEASE)
 174     if not os.path.exists(sdk_fullver_path):
 175         raise Exception('%s does not exist; Wrong SDK version installed?' %
 176                         sdk_fullver_path)
 177
 178     return vs_path, sdk_path
 179
 180
 181 def resolve_files():
 182     """Resolve the files that constitute a standalone toolchain.
 183
 184     This is a generator of (dest path, file) where the destination
 185     path is relative and the file instance is a BaseFile from mozpack.
 186     """
 187     vs_path, sdk_path = find_vs_paths()
 188
 189     for entry in PATTERNS:
 190         fullpath = entry['srcdir'] % {
 191             'vs_path': vs_path,
 192             'sdk_path': sdk_path,
 193         }
 194         for pattern in entry['files']:
 195             finder = FileFinder(fullpath, ignore=pattern.get('ignore', []))
 196             for p, f in finder.find(pattern['pattern']):
 197                 dstpath = '%s/%s' % (entry['dstdir'], p)
 198                 yield dstpath.encode('utf-8'), f
 199
 200
 201 def resolve_files_and_hash(manifest):
 202     """Resolve files and hash their data.
 203
 204     This is a generator of 3-tuples of (relpath, data, mode).
 205
 206     As data is read, the manifest is populated with metadata.
 207     Keys are set to the relative file path. Values are 2-tuples
 208     of (data length, sha-256).
 209     """
 210     assert manifest == {}
 211     for p, f in resolve_files():
 212         data = f.read()
 213
 214         sha256 = hashlib.sha256()
 215         sha256.update(data)
 216         manifest[p] = (len(data), sha256.hexdigest())
 217
 218         yield p, data, f.mode
 219
 220
 221 def format_manifest(manifest):
 222     """Return formatted SHA-256 manifests as a byte strings."""
 223     sha256_lines = []
 224     for path, (length, sha256) in sorted(manifest.items()):
 225         sha256_lines.append(b'%s\t%d\t%s' % (sha256, length, path))
 226
 227     # Trailing newline.
 228     sha256_lines.append(b'')
 229
 230     return b'\n'.join(sha256_lines)
 231
 232
 233 def write_zip(zip_path, prefix=None):
 234     """Write toolchain data to a zip file."""
 235     if isinstance(prefix, unicode): # noqa Special case for Python 2
 236         prefix = prefix.encode('utf-8')
 237
 238     with JarWriter(file=zip_path, compress_level=5) as zip:
 239         manifest = {}
 240         for p, data, mode in resolve_files_and_hash(manifest):
 241             print(p)
 242             if prefix:
 243                 p = mozpath.join(prefix, p)
 244
 245             zip.add(p, data, mode=mode)
 246
 247         sha256_manifest = format_manifest(manifest)
 248
 249         sdk_path = b'SDK_VERSION'
 250         sha256_path = b'MANIFEST.SHA256'
 251         if prefix:
 252             sdk_path = mozpath.join(prefix, sdk_path)
 253             sha256_path = mozpath.join(prefix, sha256_path)
 254
 255         zip.add(sdk_path, SDK_RELEASE.encode('utf-8'))
 256         zip.add(sha256_path, sha256_manifest)
 257
 258
 259 if __name__ == '__main__':
 260     if len(sys.argv) != 3:
 261         print('usage: %s create-zip <path-prefix>' % sys.argv[0])
 262         sys.exit(1)
 263
 264     assert sys.argv[1] == 'create-zip'
 265     prefix = os.path.basename(sys.argv[2])
 266     destzip = '%s.zip' % sys.argv[2]
 267     write_zip(destzip, prefix=prefix)
 268
 269     sha1 = hashlib.sha1()
 270     sha256 = hashlib.sha256()
 271     sha512 = hashlib.sha512()
 272
 273     with open(destzip, 'rb') as fh:
 274         data = fh.read()
 275         sha1.update(data)
 276         sha256.update(data)
 277         sha512.update(data)
 278
 279     print('Hashes of %s (size=%d)' % (destzip, len(data)))
 280     print('SHA-1:   %s' % sha1.hexdigest())
 281     print('SHA-256: %s' % sha256.hexdigest())
 282     print('SHA-512: %s' % sha512.hexdigest())