build/windows_toolchain.py

   1 #!/usr/bin/env python2.7
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # This script is used to create and manipulate archives containing
   7 # files necessary to build Firefox on Windows (referred to as the
   8 # "Windows toolchain").
   9 #
  10 # When updating behavior of this script, remember to update the docs
  11 # in ``build/docs/toolchains.rst``.
  12
  13 from __future__ import absolute_import, unicode_literals
  14
  15 import hashlib
  16 import os
  17 import sys
  18
  19 from mozpack.files import FileFinder
  20 from mozpack.mozjar import JarWriter
  21 import mozpack.path as mozpath
  22
  23 SDK_RELEASE = "10.0.17134.0"
  24
  25 PATTERNS = [
  26     {
  27         "srcdir": "%(vs_path)s/DIA SDK",
  28         "dstdir": "DIA SDK",
  29         "files": [
  30             {
  31                 "pattern": "bin/**",
  32                 "ignore": ("bin/arm/**",),
  33             },
  34             {
  35                 "pattern": "idl/**",
  36             },
  37             {
  38                 "pattern": "include/**",
  39             },
  40             {
  41                 "pattern": "lib/**",
  42                 "ignore": ("lib/arm/**",),
  43             },
  44         ],
  45     },
  46     {
  47         "srcdir": "%(vs_path)s/VC/Tools/MSVC/14.16.27023",
  48         "dstdir": "VC",
  49         "files": [
  50             # ATL is needed by Breakpad.
  51             {
  52                 "pattern": "atlmfc/include/**",
  53             },
  54             {
  55                 "pattern": "atlmfc/lib/arm64/atls.*",
  56             },
  57             {
  58                 "pattern": "atlmfc/lib/x64/atls.*",
  59             },
  60             {
  61                 "pattern": "atlmfc/lib/x86/atls.*",
  62             },
  63             # ARM64 PGO-instrumented builds require ARM64 pgort140.dll.
  64             {
  65                 "pattern": "bin/arm64/pgort140.dll",
  66             },
  67             {
  68                 "pattern": "bin/Hostx64/**",
  69             },
  70             # 32-bit PGO-instrumented builds require 32-bit pgort140.dll.
  71             {
  72                 "pattern": "bin/Hostx86/x86/pgort140.dll",
  73             },
  74             {
  75                 "pattern": "include/**",
  76             },
  77             {
  78                 "pattern": "lib/**",
  79                 "ignore": (
  80                     "lib/arm64/store/**",
  81                     "lib/onecore/**",
  82                     "lib/x64/store/**",
  83                     "lib/x86/store/**",
  84                 ),
  85             },
  86         ],
  87     },
  88     {
  89         "srcdir": "%(vs_path)s/VC/Redist/MSVC/14.16.27012",
  90         "dstdir": "VC/redist",
  91         "files": [
  92             {
  93                 "pattern": "arm64/Microsoft.VC141.CRT/**",
  94             },
  95             {
  96                 "pattern": "x64/Microsoft.VC141.CRT/**",
  97             },
  98             {
  99                 "pattern": "x86/Microsoft.VC141.CRT/**",
 100             },
 101         ],
 102     },
 103     {
 104         "srcdir": "%(sdk_path)s",
 105         "dstdir": "SDK",
 106         "files": [
 107             {
 108                 "pattern": "bin/%s/x64/**" % SDK_RELEASE,
 109             },
 110             {
 111                 "pattern": "Include/%s/**" % SDK_RELEASE,
 112             },
 113             {
 114                 "pattern": "Lib/%s/ucrt/arm64/**" % SDK_RELEASE,
 115             },
 116             {
 117                 "pattern": "Lib/%s/ucrt/x64/**" % SDK_RELEASE,
 118             },
 119             {
 120                 "pattern": "Lib/%s/ucrt/x86/**" % SDK_RELEASE,
 121             },
 122             {
 123                 "pattern": "Lib/%s/um/arm64/**" % SDK_RELEASE,
 124             },
 125             {
 126                 "pattern": "Lib/%s/um/x64/**" % SDK_RELEASE,
 127             },
 128             {
 129                 "pattern": "Lib/%s/um/x86/**" % SDK_RELEASE,
 130             },
 131             {
 132                 "pattern": "Redist/D3D/**",
 133             },
 134             {
 135                 "pattern": "Redist/ucrt/DLLs/x64/**",
 136             },
 137             {
 138                 "pattern": "Redist/ucrt/DLLs/x86/**",
 139             },
 140         ],
 141     },
 142 ]
 143
 144
 145 def find_vs_paths():
 146     """Resolve source locations of files.
 147
 148     Returns a 2-tuple of (Visual Studio Path, SDK Path).
 149     """
 150     pf = os.environ.get("ProgramFiles(x86)")
 151     if not pf:
 152         raise Exception(
 153             'No "ProgramFiles(x86)" environment variable. '
 154             "Not running on 64-bit Windows?"
 155         )
 156
 157     vs_path = os.path.join(pf, "Microsoft Visual Studio", "2017", "Community")
 158     if not os.path.exists(vs_path):
 159         raise Exception(
 160             "%s does not exist; Visual Studio 2017 not installed?" % vs_path
 161         )
 162
 163     sdk_path = os.path.join(pf, "Windows Kits", "10")
 164     if not os.path.exists(sdk_path):
 165         raise Exception("%s does not exist; Windows 10 SDK not installed?" % sdk_path)
 166
 167     sdk_fullver_path = os.path.join(sdk_path, "Include", SDK_RELEASE)
 168     if not os.path.exists(sdk_fullver_path):
 169         raise Exception(
 170             "%s does not exist; Wrong SDK version installed?" % sdk_fullver_path
 171         )
 172
 173     return vs_path, sdk_path
 174
 175
 176 def resolve_files():
 177     """Resolve the files that constitute a standalone toolchain.
 178
 179     This is a generator of (dest path, file) where the destination
 180     path is relative and the file instance is a BaseFile from mozpack.
 181     """
 182     vs_path, sdk_path = find_vs_paths()
 183
 184     for entry in PATTERNS:
 185         fullpath = entry["srcdir"] % {
 186             "vs_path": vs_path,
 187             "sdk_path": sdk_path,
 188         }
 189         for pattern in entry["files"]:
 190             finder = FileFinder(fullpath, ignore=pattern.get("ignore", []))
 191             for p, f in finder.find(pattern["pattern"]):
 192                 dstpath = "%s/%s" % (entry["dstdir"], p)
 193                 yield dstpath.encode("utf-8"), f
 194
 195
 196 def resolve_files_and_hash(manifest):
 197     """Resolve files and hash their data.
 198
 199     This is a generator of 3-tuples of (relpath, data, mode).
 200
 201     As data is read, the manifest is populated with metadata.
 202     Keys are set to the relative file path. Values are 2-tuples
 203     of (data length, sha-256).
 204     """
 205     assert manifest == {}
 206     for p, f in resolve_files():
 207         data = f.read()
 208
 209         sha256 = hashlib.sha256()
 210         sha256.update(data)
 211         manifest[p] = (len(data), sha256.hexdigest())
 212
 213         yield p, data, f.mode
 214
 215
 216 def format_manifest(manifest):
 217     """Return formatted SHA-256 manifests as a byte strings."""
 218     sha256_lines = []
 219     for path, (length, sha256) in sorted(manifest.items()):
 220         sha256_lines.append(b"%s\t%d\t%s" % (sha256, length, path))
 221
 222     # Trailing newline.
 223     sha256_lines.append(b"")
 224
 225     return b"\n".join(sha256_lines)
 226
 227
 228 def write_zip(zip_path, prefix=None):
 229     """Write toolchain data to a zip file."""
 230     if isinstance(prefix, unicode):  # noqa Special case for Python 2
 231         prefix = prefix.encode("utf-8")
 232
 233     with JarWriter(file=zip_path, compress_level=5) as zip:
 234         manifest = {}
 235         for p, data, mode in resolve_files_and_hash(manifest):
 236             print(p)
 237             if prefix:
 238                 p = mozpath.join(prefix, p)
 239
 240             zip.add(p, data, mode=mode)
 241
 242         sha256_manifest = format_manifest(manifest)
 243
 244         sdk_path = b"SDK_VERSION"
 245         sha256_path = b"MANIFEST.SHA256"
 246         if prefix:
 247             sdk_path = mozpath.join(prefix, sdk_path)
 248             sha256_path = mozpath.join(prefix, sha256_path)
 249
 250         zip.add(sdk_path, SDK_RELEASE.encode("utf-8"))
 251         zip.add(sha256_path, sha256_manifest)
 252
 253
 254 if __name__ == "__main__":
 255     if len(sys.argv) != 3:
 256         print("usage: %s create-zip <path-prefix>" % sys.argv[0])
 257         sys.exit(1)
 258
 259     assert sys.argv[1] == "create-zip"
 260     prefix = os.path.basename(sys.argv[2])
 261     destzip = "%s.zip" % sys.argv[2]
 262     write_zip(destzip, prefix=prefix)
 263
 264     sha1 = hashlib.sha1()
 265     sha256 = hashlib.sha256()
 266     sha512 = hashlib.sha512()
 267
 268     with open(destzip, "rb") as fh:
 269         data = fh.read()
 270         sha1.update(data)
 271         sha256.update(data)
 272         sha512.update(data)
 273
 274     print("Hashes of %s (size=%d)" % (destzip, len(data)))
 275     print("SHA-1:   %s" % sha1.hexdigest())
 276     print("SHA-256: %s" % sha256.hexdigest())
 277     print("SHA-512: %s" % sha512.hexdigest())