build/windows_toolchain.py

   1 #!/usr/bin/env python2.7
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # This script is used to create and manipulate archives containing
   7 # files necessary to build Firefox on Windows (referred to as the
   8 # "Windows toolchain").
   9 #
  10 # When updating behavior of this script, remember to update the docs
  11 # in ``build/docs/toolchains.rst``.
  12
  13 from __future__ import absolute_import, unicode_literals
  14
  15 import hashlib
  16 import os
  17 import sys
  18
  19 import six
  20
  21 from mozpack.files import FileFinder
  22 from mozpack.mozjar import JarWriter
  23 import mozpack.path as mozpath
  24
  25 SDK_RELEASE = "10.0.17134.0"
  26
  27 PATTERNS = [
  28     {
  29         "srcdir": "%(vs_path)s/DIA SDK",
  30         "dstdir": "DIA SDK",
  31         "files": [
  32             {
  33                 "pattern": "bin/**",
  34                 "ignore": ("bin/arm/**",),
  35             },
  36             {
  37                 "pattern": "idl/**",
  38             },
  39             {
  40                 "pattern": "include/**",
  41             },
  42             {
  43                 "pattern": "lib/**",
  44                 "ignore": ("lib/arm/**",),
  45             },
  46         ],
  47     },
  48     {
  49         "srcdir": "%(vs_path)s/VC/Tools/MSVC/14.16.27023",
  50         "dstdir": "VC",
  51         "files": [
  52             # ATL is needed by Breakpad.
  53             {
  54                 "pattern": "atlmfc/include/**",
  55             },
  56             {
  57                 "pattern": "atlmfc/lib/arm64/atls.*",
  58             },
  59             {
  60                 "pattern": "atlmfc/lib/x64/atls.*",
  61             },
  62             {
  63                 "pattern": "atlmfc/lib/x86/atls.*",
  64             },
  65             # ARM64 PGO-instrumented builds require ARM64 pgort140.dll.
  66             {
  67                 "pattern": "bin/arm64/pgort140.dll",
  68             },
  69             {
  70                 "pattern": "bin/Hostx64/**",
  71             },
  72             # 32-bit PGO-instrumented builds require 32-bit pgort140.dll.
  73             {
  74                 "pattern": "bin/Hostx86/x86/pgort140.dll",
  75             },
  76             {
  77                 "pattern": "include/**",
  78             },
  79             {
  80                 "pattern": "lib/**",
  81                 "ignore": (
  82                     "lib/arm64/store/**",
  83                     "lib/onecore/**",
  84                     "lib/x64/store/**",
  85                     "lib/x86/store/**",
  86                 ),
  87             },
  88         ],
  89     },
  90     {
  91         "srcdir": "%(vs_path)s/VC/Redist/MSVC/14.16.27012",
  92         "dstdir": "VC/redist",
  93         "files": [
  94             {
  95                 "pattern": "arm64/Microsoft.VC141.CRT/**",
  96             },
  97             {
  98                 "pattern": "x64/Microsoft.VC141.CRT/**",
  99             },
 100             {
 101                 "pattern": "x86/Microsoft.VC141.CRT/**",
 102             },
 103         ],
 104     },
 105     {
 106         "srcdir": "%(sdk_path)s",
 107         "dstdir": "SDK",
 108         "files": [
 109             {
 110                 "pattern": "bin/%s/x64/**" % SDK_RELEASE,
 111             },
 112             {
 113                 "pattern": "Include/%s/**" % SDK_RELEASE,
 114             },
 115             {
 116                 "pattern": "Lib/%s/ucrt/arm64/**" % SDK_RELEASE,
 117             },
 118             {
 119                 "pattern": "Lib/%s/ucrt/x64/**" % SDK_RELEASE,
 120             },
 121             {
 122                 "pattern": "Lib/%s/ucrt/x86/**" % SDK_RELEASE,
 123             },
 124             {
 125                 "pattern": "Lib/%s/um/arm64/**" % SDK_RELEASE,
 126             },
 127             {
 128                 "pattern": "Lib/%s/um/x64/**" % SDK_RELEASE,
 129             },
 130             {
 131                 "pattern": "Lib/%s/um/x86/**" % SDK_RELEASE,
 132             },
 133             {
 134                 "pattern": "Redist/D3D/**",
 135             },
 136             {
 137                 "pattern": "Redist/ucrt/DLLs/x64/**",
 138             },
 139             {
 140                 "pattern": "Redist/ucrt/DLLs/x86/**",
 141             },
 142         ],
 143     },
 144 ]
 145
 146
 147 def find_vs_paths():
 148     """Resolve source locations of files.
 149
 150     Returns a 2-tuple of (Visual Studio Path, SDK Path).
 151     """
 152     pf = os.environ.get("ProgramFiles(x86)")
 153     if not pf:
 154         raise Exception(
 155             'No "ProgramFiles(x86)" environment variable. '
 156             "Not running on 64-bit Windows?"
 157         )
 158
 159     vs_path = os.path.join(pf, "Microsoft Visual Studio", "2017", "Community")
 160     if not os.path.exists(vs_path):
 161         raise Exception(
 162             "%s does not exist; Visual Studio 2017 not installed?" % vs_path
 163         )
 164
 165     sdk_path = os.path.join(pf, "Windows Kits", "10")
 166     if not os.path.exists(sdk_path):
 167         raise Exception("%s does not exist; Windows 10 SDK not installed?" % sdk_path)
 168
 169     sdk_fullver_path = os.path.join(sdk_path, "Include", SDK_RELEASE)
 170     if not os.path.exists(sdk_fullver_path):
 171         raise Exception(
 172             "%s does not exist; Wrong SDK version installed?" % sdk_fullver_path
 173         )
 174
 175     return vs_path, sdk_path
 176
 177
 178 def resolve_files():
 179     """Resolve the files that constitute a standalone toolchain.
 180
 181     This is a generator of (dest path, file) where the destination
 182     path is relative and the file instance is a BaseFile from mozpack.
 183     """
 184     vs_path, sdk_path = find_vs_paths()
 185
 186     for entry in PATTERNS:
 187         fullpath = entry["srcdir"] % {
 188             "vs_path": vs_path,
 189             "sdk_path": sdk_path,
 190         }
 191         for pattern in entry["files"]:
 192             finder = FileFinder(fullpath, ignore=pattern.get("ignore", []))
 193             for p, f in finder.find(pattern["pattern"]):
 194                 dstpath = "%s/%s" % (entry["dstdir"], p)
 195                 yield dstpath.encode("utf-8"), f
 196
 197
 198 def resolve_files_and_hash(manifest):
 199     """Resolve files and hash their data.
 200
 201     This is a generator of 3-tuples of (relpath, data, mode).
 202
 203     As data is read, the manifest is populated with metadata.
 204     Keys are set to the relative file path. Values are 2-tuples
 205     of (data length, sha-256).
 206     """
 207     assert manifest == {}
 208     for p, f in resolve_files():
 209         data = f.read()
 210
 211         sha256 = hashlib.sha256()
 212         sha256.update(data)
 213         manifest[p] = (len(data), sha256.hexdigest().encode("utf-8"))
 214
 215         yield p, data, f.mode
 216
 217
 218 def format_manifest(manifest):
 219     """Return formatted SHA-256 manifests as a byte strings."""
 220     sha256_lines = []
 221     for path, (length, sha256) in sorted(manifest.items()):
 222         sha256_lines.append(b"%s\t%d\t%s" % (sha256, length, path))
 223
 224     # Trailing newline.
 225     sha256_lines.append(b"")
 226
 227     return b"\n".join(sha256_lines)
 228
 229
 230 def write_zip(zip_path, prefix=None):
 231     """Write toolchain data to a zip file."""
 232     prefix = six.ensure_binary(prefix, encoding="utf-8")
 233
 234     with JarWriter(file=zip_path, compress_level=5) as zip:
 235         manifest = {}
 236         for p, data, mode in resolve_files_and_hash(manifest):
 237             print(p)
 238             if prefix:
 239                 p = mozpath.join(prefix, p)
 240
 241             zip.add(p, data, mode=mode)
 242
 243         sha256_manifest = format_manifest(manifest)
 244
 245         sdk_path = b"SDK_VERSION"
 246         sha256_path = b"MANIFEST.SHA256"
 247         if prefix:
 248             sdk_path = mozpath.join(prefix, sdk_path)
 249             sha256_path = mozpath.join(prefix, sha256_path)
 250
 251         zip.add(sdk_path, SDK_RELEASE.encode("utf-8"))
 252         zip.add(sha256_path, sha256_manifest)
 253
 254
 255 if __name__ == "__main__":
 256     if len(sys.argv) != 3:
 257         print("usage: %s create-zip <path-prefix>" % sys.argv[0])
 258         sys.exit(1)
 259
 260     assert sys.argv[1] == "create-zip"
 261     prefix = os.path.basename(sys.argv[2])
 262     destzip = "%s.zip" % sys.argv[2]
 263     write_zip(destzip, prefix=prefix)
 264
 265     sha1 = hashlib.sha1()
 266     sha256 = hashlib.sha256()
 267     sha512 = hashlib.sha512()
 268
 269     with open(destzip, "rb") as fh:
 270         data = fh.read()
 271         sha1.update(data)
 272         sha256.update(data)
 273         sha512.update(data)
 274
 275     print("Hashes of %s (size=%d)" % (destzip, len(data)))
 276     print("SHA-1:   %s" % sha1.hexdigest())
 277     print("SHA-256: %s" % sha256.hexdigest())
 278     print("SHA-512: %s" % sha512.hexdigest())