gfx/angle/update-angle.py

   1 #! /usr/bin/env python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 assert __name__ == "__main__"
   7
   8 r"""
   9 To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
  10 python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
  11 before python3 in your path.
  12
  13 Upstream: https://chromium.googlesource.com/angle/angle
  14
  15 Our repo: https://github.com/mozilla/angle
  16 It has branches like 'firefox-60' which is the branch we use for pulling into
  17 Gecko with this script.
  18
  19 This script leaves a record of the merge-base and cherry-picks that we pull into
  20 Gecko. (gfx/angle/cherries.log)
  21
  22 ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
  23 An easy choice is to grab Chrome's Beta's ANGLE branch.
  24
  25 ## Usage
  26
  27 Prepare your env:
  28
  29 * If in `cmd`:
  30     `export PATH="$PATH:/path/to/depot_tools"`
  31 * If in `powershell`:
  32     `$env:Path += ";C:\path\to\depot_tools"`
  33
  34 If this is a new repo, don't forget:
  35
  36 ~~~
  37 # In the angle repo:
  38 ./scripts/bootstrap.py
  39 gclient sync
  40 ~~~
  41
  42 Update: (in the angle repo)
  43
  44 ~~~
  45 # In the angle repo:
  46 /path/to/gecko/gfx/angle/update-angle.py origin
  47 git push moz # Push the firefox-XX branch to github.com/mozilla/angle
  48 ~~~~
  49
  50 """
  51
  52 import json
  53 import os
  54 import pathlib
  55 import re
  56 import shutil
  57 import subprocess
  58 import sys
  59 from typing import *  # mypy annotations
  60
  61 REPO_DIR = pathlib.Path.cwd()
  62 GECKO_ANGLE_DIR = pathlib.Path(__file__).parent
  63
  64 OUT_DIR = pathlib.Path("out")
  65
  66 COMMON_HEADER = [
  67     "# Generated by update-angle.py",
  68     "",
  69     'include("../../moz.build.common")',
  70 ]
  71
  72 ROOTS = ["//:translator", "//:libEGL", "//:libGLESv2"]
  73
  74 CHECK_ONLY = False
  75 args = sys.argv[1:]
  76 while True:
  77     arg = args.pop(0)
  78     if arg == "--check":
  79         CHECK_ONLY = True
  80         continue
  81     args.insert(0, arg)
  82     break
  83
  84 GN_ENV = dict(os.environ)
  85 GN_ENV["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0"
  86
  87 (GIT_REMOTE,) = args  # Not always 'origin'!
  88
  89 # ------------------------------------------------------------------------------
  90
  91
  92 def run_checked(*args, **kwargs):
  93     print(" ", args)
  94     sys.stdout.flush()
  95     return subprocess.run(args, check=True, **kwargs)
  96
  97
  98 def sorted_items(x):
  99     for k in sorted(x.keys()):
 100         yield (k, x[k])
 101
 102
 103 def collapse_dotdots(path):
 104     split = path.split("/")
 105
 106     ret = []
 107     for x in split:
 108         if x == ".." and ret:
 109             ret.pop()
 110             continue
 111         ret.append(x)
 112         continue
 113
 114     return "/".join(ret)
 115
 116
 117 def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]):
 118     visited_keys: Set[str] = set()
 119
 120     def recurse(key):
 121         if key in visited_keys:
 122             return
 123         visited_keys.add(key)
 124
 125         t = pre_recurse_func(key)
 126         try:
 127             (next_keys, post_recurse_func) = t
 128         except ValueError:
 129             (next_keys,) = t
 130             post_recurse_func = None
 131
 132         for x in next_keys:
 133             recurse(x)
 134
 135         if post_recurse_func:
 136             post_recurse_func(key)
 137         return
 138
 139     for x in root_keys:
 140         recurse(x)
 141     return
 142
 143
 144 # ------------------------------------------------------------------------------
 145
 146 print("Importing graph")
 147
 148 # shutil.rmtree(str(OUT_DIR), True)
 149 OUT_DIR.mkdir(exist_ok=True)
 150
 151 GN_ARGS = b"""
 152 # Build arguments go here.
 153 # See "gn args <out_dir> --list" for available build arguments.
 154 is_clang = true
 155 is_debug = false
 156 angle_build_all = false
 157 angle_enable_abseil = false
 158 angle_enable_apple_translator_workarounds = true
 159 angle_enable_essl = true
 160 angle_enable_gl = false
 161 angle_enable_gl_desktop_frontend = false
 162 angle_enable_glsl = true
 163 angle_enable_null = false
 164 angle_enable_share_context_lock = true
 165 angle_enable_vulkan = false
 166 angle_has_astc_encoder = false
 167 use_custom_libcxx = false
 168 """[
 169     1:
 170 ]
 171 args_gn_path = OUT_DIR / "args.gn"
 172 args_gn_path.write_bytes(GN_ARGS)
 173
 174 try:
 175     run_checked("gn", "gen", str(OUT_DIR), shell=True, env=GN_ENV)
 176 except subprocess.CalledProcessError:
 177     sys.stderr.buffer.write(b"`gn` failed. Is depot_tools in your PATH?\n")
 178     exit(1)
 179
 180 p = run_checked(
 181     "python3",
 182     "scripts/export_targets.py",
 183     str(OUT_DIR),
 184     *ROOTS,
 185     stdout=subprocess.PIPE,
 186     shell=True,
 187     env=GN_ENV,
 188 )
 189
 190 # -
 191
 192 print("\nProcessing graph")
 193 libraries = json.loads(p.stdout.decode())
 194
 195 # -
 196 # HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different
 197 # platforms.
 198 # descs["//:angle_common"]["sources"] +=
 199 EXTRA_ANGLE_COMMON_SOURCES = [
 200     "//src/common/system_utils_apple.cpp",
 201     "//src/common/system_utils_linux.cpp",
 202     "//src/common/system_utils_mac.cpp",
 203     "//src/common/system_utils_posix.cpp",
 204 ]
 205
 206 angle_common = libraries["//:angle_common"]
 207 angle_common["sources"] += EXTRA_ANGLE_COMMON_SOURCES
 208 angle_common["sources"] = sorted(angle_common["sources"])
 209
 210 # -
 211 # Reuse our own zlib
 212
 213 del libraries["//third_party/zlib:zlib"]
 214
 215 # -
 216
 217 if CHECK_ONLY:
 218     print("\n--check complete.")
 219     exit(0)
 220
 221 # ------------------------------------------------------------------------------
 222 # Output to moz.builds
 223
 224 import vendor_from_git
 225
 226 print("")
 227 vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE)
 228
 229 # --
 230
 231
 232 def sortedi(x):
 233     return sorted(x, key=str.lower)
 234
 235
 236 def append_arr(dest, name, vals, indent=0):
 237     if not vals:
 238         return
 239
 240     dest.append("{}{} += [".format(" " * 4 * indent, name))
 241     for x in sortedi(vals):
 242         dest.append('{}"{}",'.format(" " * 4 * (indent + 1), x))
 243     dest.append("{}]".format(" " * 4 * indent))
 244     dest.append("")
 245     return
 246
 247
 248 REGISTERED_DEFINES = {
 249     "ADLER32_SIMD_SSSE3": False,
 250     "ANGLE_CAPTURE_ENABLED": True,
 251     "ANGLE_DISABLE_POOL_ALLOC": True,
 252     "ANGLE_EGL_LIBRARY_NAME": False,
 253     "ANGLE_ENABLE_APPLE_WORKAROUNDS": True,
 254     "ANGLE_ENABLE_D3D11": True,
 255     "ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True,
 256     "ANGLE_ENABLE_D3D9": True,
 257     "ANGLE_ENABLE_DEBUG_ANNOTATIONS": True,
 258     "ANGLE_ENABLE_NULL": False,
 259     "ANGLE_ENABLE_OPENGL": False,
 260     "ANGLE_ENABLE_OPENGL_NULL": False,
 261     "ANGLE_ENABLE_ESSL": True,
 262     "ANGLE_ENABLE_GLSL": True,
 263     "ANGLE_ENABLE_HLSL": True,
 264     "ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True,
 265     "ANGLE_GENERATE_SHADER_DEBUG_INFO": True,
 266     "ANGLE_GLESV2_LIBRARY_NAME": True,
 267     "ANGLE_HAS_ASTCENC": True,
 268     "ANGLE_HAS_VULKAN_SYSTEM_INFO": False,
 269     "ANGLE_IS_64_BIT_CPU": False,
 270     "ANGLE_IS_WIN": False,
 271     "ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False,
 272     "ANGLE_SHARED_LIBVULKAN": True,
 273     "ANGLE_USE_CUSTOM_LIBVULKAN": True,
 274     "ANGLE_USE_EGL_LOADER": True,
 275     "ANGLE_VK_LAYERS_DIR": True,
 276     "ANGLE_VK_MOCK_ICD_JSON": True,
 277     "ANGLE_VMA_VERSION": True,
 278     "ASTCENC_DECOMPRESS_ONLY": True,
 279     "CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False,
 280     "CHROMIUM_BUILD": False,
 281     "COMPONENT_BUILD": False,
 282     "CRC32_SIMD_SSE42_PCLMUL": False,
 283     "DEFLATE_FILL_WINDOW_SSE2": False,
 284     "DYNAMIC_ANNOTATIONS_ENABLED": True,
 285     "EGL_EGL_PROTOTYPES": True,
 286     "EGL_EGLEXT_PROTOTYPES": True,
 287     "EGLAPI": True,
 288     "FIELDTRIAL_TESTING_ENABLED": False,
 289     "FULL_SAFE_BROWSING": False,
 290     "GL_API": True,
 291     "GL_APICALL": True,
 292     "GL_GLES_PROTOTYPES": True,
 293     "GL_GLEXT_PROTOTYPES": True,
 294     "GPU_INFO_USE_SETUPAPI": True,
 295     "INFLATE_CHUNK_READ_64LE": False,
 296     "INFLATE_CHUNK_SIMD_SSE2": False,
 297     "LIBANGLE_IMPLEMENTATION": True,
 298     "LIBEGL_IMPLEMENTATION": True,
 299     "LIBGLESV2_IMPLEMENTATION": True,
 300     "NOMINMAX": True,
 301     "NO_TCMALLOC": False,
 302     # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
 303     "NTDDI_VERSION": False,
 304     "PSAPI_VERSION": False,
 305     "SAFE_BROWSING_CSD": False,
 306     "SAFE_BROWSING_DB_LOCAL": False,
 307     "UNICODE": True,
 308     "USE_AURA": False,
 309     "V8_DEPRECATION_WARNINGS": False,
 310     "VK_USE_PLATFORM_WIN32_KHR": False,
 311     "WIN32": False,
 312     "WIN32_LEAN_AND_MEAN": False,
 313     "WINAPI_FAMILY": False,
 314     "WINVER": True,
 315     # Otherwise:
 316     # gfx/angle/targets/libANGLE
 317     # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10:
 318     # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17:
 319     # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO'
 320     #         _In_ const POINTER_INFO& pointerInfo
 321     #                    ^
 322     "WTF_USE_DYNAMIC_ANNOTATIONS": False,
 323     "X86_WINDOWS": False,
 324     "ZLIB_IMPLEMENTATION": True,
 325     "_ATL_NO_OPENGL": True,
 326     "_CRT_NONSTDC_NO_DEPRECATE": True,
 327     "_CRT_NONSTDC_NO_WARNINGS": True,
 328     "_CRT_RAND_S": True,
 329     "_CRT_SECURE_NO_DEPRECATE": True,
 330     "_DEBUG": False,
 331     "_HAS_EXCEPTIONS": True,
 332     "_HAS_ITERATOR_DEBUGGING": False,
 333     "_SCL_SECURE_NO_DEPRECATE": True,
 334     "_SECURE_ATL": True,
 335     "_UNICODE": True,
 336     "_USING_V110_SDK71_": False,
 337     "_WIN32_WINNT": False,
 338     "_WINDOWS": False,
 339     "_WINSOCK_DEPRECATED_NO_WARNINGS": True,
 340     "__STD_C": False,
 341     # clang specific
 342     "CR_CLANG_REVISION": True,
 343     "NDEBUG": False,
 344     "NVALGRIND": False,
 345     "_HAS_NODISCARD": False,
 346 }
 347
 348 # -
 349
 350 print("\nRun actions")
 351 required_files: Set[str] = set()
 352 required_files.add("//LICENSE")
 353
 354 run_checked("ninja", "-C", str(OUT_DIR), ":angle_commit_id", shell=True)
 355 required_files.add("//out/gen/angle/angle_commit.h")
 356
 357 # -
 358
 359 # Export our targets
 360 print("\nExport targets")
 361
 362 # Clear our dest directories
 363 targets_dir = pathlib.Path(GECKO_ANGLE_DIR, "targets")
 364 checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, "checkout")
 365
 366 shutil.rmtree(targets_dir, True)
 367 shutil.rmtree(checkout_dir, True)
 368 targets_dir.mkdir(exist_ok=True)
 369 checkout_dir.mkdir(exist_ok=True)
 370
 371 # -
 372
 373 RE_TARGET_NAME = re.compile("//(.*):(.+)")
 374
 375
 376 def export_target(target_full_name) -> Set[str]:
 377     # print(' ', target_full_name)
 378     descs = libraries
 379     desc = descs[target_name]
 380     flat = desc
 381
 382     m = RE_TARGET_NAME.match(target_name)
 383     assert m, target_name
 384     name = m.group(2)
 385
 386     required_files: Set[str] = set(flat["sources"])
 387
 388     # Create our manifest lines
 389     target_dir = targets_dir / name
 390     target_dir.mkdir(exist_ok=True)
 391
 392     lines = list(COMMON_HEADER)
 393     lines.append("")
 394
 395     for x in sorted(set(desc["defines"])):
 396         try:
 397             (k, v) = x.split("=", 1)
 398             if v.startswith('"'):
 399                 v = f"'{v}'"
 400             else:
 401                 v = f'"{v}"'
 402         except ValueError:
 403             (k, v) = (x, "True")
 404
 405         line = f'DEFINES["{k}"] = {v}'
 406         try:
 407             if REGISTERED_DEFINES[k] == False:
 408                 line = "# " + line
 409         except KeyError:
 410             print(f"[{name}] Unrecognized define: {k}")
 411             line = "# Unrecognized: " + line
 412         lines.append(line)
 413     lines.append("")
 414
 415     cxxflags = set(desc["cflags"] + desc["cflags_cc"])
 416
 417     def fixup_paths(listt):
 418         for x in set(listt):
 419             assert x.startswith("//"), x
 420             yield "../../checkout/" + x[2:]
 421
 422     sources_by_config: Dict[str, List[str]] = {}
 423     extras: Dict[str, str] = dict()
 424     for x in fixup_paths(flat["sources"]):
 425         # print(' '*5, x)
 426         (b, e) = x.rsplit(".", 1)
 427         if e in ["h", "hpp", "y", "l", "inc", "inl"]:
 428             continue
 429         elif e in ["cpp", "cc", "c"]:
 430             if b.endswith("_win") or b.endswith("_win32"):
 431                 config = 'CONFIG["OS_ARCH"] == "WINNT"'
 432             elif b.endswith("_linux"):
 433                 # Include these on BSDs too.
 434                 config = 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")'
 435             elif b.endswith("_apple") or b.endswith("_mac"):
 436                 config = 'CONFIG["OS_ARCH"] == "Darwin"'
 437             elif b.endswith("_posix"):
 438                 config = 'CONFIG["OS_ARCH"] != "WINNT"'
 439             else:
 440                 config = ""  # None can't compare against str.
 441
 442             sources_by_config.setdefault(config, []).append(x)
 443             continue
 444         elif e == "rc":
 445             assert "RCFILE" not in extras, (target_name, extras["RCFILE"], x)
 446             extras["RCFILE"] = f'"{x}"'
 447             continue
 448         elif e == "def":
 449             assert "DEFFILE" not in extras, (target_name, extras["DEFFILE"], x)
 450             extras["DEFFILE"] = f'"{x}"'
 451             continue
 452         else:
 453             assert False, ("Unhandled ext:", x)
 454
 455     ldflags = set(desc["ldflags"])
 456     DEF_PREFIX = "/DEF:"
 457     for x in set(ldflags):
 458         if x.startswith(DEF_PREFIX):
 459             def_path = x[len(DEF_PREFIX) :]
 460             required_files.add(def_path)
 461             assert "DEFFILE" not in extras
 462             ldflags.remove(x)
 463
 464             def_path = str(OUT_DIR) + "/" + def_path
 465             def_path = "//" + collapse_dotdots(def_path)
 466
 467             def_rel_path = list(fixup_paths([def_path]))[0]
 468             extras["DEFFILE"] = '"{}"'.format(def_rel_path)
 469         elif x.startswith("/PDBSourcePath:"):
 470             ldflags.remove(x)
 471
 472     os_libs = list(map(lambda x: x[: -len(".lib")], set(desc.get("libs", []))))
 473
 474     def append_arr_commented(dest, name, src):
 475         lines = []
 476         append_arr(lines, name, src)
 477
 478         def comment(x):
 479             if x:
 480                 x = "# " + x
 481             return x
 482
 483         lines = map(comment, lines)
 484         dest += lines
 485
 486     append_arr(lines, "LOCAL_INCLUDES", fixup_paths(desc["include_dirs"]))
 487     append_arr_commented(lines, "CXXFLAGS", cxxflags)
 488
 489     for config, v in sorted_items(sources_by_config):
 490         indent = 0
 491         if config:
 492             lines.append("if {}:".format(config))
 493             indent = 1
 494         append_arr(lines, "SOURCES", v, indent=indent)
 495
 496     dep_libs: Set[str] = set()
 497     for dep_full_name in set(flat["dep_libs"]):
 498         assert dep_full_name.startswith("//"), dep_name
 499         (_, dep_name) = dep_full_name.split(":")
 500         dep_libs.add(dep_name)
 501
 502     dep_dirs = set(dep_libs)
 503     dep_dirs.discard("zlib")
 504
 505     # Those directories are added by gfx/angle/moz.build.
 506     already_added_dirs = [
 507         "angle_common",
 508         "translator",
 509         "libEGL",
 510         "libGLESv2",
 511     ]
 512
 513     append_arr(lines, "USE_LIBS", dep_libs)
 514     append_arr(
 515         lines, "DIRS", ["../" + x for x in dep_dirs if x not in already_added_dirs]
 516     )
 517     append_arr(lines, "OS_LIBS", os_libs)
 518     append_arr_commented(lines, "LDFLAGS", ldflags)
 519
 520     for k, v in sorted(extras.items()):
 521         lines.append("{} = {}".format(k, v))
 522
 523     lib_type = desc["type"]
 524     if lib_type == "shared_library":
 525         lines.append(f'GeckoSharedLibrary("{name}", linkage=None)')
 526     elif lib_type == "static_library":
 527         lines.append(f'Library("{name}")')
 528     else:
 529         assert False, lib_type
 530
 531     lines.append("")
 532     # EOF newline
 533
 534     # Write it out
 535
 536     mozbuild = target_dir / "moz.build"
 537     print(" ", " ", f"Writing {mozbuild}")
 538     data = b"\n".join((x.encode() for x in lines))
 539     mozbuild.write_bytes(data)
 540
 541     return required_files
 542
 543
 544 # -
 545
 546 for target_name in libraries:
 547     reqs = export_target(target_name)
 548     required_files |= reqs
 549
 550 # Copy all the files
 551
 552 print("\nMigrate required files")
 553
 554 i = 0
 555 for x in required_files:
 556     i += 1
 557     sys.stdout.write(f"\r  Copying {i}/{len(required_files)}")
 558     sys.stdout.flush()
 559     assert x.startswith("//"), x
 560     x = x[2:]
 561
 562     src = REPO_DIR / x
 563     dest = checkout_dir / x
 564
 565     dest.parent.mkdir(parents=True, exist_ok=True)
 566     data = src.read_bytes()
 567     data = data.replace(b"\r\n", b"\n")
 568     dest.write_bytes(data)
 569
 570 print("\n\nDone")