1 #! /usr/bin/env python3
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this
4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 assert __name__
== "__main__"
9 To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
10 python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
11 before python3 in your path.
13 Upstream: https://chromium.googlesource.com/angle/angle
15 Our repo: https://github.com/mozilla/angle
16 It has branches like 'firefox-60' which is the branch we use for pulling into
17 Gecko with this script.
19 This script leaves a record of the merge-base and cherry-picks that we pull into
20 Gecko. (gfx/angle/cherries.log)
22 ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
23 An easy choice is to grab Chrome's Beta's ANGLE branch.
30 export PATH="$PATH:/path/to/depot_tools"
33 If this is a new repo, don't forget:
37 ./scripts/bootstrap.py
41 Update: (in the angle repo)
45 /path/to/gecko/gfx/angle/update-angle.py origin
46 git push moz # Push the firefox-XX branch to github.com/mozilla/angle
58 from typing
import * # mypy annotations
60 REPO_DIR
= pathlib
.Path
.cwd()
61 GECKO_ANGLE_DIR
= pathlib
.Path(__file__
).parent
63 OUT_DIR
= pathlib
.Path("out")
66 "# Generated by update-angle.py",
68 'include("../../moz.build.common")',
71 ROOTS
= ["//:translator", "//:libEGL", "//:libGLESv2"]
83 GN_ENV
= dict(os
.environ
)
84 GN_ENV
["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0"
86 (GIT_REMOTE
,) = args
# Not always 'origin'!
88 # ------------------------------------------------------------------------------
91 def run_checked(*args
, **kwargs
):
94 return subprocess
.run(args
, check
=True, **kwargs
)
98 for k
in sorted(x
.keys()):
102 def collapse_dotdots(path
):
103 split
= path
.split("/")
107 if x
== ".." and ret
:
116 def dag_traverse(root_keys
: Sequence
[str], pre_recurse_func
: Callable
[[str], list]):
117 visited_keys
: Set
[str] = set()
120 if key
in visited_keys
:
122 visited_keys
.add(key
)
124 t
= pre_recurse_func(key
)
126 (next_keys
, post_recurse_func
) = t
129 post_recurse_func
= None
134 if post_recurse_func
:
135 post_recurse_func(key
)
143 # ------------------------------------------------------------------------------
145 print("Importing graph")
147 # shutil.rmtree(str(OUT_DIR), True)
148 OUT_DIR
.mkdir(exist_ok
=True)
151 # Build arguments go here.
152 # See "gn args <out_dir> --list" for available build arguments.
155 angle_build_all = false
156 angle_enable_abseil = false
157 angle_enable_apple_translator_workarounds = true
158 angle_enable_essl = true
159 angle_enable_gl = false
160 angle_enable_gl_desktop_frontend = false
161 angle_enable_glsl = true
162 angle_enable_null = false
163 angle_enable_share_context_lock = true
164 angle_enable_vulkan = false
165 angle_has_astc_encoder = false
166 use_custom_libcxx = false
170 args_gn_path
= OUT_DIR
/ "args.gn"
171 args_gn_path
.write_bytes(GN_ARGS
)
174 run_checked("gn", "gen", str(OUT_DIR
), shell
=True, env
=GN_ENV
)
175 except subprocess
.CalledProcessError
:
176 sys
.stderr
.buffer.write(b
"`gn` failed. Is depot_tools in your PATH?\n")
181 "scripts/export_targets.py",
184 stdout
=subprocess
.PIPE
,
191 print("\nProcessing graph")
192 libraries
= json
.loads(p
.stdout
.decode())
195 # HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different
197 # descs["//:angle_common"]["sources"] +=
198 EXTRA_ANGLE_COMMON_SOURCES
= [
199 "//src/common/system_utils_apple.cpp",
200 "//src/common/system_utils_linux.cpp",
201 "//src/common/system_utils_mac.cpp",
202 "//src/common/system_utils_posix.cpp",
205 angle_common
= libraries
["//:angle_common"]
206 angle_common
["sources"] += EXTRA_ANGLE_COMMON_SOURCES
207 angle_common
["sources"] = sorted(angle_common
["sources"])
212 del libraries
["//third_party/zlib:zlib"]
217 print("\n--check complete.")
220 # ------------------------------------------------------------------------------
221 # Output to moz.builds
223 import vendor_from_git
226 vendor_from_git
.record_cherry_picks(GECKO_ANGLE_DIR
, GIT_REMOTE
)
232 return sorted(x
, key
=str.lower
)
235 def append_arr(dest
, name
, vals
, indent
=0):
239 dest
.append("{}{} += [".format(" " * 4 * indent
, name
))
240 for x
in sortedi(vals
):
241 dest
.append('{}"{}",'.format(" " * 4 * (indent
+ 1), x
))
242 dest
.append("{}]".format(" " * 4 * indent
))
247 REGISTERED_DEFINES
= {
248 "ADLER32_SIMD_SSSE3": False,
249 "ANGLE_CAPTURE_ENABLED": True,
250 "ANGLE_DISABLE_POOL_ALLOC": True,
251 "ANGLE_EGL_LIBRARY_NAME": False,
252 "ANGLE_ENABLE_APPLE_WORKAROUNDS": True,
253 "ANGLE_ENABLE_D3D11": True,
254 "ANGLE_ENABLE_D3D11_COMPOSITOR_NATIVE_WINDOW": True,
255 "ANGLE_ENABLE_D3D9": True,
256 "ANGLE_ENABLE_DEBUG_ANNOTATIONS": True,
257 "ANGLE_ENABLE_NULL": False,
258 "ANGLE_ENABLE_OPENGL": False,
259 "ANGLE_ENABLE_OPENGL_NULL": False,
260 "ANGLE_ENABLE_ESSL": True,
261 "ANGLE_ENABLE_GLSL": True,
262 "ANGLE_ENABLE_HLSL": True,
263 "ANGLE_ENABLE_SHARE_CONTEXT_LOCK": True,
264 "ANGLE_GENERATE_SHADER_DEBUG_INFO": True,
265 "ANGLE_GLESV2_LIBRARY_NAME": True,
266 "ANGLE_HAS_ASTCENC": True,
267 "ANGLE_HAS_VULKAN_SYSTEM_INFO": False,
268 "ANGLE_IS_64_BIT_CPU": False,
269 "ANGLE_IS_WIN": False,
270 "ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES": False,
271 "ANGLE_SHARED_LIBVULKAN": True,
272 "ANGLE_USE_CUSTOM_LIBVULKAN": True,
273 "ANGLE_USE_EGL_LOADER": True,
274 "ANGLE_VK_LAYERS_DIR": True,
275 "ANGLE_VK_MOCK_ICD_JSON": True,
276 "ANGLE_VMA_VERSION": True,
277 "ASTCENC_DECOMPRESS_ONLY": True,
278 "CERT_CHAIN_PARA_HAS_EXTRA_FIELDS": False,
279 "CHROMIUM_BUILD": False,
280 "COMPONENT_BUILD": False,
281 "CRC32_SIMD_SSE42_PCLMUL": False,
282 "DEFLATE_FILL_WINDOW_SSE2": False,
283 "DYNAMIC_ANNOTATIONS_ENABLED": True,
284 "EGL_EGL_PROTOTYPES": True,
285 "EGL_EGLEXT_PROTOTYPES": True,
287 "FIELDTRIAL_TESTING_ENABLED": False,
288 "FULL_SAFE_BROWSING": False,
291 "GL_GLES_PROTOTYPES": True,
292 "GL_GLEXT_PROTOTYPES": True,
293 "GPU_INFO_USE_SETUPAPI": True,
294 "INFLATE_CHUNK_READ_64LE": False,
295 "INFLATE_CHUNK_SIMD_SSE2": False,
296 "LIBANGLE_IMPLEMENTATION": True,
297 "LIBEGL_IMPLEMENTATION": True,
298 "LIBGLESV2_IMPLEMENTATION": True,
300 "NO_TCMALLOC": False,
301 # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
302 "NTDDI_VERSION": False,
303 "PSAPI_VERSION": False,
304 "SAFE_BROWSING_CSD": False,
305 "SAFE_BROWSING_DB_LOCAL": False,
308 "V8_DEPRECATION_WARNINGS": False,
309 "VK_USE_PLATFORM_WIN32_KHR": False,
311 "WIN32_LEAN_AND_MEAN": False,
312 "WINAPI_FAMILY": False,
315 # gfx/angle/targets/libANGLE
316 # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10:
317 # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17:
318 # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO'
319 # _In_ const POINTER_INFO& pointerInfo
321 "WTF_USE_DYNAMIC_ANNOTATIONS": False,
322 "X86_WINDOWS": False,
323 "ZLIB_IMPLEMENTATION": True,
324 "_ATL_NO_OPENGL": True,
325 "_CRT_NONSTDC_NO_DEPRECATE": True,
326 "_CRT_NONSTDC_NO_WARNINGS": True,
328 "_CRT_SECURE_NO_DEPRECATE": True,
330 "_HAS_EXCEPTIONS": True,
331 "_HAS_ITERATOR_DEBUGGING": False,
332 "_SCL_SECURE_NO_DEPRECATE": True,
335 "_USING_V110_SDK71_": False,
336 "_WIN32_WINNT": False,
338 "_WINSOCK_DEPRECATED_NO_WARNINGS": True,
341 "CR_CLANG_REVISION": True,
344 "_HAS_NODISCARD": False,
349 print("\nRun actions")
350 required_files
: Set
[str] = set()
351 required_files
.add("//LICENSE")
353 run_checked("ninja", "-C", str(OUT_DIR
), ":angle_commit_id", shell
=True)
354 required_files
.add("//out/gen/angle/angle_commit.h")
359 print("\nExport targets")
361 # Clear our dest directories
362 targets_dir
= pathlib
.Path(GECKO_ANGLE_DIR
, "targets")
363 checkout_dir
= pathlib
.Path(GECKO_ANGLE_DIR
, "checkout")
365 shutil
.rmtree(targets_dir
, True)
366 shutil
.rmtree(checkout_dir
, True)
367 targets_dir
.mkdir(exist_ok
=True)
368 checkout_dir
.mkdir(exist_ok
=True)
372 RE_TARGET_NAME
= re
.compile("//(.*):(.+)")
375 def export_target(target_full_name
) -> Set
[str]:
376 # print(' ', target_full_name)
378 desc
= descs
[target_name
]
381 m
= RE_TARGET_NAME
.match(target_name
)
382 assert m
, target_name
385 required_files
: Set
[str] = set(flat
["sources"])
387 # Create our manifest lines
388 target_dir
= targets_dir
/ name
389 target_dir
.mkdir(exist_ok
=True)
391 lines
= list(COMMON_HEADER
)
394 for x
in sorted(set(desc
["defines"])):
396 (k
, v
) = x
.split("=", 1)
397 if v
.startswith('"'):
404 line
= f
'DEFINES["{k}"] = {v}'
406 if REGISTERED_DEFINES
[k
] == False:
409 print(f
"[{name}] Unrecognized define: {k}")
410 line
= "# Unrecognized: " + line
414 cxxflags
= set(desc
["cflags"] + desc
["cflags_cc"])
416 def fixup_paths(listt
):
418 assert x
.startswith("//"), x
419 yield "../../checkout/" + x
[2:]
421 sources_by_config
: Dict
[str, List
[str]] = {}
422 extras
: Dict
[str, str] = dict()
423 for x
in fixup_paths(flat
["sources"]):
425 (b
, e
) = x
.rsplit(".", 1)
426 if e
in ["h", "hpp", "y", "l", "inc", "inl"]:
428 elif e
in ["cpp", "cc", "c"]:
429 if b
.endswith("_win") or b
.endswith("_win32"):
430 config
= 'CONFIG["OS_ARCH"] == "WINNT"'
431 elif b
.endswith("_linux"):
432 # Include these on BSDs too.
433 config
= 'CONFIG["OS_ARCH"] not in ("Darwin", "WINNT")'
434 elif b
.endswith("_apple") or b
.endswith("_mac"):
435 config
= 'CONFIG["OS_ARCH"] == "Darwin"'
436 elif b
.endswith("_posix"):
437 config
= 'CONFIG["OS_ARCH"] != "WINNT"'
439 config
= "" # None can't compare against str.
441 sources_by_config
.setdefault(config
, []).append(x
)
444 assert "RCFILE" not in extras
, (target_name
, extras
["RCFILE"], x
)
445 extras
["RCFILE"] = f
'"{x}"'
448 assert "DEFFILE" not in extras
, (target_name
, extras
["DEFFILE"], x
)
449 extras
["DEFFILE"] = f
'"{x}"'
452 assert False, ("Unhandled ext:", x
)
454 ldflags
= set(desc
["ldflags"])
456 for x
in set(ldflags
):
457 if x
.startswith(DEF_PREFIX
):
458 def_path
= x
[len(DEF_PREFIX
) :]
459 required_files
.add(def_path
)
460 assert "DEFFILE" not in extras
463 def_path
= str(OUT_DIR
) + "/" + def_path
464 def_path
= "//" + collapse_dotdots(def_path
)
466 def_rel_path
= list(fixup_paths([def_path
]))[0]
467 extras
["DEFFILE"] = '"{}"'.format(def_rel_path
)
468 elif x
.startswith("/PDBSourcePath:"):
471 os_libs
= list(map(lambda x
: x
[: -len(".lib")], set(desc
.get("libs", []))))
473 def append_arr_commented(dest
, name
, src
):
475 append_arr(lines
, name
, src
)
482 lines
= map(comment
, lines
)
485 append_arr(lines
, "LOCAL_INCLUDES", fixup_paths(desc
["include_dirs"]))
486 append_arr_commented(lines
, "CXXFLAGS", cxxflags
)
488 for config
, v
in sorted_items(sources_by_config
):
491 lines
.append("if {}:".format(config
))
493 append_arr(lines
, "SOURCES", v
, indent
=indent
)
495 dep_libs
: Set
[str] = set()
496 for dep_full_name
in set(flat
["dep_libs"]):
497 assert dep_full_name
.startswith("//"), dep_name
498 (_
, dep_name
) = dep_full_name
.split(":")
499 dep_libs
.add(dep_name
)
501 dep_dirs
= set(dep_libs
)
502 dep_dirs
.discard("zlib")
504 append_arr(lines
, "USE_LIBS", dep_libs
)
505 append_arr(lines
, "DIRS", ["../" + x
for x
in dep_dirs
])
506 append_arr(lines
, "OS_LIBS", os_libs
)
507 append_arr_commented(lines
, "LDFLAGS", ldflags
)
509 for k
, v
in sorted(extras
.items()):
510 lines
.append("{} = {}".format(k
, v
))
512 lib_type
= desc
["type"]
513 if lib_type
== "shared_library":
514 lines
.append(f
'GeckoSharedLibrary("{name}", linkage=None)')
515 elif lib_type
== "static_library":
516 lines
.append(f
'Library("{name}")')
518 assert False, lib_type
525 mozbuild
= target_dir
/ "moz.build"
526 print(" ", " ", f
"Writing {mozbuild}")
527 data
= b
"\n".join((x
.encode() for x
in lines
))
528 mozbuild
.write_bytes(data
)
530 return required_files
535 for target_name
in libraries
:
536 reqs
= export_target(target_name
)
537 required_files |
= reqs
541 print("\nMigrate required files")
544 for x
in required_files
:
546 sys
.stdout
.write(f
"\r Copying {i}/{len(required_files)}")
548 assert x
.startswith("//"), x
552 dest
= checkout_dir
/ x
554 dest
.parent
.mkdir(parents
=True, exist_ok
=True)
555 data
= src
.read_bytes()
556 data
= data
.replace(b
"\r\n", b
"\n")
557 dest
.write_bytes(data
)