gfx/angle/update-angle.py

   1 #! /usr/bin/env python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 assert __name__ == '__main__'
   7
   8 '''
   9 To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
  10 python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
  11 before python3 in your path.
  12
  13 Upstream: https://chromium.googlesource.com/angle/angle
  14
  15 Our repo: https://github.com/mozilla/angle
  16 It has branches like 'firefox-60' which is the branch we use for pulling into
  17 Gecko with this script.
  18
  19 This script leaves a record of the merge-base and cherry-picks that we pull into
  20 Gecko. (gfx/angle/cherries.log)
  21
  22 ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
  23 An easy choice is to grab Chrome's Beta's ANGLE branch.
  24
  25 ## Usage
  26
  27 Prepare your env:
  28
  29 ~~~
  30 export PATH="$PATH:/path/to/depot_tools"
  31 ~~~
  32
  33 If this is a new repo, don't forget:
  34
  35 ~~~
  36 # In the angle repo:
  37 ./scripts/bootstrap.py
  38 gclient sync
  39 ~~~
  40
  41 Update: (in the angle repo)
  42
  43 ~~~
  44 # In the angle repo:
  45 /path/to/gecko/gfx/angle/update-angle.py origin/chromium/XXXX
  46 git push moz # Push the firefox-XX branch to github.com/mozilla/angle
  47 ~~~~
  48
  49 '''
  50
  51 import json
  52 import os
  53 import pathlib
  54 import re
  55 import shutil
  56 import subprocess
  57 import sys
  58 from typing import * # mypy annotations
  59
  60 REPO_DIR = pathlib.Path.cwd()
  61 GECKO_ANGLE_DIR = pathlib.Path(__file__).parent
  62
  63 OUT_DIR = pathlib.Path('out')
  64
  65 COMMON_HEADER = [
  66     '# Generated by update-angle.py',
  67     '',
  68     "include('../../moz.build.common')",
  69 ]
  70
  71 ROOTS = ['//:translator', '//:libEGL', '//:libGLESv2']
  72
  73 CHECK_ONLY = False
  74 args = sys.argv[1:]
  75 while True:
  76     arg = args.pop(0)
  77     if arg == '--check':
  78         CHECK_ONLY = True
  79         continue
  80     args.insert(0, arg)
  81     break
  82
  83 GN_ENV = dict(os.environ)
  84 GN_ENV['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0'
  85
  86 (GIT_REMOTE, ) = args # Not always 'origin'!
  87
  88 # ------------------------------------------------------------------------------
  89
  90 def run_checked(*args, **kwargs):
  91     print(' ', args)
  92     sys.stdout.flush()
  93     return subprocess.run(args, check=True, **kwargs)
  94
  95
  96 def sorted_items(x):
  97     for k in sorted(x.keys()):
  98         yield (k, x[k])
  99
 100
 101 def collapse_dotdots(path):
 102     split = path.split('/')
 103
 104     ret = []
 105     for x in split:
 106         if x == '..' and ret:
 107             ret.pop()
 108             continue
 109         ret.append(x)
 110         continue
 111
 112     return '/'.join(ret)
 113
 114
 115 def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]):
 116     visited_keys: Set[str] = set()
 117
 118     def recurse(key):
 119         if key in visited_keys:
 120             return
 121         visited_keys.add(key)
 122
 123         t = pre_recurse_func(key)
 124         try:
 125             (next_keys, post_recurse_func) = t
 126         except ValueError:
 127             (next_keys,) = t
 128             post_recurse_func = None
 129
 130         for x in next_keys:
 131             recurse(x)
 132
 133         if post_recurse_func:
 134             post_recurse_func(key)
 135         return
 136
 137     for x in root_keys:
 138         recurse(x)
 139     return
 140
 141 # ------------------------------------------------------------------------------
 142
 143 print('Importing graph')
 144
 145 #shutil.rmtree(str(OUT_DIR), True)
 146 OUT_DIR.mkdir(exist_ok=True)
 147
 148 GN_ARGS = b'''
 149 # Build arguments go here.
 150 # See "gn args <out_dir> --list" for available build arguments.
 151 is_clang = true
 152 is_debug = false
 153 angle_enable_gl = false
 154 angle_enable_gl_null = false
 155 angle_enable_null = false
 156 angle_enable_vulkan = false
 157 '''[1:]
 158 args_gn_path = OUT_DIR / 'args.gn'
 159 args_gn_path.write_bytes(GN_ARGS)
 160
 161 try:
 162     run_checked('gn', 'gen', str(OUT_DIR), shell=True, env=GN_ENV)
 163 except subprocess.CalledProcessError:
 164     sys.stderr.buffer.write(b'`gn` failed. Is depot_tools in your PATH?\n')
 165     exit(1)
 166
 167 p = run_checked('gn', 'desc', '--format=json', str(OUT_DIR), '*', stdout=subprocess.PIPE,
 168                 shell=True, env=GN_ENV)
 169
 170 # -
 171
 172 print('\nProcessing graph')
 173 descs = json.loads(p.stdout.decode())
 174
 175 # -
 176 # HACKHACKHACK: Inject linux/mac sources instead of trying to merge graphs of different
 177 # platforms.
 178 descs['//:angle_common']['sources'] += [
 179     '//src/common/system_utils_linux.cpp',
 180     '//src/common/system_utils_mac.cpp',
 181     '//src/common/system_utils_posix.cpp',
 182 ]
 183
 184 # Ready to traverse
 185 # ------------------------------------------------------------------------------
 186
 187 LIBRARY_TYPES = ('shared_library', 'static_library')
 188
 189 def flattened_target(target_name: str, descs: dict, stop_at_lib: bool =True) -> dict:
 190     flattened = dict(descs[target_name])
 191
 192     EXPECTED_TYPES = LIBRARY_TYPES + ('source_set', 'group', 'action')
 193
 194     def pre(k):
 195         dep = descs[k]
 196
 197         dep_type = dep['type']
 198         deps = dep['deps']
 199         if stop_at_lib and dep_type in LIBRARY_TYPES:
 200             return ((),)
 201
 202         if dep_type == 'copy':
 203             assert not deps, (target_name, dep['deps'])
 204         else:
 205             assert dep_type in EXPECTED_TYPES, (k, dep_type)
 206             for (k,v) in dep.items():
 207                 if type(v) in (list, tuple):
 208                     flattened[k] = flattened.get(k, []) + v
 209                 else:
 210                     #flattened.setdefault(k, v)
 211                     pass
 212         return (deps,)
 213
 214     dag_traverse(descs[target_name]['deps'], pre)
 215     return flattened
 216
 217 # ------------------------------------------------------------------------------
 218 # Check that includes are valid. (gn's version of this check doesn't seem to work!)
 219
 220 INCLUDE_REGEX = re.compile(b'(?:^|\\n) *# *include +([<"])([^>"]+)[>"]')
 221 assert INCLUDE_REGEX.match(b'#include "foo"')
 222 assert INCLUDE_REGEX.match(b'\n#include "foo"')
 223
 224 IGNORED_INCLUDES = {
 225     b'compiler/translator/TranslatorVulkan.h',
 226     b'libANGLE/renderer/d3d/d3d11/winrt/NativeWindow11WinRT.h',
 227     b'libANGLE/renderer/gl/glx/DisplayGLX.h',
 228     b'libANGLE/renderer/gl/cgl/DisplayCGL.h',
 229     b'libANGLE/renderer/gl/egl/ozone/DisplayOzone.h',
 230     b'libANGLE/renderer/gl/egl/android/DisplayAndroid.h',
 231     b'libANGLE/renderer/gl/wgl/DisplayWGL.h',
 232     b'libANGLE/renderer/null/DisplayNULL.h',
 233     b'libANGLE/renderer/vulkan/android/DisplayVkAndroid.h',
 234     b'libANGLE/renderer/vulkan/fuchsia/DisplayVkFuchsia.h',
 235     b'libANGLE/renderer/vulkan/win32/DisplayVkWin32.h',
 236     b'libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h',
 237     b'kernel/image.h',
 238 }
 239
 240 IGNORED_INCLUDE_PREFIXES = {
 241     b'android',
 242     b'Carbon',
 243     b'CoreFoundation',
 244     b'CoreServices',
 245     b'IOSurface',
 246     b'mach',
 247     b'mach-o',
 248     b'OpenGL',
 249     b'pci',
 250     b'sys',
 251     b'wrl',
 252     b'X11',
 253 }
 254
 255 def has_all_includes(target_name: str, descs: dict) -> bool:
 256     flat = flattened_target(target_name, descs, stop_at_lib=False)
 257     acceptable_sources = flat.get('sources', []) + flat.get('outputs', [])
 258     acceptable_sources = (x.rsplit('/', 1)[-1].encode() for x in acceptable_sources)
 259     acceptable_sources = set(acceptable_sources)
 260
 261     ret = True
 262     desc = descs[target_name]
 263     for cur_file in desc.get('sources', []):
 264         assert cur_file.startswith('/'), cur_file
 265         if not cur_file.startswith('//'):
 266             continue
 267         cur_file = pathlib.Path(cur_file[2:])
 268         text = cur_file.read_bytes()
 269         for m in INCLUDE_REGEX.finditer(text):
 270             if m.group(1) == b'<':
 271                 continue
 272             include = m.group(2)
 273             if include in IGNORED_INCLUDES:
 274                 continue
 275             try:
 276                 (prefix, _) = include.split(b'/', 1)
 277                 if prefix in IGNORED_INCLUDE_PREFIXES:
 278                     continue
 279             except ValueError:
 280                 pass
 281
 282             include_file = include.rsplit(b'/', 1)[-1]
 283             if include_file not in acceptable_sources:
 284                 #print('  acceptable_sources:')
 285                 #for x in sorted(acceptable_sources):
 286                 #    print('   ', x)
 287                 print('Warning in {}: {}: Invalid include: {}'.format(target_name, cur_file, include))
 288                 ret = False
 289             #print('Looks valid:', m.group())
 290             continue
 291
 292     return ret
 293
 294 # -
 295 # Gather real targets:
 296
 297 def gather_libraries(roots: Sequence[str], descs: dict) -> Set[str]:
 298     libraries = set()
 299     def fn(target_name):
 300         cur = descs[target_name]
 301         print('  ' + cur['type'], target_name)
 302         assert has_all_includes(target_name, descs), target_name
 303
 304         if cur['type'] in ('shared_library', 'static_library'):
 305             libraries.add(target_name)
 306         return (cur['deps'], )
 307
 308     dag_traverse(roots, fn)
 309     return libraries
 310
 311 # -
 312
 313 libraries = gather_libraries(ROOTS, descs)
 314 print(f'\n{len(libraries)} libraries:')
 315 for k in libraries:
 316     print(' ', k)
 317
 318 if CHECK_ONLY:
 319     print('\n--check complete.')
 320     exit(0)
 321
 322 # ------------------------------------------------------------------------------
 323 # Output to moz.builds
 324
 325 import vendor_from_git
 326
 327 print('')
 328 vendor_from_git.record_cherry_picks(GECKO_ANGLE_DIR, GIT_REMOTE)
 329
 330 # --
 331
 332 def sortedi(x):
 333     return sorted(x, key=str.lower)
 334
 335 def append_arr(dest, name, vals, indent=0):
 336     if not vals:
 337         return
 338
 339     dest.append('{}{} += ['.format(' '*4*indent, name))
 340     for x in sortedi(vals):
 341         dest.append("{}'{}',".format(' '*4*(indent+1), x))
 342     dest.append('{}]'.format(' '*4*indent))
 343     dest.append('')
 344     return
 345
 346 REGISTERED_DEFINES = {
 347     'ANGLE_CAPTURE_ENABLED': True,
 348     'ANGLE_EGL_LIBRARY_NAME': False,
 349     'ANGLE_ENABLE_D3D11': True,
 350     'ANGLE_ENABLE_D3D9': True,
 351     'ANGLE_ENABLE_DEBUG_ANNOTATIONS': True,
 352     'ANGLE_ENABLE_NULL': False,
 353     'ANGLE_ENABLE_OPENGL': False,
 354     'ANGLE_ENABLE_OPENGL_NULL': False,
 355     'ANGLE_ENABLE_ESSL': True,
 356     'ANGLE_ENABLE_GLSL': True,
 357     'ANGLE_ENABLE_HLSL': True,
 358     'ANGLE_GENERATE_SHADER_DEBUG_INFO': True,
 359     'ANGLE_GLESV2_LIBRARY_NAME': True,
 360     'ANGLE_IS_64_BIT_CPU': False,
 361     'ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES': False,
 362     'ANGLE_USE_EGL_LOADER': True,
 363     'CERT_CHAIN_PARA_HAS_EXTRA_FIELDS': False,
 364     'CHROMIUM_BUILD': False,
 365     'COMPONENT_BUILD': False,
 366     'DYNAMIC_ANNOTATIONS_ENABLED': True,
 367     'EGL_EGL_PROTOTYPES': True,
 368     'EGL_EGLEXT_PROTOTYPES': True,
 369     'EGLAPI': True,
 370     'FIELDTRIAL_TESTING_ENABLED': False,
 371     'FULL_SAFE_BROWSING': False,
 372     'GL_API': True,
 373     'GL_APICALL': True,
 374     'GL_GLES_PROTOTYPES': True,
 375     'GL_GLEXT_PROTOTYPES': True,
 376     'GPU_INFO_USE_SETUPAPI': True,
 377     'LIBANGLE_IMPLEMENTATION': True,
 378     'LIBEGL_IMPLEMENTATION': True,
 379     'LIBGLESV2_IMPLEMENTATION': True,
 380     'NOMINMAX': True,
 381     'NO_TCMALLOC': False,
 382
 383     # Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
 384     'NTDDI_VERSION': True,
 385
 386     'PSAPI_VERSION': False,
 387     'SAFE_BROWSING_CSD': False,
 388     'SAFE_BROWSING_DB_LOCAL': False,
 389     'UNICODE': True,
 390     'USE_AURA': False,
 391     'V8_DEPRECATION_WARNINGS': False,
 392     'WIN32': False,
 393     'WIN32_LEAN_AND_MEAN': False,
 394     'WINAPI_FAMILY': False,
 395
 396     'WINVER': True,
 397     # Otherwise:
 398     # gfx/angle/targets/libANGLE
 399     # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.cpp:10:
 400     # In file included from c:/dev/mozilla/gecko4/gfx/angle/checkout/src\libANGLE/renderer/d3d/d3d11/converged/CompositorNativeWindow11.h:17:
 401     # C:\Program Files (x86)\Windows Kits\10\include\10.0.17763.0\winrt\Windows.ui.composition.interop.h(103,20): error: unknown type name 'POINTER_INFO'
 402     #         _In_ const POINTER_INFO& pointerInfo
 403     #                    ^
 404
 405     'WTF_USE_DYNAMIC_ANNOTATIONS': False,
 406     '_ATL_NO_OPENGL': True,
 407     '_CRT_RAND_S': True,
 408     '_CRT_SECURE_NO_DEPRECATE': True,
 409     '_DEBUG': False,
 410     '_HAS_EXCEPTIONS': True,
 411     '_HAS_ITERATOR_DEBUGGING': False,
 412     '_SCL_SECURE_NO_DEPRECATE': True,
 413     '_SECURE_ATL': True,
 414     '_UNICODE': True,
 415     '_USING_V110_SDK71_': False,
 416     '_WIN32_WINNT': False,
 417     '_WINDOWS': False,
 418     '__STD_C': False,
 419
 420     # clang specific
 421     'CR_CLANG_REVISION': True,
 422     'NDEBUG': False,
 423     'NVALGRIND': False,
 424     '_HAS_NODISCARD': False,
 425 }
 426
 427 # -
 428
 429 print('\nRun actions')
 430 required_files: Set[str] = set()
 431
 432 run_checked('ninja', '-C', str(OUT_DIR), ':commit_id')
 433 required_files |= set(descs['//:commit_id']['outputs'])
 434
 435 # -
 436
 437 # Export our targets
 438 print('\nExport targets')
 439
 440 # Clear our dest directories
 441 targets_dir = pathlib.Path(GECKO_ANGLE_DIR, 'targets')
 442 checkout_dir = pathlib.Path(GECKO_ANGLE_DIR, 'checkout')
 443
 444 shutil.rmtree(targets_dir, True)
 445 shutil.rmtree(checkout_dir, True)
 446 targets_dir.mkdir(exist_ok=True)
 447 checkout_dir.mkdir(exist_ok=True)
 448
 449 # -
 450
 451 def export_target(target_name) -> Set[str]:
 452     #print(' ', target_name)
 453     desc = descs[target_name]
 454     flat = flattened_target(target_name, descs)
 455     assert target_name.startswith('//:'), target_name
 456     name = target_name[3:]
 457
 458     required_files: Set[str] = set(flat['sources'])
 459
 460     # Create our manifest lines
 461     target_dir = targets_dir / name
 462     target_dir.mkdir(exist_ok=True)
 463
 464     lines = list(COMMON_HEADER)
 465     lines.append('')
 466
 467     for x in sorted(set(desc['defines'])):
 468         try:
 469             (k, v) = x.split('=', 1)
 470             v = f"'{v}'"
 471         except ValueError:
 472             (k, v) = (x, 'True')
 473         try:
 474             line = f"DEFINES['{k}'] = {v}"
 475             if REGISTERED_DEFINES[k] == False:
 476                 line = '#' + line
 477             lines.append(line)
 478         except KeyError:
 479             print(f'[{name}] Unrecognized define: {k}')
 480     lines.append('')
 481
 482     cxxflags = set(desc['cflags'] + desc['cflags_cc'])
 483
 484     def fixup_paths(listt):
 485         for x in set(listt):
 486             assert x.startswith('//'), x
 487             yield '../../checkout/' + x[2:]
 488
 489     sources_by_config: Dict[str,List[str]] = {}
 490     extras: Dict[str,str] = dict()
 491     for x in fixup_paths(flat['sources']):
 492         #print(' '*5, x)
 493         (b, e) = x.rsplit('.', 1)
 494         if e in ['h', 'y', 'l', 'inc', 'inl']:
 495             continue
 496         elif e in ['cpp', 'cc', 'c']:
 497             if b.endswith('_win'):
 498                 config = "CONFIG['OS_ARCH'] == 'WINNT'"
 499             elif b.endswith('_linux'):
 500                 # Include these on BSDs too.
 501                 config = "CONFIG['OS_ARCH'] not in ('Darwin', 'WINNT')"
 502             elif b.endswith('_mac'):
 503                 config = "CONFIG['OS_ARCH'] == 'Darwin'"
 504             elif b.endswith('_posix'):
 505                 config = "CONFIG['OS_ARCH'] != 'WINNT'"
 506             else:
 507                 config = '' # None can't compare against str.
 508
 509             sources_by_config.setdefault(config, []).append(x)
 510             continue
 511         elif e == 'rc':
 512             assert 'RCFILE' not in extras, (target_name, extras['RCFILE'], x)
 513             extras['RCFILE'] = f"'{x}'"
 514             continue
 515         elif e == 'def':
 516             assert 'DEFFILE' not in extras, (target_name, extras['DEFFILE'], x)
 517             extras['DEFFILE'] = f"'{x}'"
 518             continue
 519         else:
 520             assert False, ("Unhandled ext:", x)
 521
 522     ldflags = set(desc['ldflags'])
 523     DEF_PREFIX = '/DEF:'
 524     for x in set(ldflags):
 525         if x.startswith(DEF_PREFIX):
 526             def_path = x[len(DEF_PREFIX):]
 527             required_files.add(def_path)
 528             assert 'DEFFILE' not in extras
 529             ldflags.remove(x)
 530
 531             def_path = str(OUT_DIR) + '/' + def_path
 532             def_path = '//' + collapse_dotdots(def_path)
 533
 534             def_rel_path = list(fixup_paths([def_path]))[0]
 535             extras['DEFFILE'] = "'{}'".format(def_rel_path)
 536
 537     os_libs = list(map( lambda x: x[:-len('.lib')], set(desc.get('libs', [])) ))
 538
 539     def append_arr_commented(dest, name, src):
 540         lines = []
 541         append_arr(lines, name, src)
 542         def comment(x):
 543             if x:
 544                 x = '#' + x
 545             return x
 546         lines = map(comment, lines)
 547         dest += lines
 548
 549     append_arr(lines, 'LOCAL_INCLUDES', fixup_paths(desc['include_dirs']))
 550     append_arr_commented(lines, 'CXXFLAGS', cxxflags)
 551
 552     for (config,v) in sorted_items(sources_by_config):
 553         indent = 0
 554         if config:
 555             lines.append("if {}:".format(config))
 556             indent = 1
 557         append_arr(lines, 'SOURCES', v, indent=indent)
 558
 559     dep_libs: Set[str] = set()
 560     for dep_name in set(flat['deps']):
 561         dep = descs[dep_name]
 562         if dep['type'] in LIBRARY_TYPES:
 563             assert dep_name.startswith('//:'), dep_name
 564             dep_libs.add(dep_name[3:])
 565
 566     append_arr(lines, 'USE_LIBS', dep_libs)
 567     append_arr(lines, 'DIRS', ['../' + x for x in dep_libs])
 568     append_arr(lines, 'OS_LIBS', os_libs)
 569     append_arr_commented(lines, 'LDFLAGS', ldflags)
 570
 571     for (k,v) in sorted(extras.items()):
 572         lines.append('{} = {}'.format(k, v))
 573
 574     lib_type = desc['type']
 575     if lib_type == 'shared_library':
 576         lines.append(f"GeckoSharedLibrary('{name}', linkage=None)")
 577     elif lib_type == 'static_library':
 578         lines.append(f"Library('{name}')")
 579     else:
 580         assert False, lib_type
 581
 582     # Write it out
 583
 584     mozbuild = target_dir / 'moz.build'
 585     print(' ', ' ', f'Writing {mozbuild}')
 586     data = b'\n'.join((x.encode() for x in lines))
 587     mozbuild.write_bytes(data)
 588
 589     return required_files
 590
 591 # -
 592
 593 for target_name in libraries:
 594     reqs = export_target(target_name)
 595     required_files |= reqs
 596
 597 # Copy all the files
 598
 599 print('\nMigrate required files')
 600
 601 i = 0
 602 for x in required_files:
 603     i += 1
 604     sys.stdout.write(f'\r  Copying {i}/{len(required_files)}')
 605     sys.stdout.flush()
 606     assert x.startswith('//'), x
 607     x = x[2:]
 608
 609     src = REPO_DIR / x
 610     dest = checkout_dir / x
 611
 612     dest.parent.mkdir(parents=True, exist_ok=True)
 613     data = src.read_bytes()
 614     data = data.replace(b'\r\n', b'\n')
 615     dest.write_bytes(data)
 616
 617 print('\n\nDone')