intl/icu_sources_data.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6 # Generate SOURCES in sources.mozbuild files from ICU's Makefile.in
   7 # files, and also build a standalone copy of ICU using its build
   8 # system to generate a new copy of the in-tree ICU data file.
   9 #
  10 # This script expects to be run from `update-icu.sh` after the in-tree
  11 # copy of ICU has been updated.
  12
  13 import glob
  14 import multiprocessing
  15 import os
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tempfile
  20
  21 from mozpack import path as mozpath
  22
  23 # The following files have been determined to be dead/unused by a
  24 # semi-automated analysis. You can just remove any of the files below
  25 # if you need them. However, files marked with a "Cluster" comment
  26 # can only be removed together, as they have (directional) dependencies.
  27 # If you want to rerun this analysis, contact :decoder.
  28 UNUSED_SOURCES = set(
  29     [
  30         "intl/icu/source/common/bytestrieiterator.cpp",
  31         "intl/icu/source/common/cstr.cpp",
  32         "intl/icu/source/common/cwchar.cpp",
  33         "intl/icu/source/common/icudataver.cpp",
  34         "intl/icu/source/common/icuplug.cpp",
  35         "intl/icu/source/common/pluralmap.cpp",
  36         "intl/icu/source/common/ucat.cpp",
  37         "intl/icu/source/common/ucnv2022.cpp",
  38         "intl/icu/source/common/ucnv_ct.cpp",
  39         "intl/icu/source/common/ucnvdisp.cpp",
  40         "intl/icu/source/common/ucnv_ext.cpp",
  41         "intl/icu/source/common/ucnvhz.cpp",
  42         "intl/icu/source/common/ucnvisci.cpp",
  43         "intl/icu/source/common/ucnv_lmb.cpp",
  44         "intl/icu/source/common/ucnvmbcs.cpp",
  45         "intl/icu/source/common/uidna.cpp",
  46         "intl/icu/source/common/unorm.cpp",
  47         "intl/icu/source/common/usc_impl.cpp",
  48         "intl/icu/source/common/ustr_wcs.cpp",
  49         "intl/icu/source/common/util_props.cpp",
  50         "intl/icu/source/i18n/anytrans.cpp",
  51         "intl/icu/source/i18n/brktrans.cpp",
  52         "intl/icu/source/i18n/casetrn.cpp",
  53         "intl/icu/source/i18n/cpdtrans.cpp",
  54         "intl/icu/source/i18n/esctrn.cpp",
  55         "intl/icu/source/i18n/fmtable_cnv.cpp",
  56         "intl/icu/source/i18n/funcrepl.cpp",
  57         "intl/icu/source/i18n/gender.cpp",
  58         "intl/icu/source/i18n/name2uni.cpp",
  59         "intl/icu/source/i18n/nortrans.cpp",
  60         "intl/icu/source/i18n/nultrans.cpp",
  61         "intl/icu/source/i18n/quant.cpp",
  62         "intl/icu/source/i18n/rbt.cpp",
  63         "intl/icu/source/i18n/rbt_data.cpp",
  64         "intl/icu/source/i18n/rbt_pars.cpp",
  65         "intl/icu/source/i18n/rbt_rule.cpp",
  66         "intl/icu/source/i18n/rbt_set.cpp",
  67         "intl/icu/source/i18n/regexcmp.cpp",
  68         "intl/icu/source/i18n/regeximp.cpp",
  69         "intl/icu/source/i18n/regexst.cpp",
  70         "intl/icu/source/i18n/regextxt.cpp",
  71         "intl/icu/source/i18n/rematch.cpp",
  72         "intl/icu/source/i18n/remtrans.cpp",
  73         "intl/icu/source/i18n/repattrn.cpp",
  74         "intl/icu/source/i18n/scientificnumberformatter.cpp",
  75         "intl/icu/source/i18n/strmatch.cpp",
  76         "intl/icu/source/i18n/strrepl.cpp",
  77         "intl/icu/source/i18n/titletrn.cpp",
  78         "intl/icu/source/i18n/tolowtrn.cpp",
  79         "intl/icu/source/i18n/toupptrn.cpp",
  80         "intl/icu/source/i18n/translit.cpp",
  81         "intl/icu/source/i18n/transreg.cpp",
  82         "intl/icu/source/i18n/tridpars.cpp",
  83         "intl/icu/source/i18n/unesctrn.cpp",
  84         "intl/icu/source/i18n/uni2name.cpp",
  85         "intl/icu/source/i18n/uregexc.cpp",
  86         "intl/icu/source/i18n/uregex.cpp",
  87         "intl/icu/source/i18n/uregion.cpp",
  88         "intl/icu/source/i18n/uspoof_build.cpp",
  89         "intl/icu/source/i18n/uspoof_conf.cpp",
  90         "intl/icu/source/i18n/utrans.cpp",
  91         "intl/icu/source/i18n/vzone.cpp",
  92         "intl/icu/source/i18n/zrule.cpp",
  93         "intl/icu/source/i18n/ztrans.cpp",
  94         # Cluster
  95         "intl/icu/source/common/resbund_cnv.cpp",
  96         "intl/icu/source/common/ures_cnv.cpp",
  97         # Cluster
  98         "intl/icu/source/common/propsvec.cpp",
  99         "intl/icu/source/common/ucnvsel.cpp",
 100         "intl/icu/source/common/ucnv_set.cpp",
 101         # Cluster
 102         "intl/icu/source/common/ubiditransform.cpp",
 103         "intl/icu/source/common/ushape.cpp",
 104         # Cluster
 105         "intl/icu/source/i18n/csdetect.cpp",
 106         "intl/icu/source/i18n/csmatch.cpp",
 107         "intl/icu/source/i18n/csr2022.cpp",
 108         "intl/icu/source/i18n/csrecog.cpp",
 109         "intl/icu/source/i18n/csrmbcs.cpp",
 110         "intl/icu/source/i18n/csrsbcs.cpp",
 111         "intl/icu/source/i18n/csrucode.cpp",
 112         "intl/icu/source/i18n/csrutf8.cpp",
 113         "intl/icu/source/i18n/inputext.cpp",
 114         "intl/icu/source/i18n/ucsdet.cpp",
 115         # Cluster
 116         "intl/icu/source/i18n/alphaindex.cpp",
 117         "intl/icu/source/i18n/ulocdata.cpp",
 118     ]
 119     +
 120     # We use the version of double-conversion vendored in mfbt instead
 121     [
 122         "intl/icu/source/i18n/double-conversion-bignum-dtoa.cpp",
 123         "intl/icu/source/i18n/double-conversion-bignum-dtoa.h",
 124         "intl/icu/source/i18n/double-conversion-bignum.cpp",
 125         "intl/icu/source/i18n/double-conversion-bignum.h",
 126         "intl/icu/source/i18n/double-conversion-cached-powers.cpp",
 127         "intl/icu/source/i18n/double-conversion-cached-powers.h",
 128         "intl/icu/source/i18n/double-conversion-diy-fp.h",
 129         "intl/icu/source/i18n/double-conversion-double-to-string.cpp",
 130         "intl/icu/source/i18n/double-conversion-double-to-string.h",
 131         "intl/icu/source/i18n/double-conversion-fast-dtoa.cpp",
 132         "intl/icu/source/i18n/double-conversion-fast-dtoa.h",
 133         "intl/icu/source/i18n/double-conversion-ieee.h",
 134         "intl/icu/source/i18n/double-conversion-string-to-double.cpp",
 135         "intl/icu/source/i18n/double-conversion-string-to-double.h",
 136         "intl/icu/source/i18n/double-conversion-strtod.cpp",
 137         "intl/icu/source/i18n/double-conversion-strtod.h",
 138         "intl/icu/source/i18n/double-conversion-utils.h",
 139         "intl/icu/source/i18n/double-conversion.h",
 140     ]
 141 )
 142
 143
 144 def ensure_source_file_exists(dir, filename):
 145     f = mozpath.join(dir, filename)
 146     if os.path.isfile(f):
 147         return f
 148     raise Exception("Couldn't find source file for: %s" % filename)
 149
 150
 151 def get_sources(sources_file):
 152     srcdir = os.path.dirname(sources_file)
 153     with open(sources_file) as f:
 154         return sorted(
 155             (ensure_source_file_exists(srcdir, name.strip()) for name in f),
 156             key=lambda x: x.lower(),
 157         )
 158
 159
 160 def list_headers(path):
 161     result = []
 162     for name in os.listdir(path):
 163         f = mozpath.join(path, name)
 164         if os.path.isfile(f):
 165             result.append(f)
 166     return sorted(result, key=lambda x: x.lower())
 167
 168
 169 def write_sources(mozbuild, sources, headers):
 170     with open(mozbuild, "w", newline="\n", encoding="utf-8") as f:
 171         f.write(
 172             "# THIS FILE IS GENERATED BY /intl/icu_sources_data.py " + "DO NOT EDIT\n"
 173         )
 174
 175         def write_list(name, content):
 176             if content:
 177                 f.write("%s %s [\n" % (name, "=" if name.islower() else "+="))
 178                 f.write("".join('    "/%s",\n' % s for s in content))
 179                 f.write("]\n")
 180
 181         write_list("sources", [s for s in sources if s not in UNUSED_SOURCES])
 182         write_list("other_sources", [s for s in sources if s in UNUSED_SOURCES])
 183         write_list("EXPORTS.unicode", headers)
 184
 185
 186 def update_sources(topsrcdir):
 187     print("Updating ICU sources lists...")
 188     for d in ["common", "i18n", "tools/toolutil", "tools/icupkg"]:
 189         base_path = mozpath.join(topsrcdir, "intl/icu/source/%s" % d)
 190         sources_file = mozpath.join(base_path, "sources.txt")
 191         mozbuild = mozpath.join(
 192             topsrcdir, "config/external/icu/%s/sources.mozbuild" % mozpath.basename(d)
 193         )
 194         sources = [mozpath.relpath(s, topsrcdir) for s in get_sources(sources_file)]
 195         unicode_dir = mozpath.join(base_path, "unicode")
 196         if os.path.exists(unicode_dir):
 197             headers = [
 198                 mozpath.normsep(os.path.relpath(s, topsrcdir))
 199                 for s in list_headers(unicode_dir)
 200             ]
 201         else:
 202             headers = None
 203         write_sources(mozbuild, sources, headers)
 204
 205
 206 def try_run(name, command, cwd=None, **kwargs):
 207     try:
 208         with tempfile.NamedTemporaryFile(prefix=name, delete=False) as f:
 209             subprocess.check_call(
 210                 command, cwd=cwd, stdout=f, stderr=subprocess.STDOUT, **kwargs
 211             )
 212     except subprocess.CalledProcessError:
 213         print(
 214             """Error running "{}" in directory {}
 215     See output in {}""".format(
 216                 " ".join(command), cwd, f.name
 217             ),
 218             file=sys.stderr,
 219         )
 220         return False
 221     else:
 222         os.unlink(f.name)
 223         return True
 224
 225
 226 def get_data_file(data_dir):
 227     files = glob.glob(mozpath.join(data_dir, "icudt*.dat"))
 228     return files[0] if files else None
 229
 230
 231 def update_data_file(topsrcdir):
 232     objdir = tempfile.mkdtemp(prefix="icu-obj-")
 233     configure = mozpath.join(topsrcdir, "intl/icu/source/configure")
 234     env = dict(os.environ)
 235     # bug 1262101 - these should be shared with the moz.build files
 236     env.update(
 237         {
 238             "CPPFLAGS": (
 239                 "-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 "
 240                 + "-DU_HIDE_OBSOLETE_UTF_OLD_H=1 "
 241                 + "-DUCONFIG_NO_LEGACY_CONVERSION "
 242                 + "-DUCONFIG_NO_TRANSLITERATION "
 243                 + "-DUCONFIG_NO_REGULAR_EXPRESSIONS "
 244                 + "-DUCONFIG_NO_BREAK_ITERATION "
 245                 + "-DU_CHARSET_IS_UTF8 "
 246             )
 247         }
 248     )
 249
 250     # Exclude data that we currently don't need.
 251     #
 252     # The file format for ICU's data build tool is described at
 253     # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
 254     env["ICU_DATA_FILTER_FILE"] = mozpath.join(topsrcdir, "intl/icu/data_filter.json")
 255
 256     print("Running ICU configure...")
 257     if not try_run(
 258         "icu-configure",
 259         [
 260             "sh",
 261             configure,
 262             "--with-data-packaging=archive",
 263             "--enable-static",
 264             "--disable-shared",
 265             "--disable-extras",
 266             "--disable-icuio",
 267             "--disable-layout",
 268             "--disable-layoutex",
 269             "--disable-tests",
 270             "--disable-samples",
 271             "--disable-strict",
 272         ],
 273         cwd=objdir,
 274         env=env,
 275     ):
 276         return False
 277     print("Running ICU make...")
 278     if not try_run(
 279         "icu-make",
 280         ["make", "--jobs=%d" % multiprocessing.cpu_count(), "--output-sync"],
 281         cwd=objdir,
 282     ):
 283         return False
 284     print("Copying ICU data file...")
 285     tree_data_path = mozpath.join(topsrcdir, "config/external/icu/data/")
 286     old_data_file = get_data_file(tree_data_path)
 287     if not old_data_file:
 288         print("Error: no ICU data file in %s" % tree_data_path, file=sys.stderr)
 289         return False
 290     new_data_file = get_data_file(mozpath.join(objdir, "data/out"))
 291     if not new_data_file:
 292         print("Error: no ICU data in ICU objdir", file=sys.stderr)
 293         return False
 294     if os.path.basename(old_data_file) != os.path.basename(new_data_file):
 295         # Data file name has the major version number embedded.
 296         os.unlink(old_data_file)
 297     shutil.copy(new_data_file, tree_data_path)
 298     try:
 299         shutil.rmtree(objdir)
 300     except Exception:
 301         print("Warning: failed to remove %s" % objdir, file=sys.stderr)
 302     return True
 303
 304
 305 def main():
 306     if len(sys.argv) != 2:
 307         print("Usage: icu_sources_data.py <mozilla topsrcdir>", file=sys.stderr)
 308         sys.exit(1)
 309
 310     topsrcdir = mozpath.abspath(sys.argv[1])
 311     update_sources(topsrcdir)
 312     if not update_data_file(topsrcdir):
 313         print("Error updating ICU data file", file=sys.stderr)
 314         sys.exit(1)
 315
 316
 317 if __name__ == "__main__":
 318     main()