intl/icu_sources_data.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6 # Generate SOURCES in sources.mozbuild files from ICU's Makefile.in
   7 # files, and also build a standalone copy of ICU using its build
   8 # system to generate a new copy of the in-tree ICU data file.
   9 #
  10 # This script expects to be run from `update-icu.sh` after the in-tree
  11 # copy of ICU has been updated.
  12
  13 import glob
  14 import multiprocessing
  15 import os
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tempfile
  20
  21 from mozpack import path as mozpath
  22
  23 # The following files have been determined to be dead/unused by a
  24 # semi-automated analysis. You can just remove any of the files below
  25 # if you need them. However, files marked with a "Cluster" comment
  26 # can only be removed together, as they have (directional) dependencies.
  27 # If you want to rerun this analysis, contact :decoder.
  28 UNUSED_SOURCES = set(
  29     [
  30         "intl/icu/source/common/bytestrieiterator.cpp",
  31         "intl/icu/source/common/cstr.cpp",
  32         "intl/icu/source/common/cwchar.cpp",
  33         "intl/icu/source/common/icudataver.cpp",
  34         "intl/icu/source/common/icuplug.cpp",
  35         "intl/icu/source/common/pluralmap.cpp",
  36         "intl/icu/source/common/ucat.cpp",
  37         "intl/icu/source/common/ucnv2022.cpp",
  38         "intl/icu/source/common/ucnv_ct.cpp",
  39         "intl/icu/source/common/ucnvdisp.cpp",
  40         "intl/icu/source/common/ucnv_ext.cpp",
  41         "intl/icu/source/common/ucnvhz.cpp",
  42         "intl/icu/source/common/ucnvisci.cpp",
  43         "intl/icu/source/common/ucnv_lmb.cpp",
  44         "intl/icu/source/common/ucnvmbcs.cpp",
  45         "intl/icu/source/common/uidna.cpp",
  46         "intl/icu/source/common/unorm.cpp",
  47         "intl/icu/source/common/usc_impl.cpp",
  48         "intl/icu/source/common/ustr_wcs.cpp",
  49         "intl/icu/source/common/util_props.cpp",
  50         "intl/icu/source/i18n/anytrans.cpp",
  51         "intl/icu/source/i18n/brktrans.cpp",
  52         "intl/icu/source/i18n/casetrn.cpp",
  53         "intl/icu/source/i18n/cpdtrans.cpp",
  54         "intl/icu/source/i18n/esctrn.cpp",
  55         "intl/icu/source/i18n/fmtable_cnv.cpp",
  56         "intl/icu/source/i18n/funcrepl.cpp",
  57         "intl/icu/source/i18n/gender.cpp",
  58         "intl/icu/source/i18n/name2uni.cpp",
  59         "intl/icu/source/i18n/nortrans.cpp",
  60         "intl/icu/source/i18n/nultrans.cpp",
  61         "intl/icu/source/i18n/quant.cpp",
  62         "intl/icu/source/i18n/rbt.cpp",
  63         "intl/icu/source/i18n/rbt_data.cpp",
  64         "intl/icu/source/i18n/rbt_pars.cpp",
  65         "intl/icu/source/i18n/rbt_rule.cpp",
  66         "intl/icu/source/i18n/rbt_set.cpp",
  67         "intl/icu/source/i18n/regexcmp.cpp",
  68         "intl/icu/source/i18n/regeximp.cpp",
  69         "intl/icu/source/i18n/regexst.cpp",
  70         "intl/icu/source/i18n/regextxt.cpp",
  71         "intl/icu/source/i18n/rematch.cpp",
  72         "intl/icu/source/i18n/remtrans.cpp",
  73         "intl/icu/source/i18n/repattrn.cpp",
  74         "intl/icu/source/i18n/scientificnumberformatter.cpp",
  75         "intl/icu/source/i18n/strmatch.cpp",
  76         "intl/icu/source/i18n/strrepl.cpp",
  77         "intl/icu/source/i18n/titletrn.cpp",
  78         "intl/icu/source/i18n/tolowtrn.cpp",
  79         "intl/icu/source/i18n/toupptrn.cpp",
  80         "intl/icu/source/i18n/translit.cpp",
  81         "intl/icu/source/i18n/transreg.cpp",
  82         "intl/icu/source/i18n/tridpars.cpp",
  83         "intl/icu/source/i18n/unesctrn.cpp",
  84         "intl/icu/source/i18n/uni2name.cpp",
  85         "intl/icu/source/i18n/uregexc.cpp",
  86         "intl/icu/source/i18n/uregex.cpp",
  87         "intl/icu/source/i18n/uregion.cpp",
  88         "intl/icu/source/i18n/uspoof_build.cpp",
  89         "intl/icu/source/i18n/uspoof_conf.cpp",
  90         "intl/icu/source/i18n/utrans.cpp",
  91         "intl/icu/source/i18n/vzone.cpp",
  92         "intl/icu/source/i18n/zrule.cpp",
  93         "intl/icu/source/i18n/ztrans.cpp",
  94         # Cluster
  95         "intl/icu/source/common/resbund_cnv.cpp",
  96         "intl/icu/source/common/ures_cnv.cpp",
  97         # Cluster
  98         "intl/icu/source/common/propsvec.cpp",
  99         "intl/icu/source/common/ucnvsel.cpp",
 100         "intl/icu/source/common/ucnv_set.cpp",
 101         # Cluster
 102         "intl/icu/source/common/ubiditransform.cpp",
 103         "intl/icu/source/common/ushape.cpp",
 104         # Cluster
 105         "intl/icu/source/i18n/csdetect.cpp",
 106         "intl/icu/source/i18n/csmatch.cpp",
 107         "intl/icu/source/i18n/csr2022.cpp",
 108         "intl/icu/source/i18n/csrecog.cpp",
 109         "intl/icu/source/i18n/csrmbcs.cpp",
 110         "intl/icu/source/i18n/csrsbcs.cpp",
 111         "intl/icu/source/i18n/csrucode.cpp",
 112         "intl/icu/source/i18n/csrutf8.cpp",
 113         "intl/icu/source/i18n/inputext.cpp",
 114         "intl/icu/source/i18n/ucsdet.cpp",
 115         # Cluster
 116         "intl/icu/source/i18n/alphaindex.cpp",
 117         "intl/icu/source/i18n/ulocdata.cpp",
 118     ]
 119 )
 120
 121
 122 def ensure_source_file_exists(dir, filename):
 123     f = mozpath.join(dir, filename)
 124     if os.path.isfile(f):
 125         return f
 126     raise Exception("Couldn't find source file for: %s" % filename)
 127
 128
 129 def get_sources(sources_file):
 130     srcdir = os.path.dirname(sources_file)
 131     with open(sources_file) as f:
 132         return sorted(
 133             (ensure_source_file_exists(srcdir, name.strip()) for name in f),
 134             key=lambda x: x.lower(),
 135         )
 136
 137
 138 def list_headers(path):
 139     result = []
 140     for name in os.listdir(path):
 141         f = mozpath.join(path, name)
 142         if os.path.isfile(f):
 143             result.append(f)
 144     return sorted(result, key=lambda x: x.lower())
 145
 146
 147 def write_sources(mozbuild, sources, headers):
 148     with open(mozbuild, "w", newline="\n", encoding="utf-8") as f:
 149         f.write(
 150             "# THIS FILE IS GENERATED BY /intl/icu_sources_data.py " + "DO NOT EDIT\n"
 151         )
 152
 153         def write_list(name, content):
 154             if content:
 155                 f.write("%s %s [\n" % (name, "=" if name.islower() else "+="))
 156                 f.write("".join("   '/%s',\n" % s for s in content))
 157                 f.write("]\n")
 158
 159         write_list("sources", [s for s in sources if s not in UNUSED_SOURCES])
 160         write_list("other_sources", [s for s in sources if s in UNUSED_SOURCES])
 161         write_list("EXPORTS.unicode", headers)
 162
 163
 164 def update_sources(topsrcdir):
 165     print("Updating ICU sources lists...")
 166     for d in ["common", "i18n", "tools/toolutil", "tools/icupkg"]:
 167         base_path = mozpath.join(topsrcdir, "intl/icu/source/%s" % d)
 168         sources_file = mozpath.join(base_path, "sources.txt")
 169         mozbuild = mozpath.join(
 170             topsrcdir, "config/external/icu/%s/sources.mozbuild" % mozpath.basename(d)
 171         )
 172         sources = [mozpath.relpath(s, topsrcdir) for s in get_sources(sources_file)]
 173         unicode_dir = mozpath.join(base_path, "unicode")
 174         if os.path.exists(unicode_dir):
 175             headers = [
 176                 mozpath.normsep(os.path.relpath(s, topsrcdir))
 177                 for s in list_headers(unicode_dir)
 178             ]
 179         else:
 180             headers = None
 181         write_sources(mozbuild, sources, headers)
 182
 183
 184 def try_run(name, command, cwd=None, **kwargs):
 185     try:
 186         with tempfile.NamedTemporaryFile(prefix=name, delete=False) as f:
 187             subprocess.check_call(
 188                 command, cwd=cwd, stdout=f, stderr=subprocess.STDOUT, **kwargs
 189             )
 190     except subprocess.CalledProcessError:
 191         print(
 192             """Error running "{}" in directory {}
 193     See output in {}""".format(
 194                 " ".join(command), cwd, f.name
 195             ),
 196             file=sys.stderr,
 197         )
 198         return False
 199     else:
 200         os.unlink(f.name)
 201         return True
 202
 203
 204 def get_data_file(data_dir):
 205     files = glob.glob(mozpath.join(data_dir, "icudt*.dat"))
 206     return files[0] if files else None
 207
 208
 209 def update_data_file(topsrcdir):
 210     objdir = tempfile.mkdtemp(prefix="icu-obj-")
 211     configure = mozpath.join(topsrcdir, "intl/icu/source/configure")
 212     env = dict(os.environ)
 213     # bug 1262101 - these should be shared with the moz.build files
 214     env.update(
 215         {
 216             "CPPFLAGS": (
 217                 "-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 "
 218                 + "-DU_HIDE_OBSOLETE_UTF_OLD_H=1"
 219                 + "-DUCONFIG_NO_LEGACY_CONVERSION "
 220                 + "-DUCONFIG_NO_TRANSLITERATION "
 221                 + "-DUCONFIG_NO_REGULAR_EXPRESSIONS "
 222                 + "-DUCONFIG_NO_BREAK_ITERATION "
 223                 + "-DU_CHARSET_IS_UTF8"
 224             )
 225         }
 226     )
 227
 228     # Exclude data that we currently don't need.
 229     #
 230     # The file format for ICU's data build tool is described at
 231     # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
 232     env["ICU_DATA_FILTER_FILE"] = mozpath.join(topsrcdir, "intl/icu/data_filter.json")
 233
 234     print("Running ICU configure...")
 235     if not try_run(
 236         "icu-configure",
 237         [
 238             "sh",
 239             configure,
 240             "--with-data-packaging=archive",
 241             "--enable-static",
 242             "--disable-shared",
 243             "--disable-extras",
 244             "--disable-icuio",
 245             "--disable-layout",
 246             "--disable-layoutex",
 247             "--disable-tests",
 248             "--disable-samples",
 249             "--disable-strict",
 250         ],
 251         cwd=objdir,
 252         env=env,
 253     ):
 254         return False
 255     print("Running ICU make...")
 256     if not try_run(
 257         "icu-make",
 258         ["make", "--jobs=%d" % multiprocessing.cpu_count(), "--output-sync"],
 259         cwd=objdir,
 260     ):
 261         return False
 262     print("Copying ICU data file...")
 263     tree_data_path = mozpath.join(topsrcdir, "config/external/icu/data/")
 264     old_data_file = get_data_file(tree_data_path)
 265     if not old_data_file:
 266         print("Error: no ICU data file in %s" % tree_data_path, file=sys.stderr)
 267         return False
 268     new_data_file = get_data_file(mozpath.join(objdir, "data/out"))
 269     if not new_data_file:
 270         print("Error: no ICU data in ICU objdir", file=sys.stderr)
 271         return False
 272     if os.path.basename(old_data_file) != os.path.basename(new_data_file):
 273         # Data file name has the major version number embedded.
 274         os.unlink(old_data_file)
 275     shutil.copy(new_data_file, tree_data_path)
 276     try:
 277         shutil.rmtree(objdir)
 278     except Exception:
 279         print("Warning: failed to remove %s" % objdir, file=sys.stderr)
 280     return True
 281
 282
 283 def main():
 284     if len(sys.argv) != 2:
 285         print("Usage: icu_sources_data.py <mozilla topsrcdir>", file=sys.stderr)
 286         sys.exit(1)
 287
 288     topsrcdir = mozpath.abspath(sys.argv[1])
 289     update_sources(topsrcdir)
 290     if not update_data_file(topsrcdir):
 291         print("Error updating ICU data file", file=sys.stderr)
 292         sys.exit(1)
 293
 294
 295 if __name__ == "__main__":
 296     main()