intl/icu_sources_data.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6 # Generate SOURCES in sources.mozbuild files from ICU's Makefile.in
   7 # files, and also build a standalone copy of ICU using its build
   8 # system to generate a new copy of the in-tree ICU data file.
   9 #
  10 # This script expects to be run from `update-icu.sh` after the in-tree
  11 # copy of ICU has been updated.
  12
  13 from __future__ import absolute_import
  14 from __future__ import print_function
  15
  16 import glob
  17 import multiprocessing
  18 import os
  19 import shutil
  20 import subprocess
  21 import sys
  22 import tempfile
  23
  24 from mozpack import path as mozpath
  25
  26 # The following files have been determined to be dead/unused by a
  27 # semi-automated analysis. You can just remove any of the files below
  28 # if you need them. However, files marked with a "Cluster" comment
  29 # can only be removed together, as they have (directional) dependencies.
  30 # If you want to rerun this analysis, contact :decoder.
  31 UNUSED_SOURCES = set(
  32     [
  33         "intl/icu/source/common/bytestrieiterator.cpp",
  34         "intl/icu/source/common/cstr.cpp",
  35         "intl/icu/source/common/cwchar.cpp",
  36         "intl/icu/source/common/icudataver.cpp",
  37         "intl/icu/source/common/icuplug.cpp",
  38         "intl/icu/source/common/pluralmap.cpp",
  39         "intl/icu/source/common/ucat.cpp",
  40         "intl/icu/source/common/ucnv2022.cpp",
  41         "intl/icu/source/common/ucnv_ct.cpp",
  42         "intl/icu/source/common/ucnvdisp.cpp",
  43         "intl/icu/source/common/ucnv_ext.cpp",
  44         "intl/icu/source/common/ucnvhz.cpp",
  45         "intl/icu/source/common/ucnvisci.cpp",
  46         "intl/icu/source/common/ucnv_lmb.cpp",
  47         "intl/icu/source/common/ucnvmbcs.cpp",
  48         "intl/icu/source/common/uidna.cpp",
  49         "intl/icu/source/common/unorm.cpp",
  50         "intl/icu/source/common/usc_impl.cpp",
  51         "intl/icu/source/common/ustr_wcs.cpp",
  52         "intl/icu/source/common/util_props.cpp",
  53         "intl/icu/source/i18n/anytrans.cpp",
  54         "intl/icu/source/i18n/brktrans.cpp",
  55         "intl/icu/source/i18n/casetrn.cpp",
  56         "intl/icu/source/i18n/cpdtrans.cpp",
  57         "intl/icu/source/i18n/esctrn.cpp",
  58         "intl/icu/source/i18n/fmtable_cnv.cpp",
  59         "intl/icu/source/i18n/funcrepl.cpp",
  60         "intl/icu/source/i18n/gender.cpp",
  61         "intl/icu/source/i18n/name2uni.cpp",
  62         "intl/icu/source/i18n/nortrans.cpp",
  63         "intl/icu/source/i18n/nultrans.cpp",
  64         "intl/icu/source/i18n/quant.cpp",
  65         "intl/icu/source/i18n/rbt.cpp",
  66         "intl/icu/source/i18n/rbt_data.cpp",
  67         "intl/icu/source/i18n/rbt_pars.cpp",
  68         "intl/icu/source/i18n/rbt_rule.cpp",
  69         "intl/icu/source/i18n/rbt_set.cpp",
  70         "intl/icu/source/i18n/regexcmp.cpp",
  71         "intl/icu/source/i18n/regeximp.cpp",
  72         "intl/icu/source/i18n/regexst.cpp",
  73         "intl/icu/source/i18n/regextxt.cpp",
  74         "intl/icu/source/i18n/rematch.cpp",
  75         "intl/icu/source/i18n/remtrans.cpp",
  76         "intl/icu/source/i18n/repattrn.cpp",
  77         "intl/icu/source/i18n/scientificnumberformatter.cpp",
  78         "intl/icu/source/i18n/strmatch.cpp",
  79         "intl/icu/source/i18n/strrepl.cpp",
  80         "intl/icu/source/i18n/titletrn.cpp",
  81         "intl/icu/source/i18n/tolowtrn.cpp",
  82         "intl/icu/source/i18n/toupptrn.cpp",
  83         "intl/icu/source/i18n/translit.cpp",
  84         "intl/icu/source/i18n/transreg.cpp",
  85         "intl/icu/source/i18n/tridpars.cpp",
  86         "intl/icu/source/i18n/unesctrn.cpp",
  87         "intl/icu/source/i18n/uni2name.cpp",
  88         "intl/icu/source/i18n/uregexc.cpp",
  89         "intl/icu/source/i18n/uregex.cpp",
  90         "intl/icu/source/i18n/uregion.cpp",
  91         "intl/icu/source/i18n/uspoof_build.cpp",
  92         "intl/icu/source/i18n/uspoof_conf.cpp",
  93         "intl/icu/source/i18n/utrans.cpp",
  94         "intl/icu/source/i18n/vzone.cpp",
  95         "intl/icu/source/i18n/zrule.cpp",
  96         "intl/icu/source/i18n/ztrans.cpp",
  97         # Cluster
  98         "intl/icu/source/common/resbund_cnv.cpp",
  99         "intl/icu/source/common/ures_cnv.cpp",
 100         # Cluster
 101         "intl/icu/source/common/propsvec.cpp",
 102         "intl/icu/source/common/ucnvsel.cpp",
 103         "intl/icu/source/common/ucnv_set.cpp",
 104         # Cluster
 105         "intl/icu/source/common/ubiditransform.cpp",
 106         "intl/icu/source/common/ushape.cpp",
 107         # Cluster
 108         "intl/icu/source/i18n/csdetect.cpp",
 109         "intl/icu/source/i18n/csmatch.cpp",
 110         "intl/icu/source/i18n/csr2022.cpp",
 111         "intl/icu/source/i18n/csrecog.cpp",
 112         "intl/icu/source/i18n/csrmbcs.cpp",
 113         "intl/icu/source/i18n/csrsbcs.cpp",
 114         "intl/icu/source/i18n/csrucode.cpp",
 115         "intl/icu/source/i18n/csrutf8.cpp",
 116         "intl/icu/source/i18n/inputext.cpp",
 117         "intl/icu/source/i18n/ucsdet.cpp",
 118         # Cluster
 119         "intl/icu/source/i18n/alphaindex.cpp",
 120         "intl/icu/source/i18n/ulocdata.cpp",
 121     ]
 122 )
 123
 124
 125 def find_source_file(dir, filename):
 126     base = os.path.splitext(filename)[0]
 127     for ext in (".cpp", ".c"):
 128         f = mozpath.join(dir, base + ext)
 129         if os.path.isfile(f):
 130             return f
 131     raise Exception("Couldn't find source file for: %s" % filename)
 132
 133
 134 def get_sources_from_makefile(makefile):
 135     srcdir = os.path.dirname(makefile)
 136     with open(makefile) as f:
 137         contents = f.read().replace("\\\n", "").split("\n")
 138     for line in contents:
 139         if line.startswith("OBJECTS ="):
 140             return sorted(
 141                 (
 142                     find_source_file(srcdir, s)
 143                     for s in line[len("OBJECTS =") :].strip().split()
 144                 ),
 145                 key=lambda x: x.lower(),
 146             )
 147     raise AssertionError("OBJECTS definition not found in file %s" % makefile)
 148
 149
 150 def list_headers(path):
 151     result = []
 152     for name in os.listdir(path):
 153         f = mozpath.join(path, name)
 154         if os.path.isfile(f):
 155             result.append(f)
 156     return sorted(result, key=lambda x: x.lower())
 157
 158
 159 def write_sources(mozbuild, sources, headers):
 160     with open(mozbuild, "w", newline="\n", encoding="utf-8") as f:
 161         f.write(
 162             "# THIS FILE IS GENERATED BY /intl/icu_sources_data.py " + "DO NOT EDIT\n"
 163         )
 164
 165         def write_list(name, content):
 166             if content:
 167                 f.write("%s %s [\n" % (name, "=" if name.islower() else "+="))
 168                 f.write("".join("   '/%s',\n" % s for s in content))
 169                 f.write("]\n")
 170
 171         write_list("sources", [s for s in sources if s not in UNUSED_SOURCES])
 172         write_list("other_sources", [s for s in sources if s in UNUSED_SOURCES])
 173         write_list("EXPORTS.unicode", headers)
 174
 175
 176 def update_sources(topsrcdir):
 177     print("Updating ICU sources lists...")
 178     for d in ["common", "i18n", "tools/toolutil", "tools/icupkg"]:
 179         base_path = mozpath.join(topsrcdir, "intl/icu/source/%s" % d)
 180         makefile = mozpath.join(base_path, "Makefile.in")
 181         mozbuild = mozpath.join(
 182             topsrcdir, "config/external/icu/%s/sources.mozbuild" % mozpath.basename(d)
 183         )
 184         sources = [
 185             mozpath.relpath(s, topsrcdir) for s in get_sources_from_makefile(makefile)
 186         ]
 187         unicode_dir = mozpath.join(base_path, "unicode")
 188         if os.path.exists(unicode_dir):
 189             headers = [
 190                 mozpath.normsep(os.path.relpath(s, topsrcdir))
 191                 for s in list_headers(unicode_dir)
 192             ]
 193         else:
 194             headers = None
 195         write_sources(mozbuild, sources, headers)
 196
 197
 198 def try_run(name, command, cwd=None, **kwargs):
 199     try:
 200         with tempfile.NamedTemporaryFile(prefix=name, delete=False) as f:
 201             subprocess.check_call(
 202                 command, cwd=cwd, stdout=f, stderr=subprocess.STDOUT, **kwargs
 203             )
 204     except subprocess.CalledProcessError:
 205         print(
 206             """Error running "{}" in directory {}
 207     See output in {}""".format(
 208                 " ".join(command), cwd, f.name
 209             ),
 210             file=sys.stderr,
 211         )
 212         return False
 213     else:
 214         os.unlink(f.name)
 215         return True
 216
 217
 218 def get_data_file(data_dir):
 219     files = glob.glob(mozpath.join(data_dir, "icudt*.dat"))
 220     return files[0] if files else None
 221
 222
 223 def update_data_file(topsrcdir):
 224     objdir = tempfile.mkdtemp(prefix="icu-obj-")
 225     configure = mozpath.join(topsrcdir, "intl/icu/source/configure")
 226     env = dict(os.environ)
 227     # bug 1262101 - these should be shared with the moz.build files
 228     env.update(
 229         {
 230             "CPPFLAGS": (
 231                 "-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 "
 232                 + "-DU_HIDE_OBSOLETE_UTF_OLD_H=1"
 233                 + "-DUCONFIG_NO_LEGACY_CONVERSION "
 234                 + "-DUCONFIG_NO_TRANSLITERATION "
 235                 + "-DUCONFIG_NO_REGULAR_EXPRESSIONS "
 236                 + "-DUCONFIG_NO_BREAK_ITERATION "
 237                 + "-DU_CHARSET_IS_UTF8"
 238             )
 239         }
 240     )
 241
 242     # Exclude data that we currently don't need.
 243     #
 244     # The file format for ICU's data build tool is described at
 245     # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
 246     env["ICU_DATA_FILTER_FILE"] = mozpath.join(topsrcdir, "intl/icu/data_filter.json")
 247
 248     print("Running ICU configure...")
 249     if not try_run(
 250         "icu-configure",
 251         [
 252             "sh",
 253             configure,
 254             "--with-data-packaging=archive",
 255             "--enable-static",
 256             "--disable-shared",
 257             "--disable-extras",
 258             "--disable-icuio",
 259             "--disable-layout",
 260             "--disable-layoutex",
 261             "--disable-tests",
 262             "--disable-samples",
 263             "--disable-strict",
 264         ],
 265         cwd=objdir,
 266         env=env,
 267     ):
 268         return False
 269     print("Running ICU make...")
 270     if not try_run(
 271         "icu-make",
 272         ["make", "--jobs=%d" % multiprocessing.cpu_count(), "--output-sync"],
 273         cwd=objdir,
 274     ):
 275         return False
 276     print("Copying ICU data file...")
 277     tree_data_path = mozpath.join(topsrcdir, "config/external/icu/data/")
 278     old_data_file = get_data_file(tree_data_path)
 279     if not old_data_file:
 280         print("Error: no ICU data file in %s" % tree_data_path, file=sys.stderr)
 281         return False
 282     new_data_file = get_data_file(mozpath.join(objdir, "data/out"))
 283     if not new_data_file:
 284         print("Error: no ICU data in ICU objdir", file=sys.stderr)
 285         return False
 286     if os.path.basename(old_data_file) != os.path.basename(new_data_file):
 287         # Data file name has the major version number embedded.
 288         os.unlink(old_data_file)
 289     shutil.copy(new_data_file, tree_data_path)
 290     try:
 291         shutil.rmtree(objdir)
 292     except Exception:
 293         print("Warning: failed to remove %s" % objdir, file=sys.stderr)
 294     return True
 295
 296
 297 def main():
 298     if len(sys.argv) != 2:
 299         print("Usage: icu_sources_data.py <mozilla topsrcdir>", file=sys.stderr)
 300         sys.exit(1)
 301
 302     topsrcdir = mozpath.abspath(sys.argv[1])
 303     update_sources(topsrcdir)
 304     if not update_data_file(topsrcdir):
 305         print("Error updating ICU data file", file=sys.stderr)
 306         sys.exit(1)
 307
 308
 309 if __name__ == "__main__":
 310     main()