intl/icu_sources_data.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6 # Generate SOURCES in sources.mozbuild files from ICU's Makefile.in
   7 # files, and also build a standalone copy of ICU using its build
   8 # system to generate a new copy of the in-tree ICU data file.
   9 #
  10 # This script expects to be run from `update-icu.sh` after the in-tree
  11 # copy of ICU has been updated.
  12
  13 from __future__ import absolute_import
  14 from __future__ import print_function
  15
  16 import glob
  17 import multiprocessing
  18 import os
  19 import sets
  20 import shutil
  21 import subprocess
  22 import sys
  23 import tempfile
  24
  25 from mozpack import path as mozpath
  26
  27 # The following files have been determined to be dead/unused by a
  28 # semi-automated analysis. You can just remove any of the files below
  29 # if you need them. However, files marked with a "Cluster" comment
  30 # can only be removed together, as they have (directional) dependencies.
  31 # If you want to rerun this analysis, contact :decoder.
  32 UNUSED_SOURCES = sets.Set([
  33     'intl/icu/source/common/bytestrieiterator.cpp',
  34     'intl/icu/source/common/cstr.cpp',
  35     'intl/icu/source/common/cwchar.cpp',
  36     'intl/icu/source/common/icudataver.cpp',
  37     'intl/icu/source/common/icuplug.cpp',
  38     'intl/icu/source/common/pluralmap.cpp',
  39     'intl/icu/source/common/ucat.cpp',
  40     'intl/icu/source/common/ucnv2022.cpp',
  41     'intl/icu/source/common/ucnv_ct.cpp',
  42     'intl/icu/source/common/ucnvdisp.cpp',
  43     'intl/icu/source/common/ucnv_ext.cpp',
  44     'intl/icu/source/common/ucnvhz.cpp',
  45     'intl/icu/source/common/ucnvisci.cpp',
  46     'intl/icu/source/common/ucnv_lmb.cpp',
  47     'intl/icu/source/common/ucnvmbcs.cpp',
  48     'intl/icu/source/common/uidna.cpp',
  49     'intl/icu/source/common/unorm.cpp',
  50     'intl/icu/source/common/usc_impl.cpp',
  51     'intl/icu/source/common/ustr_wcs.cpp',
  52     'intl/icu/source/common/util_props.cpp',
  53     'intl/icu/source/i18n/anytrans.cpp',
  54     'intl/icu/source/i18n/brktrans.cpp',
  55     'intl/icu/source/i18n/casetrn.cpp',
  56     'intl/icu/source/i18n/cpdtrans.cpp',
  57     'intl/icu/source/i18n/esctrn.cpp',
  58     'intl/icu/source/i18n/fmtable_cnv.cpp',
  59     'intl/icu/source/i18n/funcrepl.cpp',
  60     'intl/icu/source/i18n/gender.cpp',
  61     'intl/icu/source/i18n/name2uni.cpp',
  62     'intl/icu/source/i18n/nortrans.cpp',
  63     'intl/icu/source/i18n/nultrans.cpp',
  64     'intl/icu/source/i18n/quant.cpp',
  65     'intl/icu/source/i18n/rbt.cpp',
  66     'intl/icu/source/i18n/rbt_data.cpp',
  67     'intl/icu/source/i18n/rbt_pars.cpp',
  68     'intl/icu/source/i18n/rbt_rule.cpp',
  69     'intl/icu/source/i18n/rbt_set.cpp',
  70     'intl/icu/source/i18n/regexcmp.cpp',
  71     'intl/icu/source/i18n/regeximp.cpp',
  72     'intl/icu/source/i18n/regexst.cpp',
  73     'intl/icu/source/i18n/regextxt.cpp',
  74     'intl/icu/source/i18n/rematch.cpp',
  75     'intl/icu/source/i18n/remtrans.cpp',
  76     'intl/icu/source/i18n/repattrn.cpp',
  77     'intl/icu/source/i18n/scientificnumberformatter.cpp',
  78     'intl/icu/source/i18n/strmatch.cpp',
  79     'intl/icu/source/i18n/strrepl.cpp',
  80     'intl/icu/source/i18n/titletrn.cpp',
  81     'intl/icu/source/i18n/tolowtrn.cpp',
  82     'intl/icu/source/i18n/toupptrn.cpp',
  83     'intl/icu/source/i18n/translit.cpp',
  84     'intl/icu/source/i18n/transreg.cpp',
  85     'intl/icu/source/i18n/tridpars.cpp',
  86     'intl/icu/source/i18n/udateintervalformat.cpp',
  87     'intl/icu/source/i18n/unesctrn.cpp',
  88     'intl/icu/source/i18n/uni2name.cpp',
  89     'intl/icu/source/i18n/uregexc.cpp',
  90     'intl/icu/source/i18n/uregex.cpp',
  91     'intl/icu/source/i18n/uregion.cpp',
  92     'intl/icu/source/i18n/uspoof_build.cpp',
  93     'intl/icu/source/i18n/uspoof_conf.cpp',
  94     'intl/icu/source/i18n/utrans.cpp',
  95     'intl/icu/source/i18n/vzone.cpp',
  96     'intl/icu/source/i18n/zrule.cpp',
  97     'intl/icu/source/i18n/ztrans.cpp',
  98
  99     # Cluster
 100     'intl/icu/source/common/resbund_cnv.cpp',
 101     'intl/icu/source/common/ures_cnv.cpp',
 102
 103     # Cluster
 104     'intl/icu/source/common/propsvec.cpp',
 105     'intl/icu/source/common/ucnvsel.cpp',
 106     'intl/icu/source/common/ucnv_set.cpp',
 107
 108     # Cluster
 109     'intl/icu/source/common/ubiditransform.cpp',
 110     'intl/icu/source/common/ushape.cpp',
 111
 112     # Cluster
 113     'intl/icu/source/i18n/csdetect.cpp',
 114     'intl/icu/source/i18n/csmatch.cpp',
 115     'intl/icu/source/i18n/csr2022.cpp',
 116     'intl/icu/source/i18n/csrecog.cpp',
 117     'intl/icu/source/i18n/csrmbcs.cpp',
 118     'intl/icu/source/i18n/csrsbcs.cpp',
 119     'intl/icu/source/i18n/csrucode.cpp',
 120     'intl/icu/source/i18n/csrutf8.cpp',
 121     'intl/icu/source/i18n/inputext.cpp',
 122     'intl/icu/source/i18n/ucsdet.cpp',
 123
 124     # Cluster
 125     'intl/icu/source/i18n/alphaindex.cpp',
 126     'intl/icu/source/i18n/ulocdata.cpp',
 127 ])
 128
 129
 130 def find_source_file(dir, filename):
 131     base = os.path.splitext(filename)[0]
 132     for ext in ('.cpp', '.c'):
 133         f = mozpath.join(dir, base + ext)
 134         if os.path.isfile(f):
 135             return f
 136     raise Exception("Couldn't find source file for: %s" % filename)
 137
 138
 139 def get_sources_from_makefile(makefile):
 140     import pymake.parser
 141     from pymake.parserdata import SetVariable
 142     srcdir = os.path.dirname(makefile)
 143     for statement in pymake.parser.parsefile(makefile):
 144         if (isinstance(statement, SetVariable) and
 145                 statement.vnameexp.is_static_string and
 146                 statement.vnameexp.s == 'OBJECTS'):
 147             return sorted((find_source_file(srcdir, s)
 148                            for s in statement.value.split()),
 149                           key=lambda x: x.lower())
 150
 151
 152 def list_headers(path):
 153     result = []
 154     for name in os.listdir(path):
 155         f = mozpath.join(path, name)
 156         if os.path.isfile(f):
 157             result.append(f)
 158     return sorted(result, key=lambda x: x.lower())
 159
 160
 161 def write_sources(mozbuild, sources, headers):
 162     with open(mozbuild, 'wb') as f:
 163         f.write('# THIS FILE IS GENERATED BY /intl/icu_sources_data.py ' +
 164                 'DO NOT EDIT\n' +
 165                 'SOURCES += [\n')
 166         f.write(''.join("   '/%s',\n" % s for s in sources))
 167         f.write(']\n\n')
 168         f.write('EXPORTS.unicode += [\n')
 169         f.write(''.join("   '/%s',\n" % s for s in headers))
 170         f.write(']\n')
 171
 172
 173 def update_sources(topsrcdir):
 174     print('Updating ICU sources lists...')
 175     sys.path.append(mozpath.join(topsrcdir, 'build/pymake'))
 176     for d in ['common', 'i18n']:
 177         base_path = mozpath.join(topsrcdir, 'intl/icu/source/%s' % d)
 178         makefile = mozpath.join(base_path, 'Makefile.in')
 179         mozbuild = mozpath.join(topsrcdir,
 180                                 'config/external/icu/%s/sources.mozbuild' % d)
 181         sources = [mozpath.relpath(s, topsrcdir)
 182                    for s in get_sources_from_makefile(makefile)]
 183         sources = filter(lambda x: x not in UNUSED_SOURCES, sources)
 184         headers = [mozpath.normsep(os.path.relpath(s, topsrcdir))
 185                    for s in list_headers(mozpath.join(base_path, 'unicode'))]
 186         write_sources(mozbuild, sources, headers)
 187
 188
 189 def try_run(name, command, cwd=None, **kwargs):
 190     try:
 191         with tempfile.NamedTemporaryFile(prefix=name, delete=False) as f:
 192             subprocess.check_call(command, cwd=cwd, stdout=f,
 193                                   stderr=subprocess.STDOUT, **kwargs)
 194     except subprocess.CalledProcessError:
 195         print('''Error running "{}" in directory {}
 196     See output in {}'''.format(' '.join(command), cwd, f.name),
 197               file=sys.stderr)
 198         return False
 199     else:
 200         os.unlink(f.name)
 201         return True
 202
 203
 204 def get_data_file(data_dir):
 205     files = glob.glob(mozpath.join(data_dir, 'icudt*.dat'))
 206     return files[0] if files else None
 207
 208
 209 def update_data_file(topsrcdir):
 210     objdir = tempfile.mkdtemp(prefix='icu-obj-')
 211     configure = mozpath.join(topsrcdir, 'intl/icu/source/configure')
 212     env = dict(os.environ)
 213     # bug 1262101 - these should be shared with the moz.build files
 214     env.update({
 215         'CPPFLAGS': ('-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 ' +
 216                      '-DUCONFIG_NO_LEGACY_CONVERSION ' +
 217                      '-DUCONFIG_NO_TRANSLITERATION ' +
 218                      '-DUCONFIG_NO_REGULAR_EXPRESSIONS ' +
 219                      '-DUCONFIG_NO_BREAK_ITERATION ' +
 220                      '-DU_CHARSET_IS_UTF8')
 221     })
 222
 223     # Exclude data that we currently don't need.
 224     #
 225     # The file format for ICU's data build tool is described at
 226     # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
 227     env["ICU_DATA_FILTER_FILE"] = mozpath.join(topsrcdir, 'intl/icu/data_filter.json')
 228
 229     print('Running ICU configure...')
 230     if not try_run(
 231             'icu-configure',
 232             ['sh', configure,
 233              '--with-data-packaging=archive',
 234              '--enable-static',
 235              '--disable-shared',
 236              '--disable-extras',
 237              '--disable-icuio',
 238              '--disable-layout',
 239              '--disable-layoutex',
 240              '--disable-tests',
 241              '--disable-samples',
 242              '--disable-strict'],
 243             cwd=objdir,
 244             env=env):
 245         return False
 246     print('Running ICU make...')
 247     if not try_run(
 248             'icu-make',
 249             ['make',
 250              '--jobs=%d' % multiprocessing.cpu_count(),
 251              '--output-sync'],
 252             cwd=objdir):
 253         return False
 254     print('Copying ICU data file...')
 255     tree_data_path = mozpath.join(topsrcdir,
 256                                   'config/external/icu/data/')
 257     old_data_file = get_data_file(tree_data_path)
 258     if not old_data_file:
 259         print('Error: no ICU data file in %s' % tree_data_path,
 260               file=sys.stderr)
 261         return False
 262     new_data_file = get_data_file(mozpath.join(objdir, 'data/out'))
 263     if not new_data_file:
 264         print('Error: no ICU data in ICU objdir', file=sys.stderr)
 265         return False
 266     if os.path.basename(old_data_file) != os.path.basename(new_data_file):
 267         # Data file name has the major version number embedded.
 268         os.unlink(old_data_file)
 269     shutil.copy(new_data_file, tree_data_path)
 270     try:
 271         shutil.rmtree(objdir)
 272     except Exception:
 273         print('Warning: failed to remove %s' % objdir, file=sys.stderr)
 274     return True
 275
 276
 277 def main():
 278     if len(sys.argv) != 2:
 279         print('Usage: icu_sources_data.py <mozilla topsrcdir>',
 280               file=sys.stderr)
 281         sys.exit(1)
 282
 283     topsrcdir = mozpath.abspath(sys.argv[1])
 284     update_sources(topsrcdir)
 285     if not update_data_file(topsrcdir):
 286         print('Error updating ICU data file', file=sys.stderr)
 287         sys.exit(1)
 288
 289
 290 if __name__ == '__main__':
 291     main()