3 # Any copyright is dedicated to the Public Domain.
4 # http://creativecommons.org/publicdomain/zero/1.0/
6 # Generate SOURCES in sources.mozbuild files from ICU's Makefile.in
7 # files, and also build a standalone copy of ICU using its build
8 # system to generate a new copy of the in-tree ICU data file.
10 # This script expects to be run from `update-icu.sh` after the in-tree
11 # copy of ICU has been updated.
13 from __future__
import absolute_import
14 from __future__
import print_function
17 import multiprocessing
24 from mozpack
import path
as mozpath
26 # The following files have been determined to be dead/unused by a
27 # semi-automated analysis. You can just remove any of the files below
28 # if you need them. However, files marked with a "Cluster" comment
29 # can only be removed together, as they have (directional) dependencies.
30 # If you want to rerun this analysis, contact :decoder.
33 "intl/icu/source/common/bytestrieiterator.cpp",
34 "intl/icu/source/common/cstr.cpp",
35 "intl/icu/source/common/cwchar.cpp",
36 "intl/icu/source/common/icudataver.cpp",
37 "intl/icu/source/common/icuplug.cpp",
38 "intl/icu/source/common/pluralmap.cpp",
39 "intl/icu/source/common/ucat.cpp",
40 "intl/icu/source/common/ucnv2022.cpp",
41 "intl/icu/source/common/ucnv_ct.cpp",
42 "intl/icu/source/common/ucnvdisp.cpp",
43 "intl/icu/source/common/ucnv_ext.cpp",
44 "intl/icu/source/common/ucnvhz.cpp",
45 "intl/icu/source/common/ucnvisci.cpp",
46 "intl/icu/source/common/ucnv_lmb.cpp",
47 "intl/icu/source/common/ucnvmbcs.cpp",
48 "intl/icu/source/common/uidna.cpp",
49 "intl/icu/source/common/unorm.cpp",
50 "intl/icu/source/common/usc_impl.cpp",
51 "intl/icu/source/common/ustr_wcs.cpp",
52 "intl/icu/source/common/util_props.cpp",
53 "intl/icu/source/i18n/anytrans.cpp",
54 "intl/icu/source/i18n/brktrans.cpp",
55 "intl/icu/source/i18n/casetrn.cpp",
56 "intl/icu/source/i18n/cpdtrans.cpp",
57 "intl/icu/source/i18n/esctrn.cpp",
58 "intl/icu/source/i18n/fmtable_cnv.cpp",
59 "intl/icu/source/i18n/funcrepl.cpp",
60 "intl/icu/source/i18n/gender.cpp",
61 "intl/icu/source/i18n/name2uni.cpp",
62 "intl/icu/source/i18n/nortrans.cpp",
63 "intl/icu/source/i18n/nultrans.cpp",
64 "intl/icu/source/i18n/quant.cpp",
65 "intl/icu/source/i18n/rbt.cpp",
66 "intl/icu/source/i18n/rbt_data.cpp",
67 "intl/icu/source/i18n/rbt_pars.cpp",
68 "intl/icu/source/i18n/rbt_rule.cpp",
69 "intl/icu/source/i18n/rbt_set.cpp",
70 "intl/icu/source/i18n/regexcmp.cpp",
71 "intl/icu/source/i18n/regeximp.cpp",
72 "intl/icu/source/i18n/regexst.cpp",
73 "intl/icu/source/i18n/regextxt.cpp",
74 "intl/icu/source/i18n/rematch.cpp",
75 "intl/icu/source/i18n/remtrans.cpp",
76 "intl/icu/source/i18n/repattrn.cpp",
77 "intl/icu/source/i18n/scientificnumberformatter.cpp",
78 "intl/icu/source/i18n/strmatch.cpp",
79 "intl/icu/source/i18n/strrepl.cpp",
80 "intl/icu/source/i18n/titletrn.cpp",
81 "intl/icu/source/i18n/tolowtrn.cpp",
82 "intl/icu/source/i18n/toupptrn.cpp",
83 "intl/icu/source/i18n/translit.cpp",
84 "intl/icu/source/i18n/transreg.cpp",
85 "intl/icu/source/i18n/tridpars.cpp",
86 "intl/icu/source/i18n/unesctrn.cpp",
87 "intl/icu/source/i18n/uni2name.cpp",
88 "intl/icu/source/i18n/uregexc.cpp",
89 "intl/icu/source/i18n/uregex.cpp",
90 "intl/icu/source/i18n/uregion.cpp",
91 "intl/icu/source/i18n/uspoof_build.cpp",
92 "intl/icu/source/i18n/uspoof_conf.cpp",
93 "intl/icu/source/i18n/utrans.cpp",
94 "intl/icu/source/i18n/vzone.cpp",
95 "intl/icu/source/i18n/zrule.cpp",
96 "intl/icu/source/i18n/ztrans.cpp",
98 "intl/icu/source/common/resbund_cnv.cpp",
99 "intl/icu/source/common/ures_cnv.cpp",
101 "intl/icu/source/common/propsvec.cpp",
102 "intl/icu/source/common/ucnvsel.cpp",
103 "intl/icu/source/common/ucnv_set.cpp",
105 "intl/icu/source/common/ubiditransform.cpp",
106 "intl/icu/source/common/ushape.cpp",
108 "intl/icu/source/i18n/csdetect.cpp",
109 "intl/icu/source/i18n/csmatch.cpp",
110 "intl/icu/source/i18n/csr2022.cpp",
111 "intl/icu/source/i18n/csrecog.cpp",
112 "intl/icu/source/i18n/csrmbcs.cpp",
113 "intl/icu/source/i18n/csrsbcs.cpp",
114 "intl/icu/source/i18n/csrucode.cpp",
115 "intl/icu/source/i18n/csrutf8.cpp",
116 "intl/icu/source/i18n/inputext.cpp",
117 "intl/icu/source/i18n/ucsdet.cpp",
119 "intl/icu/source/i18n/alphaindex.cpp",
120 "intl/icu/source/i18n/ulocdata.cpp",
125 def find_source_file(dir, filename
):
126 base
= os
.path
.splitext(filename
)[0]
127 for ext
in (".cpp", ".c"):
128 f
= mozpath
.join(dir, base
+ ext
)
129 if os
.path
.isfile(f
):
131 raise Exception("Couldn't find source file for: %s" % filename
)
134 def get_sources_from_makefile(makefile
):
135 srcdir
= os
.path
.dirname(makefile
)
136 with
open(makefile
) as f
:
137 contents
= f
.read().replace("\\\n", "").split("\n")
138 for line
in contents
:
139 if line
.startswith("OBJECTS ="):
142 find_source_file(srcdir
, s
)
143 for s
in line
[len("OBJECTS =") :].strip().split()
145 key
=lambda x
: x
.lower(),
147 raise AssertionError("OBJECTS definition not found in file %s" % makefile
)
150 def list_headers(path
):
152 for name
in os
.listdir(path
):
153 f
= mozpath
.join(path
, name
)
154 if os
.path
.isfile(f
):
156 return sorted(result
, key
=lambda x
: x
.lower())
159 def write_sources(mozbuild
, sources
, headers
):
160 with
open(mozbuild
, "w", newline
="\n", encoding
="utf-8") as f
:
162 "# THIS FILE IS GENERATED BY /intl/icu_sources_data.py " + "DO NOT EDIT\n"
165 def write_list(name
, content
):
167 f
.write("%s %s [\n" % (name
, "=" if name
.islower() else "+="))
168 f
.write("".join(" '/%s',\n" % s
for s
in content
))
171 write_list("sources", [s
for s
in sources
if s
not in UNUSED_SOURCES
])
172 write_list("other_sources", [s
for s
in sources
if s
in UNUSED_SOURCES
])
173 write_list("EXPORTS.unicode", headers
)
176 def update_sources(topsrcdir
):
177 print("Updating ICU sources lists...")
178 for d
in ["common", "i18n", "tools/toolutil", "tools/icupkg"]:
179 base_path
= mozpath
.join(topsrcdir
, "intl/icu/source/%s" % d
)
180 makefile
= mozpath
.join(base_path
, "Makefile.in")
181 mozbuild
= mozpath
.join(
182 topsrcdir
, "config/external/icu/%s/sources.mozbuild" % mozpath
.basename(d
)
185 mozpath
.relpath(s
, topsrcdir
) for s
in get_sources_from_makefile(makefile
)
187 unicode_dir
= mozpath
.join(base_path
, "unicode")
188 if os
.path
.exists(unicode_dir
):
190 mozpath
.normsep(os
.path
.relpath(s
, topsrcdir
))
191 for s
in list_headers(unicode_dir
)
195 write_sources(mozbuild
, sources
, headers
)
198 def try_run(name
, command
, cwd
=None, **kwargs
):
200 with tempfile
.NamedTemporaryFile(prefix
=name
, delete
=False) as f
:
201 subprocess
.check_call(
202 command
, cwd
=cwd
, stdout
=f
, stderr
=subprocess
.STDOUT
, **kwargs
204 except subprocess
.CalledProcessError
:
206 """Error running "{}" in directory {}
207 See output in {}""".format(
208 " ".join(command
), cwd
, f
.name
218 def get_data_file(data_dir
):
219 files
= glob
.glob(mozpath
.join(data_dir
, "icudt*.dat"))
220 return files
[0] if files
else None
223 def update_data_file(topsrcdir
):
224 objdir
= tempfile
.mkdtemp(prefix
="icu-obj-")
225 configure
= mozpath
.join(topsrcdir
, "intl/icu/source/configure")
226 env
= dict(os
.environ
)
227 # bug 1262101 - these should be shared with the moz.build files
231 "-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 "
232 + "-DU_HIDE_OBSOLETE_UTF_OLD_H=1"
233 + "-DUCONFIG_NO_LEGACY_CONVERSION "
234 + "-DUCONFIG_NO_TRANSLITERATION "
235 + "-DUCONFIG_NO_REGULAR_EXPRESSIONS "
236 + "-DUCONFIG_NO_BREAK_ITERATION "
237 + "-DU_CHARSET_IS_UTF8"
242 # Exclude data that we currently don't need.
244 # The file format for ICU's data build tool is described at
245 # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
246 env
["ICU_DATA_FILTER_FILE"] = mozpath
.join(topsrcdir
, "intl/icu/data_filter.json")
248 print("Running ICU configure...")
254 "--with-data-packaging=archive",
260 "--disable-layoutex",
269 print("Running ICU make...")
272 ["make", "--jobs=%d" % multiprocessing
.cpu_count(), "--output-sync"],
276 print("Copying ICU data file...")
277 tree_data_path
= mozpath
.join(topsrcdir
, "config/external/icu/data/")
278 old_data_file
= get_data_file(tree_data_path
)
279 if not old_data_file
:
280 print("Error: no ICU data file in %s" % tree_data_path
, file=sys
.stderr
)
282 new_data_file
= get_data_file(mozpath
.join(objdir
, "data/out"))
283 if not new_data_file
:
284 print("Error: no ICU data in ICU objdir", file=sys
.stderr
)
286 if os
.path
.basename(old_data_file
) != os
.path
.basename(new_data_file
):
287 # Data file name has the major version number embedded.
288 os
.unlink(old_data_file
)
289 shutil
.copy(new_data_file
, tree_data_path
)
291 shutil
.rmtree(objdir
)
293 print("Warning: failed to remove %s" % objdir
, file=sys
.stderr
)
298 if len(sys
.argv
) != 2:
299 print("Usage: icu_sources_data.py <mozilla topsrcdir>", file=sys
.stderr
)
302 topsrcdir
= mozpath
.abspath(sys
.argv
[1])
303 update_sources(topsrcdir
)
304 if not update_data_file(topsrcdir
):
305 print("Error updating ICU data file", file=sys
.stderr
)
309 if __name__
== "__main__":