oleaut32: Also register typelibs in the opposite registry mode.
[wine.git] / tools / make_unicode
blob07cd5baf952250db0ed9ef93bf82658bf66659f9
1 #!/usr/bin/perl -w
3 # Generate code page .c files from ftp.unicode.org descriptions
5 # Copyright 2000 Alexandre Julliard
7 # This library is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU Lesser General Public
9 # License as published by the Free Software Foundation; either
10 # version 2.1 of the License, or (at your option) any later version.
12 # This library is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 # Lesser General Public License for more details.
17 # You should have received a copy of the GNU Lesser General Public
18 # License along with this library; if not, write to the Free Software
19 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
22 use strict;
24 # base URLs for www.unicode.org files
25 my $MAPPINGS = "http://www.unicode.org/Public/MAPPINGS";
26 my $UNIDATA = "http://www.unicode.org/Public/6.3.0/ucd";
27 my $REPORTS = "http://www.unicode.org/reports";
28 my $RFCS = "http://www.rfc-editor.org/rfc";
30 # Sort keys file
31 my $SORTKEYS = "tr10/allkeys.txt";
33 # RFC3454 (stringprep data)
34 my $STRINGPREP = "rfc3454.txt";
36 # Defaults mapping
37 my $DEFAULTS = "tools/unicode-defaults";
39 # Default char for undefined mappings
40 my $DEF_CHAR = ord '?';
42 my @allfiles =
44 [ 37, "VENDORS/MICSFT/EBCDIC/CP037.TXT", 0, "IBM EBCDIC US Canada", 0, 0x6f ],
45 [ 424, "VENDORS/MISC/CP424.TXT", 0, "IBM EBCDIC Hebrew" ],
46 [ 437, "VENDORS/MICSFT/PC/CP437.TXT", 1, "OEM United States" ],
47 [ 500, "VENDORS/MICSFT/EBCDIC/CP500.TXT", 0, "IBM EBCDIC International", 0, 0x6f ],
48 [ 737, "VENDORS/MICSFT/PC/CP737.TXT", 1, "OEM Greek 437G" ],
49 [ 775, "VENDORS/MICSFT/PC/CP775.TXT", 1, "OEM Baltic" ],
50 [ 850, "VENDORS/MICSFT/PC/CP850.TXT", 1, "OEM Multilingual Latin 1" ],
51 [ 852, "VENDORS/MICSFT/PC/CP852.TXT", 1, "OEM Slovak Latin 2" ],
52 [ 855, "VENDORS/MICSFT/PC/CP855.TXT", 1, "OEM Cyrillic" ],
53 [ 856, "VENDORS/MISC/CP856.TXT", 0, "Hebrew PC" ],
54 [ 857, "VENDORS/MICSFT/PC/CP857.TXT", 1, "OEM Turkish", 0xf8bb ],
55 [ 860, "VENDORS/MICSFT/PC/CP860.TXT", 1, "OEM Portuguese" ],
56 [ 861, "VENDORS/MICSFT/PC/CP861.TXT", 1, "OEM Icelandic" ],
57 [ 862, "VENDORS/MICSFT/PC/CP862.TXT", 1, "OEM Hebrew" ],
58 [ 863, "VENDORS/MICSFT/PC/CP863.TXT", 1, "OEM Canadian French" ],
59 [ 864, "VENDORS/MICSFT/PC/CP864.TXT", 0, "OEM Arabic", 0xf8bb ],
60 [ 865, "VENDORS/MICSFT/PC/CP865.TXT", 1, "OEM Nordic" ],
61 [ 866, "VENDORS/MICSFT/PC/CP866.TXT", 1, "OEM Russian" ],
62 [ 869, "VENDORS/MICSFT/PC/CP869.TXT", 1, "OEM Greek" ],
63 [ 874, "VENDORS/MICSFT/WindowsBestFit/bestfit874.txt", 1, "ANSI/OEM Thai" ],
64 [ 875, "VENDORS/MICSFT/EBCDIC/CP875.TXT", 0, "IBM EBCDIC Greek", 0, 0x6f ],
65 [ 878, "VENDORS/MISC/KOI8-R.TXT", 0, "Russian KOI8" ],
66 [ 932, "VENDORS/MICSFT/WindowsBestFit/bestfit932.txt", 0, "ANSI/OEM Japanese Shift-JIS" ],
67 [ 936, "VENDORS/MICSFT/WindowsBestFit/bestfit936.txt", 0, "ANSI/OEM Simplified Chinese GBK" ],
68 [ 949, "VENDORS/MICSFT/WindowsBestFit/bestfit949.txt", 0, "ANSI/OEM Korean Unified Hangul" ],
69 [ 950, "VENDORS/MICSFT/WindowsBestFit/bestfit950.txt", 0, "ANSI/OEM Traditional Chinese Big5" ],
70 [ 1006, "VENDORS/MISC/CP1006.TXT", 0, "IBM Arabic" ],
71 [ 1026, "VENDORS/MICSFT/EBCDIC/CP1026.TXT", 0, "IBM EBCDIC Latin 5 Turkish", 0, 0x6f ],
72 [ 1250, "VENDORS/MICSFT/WindowsBestFit/bestfit1250.txt", 0, "ANSI Eastern Europe" ],
73 [ 1251, "VENDORS/MICSFT/WindowsBestFit/bestfit1251.txt", 0, "ANSI Cyrillic" ],
74 [ 1252, "VENDORS/MICSFT/WindowsBestFit/bestfit1252.txt", 0, "ANSI Latin 1" ],
75 [ 1253, "VENDORS/MICSFT/WindowsBestFit/bestfit1253.txt", 0, "ANSI Greek" ],
76 [ 1254, "VENDORS/MICSFT/WindowsBestFit/bestfit1254.txt", 0, "ANSI Turkish" ],
77 [ 1255, "VENDORS/MICSFT/WindowsBestFit/bestfit1255.txt", 0, "ANSI Hebrew" ],
78 [ 1256, "VENDORS/MICSFT/WindowsBestFit/bestfit1256.txt", 0, "ANSI Arabic" ],
79 [ 1257, "VENDORS/MICSFT/WindowsBestFit/bestfit1257.txt", 0, "ANSI Baltic" ],
80 [ 1258, "VENDORS/MICSFT/WindowsBestFit/bestfit1258.txt", 0, "ANSI/OEM Viet Nam" ],
81 [ 1361, "VENDORS/MICSFT/WindowsBestFit/bestfit1361.txt", 0, "Korean Johab" ],
82 [ 10000, "VENDORS/MICSFT/MAC/ROMAN.TXT", 0, "Mac Roman" ],
83 [ 10001, "VENDORS/APPLE/JAPANESE.TXT", 0, "Mac Japanese" ],
84 [ 10002, "VENDORS/APPLE/CHINTRAD.TXT", 0, "Mac Traditional Chinese" ],
85 [ 10003, "VENDORS/APPLE/KOREAN.TXT", 0, "Mac Korean", 0xf8e5 ],
86 [ 10004, "VENDORS/APPLE/ARABIC.TXT", 0, "Mac Arabic", 0xf826 ],
87 [ 10005, "VENDORS/APPLE/HEBREW.TXT", 0, "Mac Hebrew", 0xf7fa ],
88 [ 10006, "VENDORS/MICSFT/MAC/GREEK.TXT", 0, "Mac Greek" ],
89 [ 10007, "VENDORS/MICSFT/MAC/CYRILLIC.TXT", 0, "Mac Cyrillic" ],
90 [ 10008, "VENDORS/APPLE/CHINSIMP.TXT", 0, "Mac Simplified Chinese" ],
91 [ 10010, "VENDORS/APPLE/ROMANIAN.TXT", 0, "Mac Romanian" ],
92 [ 10017, "VENDORS/APPLE/CYRILLIC.TXT", 0, "Mac Ukrainian" ],
93 [ 10021, "VENDORS/APPLE/THAI.TXT", 0, "Mac Thai" ],
94 [ 10029, "VENDORS/MICSFT/MAC/LATIN2.TXT", 0, "Mac Latin 2" ],
95 [ 10079, "VENDORS/MICSFT/MAC/ICELAND.TXT", 0, "Mac Icelandic" ],
96 [ 10081, "VENDORS/MICSFT/MAC/TURKISH.TXT", 0, "Mac Turkish", 0xf89f ],
97 [ 10082, "VENDORS/APPLE/CROATIAN.TXT", 0, "Mac Croatian" ],
98 [ 20127, undef, 0, "US-ASCII (7bit)" ],
99 [ 20866, "VENDORS/MISC/KOI8-R.TXT", 0, "Russian KOI8" ],
100 [ 20932, "OBSOLETE/EASTASIA/JIS/JIS0208.TXT", 0, "EUC-JP", 0, 0x3f, 0x30fb ],
101 [ 21866, "VENDORS/MISC/KOI8-U.TXT", 0, "Ukrainian KOI8" ],
102 [ 28591, "ISO8859/8859-1.TXT", 0, "ISO 8859-1 Latin 1" ],
103 [ 28592, "ISO8859/8859-2.TXT", 0, "ISO 8859-2 Latin 2 (East European)" ],
104 [ 28593, "ISO8859/8859-3.TXT", 0, "ISO 8859-3 Latin 3 (South European)", 0xf7f5 ],
105 [ 28594, "ISO8859/8859-4.TXT", 0, "ISO 8859-4 Latin 4 (Baltic old)" ],
106 [ 28595, "ISO8859/8859-5.TXT", 0, "ISO 8859-5 Cyrillic" ],
107 [ 28596, "ISO8859/8859-6.TXT", 0, "ISO 8859-6 Arabic", 0xf7c8 ],
108 [ 28597, "ISO8859/8859-7.TXT", 0, "ISO 8859-7 Greek", 0xf7c5 ],
109 [ 28598, "ISO8859/8859-8.TXT", 0, "ISO 8859-8 Hebrew", 0xf79c ],
110 [ 28599, "ISO8859/8859-9.TXT", 0, "ISO 8859-9 Latin 5 (Turkish)" ],
111 [ 28600, "ISO8859/8859-10.TXT", 0, "ISO 8859-10 Latin 6 (Nordic)" ],
112 [ 28603, "ISO8859/8859-13.TXT", 0, "ISO 8859-13 Latin 7 (Baltic)" ],
113 [ 28604, "ISO8859/8859-14.TXT", 0, "ISO 8859-14 Latin 8 (Celtic)" ],
114 [ 28605, "ISO8859/8859-15.TXT", 0, "ISO 8859-15 Latin 9 (Euro)" ],
115 [ 28606, "ISO8859/8859-16.TXT", 0, "ISO 8859-16 Latin 10 (Balkan)" ]
119 my %ctype =
121 "upper" => 0x0001,
122 "lower" => 0x0002,
123 "digit" => 0x0004,
124 "space" => 0x0008,
125 "punct" => 0x0010,
126 "cntrl" => 0x0020,
127 "blank" => 0x0040,
128 "xdigit" => 0x0080,
129 "alpha" => 0x0100,
130 "defin" => 0x0200
133 my %indic_types =
135 "Other" => 0x0000,
136 "Bindu" => 0x0001,
137 "Visarga" => 0x0002,
138 "Avagraha" => 0x0003,
139 "Nukta" => 0x0004,
140 "Virama" => 0x0005,
141 "Vowel_Independent" => 0x0006,
142 "Vowel_Dependent" => 0x0007,
143 "Vowel" => 0x0008,
144 "Consonant_Placeholder" => 0x0009,
145 "Consonant" => 0x000a,
146 "Consonant_Dead" => 0x000b,
147 "Consonant_Repha" => 0x000c,
148 "Consonant_Subjoined" => 0x000d,
149 "Consonant_Medial" => 0x000e,
150 "Consonant_Final" => 0x000f,
151 "Consonant_Head_Letter" => 0x0010,
152 "Modifying_Letter" => 0x0011,
153 "Tone_Letter" => 0x0012,
154 "Tone_Mark" => 0x0013,
155 "Register_Shifter" => 0x0014
158 my %matra_types =
160 "Right" => 0x01,
161 "Left" => 0x02,
162 "Visual_Order_Left" => 0x03,
163 "Left_And_Right" => 0x04,
164 "Top" => 0x05,
165 "Bottom" => 0x06,
166 "Top_And_Bottom" => 0x07,
167 "Top_And_Right" => 0x08,
168 "Top_And_Left" => 0x09,
169 "Top_And_Left_And_Right" => 0x0a,
170 "Bottom_And_Right" => 0x0b,
171 "Top_And_Bottom_And_Right" => 0x0c,
172 "Overstruck" => 0x0d,
173 "Invisible" => 0x0e
176 my %nameprep_flags =
178 "unassigned" => 0x01,
179 "prohibited" => 0x02,
180 "bidi_ral" => 0x04,
181 "bidi_l" => 0x08
184 my %break_types =
186 "BK" => 0x0001,
187 "CR" => 0x0002,
188 "LF" => 0x0003,
189 "CM" => 0x0004,
190 "SG" => 0x0005,
191 "GL" => 0x0006,
192 "CB" => 0x0007,
193 "SP" => 0x0008,
194 "ZW" => 0x0009,
195 "NL" => 0x000a,
196 "WJ" => 0x000b,
197 "JL" => 0x000c,
198 "JV" => 0x000d,
199 "JT" => 0x000e,
200 "H2" => 0x000f,
201 "H3" => 0x0010,
202 "XX" => 0x0011,
203 "OP" => 0x0012,
204 "CL" => 0x0013,
205 "CP" => 0x0014,
206 "QU" => 0x0015,
207 "NS" => 0x0016,
208 "EX" => 0x0017,
209 "SY" => 0x0018,
210 "IS" => 0x0019,
211 "PR" => 0x001a,
212 "PO" => 0x001b,
213 "NU" => 0x001c,
214 "AL" => 0x001d,
215 "ID" => 0x001e,
216 "IN" => 0x001f,
217 "HY" => 0x0020,
218 "BB" => 0x0021,
219 "BA" => 0x0022,
220 "SA" => 0x0023,
221 "AI" => 0x0024,
222 "B2" => 0x0025,
223 "HL" => 0x0026,
224 "CJ" => 0x0027,
225 "RI" => 0x0028,
228 my %categories =
230 "Lu" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"upper"}, # Letter, Uppercase
231 "Ll" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"lower"}, # Letter, Lowercase
232 "Lt" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"upper"}|$ctype{"lower"}, # Letter, Titlecase
233 "Mn" => $ctype{"defin"}, # Mark, Non-Spacing
234 "Mc" => $ctype{"defin"}, # Mark, Spacing Combining
235 "Me" => $ctype{"defin"}, # Mark, Enclosing
236 "Nd" => $ctype{"defin"}|$ctype{"digit"}, # Number, Decimal Digit
237 "Nl" => $ctype{"defin"}|$ctype{"alpha"}, # Number, Letter
238 "No" => $ctype{"defin"}, # Number, Other
239 "Zs" => $ctype{"defin"}|$ctype{"space"}, # Separator, Space
240 "Zl" => $ctype{"defin"}|$ctype{"space"}, # Separator, Line
241 "Zp" => $ctype{"defin"}|$ctype{"space"}, # Separator, Paragraph
242 "Cc" => $ctype{"defin"}|$ctype{"cntrl"}, # Other, Control
243 "Cf" => $ctype{"defin"}|$ctype{"cntrl"}, # Other, Format
244 "Cs" => $ctype{"defin"}, # Other, Surrogate
245 "Co" => $ctype{"defin"}, # Other, Private Use
246 "Cn" => $ctype{"defin"}, # Other, Not Assigned
247 "Lm" => $ctype{"defin"}|$ctype{"alpha"}, # Letter, Modifier
248 "Lo" => $ctype{"defin"}|$ctype{"alpha"}, # Letter, Other
249 "Pc" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Connector
250 "Pd" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Dash
251 "Ps" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Open
252 "Pe" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Close
253 "Pi" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Initial quote
254 "Pf" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Final quote
255 "Po" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Other
256 "Sm" => $ctype{"defin"}, # Symbol, Math
257 "Sc" => $ctype{"defin"}, # Symbol, Currency
258 "Sk" => $ctype{"defin"}, # Symbol, Modifier
259 "So" => $ctype{"defin"} # Symbol, Other
262 # a few characters need additional categories that cannot be determined automatically
263 my %special_categories =
265 "xdigit" => [ ord('0')..ord('9'),ord('A')..ord('F'),ord('a')..ord('f'),
266 0xff10..0xff19, 0xff21..0xff26, 0xff41..0xff46 ],
267 "space" => [ 0x09..0x0d, 0x85 ],
268 "blank" => [ 0x09, 0x20, 0xa0, 0x3000, 0xfeff ],
269 "cntrl" => [ 0x070f, 0x200c, 0x200d,
270 0x200e, 0x200f, 0x202a, 0x202b, 0x202c, 0x202d, 0x202e,
271 0x206a, 0x206b, 0x206c, 0x206d, 0x206e, 0x206f, 0xfeff,
272 0xfff9, 0xfffa, 0xfffb ],
273 "punct" => [ 0x24, 0x2b, 0x3c..0x3e, 0x5e, 0x60, 0x7c, 0x7e, 0xa2..0xbe,
274 0xd7, 0xf7 ],
275 "digit" => [ 0xb2, 0xb3, 0xb9 ],
276 "lower" => [ 0x2071, 0x207f ]
279 my %directions =
281 "L" => 1, # Left-to-Right
282 "R" => 2, # Right-to-Left
283 "AL" => 12, # Right-to-Left Arabic
284 "EN" => 3, # European Number
285 "ES" => 4, # European Number Separator
286 "ET" => 5, # European Number Terminator
287 "AN" => 6, # Arabic Number
288 "CS" => 7, # Common Number Separator
289 "NSM" => 13, # Non-Spacing Mark
290 "BN" => 14, # Boundary Neutral
291 "B" => 8, # Paragraph Separator
292 "S" => 9, # Segment Separator
293 "WS" => 10, # Whitespace
294 "ON" => 11, # Other Neutrals
295 "LRE" => 15, # Left-to-Right Embedding
296 "LRO" => 15, # Left-to-Right Override
297 "RLE" => 15, # Right-to-Left Embedding
298 "RLO" => 15, # Right-to-Left Override
299 "PDF" => 15, # Pop Directional Format
300 "LRI" => 15, # Left-to-Right Isolate
301 "RLI" => 15, # Right-to-Left Isolate
302 "FSI" => 15, # First Strong Isolate
303 "PDI" => 15 # Pop Directional Isolate
306 my %joining_types =
308 "U" => 0, # Non_Joining
309 "T" => 1, # Transparent
310 "R" => 2, # Right_Joining
311 "L" => 3, # Left_Joining
312 "D" => 4, # Dual_Joining
313 "C" => 5, # Join_Causing
316 my @cp2uni = ();
317 my @lead_bytes = ();
318 my @uni2cp = ();
319 my @unicode_defaults = ();
320 my @unicode_aliases = ();
321 my @tolower_table = ();
322 my @toupper_table = ();
323 my @digitmap_table = ();
324 my @compatmap_table = ();
325 my @category_table = (0) x 65536;
326 my @joining_table = (0) x 65536;
327 my @direction_table = ();
328 my @decomp_table = ();
329 my @compose_table = ();
330 my $default_char;
331 my $default_wchar;
333 my %joining_forms =
335 "isolated" => [],
336 "final" => [],
337 "initial" => [],
338 "medial" => []
341 ################################################################
342 # fetch a unicode.org file and open it
343 sub open_data_file($$)
345 my ($base, $name) = @_;
346 (my $dir = "data/$name") =~ s/\/[^\/]+$//;
347 local *FILE;
348 unless (-f "data/$name")
350 print "Fetching $base/$name...\n";
351 system "mkdir", "-p", $dir;
352 !system "wget", "-q", "-O", "data/$name", "$base/$name" or die "cannot fetch $base/$name";
354 open FILE, "<data/$name" or die "cannot open data/$name";
355 return *FILE;
358 ################################################################
359 # read in the defaults file
360 sub READ_DEFAULTS($)
362 my $filename = shift;
363 my $start;
365 # first setup a few default mappings
367 open DEFAULTS, "$filename" or die "Cannot open $filename";
368 print "Loading $filename\n";
369 while (<DEFAULTS>)
371 next if /^\#/; # skip comments
372 next if /^$/; # skip empty lines
373 if (/^(([0-9a-fA-F]+)(,[0-9a-fA-F]+)*)\s+([0-9a-fA-F]+|'.'|none)\s+(\#.*)?/)
375 my @src = map hex, split /,/,$1;
376 my $dst = $4;
377 my $comment = $5;
378 if ($#src > 0) { push @unicode_aliases, \@src; }
379 next if ($dst eq "none");
380 $dst = ($dst =~ /\'.\'/) ? ord substr($dst,1,1) : hex $dst;
381 foreach my $src (@src)
383 die "Duplicate value" if defined($unicode_defaults[$src]);
384 $unicode_defaults[$src] = $dst;
386 next;
388 die "Unrecognized line $_\n";
390 close DEFAULTS;
392 # now build mappings from the decomposition field of the Unicode database
394 my $UNICODE_DATA = open_data_file( $UNIDATA, "UnicodeData.txt" );
395 while (<$UNICODE_DATA>)
397 # Decode the fields ...
398 my ($code, $name, $cat, $comb, $bidi,
399 $decomp, $dec, $dig, $num, $mirror,
400 $oldname, $comment, $upper, $lower, $title) = split /;/;
401 my $dst;
402 my $src = hex $code;
404 die "unknown category $cat" unless defined $categories{$cat};
405 die "unknown directionality $bidi" unless defined $directions{$bidi};
407 $category_table[$src] = $categories{$cat};
408 $direction_table[$src] = $directions{$bidi};
409 $joining_table[$src] = $joining_types{"T"} if $cat eq "Mn" || $cat eq "Me" || $cat eq "Cf";
411 if ($lower ne "")
413 $tolower_table[$src] = hex $lower;
415 if ($upper ne "")
417 $toupper_table[$src] = hex $upper;
419 if ($dec ne "")
421 $category_table[$src] |= $ctype{"digit"};
423 if ($dig ne "")
425 $digitmap_table[$src] = ord $dig;
428 # copy the category and direction for everything between First/Last pairs
429 if ($name =~ /, First>/) { $start = $src; }
430 if ($name =~ /, Last>/)
432 while ($start < $src)
434 $category_table[$start] = $category_table[$src];
435 $direction_table[$start] = $direction_table[$src];
436 $start++;
440 next if $decomp eq ""; # no decomposition, skip it
442 if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
444 # decomposition of the form "<foo> 1234" -> use char if type is known
445 if (($src >= 0xf900 && $src < 0xfb00) || ($src >= 0xfe30 && $src < 0xfffd))
447 # Single char decomposition in the compatibility range
448 $compatmap_table[$src] = hex $2;
450 if ($1 eq "isolated" || $1 eq "final" || $1 eq "initial" || $1 eq "medial")
452 ${joining_forms{$1}}[hex $2] = $src;
453 next;
455 next unless ($1 eq "font" ||
456 $1 eq "noBreak" ||
457 $1 eq "circle" ||
458 $1 eq "super" ||
459 $1 eq "sub" ||
460 $1 eq "wide" ||
461 $1 eq "narrow" ||
462 $1 eq "compat" ||
463 $1 eq "small");
464 $dst = hex $2;
466 elsif ($decomp =~ /^<compat>\s+0020\s+([0-9a-fA-F]+)/)
468 # decomposition "<compat> 0020 1234" -> combining accent
469 $dst = hex $1;
471 elsif ($decomp =~ /^([0-9a-fA-F]+)/)
473 # decomposition contains only char values without prefix -> use first char
474 $dst = hex $1;
475 $category_table[$src] |= $category_table[$dst] if defined $category_table[$dst];
476 # store decomposition if it contains two chars
477 if ($decomp =~ /^([0-9a-fA-F]+)\s+([0-9a-fA-F]+)$/)
479 $decomp_table[$src] = [ hex $1, hex $2 ];
480 push @compose_table, [ hex $1, hex $2, $src ];
482 elsif ($decomp =~ /^(<[a-z]+>\s)*([0-9a-fA-F]+)$/ &&
483 (($src >= 0xf900 && $src < 0xfb00) || ($src >= 0xfe30 && $src < 0xfffd)))
485 # Single char decomposition in the compatibility range
486 $compatmap_table[$src] = hex $2;
489 else
491 next;
494 next if defined($unicode_defaults[$src]); # may have been set in the defaults file
496 # check for loops
497 for (my $i = $dst; ; $i = $unicode_defaults[$i])
499 die sprintf("loop detected for %04x -> %04x",$src,$dst) if $i == $src;
500 last unless defined($unicode_defaults[$i]);
502 $unicode_defaults[$src] = $dst;
504 close $UNICODE_DATA;
506 # patch the category of some special characters
508 foreach my $cat (keys %special_categories)
510 my $flag = $ctype{$cat};
511 foreach my $i (@{$special_categories{$cat}}) { $category_table[$i] |= $flag; }
516 ################################################################
517 # define a new lead byte
518 sub add_lead_byte($)
520 my $ch = shift;
521 push @lead_bytes, $ch;
522 $cp2uni[$ch] = 0;
525 ################################################################
526 # define a new char mapping
527 sub add_mapping($$)
529 my ($cp, $uni) = @_;
530 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
531 $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
532 if ($cp > 0xff && !defined($cp2uni[$cp >> 8]))
534 add_lead_byte( $cp >> 8 );
538 ################################################################
539 # parse a standard codepage file
540 sub read_codepage_file($)
542 my $name = shift;
543 my $INPUT = open_data_file( $MAPPINGS, $name );
545 while (<$INPUT>)
547 next if /^\#/; # skip comments
548 next if /^$/; # skip empty lines
549 next if /\x1a/; # skip ^Z
550 next if (/^0x([0-9a-fA-F]+)\s+\#UNDEFINED/); # undefined char
552 if (/^0x([0-9a-fA-F]+)\s+\#DBCS LEAD BYTE/)
554 add_lead_byte( hex $1 );
555 next;
557 # 0x12 0x3456
558 if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
560 add_mapping( hex $1, hex $2 );
561 next;
563 # 0x12 0x3456+0x7890
564 if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\+.*\s+(\#.*)?/)
566 add_mapping( hex $1, hex $2 );
567 next;
569 # 0x12 <LR>+0x3456
570 if (/^0x([0-9a-fA-F]+)\s+<LR>\+0x([0-9a-fA-F]+)\s+(\#.*)?/)
572 add_mapping( hex $1, hex $2 );
573 next;
575 # 0x12 <RL>+0x3456
576 if (/^0x([0-9a-fA-F]+)\s+<RL>\+0x([0-9a-fA-F]+)\s+(\#.*)?/)
578 my $uni = hex $2;
579 add_mapping( hex $1, $uni ) if $uni >= 0x80;
580 next;
582 die "$name: Unrecognized line $_\n";
584 close $INPUT;
588 ################################################################
589 # fill input data for the 20127 (us-ascii) codepage
590 sub fill_20127_codepage()
592 for (my $i = 0; $i < 128; $i++) { $cp2uni[$i] = $uni2cp[$i] = $i; }
593 for (my $i = 128; $i < 256; $i++) { $cp2uni[$i] = $i & 0x7f; }
596 ################################################################
597 # get a mapping including glyph chars for MB_USEGLYPHCHARS
599 sub get_glyphs_mapping(@)
601 $_[0x01] = 0x263a; # (WHITE SMILING FACE)
602 $_[0x02] = 0x263b; # (BLACK SMILING FACE)
603 $_[0x03] = 0x2665; # (BLACK HEART SUIT)
604 $_[0x04] = 0x2666; # (BLACK DIAMOND SUIT)
605 $_[0x05] = 0x2663; # (BLACK CLUB SUIT)
606 $_[0x06] = 0x2660; # (BLACK SPADE SUIT)
607 $_[0x07] = 0x2022; # (BULLET)
608 $_[0x08] = 0x25d8; # (INVERSE BULLET)
609 $_[0x09] = 0x25cb; # (WHITE CIRCLE)
610 $_[0x0a] = 0x25d9; # (INVERSE WHITE CIRCLE)
611 $_[0x0b] = 0x2642; # (MALE SIGN)
612 $_[0x0c] = 0x2640; # (FEMALE SIGN)
613 $_[0x0d] = 0x266a; # (EIGHTH NOTE)
614 $_[0x0e] = 0x266b; # (BEAMED EIGHTH NOTES)
615 $_[0x0f] = 0x263c; # (WHITE SUN WITH RAYS)
616 $_[0x10] = 0x25ba; # (BLACK RIGHT-POINTING POINTER)
617 $_[0x11] = 0x25c4; # (BLACK LEFT-POINTING POINTER)
618 $_[0x12] = 0x2195; # (UP DOWN ARROW)
619 $_[0x13] = 0x203c; # (DOUBLE EXCLAMATION MARK)
620 $_[0x14] = 0x00b6; # (PILCROW SIGN)
621 $_[0x15] = 0x00a7; # (SECTION SIGN)
622 $_[0x16] = 0x25ac; # (BLACK RECTANGLE)
623 $_[0x17] = 0x21a8; # (UP DOWN ARROW WITH BASE)
624 $_[0x18] = 0x2191; # (UPWARDS ARROW)
625 $_[0x19] = 0x2193; # (DOWNWARDS ARROW)
626 $_[0x1a] = 0x2192; # (RIGHTWARDS ARROW)
627 $_[0x1b] = 0x2190; # (LEFTWARDS ARROW)
628 $_[0x1c] = 0x221f; # (RIGHT ANGLE)
629 $_[0x1d] = 0x2194; # (LEFT RIGHT ARROW)
630 $_[0x1e] = 0x25b2; # (BLACK UP-POINTING TRIANGLE)
631 $_[0x1f] = 0x25bc; # (BLACK DOWN-POINTING TRIANGLE)
632 $_[0x7f] = 0x2302; # (HOUSE)
633 return @_;
636 ################################################################
637 # build EUC-JP table from the JIS 0208 file
638 # FIXME: for proper EUC-JP we should probably read JIS 0212 too
639 # but this would require 3-byte DBCS characters
640 sub READ_JIS0208_FILE($)
642 my $name = shift;
644 # ASCII chars
645 for (my $i = 0x00; $i <= 0x7f; $i++)
647 $cp2uni[$i] = $i;
648 $uni2cp[$i] = $i;
651 # JIS X 0201 right plane
652 for (my $i = 0xa1; $i <= 0xdf; $i++)
654 $cp2uni[0x8e00 + $i] = 0xfec0 + $i;
655 $uni2cp[0xfec0 + $i] = 0x8e00 + $i;
658 # lead bytes
659 foreach my $i (0x8e, 0x8f, 0xa1 .. 0xfe)
661 push @lead_bytes,$i;
662 $cp2uni[$i] = 0;
665 # undefined chars
666 foreach my $i (0x80 .. 0x8d, 0x90 .. 0xa0, 0xff)
668 $cp2uni[$i] = $DEF_CHAR;
671 # Shift-JIS compatibility
672 $uni2cp[0x00a5] = 0x5c;
673 $uni2cp[0x203e] = 0x7e;
675 # Fix backslash conversion
676 $cp2uni[0xa1c0] = 0xff3c;
677 $uni2cp[0xff3c] = 0xa1c0;
679 my $INPUT = open_data_file( $MAPPINGS, $name );
680 while (<$INPUT>)
682 next if /^\#/; # skip comments
683 next if /^$/; # skip empty lines
684 next if /\x1a/; # skip ^Z
685 if (/^0x[0-9a-fA-F]+\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
687 my $cp = 0x8080 + hex $1;
688 my $uni = hex $2;
689 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
690 $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
691 next;
693 die "$name: Unrecognized line $_\n";
695 close $INPUT;
699 ################################################################
700 # build the sort keys table
701 sub READ_SORTKEYS_FILE()
703 my @sortkeys = ();
704 for (my $i = 0; $i < 65536; $i++) { $sortkeys[$i] = [ -1, 0, 0, 0, 0 ] };
706 my $INPUT = open_data_file( $REPORTS, $SORTKEYS );
707 while (<$INPUT>)
709 next if /^\#/; # skip comments
710 next if /^$/; # skip empty lines
711 next if /\x1a/; # skip ^Z
712 next if /^\@version/; # skip @version header
713 if (/^([0-9a-fA-F]+)\s+;\s+\[([*.])([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]+)\]/)
715 my ($uni,$variable) = (hex $1, $2);
716 next if $uni > 65535;
717 $sortkeys[$uni] = [ $uni, hex $3, hex $4, hex $5, hex $6 ];
718 next;
720 if (/^([0-9a-fA-F]+\s+)+;\s+\[[*.]([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]+)\]/)
722 # multiple character sequence, ignored for now
723 next;
725 die "$SORTKEYS: Unrecognized line $_\n";
727 close $INPUT;
729 # compress the keys to 32 bit:
730 # key 1 to 16 bits, key 2 to 8 bits, key 3 to 4 bits, key 4 to 1 bit
732 @sortkeys = sort { ${$a}[1] <=> ${$b}[1] or
733 ${$a}[2] <=> ${$b}[2] or
734 ${$a}[3] <=> ${$b}[3] or
735 ${$a}[4] <=> ${$b}[4] or
736 $a cmp $b; } @sortkeys;
738 my ($n2, $n3) = (1, 1);
739 my @keys = (-1, -1, -1, -1, -1 );
740 my @flatkeys = ();
742 for (my $i = 0; $i < 65536; $i++)
744 my @current = @{$sortkeys[$i]};
745 next if $current[0] == -1;
746 if ($current[1] == $keys[1])
748 if ($current[2] == $keys[2])
750 if ($current[3] == $keys[3])
752 # nothing
754 else
756 $keys[3] = $current[3];
757 $n3++;
758 die if ($n3 >= 16);
761 else
763 $keys[2] = $current[2];
764 $keys[3] = $current[3];
765 $n2++;
766 $n3 = 1;
767 die if ($n2 >= 256);
770 else
772 $keys[1] = $current[1];
773 $keys[2] = $current[2];
774 $keys[3] = $current[3];
775 $n2 = 1;
776 $n3 = 1;
779 if ($current[2]) { $current[2] = $n2; }
780 if ($current[3]) { $current[3] = $n3; }
781 if ($current[4]) { $current[4] = 1; }
783 $flatkeys[$current[0]] = ($current[1] << 16) | ($current[2] << 8) | ($current[3] << 4) | $current[4];
785 return @flatkeys;
789 ################################################################
790 # build the sort keys table
791 sub DUMP_SORTKEYS($@)
793 my ($filename, @keys) = @_;
795 # count the number of 256-key ranges that contain something
797 my @offsets = ();
798 my $ranges = 2;
799 for (my $i = 0; $i < 256; $i++) { $offsets[$i] = 256; }
800 for (my $i = 0; $i < 65536; $i++)
802 next unless defined $keys[$i];
803 $offsets[$i >> 8] = $ranges * 256;
804 $ranges++;
805 $i |= 255;
808 # output the range offsets
810 open OUTPUT,">$filename.new" or die "Cannot create $filename";
811 printf "Building $filename\n";
812 printf OUTPUT "/* Unicode collation element table */\n";
813 printf OUTPUT "/* generated from %s */\n", "$REPORTS/$SORTKEYS";
814 printf OUTPUT "/* DO NOT EDIT!! */\n\n";
816 printf OUTPUT "const unsigned int collation_table[%d] =\n{\n", $ranges*256;
817 printf OUTPUT " /* index */\n";
818 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%08x", 0, @offsets );
820 # output the default values
822 printf OUTPUT " /* defaults */\n";
823 printf OUTPUT "%s", DUMP_ARRAY( "0x%08x", 0, (0xffffffff) x 256 );
825 # output all the key ranges
827 for (my $i = 0; $i < 256; $i++)
829 next if $offsets[$i] == 256;
830 printf OUTPUT ",\n /* 0x%02x00 .. 0x%02xff */\n", $i, $i;
831 printf OUTPUT "%s", DUMP_ARRAY( "0x%08x", 0xffffffff, @keys[($i<<8) .. ($i<<8)+255] );
833 printf OUTPUT "\n};\n";
834 close OUTPUT;
835 save_file($filename);
839 ################################################################
840 # add default mappings once the file had been read
841 sub add_default_mappings($)
843 my $first_private = shift;
845 # Apply aliases
847 foreach my $alias (@unicode_aliases)
849 my $target = undef;
850 foreach my $src (@$alias)
852 if (defined($uni2cp[$src]))
854 $target = $uni2cp[$src];
855 last;
858 next unless defined($target);
860 # At least one char of the alias set is defined, set the others to the same value
861 foreach my $src (@$alias)
863 $uni2cp[$src] = $target unless defined($uni2cp[$src]);
867 # For every src -> target mapping in the defaults table,
868 # make uni2cp[src] = uni2cp[target] if uni2cp[target] is defined
870 for (my $src = 0; $src < 65536; $src++)
872 next if defined($uni2cp[$src]); # source has a definition already
873 next unless defined($unicode_defaults[$src]); # no default for this char
874 my $target = $unicode_defaults[$src];
876 # do a recursive mapping until we find a target char that is defined
877 while (!defined($uni2cp[$target]) &&
878 defined($unicode_defaults[$target])) { $target = $unicode_defaults[$target]; }
880 if (defined($uni2cp[$target])) { $uni2cp[$src] = $uni2cp[$target]; }
883 # Add a private char mapping for undefined chars
885 if ($first_private)
887 for (my $i = 32; $i < 256; $i++)
889 next if defined($cp2uni[$i]);
890 $cp2uni[$i] = $first_private;
891 $uni2cp[$first_private] = $i;
892 $first_private++;
896 # Add an identity mapping for all undefined chars
898 for (my $i = 0; $i < 256; $i++)
900 next if defined($cp2uni[$i]);
901 next if defined($uni2cp[$i]);
902 $cp2uni[$i] = $uni2cp[$i] = $i;
906 ################################################################
907 # dump an array of integers
908 sub DUMP_ARRAY($$@)
910 my ($format,$default,@array) = @_;
911 my $i;
912 my $ret = " ";
913 for ($i = 0; $i < $#array; $i++)
915 $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
916 $ret .= (($i % 8) != 7) ? ", " : ",\n ";
918 $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
919 return $ret;
923 ################################################################
924 # dump a unicode->ascii mapping table
925 sub dump_uni2cp_table($$)
927 my ($width, $def) = @_;
929 # count the number of unicode->ascii subtables that contain something
931 my @lowerbounds;
932 my @upperbounds;
933 my $index = 0;
934 my @filled = (-1) x 256;
935 for (my $i = 0; $i < 65536; $i++)
937 next unless defined $uni2cp[$i];
938 if ($filled[$i >> 8] == -1)
940 $lowerbounds[$index] = $i & 0xff;
941 $upperbounds[$index] = 0xff - $lowerbounds[$index];
942 $filled[$i >> 8] = $index * 256;
943 $index++;
945 else
947 $upperbounds[$index-1] = 0xff - ($i & 0xff);
951 # add defaults mapping at the end
952 $filled[256] = $index * 256;
953 $lowerbounds[$index] = 255;
954 $upperbounds[$index] = 0;
955 $index++;
957 # collapse blocks upwards if possible
958 my $removed = 0;
959 $index = 0;
960 for (my $i = 0; $i < 257; $i++)
962 next if $filled[$i] == -1;
963 if ($upperbounds[$index - 1] > $lowerbounds[$index])
965 $removed += $lowerbounds[$index];
967 else
969 $removed += $upperbounds[$index - 1];
970 $lowerbounds[$index] = $upperbounds[$index - 1];
972 $filled[$i] -= $removed;
973 $index++;
976 # output all the subtables into a single array
978 printf OUTPUT "static const unsigned %s uni2cp_low[%d] =\n{\n",
979 $width == 8 ? "char" : "short", ($index + 1) * 256 - $removed;
981 my $format = $width == 8 ? "0x%02x" : "0x%04x";
982 for (my $i = $index = 0; $i < 257; $i++)
984 next if $filled[$i] == -1;
985 my $start = ($i << 8) + $lowerbounds[$index];
986 my $end = ($i << 8) + 255;
987 if ($i == 256)
989 print OUTPUT " /* defaults */\n";
990 printf OUTPUT "%s\n};\n\n", DUMP_ARRAY( $format, 0, ($def) x ($end - $start + 1) );
992 else
994 printf OUTPUT " /* 0x%04x .. 0x%04x */\n", $start, $end;
995 printf OUTPUT "%s,\n", DUMP_ARRAY( $format, $def, @uni2cp[$start .. $end] );
997 $index++;
1000 # output a table of the offsets of the subtables in the previous array
1002 for (my $i = 0; $i < 256; $i++)
1004 if ($filled[$i] == -1) { $filled[$i] = $filled[256]; }
1006 printf OUTPUT "static const unsigned short uni2cp_high[256] =\n";
1007 printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", 0, @filled[0..255] );
1010 ################################################################
1011 # dump an SBCS mapping table
1012 sub dump_sbcs_table($$$$$)
1014 my ($codepage, $has_glyphs, $name, $def, $defw) = @_;
1015 my $i;
1017 # output the ascii->unicode table
1019 if ($has_glyphs)
1021 printf OUTPUT "static const WCHAR cp2uni[512] =\n";
1022 printf OUTPUT "{\n%s", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[0 .. 255] );
1023 printf OUTPUT ",\n /* glyphs */\n%s\n};\n\n",
1024 DUMP_ARRAY( "0x%04x", $defw, get_glyphs_mapping(@cp2uni[0 .. 255]) );
1026 else
1028 printf OUTPUT "static const WCHAR cp2uni[256] =\n";
1029 printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[0 .. 255] );
1032 dump_uni2cp_table( 8, $def );
1034 # output the code page descriptor
1036 printf OUTPUT "const struct sbcs_table cptable_%03d =\n{\n", $codepage;
1037 printf OUTPUT " { %d, 1, 0x%04x, 0x%04x, \"%s\" },\n",
1038 $codepage, $def, $defw, $name;
1039 printf OUTPUT " cp2uni,\n";
1040 if ($has_glyphs) { printf OUTPUT " cp2uni + 256,\n"; }
1041 else { printf OUTPUT " cp2uni,\n"; }
1042 printf OUTPUT " uni2cp_low,\n";
1043 printf OUTPUT " uni2cp_high\n};\n";
1047 ################################################################
1048 # dump a DBCS mapping table
1049 sub dump_dbcs_table($$$$@)
1051 my ($codepage, $name, $def, $defw, @lb_ranges) = @_;
1053 # build a list of lead bytes that are actually used
1055 my @lblist = ();
1056 LBLOOP: for (my $y = 0; $y <= $#lead_bytes; $y++)
1058 my $base = $lead_bytes[$y] << 8;
1059 for (my $x = 0; $x < 256; $x++)
1061 if (defined $cp2uni[$base+$x])
1063 push @lblist,$lead_bytes[$y];
1064 next LBLOOP;
1068 my $unused = ($#lead_bytes > $#lblist);
1070 # output the ascii->unicode table for the single byte chars
1072 printf OUTPUT "static const WCHAR cp2uni[%d] =\n", 256 * ($#lblist + 2 + $unused);
1073 printf OUTPUT "{\n%s,\n", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[0 .. 255] );
1075 # output the default table for unused lead bytes
1077 if ($unused)
1079 printf OUTPUT " /* unused lead bytes */\n";
1080 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", 0, ($defw) x 256 );
1083 # output the ascii->unicode table for each DBCS lead byte
1085 for (my $y = 0; $y <= $#lblist; $y++)
1087 my $base = $lblist[$y] << 8;
1088 printf OUTPUT " /* lead byte %02x */\n", $lblist[$y];
1089 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[$base .. $base+255] );
1090 printf OUTPUT ($y < $#lblist) ? ",\n" : "\n};\n\n";
1093 # output the lead byte subtables offsets
1095 my @offsets = (0) x 256;
1096 for (my $x = 0; $x <= $#lblist; $x++) { $offsets[$lblist[$x]] = $x + 1; }
1097 if ($unused)
1099 # increment all lead bytes offset to take into account the unused table
1100 for (my $x = 0; $x <= $#lead_bytes; $x++) { $offsets[$lead_bytes[$x]]++; }
1102 printf OUTPUT "static const unsigned char cp2uni_leadbytes[256] =\n";
1103 printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%02x", 0, @offsets );
1105 dump_uni2cp_table( 16, $def );
1107 # output the code page descriptor
1109 printf OUTPUT "const struct dbcs_table cptable_%03d =\n{\n", $codepage;
1110 printf OUTPUT " { %d, 2, 0x%04x, 0x%04x, \"%s\" },\n",
1111 $codepage, $def, $defw, $name;
1112 printf OUTPUT " cp2uni,\n";
1113 printf OUTPUT " cp2uni_leadbytes,\n";
1114 printf OUTPUT " uni2cp_low,\n";
1115 printf OUTPUT " uni2cp_high,\n";
1116 printf OUTPUT " {\n %s\n }\n", DUMP_ARRAY( "0x%02x", 0, @lb_ranges, 0, 0 );
1117 printf OUTPUT "};\n";
1121 ################################################################
1122 # get the list of defined lead byte ranges
1123 sub get_lb_ranges()
1125 my @list = ();
1126 my @ranges = ();
1127 my $i = 0;
1128 foreach $i (@lead_bytes) { $list[$i] = 1; }
1129 my $on = 0;
1130 for (my $i = 0; $i < 256; $i++)
1132 if ($on)
1134 if (!defined $list[$i]) { push @ranges, $i-1; $on = 0; }
1136 else
1138 if ($list[$i]) { push @ranges, $i; $on = 1; }
1141 if ($on) { push @ranges, 0xff; }
1142 return @ranges;
1145 ################################################################
1146 # dump the Indic Syllabic Category table
1147 sub dump_indic($)
1149 my $filename = shift;
1150 my @indic_table = ($indic_types{'Other'}) x 65536;;
1152 my $INPUT = open_data_file( $UNIDATA, "IndicSyllabicCategory.txt" );
1153 while (<$INPUT>)
1155 next if /^\#/; # skip comments
1156 next if /^\s*$/; # skip empty lines
1157 next if /\x1a/; # skip ^Z
1158 if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*#/)
1160 my $type = $2;
1161 die "unknown indic $type" unless defined $indic_types{$type};
1162 if (hex $1 < 65536)
1164 $indic_table[hex $1] = $indic_types{$type};
1166 next;
1168 elsif (/^\s*([0-9a-fA-F]+)..\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*#/)
1170 my $type = $3;
1171 die "unknown indic $type" unless defined $indic_types{$type};
1172 if (hex $1 < 65536 and hex $2 < 6536)
1174 foreach my $i (hex $1 .. hex $2)
1176 $indic_table[$i] = $indic_types{$type};
1179 next;
1181 die "malformed line $_";
1183 close $INPUT;
1185 $INPUT = open_data_file( $UNIDATA, "IndicMatraCategory.txt" );
1186 while (<$INPUT>)
1188 next if /^\#/; # skip comments
1189 next if /^\s*$/; # skip empty lines
1190 next if /\x1a/; # skip ^Z
1191 if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z]+)\s*#/)
1193 my $type = $2;
1194 die "unknown matra $type" unless defined $matra_types{$type};
1195 $indic_table[hex $1] += $matra_types{$type} << 8;
1196 next;
1198 elsif (/^\s*([0-9a-fA-F]+)..\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*#/)
1200 my $type = $3;
1201 die "unknown matra $type" unless defined $matra_types{$type};
1202 foreach my $i (hex $1 .. hex $2)
1204 $indic_table[$i] += $matra_types{$type} << 8;
1206 next;
1208 die "malformed line $_";
1210 close $INPUT;
1212 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1213 print "Building $filename\n";
1214 print OUTPUT "/* Unicode Indic Syllabic Category */\n";
1215 print OUTPUT "/* generated from $UNIDATA/IndicSyllabicCategory.txt */\n";
1216 print OUTPUT "/* and from $UNIDATA/IndicMatraCategory.txt */\n";
1217 print OUTPUT "/* DO NOT EDIT!! */\n\n";
1218 print OUTPUT "#include \"wine/unicode.h\"\n\n";
1220 dump_two_level_mapping( "indic_syllabic_table", @indic_table);
1222 close OUTPUT;
1223 save_file($filename);
1226 ################################################################
1227 # dump the Line Break Properties table
1228 sub dump_linebreak($)
1230 my $filename = shift;
1231 my @break_table = ($break_types{'XX'}) x 65536;;
1232 my $next_group = 0;
1234 my $INPUT = open_data_file( $UNIDATA, "LineBreak.txt" );
1235 while (<$INPUT>)
1237 next if /^\#/; # skip comments
1238 next if /^\s*$/; # skip empty lines
1239 next if /\x1a/; # skip ^Z
1240 if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z])+\s*/)
1242 my $type = $2;
1243 die "unknown breaktype $type" unless defined $break_types{$type};
1244 $break_table[hex $1] = $break_types{$type};
1245 next;
1247 elsif (/^\s*([0-9a-fA-F]+)..\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z])+\s*/)
1249 my $type = $3;
1250 die "unknown breaktype $type" unless defined $break_types{$type};
1251 foreach my $i (hex $1 .. hex $2)
1253 $break_table[$i] = $break_types{$type};
1255 next;
1257 die "malformed line $_";
1259 close $INPUT;
1261 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1262 print "Building $filename\n";
1263 print OUTPUT "/* Unicode Line Break Properties */\n";
1264 print OUTPUT "/* generated from $UNIDATA/LineBreak.txt */\n";
1265 print OUTPUT "/* DO NOT EDIT!! */\n\n";
1266 print OUTPUT "#include \"wine/unicode.h\"\n\n";
1268 dump_two_level_mapping( "wine_linebreak_table", @break_table);
1270 close OUTPUT;
1271 save_file($filename);
1275 ################################################################
1276 # dump the BiDi mirroring table
1277 sub dump_mirroring($)
1279 my $filename = shift;
1280 my @mirror_table = ();
1282 my $INPUT = open_data_file( $UNIDATA, "BidiMirroring.txt" );
1283 while (<$INPUT>)
1285 next if /^\#/; # skip comments
1286 next if /^$/; # skip empty lines
1287 next if /\x1a/; # skip ^Z
1288 if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9a-fA-F]+)/)
1290 $mirror_table[hex $1] = hex $2;
1291 next;
1293 die "malformed line $_";
1295 close $INPUT;
1297 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1298 print "Building $filename\n";
1299 print OUTPUT "/* Unicode BiDi mirroring */\n";
1300 print OUTPUT "/* generated from $UNIDATA/BidiMirroring.txt */\n";
1301 print OUTPUT "/* DO NOT EDIT!! */\n\n";
1302 print OUTPUT "#include \"wine/unicode.h\"\n\n";
1303 DUMP_CASE_TABLE( "wine_mirror_map", @mirror_table );
1304 close OUTPUT;
1305 save_file($filename);
1309 ################################################################
1310 # dump the Arabic shaping table
1311 sub dump_shaping($)
1313 my $filename = shift;
1314 my %groups;
1315 my $next_group = 0;
1317 $groups{"No_Joining_Group"} = $next_group++;
1319 my $INPUT = open_data_file( $UNIDATA, "ArabicShaping.txt" );
1320 while (<$INPUT>)
1322 next if /^\#/; # skip comments
1323 next if /^\s*$/; # skip empty lines
1324 next if /\x1a/; # skip ^Z
1325 if (/^\s*([0-9a-fA-F]+)\s*;.*;\s*([RLDCUT])\s*;\s*(\w+)/)
1327 my $type = $2;
1328 my $group = $3;
1329 $groups{$group} = $next_group++ unless defined $groups{$group};
1330 $joining_table[hex $1] = $joining_types{$type} | ($groups{$group} << 8);
1331 next;
1333 die "malformed line $_";
1335 close $INPUT;
1337 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1338 print "Building $filename\n";
1339 print OUTPUT "/* Unicode Arabic shaping */\n";
1340 print OUTPUT "/* generated from $UNIDATA/ArabicShaping.txt */\n";
1341 print OUTPUT "/* DO NOT EDIT!! */\n\n";
1342 print OUTPUT "#include \"wine/unicode.h\"\n\n";
1344 dump_two_level_mapping( "wine_shaping_table", @joining_table );
1346 print OUTPUT "\nconst unsigned short wine_shaping_forms[256][4] =\n{\n";
1347 for (my $i = 0x600; $i <= 0x6ff; $i++)
1349 printf OUTPUT " { 0x%04x, 0x%04x, 0x%04x, 0x%04x },\n",
1350 ${joining_forms{"isolated"}}[$i] || $i,
1351 ${joining_forms{"final"}}[$i] || $i,
1352 ${joining_forms{"initial"}}[$i] || $i,
1353 ${joining_forms{"medial"}}[$i] || $i;
1355 print OUTPUT "};\n";
1357 close OUTPUT;
1358 save_file($filename);
1362 ################################################################
1363 # dump the case mapping tables
1364 sub DUMP_CASE_MAPPINGS($)
1366 my $filename = shift;
1367 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1368 printf "Building $filename\n";
1369 printf OUTPUT "/* Unicode case mappings */\n";
1370 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
1371 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1373 DUMP_CASE_TABLE( "wine_casemap_lower", @tolower_table );
1374 DUMP_CASE_TABLE( "wine_casemap_upper", @toupper_table );
1375 DUMP_CASE_TABLE( "wine_digitmap", @digitmap_table );
1376 DUMP_CASE_TABLE( "wine_compatmap", @compatmap_table );
1377 close OUTPUT;
1378 save_file($filename);
1382 ################################################################
1383 # dump a case mapping table
1384 sub DUMP_CASE_TABLE($@)
1386 my ($name,@table) = @_;
1388 # count the number of sub tables that contain something
1389 # also compute the low and upper populated bounds
1391 my @lowerbounds = ( 0, 0 );
1392 my @upperbounds = ( 0, 255 );
1393 my $index = 0;
1394 my @filled = ();
1395 for (my $i = 0; $i < 65536; $i++)
1397 next unless defined $table[$i];
1398 if (!defined $filled[$i >> 8])
1400 $lowerbounds[$index] = $i & 0xff;
1401 $upperbounds[$index] = 0xff - $lowerbounds[$index];
1402 $filled[$i >> 8] = $index * 256 + 512;
1403 $index++;
1405 else
1407 $upperbounds[$index-1] = 0xff - ($i & 0xff);
1409 $table[$i] = ($table[$i] - $i) & 0xffff;
1412 # Collapse blocks upwards if possible
1413 my $removed = 0;
1414 $index = 0;
1415 for (my $i = 0; $i < 256; $i++)
1417 next unless defined $filled[$i];
1418 if ($upperbounds[$index - 1] > $lowerbounds[$index])
1420 $removed = $removed + $lowerbounds[$index];
1422 else
1424 $removed = $removed + $upperbounds[$index - 1];
1425 $lowerbounds[$index] = $upperbounds[$index - 1];
1427 $filled[$i] = $filled[$i] - $removed;
1428 $index++;
1431 # dump the table
1433 printf OUTPUT "const WCHAR %s[%d] =\n", $name, $index * 256 + 512 - $removed;
1434 printf OUTPUT "{\n /* index */\n";
1435 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", 256, @filled );
1436 printf OUTPUT " /* defaults */\n";
1437 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, (0) x 256 );
1438 $index = 0;
1439 for (my $i = 0; $i < 256; $i++)
1441 next unless $filled[$i];
1442 printf OUTPUT ",\n /* 0x%02x%02x .. 0x%02xff */\n", $i, $lowerbounds[$index], $i;
1443 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0,
1444 @table[($i<<8) + $lowerbounds[$index] .. ($i<<8)+255] );
1445 $index++;
1447 printf OUTPUT "\n};\n";
1450 ################################################################
1451 # compress a mapping table by removing identical rows
1452 sub compress_array($@)
1454 my $rows = shift;
1455 my @table = @_;
1456 my $len = @table / $rows;
1457 my @array = (0) x $rows;
1458 my %sequences;
1460 # try to merge table rows
1461 for (my $row = 0; $row < $rows; $row++)
1463 my $rowtxt = pack "S*", @table[($row * $len)..($row * $len + $len - 1)];
1464 if (defined($sequences{$rowtxt}))
1466 # reuse an existing row
1467 $array[$row] = $sequences{$rowtxt};
1469 else
1471 # create a new row
1472 $sequences{$rowtxt} = $array[$row] = $#array + 1;
1473 push @array, @table[$row * $len..$row * $len + $len - 1];
1476 return @array;
1479 ################################################################
1480 # dump a simple char -> 16-bit value mapping table
1481 sub dump_simple_mapping($@)
1483 my $name = shift;
1484 my @array = compress_array( 256, @_[0..65535] );
1486 printf OUTPUT "const unsigned short %s[%d] =\n{\n", $name, $#array+1;
1487 printf OUTPUT " /* offsets */\n%s,\n", DUMP_ARRAY( "0x%04x", 0, @array[0..255] );
1488 printf OUTPUT " /* values */\n%s\n};\n", DUMP_ARRAY( "0x%04x", 0, @array[256..$#array] );
1491 ################################################################
1492 # dump a char -> 16-bit value mapping table using two-level tables
1493 sub dump_two_level_mapping($@)
1495 my $name = shift;
1496 my @row_array = compress_array( 4096, @_[0..65535] );
1497 my @array = compress_array( 256, @row_array[0..4095] );
1499 for (my $i = 256; $i < @array; $i++) { $array[$i] += @array - 4096; }
1501 printf OUTPUT "const unsigned short %s[%d] =\n{\n", $name, @array + @row_array - 4096;
1502 printf OUTPUT " /* level 1 offsets */\n%s,\n", DUMP_ARRAY( "0x%04x", 0, @array[0..255] );
1503 printf OUTPUT " /* level 2 offsets */\n%s,\n", DUMP_ARRAY( "0x%04x", 0, @array[256..$#array] );
1504 printf OUTPUT " /* values */\n%s\n};\n", DUMP_ARRAY( "0x%04x", 0, @row_array[4096..$#row_array] );
1507 ################################################################
1508 # dump a binary case mapping table in l_intl.nls format
1509 sub dump_binary_case_table(@)
1511 my (@table) = @_;
1513 my %difftables_hash = ();
1514 my @difftables;
1515 my %offtables2_hash = ();
1516 my @offtables2 = ();
1518 my @offtable = ();
1519 for (my $i = 0; $i < 256; $i++)
1521 my @offtable2 = ();
1522 for(my $j = 0; $j < 16; $j++) # offset table for xx00-xxFF characters
1524 my @difftable;
1525 for (my $k = 0; $k < 16; $k++) # case map table for xxx0-xxxF characters
1527 my $char = ($i<<8) + ($j<<4) + $k;
1528 $difftable[$k] = (defined $table[$char]) ? (($table[$char]-$char) & 0xffff) : 0;
1531 my $diff_key = pack "S*", @difftable;
1532 my $offset3 = $difftables_hash{$diff_key};
1533 if (!defined $offset3)
1535 $offset3 = scalar @difftables;
1536 $difftables_hash{$diff_key} = $offset3;
1537 push @difftables, @difftable;
1539 $offtable2[$j] = $offset3;
1542 my $offtable2_key = pack "S*", @offtable2;
1543 my $offset2 = $offtables2_hash{$offtable2_key};
1544 if (!defined $offset2)
1546 $offset2 = scalar @offtables2;
1547 $offtables2_hash{$offtable2_key} = $offset2;
1548 push @offtables2, \@offtable2;
1550 $offtable[$i] = $offset2;
1553 my @output;
1554 my $offset = 0x100; # offset of first subtable in words
1555 foreach (@offtable)
1557 push @output, 0x10 * $_ + $offset; # offset of subtable in words
1560 $offset = 0x100 + 0x10 * scalar @offtables2; # offset of first difftable in words
1561 foreach(@offtables2)
1563 my $table = $_;
1564 foreach(@$table)
1566 push @output, $_ + $offset; # offset of difftable in words
1570 my $len = 1 + scalar @output + scalar @difftables;
1571 return pack "S<*", $len, @output, @difftables;
1575 ################################################################
1576 # dump case mappings for l_intl.nls
1577 sub dump_intl_nls($)
1579 my $filename = shift;
1580 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1581 printf "Building $filename\n";
1583 binmode OUTPUT;
1584 print OUTPUT pack "S<", 1; # version
1585 print OUTPUT dump_binary_case_table( @toupper_table );
1586 print OUTPUT dump_binary_case_table( @tolower_table );
1587 close OUTPUT;
1588 save_file($filename);
1592 sub load_nameprep_range_table($$$)
1594 my ($INPUT, $val, $table_ref) = @_;
1596 while (<$INPUT>)
1598 if (/^\s*([0-9a-fA-F]+)-([0-9a-fA-F]+)/)
1600 my $last = hex $2;
1601 $last = 65535 if($last >= 65536);
1602 foreach my $i (hex $1 .. $last)
1604 $table_ref->[$i] |= $val;
1606 next;
1608 elsif (/^\s*([0-9a-fA-F]+)/)
1610 if (hex $1 < 65536)
1612 $table_ref->[hex $1] |= $val;
1614 next;
1617 return if (/End\sTable/);
1621 sub load_nameprep_map_table($$)
1623 my ($INPUT, $table_ref) = @_;
1625 while (<$INPUT>)
1627 if (/^\s*([0-9a-fA-F]+);\s;/)
1629 # special value for map to nothing
1630 $table_ref->[hex $1] = [0xffff, 0xffff, 0xffff];
1631 next;
1633 elsif (/^\s*([0-9a-fA-F]+);\s([0-9a-fA-F]+);/)
1635 $table_ref->[hex $1] = [hex $2, 0, 0];
1636 next;
1638 elsif (/^\s*([0-9a-fA-F]+);\s([0-9a-fA-F]+)\s([0-9a-fA-F]+);/)
1640 $table_ref->[hex $1] = [hex $2, hex $3, 0];
1641 next;
1643 elsif (/^\s*([0-9a-fA-F]+);\s([0-9a-fA-F]+)\s([0-9a-fA-F]+)\s([0-9a-fA-F]+);/)
1645 $table_ref->[hex $1] = [hex $2, hex $3, hex $4];
1646 next;
1649 return if (/End\sTable/);
1653 ################################################################
1654 # dump mapping table, prohibited characters set, unassigned
1655 # characters, bidirectional rules used by nameprep algorithm
1656 sub dump_nameprep($)
1658 my $filename = shift;
1659 my @mapping_table = ();
1660 my @flags_table = (0) x 65536;
1662 my $INPUT = open_data_file( $RFCS, $STRINGPREP );
1663 while (<$INPUT>)
1665 next unless /Start\sTable/;
1667 load_nameprep_range_table($INPUT, $nameprep_flags{"unassigned"}, \@flags_table) if (/A.1/);
1668 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.1.2/);
1669 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.2.2/);
1670 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.3/);
1671 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.4/);
1672 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.5/);
1673 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.6/);
1674 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.7/);
1675 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.8/);
1676 load_nameprep_range_table($INPUT, $nameprep_flags{"prohibited"}, \@flags_table) if (/C.9/);
1677 load_nameprep_range_table($INPUT, $nameprep_flags{"bidi_ral"}, \@flags_table) if (/D.1/);
1678 load_nameprep_range_table($INPUT, $nameprep_flags{"bidi_l"}, \@flags_table) if (/D.2/);
1680 load_nameprep_map_table($INPUT, \@mapping_table) if (/B.1/);
1681 load_nameprep_map_table($INPUT, \@mapping_table) if (/B.2/);
1683 close $INPUT;
1685 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1686 print "Building $filename\n";
1687 print OUTPUT "/* Nameprep algorithm related data */\n";
1688 print OUTPUT "/* generated from $RFCS/$STRINGPREP */\n";
1689 print OUTPUT "/* DO NOT EDIT!! */\n\n";
1690 print OUTPUT "#include \"wine/unicode.h\"\n\n";
1692 dump_two_level_mapping( "nameprep_char_type", @flags_table );
1694 ######### mapping table
1695 # first determine all the 16-char subsets that contain something
1696 my @filled = ();
1697 my $pos = 16*3; # for the null subset
1698 for (my $i = 0; $i < 65536; $i++)
1700 next unless defined $mapping_table[$i];
1701 $filled[$i >> 4] = $pos;
1702 $pos += 16*3;
1703 $i |= 15;
1705 my $total = $pos;
1707 # now count the 256-char subsets that contain something
1708 my @filled_idx = (256) x 256;
1709 $pos = 256 + 16;
1710 for (my $i = 0; $i < 4096; $i++)
1712 next unless $filled[$i];
1713 $filled_idx[$i >> 4] = $pos;
1714 $pos += 16;
1715 $i |= 15;
1717 my $null_offset = $pos;
1718 $total += $pos;
1720 # add the index offsets to the subsets positions
1721 for (my $i = 0; $i < 4096; $i++)
1723 next unless $filled[$i];
1724 $filled[$i] += $null_offset;
1727 # dump the main index
1728 printf OUTPUT "const WCHAR nameprep_mapping[%d] =\n", $total;
1729 printf OUTPUT "{\n /* index */\n";
1730 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @filled_idx );
1731 printf OUTPUT ",\n /* null sub-index */\n%s", DUMP_ARRAY( "0x%04x", 0, ($null_offset) x 16 );
1733 # dump the second-level indexes
1734 for (my $i = 0; $i < 256; $i++)
1736 next unless ($filled_idx[$i] > 256);
1737 my @table = @filled[($i<<4)..($i<<4)+15];
1738 for (my $j = 0; $j < 16; $j++) { $table[$j] ||= $null_offset; }
1739 printf OUTPUT ",\n /* sub-index %02x */\n", $i;
1740 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
1743 # dump the 16-char subsets
1744 printf OUTPUT ",\n /* null mapping */\n";
1745 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, (0) x 48 );
1747 for (my $i = 0; $i < 4096; $i++)
1749 next unless $filled[$i];
1750 my @table = (0) x 48;
1751 for (my $j = 0; $j < 16; $j++)
1753 if (defined $mapping_table[($i<<4) + $j])
1755 $table[3 * $j] = ${$mapping_table[($i << 4) + $j]}[0];
1756 $table[3 * $j + 1] = ${$mapping_table[($i << 4) + $j]}[1];
1757 $table[3 * $j + 2] = ${$mapping_table[($i << 4) + $j]}[2];
1760 printf OUTPUT ",\n /* 0x%03x0 .. 0x%03xf */\n", $i, $i;
1761 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
1764 printf OUTPUT "\n};\n";
1766 close OUTPUT;
1767 save_file($filename);
1770 ################################################################
1771 # dump the ctype tables
1772 sub DUMP_CTYPE_TABLES($)
1774 my $filename = shift;
1775 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1776 printf "Building $filename\n";
1777 printf OUTPUT "/* Unicode ctype tables */\n";
1778 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
1779 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1781 # add the direction in the high 4 bits of the category
1782 for (my $i = 0; $i < 65536; $i++)
1784 $category_table[$i] |= $direction_table[$i] << 12 if defined $direction_table[$i];
1787 dump_simple_mapping( "wine_wctype_table", @category_table );
1789 close OUTPUT;
1790 save_file($filename);
1794 ################################################################
1795 # dump the char composition tables
1796 sub DUMP_COMPOSE_TABLES($)
1798 my $filename = shift;
1800 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1801 printf "Building $filename\n";
1802 printf OUTPUT "/* Unicode char composition */\n";
1803 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
1804 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1806 ######### composition table
1808 my @filled = ();
1809 foreach my $i (@compose_table)
1811 my @comp = @$i;
1812 push @{$filled[$comp[1]]}, [ $comp[0], $comp[2] ];
1815 # count how many different second chars we have
1817 my $count = 0;
1818 for (my $i = 0; $i < 65536; $i++)
1820 next unless defined $filled[$i];
1821 $count++;
1824 # build the table of second chars and offsets
1826 my $pos = $count + 1;
1827 my @table = ();
1828 for (my $i = 0; $i < 65536; $i++)
1830 next unless defined $filled[$i];
1831 push @table, $i, $pos;
1832 $pos += @{$filled[$i]};
1834 # terminator with last position
1835 push @table, 0, $pos;
1836 printf OUTPUT "const WCHAR unicode_compose_table[0x%x] =\n{\n", 2*$pos;
1837 printf OUTPUT " /* second chars + offsets */\n%s", DUMP_ARRAY( "0x%04x", 0, @table );
1839 # build the table of first chars and mappings
1841 for (my $i = 0; $i < 65536; $i++)
1843 next unless defined $filled[$i];
1844 my @table = ();
1845 my @list = sort { $a->[0] <=> $b->[0] } @{$filled[$i]};
1846 for (my $j = 0; $j <= $#list; $j++)
1848 push @table, $list[$j][0], $list[$j][1];
1850 printf OUTPUT ",\n /* 0x%04x */\n%s", $i, DUMP_ARRAY( "0x%04x", 0, @table );
1852 printf OUTPUT "\n};\n\nconst unsigned int unicode_compose_table_size = %d;\n\n", $count;
1854 ######### decomposition table
1856 # first determine all the 16-char subsets that contain something
1858 @filled = (0) x 4096;
1859 $pos = 16*2; # for the null subset
1860 for (my $i = 0; $i < 65536; $i++)
1862 next unless defined $decomp_table[$i];
1863 $filled[$i >> 4] = $pos;
1864 $pos += 16*2;
1865 $i |= 15;
1867 my $total = $pos;
1869 # now count the 256-char subsets that contain something
1871 my @filled_idx = (256) x 256;
1872 $pos = 256 + 16;
1873 for (my $i = 0; $i < 4096; $i++)
1875 next unless $filled[$i];
1876 $filled_idx[$i >> 4] = $pos;
1877 $pos += 16;
1878 $i |= 15;
1880 my $null_offset = $pos; # null mapping
1881 $total += $pos;
1883 # add the index offsets to the subsets positions
1885 for (my $i = 0; $i < 4096; $i++)
1887 next unless $filled[$i];
1888 $filled[$i] += $null_offset;
1891 # dump the main index
1893 printf OUTPUT "const WCHAR unicode_decompose_table[%d] =\n", $total;
1894 printf OUTPUT "{\n /* index */\n";
1895 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @filled_idx );
1896 printf OUTPUT ",\n /* null sub-index */\n%s", DUMP_ARRAY( "0x%04x", 0, ($null_offset) x 16 );
1898 # dump the second-level indexes
1900 for (my $i = 0; $i < 256; $i++)
1902 next unless ($filled_idx[$i] > 256);
1903 my @table = @filled[($i<<4)..($i<<4)+15];
1904 for (my $j = 0; $j < 16; $j++) { $table[$j] ||= $null_offset; }
1905 printf OUTPUT ",\n /* sub-index %02x */\n", $i;
1906 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
1909 # dump the 16-char subsets
1911 printf OUTPUT ",\n /* null mapping */\n";
1912 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, (0) x 32 );
1914 for (my $i = 0; $i < 4096; $i++)
1916 next unless $filled[$i];
1917 my @table = (0) x 32;
1918 for (my $j = 0; $j < 16; $j++)
1920 if (defined $decomp_table[($i<<4) + $j])
1922 $table[2 * $j] = ${$decomp_table[($i << 4) + $j]}[0];
1923 $table[2 * $j + 1] = ${$decomp_table[($i << 4) + $j]}[1];
1926 printf OUTPUT ",\n /* 0x%03x0 .. 0x%03xf */\n", $i, $i;
1927 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
1930 printf OUTPUT "\n};\n";
1931 close OUTPUT;
1932 save_file($filename);
1935 ################################################################
1936 # output a codepage definition file from the global tables
1937 sub output_codepage_file($$$$)
1939 my ($codepage, $filename, $comment, $has_glyphs) = @_;
1941 my $output = sprintf "libs/wine/c_%03d.c", $codepage;
1942 open OUTPUT,">$output.new" or die "Cannot create $output";
1944 printf "Building %s from %s (%s)\n", $output, $filename || "hardcoded data", $comment;
1946 # dump all tables
1948 printf OUTPUT "/* code page %03d (%s) */\n", $codepage, $comment;
1949 if ($filename)
1951 print OUTPUT "/* generated from $MAPPINGS/$filename */\n";
1952 print OUTPUT "/* DO NOT EDIT!! */\n\n";
1954 else
1956 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
1958 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1960 if (!@lead_bytes) { dump_sbcs_table( $codepage, $has_glyphs, $comment, $default_char, $default_wchar ); }
1961 else { dump_dbcs_table( $codepage, $comment, $default_char, $default_wchar, get_lb_ranges() ); }
1962 close OUTPUT;
1963 save_file($output);
1966 ################################################################
1967 # read a "bestfit" Windows mapping file
1968 sub read_bestfit_file($)
1970 my ($filename) = @_;
1971 my $state = "";
1972 my ($codepage, $width, $count);
1973 my ($lb_cur, $lb_end);
1975 my $INPUT = open_data_file( $MAPPINGS, $filename ) or die "Cannot open $filename";
1977 while (<$INPUT>)
1979 next if /^;/; # skip comments
1980 next if /^\s*$/; # skip empty lines
1981 next if /\x1a/; # skip ^Z
1982 last if /^ENDCODEPAGE/;
1984 if (/^CODEPAGE\s+(\d+)/)
1986 $codepage = $1;
1987 next;
1989 if (/^CPINFO\s+(\d+)\s+0x([0-9a-fA-f]+)\s+0x([0-9a-fA-F]+)/)
1991 $width = $1;
1992 $default_char = hex $2;
1993 $default_wchar = hex $3;
1994 next;
1996 if (/^(MBTABLE|WCTABLE|DBCSRANGE|DBCSTABLE)\s+(\d+)/)
1998 $state = $1;
1999 $count = $2;
2000 next;
2002 if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)/)
2004 if ($state eq "MBTABLE")
2006 my $cp = hex $1;
2007 my $uni = hex $2;
2008 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
2009 next;
2011 if ($state eq "WCTABLE")
2013 my $uni = hex $1;
2014 my $cp = hex $2;
2015 $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
2016 next;
2018 if ($state eq "DBCSRANGE")
2020 my $start = hex $1;
2021 my $end = hex $2;
2022 for (my $i = $start; $i <= $end; $i++) { add_lead_byte( $i ); }
2023 $lb_cur = $start;
2024 $lb_end = $end;
2025 next;
2027 if ($state eq "DBCSTABLE")
2029 my $mb = hex $1;
2030 my $uni = hex $2;
2031 my $cp = ($lb_cur << 8) | $mb;
2032 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
2033 if (!--$count)
2035 if (++$lb_cur > $lb_end) { $state = "DBCSRANGE"; }
2037 next;
2040 die "$filename: Unrecognized line $_\n";
2042 close $INPUT;
2046 ################################################################
2047 # read an input file and generate the corresponding .c file
2048 sub HANDLE_FILE(@)
2050 my ($codepage,$filename,$has_glyphs,$comment,$first_private,$def,$defw) = @_;
2052 @cp2uni = ();
2053 @lead_bytes = ();
2054 @uni2cp = ();
2055 $default_char = $def || $DEF_CHAR;
2056 $default_wchar = $defw || $DEF_CHAR;
2058 # some codepage files are special
2059 if ($codepage == 20932)
2061 READ_JIS0208_FILE( $filename );
2062 add_default_mappings( $first_private );
2064 elsif ($codepage == 20127)
2066 fill_20127_codepage();
2067 add_default_mappings( $first_private );
2069 elsif ($filename =~ /\/bestfit/)
2071 read_bestfit_file( $filename );
2073 else
2075 read_codepage_file( $filename );
2076 if ($codepage == 10001)
2078 # add Shift-JIS mappings
2079 read_bestfit_file( "VENDORS/MICSFT/WindowsBestFit/bestfit932.txt" );
2081 else
2083 add_default_mappings( $first_private );
2087 output_codepage_file( $codepage, $filename, $comment, $has_glyphs );
2091 ################################################################
2092 # save a file if modified
2093 sub save_file($)
2095 my $file = shift;
2096 if (-f $file && !system "cmp $file $file.new >/dev/null")
2098 unlink "$file.new";
2100 else
2102 rename "$file.new", "$file";
2107 ################################################################
2108 # output the list of codepage tables into the cptable.c file
2109 sub output_cptable($)
2111 my $output = shift;
2112 my @tables_decl = ();
2114 printf "Building %s\n", $output;
2116 foreach my $file (@allfiles)
2118 my ($codepage,$filename) = @$file;
2119 push @tables_decl, sprintf("extern union cptable cptable_%03d;\n",$codepage);
2122 push @tables_decl, sprintf("\nstatic const union cptable * const cptables[%d] =\n{\n",$#allfiles+1);
2123 foreach my $file (@allfiles)
2125 my ($codepage,$filename) = @$file;
2126 push @tables_decl, sprintf(" &cptable_%03d,\n", $codepage);
2128 push @tables_decl, "};";
2129 REPLACE_IN_FILE( $output, @tables_decl );
2132 ################################################################
2133 # replace the contents of a file between ### cpmap ### marks
2135 sub REPLACE_IN_FILE($@)
2137 my $name = shift;
2138 my @data = @_;
2139 my @lines = ();
2140 open(FILE,$name) or die "Can't open $name";
2141 while (<FILE>)
2143 push @lines, $_;
2144 last if /\#\#\# cpmap begin \#\#\#/;
2146 push @lines, @data;
2147 while (<FILE>)
2149 if (/\#\#\# cpmap end \#\#\#/) { push @lines, "\n", $_; last; }
2151 push @lines, <FILE>;
2152 open(FILE,">$name.new") or die "Can't modify $name";
2153 print FILE @lines;
2154 close(FILE);
2155 save_file($name);
2158 ################################################################
2159 # main routine
2161 chdir ".." if -f "./make_unicode";
2162 READ_DEFAULTS( $DEFAULTS );
2163 DUMP_CASE_MAPPINGS( "libs/wine/casemap.c" );
2164 DUMP_SORTKEYS( "libs/wine/collation.c", READ_SORTKEYS_FILE() );
2165 DUMP_COMPOSE_TABLES( "libs/wine/compose.c" );
2166 DUMP_CTYPE_TABLES( "libs/wine/wctype.c" );
2167 dump_mirroring( "dlls/usp10/mirror.c" );
2168 dump_shaping( "dlls/usp10/shaping.c" );
2169 dump_linebreak( "dlls/usp10/linebreak.c" );
2170 dump_indic( "dlls/usp10/indicsyllable.c" );
2171 dump_intl_nls("loader/l_intl.nls");
2172 dump_nameprep( "dlls/kernel32/nameprep.c" );
2174 foreach my $file (@allfiles) { HANDLE_FILE( @{$file} ); }
2176 output_cptable("libs/wine/cptable.c");
2178 exit 0;
2180 # Local Variables:
2181 # compile-command: "./make_unicode"
2182 # End: