?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
push 2f3ca95c4974ba229fa47d638b3044f50788f3bd
[wine/hacks.git] / libs / wine / cpmap.pl
blobf27c5f206ed6c6b4b0e8b7d7fab7807eb5813c50
1 #!/usr/bin/perl -w
2 #
3 # Generate code page .c files from ftp.unicode.org descriptions
4 #
5 # Copyright 2000 Alexandre Julliard
6 #
7 # This library is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU Lesser General Public
9 # License as published by the Free Software Foundation; either
10 # version 2.1 of the License, or (at your option) any later version.
11 #
12 # This library is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 # Lesser General Public License for more details.
16 #
17 # You should have received a copy of the GNU Lesser General Public
18 # License along with this library; if not, write to the Free Software
19 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
20 #
21
22 use strict;
23
24 # base directory for ftp.unicode.org files
25 my $BASEDIR = "ftp.unicode.org/Public/";
26 my $MAPPREFIX = $BASEDIR . "MAPPINGS/";
27
28 # UnicodeData file
29 my $UNICODEDATA = $BASEDIR . "UNIDATA/UnicodeData.txt";
30
31 # Sort keys file
32 my $SORTKEYS = "www.unicode.org/reports/tr10/allkeys.txt";
33
34 # Defaults mapping
35 my $DEFAULTS = "./defaults";
36
37 # Default char for undefined mappings
38 my $DEF_CHAR = ord '?';
39
40 my @allfiles =
41 (
42 [ 37, "VENDORS/MICSFT/EBCDIC/CP037.TXT", 0, "IBM EBCDIC US Canada" ],
43 [ 424, "VENDORS/MISC/CP424.TXT", 0, "IBM EBCDIC Hebrew" ],
44 [ 437, "VENDORS/MICSFT/PC/CP437.TXT", 1, "OEM United States" ],
45 [ 500, "VENDORS/MICSFT/EBCDIC/CP500.TXT", 0, "IBM EBCDIC International" ],
46 [ 737, "VENDORS/MICSFT/PC/CP737.TXT", 1, "OEM Greek 437G" ],
47 [ 775, "VENDORS/MICSFT/PC/CP775.TXT", 1, "OEM Baltic" ],
48 [ 850, "VENDORS/MICSFT/PC/CP850.TXT", 1, "OEM Multilingual Latin 1" ],
49 [ 852, "VENDORS/MICSFT/PC/CP852.TXT", 1, "OEM Slovak Latin 2" ],
50 [ 855, "VENDORS/MICSFT/PC/CP855.TXT", 1, "OEM Cyrillic" ],
51 [ 856, "VENDORS/MISC/CP856.TXT", 0, "Hebrew PC" ],
52 [ 857, "VENDORS/MICSFT/PC/CP857.TXT", 1, "OEM Turkish" ],
53 [ 860, "VENDORS/MICSFT/PC/CP860.TXT", 1, "OEM Portuguese" ],
54 [ 861, "VENDORS/MICSFT/PC/CP861.TXT", 1, "OEM Icelandic" ],
55 [ 862, "VENDORS/MICSFT/PC/CP862.TXT", 1, "OEM Hebrew" ],
56 [ 863, "VENDORS/MICSFT/PC/CP863.TXT", 1, "OEM Canadian French" ],
57 [ 864, "VENDORS/MICSFT/PC/CP864.TXT", 0, "OEM Arabic" ],
58 [ 865, "VENDORS/MICSFT/PC/CP865.TXT", 1, "OEM Nordic" ],
59 [ 866, "VENDORS/MICSFT/PC/CP866.TXT", 1, "OEM Russian" ],
60 [ 869, "VENDORS/MICSFT/PC/CP869.TXT", 1, "OEM Greek" ],
61 [ 874, "VENDORS/MICSFT/WindowsBestFit/bestfit874.txt", 1, "ANSI/OEM Thai" ],
62 [ 875, "VENDORS/MICSFT/EBCDIC/CP875.TXT", 0, "IBM EBCDIC Greek" ],
63 [ 878, "VENDORS/MISC/KOI8-R.TXT", 0, "Russian KOI8" ],
64 [ 932, "VENDORS/MICSFT/WindowsBestFit/bestfit932.txt", 0, "ANSI/OEM Japanese Shift-JIS" ],
65 [ 936, "VENDORS/MICSFT/WindowsBestFit/bestfit936.txt", 0, "ANSI/OEM Simplified Chinese GBK" ],
66 [ 949, "VENDORS/MICSFT/WindowsBestFit/bestfit949.txt", 0, "ANSI/OEM Korean Unified Hangul" ],
67 [ 950, "VENDORS/MICSFT/WindowsBestFit/bestfit950.txt", 0, "ANSI/OEM Traditional Chinese Big5" ],
68 [ 1006, "VENDORS/MISC/CP1006.TXT", 0, "IBM Arabic" ],
69 [ 1026, "VENDORS/MICSFT/EBCDIC/CP1026.TXT", 0, "IBM EBCDIC Latin 5 Turkish" ],
70 [ 1250, "VENDORS/MICSFT/WindowsBestFit/bestfit1250.txt", 0, "ANSI Eastern Europe" ],
71 [ 1251, "VENDORS/MICSFT/WindowsBestFit/bestfit1251.txt", 0, "ANSI Cyrillic" ],
72 [ 1252, "VENDORS/MICSFT/WindowsBestFit/bestfit1252.txt", 0, "ANSI Latin 1" ],
73 [ 1253, "VENDORS/MICSFT/WindowsBestFit/bestfit1253.txt", 0, "ANSI Greek" ],
74 [ 1254, "VENDORS/MICSFT/WindowsBestFit/bestfit1254.txt", 0, "ANSI Turkish" ],
75 [ 1255, "VENDORS/MICSFT/WindowsBestFit/bestfit1255.txt", 0, "ANSI Hebrew" ],
76 [ 1256, "VENDORS/MICSFT/WindowsBestFit/bestfit1256.txt", 0, "ANSI Arabic" ],
77 [ 1257, "VENDORS/MICSFT/WindowsBestFit/bestfit1257.txt", 0, "ANSI Baltic" ],
78 [ 1258, "VENDORS/MICSFT/WindowsBestFit/bestfit1258.txt", 0, "ANSI/OEM Viet Nam" ],
79 [ 1361, "OBSOLETE/EASTASIA/KSC/JOHAB.TXT", 0, "Korean Johab" ],
80 [ 10000, "VENDORS/MICSFT/MAC/ROMAN.TXT", 0, "Mac Roman" ],
81 [ 10006, "VENDORS/MICSFT/MAC/GREEK.TXT", 0, "Mac Greek" ],
82 [ 10007, "VENDORS/MICSFT/MAC/CYRILLIC.TXT", 0, "Mac Cyrillic" ],
83 [ 10029, "VENDORS/MICSFT/MAC/LATIN2.TXT", 0, "Mac Latin 2" ],
84 [ 10079, "VENDORS/MICSFT/MAC/ICELAND.TXT", 0, "Mac Icelandic" ],
85 [ 10081, "VENDORS/MICSFT/MAC/TURKISH.TXT", 0, "Mac Turkish" ],
86 [ 20127, undef, 0, "US-ASCII (7bit)" ],
87 [ 20866, "VENDORS/MISC/KOI8-R.TXT", 0, "Russian KOI8" ],
88 [ 20932, "OBSOLETE/EASTASIA/JIS/JIS0208.TXT", 0, "EUC-JP" ],
89 [ 21866, "VENDORS/MISC/KOI8-U.TXT", 0, "Ukrainian KOI8" ],
90 [ 28591, "ISO8859/8859-1.TXT", 0, "ISO 8859-1 Latin 1" ],
91 [ 28592, "ISO8859/8859-2.TXT", 0, "ISO 8859-2 Latin 2 (East European)" ],
92 [ 28593, "ISO8859/8859-3.TXT", 0, "ISO 8859-3 Latin 3 (South European)" ],
93 [ 28594, "ISO8859/8859-4.TXT", 0, "ISO 8859-4 Latin 4 (Baltic old)" ],
94 [ 28595, "ISO8859/8859-5.TXT", 0, "ISO 8859-5 Cyrillic" ],
95 [ 28596, "ISO8859/8859-6.TXT", 0, "ISO 8859-6 Arabic" ],
96 [ 28597, "ISO8859/8859-7.TXT", 0, "ISO 8859-7 Greek" ],
97 [ 28598, "ISO8859/8859-8.TXT", 0, "ISO 8859-8 Hebrew" ],
98 [ 28599, "ISO8859/8859-9.TXT", 0, "ISO 8859-9 Latin 5 (Turkish)" ],
99 [ 28600, "ISO8859/8859-10.TXT", 0, "ISO 8859-10 Latin 6 (Nordic)" ],
100 [ 28603, "ISO8859/8859-13.TXT", 0, "ISO 8859-13 Latin 7 (Baltic)" ],
101 [ 28604, "ISO8859/8859-14.TXT", 0, "ISO 8859-14 Latin 8 (Celtic)" ],
102 [ 28605, "ISO8859/8859-15.TXT", 0, "ISO 8859-15 Latin 9 (Euro)" ],
103 [ 28606, "ISO8859/8859-16.TXT", 0, "ISO 8859-16 Latin 10 (Balkan)" ]
104 );
105
106
107 my %ctype =
108 (
109 "upper" => 0x0001,
110 "lower" => 0x0002,
111 "digit" => 0x0004,
112 "space" => 0x0008,
113 "punct" => 0x0010,
114 "cntrl" => 0x0020,
115 "blank" => 0x0040,
116 "xdigit" => 0x0080,
117 "alpha" => 0x0100
118 );
119
120 my %categories =
121 (
122 "Lu" => $ctype{"alpha"}|$ctype{"upper"}, # Letter, Uppercase
123 "Ll" => $ctype{"alpha"}|$ctype{"lower"}, # Letter, Lowercase
124 "Lt" => $ctype{"alpha"}, # Letter, Titlecase
125 "Mn" => $ctype{"punct"}, # Mark, Non-Spacing
126 "Mc" => $ctype{"punct"}, # Mark, Spacing Combining
127 "Me" => $ctype{"punct"}, # Mark, Enclosing
128 "Nd" => $ctype{"digit"}, # Number, Decimal Digit
129 "Nl" => $ctype{"punct"}, # Number, Letter
130 "No" => $ctype{"punct"}, # Number, Other
131 "Zs" => $ctype{"space"}, # Separator, Space
132 "Zl" => $ctype{"space"}, # Separator, Line
133 "Zp" => $ctype{"space"}, # Separator, Paragraph
134 "Cc" => $ctype{"cntrl"}, # Other, Control
135 "Cf" => 0, # Other, Format
136 "Cs" => 0, # Other, Surrogate
137 "Co" => 0, # Other, Private Use
138 "Cn" => 0, # Other, Not Assigned
139 "Lm" => $ctype{"punct"}, # Letter, Modifier
140 "Lo" => $ctype{"alpha"}, # Letter, Other
141 "Pc" => $ctype{"punct"}, # Punctuation, Connector
142 "Pd" => $ctype{"punct"}, # Punctuation, Dash
143 "Ps" => $ctype{"punct"}, # Punctuation, Open
144 "Pe" => $ctype{"punct"}, # Punctuation, Close
145 "Pi" => $ctype{"punct"}, # Punctuation, Initial quote
146 "Pf" => $ctype{"punct"}, # Punctuation, Final quote
147 "Po" => $ctype{"punct"}, # Punctuation, Other
148 "Sm" => $ctype{"punct"}, # Symbol, Math
149 "Sc" => $ctype{"punct"}, # Symbol, Currency
150 "Sk" => $ctype{"punct"}, # Symbol, Modifier
151 "So" => $ctype{"punct"} # Symbol, Other
152 );
153
154 # a few characters need additional categories that cannot be determined automatically
155 my %special_categories =
156 (
157 "xdigit" => [ ord('0')..ord('9'),ord('A')..ord('F'),ord('a')..ord('f'),
158 0xff10..0xff19, 0xff21..0xff26, 0xff41..0xff46 ],
159 "space" => [ 0x09..0x0d, 0x85 ],
160 "blank" => [ 0x09, 0x20, 0xa0, 0x3000, 0xfeff ],
161 "cntrl" => [ 0x070f, 0x180b, 0x180c, 0x180d, 0x180e, 0x200c, 0x200d,
162 0x200e, 0x200f, 0x202a, 0x202b, 0x202c, 0x202d, 0x202e,
163 0x206a, 0x206b, 0x206c, 0x206d, 0x206e, 0x206f, 0xfeff,
164 0xfff9, 0xfffa, 0xfffb ]
165 );
166
167 my %directions =
168 (
169 "L" => 1, # Left-to-Right
170 "LRE" => 11, # Left-to-Right Embedding
171 "LRO" => 11, # Left-to-Right Override
172 "R" => 2, # Right-to-Left
173 "AL" => 2, # Right-to-Left Arabic
174 "RLE" => 11, # Right-to-Left Embedding
175 "RLO" => 11, # Right-to-Left Override
176 "PDF" => 11, # Pop Directional Format
177 "EN" => 3, # European Number
178 "ES" => 4, # European Number Separator
179 "ET" => 5, # European Number Terminator
180 "AN" => 6, # Arabic Number
181 "CS" => 7, # Common Number Separator
182 "NSM" => 0, # Non-Spacing Mark
183 "BN" => 0, # Boundary Neutral
184 "B" => 8, # Paragraph Separator
185 "S" => 9, # Segment Separator
186 "WS" => 10, # Whitespace
187 "ON" => 11 # Other Neutrals
188 );
189
190 my @cp2uni = ();
191 my @lead_bytes = ();
192 my @uni2cp = ();
193 my @unicode_defaults = ();
194 my @unicode_aliases = ();
195 my @tolower_table = ();
196 my @toupper_table = ();
197 my @digitmap_table = ();
198 my @compatmap_table = ();
199 my @category_table = (0) x 65536;
200 my @direction_table = ();
201 my @decomp_table = ();
202 my @compose_table = ();
203
204
205 ################################################################
206 # read in the defaults file
207 sub READ_DEFAULTS($)
208 {
209 my $filename = shift;
210 my $start;
211
212 # first setup a few default mappings
213
214 open DEFAULTS, "$filename" or die "Cannot open $filename";
215 print "Loading $filename\n";
216 while (<DEFAULTS>)
217 {
218 next if /^\#/; # skip comments
219 next if /^$/; # skip empty lines
220 if (/^(([0-9a-fA-F]+)(,[0-9a-fA-F]+)*)\s+([0-9a-fA-F]+|'.'|none)\s+(\#.*)?/)
221 {
222 my @src = map hex, split /,/,$1;
223 my $dst = $4;
224 my $comment = $5;
225 if ($#src > 0) { push @unicode_aliases, \@src; }
226 next if ($dst eq "none");
227 $dst = ($dst =~ /\'.\'/) ? ord substr($dst,1,1) : hex $dst;
228 foreach my $src (@src)
229 {
230 die "Duplicate value" if defined($unicode_defaults[$src]);
231 $unicode_defaults[$src] = $dst;
232 }
233 next;
234 }
235 die "Unrecognized line $_\n";
236 }
237
238 # now build mappings from the decomposition field of the Unicode database
239
240 open UNICODEDATA, "$UNICODEDATA" or die "Cannot open $UNICODEDATA";
241 print "Loading $UNICODEDATA\n";
242 while (<UNICODEDATA>)
243 {
244 # Decode the fields ...
245 my ($code, $name, $cat, $comb, $bidi,
246 $decomp, $dec, $dig, $num, $mirror,
247 $oldname, $comment, $upper, $lower, $title) = split /;/;
248 my $dst;
249 my $src = hex $code;
250
251 die "unknown category $cat" unless defined $categories{$cat};
252 die "unknown directionality $bidi" unless defined $directions{$bidi};
253
254 $category_table[$src] = $categories{$cat};
255 $direction_table[$src] = $directions{$bidi};
256
257 if ($lower ne "")
258 {
259 $tolower_table[$src] = hex $lower;
260 $category_table[$src] |= $ctype{"upper"}|$ctype{"alpha"};
261 }
262 if ($upper ne "")
263 {
264 $toupper_table[$src] = hex $upper;
265 $category_table[$src] |= $ctype{"lower"}|$ctype{"alpha"};
266 }
267 if ($dec ne "")
268 {
269 $category_table[$src] |= $ctype{"digit"};
270 }
271 if ($dig ne "")
272 {
273 $digitmap_table[$src] = ord $dig;
274 }
275
276 # copy the category and direction for everything between First/Last pairs
277 if ($name =~ /, First>/) { $start = $src; }
278 if ($name =~ /, Last>/)
279 {
280 while ($start < $src)
281 {
282 $category_table[$start] = $category_table[$src];
283 $direction_table[$start] = $direction_table[$src];
284 $start++;
285 }
286 }
287
288 next if $decomp eq ""; # no decomposition, skip it
289
290 if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
291 {
292 # decomposition of the form "<foo> 1234" -> use char if type is known
293 if (($src >= 0xf900 && $src < 0xfb00) || ($src >= 0xfe30 && $src < 0xfffd))
294 {
295 # Single char decomposition in the compatibility range
296 $compatmap_table[$src] = hex $2;
297 }
298 next unless ($1 eq "font" ||
299 $1 eq "noBreak" ||
300 $1 eq "circle" ||
301 $1 eq "super" ||
302 $1 eq "sub" ||
303 $1 eq "wide" ||
304 $1 eq "narrow" ||
305 $1 eq "compat" ||
306 $1 eq "small");
307 $dst = hex $2;
308 }
309 elsif ($decomp =~ /^<compat>\s+0020\s+([0-9a-fA-F]+)/)
310 {
311 # decomposition "<compat> 0020 1234" -> combining accent
312 $dst = hex $1;
313 }
314 elsif ($decomp =~ /^([0-9a-fA-F]+)/)
315 {
316 # decomposition contains only char values without prefix -> use first char
317 $dst = hex $1;
318 $category_table[$src] |= $category_table[$dst] if defined $category_table[$dst];
319 # store decomposition if it contains two chars
320 if ($decomp =~ /^([0-9a-fA-F]+)\s+([0-9a-fA-F]+)$/)
321 {
322 $decomp_table[$src] = [ hex $1, hex $2 ];
323 push @compose_table, [ hex $1, hex $2, $src ];
324 }
325 elsif ($decomp =~ /^(<[a-z]+>\s)*([0-9a-fA-F]+)$/ &&
326 (($src >= 0xf900 && $src < 0xfb00) || ($src >= 0xfe30 && $src < 0xfffd)))
327 {
328 # Single char decomposition in the compatibility range
329 $compatmap_table[$src] = hex $2;
330 }
331 }
332 else
333 {
334 next;
335 }
336
337 next if defined($unicode_defaults[$src]); # may have been set in the defaults file
338
339 # check for loops
340 for (my $i = $dst; ; $i = $unicode_defaults[$i])
341 {
342 die sprintf("loop detected for %04x -> %04x",$src,$dst) if $i == $src;
343 last unless defined($unicode_defaults[$i]);
344 }
345 $unicode_defaults[$src] = $dst;
346 }
347
348 # patch the category of some special characters
349
350 foreach my $cat (keys %special_categories)
351 {
352 my $flag = $ctype{$cat};
353 foreach my $i (@{$special_categories{$cat}}) { $category_table[$i] |= $flag; }
354 }
355 }
356
357
358 ################################################################
359 # parse the input file
360 sub READ_FILE($)
361 {
362 my $name = shift;
363 open INPUT,$name or die "Cannot open $name";
364
365 while (<INPUT>)
366 {
367 next if /^\#/; # skip comments
368 next if /^$/; # skip empty lines
369 next if /\x1a/; # skip ^Z
370 next if (/^0x([0-9a-fA-F]+)\s+\#UNDEFINED/); # undefined char
371
372 if (/^0x([0-9a-fA-F]+)\s+\#DBCS LEAD BYTE/)
373 {
374 my $cp = hex $1;
375 push @lead_bytes,$cp;
376 $cp2uni[$cp] = 0;
377 next;
378 }
379 if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
380 {
381 my $cp = hex $1;
382 my $uni = hex $2;
383 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
384 $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
385 if ($cp > 0xff && !defined($cp2uni[$cp >> 8]))
386 {
387 push @lead_bytes,$cp >> 8;
388 $cp2uni[$cp >> 8] = 0;
389 }
390 next;
391 }
392 die "$name: Unrecognized line $_\n";
393 }
394 }
395
396
397 ################################################################
398 # fill input data for the 20127 (us-ascii) codepage
399 sub fill_20127_codepage()
400 {
401 for (my $i = 0; $i < 128; $i++) { $cp2uni[$i] = $uni2cp[$i] = $i; }
402 for (my $i = 128; $i < 256; $i++) { $cp2uni[$i] = $i & 0x7f; }
403 }
404
405 ################################################################
406 # get a mapping including glyph chars for MB_USEGLYPHCHARS
407
408 sub get_glyphs_mapping(@)
409 {
410 $_[0x01] = 0x263a; # (WHITE SMILING FACE)
411 $_[0x02] = 0x263b; # (BLACK SMILING FACE)
412 $_[0x03] = 0x2665; # (BLACK HEART SUIT)
413 $_[0x04] = 0x2666; # (BLACK DIAMOND SUIT)
414 $_[0x05] = 0x2663; # (BLACK CLUB SUIT)
415 $_[0x06] = 0x2660; # (BLACK SPADE SUIT)
416 $_[0x07] = 0x2022; # (BULLET)
417 $_[0x08] = 0x25d8; # (INVERSE BULLET)
418 $_[0x09] = 0x25cb; # (WHITE CIRCLE)
419 $_[0x0a] = 0x25d9; # (INVERSE WHITE CIRCLE)
420 $_[0x0b] = 0x2642; # (MALE SIGN)
421 $_[0x0c] = 0x2640; # (FEMALE SIGN)
422 $_[0x0d] = 0x266a; # (EIGHTH NOTE)
423 $_[0x0e] = 0x266b; # (BEAMED EIGHTH NOTES)
424 $_[0x0f] = 0x263c; # (WHITE SUN WITH RAYS)
425 $_[0x10] = 0x25ba; # (BLACK RIGHT-POINTING POINTER)
426 $_[0x11] = 0x25c4; # (BLACK LEFT-POINTING POINTER)
427 $_[0x12] = 0x2195; # (UP DOWN ARROW)
428 $_[0x13] = 0x203c; # (DOUBLE EXCLAMATION MARK)
429 $_[0x14] = 0x00b6; # (PILCROW SIGN)
430 $_[0x15] = 0x00a7; # (SECTION SIGN)
431 $_[0x16] = 0x25ac; # (BLACK RECTANGLE)
432 $_[0x17] = 0x21a8; # (UP DOWN ARROW WITH BASE)
433 $_[0x18] = 0x2191; # (UPWARDS ARROW)
434 $_[0x19] = 0x2193; # (DOWNWARDS ARROW)
435 $_[0x1a] = 0x2192; # (RIGHTWARDS ARROW)
436 $_[0x1b] = 0x2190; # (LEFTWARDS ARROW)
437 $_[0x1c] = 0x221f; # (RIGHT ANGLE)
438 $_[0x1d] = 0x2194; # (LEFT RIGHT ARROW)
439 $_[0x1e] = 0x25b2; # (BLACK UP-POINTING TRIANGLE)
440 $_[0x1f] = 0x25bc; # (BLACK DOWN-POINTING TRIANGLE)
441 $_[0x7f] = 0x2302; # (HOUSE)
442 return @_;
443 }
444
445 ################################################################
446 # build EUC-JP table from the JIS 0208 file
447 # FIXME: for proper EUC-JP we should probably read JIS 0212 too
448 # but this would require 3-byte DBCS characters
449 sub READ_JIS0208_FILE($)
450 {
451 my $name = shift;
452
453 # ASCII chars
454 for (my $i = 0x00; $i <= 0x7f; $i++)
455 {
456 $cp2uni[$i] = $i;
457 $uni2cp[$i] = $i;
458 }
459
460 # JIS X 0201 right plane
461 for (my $i = 0xa1; $i <= 0xdf; $i++)
462 {
463 $cp2uni[0x8e00 + $i] = 0xfec0 + $i;
464 $uni2cp[0xfec0 + $i] = 0x8e00 + $i;
465 }
466
467 # lead bytes
468 foreach my $i (0x8e, 0x8f, 0xa1 .. 0xfe)
469 {
470 push @lead_bytes,$i;
471 $cp2uni[$i] = 0;
472 }
473
474 # undefined chars
475 foreach my $i (0x80 .. 0x8d, 0x90 .. 0xa0, 0xff)
476 {
477 $cp2uni[$i] = $DEF_CHAR;
478 }
479
480 # Shift-JIS compatibility
481 $uni2cp[0x00a5] = 0x5c;
482 $uni2cp[0x203e] = 0x7e;
483
484 # Fix backslash conversion
485 $cp2uni[0xa1c0] = 0xff3c;
486 $uni2cp[0xff3c] = 0xa1c0;
487
488 open INPUT, "$name" or die "Cannot open $name";
489 while (<INPUT>)
490 {
491 next if /^\#/; # skip comments
492 next if /^$/; # skip empty lines
493 next if /\x1a/; # skip ^Z
494 if (/^0x[0-9a-fA-F]+\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
495 {
496 my $cp = 0x8080 + hex $1;
497 my $uni = hex $2;
498 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
499 $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
500 next;
501 }
502 die "$name: Unrecognized line $_\n";
503 }
504 }
505
506
507 ################################################################
508 # build the sort keys table
509 sub READ_SORTKEYS_FILE()
510 {
511 my @sortkeys = ();
512 for (my $i = 0; $i < 65536; $i++) { $sortkeys[$i] = [ -1, 0, 0, 0, 0 ] };
513
514 open INPUT, "$SORTKEYS" or die "Cannot open $SORTKEYS";
515 print "Loading $SORTKEYS\n";
516 while (<INPUT>)
517 {
518 next if /^\#/; # skip comments
519 next if /^$/; # skip empty lines
520 next if /\x1a/; # skip ^Z
521 next if /^\@version/; # skip @version header
522 if (/^([0-9a-fA-F]+)\s+;\s+\[([*.])([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]+)\]/)
523 {
524 my ($uni,$variable) = (hex $1, $2);
525 next if $uni > 65535;
526 $sortkeys[$uni] = [ $uni, hex $3, hex $4, hex $5, hex $6 ];
527 next;
528 }
529 if (/^([0-9a-fA-F]+\s+)+;\s+\[[*.]([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]+)\]/)
530 {
531 # multiple character sequence, ignored for now
532 next;
533 }
534 die "$SORTKEYS: Unrecognized line $_\n";
535 }
536 close INPUT;
537
538 # compress the keys to 32 bit:
539 # key 1 to 16 bits, key 2 to 8 bits, key 3 to 4 bits, key 4 to 1 bit
540
541 @sortkeys = sort { ${$a}[1] <=> ${$b}[1] or
542 ${$a}[2] <=> ${$b}[2] or
543 ${$a}[3] <=> ${$b}[3] or
544 ${$a}[4] <=> ${$b}[4] or
545 $a cmp $b; } @sortkeys;
546
547 my ($n2, $n3) = (1, 1);
548 my @keys = (-1, -1, -1, -1, -1 );
549 my @flatkeys = ();
550
551 for (my $i = 0; $i < 65536; $i++)
552 {
553 my @current = @{$sortkeys[$i]};
554 next if $current[0] == -1;
555 if ($current[1] == $keys[1])
556 {
557 if ($current[2] == $keys[2])
558 {
559 if ($current[3] == $keys[3])
560 {
561 # nothing
562 }
563 else
564 {
565 $keys[3] = $current[3];
566 $n3++;
567 die if ($n3 >= 16);
568 }
569 }
570 else
571 {
572 $keys[2] = $current[2];
573 $keys[3] = $current[3];
574 $n2++;
575 $n3 = 1;
576 die if ($n2 >= 256);
577 }
578 }
579 else
580 {
581 $keys[1] = $current[1];
582 $keys[2] = $current[2];
583 $keys[3] = $current[3];
584 $n2 = 1;
585 $n3 = 1;
586 }
587
588 if ($current[2]) { $current[2] = $n2; }
589 if ($current[3]) { $current[3] = $n3; }
590 if ($current[4]) { $current[4] = 1; }
591
592 $flatkeys[$current[0]] = ($current[1] << 16) | ($current[2] << 8) | ($current[3] << 4) | $current[4];
593 }
594 return @flatkeys;
595 }
596
597
598 ################################################################
599 # build the sort keys table
600 sub DUMP_SORTKEYS($@)
601 {
602 my ($filename, @keys) = @_;
603
604 # count the number of 256-key ranges that contain something
605
606 my @offsets = ();
607 my $ranges = 2;
608 for (my $i = 0; $i < 256; $i++) { $offsets[$i] = 256; }
609 for (my $i = 0; $i < 65536; $i++)
610 {
611 next unless defined $keys[$i];
612 $offsets[$i >> 8] = $ranges * 256;
613 $ranges++;
614 $i |= 255;
615 }
616
617 # output the range offsets
618
619 open OUTPUT,">$filename.new" or die "Cannot create $filename";
620 printf "Building $filename\n";
621 printf OUTPUT "/* Unicode collation element table */\n";
622 printf OUTPUT "/* generated from %s */\n", $SORTKEYS;
623 printf OUTPUT "/* DO NOT EDIT!! */\n\n";
624
625 printf OUTPUT "const unsigned int collation_table[%d] =\n{\n", $ranges*256;
626 printf OUTPUT " /* index */\n";
627 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%08x", 0, @offsets );
628
629 # output the default values
630
631 printf OUTPUT " /* defaults */\n";
632 printf OUTPUT "%s", DUMP_ARRAY( "0x%08x", 0, (0xffffffff) x 256 );
633
634 # output all the key ranges
635
636 for (my $i = 0; $i < 256; $i++)
637 {
638 next if $offsets[$i] == 256;
639 printf OUTPUT ",\n /* 0x%02x00 .. 0x%02xff */\n", $i, $i;
640 printf OUTPUT "%s", DUMP_ARRAY( "0x%08x", 0xffffffff, @keys[($i<<8) .. ($i<<8)+255] );
641 }
642 printf OUTPUT "\n};\n";
643 close OUTPUT;
644 save_file($filename);
645 }
646
647
648 ################################################################
649 # add default mappings once the file had been read
650 sub ADD_DEFAULT_MAPPINGS()
651 {
652 # Apply aliases
653
654 foreach my $alias (@unicode_aliases)
655 {
656 my $target = undef;
657 foreach my $src (@$alias)
658 {
659 if (defined($uni2cp[$src]))
660 {
661 $target = $uni2cp[$src];
662 last;
663 }
664 }
665 next unless defined($target);
666
667 # At least one char of the alias set is defined, set the others to the same value
668 foreach my $src (@$alias)
669 {
670 $uni2cp[$src] = $target unless defined($uni2cp[$src]);
671 }
672 }
673
674 # For every src -> target mapping in the defaults table,
675 # make uni2cp[src] = uni2cp[target] if uni2cp[target] is defined
676
677 for (my $src = 0; $src < 65536; $src++)
678 {
679 next if defined($uni2cp[$src]); # source has a definition already
680 next unless defined($unicode_defaults[$src]); # no default for this char
681 my $target = $unicode_defaults[$src];
682
683 # do a recursive mapping until we find a target char that is defined
684 while (!defined($uni2cp[$target]) &&
685 defined($unicode_defaults[$target])) { $target = $unicode_defaults[$target]; }
686
687 if (defined($uni2cp[$target])) { $uni2cp[$src] = $uni2cp[$target]; }
688 }
689
690 # Add an identity mapping for all undefined chars
691
692 for (my $i = 0; $i < 256; $i++)
693 {
694 next if defined($cp2uni[$i]);
695 next if defined($uni2cp[$i]);
696 $cp2uni[$i] = $uni2cp[$i] = $i;
697 }
698 }
699
700 ################################################################
701 # dump an array of integers
702 sub DUMP_ARRAY($$@)
703 {
704 my ($format,$default,@array) = @_;
705 my $i;
706 my $ret = " ";
707 for ($i = 0; $i < $#array; $i++)
708 {
709 $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
710 $ret .= (($i % 8) != 7) ? ", " : ",\n ";
711 }
712 $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
713 return $ret;
714 }
715
716 ################################################################
717 # dump an SBCS mapping table
718 sub dump_sbcs_table($$$$$)
719 {
720 my ($codepage, $has_glyphs, $name, $def, $defw) = @_;
721 my $i;
722
723 # output the ascii->unicode table
724
725 if ($has_glyphs)
726 {
727 printf OUTPUT "static const WCHAR cp2uni[512] =\n";
728 printf OUTPUT "{\n%s", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[0 .. 255] );
729 printf OUTPUT ",\n /* glyphs */\n%s\n};\n\n",
730 DUMP_ARRAY( "0x%04x", $defw, get_glyphs_mapping(@cp2uni[0 .. 255]) );
731 }
732 else
733 {
734 printf OUTPUT "static const WCHAR cp2uni[256] =\n";
735 printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[0 .. 255] );
736 }
737
738 # count the number of unicode->ascii subtables that contain something
739
740 my @filled = ();
741 my $subtables = 1;
742 for (my $i = 0; $i < 65536; $i++)
743 {
744 next unless defined $uni2cp[$i];
745 $filled[$i >> 8] = 1;
746 $subtables++;
747 $i |= 255;
748 }
749
750 # output all the subtables into a single array
751
752 printf OUTPUT "static const unsigned char uni2cp_low[%d] =\n{\n", $subtables*256;
753 for (my $i = 0; $i < 256; $i++)
754 {
755 next unless $filled[$i];
756 printf OUTPUT " /* 0x%02x00 .. 0x%02xff */\n", $i, $i;
757 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%02x", $def, @uni2cp[($i<<8) .. ($i<<8)+255] );
758 }
759 printf OUTPUT " /* defaults */\n";
760 printf OUTPUT "%s\n};\n\n", DUMP_ARRAY( "0x%02x", 0, ($def) x 256 );
761
762 # output a table of the offsets of the subtables in the previous array
763
764 my $pos = 0;
765 my @offsets = ();
766 for (my $i = 0; $i < 256; $i++)
767 {
768 if ($filled[$i]) { push @offsets, $pos; $pos += 256; }
769 else { push @offsets, ($subtables-1) * 256; }
770 }
771 printf OUTPUT "static const unsigned short uni2cp_high[256] =\n";
772 printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", 0, @offsets );
773
774 # output the code page descriptor
775
776 printf OUTPUT "const struct sbcs_table cptable_%03d =\n{\n", $codepage;
777 printf OUTPUT " { %d, 1, 0x%04x, 0x%04x, \"%s\" },\n",
778 $codepage, $def, $defw, $name;
779 printf OUTPUT " cp2uni,\n";
780 if ($has_glyphs) { printf OUTPUT " cp2uni + 256,\n"; }
781 else { printf OUTPUT " cp2uni,\n"; }
782 printf OUTPUT " uni2cp_low,\n";
783 printf OUTPUT " uni2cp_high\n};\n";
784 }
785
786
787 ################################################################
788 # dump a DBCS mapping table
789 sub dump_dbcs_table($$$$@)
790 {
791 my ($codepage, $name, $def, $defw, @lb_ranges) = @_;
792
793 # build a list of lead bytes that are actually used
794
795 my @lblist = ();
796 LBLOOP: for (my $y = 0; $y <= $#lead_bytes; $y++)
797 {
798 my $base = $lead_bytes[$y] << 8;
799 for (my $x = 0; $x < 256; $x++)
800 {
801 if (defined $cp2uni[$base+$x])
802 {
803 push @lblist,$lead_bytes[$y];
804 next LBLOOP;
805 }
806 }
807 }
808 my $unused = ($#lead_bytes > $#lblist);
809
810 # output the ascii->unicode table for the single byte chars
811
812 printf OUTPUT "static const WCHAR cp2uni[%d] =\n", 256 * ($#lblist + 2 + $unused);
813 printf OUTPUT "{\n%s,\n", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[0 .. 255] );
814
815 # output the default table for unused lead bytes
816
817 if ($unused)
818 {
819 printf OUTPUT " /* unused lead bytes */\n";
820 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", 0, ($defw) x 256 );
821 }
822
823 # output the ascii->unicode table for each DBCS lead byte
824
825 for (my $y = 0; $y <= $#lblist; $y++)
826 {
827 my $base = $lblist[$y] << 8;
828 printf OUTPUT " /* lead byte %02x */\n", $lblist[$y];
829 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", $defw, @cp2uni[$base .. $base+255] );
830 printf OUTPUT ($y < $#lblist) ? ",\n" : "\n};\n\n";
831 }
832
833 # output the lead byte subtables offsets
834
835 my @offsets = ();
836 for (my $x = 0; $x < 256; $x++) { $offsets[$x] = 0; }
837 for (my $x = 0; $x <= $#lblist; $x++) { $offsets[$lblist[$x]] = $x + 1; }
838 if ($unused)
839 {
840 # increment all lead bytes offset to take into account the unused table
841 for (my $x = 0; $x <= $#lead_bytes; $x++) { $offsets[$lead_bytes[$x]]++; }
842 }
843 printf OUTPUT "static const unsigned char cp2uni_leadbytes[256] =\n";
844 printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%02x", 0, @offsets );
845
846 # count the number of unicode->ascii subtables that contain something
847
848 my @filled = ();
849 my $subtables = 1;
850 for (my $i = 0; $i < 65536; $i++)
851 {
852 next unless defined $uni2cp[$i];
853 $filled[$i >> 8] = 1;
854 $subtables++;
855 $i |= 255;
856 }
857
858 # output all the subtables into a single array
859
860 printf OUTPUT "static const unsigned short uni2cp_low[%d] =\n{\n", $subtables*256;
861 for (my $y = 0; $y < 256; $y++)
862 {
863 next unless $filled[$y];
864 printf OUTPUT " /* 0x%02x00 .. 0x%02xff */\n", $y, $y;
865 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", $def, @uni2cp[($y<<8) .. ($y<<8)+255] );
866 }
867 printf OUTPUT " /* defaults */\n";
868 printf OUTPUT "%s\n};\n\n", DUMP_ARRAY( "0x%04x", 0, ($def) x 256 );
869
870 # output a table of the offsets of the subtables in the previous array
871
872 my $pos = 0;
873 @offsets = ();
874 for (my $y = 0; $y < 256; $y++)
875 {
876 if ($filled[$y]) { push @offsets, $pos; $pos += 256; }
877 else { push @offsets, ($subtables-1) * 256; }
878 }
879 printf OUTPUT "static const unsigned short uni2cp_high[256] =\n";
880 printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", 0, @offsets );
881
882 # output the code page descriptor
883
884 printf OUTPUT "const struct dbcs_table cptable_%03d =\n{\n", $codepage;
885 printf OUTPUT " { %d, 2, 0x%04x, 0x%04x, \"%s\" },\n",
886 $codepage, $def, $defw, $name;
887 printf OUTPUT " cp2uni,\n";
888 printf OUTPUT " cp2uni_leadbytes,\n";
889 printf OUTPUT " uni2cp_low,\n";
890 printf OUTPUT " uni2cp_high,\n";
891 printf OUTPUT " {\n %s\n }\n", DUMP_ARRAY( "0x%02x", 0, @lb_ranges, 0, 0 );
892 printf OUTPUT "};\n";
893 }
894
895
896 ################################################################
897 # get the list of defined lead byte ranges
898 sub get_lb_ranges()
899 {
900 my @list = ();
901 my @ranges = ();
902 my $i = 0;
903 foreach $i (@lead_bytes) { $list[$i] = 1; }
904 my $on = 0;
905 for (my $i = 0; $i < 256; $i++)
906 {
907 if ($on)
908 {
909 if (!defined $list[$i]) { push @ranges, $i-1; $on = 0; }
910 }
911 else
912 {
913 if ($list[$i]) { push @ranges, $i; $on = 1; }
914 }
915 }
916 if ($on) { push @ranges, 0xff; }
917 return @ranges;
918 }
919
920
921 ################################################################
922 # dump the case mapping tables
923 sub DUMP_CASE_MAPPINGS($)
924 {
925 my $filename = shift;
926 open OUTPUT,">$filename.new" or die "Cannot create $filename";
927 printf "Building $filename\n";
928 printf OUTPUT "/* Unicode case mappings */\n";
929 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
930 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
931
932 DUMP_CASE_TABLE( "wine_casemap_lower", @tolower_table );
933 DUMP_CASE_TABLE( "wine_casemap_upper", @toupper_table );
934 DUMP_CASE_TABLE( "wine_digitmap", @digitmap_table );
935 DUMP_CASE_TABLE( "wine_compatmap", @compatmap_table );
936 close OUTPUT;
937 save_file($filename);
938 }
939
940
941 ################################################################
942 # dump a case mapping table
943 sub DUMP_CASE_TABLE($@)
944 {
945 my ($name,@table) = @_;
946
947 # count the number of sub tables that contain something
948 # also compute the low and upper populated bounds
949
950 my @lowerbounds = ( 0, 0 );
951 my @upperbounds = ( 0, 255 );
952 my $index = 0;
953 my @filled = ();
954 for (my $i = 0; $i < 65536; $i++)
955 {
956 next unless defined $table[$i];
957 if (!defined $filled[$i >> 8])
958 {
959 $lowerbounds[$index] = $i & 0xff;
960 $upperbounds[$index] = 0xff - $lowerbounds[$index];
961 $filled[$i >> 8] = $index * 256 + 512;
962 $index++;
963 }
964 else
965 {
966 $upperbounds[$index-1] = 0xff - ($i & 0xff);
967 }
968 $table[$i] = ($table[$i] - $i) & 0xffff;
969 }
970
971 # Collapse blocks upwards if possible
972 my $removed = 0;
973 $index = 0;
974 for (my $i = 0; $i < 256; $i++)
975 {
976 next unless defined $filled[$i];
977 if ($upperbounds[$index - 1] > $lowerbounds[$index])
978 {
979 $removed = $removed + $lowerbounds[$index];
980 }
981 else
982 {
983 $removed = $removed + $upperbounds[$index - 1];
984 $lowerbounds[$index] = $upperbounds[$index - 1];
985 }
986 $filled[$i] = $filled[$i] - $removed;
987 $index++;
988 }
989
990 # dump the table
991
992 printf OUTPUT "const WCHAR %s[%d] =\n", $name, $index * 256 + 512 - $removed;
993 printf OUTPUT "{\n /* index */\n";
994 printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", 256, @filled );
995 printf OUTPUT " /* defaults */\n";
996 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, (0) x 256 );
997 $index = 0;
998 for (my $i = 0; $i < 256; $i++)
999 {
1000 next unless $filled[$i];
1001 printf OUTPUT ",\n /* 0x%02x%02x .. 0x%02xff */\n", $i, $lowerbounds[$index], $i;
1002 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0,
1003 @table[($i<<8) + $lowerbounds[$index] .. ($i<<8)+255] );
1004 $index++;
1005 }
1006 printf OUTPUT "\n};\n";
1007 }
1008
1009
1010 ################################################################
1011 # dump the ctype tables
1012 sub DUMP_CTYPE_TABLES($)
1013 {
1014 my $filename = shift;
1015 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1016 printf "Building $filename\n";
1017 printf OUTPUT "/* Unicode ctype tables */\n";
1018 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
1019 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1020
1021 my @array = (0) x 256;
1022 my %sequences;
1023
1024 # add the direction in the high 4 bits of the category
1025 for (my $i = 0; $i < 65536; $i++)
1026 {
1027 $category_table[$i] |= $direction_table[$i] << 12 if defined $direction_table[$i];
1028 }
1029
1030 # try to merge table rows
1031 for (my $row = 0; $row < 256; $row++)
1032 {
1033 my $rowtxt = sprintf "%04x" x 256, @category_table[($row<<8)..($row<<8)+255];
1034 if (defined($sequences{$rowtxt}))
1035 {
1036 # reuse an existing row
1037 $array[$row] = $sequences{$rowtxt};
1038 }
1039 else
1040 {
1041 # create a new row
1042 $sequences{$rowtxt} = $array[$row] = $#array + 1;
1043 push @array, @category_table[($row<<8)..($row<<8)+255];
1044 }
1045 }
1046
1047 printf OUTPUT "const unsigned short wine_wctype_table[%d] =\n{\n", $#array+1;
1048 printf OUTPUT " /* offsets */\n%s,\n", DUMP_ARRAY( "0x%04x", 0, @array[0..255] );
1049 printf OUTPUT " /* values */\n%s\n};\n", DUMP_ARRAY( "0x%04x", 0, @array[256..$#array] );
1050
1051 close OUTPUT;
1052 save_file($filename);
1053 }
1054
1055
1056 ################################################################
1057 # dump the char composition tables
1058 sub DUMP_COMPOSE_TABLES($)
1059 {
1060 my $filename = shift;
1061
1062 open OUTPUT,">$filename.new" or die "Cannot create $filename";
1063 printf "Building $filename\n";
1064 printf OUTPUT "/* Unicode char composition */\n";
1065 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
1066 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1067
1068 ######### composition table
1069
1070 my @filled = ();
1071 foreach my $i (@compose_table)
1072 {
1073 my @comp = @$i;
1074 push @{$filled[$comp[1]]}, [ $comp[0], $comp[2] ];
1075 }
1076
1077 # count how many different second chars we have
1078
1079 my $count = 0;
1080 for (my $i = 0; $i < 65536; $i++)
1081 {
1082 next unless defined $filled[$i];
1083 $count++;
1084 }
1085
1086 # build the table of second chars and offsets
1087
1088 my $pos = $count + 1;
1089 my @table = ();
1090 for (my $i = 0; $i < 65536; $i++)
1091 {
1092 next unless defined $filled[$i];
1093 push @table, $i, $pos;
1094 $pos += @{$filled[$i]};
1095 }
1096 # terminator with last position
1097 push @table, 0, $pos;
1098 printf OUTPUT "const WCHAR unicode_compose_table[0x%x] =\n{\n", 2*$pos;
1099 printf OUTPUT " /* second chars + offsets */\n%s", DUMP_ARRAY( "0x%04x", 0, @table );
1100
1101 # build the table of first chars and mappings
1102
1103 for (my $i = 0; $i < 65536; $i++)
1104 {
1105 next unless defined $filled[$i];
1106 my @table = ();
1107 my @list = sort { $a->[0] <=> $b->[0] } @{$filled[$i]};
1108 for (my $j = 0; $j <= $#list; $j++)
1109 {
1110 push @table, $list[$j][0], $list[$j][1];
1111 }
1112 printf OUTPUT ",\n /* 0x%04x */\n%s", $i, DUMP_ARRAY( "0x%04x", 0, @table );
1113 }
1114 printf OUTPUT "\n};\n\nconst unsigned int unicode_compose_table_size = %d;\n\n", $count;
1115
1116 ######### decomposition table
1117
1118 # first determine all the 16-char subsets that contain something
1119
1120 @filled = (0) x 4096;
1121 $pos = 16*2; # for the null subset
1122 for (my $i = 0; $i < 65536; $i++)
1123 {
1124 next unless defined $decomp_table[$i];
1125 $filled[$i >> 4] = $pos;
1126 $pos += 16*2;
1127 $i |= 15;
1128 }
1129 my $total = $pos;
1130
1131 # now count the 256-char subsets that contain something
1132
1133 my @filled_idx = (256) x 256;
1134 $pos = 256 + 16;
1135 for (my $i = 0; $i < 4096; $i++)
1136 {
1137 next unless $filled[$i];
1138 $filled_idx[$i >> 4] = $pos;
1139 $pos += 16;
1140 $i |= 15;
1141 }
1142 my $null_offset = $pos; # null mapping
1143 $total += $pos;
1144
1145 # add the index offsets to the subsets positions
1146
1147 for (my $i = 0; $i < 4096; $i++)
1148 {
1149 next unless $filled[$i];
1150 $filled[$i] += $null_offset;
1151 }
1152
1153 # dump the main index
1154
1155 printf OUTPUT "const WCHAR unicode_decompose_table[%d] =\n", $total;
1156 printf OUTPUT "{\n /* index */\n";
1157 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @filled_idx );
1158 printf OUTPUT ",\n /* null sub-index */\n%s", DUMP_ARRAY( "0x%04x", 0, ($null_offset) x 16 );
1159
1160 # dump the second-level indexes
1161
1162 for (my $i = 0; $i < 256; $i++)
1163 {
1164 next unless ($filled_idx[$i] > 256);
1165 my @table = @filled[($i<<4)..($i<<4)+15];
1166 for (my $j = 0; $j < 16; $j++) { $table[$j] ||= $null_offset; }
1167 printf OUTPUT ",\n /* sub-index %02x */\n", $i;
1168 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
1169 }
1170
1171 # dump the 16-char subsets
1172
1173 printf OUTPUT ",\n /* null mapping */\n";
1174 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, (0) x 32 );
1175
1176 for (my $i = 0; $i < 4096; $i++)
1177 {
1178 next unless $filled[$i];
1179 my @table = (0) x 32;
1180 for (my $j = 0; $j < 16; $j++)
1181 {
1182 if (defined $decomp_table[($i<<4) + $j])
1183 {
1184 $table[2 * $j] = ${$decomp_table[($i << 4) + $j]}[0];
1185 $table[2 * $j + 1] = ${$decomp_table[($i << 4) + $j]}[1];
1186 }
1187 }
1188 printf OUTPUT ",\n /* 0x%03x0 .. 0x%03xf */\n", $i, $i;
1189 printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table );
1190 }
1191
1192 printf OUTPUT "\n};\n";
1193 close OUTPUT;
1194 save_file($filename);
1195 }
1196
1197
1198 ################################################################
1199 # handle a "bestfit" Windows mapping file
1200
1201 sub handle_bestfit_file($$$)
1202 {
1203 my ($filename, $has_glyphs, $comment) = @_;
1204 my $state = "";
1205 my ($codepage, $width, $def, $defw, $count);
1206 my ($lb_cur, $lb_end);
1207 my @lb_ranges = ();
1208
1209 open INPUT,$MAPPREFIX . $filename or die "Cannot open $filename";
1210
1211 while (<INPUT>)
1212 {
1213 next if /^;/; # skip comments
1214 next if /^\s*$/; # skip empty lines
1215 next if /\x1a/; # skip ^Z
1216 last if /^ENDCODEPAGE/;
1217
1218 if (/^CODEPAGE\s+(\d+)/)
1219 {
1220 $codepage = $1;
1221 next;
1222 }
1223 if (/^CPINFO\s+(\d+)\s+0x([0-9a-fA-f]+)\s+0x([0-9a-fA-F]+)/)
1224 {
1225 $width = $1;
1226 $def = hex $2;
1227 $defw = hex $3;
1228 next;
1229 }
1230 if (/^(MBTABLE|WCTABLE|DBCSRANGE|DBCSTABLE)\s+(\d+)/)
1231 {
1232 $state = $1;
1233 $count = $2;
1234 next;
1235 }
1236 if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)/)
1237 {
1238 if ($state eq "MBTABLE")
1239 {
1240 my $cp = hex $1;
1241 my $uni = hex $2;
1242 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
1243 next;
1244 }
1245 if ($state eq "WCTABLE")
1246 {
1247 my $uni = hex $1;
1248 my $cp = hex $2;
1249 $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
1250 next;
1251 }
1252 if ($state eq "DBCSRANGE")
1253 {
1254 my $start = hex $1;
1255 my $end = hex $2;
1256 push @lb_ranges, $start, $end;
1257 for (my $i = $start; $i <= $end; $i++)
1258 {
1259 push @lead_bytes, $i;
1260 $cp2uni[$i] = 0;
1261 }
1262 $lb_cur = $start;
1263 $lb_end = $end;
1264 next;
1265 }
1266 if ($state eq "DBCSTABLE")
1267 {
1268 my $mb = hex $1;
1269 my $uni = hex $2;
1270 my $cp = ($lb_cur << 8) | $mb;
1271 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
1272 if (!--$count)
1273 {
1274 if (++$lb_cur > $lb_end) { $state = "DBCSRANGE"; }
1275 }
1276 next;
1277 }
1278 }
1279 die "$filename: Unrecognized line $_\n";
1280 }
1281 close INPUT;
1282
1283 my $output = sprintf "c_%03d.c", $codepage;
1284 open OUTPUT,">$output.new" or die "Cannot create $output";
1285
1286 printf "Building %s from %s (%s)\n", $output, $filename, $comment;
1287
1288 # dump all tables
1289
1290 printf OUTPUT "/* code page %03d (%s) */\n", $codepage, $comment;
1291 printf OUTPUT "/* generated from %s */\n", $MAPPREFIX . $filename;
1292 printf OUTPUT "/* DO NOT EDIT!! */\n\n";
1293 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1294
1295 if ($width == 1) { dump_sbcs_table( $codepage, $has_glyphs, $comment, $def, $defw ); }
1296 else { dump_dbcs_table( $codepage, $comment, $def, $defw, @lb_ranges ); }
1297 close OUTPUT;
1298 save_file($output);
1299 }
1300
1301
1302 ################################################################
1303 # read an input file and generate the corresponding .c file
1304 sub HANDLE_FILE(@)
1305 {
1306 my ($codepage,$filename,$has_glyphs,$comment) = @_;
1307
1308 @cp2uni = ();
1309 @lead_bytes = ();
1310 @uni2cp = ();
1311
1312 # symbol codepage file is special
1313 if ($codepage == 20932) { READ_JIS0208_FILE($MAPPREFIX . $filename); }
1314 elsif ($codepage == 20127) { fill_20127_codepage(); }
1315 elsif ($filename =~ /\/bestfit/)
1316 {
1317 handle_bestfit_file( $filename, $has_glyphs, $comment );
1318 return;
1319 }
1320 else { READ_FILE($MAPPREFIX . $filename); }
1321
1322 ADD_DEFAULT_MAPPINGS();
1323
1324 my $output = sprintf "c_%03d.c", $codepage;
1325 open OUTPUT,">$output.new" or die "Cannot create $output";
1326
1327 printf "Building %s from %s (%s)\n", $output, $filename || "hardcoded data", $comment;
1328
1329 # dump all tables
1330
1331 printf OUTPUT "/* code page %03d (%s) */\n", $codepage, $comment;
1332 if ($filename)
1333 {
1334 printf OUTPUT "/* generated from %s */\n", $MAPPREFIX . $filename;
1335 printf OUTPUT "/* DO NOT EDIT!! */\n\n";
1336 }
1337 else
1338 {
1339 printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
1340 }
1341 printf OUTPUT "#include \"wine/unicode.h\"\n\n";
1342
1343 if (!@lead_bytes) { dump_sbcs_table( $codepage, $has_glyphs, $comment, $DEF_CHAR, $DEF_CHAR ); }
1344 else { dump_dbcs_table( $codepage, $comment, $DEF_CHAR, $DEF_CHAR, get_lb_ranges() ); }
1345 close OUTPUT;
1346 save_file($output);
1347 }
1348
1349
1350 ################################################################
1351 # save a file if modified
1352 sub save_file($)
1353 {
1354 my $file = shift;
1355 if (-f $file && !system "cmp $file $file.new >/dev/null")
1356 {
1357 unlink "$file.new";
1358 }
1359 else
1360 {
1361 rename "$file.new", "$file";
1362 }
1363 }
1364
1365
1366 ################################################################
1367 # output the list of codepage tables into the cptable.c file
1368 sub OUTPUT_CPTABLE()
1369 {
1370 my @tables_decl = ();
1371
1372 foreach my $file (@allfiles)
1373 {
1374 my ($codepage,$filename,$comment) = @$file;
1375 push @tables_decl, sprintf("extern union cptable cptable_%03d;\n",$codepage);
1376 }
1377
1378 push @tables_decl, sprintf("\nstatic const union cptable * const cptables[%d] =\n{\n",$#allfiles+1);
1379 foreach my $file (@allfiles)
1380 {
1381 my ($codepage,$filename,$comment) = @$file;
1382 push @tables_decl, sprintf(" &cptable_%03d,\n", $codepage);
1383 }
1384 push @tables_decl, "};";
1385 REPLACE_IN_FILE( "cptable.c", @tables_decl );
1386 }
1387
1388 ################################################################
1389 # replace the contents of a file between ### cpmap ### marks
1390
1391 sub REPLACE_IN_FILE($@)
1392 {
1393 my $name = shift;
1394 my @data = @_;
1395 my @lines = ();
1396 open(FILE,$name) or die "Can't open $name";
1397 while (<FILE>)
1398 {
1399 push @lines, $_;
1400 last if /\#\#\# cpmap begin \#\#\#/;
1401 }
1402 push @lines, @data;
1403 while (<FILE>)
1404 {
1405 if (/\#\#\# cpmap end \#\#\#/) { push @lines, "\n", $_; last; }
1406 }
1407 push @lines, <FILE>;
1408 open(FILE,">$name.new") or die "Can't modify $name";
1409 print FILE @lines;
1410 close(FILE);
1411 save_file($name);
1412 }
1413
1414 ################################################################
1415 # main routine
1416
1417 READ_DEFAULTS( $DEFAULTS );
1418 DUMP_CASE_MAPPINGS( "casemap.c" );
1419 DUMP_SORTKEYS( "collation.c", READ_SORTKEYS_FILE() );
1420 DUMP_COMPOSE_TABLES( "compose.c" );
1421 DUMP_CTYPE_TABLES( "wctype.c" );
1422
1423 foreach my $file (@allfiles) { HANDLE_FILE( @{$file} ); }
1424
1425 OUTPUT_CPTABLE();
1426
1427 exit 0;
1428
1429 # Local Variables:
1430 # compile-command: "./cpmap.pl && make -k"
1431 # End:
useful hacks and other patches not (yet) suitable for mainline