unicode/cpmap.pl

   1 #!/usr/bin/perl
   2 #
   3 # Generate code page .c files from ftp.unicode.org descriptions
   4 #
   5 # Copyright 2000 Alexandre Julliard
   6 #
   7
   8 # base directory for ftp.unicode.org files
   9 $BASEDIR = "ftp.unicode.org/Public/";
  10 $MAPPREFIX = $BASEDIR . "MAPPINGS/";
  11
  12 # UnicodeData file
  13 $UNICODEDATA = $BASEDIR . "UNIDATA/UnicodeData.txt";
  14
  15 # Defaults mapping
  16 $DEFAULTS = "./defaults";
  17
  18 # Default char for undefined mappings
  19 $DEF_CHAR = ord '?';
  20
  21 @allfiles =
  22 (
  23     [ 37,    "VENDORS/MICSFT/EBCDIC/CP037.TXT",   "IBM EBCDIC US Canada" ],
  24     [ 42,    "VENDORS/ADOBE/symbol.txt",          "Symbol" ],
  25     [ 424,   "VENDORS/MISC/CP424.TXT",            "IBM EBCDIC Hebrew" ],
  26     [ 437,   "VENDORS/MICSFT/PC/CP437.TXT",       "OEM United States" ],
  27     [ 500,   "VENDORS/MICSFT/EBCDIC/CP500.TXT",   "IBM EBCDIC International" ],
  28     [ 737,   "VENDORS/MICSFT/PC/CP737.TXT",       "OEM Greek 437G" ],
  29     [ 775,   "VENDORS/MICSFT/PC/CP775.TXT",       "OEM Baltic" ],
  30     [ 850,   "VENDORS/MICSFT/PC/CP850.TXT",       "OEM Multilingual Latin 1" ],
  31     [ 852,   "VENDORS/MICSFT/PC/CP852.TXT",       "OEM Slovak Latin 2" ],
  32     [ 855,   "VENDORS/MICSFT/PC/CP855.TXT",       "OEM Cyrillic" ],
  33     [ 856,   "VENDORS/MISC/CP856.TXT",            "Hebrew PC" ],
  34     [ 857,   "VENDORS/MICSFT/PC/CP857.TXT",       "OEM Turkish" ],
  35     [ 860,   "VENDORS/MICSFT/PC/CP860.TXT",       "OEM Portuguese" ],
  36     [ 861,   "VENDORS/MICSFT/PC/CP861.TXT",       "OEM Icelandic" ],
  37     [ 862,   "VENDORS/MICSFT/PC/CP862.TXT",       "OEM Hebrew" ],
  38     [ 863,   "VENDORS/MICSFT/PC/CP863.TXT",       "OEM Canadian French" ],
  39     [ 864,   "VENDORS/MICSFT/PC/CP864.TXT",       "OEM Arabic" ],
  40     [ 865,   "VENDORS/MICSFT/PC/CP865.TXT",       "OEM Nordic" ],
  41     [ 866,   "VENDORS/MICSFT/PC/CP866.TXT",       "OEM Russian" ],
  42     [ 869,   "VENDORS/MICSFT/PC/CP869.TXT",       "OEM Greek" ],
  43     [ 874,   "VENDORS/MICSFT/PC/CP874.TXT",       "ANSI/OEM Thai" ],
  44     [ 875,   "VENDORS/MICSFT/EBCDIC/CP875.TXT",   "IBM EBCDIC Greek" ],
  45     [ 878,   "VENDORS/MISC/KOI8-R.TXT",           "Russian KOI8" ],
  46     [ 932,   "VENDORS/MICSFT/WINDOWS/CP932.TXT",  "ANSI/OEM Japanese Shift-JIS" ],
  47     [ 936,   "VENDORS/MICSFT/WINDOWS/CP936.TXT",  "ANSI/OEM Simplified Chinese GBK" ],
  48     [ 949,   "VENDORS/MICSFT/WINDOWS/CP949.TXT",  "ANSI/OEM Korean Unified Hangul" ],
  49     [ 950,   "VENDORS/MICSFT/WINDOWS/CP950.TXT",  "ANSI/OEM Traditional Chinese Big5" ],
  50     [ 1006,  "VENDORS/MISC/CP1006.TXT",           "IBM Arabic" ],
  51     [ 1026,  "VENDORS/MICSFT/EBCDIC/CP1026.TXT",  "IBM EBCDIC Latin 5 Turkish" ],
  52     [ 1250,  "VENDORS/MICSFT/WINDOWS/CP1250.TXT", "ANSI Eastern Europe" ],
  53     [ 1251,  "VENDORS/MICSFT/WINDOWS/CP1251.TXT", "ANSI Cyrillic" ],
  54     [ 1252,  "VENDORS/MICSFT/WINDOWS/CP1252.TXT", "ANSI Latin 1" ],
  55     [ 1253,  "VENDORS/MICSFT/WINDOWS/CP1253.TXT", "ANSI Greek" ],
  56     [ 1254,  "VENDORS/MICSFT/WINDOWS/CP1254.TXT", "ANSI Turkish" ],
  57     [ 1255,  "VENDORS/MICSFT/WINDOWS/CP1255.TXT", "ANSI Hebrew" ],
  58     [ 1256,  "VENDORS/MICSFT/WINDOWS/CP1256.TXT", "ANSI Arabic" ],
  59     [ 1257,  "VENDORS/MICSFT/WINDOWS/CP1257.TXT", "ANSI Baltic" ],
  60     [ 1258,  "VENDORS/MICSFT/WINDOWS/CP1258.TXT", "ANSI/OEM Viet Nam" ],
  61     [ 10000, "VENDORS/MICSFT/MAC/ROMAN.TXT",      "Mac Roman" ],
  62     [ 10006, "VENDORS/MICSFT/MAC/GREEK.TXT",      "Mac Greek" ],
  63     [ 10007, "VENDORS/MICSFT/MAC/CYRILLIC.TXT",   "Mac Cyrillic" ],
  64     [ 10029, "VENDORS/MICSFT/MAC/LATIN2.TXT",     "Mac Latin 2" ],
  65     [ 10079, "VENDORS/MICSFT/MAC/ICELAND.TXT",    "Mac Icelandic" ],
  66     [ 10081, "VENDORS/MICSFT/MAC/TURKISH.TXT",    "Mac Turkish" ],
  67     [ 20866, "VENDORS/MISC/KOI8-R.TXT",           "Russian KOI8" ],
  68     [ 28591, "ISO8859/8859-1.TXT",                "ISO 8859-1 Latin 1" ],
  69     [ 28592, "ISO8859/8859-2.TXT",                "ISO 8859-2 Eastern Europe" ],
  70     [ 28593, "ISO8859/8859-3.TXT",                "ISO 8859-3 Turkish" ],
  71     [ 28594, "ISO8859/8859-4.TXT",                "ISO 8859-4 Baltic" ],
  72     [ 28595, "ISO8859/8859-5.TXT",                "ISO 8859-5 Cyrillic" ],
  73     [ 28596, "ISO8859/8859-6.TXT",                "ISO 8859-6 Arabic" ],
  74     [ 28597, "ISO8859/8859-7.TXT",                "ISO 8859-7 Greek" ],
  75     [ 28598, "ISO8859/8859-8.TXT",                "ISO 8859-8 Hebrew" ],
  76     [ 28599, "ISO8859/8859-9.TXT",                "ISO 8859-9 Latin 5" ]
  77 );
  78
  79
  80 %ctype =
  81 (
  82     "upper"  => 0x0001,
  83     "lower"  => 0x0002,
  84     "digit"  => 0x0004,
  85     "space"  => 0x0008,
  86     "punct"  => 0x0010,
  87     "cntrl"  => 0x0020,
  88     "blank"  => 0x0040,
  89     "xdigit" => 0x0080,
  90     "alpha"  => 0x0100
  91 );
  92
  93 %categories =
  94 (
  95     "Lu" => $ctype{"alpha"}|$ctype{"upper"}, # Letter, Uppercase
  96     "Ll" => $ctype{"alpha"}|$ctype{"lower"}, # Letter, Lowercase
  97     "Lt" => $ctype{"alpha"},    # Letter, Titlecase
  98     "Mn" => $ctype{"punct"},    # Mark, Non-Spacing
  99     "Mc" => $ctype{"punct"},    # Mark, Spacing Combining
 100     "Me" => $ctype{"punct"},    # Mark, Enclosing
 101     "Nd" => $ctype{"digit"},    # Number, Decimal Digit
 102     "Nl" => $ctype{"punct"},    # Number, Letter
 103     "No" => $ctype{"punct"},    # Number, Other
 104     "Zs" => $ctype{"space"},    # Separator, Space
 105     "Zl" => 0,                  # Separator, Line
 106     "Zp" => 0,                  # Separator, Paragraph
 107     "Cc" => $ctype{"cntrl"},    # Other, Control
 108     "Cf" => 0,                  # Other, Format
 109     "Cs" => 0,                  # Other, Surrogate
 110     "Co" => 0,                  # Other, Private Use
 111     "Cn" => 0,                  # Other, Not Assigned
 112     "Lm" => $ctype{"punct"},    # Letter, Modifier
 113     "Lo" => $ctype{"alpha"},    # Letter, Other
 114     "Pc" => $ctype{"punct"},    # Punctuation, Connector
 115     "Pd" => $ctype{"punct"},    # Punctuation, Dash
 116     "Ps" => $ctype{"punct"},    # Punctuation, Open
 117     "Pe" => $ctype{"punct"},    # Punctuation, Close
 118     "Pi" => $ctype{"punct"},    # Punctuation, Initial quote
 119     "Pf" => $ctype{"punct"},    # Punctuation, Final quote
 120     "Po" => $ctype{"punct"},    # Punctuation, Other
 121     "Sm" => $ctype{"punct"},    # Symbol, Math
 122     "Sc" => $ctype{"punct"},    # Symbol, Currency
 123     "Sk" => $ctype{"punct"},    # Symbol, Modifier
 124     "So" => $ctype{"punct"}     # Symbol, Other
 125 );
 126
 127 # a few characters need additional categories that cannot be determined automatically
 128 %special_categories =
 129 (
 130     "xdigit" => [ ord('0')..ord('9'),ord('A')..ord('F'),ord('a')..ord('f'),
 131                   0xff10..0xff19, 0xff21..0xff26, 0xff41..0xff46 ],
 132     "space"  => [ 0x09..0x0d, 0xfeff ],
 133     "blank"  => [ 0x09, 0x20, 0xa0, 0xfeff ]
 134 );
 135
 136 %directions =
 137 (
 138     "L"   => 1,    # Left-to-Right
 139     "LRE" => 11,   # Left-to-Right Embedding
 140     "LRO" => 11,   # Left-to-Right Override
 141     "R"   => 2,    # Right-to-Left
 142     "AL"  => 2,    # Right-to-Left Arabic
 143     "RLE" => 11,   # Right-to-Left Embedding
 144     "RLO" => 11,   # Right-to-Left Override
 145     "PDF" => 11,   # Pop Directional Format
 146     "EN"  => 3,    # European Number
 147     "ES"  => 4,    # European Number Separator
 148     "ET"  => 5,    # European Number Terminator
 149     "AN"  => 6,    # Arabic Number
 150     "CS"  => 7,    # Common Number Separator
 151     "NSM" => 0,    # Non-Spacing Mark
 152     "BN"  => 0,    # Boundary Neutral
 153     "B"   => 8,    # Paragraph Separator
 154     "S"   => 9,    # Segment Separator
 155     "WS"  => 10,   # Whitespace
 156     "ON"  => 11    # Other Neutrals
 157 );
 158
 159
 160 ################################################################
 161 # main routine
 162
 163 READ_DEFAULTS();
 164 DUMP_CASE_MAPPINGS();
 165 DUMP_CTYPE_TABLES();
 166
 167 foreach $file (@allfiles) { HANDLE_FILE( @$file ); }
 168
 169 OUTPUT_CPTABLE();
 170
 171 exit(0);
 172
 173
 174 ################################################################
 175 # read in the defaults file
 176 sub READ_DEFAULTS
 177 {
 178     @unicode_defaults = ();
 179     @unicode_aliases = ();
 180     @tolower_table = ();
 181     @toupper_table = ();
 182     @category_table = ();
 183     @direction_table = ();
 184
 185     # first setup a few default mappings
 186
 187     open DEFAULTS or die "Cannot open $DEFAULTS";
 188     print "Loading $DEFAULTS\n";
 189     while (<DEFAULTS>)
 190     {
 191         next if /^\#/;  # skip comments
 192         next if /^$/;  # skip empty lines
 193         if (/^(([0-9a-fA-F]+)(,[0-9a-fA-F]+)*)\s+([0-9a-fA-F]+|'.'|none)\s+(\#.*)?/)
 194         {
 195             my @src = map hex, split /,/,$1;
 196             my $dst = $4;
 197             my $comment = $5;
 198             if ($#src > 0) { push @unicode_aliases, \@src; }
 199             next if ($dst eq "none");
 200             $dst = ($dst =~ /\'.\'/) ? ord substr($dst,1,1) : hex $dst;
 201             foreach $src (@src)
 202             {
 203                 die "Duplicate value" if defined($unicode_defaults[$src]);
 204                 $unicode_defaults[$src] = $dst;
 205             }
 206             next;
 207         }
 208         die "Unrecognized line $_\n";
 209     }
 210
 211     # now build mappings from the decomposition field of the Unicode database
 212
 213     open UNICODEDATA or die "Cannot open $UNICODEDATA";
 214     print "Loading $UNICODEDATA\n";
 215     while (<UNICODEDATA>)
 216     {
 217         # Decode the fields ...
 218         ($code, $name, $cat, $comb, $bidi,
 219          $decomp, $dec, $dig, $num, $mirror,
 220          $oldname, $comment, $upper, $lower, $title) = split /;/;
 221
 222         my $src = hex $code;
 223
 224         die "unknown category $cat" unless defined $categories{$cat};
 225         die "unknown directionality $bidi" unless defined $directions{$bidi};
 226
 227         $uniname[$src] = $name;
 228         $category_table[$src] = $categories{$cat};
 229         $direction_table[$src] = $directions{$bidi};
 230
 231         if ($lower ne "")
 232         {
 233             $tolower_table[$src] = hex $lower;
 234             $category_table[$src] |= $ctype{"upper"}|$ctype{"alpha"};
 235         }
 236         if ($upper ne "")
 237         {
 238             $toupper_table[$src] = hex $upper;
 239             $category_table[$src] |= $ctype{"lower"}|$ctype{"alpha"};
 240         }
 241         if ($dec ne "")
 242         {
 243             $category_table[$src] |= $ctype{"digit"};
 244         }
 245
 246         # copy the category and direction for everything between First/Last pairs
 247         if ($name =~ /, First>/) { $start = $src; }
 248         if ($name =~ /, Last>/)
 249         {
 250             while ($start < $src)
 251             {
 252                 $category_table[$start] = $category_table[$src];
 253                 $direction_table[$start] = $direction_table[$src];
 254                 $start++;
 255             }
 256         }
 257
 258         next if $decomp eq "";  # no decomposition, skip it
 259
 260         if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
 261         {
 262             # decomposition of the form "<foo> 1234" -> use char if type is known
 263             next unless ($1 eq "font" ||
 264                          $1 eq "noBreak" ||
 265                          $1 eq "circle" ||
 266                          $1 eq "super" ||
 267                          $1 eq "sub" ||
 268                          $1 eq "wide" ||
 269                          $1 eq "narrow" ||
 270                          $1 eq "compat" ||
 271                          $1 eq "small");
 272             $dst = hex $2;
 273         }
 274         elsif ($decomp =~ /^<compat>\s+0020\s+([0-9a-fA-F]+)/)
 275         {
 276             # decomposition "<compat> 0020 1234" -> combining accent
 277             $dst = hex $1;
 278         }
 279         elsif ($decomp =~ /^([0-9a-fA-F]+)/)
 280         {
 281             # decomposition contains only char values without prefix -> use first char
 282             $dst = hex $1;
 283             $category_table[$src] |= $category_table[$dst];
 284         }
 285         else
 286         {
 287             next;
 288         }
 289
 290         next if defined($unicode_defaults[$src]);  # may have been set in the defaults file
 291
 292         # check for loops
 293         for ($i = $dst; ; $i = $unicode_defaults[$i])
 294         {
 295             die sprintf("loop detected for %04x -> %04x",$src,$dst) if $i == $src;
 296             last unless defined($unicode_defaults[$i]);
 297         }
 298         $unicode_defaults[$src] = $dst;
 299     }
 300
 301     # patch the category of some special characters
 302
 303     foreach $cat (keys %special_categories)
 304     {
 305         my $flag = $ctype{$cat};
 306         foreach $i (@{$special_categories{$cat}}) { $category_table[$i] |= $flag; }
 307     }
 308 }
 309
 310
 311 ################################################################
 312 # parse the input file
 313 sub READ_FILE
 314 {
 315     my $name = shift;
 316     open INPUT,$name or die "Cannot open $name";
 317     @cp2uni = ();
 318     @lead_bytes = ();
 319     @uni2cp = ();
 320
 321     while (<INPUT>)
 322     {
 323         next if /^\#/;  # skip comments
 324         next if /^$/;  # skip empty lines
 325         next if /\x1a/;  # skip ^Z
 326         next if (/^0x([0-9a-fA-F]+)\s+\#UNDEFINED/);  # undefined char
 327
 328         if (/^0x([0-9a-fA-F]+)\s+\#DBCS LEAD BYTE/)
 329         {
 330             $cp = hex $1;
 331             push @lead_bytes,$cp;
 332             $cp2uni[$cp] = 0;
 333             next;
 334         }
 335         if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
 336         {
 337             $cp = hex $1;
 338             $uni = hex $2;
 339             $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
 340             $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
 341             next;
 342         }
 343         die "$name: Unrecognized line $_\n";
 344     }
 345 }
 346
 347
 348 ################################################################
 349 # parse the symbol.txt file, since its syntax is different from the other ones
 350 sub READ_SYMBOL_FILE
 351 {
 352     my $name = shift;
 353     open INPUT,$name or die "Cannot open $name";
 354     @cp2uni = ();
 355     @lead_bytes = ();
 356     @uni2cp = ();
 357
 358     while (<INPUT>)
 359     {
 360         next if /^\#/;  # skip comments
 361         next if /^$/;  # skip empty lines
 362         next if /\x1a/;  # skip ^Z
 363         if (/^([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+(\#.*)?/)
 364         {
 365             $uni = hex $1;
 366             $cp = hex $2;
 367             $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
 368             $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
 369             next;
 370         }
 371         die "$name: Unrecognized line $_\n";
 372     }
 373 }
 374
 375
 376 ################################################################
 377 # add default mappings once the file had been read
 378 sub ADD_DEFAULT_MAPPINGS
 379 {
 380     # Apply aliases
 381
 382     foreach $alias (@unicode_aliases)
 383     {
 384         my $target = undef;
 385         foreach $src (@$alias)
 386         {
 387             if (defined($uni2cp[$src]))
 388             {
 389                 $target = $uni2cp[$src];
 390                 last;
 391             }
 392         }
 393         next unless defined($target);
 394
 395         # At least one char of the alias set is defined, set the others to the same value
 396         foreach $src (@$alias)
 397         {
 398             $uni2cp[$src] = $target unless defined($uni2cp[$src]);
 399         }
 400     }
 401
 402     # For every src -> target mapping in the defaults table,
 403     # make uni2cp[src] = uni2cp[target] if uni2cp[target] is defined
 404
 405     for ($src = 0; $src < 65536; $src++)
 406     {
 407         next if defined($uni2cp[$src]);  # source has a definition already
 408         next unless defined($unicode_defaults[$src]);  # no default for this char
 409         my $target = $unicode_defaults[$src];
 410
 411         # do a recursive mapping until we find a target char that is defined
 412         while (!defined($uni2cp[$target]) &&
 413                defined($unicode_defaults[$target])) { $target = $unicode_defaults[$target]; }
 414
 415         if (defined($uni2cp[$target])) { $uni2cp[$src] = $uni2cp[$target]; }
 416     }
 417
 418     # Add an identity mapping for all undefined chars
 419
 420     for ($i = 0; $i < 256; $i++)
 421     {
 422         next if defined($cp2uni[$i]);
 423         next if defined($uni2cp[$i]);
 424         $cp2uni[$i] = $uni2cp[$i] = $i;
 425     }
 426 }
 427
 428 ################################################################
 429 # dump an array of integers
 430 sub DUMP_ARRAY
 431 {
 432     my ($format,$default,@array) = @_;
 433     my $i, $ret = "    ";
 434     for ($i = 0; $i < $#array; $i++)
 435     {
 436         $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
 437         $ret .= (($i % 8) != 7) ? ", " : ",\n    ";
 438     }
 439     $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
 440     return $ret;
 441 }
 442
 443 ################################################################
 444 # dump an SBCS mapping table
 445 sub DUMP_SBCS_TABLE
 446 {
 447     my ($codepage, $name) = @_;
 448     my $i;
 449
 450     # output the ascii->unicode table
 451
 452     printf OUTPUT "static const WCHAR cp2uni[256] =\n";
 453     printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", $DEF_CHAR, @cp2uni[0 .. 255] );
 454
 455     # count the number of unicode->ascii subtables that contain something
 456
 457     my @filled = ();
 458     my $subtables = 1;
 459     for ($i = 0; $i < 65536; $i++)
 460     {
 461         next unless defined $uni2cp[$i];
 462         $filled[$i >> 8] = 1;
 463         $subtables++;
 464         $i = ($i & ~255) + 256;
 465     }
 466
 467     # output all the subtables into a single array
 468
 469     printf OUTPUT "static const unsigned char uni2cp_low[%d] =\n{\n", $subtables*256;
 470     for ($i = 0; $i < 256; $i++)
 471     {
 472         next unless $filled[$i];
 473         printf OUTPUT "    /* 0x%02x00 .. 0x%02xff */\n", $i, $i;
 474         printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%02x", $DEF_CHAR, @uni2cp[($i<<8) .. ($i<<8)+255] );
 475     }
 476     printf OUTPUT "    /* defaults */\n";
 477     printf OUTPUT "%s\n};\n\n", DUMP_ARRAY( "0x%02x", 0, ($DEF_CHAR) x 256 );
 478
 479     # output a table of the offsets of the subtables in the previous array
 480
 481     my $pos = 0;
 482     my @offsets = ();
 483     for ($i = 0; $i < 256; $i++)
 484     {
 485         if ($filled[$i]) { push @offsets, $pos; $pos += 256; }
 486         else { push @offsets, ($subtables-1) * 256; }
 487     }
 488     printf OUTPUT "static const unsigned short uni2cp_high[256] =\n";
 489     printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", 0, @offsets );
 490
 491     # output the code page descriptor
 492
 493     printf OUTPUT "const struct sbcs_table cptable_%03d =\n{\n", $codepage;
 494     printf OUTPUT "    { %d, 1, 0x%04x, 0x%04x, \"%s\" },\n",
 495                   $codepage, $DEF_CHAR, $DEF_CHAR, $name;
 496     printf OUTPUT "    cp2uni,\n";
 497     printf OUTPUT "    uni2cp_low,\n";
 498     printf OUTPUT "    uni2cp_high\n};\n";
 499 }
 500
 501
 502 ################################################################
 503 # dump a DBCS mapping table
 504 sub DUMP_DBCS_TABLE
 505 {
 506     my ($codepage, $name) = @_;
 507     my $i, $x, $y;
 508
 509     # build a list of lead bytes that are actually used
 510
 511     my @lblist = ();
 512     LBLOOP: for ($y = 0; $y <= $#lead_bytes; $y++)
 513     {
 514         my $base = $lead_bytes[$y] << 8;
 515         for ($x = 0; $x < 256; $x++)
 516         {
 517             if (defined $cp2uni[$base+$x])
 518             {
 519                 push @lblist,$lead_bytes[$y];
 520                 next LBLOOP;
 521             }
 522         }
 523     }
 524     my $unused = ($#lead_bytes > $#lblist);
 525
 526     # output the ascii->unicode table for the single byte chars
 527
 528     printf OUTPUT "static const WCHAR cp2uni[%d] =\n", 256 * ($#lblist + 2 + $unused);
 529     printf OUTPUT "{\n%s,\n", DUMP_ARRAY( "0x%04x", $DEF_CHAR, @cp2uni[0 .. 255] );
 530
 531     # output the default table for unused lead bytes
 532
 533     if ($unused)
 534     {
 535         printf OUTPUT "    /* unused lead bytes */\n";
 536         printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", 0, ($DEF_CHAR) x 256 );
 537     }
 538
 539     # output the ascii->unicode table for each DBCS lead byte
 540
 541     for ($y = 0; $y <= $#lblist; $y++)
 542     {
 543         my $base = $lblist[$y] << 8;
 544         printf OUTPUT "    /* lead byte %02x */\n", $lblist[$y];
 545         printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", $DEF_CHAR, @cp2uni[$base .. $base+255] );
 546         printf OUTPUT ($y < $#lblist) ? ",\n" : "\n};\n\n";
 547     }
 548
 549     # output the lead byte subtables offsets
 550
 551     my @offsets = ();
 552     for ($x = 0; $x < 256; $x++) { $offsets[$x] = 0; }
 553     for ($x = 0; $x <= $#lblist; $x++) { $offsets[$lblist[$x]] = $x + 1; }
 554     if ($unused)
 555     {
 556         # increment all lead bytes offset to take into account the unused table
 557         for ($x = 0; $x <= $#lead_bytes; $x++) { $offsets[$lead_bytes[$x]]++; }
 558     }
 559     printf OUTPUT "static const unsigned char cp2uni_leadbytes[256] =\n";
 560     printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%02x", 0, @offsets );
 561
 562     # count the number of unicode->ascii subtables that contain something
 563
 564     my @filled = ();
 565     my $subtables = 1;
 566     for ($i = 0; $i < 65536; $i++)
 567     {
 568         next unless defined $uni2cp[$i];
 569         $filled[$i >> 8] = 1;
 570         $subtables++;
 571         $i = ($i & ~255) + 256;
 572     }
 573
 574     # output all the subtables into a single array
 575
 576     printf OUTPUT "static const unsigned short uni2cp_low[%d] =\n{\n", $subtables*256;
 577     for ($y = 0; $y < 256; $y++)
 578     {
 579         next unless $filled[$y];
 580         printf OUTPUT "    /* 0x%02x00 .. 0x%02xff */\n", $y, $y;
 581         printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", $DEF_CHAR, @uni2cp[($y<<8) .. ($y<<8)+255] );
 582     }
 583     printf OUTPUT "    /* defaults */\n";
 584     printf OUTPUT "%s\n};\n\n", DUMP_ARRAY( "0x%04x", 0, ($DEF_CHAR) x 256 );
 585
 586     # output a table of the offsets of the subtables in the previous array
 587
 588     my $pos = 0;
 589     my @offsets = ();
 590     for ($y = 0; $y < 256; $y++)
 591     {
 592         if ($filled[$y]) { push @offsets, $pos; $pos += 256; }
 593         else { push @offsets, ($subtables-1) * 256; }
 594     }
 595     printf OUTPUT "static const unsigned short uni2cp_high[256] =\n";
 596     printf OUTPUT "{\n%s\n};\n\n", DUMP_ARRAY( "0x%04x", 0, @offsets );
 597
 598     # output the code page descriptor
 599
 600     printf OUTPUT "const struct dbcs_table cptable_%03d =\n{\n", $codepage;
 601     printf OUTPUT "    { %d, 2, 0x%04x, 0x%04x, \"%s\" },\n",
 602                   $codepage, $DEF_CHAR, $DEF_CHAR, $name;
 603     printf OUTPUT "    cp2uni,\n";
 604     printf OUTPUT "    cp2uni_leadbytes,\n";
 605     printf OUTPUT "    uni2cp_low,\n";
 606     printf OUTPUT "    uni2cp_high,\n";
 607     DUMP_LB_RANGES();
 608     printf OUTPUT "};\n";
 609 }
 610
 611
 612 ################################################################
 613 # dump the list of defined lead byte ranges
 614 sub DUMP_LB_RANGES
 615 {
 616     my @list = ();
 617     my $i = 0;
 618     foreach $i (@lead_bytes) { $list[$i] = 1; }
 619     my $on = 0;
 620     printf OUTPUT "    { ";
 621     for ($i = 0; $i < 256; $i++)
 622     {
 623         if ($on)
 624         {
 625             if (!defined $list[$i]) { printf OUTPUT "0x%02x, ", $i-1; $on = 0; }
 626         }
 627         else
 628         {
 629             if ($list[$i]) { printf OUTPUT "0x%02x, ", $i; $on = 1; }
 630         }
 631     }
 632     if ($on) { printf OUTPUT "0xff, "; }
 633     printf OUTPUT "0x00, 0x00 }\n";
 634 }
 635
 636
 637 ################################################################
 638 # dump the case mapping tables
 639 sub DUMP_CASE_MAPPINGS
 640 {
 641     open OUTPUT,">casemap.c" or die "Cannot create casemap.c";
 642     printf "Building casemap.c\n";
 643     printf OUTPUT "/* Unicode case mappings */\n";
 644     printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
 645     printf OUTPUT "#include \"wine/unicode.h\"\n\n";
 646
 647     DUMP_CASE_TABLE( "casemap_lower", @tolower_table );
 648     DUMP_CASE_TABLE( "casemap_upper", @toupper_table );
 649     close OUTPUT;
 650 }
 651
 652
 653 ################################################################
 654 # dump a case mapping table
 655 sub DUMP_CASE_TABLE
 656 {
 657     my ($name,@table) = @_;
 658
 659     # count the number of sub tables that contain something
 660
 661     my @filled = ();
 662     my $pos = 512;
 663     for ($i = 0; $i < 65536; $i++)
 664     {
 665         next unless defined $table[$i];
 666         $filled[$i >> 8] = $pos;
 667         $pos += 256;
 668         $i = ($i & ~255) + 256;
 669     }
 670     for ($i = 0; $i < 65536; $i++)
 671     {
 672         next unless defined $table[$i];
 673         $table[$i] = ($table[$i] - $i) & 0xffff;
 674     }
 675
 676     # dump the table
 677
 678     printf OUTPUT "const WCHAR %s[%d] =\n", $name, $pos;
 679     printf OUTPUT "{\n    /* index */\n";
 680     printf OUTPUT "%s,\n", DUMP_ARRAY( "0x%04x", 256, @filled );
 681     printf OUTPUT "    /* defaults */\n";
 682     printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, (0) x 256 );
 683     for ($i = 0; $i < 256; $i++)
 684     {
 685         next unless $filled[$i];
 686         printf OUTPUT ",\n    /* 0x%02x00 .. 0x%02xff */\n", $i, $i;
 687         printf OUTPUT "%s", DUMP_ARRAY( "0x%04x", 0, @table[($i<<8) .. ($i<<8)+255] );
 688     }
 689     printf OUTPUT "\n};\n";
 690 }
 691
 692
 693 ################################################################
 694 # dump the ctype tables
 695 sub DUMP_CTYPE_TABLES
 696 {
 697     open OUTPUT,">wctype.c" or die "Cannot create casemap.c";
 698     printf "Building wctype.c\n";
 699     printf OUTPUT "/* Unicode ctype tables */\n";
 700     printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
 701     printf OUTPUT "#include \"wine/unicode.h\"\n\n";
 702
 703     my $i;
 704     my @array = (0) x 256;
 705
 706     # add the direction in the high 4 bits of the category
 707     for ($i = 0; $i < 65536; $i++)
 708     {
 709         $category_table[$i] |= $direction_table[$i] << 12;
 710     }
 711
 712     # try to merge table rows
 713     for ($row = 0; $row < 256; $row++)
 714     {
 715         my $rowtxt = sprintf "%04x" x 256, @category_table[($row<<8)..($row<<8)+255];
 716         if (defined($sequences{$rowtxt}))
 717         {
 718             # reuse an existing row
 719             $array[$row] = $sequences{$rowtxt};
 720         }
 721         else
 722         {
 723             # create a new row
 724             $sequences{$rowtxt} = $array[$row] = $#array + 1;
 725             push @array, @category_table[($row<<8)..($row<<8)+255];
 726         }
 727     }
 728
 729     printf OUTPUT "const unsigned short wctype_table[%d] =\n{\n", $#array+1;
 730     printf OUTPUT "    /* offsets */\n%s,\n", DUMP_ARRAY( "0x%04x", 0, @array[0..255] );
 731     printf OUTPUT "    /* values */\n%s\n};\n", DUMP_ARRAY( "0x%04x", 0, @array[256..$#array] );
 732
 733     close OUTPUT;
 734 }
 735
 736 ################################################################
 737 # read an input file and generate the corresponding .c file
 738 sub HANDLE_FILE
 739 {
 740     my ($codepage,$filename,$comment) = @_;
 741
 742     # symbol codepage file is special
 743     if ($codepage == 42) { READ_SYMBOL_FILE($MAPPREFIX . $filename); }
 744     else { READ_FILE($MAPPREFIX . $filename); }
 745
 746     ADD_DEFAULT_MAPPINGS();
 747
 748     my $output = sprintf "c_%03d.c", $codepage;
 749     open OUTPUT,">$output" or die "Cannot create $output";
 750
 751     printf "Building %s from %s (%s)\n", $output, $filename, $comment;
 752
 753     # dump all tables
 754
 755     printf OUTPUT "/* code page %03d (%s) */\n", $codepage, $comment;
 756     printf OUTPUT "/* generated from %s */\n", $MAPPREFIX . $filename;
 757     printf OUTPUT "/* DO NOT EDIT!! */\n\n";
 758     printf OUTPUT "#include \"wine/unicode.h\"\n\n";
 759
 760     if ($#lead_bytes == -1) { DUMP_SBCS_TABLE( $codepage, $comment ); }
 761     else { DUMP_DBCS_TABLE( $codepage, $comment ); }
 762     close OUTPUT;
 763 }
 764
 765
 766 ################################################################
 767 # output the list of codepage tables into the cptable.c file
 768 sub OUTPUT_CPTABLE
 769 {
 770     @tables_decl = ();
 771
 772     foreach $file (@allfiles)
 773     {
 774         my ($codepage,$filename,$comment) = @$file;
 775         push @tables_decl, sprintf("extern union cptable cptable_%03d;\n",$codepage);
 776     }
 777
 778     push @tables_decl, sprintf("\nstatic const union cptable * const cptables[%d] =\n{\n",$#allfiles+1);
 779     foreach $file (@allfiles)
 780     {
 781         my ($codepage,$filename,$comment) = @$file;
 782         push @tables_decl, sprintf("    &cptable_%03d,\n", $codepage);
 783     }
 784     push @tables_decl, "};";
 785     REPLACE_IN_FILE( "cptable.c", @tables_decl );
 786 }
 787
 788 ################################################################
 789 # replace the contents of a file between ### cpmap ### marks
 790
 791 sub REPLACE_IN_FILE
 792 {
 793     my $name = shift;
 794     my @data = @_;
 795     my @lines = ();
 796     open(FILE,$name) or die "Can't open $name";
 797     while (<FILE>)
 798     {
 799         push @lines, $_;
 800         last if /\#\#\# cpmap begin \#\#\#/;
 801     }
 802     push @lines, @data;
 803     while (<FILE>)
 804     {
 805         if (/\#\#\# cpmap end \#\#\#/) { push @lines, "\n", $_; last; }
 806     }
 807     push @lines, <FILE>;
 808     open(FILE,">$name") or die "Can't modify $name";
 809     print FILE @lines;
 810     close(FILE);
 811 }