tools/make_unicode

   1 #!/usr/bin/perl -w
   2 #
   3 # Generate code page .c files from ftp.unicode.org descriptions
   4 #
   5 # Copyright 2000 Alexandre Julliard
   6 #
   7 # This library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # This library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with this library; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  20 #
  21
  22 use strict;
  23 use XML::LibXML;
  24 use Digest::SHA;
  25 use Encode;
  26 use Time::Local qw(timegm_modern);
  27
  28 my $UNIVERSION = "15.1.0";
  29 my $CLDRVERSION = "43";
  30 my $ISO639VERSION = "20230123";
  31 my $TZVERSION = "2023c";
  32
  33 my %data_files =
  34 (
  35  ucd       => { url  => "https://www.unicode.org/Public/$UNIVERSION/ucd/UCD.zip", name => "UCD-$UNIVERSION.zip",
  36                 sha  => "cb1c663d053926500cd501229736045752713a066bd75802098598b7a7056177" },
  37  unihan    => { url  => "https://www.unicode.org/Public/$UNIVERSION/ucd/Unihan.zip", name => "Unihan-$UNIVERSION.zip",
  38                 sha  => "a0226610e324bcf784ac380e11f4cbf533ee1e6b3d028b0991bf8c0dc3f85853" },
  39  idna      => { url  => "https://www.unicode.org/Public/idna/$UNIVERSION/IdnaMappingTable.txt", name => "IdnaMappingTable-$UNIVERSION.txt",
  40                 sha  => "402cbd285f1f952fcd0834b63541d54f69d3d8f1b8f8599bf71a1a14935f82c4" },
  41  cldr      => { url  => "https://github.com/unicode-org/cldr/archive/refs/tags/release-$CLDRVERSION.zip",
  42                 sha  => "132cdd24e479abb6e86db1429931cec3dada485fd41da39ece3c08e531c477df" },
  43  cldr33    => { url  => "https://www.unicode.org/Public/cldr/33/cldr-common-33.0.zip",
  44                 sha  => "fa3490082c086d21257153609642f54fcf788fcfda4966fe67f3f6daca0d58b9" },
  45  sorting   => { url  => "https://download.microsoft.com/download/C/F/7/CF713A5E-9FBC-4FD6-9246-275F65C0E498/Windows 10 Sorting Weight Table.txt",
  46                 sha  => "81fcfa1e5ed3e3a94d329959ff7d97d522ddf9d653d2c4d6ddcccc5cd4df663f" },
  47  codepages => { url  => "https://download.microsoft.com/download/C/F/7/CF713A5E-9FBC-4FD6-9246-275F65C0E498/Windows Supported Code Page Data Files.zip",
  48                 sha  => "5074e6dd253056ba61fc6c870c9a955467855129c6ad3a51761c386b301b125a" },
  49  iso639    => { url  => "https://iso639-3.sil.org/sites/iso639-3/files/downloads/iso-639-3_Code_Tables_$ISO639VERSION.zip",
  50                 sha  => "884faa6cc5ac5181ed7969eed75355c1bc665447614cf4c06c62e87b38fe6a97" },
  51  ksx1001   => { url  => "https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/KSC/KSX1001.TXT",
  52                 sha  => "d8d2a35206ac0ea2865f5d801c9d6717f735bf46f263a658a64a960abe59e371" },
  53  jis0208   => { url  => "https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/JIS0208.TXT",
  54                 sha  => "1c571870457f19c97720631fa83ee491549a96ba1436da1296786a67d8632e87" },
  55  jis0212   => { url  => "https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/JIS0212.TXT",
  56                 sha  => "477820bb3055bbcc90880d788cd95607d221dc94457bae249231adecf13c12e6" },
  57  tzdata    => { url  => "https://data.iana.org/time-zones/releases/tzdata$TZVERSION.tar.gz",
  58                 sha  => "3f510b5d1b4ae9bb38e485aa302a776b317fb3637bdb6404c4adf7b6cadd965c" },
  59 );
  60
  61
  62 # Default char for undefined mappings
  63 my $DEF_CHAR = ord '?';
  64
  65 # Last valid Unicode character
  66 my $MAX_CHAR = 0x10ffff;
  67
  68 my $nlskey = "-SYSTEM\\-CurrentControlSet\\-Control\\-Nls";
  69 my $zonekey = "-Software\\-Microsoft\\-Windows NT\\-CurrentVersion\\Time Zones";
  70
  71 my @allfiles =
  72 (
  73     "CodpageFiles/037.txt",
  74     "CodpageFiles/437.txt",
  75     "CodpageFiles/500.txt",
  76     "CodpageFiles/708.txt",
  77     "CodpageFiles/720.txt",
  78     "CodpageFiles/737.txt",
  79     "CodpageFiles/775.txt",
  80     "CodpageFiles/850.txt",
  81     "CodpageFiles/852.txt",
  82     "CodpageFiles/855.txt",
  83     "CodpageFiles/857.txt",
  84     "CodpageFiles/860.txt",
  85     "CodpageFiles/861.txt",
  86     "CodpageFiles/862.txt",
  87     "CodpageFiles/863.txt",
  88     "CodpageFiles/864.txt",
  89     "CodpageFiles/865.txt",
  90     "CodpageFiles/866.txt",
  91     "CodpageFiles/869.txt",
  92     "CodpageFiles/874.txt",
  93     "CodpageFiles/875.txt",
  94     "CodpageFiles/932.txt",
  95     "CodpageFiles/936.txt",
  96     "CodpageFiles/949.txt",
  97     "CodpageFiles/950.txt",
  98     "CodpageFiles/1026.txt",
  99     "CodpageFiles/1250.txt",
 100     "CodpageFiles/1251.txt",
 101     "CodpageFiles/1252.txt",
 102     "CodpageFiles/1253.txt",
 103     "CodpageFiles/1254.txt",
 104     "CodpageFiles/1255.txt",
 105     "CodpageFiles/1256.txt",
 106     "CodpageFiles/1257.txt",
 107     "CodpageFiles/1258.txt",
 108     "CodpageFiles/1361.txt",
 109     "CodpageFiles/10000.txt",
 110     "CodpageFiles/10001.txt",
 111     "CodpageFiles/10002.txt",
 112     "CodpageFiles/10003.txt",
 113     "CodpageFiles/10004.txt",
 114     "CodpageFiles/10005.txt",
 115     "CodpageFiles/10006.txt",
 116     "CodpageFiles/10007.txt",
 117     "CodpageFiles/10008.txt",
 118     "CodpageFiles/10010.txt",
 119     "CodpageFiles/10017.txt",
 120     "CodpageFiles/10021.txt",
 121     "CodpageFiles/10029.txt",
 122     "CodpageFiles/10079.txt",
 123     "CodpageFiles/10081.txt",
 124     "CodpageFiles/10082.txt",
 125     "CodpageFiles/20127.txt",
 126     "CodpageFiles/20866.txt",
 127     "CodpageFiles/21866.txt",
 128     "CodpageFiles/28591.txt",
 129     "CodpageFiles/28592.txt",
 130     "CodpageFiles/28593.txt",
 131     "CodpageFiles/28594.txt",
 132     "CodpageFiles/28595.txt",
 133     "CodpageFiles/28596.txt",
 134     "CodpageFiles/28597.txt",
 135     "CodpageFiles/28598.txt",
 136     "CodpageFiles/28599.txt",
 137     "CodpageFiles/28603.txt",
 138     "CodpageFiles/28605.txt",
 139 );
 140
 141 my @timezone_files = qw(africa antarctica asia australasia europe northamerica southamerica etcetera backward);
 142
 143 my %ctype =
 144 (
 145      # CT_CTYPE1
 146     "upper"  => 0x0001,
 147     "lower"  => 0x0002,
 148     "digit"  => 0x0004,
 149     "space"  => 0x0008,
 150     "punct"  => 0x0010,
 151     "cntrl"  => 0x0020,
 152     "blank"  => 0x0040,
 153     "xdigit" => 0x0080,
 154     "alpha"  => 0x0100 | 0x80000000,
 155     "defin"  => 0x0200,
 156      # CT_CTYPE3 in high 16 bits
 157     "nonspacing"    => 0x00010000,
 158     "diacritic"     => 0x00020000,
 159     "vowelmark"     => 0x00040000,
 160     "symbol"        => 0x00080000,
 161     "katakana"      => 0x00100000,
 162     "hiragana"      => 0x00200000,
 163     "halfwidth"     => 0x00400000,
 164     "fullwidth"     => 0x00800000,
 165     "ideograph"     => 0x01000000,
 166     "kashida"       => 0x02000000,
 167     "lexical"       => 0x04000000,
 168     "highsurrogate" => 0x08000000,
 169     "lowsurrogate"  => 0x10000000,
 170 );
 171
 172 my %bracket_types =
 173 (
 174     "o" => 0x0000,
 175     "c" => 0x0001,
 176 );
 177
 178 my %indic_types =
 179 (
 180     "Other"    => 0x0000,
 181     "Bindu"    => 0x0001,
 182     "Visarga"  => 0x0002,
 183     "Avagraha" => 0x0003,
 184     "Nukta"    => 0x0004,
 185     "Virama"   => 0x0005,
 186     "Vowel_Independent"  => 0x0006,
 187     "Vowel_Dependent"  => 0x0007,
 188     "Vowel"  => 0x0008,
 189     "Consonant_Placeholder"  => 0x0009,
 190     "Consonant"  => 0x000a,
 191     "Consonant_Dead"  => 0x000b,
 192     "Consonant_Succeeding_Repha" => 0x000c,
 193     "Consonant_Subjoined"  => 0x000d,
 194     "Consonant_Medial"  => 0x000e,
 195     "Consonant_Final"  => 0x000f,
 196     "Consonant_Head_Letter"  => 0x0010,
 197     "Modifying_Letter"  => 0x0011,
 198     "Tone_Letter"  => 0x0012,
 199     "Tone_Mark"  => 0x0013,
 200     "Register_Shifter"  => 0x0014,
 201     "Consonant_Preceding_Repha" => 0x0015,
 202     "Pure_Killer" => 0x0016,
 203     "Invisible_Stacker" => 0x0017,
 204     "Gemination_Mark" => 0x0018,
 205     "Cantillation_Mark" => 0x0019,
 206     "Non_Joiner" => 0x001a,
 207     "Joiner" => 0x001b,
 208     "Number_Joiner" => 0x001c,
 209     "Number" => 0x001d,
 210     "Brahmi_Joining_Number" => 0x001e,
 211     "Consonant_With_Stacker" => 0x001f,
 212     "Consonant_Prefixed" => 0x0020,
 213     "Syllable_Modifier" => 0x0021,
 214     "Consonant_Killer" => 0x0022,
 215     "Consonant_Initial_Postfixed" => 0x0023,
 216 );
 217
 218 my %matra_types =
 219 (
 220     "Right"    => 0x01,
 221     "Left"  => 0x02,
 222     "Visual_Order_Left" => 0x03,
 223     "Left_And_Right"    => 0x04,
 224     "Top"   => 0x05,
 225     "Bottom"  => 0x06,
 226     "Top_And_Bottom"  => 0x07,
 227     "Top_And_Right"  => 0x08,
 228     "Top_And_Left"  => 0x09,
 229     "Top_And_Left_And_Right"  => 0x0a,
 230     "Bottom_And_Right"  => 0x0b,
 231     "Top_And_Bottom_And_Right"  => 0x0c,
 232     "Overstruck"  => 0x0d,
 233     "Invisible"  => 0x0e,
 234     "Bottom_And_Left"  => 0x0f,
 235     "Top_And_Bottom_And_Left"  => 0x10,
 236 );
 237
 238 my %break_types =
 239 (
 240     "BK"  => 0x0001,
 241     "CR"  => 0x0002,
 242     "LF"  => 0x0003,
 243     "CM"  => 0x0004,
 244     "SG"  => 0x0005,
 245     "GL"  => 0x0006,
 246     "CB"  => 0x0007,
 247     "SP"  => 0x0008,
 248     "ZW"  => 0x0009,
 249     "NL"  => 0x000a,
 250     "WJ"  => 0x000b,
 251     "JL"  => 0x000c,
 252     "JV"  => 0x000d,
 253     "JT"  => 0x000e,
 254     "H2"  => 0x000f,
 255     "H3"  => 0x0010,
 256     "XX"  => 0x0011,
 257     "OP"  => 0x0012,
 258     "CL"  => 0x0013,
 259     "CP"  => 0x0014,
 260     "QU"  => 0x0015,
 261     "NS"  => 0x0016,
 262     "EX"  => 0x0017,
 263     "SY"  => 0x0018,
 264     "IS"  => 0x0019,
 265     "PR"  => 0x001a,
 266     "PO"  => 0x001b,
 267     "NU"  => 0x001c,
 268     "AL"  => 0x001d,
 269     "ID"  => 0x001e,
 270     "IN"  => 0x001f,
 271     "HY"  => 0x0020,
 272     "BB"  => 0x0021,
 273     "BA"  => 0x0022,
 274     "SA"  => 0x0023,
 275     "AI"  => 0x0024,
 276     "B2"  => 0x0025,
 277     "HL"  => 0x0026,
 278     "CJ"  => 0x0027,
 279     "RI"  => 0x0028,
 280     "EB"  => 0x0029,
 281     "EM"  => 0x002a,
 282     "ZWJ" => 0x002b,
 283     "AK"  => 0x002c,
 284     "AP"  => 0x002d,
 285     "AS"  => 0x002e,
 286     "VF"  => 0x002f,
 287     "VI"  => 0x0030,
 288 );
 289
 290 my %vertical_types =
 291 (
 292     "R"  => 0x0000,
 293     "U"  => 0x0001,
 294     "Tr" => 0x0002,
 295     "Tu" => 0x0003,
 296 );
 297
 298 my %categories =
 299 (
 300     "Lu" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"upper"}, # Letter, Uppercase
 301     "Ll" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"lower"}, # Letter, Lowercase
 302     "Lt" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"upper"}|$ctype{"lower"},    # Letter, Titlecase
 303     "Mn" => $ctype{"defin"}|$ctype{"nonspacing"}, # Mark, Non-Spacing
 304     "Mc" => $ctype{"defin"},                    # Mark, Spacing Combining
 305     "Me" => $ctype{"defin"},                    # Mark, Enclosing
 306     "Nd" => $ctype{"defin"}|$ctype{"digit"},    # Number, Decimal Digit
 307     "Nl" => $ctype{"defin"}|$ctype{"alpha"},    # Number, Letter
 308     "No" => $ctype{"defin"},                    # Number, Other
 309     "Zs" => $ctype{"defin"}|$ctype{"space"},    # Separator, Space
 310     "Zl" => $ctype{"defin"}|$ctype{"space"},    # Separator, Line
 311     "Zp" => $ctype{"defin"}|$ctype{"space"},    # Separator, Paragraph
 312     "Cc" => $ctype{"defin"}|$ctype{"cntrl"},    # Other, Control
 313     "Cf" => $ctype{"defin"}|$ctype{"cntrl"},    # Other, Format
 314     "Cs" => $ctype{"defin"},                    # Other, Surrogate
 315     "Co" => $ctype{"defin"},                    # Other, Private Use
 316     "Cn" => $ctype{"defin"},                    # Other, Not Assigned
 317     "Lm" => $ctype{"defin"}|$ctype{"alpha"},    # Letter, Modifier
 318     "Lo" => $ctype{"defin"}|$ctype{"alpha"},    # Letter, Other
 319     "Pc" => $ctype{"defin"}|$ctype{"punct"},    # Punctuation, Connector
 320     "Pd" => $ctype{"defin"}|$ctype{"punct"},    # Punctuation, Dash
 321     "Ps" => $ctype{"defin"}|$ctype{"punct"},    # Punctuation, Open
 322     "Pe" => $ctype{"defin"}|$ctype{"punct"},    # Punctuation, Close
 323     "Pi" => $ctype{"defin"}|$ctype{"punct"},    # Punctuation, Initial quote
 324     "Pf" => $ctype{"defin"}|$ctype{"punct"},    # Punctuation, Final quote
 325     "Po" => $ctype{"defin"}|$ctype{"punct"},    # Punctuation, Other
 326     "Sm" => $ctype{"defin"}|$ctype{"symbol"},   # Symbol, Math
 327     "Sc" => $ctype{"defin"}|$ctype{"symbol"},   # Symbol, Currency
 328     "Sk" => $ctype{"defin"}|$ctype{"symbol"},   # Symbol, Modifier
 329     "So" => $ctype{"defin"}|$ctype{"symbol"}    # Symbol, Other
 330 );
 331
 332 # a few characters need additional categories that cannot be determined automatically
 333 my %special_categories =
 334 (
 335     "xdigit" => [ ord('0')..ord('9'),ord('A')..ord('F'),ord('a')..ord('f'),
 336                   0xff10..0xff19, 0xff21..0xff26, 0xff41..0xff46 ],
 337     "space"  => [ 0x09..0x0d, 0x85 ],
 338     "blank"  => [ 0x09, 0x20, 0xa0, 0x3000, 0xfeff ],
 339     "cntrl"  => [ 0x070f, 0x200c, 0x200d,
 340                   0x200e, 0x200f, 0x202a, 0x202b, 0x202c, 0x202d, 0x202e,
 341                   0x206a, 0x206b, 0x206c, 0x206d, 0x206e, 0x206f, 0xfeff,
 342                   0xfff9, 0xfffa, 0xfffb ],
 343     "punct"  => [ 0x24, 0x2b, 0x3c..0x3e, 0x5e, 0x60, 0x7c, 0x7e, 0xa2..0xbe,
 344                   0xd7, 0xf7 ],
 345     "digit"  => [ 0xb2, 0xb3, 0xb9 ],
 346     "lower"  => [ 0xaa, 0xba, 0x2071, 0x207f ],
 347     "nonspacing" => [ 0xc0..0xc5, 0xc7..0xcf, 0xd1..0xd6, 0xd8..0xdd, 0xe0..0xe5, 0xe7..0xef,
 348                       0xf1..0xf6, 0xf8..0xfd, 0xff, 0x6de, 0x1929..0x192b, 0x302e..0x302f ],
 349     "diacritic" => [ 0x5e, 0x60, 0xb7, 0xd8, 0xf8 ],
 350     "symbol" => [ 0x09..0x0d, 0x20..0x23, 0x25, 0x26, 0x28..0x2a, 0x2c, 0x2e..0x2f, 0x3a..0x40,
 351                   0x5b..0x60, 0x7b..0x7e, 0xa0..0xa9, 0xab..0xb1, 0xb4..0xb8, 0xbb, 0xbf,
 352                   0x02b9..0x02ba, 0x02c6..0x02cf ],
 353     "halfwidth" => [ 0x20..0x7e, 0xa2..0xa3, 0xa5..0xa6, 0xac, 0xaf, 0x20a9 ],
 354     "fullwidth" => [ 0x2018..0x2019, 0x201c..0x201d, 0x3000..0x3002, 0x300c..0x300d, 0x309b..0x309c,
 355                      0x30a1..0x30ab, 0x30ad, 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9,
 356                      0x30bb, 0x30bd, 0x30bf, 0x30c1, 0x30c3, 0x30c4, 0x30c6, 0x30c8, 0x30ca..0x30cf,
 357                      0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de..0x30ed, 0x30ef, 0x30f2..0x30f3, 0x30fb,
 358                      0x3131..0x3164 ],
 359     "ideograph" => [ 0x3006..0x3007 ],
 360     "lexical" => [ 0x22, 0x24, 0x27, 0x2d, 0x2f, 0x3d, 0x40, 0x5c, 0x5e..0x60, 0x7e,
 361                    0xa8, 0xaa, 0xad, 0xaf, 0xb4, 0xb8, 0xba,
 362                    0x02b0..0x02b8, 0x02bc, 0x02c7, 0x02ca..0x02cb, 0x02cf, 0x02d8..0x02dd, 0x02e0..0x02e3,
 363                    0x037a, 0x0384..0x0385, 0x0387, 0x0559..0x055a, 0x0640, 0x1fbd..0x1fc1,
 364                    0x1fcd..0x1fcf, 0x1fdd..0x1fdf, 0x1fed..0x1fef, 0x1ffd..0x1ffe, 0x2010..0x2015,
 365                    0x2032..0x2034, 0x2038, 0x2043..0x2044, 0x207b..0x207c, 0x207f, 0x208b..0x208c,
 366                    0x2212, 0x2215..0x2216, 0x2500, 0x2504..0x2505, 0x2508..0x2509, 0x254c..0x254d,
 367                    0x3003, 0x301c, 0x3030..0x3035, 0x309b..0x309e, 0x30fd..0x30fe, 0xfe31..0xfe32,
 368                    0xfe58, 0xfe63, 0xfe66, 0xfe68..0xfe69, 0xfe6b, 0xff04, 0xff07, 0xff0d, 0xff0f,
 369                    0xff1d, 0xff20, 0xff3c, 0xff3e, 0xff40, 0xff5e ],
 370     "kashida" => [ 0x0640 ],
 371 );
 372
 373 my %directions =
 374 (
 375     "L"   => 1,    # Left-to-Right
 376     "R"   => 2,    # Right-to-Left
 377     "AL"  => 12,   # Right-to-Left Arabic
 378     "EN"  => 3,    # European Number
 379     "ES"  => 4,    # European Number Separator
 380     "ET"  => 5,    # European Number Terminator
 381     "AN"  => 6,    # Arabic Number
 382     "CS"  => 7,    # Common Number Separator
 383     "NSM" => 13,   # Non-Spacing Mark
 384     "BN"  => 14,   # Boundary Neutral
 385     "B"   => 8,    # Paragraph Separator
 386     "S"   => 9,    # Segment Separator
 387     "WS"  => 10,   # Whitespace
 388     "ON"  => 11,   # Other Neutrals
 389     "LRE" => 15,   # Left-to-Right Embedding
 390     "LRO" => 15,   # Left-to-Right Override
 391     "RLE" => 15,   # Right-to-Left Embedding
 392     "RLO" => 15,   # Right-to-Left Override
 393     "PDF" => 15,   # Pop Directional Format
 394     "LRI" => 15,   # Left-to-Right Isolate
 395     "RLI" => 15,   # Right-to-Left Isolate
 396     "FSI" => 15,   # First Strong Isolate
 397     "PDI" => 15    # Pop Directional Isolate
 398 );
 399
 400 my %c2_types =
 401 (
 402     "L"   => 1,    # C2_LEFTTORIGHT
 403     "R"   => 2,    # C2_RIGHTTOLEFT
 404     "AL"  => 2,    # C2_RIGHTTOLEFT
 405     "EN"  => 3,    # C2_EUROPENUMBER
 406     "ES"  => 4,    # C2_EUROPESEPARATOR
 407     "ET"  => 5,    # C2_EUROPETERMINATOR
 408     "AN"  => 6,    # C2_ARABICNUMBER
 409     "CS"  => 7,    # C2_COMMONSEPARATOR
 410     "NSM" => 11,   # C2_OTHERNEUTRAL
 411     "BN"  => 0,    # C2_NOTAPPLICABLE
 412     "B"   => 8,    # C2_BLOCKSEPARATOR
 413     "S"   => 9,    # C2_SEGMENTSEPARATOR
 414     "WS"  => 10,   # C2_WHITESPACE
 415     "ON"  => 11,   # C2_OTHERNEUTRAL
 416     "LRE" => 11,   # C2_OTHERNEUTRAL
 417     "LRO" => 11,   # C2_OTHERNEUTRAL
 418     "RLE" => 11,   # C2_OTHERNEUTRAL
 419     "RLO" => 11,   # C2_OTHERNEUTRAL
 420     "PDF" => 11,   # C2_OTHERNEUTRAL
 421     "LRI" => 11,   # C2_OTHERNEUTRAL
 422     "RLI" => 11,   # C2_OTHERNEUTRAL
 423     "FSI" => 11,   # C2_OTHERNEUTRAL
 424     "PDI" => 11    # C2_OTHERNEUTRAL
 425 );
 426
 427 my %bidi_types =
 428 (
 429     "ON"  => 0,    # Other Neutrals
 430     "L"   => 1,    # Left-to-Right
 431     "R"   => 2,    # Right-to-Left
 432     "AN"  => 3,    # Arabic Number
 433     "EN"  => 4,    # European Number
 434     "AL"  => 5,    # Right-to-Left Arabic
 435     "NSM" => 6,    # Non-Spacing Mark
 436     "CS"  => 7,    # Common Number Separator
 437     "ES"  => 8,    # European Number Separator
 438     "ET"  => 9,    # European Number Terminator
 439     "BN"  => 10,   # Boundary Neutral
 440     "S"   => 11,   # Segment Separator
 441     "WS"  => 12,   # Whitespace
 442     "B"   => 13,   # Paragraph Separator
 443     "RLO" => 14,   # Right-to-Left Override
 444     "RLE" => 15,   # Right-to-Left Embedding
 445     "LRO" => 16,   # Left-to-Right Override
 446     "LRE" => 17,   # Left-to-Right Embedding
 447     "PDF" => 18,   # Pop Directional Format
 448     "LRI" => 19,   # Left-to-Right Isolate
 449     "RLI" => 20,   # Right-to-Left Isolate
 450     "FSI" => 21,   # First Strong Isolate
 451     "PDI" => 22    # Pop Directional Isolate
 452 );
 453
 454 my %joining_types =
 455 (
 456    "U" => 0,           # Non_Joining
 457    "L" => 1,           # Left_Joining
 458    "R" => 2,           # Right_Joining
 459    "D" => 3,           # Dual_Joining
 460    "C" => 3,           # Join_Causing
 461    "ALAPH" => 4,       # Syriac ALAPH
 462    "DALATH RISH" => 5, # Syriac DALATH RISH group
 463    "T" => 6,           # Transparent
 464 );
 465
 466 my @locales =
 467 (
 468  { name => "", lcid => 0x0000007f, file => "root", territory => "IV", sabbrevlangname => "IVL", sopentypelang =>"dflt" },
 469  { name => "aa", sopentypelang => "AFR" },
 470  { name => "aa-DJ" },
 471  { name => "aa-ER" },
 472  { name => "aa-ET" },
 473  { name => "af", lcid => 0x00000036, oemcp => 850, sabbrevlangname => "AFK", sopentypelang => "AFK" },
 474  { name => "af-NA" },
 475  { name => "af-ZA", lcid => 0x00000436 },
 476  { name => "agq" },
 477  { name => "agq-CM" },
 478  { name => "ak", sopentypelang => "TWI" },
 479  { name => "ak-GH" },
 480  { name => "am", lcid => 0x0000005e, sabbrevlangname => "AMH" },
 481  { name => "am-ET", lcid => 0x0000045e },
 482  { name => "ar", lcid => 0x00000001, territory => "SA", oemcp => 720, group => 13 },
 483  { name => "ar-001" },
 484  { name => "ar-AE", lcid => 0x00003801, sabbrevlangname => "ARU" },
 485  { name => "ar-BH", lcid => 0x00003c01, sabbrevlangname => "ARH" },
 486  { name => "ar-DJ" },
 487  { name => "ar-DZ", lcid => 0x00001401, sabbrevlangname => "ARG", nativedigits => "0123456789" },
 488  { name => "ar-EG", lcid => 0x00000c01, sabbrevlangname => "ARE" },
 489  { name => "ar-EH" },
 490  { name => "ar-ER" },
 491  { name => "ar-IL" },
 492  { name => "ar-IQ", lcid => 0x00000801, sabbrevlangname => "ARI" },
 493  { name => "ar-JO", lcid => 0x00002c01, sabbrevlangname => "ARJ" },
 494  { name => "ar-KM" },
 495  { name => "ar-KW", lcid => 0x00003401, sabbrevlangname => "ARK" },
 496  { name => "ar-LB", lcid => 0x00003001, sabbrevlangname => "ARB" },
 497  { name => "ar-LY", lcid => 0x00001001, sabbrevlangname => "ARL", nativedigits => "0123456789" },
 498  { name => "ar-MA", lcid => 0x00001801, sabbrevlangname => "ARM", nativedigits => "0123456789" },
 499  { name => "ar-MR" },
 500  { name => "ar-OM", lcid => 0x00002001, sabbrevlangname => "ARO" },
 501  { name => "ar-PS" },
 502  { name => "ar-QA", lcid => 0x00004001, sabbrevlangname => "ARQ" },
 503  { name => "ar-SA", lcid => 0x00000401, sabbrevlangname => "ARA" },
 504  { name => "ar-SD" },
 505  { name => "ar-SO" },
 506  { name => "ar-SS" },
 507  { name => "ar-SY", lcid => 0x00002801, sabbrevlangname => "ARS" },
 508  { name => "ar-TD" },
 509  { name => "ar-TN", lcid => 0x00001c01, sabbrevlangname => "ART", nativedigits => "0123456789" },
 510  { name => "ar-YE", lcid => 0x00002401, sabbrevlangname => "ARY" },
 511  { name => "arn", lcid => 0x0000007a, oemcp => 850, ebcdiccp => 20284, slist => ",", sabbrevlangname => "MPD", sopentypelang => "MAP" },
 512  { name => "arn-CL", lcid => 0x0000047a },
 513  { name => "arn-Latn", alias => "arn" },
 514  { name => "arn-Latn-CL", alias => "arn-CL" },
 515  { name => "as", lcid => 0x0000004d, slist => ",", group => 15 },
 516  { name => "as-IN", lcid => 0x0000044d },
 517  { name => "asa" },
 518  { name => "asa-TZ" },
 519  { name => "ast" },
 520  { name => "ast-ES" },
 521  { name => "az", lcid => 0x0000002c, oemcp => 857, ebcdiccp => 20905, group => 2 },
 522  { name => "az-Cyrl", lcid => 0x0000742c, oemcp => 866, ebcdiccp => 20880, group => 5, sabbrevlangname => "AZC" },
 523  { name => "az-Cyrl-AZ", lcid => 0x0000082c },
 524  { name => "az-Latn", lcid => 0x0000782c },
 525  { name => "az-Latn-AZ", lcid => 0x0000042c },
 526  { name => "ba", lcid => 0x0000006d, oemcp => 866, group => 5, sabbrevlangname => "BAS", sopentypelang => "BSH" },
 527  { name => "ba-Cyrl", alias => "ba" },
 528  { name => "ba-Cyrl-RU", alias => "ba-RU" },
 529  { name => "ba-RU", lcid => 0x0000046d },
 530  { name => "bas" },
 531  { name => "bas-CM" },
 532  { name => "be", lcid => 0x00000023, oemcp => 866, ebcdiccp => 500, group => 5 },
 533  { name => "be-BY", lcid => 0x00000423 },
 534  { name => "bem" },
 535  { name => "bem-ZM" },
 536  { name => "bez" },
 537  { name => "bez-TZ" },
 538  { name => "bg", lcid => 0x00000002, oemcp => 866, ebcdiccp => 21025, group => 5, sabbrevlangname => "BGR", sopentypelang => "BGR" },
 539  { name => "bg-BG", lcid => 0x00000402 },
 540  { name => "bin", lcid => 0x00000066, oemcp => 850, dir => "exemplars", sabbrevlangname => "ZZZ", sopentypelang => "EDO" },
 541  { name => "bin-NG", lcid => 0x00000466, file => "bin", dir => "exemplars" },
 542  { name => "bm", sopentypelang => "BMB" },
 543  { name => "bm-Latn", file => "bm" },
 544  { name => "bm-Latn-ML", file => "bm_ML" },
 545  { name => "bm-ML", alias => "bm-Latn-ML" },
 546  { name => "bn", lcid => 0x00000045, slist => ",", group => 15, sabbrevlangname => "BNB" },
 547  { name => "bn-BD", lcid => 0x00000845 },
 548  { name => "bn-IN", lcid => 0x00000445, sabbrevlangname => "BNG" },
 549  { name => "bo", lcid => 0x00000051, slist => ",", group => 15, sabbrevlangname => "BOB", sopentypelang => "TIB" },
 550  { name => "bo-CN", lcid => 0x00000451 },
 551  { name => "bo-IN", slist => "," },
 552  { name => "bo-Tibt", alias => "bo" },
 553  { name => "bo-Tibt-CN", alias => "bo-CN" },
 554  { name => "bo-Tibt-IN", alias => "bo-IN" },
 555  { name => "br", lcid => 0x0000007e, oemcp => 850, ebcdiccp => 20297 },
 556  { name => "br-FR", lcid => 0x0000047e },
 557  { name => "br-Latn", alias => "br" },
 558  { name => "br-Latn-FR", alias => "br-FR" },
 559  { name => "brx" },
 560  { name => "brx-IN" },
 561  { name => "bs", lcid => 0x0000781a, oemcp => 852, maccp => 10082, ebcdiccp => 870, group => 2, sabbrevlangname => "BSB" },
 562  { name => "bs-Cyrl", lcid => 0x0000641a, oemcp => 855, group => 5, sabbrevlangname => "BSC" },
 563  { name => "bs-Cyrl-BA", lcid => 0x0000201a },
 564  { name => "bs-Latn", lcid => 0x0000681a },
 565  { name => "bs-Latn-BA", lcid => 0x0000141a },
 566  { name => "byn", sopentypelang => "BIL" },
 567  { name => "byn-ER" },
 568  { name => "ca", lcid => 0x00000003, oemcp => 850 },
 569  { name => "ca-AD", maccp => 65001 },
 570  { name => "ca-ES", lcid => 0x00000403 },
 571  { name => "ca-ES-valencia", lcid => 0x00000803, file => "ca_ES_VALENCIA", sabbrevlangname => "VAL" },
 572  { name => "ca-FR", maccp => 65001 },
 573  { name => "ca-IT", maccp => 65001 },
 574  { name => "ccp" },
 575  { name => "ccp-BD", alias => "ccp-Cakm-BD" },
 576  { name => "ccp-Cakm", file => "ccp" },
 577  { name => "ccp-Cakm-BD", file => "ccp_BD" },
 578  { name => "ccp-Cakm-IN", file => "ccp_IN" },
 579  { name => "ccp-IN", alias => "ccp-Cakm-IN" },
 580  { name => "ce" },
 581  { name => "ce-RU" },
 582  { name => "ceb" },
 583  { name => "ceb-Latn", file => "ceb" },
 584  { name => "ceb-Latn-PH", file => "ceb_PH" },
 585  { name => "ceb-PH", alias => "ceb-Latn-PH" },
 586  { name => "cgg" },
 587  { name => "cgg-UG" },
 588  { name => "chr", lcid => 0x0000005c, slist => ",", sabbrevlangname => "CRE" },
 589  { name => "chr-Cher", lcid => 0x00007c5c, file => "chr" },
 590  { name => "chr-Cher-US", lcid => 0x0000045c, file => "chr_US" },
 591  { name => "chr-US", alias => "chr-Cher-US" },
 592  { name => "ckb", alias => "ku" },
 593  { name => "ckb-IQ", alias => "ku-Arab-IQ" },
 594  { name => "ckb-IR", alias => "ku-Arab-IR" },
 595  { name => "co", lcid => 0x00000083, oemcp => 850, ebcdiccp => 20297 },
 596  { name => "co-FR", lcid => 0x00000483 },
 597  { name => "co-Latn", alias => "co" },
 598  { name => "co-Latn-FR", alias => "co-FR" },
 599  { name => "cs", lcid => 0x00000005, oemcp => 852, group => 2, sabbrevlangname => "CSY", sopentypelang => "CSY" },
 600  { name => "cs-CZ", lcid => 0x00000405 },
 601  { name => "cu", sopentypelang => "CSL" },
 602  { name => "cu-RU" },
 603  { name => "cy", lcid => 0x00000052, oemcp => 850, ebcdiccp => 20285, sabbrevlangname => "CYM", sopentypelang => "WEL" },
 604  { name => "cy-GB", lcid => 0x00000452 },
 605  { name => "da", lcid => 0x00000006, oemcp => 850, ebcdiccp => 20277 },
 606  { name => "da-DK", lcid => 0x00000406 },
 607  { name => "da-GL", maccp => 65001 },
 608  { name => "dav" },
 609  { name => "dav-KE" },
 610  { name => "de", lcid => 0x00000007, oemcp => 850, ebcdiccp => 20273 },
 611  { name => "de-AT", lcid => 0x00000c07, sabbrevlangname => "DEA" },
 612  { name => "de-BE" },
 613  { name => "de-CH", lcid => 0x00000807, sabbrevlangname => "DES" },
 614  { name => "de-DE", lcid => 0x00000407 },
 615  { name => "de-DE_phoneb", lcid => 0x00010407, alias => "de-DE" },
 616  { name => "de-DE-u-co-phonebk", alias => "de-DE_phoneb" },
 617  { name => "de-IT", oemcp => 65001 },
 618  { name => "de-LI", lcid => 0x00001407, sabbrevlangname => "DEC" },
 619  { name => "de-LU", lcid => 0x00001007, sabbrevlangname => "DEL" },
 620  { name => "dje", sopentypelang => "DJR" },
 621  { name => "dje-NE" },
 622  { name => "doi", sopentypelang => "DGR" },
 623  { name => "doi-IN", alias => "doi-Deva-IN" },
 624  { name => "doi-Deva", file => "doi" },
 625  { name => "doi-Deva-IN", file => "doi_IN" },
 626  { name => "dsb", lcid => 0x00007c2e, sparent => "hsb", oemcp => 850, ebcdiccp => 870, sabbrevlangname => "DSB", sopentypelang => "LSB" },
 627  { name => "dsb-DE", lcid => 0x0000082e },
 628  { name => "dua" },
 629  { name => "dua-CM" },
 630  { name => "dv", lcid => 0x00000065, slist => "\x{060c}", group => 13, nativedigits => "0123456789" },
 631  { name => "dv-MV", lcid => 0x00000465 },
 632  { name => "dyo" },
 633  { name => "dyo-SN" },
 634  { name => "dz", sopentypelang => "DZN" },
 635  { name => "dz-BT", lcid => 0x00000c51, sabbrevlangname => "ZZZ" },
 636  { name => "ebu" },
 637  { name => "ebu-KE" },
 638  { name => "ee" },
 639  { name => "ee-GH" },
 640  { name => "ee-TG" },
 641  { name => "el", lcid => 0x00000008, oemcp => 737, group => 4 },
 642  { name => "el-CY" },
 643  { name => "el-GR", lcid => 0x00000408 },
 644  { name => "en", lcid => 0x00000009, oemcp => 437, slist => ",", sabbrevlangname => "ENU" },
 645  { name => "en-001", oemcp => 850 },
 646  { name => "en-029", lcid => 0x00002409, file => "en", oemcp => 850, sabbrevlangname => "ENB" },
 647  { name => "en-150", oemcp => 65001 },
 648  { name => "en-AE", lcid => 0x00004c09, oemcp => 65001, sabbrevlangname => "ZZZ" },
 649  { name => "en-AG", oemcp => 850 },
 650  { name => "en-AI", oemcp => 850 },
 651  { name => "en-AS", oemcp => 850 },
 652  { name => "en-AT", oemcp => 65001 },
 653  { name => "en-AU", lcid => 0x00000c09, oemcp => 850, sabbrevlangname => "ENA" },
 654  { name => "en-BB", oemcp => 850 },
 655  { name => "en-BE", oemcp => 850 },
 656  { name => "en-BI", oemcp => 65001 },
 657  { name => "en-BM", oemcp => 850 },
 658  { name => "en-BS", oemcp => 850 },
 659  { name => "en-BW", oemcp => 850 },
 660  { name => "en-BZ", lcid => 0x00002809, oemcp => 850, sabbrevlangname => "ENL" },
 661  { name => "en-CA", lcid => 0x00001009, oemcp => 850, ebcdiccp => 37, sabbrevlangname => "ENC" },
 662  { name => "en-CC", oemcp => 850 },
 663  { name => "en-CH", oemcp => 65001 },
 664  { name => "en-CK", oemcp => 850 },
 665  { name => "en-CM", oemcp => 850 },
 666  { name => "en-CX", oemcp => 850 },
 667  { name => "en-CY", oemcp => 65001 },
 668  { name => "en-DE", oemcp => 65001 },
 669  { name => "en-DG", oemcp => 850 },
 670  { name => "en-DK", oemcp => 65001 },
 671  { name => "en-DM", oemcp => 850 },
 672  { name => "en-ER", oemcp => 850 },
 673  { name => "en-FI", oemcp => 65001 },
 674  { name => "en-FJ", oemcp => 850 },
 675  { name => "en-FK", oemcp => 850 },
 676  { name => "en-FM", oemcp => 850 },
 677  { name => "en-GB", lcid => 0x00000809, oemcp => 850, ebcdiccp => 20285, sabbrevlangname => "ENG" },
 678  { name => "en-GD", oemcp => 850 },
 679  { name => "en-GG", oemcp => 850 },
 680  { name => "en-GH", oemcp => 850 },
 681  { name => "en-GI", oemcp => 850 },
 682  { name => "en-GM", oemcp => 850 },
 683  { name => "en-GU", oemcp => 850 },
 684  { name => "en-GY", oemcp => 850 },
 685  { name => "en-HK", lcid => 0x00003c09, oemcp => 850, sabbrevlangname => "ENH" },
 686  { name => "en-ID", lcid => 0x00003809, file => "en", oemcp => 850, sabbrevlangname => "ZZZ" },
 687  { name => "en-IE", lcid => 0x00001809, oemcp => 850, sabbrevlangname => "ENI" },
 688  { name => "en-IL", oemcp => 65001 },
 689  { name => "en-IM", oemcp => 850 },
 690  { name => "en-IN", lcid => 0x00004009, sabbrevlangname => "ENN" },
 691  { name => "en-IO", oemcp => 850 },
 692  { name => "en-JE", oemcp => 850 },
 693  { name => "en-JM", lcid => 0x00002009, oemcp => 850, sabbrevlangname => "ENJ" },
 694  { name => "en-KE", oemcp => 850 },
 695  { name => "en-KI", oemcp => 850 },
 696  { name => "en-KN", oemcp => 850 },
 697  { name => "en-KY", oemcp => 850 },
 698  { name => "en-LC", oemcp => 850 },
 699  { name => "en-LR", oemcp => 850 },
 700  { name => "en-LS", oemcp => 850 },
 701  { name => "en-MG", oemcp => 850 },
 702  { name => "en-MH", oemcp => 850 },
 703  { name => "en-MO", oemcp => 850 },
 704  { name => "en-MP", oemcp => 850 },
 705  { name => "en-MS", oemcp => 850 },
 706  { name => "en-MT", oemcp => 850 },
 707  { name => "en-MU", oemcp => 850 },
 708  { name => "en-MW", oemcp => 850 },
 709  { name => "en-MY", lcid => 0x00004409, sabbrevlangname => "ENM" },
 710  { name => "en-NA", oemcp => 850 },
 711  { name => "en-NF", oemcp => 850 },
 712  { name => "en-NG", oemcp => 850 },
 713  { name => "en-NL", oemcp => 65001 },
 714  { name => "en-NR", oemcp => 850 },
 715  { name => "en-NU", oemcp => 850 },
 716  { name => "en-NZ", lcid => 0x00001409, oemcp => 850, sabbrevlangname => "ENZ" },
 717  { name => "en-PG", oemcp => 850 },
 718  { name => "en-PH", lcid => 0x00003409, ebcdiccp => 500, sabbrevlangname => "ENP" },
 719  { name => "en-PK", oemcp => 850 },
 720  { name => "en-PN", oemcp => 850 },
 721  { name => "en-PR", oemcp => 850 },
 722  { name => "en-PW", oemcp => 850 },
 723  { name => "en-RW", oemcp => 850 },
 724  { name => "en-SB", oemcp => 850 },
 725  { name => "en-SC", oemcp => 850 },
 726  { name => "en-SD", oemcp => 850 },
 727  { name => "en-SE", oemcp => 65001 },
 728  { name => "en-SG", lcid => 0x00004809, sabbrevlangname => "ENE" },
 729  { name => "en-SH", oemcp => 850 },
 730  { name => "en-SI", oemcp => 65001 },
 731  { name => "en-SL", oemcp => 850 },
 732  { name => "en-SS", oemcp => 850 },
 733  { name => "en-SX", oemcp => 850 },
 734  { name => "en-SZ", oemcp => 850 },
 735  { name => "en-TC", oemcp => 850 },
 736  { name => "en-TK", oemcp => 850 },
 737  { name => "en-TO", oemcp => 850 },
 738  { name => "en-TT", lcid => 0x00002c09, oemcp => 850, sabbrevlangname => "ENT" },
 739  { name => "en-TV", oemcp => 850 },
 740  { name => "en-TZ", oemcp => 850 },
 741  { name => "en-UG", oemcp => 850 },
 742  { name => "en-UM", oemcp => 850 },
 743  { name => "en-US", lcid => 0x00000409 },
 744  { name => "en-VC", oemcp => 850 },
 745  { name => "en-VG", oemcp => 850 },
 746  { name => "en-VI", oemcp => 850 },
 747  { name => "en-VU", oemcp => 850 },
 748  { name => "en-WS", oemcp => 850 },
 749  { name => "en-ZA", lcid => 0x00001c09, ebcdiccp => 500, sabbrevlangname => "ENS" },
 750  { name => "en-ZM", oemcp => 850 },
 751  { name => "en-ZW", lcid => 0x00003009, ebcdiccp => 500, sabbrevlangname => "ENW" },
 752  { name => "eo", sopentypelang => "NTO" },
 753  { name => "eo-001" },
 754  { name => "es", lcid => 0x0000000a, oemcp => 850, ebcdiccp => 20284, sabbrevlangname => "ESP", sopentypelang => "ESP" },
 755  { name => "es-419", lcid => 0x0000580a, sabbrevlangname => "ESJ" },
 756  { name => "es-AR", lcid => 0x00002c0a, sabbrevlangname => "ESS" },
 757  { name => "es-BO", lcid => 0x0000400a, sabbrevlangname => "ESB" },
 758  { name => "es-BR", oemcp => 65001 },
 759  { name => "es-BZ", oemcp => 65001 },
 760  { name => "es-CL", lcid => 0x0000340a, sabbrevlangname => "ESL" },
 761  { name => "es-CO", lcid => 0x0000240a, sabbrevlangname => "ESO" },
 762  { name => "es-CR", lcid => 0x0000140a, sabbrevlangname => "ESC" },
 763  { name => "es-CU", lcid => 0x00005c0a, sabbrevlangname => "ESK" },
 764  { name => "es-DO", lcid => 0x00001c0a, sabbrevlangname => "ESD" },
 765  { name => "es-EA" },
 766  { name => "es-EC", lcid => 0x0000300a, sabbrevlangname => "ESF" },
 767  { name => "es-ES", lcid => 0x00000c0a, sabbrevlangname => "ESN" },
 768  { name => "es-ES_tradnl", lcid => 0x0000040a, file => "es_ES" },
 769  { name => "es-ES-u-co-trad", alias => "es-ES_tradnl" },
 770  { name => "es-GQ" },
 771  { name => "es-GT", lcid => 0x0000100a, sabbrevlangname => "ESG" },
 772  { name => "es-HN", lcid => 0x0000480a, sabbrevlangname => "ESH" },
 773  { name => "es-IC" },
 774  { name => "es-MX", lcid => 0x0000080a, sabbrevlangname => "ESM" },
 775  { name => "es-NI", lcid => 0x00004c0a, sabbrevlangname => "ESI" },
 776  { name => "es-PA", lcid => 0x0000180a, sabbrevlangname => "ESA" },
 777  { name => "es-PE", lcid => 0x0000280a, sabbrevlangname => "ESR" },
 778  { name => "es-PH" },
 779  { name => "es-PR", lcid => 0x0000500a, sabbrevlangname => "ESU" },
 780  { name => "es-PY", lcid => 0x00003c0a, sabbrevlangname => "ESZ" },
 781  { name => "es-SV", lcid => 0x0000440a, sabbrevlangname => "ESE" },
 782  { name => "es-US", lcid => 0x0000540a, sabbrevlangname => "EST" },
 783  { name => "es-UY", lcid => 0x0000380a, sabbrevlangname => "ESY" },
 784  { name => "es-VE", lcid => 0x0000200a, sabbrevlangname => "ESV" },
 785  { name => "et", lcid => 0x00000025, oemcp => 775, group => 3, sabbrevlangname => "ETI", sopentypelang => "ETI" },
 786  { name => "et-EE", lcid => 0x00000425 },
 787  { name => "eu", lcid => 0x0000002d, oemcp => 850, maccp => 65001, sabbrevlangname => "EUQ", sopentypelang => "EUQ" },
 788  { name => "eu-ES", lcid => 0x0000042d },
 789  { name => "ewo" },
 790  { name => "ewo-CM" },
 791  { name => "fa", lcid => 0x00000029, inegnumber => 3, oemcp => 720, slist => "\x{061b}", group => 13, sabbrevlangname => "FAR", sopentypelang => "FAR" },
 792  { name => "fa-AF", alias => "prs-AF" },
 793  { name => "fa-IR", lcid => 0x00000429 },
 794  { name => "ff", lcid => 0x00000067, oemcp => 850, ebcdiccp => 20297 },
 795  { name => "ff-CM", alias => "ff-Latn-CM" },
 796  { name => "ff-GN", alias => "ff-Latn-GN" },
 797  { name => "ff-MR", alias => "ff-Latn-MR" },
 798  { name => "ff-NG", alias => "ff-Latn-NG" },
 799  { name => "ff-SN", alias => "ff-Latn-SN" },
 800  { name => "ff-Adlm", oemcp => 65001 },
 801  { name => "ff-Adlm-BF" },
 802  { name => "ff-Adlm-CM" },
 803  { name => "ff-Adlm-GH" },
 804  { name => "ff-Adlm-GM" },
 805  { name => "ff-Adlm-GN" },
 806  { name => "ff-Adlm-GW" },
 807  { name => "ff-Adlm-LR" },
 808  { name => "ff-Adlm-MR" },
 809  { name => "ff-Adlm-NE" },
 810  { name => "ff-Adlm-NG" },
 811  { name => "ff-Adlm-SL" },
 812  { name => "ff-Adlm-SN" },
 813  { name => "ff-Latn", lcid => 0x00007c67 },
 814  { name => "ff-Latn-BF", oemcp => 65001 },
 815  { name => "ff-Latn-CM" },
 816  { name => "ff-Latn-GH", oemcp => 65001 },
 817  { name => "ff-Latn-GM", oemcp => 65001 },
 818  { name => "ff-Latn-GN" },
 819  { name => "ff-Latn-GW", oemcp => 65001 },
 820  { name => "ff-Latn-LR", oemcp => 65001 },
 821  { name => "ff-Latn-MR" },
 822  { name => "ff-Latn-NE", oemcp => 65001 },
 823  { name => "ff-Latn-NG", lcid => 0x00000467, sabbrevlangname => "ZZZ" },
 824  { name => "ff-Latn-SL", oemcp => 65001 },
 825  { name => "ff-Latn-SN", lcid => 0x00000867 },
 826  { name => "fi", lcid => 0x0000000b, oemcp => 850, ebcdiccp => 20278 },
 827  { name => "fi-FI", lcid => 0x0000040b },
 828  { name => "fil", lcid => 0x00000064, oemcp => 437, ebcdiccp => 500, sabbrevlangname => "FPO", sopentypelang => "PIL" },
 829  { name => "fil-PH", lcid => 0x00000464 },
 830  { name => "fil-Latn", alias => "fil" },
 831  { name => "fil-Latn-PH", alias => "fil-PH" },
 832  { name => "fo", lcid => 0x00000038, oemcp => 850, maccp => 10079, ebcdiccp => 20277, sabbrevlangname => "FOS", sopentypelang => "FOS" },
 833  { name => "fo-DK", oemcp => 65001, maccp => 65001 },
 834  { name => "fo-FO", lcid => 0x00000438 },
 835  { name => "fr", lcid => 0x0000000c, oemcp => 850, ebcdiccp => 20297 },
 836  { name => "fr-029", lcid => 0x00001c0c, file => "fr", sabbrevlangname => "ZZZ" },
 837  { name => "fr-BE", lcid => 0x0000080c, sabbrevlangname => "FRB" },
 838  { name => "fr-BF" },
 839  { name => "fr-BI" },
 840  { name => "fr-BJ" },
 841  { name => "fr-BL" },
 842  { name => "fr-CA", lcid => 0x00000c0c, sabbrevlangname => "FRC" },
 843  { name => "fr-CD", lcid => 0x0000240c, sabbrevlangname => "FRD" },
 844  { name => "fr-CF" },
 845  { name => "fr-CG" },
 846  { name => "fr-CH", lcid => 0x0000100c, sabbrevlangname => "FRS" },
 847  { name => "fr-CI", lcid => 0x0000300c, sabbrevlangname => "FRI" },
 848  { name => "fr-CM", lcid => 0x00002c0c, sabbrevlangname => "FRE" },
 849  { name => "fr-DJ" },
 850  { name => "fr-DZ" },
 851  { name => "fr-FR", lcid => 0x0000040c },
 852  { name => "fr-GA" },
 853  { name => "fr-GF" },
 854  { name => "fr-GN" },
 855  { name => "fr-GP" },
 856  { name => "fr-GQ" },
 857  { name => "fr-HT", lcid => 0x00003c0c, sabbrevlangname => "FRH" },
 858  { name => "fr-KM" },
 859  { name => "fr-LU", lcid => 0x0000140c, sabbrevlangname => "FRL" },
 860  { name => "fr-MA", lcid => 0x0000380c, sabbrevlangname => "FRO" },
 861  { name => "fr-MC", lcid => 0x0000180c, sabbrevlangname => "FRM" },
 862  { name => "fr-MF" },
 863  { name => "fr-MG" },
 864  { name => "fr-ML", lcid => 0x0000340c, sabbrevlangname => "FRF" },
 865  { name => "fr-MQ" },
 866  { name => "fr-MR" },
 867  { name => "fr-MU" },
 868  { name => "fr-NC" },
 869  { name => "fr-NE" },
 870  { name => "fr-PF" },
 871  { name => "fr-PM" },
 872  { name => "fr-RE", lcid => 0x0000200c, sabbrevlangname => "FRR" },
 873  { name => "fr-RW" },
 874  { name => "fr-SC" },
 875  { name => "fr-SN", lcid => 0x0000280c, sabbrevlangname => "FRN" },
 876  { name => "fr-SY" },
 877  { name => "fr-TD" },
 878  { name => "fr-TG" },
 879  { name => "fr-TN" },
 880  { name => "fr-VU" },
 881  { name => "fr-WF" },
 882  { name => "fr-YT" },
 883  { name => "fur", sopentypelang => "FRL" },
 884  { name => "fur-IT" },
 885  { name => "fuv-NG", alias => "ff-Latn-NG" },
 886  { name => "fy", lcid => 0x00000062, oemcp => 850, sabbrevlangname => "FYN", sopentypelang => "FRI" },
 887  { name => "fy-NL", lcid => 0x00000462 },
 888  { name => "ga", lcid => 0x0000003c, oemcp => 850, sabbrevlangname => "IRE", sopentypelang => "IRI" },
 889  { name => "ga-GB", oemcp => 65001 },
 890  { name => "ga-IE", lcid => 0x0000083c },
 891  { name => "gd", lcid => 0x00000091, oemcp => 850, ebcdiccp => 20285, sopentypelang => "GAE" },
 892  { name => "gd-GB", lcid => 0x00000491 },
 893  { name => "gd-Latn", alias => "gd" },
 894  { name => "gl", lcid => 0x00000056, oemcp => 850, sabbrevlangname => "GLC", sopentypelang => "GAL" },
 895  { name => "gl-ES", lcid => 0x00000456 },
 896  { name => "gn", lcid => 0x00000074, oemcp => 850, ebcdiccp => 20284, slist => ",", sopentypelang => "GUA" },
 897  { name => "gn-PY", lcid => 0x00000474 },
 898  { name => "gsw", lcid => 0x00000084, oemcp => 850, ebcdiccp => 20297, sabbrevlangname => "ZZZ", sopentypelang => "ALS" },
 899  { name => "gsw-CH" },
 900  { name => "gsw-FR", lcid => 0x00000484, sabbrevlangname => "GSW" },
 901  { name => "gsw-LI" },
 902  { name => "gu", lcid => 0x00000047, slist => ",", group => 15 },
 903  { name => "gu-IN", lcid => 0x00000447 },
 904  { name => "guz" },
 905  { name => "guz-KE" },
 906  { name => "gv", sopentypelang => "MNX" },
 907  { name => "gv-GB", file => "gv" },
 908  { name => "gv-IM" },
 909  { name => "ha", lcid => 0x00000068, oemcp => 437 },
 910  { name => "ha-GH", alias => "ha-Latn-GH" },
 911  { name => "ha-Latn", lcid => 0x00007c68, file => "ha" },
 912  { name => "ha-Latn-GH", file => "ha_GH", ebcdiccp => 500 },
 913  { name => "ha-Latn-NE", file => "ha_NE", ebcdiccp => 500 },
 914  { name => "ha-Latn-NG", lcid => 0x00000468, file => "ha_NG" },
 915  { name => "ha-NE", alias => "ha-Latn-NE" },
 916  { name => "ha-NG", alias => "ha-Latn-NG" },
 917  { name => "haw", lcid => 0x00000075, oemcp => 437 },
 918  { name => "haw-Latn", alias => "haw" },
 919  { name => "haw-Latn-US", alias => "haw-US" },
 920  { name => "haw-US", lcid => 0x00000475 },
 921  { name => "he", lcid => 0x0000000d, oemcp => 862, slist => ",", group => 12, sopentypelang => "IWR" },
 922  { name => "he-IL", lcid => 0x0000040d },
 923  { name => "hi", lcid => 0x00000039, slist => ",", group => 15 },
 924  { name => "hi-IN", lcid => 0x00000439 },
 925  { name => "hr", lcid => 0x0000001a, inegnumber => 2, oemcp => 852, maccp => 10082, group => 2 },
 926  { name => "hr-BA", lcid => 0x0000101a, ebcdiccp => 870, inegnumber => 1, sabbrevlangname => "HRB" },
 927  { name => "hr-HR", lcid => 0x0000041a },
 928  { name => "hsb", lcid => 0x0000002e, oemcp => 850, ebcdiccp => 870, sopentypelang => "USB" },
 929  { name => "hsb-DE", lcid => 0x0000042e },
 930  { name => "hu", lcid => 0x0000000e, oemcp => 852, group => 2 },
 931  { name => "hu-HU", lcid => 0x0000040e },
 932  { name => "hu-HU_technl", lcid => 0x0001040e, alias => "hu-HU" },
 933  { name => "hy", lcid => 0x0000002b, slist => ",", group => 17 },
 934  { name => "hy-AM", lcid => 0x0000042b },
 935  { name => "ia" },
 936  { name => "ia-001" },
 937 ## name => "ibb", lcid => 0x00000069 },
 938 ## name => "ibb-NG", lcid => 0x00000469 },
 939  { name => "id", lcid => 0x00000021, oemcp => 850 },
 940  { name => "id-ID", lcid => 0x00000421 },
 941  { name => "ig", lcid => 0x00000070, oemcp => 437 },
 942  { name => "ig-Latn", alias => "ig" },
 943  { name => "ig-Latn-NG", alias => "ig-NG" },
 944  { name => "ig-NG", lcid => 0x00000470 },
 945  { name => "ii", lcid => 0x00000078, group => 9, sopentypelang => "YIM" },
 946  { name => "ii-CN", lcid => 0x00000478 },
 947  { name => "ii-Yiii", alias => "ii" },
 948  { name => "ii-Yiii-CN", alias => "ii-CN" },
 949  { name => "is", lcid => 0x0000000f, oemcp => 850, maccp => 10079, ebcdiccp => 20871 },
 950  { name => "is-IS", lcid => 0x0000040f },
 951  { name => "it", lcid => 0x00000010, oemcp => 850, ebcdiccp => 20280 },
 952  { name => "it-CH", lcid => 0x00000810, ebcdiccp => 500, sabbrevlangname => "ITS" },
 953  { name => "it-IT", lcid => 0x00000410 },
 954  { name => "it-SM" },
 955  { name => "it-VA", oemcp => 65001 },
 956  { name => "iu", lcid => 0x0000005d, oemcp => 437, slist => ",", sortlocale => "iu-Latn-CA", sabbrevlangname => "IUK", sopentypelang => "INU" },
 957  { name => "iu-Cans", lcid => 0x0000785d, file => "iu", oemcp => 65001, sabbrevlangname => "IUS" },
 958  { name => "iu-Cans-CA", lcid => 0x0000045d, file => "iu_CA" },
 959  { name => "iu-Latn", lcid => 0x00007c5d },
 960  { name => "iu-Latn-CA", lcid => 0x0000085d },
 961  { name => "ja", lcid => 0x00000011, ireadinglayout => 2, oemcp => 932, slist => ",",  sscripts => "Hani Hira Jpan Kana", group => 7, sopentypelang => "JAN" },
 962  { name => "ja-JP", lcid => 0x00000411 },
 963  { name => "ja-JP_radstr", lcid => 0x00040411, alias => "ja-JP" },
 964  { name => "ja-JP-u-co-unihan", alias => "ja-JP_radstr" },
 965  { name => "jgo" },
 966  { name => "jgo-CM" },
 967  { name => "jmc" },
 968  { name => "jmc-TZ" },
 969  { name => "jv", oemcp => 850, nativedigits => "0123456789" },
 970  { name => "jv-ID", alias => "jv-Latn-ID" },
 971 ## name => "jv-Java" },
 972 ## name => "jv-Java-ID" },
 973  { name => "jv-Latn", file => "jv" },
 974  { name => "jv-Latn-ID", file => "jv_ID" },
 975  { name => "ka", lcid => 0x00000037, group => 16 },
 976  { name => "ka-GE", lcid => 0x00000437 },
 977  { name => "ka-GE_modern", lcid => 0x00010437, alias => "ka-GE" },
 978  { name => "kab", sopentypelang => "KAB0" },
 979  { name => "kab-DZ" },
 980  { name => "kam", sopentypelang => "KMB" },
 981  { name => "kam-KE" },
 982  { name => "kde" },
 983  { name => "kde-TZ" },
 984  { name => "kea" },
 985  { name => "kea-CV" },
 986  { name => "kgp" },
 987  { name => "kgp-BR" },
 988  { name => "khq" },
 989  { name => "khq-ML" },
 990  { name => "ki" },
 991  { name => "ki-KE" },
 992  { name => "kk", lcid => 0x0000003f, group => 5, sabbrevlangname => "KKZ" },
 993  { name => "kk-Cyrl", alias => "kk" },
 994  { name => "kk-Cyrl-KZ", alias => "kk-KZ" },
 995  { name => "kk-KZ", lcid => 0x0000043f },
 996  { name => "kkj" },
 997  { name => "kkj-CM" },
 998  { name => "kl", lcid => 0x0000006f, oemcp => 850, ebcdiccp => 20277, sopentypelang => "GRN" },
 999  { name => "kl-GL", lcid => 0x0000046f },
1000  { name => "kln", sopentypelang => "KAL" },
1001  { name => "kln-KE" },
1002  { name => "km", lcid => 0x00000053, inegnumber => 2, slist => ",", group => 15 },
1003  { name => "km-KH", lcid => 0x00000453 },
1004  { name => "kn", lcid => 0x0000004b, slist => ",", group => 15, sabbrevlangname => "KDI" },
1005  { name => "kn-IN", lcid => 0x0000044b },
1006  { name => "ko", lcid => 0x00000012, ireadinglayout => 2, slist => ",", oemcp => 949, ebcdiccp => 20833, sscripts => "Hang Hani Kore", group => 8 },
1007  { name => "ko-KP", oemcp => 65001 },
1008  { name => "ko-KR", lcid => 0x00000412 },
1009  { name => "kok", lcid => 0x00000057, slist => ",", group => 15, sabbrevlangname => "KNK" },
1010  { name => "kok-IN", lcid => 0x00000457 },
1011  { name => "kr", lcid => 0x00000071, sortlocale => "kr-Latn-NG", oemcp => 850, dir => "exemplars", sabbrevlangname => "ZZZ", sopentypelang => "KNR" },
1012  { name => "kr-Latn", file => "kr", dir => "exemplars" },
1013  { name => "kr-Latn-NG", lcid => 0x00000471, file => "kr", dir => "exemplars" },
1014  { name => "kr-NG", alias => "kr-Latn-NG" },
1015  { name => "ks", lcid => 0x00000060, group => 15, sabbrevlangname => "ZZZ", sopentypelang => "KSH" },
1016  { name => "ks-Arab", lcid => 0x00000460 },
1017  { name => "ks-Arab-IN" },
1018  { name => "ks-Deva", slist => "," },
1019  { name => "ks-Deva-IN", lcid => 0x00000860 },
1020  { name => "ks-IN", alias => "ks-Arab-IN" },
1021  { name => "ksb" },
1022  { name => "ksb-TZ" },
1023  { name => "ksf" },
1024  { name => "ksf-CM" },
1025  { name => "ksh", sopentypelang => "KSH0" },
1026  { name => "ksh-DE" },
1027  { name => "ku", lcid => 0x00000092, file => "ckb", slist => "\x{061b}", sortlocale => "ku-Arab-IQ", oemcp => 720 },
1028  { name => "ku-Arab", lcid => 0x00007c92, file => "ckb", group => 13 },
1029  { name => "ku-Arab-IQ", lcid => 0x00000492, file => "ckb_IQ" },
1030  { name => "ku-Arab-IR", file => "ckb_IR", oemcp => 65001 },
1031  { name => "kw" },
1032  { name => "kw-GB" },
1033  { name => "ky", lcid => 0x00000040, oemcp => 866, group => 5, sabbrevlangname => "KYR" },
1034  { name => "ky-Cyrl", alias => "ky" },
1035  { name => "ky-Cyrl-KG", alias => "ky-KG" },
1036  { name => "ky-KG", lcid => 0x00000440 },
1037  { name => "la", lcid => 0x00000076, oemcp => 437, slist => ",", sabbrevlangname => "ZZZ" },
1038  { name => "la-VA", lcid => 0x00000476 },
1039  { name => "la-001", alias => "la-VA" },
1040  { name => "lag" },
1041  { name => "lag-TZ" },
1042  { name => "lb", lcid => 0x0000006e, oemcp => 850, ebcdiccp => 20297, sabbrevlangname => "LBX" },
1043  { name => "lb-LU", lcid => 0x0000046e },
1044  { name => "lg" },
1045  { name => "lg-UG" },
1046  { name => "lkt" },
1047  { name => "lkt-US" },
1048  { name => "ln" },
1049  { name => "ln-AO" },
1050  { name => "ln-CD" },
1051  { name => "ln-CF" },
1052  { name => "ln-CG" },
1053  { name => "lo", lcid => 0x00000054, group => 15 },
1054  { name => "lo-LA", lcid => 0x00000454 },
1055  { name => "lrc" },
1056  { name => "lrc-IQ" },
1057  { name => "lrc-IR" },
1058  { name => "lt", lcid => 0x00000027, oemcp => 775, group => 3, sabbrevlangname => "LTH", sopentypelang => "LTH" },
1059  { name => "lt-LT", lcid => 0x00000427 },
1060  { name => "lu" },
1061  { name => "lu-CD" },
1062  { name => "luo" },
1063  { name => "luo-KE" },
1064  { name => "luy", sopentypelang => "LUH" },
1065  { name => "luy-KE" },
1066  { name => "lv", lcid => 0x00000026, oemcp => 775, group => 3, sabbrevlangname => "LVI", sopentypelang => "LVI" },
1067  { name => "lv-LV", lcid => 0x00000426 },
1068  { name => "mai" },
1069  { name => "mai-IN" },
1070  { name => "mas" },
1071  { name => "mas-KE" },
1072  { name => "mas-TZ" },
1073  { name => "mer" },
1074  { name => "mer-KE" },
1075  { name => "mfe" },
1076  { name => "mfe-MU" },
1077  { name => "mg" },
1078  { name => "mg-MG" },
1079  { name => "mgh" },
1080  { name => "mgh-MZ" },
1081  { name => "mgo" },
1082  { name => "mgo-CM" },
1083  { name => "mi", lcid => 0x00000081, slist => "," },
1084  { name => "mi-Latn", alias => "mi" },
1085  { name => "mi-Latn-NZ", alias => "mi-NZ" },
1086  { name => "mi-NZ", lcid => 0x00000481 },
1087  { name => "mk", lcid => 0x0000002f, oemcp => 866, ebcdiccp => 500, group => 5, sabbrevlangname => "MKI" },
1088  { name => "mk-MK", lcid => 0x0000042f },
1089  { name => "ml", lcid => 0x0000004c, group => 15, sabbrevlangname => "MYM", sopentypelang => "MLR" },
1090  { name => "ml-IN", lcid => 0x0000044c },
1091  { name => "mn", lcid => 0x00000050, oemcp => 866, sopentypelang => "MNG" },
1092  { name => "mn-Cyrl", lcid => 0x00007850, file => "mn", sabbrevlangname => "MNN" },
1093  { name => "mn-Cyrl-MN", alias => "mn-MN" },
1094  { name => "mn-MN", lcid => 0x00000450, sparent => "mn-Cyrl", group => 5 },
1095  { name => "mn-Mong", lcid => 0x00007c50, oemcp => 65001, slist => ",", group => 15, sabbrevlangname => "MNG", nativedigits => "0123456789" },
1096  { name => "mn-Mong-CN", lcid => 0x00000850 },
1097  { name => "mn-Mong-MN", lcid => 0x00000c50, sabbrevlangname => "MNM" },
1098  { name => "mni", lcid => 0x00000058, slist => ",", sabbrevlangname => "ZZZ" },
1099  { name => "mni-IN", lcid => 0x00000458, file => "mni_Beng_IN" },
1100  { name => "mni-Beng" },
1101  { name => "mni-Beng-IN", alias => "mni-IN" },
1102  { name => "moh", lcid => 0x0000007c, oemcp => 850, ebcdiccp => 37, slist => ",", sabbrevlangname => "MWK" },
1103  { name => "moh-CA", lcid => 0x0000047c },
1104  { name => "moh-Latn", alias => "moh" },
1105  { name => "moh-Latn-CA", alias => "moh-CA" },
1106  { name => "mr", lcid => 0x0000004e, slist => ",", group => 15 },
1107  { name => "mr-IN", lcid => 0x0000044e },
1108  { name => "ms", lcid => 0x0000003e, oemcp => 850, sabbrevlangname => "MSL", sopentypelang => "MLY" },
1109  { name => "ms-BN", lcid => 0x0000083e, sabbrevlangname => "MSB" },
1110  { name => "ms-ID" },
1111  { name => "ms-Latn", alias => "ms" },
1112  { name => "ms-Latn-BN", alias => "ms-BN" },
1113  { name => "ms-Latn-MY", alias => "ms-MY" },
1114  { name => "ms-Latn-SG", alias => "ms-SG" },
1115  { name => "ms-MY", lcid => 0x0000043e },
1116  { name => "ms-SG" },
1117  { name => "mt", lcid => 0x0000003a, sopentypelang => "MTS" },
1118  { name => "mt-MT", lcid => 0x0000043a },
1119  { name => "mua" },
1120  { name => "mua-CM" },
1121  { name => "my", lcid => 0x00000055, sopentypelang => "BRM" },
1122  { name => "my-MM", lcid => 0x00000455 },
1123  { name => "mzn" },
1124  { name => "mzn-IR" },
1125  { name => "naq" },
1126  { name => "naq-NA" },
1127  { name => "nb", lcid => 0x00007c14, oemcp => 850, ebcdiccp => 20277, sabbrevlangname => "NOR", sopentypelang => "NOR" },
1128  { name => "nb-NO", lcid => 0x00000414 },
1129  { name => "nb-SJ" },
1130  { name => "nd", sopentypelang => "NDB" },
1131  { name => "nd-ZW" },
1132  { name => "nds" },
1133  { name => "nds-DE" },
1134  { name => "nds-NL" },
1135  { name => "ne", lcid => 0x00000061, slist => "," },
1136  { name => "ne-IN", lcid => 0x00000861, sabbrevlangname => "NEI" },
1137  { name => "ne-NP", lcid => 0x00000461, group => 15 },
1138  { name => "nl", lcid => 0x00000013, oemcp => 850 },
1139  { name => "nl-AW" },
1140  { name => "nl-BE", lcid => 0x00000813, sabbrevlangname => "NLB" },
1141  { name => "nl-BQ" },
1142  { name => "nl-CW" },
1143  { name => "nl-NL", lcid => 0x00000413 },
1144  { name => "nl-SR" },
1145  { name => "nl-SX" },
1146  { name => "nmg" },
1147  { name => "nmg-CM" },
1148  { name => "nn", lcid => 0x00007814, oemcp => 850, ebcdiccp => 20277, sabbrevlangname => "NON", sopentypelang => "NYN" },
1149  { name => "nn-NO", lcid => 0x00000814 },
1150  { name => "nnh" },
1151  { name => "nnh-CM" },
1152  { name => "no", lcid => 0x00000014, oemcp => 850, ebcdiccp => 20277, sortlocale => "nb-NO" },
1153  { name => "nqo", idigits => 3, inegnumber => 3, slist => "\x{060c}", sopentypelang => "NKO" },
1154  { name => "nqo-GN" },
1155  { name => "nr", sopentypelang => "NDB" },
1156  { name => "nr-ZA" },
1157  { name => "nso", lcid => 0x0000006c, oemcp => 850, sopentypelang => "SOT" },
1158  { name => "nso-ZA", lcid => 0x0000046c },
1159  { name => "nus" },
1160  { name => "nus-SD", alias => "nus-SS" },
1161  { name => "nus-SS" },
1162  { name => "nyn", sopentypelang => "NKL" },
1163  { name => "nyn-UG" },
1164  { name => "oc", lcid => 0x00000082, oemcp => 850, ebcdiccp => 20297 },
1165  { name => "oc-FR", lcid => 0x00000482 },
1166  { name => "oc-Latn", alias => "oc" },
1167  { name => "oc-Latn-FR", alias => "oc-FR" },
1168  { name => "om", lcid => 0x00000072, sopentypelang => "ORO" },
1169  { name => "om-ET", lcid => 0x00000472 },
1170  { name => "om-KE" },
1171  { name => "or", lcid => 0x00000048, slist => ",", group => 15 },
1172  { name => "or-IN", lcid => 0x00000448 },
1173  { name => "os" },
1174  { name => "os-GE" },
1175  { name => "os-RU" },
1176  { name => "pa", lcid => 0x00000046, slist => "," },
1177  { name => "pa-Arab", lcid => 0x00007c46, slist => ";", inegnumber => 2, oemcp => 720, group => 13, sabbrevlangname => "PAP" },
1178  { name => "pa-Arab-PK", lcid => 0x00000846 },
1179  { name => "pa-Guru" },
1180  { name => "pa-Guru-IN", alias => "pa-IN" },
1181  { name => "pa-IN", lcid => 0x00000446, sparent => "pa-Guru", file => "pa_Guru_IN", group => 15 },
1182  { name => "pap", lcid => 0x00000079, oemcp => 850, sopentypelang => "PAP0" },
1183 ## name => "pap-029", lcid => 0x00000479 },
1184  { name => "pcm" },
1185  { name => "pcm-NG", alias => "pcm-Latn-NG" },
1186  { name => "pcm-Latn", file => "pcm" },
1187  { name => "pcm-Latn-NG", file => "pcm_NG" },
1188  { name => "pl", lcid => 0x00000015, oemcp => 852, ebcdiccp => 20880, group => 2, sabbrevlangname => "PLK", sopentypelang => "PLK" },
1189  { name => "pl-PL", lcid => 0x00000415 },
1190  { name => "prg" },
1191  { name => "prg-001" },
1192  { name => "prs", lcid => 0x0000008c, file => "fa", inegnumber => 3, oemcp => 720, group => 13, sopentypelang => "DRI" },
1193  { name => "prs-AF", lcid => 0x0000048c, file => "fa_AF" },
1194  { name => "prs-Arab", alias => "prs" },
1195  { name => "prs-Arab-AF", alias => "prs-AF" },
1196  { name => "ps", lcid => 0x00000063, group => 13, sabbrevlangname => "PAS", sopentypelang => "PAS" },
1197  { name => "ps-AF", lcid => 0x00000463 },
1198  { name => "ps-PK" },
1199  { name => "pt", lcid => 0x00000016, oemcp => 850, sabbrevlangname => "PTB", sopentypelang => "PTG" },
1200  { name => "pt-AO" },
1201  { name => "pt-BR", lcid => 0x00000416 },
1202  { name => "pt-CH", oemcp => 65001 },
1203  { name => "pt-CV" },
1204  { name => "pt-GQ", oemcp => 65001 },
1205  { name => "pt-GW" },
1206  { name => "pt-LU", oemcp => 65001 },
1207  { name => "pt-MO" },
1208  { name => "pt-MZ" },
1209  { name => "pt-PT", lcid => 0x00000816, sabbrevlangname => "PTG" },
1210  { name => "pt-ST" },
1211  { name => "pt-TL" },
1212 ## name => qps-Latn-x-sh", lcid => 0x80000901 },
1213 ## name => qps-ploc", lcid => 0x80000501 },
1214 ## name => qps-ploca", lcid => 0x800005fe },
1215 ## name => qps-plocm", lcid => 0x800009ff },
1216  { name => "qu", alias => "quz" },
1217  { name => "qu-BO", alias => "quz-BO" },
1218  { name => "qu-EC", alias => "quz-EC" },
1219  { name => "qu-PE", alias => "quz-PE" },
1220  { name => "quc", lcid => 0x00000086, oemcp => 850, ebcdiccp => 20284, slist => "," },
1221  { name => "quc-Latn", lcid => 0x00007c86, file => "quc" },
1222  { name => "quc-Latn-GT", lcid => 0x00000486, file => "quc_GT" },
1223  { name => "qut", alias => "quc" },
1224  { name => "qut-GT", alias => "quc-Latn-GT" },
1225  { name => "quz", lcid => 0x0000006b, file => "qu", territory => "BO", oemcp => 850, ebcdiccp => 20284, slist => "," },
1226  { name => "quz-BO", lcid => 0x0000046b, file => "qu_BO" },
1227  { name => "quz-EC", lcid => 0x0000086b, file => "qu_EC" },
1228  { name => "quz-Latn", alias => "quz" },
1229  { name => "quz-Latn-BO", alias => "quz-BO" },
1230  { name => "quz-Latn-EC", alias => "quz-EC" },
1231  { name => "quz-Latn-PE", alias => "quz-PE" },
1232  { name => "quz-PE", lcid => 0x00000c6b, file => "qu_PE" },
1233  { name => "rm", lcid => 0x00000017, oemcp => 850, ebcdiccp => 20273, sabbrevlangname => "RMC", sopentypelang => "RMS" },
1234  { name => "rm-CH", lcid => 0x00000417 },
1235  { name => "rn" },
1236  { name => "rn-BI" },
1237  { name => "ro", lcid => 0x00000018, oemcp => 852, ebcdiccp => 20880, sabbrevlangname => "ROM", sopentypelang => "ROM" },
1238  { name => "ro-MD", lcid => 0x00000818, maccp => 65001, sabbrevlangname => "ROD" },
1239  { name => "ro-RO", lcid => 0x00000418, group => 2 },
1240  { name => "rof" },
1241  { name => "rof-TZ" },
1242  { name => "ru", lcid => 0x00000019, oemcp => 866 },
1243  { name => "ru-BY", maccp => 65001 },
1244  { name => "ru-KG", maccp => 65001 },
1245  { name => "ru-KZ", maccp => 65001 },
1246  { name => "ru-MD", lcid => 0x00000819, maccp => 65001, sabbrevlangname => "RUM" },
1247  { name => "ru-RU", lcid => 0x00000419, group => 5 },
1248  { name => "ru-UA", maccp => 65001 },
1249  { name => "rw", lcid => 0x00000087, oemcp => 437, sopentypelang => "RUA" },
1250  { name => "rw-RW", lcid => 0x00000487 },
1251  { name => "rwk" },
1252  { name => "rwk-TZ" },
1253  { name => "sa", lcid => 0x0000004f, slist => ",", group => 15 },
1254  { name => "sa-Deva", alias => "sa" },
1255  { name => "sa-Deva-IN", alias => "sa-IN" },
1256  { name => "sa-IN", lcid => 0x0000044f },
1257  { name => "sah", lcid => 0x00000085, oemcp => 866, group => 5, sopentypelang => "YAK" },
1258  { name => "sah-Cyrl", alias => "sah" },
1259  { name => "sah-Cyrl-RU", alias => "sah-RU" },
1260  { name => "sah-RU", lcid => 0x00000485 },
1261  { name => "saq" },
1262  { name => "saq-KE" },
1263  { name => "sat" },
1264  { name => "sat-Olck" },
1265  { name => "sat-Olck-IN" },
1266  { name => "sbp" },
1267  { name => "sbp-TZ" },
1268  { name => "sc" },
1269  { name => "sc-IT" },
1270  { name => "sd", lcid => 0x00000059, inegnumber => 3, oemcp => 720, sabbrevlangname => "SIP" },
1271  { name => "sd-Arab", lcid => 0x00007c59, group => 13 },
1272  { name => "sd-Arab-PK", lcid => 0x00000859 },
1273  { name => "sd-Deva", inegnumber => 1, slist => ",", oemcp => 65001, group => 15 },
1274  { name => "sd-Deva-IN", lcid => 0x00000459, sabbrevlangname => "ZZZ" },
1275  { name => "sd-PK", alias => "sd-Arab-PK" },
1276  { name => "se", lcid => 0x0000003b, oemcp => 850, ebcdiccp => 20277, sopentypelang => "NSM" },
1277  { name => "se-FI", lcid => 0x00000c3b, ebcdiccp => 20278, sabbrevlangname => "SMG" },
1278  { name => "se-NO", lcid => 0x0000043b },
1279  { name => "se-SE", lcid => 0x0000083b, ebcdiccp => 20278, sabbrevlangname => "SMF" },
1280  { name => "se-Latn", alias => "se" },
1281  { name => "se-Latn-FI", alias => "se-FI" },
1282  { name => "se-Latn-NO", alias => "se-NO" },
1283  { name => "se-Latn-SE", alias => "se-SE" },
1284  { name => "seh" },
1285  { name => "seh-MZ" },
1286  { name => "ses" },
1287  { name => "ses-ML" },
1288  { name => "sg", sopentypelang => "SGO" },
1289  { name => "sg-CF" },
1290  { name => "shi" },
1291  { name => "shi-Latn" },
1292  { name => "shi-Latn-MA" },
1293  { name => "shi-Tfng" },
1294  { name => "shi-Tfng-MA" },
1295  { name => "si", lcid => 0x0000005b, group => 15, sopentypelang => "SNH" },
1296  { name => "si-LK", lcid => 0x0000045b },
1297  { name => "sk", lcid => 0x0000001b, oemcp => 852, ebcdiccp => 20880, group => 2, sabbrevlangname => "SKY", sopentypelang => "SKY" },
1298  { name => "sk-SK", lcid => 0x0000041b },
1299  { name => "sl", lcid => 0x00000024, oemcp => 852, ebcdiccp => 20880, group => 2 },
1300  { name => "sl-SI", lcid => 0x00000424 },
1301  { name => "sma", lcid => 0x0000783b, sparent => "se", ebcdiccp => 20278, sabbrevlangname => "SMB", sopentypelang => "SSM" },
1302  { name => "sma-Latn", alias => "sma" },
1303  { name => "sma-Latn-NO", alias => "sma-NO" },
1304  { name => "sma-Latn-SE", alias => "sma-SE" },
1305  { name => "sma-NO", lcid => 0x0000183b, ebcdiccp => 20277, sabbrevlangname => "SMA" },
1306  { name => "sma-SE", lcid => 0x00001c3b },
1307  { name => "smj", lcid => 0x00007c3b, sparent => "se", ebcdiccp => 20278, sabbrevlangname => "SMK", sopentypelang => "LSM" },
1308  { name => "smj-Latn", alias => "smj" },
1309  { name => "smj-Latn-NO", alias => "smj-NO" },
1310  { name => "smj-Latn-SE", alias => "smj-SE" },
1311  { name => "smj-NO", lcid => 0x0000103b, ebcdiccp => 20277, sabbrevlangname => "SMJ" },
1312  { name => "smj-SE", lcid => 0x0000143b },
1313  { name => "smn", lcid => 0x0000703b, sparent => "se", ebcdiccp => 20278, sopentypelang => "ISM" },
1314  { name => "smn-FI", lcid => 0x0000243b },
1315  { name => "smn-Latn", alias => "smn" },
1316  { name => "smn-Latn-FI", alias => "smn-FI" },
1317  { name => "sms", lcid => 0x0000743b, sparent => "se", ebcdiccp => 20278, sopentypelang => "SKS" },
1318  { name => "sms-FI", lcid => 0x0000203b },
1319  { name => "sms-Latn", alias => "sms" },
1320  { name => "sms-Latn-FI", alias => "sms-FI" },
1321  { name => "sn", sopentypelang => "SNA0" },
1322  { name => "sn-Latn", file => "sn" },
1323  { name => "sn-Latn-ZW", file => "sn_ZW" },
1324  { name => "sn-ZW", alias => "sn-Latn-ZW" },
1325  { name => "so", lcid => 0x00000077, sopentypelang => "SML" },
1326  { name => "so-DJ" },
1327  { name => "so-ET" },
1328  { name => "so-KE" },
1329  { name => "so-SO", lcid => 0x00000477 },
1330  { name => "sq", lcid => 0x0000001c, oemcp => 852, ebcdiccp => 20880, group => 2 },
1331  { name => "sq-AL", lcid => 0x0000041c },
1332  { name => "sq-MK" },
1333  { name => "sq-XK" },
1334  { name => "sr", lcid => 0x00007c1a, sortlocale => "sr-Latn-RS", oemcp => 852, group => 2, sabbrevlangname => "SRB", sopentypelang => "SRB" },
1335  { name => "sr-Cyrl", lcid => 0x00006c1a, oemcp => 855, ebcdiccp => 21025, group => 5, sabbrevlangname => "SRO" },
1336  { name => "sr-Cyrl-BA", lcid => 0x00001c1a, sabbrevlangname => "SRN" },
1337  { name => "sr-Cyrl-ME", lcid => 0x0000301a, sabbrevlangname => "SRQ" },
1338  { name => "sr-Cyrl-RS", lcid => 0x0000281a },
1339  { name => "sr-Cyrl-XK" },
1340  { name => "sr-Latn", lcid => 0x0000701a, sabbrevlangname => "SRM" },
1341  { name => "sr-Latn-BA", lcid => 0x0000181a, maccp => 10082, ebcdiccp => 870, sabbrevlangname => "SRS" },
1342  { name => "sr-Latn-ME", lcid => 0x00002c1a, sabbrevlangname => "SRP" },
1343  { name => "sr-Latn-RS", lcid => 0x0000241a, sabbrevlangname => "SRM" },
1344  { name => "sr-Latn-XK" },
1345 ## name => "sr-Cyrl-CS", lcid => 0x00000c1a },
1346 ## name => "sr-Latn-CS", lcid => 0x0000081a },
1347  { name => "ss", sopentypelang => "SWZ" },
1348  { name => "ss-SZ" },
1349  { name => "ss-ZA" },
1350  { name => "ssy" },
1351  { name => "ssy-ER" },
1352  { name => "st", lcid => 0x00000030 },
1353  { name => "st-LS" },
1354  { name => "st-ZA", lcid => 0x00000430 },
1355  { name => "su" },
1356  { name => "su-Latn" },
1357  { name => "su-Latn-ID" },
1358  { name => "sv", lcid => 0x0000001d, oemcp => 850, ebcdiccp => 20278, sabbrevlangname => "SVE", sopentypelang => "SVE" },
1359  { name => "sv-AX" },
1360  { name => "sv-FI", lcid => 0x0000081d, sabbrevlangname => "SVF" },
1361  { name => "sv-SE", lcid => 0x0000041d, sabbrevlangname => "SVE" },
1362  { name => "sw", lcid => 0x00000041, territory => "KE", oemcp => 437, ebcdiccp => 500, sabbrevlangname => "SWK", sopentypelang => "SWK" },
1363  { name => "sw-CD" },
1364  { name => "sw-KE", lcid => 0x00000441 },
1365  { name => "sw-TZ" },
1366  { name => "sw-UG" },
1367  { name => "swc-CD", alias => "sw-CD" },
1368  { name => "syr", lcid => 0x0000005a, slist => ",", group => 13 },
1369  { name => "syr-SY", lcid => 0x0000045a },
1370  { name => "syr-Syrc", alias => "syr" },
1371  { name => "syr-Syrc-SY", alias => "syr-SY" },
1372  { name => "ta", lcid => 0x00000049, slist => ",", group => 15, sabbrevlangname => "TAI" },
1373  { name => "ta-IN", lcid => 0x00000449 },
1374  { name => "ta-LK", lcid => 0x00000849, sabbrevlangname => "TAM" },
1375  { name => "ta-MY" },
1376  { name => "ta-SG" },
1377  { name => "te", lcid => 0x0000004a, group => 15 },
1378  { name => "te-IN", lcid => 0x0000044a },
1379  { name => "teo" },
1380  { name => "teo-KE" },
1381  { name => "teo-UG" },
1382  { name => "tg", lcid => 0x00000028, oemcp => 866, group => 5, sabbrevlangname => "TAJ", sopentypelang => "TAJ" },
1383  { name => "tg-Cyrl", lcid => 0x00007c28, file => "tg" },
1384  { name => "tg-Cyrl-TJ", lcid => 0x00000428, file => "tg_TJ" },
1385  { name => "tg-TJ", alias => "tg-Cyrl-TJ" },
1386  { name => "th", lcid => 0x0000001e, oemcp => 874, ebcdiccp => 20838, slist => ",", group => 11 },
1387  { name => "th-TH", lcid => 0x0000041e },
1388  { name => "ti", lcid => 0x00000073, territory => "ER", sopentypelang => "TGY" },
1389  { name => "ti-ER", lcid => 0x00000873 },
1390  { name => "ti-ET", lcid => 0x00000473, sabbrevlangname => "TIE" },
1391  { name => "tig", sopentypelang => "TGR" },
1392  { name => "tig-ER" },
1393  { name => "tig-Ethi-ER", alias => "tig-ER" },
1394  { name => "tk", lcid => 0x00000042, oemcp => 852, ebcdiccp => 20880, group => 2, sopentypelang => "TKM" },
1395  { name => "tk-Latn", alias => "tk" },
1396  { name => "tk-Latn-TM", alias => "tk-TM" },
1397  { name => "tk-TM", lcid => 0x00000442 },
1398  { name => "tn", lcid => 0x00000032, oemcp => 850, sopentypelang => "TNA" },
1399  { name => "tn-BW", lcid => 0x00000832, sabbrevlangname => "TSB" },
1400  { name => "tn-ZA", lcid => 0x00000432 },
1401  { name => "to", sopentypelang => "TGN" },
1402  { name => "to-TO" },
1403  { name => "tr", lcid => 0x0000001f, oemcp => 857, ebcdiccp => 20905, group => 6, sabbrevlangname => "TRK", sopentypelang => "TRK" },
1404  { name => "tr-CY" },
1405  { name => "tr-TR", lcid => 0x0000041f },
1406  { name => "ts", lcid => 0x00000031, sopentypelang => "TSG" },
1407  { name => "ts-ZA", lcid => 0x00000431 },
1408  { name => "tt", lcid => 0x00000044, oemcp => 866, group => 5, sabbrevlangname => "TTT" },
1409  { name => "tt-Cyrl", alias => "tt" },
1410  { name => "tt-Cyrl-RU", alias => "tt-RU" },
1411  { name => "tt-RU", lcid => 0x00000444 },
1412  { name => "twq" },
1413  { name => "twq-NE" },
1414  { name => "tzm", lcid => 0x0000005f, sortlocale => "tzm-Latn-DZ", oemcp => 850, ebcdiccp => 20297, sabbrevlangname => "TZA" },
1415  { name => "tzm-Latn", lcid => 0x00007c5f, territory => "DZ", file => "tzm" },
1416  { name => "tzm-Latn-MA", file => "tzm_MA", oemcp => 65001 },
1417  { name => "tzm-Latn-DZ", lcid => 0x0000085f, file => "tzm" },
1418  { name => "tzm-MA", alias => "tzm-Latn-MA" },
1419  { name => "tzm-DZ", alias => "tzm-Latn-DZ" },
1420 ## name => "tzm-Arab", group => 13 },
1421 ## name => "tzm-Arab-MA", lcid => 0x0000045f },
1422 ## name => "tzm-Tfng", lcid => 0x0000785f },
1423 ## name => "tzm-Tfng-MA", lcid => 0x0000105f },
1424  { name => "ug", lcid => 0x00000080, oemcp => 720, slist => ",", group => 13, sopentypelang => "UYG", nativedigits => "0123456789" },
1425  { name => "ug-Arab", alias => "ug" },
1426  { name => "ug-Arab-CN", alias => "ug-CN" },
1427  { name => "ug-CN", lcid => 0x00000480 },
1428  { name => "uk", lcid => 0x00000022, oemcp => 866, maccp => 10017, ebcdiccp => 500, group => 5 },
1429  { name => "uk-UA", lcid => 0x00000422 },
1430  { name => "ur", lcid => 0x00000020, oemcp => 720 },
1431  { name => "ur-IN", lcid => 0x00000820, maccp => 65001, sabbrevlangname => "URI" },
1432  { name => "ur-PK", lcid => 0x00000420, group => 13 },
1433  { name => "uz", lcid => 0x00000043, oemcp => 857, maccp => 10029, group => 2 },
1434  { name => "uz-Arab", oemcp => 65001, maccp => 65001 },
1435  { name => "uz-Arab-AF" },
1436  { name => "uz-Cyrl", lcid => 0x00007843, oemcp => 866, maccp => 10007, group => 5, sabbrevlangname => "UZC" },
1437  { name => "uz-Cyrl-UZ", lcid => 0x00000843 },
1438  { name => "uz-Latn", lcid => 0x00007c43 },
1439  { name => "uz-Latn-UZ", lcid => 0x00000443 },
1440  { name => "vai" },
1441  { name => "vai-Latn" },
1442  { name => "vai-Latn-LR" },
1443  { name => "vai-Vaii" },
1444  { name => "vai-Vaii-LR" },
1445  { name => "ve", lcid => 0x00000033, sabbrevlangname => "ZZZ" },
1446  { name => "ve-ZA", lcid => 0x00000433 },
1447  { name => "vi", lcid => 0x0000002a, oemcp => 1258, slist => ",", group => 14, sabbrevlangname => "VIT", sopentypelang => "VIT" },
1448  { name => "vi-VN", lcid => 0x0000042a },
1449  { name => "vo" },
1450  { name => "vo-001" },
1451  { name => "vun" },
1452  { name => "vun-TZ" },
1453  { name => "wa", oemcp => 850 },
1454  { name => "wa-BE" },
1455  { name => "wae" },
1456  { name => "wae-CH" },
1457  { name => "wal" },
1458  { name => "wal-ET" },
1459  { name => "wo", lcid => 0x00000088, oemcp => 850, ebcdiccp => 20297, sopentypelang => "WLF" },
1460  { name => "wo-Latn", alias => "wo" },
1461  { name => "wo-Latn-SN", alias => "wo-SN" },
1462  { name => "wo-SN", lcid => 0x00000488 },
1463  { name => "x-IV_mathan", lcid => 0x0001007f, alias => "" },
1464  { name => "xh", lcid => 0x00000034, oemcp => 850, sopentypelang => "XHS" },
1465  { name => "xh-ZA", lcid => 0x00000434 },
1466  { name => "xog" },
1467  { name => "xog-UG" },
1468  { name => "yav" },
1469  { name => "yav-CM" },
1470  { name => "yi", lcid => 0x0000003d, sabbrevlangname => "ZZZ", sopentypelang => "JII" },
1471  { name => "yi-001", lcid => 0x0000043d },
1472  { name => "yo", lcid => 0x0000006a, oemcp => 437, sopentypelang => "YBA" },
1473  { name => "yo-BJ", ebcdiccp => 500 },
1474  { name => "yo-Latn", alias => "yo" },
1475  { name => "yo-Latn-NG", alias => "yo-NG" },
1476  { name => "yo-NG", lcid => 0x0000046a },
1477  { name => "yrl" },
1478  { name => "yrl-BR" },
1479  { name => "yrl-CO" },
1480  { name => "yrl-VE" },
1481  { name => "yue" },
1482  { name => "yue-Hans" },
1483  { name => "yue-Hans-CN" },
1484  { name => "yue-Hant" },
1485  { name => "yue-Hant-HK" },
1486  { name => "zgh" },
1487  { name => "zgh-MA", alias => "zgh-Tfng-MA" },
1488  { name => "zgh-Tfng", file => "zgh" },
1489  { name => "zgh-Tfng-MA", file => "zgh_MA" },
1490  { name => "zh", lcid => 0x00007804, ireadinglayout => 2, oemcp => 936, slist => ",", sscripts => "Hani Hans", sabbrevlangname => "CHS", sopentypelang => "ZHS", nativedigits => "0123456789" },
1491  { name => "zh-CN", lcid => 0x00000804, file => "zh_Hans_CN", sparent => "zh-Hans" },
1492  { name => "zh-CN_phoneb", lcid => 0x00050804, alias => "zh-CN" },
1493  { name => "zh-CN_stroke", lcid => 0x00020804, alias => "zh-CN" },
1494  { name => "zh-Hans", lcid => 0x00000004, group => 10 },
1495  { name => "zh-Hans-CN", alias => "zh-CN" },
1496  { name => "zh-Hans-CN-u-co-phonebk", alias => "zh-CN_phoneb" },
1497  { name => "zh-Hans-CN-u-co-stroke", alias => "zh-CN_stroke" },
1498  { name => "zh-Hans-HK", slist => ";", nativedigits => "" },
1499  { name => "zh-Hans-MO", slist => ";", nativedigits => "" },
1500  { name => "zh-Hans-SG", alias => "zh-SG" },
1501  { name => "zh-Hans-SG-u-co-phonebk", alias => "zh-SG_phoneb" },
1502  { name => "zh-Hans-SG-u-co-stroke", alias => "zh-SG_stroke" },
1503  { name => "zh-Hant", lcid => 0x00007c04, sortlocale => "zh-HK", ireadinglayout => 2, oemcp => 950, slist => ",", sscripts => "Hani Hant", group => 9, sabbrevlangname => "CHT", sopentypelang => "ZHH" },
1504  { name => "zh-Hant-HK", alias => "zh-HK" },
1505  { name => "zh-Hant-HK-u-co-unihan", alias => "zh-HK_radstr" },
1506  { name => "zh-Hant-MO", alias => "zh-MO" },
1507  { name => "zh-Hant-MO-u-co-stroke", alias => "zh-MO_stroke" },
1508  { name => "zh-Hant-MO-u-co-unihan", alias => "zh-MO_radstr" },
1509  { name => "zh-Hant-TW", alias => "zh-TW" },
1510  { name => "zh-Hant-TW-u-co-phonetic", alias => "zh-TW_pronun" },
1511  { name => "zh-Hant-TW-u-co-unihan", alias => "zh-TW_radstr" },
1512  { name => "zh-HK", lcid => 0x00000c04, file => "zh_Hant_HK", sparent => "zh-Hant", sabbrevlangname => "ZHH" },
1513  { name => "zh-HK_radstr", lcid => 0x00040c04, alias => "zh-HK" },
1514  { name => "zh-MO", lcid => 0x00001404, file => "zh_Hant_MO", sparent => "zh-Hant", sabbrevlangname => "ZHM", sopentypelang => "ZHT" },
1515  { name => "zh-MO_radstr", lcid => 0x00041404, alias => "zh-MO" },
1516  { name => "zh-MO_stroke", lcid => 0x00021404, alias => "zh-MO" },
1517  { name => "zh-SG", lcid => 0x00001004, file => "zh_Hans_SG", sparent => "zh-Hans", sabbrevlangname => "ZHI" },
1518  { name => "zh-SG_phoneb", lcid => 0x00051004, alias => "zh-SG" },
1519  { name => "zh-SG_stroke", lcid => 0x00021004, alias => "zh-SG" },
1520  { name => "zh-TW", lcid => 0x00000404, file => "zh_Hant_TW", sparent => "zh-Hant", sopentypelang => "ZHT" },
1521  { name => "zh-TW_pronun", lcid => 0x00030404, alias => "zh-TW" },
1522  { name => "zh-TW_radstr", lcid => 0x00040404, alias => "zh-TW" },
1523  { name => "zu", lcid => 0x00000035, oemcp => 850 },
1524  { name => "zu-ZA", lcid => 0x00000435 },
1525 );
1526
1527 my @calendars =
1528 (
1529  { id => 1, name => "Gregorian", itwodigityearmax => 2049 },
1530  { id => 2, type => "gregorian", locale => "en-US", itwodigityearmax => 2049 },
1531  { id => 3, type => "japanese", locale => "ja-JP", eras => [ 232..236 ] },
1532  { id => 4, type => "roc", locale => "zh-TW", eras => [ 1 ] },
1533  { id => 5, type => "dangi", locale => "ko-KR", eras => [ 0 ] },
1534  { id => 6, type => "islamic", locale => "ar-SA", itwodigityearmax => 1451 },
1535  { id => 7, type => "buddhist", locale => "th-TH", eras => [ 0 ] },
1536  { id => 8, type => "hebrew", locale => "he-IL", itwodigityearmax => 5810 },
1537  { id => 9, type => "gregorian", locale => "fr-FR", itwodigityearmax => 2049 },
1538  { id => 10, type => "gregorian", locale => "ar-SA", itwodigityearmax => 2049 },
1539  { id => 11, type => "gregorian", locale => "ar-SA", itwodigityearmax => 2049 },
1540  { id => 12, type => "gregorian", locale => "ar-SA", itwodigityearmax => 2049 },
1541  { id => 13, name => "Julian", locale => "en-US", itwodigityearmax => 2049 },
1542  { id => 14, name => "Japanese Lunisolar" },
1543  { id => 15, name => "Chinese Lunisolar" },
1544  { id => 16, name => "Saka" },
1545  { id => 17, name => "Lunar ETO Chinese" },
1546  { id => 18, name => "Lunar ETO Korean" },
1547  { id => 19, name => "Lunar ETO Rokuyou" },
1548  { id => 20, name => "Korean Lunisolar" },
1549  { id => 21, name => "Taiwan Lunisolar" },
1550  { id => 22, type => "persian", locale => "prs-AF", itwodigityearmax => 1429 },
1551  { id => 23, type => "islamic-umalqura", locale => "ar-SA", itwodigityearmax => 1451 },
1552 );
1553
1554 my @geoids =
1555 (
1556  { id => 2,   name => "AG" },     # Antigua and Barbuda
1557  { id => 3,   name => "AF" },     # Afghanistan
1558  { id => 4,   name => "DZ" },     # Algeria
1559  { id => 5,   name => "AZ" },     # Azerbaijan
1560  { id => 6,   name => "AL" },     # Albania
1561  { id => 7,   name => "AM" },     # Armenia
1562  { id => 8,   name => "AD" },     # Andorra
1563  { id => 9,   name => "AO" },     # Angola
1564  { id => 10,  name => "AS" },     # American Samoa
1565  { id => 11,  name => "AR" },     # Argentina
1566  { id => 12,  name => "AU" },     # Australia
1567  { id => 14,  name => "AT" },     # Austria
1568  { id => 17,  name => "BH" },     # Bahrain
1569  { id => 18,  name => "BB" },     # Barbados
1570  { id => 19,  name => "BW" },     # Botswana
1571  { id => 20,  name => "BM" },     # Bermuda
1572  { id => 21,  name => "BE" },     # Belgium
1573  { id => 22,  name => "BS" },     # Bahamas, The
1574  { id => 23,  name => "BD" },     # Bangladesh
1575  { id => 24,  name => "BZ" },     # Belize
1576  { id => 25,  name => "BA" },     # Bosnia and Herzegovina
1577  { id => 26,  name => "BO" },     # Bolivia
1578  { id => 27,  name => "MM" },     # Myanmar
1579  { id => 28,  name => "BJ" },     # Benin
1580  { id => 29,  name => "BY" },     # Belarus
1581  { id => 30,  name => "SB" },     # Solomon Islands
1582  { id => 32,  name => "BR" },     # Brazil
1583  { id => 34,  name => "BT" },     # Bhutan
1584  { id => 35,  name => "BG" },     # Bulgaria
1585  { id => 37,  name => "BN" },     # Brunei
1586  { id => 38,  name => "BI" },     # Burundi
1587  { id => 39,  name => "CA" },     # Canada
1588  { id => 40,  name => "KH" },     # Cambodia
1589  { id => 41,  name => "TD" },     # Chad
1590  { id => 42,  name => "LK" },     # Sri Lanka
1591  { id => 43,  name => "CG" },     # Congo
1592  { id => 44,  name => "CD" },     # Congo (DRC)
1593  { id => 45,  name => "CN" },     # China
1594  { id => 46,  name => "CL" },     # Chile
1595  { id => 49,  name => "CM" },     # Cameroon
1596  { id => 50,  name => "KM" },     # Comoros
1597  { id => 51,  name => "CO" },     # Colombia
1598  { id => 54,  name => "CR" },     # Costa Rica
1599  { id => 55,  name => "CF" },     # Central African Republic
1600  { id => 56,  name => "CU" },     # Cuba
1601  { id => 57,  name => "CV" },     # Cape Verde
1602  { id => 59,  name => "CY" },     # Cyprus
1603  { id => 61,  name => "DK" },     # Denmark
1604  { id => 62,  name => "DJ" },     # Djibouti
1605  { id => 63,  name => "DM" },     # Dominica
1606  { id => 65,  name => "DO" },     # Dominican Republic
1607  { id => 66,  name => "EC" },     # Ecuador
1608  { id => 67,  name => "EG" },     # Egypt
1609  { id => 68,  name => "IE" },     # Ireland
1610  { id => 69,  name => "GQ" },     # Equatorial Guinea
1611  { id => 70,  name => "EE" },     # Estonia
1612  { id => 71,  name => "ER" },     # Eritrea
1613  { id => 72,  name => "SV" },     # El Salvador
1614  { id => 73,  name => "ET" },     # Ethiopia
1615  { id => 75,  name => "CZ" },     # Czech Republic
1616  { id => 77,  name => "FI" },     # Finland
1617  { id => 78,  name => "FJ" },     # Fiji Islands
1618  { id => 80,  name => "FM" },     # Micronesia
1619  { id => 81,  name => "FO" },     # Faroe Islands
1620  { id => 84,  name => "FR" },     # France
1621  { id => 86,  name => "GM" },     # Gambia, The
1622  { id => 87,  name => "GA" },     # Gabon
1623  { id => 88,  name => "GE" },     # Georgia
1624  { id => 89,  name => "GH" },     # Ghana
1625  { id => 90,  name => "GI" },     # Gibraltar
1626  { id => 91,  name => "GD" },     # Grenada
1627  { id => 93,  name => "GL" },     # Greenland
1628  { id => 94,  name => "DE" },     # Germany
1629  { id => 98,  name => "GR" },     # Greece
1630  { id => 99,  name => "GT" },     # Guatemala
1631  { id => 100, name => "GN" },     # Guinea
1632  { id => 101, name => "GY" },     # Guyana
1633  { id => 103, name => "HT" },     # Haiti
1634  { id => 104, name => "HK" },     # Hong Kong S.A.R.
1635  { id => 106, name => "HN" },     # Honduras
1636  { id => 108, name => "HR" },     # Croatia
1637  { id => 109, name => "HU" },     # Hungary
1638  { id => 110, name => "IS" },     # Iceland
1639  { id => 111, name => "ID" },     # Indonesia
1640  { id => 113, name => "IN" },     # India
1641  { id => 114, name => "IO" },     # British Indian Ocean Territory
1642  { id => 116, name => "IR" },     # Iran
1643  { id => 117, name => "IL" },     # Israel
1644  { id => 118, name => "IT" },     # Italy
1645  { id => 119, name => "CI" },     # Côte d'Ivoire
1646  { id => 121, name => "IQ" },     # Iraq
1647  { id => 122, name => "JP" },     # Japan
1648  { id => 124, name => "JM" },     # Jamaica
1649  { id => 125, name => "SJ" },     # Jan Mayen
1650  { id => 126, name => "JO" },     # Jordan
1651  { id => 127, parent => "UM" },   # Johnston Atoll
1652  { id => 129, name => "KE" },     # Kenya
1653  { id => 130, name => "KG" },     # Kyrgyzstan
1654  { id => 131, name => "KP" },     # North Korea
1655  { id => 133, name => "KI" },     # Kiribati
1656  { id => 134, name => "KR" },     # Korea
1657  { id => 136, name => "KW" },     # Kuwait
1658  { id => 137, name => "KZ" },     # Kazakhstan
1659  { id => 138, name => "LA" },     # Laos
1660  { id => 139, name => "LB" },     # Lebanon
1661  { id => 140, name => "LV" },     # Latvia
1662  { id => 141, name => "LT" },     # Lithuania
1663  { id => 142, name => "LR" },     # Liberia
1664  { id => 143, name => "SK" },     # Slovakia
1665  { id => 145, name => "LI" },     # Liechtenstein
1666  { id => 146, name => "LS" },     # Lesotho
1667  { id => 147, name => "LU" },     # Luxembourg
1668  { id => 148, name => "LY" },     # Libya
1669  { id => 149, name => "MG" },     # Madagascar
1670  { id => 151, name => "MO" },     # Macao S.A.R.
1671  { id => 152, name => "MD" },     # Moldova
1672  { id => 154, name => "MN" },     # Mongolia
1673  { id => 156, name => "MW" },     # Malawi
1674  { id => 157, name => "ML" },     # Mali
1675  { id => 158, name => "MC" },     # Monaco
1676  { id => 159, name => "MA" },     # Morocco
1677  { id => 160, name => "MU" },     # Mauritius
1678  { id => 162, name => "MR" },     # Mauritania
1679  { id => 163, name => "MT" },     # Malta
1680  { id => 164, name => "OM" },     # Oman
1681  { id => 165, name => "MV" },     # Maldives
1682  { id => 166, name => "MX" },     # Mexico
1683  { id => 167, name => "MY" },     # Malaysia
1684  { id => 168, name => "MZ" },     # Mozambique
1685  { id => 173, name => "NE" },     # Niger
1686  { id => 174, name => "VU" },     # Vanuatu
1687  { id => 175, name => "NG" },     # Nigeria
1688  { id => 176, name => "NL" },     # Netherlands
1689  { id => 177, name => "NO" },     # Norway
1690  { id => 178, name => "NP" },     # Nepal
1691  { id => 180, name => "NR" },     # Nauru
1692  { id => 181, name => "SR" },     # Suriname
1693  { id => 182, name => "NI" },     # Nicaragua
1694  { id => 183, name => "NZ" },     # New Zealand
1695  { id => 184, name => "PS" },     # Palestinian Authority
1696  { id => 185, name => "PY" },     # Paraguay
1697  { id => 187, name => "PE" },     # Peru
1698  { id => 190, name => "PK" },     # Pakistan
1699  { id => 191, name => "PL" },     # Poland
1700  { id => 192, name => "PA" },     # Panama
1701  { id => 193, name => "PT" },     # Portugal
1702  { id => 194, name => "PG" },     # Papua New Guinea
1703  { id => 195, name => "PW" },     # Palau
1704  { id => 196, name => "GW" },     # Guinea-Bissau
1705  { id => 197, name => "QA" },     # Qatar
1706  { id => 198, name => "RE" },     # Reunion
1707  { id => 199, name => "MH" },     # Marshall Islands
1708  { id => 200, name => "RO" },     # Romania
1709  { id => 201, name => "PH" },     # Philippines
1710  { id => 202, name => "PR" },     # Puerto Rico
1711  { id => 203, name => "RU" },     # Russia
1712  { id => 204, name => "RW" },     # Rwanda
1713  { id => 205, name => "SA" },     # Saudi Arabia
1714  { id => 206, name => "PM" },     # St. Pierre and Miquelon
1715  { id => 207, name => "KN" },     # St. Kitts and Nevis
1716  { id => 208, name => "SC" },     # Seychelles
1717  { id => 209, name => "ZA" },     # South Africa
1718  { id => 210, name => "SN" },     # Senegal
1719  { id => 212, name => "SI" },     # Slovenia
1720  { id => 213, name => "SL" },     # Sierra Leone
1721  { id => 214, name => "SM" },     # San Marino
1722  { id => 215, name => "SG" },     # Singapore
1723  { id => 216, name => "SO" },     # Somalia
1724  { id => 217, name => "ES" },     # Spain
1725  { id => 218, name => "LC" },     # St. Lucia
1726  { id => 219, name => "SD" },     # Sudan
1727  { id => 220, name => "SJ" },     # Svalbard
1728  { id => 221, name => "SE" },     # Sweden
1729  { id => 222, name => "SY" },     # Syria
1730  { id => 223, name => "CH" },     # Switzerland
1731  { id => 224, name => "AE" },     # United Arab Emirates
1732  { id => 225, name => "TT" },     # Trinidad and Tobago
1733  { id => 227, name => "TH" },     # Thailand
1734  { id => 228, name => "TJ" },     # Tajikistan
1735  { id => 231, name => "TO" },     # Tonga
1736  { id => 232, name => "TG" },     # Togo
1737  { id => 233, name => "ST" },     # São Tomé and Príncipe
1738  { id => 234, name => "TN" },     # Tunisia
1739  { id => 235, name => "TR" },     # Turkey
1740  { id => 236, name => "TV" },     # Tuvalu
1741  { id => 237, name => "TW" },     # Taiwan
1742  { id => 238, name => "TM" },     # Turkmenistan
1743  { id => 239, name => "TZ" },     # Tanzania
1744  { id => 240, name => "UG" },     # Uganda
1745  { id => 241, name => "UA" },     # Ukraine
1746  { id => 242, name => "GB" },     # United Kingdom
1747  { id => 244, name => "US" },     # United States
1748  { id => 245, name => "BF" },     # Burkina Faso
1749  { id => 246, name => "UY" },     # Uruguay
1750  { id => 247, name => "UZ" },     # Uzbekistan
1751  { id => 248, name => "VC" },     # St. Vincent and the Grenadines
1752  { id => 249, name => "VE" },     # Bolivarian Republic of Venezuela
1753  { id => 251, name => "VN" },     # Vietnam
1754  { id => 252, name => "VI" },     # Virgin Islands
1755  { id => 253, name => "VA" },     # Vatican City
1756  { id => 254, name => "NA" },     # Namibia
1757  { id => 257, name => "EH" },     # Western Sahara (disputed)
1758  { id => 258, parent => "UM" },   # Wake Island
1759  { id => 259, name => "WS" },     # Samoa
1760  { id => 260, name => "SZ" },     # Swaziland
1761  { id => 261, name => "YE" },     # Yemen
1762  { id => 263, name => "ZM" },     # Zambia
1763  { id => 264, name => "ZW" },     # Zimbabwe
1764  { id => 269, name => "CS" },     # Serbia and Montenegro (Former)
1765  { id => 270, name => "ME" },     # Montenegro
1766  { id => 271, name => "RS" },     # Serbia
1767  { id => 273, name => "CW" },     # Curaçao
1768  { id => 276, name => "SS" },     # South Sudan
1769  { id => 300, name => "AI" },     # Anguilla
1770  { id => 301, name => "AQ" },     # Antarctica
1771  { id => 302, name => "AW" },     # Aruba
1772  { id => 303, parent => "SH" },   # Ascension Island
1773  { id => 304, parent => "053" },  # Ashmore and Cartier Islands
1774  { id => 305, parent => "UM" },   # Baker Island
1775  { id => 306, name => "BV" },     # Bouvet Island
1776  { id => 307, name => "KY" },     # Cayman Islands
1777  { id => 308, name => "830", parent => "155" }, # Channel Islands
1778  { id => 309, name => "CX" },     # Christmas Island
1779  { id => 310, parent => "009" },  # Clipperton Island
1780  { id => 311, name => "CC" },     # Cocos (Keeling) Islands
1781  { id => 312, name => "CK" },     # Cook Islands
1782  { id => 313, parent => "053" },  # Coral Sea Islands
1783  { id => 314, parent => "IO" },   # Diego Garcia
1784  { id => 315, name => "FK" },     # Falkland Islands (Islas Malvinas)
1785  { id => 317, name => "GF" },     # French Guiana
1786  { id => 318, name => "PF" },     # French Polynesia
1787  { id => 319, name => "TF" },     # French Southern and Antarctic Lands
1788  { id => 321, name => "GP" },     # Guadeloupe
1789  { id => 322, name => "GU" },     # Guam
1790  { id => 323 },                   # Guantanamo Bay
1791  { id => 324, name => "GG" },     # Guernsey
1792  { id => 325, name => "HM" },     # Heard Island and McDonald Islands
1793  { id => 326, parent => "UM" },   # Howland Island
1794  { id => 327, parent => "UM" },   # Jarvis Island
1795  { id => 328, name => "JE" },     # Jersey
1796  { id => 329, parent => "UM" },   # Kingman Reef
1797  { id => 330, name => "MQ" },     # Martinique
1798  { id => 331, name => "YT" },     # Mayotte
1799  { id => 332, name => "MS" },     # Montserrat
1800  { id => 333, name => "AN", region => 1 }, # Netherlands Antilles (Former)
1801  { id => 334, name => "NC" },     # New Caledonia
1802  { id => 335, name => "NU" },     # Niue
1803  { id => 336, name => "NF" },     # Norfolk Island
1804  { id => 337, name => "MP" },     # Northern Mariana Islands
1805  { id => 338, parent => "UM" },   # Palmyra Atoll
1806  { id => 339, name => "PN" },     # Pitcairn Islands
1807  { id => 340, parent => "MP" },   # Rota Island
1808  { id => 341, parent => "MP" },   # Saipan
1809  { id => 342, name => "GS" },     # South Georgia and the South Sandwich Islands
1810  { id => 343, name => "SH" },     # St. Helena
1811  { id => 346, parent => "MP" },   # Tinian Island
1812  { id => 347, name => "TK" },     # Tokelau
1813  { id => 348, parent => "SH" },   # Tristan da Cunha
1814  { id => 349, name => "TC" },     # Turks and Caicos Islands
1815  { id => 351, name => "VG" },     # Virgin Islands, British
1816  { id => 352, name => "WF" },     # Wallis and Futuna
1817  { id => 742, name => "002" },    # Africa
1818  { id => 2129, name => "142" },   # Asia
1819  { id => 10541, name => "150" },  # Europe
1820  { id => 15126, name => "IM" },   # Man, Isle of
1821  { id => 19618, name => "MK" },   # Macedonia, Former Yugoslav Republic of
1822  { id => 20900, name => "054" },  # Melanesia
1823  { id => 21206, name => "057" },  # Micronesia
1824  { id => 21242, parent => "UM" }, # Midway Islands
1825  { id => 23581, name => "021" },  # Northern America
1826  { id => 26286, name => "061" },  # Polynesia
1827  { id => 27082, name => "013" },  # Central America
1828  { id => 27114, name => "009" },  # Oceania
1829  { id => 30967, name => "SX" },   # Sint Maarten (Dutch part)
1830  { id => 31396, name => "005" },  # South America
1831  { id => 31706, name => "MF" },   # Saint Martin (French part)
1832  { id => 39070, name => "001" },  # World
1833  { id => 42483, name => "011" },  # Western Africa
1834  { id => 42484, name => "017" },  # Middle Africa
1835  { id => 42487, name => "015" },  # Northern Africa
1836  { id => 47590, name => "143" },  # Central Asia
1837  { id => 47599, name => "035" },  # South-Eastern Asia
1838  { id => 47600, name => "030" },  # Eastern Asia
1839  { id => 47603, name => "014" },  # Eastern Africa
1840  { id => 47609, name => "151" },  # Eastern Europe
1841  { id => 47610, name => "039" },  # Southern Europe
1842  { id => 47611, name => "145" },  # Middle East
1843  { id => 47614, name => "034" },  # Southern Asia
1844  { id => 7299303, name => "TL" }, # Democratic Republic of Timor-Leste
1845  { id => 9914689, name => "XK" }, # Kosovo
1846  { id => 10026358, name => "019" }, # Americas
1847  { id => 10028789, name => "AX" },  # Åland Islands
1848  { id => 10039880, name => "029", sintlsymbol => "XCD" }, # Caribbean
1849  { id => 10039882, name => "154" }, # Northern Europe
1850  { id => 10039883, name => "018" }, # Southern Africa
1851  { id => 10210824, name => "155" }, # Western Europe
1852  { id => 10210825, name => "053" }, # Australia and New Zealand
1853  { id => 161832015, name => "BL" }, # Saint Barthélemy
1854  { id => 161832256, name => "UM" }, # U.S. Minor Outlying Islands
1855  { id => 161832257, name => "419", parent => "019" }, # Latin America and the Caribbean
1856  { id => 161832258, name => "BQ" },   # Bonaire, Sint Eustatius and Saba
1857 );
1858
1859 my @cp2uni = ();
1860 my @glyph2uni = ();
1861 my @lead_bytes = ();
1862 my @uni2cp = ();
1863 my @tolower_table = ();
1864 my @toupper_table = ();
1865 my @digitmap_table = ();
1866 my @halfwidth_table = ();
1867 my @fullwidth_table = ();
1868 my @cjk_compat_table = ();
1869 my @chinese_traditional_table = ();
1870 my @chinese_simplified_table = ();
1871 my @category_table = ();
1872 my @initial_joining_table = ();
1873 my @direction_table = ();
1874 my @decomp_table = ();
1875 my @combining_class_table = ();
1876 my @decomp_compat_table = ();
1877 my @comp_exclusions = ();
1878 my @idna_decomp_table = ();
1879 my @idna_disallowed = ();
1880 my %registry_keys;
1881 my $default_char;
1882 my $default_wchar;
1883
1884 my %joining_forms =
1885 (
1886    "isolated" => [],
1887    "final" => [],
1888    "initial" => [],
1889    "medial" => []
1890 );
1891
1892 my $current_data_file;
1893
1894 sub to_utf16(@)
1895 {
1896     my @ret;
1897     foreach my $ch (@_)
1898     {
1899         if ($ch < 0x10000)
1900         {
1901             push @ret, $ch;
1902         }
1903         else
1904         {
1905             my $val = $ch - 0x10000;
1906             push @ret, 0xd800 | ($val >> 10), 0xdc00 | ($val & 0x3ff);
1907         }
1908     }
1909     return @ret;
1910 }
1911
1912 ################################################################
1913 # fetch a unicode.org file and open it
1914 sub open_data_file($@)
1915 {
1916     my ($id, $name) = @_;
1917     my $data = $data_files{$id};
1918     my $cache = ($ENV{XDG_CACHE_HOME} || "$ENV{HOME}/.cache") . "/wine";
1919     local *FILE;
1920
1921     my $url = $data->{url};
1922     my $filename = "$cache/" . ($data->{name} || ($url =~ s/.*\/([^\/]+)$/$1/r));
1923     unless (-f $filename)
1924     {
1925         print "Fetching $url...\n";
1926         system "mkdir", "-p", $cache;
1927         !system "wget", "-q", "-O", $filename, $url or die "cannot fetch $url";
1928     }
1929
1930     my $sha = Digest::SHA->new( "sha256" )->addfile( $filename )->hexdigest;
1931     die "invalid checksum $sha for $filename" unless $sha eq $data->{sha};
1932
1933     if ($filename =~ /\.zip$/)
1934     {
1935         open FILE, "-|", "unzip", "-p", $filename, $name or die "cannot extract $name from $filename";
1936     }
1937     elsif ($filename =~ /\.tar\.gz$/)
1938     {
1939         open FILE, "-|", "tar", "-x", "-f", $filename, "-O", $name or die "cannot extract $name from $filename";
1940     }
1941     else
1942     {
1943         open FILE, "<$filename" or die "cannot open $filename";
1944     }
1945     $current_data_file = $name ? "$url:$name" : $url;
1946     return *FILE;
1947 }
1948
1949 ################################################################
1950 # load a unicode.org file as XML data
1951 sub load_xml_data_file($@)
1952 {
1953     my ($id, $name) = @_;
1954     my $FILE = open_data_file( $id, $name );
1955     my $xml = XML::LibXML->load_xml( IO => $FILE );
1956     close FILE;
1957     return $xml;
1958 }
1959
1960 ################################################################
1961 # recursively get the decomposition for a character
1962 sub get_decomposition($$);
1963 sub get_decomposition($$)
1964 {
1965     my ($char, $table) = @_;
1966     my @ret;
1967
1968     return $char unless defined $table->[$char];
1969     foreach my $ch (@{$table->[$char]})
1970     {
1971         push @ret, get_decomposition( $ch, $table );
1972     }
1973     return @ret;
1974 }
1975
1976 ################################################################
1977 # get the composition that results in a given character
1978 sub get_composition($$)
1979 {
1980     my ($ch, $compat) = @_;
1981     return () unless defined $decomp_table[$ch];  # no decomposition
1982     my @ret = @{$decomp_table[$ch]};
1983     return () if @ret < 2;                        # singleton decomposition
1984     return () if $comp_exclusions[$ch];           # composition exclusion
1985     return () if $combining_class_table[$ch];     # non-starter
1986     return () if $combining_class_table[$ret[0]]; # first char is non-starter
1987     return () if $compat == 1 && !defined $decomp_table[$ret[0]] &&
1988         defined $decomp_compat_table[$ret[0]];    # first char has compat decomposition
1989     return () if $compat == 2 && !defined $decomp_table[$ret[0]] &&
1990         defined $idna_decomp_table[$ret[0]];      # first char has IDNA decomposition
1991     return () if $compat == 2 && defined $idna_decomp_table[$ret[0]] &&
1992         defined $idna_decomp_table[$idna_decomp_table[$ret[0]]->[0]];  # first char's decomposition has IDNA decomposition
1993     return () if $compat == 2 && defined $idna_decomp_table[$ret[1]];  # second char has IDNA decomposition
1994     return @ret;
1995 }
1996
1997 ################################################################
1998 # recursively build decompositions
1999 sub build_decompositions(@)
2000 {
2001     my @src = @_;
2002     my @dst;
2003
2004     for (my $i = 0; $i < @src; $i++)
2005     {
2006         next unless defined $src[$i];
2007         my @decomp = to_utf16( get_decomposition( $i, \@src ));
2008         $dst[$i] = \@decomp;
2009     }
2010     return @dst;
2011 }
2012
2013 ################################################################
2014 # compose Hangul sequences
2015 sub compose_hangul(@)
2016 {
2017     my $SBASE  = 0xac00;
2018     my $LBASE  = 0x1100;
2019     my $VBASE  = 0x1161;
2020     my $TBASE  = 0x11a7;
2021     my $LCOUNT = 19;
2022     my $VCOUNT = 21;
2023     my $TCOUNT = 28;
2024     my $NCOUNT = $VCOUNT * $TCOUNT;
2025     my $SCOUNT = $LCOUNT * $NCOUNT;
2026
2027     my @seq = @_;
2028     my @ret;
2029     my $i;
2030
2031     for ($i = 0; $i < @seq; $i++)
2032     {
2033         my $ch = $seq[$i];
2034         if ($ch >= $LBASE && $ch < $LBASE + $LCOUNT && $i < @seq - 1 &&
2035             $seq[$i+1] >= $VBASE && $seq[$i+1] < $VBASE + $VCOUNT)
2036         {
2037             $ch = $SBASE + (($seq[$i] - $LBASE) * $VCOUNT + ($seq[$i+1] - $VBASE)) * $TCOUNT;
2038             $i++;
2039         }
2040         if ($ch >= $SBASE && $ch < $SBASE + $SCOUNT && !(($ch - $SBASE) % $TCOUNT) && $i < @seq - 1 &&
2041             $seq[$i+1] > $TBASE && $seq[$i+1] < $TBASE + $TCOUNT)
2042         {
2043             $ch += $seq[$i+1] - $TBASE;
2044             $i++;
2045         }
2046         push @ret, $ch;
2047     }
2048     return @ret;
2049 }
2050
2051 ################################################################
2052 # remove linguistic-only mappings from the case table
2053 sub remove_linguistic_mappings($$)
2054 {
2055     my ($upper, $lower) = @_;
2056
2057     # remove case mappings that don't round-trip
2058
2059     for (my $i = 0; $i < @{$upper}; $i++)
2060     {
2061         next unless defined ${$upper}[$i];
2062         my $ch = ${$upper}[$i];
2063         ${$upper}[$i] = undef unless defined ${$lower}[$ch] && ${$lower}[$ch] == $i;
2064     }
2065     for (my $i = 0; $i < @{$lower}; $i++)
2066     {
2067         next unless defined ${$lower}[$i];
2068         my $ch = ${$lower}[$i];
2069         ${$lower}[$i] = undef unless defined ${$upper}[$ch] && ${$upper}[$ch] == $i;
2070     }
2071 }
2072
2073 ################################################################
2074 # read in the Unicode database files
2075 sub load_data()
2076 {
2077     my $start;
2078
2079     # now build mappings from the decomposition field of the Unicode database
2080
2081     my $UNICODE_DATA = open_data_file( "ucd", "UnicodeData.txt" );
2082     while (<$UNICODE_DATA>)
2083     {
2084         # Decode the fields ...
2085         my ($code, $name, $cat, $comb, $bidi,
2086             $decomp, $dec, $dig, $num, $mirror,
2087             $oldname, $comment, $upper, $lower, $title) = split /;/;
2088         my $src = hex $code;
2089
2090         die "unknown category $cat" unless defined $categories{$cat};
2091         die "unknown directionality $bidi" unless defined $directions{$bidi};
2092
2093         $category_table[$src] = $categories{$cat};
2094         $direction_table[$src] = $bidi;
2095         if ($cat eq "Mn" || $cat eq "Me" || $cat eq "Cf")
2096         {
2097             $initial_joining_table[$src] = $joining_types{"T"};
2098         }
2099         else
2100         {
2101             $initial_joining_table[$src] = $joining_types{"U"};
2102         }
2103
2104         if ($lower ne "")
2105         {
2106             $tolower_table[$src] = hex $lower;
2107         }
2108         if ($upper ne "")
2109         {
2110             $toupper_table[$src] = hex $upper;
2111         }
2112         if ($dec ne "")
2113         {
2114             $category_table[$src] |= $ctype{"digit"};
2115         }
2116         if ($dig ne "")
2117         {
2118             $digitmap_table[$src] = ord $dig;
2119         }
2120         $combining_class_table[$src] = ($cat ne "Co") ? $comb : 0x100; # Private Use
2121
2122         $category_table[$src] |= $ctype{"nonspacing"}    if $bidi eq "NSM";
2123         $category_table[$src] |= $ctype{"diacritic"}     if $name =~ /^(COMBINING)|(MODIFIER LETTER)\W/;
2124         $category_table[$src] |= $ctype{"vowelmark"}     if $name =~ /\sVOWEL/ || $oldname =~ /\sVOWEL/;
2125         $category_table[$src] |= $ctype{"halfwidth"}     if $name =~ /^HALFWIDTH\s/;
2126         $category_table[$src] |= $ctype{"fullwidth"}     if $name =~ /^FULLWIDTH\s/;
2127         $category_table[$src] |= $ctype{"hiragana"}      if $name =~ /(HIRAGANA)|(\WKANA\W)/;
2128         $category_table[$src] |= $ctype{"katakana"}      if $name =~ /(KATAKANA)|(\WKANA\W)/;
2129         $category_table[$src] |= $ctype{"ideograph"}     if $name =~ /^<CJK Ideograph/;
2130         $category_table[$src] |= $ctype{"ideograph"}     if $name =~ /^CJK COMPATIBILITY IDEOGRAPH/;
2131         $category_table[$src] |= $ctype{"ideograph"}     if $name =~ /^HANGZHOU/;
2132         $category_table[$src] |= $ctype{"highsurrogate"} if $name =~ /High Surrogate/;
2133         $category_table[$src] |= $ctype{"lowsurrogate"}  if $name =~ /Low Surrogate/;
2134
2135         # copy the category and direction for everything between First/Last pairs
2136         if ($name =~ /, First>/) { $start = $src; }
2137         if ($name =~ /, Last>/)
2138         {
2139             while ($start < $src)
2140             {
2141                 $category_table[$start] = $category_table[$src];
2142                 $direction_table[$start] = $direction_table[$src];
2143                 $combining_class_table[$start] = $combining_class_table[$src];
2144                 $start++;
2145             }
2146         }
2147
2148         next if $decomp eq "";  # no decomposition, skip it
2149
2150         if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)/)
2151         {
2152             my @seq = map { hex $_; } (split /\s+/, (split /\s+/, $decomp, 2)[1]);
2153             $decomp_compat_table[$src] = \@seq;
2154         }
2155
2156         if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
2157         {
2158             # decomposition of the form "<foo> 1234" -> use char if type is known
2159             my $dst = hex $2;
2160             if ($1 eq "narrow")
2161             {
2162                 $halfwidth_table[$dst] = $src;
2163                 $fullwidth_table[$src] = $dst;
2164             }
2165             elsif ($1 eq "wide")
2166             {
2167                 next if $dst == 0x5c; # don't remap backslash
2168                 $fullwidth_table[$dst] = $src;
2169                 $halfwidth_table[$src] = $dst;
2170             }
2171             elsif ($1 eq "font" || $1 eq "square" || $1 eq "circle")
2172             {
2173                 $fullwidth_table[$src] = $dst if $src >= 0x10000;
2174             }
2175             elsif ($1 eq "isolated" || $1 eq "final" || $1 eq "initial" || $1 eq "medial")
2176             {
2177                 ${joining_forms{$1}}[$dst] = $src;
2178             }
2179         }
2180         elsif ($decomp =~ /^<compat>\s+0020\s+([0-9a-fA-F]+)/)
2181         {
2182             # decomposition "<compat> 0020 1234" -> combining accent
2183         }
2184         elsif ($decomp =~ /^([0-9a-fA-F]+)/)
2185         {
2186             # store decomposition
2187             if ($decomp =~ /^([0-9a-fA-F]+)\s+([0-9a-fA-F]+)$/)
2188             {
2189                 $decomp_table[$src] = $decomp_compat_table[$src] = [ hex $1, hex $2 ];
2190             }
2191             elsif ($decomp =~ /^([0-9a-fA-F]+)$/)
2192             {
2193                 my $dst = hex $1;
2194                 # Single char decomposition
2195                 $decomp_table[$src] = $decomp_compat_table[$src] = [ $dst ];
2196                 if ($name =~ /^CJK COMPATIBILITY IDEOGRAPH/)
2197                 {
2198                     $cjk_compat_table[$src] = $dst;
2199                     $fullwidth_table[$src] = $dst if $src >= 0x10000;
2200                 }
2201             }
2202         }
2203     }
2204     close $UNICODE_DATA;
2205
2206     # patch the category of some special characters
2207
2208     for (my $i = 0; $i < @decomp_table; $i++)
2209     {
2210         next unless defined $decomp_table[$i];
2211         $category_table[$i] |= $category_table[$decomp_table[$i]->[0]];
2212     }
2213     foreach my $cat (keys %special_categories)
2214     {
2215         my $flag = $ctype{$cat};
2216         foreach my $i (@{$special_categories{$cat}}) { $category_table[$i] |= $flag; }
2217     }
2218     for (my $i = 0; $i < @decomp_compat_table; $i++)
2219     {
2220         next unless defined $decomp_compat_table[$i];
2221         next unless @{$decomp_compat_table[$i]} == 2;
2222         $category_table[$i] |= $category_table[$decomp_compat_table[$i]->[1]] & $ctype{"diacritic"};
2223     }
2224
2225     # load the composition exclusions
2226
2227     my $EXCL = open_data_file( "ucd", "CompositionExclusions.txt" );
2228     while (<$EXCL>)
2229     {
2230         s/\#.*//;  # remove comments
2231         if (/^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+)\s*$/)
2232         {
2233             foreach my $i (hex $1 .. hex $2) { $comp_exclusions[$i] = 1; }
2234         }
2235         elsif (/^([0-9a-fA-F]+)\s*$/)
2236         {
2237             $comp_exclusions[hex $1] = 1;
2238         }
2239     }
2240     close $EXCL;
2241
2242     # load the IDNA mappings
2243
2244     @idna_decomp_table = @decomp_compat_table;
2245     my $IDNA = open_data_file( "idna", "IdnaMappingTable.txt" );
2246     while (<$IDNA>)
2247     {
2248         s/\#.*//;  # remove comments
2249         next if /^\s*$/;
2250         my ($char, $type, $mapping) = split /;/;
2251         my ($ch1, $ch2);
2252         if ($char =~ /([0-9a-fA-F]+)\.\.([0-9a-fA-F]+)/)
2253         {
2254             $ch1 = hex $1;
2255             $ch2 = hex $2;
2256         }
2257         elsif ($char =~ /([0-9a-fA-F]+)/)
2258         {
2259             $ch1 = $ch2 = hex $1;
2260         }
2261
2262         if ($type =~ /mapped/ || $type =~ /deviation/)
2263         {
2264             $mapping =~ s/^\s*(([0-9a-fA-F]+\s+)+)\s*$/$1/;
2265             my @seq = map { hex $_; } split /\s+/, $mapping;
2266             foreach my $i ($ch1 .. $ch2) { $idna_decomp_table[$i] = @seq ? \@seq : [ 0 ]; }
2267         }
2268         elsif ($type =~ /valid/)
2269         {
2270         }
2271         elsif ($type =~ /ignored/)
2272         {
2273             foreach my $i ($ch1 .. $ch2) { $idna_decomp_table[$i] = [ 0 ]; }
2274         }
2275         elsif ($type =~ /disallowed/)
2276         {
2277             foreach my $i ($ch1 .. $ch2)
2278             {
2279                 $idna_decomp_table[$i] = undef;
2280                 $idna_disallowed[$i] = 1;
2281             }
2282         }
2283     }
2284     close $IDNA;
2285
2286     # load the Unihan mappings
2287
2288     my $UNIHAN = open_data_file( "unihan", "Unihan_Variants.txt" );
2289     while (<$UNIHAN>)
2290     {
2291         s/\#.*//;  # remove comments
2292         next if /^\s*$/;
2293         if (/^U\+([0-9a-fA-F]{4})\s+kTraditionalVariant\s+U\+([0-9a-fA-F]{4})$/)
2294         {
2295             next if hex $1 < 0x4dc0;  # skip extension A
2296             $chinese_traditional_table[hex $1] = hex $2;
2297         }
2298         elsif (/^U\+([0-9a-fA-F]{4})\s+kSimplifiedVariant\s+U\+([0-9a-fA-F]{4})$/)
2299         {
2300             next if hex $1 < 0x4dc0;  # skip extension A
2301             $chinese_simplified_table[hex $1] = hex $2;
2302         }
2303     }
2304     close $UNIHAN;
2305     foreach my $i (0xf900..0xfaff)
2306     {
2307         next unless defined $cjk_compat_table[$i];
2308         next if defined $chinese_simplified_table[$cjk_compat_table[$i]];
2309         $chinese_simplified_table[$i] = $cjk_compat_table[$i];
2310     }
2311 }
2312
2313
2314 ################################################################
2315 # add a new registry key
2316 sub add_registry_key($$$)
2317 {
2318     my ($base, $key, $defval) = @_;
2319     $registry_keys{"$base\\$key"} = [ $defval ] unless defined $registry_keys{"$base\\$key"};
2320 }
2321
2322 ################################################################
2323 # add a new registry value with explicit type
2324 sub add_registry_value($$$$)
2325 {
2326     my ($base, $key, $name, $value) = @_;
2327     add_registry_key( $base, $key, undef );
2328     push @{$registry_keys{"$base\\$key"}}, "'$name' = $value";
2329 }
2330
2331 ################################################################
2332 # add a new registry string value
2333 sub add_registry_string_value($$$$)
2334 {
2335     my ($base, $key, $name, $value) = @_;
2336     $value =~ s/\'/\'\'/g;
2337     add_registry_value( $base, $key, $name, "s '$value'" );
2338 }
2339
2340 ################################################################
2341 # add a new registry dword value
2342 sub add_registry_dword_value($$$$)
2343 {
2344     my ($base, $key, $name, $value) = @_;
2345     add_registry_value( $base, $key, $name, "d $value" );
2346 }
2347
2348 ################################################################
2349 # add a new registry binary value
2350 sub add_registry_binary_value($$$$)
2351 {
2352     my ($base, $key, $name, $value) = @_;
2353     add_registry_value( $base, $key, $name, "b " . join "", map { sprintf "%02x", $_; } unpack( "C*", $value ));
2354 }
2355
2356 ################################################################
2357 # define a new lead byte
2358 sub add_lead_byte($)
2359 {
2360     my $ch = shift;
2361     return if defined $cp2uni[$ch];
2362     push @lead_bytes, $ch;
2363     $cp2uni[$ch] = 0;
2364 }
2365
2366 ################################################################
2367 # define a new char mapping
2368 sub add_mapping($$)
2369 {
2370     my ($cp, $uni) = @_;
2371     $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
2372     $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
2373     if ($cp > 0xff) { add_lead_byte( $cp >> 8 ); }
2374 }
2375
2376 ################################################################
2377 # get a mapping including glyph chars for MB_USEGLYPHCHARS
2378 sub get_glyphs_mapping(@)
2379 {
2380     my @table = @_;
2381
2382     for (my $i = 0; $i < @glyph2uni; $i++)
2383     {
2384         $table[$i] = $glyph2uni[$i] if defined $glyph2uni[$i];
2385     }
2386     return @table;
2387 }
2388
2389 ################################################################
2390 # build EUC-JP table from the JIS 0208/0212 files
2391 sub dump_eucjp_codepage()
2392 {
2393     @cp2uni = ();
2394     @glyph2uni = ();
2395     @lead_bytes = ();
2396     @uni2cp = ();
2397     $default_char = $DEF_CHAR;
2398     $default_wchar = 0x30fb;
2399
2400     # ASCII chars
2401     foreach my $i (0x00 .. 0x7f) { add_mapping( $i, $i ); }
2402
2403     # lead bytes
2404     foreach my $i (0x8e, 0xa1 .. 0xfe) { add_lead_byte($i); }
2405
2406     # JIS X 0201 right plane
2407     foreach my $i (0xa1 .. 0xdf) { add_mapping( 0x8e00 + $i, 0xfec0 + $i ); }
2408
2409     # undefined chars
2410     foreach my $i (0x80 .. 0x8d, 0x8f .. 0x9f) { $cp2uni[$i] = $i; }
2411     $cp2uni[0xa0] = 0xf8f0;
2412     $cp2uni[0xff] = 0xf8f3;
2413
2414     # Fix backslash conversion
2415     add_mapping( 0xa1c0, 0xff3c );
2416
2417     # Add private mappings for rows undefined in JIS 0208/0212
2418     my $private = 0xe000;
2419     foreach my $hi (0xf5 .. 0xfe)
2420     {
2421         foreach my $lo (0xa1 .. 0xfe)
2422         {
2423             add_mapping( ($hi << 8) + $lo, $private++ );
2424         }
2425     }
2426     foreach my $hi (0xf5 .. 0xfe)
2427     {
2428         foreach my $lo (0x21 .. 0x7e)
2429         {
2430             add_mapping( ($hi << 8) + $lo, $private++ );
2431         }
2432     }
2433
2434     my $INPUT = open_data_file( "jis0208" );
2435     while (<$INPUT>)
2436     {
2437         next if /^\#/;  # skip comments
2438         next if /^$/;  # skip empty lines
2439         next if /\x1a/;  # skip ^Z
2440         if (/^0x[0-9a-fA-F]+\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
2441         {
2442             add_mapping( 0x8080 + hex $1, hex $2 );
2443             next;
2444         }
2445         die "Unrecognized line $_\n";
2446     }
2447     close $INPUT;
2448
2449     $INPUT = open_data_file( "jis0212" );
2450     while (<$INPUT>)
2451     {
2452         next if /^\#/;  # skip comments
2453         next if /^$/;  # skip empty lines
2454         next if /\x1a/;  # skip ^Z
2455         if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
2456         {
2457             add_mapping( 0x8000 + hex $1, hex $2 );
2458             next;
2459         }
2460         die "Unrecognized line $_\n";
2461     }
2462     close $INPUT;
2463
2464     output_codepage_file( 20932 );
2465 }
2466
2467 ################################################################
2468 # build Korean Wansung table from the KSX1001 file
2469 sub dump_krwansung_codepage(@)
2470 {
2471     my @cp949 = @_;
2472     @cp2uni = ();
2473     @glyph2uni = ();
2474     @lead_bytes = ();
2475     @uni2cp = ();
2476     $default_char = 0x3f;
2477     $default_wchar = 0x003f;
2478
2479     # ASCII and undefined chars
2480     foreach my $i (0x00 .. 0x9f) { add_mapping( $i, $i ); }
2481     add_mapping( 0xa0, 0xf8e6 );
2482     add_mapping( 0xad, 0xf8e7 );
2483     add_mapping( 0xae, 0xf8e8 );
2484     add_mapping( 0xaf, 0xf8e9 );
2485     add_mapping( 0xfe, 0xf8ea );
2486     add_mapping( 0xff, 0xf8eb );
2487
2488     my $INPUT = open_data_file( "ksx1001" );
2489     while (<$INPUT>)
2490     {
2491         next if /^\#/;  # skip comments
2492         next if /^$/;  # skip empty lines
2493         next if /\x1a/;  # skip ^Z
2494         if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
2495         {
2496             add_mapping( 0x8080 + hex $1, hex $2 );
2497             next;
2498         }
2499         die "Unrecognized line $_\n";
2500     }
2501     close $INPUT;
2502
2503     # get some extra mappings from cp 949
2504     my @defined_lb;
2505     map { $defined_lb[$_] = 1; } @lead_bytes;
2506     foreach my $i (0x0000 .. 0xffff)
2507     {
2508         next if ($i >= 0x1100 && $i <= 0x11ff);  # range not used in 20949
2509         next unless defined $cp949[$i];
2510         if ($cp949[$i] >= 0xff)
2511         {
2512             # only add chars for lead bytes that exist in 20949
2513             my $hi = $cp949[$i] >> 8;
2514             my $lo = $cp949[$i] & 0xff;
2515             next unless $defined_lb[$hi];
2516             next unless $lo >= 0xa1 && $lo <= 0xfe;
2517         }
2518         add_mapping( $cp949[$i], $i );
2519     }
2520
2521     output_codepage_file( 20949 );
2522 }
2523
2524
2525 ################################################################
2526 # dump an array of integers
2527 sub dump_array($$@)
2528 {
2529     my ($bit_width, $default, @array) = @_;
2530     my $format = sprintf "0x%%0%ux", $bit_width / 4;
2531     my $i;
2532     my $ret = "    ";
2533     for ($i = 0; $i < $#array; $i++)
2534     {
2535         $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
2536         $ret .= (($i % 8) != 7) ? ", " : ",\n    ";
2537     }
2538     $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
2539     return $ret;
2540 }
2541
2542
2543 ################################################################
2544 # dump an SBCS mapping table in binary format
2545 sub dump_binary_sbcs_table($)
2546 {
2547     my $codepage = shift;
2548
2549     my @header = ( 13, $codepage, 1, $default_char, $default_wchar, $cp2uni[$default_char], $uni2cp[$default_wchar] );
2550     my $wc_offset = 256 + 3 + (@glyph2uni ? 256 : 0);
2551
2552     print OUTPUT pack "S<*", @header;
2553     print OUTPUT pack "C12", (0) x 12;
2554     print OUTPUT pack "S<*", $wc_offset, map { $_ || 0; } @cp2uni[0 .. 255];
2555
2556     if (@glyph2uni)
2557     {
2558         print OUTPUT pack "S<*", 256, get_glyphs_mapping(@cp2uni[0 .. 255]);
2559     }
2560     else
2561     {
2562         print OUTPUT pack "S<*", 0;
2563     }
2564
2565     print OUTPUT pack "S<*", 0, 0;
2566
2567     print OUTPUT pack "C*", map { defined $_ ? $_ : $default_char; } @uni2cp[0 .. 65535];
2568 }
2569
2570
2571 ################################################################
2572 # dump a DBCS mapping table in binary format
2573 sub dump_binary_dbcs_table($)
2574 {
2575     my $codepage = shift;
2576     my @lb_ranges = get_lb_ranges();
2577     my @header = ( 13, $codepage, 2, $default_char, $default_wchar, $cp2uni[$default_char], $uni2cp[$default_wchar] );
2578
2579     my @offsets = (0) x 256;
2580     my $pos = 0;
2581     foreach my $i (@lead_bytes)
2582     {
2583         $offsets[$i] = ($pos += 256);
2584         $cp2uni[$i] = 0;
2585     }
2586
2587     my $wc_offset = 256 + 3 + 256 * (1 + scalar @lead_bytes);
2588
2589     print OUTPUT pack "S<*", @header;
2590     print OUTPUT pack "C12", @lb_ranges, 0 x 12;
2591     print OUTPUT pack "S<*", $wc_offset, map { $_ || 0; } @cp2uni[0 .. 255];
2592     print OUTPUT pack "S<*", 0, scalar @lb_ranges / 2, @offsets;
2593
2594     foreach my $i (@lead_bytes)
2595     {
2596         my $base = $i << 8;
2597         print OUTPUT pack "S<*", map { defined $_ ? $_ : $default_wchar; } @cp2uni[$base .. $base + 255];
2598     }
2599
2600     print OUTPUT pack "S<", 4;
2601     print OUTPUT pack "S<*", map { defined $_ ? $_ : $default_char; } @uni2cp[0 .. 65535];
2602 }
2603
2604
2605 ################################################################
2606 # get the list of defined lead byte ranges
2607 sub get_lb_ranges()
2608 {
2609     my @list = ();
2610     my @ranges = ();
2611
2612     foreach my $i (@lead_bytes) { $list[$i] = 1; }
2613     my $on = 0;
2614     for (my $i = 0; $i < 256; $i++)
2615     {
2616         if ($on)
2617         {
2618             if (!defined $list[$i]) { push @ranges, $i-1; $on = 0; }
2619         }
2620         else
2621         {
2622             if ($list[$i]) { push @ranges, $i; $on = 1; }
2623         }
2624     }
2625     if ($on) { push @ranges, 0xff; }
2626     return @ranges;
2627 }
2628
2629 ################################################################
2630 # dump the Indic Syllabic Category table
2631 sub dump_indic($)
2632 {
2633     my $filename = shift;
2634     my @indic_table;
2635
2636     my $INPUT = open_data_file( "ucd", "IndicSyllabicCategory.txt" );
2637     while (<$INPUT>)
2638     {
2639         next if /^\#/;  # skip comments
2640         next if /^\s*$/;  # skip empty lines
2641         next if /\x1a/;  # skip ^Z
2642         if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*#/)
2643         {
2644             my $type = $2;
2645             die "unknown indic $type" unless defined $indic_types{$type};
2646             if (hex $1 < 65536)
2647             {
2648                 $indic_table[hex $1] = $indic_types{$type};
2649             }
2650             next;
2651         }
2652         elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*#/)
2653         {
2654             my $type = $3;
2655             die "unknown indic $type" unless defined $indic_types{$type};
2656             if (hex $1 < 65536 and hex $2 < 65536)
2657             {
2658                 foreach my $i (hex $1 .. hex $2)
2659                 {
2660                     $indic_table[$i] = $indic_types{$type};
2661                 }
2662             }
2663             next;
2664         }
2665         die "malformed line $_";
2666     }
2667     close $INPUT;
2668
2669     my $prev_data_file = $current_data_file;
2670     $INPUT = open_data_file( "ucd", "IndicPositionalCategory.txt" );
2671     while (<$INPUT>)
2672     {
2673         next if /^\#/;  # skip comments
2674         next if /^\s*$/;  # skip empty lines
2675         next if /\x1a/;  # skip ^Z
2676         if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*#/)
2677         {
2678             my $type = $2;
2679             die "unknown matra $type" unless defined $matra_types{$type};
2680             $indic_table[hex $1] |= $matra_types{$type} << 8;
2681             next;
2682         }
2683         elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*#/)
2684         {
2685             my $type = $3;
2686             die "unknown matra $type" unless defined $matra_types{$type};
2687             foreach my $i (hex $1 .. hex $2)
2688             {
2689                 $indic_table[$i] |= $matra_types{$type} << 8;
2690             }
2691             next;
2692         }
2693         die "malformed line $_";
2694     }
2695     close $INPUT;
2696
2697     open OUTPUT,">$filename.new" or die "Cannot create $filename";
2698     print "Building $filename\n";
2699     print OUTPUT "/* Unicode Indic Syllabic Category */\n";
2700     print OUTPUT "/* generated from $prev_data_file */\n";
2701     print OUTPUT "/*       and from $current_data_file */\n";
2702     print OUTPUT "/* DO NOT EDIT!! */\n\n";
2703     print OUTPUT "#include \"windef.h\"\n\n";
2704
2705     dump_two_level_mapping( "indic_syllabic_table", $indic_types{'Other'}, 16, @indic_table );
2706
2707     close OUTPUT;
2708     save_file($filename);
2709 }
2710
2711 ################################################################
2712 # dump the Line Break Properties table
2713 sub dump_linebreak($)
2714 {
2715     my $filename = shift;
2716     my @break_table;
2717
2718     my $INPUT = open_data_file( "ucd", "LineBreak.txt" );
2719     while (<$INPUT>)
2720     {
2721         next if /^\#/;  # skip comments
2722         next if /^\s*$/;  # skip empty lines
2723         next if /\x1a/;  # skip ^Z
2724         if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z][0-9A-Z])+\s*/)
2725         {
2726             my $type = $2;
2727             die "unknown breaktype $type" unless defined $break_types{$type};
2728             $break_table[hex $1] = $break_types{$type};
2729             next;
2730         }
2731         elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z][0-9A-Z])+\s*/)
2732         {
2733             my $type = $3;
2734             die "unknown breaktype $type" unless defined $break_types{$type};
2735             foreach my $i (hex $1 .. hex $2)
2736             {
2737                 $break_table[$i] = $break_types{$type};
2738             }
2739             next;
2740         }
2741         elsif (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z])+\s*/)
2742         {
2743             my $type = $2;
2744             die "unknown breaktype $type" unless defined $break_types{$type};
2745             $break_table[hex $1] = $break_types{$type};
2746             next;
2747         }
2748         elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z])+\s*/)
2749         {
2750             my $type = $3;
2751             die "unknown breaktype $type" unless defined $break_types{$type};
2752             foreach my $i (hex $1 .. hex $2)
2753             {
2754                 $break_table[$i] = $break_types{$type};
2755             }
2756             next;
2757         }
2758         die "malformed line $_";
2759     }
2760     close $INPUT;
2761
2762     open OUTPUT,">$filename.new" or die "Cannot create $filename";
2763     print "Building $filename\n";
2764     print OUTPUT "/* Unicode Line Break Properties */\n";
2765     print OUTPUT "/* generated from $current_data_file */\n";
2766     print OUTPUT "/* DO NOT EDIT!! */\n\n";
2767     print OUTPUT "#include \"windef.h\"\n\n";
2768
2769     dump_three_level_mapping( "wine_linebreak_table", $break_types{'XX'}, 16, @break_table );
2770
2771     close OUTPUT;
2772     save_file($filename);
2773 }
2774
2775 my %scripts =
2776 (
2777     "Unknown"                => 0,
2778     "Common"                 => 1,
2779     "Inherited"              => 2,
2780     "Arabic"                 => 3,
2781     "Armenian"               => 4,
2782     "Avestan"                => 5,
2783     "Balinese"               => 6,
2784     "Bamum"                  => 7,
2785     "Batak"                  => 8,
2786     "Bengali"                => 9,
2787     "Bopomofo"               => 10,
2788     "Brahmi"                 => 11,
2789     "Braille"                => 12,
2790     "Buginese"               => 13,
2791     "Buhid"                  => 14,
2792     "Canadian_Aboriginal"    => 15,
2793     "Carian"                 => 16,
2794     "Cham"                   => 17,
2795     "Cherokee"               => 18,
2796     "Coptic"                 => 19,
2797     "Cuneiform"              => 20,
2798     "Cypriot"                => 21,
2799     "Cyrillic"               => 22,
2800     "Deseret"                => 23,
2801     "Devanagari"             => 24,
2802     "Egyptian_Hieroglyphs"   => 25,
2803     "Ethiopic"               => 26,
2804     "Georgian"               => 27,
2805     "Glagolitic"             => 28,
2806     "Gothic"                 => 29,
2807     "Greek"                  => 30,
2808     "Gujarati"               => 31,
2809     "Gurmukhi"               => 32,
2810     "Han"                    => 33,
2811     "Hangul"                 => 34,
2812     "Hanunoo"                => 35,
2813     "Hebrew"                 => 36,
2814     "Hiragana"               => 37,
2815     "Imperial_Aramaic"       => 38,
2816     "Inscriptional_Pahlavi"  => 39,
2817     "Inscriptional_Parthian" => 40,
2818     "Javanese"               => 41,
2819     "Kaithi"                 => 42,
2820     "Kannada"                => 43,
2821     "Katakana"               => 44,
2822     "Kayah_Li"               => 45,
2823     "Kharoshthi"             => 46,
2824     "Khmer"                  => 47,
2825     "Lao"                    => 48,
2826     "Latin"                  => 49,
2827     "Lepcha"                 => 50,
2828     "Limbu"                  => 51,
2829     "Linear_B"               => 52,
2830     "Lisu"                   => 53,
2831     "Lycian"                 => 54,
2832     "Lydian"                 => 55,
2833     "Malayalam"              => 56,
2834     "Mandaic"                => 57,
2835     "Meetei_Mayek"           => 58,
2836     "Mongolian"              => 59,
2837     "Myanmar"                => 60,
2838     "New_Tai_Lue"            => 61,
2839     "Nko"                    => 62,
2840     "Ogham"                  => 63,
2841     "Ol_Chiki"               => 64,
2842     "Old_Italic"             => 65,
2843     "Old_Persian"            => 66,
2844     "Old_South_Arabian"      => 67,
2845     "Old_Turkic"             => 68,
2846     "Oriya"                  => 69,
2847     "Osmanya"                => 70,
2848     "Phags_Pa"               => 71,
2849     "Phoenician"             => 72,
2850     "Rejang"                 => 73,
2851     "Runic"                  => 74,
2852     "Samaritan"              => 75,
2853     "Saurashtra"             => 76,
2854     "Shavian"                => 77,
2855     "Sinhala"                => 78,
2856     "Sundanese"              => 79,
2857     "Syloti_Nagri"           => 80,
2858     "Syriac"                 => 81,
2859     "Tagalog"                => 82,
2860     "Tagbanwa"               => 83,
2861     "Tai_Le"                 => 84,
2862     "Tai_Tham"               => 85,
2863     "Tai_Viet"               => 86,
2864     "Tamil"                  => 87,
2865     "Telugu"                 => 88,
2866     "Thaana"                 => 89,
2867     "Thai"                   => 90,
2868     "Tibetan"                => 91,
2869     "Tifinagh"               => 92,
2870     "Ugaritic"               => 93,
2871     "Vai"                    => 94,
2872     "Yi"                     => 95,
2873     # Win8/Win8.1
2874     "Chakma"                 => 96,
2875     "Meroitic_Cursive"       => 97,
2876     "Meroitic_Hieroglyphs"   => 98,
2877     "Miao"                   => 99,
2878     "Sharada"                => 100,
2879     "Sora_Sompeng"           => 101,
2880     "Takri"                  => 102,
2881     # Win10
2882     "Bassa_Vah"              => 103,
2883     "Caucasian_Albanian"     => 104,
2884     "Duployan"               => 105,
2885     "Elbasan"                => 106,
2886     "Grantha"                => 107,
2887     "Khojki"                 => 108,
2888     "Khudawadi"              => 109,
2889     "Linear_A"               => 110,
2890     "Mahajani"               => 111,
2891     "Manichaean"             => 112,
2892     "Mende_Kikakui"          => 113,
2893     "Modi"                   => 114,
2894     "Mro"                    => 115,
2895     "Nabataean"              => 116,
2896     "Old_North_Arabian"      => 117,
2897     "Old_Permic"             => 118,
2898     "Pahawh_Hmong"           => 119,
2899     "Palmyrene"              => 120,
2900     "Pau_Cin_Hau"            => 121,
2901     "Psalter_Pahlavi"        => 122,
2902     "Siddham"                => 123,
2903     "Tirhuta"                => 124,
2904     "Warang_Citi"            => 125,
2905     # Win10 RS1
2906     "Adlam"                  => 126,
2907     "Ahom"                   => 127,
2908     "Anatolian_Hieroglyphs"  => 128,
2909     "Bhaiksuki"              => 129,
2910     "Hatran"                 => 130,
2911     "Marchen"                => 131,
2912     "Multani"                => 132,
2913     "Newa"                   => 133,
2914     "Old_Hungarian"          => 134,
2915     "Osage"                  => 135,
2916     "SignWriting"            => 136,
2917     "Tangut"                 => 137,
2918     # Win10 RS4
2919     "Masaram_Gondi"          => 138,
2920     "Nushu"                  => 139,
2921     "Soyombo"                => 140,
2922     "Zanabazar_Square"       => 141,
2923     # Win10 1903
2924     "Dogra"                  => 142,
2925     "Gunjala_Gondi"          => 143,
2926     "Hanifi_Rohingya"        => 144,
2927     "Makasar"                => 145,
2928     "Medefaidrin"            => 146,
2929     "Old_Sogdian"            => 147,
2930     "Sogdian"                => 148,
2931     # Win10 2004
2932     "Elymaic"                => 149,
2933     "Nyiakeng_Puachue_Hmong" => 150,
2934     "Nandinagari"            => 151,
2935     "Wancho"                 => 152,
2936     # Win11
2937     "Chorasmian"             => 153,
2938     "Dives_Akuru"            => 154,
2939     "Khitan_Small_Script"    => 155,
2940     "Yezidi"                 => 156,
2941 );
2942
2943 ################################################################
2944 # dump Script IDs table
2945 sub dump_scripts($)
2946 {
2947     my $filename = shift;
2948     my $header = $filename;
2949     my @scripts_table;
2950     my $script_index;
2951     my $i;
2952
2953     my $INPUT = open_data_file( "ucd", "Scripts.txt" );
2954     # Fill the table
2955     # Unknown script id is always 0, so undefined scripts are automatically treated as such
2956     while (<$INPUT>)
2957     {
2958         my $type = "";
2959
2960         next if /^\#/;  # skip comments
2961         next if /^\s*$/;  # skip empty lines
2962         next if /\x1a/;  # skip ^Z
2963         if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*/)
2964         {
2965             $type = $2;
2966             if (defined $scripts{$type})
2967             {
2968                 $scripts_table[hex $1] = $scripts{$type};
2969             }
2970             next;
2971         }
2972         elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*/)
2973         {
2974             $type = $3;
2975             if (defined $scripts{$type})
2976             {
2977                 foreach my $i (hex $1 .. hex $2)
2978                 {
2979                     $scripts_table[$i] = $scripts{$type};
2980                 }
2981             }
2982             next;
2983         }
2984     }
2985
2986     close $INPUT;
2987
2988     $header = "$filename.h";
2989     open OUTPUT,">$header.new" or die "Cannot create $header";
2990     print "Building $header\n";
2991     print OUTPUT "/* Unicode Script IDs */\n";
2992     print OUTPUT "/* generated from $current_data_file */\n";
2993     print OUTPUT "/* DO NOT EDIT!! */\n\n";
2994
2995     print OUTPUT "enum unicode_script_id {\n";
2996     foreach my $script (sort { $scripts{$a} <=> $scripts{$b} } keys %scripts)
2997     {
2998         print OUTPUT "    Script_$script = $scripts{$script},\n";
2999     }
3000     print OUTPUT "    Script_LastId = ", (scalar keys %scripts) - 1, "\n";
3001     print OUTPUT "};\n";
3002
3003     close OUTPUT;
3004     save_file($header);
3005
3006     $filename = "$filename.c";
3007     open OUTPUT,">$filename.new" or die "Cannot create $header";
3008     print "Building $filename\n";
3009     print OUTPUT "/* Unicode Script IDs */\n";
3010     print OUTPUT "/* generated from $current_data_file */\n";
3011     print OUTPUT "/* DO NOT EDIT!! */\n\n";
3012     print OUTPUT "#include \"windef.h\"\n\n";
3013
3014     dump_three_level_mapping( "wine_scripts_table", 0, 16, @scripts_table );
3015     close OUTPUT;
3016     save_file($filename);
3017 }
3018
3019 ################################################################
3020 # dump the BiDi mirroring table
3021 sub dump_mirroring($)
3022 {
3023     my $filename = shift;
3024     my @mirror_table = ();
3025
3026     my $INPUT = open_data_file( "ucd", "BidiMirroring.txt" );
3027     while (<$INPUT>)
3028     {
3029         next if /^\#/;  # skip comments
3030         next if /^$/;  # skip empty lines
3031         next if /\x1a/;  # skip ^Z
3032         if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9a-fA-F]+)/)
3033         {
3034             $mirror_table[hex $1] = hex $2;
3035             next;
3036         }
3037         die "malformed line $_";
3038     }
3039     close $INPUT;
3040
3041     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3042     print "Building $filename\n";
3043     print OUTPUT "/* Unicode BiDi mirroring */\n";
3044     print OUTPUT "/* generated from $current_data_file */\n";
3045     print OUTPUT "/* DO NOT EDIT!! */\n\n";
3046     print OUTPUT "#include \"windef.h\"\n\n";
3047     dump_two_level_mapping( "wine_mirror_map", 0, 16, @mirror_table );
3048     close OUTPUT;
3049     save_file($filename);
3050 }
3051
3052 ################################################################
3053 # dump the Bidi Brackets
3054 sub dump_bracket($)
3055 {
3056     my $filename = shift;
3057     my @bracket_table;
3058
3059     my $INPUT = open_data_file( "ucd", "BidiBrackets.txt" );
3060     while (<$INPUT>)
3061     {
3062         next if /^\#/;  # skip comments
3063         next if /^\s*$/;  # skip empty lines
3064         next if /\x1a/;  # skip ^Z
3065         if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9a-fA-F]+);\s*([con])/)
3066         {
3067             my $type = $3;
3068             die "unknown bracket $type" unless defined $bracket_types{$type};
3069             die "characters too distant $1 and $2" if abs(hex($2) - hex($1)) >= 128;
3070             $bracket_table[hex $1] = (hex($2) - hex($1)) % 255;
3071             $bracket_table[hex $1] += $bracket_types{$type} << 8;
3072             next;
3073         }
3074         die "malformed line $_";
3075     }
3076     close $INPUT;
3077
3078     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3079     print "Building $filename\n";
3080     print OUTPUT "/* Unicode Bidirectional Bracket table */\n";
3081     print OUTPUT "/* generated from $current_data_file */\n";
3082     print OUTPUT "/* DO NOT EDIT!! */\n\n";
3083     print OUTPUT "#include \"windef.h\"\n\n";
3084
3085     dump_two_level_mapping( "bidi_bracket_table", 0, 16, @bracket_table );
3086
3087     close OUTPUT;
3088     save_file($filename);
3089 }
3090
3091 ################################################################
3092 # dump the Arabic shaping table
3093 sub dump_shaping($)
3094 {
3095     my $filename = shift;
3096     my @joining_table = @initial_joining_table;
3097
3098     my $INPUT = open_data_file( "ucd", "ArabicShaping.txt" );
3099     while (<$INPUT>)
3100     {
3101         next if /^\#/;  # skip comments
3102         next if /^\s*$/;  # skip empty lines
3103         next if /\x1a/;  # skip ^Z
3104         if (/^\s*([0-9a-fA-F]+)\s*;.*;\s*([RLDCUT])\s*;\s*(\w+)/)
3105         {
3106             my $type = $2;
3107             $joining_table[hex $1] = $joining_types{$type};
3108             next;
3109         }
3110         die "malformed line $_";
3111     }
3112     close $INPUT;
3113
3114     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3115     print "Building $filename\n";
3116     print OUTPUT "/* Unicode Arabic shaping */\n";
3117     print OUTPUT "/* generated from $current_data_file */\n";
3118     print OUTPUT "/* DO NOT EDIT!! */\n\n";
3119     print OUTPUT "#include \"windef.h\"\n\n";
3120
3121     dump_two_level_mapping( "wine_shaping_table", 0, 16, @joining_table );
3122
3123     print OUTPUT "\nconst unsigned short wine_shaping_forms[256][4] =\n{\n";
3124     for (my $i = 0x600; $i <= 0x6ff; $i++)
3125     {
3126         printf OUTPUT "    { 0x%04x, 0x%04x, 0x%04x, 0x%04x },\n",
3127             ${joining_forms{"isolated"}}[$i] || $i,
3128             ${joining_forms{"final"}}[$i] || $i,
3129             ${joining_forms{"initial"}}[$i] || $i,
3130             ${joining_forms{"medial"}}[$i] || $i;
3131     }
3132     print OUTPUT "};\n";
3133
3134     close OUTPUT;
3135     save_file($filename);
3136 }
3137
3138 ################################################################
3139 # dump the Arabic shaping table
3140 sub dump_arabic_shaping($)
3141 {
3142     my $filename = shift;
3143     my @joining_table = @initial_joining_table;
3144
3145     my $INPUT = open_data_file( "ucd", "ArabicShaping.txt" );
3146     while (<$INPUT>)
3147     {
3148         next if /^\#/;  # skip comments
3149         next if /^\s*$/;  # skip empty lines
3150         next if /\x1a/;  # skip ^Z
3151         if (/^\s*([0-9a-fA-F]+)\s*;.*;\s*([RLDCUT])\s*;\s*(\w+)/)
3152         {
3153             my $type = $2;
3154             my $group = $3;
3155
3156             if ($group eq "ALAPH" || $group eq "DALATH RISH")
3157             {
3158                 $joining_table[hex $1] = $joining_types{$group};
3159             }
3160             else
3161             {
3162                 $joining_table[hex $1] = $joining_types{$type};
3163             }
3164
3165             next;
3166         }
3167         die "malformed line $_";
3168     }
3169     close $INPUT;
3170
3171     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3172     print "Building $filename\n";
3173     print OUTPUT "/* Unicode Arabic shaping */\n";
3174     print OUTPUT "/* generated from $current_data_file */\n";
3175     print OUTPUT "/* DO NOT EDIT!! */\n\n";
3176     print OUTPUT "#include \"windef.h\"\n\n";
3177
3178     dump_three_level_mapping( "arabic_shaping_table", 0, 16, @joining_table );
3179
3180     close OUTPUT;
3181     save_file($filename);
3182 }
3183
3184 ################################################################
3185 # dump the Vertical Orientation table
3186 sub dump_vertical($$)
3187 {
3188     my ($filename, $unix) = @_;
3189     my @vertical_table;
3190
3191     my $INPUT = open_data_file( "ucd", "VerticalOrientation.txt" );
3192     while (<$INPUT>)
3193     {
3194         next if /^\#/;  # skip comments
3195         next if /^\s*$/;  # skip empty lines
3196         next if /\x1a/;  # skip ^Z
3197         if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*/)
3198         {
3199             my $type = $2;
3200             die "unknown vertical $type" unless defined $vertical_types{$type};
3201             if (hex $1 < 65536)
3202             {
3203                 $vertical_table[hex $1] = $vertical_types{$type};
3204             }
3205             next;
3206         }
3207         elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*/)
3208         {
3209             my $type = $3;
3210             die "unknown vertical $type" unless defined $vertical_types{$type};
3211             foreach my $i (hex $1 .. hex $2)
3212             {
3213                 $vertical_table[$i] = $vertical_types{$type};
3214             }
3215             next;
3216         }
3217         die "malformed line $_";
3218     }
3219     close $INPUT;
3220
3221     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3222     print "Building $filename\n";
3223     print OUTPUT "/* Unicode Vertical Orientation */\n";
3224     print OUTPUT "/* generated from $current_data_file */\n";
3225     print OUTPUT "/* DO NOT EDIT!! */\n\n";
3226     if ($unix)
3227     {
3228         print OUTPUT "#if 0\n";
3229         print OUTPUT "#pragma makedep unix\n";
3230         print OUTPUT "#endif\n\n";
3231     }
3232     print OUTPUT "#include \"windef.h\"\n\n";
3233
3234     dump_two_level_mapping( "vertical_orientation_table", $vertical_types{'R'}, 16, @vertical_table );
3235
3236     close OUTPUT;
3237     save_file($filename);
3238 }
3239
3240 ################################################################
3241 # compress a mapping table by removing identical rows
3242 sub compress_array($$@)
3243 {
3244     my $rows = shift;
3245     my $def = shift;
3246     my @table = @_;
3247     my $len = @table / $rows;
3248     my @array;
3249     my $data = "";
3250
3251     # try to merge table rows
3252     for (my $row = 0; $row < $rows; $row++)
3253     {
3254         my $rowtxt = pack "U*", map { defined($_) ? $_ : $def; } @table[($row * $len)..(($row + 1) * $len - 1)];
3255         my $pos = index $data, $rowtxt;
3256         if ($pos == -1)
3257         {
3258             # check if the tail of the data can match the start of the new row
3259             my $first = substr( $rowtxt, 0, 1 );
3260             for (my $i = length($data) - 1; $i > 0; $i--)
3261             {
3262                 $pos = index( substr( $data, -$i ), $first );
3263                 last if $pos == -1;
3264                 $i -= $pos;
3265                 next unless substr( $data, -$i ) eq substr( $rowtxt, 0, $i );
3266                 substr( $data, -$i ) = "";
3267                 last;
3268             }
3269             $pos = length $data;
3270             $data .= $rowtxt;
3271         }
3272         $array[$row] = $rows + $pos;
3273     }
3274     return @array, unpack "U*", $data;
3275 }
3276
3277 ################################################################
3278 # dump a char -> value mapping table using two-level tables
3279 sub dump_two_level_mapping($$$@)
3280 {
3281     my $name = shift;
3282     my $def = shift;
3283     my $size = shift;
3284     my $type = $size == 16 ? "unsigned short" : "unsigned int";
3285     my (@array, @row_array, @data, @row_data);
3286     (@row_array[0..4095], @data) = compress_array( 4096, $def, @_[0..65535] );
3287     (@array[0..255], @row_data) = compress_array( 256, 0, @row_array );
3288
3289     for (my $i = 0; $i < @row_data; $i++) { $row_data[$i] += @row_data + 256 - 4096; }
3290
3291     printf OUTPUT "const %s %s[%d] =\n{\n", $type, $name, @array + @row_data + @data;
3292     printf OUTPUT "    /* level 1 offsets */\n%s,\n", dump_array( $size, 0, @array );
3293     printf OUTPUT "    /* level 2 offsets */\n%s,\n", dump_array( $size, 0, @row_data );
3294     printf OUTPUT "    /* values */\n%s\n};\n", dump_array( $size, 0, @data );
3295 }
3296
3297 ################################################################
3298 # dump a char -> value mapping table using three-level tables
3299 sub dump_three_level_mapping($$@)
3300 {
3301     my $name = shift;
3302     my $def = shift;
3303     my $size = shift;
3304     my $type = $size == 16 ? "unsigned short" : "unsigned int";
3305     my $level3 = ($MAX_CHAR + 1) / 16;
3306     my $level2 = $level3 / 16;
3307     my $level1 = $level2 / 16;
3308     my @array3 = compress_array( $level3, $def, @_[0..$MAX_CHAR] );
3309     my @array2 = compress_array( $level2, 0, @array3[0..$level3-1] );
3310     my @array1 = compress_array( $level1, 0, @array2[0..$level2-1] );
3311
3312     for (my $i = $level2; $i < @array2; $i++) { $array2[$i] += @array1 + @array2 - $level2 - $level3; }
3313     for (my $i = $level1; $i < @array1; $i++) { $array1[$i] += @array1 - $level2; }
3314
3315     printf OUTPUT "const %s %s[%u] =\n{\n", $type, $name, @array1 + (@array2 - $level2) + (@array3 - $level3);
3316     printf OUTPUT "    /* level 1 offsets */\n%s,\n", dump_array( $size, 0, @array1[0..$level1-1] );
3317     printf OUTPUT "    /* level 2 offsets */\n%s,\n", dump_array( $size, 0, @array1[$level1..$#array1] );
3318     printf OUTPUT "    /* level 3 offsets */\n%s,\n", dump_array( $size, 0, @array2[$level2..$#array2] );
3319     printf OUTPUT "    /* values */\n%s\n};\n", dump_array( $size, 0, @array3[$level3..$#array3] );
3320 }
3321
3322 ################################################################
3323 # dump a binary case mapping table in l_intl.nls format
3324 sub dump_binary_case_table(@)
3325 {
3326     my (@table) = @_;
3327     my @difftable;
3328     my @res;
3329
3330     for (my $i = 0; $i < @table; $i++)
3331     {
3332         next unless defined $table[$i];
3333         $difftable[$i] = ($table[$i] - $i) & 0xffffffff;
3334     }
3335
3336     my (@low_array1, @low_array2, @low_data, @low_row_data);
3337     (@low_array2[0..4095], @low_data) = compress_array( 4096, 0, @difftable[0..65535] );
3338     (@low_array1[0..255], @low_row_data) = compress_array( 256, 0, @low_array2 );
3339
3340     if (scalar @table > 0x10000)
3341     {
3342         my (@high_array1, @high_array2, @high_data, @high_row_data);
3343         (@high_array2[0..32767], @high_data) = compress_array( 32768, 0, @difftable[65536..$MAX_CHAR] );
3344         (@high_array1[0..1023], @high_row_data) = compress_array( 1024, 0, @high_array2 );
3345
3346         push @res, map { $_ + 1024; } @low_array1;
3347         push @res, map { $_ + @res + @low_row_data + @low_data; } @high_array1;
3348         push @res, map { $_ + @res + @low_row_data - 4096; } @low_row_data;
3349         push @res, @low_data;
3350         push @res, map { 2 * ($_ - 32768) + @res + @high_row_data; } @high_row_data;
3351         return pack( "S<*", 1 + scalar @res  + 2 * scalar @high_data, @res ) . pack( "L<*", @high_data );
3352     }
3353     else
3354     {
3355         push @res, @low_array1;
3356         push @res, map { $_ + @res + @low_row_data - 4096; } @low_row_data;
3357         push @res, @low_data;
3358         return pack "S<*", 1 + scalar @res, @res;
3359     }
3360 }
3361
3362 ################################################################
3363 # dump case mappings for l_intl.nls
3364 sub dump_intl_nls($)
3365 {
3366     my @upper_table = @toupper_table;
3367     my @lower_table = @tolower_table;
3368     remove_linguistic_mappings( \@upper_table, \@lower_table );
3369
3370     my $upper = dump_binary_case_table( @upper_table[0..65535] );
3371     my $lower = dump_binary_case_table( @lower_table[0..65535] );
3372
3373     my $filename = shift;
3374     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3375     printf "Building $filename\n";
3376
3377     binmode OUTPUT;
3378     print OUTPUT pack "S<", 1;  # version
3379     print OUTPUT $upper;
3380     print OUTPUT $lower;
3381     close OUTPUT;
3382     save_file($filename);
3383 }
3384
3385
3386 ################################################################
3387 # dump the bidi direction table
3388 sub dump_bidi_dir_table($)
3389 {
3390     my $filename = shift;
3391     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3392     printf "Building $filename\n";
3393     printf OUTPUT "/* Unicode BiDi direction table */\n";
3394     printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
3395     printf OUTPUT "#include \"windef.h\"\n\n";
3396
3397     my @table;
3398
3399     for (my $i = 0; $i < @direction_table; $i++)
3400     {
3401         $table[$i] = $bidi_types{$direction_table[$i]} if defined $direction_table[$i];
3402     }
3403
3404     dump_three_level_mapping( "bidi_direction_table", $bidi_types{"L"}, 16, @table );
3405
3406     close OUTPUT;
3407     save_file($filename);
3408 }
3409
3410
3411 sub rol($$)
3412 {
3413     my ($byte, $count) = @_;
3414     return (($byte << $count) | ($byte >> (8 - $count))) & 0xff;
3415 }
3416
3417 ################################################################
3418 # compress the character properties table
3419 sub compress_char_props_table($@)
3420 {
3421     my $rows = shift;
3422     my @table = @_;
3423     my $len = @table / $rows;
3424     my $pos = 0;
3425     my @array = (0) x $rows;
3426     my %sequences;
3427
3428     # add some predefined sequences
3429     foreach my $i (0, 0xfb .. 0xff) { $sequences{pack "L*", (rol($i,5)) x $len} = $i; }
3430
3431     # try to merge table rows
3432     for (my $row = 0; $row < $rows; $row++)
3433     {
3434         my @table_row = map { defined $_ ? $_ : 0x7f; } @table[($row * $len)..(($row + 1) * $len - 1)];
3435         my $rowtxt = pack "L*", @table_row;
3436         if (defined($sequences{$rowtxt}))
3437         {
3438             # reuse an existing row
3439             $array[$row] = $sequences{$rowtxt};
3440         }
3441         else
3442         {
3443             # create a new row
3444             $sequences{$rowtxt} = $array[$row] = ++$pos;
3445             push @array, @table_row;
3446         }
3447     }
3448     return @array;
3449 }
3450
3451 ################################################################
3452 # dump a normalization table in binary format
3453 sub dump_norm_table($)
3454 {
3455     my $filename = shift;
3456
3457     my %forms  = ( "nfc" => 1, "nfd" => 2, "nfkc" => 5, "nfkd" => 6, "idna" => 13 );
3458     my %decomp = ( "nfc" => \@decomp_table,
3459                    "nfd" => \@decomp_table,
3460                    "nfkc" => \@decomp_compat_table,
3461                    "nfkd" => \@decomp_compat_table ,
3462                    "idna" => \@idna_decomp_table );
3463
3464     open OUTPUT,">$filename.new" or die "Cannot create $filename";
3465     print "Building $filename\n";
3466
3467     my $type = $filename;
3468     $type =~ s!.*/norm(\w+)\.nls!$1!;
3469
3470     my $compose = $forms{$type} & 1;
3471     my $compat = !!($forms{$type} & 4) + ($type eq "idna");
3472
3473     my @version = split /\./, $UNIVERSION;
3474
3475     # combining classes
3476
3477     my @classes;
3478     my @class_values;
3479
3480     foreach my $c (grep defined, @combining_class_table)
3481     {
3482         $classes[$c] = 1 if $c < 0x100;
3483     }
3484     for (my $i = 0; $i < @classes; $i++)
3485     {
3486         next unless defined $classes[$i];
3487         $classes[$i] = @class_values;
3488         push @class_values, $i;
3489     }
3490     push @class_values, 0 if (@class_values % 2);
3491     die "too many classes" if @class_values >= 0x40;
3492
3493     # character properties
3494
3495     my @char_props;
3496     my @decomposed;
3497     my @comp_hash_table;
3498     my $comp_hash_size = $compose ? 254 : 0;
3499
3500     for (my $i = 0; $i <= $MAX_CHAR; $i++)
3501     {
3502         next unless defined $combining_class_table[$i];
3503         if (defined $decomp{$type}->[$i])
3504         {
3505             my @dec = get_decomposition( $i, $decomp{$type} );
3506             if ($compose && (my @comp = get_composition( $i, $compat )))
3507             {
3508                 my $hash = ($comp[0] + 95 * $comp[1]) % $comp_hash_size;
3509                 push @{$comp_hash_table[$hash]}, to_utf16( @comp, $i );
3510
3511                 my $val = 0;
3512                 foreach my $d (@dec)
3513                 {
3514                     $val = $combining_class_table[$d];
3515                     last if $val;
3516                 }
3517                 $char_props[$i] = $classes[$val];
3518             }
3519             else
3520             {
3521                 $char_props[$i] = 0xbf;
3522             }
3523             @dec = compose_hangul( @dec ) if $compose;
3524             @dec = to_utf16( @dec );
3525             push @dec, 0 if @dec >= 7;
3526             $decomposed[$i] = \@dec;
3527         }
3528         else
3529         {
3530             if ($combining_class_table[$i] == 0x100)
3531             {
3532                 $char_props[$i] = 0x7f;
3533             }
3534             elsif ($combining_class_table[$i])
3535             {
3536                 $char_props[$i] = $classes[$combining_class_table[$i]] | 0x80;
3537             }
3538             elsif ($type eq "idna" && defined $idna_disallowed[$i])
3539             {
3540                 $char_props[$i] = 0xff;
3541             }
3542             else
3543             {
3544                 $char_props[$i] = 0;
3545             }
3546         }
3547     }
3548
3549     if ($compose)
3550     {
3551         for (my $i = 0; $i <= $MAX_CHAR; $i++)
3552         {
3553             my @comp = get_composition( $i, $compat );
3554             next unless @comp;
3555             if ($combining_class_table[$comp[1]])
3556             {
3557                 $char_props[$comp[0]] |= 0x40 unless $char_props[$comp[0]] & 0x80;
3558                 $char_props[$comp[1]] |= 0x40;
3559             }
3560             else
3561             {
3562                 $char_props[$comp[0]] = ($char_props[$comp[0]] & ~0x40) | 0x80;
3563                 $char_props[$comp[1]] |= 0xc0;
3564             }
3565         }
3566     }
3567
3568     # surrogates
3569     foreach my $i (0xd800..0xdbff) { $char_props[$i] = 0xdf; }
3570     foreach my $i (0xdc00..0xdfff) { $char_props[$i] = 0x9f; }
3571
3572     # Hangul
3573     if ($type eq "nfc") { foreach my $i (0x1100..0x117f) { $char_props[$i] = 0xff; } }
3574     elsif ($compose) { foreach my $i (0x1100..0x11ff) { $char_props[$i] = 0xff; } }
3575     foreach my $i (0xac00..0xd7ff) { $char_props[$i] = 0xff; }
3576
3577     # invalid chars
3578     if ($type eq "idna") { foreach my $i (0x00..0x1f, 0x7f) { $char_props[$i] = 0xff; } }
3579     foreach my $i (0xfdd0..0xfdef) { $char_props[$i] = 0xff; }
3580     foreach my $i (0x00..0x10)
3581     {
3582         $char_props[($i << 16) | 0xfffe] = 0xff;
3583         $char_props[($i << 16) | 0xffff] = 0xff;
3584     }
3585
3586     # decomposition hash table
3587
3588     my @decomp_hash_table;
3589     my @decomp_hash_index;
3590     my @decomp_hash_data;
3591     my $decomp_hash_size = 944;
3592
3593     # build string of character data, reusing substrings when possible
3594     my $decomp_char_data = "";
3595     foreach my $i (sort { @{$b} <=> @{$a} } grep defined, @decomposed)
3596     {
3597         my $str = pack "U*", @{$i};
3598         $decomp_char_data .= $str if index( $decomp_char_data, $str) == -1;
3599     }
3600     for (my $i = 0; $i < @decomposed; $i++)
3601     {
3602         next unless defined $decomposed[$i];
3603         my $pos = index( $decomp_char_data, pack( "U*", @{$decomposed[$i]} ));
3604         die "sequence not found" if $pos == -1;
3605         my $len = @{$decomposed[$i]};
3606         $len = 7 if $len > 7;
3607         my $hash = $i % $decomp_hash_size;
3608         push @{$decomp_hash_table[$hash]}, [ $i, ($len << 13) | $pos ];
3609     }
3610     for (my $i = 0; $i < $decomp_hash_size; $i++)
3611     {
3612         $decomp_hash_index[$i] = @decomp_hash_data / 2;
3613         next unless defined $decomp_hash_table[$i];
3614         if (@{$decomp_hash_table[$i]} == 1)
3615         {
3616             my $entry = $decomp_hash_table[$i]->[0];
3617             if ($char_props[$entry->[0]] == 0xbf)
3618             {
3619                 $decomp_hash_index[$i] = $entry->[1];
3620                 next;
3621             }
3622         }
3623         foreach my $entry (@{$decomp_hash_table[$i]})
3624         {
3625             push @decomp_hash_data, $entry->[0] & 0xffff, $entry->[1];
3626         }
3627     }
3628     push @decomp_hash_data, 0, 0;
3629
3630     # composition hash table
3631
3632     my @comp_hash_index;
3633     my @comp_hash_data;
3634     if (@comp_hash_table)
3635     {
3636         for (my $i = 0; $i < $comp_hash_size; $i++)
3637         {
3638             $comp_hash_index[$i] = @comp_hash_data;
3639             push @comp_hash_data, @{$comp_hash_table[$i]} if defined $comp_hash_table[$i];
3640         }
3641         $comp_hash_index[$comp_hash_size] = @comp_hash_data;
3642         push @comp_hash_data, 0, 0, 0;
3643     }
3644
3645     my $level1 = ($MAX_CHAR + 1) / 128;
3646     my @rows = compress_char_props_table( $level1, @char_props[0..$MAX_CHAR] );
3647
3648     my @header = ( $version[0], $version[1], $version[2], 0, $forms{$type}, $compat ? 18 : 3,
3649                    0, $decomp_hash_size, $comp_hash_size, 0 );
3650     my @tables = (0) x 8;
3651
3652     $tables[0] = 16 + @header + @tables;
3653     $tables[1] = $tables[0] + @class_values / 2;
3654     $tables[2] = $tables[1] + $level1 / 2;
3655     $tables[3] = $tables[2] + (@rows - $level1) / 2;
3656     $tables[4] = $tables[3] + @decomp_hash_index;
3657     $tables[5] = $tables[4] + @decomp_hash_data;
3658     $tables[6] = $tables[5] + length $decomp_char_data;
3659     $tables[7] = $tables[6] + @comp_hash_index;
3660
3661     print OUTPUT pack "S<16", unpack "U*", "norm$type.nlp";
3662     print OUTPUT pack "S<*", @header;
3663     print OUTPUT pack "S<*", @tables;
3664     print OUTPUT pack "C*", @class_values;
3665
3666     print OUTPUT pack "C*", @rows[0..$level1-1];
3667     print OUTPUT pack "C*", @rows[$level1..$#rows];
3668     print OUTPUT pack "S<*", @decomp_hash_index;
3669     print OUTPUT pack "S<*", @decomp_hash_data;
3670     print OUTPUT pack "S<*", unpack "U*", $decomp_char_data;
3671     print OUTPUT pack "S<*", @comp_hash_index;
3672     print OUTPUT pack "S<*", @comp_hash_data;
3673
3674     close OUTPUT;
3675     save_file($filename);
3676
3677     add_registry_string_value( $nlskey, "Normalization", sprintf( "%x", $forms{$type} ), "norm$type.nls" );
3678 }
3679
3680
3681 ################################################################
3682 # output a codepage definition file from the global tables
3683 sub output_codepage_file($)
3684 {
3685     my $codepage = shift;
3686
3687     my $output = sprintf "nls/c_%03d.nls", $codepage;
3688     open OUTPUT,">$output.new" or die "Cannot create $output";
3689
3690     printf "Building %s\n", $output;
3691     if (!@lead_bytes) { dump_binary_sbcs_table( $codepage ); }
3692     else { dump_binary_dbcs_table( $codepage ); }
3693
3694     close OUTPUT;
3695     save_file($output);
3696
3697     add_registry_string_value( $nlskey, "Codepage", sprintf( "%d", $codepage ), sprintf( "c_%03d.nls", $codepage ));
3698 }
3699
3700 ################################################################
3701 # output a codepage table from a Microsoft-style mapping file
3702 sub dump_msdata_codepage($)
3703 {
3704     my $filename = shift;
3705
3706     my $state = "";
3707     my ($codepage, $width, $count);
3708     my ($lb_cur, $lb_end);
3709
3710     @cp2uni = ();
3711     @glyph2uni = ();
3712     @lead_bytes = ();
3713     @uni2cp = ();
3714     $default_char = $DEF_CHAR;
3715     $default_wchar = $DEF_CHAR;
3716
3717     my $INPUT = open_data_file( "codepages", $filename );
3718
3719     while (<$INPUT>)
3720     {
3721         next if /^;/;  # skip comments
3722         next if /^\s*$/;  # skip empty lines
3723         next if /\x1a/;  # skip ^Z
3724         last if /^ENDCODEPAGE/;
3725
3726         if (/^CODEPAGE\s+(\d+)/)
3727         {
3728             $codepage = $1;
3729             next;
3730         }
3731         if (/^CPINFO\s+(\d+)\s+0x([0-9a-fA-f]+)\s+0x([0-9a-fA-F]+)/)
3732         {
3733             $width = $1;
3734             $default_char = hex $2;
3735             $default_wchar = hex $3;
3736             next;
3737         }
3738         if (/^(MBTABLE|GLYPHTABLE|WCTABLE|DBCSRANGE|DBCSTABLE)\s+(\d+)/)
3739         {
3740             $state = $1;
3741             $count = $2;
3742             next;
3743         }
3744         if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)/)
3745         {
3746             if ($state eq "MBTABLE")
3747             {
3748                 my $cp = hex $1;
3749                 my $uni = hex $2;
3750                 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
3751                 next;
3752             }
3753             if ($state eq "GLYPHTABLE")
3754             {
3755                 my $cp = hex $1;
3756                 my $uni = hex $2;
3757                 $glyph2uni[$cp] = $uni unless defined($glyph2uni[$cp]);
3758                 next;
3759             }
3760             if ($state eq "WCTABLE")
3761             {
3762                 my $uni = hex $1;
3763                 my $cp = hex $2;
3764                 $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
3765                 next;
3766             }
3767             if ($state eq "DBCSRANGE")
3768             {
3769                 my $start = hex $1;
3770                 my $end = hex $2;
3771                 for (my $i = $start; $i <= $end; $i++) { add_lead_byte( $i ); }
3772                 $lb_cur = $start;
3773                 $lb_end = $end;
3774                 next;
3775             }
3776             if ($state eq "DBCSTABLE")
3777             {
3778                 my $mb = hex $1;
3779                 my $uni = hex $2;
3780                 my $cp = ($lb_cur << 8) | $mb;
3781                 $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
3782                 if (!--$count)
3783                 {
3784                     if (++$lb_cur > $lb_end) { $state = "DBCSRANGE"; }
3785                 }
3786                 next;
3787             }
3788         }
3789         die "$filename: Unrecognized line $_\n";
3790     }
3791     close $INPUT;
3792
3793     output_codepage_file( $codepage );
3794
3795     if ($codepage == 949) { dump_krwansung_codepage( @uni2cp ); }
3796 }
3797
3798 ################################################################
3799 # align a string length
3800 sub align_string($$)
3801 {
3802     my ($align, $str) = @_;
3803     $str .= pack "C*", (0) x ($align - length($str) % $align) if length($str) % $align;
3804     return $str;
3805 }
3806
3807 ################################################################
3808 # pad a string with zeros
3809 sub pad_string($$)
3810 {
3811     my ($pad, $str) = @_;
3812     $str .= pack "C*", (0) x ($pad - length($str)) if length($str) < $pad;
3813     return $str;
3814 }
3815
3816 ################################################################
3817 # pack a GUID string
3818 sub pack_guid($)
3819 {
3820     $_ = shift;
3821     /([0-9A-Fa-f]{8})-([0-9A-Fa-f]{4})-([0-9A-Fa-f]{4})-([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})-([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})/;
3822     return pack "L<S<2C8", hex $1, hex $2, hex $3, hex $4, hex $5, hex $6, hex $7, hex $8, hex $9, hex $10, hex $11;
3823 }
3824
3825 ################################################################
3826 # comparison function for compression sort
3827 sub cmp_compression
3828 {
3829     return scalar @{$a} <=> scalar @{$b} ||
3830         $a->[4] <=> $b->[4] ||
3831         $a->[5] <=> $b->[5] ||
3832         $a->[6] <=> $b->[6] ||
3833         $a->[7] <=> $b->[7] ||
3834         $a->[8] <=> $b->[8] ||
3835         $a->[9] <=> $b->[9] ||
3836         $a->[10] <=> $b->[10] ||
3837         $a->[11] <=> $b->[11] ||
3838         $a->[12] <=> $b->[12];
3839 }
3840
3841 ################################################################
3842 # build a binary sort keys table
3843 sub dump_sortkey_table($)
3844 {
3845     my $filename = shift;
3846     my @keys;
3847     my ($part, $section, $subsection, $guid, $version, $ling_flag);
3848     my @multiple_weights;
3849     my @expansions;
3850     my @compressions;
3851     my %exceptions;
3852     my %guids;
3853     my %compr_flags;
3854     my %locales;
3855     my $default_guid = "00000001-57ee-1e5c-00b4-d0000bb1e11e";
3856     my $jamostr = "";
3857
3858     my $re_hex = '0x[0-9A-Fa-f]+';
3859     my $re_key = '(\d+\s+\d+\s+\d+\s+\d+)';
3860     $guids{$default_guid} = { };
3861
3862     my %flags = ( "HAS_3_BYTE_WEIGHTS" => 0x01, "REVERSEDIACRITICS" => 0x10, "DOUBLECOMPRESSION" => 0x20, "INVERSECASING" => 0x40 );
3863
3864     my $KEYS = open_data_file( "sorting" );
3865
3866     printf "Building $filename\n";
3867
3868     while (<$KEYS>)
3869     {
3870         s/\s*;.*$//;
3871         next if /^\s*$/;  # skip empty lines
3872         if (/^\s*(SORTKEY|SORTTABLES)/)
3873         {
3874             $part = $1;
3875             next;
3876         }
3877         if (/^\s*(ENDSORTKEY|ENDSORTTABLES)/)
3878         {
3879             $part = $section = "";
3880             next;
3881         }
3882         if (/^\s*(DEFAULT|RELEASE|REVERSEDIACRITICS|DOUBLECOMPRESSION|INVERSECASING|MULTIPLEWEIGHTS|EXPANSION|COMPATIBILITY|COMPRESSION|EXCEPTION|JAMOSORT)\s+/)
3883         {
3884             $section = $1;
3885             $guid = undef;
3886             next;
3887         }
3888         next unless $part;
3889         if ("$part.$section" eq "SORTKEY.DEFAULT")
3890         {
3891             if (/^\s*($re_hex)\s+$re_key/)
3892             {
3893                 $keys[hex $1] = [ split(/\s+/,$2) ];
3894                 next;
3895             }
3896         }
3897         elsif ("$part.$section" eq "SORTTABLES.RELEASE")
3898         {
3899             if (/^\s*NLSVERSION\s+0x([0-9A-Fa-f]+)/)
3900             {
3901                 $version = hex $1;
3902                 next;
3903             }
3904             if (/^\s*DEFINEDVERSION\s+0x([0-9A-Fa-f]+)/)
3905             {
3906                 # ignore for now
3907                 next;
3908             }
3909         }
3910         elsif ("$part.$section" eq "SORTTABLES.REVERSEDIACRITICS" ||
3911                "$part.$section" eq "SORTTABLES.DOUBLECOMPRESSION" ||
3912                "$part.$section" eq "SORTTABLES.INVERSECASING")
3913         {
3914             if (/^\s*SORTGUID\s+([-0-9A-Fa-f]+)/)
3915             {
3916                 $guid = lc $1;
3917                 $guids{$guid} = { } unless defined $guids{$guid};
3918                 $guids{$guid}->{flags} |= $flags{$section};
3919                 next;
3920             }
3921             if (/^\s*LOCALENAME\s+([A-Za-z0-9-_]+)/)
3922             {
3923                 $locales{$1} = $guid;
3924                 next;
3925             }
3926         }
3927         elsif ("$part.$section" eq "SORTTABLES.MULTIPLEWEIGHTS")
3928         {
3929             if (/^\s*(\d+)\s+(\d+)/)
3930             {
3931                 push @multiple_weights, $1, $2;
3932                 next;
3933             }
3934         }
3935         elsif ("$part.$section" eq "SORTTABLES.EXPANSION")
3936         {
3937             if (/^\s*0x([0-9A-Fa-f]+)\s+0x([0-9A-Fa-f]+)\s+0x([0-9A-Fa-f]+)/)
3938             {
3939                 my $pos = scalar @expansions / 2;
3940                 $keys[hex $1] = [ 2, 0, $pos & 0xff, $pos >> 8 ] unless defined $keys[hex $1];
3941                 push @expansions, hex $2, hex $3;
3942                 next;
3943             }
3944         }
3945         elsif ("$part.$section" eq "SORTTABLES.COMPATIBILITY")
3946         {
3947             if (/^\s*0x([0-9A-Fa-f]+)\s+0x([0-9A-Fa-f]+)/)
3948             {
3949                 $keys[hex $1] = $keys[hex $2];
3950                 next;
3951             }
3952         }
3953         elsif ("$part.$section" eq "SORTTABLES.COMPRESSION")
3954         {
3955             if (/^\s*SORTGUID\s+([-0-9A-Fa-f]+)\s+\d*\s*([A-Z0-9_]+)?/)
3956             {
3957                 if ($subsection || !$guid)  # start a new one
3958                 {
3959                     $guid = lc $1;
3960                     $subsection = "";
3961                     $guids{$guid} = { } unless defined $guids{$guid};
3962                     $guids{$guid}->{flags} |= $flags{$2} if $2;
3963                     $guids{$guid}->{compr} = @compressions;
3964                     $exceptions{"$guid-"} = [ ] unless defined $exceptions{"$guid-"};
3965                     $compr_flags{$guid} = [ ] unless defined $compr_flags{$guid};
3966                     push @compressions, [ ];
3967                 }
3968                 else  # merge with current one
3969                 {
3970                     $guids{lc $1} = { } unless defined $guids{lc $1};
3971                     $guids{lc $1}->{flags} |= $flags{$2} if $2;
3972                     $guids{lc $1}->{compr} = $guids{$guid}->{compr};
3973                     $compr_flags{lc $1} = $compr_flags{$guid};
3974                 }
3975                 next;
3976             }
3977             if (/^\s*LOCALENAME\s+([A-Za-z0-9-_]+)/)
3978             {
3979                 $locales{$1} = $guid;
3980                 next;
3981             }
3982             if (/^\s*(TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT)/)
3983             {
3984                 $subsection = $1;
3985                 next;
3986             }
3987             if ($subsection && /^\s*(($re_hex\s+){2,8})$re_key/)
3988             {
3989                 my @comp = map { hex $_; } split(/\s+/,$1);
3990                 push @{$compressions[$#compressions]}, [ split(/\s+/,$3), @comp ];
3991                 # add compression flags
3992                 $compr_flags{$guid}->[$comp[0]] |= @comp >= 6 ? 0xc0 : @comp >= 4 ? 0x80 : 0x40;
3993                 next;
3994             }
3995         }
3996         elsif ("$part.$section" eq "SORTTABLES.EXCEPTION")
3997         {
3998             if (/^\s*SORTGUID\s+([-0-9A-Fa-f]+)\s+\d*\s*(LINGUISTIC_CASING)?/)
3999             {
4000                 $guid = lc $1;
4001                 $guids{$guid} = { } unless defined $guids{lc $1};
4002                 $ling_flag = ($2 ? "+" : "-");
4003                 $exceptions{"$guid$ling_flag"} = [ ] unless defined $exceptions{"$guid$ling_flag"};
4004                 next;
4005             }
4006             if (/^\s*LOCALENAME\s+([A-Za-z0-9-_]+)/)
4007             {
4008                 $locales{$1} = $guid;
4009                 next;
4010             }
4011             if (/^\s*($re_hex)\s+$re_key/)
4012             {
4013                 $exceptions{"$guid$ling_flag"}->[hex $1] = [ split(/\s+/,$2) ];
4014                 next;
4015             }
4016         }
4017         elsif ("$part.$section" eq "SORTTABLES.JAMOSORT")
4018         {
4019             if (/^\s*$re_hex\s+(($re_hex\s*){5})/)
4020             {
4021                 $jamostr .= pack "C8", map { hex $_; } split /\s+/, $1;
4022                 next;
4023             }
4024         }
4025         die "$current_data_file: $part.$section: unrecognized line $_\n";
4026     }
4027     close $KEYS;
4028
4029     # Sortkey table
4030
4031     my $table;
4032     for (my $i = 0; $i < 0x10000; $i++)
4033     {
4034         my @k = defined $keys[$i] ? @{$keys[$i]} : (0) x 4;
4035         $table .= pack "C4", $k[1], $k[0], $k[2], $k[3];
4036     }
4037
4038     foreach my $id (sort keys %exceptions)
4039     {
4040         my $pos = length($table) / 4;
4041         my @exc = @{$exceptions{$id}};
4042         my @filled;
4043         my $key = (substr( $id, -1 ) eq "+" ? "ling_except" : "except");
4044         my $guid = substr( $id, 0, -1 );
4045         $guids{$guid}->{$key} = $pos;
4046         $pos += 0x100;
4047         my @flags = @{$compr_flags{$guid}} if defined $compr_flags{$guid};
4048         for (my $j = 0; $j < 0x10000; $j++)
4049         {
4050             next unless defined $exc[$j] || defined $flags[$j];
4051             $filled[$j >> 8] = 1;
4052             $j |= 0xff;
4053         }
4054         for (my $j = 0; $j < 0x100; $j++)
4055         {
4056             $table .= pack "L<", $filled[$j] ? $pos : $j * 0x100;
4057             $pos += 0x100 if $filled[$j];
4058         }
4059         for (my $j = 0; $j < 0x10000; $j++)
4060         {
4061             next unless $filled[$j >> 8];
4062             my @k = defined $exc[$j] ? @{$exc[$j]} : defined $keys[$j] ? @{$keys[$j]} : (0) x 4;
4063             $k[3] |= $flags[$j] || 0;
4064             $table .= pack "C4", $k[1], $k[0], $k[2], $k[3];
4065         }
4066     }
4067
4068     # Case mapping tables
4069
4070     # standard table
4071     my @casemaps;
4072     my @upper = @toupper_table;
4073     my @lower = @tolower_table;
4074     remove_linguistic_mappings( \@upper, \@lower );
4075     $casemaps[0] = pack( "S<*", 1) . dump_binary_case_table( @upper ) . dump_binary_case_table( @lower );
4076
4077     # linguistic table
4078     $casemaps[1] = pack( "S<*", 1) . dump_binary_case_table( @toupper_table ) . dump_binary_case_table( @tolower_table );
4079
4080     # Turkish table
4081     @upper = @toupper_table;
4082     @lower = @tolower_table;
4083     $upper[ord 'i'] = 0x130; # LATIN CAPITAL LETTER I WITH DOT ABOVE
4084     $lower[ord 'I'] = 0x131; # LATIN SMALL LETTER DOTLESS I
4085     $casemaps[2] = pack( "S<*", 1) . dump_binary_case_table( @upper ) . dump_binary_case_table( @lower );
4086     my $casemaps = align_string( 8, $casemaps[0] . $casemaps[1] . $casemaps[2] );
4087
4088     # Char type table
4089
4090     my @table;
4091     my $types = "";
4092     my %typestr;
4093     for (my $i = 0; $i < 0x10000; $i++)
4094     {
4095         my $str = pack "S<3",
4096             ($category_table[$i] || 0) & 0xffff,
4097             defined($direction_table[$i]) ? $c2_types{$direction_table[$i]} : 0,
4098             ($category_table[$i] || 0) >> 16;
4099
4100         if (!defined($typestr{$str}))
4101         {
4102             $typestr{$str} = length($types) / 6;
4103             $types .= $str;
4104         }
4105         $table[$i] = $typestr{$str};
4106     }
4107
4108     my (@rows, @array, @data, @row_data);
4109     (@rows[0..4095], @data) = compress_array( 4096, 0, @table[0..65535] );
4110     (@array[0..255], @row_data) = compress_array( 256, 0, @rows );
4111     for (my $i = 0; $i < 256; $i++) { $array[$i] *= 2; }  # we need byte offsets
4112     for (my $i = 0; $i < @row_data; $i++) { $row_data[$i] += 2 * @row_data + 512 - 4096; }
4113
4114     my $arraystr = pack("S<*", @array, @row_data) . pack("C*", @data);
4115     my $chartypes = pack "S<2", 4 + length($types) + length($arraystr), 2 + length($types);
4116     $chartypes = align_string( 8, $chartypes . $types . $arraystr );
4117
4118     # Sort tables
4119
4120     # guids
4121     my $sorttables = pack "L<2", $version, scalar %guids;
4122     foreach my $id (sort keys %guids)
4123     {
4124         my %guid = %{$guids{$id}};
4125         my $flags = $guid{flags} || 0;
4126         my $map = length($casemaps[0]) + (defined $guid{ling_except} ? length($casemaps[1]) : 0);
4127         $sorttables .= pack_guid($id) . pack "L<5",
4128             $flags,
4129             defined($guid{compr}) ? $guid{compr} : 0xffffffff,
4130             $guid{except} || 0,
4131             $guid{ling_except} || 0,
4132             $map / 2;
4133     }
4134
4135     # expansions
4136     $sorttables .= pack "L<S<*", scalar @expansions / 2, @expansions;
4137
4138     # compressions
4139     $sorttables .= pack "L<", scalar @compressions;
4140     my $rowstr = "";
4141     foreach my $c (@compressions)
4142     {
4143         my $pos = length($rowstr) / 2;
4144         my $min = 0xffff;
4145         my $max = 0;
4146         my @lengths = (0) x 8;
4147         foreach my $r (sort cmp_compression @{$c})
4148         {
4149             my @row = @{$r};
4150             $lengths[scalar @row - 6]++;
4151             foreach my $val (@row[4..$#row])
4152             {
4153                 $min = $val if $min > $val;
4154                 $max = $val if $max < $val;
4155             }
4156             $rowstr .= align_string( 4, pack "S<*", @row[4..$#row] );
4157             $rowstr .= pack "C4", $row[1], $row[0], $row[2], $row[3];
4158         }
4159         $sorttables .= pack "L<S<10", $pos, $min, $max, @lengths;
4160     }
4161     $sorttables .= $rowstr;
4162
4163     # multiple weights
4164     $sorttables .= align_string( 4, pack "L<C*", scalar @multiple_weights / 2, @multiple_weights );
4165
4166     # jamo sort
4167     $sorttables .= pack("L<", length($jamostr) / 8) . $jamostr;
4168
4169     # Locales
4170
4171     add_registry_key( $nlskey, "Sorting\\Ids", "{$default_guid}" );
4172     foreach my $loc (sort keys %locales)
4173     {
4174         # skip specific locales that match more general ones
4175         my @parts = split /[-_]/, $loc;
4176         next if @parts > 1 && defined($locales{$parts[0]}) && $locales{$parts[0]} eq $locales{$loc};
4177         next if @parts > 2 && defined($locales{"$parts[0]-$parts[1]"}) && $locales{"$parts[0]-$parts[1]"} eq $locales{$loc};
4178         add_registry_string_value( $nlskey, "Sorting\\Ids", $loc, "\{$locales{$loc}\}" );
4179     }
4180
4181     # File header
4182
4183     my @header;
4184     $header[0] = 16;
4185     $header[1] = $header[0] + length $table;
4186     $header[2] = $header[1] + length $casemaps;
4187     $header[3] = $header[2] + length $chartypes;
4188
4189     open OUTPUT, ">$filename.new" or die "Cannot create $filename";
4190     print OUTPUT pack "L<*", @header;
4191     print OUTPUT $table, $casemaps, $chartypes, $sorttables;
4192     close OUTPUT;
4193     save_file($filename);
4194     return $chartypes;
4195 }
4196
4197
4198 my %lcnames;
4199
4200 sub locale_parent($)
4201 {
4202     my $loc = shift;
4203
4204     return undef unless $loc;
4205     return $lcnames{$loc}->{sparent} if defined $lcnames{$loc} && defined $lcnames{$loc}->{sparent};
4206     return $lcnames{$loc}->{parent} if defined $lcnames{$loc} && defined $lcnames{$loc}->{parent};
4207     if ($loc =~ /(.*)-[0-9A-Za-z]+/) { return $1; }
4208     return "";
4209 }
4210
4211 sub compare_locales
4212 {
4213     (my $n1 = $a) =~ tr/A-Z_/a-z-/;
4214     (my $n2 = $b) =~ tr/A-Z_/a-z-/;
4215     return $n1 cmp $n2;
4216 }
4217
4218 # query an xml key
4219 sub xml_query($$)
4220 {
4221     my ($xml, $query) = @_;
4222     my $ret = $xml->find( $query );
4223     return undef unless $ret;
4224     printf STDERR "multiple entries for %s\n", $query if (@{$ret} > 1);
4225     return @{$ret}[0]->textContent;
4226 }
4227
4228 # query an xml key for a locale, with fallback to the parents
4229 sub loc_query($$)
4230 {
4231     my ($loc, $query) = @_;
4232
4233     $loc = $lcnames{"en-US"} unless $loc->{name};  # fallback to "en-US" for root locale
4234
4235     for (my $cur = $loc->{name}; defined $cur; $cur = locale_parent( $cur ))
4236     {
4237         next unless defined $lcnames{$cur};
4238         my $xml = $lcnames{$cur}->{xml};
4239         my $ret = $xml->find( $query );
4240         next unless $ret;
4241         printf STDERR "%s: multiple entries for %s\n", $cur, $query if (@{$ret} > 1);
4242         next if @{$ret}[0]->textContent eq "\x{2191}\x{2191}\x{2191}"; # "↑↑↑"
4243         return @{$ret}[0]->textContent;
4244     }
4245     return undef;
4246 }
4247
4248 # retrieve a locale field entry by going up the parents tree
4249 sub locale_entry($$$)
4250 {
4251     my ($loc, $field, $def) = @_;
4252
4253     return $loc->{$field} if defined $loc->{$field};
4254
4255     unless ($loc->{name})  # fallback to "en-US" for root locale
4256     {
4257         $loc = $lcnames{"en-US"};
4258         return $loc->{$field} if defined $loc->{$field};
4259     }
4260     while (defined $loc->{alias})  # resolve aliases
4261     {
4262         $loc = $lcnames{$loc->{alias}};
4263         return $loc->{$field} if defined $loc->{$field};
4264     }
4265     my $cur = $loc->{name};
4266     while ($cur)
4267     {
4268         if (defined $lcnames{$cur} && defined $lcnames{$cur}->{sparent})
4269         {
4270             $cur = $lcnames{$cur}->{sparent};
4271         }
4272         elsif ($cur =~ /(.*)-[0-9A-Za-z]+/)
4273         {
4274             $cur = $1;
4275         }
4276         else
4277         {
4278             return $def;
4279         }
4280         return $lcnames{$cur}->{$field} if defined $lcnames{$cur} && defined $lcnames{$cur}->{$field};
4281     }
4282     return $def;
4283 }
4284
4285 my $string_data;
4286
4287 sub add_str_data($)
4288 {
4289     my $txt = shift;
4290     my $ret = index( $string_data, $txt );
4291     if ($ret == -1)
4292     {
4293         $ret = length($string_data);
4294         $string_data .= $txt
4295     }
4296     return $ret / 2;
4297 }
4298
4299 sub add_string($)
4300 {
4301     my $str = shift;
4302     return 0 unless defined($str) && $str ne "";
4303     my $utf = encode( "UTF16LE", $str );
4304     return add_str_data( (pack "S<", length($utf) / 2) . $utf . (pack "S", 0) );
4305 }
4306
4307 sub add_fontsig(@)
4308 {
4309     return add_str_data( pack "S<L<*", scalar(@_) * 2, @_ );
4310 }
4311
4312 sub add_strarray(@)
4313 {
4314     return 0 unless @_;
4315     return add_str_data( pack "S<L<*", scalar @_, map { add_string($_) } @_);
4316 }
4317
4318 sub format_to_grouping($)
4319 {
4320     my $format = shift;
4321     if ($format =~ /#,(#+),(#+0)/) { return chr(length($2)) . chr(length($1)); }
4322     if ($format =~ /#,(#+0)/) { return chr(length($1)); }
4323 #    printf STDERR "unknown format %s\n", $format;
4324     return chr(3);
4325 }
4326
4327 sub parse_currency_format($$)
4328 {
4329     my $name = shift;
4330     my ($posfmt, $negfmt) = split /;/, shift;
4331     my @pospatterns = ( "\xa4[^\xa0]*#",      # $1.1
4332                         "00[^\xa0]*\xa4",     # 1.1$
4333                         "\xa4.*\xa0.*#",      # $ 1.1
4334                         "00.*\xa0.*\xa4" );   # 1.1 $
4335     my @negpatterns = ( "\\(\xa4[^\xa0]*#",   # ($1.1)
4336                         "-\xa4[^\xa0]*#",     # -$1.1
4337                         "\xa4[^\xa0]*-#",     # $-1.1
4338                         "\xa4[^\xa0]*#.*00-", # $1.1-
4339                         "00[^\xa0]*\xa4\\)",  # (1.1$)
4340                         "-#.*00[^\xa0]*\xa4", # -1.1$
4341                         "00-[^\xa0]*\xa4",    # 1.1-$
4342                         "00[^\xa0]*\xa4-",    # 1.1$-
4343                         "-#.*00.*\xa0.*\xa4", # -1.1 $
4344                         "-\xa4.*\xa0.*#",     # -$ 1.1
4345                         "00.*\xa0.*\xa4-",    # 1.1 $-
4346                         "\xa4.*\xa0.*#.*00-", # $ 1.1-
4347                         "\xa4.*\xa0.*-#",     # $ -1.1
4348                         "00-.*\xa0.*\xa4",    # 1.1- $
4349                         "\\(\xa4.*\xa0.*#",   # ($ 1.1)
4350                         "00.*\xa0.*\xa4\\)"); # (1.1 $)
4351     my ($pos, $neg);
4352
4353     for ($pos = 0; $pos < @pospatterns; $pos++)
4354     {
4355         last if ($posfmt =~ /$pospatterns[$pos]/);
4356     }
4357     #printf STDERR "$name: unknown format '%s'\n", $posfmt if ($pos == @pospatterns);
4358     $pos = 0 if ($pos == @pospatterns);
4359
4360     if (defined $negfmt)
4361     {
4362         for ($neg = 0; $neg < @negpatterns; $neg++)
4363         {
4364             last if ($negfmt =~ /$negpatterns[$neg]/);
4365         }
4366         #printf STDERR "$name: unknown format '%s'\n", $negfmt if ($neg == @negpatterns);
4367         $neg = 0 if ($neg == @negpatterns);
4368     }
4369     elsif ($pos == 0) { $neg = 1; }
4370     elsif ($pos == 1) { $neg = 5; }
4371     elsif ($pos == 2) { $neg = 9; }
4372     elsif ($pos == 3) { $neg = 8; }
4373
4374     return ($pos, $neg);
4375 }
4376
4377 sub parse_percent_format($)
4378 {
4379     my $fmt = shift;
4380     my @patterns = ( "0.+%",    # 1 %
4381                      "0%",      # 1%
4382                      "%#",      # %1
4383                      "%.+#" );  # % 1
4384     my $pos;
4385     for ($pos = 0; $pos < @patterns; $pos++)
4386     {
4387         last if ($fmt =~ /$patterns[$pos]/);
4388     }
4389     printf STDERR "unknown format '%s'\n", $fmt if ($pos == @patterns);
4390     return ($pos, ($pos == 3) ? 7 : $pos);
4391 }
4392
4393 sub convert_date_format($)
4394 {
4395     my $fmt = shift;
4396     $fmt =~ s/G+/gg/;
4397     $fmt =~ s/LLLL/MMMM/;
4398     $fmt =~ s/LLL/MMM/;
4399     $fmt =~ s/E+/dddd/;
4400     $fmt =~ s/ccc+/dddd/;
4401     $fmt =~ s/([^gy])y([^y])/$1yyyy$2/;
4402     $fmt =~ s/^y([^y])/yyyy$1/;
4403     $fmt =~ s/([^gy])y$/$1yyyy/;
4404     return $fmt;
4405 }
4406
4407 sub convert_time_format($)
4408 {
4409     my $fmt = shift;
4410     $fmt =~ s/a+/tt/;
4411     $fmt =~ s/B+/tt/;
4412     $fmt =~ s/\x{202f}/ /;
4413     return $fmt;
4414 }
4415
4416 sub load_iso639()
4417 {
4418     my %iso639;
4419     my $DATA = open_data_file( "iso639", "iso-639-3_Code_Tables_$ISO639VERSION/iso-639-3_$ISO639VERSION.tab" );
4420     while (<$DATA>)
4421     {
4422         if (/^\s*[a-z]{3}\s+[a-z]{3}\s+([a-z]{3})\s+([a-z]{2})\s/) { $iso639{$2} = $1; }
4423     }
4424     close $DATA;
4425     return %iso639;
4426 }
4427
4428
4429 ################################################################
4430 # build the locale table for locale.nls
4431 sub build_locale_data()
4432 {
4433     my $base = "cldr-release-$CLDRVERSION";
4434     my $suppl = load_xml_data_file( "cldr", "$base/common/supplemental/supplementalData.xml" );
4435     my $subtags = load_xml_data_file( "cldr", "$base/common/supplemental/likelySubtags.xml" );
4436     my $numbers = load_xml_data_file( "cldr", "$base/common/supplemental/numberingSystems.xml" );
4437     # obsolete phone data from CLDR version 33
4438     my $phone = load_xml_data_file( "cldr33", "common/supplemental/telephoneCodeData.xml" );
4439     my %iso639 = load_iso639();
4440     $string_data = pack "S2", 0, 0;  # offset 0 == empty string
4441
4442     %lcnames = map { $_->{name} => $_ } @locales;
4443
4444     my %lcids;
4445     foreach my $loc (@locales) { $lcids{$loc->{lcid}} = $loc if defined $loc->{lcid}; }
4446
4447     my %days = ( "sun" => 0, "mon" => 1, "tue" => 2, "wed" => 3, "thu" => 4, "fri" => 5, "sat" => 6 );
4448
4449     # assign locale parents
4450
4451     foreach my $loc (@locales)
4452     {
4453         next if $loc->{name} eq "";
4454         next if defined $loc->{parent};
4455         (my $unix_name = $loc->{name}) =~ s/-/_/g;
4456         my $parent = xml_query( $suppl, "/supplementalData/parentLocales[not(\@component)]/parentLocale[contains(concat(' ',\@locales,' '),' $unix_name ')]/\@parent" );
4457         if ($parent)
4458         {
4459             $parent =~ s/_/-/g;
4460             $parent = "" if $parent eq "root";
4461         }
4462         elsif ($loc->{name} =~ /(.*)-[0-9A-Za-z]+/) { $parent = $1; }
4463         $loc->{parent} = $parent || "";
4464     }
4465
4466     # load per-locale XML files
4467
4468     foreach my $loc (@locales)
4469     {
4470         next if defined $loc->{alias};
4471         (my $file = $loc->{file} || $loc->{name}) =~ s/-/_/g;
4472         $file = "$base/" . ($loc->{dir} || "common") . "/main/$file.xml";
4473         my $xml = load_xml_data_file( "cldr", $file );
4474         $loc->{xml} = $xml;
4475         $loc->{language} ||= xml_query( $xml, "/ldml/identity/language/\@type" );
4476         $loc->{territory} ||= xml_query( $xml, "/ldml/identity/territory/\@type" );
4477         $loc->{script} = xml_query( $xml, "/ldml/identity/script/\@type" );
4478         if (!defined($loc->{territory}) && $loc->{name} =~ /-([A-Z]{2}|[0-9]{3})$/) { $loc->{territory} = $1; }
4479         if (!defined($loc->{script}) && $loc->{name} =~ /-([A-Z][a-z]{3})(-[A-Z]{2})?$/) { $loc->{script} = $1; }
4480     }
4481
4482     # assign a default territory and sort locale
4483
4484     foreach my $loc (@locales)
4485     {
4486         next if defined $loc->{alias};
4487         next if defined $loc->{territory};
4488         my $id = $loc->{sortlocale};
4489         if (defined $id && ($id =~ /[-_]([A-Z0-9]+)$/))
4490         {
4491             $loc->{territory} = $1;
4492             next;
4493         }
4494         my @children = grep /^$loc->{name}-[A-Z0-9]+$/ && !defined $lcnames{$_}->{alias}, keys %lcnames;
4495         if (@children == 1)
4496         {
4497             $id = $children[0];
4498         }
4499         else
4500         {
4501             my $name = $loc->{file} || $loc->{name};
4502             $name =~ s/-(Arab|Beng|Cyrl|Deva|Guru|Hans|Hant|Latn|Tfng|Vaii)$//;
4503             $name =~ s/-/_/g;
4504             $id = xml_query( $subtags, "/supplementalData/likelySubtags/likelySubtag[\@from='$name']/\@to" );
4505             $id =~ s/_/-/g if $id;
4506         }
4507         if ($id =~ /[-_]([A-Z0-9]+)$/)
4508         {
4509             $loc->{territory} = $1;
4510             next if defined $loc->{sortlocale};
4511             next unless $id =~ /^$loc->{name}/;
4512             while (defined $lcnames{$id} && defined $lcnames{$id}->{alias}) { $id = $lcnames{$id}->{alias}; }
4513             $loc->{sortlocale} = $id if defined $lcnames{$id};
4514             next;
4515         }
4516         print STDERR "no territory found for $loc->{name}\n";
4517     }
4518
4519     # fill geoid table
4520
4521     my %geotable;
4522     foreach my $geo (@geoids)
4523     {
4524         my $name = $geo->{name};
4525         next unless defined $name;
4526         $geo->{alias} = $geotable{$name} if defined $geotable{$name};
4527         $geotable{$name} ||= $geo;
4528     }
4529     foreach my $loc (@locales)
4530     {
4531         next if defined $loc->{alias};
4532         my $territory = $loc->{territory};
4533         $geotable{$territory} ||= { name => $territory };
4534     }
4535     foreach my $name (keys %geotable)
4536     {
4537         my $geo = $geotable{$name};
4538         $geo->{dialcode} = xml_query( $phone, "(/supplementalData/telephoneCodeData/codesByTerritory[\@territory='$name']/telephoneCountryCode)[1]/\@code" );
4539         if ($name =~ /\d+/)
4540         {
4541             $geo->{uncode} = $name;
4542             next;
4543         }
4544         $geo->{iso2} = $name;
4545         $geo->{iso3} = xml_query( $suppl, "/supplementalData/codeMappings/territoryCodes[\@type='$name']/\@alpha3");
4546         $geo->{uncode} = xml_query( $suppl, "/supplementalData/codeMappings/territoryCodes[\@type='$name']/\@numeric");
4547         $geo->{sintlsymbol} ||= xml_query( $suppl, "(/supplementalData/currencyData/region[\@iso3166='$name']/currency[not(\@to)])[1]/\@iso4217") || "XXX";
4548         $geo->{sintlsymbol} =~ s/XXX/XDR/;
4549     }
4550     foreach my $geo (@geoids)
4551     {
4552         $geo->{parentid} = $geotable{$geo->{parent}}->{id} if defined $geo->{parent};
4553         next if defined $geo->{iso2};
4554         next if defined $geo->{alias};
4555         next unless defined $geo->{uncode};
4556         my @contains;
4557         my $list = xml_query( $suppl, "/supplementalData/territoryContainment/group[\@type='$geo->{uncode}' and not(\@status)]/\@contains");
4558         push @contains, split /\s+/, $list if defined $list;
4559         $list = xml_query( $suppl, "/supplementalData/territoryContainment/group[\@type='$geo->{uncode}' and \@status='deprecated']/\@contains");
4560         push @contains, split /\s+/, $list if defined $list;
4561         while (@contains)
4562         {
4563             my $territory = pop @contains;
4564             if (defined $geotable{$territory})
4565             {
4566                 $geotable{$territory}->{parentid} ||= $geo->{id};
4567             }
4568             elsif ($territory =~ /\d+/)
4569             {
4570                 # expand region recursively
4571                 $list = xml_query( $suppl, "/supplementalData/territoryContainment/group[\@type='$territory' and not(\@status)]/\@contains" );
4572                 push @contains, split /\s+/, $list if defined $list;
4573             }
4574         }
4575     }
4576
4577     # assign calendars to their locale
4578
4579     foreach my $cal (@calendars)
4580     {
4581         next unless defined $cal->{locale};
4582         my $loc = $lcnames{$cal->{locale}};
4583         $loc->{calendar} = [ ] unless defined $loc->{calendar};
4584         push @{$loc->{calendar}}, $cal;
4585     }
4586
4587     # assign default lcid to aliases
4588
4589     foreach my $loc (@locales)
4590     {
4591         next unless defined $loc->{alias};
4592         next if defined $loc->{lcid};
4593         my $alias = $loc->{alias};
4594         my $lcid = $lcnames{$alias}->{lcid} || 0x1000;
4595         $loc->{lcid} = $lcid | 0x80000000;
4596     }
4597
4598     # assign sort aliases to parent locale
4599
4600     foreach my $loc (@locales)
4601     {
4602         next unless $loc->{name} =~ /_/;
4603         next unless defined $loc->{alias};
4604         my $alias = $loc->{alias};
4605         my $parent = $lcnames{$alias};
4606         my $basename = $parent->{name};
4607         while (1)
4608         {
4609             @{$parent->{sortnames}}[($loc->{lcid} >> 16) - 1] = $loc->{name};
4610             $alias = locale_parent( $alias );
4611             last unless $alias && defined $lcnames{$alias};
4612             $parent = $lcnames{$alias};
4613             last if defined $parent->{sortbase} && $parent->{sortbase} ne $basename;
4614             $parent->{sortbase} = $basename;
4615         }
4616     }
4617
4618     # assign an array index to all locales
4619
4620     my $idx = 0;
4621     foreach my $loc (@locales)
4622     {
4623         next if defined $loc->{alias};
4624         $loc->{idx} = $idx++;
4625     }
4626     foreach my $loc (@locales)
4627     {
4628         my $alias = $loc->{alias};
4629         next unless defined $alias;
4630         while (defined $lcnames{$alias}->{alias}) { $alias = $lcnames{$alias}->{alias}; }
4631         $loc->{idx} = $lcnames{$alias}->{idx};
4632     }
4633
4634     # output lcids table
4635
4636     my $lcid_data = "";
4637     foreach my $id (sort { $a <=> $b } keys %lcids)
4638     {
4639         my $loc = $lcids{$id};
4640         $lcid_data .= pack "L<S<2", $id, $loc->{idx}, add_string($loc->{name});
4641     }
4642
4643     # output lcnames table
4644
4645     my $lcname_data = "";
4646     foreach my $name (sort compare_locales keys %lcnames)
4647     {
4648         my $loc = $lcnames{$name};
4649         $lcname_data .= pack "S<2L<", add_string($name), $loc->{idx}, $loc->{lcid} || 0x1000;
4650     }
4651
4652     # output locales array
4653
4654     my $locale_data = "";
4655     my $default_lcid = 0x8001;
4656     foreach my $loc (@locales)
4657     {
4658         next if defined $loc->{alias};
4659         my $sname = $loc->{name};
4660         my $language = $loc->{language};
4661         my $territory = $loc->{territory};
4662         my $script = $loc->{script};
4663         my $neutral = ($sname && $sname !~ /-$territory/);
4664         my $sparent = $loc->{sparent} || (($sname =~ /(.*)-[0-9A-Za-z]+/) ? $1 : $loc->{parent});
4665         my $unique_lcid = $loc->{lcid};
4666         unless (defined $unique_lcid) { $unique_lcid = $default_lcid++; }
4667         my $geo = $geotable{$territory};
4668         my $territory_match = "contains(concat(' ',normalize-space(\@territories),' '),' $territory ')";
4669
4670         # languages and scripts
4671
4672         my $ssortlocale = $loc->{sortlocale} || ($neutral ? "$sname-$territory" : $sname);
4673         my $idefaultlanguage = defined $lcnames{$ssortlocale} ? $lcnames{$ssortlocale}->{lcid} : undef;
4674         $idefaultlanguage = $lcnames{"en-US"}->{lcid} unless $ssortlocale;
4675         (my $siso639langname = $sname) =~ s/-.*$//;
4676         my $siso639langname2 = $iso639{$siso639langname} || $siso639langname;
4677         my $sopentypelang = sprintf "%-4s", locale_entry( $loc, "sopentypelang", uc $siso639langname2 );
4678         my $sabbrevlangname = defined $loc->{lcid} ? locale_entry( $loc, "sabbrevlangname", uc $siso639langname2 ) : "ZZZ";
4679         my $siso3166ctryname2 = $geo->{iso3} || $geo->{uncode};
4680         my $senglanguage = loc_query( $lcnames{en}, "/ldml/localeDisplayNames/languages/language[\@type='$language' and not(\@alt)]" ) || "";
4681         my $sengcountry = loc_query( $lcnames{en}, "/ldml/localeDisplayNames/territories/territory[\@type='$territory' and not(\@alt)]" ) || "";
4682         my $snativelangname = loc_query( $loc, "/ldml/localeDisplayNames/languages/language[\@type='$language' and not(\@alt)]" );
4683         my $snativectryname = loc_query( $loc, "/ldml/localeDisplayNames/territories/territory[\@type='$territory' and not(\@alt)]" );
4684         $sengcountry =~ s/South Korea/Korea/;
4685         $sengcountry =~ s/T\xfcrkiye/Turkey/;
4686         $snativelangname ||= $senglanguage;
4687         $snativectryname ||= $sengcountry;
4688         if ($script)
4689         {
4690             my $engscript = loc_query( $lcnames{en}, "/ldml/localeDisplayNames/scripts/script[\@type='$script' and not(\@alt)]" );
4691             my $nativescript = loc_query( $loc, "/ldml/localeDisplayNames/scripts/script[\@type='$script' and not(\@alt)]" );
4692             $senglanguage .= " ($engscript)" if $engscript;
4693             $snativelangname .= " ($nativescript)" if $nativescript;
4694         }
4695         my $sengdisplayname = $neutral ? $senglanguage : "$senglanguage ($sengcountry)";
4696         my $snativedisplayname = $neutral ? $snativelangname : "$snativelangname ($snativectryname)";
4697         $sengdisplayname =~ s/\) \(/, /;
4698         $snativedisplayname =~ s/\) \(/, /;
4699         my $sscripts = locale_entry( $loc, "sscripts", $script ) || xml_query( $suppl, "/supplementalData/languageData/language[\@type='$language' and not(\@alt)]/\@scripts" );
4700         $sscripts = (join ";", (sort split / /, ($sscripts || "Latn"))) . ";";
4701         my $ireadinglayout = locale_entry( $loc, "ireadinglayout", 0 );
4702         my $charlayout = loc_query( $loc, "/ldml/layout/orientation/characterOrder" );
4703         if ($charlayout eq "right-to-left")
4704         {
4705             $ireadinglayout = 1;
4706         }
4707         elsif ($charlayout eq "top-to-bottom")
4708         {
4709             my $linelayout = loc_query( $loc, "/ldml/layout/orientation/lineOrder" );
4710             $ireadinglayout = $linelayout eq "right-to-left" ? 2 : 3;
4711         }
4712         my $igeoid = $geo->{id} || 0;
4713
4714         # numbers
4715
4716         my $sdecimal = loc_query( $loc, "/ldml/numbers/symbols[\@numberSystem='latn']/decimal" );
4717         my $slist = locale_entry( $loc, "slist", ";" );
4718         my $smondecimalsep = loc_query( $loc, "/ldml/numbers/symbols[\@numberSystem='latn']/currencyDecimal" ) || $sdecimal;
4719         my $sthousand = loc_query( $loc, "/ldml/numbers/symbols[\@numberSystem='latn']/group" );
4720         $sthousand =~ s/\x{202f}/\x{00a0}/;
4721         my $smonthousandsep = loc_query( $loc, "/ldml/numbers/symbols[\@numberSystem='latn']/currencyGroup" ) || $sthousand;
4722         my $spositivesign = "";
4723         my $snegativesign = "-";
4724         my $spercent = loc_query( $loc, "/ldml/numbers/symbols[\@numberSystem='latn']/percentSign" );
4725         my $snan = loc_query( $loc, "/ldml/numbers/symbols[\@numberSystem='latn']/nan" );
4726         my $sposinfinity = loc_query( $loc, "/ldml/numbers/symbols[\@numberSystem='latn']/infinity" );
4727         my $sneginfinity = $sposinfinity ? "-$sposinfinity" : "";
4728         my $sgrouping = format_to_grouping( loc_query( $loc, "/ldml/numbers/decimalFormats[\@numberSystem='latn']/decimalFormatLength[not(\@type)]/decimalFormat/pattern" ));
4729         my $percentformat = loc_query( $loc, "/ldml/numbers/percentFormats[\@numberSystem='latn']/percentFormatLength[not(\@type)]/percentFormat/pattern" );
4730         my $currencyformat = loc_query( $loc, "/ldml/numbers/currencyFormats[\@numberSystem='latn']/currencyFormatLength[not(\@type)]/currencyFormat[\@type='accounting']/pattern[not(\@alt)]" ) ||
4731                              loc_query( $loc, "/ldml/numbers/currencyFormats[\@numberSystem='latn']/currencyFormatLength[not(\@type)]/currencyFormat[\@type='standard']/pattern[not(\@alt)]" );
4732         my $smongrouping = format_to_grouping( $currencyformat );
4733         my ($icurrency, $inegcurr) = parse_currency_format( $sname, $currencyformat );
4734         my ($ipospercent, $inegpercent) = parse_percent_format( $percentformat );
4735         my $native_numbering = loc_query( $loc, "/ldml/numbers/otherNumberingSystems/native" );
4736         my @snativedigits = split //, (locale_entry( $loc, "nativedigits", "" ) || xml_query( $numbers, "/supplementalData/numberingSystems/numberingSystem[\@id='$native_numbering']/\@digits" ));
4737         my $digitsubstitution = !(ord($snativedigits[0]) >= 0x600 && ord($snativedigits[0]) <= 0x6ff);
4738         my $measure = defined xml_query( $suppl, "/supplementalData/measurementData/measurementSystem[\@type='US' and $territory_match]" );
4739         my $papersize = defined xml_query( $suppl, "/supplementalData/measurementData/paperSize[\@type='US-Letter' and $territory_match]" );
4740
4741         # currencies
4742
4743         my $sintlsymbol = $geo->{sintlsymbol} || "XDR";
4744         my $scurrency = $geo->{scurrency} || loc_query( $loc, "/ldml/numbers/currencies/currency[\@type='$sintlsymbol']/symbol[\@alt='narrow']" );
4745         $scurrency ||= loc_query( $loc, "/ldml/numbers/currencies/currency[\@type='$sintlsymbol']/symbol[not(\@alt)]" );
4746         $geo->{scurrency} = $scurrency if $scurrency;
4747         $scurrency ||= $sintlsymbol;
4748         my $sengcurrname = $loc->{sengcurrname} || loc_query( $lcnames{en}, "/ldml/numbers/currencies/currency[\@type='$sintlsymbol']/displayName[not(\@count)]" );
4749         my $snativecurrname = $loc->{sengcurrname} || loc_query( $loc, "/ldml/numbers/currencies/currency[\@type='$sintlsymbol']/displayName[not(\@count)]" ) || $sengcurrname;
4750         my $icurrdigits = xml_query( $suppl, "/supplementalData/currencyData/fractions/info[\@iso4217='$sintlsymbol']/\@digits" );
4751         $icurrdigits = 2 unless defined $icurrdigits;
4752
4753         # calendars
4754
4755         my $firstday = xml_query( $suppl, "/supplementalData/weekData/firstDay[not(\@alt) and $territory_match]/\@day" );
4756         my $ifirstdayofweek = $firstday ? $days{$firstday} : 1;
4757         my $firstweekofyear = (xml_query( $suppl, "/supplementalData/weekData/minDays[$territory_match]/\@count" ) || 0) == 4 ? 2 : 0;
4758         my $serastring = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/eras/eraAbbr/era[\@type='1' and not(\@alt)]" );
4759         my (@sdayname, @sabbrevdayname, @sshortestdayname);
4760         foreach my $d (sort { $days{$a} <=> $days{$b} } keys %days)
4761         {
4762             my $n = $days{$d};
4763             my %name;
4764             foreach my $type (qw(wide abbreviated short))
4765             {
4766                 $name{$type} = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/days/dayContext[\@type='format']/dayWidth[\@type='$type']/day[\@type='$d' and not(\@alt)]" );
4767             }
4768             push @sdayname, $name{wide};
4769             push @sabbrevdayname, $name{abbreviated} || $name{wide};
4770             push @sshortestdayname, $name{short} || $name{abbreviated} || $name{wide};
4771         }
4772         my (@smonthname, @sabbrevmonthname, @sgenitivemonth, @sabbrevgenitivemonth);
4773         foreach my $n (1..13)
4774         {
4775             my $name = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/months/monthContext[\@type='stand-alone']/monthWidth[\@type='wide']/month[\@type='$n']" );
4776             my $abbrev = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/months/monthContext[\@type='stand-alone']/monthWidth[\@type='abbreviated']/month[\@type='$n']" );
4777             my $genitive = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/months/monthContext[\@type='format']/monthWidth[\@type='wide']/month[\@type='$n']" );
4778             my $abbrevgen = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/months/monthContext[\@type='format']/monthWidth[\@type='abbreviated']/month[\@type='$n']" );
4779             push @smonthname, $name || $genitive || "";
4780             push @sabbrevmonthname, $abbrev || $abbrevgen || $name || $genitive || "";
4781             push @sgenitivemonth, $genitive || "";
4782             push @sabbrevgenitivemonth, $abbrevgen || $genitive || "";
4783         }
4784         @sgenitivemonth = () if join("|",@smonthname) eq join("|",@sgenitivemonth);
4785         @sabbrevgenitivemonth = () if join("|",@sabbrevmonthname) eq join("|",@sabbrevgenitivemonth);
4786         my %caltypes = ( "gregorian" => 1, "japanese" => 3, "chinese" => 4, "dangi" => 5, "islamic" => 6, "buddhist" => 7, "hebrew" => 8,
4787                          "persian" => 22, "islamic-civil" => 23, "islamic-umalqura" => 23 );
4788         my $calpref = xml_query( $suppl, "/supplementalData/calendarPreferenceData/calendarPreference[$territory_match]/\@ordering" ) || "gregorian";
4789         my $icalendartype;
4790         my @scalnames;
4791         foreach my $c (split /\s+/, $calpref)
4792         {
4793             next unless defined $caltypes{$c};
4794             $icalendartype .= chr($caltypes{$c});
4795             $scalnames[$caltypes{$c} - 1] = loc_query( $loc, "/ldml/localeDisplayNames/types/type[\@key='calendar' and \@type='$c']" );
4796         }
4797
4798         # date/time formats
4799
4800         my $s1159 = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dayPeriods/dayPeriodContext[\@type='format']/dayPeriodWidth[\@type='abbreviated']/dayPeriod[\@type='am' and not(\@alt)]" );
4801         my $s2359 = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dayPeriods/dayPeriodContext[\@type='format']/dayPeriodWidth[\@type='abbreviated']/dayPeriod[\@type='pm' and not (\@alt)]" );
4802         my $sshortestam = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dayPeriods/dayPeriodContext[\@type='format']/dayPeriodWidth[\@type='narrow']/dayPeriod[\@type='am' and not(\@alt)]" );
4803         my $sshortestpm = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dayPeriods/dayPeriodContext[\@type='format']/dayPeriodWidth[\@type='narrow']/dayPeriod[\@type='pm' and not (\@alt)]" );
4804         my @stimeformat = (loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/timeFormats/timeFormatLength[\@type='medium']/timeFormat/pattern[not(\@alt)]" ));
4805         push @stimeformat, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='Hms' and not(\@alt)]" );
4806         pop @stimeformat if $stimeformat[0] eq $stimeformat[1];
4807         @stimeformat = map convert_time_format($_), @stimeformat;
4808         my @sshorttime = (loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/timeFormats/timeFormatLength[\@type='short']/timeFormat/pattern[not(\@alt)]" ));
4809         push @sshorttime, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='Hm' and not(\@alt)]" );
4810         pop @sshorttime if $sshorttime[0] eq $sshorttime[1];
4811         @sshorttime = map convert_time_format($_), @sshorttime;
4812         my @sshortdate = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='yMd' and not(\@alt)]" );
4813         push @sshortdate, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='yMMMd' and not(\@alt)]" );
4814         @sshortdate = map convert_date_format($_), @sshortdate;
4815         my @slongdate = (loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateFormats/dateFormatLength[\@type='full']/dateFormat/pattern[not(\@alt)]" ));
4816         push @slongdate, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateFormats/dateFormatLength[\@type='long']/dateFormat/pattern[not(\@alt)]" );
4817         @slongdate = map convert_date_format($_), @slongdate;
4818         my @smonthday = (loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='MMMMd' and not(\@alt)]" ));
4819         push @smonthday, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='Md' and not(\@alt)]" );
4820         push @smonthday, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='MMMd' and not(\@alt)]" );
4821         @smonthday = map convert_date_format($_), @smonthday;
4822         my @syearmonth = map convert_date_format($_), loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='yMMMM' and not(\@alt)]" );
4823         my @sduration = map convert_time_format( lc $_ ), loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='Hms' and not(\@alt)]" );
4824         my $srelativelongdate = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='MMMMEd' and not(\@alt)]" ) ||
4825                                 loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='gregorian']/dateTimeFormats/availableFormats/dateFormatItem[\@id='MMMEd' and not(\@alt)]" );
4826         $srelativelongdate = convert_date_format( $srelativelongdate );
4827
4828         if (defined $loc->{calendar})
4829         {
4830             foreach my $cal (@{$loc->{calendar}})
4831             {
4832                 $cal->{sshortdate} = \@sshortdate;
4833                 $cal->{syearmonth} = \@syearmonth;
4834                 $cal->{slongdate} = \@slongdate;
4835                 $cal->{serastring} = [ $serastring ];
4836                 $cal->{sdayname} = \@sdayname;
4837                 $cal->{sabbrevdayname} = \@sabbrevdayname;
4838                 $cal->{smonthname} = \@smonthname;
4839                 $cal->{sabbrevmonthname} = \@sabbrevmonthname;
4840                 $cal->{scalname} = $scalnames[$cal->{id}];
4841                 $cal->{smonthday} = \@smonthday;
4842                 $cal->{sshortestdayname} = \@sshortestdayname;
4843                 $cal->{sabbreverastring} = [ $serastring ];
4844                 $cal->{sshortestdayname} = \@sshortestdayname;
4845                 $cal->{srelativelongdate} = $srelativelongdate;
4846             }
4847         }
4848
4849         # codepages
4850
4851         my %ansicpmap = ( 437 => 1252, 720 => 1256, 737 => 1253, 775 => 1257, 850 => 1252,
4852                           852 => 1250, 855 => 1251, 866 => 1251, 857 => 1254, 862 => 1255 );
4853         my %maccpmap = ( 437 => 10000, 720 => 10004, 737 => 10006, 775 => 10029, 850 => 10000,
4854                          852 => 10029, 855 => 10007, 857 => 10081, 862 => 10005, 866 => 10007,
4855                          874 => 10021, 932 => 10001, 936 => 10008, 949 => 10003, 950 => 10002,
4856                          1258 => 10000 );
4857         my %ebcdiccpmap = ( 437 => 37, 720 => 20420, 737 => 20273, 866 => 20880, 932 => 20290 );
4858         my %codepagemasks = ( 874 =>   [ 0x01000000, 0x00000000, 0x00000000, 0, 0x00010000, 0x00000000, 0x00010000, 0x00000000 ],
4859                               932 =>   [ 0x00000000, 0x28c70000, 0x00000010, 0, 0x00020000, 0x00000000, 0x00020000, 0x00000000 ],
4860                               936 =>   [ 0x00000000, 0x28010000, 0x00000002, 0, 0x00040000, 0x00000000, 0x00040000, 0x00000000 ],
4861                               949 =>   [ 0x00000000, 0x00000000, 0x00000000, 0, 0x00080000, 0x00000000, 0x00080000, 0x00000000 ],
4862                               950 =>   [ 0x00000000, 0x28c10000, 0x00000012, 0, 0x00100000, 0x00000000, 0x00100000, 0x00000000 ],
4863                               1258 =>  [ 0x2000000f, 0x00000000, 0x00000000, 0, 0x00000100, 0x00008000, 0x00000100, 0x00008000 ],
4864                               866 =>   [ 0x00000200, 0x00000000, 0x00000000, 0, 0x00000004, 0x00020000, 0x00000004, 0x02020000 ],
4865                               862 =>   [ 0x00000800, 0x40000000, 0x00000000, 0, 0x00000020, 0x00200000, 0x00000020, 0x00200000 ],
4866                               857 =>   [ 0x0000001f, 0x00000000, 0x00000000, 0, 0x00000010, 0x01000000, 0x00000010, 0x01000000 ],
4867                               855 =>   [ 0x00000200, 0x00000000, 0x00000000, 0, 0x00000004, 0x02000000, 0x00000004, 0x02000000 ],
4868                               852 =>   [ 0x00000027, 0x00000000, 0x00000000, 0, 0x00000002, 0x04000000, 0x00000002, 0x04000000 ],
4869                               775 =>   [ 0x00000007, 0x00000000, 0x00000000, 0, 0x00000080, 0x08000000, 0x00000080, 0x08000000 ],
4870                               737 =>   [ 0x00000080, 0x00000000, 0x00000000, 0, 0x00000008, 0x10000000, 0x00000008, 0x10010000 ],
4871                               720 =>   [ 0x00002000, 0x00000000, 0x00000000, 0, 0x00000040, 0x20000000, 0x00000040, 0x20080000 ],
4872                               850 =>   [ 0x00000003, 0x00000000, 0x00000000, 0, 0x00000001, 0x40000000, 0x0000019f, 0xdfd70000 ],
4873                               437 =>   [ 0x00000003, 0x00000000, 0x00000000, 0, 0x00000001, 0x80000000, 0x0000019f, 0xdfd70000 ],
4874                               65001 => [ 0x00000000, 0x00000000, 0x00000000, 0, 0x00000000, 0x00000000, 0x0000019f, 0xdfd70000 ] );
4875         my $oemcp = locale_entry( $loc, "oemcp", 65001 );
4876         my $maccp = locale_entry( $loc, "maccp", undef ) || $maccpmap{$oemcp} || 65001;
4877         my $ebcdiccp = locale_entry( $loc, "ebcdiccp", undef ) || $ebcdiccpmap{$oemcp} || 500;
4878         $ebcdiccp = 500 if (defined $loc->{oemcp} && $loc->{oemcp} == 65001) || (defined $loc->{maccp} && $loc->{maccp} == 65001);
4879         my $ansicp = $ansicpmap{$oemcp} || $oemcp;
4880         my @fontsig = (0) x 8;
4881         my $sig = locale_entry( $loc, "fontsig", [] );
4882         foreach my $i (0..7) { $fontsig[$i] |= $codepagemasks{$oemcp}->[$i]; }
4883         foreach my $i (0..$#{$sig}) { $fontsig[$i] |= $sig->[$i]; }
4884         $fontsig[3] |= 1 << 31;
4885         $fontsig[3] |= 1 << 27 if $ireadinglayout == 1;
4886         $fontsig[3] |= 1 << 28 if $ireadinglayout == 3;
4887
4888         # special cases for invariant locale
4889
4890         unless ($loc->{name})
4891         {
4892             $siso639langname = "iv";
4893             $siso639langname2 = "ivl";
4894             $senglanguage = $snativelangname = "Invariant Language";
4895             $sengcountry = $snativectryname = "Invariant Country";
4896             $sengdisplayname = "Invariant Language (Invariant Country)";
4897             $snativedisplayname = "Invariant Language (Invariant Region)";
4898             $sengcurrname = $snativecurrname = "International Monetary Fund";
4899             $scurrency = "\x{00a4}";
4900             $ifirstdayofweek = 0;
4901             $igeoid = $geotable{"US"}->{id};
4902             @stimeformat = ("HH:mm:ss");
4903             @sshortdate = ("MM/dd/yyyy", "yyyy-MM-dd");
4904             @slongdate = ("dddd, dd MMMM yyyy");
4905             @syearmonth = ("yyyy MMMM");
4906             @smonthday = ("MMMM dd", "MMMM d", "M/d", "MMM d");
4907             @sshorttime = ("HH:mm", "hh:mm tt", "H:mm", "h:mm tt");
4908             $srelativelongdate = "dddd, MMMM dd";
4909             $sposinfinity = "Infinity";
4910             $sneginfinity = "-Infinity";
4911             $spositivesign = "+";
4912             $ipospercent = $inegpercent = 0;
4913         }
4914
4915         # output data
4916
4917         $locale_data .= pack "L<2",
4918             add_string( $sname ),                  # name
4919             add_string( $sopentypelang );          # LOCALE_SOPENTYPELANGUAGETAG
4920
4921         $locale_data .= pack "S<14",
4922             $loc->{lcid} || 0x1000,                # LOCALE_ILANGUAGE
4923             $unique_lcid,                          # unique_lcid
4924             locale_entry( $loc, "idigits", 2 ),    # LOCALE_IDIGITS
4925             locale_entry( $loc, "inegnumber", 1 ), # LOCALE_INEGNUMBER
4926             $icurrdigits,                          # LOCALE_ICURRDIGITS
4927             $icurrency,                            # LOCALE_ICURRENCY
4928             $inegcurr,                             # LOCALE_INEGCURR
4929             locale_entry( $loc, "ilzero", 1 ),     # LOCALE_ILZERO
4930             !$neutral,                             # LOCALE_INEUTRAL
4931             $ifirstdayofweek,                      # LOCALE_IFIRSTDAYOFWEEK
4932             $firstweekofyear,                      # LOCALE_IFIRSTWEEKOFYEAR
4933             $geo->{dialcode} || 1 ,                # LOCALE_ICOUNTRY,
4934             $measure,                              # LOCALE_IMEASURE
4935             $digitsubstitution;                    # LOCALE_IDIGITSUBSTITUTION
4936
4937         $locale_data .= pack "L<18",
4938             add_string( $sgrouping ),              # LOCALE_SGROUPING
4939             add_string( $smongrouping ),           # LOCALE_SMONGROUPING
4940             add_string( $slist ),                  # LOCALE_SLIST
4941             add_string( $sdecimal ),               # LOCALE_SDECIMAL
4942             add_string( $sthousand ),              # LOCALE_STHOUSAND
4943             add_string( $scurrency ),              # LOCALE_SCURRENCY
4944             add_string( $smondecimalsep ),         # LOCALE_SMONDECIMALSEP
4945             add_string( $smonthousandsep ),        # LOCALE_SMONTHOUSANDSEP
4946             add_string( $spositivesign ),          # LOCALE_SPOSITIVESIGN
4947             add_string( $snegativesign ),          # LOCALE_SNEGATIVESIGN
4948             add_string( $s1159 ),                  # LOCALE_S1159
4949             add_string( $s2359 ),                  # LOCALE_S2359
4950             add_strarray( @snativedigits ),        # LOCALE_SNATIVEDIGITS
4951             add_strarray( @stimeformat ),          # LOCALE_STIMEFORMAT
4952             add_strarray( @sshortdate ),           # LOCALE_SSHORTDATE
4953             add_strarray( @slongdate ),            # LOCALE_SLONGDATE
4954             add_strarray( @syearmonth ),           # LOCALE_SYEARMONTH
4955             add_strarray( @sduration );            # LOCALE_SDURATION
4956
4957         $locale_data .= pack "S<8",
4958             $idefaultlanguage || 0x1000,           # LOCALE_IDEFAULTLANGUAGE
4959             $ansicp,                               # LOCALE_IDEFAULTANSICODEPAGE
4960             $oemcp,                                # LOCALE_IDEFAULTCODEPAGE
4961             $maccp,                                # LOCALE_IDEFAULTMACCODEPAGE
4962             $ebcdiccp,                             # LOCALE_IDEFAULTEBCDICCODEPAGE
4963             $igeoid < 65536 ? $igeoid : 39070,     # old_geoid
4964             $papersize ? 1 : 9,                    # LOCALE_IPAPERSIZE
4965             0;  # FIXME                            # islamic_cal
4966
4967         $locale_data .= pack "L<24",
4968             add_string( $icalendartype ),          # LOCALE_ICALENDARTYPE
4969             add_string( $sabbrevlangname ),        # LOCALE_SABBREVLANGNAME
4970             add_string( $siso639langname ),        # LOCALE_SISO639LANGNAME
4971             add_string( $senglanguage ),           # LOCALE_SENGLANGUAGE
4972             add_string( $snativelangname ),        # LOCALE_SNATIVELANGNAME
4973             add_string( $sengcountry ),            # LOCALE_SENGCOUNTRY
4974             add_string( $snativectryname ),        # LOCALE_SNATIVECTRYNAME
4975             add_string( $siso3166ctryname2 ),      # LOCALE_SABBREVCTRYNAME
4976             add_string( $territory ),              # LOCALE_SISO3166CTRYNAME
4977             add_string( $sintlsymbol ),            # LOCALE_SINTLSYMBOL
4978             add_string( $sengcurrname ),           # LOCALE_SENGCURRNAME
4979             add_string( $snativecurrname ),        # LOCALE_SNATIVECURRNAME
4980             add_fontsig( @fontsig ),               # LOCALE_FONTSIGNATURE
4981             add_string( $siso639langname2 ),       # LOCALE_SISO639LANGNAME2
4982             add_string( $siso3166ctryname2 ),      # LOCALE_SISO3166CTRYNAME2
4983             add_string( $sparent ),                # LOCALE_SPARENT
4984             add_strarray( @sdayname ),             # LOCALE_SDAYNAME
4985             add_strarray( @sabbrevdayname ),       # LOCALE_SABBREVDAYNAME
4986             add_strarray( @smonthname ),           # LOCALE_SMONTHNAME
4987             add_strarray( @sabbrevmonthname ),     # LOCALE_SABBREVMONTHNAME
4988             add_strarray( @sgenitivemonth ),       # LOCALE_SGENITIVEMONTH
4989             add_strarray( @sabbrevgenitivemonth ), # LOCALE_SABBREVGENITIVEMONTH
4990             add_strarray( @scalnames ),            # LOCALE_SCALNAMES
4991             add_strarray( @{$loc->{sortnames}} );  # LOCALE_SSORTNAMES
4992
4993         $locale_data .= pack "S<6",
4994             $inegpercent,                          # LOCALE_INEGATIVEPERCENT
4995             $ipospercent,                          # LOCALE_IPOSITIVEPERCENT
4996             0,                                     # unknown
4997             $ireadinglayout,                       # LOCALE_IREADINGLAYOUT
4998             0x2a,                                  # unknown
4999             0x2a;                                  # unknown
5000
5001         $locale_data .= pack "L<24",
5002             0,                                     # unknown
5003             add_string( $sengdisplayname ),        # LOCALE_SENGLISHDISPLAYNAME
5004             add_string( $snativedisplayname ),     # LOCALE_SNATIVEDISPLAYNAME
5005             add_string( $spercent ),               # LOCALE_SPERCENT
5006             add_string( $snan ),                   # LOCALE_SNAN
5007             add_string( $sposinfinity ),           # LOCALE_SPOSINFINITY
5008             add_string( $sneginfinity ),           # LOCALE_SNEGINFINITY
5009             0,                                     # unknown
5010             add_string( $serastring ),             # CAL_SERASTRING
5011             add_string( $serastring ),             # CAL_SABBREVERASTRING
5012             0,                                     # unknown
5013             add_string( $ssortlocale ),            # LOCALE_SCONSOLEFALLBACKNAME
5014             add_strarray( @sshorttime ),           # LOCALE_SSHORTTIME
5015             add_strarray( @sshortestdayname ),     # CAL_SSHORTESTDAYNAME
5016             0,                                     # unknown
5017             add_string( $ssortlocale ),            # LOCALE_SSORTLOCALE
5018             add_string( "0409:00000409" ), # FIXME # LOCALE_SKEYBOARDSTOINSTALL
5019             add_string( $sscripts ),               # LOCALE_SSCRIPTS
5020             add_string( $srelativelongdate ),      # LOCALE_SRELATIVELONGDATE
5021             $igeoid,                               # LOCALE_IGEOID
5022             add_string( $sshortestam || "a" ),     # LOCALE_SSHORTESTAM
5023             add_string( $sshortestpm || "p" ),     # LOCALE_SSHORTESTPM
5024             add_strarray( @smonthday ),            # LOCALE_SMONTHDAY
5025             add_string( "k0-windows-us" ) # FIXME  # keyboard_layout
5026     }
5027
5028     # output language groups
5029
5030     my %groups;
5031     add_registry_key( $nlskey, "Locale", "00000409" );
5032     foreach my $loc (@locales)
5033     {
5034         next unless defined $loc->{lcid};
5035         next if ($loc->{lcid} & 0x80000000);
5036         next if !defined($loc->{alias}) && $loc->{name} !~ /-$loc->{territory}/; # skip neutral locales
5037         my $group = locale_entry( $loc, "group", 1 );
5038         my $name = sprintf( "%08x", $loc->{lcid} );
5039         my $val = sprintf( "%x", $group );
5040         add_registry_string_value( $nlskey, "Locale", $name, $val ) unless ($loc->{lcid} & 0x000f0000);
5041         add_registry_string_value( $nlskey, "Locale\\Alternate Sorts", $name, $val ) if $loc->{name} =~ /_/;
5042         $groups{$val} = 1;
5043     }
5044     foreach my $group (keys %groups) { add_registry_string_value( $nlskey, "Language Groups", $group, "1" ); }
5045
5046     # output calendar data
5047
5048     my $calendar_data = "";
5049     foreach my $cal (@calendars)
5050     {
5051         my $scalname = $cal->{name};
5052         my $iyearoffsetrange = 0;
5053         my $itwodigityearmax = $cal->{itwodigityearmax};
5054         my @sshortdate;
5055         my @syearmonth;
5056         my @slongdate;
5057         my @serastring;
5058         my @sdayname;
5059         my @sabbrevdayname;
5060         my @smonthname;
5061         my @sabbrevmonthname;
5062         my @smonthday;
5063         my @sabbreverastring;
5064         my @sshortestdayname;
5065
5066         my $type = $cal->{type};
5067         if (defined $cal->{locale} && defined $type)
5068         {
5069             my $loc = $lcnames{$cal->{locale}};
5070             my $fmt = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/dateTimeFormats/availableFormats/dateFormatItem[\@id='yMd' and not(\@alt)]" );
5071             push @sshortdate, $fmt if $fmt;
5072             $fmt = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/dateTimeFormats/availableFormats/dateFormatItem[\@id='yyyyMd' and not(\@alt)]" );
5073             push @sshortdate, $fmt if $fmt;
5074             $fmt = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/dateTimeFormats/availableFormats/dateFormatItem[\@id='yMMMd' and not(\@alt)]" );
5075             push @sshortdate, $fmt if $fmt;
5076             $fmt = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/dateTimeFormats/availableFormats/dateFormatItem[\@id='yyyyMMMd' and not(\@alt)]" );
5077             push @sshortdate, $fmt if $fmt;
5078             @sshortdate = map convert_date_format($_), @sshortdate;
5079             $fmt = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/dateFormats/dateFormatLength[\@type='full']/dateFormat/pattern[not(\@alt)]" );
5080             push @slongdate, $fmt if $fmt;
5081             $fmt = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/dateFormats/dateFormatLength[\@type='long']/dateFormat/pattern[not(\@alt)]" );
5082             push @slongdate, $fmt if $fmt;
5083             @slongdate = map convert_date_format($_), @slongdate;
5084
5085             foreach my $n (1..13)
5086             {
5087                 my $name = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/months/monthContext[\@type='format']/monthWidth[\@type='wide']/month[\@type='$n' and not(\@yeartype)]" );
5088                 my $abbrev = loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/months/monthContext[\@type='format']/monthWidth[\@type='abbreviated']/month[\@type='$n' and not(\@yeartype)]" );
5089                 push @smonthname, $name || "";
5090                 push @sabbrevmonthname, $abbrev || $name || "";
5091             }
5092
5093             $scalname ||= loc_query( $loc, "/ldml/localeDisplayNames/types/type[\@key='calendar' and \@type='$type']" );
5094             if (defined $cal->{eras})
5095             {
5096                 my @eras;
5097                 my $idx = 1;
5098                 foreach my $era (@{$cal->{eras}})
5099                 {
5100                     my $start = xml_query( $suppl, "/supplementalData/calendarData/calendar[\@type='$type']/eras/era[\@type='$era']/\@start" );
5101                     next unless $start =~ /^(-?\d+)-(\d+)-(\d+)/;
5102                     my ($year, $mon, $day, $zero, $first) = ($1, $2, $3, $1 - 1, 1);
5103                     if ($zero < 0)
5104                     {
5105                         $first -= $zero;
5106                         $year = 1;
5107                         $itwodigityearmax = 2049 - $zero;
5108                     }
5109                     unshift @eras, pack( "S<8", 6, $idx++, $year, $mon, $day, $zero, $first, 0 );
5110                     push @serastring, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/eras/eraAbbr/era[\@type='$era']" );
5111                     push @sabbreverastring, loc_query( $loc, "/ldml/dates/calendars/calendar[\@type='$type']/eras/eraNarrow/era[\@type='$era']" );
5112                 }
5113                 $iyearoffsetrange = add_str_data( pack "S<L<*", scalar @eras, map { add_str_data($_); } @eras );
5114             }
5115         }
5116
5117         @sshortdate = @{$cal->{sshortdate}} if defined $cal->{sshortdate} && !@sshortdate;
5118         @syearmonth = @{$cal->{syearmonth}} if defined $cal->{syearmonth};
5119         @slongdate = @{$cal->{slongdate}} if defined $cal->{slongdate} && !@slongdate;
5120         @serastring = @{$cal->{serastring}} if defined $cal->{serastring} && !@serastring;
5121         @sdayname = @{$cal->{sdayname}} if defined $cal->{sdayname};
5122         @sabbrevdayname = @{$cal->{sabbrevdayname}} if defined $cal->{sabbrevdayname};
5123         @smonthname = @{$cal->{smonthname}} if defined $cal->{smonthname} && !join("",@smonthname);
5124         @sabbrevmonthname = @{$cal->{sabbrevmonthname}} if defined $cal->{sabbrevmonthname} && !join("",@sabbrevmonthname);
5125         @smonthday = @{$cal->{smonthday}} if defined $cal->{smonthday};
5126         @sabbreverastring = @{$cal->{sabbreverastring}} if defined $cal->{sabbreverastring} && !@sabbreverastring;
5127         @sshortestdayname = @{$cal->{sshortestdayname}} if defined $cal->{sshortestdayname};
5128         my $srelativelongdate = $cal->{srelativelongdate};
5129
5130         @serastring = ("A.D.") unless @serastring;
5131         @sabbreverastring = ("AD") unless @sabbreverastring;
5132
5133         if ($cal->{id} != 1)  # calendar 1 is a placeholder, information is fetched from locale instead
5134         {
5135             @sshortdate = ("") unless @sshortdate;
5136             @syearmonth = ("") unless @syearmonth;
5137             @slongdate = ("") unless @slongdate;
5138             @sdayname = ("") x 7 unless @sdayname;
5139             @sabbrevdayname = ("") x 7 unless @sabbrevdayname;
5140             @sshortestdayname = ("") x 7 unless @sshortestdayname;
5141             @smonthname = ("") x 13 unless @smonthname;
5142             @sabbrevmonthname = ("") x 13 unless @sabbrevmonthname;
5143             @smonthday = ("") unless @smonthday;
5144         }
5145
5146         $calendar_data .= pack "S<2L<17",
5147             $cal->{id},                        # CAL_ICALINTVALUE
5148             $itwodigityearmax || 99,           # CAL_ITWODIGITYEARMAX
5149             add_strarray( @sshortdate ),       # CAL_SSHORTDATE
5150             add_strarray( @syearmonth ),       # CAL_SYEARMONTH
5151             add_strarray( @slongdate ),        # CAL_SLONGDATE
5152             add_strarray( @serastring ),       # CAL_SERASTRING
5153             $iyearoffsetrange,                 # CAL_IYEAROFFSETRANGE
5154             add_strarray( @sdayname ),         # CAL_SDAYNAME
5155             add_strarray( @sabbrevdayname ),   # CAL_SABBREVDAYNAME
5156             add_strarray( @smonthname ),       # CAL_SMONTHNAME
5157             add_strarray( @sabbrevmonthname ), # CAL_SABBREVMONTHNAME
5158             add_string( $scalname ),           # CAL_SCALNAME
5159             add_strarray( @smonthday ),        # CAL_SMONTHDAY
5160             add_strarray( @sabbreverastring ), # CAL_SABBREVERASTRING
5161             add_strarray( @sshortestdayname ), # CAL_SSHORTESTDAYNAME
5162             add_string( $srelativelongdate );  # CAL_SRELATIVELONGDATE
5163     }
5164
5165     # output locale header
5166
5167     my $nb_lcids = scalar keys %lcids;
5168     my $nb_locales = scalar grep { !defined $_->{alias} } @locales;
5169     my $nb_lcnames = scalar keys %lcnames;
5170     my $locale_size = length($locale_data) / $nb_locales;
5171     my $nb_calendars = scalar @calendars;
5172     my $calendar_size = length($calendar_data) / $nb_calendars;
5173     my $lcids_offset = 19 * 4;  # size of header
5174     my $lcnames_offset = $lcids_offset + length $lcid_data;
5175     my $locales_offset = $lcnames_offset + length $lcname_data;
5176     my $calendar_offset = $locales_offset + length $locale_data;
5177     my $strings_offset = $calendar_offset + length $calendar_data;
5178
5179     my $locale_header = pack "L<7S<4L<S<2L<3S<2L<4",
5180         8,  # offset
5181         0,
5182         7,  # version
5183         0x5344534e,  # magic
5184         0, 0, 0,
5185         0,
5186         $nb_lcids,
5187         $nb_locales,
5188         $locale_size,
5189         $locales_offset,
5190         $nb_lcnames,
5191         0,
5192         $lcids_offset,
5193         $lcnames_offset,
5194         0,
5195         $nb_calendars,
5196         $calendar_size,
5197         $calendar_offset,
5198         $strings_offset,
5199         0, 0;
5200
5201     return align_string( 4, $locale_header . $lcid_data . $lcname_data . $locale_data . $calendar_data . $string_data );
5202 }
5203
5204
5205 ################################################################
5206 # build the charmaps table for locale.nls
5207 sub build_charmaps_data()
5208 {
5209     my $data = "";
5210
5211     # MAP_FOLDDIGITS
5212     my @digits = (ord('0') .. ord('9'));
5213     $digitmap_table[0x3007] = $digits[0];               # Ideographic Zero
5214     @digitmap_table[0x0c78..0x0c7b] = @digits[0..3];    # Telugu Fraction Digits
5215     @digitmap_table[0x0c7c..0x0c7e] = @digits[1..3];    # Telugu Fraction Digits
5216     @digitmap_table[0x3021..0x3029] = @digits[1..9];    # Hangzhou Numerals
5217     @digitmap_table[0xa8e0..0xa8e9] = @digits;          # Combining Devanagari Digits
5218     @digitmap_table[0x10107..0x1010f] = @digits[1..9];  # Aegean Numbers
5219     $digitmap_table[0x10320] = $digits[1];              # Old Italic Numerals
5220     $digitmap_table[0x10321] = $digits[5];              # Old Italic Numerals
5221     $data .= dump_binary_case_table( @digitmap_table );
5222
5223     # CJK compatibility map
5224     $data .= dump_binary_case_table( @cjk_compat_table );
5225
5226     # LCMAP_HIRAGANA/KATAKANA
5227     my (@hiragana_table, @katakana_table);
5228     foreach my $ch (0x3041..0x3096, 0x309d..0x309e)
5229     {
5230         $hiragana_table[$ch + 0x60] = $ch;
5231         $katakana_table[$ch] = $ch + 0x60;
5232     }
5233     $data .= dump_binary_case_table( @hiragana_table ) . dump_binary_case_table( @katakana_table );
5234
5235     # LCMAP_HALFWIDTH/FULLWIDTH
5236     $halfwidth_table[0x2018] = 0x0027;
5237     $halfwidth_table[0x2019] = 0x0027;
5238     $halfwidth_table[0x201c] = 0x0022;
5239     $halfwidth_table[0x201d] = 0x0022;
5240     $halfwidth_table[0x309b] = 0xff9e;
5241     $halfwidth_table[0x309c] = 0xff9f;
5242     $fullwidth_table[0x309b] = 0x3099;
5243     $fullwidth_table[0x309c] = 0x309a;
5244     $data .= dump_binary_case_table( @halfwidth_table ) . dump_binary_case_table( @fullwidth_table );
5245
5246     # LCMAP_TRADITIONAL/SIMPLIFIED_CHINESE
5247     $data .= dump_binary_case_table( @chinese_traditional_table ) . dump_binary_case_table( @chinese_simplified_table );
5248
5249     # FIXME: some more unknown tables here
5250
5251     return $data;
5252 }
5253
5254
5255 ################################################################
5256 # build the geoids table for locale.nls
5257 sub build_geoids_data()
5258 {
5259     my $data = "";
5260     my %index;
5261     my $idx = 0;
5262     my @geo_header = (0x00650067, 0x0000006f, 0, 4 * 7, scalar @geoids, 0, 0);
5263
5264     foreach my $geo (@geoids)
5265     {
5266         my $id = $geo->{id};
5267         $geo = $geo->{alias} if defined $geo->{alias};
5268         my $lat = "0.000";
5269         my $long = "0.000";
5270         my $iso2 = $geo->{iso2} || "XX";
5271         my $iso3 = $geo->{iso3} || "XX";
5272         my $isregion = $geo->{region} || (defined $geo->{uncode} && !defined $geo->{iso2});
5273         my $sintlsymbol = $geo->{sintlsymbol} || "XDR";
5274         my $scurrency = $geo->{scurrency} || "\x{00a4}";
5275
5276         $data .= pack( "L<", $id );
5277         $data .= pad_string( 24, encode( "UTF16LE", $lat ));
5278         $data .= pad_string( 24, encode( "UTF16LE", $long ));
5279         $data .= pack( "L<2", $isregion ? 14 : 16, $geo->{parentid} || 39070 );
5280         $data .= pad_string( 8, encode( "UTF16LE", $iso2 ));
5281         $data .= pad_string( 8, encode( "UTF16LE", $iso3 ));
5282         $data .= pack( "S<2", $geo->{uncode} || 0, $geo->{dialcode} || 0 );
5283         $data .= pad_string( 8, encode( "UTF16LE", $sintlsymbol ));
5284         $data .= pad_string( 16, encode( "UTF16LE", $scurrency ));
5285         $index{$geo->{name}} = $idx if $geo->{name};
5286         $idx++;
5287     }
5288     $index{"XX"} = $index{"001"};
5289
5290     $geo_header[5] = $geo_header[3] + length $data;
5291     $geo_header[6] = scalar keys %index;
5292
5293     foreach my $name (sort keys %index)
5294     {
5295         $data .= pad_string( 8, encode( "UTF16LE", $name ));
5296         $data .= pack "L<", $index{$name};
5297     }
5298
5299     $geo_header[2] = $geo_header[3] + length $data;
5300     return pack( "L<7", @geo_header ) . $data;
5301 }
5302
5303
5304 ################################################################
5305 # build a binary locale table
5306 sub dump_locales($$)
5307 {
5308     my ($filename, $chartypes) = @_;
5309
5310     printf "Building $filename\n";
5311
5312     my $locale_data = build_locale_data();
5313     my $charmaps_data = build_charmaps_data();
5314     my $geoids_data = build_geoids_data();
5315     my $scripts_data = "";  # FIXME
5316
5317     my @header = ( 0 ) x 8;
5318     $header[0] = 4 * scalar @header;  # chartypes offset
5319     $header[4] = $header[0] + length $chartypes;  # locales offset
5320     $header[5] = $header[4] + length $locale_data;  # charmaps offset
5321     $header[6] = $header[5] + length $charmaps_data;  # geoids offset
5322     $header[7] = $header[6] + length $geoids_data;  # scripts offset
5323
5324     open OUTPUT, ">$filename.new" or die "Cannot create $filename";
5325     print OUTPUT pack "L<*", @header;
5326     print OUTPUT $chartypes, $locale_data, $charmaps_data, $geoids_data, $scripts_data;
5327     close OUTPUT;
5328     save_file($filename);
5329 }
5330
5331
5332 ################################################################
5333 # return the day of week of the first of the month
5334 sub month_first_dow($$)
5335 {
5336     my ($year, $month) = @_;
5337     my @time = gmtime( timegm_modern( 0, 0, 0, 1, $month - 1, $year ));
5338     return $time[6];
5339 }
5340
5341
5342 ################################################################
5343 # compare system time values
5344 sub compare_systime($$)
5345 {
5346     my ($a, $b) = @_;
5347     return $a->[0] <=> $b->[0] ||
5348            $a->[1] <=> $b->[1] ||
5349            $a->[2] <=> $b->[2] ||
5350            $a->[3] <=> $b->[3] ||
5351            $a->[4] <=> $b->[4] ||
5352            $a->[5] <=> $b->[5] ||
5353            $a->[6] <=> $b->[6];
5354 }
5355
5356
5357 ################################################################
5358 # compare the zone transition date with the rule date
5359 sub compare_transition_date($$$$)
5360 {
5361     my ($stdoff, $isdst, $zone, $rule) = @_;
5362
5363     if (scalar @{$zone} <= 1)
5364     {
5365         return (!defined($zone->[0]) || $zone->[0] > $rule->[0]) ? 1 : -1;
5366     }
5367
5368     my @date = parse_transition_date( $stdoff, $isdst, $zone->[0], $zone->[1], $zone->[2], $zone->[3] || 0 );
5369     return compare_systime( \@date, $rule );
5370 }
5371
5372
5373 ################################################################
5374 # get the Windows zone names from the CLDR data
5375 sub load_windows_zones()
5376 {
5377     my $current_name;
5378     my %names;
5379     my $base = "cldr-release-$CLDRVERSION";
5380     my $INPUT = open_data_file( "cldr", "$base/common/supplemental/windowsZones.xml" );
5381     while (<$INPUT>)
5382     {
5383         if (/<!-- +(\(UTC.*) -->.*/)
5384         {
5385             $current_name = $1;
5386         }
5387         if (/<mapZone other="(.*)" territory="001" type="(.*)"\/>/)
5388         {
5389             $names{$1} = [ $current_name, $2 ];
5390         }
5391     }
5392     close $INPUT;
5393     return %names;
5394 }
5395
5396
5397 ################################################################
5398 # parse a transition date specification from the tzdata files
5399 sub parse_transition_date($$@)
5400 {
5401     use integer;
5402     my ($stdoff, $isdst, $year, $in, $on, $at) = @_;
5403
5404     $on = "1" unless defined $on;
5405     $at = "0" unless defined $at;
5406
5407     my %months = ( Jan => 1, Feb => 2, Mar => 3, Apr => 4, May => 5, Jun => 6,
5408                    Jul => 7, Aug => 8, Sep => 9, Oct => 10, Nov => 11, Dec => 12 );
5409     my %days = ( Sun => 0, Mon => 1, Tue => 2, Wed => 3, Thu => 4, Fri => 5, Sat => 6 );
5410
5411     my $mon = $in ? $months{$in} : 1;
5412     my ($week, $dow, $flag, $time, $sec);
5413     my $first = month_first_dow( $year, $mon );
5414
5415     if ($on =~ /^last(.*)$/)
5416     {
5417         $week = 5;
5418         $dow = $days{$1};
5419     }
5420     elsif ($on =~ /^(.*)>=(\d+)$/)
5421     {
5422         $dow = $days{$1};
5423         my $diff = ($first + 6 - $dow) % 7;
5424         $week = $2 >= 25 ? 5 : ($2 + 6 + $diff) / 7;
5425     }
5426     elsif ($on =~ /^(.*)<=(\d+)$/)
5427     {
5428         $dow = $days{$1};
5429         my $diff = ($first + $2 + 6 - $dow) % 7;
5430         $week = ($2 + 6 - $diff) / 7;
5431         if (!$week)
5432         {
5433             $week = 5;
5434             if (!--$mon) { $mon = 12; $year--; }
5435         }
5436     }
5437     elsif ($on =~ /^\d+$/)
5438     {
5439         $dow = ($first + $on - 1) % 7;
5440         $week = $on >= 25 ? 5 : ($on + 6) / 7;
5441     }
5442     else
5443     {
5444         die "unsupported date specification $year $in $on $at";
5445     }
5446
5447     if ($at =~ /^(\d+):(\d+):(\d+)([uws]?)$/)
5448     {
5449         $time = $1 * 60 + $2;
5450         $sec = $3;
5451         $flag = $4;
5452     }
5453     elsif ($at =~ /^(\d+):(\d+)([uws]?)$/)
5454     {
5455         $time = $1 * 60 + $2;
5456         $flag = $3;
5457     }
5458     elsif ($at =~ /^(\d+)([uws]?)$/)
5459     {
5460         $time = $1 * 60;
5461         $flag = $2;
5462     }
5463     else
5464     {
5465         die "unsupported time specification $year $in $on $at";
5466     }
5467
5468     $flag ||= "w";
5469     $time -= $stdoff if $flag eq "u";
5470     $time += 60 if !$isdst && $flag ne "w";
5471
5472     if ($time < 0)  # previous day
5473     {
5474         $week-- if $week < 5 && $dow == month_first_dow( $year, $mon );
5475         $week-- if $week == 5 && $dow == month_first_dow( $year + ($mon == 12), $mon % 12 + 1 );
5476         if (!$week)
5477         {
5478             $week = 5;
5479             if (!--$mon) { $mon = 12; $year--; }
5480         }
5481         $dow = ($dow + 6) % 7;
5482         $time += 24 * 60;
5483     }
5484
5485     return ($year, $mon, $week, $dow, $time / 60, $time % 60, $sec || 0);
5486 }
5487
5488
5489 ################################################################
5490 # parse a system time value as a SYSTEMTIME structure
5491 sub pack_systime(@)
5492 {
5493     my ($year, $mon, $week, $dow, $hour, $min, $sec) = @_;
5494     return pack "S<8", 0, $mon, $dow, $week, $hour < 24 ? ($hour, $min, $sec, 0) : (23, 59, 59, 999);
5495 }
5496
5497
5498 ################################################################
5499 # parse a timezone offset from the tzdata files
5500 sub parse_tz_offset($)
5501 {
5502     my ($hour, $min) = split /:/, shift;
5503     $min ||= 0;
5504     return $hour < 0 ? -$hour * 60 + $min : -$hour * 60 - $min;  # invert sign
5505 }
5506
5507
5508 ################################################################
5509 # build the timezone data
5510 sub dump_timezones($@)
5511 {
5512     my $filename = shift;
5513     my $FIRST_YEAR = 2000;
5514     my $LAST_YEAR = 2030;
5515
5516     my %names = load_windows_zones();
5517     my %zones;
5518     my %rules;
5519     my %links;
5520     my %res_indices;
5521
5522     printf "Building $filename\n";
5523
5524     open OUTPUT, ">$filename.new" or die "Cannot create $filename";
5525     print OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
5526     print OUTPUT "#include \"winresrc.h\"\n\n";
5527     print OUTPUT "#pragma makedep po\n\n";
5528     print OUTPUT "LANGUAGE LANG_ENGLISH, SUBLANG_DEFAULT\n\n";
5529     print OUTPUT "STRINGTABLE\n{\n";
5530
5531     # load tzdata files
5532
5533     foreach my $filename (@_)
5534     {
5535         my $FILE = open_data_file( "tzdata", $filename );
5536         my $zonename;
5537         while (<$FILE>)
5538         {
5539             chomp;
5540             s/\#.*$//;
5541             next if /^\s*$/;
5542             my @fields = split /\s+/;
5543             if ($fields[0] eq "Zone" || ($zonename && $fields[0] eq ""))
5544             {
5545                 shift @fields;
5546                 $zonename = shift @fields unless $zonename;
5547                 my ($stdoff, $rules, $dummy, @date) = @fields;
5548                 $zones{$zonename} ||= [ ];
5549                 push @{$zones{$zonename}}, [ parse_tz_offset( $stdoff ), $rules, @date ];
5550                 $zonename = undef unless @date;  # last entry doesn't have an until date
5551                 next;
5552             }
5553             if ($fields[0] eq "Rule")
5554             {
5555                 shift @fields;
5556                 my ($rulename, $from, $to, $dummy, $in, $on, $at, $save) = @fields;
5557                 $to = $from if $to eq "only";
5558                 $to = $LAST_YEAR if $to eq "max";
5559                 push @{$rules{$rulename}}, [ parse_tz_offset( $save ), $from, $to, $in, $on, $at ];
5560                 next;
5561             }
5562             if ($fields[0] eq "Link")
5563             {
5564                 $links{$fields[2]} = $fields[1];
5565                 next;
5566             }
5567             die "unrecognized line $_";
5568         }
5569         close $FILE;
5570     }
5571
5572     foreach my $name (sort { uc($a) cmp uc($b) } keys %names)
5573     {
5574         my ($display, $zone) = @{$names{$name}};
5575         $zone = $links{$zone} if defined $links{$zone};
5576
5577         # build list of transitions
5578
5579         my @transitions;
5580         my @from_date = ( 1 );
5581         my $last_stdoff = 0;
5582         for (my $i = 0; $i < scalar @{$zones{$zone}}; $i++)
5583         {
5584             my ($stdoff, $rule, @until_date) = @{$zones{$zone}->[$i]};
5585             my $isdst = ($last_stdoff != $stdoff);
5586             $from_date[0] ||= $LAST_YEAR;
5587             my @systime = parse_transition_date( $stdoff, $isdst, @from_date );
5588             push @transitions, [ $stdoff, -1, \@systime ];
5589
5590             if (defined $rules{$rule})
5591             {
5592                 foreach my $r (@{$rules{$rule}})
5593                 {
5594                     my ($offset, $from, $to, $in, $on, $at) = @{$r};
5595                     foreach my $year ($from..$to)
5596                     {
5597                         next if $year < $from_date[0];
5598                         next if $until_date[0] && $year > $until_date[0];
5599                         my @systime = parse_transition_date( $stdoff, !!$offset, $year, $in, $on, $at );
5600                         next if compare_transition_date( $stdoff, $isdst, \@until_date, \@systime ) <= 0;
5601                         my $ret = compare_transition_date( $stdoff, $isdst, \@from_date, \@systime );
5602                         next if $ret > 0;
5603                         pop @transitions if !$ret;  # remove transition if there's a dst change at the same time
5604                         push @transitions, [ $stdoff, $offset, \@systime ];
5605                     }
5606                 }
5607             }
5608             @from_date = @until_date;
5609             $last_stdoff = $stdoff;
5610         }
5611         @transitions = sort { compare_systime( $a->[2], $b->[2] ) }  @transitions;
5612
5613         # build per-year dynamic info
5614
5615         my @info;
5616         my $last_dstoff = 0;
5617         my $last_dst = 0;
5618         my $year = $FIRST_YEAR;
5619         while ($year <= $LAST_YEAR)
5620         {
5621             if (@transitions && $transitions[0]->[2]->[0] < $year)
5622             {
5623                 $last_stdoff = $transitions[0]->[0];
5624                 shift @transitions;
5625                 next;
5626             }
5627             my ($std, $dst, @trans);
5628             my $cur_stdoff = $last_stdoff;
5629             my $cur_dstoff = ($name =~ /^UTC/) ? 0 : -60;
5630             while (@transitions && $transitions[0]->[2]->[0] == $year)
5631             {
5632                 my $t = shift @transitions;
5633                 my ($stdoff, $dstoff, $systime) = @{$t};
5634                 $systime = pack_systime( @{$systime} );
5635                 if (!$dstoff)  # std
5636                 {
5637                     $cur_stdoff = $stdoff unless $std;
5638                     $std = $systime;
5639                 }
5640                 elsif ($dstoff != -1)  # dst
5641                 {
5642                     $cur_dstoff = $dstoff unless $dst;
5643                     $dst ||= $systime;
5644                 }
5645                 elsif ($stdoff != $last_stdoff)  # rule transition
5646                 {
5647                     # Handle a special case: Samoa moved to the other side of
5648                     # the date line between 2011-12-03 and 2012-01-01,
5649                     # entirely skipping the day 2011-12-31. We ignore this
5650                     # change because it happens on a year boundary and more
5651                     # importantly it would generate on offset of -25 hours,
5652                     # which some programs (e.g., Mono) do not like. See
5653                     # https://bugs.winehq.org/show_bug.cgi?id=51758
5654
5655                     if ($last_stdoff - $stdoff < 24 * 60)
5656                     {
5657                         @trans = ($last_stdoff, $stdoff, $systime);
5658                         $cur_stdoff = $stdoff;
5659                     }
5660                 }
5661                 elsif ($dst)  # rule transition with no stdoff change
5662                 {
5663                     $std = $systime;
5664                 }
5665                 $last_dstoff = ($dstoff == -1) ? 0 : $dstoff;
5666             }
5667             $last_stdoff = $cur_stdoff;
5668
5669             if ($cur_dstoff > 0)  # swap std and dst to ensure that offset is negative
5670             {
5671                 ($std, $dst) = ($dst, $std);
5672                 $cur_stdoff += $cur_dstoff;
5673                 $cur_dstoff = -$cur_dstoff;
5674             }
5675
5676             if (@trans)
5677             {
5678                 # heuristic to prefer switching dst
5679                 if ($last_dst == $year - 1 || (!$last_dst && $trans[0] > $trans[1]))
5680                 {
5681                     $dst ||= $trans[2];
5682                     $cur_stdoff = $trans[0];
5683                     $cur_dstoff = $trans[1] - $trans[0];
5684                 }
5685                 else
5686                 {
5687                     $std ||= $trans[2];
5688                     $cur_stdoff = $trans[1];
5689                     $cur_dstoff = $trans[0] - $trans[1];
5690                 }
5691             }
5692
5693             if ($std || $dst)
5694             {
5695                 $std ||= pack_systime( parse_transition_date( 0, 0, $year, "Jan", 1 ));
5696                 $dst ||= pack_systime( parse_transition_date( 0, 0, $year, "Jan", 1 ));
5697                 $last_dst = $year;
5698             }
5699             else
5700             {
5701                 $std = pack "S<8", 0;
5702                 $dst = pack "S<8", 0;
5703                 $cur_stdoff += $last_dstoff;
5704             }
5705             $info[$year++] = pack( "l<3", $cur_stdoff, 0, $cur_dstoff ) . $std . $dst;
5706         }
5707
5708         # output registry keys
5709
5710         my $std_name = $name eq "UTC" ? "Coordinated Universal Time" : $name;
5711         my $dlt_name = $std_name =~ s/Standard Time/Daylight Time/r;
5712         my $res_idx = hex( substr( Digest::SHA::sha1_hex($name), -3, 3 )) << 4;
5713         $res_idx += 16 while exists $res_indices{$res_idx};
5714         $res_indices{$res_idx} = 1;
5715
5716         add_registry_string_value( $zonekey, $name, "Display", $display );
5717         add_registry_string_value( $zonekey, $name, "Std", $std_name );
5718         add_registry_string_value( $zonekey, $name, "Dlt", $dlt_name );
5719         add_registry_string_value( $zonekey, $name, "MUI_Std", sprintf( "\@tzres.dll,-%u", $res_idx ));
5720         add_registry_string_value( $zonekey, $name, "MUI_Dlt", sprintf( "\@tzres.dll,-%u", $res_idx + 1 ));
5721         add_registry_string_value( $zonekey, $name, "MUI_Display", sprintf( "\@tzres.dll,-%u", $res_idx + 2 ));
5722         add_registry_binary_value( $zonekey, $name, "TZI", $info[$LAST_YEAR] );
5723
5724         printf OUTPUT "%7d \"#msgctxt#maximum 31 characters#%s\"\n", $res_idx, $std_name;
5725         printf OUTPUT "%7d \"#msgctxt#maximum 31 characters#%s\"\n", $res_idx + 1, $dlt_name;
5726         printf OUTPUT "%7d \"%s\"\n", $res_idx + 2, $display;
5727
5728         my $first_year = $FIRST_YEAR;
5729         my $last_year = $LAST_YEAR;
5730         $last_year-- while $last_year > $FIRST_YEAR && $info[$last_year] eq $info[$last_year - 1];
5731         $first_year++ while $first_year < $last_year && $info[$first_year] eq $info[$last_year];
5732
5733         next if $last_year <= $first_year;
5734
5735         foreach my $i ($first_year..$last_year)
5736         {
5737             add_registry_binary_value( $zonekey, "$name\\Dynamic DST", $i, $info[$i] );
5738         }
5739         add_registry_dword_value( $zonekey, "$name\\Dynamic DST", "FirstEntry", $first_year );
5740         add_registry_dword_value( $zonekey, "$name\\Dynamic DST", "LastEntry", $last_year );
5741     }
5742
5743     print OUTPUT "}\n";
5744     close OUTPUT;
5745     save_file($filename);
5746 }
5747
5748
5749 ################################################################
5750 # build the script to create registry keys
5751 sub dump_registry_script($%)
5752 {
5753     my ($filename, %keys) = @_;
5754     my $indent = 1;
5755     my @prev;
5756
5757     printf "Building %s\n", $filename;
5758     open OUTPUT, ">$filename.new" or die "Cannot create $filename";
5759     print OUTPUT "HKLM\n{\n";
5760     foreach my $k (sort { ($a =~ tr/a-z\\/A-Z\001/r) cmp ($b =~ tr/a-z\\/A-Z\001/r) } keys %keys)
5761     {
5762         my @subkeys = split /\\/, $k;
5763         while (@prev && @subkeys && $prev[0] eq $subkeys[0]) { shift @prev; shift @subkeys; }
5764         while (@prev) { printf OUTPUT "%*s}\n", 4 * --$indent, ""; shift @prev; }
5765         my ($def, @vals) = @{$keys{$k}};
5766         for (my $i = 0; $i < @subkeys; $i++)
5767         {
5768             my $name = $subkeys[$i];
5769             my $prefix = "";
5770             if ($name =~ /^-/)
5771             {
5772                 $name =~ s/^-//;
5773                 $prefix = "NoRemove ";
5774             }
5775             if ($name =~ /\s/)
5776             {
5777                 $name = "'$name'";
5778             }
5779             printf OUTPUT "%*s%s%s%s\n%*s{\n", 4 * $indent, "", $prefix, $name,
5780                 $i == $#subkeys && $def ? " = s '$def'" : "", 4 * $indent, "";
5781             $indent++;
5782         }
5783         foreach my $v (sort @vals) { printf OUTPUT "%*sval $v\n", 4 * $indent, ""; }
5784         @prev = split /\\/, $k;
5785     }
5786     while (@prev) { printf OUTPUT "%*s}\n", 4 * --$indent, ""; shift @prev; }
5787     printf OUTPUT "}\n";
5788     close OUTPUT;
5789     save_file($filename);
5790 }
5791
5792
5793 ################################################################
5794 # save a file if modified
5795 sub save_file($)
5796 {
5797     my $file = shift;
5798     if (-f $file && !system "cmp $file $file.new >/dev/null")
5799     {
5800         unlink "$file.new";
5801     }
5802     else
5803     {
5804         rename "$file.new", "$file";
5805     }
5806 }
5807
5808
5809 ################################################################
5810 # main routine
5811
5812 chdir ".." if -f "./make_unicode";
5813 load_data();
5814 dump_bidi_dir_table( "dlls/gdi32/uniscribe/direction.c" );
5815 dump_bidi_dir_table( "dlls/dwrite/direction.c" );
5816 dump_bidi_dir_table( "dlls/wineps.drv/direction.c" );
5817 dump_mirroring( "dlls/gdi32/uniscribe/mirror.c" );
5818 dump_mirroring( "dlls/dwrite/mirror.c" );
5819 dump_bracket( "dlls/gdi32/uniscribe/bracket.c" );
5820 dump_bracket( "dlls/dwrite/bracket.c" );
5821 dump_shaping( "dlls/gdi32/uniscribe/shaping.c" );
5822 dump_arabic_shaping( "dlls/dwrite/shapers/arabic_table.c" );
5823 dump_linebreak( "dlls/gdi32/uniscribe/linebreak.c" );
5824 dump_linebreak( "dlls/dwrite/linebreak.c" );
5825 dump_scripts( "dlls/dwrite/scripts" );
5826 dump_indic( "dlls/gdi32/uniscribe/indicsyllable.c" );
5827 dump_vertical( "dlls/win32u/vertical.c", 1 );
5828 dump_vertical( "dlls/wineps.drv/vertical.c", 0 );
5829 dump_intl_nls("nls/l_intl.nls");
5830 dump_norm_table( "nls/normnfc.nls" );
5831 dump_norm_table( "nls/normnfd.nls" );
5832 dump_norm_table( "nls/normnfkc.nls" );
5833 dump_norm_table( "nls/normnfkd.nls" );
5834 dump_norm_table( "nls/normidna.nls" );
5835 my $chartypes = dump_sortkey_table( "nls/sortdefault.nls" );
5836 dump_locales( "nls/locale.nls", $chartypes );
5837 foreach my $file (@allfiles) { dump_msdata_codepage( $file ); }
5838 dump_eucjp_codepage();
5839 dump_timezones( "dlls/tzres/tzres.rc", @timezone_files );
5840 dump_registry_script( "dlls/kernelbase/kernelbase.rgs", %registry_keys );
5841
5842 exit 0;
5843
5844 # Local Variables:
5845 # compile-command: "./make_unicode"
5846 # End: