Use uint32_t instead of unsigned long.
[libidn.git] / gen-unicode-tables.pl
blob590c54f0aed9cacf7430b422fcee8de9c52a9456
1 #! /usr/bin/perl -w
3 # Copyright (C) 2003 Simon Josefsson
4 # Copyright (C) 1998, 1999 Tom Tromey
5 # Copyright (C) 2001 Red Hat Software
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2, or (at your option)
10 # any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
20 # 02111-1307, USA.
22 # Contributer(s):
23 # Andrew Taylor <andrew.taylor@montage.ca>
25 # gen-unicode-tables.pl - Generate tables for libunicode from Unicode data.
26 # See http://www.unicode.org/Public/UNIDATA/UnicodeCharacterDatabase.html
27 # Usage: gen-unicode-tables.pl [-decomp | -both] UNICODE-VERSION UnicodeData.txt LineBreak.txt SpecialCasing.txt CaseFolding.txt
28 # I consider the output of this program to be unrestricted. Use it as
29 # you will.
31 # FIXME:
32 # * For decomp table it might make sense to use a shift count other
33 # than 8. We could easily compute the perfect shift count.
35 use vars qw($CODE $NAME $CATEGORY $COMBINING_CLASSES $BIDI_CATEGORY $DECOMPOSITION $DECIMAL_VALUE $DIGIT_VALUE $NUMERIC_VALUE $MIRRORED $OLD_NAME $COMMENT $UPPER $LOWER $TITLE $BREAK_CODE $BREAK_CATEGORY $BREAK_NAME $CASE_CODE $CASE_LOWER $CASE_TITLE $CASE_UPPER $CASE_CONDITION);
37 # Names of fields in Unicode data table.
38 $CODE = 0;
39 $NAME = 1;
40 $CATEGORY = 2;
41 $COMBINING_CLASSES = 3;
42 $BIDI_CATEGORY = 4;
43 $DECOMPOSITION = 5;
44 $DECIMAL_VALUE = 6;
45 $DIGIT_VALUE = 7;
46 $NUMERIC_VALUE = 8;
47 $MIRRORED = 9;
48 $OLD_NAME = 10;
49 $COMMENT = 11;
50 $UPPER = 12;
51 $LOWER = 13;
52 $TITLE = 14;
54 # Names of fields in the line break table
55 $BREAK_CODE = 0;
56 $BREAK_PROPERTY = 1;
58 # Names of fields in the SpecialCasing table
59 $CASE_CODE = 0;
60 $CASE_LOWER = 1;
61 $CASE_TITLE = 2;
62 $CASE_UPPER = 3;
63 $CASE_CONDITION = 4;
65 # Names of fields in the CaseFolding table
66 $FOLDING_CODE = 0;
67 $FOLDING_STATUS = 1;
68 $FOLDING_MAPPING = 2;
70 # Map general category code onto symbolic name.
71 %mappings =
73 # Normative.
74 'Lu' => "G_UNICODE_UPPERCASE_LETTER",
75 'Ll' => "G_UNICODE_LOWERCASE_LETTER",
76 'Lt' => "G_UNICODE_TITLECASE_LETTER",
77 'Mn' => "G_UNICODE_NON_SPACING_MARK",
78 'Mc' => "G_UNICODE_COMBINING_MARK",
79 'Me' => "G_UNICODE_ENCLOSING_MARK",
80 'Nd' => "G_UNICODE_DECIMAL_NUMBER",
81 'Nl' => "G_UNICODE_LETTER_NUMBER",
82 'No' => "G_UNICODE_OTHER_NUMBER",
83 'Zs' => "G_UNICODE_SPACE_SEPARATOR",
84 'Zl' => "G_UNICODE_LINE_SEPARATOR",
85 'Zp' => "G_UNICODE_PARAGRAPH_SEPARATOR",
86 'Cc' => "G_UNICODE_CONTROL",
87 'Cf' => "G_UNICODE_FORMAT",
88 'Cs' => "G_UNICODE_SURROGATE",
89 'Co' => "G_UNICODE_PRIVATE_USE",
90 'Cn' => "G_UNICODE_UNASSIGNED",
92 # Informative.
93 'Lm' => "G_UNICODE_MODIFIER_LETTER",
94 'Lo' => "G_UNICODE_OTHER_LETTER",
95 'Pc' => "G_UNICODE_CONNECT_PUNCTUATION",
96 'Pd' => "G_UNICODE_DASH_PUNCTUATION",
97 'Ps' => "G_UNICODE_OPEN_PUNCTUATION",
98 'Pe' => "G_UNICODE_CLOSE_PUNCTUATION",
99 'Pi' => "G_UNICODE_INITIAL_PUNCTUATION",
100 'Pf' => "G_UNICODE_FINAL_PUNCTUATION",
101 'Po' => "G_UNICODE_OTHER_PUNCTUATION",
102 'Sm' => "G_UNICODE_MATH_SYMBOL",
103 'Sc' => "G_UNICODE_CURRENCY_SYMBOL",
104 'Sk' => "G_UNICODE_MODIFIER_SYMBOL",
105 'So' => "G_UNICODE_OTHER_SYMBOL"
108 %break_mappings =
110 'BK' => "G_UNICODE_BREAK_MANDATORY",
111 'CR' => "G_UNICODE_BREAK_CARRIAGE_RETURN",
112 'LF' => "G_UNICODE_BREAK_LINE_FEED",
113 'CM' => "G_UNICODE_BREAK_COMBINING_MARK",
114 'SG' => "G_UNICODE_BREAK_SURROGATE",
115 'ZW' => "G_UNICODE_BREAK_ZERO_WIDTH_SPACE",
116 'IN' => "G_UNICODE_BREAK_INSEPARABLE",
117 'GL' => "G_UNICODE_BREAK_NON_BREAKING_GLUE",
118 'CB' => "G_UNICODE_BREAK_CONTINGENT",
119 'SP' => "G_UNICODE_BREAK_SPACE",
120 'BA' => "G_UNICODE_BREAK_AFTER",
121 'BB' => "G_UNICODE_BREAK_BEFORE",
122 'B2' => "G_UNICODE_BREAK_BEFORE_AND_AFTER",
123 'HY' => "G_UNICODE_BREAK_HYPHEN",
124 'NS' => "G_UNICODE_BREAK_NON_STARTER",
125 'OP' => "G_UNICODE_BREAK_OPEN_PUNCTUATION",
126 'CL' => "G_UNICODE_BREAK_CLOSE_PUNCTUATION",
127 'QU' => "G_UNICODE_BREAK_QUOTATION",
128 'EX' => "G_UNICODE_BREAK_EXCLAMATION",
129 'ID' => "G_UNICODE_BREAK_IDEOGRAPHIC",
130 'NU' => "G_UNICODE_BREAK_NUMERIC",
131 'IS' => "G_UNICODE_BREAK_INFIX_SEPARATOR",
132 'SY' => "G_UNICODE_BREAK_SYMBOL",
133 'AL' => "G_UNICODE_BREAK_ALPHABETIC",
134 'PR' => "G_UNICODE_BREAK_PREFIX",
135 'PO' => "G_UNICODE_BREAK_POSTFIX",
136 'SA' => "G_UNICODE_BREAK_COMPLEX_CONTEXT",
137 'AI' => "G_UNICODE_BREAK_AMBIGUOUS",
138 'XX' => "G_UNICODE_BREAK_UNKNOWN"
141 # Title case mappings.
142 %title_to_lower = ();
143 %title_to_upper = ();
145 # Maximum length of special-case strings
147 my $special_case_len = 0;
148 my @special_cases;
150 $do_decomp = 0;
151 $do_props = 1;
152 if (@ARGV && $ARGV[0] eq '-decomp')
154 $do_decomp = 1;
155 $do_props = 0;
156 shift @ARGV;
158 elsif (@ARGV && $ARGV[0] eq '-both')
160 $do_decomp = 1;
161 shift @ARGV;
164 if (@ARGV != 6) {
165 $0 =~ s@.*/@@;
166 die "Usage: $0 [-decomp | -both] UNICODE-VERSION UnicodeData.txt LineBreak.txt SpecialCasing.txt CaseFolding.txt CompositionExclusions.txt\n";
169 print "Creating decomp table\n" if ($do_decomp);
170 print "Creating property table\n" if ($do_props);
172 print "Composition exlusions from $ARGV[5]\n";
174 open (INPUT, "< $ARGV[5]") || exit 1;
176 while (<INPUT>) {
178 chop;
180 next if /^#/;
181 next if /^\s*$/;
183 s/\s*#.*//;
185 s/^\s*//;
186 s/\s*$//;
188 $composition_exclusions{hex($_)} = 1;
191 close INPUT;
193 print "Unicode data from $ARGV[1]\n";
195 open (INPUT, "< $ARGV[1]") || exit 1;
197 $last_code = -1;
198 while (<INPUT>)
200 chop;
201 @fields = split (';', $_, 30);
202 if ($#fields != 14)
204 printf STDERR ("Entry for $fields[$CODE] has wrong number of fields (%d)\n", $#fields);
207 $code = hex ($fields[$CODE]);
209 last if ($code > 0xFFFF); # ignore characters out of the basic plane
211 if ($code > $last_code + 1)
213 # Found a gap.
214 if ($fields[$NAME] =~ /Last>/)
216 # Fill the gap with the last character read,
217 # since this was a range specified in the char database
218 @gfields = @fields;
220 else
222 # The gap represents undefined characters. Only the type
223 # matters.
224 @gfields = ('', '', 'Cn', '0', '', '', '', '', '', '', '',
225 '', '', '', '');
227 for (++$last_code; $last_code < $code; ++$last_code)
229 $gfields{$CODE} = sprintf ("%04x", $last_code);
230 &process_one ($last_code, @gfields);
233 &process_one ($code, @fields);
234 $last_code = $code;
237 close INPUT;
239 @gfields = ('', '', 'Cn', '0', '', '', '', '', '', '', '',
240 '', '', '', '');
241 for (++$last_code; $last_code < 0x10000; ++$last_code)
243 $gfields{$CODE} = sprintf ("%04x", $last_code);
244 &process_one ($last_code, @gfields);
246 --$last_code; # Want last to be 0xFFFF.
248 print "Creating line break table\n";
250 print "Line break data from $ARGV[2]\n";
252 open (INPUT, "< $ARGV[2]") || exit 1;
254 $last_code = -1;
255 while (<INPUT>)
257 my ($start_code, $end_code);
259 chop;
261 next if /^#/;
263 s/\s*#.*//;
265 @fields = split (';', $_, 30);
266 if ($#fields != 1)
268 printf STDERR ("Entry for $fields[$CODE] has wrong number of fields (%d)\n", $#fields);
269 next;
272 if ($fields[$CODE] =~ /([A-F0-9]{4})..([A-F0-9]{4})/)
274 $start_code = hex ($1);
275 $end_code = hex ($2);
276 } else {
277 $start_code = $end_code = hex ($fields[$CODE]);
281 last if ($start_code > 0xFFFF); # FIXME ignore characters out of the basic plane
283 if ($start_code > $last_code + 1)
285 # The gap represents undefined characters. If assigned,
286 # they are AL, if not assigned, XX
287 for (++$last_code; $last_code < $start_code; ++$last_code)
289 if ($type[$last_code] eq 'Cn')
291 $break_props[$last_code] = 'XX';
293 else
295 $break_props[$last_code] = 'AL';
300 for ($last_code = $start_code; $last_code <= $end_code; $last_code++)
302 $break_props[$last_code] = $fields[$BREAK_PROPERTY];
305 $last_code = $end_code;
308 close INPUT;
310 for (++$last_code; $last_code < 0x10000; ++$last_code)
312 if ($type[$last_code] eq 'Cn')
314 $break_props[$last_code] = 'XX';
316 else
318 $break_props[$last_code] = 'AL';
321 --$last_code; # Want last to be 0xFFFF.
323 print STDERR "Last code is not 0xFFFF" if ($last_code != 0xFFFF);
325 print "Reading special-casing table for case conversion\n";
327 open (INPUT, "< $ARGV[3]") || exit 1;
329 while (<INPUT>)
331 my $code;
333 chop;
335 next if /^#/;
336 next if /^\s*$/;
338 s/\s*#.*//;
340 @fields = split ('\s*;\s*', $_, 30);
342 $raw_code = $fields[$CASE_CODE];
343 $code = hex ($raw_code);
345 if ($#fields != 4 && $#fields != 5)
347 printf STDERR ("Entry for $raw_code has wrong number of fields (%d)\n", $#fields);
348 next;
351 if (!defined $type[$code])
353 printf STDERR "Special case for code point: $code, which has no defined type\n";
354 next;
357 if (defined $fields[5]) {
358 # Ignore conditional special cases - we'll handle them in code
359 next;
362 if ($type[$code] eq 'Lu')
364 (hex $fields[$CASE_UPPER] == $code) || die "$raw_code is Lu and UCD_Upper($raw_code) != $raw_code";
366 &add_special_case ($code, $value[$code],$fields[$CASE_LOWER], $fields[$CASE_TITLE]);
368 } elsif ($type[$code] eq 'Lt')
370 (hex $fields[$CASE_TITLE] == $code) || die "$raw_code is Lt and UCD_Title($raw_code) != $raw_code";
372 &add_special_case ($code, undef,$fields[$CASE_LOWER], $fields[$CASE_UPPER]);
373 } elsif ($type[$code] eq 'Ll')
375 (hex $fields[$CASE_LOWER] == $code) || die "$raw_code is Ll and UCD_Lower($raw_code) != $raw_code";
377 &add_special_case ($code, $value[$code],$fields[$CASE_UPPER], $fields[$CASE_TITLE]);
378 } else {
379 printf STDERR "Special case for non-alphabetic code point: $raw_code\n";
380 next;
384 close INPUT;
386 open (INPUT, "< $ARGV[4]") || exit 1;
388 my $casefoldlen = 0;
389 my @casefold;
391 while (<INPUT>)
393 my $code;
395 chop;
397 next if /^#/;
398 next if /^\s*$/;
400 s/\s*#.*//;
402 @fields = split ('\s*;\s*', $_, 30);
404 $raw_code = $fields[$FOLDING_CODE];
405 $code = hex ($raw_code);
407 next if $code > 0xffff; # FIXME!
409 if ($#fields != 3)
411 printf STDERR ("Entry for $raw_code has wrong number of fields (%d)\n", $#fields);
412 next;
415 next if ($fields[$FOLDING_STATUS] eq 'S');
417 @values = map { hex ($_) } split /\s+/, $fields[$FOLDING_MAPPING];
419 # Check simple case
421 if (@values == 1 &&
422 !(defined $value[$code] && $value[$code] >= 0xd800 && $value[$code] < 0xdc00) &&
423 defined $type[$code]) {
425 my $lower;
426 if ($type[$code] eq 'Ll')
428 $lower = $code;
429 } elsif ($type[$code] eq 'Lt')
431 $lower = $title_to_lower{$code};
432 } elsif ($type[$code] eq 'Lu')
434 $lower = $value[$code];
435 } else {
436 $lower = $code;
439 if ($lower == $values[0]) {
440 next;
444 my $string = pack ("U*", @values);
445 if (1 + length $string > $casefoldlen) {
446 $casefoldlen = 1 + length $string;
449 push @casefold, [ $code, $string ];
452 close INPUT;
454 if ($do_props) {
455 &print_tables ($last_code)
457 if ($do_decomp) {
458 &print_decomp ($last_code);
459 &output_composition_table;
462 &print_line_break ($last_code);
464 exit 0;
466 # Process a single character.
467 sub process_one
469 my ($code, @fields) = @_;
471 $type[$code] = $fields[$CATEGORY];
472 if ($type[$code] eq 'Nd')
474 $value[$code] = int ($fields[$DECIMAL_VALUE]);
476 elsif ($type[$code] eq 'Ll')
478 $value[$code] = hex ($fields[$UPPER]);
480 elsif ($type[$code] eq 'Lu')
482 $value[$code] = hex ($fields[$LOWER]);
485 if ($type[$code] eq 'Lt')
487 $title_to_lower{$code} = hex ($fields[$LOWER]);
488 $title_to_upper{$code} = hex ($fields[$UPPER]);
491 $cclass[$code] = $fields[$COMBINING_CLASSES];
493 # Handle decompositions.
494 if ($fields[$DECOMPOSITION] ne '')
496 if ($fields[$DECOMPOSITION] =~ s/\<.*\>\s*//) {
497 $decompose_compat[$code] = 1;
498 } else {
499 $decompose_compat[$code] = 0;
501 if (!exists $composition_exclusions{$code}) {
502 $compositions{$code} = $fields[$DECOMPOSITION];
505 $decompositions[$code] = $fields[$DECOMPOSITION];
509 sub print_tables
511 my ($last) = @_;
512 my ($outfile) = "gunichartables.h";
514 local ($bytes_out) = 0;
516 print "Writing $outfile...\n";
518 open (OUT, "> $outfile");
520 print OUT "/* This file is automatically generated. DO NOT EDIT!\n";
521 print OUT " Instead, edit gen-unicode-tables.pl and re-run. */\n\n";
523 print OUT "#ifndef CHARTABLES_H\n";
524 print OUT "#define CHARTABLES_H\n\n";
526 print OUT "#define G_UNICODE_DATA_VERSION \"$ARGV[0]\"\n\n";
528 printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last;
530 printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 1000\n\n";
532 $table_index = 0;
533 printf OUT "static const char type_data[][256] = {\n";
534 for ($count = 0; $count <= $last; $count += 256)
536 $row[$count / 256] = &print_row ($count, 1, \&fetch_type);
538 printf OUT "\n};\n\n";
540 print OUT "static const short type_table[256] = {\n";
541 for ($count = 0; $count <= $last; $count += 256)
543 print OUT ",\n" if $count > 0;
544 print OUT " ", $row[$count / 256];
545 $bytes_out += 2;
547 print OUT "\n};\n\n";
551 # Now print attribute table.
554 $table_index = 0;
555 printf OUT "static const unsigned short attr_data[][256] = {\n";
556 for ($count = 0; $count <= $last; $count += 256)
558 $row[$count / 256] = &print_row ($count, 2, \&fetch_attr);
560 printf OUT "\n};\n\n";
562 print OUT "static const short attr_table[256] = {\n";
563 for ($count = 0; $count <= $last; $count += 256)
565 print OUT ",\n" if $count > 0;
566 print OUT " ", $row[$count / 256];
567 $bytes_out += 2;
569 print OUT "\n};\n\n";
572 # print title case table
575 # FIXME: type.
576 print OUT "static const unsigned short title_table[][3] = {\n";
577 my ($item);
578 my ($first) = 1;
579 foreach $item (sort keys %title_to_lower)
581 print OUT ",\n"
582 unless $first;
583 $first = 0;
584 printf OUT " { 0x%04x, 0x%04x, 0x%04x }", $item, $title_to_upper{$item}, $title_to_lower{$item};
585 $bytes_out += 6;
587 print OUT "\n};\n\n";
590 # And special case conversion table -- conversions that change length
592 &output_special_case_table (\*OUT);
593 &output_casefold_table (\*OUT);
595 print OUT "#endif /* CHARTABLES_H */\n";
597 close (OUT);
599 printf STDERR "Generated %d bytes in tables\n", $bytes_out;
602 # A fetch function for the type table.
603 sub fetch_type
605 my ($index) = @_;
606 return $mappings{$type[$index]};
609 # A fetch function for the attribute table.
610 sub fetch_attr
612 my ($index) = @_;
613 if (defined $value[$index])
615 return sprintf ("0x%04x", $value[$index]);
617 else
619 return "0x0000";
623 sub print_row
625 my ($start, $typsize, $fetcher) = @_;
627 my ($i);
628 my (@values);
629 my ($flag) = 1;
630 my ($off);
632 for ($off = 0; $off < 256; ++$off)
634 $values[$off] = $fetcher->($off + $start);
635 if ($values[$off] ne $values[0])
637 $flag = 0;
640 if ($flag)
642 return $values[0] . " + G_UNICODE_MAX_TABLE_INDEX";
645 printf OUT ",\n" if ($table_index != 0);
646 printf OUT " { /* page %d, index %d */\n ", $start / 256, $table_index;
647 my ($column) = 4;
648 for ($i = $start; $i < $start + 256; ++$i)
650 print OUT ", "
651 if $i > $start;
652 my ($text) = $values[$i - $start];
653 if (length ($text) + $column + 2 > 78)
655 print OUT "\n ";
656 $column = 4;
658 print OUT $text;
659 $column += length ($text) + 2;
661 print OUT "\n }";
663 $bytes_out += 256 * $typsize;
665 return sprintf "%d /* page %d */", $table_index++, $start / 256;
668 # Generate the character decomposition header.
669 sub print_decomp
671 my ($last) = @_;
672 my ($outfile) = "gunidecomp.h";
674 local ($bytes_out) = 0;
676 print "Writing $outfile...\n";
678 open (OUT, "> $outfile") || exit 1;
680 print OUT "/* This file is automatically generated. DO NOT EDIT! */\n\n";
681 print OUT "#ifndef DECOMP_H\n";
682 print OUT "#define DECOMP_H\n\n";
684 printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last;
686 printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 1000\n\n";
688 my ($count, @row);
689 $table_index = 0;
690 printf OUT "static const unsigned char cclass_data[][256] = {\n";
691 for ($count = 0; $count <= $last; $count += 256)
693 $row[$count / 256] = &print_row ($count, 1, \&fetch_cclass);
695 printf OUT "\n};\n\n";
697 print OUT "static const short combining_class_table[256] = {\n";
698 for ($count = 0; $count <= $last; $count += 256)
700 print OUT ",\n" if $count > 0;
701 print OUT " ", $row[$count / 256];
702 $bytes_out += 2;
704 print OUT "\n};\n\n";
706 print OUT "typedef struct\n{\n";
707 # FIXME: type.
708 print OUT " unsigned short ch;\n";
709 print OUT " unsigned char canon_offset;\n";
710 print OUT " unsigned char compat_offset;\n";
711 print OUT " unsigned short expansion_offset;\n";
712 print OUT "} decomposition;\n\n";
714 print OUT "static const decomposition decomp_table[] =\n{\n";
715 my ($iter);
716 my ($first) = 1;
717 my ($decomp_string) = "";
718 my ($decomp_string_offset) = 0;
719 for ($count = 0; $count <= $last; ++$count)
721 if (defined $decompositions[$count])
723 print OUT ",\n"
724 if ! $first;
725 $first = 0;
727 my $canon_decomp;
728 my $compat_decomp;
730 if (!$decompose_compat[$count]) {
731 $canon_decomp = make_decomp ($count, 0);
733 $compat_decomp = make_decomp ($count, 1);
735 if (defined $canon_decomp && $compat_decomp eq $canon_decomp) {
736 undef $compat_decomp;
739 my $string = "";
740 my $canon_offset = 0xff;
741 my $compat_offset = 0xff;
743 if (defined $canon_decomp) {
744 $canon_offset = 0;
745 $string .= $canon_decomp;
747 if (defined $compat_decomp) {
748 if (defined $canon_decomp) {
749 $string .= "\\x00\\x00";
751 $compat_offset = (length $string) / 4;
752 $string .= $compat_decomp;
755 if (!defined($decomp_offsets{$string})) {
756 $decomp_offsets{$string} = $decomp_string_offset;
757 $decomp_string .= "\n \"".$string."\\0\\0\" /* offset ".
758 $decomp_string_offset." */";
759 $decomp_string_offset += ((length $string) / 4) + 2;
761 $bytes_out += (length $string) / 4 + 2; # "\x20"
764 printf OUT qq( { 0x%04x, %u, %u, %d }),
765 $count, $canon_offset, $compat_offset, $decomp_offsets{$string};
766 $bytes_out += 6;
770 print OUT "\n};\n\n";
772 printf OUT "static const unsigned char decomp_expansion_string[] = %s;\n\n", $decomp_string;
774 print OUT "#endif /* DECOMP_H */\n";
776 printf STDERR "Generated %d bytes in decomp tables\n", $bytes_out;
779 sub print_line_break
781 my ($last) = @_;
782 my ($outfile) = "gunibreak.h";
784 local ($bytes_out) = 0;
786 print "Writing $outfile...\n";
788 open (OUT, "> $outfile");
790 print OUT "/* This file is automatically generated. DO NOT EDIT!\n";
791 print OUT " Instead, edit gen-unicode-tables.pl and re-run. */\n\n";
793 print OUT "#ifndef BREAKTABLES_H\n";
794 print OUT "#define BREAKTABLES_H\n\n";
796 print OUT "#define G_UNICODE_DATA_VERSION \"$ARGV[0]\"\n\n";
798 printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last;
800 printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 1000\n\n";
802 $table_index = 0;
803 printf OUT "static const char break_property_data[][256] = {\n";
804 for ($count = 0; $count <= $last; $count += 256)
806 $row[$count / 256] = &print_row ($count, 1, \&fetch_break_type);
808 printf OUT "\n};\n\n";
810 print OUT "static const short break_property_table[256] = {\n";
811 for ($count = 0; $count <= $last; $count += 256)
813 print OUT ",\n" if $count > 0;
814 print OUT " ", $row[$count / 256];
815 $bytes_out += 2;
817 print OUT "\n};\n\n";
819 print OUT "#endif /* BREAKTABLES_H */\n";
821 close (OUT);
823 printf STDERR "Generated %d bytes in break tables\n", $bytes_out;
827 # A fetch function for the break properties table.
828 sub fetch_break_type
830 my ($index) = @_;
831 return $break_mappings{$break_props[$index]};
834 # Fetcher for combining class.
835 sub fetch_cclass
837 my ($i) = @_;
838 return $cclass[$i];
841 # Expand a character decomposition recursively.
842 sub expand_decomp
844 my ($code, $compat) = @_;
846 my ($iter, $val);
847 my (@result) = ();
848 foreach $iter (split (' ', $decompositions[$code]))
850 $val = hex ($iter);
851 if (defined $decompositions[$val] &&
852 ($compat || !$decompose_compat[$val]))
854 push (@result, &expand_decomp ($val, $compat));
856 else
858 push (@result, $val);
862 return @result;
865 sub make_decomp
867 my ($code, $compat) = @_;
869 my $result = "";
870 foreach $iter (&expand_decomp ($code, $compat))
872 $result .= sprintf "\\x%02x\\x%02x", $iter / 256, $iter & 0xff;
875 $result;
877 # Generate special case data string from two fields
878 sub add_special_case
880 my ($code, $single, $field1, $field2) = @_;
882 @values = (defined $single ? $single : (),
883 (map { hex ($_) } split /\s+/, $field1),
885 (map { hex ($_) } split /\s+/, $field2));
886 $result = "";
889 for $value (@values) {
890 $result .= sprintf ("\\x%02x\\x%02x", $value / 256, $value & 0xff);
893 $result .= "\\0";
895 if (2 * @values + 2 > $special_case_len) {
896 $special_case_len = 2 * @values + 2;
899 push @special_cases, $result;
902 # We encode special cases in the surrogate pair space
904 $value[$code] = 0xD800 + scalar(@special_cases) - 1;
907 sub output_special_case_table
909 my $out = shift;
911 print $out <<EOT;
913 /* Table of special cases for case conversion; each record contains
914 * First, the best single character mapping to lowercase if Lu,
915 * and to uppercase if Ll, followed by the output mapping for the two cases
916 * other than the case of the codepoint, in the order [Ll],[Lu],[Lt],
917 * separated and terminated by a double NUL.
919 static const unsigned char special_case_table[][$special_case_len] = {
922 for $case (@special_cases) {
923 print $out qq( "$case",\n);
926 print $out <<EOT;
931 print STDERR "Generated ", ($special_case_len * scalar @special_cases), " bytes in special case table\n";
934 sub enumerate_ordered
936 my ($array) = @_;
938 my $n = 0;
939 for my $code (sort { $a <=> $b } keys %$array) {
940 if ($array->{$code} == 1) {
941 delete $array->{$code};
942 next;
944 $array->{$code} = $n++;
947 return $n;
950 sub output_composition_table
952 print STDERR "Generating composition table\n";
954 local ($bytes_out) = 0;
956 my %first;
957 my %second;
959 # First we need to go through and remove decompositions
960 # starting with a non-starter, and single-character
961 # decompositions. At the same time, record
962 # the first and second character of each decomposition
964 for $code (keys %compositions) {
965 @values = map { hex ($_) } split /\s+/, $compositions{$code};
966 if ($cclass[$values[0]]) {
967 delete $compositions{$code};
968 next;
970 if (@values == 1) {
971 delete $compositions{$code};
972 next;
974 if (@values != 2) {
975 die "$code has more than two elements in its decomposition!\n";
978 if (exists $first{$values[0]}) {
979 $first{$values[0]}++;
980 } else {
981 $first{$values[0]} = 1;
985 # Assign integer indicices, removing singletons
986 my $n_first = enumerate_ordered (\%first);
988 # Now record the second character if each (non-singleton) decomposition
989 for $code (keys %compositions) {
990 @values = map { hex ($_) } split /\s+/, $compositions{$code};
992 if (exists $first{$values[0]}) {
993 if (exists $second{$values[1]}) {
994 $second{$values[1]}++;
995 } else {
996 $second{$values[1]} = 1;
1001 # Assign integer indices, removing duplicate
1002 my $n_second = enumerate_ordered (\%second);
1004 # Build reverse table
1006 my @first_singletons;
1007 my @second_singletons;
1008 my %reverse;
1009 for $code (keys %compositions) {
1010 @values = map { hex ($_) } split /\s+/, $compositions{$code};
1012 my $first = $first{$values[0]};
1013 my $second = $second{$values[1]};
1015 if (defined $first && defined $second) {
1016 $reverse{"$first|$second"} = $code;
1017 } elsif (!defined $first) {
1018 push @first_singletons, [ $values[0], $values[1], $code ];
1019 } else {
1020 push @second_singletons, [ $values[1], $values[0], $code ];
1024 @first_singletons = sort { $a->[0] <=> $b->[0] } @first_singletons;
1025 @second_singletons = sort { $a->[0] <=> $b->[0] } @second_singletons;
1027 my %vals;
1029 open OUT, ">gunicomp.h" or die "Cannot open gunicomp.h: $!\n";
1030 printf OUT "#include \"internal.h\"\n";
1032 # Assign values in lookup table for all code points involved
1034 my $total = 1;
1035 my $last = 0;
1036 printf OUT "#define COMPOSE_FIRST_START %d\n", $total;
1037 for $code (keys %first) {
1038 $vals{$code} = $first{$code} + $total;
1039 $last = $code if $code > $last;
1041 $total += $n_first;
1042 $i = 0;
1043 printf OUT "#define COMPOSE_FIRST_SINGLE_START %d\n", $total;
1044 for $record (@first_singletons) {
1045 my $code = $record->[0];
1046 $vals{$code} = $i++ + $total;
1047 $last = $code if $code > $last;
1049 $total += @first_singletons;
1050 printf OUT "#define COMPOSE_SECOND_START %d\n", $total;
1051 for $code (keys %second) {
1052 $vals{$code} = $second{$code} + $total;
1053 $last = $code if $code > $last;
1055 $total += $n_second;
1056 $i = 0;
1057 printf OUT "#define COMPOSE_SECOND_SINGLE_START %d\n\n", $total;
1058 for $record (@second_singletons) {
1059 my $code = $record->[0];
1060 $vals{$code} = $i++ + $total;
1061 $last = $code if $code > $last;
1064 # Output lookup table
1066 my @row;
1067 $table_index = 0;
1068 printf OUT "static const int compose_data[][256] = {\n";
1069 for (my $count = 0; $count <= $last; $count += 256)
1071 $row[$count / 256] = &print_row ($count, 2, sub { exists $vals{$_[0]} ? $vals{$_[0]} : 0; });
1073 printf OUT "\n};\n\n";
1075 print OUT "static const short compose_table[256] = {\n";
1076 for (my $count = 0; $count <= $last; $count += 256)
1078 print OUT ",\n" if $count > 0;
1079 print OUT " ", $row[$count / 256];
1080 $bytes_out += 4;
1082 print OUT "\n};\n\n";
1084 # Output first singletons
1086 print OUT "static const uint32_t compose_first_single[][2] = {\n";
1087 $i = 0;
1088 for $record (@first_singletons) {
1089 print OUT ",\n" if $i++ > 0;
1090 printf OUT " { %#06x, %#06x }", $record->[1], $record->[2];
1092 print OUT "\n};\n";
1094 $bytes_out += @first_singletons * 4;
1096 # Output second singletons
1098 print OUT "static const uint32_t compose_second_single[][2] = {\n";
1099 $i = 0;
1100 for $record (@second_singletons) {
1101 print OUT ",\n" if $i++ > 0;
1102 printf OUT " { %#06x, %#06x }", $record->[1], $record->[2];
1104 print OUT "\n};\n";
1106 $bytes_out += @second_singletons * 4;
1108 # Output array of composition pairs
1110 print OUT <<EOT;
1111 static const int compose_array[$n_first][$n_second] = {
1114 for (my $i = 0; $i < $n_first; $i++) {
1115 print OUT ",\n" if $i;
1116 print OUT " { ";
1117 for (my $j = 0; $j < $n_second; $j++) {
1118 print OUT ", " if $j;
1119 if (exists $reverse{"$i|$j"}) {
1120 printf OUT "%#06x", $reverse{"$i|$j"};
1121 } else {
1122 print OUT " 0";
1125 print OUT " }";
1127 print OUT "\n";
1129 print OUT <<EOT;
1133 $bytes_out += $n_first * $n_second * 2;
1135 printf STDERR "Generated %d bytes in compose tables\n", $bytes_out;
1138 sub output_casefold_table
1140 my $out = shift;
1142 print $out <<EOT;
1144 /* Table of casefolding cases that can't be derived by lowercasing
1146 static const struct {
1147 guint16 ch;
1148 gchar data[$casefoldlen];
1149 } casefold_table[] = {
1152 @casefold = sort { $a->[0] <=> $b->[0] } @casefold;
1154 for $case (@casefold) {
1155 $code = $case->[0];
1156 $string = $case->[1];
1157 print $out sprintf(qq({ %#04x, "$string" },\n), $code);
1161 print $out <<EOT;
1166 my $recordlen = (2+$casefoldlen+1) & ~1;
1167 printf "Generated %d bytes for casefold table\n", $recordlen * @casefold;