libcpp/makeucnid.cc

   1 /* Make ucnid.h from various sources.
   2    Copyright (C) 2005-2023 Free Software Foundation, Inc.
   3
   4 This program is free software; you can redistribute it and/or modify it
   5 under the terms of the GNU General Public License as published by the
   6 Free Software Foundation; either version 3, or (at your option) any
   7 later version.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; see the file COPYING3.  If not see
  16 <http://www.gnu.org/licenses/>.  */
  17
  18 /* Run this program as
  19    ./makeucnid ucnid.tab UnicodeData.txt DerivedNormalizationProps.txt \
  20       DerivedCoreProperties.txt > ucnid.h
  21 */
  22
  23 #include <stdio.h>
  24 #include <string.h>
  25 #include <ctype.h>
  26 #include <stdbool.h>
  27 #include <stdlib.h>
  28
  29 enum {
  30   C99 = 1,
  31   CXX = 2,
  32   N99 = 4,
  33   C11 = 8,
  34   N11 = 16,
  35   CXX23 = 32,
  36   NXX23 = 64,
  37   all_languages = C99 | CXX | C11 | CXX23 | NXX23,
  38   not_NFC = 128,
  39   not_NFKC = 256,
  40   maybe_not_NFC = 512
  41 };
  42
  43 #define NUM_CODE_POINTS 0x110000
  44 #define MAX_CODE_POINT 0x10ffff
  45
  46 static unsigned flags[NUM_CODE_POINTS];
  47 static unsigned int all_decomp[NUM_CODE_POINTS][2];
  48 static unsigned int decomp[NUM_CODE_POINTS][2];
  49 static unsigned char combining_value[NUM_CODE_POINTS];
  50
  51 /* Die!  */
  52
  53 static void
  54 fail (const char *s)
  55 {
  56   fprintf (stderr, "%s\n", s);
  57   exit (1);
  58 }
  59
  60 /* Read ucnid.tab and set the flags for language versions in header[].  */
  61
  62 static void
  63 read_ucnid (const char *fname)
  64 {
  65   FILE *f = fopen (fname, "r");
  66   unsigned fl = 0;
  67
  68   if (!f)
  69     fail ("opening ucnid.tab");
  70   for (;;)
  71     {
  72       char line[256];
  73
  74       if (!fgets (line, sizeof (line), f))
  75         break;
  76       if (strcmp (line, "[C99]\n") == 0)
  77         fl = C99;
  78       else if (strcmp (line, "[C99DIG]\n") == 0)
  79         fl = C99|N99;
  80       else if (strcmp (line, "[CXX]\n") == 0)
  81         fl = CXX;
  82       else if (strcmp (line, "[C11]\n") == 0)
  83         fl = C11;
  84       else if (strcmp (line, "[C11NOSTART]\n") == 0)
  85         fl = C11|N11;
  86       else if (isxdigit (line[0]))
  87         {
  88           char *l = line;
  89           while (*l)
  90             {
  91               unsigned long start, end;
  92               char *endptr;
  93               start = strtoul (l, &endptr, 16);
  94               if (endptr == l || (*endptr != '-' && ! isspace (*endptr)))
  95                 fail ("parsing ucnid.tab [1]");
  96               l = endptr;
  97               if (*l != '-')
  98                 end = start;
  99               else
 100                 {
 101                   end = strtoul (l + 1, &endptr, 16);
 102                   if (end < start)
 103                     fail ("parsing ucnid.tab, end before start");
 104                   l = endptr;
 105                   if (! isspace (*l))
 106                     fail ("parsing ucnid.tab, junk after range");
 107                 }
 108               while (isspace (*l))
 109                 l++;
 110               if (end > MAX_CODE_POINT)
 111                 fail ("parsing ucnid.tab, end too large");
 112               while (start <= end)
 113                 flags[start++] |= fl;
 114             }
 115         }
 116     }
 117   if (ferror (f))
 118     fail ("reading ucnid.tab");
 119   fclose (f);
 120 }
 121
 122 /* Read UnicodeData.txt and fill in the 'decomp' table to be the
 123    decompositions of characters for which both the character
 124    decomposed and all the code points in the decomposition are valid
 125    for some supported language version, and the 'all_decomp' table to
 126    be the decompositions of all characters without those
 127    constraints.  */
 128
 129 static void
 130 read_table (char *fname)
 131 {
 132   FILE * f = fopen (fname, "r");
 133
 134   if (!f)
 135     fail ("opening UnicodeData.txt");
 136   for (;;)
 137     {
 138       char line[256];
 139       unsigned long codepoint, this_decomp[4];
 140       char *l;
 141       int i, j;
 142       int decomp_useful;
 143
 144       if (!fgets (line, sizeof (line), f))
 145         break;
 146       codepoint = strtoul (line, &l, 16);
 147       if (l == line || *l != ';')
 148         fail ("parsing UnicodeData.txt, reading code point");
 149       if (codepoint > MAX_CODE_POINT)
 150         fail ("parsing UnicodeData.txt, code point too large");
 151
 152       do {
 153         l++;
 154       } while (*l != ';');
 155       /* Category value.  */
 156       do {
 157         l++;
 158       } while (*l != ';');
 159       /* Canonical combining class; in NFC/NFKC, they must be increasing
 160          (or zero).  */
 161       if (! isdigit (*++l))
 162         fail ("parsing UnicodeData.txt, combining class not number");
 163       combining_value[codepoint] = strtoul (l, &l, 10);
 164       if (*l++ != ';')
 165         fail ("parsing UnicodeData.txt, junk after combining class");
 166
 167       /* Skip over bidi value.  */
 168       do {
 169         l++;
 170       } while (*l != ';');
 171
 172       /* Decomposition mapping.  */
 173       decomp_useful = flags[codepoint];
 174       if (*++l == '<')  /* Compatibility mapping. */
 175         continue;
 176       for (i = 0; i < 4; i++)
 177         {
 178           if (*l == ';')
 179             break;
 180           if (!isxdigit (*l))
 181             fail ("parsing UnicodeData.txt, decomposition format");
 182           this_decomp[i] = strtoul (l, &l, 16);
 183           decomp_useful &= flags[this_decomp[i]];
 184           while (isspace (*l))
 185             l++;
 186         }
 187       if (i > 2)  /* Decomposition too long.  */
 188         fail ("parsing UnicodeData.txt, decomposition too long");
 189       for (j = 0; j < i; j++)
 190         all_decomp[codepoint][j] = this_decomp[j];
 191       if ((flags[codepoint] & all_languages) && decomp_useful)
 192         while (--i >= 0)
 193           decomp[codepoint][i] = this_decomp[i];
 194     }
 195   if (ferror (f))
 196     fail ("reading UnicodeData.txt");
 197   fclose (f);
 198 }
 199
 200 /* Read DerivedNormalizationProps.txt and set the flags that say whether
 201    a character is in NFC, NFKC, or is context-dependent.  */
 202
 203 static void
 204 read_derived (const char *fname)
 205 {
 206   FILE * f = fopen (fname, "r");
 207
 208   if (!f)
 209     fail ("opening DerivedNormalizationProps.txt");
 210   for (;;)
 211     {
 212       char line[256];
 213       unsigned long start, end;
 214       char *l;
 215       bool not_NFC_p, not_NFKC_p, maybe_not_NFC_p;
 216
 217       if (!fgets (line, sizeof (line), f))
 218         break;
 219       not_NFC_p = (strstr (line, "; NFC_QC; N") != NULL);
 220       not_NFKC_p = (strstr (line, "; NFKC_QC; N") != NULL);
 221       maybe_not_NFC_p = (strstr (line, "; NFC_QC; M") != NULL);
 222       if (! not_NFC_p && ! not_NFKC_p && ! maybe_not_NFC_p)
 223         continue;
 224
 225       start = strtoul (line, &l, 16);
 226       if (l == line)
 227         fail ("parsing DerivedNormalizationProps.txt, reading start");
 228       if (start > MAX_CODE_POINT)
 229         fail ("parsing DerivedNormalizationProps.txt, code point too large");
 230       if (*l == '.' && l[1] == '.')
 231         end = strtoul (l + 2, &l, 16);
 232       else
 233         end = start;
 234
 235       while (start <= end)
 236         flags[start++] |= ((not_NFC_p ? not_NFC : 0)
 237                            | (not_NFKC_p ? not_NFKC : 0)
 238                            | (maybe_not_NFC_p ? maybe_not_NFC : 0)
 239                            );
 240     }
 241   if (ferror (f))
 242     fail ("reading DerivedNormalizationProps.txt");
 243   fclose (f);
 244 }
 245
 246 /* Read DerivedCoreProperties.txt and fill in languages version in
 247    flags from the XID_Start and XID_Continue properties.  */
 248
 249 static void
 250 read_derivedcore (char *fname)
 251 {
 252   FILE * f = fopen (fname, "r");
 253
 254   if (!f)
 255     fail ("opening DerivedCoreProperties.txt");
 256   for (;;)
 257     {
 258       char line[256];
 259       unsigned long codepoint_start, codepoint_end;
 260       char *l;
 261       int i, j;
 262
 263       if (!fgets (line, sizeof (line), f))
 264         break;
 265       if (line[0] == '#' || line[0] == '\n' || line[0] == '\r')
 266         continue;
 267       codepoint_start = strtoul (line, &l, 16);
 268       if (l == line)
 269         fail ("parsing DerivedCoreProperties.txt, reading code point");
 270       if (codepoint_start > MAX_CODE_POINT)
 271         fail ("parsing DerivedCoreProperties.txt, code point too large");
 272
 273       if (*l == '.' && l[1] == '.')
 274         {
 275           char *l2 = l + 2;
 276           codepoint_end = strtoul (l + 2, &l, 16);
 277           if (l == l2 || codepoint_end < codepoint_start)
 278             fail ("parsing DerivedCoreProperties.txt, reading code point");
 279           if (codepoint_end > MAX_CODE_POINT)
 280             fail ("parsing DerivedCoreProperties.txt, code point too large");
 281         }
 282       else
 283         codepoint_end = codepoint_start;
 284
 285       while (*l == ' ')
 286         l++;
 287       if (*l++ != ';')
 288         fail ("parsing DerivedCoreProperties.txt, reading code point");
 289
 290       while (*l == ' ')
 291         l++;
 292
 293       if (codepoint_end < 0x80)
 294         continue;
 295
 296       if (strncmp (l, "XID_Start ", 10) == 0)
 297         {
 298           for (; codepoint_start <= codepoint_end; codepoint_start++)
 299             flags[codepoint_start]
 300               = (flags[codepoint_start] | CXX23) & ~NXX23;
 301         }
 302       else if (strncmp (l, "XID_Continue ", 13) == 0)
 303         {
 304           for (; codepoint_start <= codepoint_end; codepoint_start++)
 305             if ((flags[codepoint_start] & CXX23) == 0)
 306               flags[codepoint_start] |= CXX23 | NXX23;
 307         }
 308     }
 309   if (ferror (f))
 310     fail ("reading DerivedCoreProperties.txt");
 311   fclose (f);
 312 }
 313
 314 /* Write out the table.
 315    The table consists of two words per entry.  The first word is the flags
 316    for the unicode code points up to and including the second word.  */
 317
 318 static void
 319 write_table (void)
 320 {
 321   unsigned i;
 322   unsigned last_flag = flags[0];
 323   bool really_safe = decomp[0][0] == 0;
 324   unsigned char last_combine = combining_value[0];
 325
 326   printf ("static const struct ucnrange ucnranges[] = {\n");
 327
 328   for (i = 1; i <= NUM_CODE_POINTS; i++)
 329     if (i == NUM_CODE_POINTS
 330         || (flags[i] != last_flag && ((flags[i] | last_flag) & all_languages))
 331         || really_safe != (decomp[i][0] == 0)
 332         || combining_value[i] != last_combine)
 333       {
 334         printf ("{ %s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s, %3d, %#06x },\n",
 335                 last_flag & C99 ? "C99" : "  0",
 336                 last_flag & N99 ? "N99" : "  0",
 337                 last_flag & CXX ? "CXX" : "  0",
 338                 last_flag & C11 ? "C11" : "  0",
 339                 last_flag & N11 ? "N11" : "  0",
 340                 last_flag & CXX23 ? "CXX23" : "    0",
 341                 last_flag & NXX23 ? "NXX23" : "    0",
 342                 really_safe ? "CID" : "  0",
 343                 last_flag & not_NFC ? "  0" : "NFC",
 344                 last_flag & not_NFKC ? "  0" : "NKC",
 345                 last_flag & maybe_not_NFC ? "CTX" : "  0",
 346                 combining_value[i - 1],
 347                 i - 1);
 348         last_flag = flags[i];
 349         last_combine = combining_value[i];
 350         really_safe = decomp[i][0] == 0;
 351       }
 352
 353   printf ("};\n");
 354 }
 355
 356 /* Return whether a given character is valid in an identifier for some
 357    supported language, either as itself or as a UCN.  */
 358
 359 static bool
 360 char_id_valid (unsigned int c)
 361 {
 362   return ((flags[c] & all_languages)
 363           || (c == 0x24)
 364           || (c >= 0x30 && c <= 0x39)
 365           || (c >= 0x41 && c <= 0x5a)
 366           || (c >= 0x61 && c <= 0x7a));
 367 }
 368
 369 /* Write out the switch statement over characters for which it is
 370    context-dependent whether they are in NFC.  */
 371
 372 static void
 373 write_context_switch (void)
 374 {
 375   unsigned i;
 376   printf ("static bool\n"
 377           "check_nfc (cpp_reader *pfile, cppchar_t c, cppchar_t p)\n"
 378           "{\n"
 379           "  switch (c)\n"
 380           "    {\n");
 381   for (i = 0; i < NUM_CODE_POINTS; i++)
 382     {
 383       bool found_case = false;
 384       unsigned j;
 385       if (!(flags[i] & all_languages) || !(flags[i] & maybe_not_NFC))
 386         continue;
 387       if ((i >= 0x1161 && i <= 0x1175) || (i >= 0x11A8 && i <= 0x11C2))
 388         continue; /* Hangul handled algorithmically.  */
 389       printf ("    case %#06x:\n"
 390               "      switch (p)\n"
 391               "\t{\n", i);
 392       /* If an NFC starter character decomposes with this character I
 393          as the second character and an NFC starter character S as the
 394          first character, that latter character as a previous
 395          character means this character is not NFC.  Furthermore, any
 396          NFC starter character K made by a series of compositions of S
 397          with combining characters whose combining class is greater
 398          than that of I also means this character is not NFC.  */
 399       for (j = 0; j < NUM_CODE_POINTS; j++)
 400         {
 401           unsigned s, k;
 402           if (all_decomp[j][1] != i)
 403             continue;
 404           s = all_decomp[j][0];
 405           if (combining_value[s] != 0 || (flags[s] & not_NFC) != 0)
 406             continue;
 407           if (char_id_valid (s))
 408             {
 409               found_case = true;
 410               printf ("\tcase %#06x:\n", s);
 411             }
 412           for (k = 0; k < NUM_CODE_POINTS; k++)
 413             {
 414               unsigned t = k;
 415               if (k == s || !char_id_valid (k))
 416                 continue;
 417               while (all_decomp[t][1] != 0
 418                      && combining_value[all_decomp[t][1]] > combining_value[i])
 419                 {
 420                   if (combining_value[t] != 0 || (flags[t] & not_NFC) != 0)
 421                     break;
 422                   t = all_decomp[t][0];
 423                 }
 424               if (t == s)
 425                 {
 426                   found_case = true;
 427                   printf ("\tcase %#06x:\n", k);
 428                 }
 429             }
 430         }
 431       if (found_case)
 432         printf ("\t  return false;\n");
 433       else
 434         printf ("\t/* Non-NFC cases not applicable to C/C++.  */\n");
 435       printf ("\tdefault:\n"
 436               "\t  return true;\n"
 437               "\t}\n\n");
 438     }
 439   printf ("    default:\n"
 440           "      cpp_error (pfile, CPP_DL_ICE, \"Character %%x might not be NFKC\", c);\n"
 441           "      return true;\n"
 442           "  }\n"
 443           "}\n");
 444 }
 445
 446 /* Print out the huge copyright notice.  */
 447
 448 static void
 449 write_copyright (void)
 450 {
 451   static const char copyright[] = "\
 452 /* Unicode characters and various properties.\n\
 453    Copyright (C) 2003-2023 Free Software Foundation, Inc.\n\
 454 \n\
 455    This program is free software; you can redistribute it and/or modify it\n\
 456    under the terms of the GNU General Public License as published by the\n\
 457    Free Software Foundation; either version 3, or (at your option) any\n\
 458    later version.\n\
 459 \n\
 460    This program is distributed in the hope that it will be useful,\n\
 461    but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
 462    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n\
 463    GNU General Public License for more details.\n\
 464 \n\
 465    You should have received a copy of the GNU General Public License\n\
 466    along with this program; see the file COPYING3.  If not see\n\
 467    <http://www.gnu.org/licenses/>.\n\
 468 \n\
 469 \n\
 470    Copyright (C) 1991-2022 Unicode, Inc.  All rights reserved.\n\
 471    Distributed under the Terms of Use in\n\
 472    http://www.unicode.org/copyright.html.\n\
 473 \n\
 474    Permission is hereby granted, free of charge, to any person\n\
 475    obtaining a copy of the Unicode data files and any associated\n\
 476    documentation (the \"Data Files\") or Unicode software and any\n\
 477    associated documentation (the \"Software\") to deal in the Data Files\n\
 478    or Software without restriction, including without limitation the\n\
 479    rights to use, copy, modify, merge, publish, distribute, and/or\n\
 480    sell copies of the Data Files or Software, and to permit persons to\n\
 481    whom the Data Files or Software are furnished to do so, provided\n\
 482    that (a) the above copyright notice(s) and this permission notice\n\
 483    appear with all copies of the Data Files or Software, (b) both the\n\
 484    above copyright notice(s) and this permission notice appear in\n\
 485    associated documentation, and (c) there is clear notice in each\n\
 486    modified Data File or in the Software as well as in the\n\
 487    documentation associated with the Data File(s) or Software that the\n\
 488    data or software has been modified.\n\
 489 \n\
 490    THE DATA FILES AND SOFTWARE ARE PROVIDED \"AS IS\", WITHOUT WARRANTY\n\
 491    OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE\n\
 492    WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n\
 493    NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE\n\
 494    COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR\n\
 495    ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY\n\
 496    DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,\n\
 497    WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS\n\
 498    ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE\n\
 499    OF THE DATA FILES OR SOFTWARE.\n\
 500 \n\
 501    Except as contained in this notice, the name of a copyright holder\n\
 502    shall not be used in advertising or otherwise to promote the sale,\n\
 503    use or other dealings in these Data Files or Software without prior\n\
 504    written authorization of the copyright holder.  */\n";
 505
 506    puts (copyright);
 507 }
 508
 509 /* Main program.  */
 510
 511 int
 512 main(int argc, char ** argv)
 513 {
 514   if (argc != 5)
 515     fail ("too few arguments to makeucn");
 516   read_ucnid (argv[1]);
 517   read_table (argv[2]);
 518   read_derived (argv[3]);
 519   read_derivedcore (argv[4]);
 520
 521   write_copyright ();
 522   write_table ();
 523   write_context_switch ();
 524   return 0;
 525 }