locale/programs/localedef.c

   1 /* Copyright (C) 1995-2020 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1995.
   4
   5    This program is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published
   7    by the Free Software Foundation; version 2 of the License, or
   8    (at your option) any later version.
   9
  10    This program is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    You should have received a copy of the GNU General Public License
  16    along with this program; if not, see <https://www.gnu.org/licenses/>.  */
  17
  18 #ifdef HAVE_CONFIG_H
  19 # include <config.h>
  20 #endif
  21
  22 #include <argp.h>
  23 #include <errno.h>
  24 #include <fcntl.h>
  25 #include <libintl.h>
  26 #include <locale.h>
  27 #include <stdbool.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <unistd.h>
  32 #include <error.h>
  33 #include <sys/mman.h>
  34 #include <sys/stat.h>
  35 #include <ctype.h>
  36
  37 #include "localedef.h"
  38 #include "charmap.h"
  39 #include "locfile.h"
  40
  41 /* Undefine the following line in the production version.  */
  42 /* #define NDEBUG 1 */
  43 #include <assert.h>
  44
  45
  46 /* List of copied locales.  */
  47 struct copy_def_list_t *copy_list;
  48
  49 /* If this is defined be POSIX conform.  */
  50 int posix_conformance;
  51
  52 /* If not zero force output even if warning were issued.  */
  53 static int force_output;
  54
  55 /* Prefix for output files.  */
  56 const char *output_prefix;
  57
  58 /* Name of the character map file.  */
  59 static const char *charmap_file;
  60
  61 /* Name of the locale definition file.  */
  62 static const char *input_file;
  63
  64 /* Name of the repertoire map file.  */
  65 const char *repertoire_global;
  66
  67 /* Name of the locale.alias file.  */
  68 const char *alias_file;
  69
  70 /* List of all locales.  */
  71 static struct localedef_t *locales;
  72
  73 /* If true don't add locale data to archive.  */
  74 bool no_archive;
  75
  76 /* If true add named locales to archive.  */
  77 static bool add_to_archive;
  78
  79 /* If true delete named locales from archive.  */
  80 static bool delete_from_archive;
  81
  82 /* If true replace archive content when adding.  */
  83 static bool replace_archive;
  84
  85 /* If true list archive content.  */
  86 static bool list_archive;
  87
  88 /* If true create hard links to other locales (default).  */
  89 bool hard_links = true;
  90
  91 /* Maximum number of retries when opening the locale archive.  */
  92 int max_locarchive_open_retry = 10;
  93
  94
  95 /* Name and version of program.  */
  96 static void print_version (FILE *stream, struct argp_state *state);
  97 void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
  98
  99 #define OPT_POSIX 301
 100 #define OPT_QUIET 302
 101 #define OPT_PREFIX 304
 102 #define OPT_NO_ARCHIVE 305
 103 #define OPT_ADD_TO_ARCHIVE 306
 104 #define OPT_REPLACE 307
 105 #define OPT_DELETE_FROM_ARCHIVE 308
 106 #define OPT_LIST_ARCHIVE 309
 107 #define OPT_LITTLE_ENDIAN 400
 108 #define OPT_BIG_ENDIAN 401
 109 #define OPT_NO_WARN 402
 110 #define OPT_WARN 403
 111 #define OPT_NO_HARD_LINKS 404
 112
 113 /* Definitions of arguments for argp functions.  */
 114 static const struct argp_option options[] =
 115 {
 116   { NULL, 0, NULL, 0, N_("Input Files:") },
 117   { "charmap", 'f', N_("FILE"), 0,
 118     N_("Symbolic character names defined in FILE") },
 119   { "inputfile", 'i', N_("FILE"), 0,
 120     N_("Source definitions are found in FILE") },
 121   { "repertoire-map", 'u', N_("FILE"), 0,
 122     N_("FILE contains mapping from symbolic names to UCS4 values") },
 123
 124   { NULL, 0, NULL, 0, N_("Output control:") },
 125   { "force", 'c', NULL, 0,
 126     N_("Create output even if warning messages were issued") },
 127   { "no-hard-links", OPT_NO_HARD_LINKS, NULL, 0,
 128     N_("Do not create hard links between installed locales") },
 129   { "prefix", OPT_PREFIX, N_("PATH"), 0, N_("Optional output file prefix") },
 130   { "posix", OPT_POSIX, NULL, 0, N_("Strictly conform to POSIX") },
 131   { "quiet", OPT_QUIET, NULL, 0,
 132     N_("Suppress warnings and information messages") },
 133   { "verbose", 'v', NULL, 0, N_("Print more messages") },
 134   { "no-warnings", OPT_NO_WARN, N_("<warnings>"), 0,
 135     N_("Comma-separated list of warnings to disable; "
 136        "supported warnings are: ascii, intcurrsym") },
 137   { "warnings", OPT_WARN, N_("<warnings>"), 0,
 138     N_("Comma-separated list of warnings to enable; "
 139        "supported warnings are: ascii, intcurrsym") },
 140
 141   { NULL, 0, NULL, 0, N_("Archive control:") },
 142   { "no-archive", OPT_NO_ARCHIVE, NULL, 0,
 143     N_("Don't add new data to archive") },
 144   { "add-to-archive", OPT_ADD_TO_ARCHIVE, NULL, 0,
 145     N_("Add locales named by parameters to archive") },
 146   { "replace", OPT_REPLACE, NULL, 0, N_("Replace existing archive content") },
 147   { "delete-from-archive", OPT_DELETE_FROM_ARCHIVE, NULL, 0,
 148     N_("Remove locales named by parameters from archive") },
 149   { "list-archive", OPT_LIST_ARCHIVE, NULL, 0, N_("List content of archive") },
 150   { "alias-file", 'A', N_("FILE"), 0,
 151     N_("locale.alias file to consult when making archive")},
 152   { "little-endian", OPT_LITTLE_ENDIAN, NULL, 0,
 153     N_("Generate little-endian output") },
 154   { "big-endian", OPT_BIG_ENDIAN, NULL, 0,
 155     N_("Generate big-endian output") },
 156   { NULL, 0, NULL, 0, NULL }
 157 };
 158
 159 /* Short description of program.  */
 160 static const char doc[] = N_("Compile locale specification");
 161
 162 /* Strings for arguments in help texts.  */
 163 static const char args_doc[] = N_("\
 164 NAME\n\
 165 [--add-to-archive|--delete-from-archive] FILE...\n\
 166 --list-archive [FILE]");
 167
 168 /* Prototype for option handler.  */
 169 static error_t parse_opt (int key, char *arg, struct argp_state *state);
 170
 171 /* Function to print some extra text in the help message.  */
 172 static char *more_help (int key, const char *text, void *input);
 173
 174 /* Data structure to communicate with argp functions.  */
 175 static struct argp argp =
 176 {
 177   options, parse_opt, args_doc, doc, NULL, more_help
 178 };
 179
 180
 181 /* Prototypes for local functions.  */
 182 static void error_print (void);
 183 static char *construct_output_path (char *path);
 184 static char *normalize_codeset (const char *codeset, size_t name_len);
 185
 186
 187 int
 188 main (int argc, char *argv[])
 189 {
 190   char *output_path;
 191   int cannot_write_why;
 192   struct charmap_t *charmap;
 193   struct localedef_t global;
 194   int remaining;
 195
 196   /* Set initial values for global variables.  */
 197   copy_list = NULL;
 198   posix_conformance = getenv ("POSIXLY_CORRECT") != NULL;
 199   error_print_progname = error_print;
 200
 201   /* Set locale.  Do not set LC_ALL because the other categories must
 202      not be affected (according to POSIX.2).  */
 203   setlocale (LC_MESSAGES, "");
 204   setlocale (LC_CTYPE, "");
 205
 206   /* Initialize the message catalog.  */
 207   textdomain (_libc_intl_domainname);
 208
 209   /* Parse and process arguments.  */
 210   argp_err_exit_status = 4;
 211   argp_parse (&argp, argc, argv, 0, &remaining, NULL);
 212
 213   /* Handle a few special cases.  */
 214   if (list_archive)
 215     show_archive_content (remaining > 1 ? argv[remaining] : NULL, verbose);
 216   if (add_to_archive)
 217     return add_locales_to_archive (argc - remaining, &argv[remaining],
 218                                    replace_archive);
 219   if (delete_from_archive)
 220     return delete_locales_from_archive (argc - remaining, &argv[remaining]);
 221
 222   /* POSIX.2 requires to be verbose about missing characters in the
 223      character map.  */
 224   verbose |= posix_conformance;
 225
 226   if (argc - remaining != 1)
 227     {
 228       /* We need exactly one non-option parameter.  */
 229       argp_help (&argp, stdout, ARGP_HELP_SEE | ARGP_HELP_EXIT_ERR,
 230                  program_invocation_short_name);
 231       exit (4);
 232     }
 233
 234   /* The parameter describes the output path of the constructed files.
 235      If the described files cannot be written return a NULL pointer.
 236      We don't free output_path because we will exit.  */
 237   output_path  = construct_output_path (argv[remaining]);
 238   if (output_path == NULL && ! no_archive)
 239     error (4, errno, _("cannot create directory for output files"));
 240   cannot_write_why = errno;
 241
 242   /* Now that the parameters are processed we have to reset the local
 243      ctype locale.  (P1003.2 4.35.5.2)  */
 244   setlocale (LC_CTYPE, "POSIX");
 245
 246   /* Look whether the system really allows locale definitions.  POSIX
 247      defines error code 3 for this situation so I think it must be
 248      a fatal error (see P1003.2 4.35.8).  */
 249   if (sysconf (_SC_2_LOCALEDEF) < 0)
 250     record_error (3, 0, _("\
 251 FATAL: system does not define `_POSIX2_LOCALEDEF'"));
 252
 253   /* Process charmap file.  */
 254   charmap = charmap_read (charmap_file, verbose, 1, be_quiet, 1);
 255
 256   /* Add the first entry in the locale list.  */
 257   memset (&global, '\0', sizeof (struct localedef_t));
 258   global.name = input_file ?: "/dev/stdin";
 259   global.needed = ALL_LOCALES;
 260   locales = &global;
 261
 262   /* Now read the locale file.  */
 263   if (locfile_read (&global, charmap) != 0)
 264     record_error (4, errno, _("\
 265 cannot open locale definition file `%s'"), input_file);
 266
 267   /* Perhaps we saw some `copy' instructions.  */
 268   while (1)
 269     {
 270       struct localedef_t *runp = locales;
 271
 272       while (runp != NULL && (runp->needed & runp->avail) == runp->needed)
 273         runp = runp->next;
 274
 275       if (runp == NULL)
 276         /* Everything read.  */
 277         break;
 278
 279       if (locfile_read (runp, charmap) != 0)
 280         record_error (4, errno, _("\
 281 cannot open locale definition file `%s'"), runp->name);
 282     }
 283
 284   /* Check the categories we processed in source form.  */
 285   check_all_categories (locales, charmap);
 286
 287   /* What we do next depends on the number of errors and warnings we
 288      have generated in processing the input files.
 289
 290      * No errors: Write the output file.
 291
 292      * Some warnings: Write the output file and exit with status 1 to
 293      indicate there may be problems using the output file e.g. missing
 294      data that makes it difficult to use
 295
 296      * Errors: We don't write the output file and we exit with status 4
 297      to indicate no output files were written.
 298
 299      The use of -c|--force writes the output file even if errors were
 300      seen.  */
 301   if (recorded_error_count == 0 || force_output != 0)
 302     {
 303       if (cannot_write_why != 0)
 304         record_error (4, cannot_write_why, _("\
 305 cannot write output files to `%s'"), output_path ? : argv[remaining]);
 306       else
 307         write_all_categories (locales, charmap, argv[remaining], output_path);
 308     }
 309   else
 310     record_error (4, 0, _("\
 311 no output file produced because errors were issued"));
 312
 313   /* This exit status is prescribed by POSIX.2 4.35.7.  */
 314   exit (recorded_warning_count != 0);
 315 }
 316
 317 /* Search warnings for matching warnings and if found enable those
 318    warnings if ENABLED is true, otherwise disable the warnings.  */
 319 static void
 320 set_warnings (char *warnings, bool enabled)
 321 {
 322   char *tok = warnings;
 323   char *copy = (char *) malloc (strlen (warnings) + 1);
 324   char *save = copy;
 325
 326   /* As we make a copy of the warnings list we remove all spaces from
 327      the warnings list to make the processing a more robust.  We don't
 328      support spaces in a warning name.  */
 329   do
 330     {
 331       while (isspace (*tok) != 0)
 332         tok++;
 333     }
 334   while ((*save++ = *tok++) != '\0');
 335
 336   warnings = copy;
 337
 338   /* Tokenize the input list of warnings to set, compare them to
 339      known warnings, and set the warning.  We purposely ignore unknown
 340      warnings, and are thus forward compatible, users can attempt to
 341      disable whaterver new warnings they know about, but we will only
 342      disable those *we* known about.  */
 343   while ((tok = strtok_r (warnings, ",", &save)) != NULL)
 344     {
 345       warnings = NULL;
 346       if (strcmp (tok, "ascii") == 0)
 347         warn_ascii = enabled;
 348       else if (strcmp (tok, "intcurrsym") == 0)
 349         warn_int_curr_symbol = enabled;
 350     }
 351
 352   free (copy);
 353 }
 354
 355 /* Handle program arguments.  */
 356 static error_t
 357 parse_opt (int key, char *arg, struct argp_state *state)
 358 {
 359   switch (key)
 360     {
 361     case OPT_QUIET:
 362       be_quiet = 1;
 363       break;
 364     case OPT_POSIX:
 365       posix_conformance = 1;
 366       break;
 367     case OPT_PREFIX:
 368       output_prefix = arg;
 369       break;
 370     case OPT_NO_ARCHIVE:
 371       no_archive = true;
 372       break;
 373     case OPT_ADD_TO_ARCHIVE:
 374       add_to_archive = true;
 375       break;
 376     case OPT_REPLACE:
 377       replace_archive = true;
 378       break;
 379     case OPT_DELETE_FROM_ARCHIVE:
 380       delete_from_archive = true;
 381       break;
 382     case OPT_LIST_ARCHIVE:
 383       list_archive = true;
 384       break;
 385     case OPT_LITTLE_ENDIAN:
 386       set_big_endian (false);
 387       break;
 388     case OPT_BIG_ENDIAN:
 389       set_big_endian (true);
 390       break;
 391     case OPT_NO_WARN:
 392       /* Disable the warnings.  */
 393       set_warnings (arg, false);
 394       break;
 395     case OPT_WARN:
 396       /* Enable the warnings.  */
 397       set_warnings (arg, true);
 398       break;
 399     case OPT_NO_HARD_LINKS:
 400       /* Do not hard link to other locales.  */
 401       hard_links = false;
 402       break;
 403     case 'c':
 404       force_output = 1;
 405       break;
 406     case 'f':
 407       charmap_file = arg;
 408       break;
 409     case 'A':
 410       alias_file = arg;
 411       break;
 412     case 'i':
 413       input_file = arg;
 414       break;
 415     case 'u':
 416       repertoire_global = arg;
 417       break;
 418     case 'v':
 419       verbose = 1;
 420       break;
 421     default:
 422       return ARGP_ERR_UNKNOWN;
 423     }
 424   return 0;
 425 }
 426
 427
 428 static char *
 429 more_help (int key, const char *text, void *input)
 430 {
 431   char *cp;
 432   char *tp;
 433
 434   switch (key)
 435     {
 436     case ARGP_KEY_HELP_EXTRA:
 437       /* We print some extra information.  */
 438       tp = xasprintf (gettext ("\
 439 For bug reporting instructions, please see:\n\
 440 %s.\n"), REPORT_BUGS_TO);
 441       cp = xasprintf (gettext ("\
 442 System's directory for character maps : %s\n\
 443                        repertoire maps: %s\n\
 444                        locale path    : %s\n\
 445 %s"),
 446                     CHARMAP_PATH, REPERTOIREMAP_PATH, LOCALE_PATH, tp);
 447       free (tp);
 448       return cp;
 449     default:
 450       break;
 451     }
 452   return (char *) text;
 453 }
 454
 455 /* Print the version information.  */
 456 static void
 457 print_version (FILE *stream, struct argp_state *state)
 458 {
 459   fprintf (stream, "localedef %s%s\n", PKGVERSION, VERSION);
 460   fprintf (stream, gettext ("\
 461 Copyright (C) %s Free Software Foundation, Inc.\n\
 462 This is free software; see the source for copying conditions.  There is NO\n\
 463 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
 464 "), "2020");
 465   fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
 466 }
 467
 468
 469 /* The address of this function will be assigned to the hook in the error
 470    functions.  */
 471 static void
 472 error_print (void)
 473 {
 474 }
 475
 476
 477 /* The parameter to localedef describes the output path.  If it does contain a
 478    '/' character it is a relative path.  Otherwise it names the locale this
 479    definition is for.   The returned path must be freed by the caller. */
 480 static char *
 481 construct_output_path (char *path)
 482 {
 483   char *result;
 484
 485   if (strchr (path, '/') == NULL)
 486     {
 487       /* This is a system path.  First examine whether the locale name
 488          contains a reference to the codeset.  This should be
 489          normalized.  */
 490       char *startp;
 491       char *endp = NULL;
 492       char *normal = NULL;
 493
 494       startp = path;
 495       /* Either we have a '@' which starts a CEN name or '.' which starts the
 496          codeset specification.  The CEN name starts with '@' and may also have
 497          a codeset specification, but we do not normalize the string after '@'.
 498          If we only find the codeset specification then we normalize only the codeset
 499          specification (but not anything after a subsequent '@').  */
 500       while (*startp != '\0' && *startp != '@' && *startp != '.')
 501         ++startp;
 502       if (*startp == '.')
 503         {
 504           /* We found a codeset specification.  Now find the end.  */
 505           endp = ++startp;
 506
 507           /* Stop at the first '@', and don't normalize anything past that.  */
 508           while (*endp != '\0' && *endp != '@')
 509             ++endp;
 510
 511           if (endp > startp)
 512             normal = normalize_codeset (startp, endp - startp);
 513         }
 514
 515       if (normal == NULL)
 516         result = xasprintf ("%s%s/%s/", output_prefix ?: "",
 517                             COMPLOCALEDIR, path);
 518       else
 519         result = xasprintf ("%s%s/%.*s%s%s/",
 520                             output_prefix ?: "", COMPLOCALEDIR,
 521                             (int) (startp - path), path, normal, endp ?: "");
 522       /* Free the allocated normalized codeset name.  */
 523       free (normal);
 524     }
 525   else
 526     {
 527       /* This is a user path.  */
 528       result = xasprintf ("%s/", path);
 529
 530       /* If the user specified an output path we cannot add the output
 531          to the archive.  */
 532       no_archive = true;
 533     }
 534
 535   errno = 0;
 536
 537   if (no_archive && euidaccess (result, W_OK) == -1)
 538     {
 539       /* Perhaps the directory does not exist now.  Try to create it.  */
 540       if (errno == ENOENT)
 541         {
 542           errno = 0;
 543           if (mkdir (result, 0777) < 0)
 544             {
 545               record_verbose (stderr,
 546                               _("cannot create output path \'%s\': %s"),
 547                               result, strerror (errno));
 548               free (result);
 549               return NULL;
 550             }
 551         }
 552       else
 553         record_verbose (stderr,
 554                         _("no write permission to output path \'%s\': %s"),
 555                         result, strerror (errno));
 556     }
 557
 558   return result;
 559 }
 560
 561
 562 /* Normalize codeset name.  There is no standard for the codeset names.
 563    Normalization allows the user to use any of the common names e.g. UTF-8,
 564    utf-8, utf8, UTF8 etc.
 565
 566    We normalize using the following rules:
 567    - Remove all non-alpha-numeric characters
 568    - Lowercase all characters.
 569    - If there are only digits assume it's an ISO standard and prefix with 'iso'
 570
 571    We return the normalized string which needs to be freed by free.  */
 572 static char *
 573 normalize_codeset (const char *codeset, size_t name_len)
 574 {
 575   int len = 0;
 576   int only_digit = 1;
 577   char *retval;
 578   char *wp;
 579   size_t cnt;
 580
 581   /* Compute the length of only the alpha-numeric characters.  */
 582   for (cnt = 0; cnt < name_len; ++cnt)
 583     if (isalnum (codeset[cnt]))
 584       {
 585         ++len;
 586
 587         if (isalpha (codeset[cnt]))
 588           only_digit = 0;
 589       }
 590
 591   /* If there were only digits we assume it's an ISO standard and we will
 592      prefix with 'iso' so include space for that.  We fill in the required
 593      space from codeset up to the converted length.  */
 594   wp = retval = xasprintf ("%s%.*s", only_digit ? "iso" : "", len, codeset);
 595
 596   /* Skip "iso".  */
 597   if (only_digit)
 598     wp += 3;
 599
 600   /* Lowercase all characters. */
 601   for (cnt = 0; cnt < name_len; ++cnt)
 602     if (isalpha (codeset[cnt]))
 603       *wp++ = tolower (codeset[cnt]);
 604     else if (isdigit (codeset[cnt]))
 605       *wp++ = codeset[cnt];
 606
 607   /* Return allocated and converted name for caller to free.  */
 608   return retval;
 609 }
 610
 611
 612 struct localedef_t *
 613 add_to_readlist (int category, const char *name, const char *repertoire_name,
 614                  int generate, struct localedef_t *copy_locale)
 615 {
 616   struct localedef_t *runp = locales;
 617
 618   while (runp != NULL && strcmp (name, runp->name) != 0)
 619     runp = runp->next;
 620
 621   if (runp == NULL)
 622     {
 623       /* Add a new entry at the end.  */
 624       struct localedef_t *newp;
 625
 626       assert (generate == 1);
 627
 628       newp = xcalloc (1, sizeof (struct localedef_t));
 629       newp->name = name;
 630       newp->repertoire_name = repertoire_name;
 631
 632       if (locales == NULL)
 633         runp = locales = newp;
 634       else
 635         {
 636           runp = locales;
 637           while (runp->next != NULL)
 638             runp = runp->next;
 639           runp = runp->next = newp;
 640         }
 641     }
 642
 643   if (generate
 644       && (runp->needed & (1 << category)) != 0
 645       && (runp->avail & (1 << category)) == 0)
 646     record_error (5, 0, _("\
 647 circular dependencies between locale definitions"));
 648
 649   if (copy_locale != NULL)
 650     {
 651       if (runp->categories[category].generic != NULL)
 652         record_error (5, 0, _("\
 653 cannot add already read locale `%s' a second time"), name);
 654       else
 655         runp->categories[category].generic =
 656           copy_locale->categories[category].generic;
 657     }
 658
 659   runp->needed |= 1 << category;
 660
 661   return runp;
 662 }
 663
 664
 665 struct localedef_t *
 666 find_locale (int category, const char *name, const char *repertoire_name,
 667              const struct charmap_t *charmap)
 668 {
 669   struct localedef_t *result;
 670
 671   /* Find the locale, but do not generate it since this would be a bug.  */
 672   result = add_to_readlist (category, name, repertoire_name, 0, NULL);
 673
 674   assert (result != NULL);
 675
 676   if ((result->avail & (1 << category)) == 0
 677       && locfile_read (result, charmap) != 0)
 678     record_error (4, errno, _("\
 679 cannot open locale definition file `%s'"), result->name);
 680
 681   return result;
 682 }
 683
 684
 685 struct localedef_t *
 686 load_locale (int category, const char *name, const char *repertoire_name,
 687              const struct charmap_t *charmap, struct localedef_t *copy_locale)
 688 {
 689   struct localedef_t *result;
 690
 691   /* Generate the locale if it does not exist.  */
 692   result = add_to_readlist (category, name, repertoire_name, 1, copy_locale);
 693
 694   assert (result != NULL);
 695
 696   if ((result->avail & (1 << category)) == 0
 697       && locfile_read (result, charmap) != 0)
 698     record_error (4, errno, _("\
 699 cannot open locale definition file `%s'"), result->name);
 700
 701   return result;
 702 }