fedora/build-locale-archive.c

   1 #define _GNU_SOURCE
   2 #include <assert.h>
   3 #include <dirent.h>
   4 #include <errno.h>
   5 #include <fcntl.h>
   6 #include <locale.h>
   7 #include <stdarg.h>
   8 #include <stdbool.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <sys/mman.h>
  13 #include <sys/stat.h>
  14 #include <unistd.h>
  15 #include "../locale/hashval.h"
  16 #define __LC_LAST 13
  17 #include "../locale/locarchive.h"
  18 #include "../crypt/md5.h"
  19
  20 const char *alias_file = DATADIR "/locale/locale.alias";
  21 const char *locar_file = PREFIX "/lib/locale/locale-archive";
  22 const char *tmpl_file = PREFIX "/lib/locale/locale-archive.tmpl";
  23 const char *loc_path = PREFIX "/lib/locale/";
  24 int be_quiet = 1;
  25 int verbose = 0;
  26 int max_locarchive_open_retry = 10;
  27 const char *output_prefix;
  28
  29 static const char *locnames[] =
  30   {
  31 #define DEFINE_CATEGORY(category, category_name, items, a) \
  32   [category] = category_name,
  33 #include "../locale/categories.def"
  34 #undef  DEFINE_CATEGORY
  35   };
  36
  37 static int
  38 is_prime (unsigned long candidate)
  39 {
  40   /* No even number and none less than 10 will be passed here.  */
  41   unsigned long int divn = 3;
  42   unsigned long int sq = divn * divn;
  43
  44   while (sq < candidate && candidate % divn != 0)
  45     {
  46       ++divn;
  47       sq += 4 * divn;
  48       ++divn;
  49     }
  50
  51   return candidate % divn != 0;
  52 }
  53
  54 unsigned long
  55 next_prime (unsigned long seed)
  56 {
  57   /* Make it definitely odd.  */
  58   seed |= 1;
  59
  60   while (!is_prime (seed))
  61     seed += 2;
  62
  63   return seed;
  64 }
  65
  66 /* xmalloc is only used by show_archive_content.  */
  67 void *
  68 xmalloc (size_t size)
  69 {
  70   (void) size;
  71   exit (255);
  72 }
  73
  74 void
  75 error (int status, int errnum, const char *message, ...)
  76 {
  77   va_list args;
  78
  79   va_start (args, message);
  80   fflush (stdout);
  81   fprintf (stderr, "%s: ", program_invocation_name);
  82   vfprintf (stderr, message, args);
  83   va_end (args);
  84   if (errnum)
  85     fprintf (stderr, ": %s", strerror (errnum));
  86   putc ('\n', stderr);
  87   fflush (stderr);
  88   if (status)
  89     exit (errnum == EROFS ? 0 : status);
  90 }
  91
  92 static void
  93 open_tmpl_archive (struct locarhandle *ah)
  94 {
  95   struct stat64 st;
  96   int fd;
  97   struct locarhead head;
  98   const char *archivefname = tmpl_file;
  99
 100   /* Open the archive.  We must have exclusive write access.  */
 101   fd = open64 (archivefname, O_RDONLY);
 102   if (fd == -1)
 103     error (EXIT_FAILURE, errno, "cannot open locale archive template file \"%s\"",
 104            archivefname);
 105
 106   if (fstat64 (fd, &st) < 0)
 107     error (EXIT_FAILURE, errno, "cannot stat locale archive template file \"%s\"",
 108            archivefname);
 109
 110   /* Read the header.  */
 111   if (TEMP_FAILURE_RETRY (read (fd, &head, sizeof (head))) != sizeof (head))
 112     error (EXIT_FAILURE, errno, "cannot read archive header");
 113
 114   ah->fd = fd;
 115   ah->len = (head.sumhash_offset
 116              + head.sumhash_size * sizeof (struct sumhashent));
 117   if (ah->len > st.st_size)
 118     error (EXIT_FAILURE, 0, "locale archite template file truncated");
 119   ah->len = st.st_size;
 120
 121   /* Now we know how large the administrative information part is.
 122      Map all of it.  */
 123   ah->addr = mmap64 (NULL, ah->len, PROT_READ, MAP_SHARED, fd, 0);
 124   if (ah->addr == MAP_FAILED)
 125     error (EXIT_FAILURE, errno, "cannot map archive header");
 126 }
 127
 128 /* Open the locale archive.  */
 129 extern void open_archive (struct locarhandle *ah, bool readonly);
 130
 131 /* Close the locale archive.  */
 132 extern void close_archive (struct locarhandle *ah);
 133
 134 /* Add given locale data to the archive.  */
 135 extern int add_locale_to_archive (struct locarhandle *ah, const char *name,
 136                                   locale_data_t data, bool replace);
 137
 138 extern void add_alias (struct locarhandle *ah, const char *alias,
 139                        bool replace, const char *oldname,
 140                        uint32_t *locrec_offset_p);
 141
 142 extern struct namehashent *
 143 insert_name (struct locarhandle *ah,
 144              const char *name, size_t name_len, bool replace);
 145
 146 struct nameent
 147 {
 148   char *name;
 149   struct locrecent *locrec;
 150 };
 151
 152 struct dataent
 153 {
 154   const unsigned char *sum;
 155   uint32_t file_offset;
 156 };
 157
 158 static int
 159 nameentcmp (const void *a, const void *b)
 160 {
 161   struct locrecent *la = ((const struct nameent *) a)->locrec;
 162   struct locrecent *lb = ((const struct nameent *) b)->locrec;
 163   uint32_t start_a = -1, end_a = 0;
 164   uint32_t start_b = -1, end_b = 0;
 165   int cnt;
 166
 167   for (cnt = 0; cnt < __LC_LAST; ++cnt)
 168     if (cnt != LC_ALL)
 169       {
 170         if (la->record[cnt].offset < start_a)
 171           start_a = la->record[cnt].offset;
 172         if (la->record[cnt].offset + la->record[cnt].len > end_a)
 173           end_a = la->record[cnt].offset + la->record[cnt].len;
 174       }
 175   assert (start_a != (uint32_t)-1);
 176   assert (end_a != 0);
 177
 178   for (cnt = 0; cnt < __LC_LAST; ++cnt)
 179     if (cnt != LC_ALL)
 180       {
 181         if (lb->record[cnt].offset < start_b)
 182           start_b = lb->record[cnt].offset;
 183         if (lb->record[cnt].offset + lb->record[cnt].len > end_b)
 184           end_b = lb->record[cnt].offset + lb->record[cnt].len;
 185       }
 186   assert (start_b != (uint32_t)-1);
 187   assert (end_b != 0);
 188
 189   if (start_a != start_b)
 190     return (int)start_a - (int)start_b;
 191   return (int)end_a - (int)end_b;
 192 }
 193
 194 static int
 195 dataentcmp (const void *a, const void *b)
 196 {
 197   if (((const struct dataent *) a)->file_offset
 198       < ((const struct dataent *) b)->file_offset)
 199     return -1;
 200
 201   if (((const struct dataent *) a)->file_offset
 202       > ((const struct dataent *) b)->file_offset)
 203     return 1;
 204
 205   return 0;
 206 }
 207
 208 static int
 209 sumsearchfn (const void *key, const void *ent)
 210 {
 211   uint32_t keyn = *(uint32_t *)key;
 212   uint32_t entn = ((struct dataent *)ent)->file_offset;
 213
 214   if (keyn < entn)
 215     return -1;
 216   if (keyn > entn)
 217     return 1;
 218   return 0;
 219 }
 220
 221 static void
 222 compute_data (struct locarhandle *ah, struct nameent *name, size_t sumused,
 223               struct dataent *files, locale_data_t data)
 224 {
 225   int cnt;
 226   struct locrecent *locrec = name->locrec;
 227   struct dataent *file;
 228   data[LC_ALL].addr = ((char *) ah->addr) + locrec->record[LC_ALL].offset;
 229   data[LC_ALL].size = locrec->record[LC_ALL].len;
 230   for (cnt = 0; cnt < __LC_LAST; ++cnt)
 231     if (cnt != LC_ALL)
 232       {
 233         data[cnt].addr = ((char *) ah->addr) + locrec->record[cnt].offset;
 234         data[cnt].size = locrec->record[cnt].len;
 235         if (data[cnt].addr >= data[LC_ALL].addr
 236             && data[cnt].addr + data[cnt].size
 237                <= data[LC_ALL].addr + data[LC_ALL].size)
 238           __md5_buffer (data[cnt].addr, data[cnt].size, data[cnt].sum);
 239         else
 240           {
 241             file = bsearch (&locrec->record[cnt].offset, files, sumused,
 242                             sizeof (*files), sumsearchfn);
 243             if (file == NULL)
 244               error (EXIT_FAILURE, 0, "inconsistent template file");
 245             memcpy (data[cnt].sum, file->sum, sizeof (data[cnt].sum));
 246           }
 247       }
 248 }
 249
 250 static int
 251 fill_archive (struct locarhandle *tmpl_ah, size_t nlist, char *list[],
 252               const char *primary)
 253 {
 254   struct locarhandle ah;
 255   struct locarhead *head;
 256   int result = 0;
 257   struct nameent *names;
 258   struct namehashent *namehashtab;
 259   size_t cnt, used;
 260   struct dataent *files;
 261   struct sumhashent *sumhashtab;
 262   size_t sumused;
 263   struct locrecent *primary_locrec = NULL;
 264   struct nameent *primary_nameent = NULL;
 265
 266   head = tmpl_ah->addr;
 267   names = (struct nameent *) malloc (head->namehash_used
 268                                      * sizeof (struct nameent));
 269   files = (struct dataent *) malloc (head->sumhash_used
 270                                      * sizeof (struct dataent));
 271   if (names == NULL || files == NULL)
 272     error (EXIT_FAILURE, errno, "could not allocate tables");
 273
 274   namehashtab = (struct namehashent *) ((char *) tmpl_ah->addr
 275                                         + head->namehash_offset);
 276   sumhashtab = (struct sumhashent *) ((char *) tmpl_ah->addr
 277                                       + head->sumhash_offset);
 278
 279   for (cnt = used = 0; cnt < head->namehash_size; ++cnt)
 280     if (namehashtab[cnt].locrec_offset != 0)
 281       {
 282         assert (used < head->namehash_used);
 283         names[used].name = tmpl_ah->addr + namehashtab[cnt].name_offset;
 284         names[used++].locrec
 285           = (struct locrecent *) ((char *) tmpl_ah->addr +
 286                                   namehashtab[cnt].locrec_offset);
 287       }
 288
 289   /* Sort the names.  */
 290   qsort (names, used, sizeof (struct nameent), nameentcmp);
 291
 292   for (cnt = sumused = 0; cnt < head->sumhash_size; ++cnt)
 293     if (sumhashtab[cnt].file_offset != 0)
 294       {
 295         assert (sumused < head->sumhash_used);
 296         files[sumused].sum = (const unsigned char *) sumhashtab[cnt].sum;
 297         files[sumused++].file_offset = sumhashtab[cnt].file_offset;
 298       }
 299
 300   /* Sort by file locations.  */
 301   qsort (files, sumused, sizeof (struct dataent), dataentcmp);
 302
 303   /* Open the archive.  This call never returns if we cannot
 304      successfully open the archive.  */
 305   open_archive (&ah, false);
 306
 307   if (primary != NULL)
 308     {
 309       for (cnt = 0; cnt < used; ++cnt)
 310         if (strcmp (names[cnt].name, primary) == 0)
 311           break;
 312       if (cnt < used)
 313         {
 314           locale_data_t data;
 315
 316           compute_data (tmpl_ah, &names[cnt], sumused, files, data);
 317           result |= add_locale_to_archive (&ah, primary, data, 0);
 318           primary_locrec = names[cnt].locrec;
 319           primary_nameent = &names[cnt];
 320         }
 321     }
 322
 323   for (cnt = 0; cnt < used; ++cnt)
 324     if (&names[cnt] == primary_nameent)
 325       continue;
 326     else if ((cnt > 0 && names[cnt - 1].locrec == names[cnt].locrec)
 327              || names[cnt].locrec == primary_locrec)
 328       {
 329         const char *oldname;
 330         struct namehashent *namehashent;
 331         uint32_t locrec_offset;
 332
 333         if (names[cnt].locrec == primary_locrec)
 334           oldname = primary;
 335         else
 336           oldname = names[cnt - 1].name;
 337         namehashent = insert_name (&ah, oldname, strlen (oldname), true);
 338         assert (namehashent->name_offset != 0);
 339         assert (namehashent->locrec_offset != 0);
 340         locrec_offset = namehashent->locrec_offset;
 341         add_alias (&ah, names[cnt].name, 0, oldname, &locrec_offset);
 342       }
 343     else
 344       {
 345         locale_data_t data;
 346
 347         compute_data (tmpl_ah, &names[cnt], sumused, files, data);
 348         result |= add_locale_to_archive (&ah, names[cnt].name, data, 0);
 349       }
 350
 351   while (nlist-- > 0)
 352     {
 353       const char *fname = *list++;
 354       size_t fnamelen = strlen (fname);
 355       struct stat64 st;
 356       DIR *dirp;
 357       struct dirent64 *d;
 358       int seen;
 359       locale_data_t data;
 360       int cnt;
 361
 362       /* First see whether this really is a directory and whether it
 363          contains all the require locale category files.  */
 364       if (stat64 (fname, &st) < 0)
 365         {
 366           error (0, 0, "stat of \"%s\" failed: %s: ignored", fname,
 367                  strerror (errno));
 368           continue;
 369         }
 370       if (!S_ISDIR (st.st_mode))
 371         {
 372           error (0, 0, "\"%s\" is no directory; ignored", fname);
 373           continue;
 374         }
 375
 376       dirp = opendir (fname);
 377       if (dirp == NULL)
 378         {
 379           error (0, 0, "cannot open directory \"%s\": %s: ignored",
 380                  fname, strerror (errno));
 381           continue;
 382         }
 383
 384       seen = 0;
 385       while ((d = readdir64 (dirp)) != NULL)
 386         {
 387           for (cnt = 0; cnt < __LC_LAST; ++cnt)
 388             if (cnt != LC_ALL)
 389               if (strcmp (d->d_name, locnames[cnt]) == 0)
 390                 {
 391                   unsigned char d_type;
 392
 393                   /* We have an object of the required name.  If it's
 394                      a directory we have to look at a file with the
 395                      prefix "SYS_".  Otherwise we have found what we
 396                      are looking for.  */
 397 #ifdef _DIRENT_HAVE_D_TYPE
 398                   d_type = d->d_type;
 399
 400                   if (d_type != DT_REG)
 401 #endif
 402                     {
 403                       char fullname[fnamelen + 2 * strlen (d->d_name) + 7];
 404
 405 #ifdef _DIRENT_HAVE_D_TYPE
 406                       if (d_type == DT_UNKNOWN)
 407 #endif
 408                         {
 409                           strcpy (stpcpy (stpcpy (fullname, fname), "/"),
 410                                   d->d_name);
 411
 412                           if (stat64 (fullname, &st) == -1)
 413                             /* We cannot stat the file, ignore it.  */
 414                             break;
 415
 416                           d_type = IFTODT (st.st_mode);
 417                         }
 418
 419                       if (d_type == DT_DIR)
 420                         {
 421                           /* We have to do more tests.  The file is a
 422                              directory and it therefore must contain a
 423                              regular file with the same name except a
 424                              "SYS_" prefix.  */
 425                           char *t = stpcpy (stpcpy (fullname, fname), "/");
 426                           strcpy (stpcpy (stpcpy (t, d->d_name), "/SYS_"),
 427                                   d->d_name);
 428
 429                           if (stat64 (fullname, &st) == -1)
 430                             /* There is no SYS_* file or we cannot
 431                                access it.  */
 432                             break;
 433
 434                           d_type = IFTODT (st.st_mode);
 435                         }
 436                     }
 437
 438                   /* If we found a regular file (eventually after
 439                      following a symlink) we are successful.  */
 440                   if (d_type == DT_REG)
 441                     ++seen;
 442                   break;
 443                 }
 444         }
 445
 446       closedir (dirp);
 447
 448       if (seen != __LC_LAST - 1)
 449         {
 450           /* We don't have all locale category files.  Ignore the name.  */
 451           error (0, 0, "incomplete set of locale files in \"%s\"",
 452                  fname);
 453           continue;
 454         }
 455
 456       /* Add the files to the archive.  To do this we first compute
 457          sizes and the MD5 sums of all the files.  */
 458       for (cnt = 0; cnt < __LC_LAST; ++cnt)
 459         if (cnt != LC_ALL)
 460           {
 461             char fullname[fnamelen + 2 * strlen (locnames[cnt]) + 7];
 462             int fd;
 463
 464             strcpy (stpcpy (stpcpy (fullname, fname), "/"), locnames[cnt]);
 465             fd = open64 (fullname, O_RDONLY);
 466             if (fd == -1 || fstat64 (fd, &st) == -1)
 467               {
 468                 /* Cannot read the file.  */
 469                 if (fd != -1)
 470                   close (fd);
 471                 break;
 472               }
 473
 474             if (S_ISDIR (st.st_mode))
 475               {
 476                 char *t;
 477                 close (fd);
 478                 t = stpcpy (stpcpy (fullname, fname), "/");
 479                 strcpy (stpcpy (stpcpy (t, locnames[cnt]), "/SYS_"),
 480                         locnames[cnt]);
 481
 482                 fd = open64 (fullname, O_RDONLY);
 483                 if (fd == -1 || fstat64 (fd, &st) == -1
 484                     || !S_ISREG (st.st_mode))
 485                   {
 486                     if (fd != -1)
 487                       close (fd);
 488                     break;
 489                   }
 490               }
 491
 492             /* Map the file.  */
 493             data[cnt].addr = mmap64 (NULL, st.st_size, PROT_READ, MAP_SHARED,
 494                                      fd, 0);
 495             if (data[cnt].addr == MAP_FAILED)
 496               {
 497                 /* Cannot map it.  */
 498                 close (fd);
 499                 break;
 500               }
 501
 502             data[cnt].size = st.st_size;
 503             __md5_buffer (data[cnt].addr, st.st_size, data[cnt].sum);
 504
 505             /* We don't need the file descriptor anymore.  */
 506             close (fd);
 507           }
 508
 509       if (cnt != __LC_LAST)
 510         {
 511           while (cnt-- > 0)
 512             if (cnt != LC_ALL)
 513               munmap (data[cnt].addr, data[cnt].size);
 514
 515           error (0, 0, "cannot read all files in \"%s\": ignored", fname);
 516
 517           continue;
 518         }
 519
 520       result |= add_locale_to_archive (&ah, basename (fname), data, 0);
 521
 522       for (cnt = 0; cnt < __LC_LAST; ++cnt)
 523         if (cnt != LC_ALL)
 524           munmap (data[cnt].addr, data[cnt].size);
 525     }
 526
 527   /* We are done.  */
 528   close_archive (&ah);
 529
 530   return result;
 531 }
 532
 533 int main ()
 534 {
 535   char path[4096];
 536   DIR *dirp;
 537   struct dirent64 *d;
 538   struct stat64 st;
 539   char *list[16384], *primary;
 540   unsigned int cnt = 0;
 541   struct locarhandle tmpl_ah;
 542   size_t loc_path_len = strlen (loc_path);
 543
 544   dirp = opendir (loc_path);
 545   if (dirp == NULL)
 546     error (EXIT_FAILURE, errno, "cannot open directory \"%s\"", loc_path);
 547
 548   open_tmpl_archive (&tmpl_ah);
 549
 550   unlink (locar_file);
 551   primary = getenv ("LC_ALL");
 552   if (primary == NULL)
 553     primary = getenv ("LANG");
 554   if (primary != NULL)
 555     {
 556       if (strncmp (primary, "ja", 2) != 0
 557           && strncmp (primary, "ko", 2) != 0
 558           && strncmp (primary, "zh", 2) != 0)
 559         {
 560           char *ptr = malloc (strlen (primary) + strlen (".utf8") + 1), *p, *q;
 561
 562           if (ptr)
 563             {
 564               p = ptr;
 565               q = primary;
 566               while (*q && *q != '.' && *q != '@')
 567                 *p++ = *q++;
 568               if (*q == '.')
 569                 while (*q && *q != '@')
 570                   q++;
 571               p = stpcpy (p, ".utf8");
 572               strcpy (p, q);
 573               primary = ptr;
 574             }
 575           else
 576             primary = NULL;
 577         }
 578     }
 579
 580   memcpy (path, loc_path, loc_path_len);
 581
 582   while ((d = readdir64 (dirp)) != NULL)
 583     {
 584       if (strcmp (d->d_name, ".") == 0 || strcmp (d->d_name, "..") == 0)
 585         continue;
 586
 587       size_t d_name_len = strlen (d->d_name);
 588       if (loc_path_len + d_name_len + 1 > sizeof (path))
 589         {
 590           error (0, 0, "too long filename \"%s\"", d->d_name);
 591           continue;
 592         }
 593
 594       memcpy (path + loc_path_len, d->d_name, d_name_len + 1);
 595       if (stat64 (path, &st) < 0)
 596         {
 597           error (0, errno, "cannot stat \"%s\"", path);
 598           continue;
 599         }
 600       if (! S_ISDIR (st.st_mode))
 601         continue;
 602       if (cnt == 16384)
 603         error (EXIT_FAILURE, 0, "too many directories in \"%s\"", loc_path);
 604       list[cnt] = strdup (path);
 605       if (list[cnt] == NULL)
 606         {
 607           error (0, errno, "cannot add file to list \"%s\"", path);
 608           continue;
 609         }
 610       if (primary != NULL && cnt > 0 && strcmp (primary, d->d_name) == 0)
 611         {
 612           char *p = list[0];
 613           list[0] = list[cnt];
 614           list[cnt] = p;
 615         }
 616       cnt++;
 617     }
 618   closedir (dirp);
 619   fill_archive (&tmpl_ah, cnt, list, primary);
 620   close_archive (&tmpl_ah);
 621   truncate (tmpl_file, 0);
 622   char *argv[] = { "/usr/sbin/tzdata-update", NULL };
 623   execve (argv[0], (char *const *)argv, (char *const *)&argv[1]);
 624   exit (0);
 625 }