fedora/build-locale-archive.c

   1 #define _GNU_SOURCE
   2 #include <assert.h>
   3 #include <dirent.h>
   4 #include <errno.h>
   5 #include <fcntl.h>
   6 #include <locale.h>
   7 #include <stdarg.h>
   8 #include <stdbool.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <sys/mman.h>
  13 #include <sys/stat.h>
  14 #include <unistd.h>
  15 #include "../locale/hashval.h"
  16 #define __LC_LAST 13
  17 #include "../locale/locarchive.h"
  18 #include "../crypt/md5.h"
  19
  20 const char *alias_file = DATADIR "/locale/locale.alias";
  21 const char *locar_file = PREFIX "/lib/locale/locale-archive";
  22 const char *tmpl_file = PREFIX "/lib/locale/locale-archive.tmpl";
  23 const char *loc_path = PREFIX "/lib/locale/";
  24 int be_quiet = 1;
  25 int verbose = 0;
  26 int max_locarchive_open_retry = 10;
  27 const char *output_prefix;
  28
  29 static const char *locnames[] =
  30   {
  31 #define DEFINE_CATEGORY(category, category_name, items, a) \
  32   [category] = category_name,
  33 #include "../locale/categories.def"
  34 #undef  DEFINE_CATEGORY
  35   };
  36
  37 static int
  38 is_prime (unsigned long candidate)
  39 {
  40   /* No even number and none less than 10 will be passed here.  */
  41   unsigned long int divn = 3;
  42   unsigned long int sq = divn * divn;
  43
  44   while (sq < candidate && candidate % divn != 0)
  45     {
  46       ++divn;
  47       sq += 4 * divn;
  48       ++divn;
  49     }
  50
  51   return candidate % divn != 0;
  52 }
  53
  54 unsigned long
  55 next_prime (unsigned long seed)
  56 {
  57   /* Make it definitely odd.  */
  58   seed |= 1;
  59
  60   while (!is_prime (seed))
  61     seed += 2;
  62
  63   return seed;
  64 }
  65
  66 void
  67 error (int status, int errnum, const char *message, ...)
  68 {
  69   va_list args;
  70
  71   va_start (args, message);
  72   fflush (stdout);
  73   fprintf (stderr, "%s: ", program_invocation_name);
  74   vfprintf (stderr, message, args);
  75   va_end (args);
  76   if (errnum)
  77     fprintf (stderr, ": %s", strerror (errnum));
  78   putc ('\n', stderr);
  79   fflush (stderr);
  80   if (status)
  81     exit (errnum == EROFS ? 0 : status);
  82 }
  83
  84 void *
  85 xmalloc (size_t size)
  86 {
  87   void *p = malloc (size);
  88   if (p == NULL)
  89     error (EXIT_FAILURE, errno, "could not allocate %zd bytes of memory", size);
  90   return p;
  91 }
  92
  93 static void
  94 open_tmpl_archive (struct locarhandle *ah)
  95 {
  96   struct stat64 st;
  97   int fd;
  98   struct locarhead head;
  99   const char *archivefname = tmpl_file;
 100
 101   /* Open the archive.  We must have exclusive write access.  */
 102   fd = open64 (archivefname, O_RDONLY);
 103   if (fd == -1)
 104     error (EXIT_FAILURE, errno, "cannot open locale archive template file \"%s\"",
 105            archivefname);
 106
 107   if (fstat64 (fd, &st) < 0)
 108     error (EXIT_FAILURE, errno, "cannot stat locale archive template file \"%s\"",
 109            archivefname);
 110
 111   /* Read the header.  */
 112   if (TEMP_FAILURE_RETRY (read (fd, &head, sizeof (head))) != sizeof (head))
 113     error (EXIT_FAILURE, errno, "cannot read archive header");
 114
 115   ah->fd = fd;
 116   ah->mmaped = (head.sumhash_offset
 117                 + head.sumhash_size * sizeof (struct sumhashent));
 118   if (ah->mmaped > (unsigned long) st.st_size)
 119     error (EXIT_FAILURE, 0, "locale archite template file truncated");
 120   ah->mmaped = st.st_size;
 121   ah->reserved = st.st_size;
 122
 123   /* Now we know how large the administrative information part is.
 124      Map all of it.  */
 125   ah->addr = mmap64 (NULL, ah->mmaped, PROT_READ, MAP_SHARED, fd, 0);
 126   if (ah->addr == MAP_FAILED)
 127     error (EXIT_FAILURE, errno, "cannot map archive header");
 128 }
 129
 130 /* Open the locale archive.  */
 131 extern void open_archive (struct locarhandle *ah, bool readonly);
 132
 133 /* Close the locale archive.  */
 134 extern void close_archive (struct locarhandle *ah);
 135
 136 /* Add given locale data to the archive.  */
 137 extern int add_locale_to_archive (struct locarhandle *ah, const char *name,
 138                                   locale_data_t data, bool replace);
 139
 140 extern void add_alias (struct locarhandle *ah, const char *alias,
 141                        bool replace, const char *oldname,
 142                        uint32_t *locrec_offset_p);
 143
 144 extern struct namehashent *
 145 insert_name (struct locarhandle *ah,
 146              const char *name, size_t name_len, bool replace);
 147
 148 struct nameent
 149 {
 150   char *name;
 151   struct locrecent *locrec;
 152 };
 153
 154 struct dataent
 155 {
 156   const unsigned char *sum;
 157   uint32_t file_offset;
 158 };
 159
 160 static int
 161 nameentcmp (const void *a, const void *b)
 162 {
 163   struct locrecent *la = ((const struct nameent *) a)->locrec;
 164   struct locrecent *lb = ((const struct nameent *) b)->locrec;
 165   uint32_t start_a = -1, end_a = 0;
 166   uint32_t start_b = -1, end_b = 0;
 167   int cnt;
 168
 169   for (cnt = 0; cnt < __LC_LAST; ++cnt)
 170     if (cnt != LC_ALL)
 171       {
 172         if (la->record[cnt].offset < start_a)
 173           start_a = la->record[cnt].offset;
 174         if (la->record[cnt].offset + la->record[cnt].len > end_a)
 175           end_a = la->record[cnt].offset + la->record[cnt].len;
 176       }
 177   assert (start_a != (uint32_t)-1);
 178   assert (end_a != 0);
 179
 180   for (cnt = 0; cnt < __LC_LAST; ++cnt)
 181     if (cnt != LC_ALL)
 182       {
 183         if (lb->record[cnt].offset < start_b)
 184           start_b = lb->record[cnt].offset;
 185         if (lb->record[cnt].offset + lb->record[cnt].len > end_b)
 186           end_b = lb->record[cnt].offset + lb->record[cnt].len;
 187       }
 188   assert (start_b != (uint32_t)-1);
 189   assert (end_b != 0);
 190
 191   if (start_a != start_b)
 192     return (int)start_a - (int)start_b;
 193   return (int)end_a - (int)end_b;
 194 }
 195
 196 static int
 197 dataentcmp (const void *a, const void *b)
 198 {
 199   if (((const struct dataent *) a)->file_offset
 200       < ((const struct dataent *) b)->file_offset)
 201     return -1;
 202
 203   if (((const struct dataent *) a)->file_offset
 204       > ((const struct dataent *) b)->file_offset)
 205     return 1;
 206
 207   return 0;
 208 }
 209
 210 static int
 211 sumsearchfn (const void *key, const void *ent)
 212 {
 213   uint32_t keyn = *(uint32_t *)key;
 214   uint32_t entn = ((struct dataent *)ent)->file_offset;
 215
 216   if (keyn < entn)
 217     return -1;
 218   if (keyn > entn)
 219     return 1;
 220   return 0;
 221 }
 222
 223 static void
 224 compute_data (struct locarhandle *ah, struct nameent *name, size_t sumused,
 225               struct dataent *files, locale_data_t data)
 226 {
 227   int cnt;
 228   struct locrecent *locrec = name->locrec;
 229   struct dataent *file;
 230   data[LC_ALL].addr = ((char *) ah->addr) + locrec->record[LC_ALL].offset;
 231   data[LC_ALL].size = locrec->record[LC_ALL].len;
 232   for (cnt = 0; cnt < __LC_LAST; ++cnt)
 233     if (cnt != LC_ALL)
 234       {
 235         data[cnt].addr = ((char *) ah->addr) + locrec->record[cnt].offset;
 236         data[cnt].size = locrec->record[cnt].len;
 237         if (data[cnt].addr >= data[LC_ALL].addr
 238             && data[cnt].addr + data[cnt].size
 239                <= data[LC_ALL].addr + data[LC_ALL].size)
 240           __md5_buffer (data[cnt].addr, data[cnt].size, data[cnt].sum);
 241         else
 242           {
 243             file = bsearch (&locrec->record[cnt].offset, files, sumused,
 244                             sizeof (*files), sumsearchfn);
 245             if (file == NULL)
 246               error (EXIT_FAILURE, 0, "inconsistent template file");
 247             memcpy (data[cnt].sum, file->sum, sizeof (data[cnt].sum));
 248           }
 249       }
 250 }
 251
 252 static int
 253 fill_archive (struct locarhandle *tmpl_ah, size_t nlist, char *list[],
 254               const char *primary)
 255 {
 256   struct locarhandle ah;
 257   struct locarhead *head;
 258   int result = 0;
 259   struct nameent *names;
 260   struct namehashent *namehashtab;
 261   size_t cnt, used;
 262   struct dataent *files;
 263   struct sumhashent *sumhashtab;
 264   size_t sumused;
 265   struct locrecent *primary_locrec = NULL;
 266   struct nameent *primary_nameent = NULL;
 267
 268   head = tmpl_ah->addr;
 269   names = (struct nameent *) malloc (head->namehash_used
 270                                      * sizeof (struct nameent));
 271   files = (struct dataent *) malloc (head->sumhash_used
 272                                      * sizeof (struct dataent));
 273   if (names == NULL || files == NULL)
 274     error (EXIT_FAILURE, errno, "could not allocate tables");
 275
 276   namehashtab = (struct namehashent *) ((char *) tmpl_ah->addr
 277                                         + head->namehash_offset);
 278   sumhashtab = (struct sumhashent *) ((char *) tmpl_ah->addr
 279                                       + head->sumhash_offset);
 280
 281   for (cnt = used = 0; cnt < head->namehash_size; ++cnt)
 282     if (namehashtab[cnt].locrec_offset != 0)
 283       {
 284         assert (used < head->namehash_used);
 285         names[used].name = tmpl_ah->addr + namehashtab[cnt].name_offset;
 286         names[used++].locrec
 287           = (struct locrecent *) ((char *) tmpl_ah->addr +
 288                                   namehashtab[cnt].locrec_offset);
 289       }
 290
 291   /* Sort the names.  */
 292   qsort (names, used, sizeof (struct nameent), nameentcmp);
 293
 294   for (cnt = sumused = 0; cnt < head->sumhash_size; ++cnt)
 295     if (sumhashtab[cnt].file_offset != 0)
 296       {
 297         assert (sumused < head->sumhash_used);
 298         files[sumused].sum = (const unsigned char *) sumhashtab[cnt].sum;
 299         files[sumused++].file_offset = sumhashtab[cnt].file_offset;
 300       }
 301
 302   /* Sort by file locations.  */
 303   qsort (files, sumused, sizeof (struct dataent), dataentcmp);
 304
 305   /* Open the archive.  This call never returns if we cannot
 306      successfully open the archive.  */
 307   open_archive (&ah, false);
 308
 309   if (primary != NULL)
 310     {
 311       for (cnt = 0; cnt < used; ++cnt)
 312         if (strcmp (names[cnt].name, primary) == 0)
 313           break;
 314       if (cnt < used)
 315         {
 316           locale_data_t data;
 317
 318           compute_data (tmpl_ah, &names[cnt], sumused, files, data);
 319           result |= add_locale_to_archive (&ah, primary, data, 0);
 320           primary_locrec = names[cnt].locrec;
 321           primary_nameent = &names[cnt];
 322         }
 323     }
 324
 325   for (cnt = 0; cnt < used; ++cnt)
 326     if (&names[cnt] == primary_nameent)
 327       continue;
 328     else if ((cnt > 0 && names[cnt - 1].locrec == names[cnt].locrec)
 329              || names[cnt].locrec == primary_locrec)
 330       {
 331         const char *oldname;
 332         struct namehashent *namehashent;
 333         uint32_t locrec_offset;
 334
 335         if (names[cnt].locrec == primary_locrec)
 336           oldname = primary;
 337         else
 338           oldname = names[cnt - 1].name;
 339         namehashent = insert_name (&ah, oldname, strlen (oldname), true);
 340         assert (namehashent->name_offset != 0);
 341         assert (namehashent->locrec_offset != 0);
 342         locrec_offset = namehashent->locrec_offset;
 343         add_alias (&ah, names[cnt].name, 0, oldname, &locrec_offset);
 344       }
 345     else
 346       {
 347         locale_data_t data;
 348
 349         compute_data (tmpl_ah, &names[cnt], sumused, files, data);
 350         result |= add_locale_to_archive (&ah, names[cnt].name, data, 0);
 351       }
 352
 353   while (nlist-- > 0)
 354     {
 355       const char *fname = *list++;
 356       size_t fnamelen = strlen (fname);
 357       struct stat64 st;
 358       DIR *dirp;
 359       struct dirent64 *d;
 360       int seen;
 361       locale_data_t data;
 362       int cnt;
 363
 364       /* First see whether this really is a directory and whether it
 365          contains all the require locale category files.  */
 366       if (stat64 (fname, &st) < 0)
 367         {
 368           error (0, 0, "stat of \"%s\" failed: %s: ignored", fname,
 369                  strerror (errno));
 370           continue;
 371         }
 372       if (!S_ISDIR (st.st_mode))
 373         {
 374           error (0, 0, "\"%s\" is no directory; ignored", fname);
 375           continue;
 376         }
 377
 378       dirp = opendir (fname);
 379       if (dirp == NULL)
 380         {
 381           error (0, 0, "cannot open directory \"%s\": %s: ignored",
 382                  fname, strerror (errno));
 383           continue;
 384         }
 385
 386       seen = 0;
 387       while ((d = readdir64 (dirp)) != NULL)
 388         {
 389           for (cnt = 0; cnt < __LC_LAST; ++cnt)
 390             if (cnt != LC_ALL)
 391               if (strcmp (d->d_name, locnames[cnt]) == 0)
 392                 {
 393                   unsigned char d_type;
 394
 395                   /* We have an object of the required name.  If it's
 396                      a directory we have to look at a file with the
 397                      prefix "SYS_".  Otherwise we have found what we
 398                      are looking for.  */
 399 #ifdef _DIRENT_HAVE_D_TYPE
 400                   d_type = d->d_type;
 401
 402                   if (d_type != DT_REG)
 403 #endif
 404                     {
 405                       char fullname[fnamelen + 2 * strlen (d->d_name) + 7];
 406
 407 #ifdef _DIRENT_HAVE_D_TYPE
 408                       if (d_type == DT_UNKNOWN)
 409 #endif
 410                         {
 411                           strcpy (stpcpy (stpcpy (fullname, fname), "/"),
 412                                   d->d_name);
 413
 414                           if (stat64 (fullname, &st) == -1)
 415                             /* We cannot stat the file, ignore it.  */
 416                             break;
 417
 418                           d_type = IFTODT (st.st_mode);
 419                         }
 420
 421                       if (d_type == DT_DIR)
 422                         {
 423                           /* We have to do more tests.  The file is a
 424                              directory and it therefore must contain a
 425                              regular file with the same name except a
 426                              "SYS_" prefix.  */
 427                           char *t = stpcpy (stpcpy (fullname, fname), "/");
 428                           strcpy (stpcpy (stpcpy (t, d->d_name), "/SYS_"),
 429                                   d->d_name);
 430
 431                           if (stat64 (fullname, &st) == -1)
 432                             /* There is no SYS_* file or we cannot
 433                                access it.  */
 434                             break;
 435
 436                           d_type = IFTODT (st.st_mode);
 437                         }
 438                     }
 439
 440                   /* If we found a regular file (eventually after
 441                      following a symlink) we are successful.  */
 442                   if (d_type == DT_REG)
 443                     ++seen;
 444                   break;
 445                 }
 446         }
 447
 448       closedir (dirp);
 449
 450       if (seen != __LC_LAST - 1)
 451         {
 452           /* We don't have all locale category files.  Ignore the name.  */
 453           error (0, 0, "incomplete set of locale files in \"%s\"",
 454                  fname);
 455           continue;
 456         }
 457
 458       /* Add the files to the archive.  To do this we first compute
 459          sizes and the MD5 sums of all the files.  */
 460       for (cnt = 0; cnt < __LC_LAST; ++cnt)
 461         if (cnt != LC_ALL)
 462           {
 463             char fullname[fnamelen + 2 * strlen (locnames[cnt]) + 7];
 464             int fd;
 465
 466             strcpy (stpcpy (stpcpy (fullname, fname), "/"), locnames[cnt]);
 467             fd = open64 (fullname, O_RDONLY);
 468             if (fd == -1 || fstat64 (fd, &st) == -1)
 469               {
 470                 /* Cannot read the file.  */
 471                 if (fd != -1)
 472                   close (fd);
 473                 break;
 474               }
 475
 476             if (S_ISDIR (st.st_mode))
 477               {
 478                 char *t;
 479                 close (fd);
 480                 t = stpcpy (stpcpy (fullname, fname), "/");
 481                 strcpy (stpcpy (stpcpy (t, locnames[cnt]), "/SYS_"),
 482                         locnames[cnt]);
 483
 484                 fd = open64 (fullname, O_RDONLY);
 485                 if (fd == -1 || fstat64 (fd, &st) == -1
 486                     || !S_ISREG (st.st_mode))
 487                   {
 488                     if (fd != -1)
 489                       close (fd);
 490                     break;
 491                   }
 492               }
 493
 494             /* Map the file.  */
 495             data[cnt].addr = mmap64 (NULL, st.st_size, PROT_READ, MAP_SHARED,
 496                                      fd, 0);
 497             if (data[cnt].addr == MAP_FAILED)
 498               {
 499                 /* Cannot map it.  */
 500                 close (fd);
 501                 break;
 502               }
 503
 504             data[cnt].size = st.st_size;
 505             __md5_buffer (data[cnt].addr, st.st_size, data[cnt].sum);
 506
 507             /* We don't need the file descriptor anymore.  */
 508             close (fd);
 509           }
 510
 511       if (cnt != __LC_LAST)
 512         {
 513           while (cnt-- > 0)
 514             if (cnt != LC_ALL)
 515               munmap (data[cnt].addr, data[cnt].size);
 516
 517           error (0, 0, "cannot read all files in \"%s\": ignored", fname);
 518
 519           continue;
 520         }
 521
 522       result |= add_locale_to_archive (&ah, basename (fname), data, 0);
 523
 524       for (cnt = 0; cnt < __LC_LAST; ++cnt)
 525         if (cnt != LC_ALL)
 526           munmap (data[cnt].addr, data[cnt].size);
 527     }
 528
 529   /* We are done.  */
 530   close_archive (&ah);
 531
 532   return result;
 533 }
 534
 535 int main ()
 536 {
 537   char path[4096];
 538   DIR *dirp;
 539   struct dirent64 *d;
 540   struct stat64 st;
 541   char *list[16384], *primary;
 542   unsigned int cnt = 0;
 543   struct locarhandle tmpl_ah;
 544   size_t loc_path_len = strlen (loc_path);
 545
 546   dirp = opendir (loc_path);
 547   if (dirp == NULL)
 548     error (EXIT_FAILURE, errno, "cannot open directory \"%s\"", loc_path);
 549
 550   open_tmpl_archive (&tmpl_ah);
 551
 552   unlink (locar_file);
 553   primary = getenv ("LC_ALL");
 554   if (primary == NULL)
 555     primary = getenv ("LANG");
 556   if (primary != NULL)
 557     {
 558       if (strncmp (primary, "ja", 2) != 0
 559           && strncmp (primary, "ko", 2) != 0
 560           && strncmp (primary, "zh", 2) != 0)
 561         {
 562           char *ptr = malloc (strlen (primary) + strlen (".utf8") + 1), *p, *q;
 563
 564           if (ptr)
 565             {
 566               p = ptr;
 567               q = primary;
 568               while (*q && *q != '.' && *q != '@')
 569                 *p++ = *q++;
 570               if (*q == '.')
 571                 while (*q && *q != '@')
 572                   q++;
 573               p = stpcpy (p, ".utf8");
 574               strcpy (p, q);
 575               primary = ptr;
 576             }
 577           else
 578             primary = NULL;
 579         }
 580     }
 581
 582   memcpy (path, loc_path, loc_path_len);
 583
 584   while ((d = readdir64 (dirp)) != NULL)
 585     {
 586       if (strcmp (d->d_name, ".") == 0 || strcmp (d->d_name, "..") == 0)
 587         continue;
 588
 589       size_t d_name_len = strlen (d->d_name);
 590       if (loc_path_len + d_name_len + 1 > sizeof (path))
 591         {
 592           error (0, 0, "too long filename \"%s\"", d->d_name);
 593           continue;
 594         }
 595
 596       memcpy (path + loc_path_len, d->d_name, d_name_len + 1);
 597       if (stat64 (path, &st) < 0)
 598         {
 599           error (0, errno, "cannot stat \"%s\"", path);
 600           continue;
 601         }
 602       if (! S_ISDIR (st.st_mode))
 603         continue;
 604       if (cnt == 16384)
 605         error (EXIT_FAILURE, 0, "too many directories in \"%s\"", loc_path);
 606       list[cnt] = strdup (path);
 607       if (list[cnt] == NULL)
 608         {
 609           error (0, errno, "cannot add file to list \"%s\"", path);
 610           continue;
 611         }
 612       if (primary != NULL && cnt > 0 && strcmp (primary, d->d_name) == 0)
 613         {
 614           char *p = list[0];
 615           list[0] = list[cnt];
 616           list[cnt] = p;
 617         }
 618       cnt++;
 619     }
 620   closedir (dirp);
 621   fill_archive (&tmpl_ah, cnt, list, primary);
 622   close_archive (&tmpl_ah);
 623   truncate (tmpl_file, 0);
 624   char *argv[] = { "/usr/sbin/tzdata-update", NULL };
 625   execve (argv[0], (char *const *)argv, (char *const *)&argv[1]);
 626   exit (0);
 627 }