tools/wmc/po.c

   1 /*
   2  * Support for po files
   3  *
   4  * Copyright 2010, 2011 Alexandre Julliard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  */
  20
  21 #include "config.h"
  22
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <stdarg.h>
  27 #include <assert.h>
  28 #include <ctype.h>
  29 #ifdef HAVE_GETTEXT_PO_H
  30 #include <gettext-po.h>
  31 #endif
  32
  33 #include "wmc.h"
  34 #include "utils.h"
  35 #include "lang.h"
  36 #include "write.h"
  37 #include "windef.h"
  38
  39 struct mo_file
  40 {
  41     unsigned int magic;
  42     unsigned int revision;
  43     unsigned int count;
  44     unsigned int msgid_off;
  45     unsigned int msgstr_off;
  46     /* ... rest of file data here */
  47 };
  48
  49 static struct lan_blk *new_top, *new_tail;
  50
  51 static BOOL is_english( int lan )
  52 {
  53     return lan == MAKELANGID( LANG_ENGLISH, SUBLANG_DEFAULT );
  54 }
  55
  56 static char *convert_msgid_ascii( const struct lanmsg *msg, int error_on_invalid_char )
  57 {
  58     int i;
  59     char *buffer = xmalloc( msg->len * 4 + 1 );
  60
  61     for (i = 0; i < msg->len; i++)
  62     {
  63         buffer[i] = msg->msg[i];
  64         if (!msg->msg[i]) break;
  65         if (msg->msg[i] >= 32 && msg->msg[i] <= 127) continue;
  66         if (msg->msg[i] == '\t' || msg->msg[i] == '\n') continue;
  67         if (error_on_invalid_char)
  68         {
  69             fprintf( stderr, "%s:%d: ", msg->file, msg->line );
  70             error( "Invalid character %04x in source string\n", msg->msg[i] );
  71         }
  72         free( buffer );
  73         return NULL;
  74     }
  75     buffer[i] = 0;
  76     return buffer;
  77 }
  78
  79 static char *get_message_context( char **msgid )
  80 {
  81     static const char magic[] = "#msgctxt#";
  82     char *id, *context;
  83
  84     if (strncmp( *msgid, magic, sizeof(magic) - 1 )) return NULL;
  85     context = *msgid + sizeof(magic) - 1;
  86     if (!(id = strchr( context, '#' ))) return NULL;
  87     *id = 0;
  88     *msgid = id + 1;
  89     return context;
  90 }
  91
  92 #ifdef HAVE_LIBGETTEXTPO
  93
  94 static po_message_t find_message( po_file_t po, const char *msgid, const char *msgctxt,
  95                                   po_message_iterator_t *iterator )
  96 {
  97     po_message_t msg;
  98     const char *context;
  99
 100     *iterator = po_message_iterator( po, NULL );
 101     while ((msg = po_next_message( *iterator )))
 102     {
 103         if (strcmp( po_message_msgid( msg ), msgid )) continue;
 104         if (!msgctxt) break;
 105         if (!(context = po_message_msgctxt( msg ))) continue;
 106         if (!strcmp( context, msgctxt )) break;
 107     }
 108     return msg;
 109 }
 110
 111 static void po_xerror( int severity, po_message_t message,
 112                        const char *filename, size_t lineno, size_t column,
 113                        int multiline_p, const char *message_text )
 114 {
 115     fprintf( stderr, "%s:%u:%u: %s\n",
 116              filename, (unsigned int)lineno, (unsigned int)column, message_text );
 117     if (severity) exit(1);
 118 }
 119
 120 static void po_xerror2( int severity, po_message_t message1,
 121                         const char *filename1, size_t lineno1, size_t column1,
 122                         int multiline_p1, const char *message_text1,
 123                         po_message_t message2,
 124                         const char *filename2, size_t lineno2, size_t column2,
 125                         int multiline_p2, const char *message_text2 )
 126 {
 127     fprintf( stderr, "%s:%u:%u: %s\n",
 128              filename1, (unsigned int)lineno1, (unsigned int)column1, message_text1 );
 129     fprintf( stderr, "%s:%u:%u: %s\n",
 130              filename2, (unsigned int)lineno2, (unsigned int)column2, message_text2 );
 131     if (severity) exit(1);
 132 }
 133
 134 static const struct po_xerror_handler po_xerror_handler = { po_xerror, po_xerror2 };
 135
 136 static void add_po_string( po_file_t po, const struct lanmsg *msgid, const struct lanmsg *msgstr )
 137 {
 138     po_message_t msg;
 139     po_message_iterator_t iterator;
 140     char *id, *id_buffer, *context, *str = NULL, *str_buffer = NULL;
 141
 142     if (msgid->len <= 1) return;
 143
 144     id_buffer = id = convert_msgid_ascii( msgid, 1 );
 145     context = get_message_context( &id );
 146
 147     if (msgstr)
 148     {
 149         int len;
 150         str_buffer = str = unicode_to_utf8( msgstr->msg, msgstr->len, &len );
 151         if (is_english( msgstr->lan )) get_message_context( &str );
 152     }
 153     if (!(msg = find_message( po, id, context, &iterator )))
 154     {
 155         msg = po_message_create();
 156         po_message_set_msgid( msg, id );
 157         po_message_set_msgstr( msg, str ? str : "" );
 158         if (context) po_message_set_msgctxt( msg, context );
 159         po_message_insert( iterator, msg );
 160     }
 161     if (msgid->file) po_message_add_filepos( msg, msgid->file, msgid->line );
 162     po_message_iterator_free( iterator );
 163     free( id_buffer );
 164     free( str_buffer );
 165 }
 166
 167 static po_file_t create_po_file(void)
 168 {
 169     po_file_t po;
 170     po_message_t msg;
 171     po_message_iterator_t iterator;
 172
 173     po = po_file_create();
 174     iterator = po_message_iterator( po, NULL );
 175     msg = po_message_create();
 176     po_message_set_msgid( msg, "" );
 177     po_message_set_msgstr( msg,
 178                            "Project-Id-Version: Wine\n"
 179                            "Report-Msgid-Bugs-To: https://bugs.winehq.org\n"
 180                            "POT-Creation-Date: N/A\n"
 181                            "PO-Revision-Date: N/A\n"
 182                            "Last-Translator: Automatically generated\n"
 183                            "Language-Team: none\n"
 184                            "MIME-Version: 1.0\n"
 185                            "Content-Type: text/plain; charset=UTF-8\n"
 186                            "Content-Transfer-Encoding: 8bit\n" );
 187     po_message_insert( iterator, msg );
 188     po_message_iterator_free( iterator );
 189     return po;
 190 }
 191
 192 void write_pot_file( const char *outname )
 193 {
 194     int i, j;
 195     struct lan_blk *lbp;
 196     po_file_t po = create_po_file();
 197
 198     for (lbp = lanblockhead; lbp; lbp = lbp->next)
 199     {
 200         if (!is_english( lbp->lan )) continue;
 201         for (i = 0; i < lbp->nblk; i++)
 202         {
 203             struct block *blk = &lbp->blks[i];
 204             for (j = 0; j < blk->nmsg; j++) add_po_string( po, blk->msgs[j], NULL );
 205         }
 206     }
 207     po_file_write( po, outname, &po_xerror_handler );
 208     po_file_free( po );
 209 }
 210
 211
 212 #else  /* HAVE_LIBGETTEXTPO */
 213
 214 void write_pot_file( const char *outname )
 215 {
 216     error( "PO files not supported in this wmc build\n" );
 217 }
 218
 219 #endif
 220
 221 static struct mo_file *mo_file;
 222
 223 static void byteswap( unsigned int *data, unsigned int count )
 224 {
 225     unsigned int i;
 226
 227     for (i = 0; i < count; i++)
 228         data[i] = data[i] >> 24 | (data[i] >> 8 & 0xff00) | (data[i] << 8 & 0xff0000) | data[i] << 24;
 229 }
 230
 231 static void load_mo_file( const char *name )
 232 {
 233     size_t size;
 234
 235     if (!(mo_file = read_file( name, &size ))) fatal_perror( "Failed to read %s", name );
 236
 237     /* sanity checks */
 238
 239     if (size < sizeof(*mo_file))
 240         error( "%s is not a valid .mo file\n", name );
 241     if (mo_file->magic == 0xde120495)
 242         byteswap( &mo_file->revision, 4 );
 243     else if (mo_file->magic != 0x950412de)
 244         error( "%s is not a valid .mo file\n", name );
 245     if ((mo_file->revision >> 16) > 1)
 246         error( "%s: unsupported file version %x\n", name, mo_file->revision );
 247     if (mo_file->msgid_off >= size ||
 248         mo_file->msgstr_off >= size ||
 249         size < sizeof(*mo_file) + 2 * 8 * mo_file->count)
 250         error( "%s: corrupted file\n", name );
 251
 252     if (mo_file->magic == 0xde120495)
 253     {
 254         byteswap( (unsigned int *)((char *)mo_file + mo_file->msgid_off), 2 * mo_file->count );
 255         byteswap( (unsigned int *)((char *)mo_file + mo_file->msgstr_off), 2 * mo_file->count );
 256     }
 257 }
 258
 259 static void free_mo_file(void)
 260 {
 261     free( mo_file );
 262     mo_file = NULL;
 263 }
 264
 265 static inline const char *get_mo_msgid( int index )
 266 {
 267     const char *base = (const char *)mo_file;
 268     const unsigned int *offsets = (const unsigned int *)(base + mo_file->msgid_off);
 269     return base + offsets[2 * index + 1];
 270 }
 271
 272 static inline const char *get_mo_msgstr( int index )
 273 {
 274     const char *base = (const char *)mo_file;
 275     const unsigned int *offsets = (const unsigned int *)(base + mo_file->msgstr_off);
 276     return base + offsets[2 * index + 1];
 277 }
 278
 279 static const char *get_msgstr( const char *msgid, const char *context, int *found )
 280 {
 281     int pos, res, min, max;
 282     const char *ret = msgid;
 283     char *id = NULL;
 284
 285     if (!mo_file)  /* strings containing a context still need to be transformed */
 286     {
 287         if (context) (*found)++;
 288         return ret;
 289     }
 290
 291     if (context) id = strmake( "%s%c%s", context, 4, msgid );
 292     min = 0;
 293     max = mo_file->count - 1;
 294     while (min <= max)
 295     {
 296         pos = (min + max) / 2;
 297         res = strcmp( get_mo_msgid(pos), id ? id : msgid );
 298         if (!res)
 299         {
 300             const char *str = get_mo_msgstr( pos );
 301             if (str[0])  /* ignore empty strings */
 302             {
 303                 ret = str;
 304                 (*found)++;
 305             }
 306             break;
 307         }
 308         if (res > 0) max = pos - 1;
 309         else min = pos + 1;
 310     }
 311     free( id );
 312     return ret;
 313 }
 314
 315 static struct lanmsg *translate_string( struct lanmsg *str, int lang, int *found )
 316 {
 317     struct lanmsg *new;
 318     const char *transl;
 319     char *buffer, *msgid, *context;
 320
 321     if (str->len <= 1 || !(buffer = convert_msgid_ascii( str, 0 ))) return str;
 322
 323     msgid = buffer;
 324     context = get_message_context( &msgid );
 325     transl = get_msgstr( msgid, context, found );
 326
 327     new = xmalloc( sizeof(*new) );
 328     new->lan  = lang;
 329     new->cp   = 0;  /* FIXME */
 330     new->file = str->file;
 331     new->line = str->line;
 332     new->msg  = utf8_to_unicode( transl, strlen(transl) + 1, &new->len );
 333     free( buffer );
 334     return new;
 335 }
 336
 337 static void translate_block( struct block *blk, struct block *new, int lang, int *found )
 338 {
 339     int i;
 340
 341     new->idlo = blk->idlo;
 342     new->idhi = blk->idhi;
 343     new->size = 0;
 344     new->msgs = xmalloc( blk->nmsg * sizeof(*new->msgs) );
 345     new->nmsg = blk->nmsg;
 346     for (i = 0; i < blk->nmsg; i++)
 347     {
 348         new->msgs[i] = translate_string( blk->msgs[i], lang, found );
 349         new->size += ((2 * new->msgs[i]->len + 3) & ~3) + 4;
 350     }
 351 }
 352
 353 static void translate_messages( int lang )
 354 {
 355     int i, found;
 356     struct lan_blk *lbp, *new;
 357
 358     for (lbp = lanblockhead; lbp; lbp = lbp->next)
 359     {
 360         if (!is_english( lbp->lan )) continue;
 361         found = 0;
 362         new = xmalloc( sizeof(*new) );
 363         /* English "translations" take precedence over the original contents */
 364         new->version = is_english( lang ) ? 1 : -1;
 365         new->lan = lang;
 366         new->blks = xmalloc( lbp->nblk * sizeof(*new->blks) );
 367         new->nblk = lbp->nblk;
 368
 369         for (i = 0; i < lbp->nblk; i++)
 370             translate_block( &lbp->blks[i], &new->blks[i], lang, &found );
 371         if (found)
 372         {
 373             if (new_tail) new_tail->next = new;
 374             else new_top = new;
 375             new->prev = new_tail;
 376             new_tail = new;
 377         }
 378         else
 379         {
 380             free( new->blks );
 381             free( new );
 382         }
 383     }
 384 }
 385
 386 /* Unix format is: lang[_country][.charset][@modifier]
 387  * Windows format is: lang[-script][-country][_modifier] */
 388 static int unix_to_win_locale( const char *unix_name, char *win_name )
 389 {
 390     static const char sep[] = "_.@";
 391     const char *extra = NULL;
 392     char buffer[LOCALE_NAME_MAX_LENGTH];
 393     char *p, *country = NULL, *modifier = NULL;
 394
 395     if (strlen( unix_name ) >= LOCALE_NAME_MAX_LENGTH) return FALSE;
 396     strcpy( buffer, unix_name );
 397     if (!(p = strpbrk( buffer, sep )))
 398     {
 399         strcpy( win_name, buffer );
 400         return TRUE;
 401     }
 402
 403     if (*p == '_')
 404     {
 405         *p++ = 0;
 406         country = p;
 407         p = strpbrk( p, sep + 1 );
 408     }
 409     if (p && *p == '.')
 410     {
 411         *p++ = 0;
 412         /* charset, ignore */
 413         p = strchr( p, '@' );
 414     }
 415     if (p)
 416     {
 417         *p++ = 0;
 418         modifier = p;
 419     }
 420
 421     /* rebuild a Windows name */
 422
 423     strcpy( win_name, buffer );
 424     if (modifier)
 425     {
 426         if (!strcmp( modifier, "arabic" )) strcat( win_name, "-Arab" );
 427         else if (!strcmp( modifier, "chakma" )) strcat( win_name, "-Cakm" );
 428         else if (!strcmp( modifier, "cherokee" )) strcat( win_name, "-Cher" );
 429         else if (!strcmp( modifier, "cyrillic" )) strcat( win_name, "-Cyrl" );
 430         else if (!strcmp( modifier, "devanagari" )) strcat( win_name, "-Deva" );
 431         else if (!strcmp( modifier, "gurmukhi" )) strcat( win_name, "-Guru" );
 432         else if (!strcmp( modifier, "javanese" )) strcat( win_name, "-Java" );
 433         else if (!strcmp( modifier, "latin" )) strcat( win_name, "-Latn" );
 434         else if (!strcmp( modifier, "mongolian" )) strcat( win_name, "-Mong" );
 435         else if (!strcmp( modifier, "syriac" )) strcat( win_name, "-Syrc" );
 436         else if (!strcmp( modifier, "tifinagh" )) strcat( win_name, "-Tfng" );
 437         else if (!strcmp( modifier, "tibetan" )) strcat( win_name, "-Tibt" );
 438         else if (!strcmp( modifier, "vai" )) strcat( win_name, "-Vaii" );
 439         else if (!strcmp( modifier, "yi" )) strcat( win_name, "-Yiii" );
 440         else if (!strcmp( modifier, "saaho" )) strcpy( win_name, "ssy" );
 441         else if (!strcmp( modifier, "valencia" )) extra = "-valencia";
 442         /* ignore unknown modifiers */
 443     }
 444     if (country)
 445     {
 446         p = win_name + strlen(win_name);
 447         *p++ = '-';
 448         strcpy( p, country );
 449     }
 450     if (extra) strcat( win_name, extra );
 451     return TRUE;
 452 }
 453
 454
 455 void add_translations( const char *po_dir )
 456 {
 457     struct lan_blk *lbp;
 458     char buffer[256];
 459     char *p, *tok, *name;
 460     FILE *f;
 461
 462     /* first check if we have English resources to translate */
 463     for (lbp = lanblockhead; lbp; lbp = lbp->next) if (is_english( lbp->lan )) break;
 464     if (!lbp) return;
 465
 466     if (!po_dir)  /* run through the translation process to remove msg contexts */
 467     {
 468         translate_messages( MAKELANGID( LANG_ENGLISH, SUBLANG_DEFAULT ));
 469         goto done;
 470     }
 471
 472     new_top = new_tail = NULL;
 473
 474     name = strmake( "%s/LINGUAS", po_dir );
 475     if (!(f = fopen( name, "r" ))) return;
 476     free( name );
 477     while (fgets( buffer, sizeof(buffer), f ))
 478     {
 479         if ((p = strchr( buffer, '#' ))) *p = 0;
 480         for (tok = strtok( buffer, " \t\r\n" ); tok; tok = strtok( NULL, " \t\r\n" ))
 481         {
 482             char locale[LOCALE_NAME_MAX_LENGTH];
 483             unsigned int lang;
 484
 485             if (!unix_to_win_locale( tok, locale ) || !(lang = get_language_from_name( locale )))
 486             {
 487                 error( "unknown language '%s'\n", tok );
 488                 continue;
 489             }
 490             name = strmake( "%s/%s.mo", po_dir, tok );
 491             load_mo_file( name );
 492             translate_messages( lang );
 493             free_mo_file();
 494             free( name );
 495         }
 496     }
 497     fclose( f );
 498
 499 done:
 500     /* prepend the translated messages to the global list */
 501     if (new_tail)
 502     {
 503         new_tail->next = lanblockhead;
 504         lanblockhead->prev = new_tail;
 505         lanblockhead = new_top;
 506     }
 507 }