src/strgen/strgen.cpp

   1 /* $Id$ */
   2
   3 /*
   4  * This file is part of OpenTTD.
   5  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
   6  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
   7  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
   8  */
   9
  10 /** @file strgen.cpp Tool to create computer readable (stand-alone) translation files. */
  11
  12 #include "../stdafx.h"
  13 #include "../core/endian_func.hpp"
  14 #include "../string_func.h"
  15 #include "../strings_type.h"
  16 #include "../misc/getoptdata.h"
  17 #include "../table/control_codes.h"
  18
  19 #include "strgen.h"
  20
  21 #include <stdarg.h>
  22 #include <exception>
  23
  24 #if (!defined(WIN32) && !defined(WIN64)) || defined(__CYGWIN__)
  25 #include <unistd.h>
  26 #include <sys/stat.h>
  27 #endif
  28
  29 #if defined WIN32 || defined __WATCOMC__
  30 #include <direct.h>
  31 #endif /* WIN32 || __WATCOMC__ */
  32
  33 #ifdef __MORPHOS__
  34 #ifdef stderr
  35 #undef stderr
  36 #endif
  37 #define stderr stdout
  38 #endif /* __MORPHOS__ */
  39
  40 #include "../table/strgen_tables.h"
  41
  42
  43 #ifdef _MSC_VER
  44 # define LINE_NUM_FMT(s) "%s (%d): warning: %s (" s ")\n"
  45 #else
  46 # define LINE_NUM_FMT(s) "%s:%d: " s ": %s\n"
  47 #endif
  48
  49 void CDECL strgen_warning(const char *s, ...)
  50 {
  51         char buf[1024];
  52         va_list va;
  53         va_start(va, s);
  54         vsnprintf(buf, lengthof(buf), s, va);
  55         va_end(va);
  56         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, buf);
  57         _warnings++;
  58 }
  59
  60 void CDECL strgen_error(const char *s, ...)
  61 {
  62         char buf[1024];
  63         va_list va;
  64         va_start(va, s);
  65         vsnprintf(buf, lengthof(buf), s, va);
  66         va_end(va);
  67         fprintf(stderr, LINE_NUM_FMT("error"), _file, _cur_line, buf);
  68         _errors++;
  69 }
  70
  71 void NORETURN CDECL strgen_fatal(const char *s, ...)
  72 {
  73         char buf[1024];
  74         va_list va;
  75         va_start(va, s);
  76         vsnprintf(buf, lengthof(buf), s, va);
  77         va_end(va);
  78         fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
  79 #ifdef _MSC_VER
  80         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  81 #endif
  82         throw std::exception();
  83 }
  84
  85 void NORETURN CDECL error(const char *s, ...)
  86 {
  87         char buf[1024];
  88         va_list va;
  89         va_start(va, s);
  90         vsnprintf(buf, lengthof(buf), s, va);
  91         va_end(va);
  92         fprintf(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, buf);
  93 #ifdef _MSC_VER
  94         fprintf(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
  95 #endif
  96         exit(2);
  97 }
  98
  99 /** A reader that simply reads using fopen. */
 100 struct FileStringReader : StringReader {
 101         FILE *fh; ///< The file we are reading.
 102
 103         /**
 104          * Create the reader.
 105          * @param data        The data to fill during reading.
 106          * @param file        The file we are reading.
 107          * @param master      Are we reading the master file?
 108          * @param translation Are we reading a translation?
 109          */
 110         FileStringReader(StringData &data, const char *file, bool master, bool translation) :
 111                         StringReader(data, file, master, translation)
 112         {
 113                 this->fh = fopen(file, "rb");
 114                 if (this->fh == NULL) error("Could not open %s", file);
 115         }
 116
 117         /** Free/close the file. */
 118         virtual ~FileStringReader()
 119         {
 120                 fclose(this->fh);
 121         }
 122
 123         /* virtual */ char *ReadLine(char *buffer, size_t size)
 124         {
 125                 return fgets(buffer, size, this->fh);
 126         }
 127
 128         /* virtual */ void HandlePragma(char *str);
 129
 130         /* virtual */ void ParseFile()
 131         {
 132                 this->StringReader::ParseFile();
 133
 134                 if (StrEmpty(_lang.name) || StrEmpty(_lang.own_name) || StrEmpty(_lang.isocode)) {
 135                         error("Language must include ##name, ##ownname and ##isocode");
 136                 }
 137         }
 138 };
 139
 140 void FileStringReader::HandlePragma(char *str)
 141 {
 142         if (!memcmp(str, "id ", 3)) {
 143                 this->data.next_string_id = strtoul(str + 3, NULL, 0);
 144         } else if (!memcmp(str, "name ", 5)) {
 145                 strecpy(_lang.name, str + 5, lastof(_lang.name));
 146         } else if (!memcmp(str, "ownname ", 8)) {
 147                 strecpy(_lang.own_name, str + 8, lastof(_lang.own_name));
 148         } else if (!memcmp(str, "isocode ", 8)) {
 149                 strecpy(_lang.isocode, str + 8, lastof(_lang.isocode));
 150         } else if (!memcmp(str, "textdir ", 8)) {
 151                 if (!memcmp(str + 8, "ltr", 3)) {
 152                         _lang.text_dir = TD_LTR;
 153                 } else if (!memcmp(str + 8, "rtl", 3)) {
 154                         _lang.text_dir = TD_RTL;
 155                 } else {
 156                         error("Invalid textdir %s", str + 8);
 157                 }
 158         } else if (!memcmp(str, "digitsep ", 9)) {
 159                 str += 9;
 160                 strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator));
 161         } else if (!memcmp(str, "digitsepcur ", 12)) {
 162                 str += 12;
 163                 strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_group_separator_currency));
 164         } else if (!memcmp(str, "decimalsep ", 11)) {
 165                 str += 11;
 166                 strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str, lastof(_lang.digit_decimal_separator));
 167         } else if (!memcmp(str, "winlangid ", 10)) {
 168                 const char *buf = str + 10;
 169                 long langid = strtol(buf, NULL, 16);
 170                 if (langid > (long)UINT16_MAX || langid < 0) {
 171                         error("Invalid winlangid %s", buf);
 172                 }
 173                 _lang.winlangid = (uint16)langid;
 174         } else if (!memcmp(str, "grflangid ", 10)) {
 175                 const char *buf = str + 10;
 176                 long langid = strtol(buf, NULL, 16);
 177                 if (langid >= 0x7F || langid < 0) {
 178                         error("Invalid grflangid %s", buf);
 179                 }
 180                 _lang.newgrflangid = (uint8)langid;
 181         } else if (!memcmp(str, "gender ", 7)) {
 182                 if (this->master) error("Genders are not allowed in the base translation.");
 183                 char *buf = str + 7;
 184
 185                 for (;;) {
 186                         const char *s = ParseWord(&buf);
 187
 188                         if (s == NULL) break;
 189                         if (_lang.num_genders >= MAX_NUM_GENDERS) error("Too many genders, max %d", MAX_NUM_GENDERS);
 190                         strecpy(_lang.genders[_lang.num_genders], s, lastof(_lang.genders[_lang.num_genders]));
 191                         _lang.num_genders++;
 192                 }
 193         } else if (!memcmp(str, "case ", 5)) {
 194                 if (this->master) error("Cases are not allowed in the base translation.");
 195                 char *buf = str + 5;
 196
 197                 for (;;) {
 198                         const char *s = ParseWord(&buf);
 199
 200                         if (s == NULL) break;
 201                         if (_lang.num_cases >= MAX_NUM_CASES) error("Too many cases, max %d", MAX_NUM_CASES);
 202                         strecpy(_lang.cases[_lang.num_cases], s, lastof(_lang.cases[_lang.num_cases]));
 203                         _lang.num_cases++;
 204                 }
 205         } else {
 206                 StringReader::HandlePragma(str);
 207         }
 208 }
 209
 210 bool CompareFiles(const char *n1, const char *n2)
 211 {
 212         FILE *f2 = fopen(n2, "rb");
 213         if (f2 == NULL) return false;
 214
 215         FILE *f1 = fopen(n1, "rb");
 216         if (f1 == NULL) error("can't open %s", n1);
 217
 218         size_t l1, l2;
 219         do {
 220                 char b1[4096];
 221                 char b2[4096];
 222                 l1 = fread(b1, 1, sizeof(b1), f1);
 223                 l2 = fread(b2, 1, sizeof(b2), f2);
 224
 225                 if (l1 != l2 || memcmp(b1, b2, l1)) {
 226                         fclose(f2);
 227                         fclose(f1);
 228                         return false;
 229                 }
 230         } while (l1 != 0);
 231
 232         fclose(f2);
 233         fclose(f1);
 234         return true;
 235 }
 236
 237 /** Base class for writing data to disk. */
 238 struct FileWriter {
 239         FILE *fh;             ///< The file handle we're writing to.
 240         const char *filename; ///< The file name we're writing to.
 241
 242         /**
 243          * Open a file to write to.
 244          * @param filename The file to open.
 245          */
 246         FileWriter(const char *filename)
 247         {
 248                 this->filename = strdup(filename);
 249                 this->fh = fopen(this->filename, "wb");
 250
 251                 if (this->fh == NULL) {
 252                         error("Could not open %s", this->filename);
 253                 }
 254         }
 255
 256         /** Finalise the writing. */
 257         void Finalise()
 258         {
 259                 fclose(this->fh);
 260                 this->fh = NULL;
 261         }
 262
 263         /** Make sure the file is closed. */
 264         virtual ~FileWriter()
 265         {
 266                 /* If we weren't closed an exception was thrown, so remove the temporary file. */
 267                 if (fh != NULL) {
 268                         fclose(this->fh);
 269                         unlink(this->filename);
 270                 }
 271                 free(this->filename);
 272         }
 273 };
 274
 275 struct HeaderFileWriter : HeaderWriter, FileWriter {
 276         /** The real file name we eventually want to write to. */
 277         const char *real_filename;
 278         /** The previous string ID that was printed. */
 279         int prev;
 280
 281         /**
 282          * Open a file to write to.
 283          * @param filename The file to open.
 284          */
 285         HeaderFileWriter(const char *filename) : FileWriter("tmp.xxx"),
 286                 real_filename(strdup(filename)), prev(0)
 287         {
 288                 fprintf(this->fh, "/* This file is automatically generated. Do not modify */\n\n");
 289                 fprintf(this->fh, "#ifndef TABLE_STRINGS_H\n");
 290                 fprintf(this->fh, "#define TABLE_STRINGS_H\n");
 291         }
 292
 293         /** Free the filename. */
 294         ~HeaderFileWriter()
 295         {
 296                 free(real_filename);
 297         }
 298
 299         void WriteStringID(const char *name, int stringid)
 300         {
 301                 if (prev + 1 != stringid) fprintf(this->fh, "\n");
 302                 fprintf(this->fh, "static const StringID %s = 0x%X;\n", name, stringid);
 303                 prev = stringid;
 304         }
 305
 306         void Finalise(const StringData &data)
 307         {
 308                 /* Find the plural form with the most amount of cases. */
 309                 int max_plural_forms = 0;
 310                 for (uint i = 0; i < lengthof(_plural_forms); i++) {
 311                         max_plural_forms = max(max_plural_forms, _plural_forms[i].plural_count);
 312                 }
 313
 314                 fprintf(this->fh,
 315                         "\n"
 316                         "static const uint LANGUAGE_PACK_VERSION     = 0x%X;\n"
 317                         "static const uint LANGUAGE_MAX_PLURAL       = %d;\n"
 318                         "static const uint LANGUAGE_MAX_PLURAL_FORMS = %d;\n\n",
 319                         (uint)data.Version(), (uint)lengthof(_plural_forms), max_plural_forms
 320                 );
 321
 322                 fprintf(this->fh, "#endif /* TABLE_STRINGS_H */\n");
 323
 324                 this->FileWriter::Finalise();
 325
 326                 if (CompareFiles(this->filename, this->real_filename)) {
 327                         /* files are equal. tmp.xxx is not needed */
 328                         unlink(this->filename);
 329                 } else {
 330                         /* else rename tmp.xxx into filename */
 331         #if defined(WIN32) || defined(WIN64)
 332                         unlink(this->real_filename);
 333         #endif
 334                         if (rename(this->filename, this->real_filename) == -1) error("rename() failed");
 335                 }
 336         }
 337 };
 338
 339 /** Class for writing a language to disk. */
 340 struct LanguageFileWriter : LanguageWriter, FileWriter {
 341         /**
 342          * Open a file to write to.
 343          * @param filename The file to open.
 344          */
 345         LanguageFileWriter(const char *filename) : FileWriter(filename)
 346         {
 347         }
 348
 349         void WriteHeader(const LanguagePackHeader *header)
 350         {
 351                 this->Write((const byte *)header, sizeof(*header));
 352         }
 353
 354         void Finalise()
 355         {
 356                 if (fputc(0, this->fh) == EOF) {
 357                         error("Could not write to %s", this->filename);
 358                 }
 359                 this->FileWriter::Finalise();
 360         }
 361
 362         void Write(const byte *buffer, size_t length)
 363         {
 364                 if (fwrite(buffer, sizeof(*buffer), length, this->fh) != length) {
 365                         error("Could not write to %s", this->filename);
 366                 }
 367         }
 368 };
 369
 370 /** Multi-OS mkdirectory function */
 371 static inline void ottd_mkdir(const char *directory)
 372 {
 373         /* Ignore directory creation errors; they'll surface later on, and most
 374          * of the time they are 'directory already exists' errors anyhow. */
 375 #if defined(WIN32) || defined(__WATCOMC__)
 376         mkdir(directory);
 377 #else
 378         mkdir(directory, 0755);
 379 #endif
 380 }
 381
 382 /**
 383  * Create a path consisting of an already existing path, a possible
 384  * path separator and the filename. The separator is only appended if the path
 385  * does not already end with a separator
 386  */
 387 static inline char *mkpath(char *buf, size_t buflen, const char *path, const char *file)
 388 {
 389         ttd_strlcpy(buf, path, buflen); // copy directory into buffer
 390
 391         char *p = strchr(buf, '\0'); // add path separator if necessary
 392         if (p[-1] != PATHSEPCHAR && (size_t)(p - buf) + 1 < buflen) *p++ = PATHSEPCHAR;
 393         ttd_strlcpy(p, file, buflen - (size_t)(p - buf)); // concatenate filename at end of buffer
 394         return buf;
 395 }
 396
 397 #if defined(__MINGW32__)
 398 /**
 399  * On MingW, it is common that both / as \ are accepted in the
 400  * params. To go with those flow, we rewrite all incoming /
 401  * simply to \, so internally we can safely assume \.
 402  */
 403 static inline char *replace_pathsep(char *s)
 404 {
 405         for (char *c = s; *c != '\0'; c++) if (*c == '/') *c = '\\';
 406         return s;
 407 }
 408 #else
 409 static inline char *replace_pathsep(char *s) { return s; }
 410 #endif
 411
 412 /** Options of strgen. */
 413 static const OptionData _opts[] = {
 414           GETOPT_NOVAL(     'v',  "--version"),
 415         GETOPT_GENERAL('C', '\0', "-export-commands", ODF_NO_VALUE),
 416         GETOPT_GENERAL('L', '\0', "-export-plurals",  ODF_NO_VALUE),
 417         GETOPT_GENERAL('P', '\0', "-export-pragmas",  ODF_NO_VALUE),
 418           GETOPT_NOVAL(     't',  "--todo"),
 419           GETOPT_NOVAL(     'w',  "--warning"),
 420           GETOPT_NOVAL(     'h',  "--help"),
 421         GETOPT_GENERAL('h', '?',  NULL,               ODF_NO_VALUE),
 422           GETOPT_VALUE(     's',  "--source_dir"),
 423           GETOPT_VALUE(     'd',  "--dest_dir"),
 424         GETOPT_END(),
 425 };
 426
 427 int CDECL main(int argc, char *argv[])
 428 {
 429         char pathbuf[MAX_PATH];
 430         const char *src_dir = ".";
 431         const char *dest_dir = NULL;
 432
 433         GetOptData mgo(argc - 1, argv + 1, _opts);
 434         for (;;) {
 435                 int i = mgo.GetOpt();
 436                 if (i == -1) break;
 437
 438                 switch (i) {
 439                         case 'v':
 440                                 puts("$Revision$");
 441                                 return 0;
 442
 443                         case 'C':
 444                                 printf("args\tflags\tcommand\treplacement\n");
 445                                 for (const CmdStruct *cs = _cmd_structs; cs < endof(_cmd_structs); cs++) {
 446                                         char flags;
 447                                         if (cs->proc == EmitGender) {
 448                                                 flags = 'g'; // Command needs number of parameters defined by number of genders
 449                                         } else if (cs->proc == EmitPlural) {
 450                                                 flags = 'p'; // Command needs number of parameters defined by plural value
 451                                         } else if (cs->flags & C_DONTCOUNT) {
 452                                                 flags = 'i'; // Command may be in the translation when it is not in base
 453                                         } else {
 454                                                 flags = '0'; // Command needs no parameters
 455                                         }
 456                                         printf("%i\t%c\t\"%s\"\t\"%s\"\n", cs->consumes, flags, cs->cmd, strstr(cs->cmd, "STRING") ? "STRING" : cs->cmd);
 457                                 }
 458                                 return 0;
 459
 460                         case 'L':
 461                                 printf("count\tdescription\tnames\n");
 462                                 for (const PluralForm *pf = _plural_forms; pf < endof(_plural_forms); pf++) {
 463                                         printf("%i\t\"%s\"\t%s\n", pf->plural_count, pf->description, pf->names);
 464                                 }
 465                                 return 0;
 466
 467                         case 'P':
 468                                 printf("name\tflags\tdefault\tdescription\n");
 469                                 for (size_t i = 0; i < lengthof(_pragmas); i++) {
 470                                         printf("\"%s\"\t%s\t\"%s\"\t\"%s\"\n",
 471                                                         _pragmas[i][0], _pragmas[i][1], _pragmas[i][2], _pragmas[i][3]);
 472                                 }
 473                                 return 0;
 474
 475                         case 't':
 476                                 _show_todo |= 1;
 477                                 break;
 478
 479                         case 'w':
 480                                 _show_todo |= 2;
 481                                 break;
 482
 483                         case 'h':
 484                                 puts(
 485                                         "strgen - $Revision$\n"
 486                                         " -v | --version    print version information and exit\n"
 487                                         " -t | --todo       replace any untranslated strings with '<TODO>'\n"
 488                                         " -w | --warning    print a warning for any untranslated strings\n"
 489                                         " -h | -? | --help  print this help message and exit\n"
 490                                         " -s | --source_dir search for english.txt in the specified directory\n"
 491                                         " -d | --dest_dir   put output file in the specified directory, create if needed\n"
 492                                         " -export-commands  export all commands and exit\n"
 493                                         " -export-plurals   export all plural forms and exit\n"
 494                                         " -export-pragmas   export all pragmas and exit\n"
 495                                         " Run without parameters and strgen will search for english.txt and parse it,\n"
 496                                         " creating strings.h. Passing an argument, strgen will translate that language\n"
 497                                         " file using english.txt as a reference and output <language>.lng."
 498                                 );
 499                                 return 0;
 500
 501                         case 's':
 502                                 src_dir = replace_pathsep(mgo.opt);
 503                                 break;
 504
 505                         case 'd':
 506                                 dest_dir = replace_pathsep(mgo.opt);
 507                                 break;
 508
 509                         case -2:
 510                                 fprintf(stderr, "Invalid arguments\n");
 511                                 return 0;
 512                 }
 513         }
 514
 515         if (dest_dir == NULL) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
 516
 517         try {
 518                 /* strgen has two modes of operation. If no (free) arguments are passed
 519                  * strgen generates strings.h to the destination directory. If it is supplied
 520                  * with a (free) parameter the program will translate that language to destination
 521                  * directory. As input english.txt is parsed from the source directory */
 522                 if (mgo.numleft == 0) {
 523                         mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
 524
 525                         /* parse master file */
 526                         StringData data(TAB_COUNT);
 527                         FileStringReader master_reader(data, pathbuf, true, false);
 528                         master_reader.ParseFile();
 529                         if (_errors != 0) return 1;
 530
 531                         /* write strings.h */
 532                         ottd_mkdir(dest_dir);
 533                         mkpath(pathbuf, lengthof(pathbuf), dest_dir, "strings.h");
 534
 535                         HeaderFileWriter writer(pathbuf);
 536                         writer.WriteHeader(data);
 537                         writer.Finalise(data);
 538                 } else if (mgo.numleft >= 1) {
 539                         char *r;
 540
 541                         mkpath(pathbuf, lengthof(pathbuf), src_dir, "english.txt");
 542
 543                         StringData data(TAB_COUNT);
 544                         /* parse master file and check if target file is correct */
 545                         FileStringReader master_reader(data, pathbuf, true, false);
 546                         master_reader.ParseFile();
 547
 548                         for (int i = 0; i < mgo.numleft; i++) {
 549                                 data.FreeTranslation();
 550
 551                                 const char *translation = replace_pathsep(mgo.argv[i]);
 552                                 const char *file = strrchr(translation, PATHSEPCHAR);
 553                                 FileStringReader translation_reader(data, translation, false, file == NULL || strcmp(file + 1, "english.txt") != 0);
 554                                 translation_reader.ParseFile(); // target file
 555                                 if (_errors != 0) return 1;
 556
 557                                 /* get the targetfile, strip any directories and append to destination path */
 558                                 r = strrchr(mgo.argv[i], PATHSEPCHAR);
 559                                 mkpath(pathbuf, lengthof(pathbuf), dest_dir, (r != NULL) ? &r[1] : mgo.argv[i]);
 560
 561                                 /* rename the .txt (input-extension) to .lng */
 562                                 r = strrchr(pathbuf, '.');
 563                                 if (r == NULL || strcmp(r, ".txt") != 0) r = strchr(pathbuf, '\0');
 564                                 ttd_strlcpy(r, ".lng", (size_t)(r - pathbuf));
 565
 566                                 LanguageFileWriter writer(pathbuf);
 567                                 writer.WriteLang(data);
 568                                 writer.Finalise();
 569
 570                                 /* if showing warnings, print a summary of the language */
 571                                 if ((_show_todo & 2) != 0) {
 572                                         fprintf(stdout, "%d warnings and %d errors for %s\n", _warnings, _errors, pathbuf);
 573                                 }
 574                         }
 575                 }
 576         } catch (...) {
 577                 return 2;
 578         }
 579
 580         return 0;
 581 }