utility/ioz.c

   1 /***********************************************************************
   2  Freeciv - Copyright (C) 1996 - A Kjeldberg, L Gregersen, P Unold
   3    This program is free software; you can redistribute it and/or modify
   4    it under the terms of the GNU General Public License as published by
   5    the Free Software Foundation; either version 2, or (at your option)
   6    any later version.
   7
   8    This program is distributed in the hope that it will be useful,
   9    but WITHOUT ANY WARRANTY; without even the implied warranty of
  10    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11    GNU General Public License for more details.
  12 ***********************************************************************/
  13
  14 /***********************************************************************
  15   An IO layer to support transparent compression/uncompression.
  16   (Currently only "required" functionality is supported.)
  17
  18   There are various reasons for making this a full-blown module
  19   instead of just defining a few macros:
  20
  21   - Ability to switch between compressed and uncompressed at run-time
  22     (zlib with compression level 0 saves uncompressed, but still with
  23     gzip header, so non-zlib server cannot read the savefile).
  24
  25   - Flexibility to add other methods if desired (eg, bzip2, arbitrary
  26     external filter program, etc).
  27
  28   FIXME: when zlib support _not_ included, should sanity check whether
  29   the first few bytes are gzip marker and complain if so.
  30 ***********************************************************************/
  31
  32 #ifdef HAVE_CONFIG_H
  33 #include <fc_config.h>
  34 #endif
  35
  36 #include "fc_prehdrs.h"
  37
  38 #include <errno.h>
  39 #include <stdarg.h>
  40 #include <stdio.h>
  41 #include <string.h>
  42
  43 #ifdef FREECIV_HAVE_LIBZ
  44 #include <zlib.h>
  45 #endif
  46
  47 #ifdef HAVE_BZLIB_H
  48 #include <bzlib.h>
  49 #endif
  50
  51 #ifdef HAVE_LZMA_H
  52 #include <lzma.h>
  53 #endif
  54
  55 /* utility */
  56 #include "log.h"
  57 #include "mem.h"
  58 #include "shared.h"
  59 #include "support.h"
  60
  61 #include "ioz.h"
  62
  63 #ifdef FREECIV_HAVE_LIBBZ2
  64 struct bzip2_struct {
  65   BZFILE *file;
  66   FILE *plain;
  67   int error;
  68   int firstbyte;
  69   bool eof;
  70 };
  71 #endif /* FREECIV_HAVE_LIBBZ2 */
  72
  73 #ifdef FREECIV_HAVE_LIBLZMA
  74
  75 #define PLAIN_FILE_BUF_SIZE (1024*1024)    /* 1024kb */
  76 #define XZ_DECODER_TEST_SIZE (4*1024)      /* 4kb */
  77
  78 /* In my tests 7Mb proved to be not enough and with 10Mb decompression
  79    succeeded in typical case. */
  80 #define XZ_DECODER_MEMLIMIT (65*1024*1024)        /* 65Mb */
  81 #define XZ_DECODER_MEMLIMIT_STEP (25*1024*1024)   /* Increase 25Mb at a time */
  82 #define XZ_DECODER_MEMLIMIT_FINAL (100*1024*1024) /* 100Mb */
  83
  84 struct xz_struct {
  85   lzma_stream stream;
  86   int out_index;
  87   uint64_t memlimit;
  88
  89   /* liblzma bug workaround. This is what stream.avail_out should be,
  90      calculated correctly. Used only when reading file. */
  91   int out_avail;
  92   int total_read;
  93
  94   FILE *plain;
  95   uint8_t *in_buf;  /* uint8_t is what xz headers use */
  96   uint8_t *out_buf;
  97   lzma_ret error;
  98
  99   /* There seems to be bug in liblzma decompression that if all the
 100      processing happened when lzma_code() last time was called with
 101      LZMA_RUN action, it does not set next_out or avail_out variables
 102      to sane values when lzma_code() is called with LZMA_FINISH.
 103      We never call lzma_code() with LZMA_RUN action with all the input
 104      we have left. This hack_byte is kept in storage in case there is no
 105      more input when we try to get it. This byte can then be provided to
 106      lzma_code(LZMA_FINISH) */
 107   char hack_byte;
 108   bool hack_byte_used;
 109 };
 110
 111 static bool xz_outbuffer_to_file(fz_FILE *fp, lzma_action action);
 112 static void xz_action(fz_FILE *fp, lzma_action action);
 113
 114 #endif /* FREECIV_HAVE_LIBLZMA */
 115
 116 struct mem_fzFILE {
 117   bool control;
 118   char *buffer;
 119   int pos;
 120   int size;
 121 };
 122
 123 struct fz_FILE_s {
 124   enum fz_method method;
 125   char mode;
 126   bool memory;
 127   union {
 128     struct mem_fzFILE mem;
 129     FILE *plain;                /* FZ_PLAIN */
 130 #ifdef FREECIV_HAVE_LIBZ
 131     gzFile zlib;                 /* FZ_ZLIB */
 132 #endif
 133 #ifdef FREECIV_HAVE_LIBBZ2
 134     struct bzip2_struct bz2;
 135 #endif
 136 #ifdef FREECIV_HAVE_LIBLZMA
 137     struct xz_struct xz;
 138 #endif
 139   } u;
 140 };
 141
 142 /****************************************************************************
 143   Validate the compression method.
 144 ****************************************************************************/
 145 static inline bool fz_method_is_valid(enum fz_method method)
 146 {
 147   switch (method) {
 148   case FZ_PLAIN:
 149 #ifdef FREECIV_HAVE_LIBZ
 150   case FZ_ZLIB:
 151 #endif
 152 #ifdef FREECIV_HAVE_LIBBZ2
 153   case FZ_BZIP2:
 154 #endif
 155 #ifdef FREECIV_HAVE_LIBLZMA
 156   case FZ_XZ:
 157 #endif
 158     return TRUE;
 159   }
 160   return FALSE;
 161 }
 162
 163 #define fz_method_validate(method)                                          \
 164     (fz_method_is_valid(method) ? method                                    \
 165      : (fc_assert_msg(TRUE == fz_method_is_valid(method),                   \
 166                       "Unsupported compress method %d, reverting to plain.",\
 167                       method), FZ_PLAIN))
 168
 169
 170 /***************************************************************
 171   Open memory buffer for reading as fz_FILE.
 172   If control is TRUE, caller gives up control of the buffer
 173   so ioz will free it when fz_FILE closed.
 174 ***************************************************************/
 175 fz_FILE *fz_from_memory(char *buffer, int size, bool control)
 176 {
 177   fz_FILE *fp;
 178
 179   fp = (fz_FILE *)fc_malloc(sizeof(*fp));
 180   fp->memory = TRUE;
 181   fp->u.mem.control = control;
 182   fp->u.mem.buffer = buffer;
 183   fp->u.mem.pos = 0;
 184   fp->u.mem.size = size;
 185
 186   return fp;
 187 }
 188
 189 /***************************************************************
 190   Open file for reading/writing, like fopen.
 191   Parameters compress_method and compress_level only apply
 192   for writing: for reading try to use the most appropriate
 193   available method.
 194   Returns NULL if there was a problem; check errno for details.
 195   (If errno is 0, and using FZ_ZLIB, probably had zlib error
 196   Z_MEM_ERROR.  Wishlist: better interface for errors?)
 197 ***************************************************************/
 198 fz_FILE *fz_from_file(const char *filename, const char *in_mode,
 199                       enum fz_method method, int compress_level)
 200 {
 201   fz_FILE *fp;
 202   char mode[64];
 203
 204   if (!is_reg_file_for_access(filename, in_mode[0] == 'w')) {
 205     return NULL;
 206   }
 207
 208   fp = (fz_FILE *)fc_malloc(sizeof(*fp));
 209   fp->memory = FALSE;
 210   sz_strlcpy(mode, in_mode);
 211
 212   if (mode[0] == 'w') {
 213     /* Writing: */
 214     fp->mode = 'w';
 215   } else {
 216 #if defined(FREECIV_HAVE_LIBBZ2) || defined(FREECIV_HAVE_LIBLZMA)
 217     char test_mode[4];
 218     sz_strlcpy(test_mode, mode);
 219     sz_strlcat(test_mode, "b");
 220 #endif /* FREECIV_HAVE_LIBBZ2 || FREECIV_HAVE_LIBLZMA */
 221
 222     /* Reading: ignore specified method and try each: */
 223     fp->mode = 'r';
 224
 225 #ifdef FREECIV_HAVE_LIBBZ2
 226     /* Try to open as bzip2 file
 227        This is simplest test, so do it first. */
 228     method = FZ_BZIP2;
 229     fp->u.bz2.plain = fc_fopen(filename, test_mode);
 230     if (fp->u.bz2.plain) {
 231       fp->u.bz2.file = BZ2_bzReadOpen(&fp->u.bz2.error, fp->u.bz2.plain, 1, 0,
 232                                       NULL, 0);
 233     } else {
 234       /* This may currently have garbage value assigned via other union member. */
 235       fp->u.bz2.file = NULL;
 236     }
 237     if (!fp->u.bz2.file) {
 238       if (fp->u.bz2.plain) {
 239         fclose(fp->u.bz2.plain);
 240       }
 241       free(fp);
 242       return NULL;
 243     } else {
 244       /* Try to read first byte out of stream so we can figure out if this
 245          really is bzip2 file or not. Store byte for later use */
 246       char tmp;
 247       int read_len;
 248
 249       /* We put error to tmp variable when we don't want to overwrite
 250        * error already in fp->u.bz2.error. So calls to fz_ferror() or
 251        * fz_strerror() will later return what originally went wrong,
 252        * and not what happened in error recovery. */
 253       int tmp_err;
 254
 255       read_len = BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, &tmp, 1);
 256       if (fp->u.bz2.error != BZ_DATA_ERROR_MAGIC) {
 257         /* bzip2 file */
 258         if (fp->u.bz2.error == BZ_STREAM_END) {
 259           /* We already reached end of file with our read of one byte */
 260           if (read_len == 0) {
 261             /* 0 byte file */
 262             fp->u.bz2.firstbyte = -1;
 263           } else {
 264             fp->u.bz2.firstbyte = tmp;
 265           }
 266           fp->u.bz2.eof = TRUE;
 267         } else if (fp->u.bz2.error != BZ_OK) {
 268           /* Read failed */
 269           BZ2_bzReadClose(&tmp_err, fp->u.bz2.file);
 270           fclose(fp->u.bz2.plain);
 271           free(fp);
 272           return NULL;
 273         } else {
 274           /* Read success and we can continue reading */
 275           fp->u.bz2.firstbyte = tmp;
 276           fp->u.bz2.eof = FALSE;
 277         }
 278         fp->method = FZ_BZIP2;
 279         return fp;
 280       }
 281
 282       /* Not bzip2 file */
 283       BZ2_bzReadClose(&tmp_err, fp->u.bz2.file);
 284       fclose(fp->u.bz2.plain);
 285     }
 286 #endif /* FREECIV_HAVE_LIBBZ2 */
 287
 288 #ifdef FREECIV_HAVE_LIBLZMA
 289     /* Try to open as xz file */
 290     fp->u.xz.memlimit = XZ_DECODER_MEMLIMIT;
 291     memset(&fp->u.xz.stream, 0, sizeof(lzma_stream));
 292     fp->u.xz.error = lzma_stream_decoder(&fp->u.xz.stream,
 293                                          fp->u.xz.memlimit,
 294                                          LZMA_CONCATENATED);
 295     if (fp->u.xz.error != LZMA_OK) {
 296       free(fp);
 297       return NULL;
 298     }
 299     fp->u.xz.plain = fc_fopen(filename, test_mode);
 300     if (fp->u.xz.plain) {
 301       size_t len = 0;
 302
 303       fp->u.xz.in_buf = fc_malloc(PLAIN_FILE_BUF_SIZE);
 304
 305       len = fread(fp->u.xz.in_buf, 1, XZ_DECODER_TEST_SIZE,
 306                   fp->u.xz.plain);
 307       if (len > 0) {
 308         lzma_action action;
 309
 310         fp->u.xz.stream.next_in = fp->u.xz.in_buf;
 311         fp->u.xz.stream.avail_in = len;
 312         fp->u.xz.out_buf = fc_malloc(PLAIN_FILE_BUF_SIZE);
 313         fp->u.xz.stream.next_out = fp->u.xz.out_buf;
 314         fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE;
 315         len = fread(&fp->u.xz.hack_byte, 1, 1, fp->u.xz.plain);
 316         if (len > 0) {
 317           fp->u.xz.hack_byte_used = TRUE;
 318           action = LZMA_RUN;
 319         } else {
 320           fp->u.xz.hack_byte_used = FALSE;
 321           action = LZMA_FINISH;
 322         }
 323         xz_action(fp, action);
 324         if (fp->u.xz.error == LZMA_OK || fp->u.xz.error == LZMA_STREAM_END) {
 325           fp->method = FZ_XZ;
 326           fp->u.xz.out_index = 0;
 327           fp->u.xz.total_read = 0;
 328           fp->u.xz.out_avail = fp->u.xz.stream.total_out;
 329           return fp;
 330         }
 331
 332         free(fp->u.xz.out_buf);
 333       }
 334       fclose(fp->u.xz.plain);
 335       lzma_end(&fp->u.xz.stream);
 336       free(fp->u.xz.in_buf);
 337     } else {
 338       free(fp);
 339       return NULL;
 340     }
 341 #endif /* FREECIV_HAVE_LIBLZMA */
 342
 343 #ifdef FREECIV_HAVE_LIBZ
 344     method = FZ_ZLIB;
 345 #else
 346     method = FZ_PLAIN;
 347 #endif
 348   }
 349
 350   fp->method = fz_method_validate(method);
 351
 352   switch (fp->method) {
 353 #ifdef FREECIV_HAVE_LIBLZMA
 354   case FZ_XZ:
 355     {
 356       lzma_ret ret;
 357
 358       /*  xz files are binary files, so we should add "b" to mode! */
 359       sz_strlcat(mode,"b");
 360       memset(&fp->u.xz.stream, 0, sizeof(lzma_stream));
 361       ret = lzma_easy_encoder(&fp->u.xz.stream, compress_level, LZMA_CHECK_CRC32);
 362       fp->u.xz.error = ret;
 363       if (ret != LZMA_OK) {
 364         free(fp);
 365         return NULL;
 366       }
 367       fp->u.xz.in_buf = fc_malloc(PLAIN_FILE_BUF_SIZE);
 368       fp->u.xz.stream.next_in = fp->u.xz.in_buf;
 369       fp->u.xz.out_buf = fc_malloc(PLAIN_FILE_BUF_SIZE);
 370       fp->u.xz.stream.next_out = fp->u.xz.out_buf;
 371       fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE;
 372       fp->u.xz.out_index = 0;
 373       fp->u.xz.total_read = 0;
 374       fp->u.xz.plain = fc_fopen(filename, mode);
 375     }
 376     return fp;
 377 #endif /* FREECIV_HAVE_LIBLZMA */
 378 #ifdef FREECIV_HAVE_LIBBZ2
 379   case FZ_BZIP2:
 380     /*  bz2 files are binary files, so we should add "b" to mode! */
 381     sz_strlcat(mode,"b");
 382     fp->u.bz2.plain = fc_fopen(filename, mode);
 383     if (fp->u.bz2.plain) {
 384       /*  Open for read handled earlier */
 385       fc_assert_ret_val('w' == mode[0], NULL);
 386       fp->u.bz2.file = BZ2_bzWriteOpen(&fp->u.bz2.error, fp->u.bz2.plain,
 387                                        compress_level, 1, 15);
 388       if (fp->u.bz2.error != BZ_OK) {
 389         int tmp_err; /* See comments for similar variable
 390                       * near BZ2_bzReadOpen() */
 391         BZ2_bzWriteClose(&tmp_err, fp->u.bz2.file, 0, NULL, NULL);
 392         fp->u.bz2.file = NULL;
 393       }
 394     }
 395     if (!fp->u.bz2.file) {
 396       if (fp->u.bz2.plain) {
 397         fclose(fp->u.bz2.plain);
 398       }
 399       free(fp);
 400       fp = NULL;
 401     }
 402     return fp;
 403 #endif /* FREECIV_HAVE_LIBBZ2 */
 404 #ifdef FREECIV_HAVE_LIBZ
 405   case FZ_ZLIB:
 406     /*  gz files are binary files, so we should add "b" to mode! */
 407     sz_strlcat(mode,"b");
 408     if (mode[0] == 'w') {
 409       cat_snprintf(mode, sizeof(mode), "%d", compress_level);
 410     }
 411     fp->u.zlib = fc_gzopen(filename, mode);
 412     if (!fp->u.zlib) {
 413       free(fp);
 414       fp = NULL;
 415     }
 416     return fp;
 417 #endif /* FREECIV_HAVE_LIBZ */
 418   case FZ_PLAIN:
 419     fp->u.plain = fc_fopen(filename, mode);
 420     if (!fp->u.plain) {
 421       free(fp);
 422       fp = NULL;
 423     }
 424     return fp;
 425   }
 426
 427   /* Should never happen */
 428   fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
 429                 __FUNCTION__, fp->method);
 430   free(fp);
 431   return NULL;
 432 }
 433
 434 /***************************************************************
 435   Open uncompressed stream for reading/writing.
 436 ***************************************************************/
 437 fz_FILE *fz_from_stream(FILE *stream)
 438 {
 439   fz_FILE *fp;
 440
 441   if (!stream) {
 442     return NULL;
 443   }
 444
 445   fp = fc_malloc(sizeof(*fp));
 446   fp->method = FZ_PLAIN;
 447   fp->memory = FALSE;
 448   fp->u.plain = stream;
 449   return fp;
 450 }
 451
 452 /***************************************************************
 453   Close file, like fclose.
 454   Returns 0 on success, or non-zero for problems (but don't call
 455   fz_ferror in that case because the struct has already been
 456   free'd;  wishlist: better interface for errors?)
 457   (For FZ_PLAIN returns EOF and could check errno;
 458   for FZ_ZLIB: returns zlib error number; see zlib.h.)
 459 ***************************************************************/
 460 int fz_fclose(fz_FILE *fp)
 461 {
 462   int error = 0;
 463
 464   fc_assert_ret_val(NULL != fp, 1);
 465
 466   if (fp->memory) {
 467     if (fp->u.mem.control) {
 468       FC_FREE(fp->u.mem.buffer);
 469     }
 470     FC_FREE(fp);
 471
 472     return 0;
 473   }
 474
 475   switch (fz_method_validate(fp->method)) {
 476 #ifdef FREECIV_HAVE_LIBLZMA
 477   case FZ_XZ:
 478     if (fp->mode == 'w' && !xz_outbuffer_to_file(fp, LZMA_FINISH)) {
 479       error = 1;
 480     }
 481     lzma_end(&fp->u.xz.stream);
 482     free(fp->u.xz.in_buf);
 483     free(fp->u.xz.out_buf);
 484     fclose(fp->u.xz.plain);
 485     free(fp);
 486     return error;
 487 #endif /* FREECIV_HAVE_LIBLZMA */
 488 #ifdef FREECIV_HAVE_LIBBZ2
 489   case FZ_BZIP2:
 490     if ('w' == fp->mode) {
 491       BZ2_bzWriteClose(&fp->u.bz2.error, fp->u.bz2.file, 0, NULL, NULL);
 492     } else {
 493       BZ2_bzReadClose(&fp->u.bz2.error, fp->u.bz2.file);
 494     }
 495     error = fp->u.bz2.error;
 496     fclose(fp->u.bz2.plain);
 497     free(fp);
 498     return BZ_OK == error ? 0 : 1;
 499 #endif /* FREECIV_HAVE_LIBBZ2 */
 500 #ifdef FREECIV_HAVE_LIBZ
 501   case FZ_ZLIB:
 502     error = gzclose(fp->u.zlib);
 503     free(fp);
 504     return 0 > error ? error : 0; /* Only negative Z values are errors. */
 505 #endif /* FREECIV_HAVE_LIBZ */
 506   case FZ_PLAIN:
 507     error = fclose(fp->u.plain);
 508     free(fp);
 509     return error;
 510   }
 511
 512   /* Should never happen */
 513   fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
 514                 __FUNCTION__, fp->method);
 515   free(fp);
 516   return 1;
 517 }
 518
 519 /***************************************************************
 520   Get a line, like fgets.
 521   Returns NULL in case of error, or when end-of-file reached
 522   and no characters have been read.
 523 ***************************************************************/
 524 char *fz_fgets(char *buffer, int size, fz_FILE *fp)
 525 {
 526   fc_assert_ret_val(NULL != fp, NULL);
 527
 528   if (fp->memory) {
 529     int i, j;
 530
 531     for (i = fp->u.mem.pos, j = 0;
 532          i < fp->u.mem.size && j < size - 1 /* Space for '\0' */
 533            && fp->u.mem.buffer[i] != '\n'
 534            && (fp->u.mem.buffer[i] != '\r'
 535                || fp->u.mem.size == i + 1
 536                || fp->u.mem.buffer[i + 1] != '\n'); i++) {
 537       buffer[j++] = fp->u.mem.buffer[i];
 538     }
 539
 540     if (j < size - 2) {
 541       /* Space for both newline and terminating '\0' */
 542       if (i + 1 < fp->u.mem.size
 543           && fp->u.mem.buffer[i] == '\r'
 544           && fp->u.mem.buffer[i + 1] == '\n') {
 545         i += 2;
 546         buffer[j++] = '\n';
 547       } else if (i < fp->u.mem.size
 548                  && fp->u.mem.buffer[i] == '\n') {
 549         i++;
 550         buffer[j++] = '\n';
 551       }
 552     }
 553
 554     if (j == 0) {
 555       return NULL;
 556     }
 557
 558     fp->u.mem.pos = i;
 559     buffer[j] = '\0';
 560
 561     return buffer;
 562   }
 563
 564   switch (fz_method_validate(fp->method)) {
 565 #ifdef FREECIV_HAVE_LIBLZMA
 566   case FZ_XZ:
 567     {
 568       int i, j;
 569
 570       for (i = 0; i < size - 1; i += j) {
 571         size_t len = 0;
 572         bool line_end;
 573
 574         for (j = 0, line_end = FALSE; fp->u.xz.out_avail > 0
 575                && !line_end
 576                && j < size - i - 1;
 577              j++, fp->u.xz.out_avail--) {
 578           buffer[i + j] = fp->u.xz.out_buf[fp->u.xz.out_index++];
 579           fp->u.xz.total_read++;
 580           if (buffer[i + j] == '\n') {
 581             line_end = TRUE;
 582           }
 583         }
 584
 585         if (line_end || size <= j + i + 1) {
 586           buffer[i + j] = '\0';
 587           return buffer;
 588         }
 589
 590         if (fp->u.xz.hack_byte_used) {
 591           size_t hblen = 0;
 592
 593           fp->u.xz.in_buf[0] = fp->u.xz.hack_byte;
 594           len = fread(fp->u.xz.in_buf + 1, 1, PLAIN_FILE_BUF_SIZE - 1,
 595                       fp->u.xz.plain);
 596           len++;
 597
 598           if (len <= 1) {
 599             hblen = fread(&fp->u.xz.hack_byte, 1, 1, fp->u.xz.plain);
 600           }
 601           if (hblen == 0) {
 602             fp->u.xz.hack_byte_used = FALSE;
 603           }
 604         }
 605         if (len == 0) {
 606           if (fp->u.xz.error == LZMA_STREAM_END) {
 607             if (i + j == 0) {
 608               /* Plain file read complete, and there was nothing in xz buffers
 609                  -> end-of-file. */
 610               return NULL;
 611             }
 612             buffer[i + j] = '\0';
 613             return buffer;
 614           } else {
 615             fp->u.xz.stream.next_out = fp->u.xz.out_buf;
 616             fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE;
 617             xz_action(fp, LZMA_FINISH);
 618             fp->u.xz.out_index = 0;
 619             fp->u.xz.out_avail =
 620               fp->u.xz.stream.total_out - fp->u.xz.total_read;
 621             if (fp->u.xz.error != LZMA_OK
 622                 && fp->u.xz.error != LZMA_STREAM_END) {
 623               return NULL;
 624             }
 625           }
 626         } else {
 627           lzma_action action;
 628
 629           fp->u.xz.stream.next_in = fp->u.xz.in_buf;
 630           fp->u.xz.stream.avail_in = len;
 631           fp->u.xz.stream.next_out = fp->u.xz.out_buf;
 632           fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE;
 633           if (fp->u.xz.hack_byte_used) {
 634             action = LZMA_RUN;
 635           } else {
 636             action = LZMA_FINISH;
 637           }
 638           xz_action(fp, action);
 639           fp->u.xz.out_avail =
 640             fp->u.xz.stream.total_out - fp->u.xz.total_read;
 641           fp->u.xz.out_index = 0;
 642           if (fp->u.xz.error != LZMA_OK && fp->u.xz.error != LZMA_STREAM_END) {
 643             return NULL;
 644           }
 645         }
 646       }
 647
 648       buffer[i] = '\0';
 649       return buffer;
 650     }
 651     break;
 652 #endif /* FREECIV_HAVE_LIBLZMA */
 653 #ifdef FREECIV_HAVE_LIBBZ2
 654   case FZ_BZIP2:
 655     {
 656       char *retval = NULL;
 657       int i = 0;
 658       int last_read;
 659
 660       /* See if first byte is already read and stored */
 661       if (fp->u.bz2.firstbyte >= 0) {
 662         buffer[0] = fp->u.bz2.firstbyte;
 663         fp->u.bz2.firstbyte = -1;
 664         i++;
 665       } else {
 666         if (!fp->u.bz2.eof) {
 667           last_read = BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, buffer + i, 1);
 668           i += last_read; /* 0 or 1 */
 669         }
 670       }
 671       if (!fp->u.bz2.eof) {
 672         /* Leave space for trailing zero */
 673         for (; i < size - 1
 674                && fp->u.bz2.error == BZ_OK && buffer[i - 1] != '\n' ;
 675              i += last_read) {
 676           last_read = BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file,
 677                                  buffer + i, 1);
 678         }
 679         if (fp->u.bz2.error != BZ_OK &&
 680             (fp->u.bz2.error != BZ_STREAM_END ||
 681              i == 0)) {
 682           retval = NULL;
 683         } else {
 684           retval = buffer;
 685         }
 686         if (fp->u.bz2.error == BZ_STREAM_END) {
 687           /* EOF reached. Do not BZ2_bzRead() any more. */
 688           fp->u.bz2.eof = TRUE;
 689         }
 690       }
 691       buffer[i] = '\0';
 692       return retval;
 693     }
 694 #endif /* FREECIV_HAVE_LIBBZ2 */
 695 #ifdef FREECIV_HAVE_LIBZ
 696   case FZ_ZLIB:
 697     return gzgets(fp->u.zlib, buffer, size);
 698 #endif /* FREECIV_HAVE_LIBZ */
 699   case FZ_PLAIN:
 700     return fgets(buffer, size, fp->u.plain);
 701   }
 702
 703   /* Should never happen */
 704   fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
 705                 __FUNCTION__, fp->method);
 706   return NULL;
 707 }
 708
 709 #ifdef FREECIV_HAVE_LIBLZMA
 710
 711 /***************************************************************
 712   Helper function to do given compression action and writing
 713   results from output buffer to file.
 714 ***************************************************************/
 715 static bool xz_outbuffer_to_file(fz_FILE *fp, lzma_action action)
 716 {
 717   do {
 718     size_t len;
 719     size_t total = 0;
 720
 721     fp->u.xz.error = lzma_code(&fp->u.xz.stream, action);
 722
 723     if (fp->u.xz.error != LZMA_OK && fp->u.xz.error != LZMA_STREAM_END) {
 724       return FALSE;
 725     }
 726
 727     while (total < PLAIN_FILE_BUF_SIZE - fp->u.xz.stream.avail_out) {
 728       len = fwrite(fp->u.xz.out_buf, 1,
 729                    PLAIN_FILE_BUF_SIZE - fp->u.xz.stream.avail_out - total,
 730                    fp->u.xz.plain);
 731       total += len;
 732       if (len == 0) {
 733         return FALSE;
 734       }
 735     }
 736     fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE;
 737     fp->u.xz.stream.next_out = fp->u.xz.out_buf;
 738   } while (fp->u.xz.stream.avail_in > 0);
 739
 740   return TRUE;
 741 }
 742
 743 /***************************************************************
 744   Helper function to do given decompression action.
 745 ***************************************************************/
 746 static void xz_action(fz_FILE *fp, lzma_action action)
 747 {
 748   fp->u.xz.error = lzma_code(&fp->u.xz.stream, action);
 749   if (fp->u.xz.error != LZMA_MEMLIMIT_ERROR) {
 750     return;
 751   }
 752
 753   while (fp->u.xz.error == LZMA_MEMLIMIT_ERROR
 754          && fp->u.xz.memlimit < XZ_DECODER_MEMLIMIT_FINAL) {
 755     fp->u.xz.memlimit += XZ_DECODER_MEMLIMIT_STEP;
 756     if (fp->u.xz.memlimit > XZ_DECODER_MEMLIMIT_FINAL) {
 757       fp->u.xz.memlimit = XZ_DECODER_MEMLIMIT_FINAL;
 758     }
 759     fp->u.xz.error = lzma_memlimit_set(&fp->u.xz.stream, fp->u.xz.memlimit);
 760   }
 761
 762   fp->u.xz.error = lzma_code(&fp->u.xz.stream, action);
 763 }
 764 #endif /* FREECIV_HAVE_LIBLZMA */
 765
 766 /***************************************************************
 767   Print formated, like fprintf.
 768
 769   Note: zlib doesn't have gzvfprintf, but thats ok because its
 770   fprintf only does similar to what we do here (print to fixed
 771   buffer), and in addition this way we get to use our safe
 772   snprintf.
 773
 774   Returns number of (uncompressed) bytes actually written, or
 775   0 on error.
 776 ***************************************************************/
 777 int fz_fprintf(fz_FILE *fp, const char *format, ...)
 778 {
 779   int num;
 780   va_list ap;
 781
 782   fc_assert_ret_val(NULL != fp, 0);
 783   fc_assert_ret_val(!fp->memory, 0);
 784
 785   switch (fz_method_validate(fp->method)) {
 786 #ifdef FREECIV_HAVE_LIBLZMA
 787   case FZ_XZ:
 788     {
 789       va_start(ap, format);
 790       num = fc_vsnprintf((char *)fp->u.xz.in_buf, PLAIN_FILE_BUF_SIZE, format, ap);
 791       va_end(ap);
 792
 793       if (num == -1) {
 794         log_error("Too much data: truncated in fz_fprintf (%u)",
 795                   PLAIN_FILE_BUF_SIZE);
 796         num = PLAIN_FILE_BUF_SIZE;
 797       }
 798       fp->u.xz.stream.next_in = fp->u.xz.in_buf;
 799       fp->u.xz.stream.avail_in = num;
 800
 801       if (!xz_outbuffer_to_file(fp, LZMA_RUN)) {
 802         return 0;
 803       } else {
 804         return strlen((char *)fp->u.xz.in_buf);
 805       }
 806     }
 807     break;
 808 #endif /* FREECIV_HAVE_LIBLZMA */
 809 #ifdef FREECIV_HAVE_LIBBZ2
 810   case FZ_BZIP2:
 811     {
 812       char buffer[65536];
 813
 814       va_start(ap, format);
 815       num = fc_vsnprintf(buffer, sizeof(buffer), format, ap);
 816       va_end(ap);
 817       if (num == -1) {
 818         log_error("Too much data: truncated in fz_fprintf (%lu)",
 819                   (unsigned long) sizeof(buffer));
 820       }
 821       BZ2_bzWrite(&fp->u.bz2.error, fp->u.bz2.file, buffer, strlen(buffer));
 822       if (fp->u.bz2.error != BZ_OK) {
 823         return 0;
 824       } else {
 825         return strlen(buffer);
 826       }
 827     }
 828 #endif /* FREECIV_HAVE_LIBBZ2 */
 829 #ifdef FREECIV_HAVE_LIBZ
 830   case FZ_ZLIB:
 831     {
 832       char buffer[65536];
 833
 834       va_start(ap, format);
 835       num = fc_vsnprintf(buffer, sizeof(buffer), format, ap);
 836       va_end(ap);
 837       if (num == -1) {
 838         log_error("Too much data: truncated in fz_fprintf (%lu)",
 839                   (unsigned long) sizeof(buffer));
 840       }
 841       return gzwrite(fp->u.zlib, buffer, (unsigned int)strlen(buffer));
 842     }
 843 #endif /* FREECIV_HAVE_LIBZ */
 844   case FZ_PLAIN:
 845     va_start(ap, format);
 846     num = vfprintf(fp->u.plain, format, ap);
 847     va_end(ap);
 848     return num;
 849   }
 850
 851   /* Should never happen */
 852   fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
 853                 __FUNCTION__, fp->method);
 854   return 0;
 855 }
 856
 857 /***************************************************************
 858   Return non-zero if there is an error status associated with
 859   this stream.  Check fz_strerror for details.
 860 ***************************************************************/
 861 int fz_ferror(fz_FILE *fp)
 862 {
 863   fc_assert_ret_val(NULL != fp, 0);
 864
 865   if (fp->memory) {
 866     return 0;
 867   }
 868
 869   switch (fz_method_validate(fp->method)) {
 870 #ifdef FREECIV_HAVE_LIBLZMA
 871   case FZ_XZ:
 872     if (fp->u.xz.error != LZMA_OK
 873         && fp->u.xz.error != LZMA_STREAM_END) {
 874       return 1;
 875     } else {
 876       return 0;
 877     }
 878     break;
 879 #endif /* FREECIV_HAVE_LZMA */
 880 #ifdef FREECIV_HAVE_LIBBZ2
 881   case FZ_BZIP2:
 882     return (BZ_OK != fp->u.bz2.error
 883             && BZ_STREAM_END != fp->u.bz2.error);
 884 #endif /* FREECIV_HAVE_LIBBZ2 */
 885 #ifdef FREECIV_HAVE_LIBZ
 886   case FZ_ZLIB:
 887     {
 888       int error;
 889
 890       (void) gzerror(fp->u.zlib, &error); /* Ignore string result here. */
 891       return 0 > error ? error : 0; /* Only negative Z values are errors. */
 892     }
 893 #endif /* FREECIV_HAVE_LIBZ */
 894   case FZ_PLAIN:
 895     return ferror(fp->u.plain);
 896     break;
 897   }
 898
 899   /* Should never happen */
 900   fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
 901                 __FUNCTION__, fp->method);
 902   return 0;
 903 }
 904
 905 /***************************************************************
 906   Return string (pointer to static memory) containing an error
 907   description associated with the file.  Should only call
 908   this is you know there is an error (eg, from fz_ferror()).
 909   Note the error string may be based on errno, so should call
 910   this immediately after problem, or possibly something else
 911   might overwrite errno.
 912 ***************************************************************/
 913 const char *fz_strerror(fz_FILE *fp)
 914 {
 915   fc_assert_ret_val(NULL != fp, NULL);
 916   fc_assert_ret_val(!fp->memory, NULL);
 917
 918   switch (fz_method_validate(fp->method)) {
 919 #ifdef FREECIV_HAVE_LIBLZMA
 920   case FZ_XZ:
 921     {
 922       static char xzerror[50];
 923       char *cleartext = NULL;
 924
 925       switch(fp->u.xz.error) {
 926        case LZMA_OK:
 927          cleartext = "OK";
 928          break;
 929        case LZMA_STREAM_END:
 930          cleartext = "Stream end";
 931          break;
 932        case LZMA_NO_CHECK:
 933          cleartext = "No integrity check";
 934          break;
 935        case LZMA_UNSUPPORTED_CHECK:
 936          cleartext = "Cannot calculate the integrity check";
 937          break;
 938        case LZMA_MEM_ERROR:
 939          cleartext = "Mem error";
 940          break;
 941        case LZMA_MEMLIMIT_ERROR:
 942          cleartext = "Memory limit reached";
 943          break;
 944        case LZMA_FORMAT_ERROR:
 945          cleartext = "Unrecognized file format";
 946          break;
 947        case LZMA_OPTIONS_ERROR:
 948          cleartext = "Unsupported options";
 949          break;
 950        case LZMA_DATA_ERROR:
 951          cleartext = "Data error";
 952          break;
 953        case LZMA_BUF_ERROR:
 954          cleartext = "Progress not possible";
 955          break;
 956        default:
 957          break;
 958       }
 959
 960       if (NULL != cleartext) {
 961         fc_snprintf(xzerror, sizeof(xzerror), "XZ: \"%s\" (%d)",
 962                     cleartext, fp->u.xz.error);
 963       } else {
 964         fc_snprintf(xzerror, sizeof(xzerror), "XZ error %d",
 965                     fp->u.xz.error);
 966       }
 967       return xzerror;
 968     }
 969     break;
 970 #endif /* FREECIV_HAVE_LIBLZMA */
 971 #ifdef FREECIV_HAVE_LIBBZ2
 972   case FZ_BZIP2:
 973     {
 974       static char bzip2error[50];
 975       const char *cleartext = NULL;
 976
 977       /* Rationale for translating these:
 978        * - Some of them provide usable information to user
 979        * - Messages still contain numerical error code for developers
 980        */
 981       switch(fp->u.bz2.error) {
 982        case BZ_OK:
 983          cleartext = "OK";
 984          break;
 985        case BZ_RUN_OK:
 986          cleartext = "Run ok";
 987          break;
 988        case BZ_FLUSH_OK:
 989          cleartext = "Flush ok";
 990          break;
 991        case BZ_FINISH_OK:
 992          cleartext = "Finish ok";
 993          break;
 994        case BZ_STREAM_END:
 995          cleartext = "Stream end";
 996          break;
 997        case BZ_CONFIG_ERROR:
 998          cleartext = "Config error";
 999          break;
1000        case BZ_SEQUENCE_ERROR:
1001          cleartext = "Sequence error";
1002          break;
1003        case BZ_PARAM_ERROR:
1004          cleartext = "Parameter error";
1005          break;
1006        case BZ_MEM_ERROR:
1007          cleartext = "Mem error";
1008          break;
1009        case BZ_DATA_ERROR:
1010          cleartext = "Data error";
1011          break;
1012        case BZ_DATA_ERROR_MAGIC:
1013          cleartext = "Not bzip2 file";
1014          break;
1015        case BZ_IO_ERROR:
1016          cleartext = "IO error";
1017          break;
1018        case BZ_UNEXPECTED_EOF:
1019          cleartext = "Unexpected EOF";
1020          break;
1021        case BZ_OUTBUFF_FULL:
1022          cleartext = "Output buffer full";
1023          break;
1024        default:
1025          break;
1026       }
1027
1028       if (cleartext != NULL) {
1029         fc_snprintf(bzip2error, sizeof(bzip2error), "Bz2: \"%s\" (%d)",
1030                     cleartext, fp->u.bz2.error);
1031       } else {
1032         fc_snprintf(bzip2error, sizeof(bzip2error), "Bz2 error %d",
1033                     fp->u.bz2.error);
1034       }
1035       return bzip2error;
1036     }
1037 #endif /* FREECIV_HAVE_LIBBZ2 */
1038 #ifdef FREECIV_HAVE_LIBZ
1039   case FZ_ZLIB:
1040     {
1041       int errnum;
1042       const char *estr = gzerror(fp->u.zlib, &errnum);
1043
1044       return Z_ERRNO == errnum ? fc_strerror(fc_get_errno()) : estr;
1045     }
1046 #endif /* FREECIV_HAVE_LIBZ */
1047   case FZ_PLAIN:
1048     return fc_strerror(fc_get_errno());
1049   }
1050
1051   /* Should never happen */
1052   fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
1053                 __FUNCTION__, fp->method);
1054   return NULL;
1055 }