src/common/torgzip.c

   1 /* Copyright (c) 2004, Roger Dingledine.
   2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
   3  * Copyright (c) 2007-2010, The Tor Project, Inc. */
   4 /* See LICENSE for licensing information */
   5
   6 /**
   7  * \file torgzip.c
   8  * \brief A simple in-memory gzip implementation.
   9  **/
  10
  11 #include "orconfig.h"
  12
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <assert.h>
  16 #include <string.h>
  17 #ifdef HAVE_NETINET_IN_H
  18 #include <netinet/in.h>
  19 #endif
  20
  21 #include "torint.h"
  22 #include "util.h"
  23 #include "torlog.h"
  24 #include "torgzip.h"
  25
  26 /* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
  27    saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
  28    that nobody will care if the compile outputs a no-such-identifier warning.
  29
  30    Sorry, but we like -Werror over here, so I guess we need to define these.
  31    I hope that zlib 1.2.6 doesn't break these too.
  32 */
  33 #ifndef _LARGEFILE64_SOURCE
  34 #define _LARGEFILE64_SOURCE 0
  35 #endif
  36 #ifndef _LFS64_LARGEFILE
  37 #define _LFS64_LARGEFILE 0
  38 #endif
  39 #ifndef _FILE_OFFSET_BITS
  40 #define _FILE_OFFSET_BITS 0
  41 #endif
  42 #ifndef off64_t
  43 #define off64_t int64_t
  44 #endif
  45
  46 #ifdef _MSC_VER
  47 #include "..\..\contrib\zlib\zlib.h"
  48 #else
  49 #include <zlib.h>
  50 #endif
  51
  52 /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't;
  53  * set to -1 if we haven't checked yet. */
  54 static int gzip_is_supported = -1;
  55
  56 /** Return true iff we support gzip-based compression.  Otherwise, we need to
  57  * use zlib. */
  58 int
  59 is_gzip_supported(void)
  60 {
  61   if (gzip_is_supported >= 0)
  62     return gzip_is_supported;
  63
  64   if (!strcmpstart(ZLIB_VERSION, "0.") ||
  65       !strcmpstart(ZLIB_VERSION, "1.0") ||
  66       !strcmpstart(ZLIB_VERSION, "1.1"))
  67     gzip_is_supported = 0;
  68   else
  69     gzip_is_supported = 1;
  70
  71   return gzip_is_supported;
  72 }
  73
  74 /** Return the 'bits' value to tell zlib to use <b>method</b>.*/
  75 static INLINE int
  76 method_bits(compress_method_t method)
  77 {
  78   /* Bits+16 means "use gzip" in zlib >= 1.2 */
  79   return method == GZIP_METHOD ? 15+16 : 15;
  80 }
  81
  82 /** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
  83  * allocated buffer, using the method described in <b>method</b>.  Store the
  84  * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
  85  * Return 0 on success, -1 on failure.
  86  */
  87 int
  88 tor_gzip_compress(char **out, size_t *out_len,
  89                   const char *in, size_t in_len,
  90                   compress_method_t method)
  91 {
  92   struct z_stream_s *stream = NULL;
  93   size_t out_size, old_size;
  94   off_t offset;
  95
  96   tor_assert(out);
  97   tor_assert(out_len);
  98   tor_assert(in);
  99   tor_assert(in_len < UINT_MAX);
 100
 101   *out = NULL;
 102
 103   if (method == GZIP_METHOD && !is_gzip_supported()) {
 104     /* Old zlib version don't support gzip in deflateInit2 */
 105     log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
 106     goto err;
 107   }
 108
 109   stream = tor_malloc_zero(sizeof(struct z_stream_s));
 110   stream->zalloc = Z_NULL;
 111   stream->zfree = Z_NULL;
 112   stream->opaque = NULL;
 113   stream->next_in = (unsigned char*) in;
 114   stream->avail_in = (unsigned int)in_len;
 115
 116   if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
 117                    method_bits(method),
 118                    8, Z_DEFAULT_STRATEGY) != Z_OK) {
 119     log_warn(LD_GENERAL, "Error from deflateInit2: %s",
 120              stream->msg?stream->msg:"<no message>");
 121     goto err;
 122   }
 123
 124   /* Guess 50% compression. */
 125   out_size = in_len / 2;
 126   if (out_size < 1024) out_size = 1024;
 127   *out = tor_malloc(out_size);
 128   stream->next_out = (unsigned char*)*out;
 129   stream->avail_out = (unsigned int)out_size;
 130
 131   while (1) {
 132     switch (deflate(stream, Z_FINISH))
 133       {
 134       case Z_STREAM_END:
 135         goto done;
 136       case Z_OK:
 137         /* In case zlib doesn't work as I think .... */
 138         if (stream->avail_out >= stream->avail_in+16)
 139           break;
 140       case Z_BUF_ERROR:
 141         offset = stream->next_out - ((unsigned char*)*out);
 142         old_size = out_size;
 143         out_size *= 2;
 144         if (out_size < old_size) {
 145           log_warn(LD_GENERAL, "Size overflow in compression.");
 146           goto err;
 147         }
 148         *out = tor_realloc(*out, out_size);
 149         stream->next_out = (unsigned char*)(*out + offset);
 150         if (out_size - offset > UINT_MAX) {
 151           log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
 152                    "uncompressing.");
 153           goto err;
 154         }
 155         stream->avail_out = (unsigned int)(out_size - offset);
 156         break;
 157       default:
 158         log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
 159                  stream->msg ? stream->msg : "<no message>");
 160         goto err;
 161       }
 162   }
 163  done:
 164   *out_len = stream->total_out;
 165 #ifdef OPENBSD
 166   /* "Hey Rocky!  Watch me change an unsigned field to a signed field in a
 167    *    third-party API!"
 168    * "Oh, that trick will just make people do unsafe casts to the unsigned
 169    *    type in their cross-platform code!"
 170    * "Don't be foolish.  I'm _sure_ they'll have the good sense to make sure
 171    *    the newly unsigned field isn't negative." */
 172   tor_assert(stream->total_out >= 0);
 173 #endif
 174   if (((size_t)stream->total_out) > out_size + 4097) {
 175     /* If we're wasting more than 4k, don't. */
 176     *out = tor_realloc(*out, stream->total_out + 1);
 177   }
 178   if (deflateEnd(stream)!=Z_OK) {
 179     log_warn(LD_BUG, "Error freeing gzip structures");
 180     goto err;
 181   }
 182   tor_free(stream);
 183
 184   return 0;
 185  err:
 186   if (stream) {
 187     deflateEnd(stream);
 188     tor_free(stream);
 189   }
 190   tor_free(*out);
 191   return -1;
 192 }
 193
 194 /** Given zero or more zlib-compressed or gzip-compressed strings of
 195  * total length
 196  * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated
 197  * buffer, using the method described in <b>method</b>.  Store the uncompressed
 198  * string in *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on
 199  * success, -1 on failure.
 200  *
 201  * If <b>complete_only</b> is true, we consider a truncated input as a
 202  * failure; otherwise we decompress as much as we can.  Warn about truncated
 203  * or corrupt inputs at <b>protocol_warn_level</b>.
 204  */
 205 int
 206 tor_gzip_uncompress(char **out, size_t *out_len,
 207                     const char *in, size_t in_len,
 208                     compress_method_t method,
 209                     int complete_only,
 210                     int protocol_warn_level)
 211 {
 212   struct z_stream_s *stream = NULL;
 213   size_t out_size, old_size;
 214   off_t offset;
 215   int r;
 216
 217   tor_assert(out);
 218   tor_assert(out_len);
 219   tor_assert(in);
 220   tor_assert(in_len < UINT_MAX);
 221
 222   if (method == GZIP_METHOD && !is_gzip_supported()) {
 223     /* Old zlib version don't support gzip in inflateInit2 */
 224     log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
 225     return -1;
 226   }
 227
 228   *out = NULL;
 229
 230   stream = tor_malloc_zero(sizeof(struct z_stream_s));
 231   stream->zalloc = Z_NULL;
 232   stream->zfree = Z_NULL;
 233   stream->opaque = NULL;
 234   stream->next_in = (unsigned char*) in;
 235   stream->avail_in = (unsigned int)in_len;
 236
 237   if (inflateInit2(stream,
 238                    method_bits(method)) != Z_OK) {
 239     log_warn(LD_GENERAL, "Error from inflateInit2: %s",
 240              stream->msg?stream->msg:"<no message>");
 241     goto err;
 242   }
 243
 244   out_size = in_len * 2;  /* guess 50% compression. */
 245   if (out_size < 1024) out_size = 1024;
 246   if (out_size > UINT_MAX)
 247     goto err;
 248
 249   *out = tor_malloc(out_size);
 250   stream->next_out = (unsigned char*)*out;
 251   stream->avail_out = (unsigned int)out_size;
 252
 253   while (1) {
 254     switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
 255       {
 256       case Z_STREAM_END:
 257         if (stream->avail_in == 0)
 258           goto done;
 259         /* There may be more compressed data here. */
 260         if ((r = inflateEnd(stream)) != Z_OK) {
 261           log_warn(LD_BUG, "Error freeing gzip structures");
 262           goto err;
 263         }
 264         if (inflateInit2(stream, method_bits(method)) != Z_OK) {
 265           log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
 266                    stream->msg?stream->msg:"<no message>");
 267           goto err;
 268         }
 269         break;
 270       case Z_OK:
 271         if (!complete_only && stream->avail_in == 0)
 272           goto done;
 273         /* In case zlib doesn't work as I think.... */
 274         if (stream->avail_out >= stream->avail_in+16)
 275           break;
 276       case Z_BUF_ERROR:
 277         if (stream->avail_out > 0) {
 278           log_fn(protocol_warn_level, LD_PROTOCOL,
 279                  "possible truncated or corrupt zlib data");
 280           goto err;
 281         }
 282         offset = stream->next_out - (unsigned char*)*out;
 283         old_size = out_size;
 284         out_size *= 2;
 285         if (out_size < old_size) {
 286           log_warn(LD_GENERAL, "Size overflow in compression.");
 287           goto err;
 288         }
 289         *out = tor_realloc(*out, out_size);
 290         stream->next_out = (unsigned char*)(*out + offset);
 291         if (out_size - offset > UINT_MAX) {
 292           log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
 293                    "uncompressing.");
 294           goto err;
 295         }
 296         stream->avail_out = (unsigned int)(out_size - offset);
 297         break;
 298       default:
 299         log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
 300                  stream->msg ? stream->msg : "<no message>");
 301         goto err;
 302       }
 303   }
 304  done:
 305   *out_len = stream->next_out - (unsigned char*)*out;
 306   r = inflateEnd(stream);
 307   tor_free(stream);
 308   if (r != Z_OK) {
 309     log_warn(LD_BUG, "Error freeing gzip structures");
 310     goto err;
 311   }
 312
 313   /* NUL-terminate output. */
 314   if (out_size == *out_len)
 315     *out = tor_realloc(*out, out_size + 1);
 316   (*out)[*out_len] = '\0';
 317
 318   return 0;
 319  err:
 320   if (stream) {
 321     inflateEnd(stream);
 322     tor_free(stream);
 323   }
 324   if (*out) {
 325     tor_free(*out);
 326   }
 327   return -1;
 328 }
 329
 330 /** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
 331  * to be compressed or not.  If it is, return the likeliest compression method.
 332  * Otherwise, return UNKNOWN_METHOD.
 333  */
 334 compress_method_t
 335 detect_compression_method(const char *in, size_t in_len)
 336 {
 337   if (in_len > 2 && !memcmp(in, "\x1f\x8b", 2)) {
 338     return GZIP_METHOD;
 339   } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
 340              (ntohs(get_uint16(in)) % 31) == 0) {
 341     return ZLIB_METHOD;
 342   } else {
 343     return UNKNOWN_METHOD;
 344   }
 345 }
 346
 347 /** Internal state for an incremental zlib compression/decompression.  The
 348  * body of this struct is not exposed. */
 349 struct tor_zlib_state_t {
 350   struct z_stream_s stream;
 351   int compress;
 352 };
 353
 354 /** Construct and return a tor_zlib_state_t object using <b>method</b>.  If
 355  * <b>compress</b>, it's for compression; otherwise it's for
 356  * decompression. */
 357 tor_zlib_state_t *
 358 tor_zlib_new(int compress, compress_method_t method)
 359 {
 360   tor_zlib_state_t *out;
 361
 362   if (method == GZIP_METHOD && !is_gzip_supported()) {
 363     /* Old zlib version don't support gzip in inflateInit2 */
 364     log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
 365     return NULL;
 366  }
 367
 368  out = tor_malloc_zero(sizeof(tor_zlib_state_t));
 369  out->stream.zalloc = Z_NULL;
 370  out->stream.zfree = Z_NULL;
 371  out->stream.opaque = NULL;
 372  out->compress = compress;
 373  if (compress) {
 374    if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
 375                     method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK)
 376      goto err;
 377  } else {
 378    if (inflateInit2(&out->stream, method_bits(method)) != Z_OK)
 379      goto err;
 380  }
 381  return out;
 382
 383  err:
 384  tor_free(out);
 385  return NULL;
 386 }
 387
 388 /** Compress/decompress some bytes using <b>state</b>.  Read up to
 389  * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
 390  * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
 391  * we've reached the end of the input.
 392  *
 393  * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression.
 394  * Return TOR_ZLIB_OK if we're processed everything from the input.
 395  * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>.
 396  * Return TOR_ZLIB_ERR if the stream is corrupt.
 397  */
 398 tor_zlib_output_t
 399 tor_zlib_process(tor_zlib_state_t *state,
 400                  char **out, size_t *out_len,
 401                  const char **in, size_t *in_len,
 402                  int finish)
 403 {
 404   int err;
 405   tor_assert(*in_len <= UINT_MAX);
 406   tor_assert(*out_len <= UINT_MAX);
 407   state->stream.next_in = (unsigned char*) *in;
 408   state->stream.avail_in = (unsigned int)*in_len;
 409   state->stream.next_out = (unsigned char*) *out;
 410   state->stream.avail_out = (unsigned int)*out_len;
 411
 412   if (state->compress) {
 413     err = deflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
 414   } else {
 415     err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
 416   }
 417
 418   *out = (char*) state->stream.next_out;
 419   *out_len = state->stream.avail_out;
 420   *in = (const char *) state->stream.next_in;
 421   *in_len = state->stream.avail_in;
 422
 423   switch (err)
 424     {
 425     case Z_STREAM_END:
 426       return TOR_ZLIB_DONE;
 427     case Z_BUF_ERROR:
 428       if (state->stream.avail_in == 0)
 429         return TOR_ZLIB_OK;
 430       return TOR_ZLIB_BUF_FULL;
 431     case Z_OK:
 432       if (state->stream.avail_out == 0 || finish)
 433         return TOR_ZLIB_BUF_FULL;
 434       return TOR_ZLIB_OK;
 435     default:
 436       log_warn(LD_GENERAL, "Gzip returned an error: %s",
 437                state->stream.msg ? state->stream.msg : "<no message>");
 438       return TOR_ZLIB_ERR;
 439     }
 440 }
 441
 442 /** Deallocate <b>state</b>. */
 443 void
 444 tor_zlib_free(tor_zlib_state_t *state)
 445 {
 446   if (!state)
 447     return;
 448
 449   if (state->compress)
 450     deflateEnd(&state->stream);
 451   else
 452     inflateEnd(&state->stream);
 453
 454   tor_free(state);
 455 }
 456