src/common/torgzip.c

   1 /* Copyright (c) 2004, Roger Dingledine.
   2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
   3  * Copyright (c) 2007-2009, The Tor Project, Inc. */
   4 /* See LICENSE for licensing information */
   5
   6 /**
   7  * \file torgzip.c
   8  * \brief A simple in-memory gzip implementation.
   9  **/
  10
  11 #include "orconfig.h"
  12
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <assert.h>
  16 #ifdef _MSC_VER
  17 #include "..\..\contrib\zlib\zlib.h"
  18 #else
  19 #include <zlib.h>
  20 #endif
  21 #include <string.h>
  22 #ifdef HAVE_NETINET_IN_H
  23 #include <netinet/in.h>
  24 #endif
  25
  26 #include "util.h"
  27 #include "log.h"
  28 #include "torgzip.h"
  29
  30 /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't;
  31  * set to -1 if we haven't checked yet. */
  32 static int gzip_is_supported = -1;
  33
  34 /** Return true iff we support gzip-based compression.  Otherwise, we need to
  35  * use zlib. */
  36 int
  37 is_gzip_supported(void)
  38 {
  39   if (gzip_is_supported >= 0)
  40     return gzip_is_supported;
  41
  42   if (!strcmpstart(ZLIB_VERSION, "0.") ||
  43       !strcmpstart(ZLIB_VERSION, "1.0") ||
  44       !strcmpstart(ZLIB_VERSION, "1.1"))
  45     gzip_is_supported = 0;
  46   else
  47     gzip_is_supported = 1;
  48
  49   return gzip_is_supported;
  50 }
  51
  52 /** Return the 'bits' value to tell zlib to use <b>method</b>.*/
  53 static INLINE int
  54 method_bits(compress_method_t method)
  55 {
  56   /* Bits+16 means "use gzip" in zlib >= 1.2 */
  57   return method == GZIP_METHOD ? 15+16 : 15;
  58 }
  59
  60 /** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
  61  * allocated buffer, using the method described in <b>method</b>.  Store the
  62  * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
  63  * Return 0 on success, -1 on failure.
  64  */
  65 int
  66 tor_gzip_compress(char **out, size_t *out_len,
  67                   const char *in, size_t in_len,
  68                   compress_method_t method)
  69 {
  70   struct z_stream_s *stream = NULL;
  71   size_t out_size, old_size;
  72   off_t offset;
  73
  74   tor_assert(out);
  75   tor_assert(out_len);
  76   tor_assert(in);
  77   tor_assert(in_len < UINT_MAX);
  78
  79   *out = NULL;
  80
  81   if (method == GZIP_METHOD && !is_gzip_supported()) {
  82     /* Old zlib version don't support gzip in deflateInit2 */
  83     log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
  84     goto err;
  85   }
  86
  87   stream = tor_malloc_zero(sizeof(struct z_stream_s));
  88   stream->zalloc = Z_NULL;
  89   stream->zfree = Z_NULL;
  90   stream->opaque = NULL;
  91   stream->next_in = (unsigned char*) in;
  92   stream->avail_in = (unsigned int)in_len;
  93
  94   if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
  95                    method_bits(method),
  96                    8, Z_DEFAULT_STRATEGY) != Z_OK) {
  97     log_warn(LD_GENERAL, "Error from deflateInit2: %s",
  98              stream->msg?stream->msg:"<no message>");
  99     goto err;
 100   }
 101
 102   /* Guess 50% compression. */
 103   out_size = in_len / 2;
 104   if (out_size < 1024) out_size = 1024;
 105   *out = tor_malloc(out_size);
 106   stream->next_out = (unsigned char*)*out;
 107   stream->avail_out = (unsigned int)out_size;
 108
 109   while (1) {
 110     switch (deflate(stream, Z_FINISH))
 111       {
 112       case Z_STREAM_END:
 113         goto done;
 114       case Z_OK:
 115         /* In case zlib doesn't work as I think .... */
 116         if (stream->avail_out >= stream->avail_in+16)
 117           break;
 118       case Z_BUF_ERROR:
 119         offset = stream->next_out - ((unsigned char*)*out);
 120         old_size = out_size;
 121         out_size *= 2;
 122         if (out_size < old_size) {
 123           log_warn(LD_GENERAL, "Size overflow in compression.");
 124           goto err;
 125         }
 126         *out = tor_realloc(*out, out_size);
 127         stream->next_out = (unsigned char*)(*out + offset);
 128         if (out_size - offset > UINT_MAX) {
 129           log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
 130                    "uncompressing.");
 131           goto err;
 132         }
 133         stream->avail_out = (unsigned int)(out_size - offset);
 134         break;
 135       default:
 136         log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
 137                  stream->msg ? stream->msg : "<no message>");
 138         goto err;
 139       }
 140   }
 141  done:
 142   *out_len = stream->total_out;
 143 #ifdef OPENBSD
 144   /* "Hey Rocky!  Watch me change an unsigned field to a signed field in a
 145    *    third-party API!"
 146    * "Oh, that trick will just make people do unsafe casts to the unsigned
 147    *    type in their cross-platform code!"
 148    * "Don't be foolish.  I'm _sure_ they'll have the good sense to make sure
 149    *    the newly unsigned field isn't negative." */
 150   tor_assert(stream->total_out >= 0);
 151 #endif
 152   if (((size_t)stream->total_out) > out_size + 4097) {
 153     /* If we're wasting more than 4k, don't. */
 154     *out = tor_realloc(*out, stream->total_out + 1);
 155   }
 156   if (deflateEnd(stream)!=Z_OK) {
 157     log_warn(LD_BUG, "Error freeing gzip structures");
 158     goto err;
 159   }
 160   tor_free(stream);
 161
 162   return 0;
 163  err:
 164   if (stream) {
 165     deflateEnd(stream);
 166     tor_free(stream);
 167   }
 168   if (*out) {
 169     tor_free(*out);
 170   }
 171   return -1;
 172 }
 173
 174 /** Given zero or more zlib-compressed or gzip-compressed strings of
 175  * total length
 176  * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated
 177  * buffer, using the method described in <b>method</b>.  Store the uncompressed
 178  * string in *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on
 179  * success, -1 on failure.
 180  *
 181  * If <b>complete_only</b> is true, we consider a truncated input as a
 182  * failure; otherwise we decompress as much as we can.  Warn about truncated
 183  * or corrupt inputs at <b>protocol_warn_level</b>.
 184  */
 185 int
 186 tor_gzip_uncompress(char **out, size_t *out_len,
 187                     const char *in, size_t in_len,
 188                     compress_method_t method,
 189                     int complete_only,
 190                     int protocol_warn_level)
 191 {
 192   struct z_stream_s *stream = NULL;
 193   size_t out_size, old_size;
 194   off_t offset;
 195   int r;
 196
 197   tor_assert(out);
 198   tor_assert(out_len);
 199   tor_assert(in);
 200   tor_assert(in_len < UINT_MAX);
 201
 202   if (method == GZIP_METHOD && !is_gzip_supported()) {
 203     /* Old zlib version don't support gzip in inflateInit2 */
 204     log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
 205     return -1;
 206   }
 207
 208   *out = NULL;
 209
 210   stream = tor_malloc_zero(sizeof(struct z_stream_s));
 211   stream->zalloc = Z_NULL;
 212   stream->zfree = Z_NULL;
 213   stream->opaque = NULL;
 214   stream->next_in = (unsigned char*) in;
 215   stream->avail_in = (unsigned int)in_len;
 216
 217   if (inflateInit2(stream,
 218                    method_bits(method)) != Z_OK) {
 219     log_warn(LD_GENERAL, "Error from inflateInit2: %s",
 220              stream->msg?stream->msg:"<no message>");
 221     goto err;
 222   }
 223
 224   out_size = in_len * 2;  /* guess 50% compression. */
 225   if (out_size < 1024) out_size = 1024;
 226   if (out_size > UINT_MAX)
 227     goto err;
 228
 229   *out = tor_malloc(out_size);
 230   stream->next_out = (unsigned char*)*out;
 231   stream->avail_out = (unsigned int)out_size;
 232
 233   while (1) {
 234     switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
 235       {
 236       case Z_STREAM_END:
 237         if (stream->avail_in == 0)
 238           goto done;
 239         /* There may be more compressed data here. */
 240         if ((r = inflateEnd(stream)) != Z_OK) {
 241           log_warn(LD_BUG, "Error freeing gzip structures");
 242           goto err;
 243         }
 244         if (inflateInit2(stream, method_bits(method)) != Z_OK) {
 245           log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
 246                    stream->msg?stream->msg:"<no message>");
 247           goto err;
 248         }
 249         break;
 250       case Z_OK:
 251         if (!complete_only && stream->avail_in == 0)
 252           goto done;
 253         /* In case zlib doesn't work as I think.... */
 254         if (stream->avail_out >= stream->avail_in+16)
 255           break;
 256       case Z_BUF_ERROR:
 257         if (stream->avail_out > 0) {
 258           log_fn(protocol_warn_level, LD_PROTOCOL,
 259                  "possible truncated or corrupt zlib data");
 260           goto err;
 261         }
 262         offset = stream->next_out - (unsigned char*)*out;
 263         old_size = out_size;
 264         out_size *= 2;
 265         if (out_size < old_size) {
 266           log_warn(LD_GENERAL, "Size overflow in compression.");
 267           goto err;
 268         }
 269         *out = tor_realloc(*out, out_size);
 270         stream->next_out = (unsigned char*)(*out + offset);
 271         if (out_size - offset > UINT_MAX) {
 272           log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
 273                    "uncompressing.");
 274           goto err;
 275         }
 276         stream->avail_out = (unsigned int)(out_size - offset);
 277         break;
 278       default:
 279         log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
 280                  stream->msg ? stream->msg : "<no message>");
 281         goto err;
 282       }
 283   }
 284  done:
 285   *out_len = stream->next_out - (unsigned char*)*out;
 286   r = inflateEnd(stream);
 287   tor_free(stream);
 288   if (r != Z_OK) {
 289     log_warn(LD_BUG, "Error freeing gzip structures");
 290     goto err;
 291   }
 292
 293   /* NUL-terminate output. */
 294   if (out_size == *out_len)
 295     *out = tor_realloc(*out, out_size + 1);
 296   (*out)[*out_len] = '\0';
 297
 298   return 0;
 299  err:
 300   if (stream) {
 301     inflateEnd(stream);
 302     tor_free(stream);
 303   }
 304   if (*out) {
 305     tor_free(*out);
 306   }
 307   return -1;
 308 }
 309
 310 /** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
 311  * to be compressed or not.  If it is, return the likeliest compression method.
 312  * Otherwise, return UNKNOWN_METHOD.
 313  */
 314 compress_method_t
 315 detect_compression_method(const char *in, size_t in_len)
 316 {
 317   if (in_len > 2 && !memcmp(in, "\x1f\x8b", 2)) {
 318     return GZIP_METHOD;
 319   } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
 320              (ntohs(get_uint16(in)) % 31) == 0) {
 321     return ZLIB_METHOD;
 322   } else {
 323     return UNKNOWN_METHOD;
 324   }
 325 }
 326
 327 /** Internal state for an incremental zlib compression/decompression.  The
 328  * body of this struct is not exposed. */
 329 struct tor_zlib_state_t {
 330   struct z_stream_s stream;
 331   int compress;
 332 };
 333
 334 /** Construct and return a tor_zlib_state_t object using <b>method</b>.  If
 335  * <b>compress</b>, it's for compression; otherwise it's for
 336  * decompression. */
 337 tor_zlib_state_t *
 338 tor_zlib_new(int compress, compress_method_t method)
 339 {
 340   tor_zlib_state_t *out;
 341
 342   if (method == GZIP_METHOD && !is_gzip_supported()) {
 343     /* Old zlib version don't support gzip in inflateInit2 */
 344     log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION);
 345     return NULL;
 346  }
 347
 348  out = tor_malloc_zero(sizeof(tor_zlib_state_t));
 349  out->stream.zalloc = Z_NULL;
 350  out->stream.zfree = Z_NULL;
 351  out->stream.opaque = NULL;
 352  out->compress = compress;
 353  if (compress) {
 354    if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
 355                     method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK)
 356      goto err;
 357  } else {
 358    if (inflateInit2(&out->stream, method_bits(method)) != Z_OK)
 359      goto err;
 360  }
 361  return out;
 362
 363  err:
 364  tor_free(out);
 365  return NULL;
 366 }
 367
 368 /** Compress/decompress some bytes using <b>state</b>.  Read up to
 369  * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
 370  * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
 371  * we've reached the end of the input.
 372  *
 373  * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression.
 374  * Return TOR_ZLIB_OK if we're processed everything from the input.
 375  * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>.
 376  * Return TOR_ZLIB_ERR if the stream is corrupt.
 377  */
 378 tor_zlib_output_t
 379 tor_zlib_process(tor_zlib_state_t *state,
 380                  char **out, size_t *out_len,
 381                  const char **in, size_t *in_len,
 382                  int finish)
 383 {
 384   int err;
 385   tor_assert(*in_len <= UINT_MAX);
 386   tor_assert(*out_len <= UINT_MAX);
 387   state->stream.next_in = (unsigned char*) *in;
 388   state->stream.avail_in = (unsigned int)*in_len;
 389   state->stream.next_out = (unsigned char*) *out;
 390   state->stream.avail_out = (unsigned int)*out_len;
 391
 392   if (state->compress) {
 393     err = deflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
 394   } else {
 395     err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
 396   }
 397
 398   *out = (char*) state->stream.next_out;
 399   *out_len = state->stream.avail_out;
 400   *in = (const char *) state->stream.next_in;
 401   *in_len = state->stream.avail_in;
 402
 403   switch (err)
 404     {
 405     case Z_STREAM_END:
 406       return TOR_ZLIB_DONE;
 407     case Z_BUF_ERROR:
 408       if (state->stream.avail_in == 0)
 409         return TOR_ZLIB_OK;
 410       return TOR_ZLIB_BUF_FULL;
 411     case Z_OK:
 412       if (state->stream.avail_out == 0 || finish)
 413         return TOR_ZLIB_BUF_FULL;
 414       return TOR_ZLIB_OK;
 415     default:
 416       log_warn(LD_GENERAL, "Gzip returned an error: %s",
 417                state->stream.msg ? state->stream.msg : "<no message>");
 418       return TOR_ZLIB_ERR;
 419     }
 420 }
 421
 422 /** Deallocate <b>state</b>. */
 423 void
 424 tor_zlib_free(tor_zlib_state_t *state)
 425 {
 426   tor_assert(state);
 427
 428   if (state->compress)
 429     deflateEnd(&state->stream);
 430   else
 431     inflateEnd(&state->stream);
 432
 433   tor_free(state);
 434 }
 435