src/common/compress_zstd.c

   1 /* Copyright (c) 2004, Roger Dingledine.
   2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
   3  * Copyright (c) 2007-2017, The Tor Project, Inc. */
   4 /* See LICENSE for licensing information */
   5
   6 /**
   7  * \file compress_zstd.c
   8  * \brief Compression backend for Zstandard.
   9  *
  10  * This module should never be invoked directly. Use the compress module
  11  * instead.
  12  **/
  13
  14 #include "orconfig.h"
  15
  16 #include "util.h"
  17 #include "torlog.h"
  18 #include "compress.h"
  19 #include "compress_zstd.h"
  20
  21 #ifdef HAVE_ZSTD
  22 #include <zstd.h>
  23 #endif
  24
  25 /** Total number of bytes allocated for Zstandard state. */
  26 static atomic_counter_t total_zstd_allocation;
  27
  28 #ifdef HAVE_ZSTD
  29 /** Given <b>level</b> return the memory level. */
  30 static int
  31 memory_level(compression_level_t level)
  32 {
  33   switch (level) {
  34     default:
  35     case BEST_COMPRESSION:
  36     case HIGH_COMPRESSION: return 9;
  37     case MEDIUM_COMPRESSION: return 8;
  38     case LOW_COMPRESSION: return 7;
  39   }
  40 }
  41 #endif // HAVE_ZSTD.
  42
  43 /** Return 1 if Zstandard compression is supported; otherwise 0. */
  44 int
  45 tor_zstd_method_supported(void)
  46 {
  47 #ifdef HAVE_ZSTD
  48   return 1;
  49 #else
  50   return 0;
  51 #endif
  52 }
  53
  54 /** Return a string representation of the version of the currently running
  55  * version of libzstd. Returns NULL if Zstandard is unsupported. */
  56 const char *
  57 tor_zstd_get_version_str(void)
  58 {
  59 #ifdef HAVE_ZSTD
  60   static char version_str[16];
  61   size_t version_number;
  62
  63   version_number = ZSTD_versionNumber();
  64   tor_snprintf(version_str, sizeof(version_str),
  65                "%lu.%lu.%lu",
  66                version_number / 10000 % 100,
  67                version_number / 100 % 100,
  68                version_number % 100);
  69
  70   return version_str;
  71 #else
  72   return NULL;
  73 #endif
  74 }
  75
  76 /** Return a string representation of the version of the version of libzstd
  77  * used at compilation time. Returns NULL if Zstandard is unsupported. */
  78 const char *
  79 tor_zstd_get_header_version_str(void)
  80 {
  81 #ifdef HAVE_ZSTD
  82   return ZSTD_VERSION_STRING;
  83 #else
  84   return NULL;
  85 #endif
  86 }
  87
  88 /** Internal Zstandard state for incremental compression/decompression.
  89  * The body of this struct is not exposed. */
  90 struct tor_zstd_compress_state_t {
  91 #ifdef HAVE_ZSTD
  92   union {
  93     /** Compression stream. Used when <b>compress</b> is true. */
  94     ZSTD_CStream *compress_stream;
  95     /** Decompression stream. Used when <b>compress</b> is false. */
  96     ZSTD_DStream *decompress_stream;
  97   } u; /**< Zstandard stream objects. */
  98 #endif // HAVE_ZSTD.
  99
 100   int compress; /**< True if we are compressing; false if we are inflating */
 101
 102   /** Number of bytes read so far.  Used to detect compression bombs. */
 103   size_t input_so_far;
 104   /** Number of bytes written so far.  Used to detect compression bombs. */
 105   size_t output_so_far;
 106
 107   /** Approximate number of bytes allocated for this object. */
 108   size_t allocation;
 109 };
 110
 111 #ifdef HAVE_ZSTD
 112 /** Return an approximate number of bytes stored in memory to hold the
 113  * Zstandard compression/decompression state. */
 114 static size_t
 115 tor_zstd_state_size_precalc(int compress, int preset)
 116 {
 117   tor_assert(preset > 0);
 118
 119   size_t memory_usage = sizeof(tor_zstd_compress_state_t);
 120
 121   // The Zstandard library provides a number of functions that would be useful
 122   // here, but they are, unfortunately, still considered experimental and are
 123   // thus only available in libzstd if we link against the library statically.
 124   //
 125   // The code in this function tries to approximate the calculations without
 126   // being able to use the following:
 127   //
 128   // - We do not have access to neither the internal members of ZSTD_CStream
 129   //   and ZSTD_DStream and their internal context objects.
 130   //
 131   // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they
 132   //   are unexposed.
 133   //
 134   // In the future it might be useful to check if libzstd have started
 135   // providing these functions in a stable manner and simplify this function.
 136   if (compress) {
 137     // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream)
 138     // function here. This function uses the following fields to make its
 139     // estimate:
 140
 141     // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine:
 142     memory_usage += 192;
 143
 144     // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to
 145     // variables that are not exposed via the public API. We use a _very_
 146     // simplified function to calculate the estimated amount of bytes used in
 147     // this struct.
 148     // memory_usage += (preset - 0.5) * 1024 * 1024;
 149     memory_usage += (preset * 1024 * 1024) - (512 * 1024);
 150     // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes.
 151     // - stream->outBuffSize: 128 KB:
 152     memory_usage += 128 * 1024;
 153     // - stream->inBuffSize: 2048 KB:
 154     memory_usage += 2048 * 1024;
 155   } else {
 156     // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream)
 157     // function here. This function uses the following fields to make its
 158     // estimate:
 159
 160     // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine:
 161     memory_usage += 208;
 162     // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB.
 163     memory_usage += 150 * 1024;
 164
 165     // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes.
 166     // - stream->inBuffSize: 0 KB.
 167     // - stream->outBuffSize: 0 KB.
 168   }
 169
 170   return memory_usage;
 171 }
 172 #endif // HAVE_ZSTD.
 173
 174 /** Construct and return a tor_zstd_compress_state_t object using
 175  * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
 176  * decompression. */
 177 tor_zstd_compress_state_t *
 178 tor_zstd_compress_new(int compress,
 179                       compress_method_t method,
 180                       compression_level_t level)
 181 {
 182   tor_assert(method == ZSTD_METHOD);
 183
 184 #ifdef HAVE_ZSTD
 185   const int preset = memory_level(level);
 186   tor_zstd_compress_state_t *result;
 187   size_t retval;
 188
 189   result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
 190   result->compress = compress;
 191   result->allocation = tor_zstd_state_size_precalc(compress, preset);
 192
 193   if (compress) {
 194     result->u.compress_stream = ZSTD_createCStream();
 195
 196     if (result->u.compress_stream == NULL) {
 197       log_warn(LD_GENERAL, "Error while creating Zstandard stream");
 198       goto err;
 199     }
 200
 201     retval = ZSTD_initCStream(result->u.compress_stream, preset);
 202
 203     if (ZSTD_isError(retval)) {
 204       log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
 205                ZSTD_getErrorName(retval));
 206       goto err;
 207     }
 208   } else {
 209     result->u.decompress_stream = ZSTD_createDStream();
 210
 211     if (result->u.decompress_stream == NULL) {
 212       log_warn(LD_GENERAL, "Error while creating Zstandard stream");
 213       goto err;
 214     }
 215
 216     retval = ZSTD_initDStream(result->u.decompress_stream);
 217
 218     if (ZSTD_isError(retval)) {
 219       log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
 220                ZSTD_getErrorName(retval));
 221       goto err;
 222     }
 223   }
 224
 225   atomic_counter_add(&total_zstd_allocation, result->allocation);
 226   return result;
 227
 228  err:
 229   if (compress) {
 230     ZSTD_freeCStream(result->u.compress_stream);
 231   } else {
 232     ZSTD_freeDStream(result->u.decompress_stream);
 233   }
 234
 235   tor_free(result);
 236   return NULL;
 237 #else // HAVE_ZSTD.
 238   (void)compress;
 239   (void)method;
 240   (void)level;
 241
 242   return NULL;
 243 #endif // HAVE_ZSTD.
 244 }
 245
 246 /** Compress/decompress some bytes using <b>state</b>.  Read up to
 247  * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
 248  * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
 249  * we've reached the end of the input.
 250  *
 251  * Return TOR_COMPRESS_DONE if we've finished the entire
 252  * compression/decompression.
 253  * Return TOR_COMPRESS_OK if we're processed everything from the input.
 254  * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
 255  * Return TOR_COMPRESS_ERROR if the stream is corrupt.
 256  */
 257 tor_compress_output_t
 258 tor_zstd_compress_process(tor_zstd_compress_state_t *state,
 259                           char **out, size_t *out_len,
 260                           const char **in, size_t *in_len,
 261                           int finish)
 262 {
 263 #ifdef HAVE_ZSTD
 264   size_t retval;
 265
 266   tor_assert(state != NULL);
 267   tor_assert(*in_len <= UINT_MAX);
 268   tor_assert(*out_len <= UINT_MAX);
 269
 270   ZSTD_inBuffer input = { *in, *in_len, 0 };
 271   ZSTD_outBuffer output = { *out, *out_len, 0 };
 272
 273   if (state->compress) {
 274     retval = ZSTD_compressStream(state->u.compress_stream,
 275                                  &output, &input);
 276   } else {
 277     retval = ZSTD_decompressStream(state->u.decompress_stream,
 278                                    &output, &input);
 279   }
 280
 281   state->input_so_far += input.pos;
 282   state->output_so_far += output.pos;
 283
 284   *out = (char *)output.dst + output.pos;
 285   *out_len = output.size - output.pos;
 286   *in = (char *)input.src + input.pos;
 287   *in_len = input.size - input.pos;
 288
 289   if (! state->compress &&
 290       tor_compress_is_compression_bomb(state->input_so_far,
 291                                        state->output_so_far)) {
 292     log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
 293     return TOR_COMPRESS_ERROR;
 294   }
 295
 296   if (ZSTD_isError(retval)) {
 297     log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.",
 298              state->compress ? "compression" : "decompression",
 299              ZSTD_getErrorName(retval));
 300     return TOR_COMPRESS_ERROR;
 301   }
 302
 303   if (state->compress && !finish) {
 304     retval = ZSTD_flushStream(state->u.compress_stream, &output);
 305
 306     *out = (char *)output.dst + output.pos;
 307     *out_len = output.size - output.pos;
 308
 309     if (ZSTD_isError(retval)) {
 310       log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.",
 311                ZSTD_getErrorName(retval));
 312       return TOR_COMPRESS_ERROR;
 313     }
 314
 315     if (retval > 0)
 316       return TOR_COMPRESS_BUFFER_FULL;
 317   }
 318
 319   if (!finish) {
 320     // We're not done with the input, so no need to flush.
 321     return TOR_COMPRESS_OK;
 322   } else if (state->compress && finish) {
 323     retval = ZSTD_endStream(state->u.compress_stream, &output);
 324
 325     *out = (char *)output.dst + output.pos;
 326     *out_len = output.size - output.pos;
 327
 328     if (ZSTD_isError(retval)) {
 329       log_warn(LD_GENERAL, "Zstandard compression unable to write "
 330                "epilogue: %s.",
 331                ZSTD_getErrorName(retval));
 332       return TOR_COMPRESS_ERROR;
 333     }
 334
 335     // endStream returns the number of bytes that is needed to write the
 336     // epilogue.
 337     if (retval > 0)
 338       return TOR_COMPRESS_BUFFER_FULL;
 339
 340     return TOR_COMPRESS_DONE;
 341   } else {
 342     // ZSTD_flushStream returns 0 if the frame is done, or >0 if it
 343     // is incomplete.
 344     return (retval == 0) ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK;
 345   }
 346
 347 #else // HAVE_ZSTD.
 348   (void)state;
 349   (void)out;
 350   (void)out_len;
 351   (void)in;
 352   (void)in_len;
 353   (void)finish;
 354
 355   return TOR_COMPRESS_ERROR;
 356 #endif // HAVE_ZSTD.
 357 }
 358
 359 /** Deallocate <b>state</b>. */
 360 void
 361 tor_zstd_compress_free(tor_zstd_compress_state_t *state)
 362 {
 363   if (state == NULL)
 364     return;
 365
 366   atomic_counter_sub(&total_zstd_allocation, state->allocation);
 367
 368 #ifdef HAVE_ZSTD
 369   if (state->compress) {
 370     ZSTD_freeCStream(state->u.compress_stream);
 371   } else {
 372     ZSTD_freeDStream(state->u.decompress_stream);
 373   }
 374 #endif // HAVE_ZSTD.
 375
 376   tor_free(state);
 377 }
 378
 379 /** Return the approximate number of bytes allocated for <b>state</b>. */
 380 size_t
 381 tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state)
 382 {
 383   tor_assert(state != NULL);
 384   return state->allocation;
 385 }
 386
 387 /** Return the approximate number of bytes allocated for all Zstandard
 388  * states. */
 389 size_t
 390 tor_zstd_get_total_allocation(void)
 391 {
 392   return atomic_counter_get(&total_zstd_allocation);
 393 }
 394
 395 /** Initialize the zstd module */
 396 void
 397 tor_zstd_init(void)
 398 {
 399   atomic_counter_init(&total_zstd_allocation);
 400 }
 401