src/common/compress_zstd.c

   1 /* Copyright (c) 2004, Roger Dingledine.
   2  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
   3  * Copyright (c) 2007-2017, The Tor Project, Inc. */
   4 /* See LICENSE for licensing information */
   5
   6 /**
   7  * \file compress_zstd.c
   8  * \brief Compression backend for Zstandard.
   9  *
  10  * This module should never be invoked directly. Use the compress module
  11  * instead.
  12  **/
  13
  14 #include "orconfig.h"
  15
  16 #include "util.h"
  17 #include "torlog.h"
  18 #include "compress.h"
  19 #include "compress_zstd.h"
  20
  21 #ifdef HAVE_ZSTD
  22 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
  23 DISABLE_GCC_WARNING(unused-const-variable)
  24 #endif
  25 #include <zstd.h>
  26 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
  27 ENABLE_GCC_WARNING(unused-const-variable)
  28 #endif
  29 #endif
  30
  31 /** Total number of bytes allocated for Zstandard state. */
  32 static atomic_counter_t total_zstd_allocation;
  33
  34 #ifdef HAVE_ZSTD
  35 /** Given <b>level</b> return the memory level. */
  36 static int
  37 memory_level(compression_level_t level)
  38 {
  39   switch (level) {
  40     default:
  41     case BEST_COMPRESSION:
  42     case HIGH_COMPRESSION: return 9;
  43     case MEDIUM_COMPRESSION: return 8;
  44     case LOW_COMPRESSION: return 7;
  45   }
  46 }
  47 #endif /* defined(HAVE_ZSTD) */
  48
  49 /** Return 1 if Zstandard compression is supported; otherwise 0. */
  50 int
  51 tor_zstd_method_supported(void)
  52 {
  53 #ifdef HAVE_ZSTD
  54   return 1;
  55 #else
  56   return 0;
  57 #endif
  58 }
  59
  60 /** Return a string representation of the version of the currently running
  61  * version of libzstd. Returns NULL if Zstandard is unsupported. */
  62 const char *
  63 tor_zstd_get_version_str(void)
  64 {
  65 #ifdef HAVE_ZSTD
  66   static char version_str[16];
  67   size_t version_number;
  68
  69   version_number = ZSTD_versionNumber();
  70   tor_snprintf(version_str, sizeof(version_str),
  71                "%d.%d.%d",
  72                (int) version_number / 10000 % 100,
  73                (int) version_number / 100 % 100,
  74                (int) version_number % 100);
  75
  76   return version_str;
  77 #else /* !(defined(HAVE_ZSTD)) */
  78   return NULL;
  79 #endif /* defined(HAVE_ZSTD) */
  80 }
  81
  82 /** Return a string representation of the version of the version of libzstd
  83  * used at compilation time. Returns NULL if Zstandard is unsupported. */
  84 const char *
  85 tor_zstd_get_header_version_str(void)
  86 {
  87 #ifdef HAVE_ZSTD
  88   return ZSTD_VERSION_STRING;
  89 #else
  90   return NULL;
  91 #endif
  92 }
  93
  94 /** Internal Zstandard state for incremental compression/decompression.
  95  * The body of this struct is not exposed. */
  96 struct tor_zstd_compress_state_t {
  97 #ifdef HAVE_ZSTD
  98   union {
  99     /** Compression stream. Used when <b>compress</b> is true. */
 100     ZSTD_CStream *compress_stream;
 101     /** Decompression stream. Used when <b>compress</b> is false. */
 102     ZSTD_DStream *decompress_stream;
 103   } u; /**< Zstandard stream objects. */
 104 #endif /* defined(HAVE_ZSTD) */
 105
 106   int compress; /**< True if we are compressing; false if we are inflating */
 107   int have_called_end; /**< True if we are compressing and we've called
 108                         * ZSTD_endStream */
 109
 110   /** Number of bytes read so far.  Used to detect compression bombs. */
 111   size_t input_so_far;
 112   /** Number of bytes written so far.  Used to detect compression bombs. */
 113   size_t output_so_far;
 114
 115   /** Approximate number of bytes allocated for this object. */
 116   size_t allocation;
 117 };
 118
 119 #ifdef HAVE_ZSTD
 120 /** Return an approximate number of bytes stored in memory to hold the
 121  * Zstandard compression/decompression state. */
 122 static size_t
 123 tor_zstd_state_size_precalc(int compress, int preset)
 124 {
 125   tor_assert(preset > 0);
 126
 127   size_t memory_usage = sizeof(tor_zstd_compress_state_t);
 128
 129   // The Zstandard library provides a number of functions that would be useful
 130   // here, but they are, unfortunately, still considered experimental and are
 131   // thus only available in libzstd if we link against the library statically.
 132   //
 133   // The code in this function tries to approximate the calculations without
 134   // being able to use the following:
 135   //
 136   // - We do not have access to neither the internal members of ZSTD_CStream
 137   //   and ZSTD_DStream and their internal context objects.
 138   //
 139   // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they
 140   //   are unexposed.
 141   //
 142   // In the future it might be useful to check if libzstd have started
 143   // providing these functions in a stable manner and simplify this function.
 144   if (compress) {
 145     // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream)
 146     // function here. This function uses the following fields to make its
 147     // estimate:
 148
 149     // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine:
 150     memory_usage += 192;
 151
 152     // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to
 153     // variables that are not exposed via the public API. We use a _very_
 154     // simplified function to calculate the estimated amount of bytes used in
 155     // this struct.
 156     // memory_usage += (preset - 0.5) * 1024 * 1024;
 157     memory_usage += (preset * 1024 * 1024) - (512 * 1024);
 158     // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes.
 159     // - stream->outBuffSize: 128 KB:
 160     memory_usage += 128 * 1024;
 161     // - stream->inBuffSize: 2048 KB:
 162     memory_usage += 2048 * 1024;
 163   } else {
 164     // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream)
 165     // function here. This function uses the following fields to make its
 166     // estimate:
 167
 168     // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine:
 169     memory_usage += 208;
 170     // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB.
 171     memory_usage += 150 * 1024;
 172
 173     // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes.
 174     // - stream->inBuffSize: 0 KB.
 175     // - stream->outBuffSize: 0 KB.
 176   }
 177
 178   return memory_usage;
 179 }
 180 #endif /* defined(HAVE_ZSTD) */
 181
 182 /** Construct and return a tor_zstd_compress_state_t object using
 183  * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
 184  * decompression. */
 185 tor_zstd_compress_state_t *
 186 tor_zstd_compress_new(int compress,
 187                       compress_method_t method,
 188                       compression_level_t level)
 189 {
 190   tor_assert(method == ZSTD_METHOD);
 191
 192 #ifdef HAVE_ZSTD
 193   const int preset = memory_level(level);
 194   tor_zstd_compress_state_t *result;
 195   size_t retval;
 196
 197   result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
 198   result->compress = compress;
 199   result->allocation = tor_zstd_state_size_precalc(compress, preset);
 200
 201   if (compress) {
 202     result->u.compress_stream = ZSTD_createCStream();
 203
 204     if (result->u.compress_stream == NULL) {
 205       // LCOV_EXCL_START
 206       log_warn(LD_GENERAL, "Error while creating Zstandard compression "
 207                "stream");
 208       goto err;
 209       // LCOV_EXCL_STOP
 210     }
 211
 212     retval = ZSTD_initCStream(result->u.compress_stream, preset);
 213
 214     if (ZSTD_isError(retval)) {
 215       // LCOV_EXCL_START
 216       log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
 217                ZSTD_getErrorName(retval));
 218       goto err;
 219       // LCOV_EXCL_STOP
 220     }
 221   } else {
 222     result->u.decompress_stream = ZSTD_createDStream();
 223
 224     if (result->u.decompress_stream == NULL) {
 225       // LCOV_EXCL_START
 226       log_warn(LD_GENERAL, "Error while creating Zstandard decompression "
 227                "stream");
 228       goto err;
 229       // LCOV_EXCL_STOP
 230     }
 231
 232     retval = ZSTD_initDStream(result->u.decompress_stream);
 233
 234     if (ZSTD_isError(retval)) {
 235       // LCOV_EXCL_START
 236       log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
 237                ZSTD_getErrorName(retval));
 238       goto err;
 239       // LCOV_EXCL_STOP
 240     }
 241   }
 242
 243   atomic_counter_add(&total_zstd_allocation, result->allocation);
 244   return result;
 245
 246  err:
 247   // LCOV_EXCL_START
 248   if (compress) {
 249     ZSTD_freeCStream(result->u.compress_stream);
 250   } else {
 251     ZSTD_freeDStream(result->u.decompress_stream);
 252   }
 253
 254   tor_free(result);
 255   return NULL;
 256   // LCOV_EXCL_STOP
 257 #else /* !(defined(HAVE_ZSTD)) */
 258   (void)compress;
 259   (void)method;
 260   (void)level;
 261
 262   return NULL;
 263 #endif /* defined(HAVE_ZSTD) */
 264 }
 265
 266 /** Compress/decompress some bytes using <b>state</b>.  Read up to
 267  * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
 268  * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
 269  * we've reached the end of the input.
 270  *
 271  * Return TOR_COMPRESS_DONE if we've finished the entire
 272  * compression/decompression.
 273  * Return TOR_COMPRESS_OK if we're processed everything from the input.
 274  * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
 275  * Return TOR_COMPRESS_ERROR if the stream is corrupt.
 276  */
 277 tor_compress_output_t
 278 tor_zstd_compress_process(tor_zstd_compress_state_t *state,
 279                           char **out, size_t *out_len,
 280                           const char **in, size_t *in_len,
 281                           int finish)
 282 {
 283 #ifdef HAVE_ZSTD
 284   size_t retval;
 285
 286   tor_assert(state != NULL);
 287   tor_assert(*in_len <= UINT_MAX);
 288   tor_assert(*out_len <= UINT_MAX);
 289
 290   ZSTD_inBuffer input = { *in, *in_len, 0 };
 291   ZSTD_outBuffer output = { *out, *out_len, 0 };
 292
 293   if (BUG(finish == 0 && state->have_called_end)) {
 294     finish = 1;
 295   }
 296
 297   if (state->compress) {
 298     if (! state->have_called_end)
 299       retval = ZSTD_compressStream(state->u.compress_stream,
 300                                    &output, &input);
 301     else
 302       retval = 0;
 303   } else {
 304     retval = ZSTD_decompressStream(state->u.decompress_stream,
 305                                    &output, &input);
 306   }
 307
 308   state->input_so_far += input.pos;
 309   state->output_so_far += output.pos;
 310
 311   *out = (char *)output.dst + output.pos;
 312   *out_len = output.size - output.pos;
 313   *in = (char *)input.src + input.pos;
 314   *in_len = input.size - input.pos;
 315
 316   if (! state->compress &&
 317       tor_compress_is_compression_bomb(state->input_so_far,
 318                                        state->output_so_far)) {
 319     log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
 320     return TOR_COMPRESS_ERROR;
 321   }
 322
 323   if (ZSTD_isError(retval)) {
 324     log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.",
 325              state->compress ? "compression" : "decompression",
 326              ZSTD_getErrorName(retval));
 327     return TOR_COMPRESS_ERROR;
 328   }
 329
 330   if (state->compress && !state->have_called_end) {
 331     retval = ZSTD_flushStream(state->u.compress_stream, &output);
 332
 333     *out = (char *)output.dst + output.pos;
 334     *out_len = output.size - output.pos;
 335
 336     if (ZSTD_isError(retval)) {
 337       log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.",
 338                ZSTD_getErrorName(retval));
 339       return TOR_COMPRESS_ERROR;
 340     }
 341
 342     // ZSTD_flushStream returns 0 if the frame is done, or >0 if it
 343     // is incomplete.
 344     if (retval > 0) {
 345       return TOR_COMPRESS_BUFFER_FULL;
 346     }
 347   }
 348
 349   if (!finish) {
 350     // The caller says we're not done with the input, so no need to write an
 351     // epilogue.
 352     return TOR_COMPRESS_OK;
 353   } else if (state->compress) {
 354     if (*in_len) {
 355       // We say that we're not done with the input, so we can't write an
 356       // epilogue.
 357       return TOR_COMPRESS_OK;
 358     }
 359
 360     retval = ZSTD_endStream(state->u.compress_stream, &output);
 361     state->have_called_end = 1;
 362     *out = (char *)output.dst + output.pos;
 363     *out_len = output.size - output.pos;
 364
 365     if (ZSTD_isError(retval)) {
 366       log_warn(LD_GENERAL, "Zstandard compression unable to write "
 367                "epilogue: %s.",
 368                ZSTD_getErrorName(retval));
 369       return TOR_COMPRESS_ERROR;
 370     }
 371
 372     // endStream returns the number of bytes that is needed to write the
 373     // epilogue.
 374     if (retval > 0)
 375       return TOR_COMPRESS_BUFFER_FULL;
 376
 377     return TOR_COMPRESS_DONE;
 378   } else /* if (!state->compress) */ {
 379     // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it
 380     // is incomplete.
 381     // We check this above.
 382     tor_assert_nonfatal(!ZSTD_isError(retval));
 383     // Start a new frame if this frame is done
 384     if (retval == 0)
 385       return TOR_COMPRESS_DONE;
 386     // Don't check out_len, it might have some space left if the next output
 387     // chunk is larger than the remaining space
 388     else if (*in_len > 0)
 389       return  TOR_COMPRESS_BUFFER_FULL;
 390     else
 391       return TOR_COMPRESS_OK;
 392   }
 393
 394 #else /* !(defined(HAVE_ZSTD)) */
 395   (void)state;
 396   (void)out;
 397   (void)out_len;
 398   (void)in;
 399   (void)in_len;
 400   (void)finish;
 401
 402   return TOR_COMPRESS_ERROR;
 403 #endif /* defined(HAVE_ZSTD) */
 404 }
 405
 406 /** Deallocate <b>state</b>. */
 407 void
 408 tor_zstd_compress_free_(tor_zstd_compress_state_t *state)
 409 {
 410   if (state == NULL)
 411     return;
 412
 413   atomic_counter_sub(&total_zstd_allocation, state->allocation);
 414
 415 #ifdef HAVE_ZSTD
 416   if (state->compress) {
 417     ZSTD_freeCStream(state->u.compress_stream);
 418   } else {
 419     ZSTD_freeDStream(state->u.decompress_stream);
 420   }
 421 #endif /* defined(HAVE_ZSTD) */
 422
 423   tor_free(state);
 424 }
 425
 426 /** Return the approximate number of bytes allocated for <b>state</b>. */
 427 size_t
 428 tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state)
 429 {
 430   tor_assert(state != NULL);
 431   return state->allocation;
 432 }
 433
 434 /** Return the approximate number of bytes allocated for all Zstandard
 435  * states. */
 436 size_t
 437 tor_zstd_get_total_allocation(void)
 438 {
 439   return atomic_counter_get(&total_zstd_allocation);
 440 }
 441
 442 /** Initialize the zstd module */
 443 void
 444 tor_zstd_init(void)
 445 {
 446   atomic_counter_init(&total_zstd_allocation);
 447 }
 448