hphp/util/brotli.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/util/brotli.h"
  18 #include <enc/encode.h>
  19 #include <folly/ScopeGuard.h>
  20
  21 // The version of brotli we are using does implement these helper
  22 // methods that we need. However the methods are not declared in
  23 // the core brotli libraries. Lets declare those here.
  24 namespace brotli {
  25 size_t CopyOneBlockToRingBuffer(BrotliIn* r,
  26                                 BrotliCompressor* compressor);
  27 bool BrotliInIsFinished(brotli::BrotliIn* r);
  28 }
  29
  30 using namespace brotli;
  31
  32
  33 namespace HPHP {
  34 ///////////////////////////////////////////////////////////////////////////////
  35
  36 const char* compressBrotli(BrotliCompressor* compressor,
  37                            const void* data,
  38                            size_t& len,
  39                            bool last) {
  40   // Brotli does not have a utility to compute max size of the buffer
  41   // in case data is incompressible. Below link discusses some numbers
  42   // and formula where 16MB block would use only 6 extra bytes.
  43   // For all practical usage we should be fine with 20 bytes.
  44   // https://github.com/google/brotli/issues/274
  45   // We should also allow 6 extra bytes for an empty meta-block at
  46   // the end of each chunk to force "flush".
  47   size_t availableBytes = len + 30;
  48   auto available = (char *)malloc(len + availableBytes);
  49   auto deleter = folly::makeGuard([&] { free(available); });
  50
  51   BrotliMemIn in(data, len);
  52   BrotliMemOut out(available, availableBytes);
  53   bool finalBlock = false;
  54   while (!finalBlock) {
  55     auto inBytes = CopyOneBlockToRingBuffer(&in, compressor);
  56     finalBlock = inBytes == 0 || BrotliInIsFinished(&in);
  57     size_t outBytes = 0;
  58     uint8_t* output = nullptr;
  59     if (!compressor->WriteBrotliData(last && finalBlock,
  60                                              /* force_flush */ finalBlock,
  61                                              &outBytes,
  62                                              &output)) {
  63       return nullptr;
  64     }
  65
  66     // This deserves an explanation as brotli's documentation is
  67     // really incomplete on the topic.
  68     // 'force_flush' is what they call a "soft flush" and it stops at the byte
  69     // boundary of the compressed buffer. As such, there is a 7/8 chance that
  70     // last few bytes will be held by the compressor. To force the compressor
  71     // stop at the byte boundary one can write an empty meta-block.
  72     if (!last && finalBlock) {
  73       size_t bytes = 6;
  74       compressor->WriteMetadata(
  75           0, nullptr, false, &bytes, output + outBytes);
  76       outBytes += bytes;
  77     }
  78
  79     if (outBytes > 0 && !out.Write(output, outBytes)) {
  80       return nullptr;
  81     }
  82   }
  83
  84   deleter.dismiss();
  85   len = out.position();
  86
  87   return available;
  88 }
  89
  90 ///////////////////////////////////////////////////////////////////////////////
  91 }