From 1f35f62b8fc0b9b43221906a267fa515b77ed693 Mon Sep 17 00:00:00 2001 From: Misha Shneerson Date: Fri, 18 Mar 2016 12:44:40 -0700 Subject: [PATCH] Brotli support Summary:Support for brotli compression. Until now we have only supported gzip encoding, and now we are adding brotli as well. Since only gzip was supported until now, we often made binary decision - we either do have compression or not. Now we need to consider which type of compression is enabled / disabled so this code got a bit more involved. On top of that, due to what seems to be a bug in FF, brotli by default is disabled for chunked responses. However, there is an option to enable it at runtime by calling: ini_set("brotli.chunked_compression", "on"); Reviewed By: markw65 Differential Revision: D3010183 fb-gh-sync-id: 9048064017fd8d2b6064f236f72e73dda8495413 shipit-source-id: 9048064017fd8d2b6064f236f72e73dda8495413 --- hphp/runtime/base/request-injection-data.cpp | 15 ++ hphp/runtime/base/request-injection-data.h | 3 + hphp/runtime/base/runtime-option.cpp | 15 ++ hphp/runtime/base/runtime-option.h | 6 + hphp/runtime/server/transport.cpp | 220 +++++++++++++++++++++------ hphp/runtime/server/transport.h | 38 ++++- hphp/util/brotli.cpp | 91 +++++++++++ hphp/util/brotli.h | 37 +++++ hphp/util/test/brotli-test.cpp | 101 ++++++++++++ 9 files changed, 473 insertions(+), 53 deletions(-) create mode 100644 hphp/util/brotli.cpp create mode 100644 hphp/util/brotli.h create mode 100644 hphp/util/test/brotli-test.cpp diff --git a/hphp/runtime/base/request-injection-data.cpp b/hphp/runtime/base/request-injection-data.cpp index 3c5bf9baa5a..9ab7a8df109 100644 --- a/hphp/runtime/base/request-injection-data.cpp +++ b/hphp/runtime/base/request-injection-data.cpp @@ -497,6 +497,21 @@ void RequestInjectionData::threadInit() { IniSetting::Bind(IniSetting::CORE, IniSetting::PHP_INI_ALL, "zlib.output_compression_level", &m_gzipCompressionLevel); + IniSetting::Bind(IniSetting::CORE, IniSetting::PHP_INI_ALL, + "brotli.chunked_compression", &m_brotliChunkedEnabled); + IniSetting::Bind( + IniSetting::CORE, + IniSetting::PHP_INI_ALL, + "brotli.compression_quality", + std::to_string(RuntimeOption::BrotliCompressionQuality).c_str(), + &m_brotliQuality); + IniSetting::Bind( + IniSetting::CORE, + IniSetting::PHP_INI_ALL, + "brotli.compression_lgwin", + std::to_string(RuntimeOption::BrotliCompressionLgWindowSize).c_str(), + &m_brotliLgWindowSize); + // Assertions IniSetting::Bind(IniSetting::CORE, IniSetting::PHP_INI_ALL, "zend.assertions", "1", diff --git a/hphp/runtime/base/request-injection-data.h b/hphp/runtime/base/request-injection-data.h index e13c58e93a7..dc0683a4ef3 100644 --- a/hphp/runtime/base/request-injection-data.h +++ b/hphp/runtime/base/request-injection-data.h @@ -304,6 +304,7 @@ private: std::string m_requestOrder; std::string m_defaultCharset; std::string m_defaultMimeType; + std::string m_brotliChunkedEnabled; std::string m_gzipCompressionLevel = "-1"; std::string m_gzipCompression; std::string m_errorLog; @@ -322,6 +323,8 @@ private: int64_t m_socketDefaultTimeout; int64_t m_maxMemoryNumeric; int64_t m_zendAssertions; + int64_t m_brotliLgWindowSize; + int64_t m_brotliQuality; /* * Keep track of the open_basedir_separator that may be used so we can diff --git a/hphp/runtime/base/runtime-option.cpp b/hphp/runtime/base/runtime-option.cpp index ee510ca188f..0119fdcf83d 100644 --- a/hphp/runtime/base/runtime-option.cpp +++ b/hphp/runtime/base/runtime-option.cpp @@ -185,6 +185,11 @@ int RuntimeOption::ServerShutdownListenWait = 0; int RuntimeOption::ServerShutdownEOMWait = 0; std::vector RuntimeOption::ServerNextProtocols; bool RuntimeOption::ServerEnableH2C = false; +int RuntimeOption::BrotliCompressionEnabled = 1; +int RuntimeOption::BrotliChunkedCompressionEnabled = -1; +int RuntimeOption::BrotliCompressionMode = 0; +int RuntimeOption::BrotliCompressionQuality = 11; +int RuntimeOption::BrotliCompressionLgWindowSize = 22; int RuntimeOption::GzipCompressionLevel = 3; int RuntimeOption::GzipMaxCompressionLevel = 9; std::string RuntimeOption::ForceCompressionURL; @@ -1306,6 +1311,16 @@ void RuntimeOption::Load( "Server.ShutdownEOMWait", 0); Config::Bind(ServerNextProtocols, ini, config, "Server.SSLNextProtocols"); Config::Bind(ServerEnableH2C, ini, config, "Server.EnableH2C"); + Config::Bind(BrotliCompressionEnabled, ini, config, + "Server.BrotliCompressionEnabled", 1); + Config::Bind(BrotliChunkedCompressionEnabled, ini, config, + "Server.BrotliChunkedCompressionEnabled", -1); + Config::Bind(BrotliCompressionLgWindowSize, ini, config, + "Server.BrotliCompressionLgWindowSize", 22); + Config::Bind(BrotliCompressionMode, ini, config, + "Server.BrotliCompressionMode", 0); + Config::Bind(BrotliCompressionQuality, ini, config, + "Server.BrotliCompressionQuality", 11); Config::Bind(GzipCompressionLevel, ini, config, "Server.GzipCompressionLevel", 3); Config::Bind(GzipMaxCompressionLevel, ini, config, diff --git a/hphp/runtime/base/runtime-option.h b/hphp/runtime/base/runtime-option.h index 1e3a2e04505..d68789c894a 100644 --- a/hphp/runtime/base/runtime-option.h +++ b/hphp/runtime/base/runtime-option.h @@ -164,6 +164,12 @@ struct RuntimeOption { static int ServerShutdownEOMWait; static std::vector ServerNextProtocols; static bool ServerEnableH2C; + static int BrotliCompressionEnabled; + static int BrotliChunkedCompressionEnabled; + static int BrotliCompressionMode; + // Base 2 logarithm of the sliding window size. Range is 10-24. + static int BrotliCompressionLgWindowSize; + static int BrotliCompressionQuality; static int GzipCompressionLevel; static int GzipMaxCompressionLevel; static std::string ForceCompressionURL; diff --git a/hphp/runtime/server/transport.cpp b/hphp/runtime/server/transport.cpp index 8e568070087..888017169c6 100644 --- a/hphp/runtime/server/transport.cpp +++ b/hphp/runtime/server/transport.cpp @@ -33,6 +33,7 @@ #include "hphp/runtime/server/http-protocol.h" #include "hphp/runtime/ext/openssl/ext_openssl.h" #include "hphp/util/compatibility.h" +#include "hphp/util/brotli.h" #include "hphp/util/compression.h" #include "hphp/util/hardware-counter.h" #include "hphp/util/logger.h" @@ -41,11 +42,14 @@ #include "hphp/util/timer.h" #include "hphp/runtime/ext/string/ext_string.h" #include +#include namespace HPHP { /////////////////////////////////////////////////////////////////////////////// static const char HTTP_RESPONSE_STATS_PREFIX[] = "http_response_"; +const char* Transport::ENCODING_TYPE_TO_NAME[CompressionType::Max + 1] = { + "br", "br", "gzip", ""}; Transport::Transport() : m_instructions(0), m_sleepTime(0), m_usleepTime(0), @@ -56,12 +60,14 @@ Transport::Transport() m_responseCode(-1), m_firstHeaderSet(false), m_firstHeaderLine(0), m_responseSize(0), m_responseTotalSize(0), m_responseSentSize(0), m_flushTimeUs(0), m_sendEnded(false), m_sendContentType(true), - m_compression(true), m_compressor(nullptr), m_isSSL(false), + m_encodingType(CompressionType::Max), m_isSSL(false), m_compressionDecision(CompressionDecision::NotDecidedYet), m_threadType(ThreadType::RequestThread) { memset(&m_queueTime, 0, sizeof(m_queueTime)); memset(&m_wallTime, 0, sizeof(m_wallTime)); memset(&m_cpuTime, 0, sizeof(m_cpuTime)); + memset(m_acceptedEncodings, 0, sizeof(m_acceptedEncodings)); + enableCompression(); m_chunksSentSizes.clear(); tvWriteUninit(&m_headerCallback); } @@ -73,9 +79,6 @@ Transport::~Transport() { if (m_postData) { free(m_postData); } - if (m_compressor) { - delete m_compressor; - } m_chunksSentSizes.clear(); } @@ -500,16 +503,38 @@ bool Transport::decideCompression() { if (!RuntimeOption::ForceCompressionURL.empty() && getCommand() == RuntimeOption::ForceCompressionURL) { + // ForceCompression exists only to support cases when proxy removes + // Accept-Encoding header but browser can read compressed data. This + // feature is much less relevant since HTTPS does not allow proxies to + // remove headers. So we won't expand support for this feature to + // new compression types. + m_acceptedEncodings[CompressionType::Gzip] = true; m_compressionDecision = CompressionDecision::HasTo; return true; } - if (acceptEncoding("gzip") || - (!RuntimeOption::ForceCompressionCookie.empty() && + bool acceptsEncoding = false; + if (acceptEncoding("br")) { + m_acceptedEncodings[CompressionType::Brotli] = true; + m_acceptedEncodings[CompressionType::BrotliChunked] = true; + acceptsEncoding = true; + } + if (acceptEncoding("gzip")) { + m_acceptedEncodings[CompressionType::Gzip] = true; + acceptsEncoding = true; + } + + if (acceptsEncoding) { + m_compressionDecision = CompressionDecision::Should; + return true; + } + + if ((!RuntimeOption::ForceCompressionCookie.empty() && cookieExists(RuntimeOption::ForceCompressionCookie.c_str())) || (!RuntimeOption::ForceCompressionParam.empty() && paramExists(RuntimeOption::ForceCompressionParam.c_str()))) { m_compressionDecision = CompressionDecision::Should; + m_acceptedEncodings[CompressionType::Gzip] = true; return true; } @@ -695,16 +720,17 @@ void Transport::prepareHeaders(bool compressed, bool chunked, * Our response may vary depending on the Accept-Encoding header if * - we compressed it, and compression was not forced; or * - we didn't compress it because the client does not accept gzip + * or brotli. */ - if (compressed ? - m_compressionDecision != CompressionDecision::HasTo : - (isCompressionEnabled() && !acceptEncoding("gzip"))) { + if (compressed ? m_compressionDecision != CompressionDecision::HasTo + : (isCompressionEnabled() && + !(acceptEncoding("gzip") || acceptEncoding("br")))) { addHeaderImpl("Vary", "Accept-Encoding"); } } if (compressed) { - addHeaderImpl("Content-Encoding", "gzip"); + addHeaderImpl("Content-Encoding", compressionName(m_encodingType)); removeHeaderImpl("Content-Length"); // Remove the Content-MD5 header coming from PHP if we compressed the data, // as the checksum is going to be invalid. @@ -798,9 +824,15 @@ void LogException(const char* msg) { } -StringHolder Transport::prepareResponse(const void *data, int size, - bool &compressed, bool last) { - StringHolder response((const char *)data, size); +const char* Transport::compressionName(CompressionType type) { + return ENCODING_TYPE_TO_NAME[static_cast(type)]; +} + +StringHolder Transport::prepareResponse(const void* data, + int size, + bool& compressed, + bool last) { + StringHolder response((const char*)data, size); // we don't use chunk encoding to send anything pre-compressed assert(!compressed || !m_chunkedEncoding); @@ -808,50 +840,148 @@ StringHolder Transport::prepareResponse(const void *data, int size, if (m_compressionDecision == CompressionDecision::NotDecidedYet) { decideCompression(); } - if (compressed || !isCompressionEnabled() || + + if (compressed) { + // pre-compressed responses are always gzip + m_encodingType = CompressionType::Gzip; + return response; + } + + if (!isCompressionEnabled() || m_compressionDecision == CompressionDecision::ShouldNot) { return response; } - // Gzip has 20 bytes header, so anything smaller than a few bytes probably - // wouldn't benefit much from compression - if (m_chunkedEncoding || size > 50 || - m_compressionDecision == CompressionDecision::HasTo) { - String compression; - int compressionLevel = RuntimeOption::GzipCompressionLevel; - IniSetting::Get("zlib.output_compression", compression); - if (compression.size() == 2 && bstrcaseeq(compression.data(), "on", 2)) { - String compressionLevelStr; - IniSetting::Get("zlib.output_compression_level", compressionLevelStr); - int level = compressionLevelStr.toInt64(); - if (level > compressionLevel && - level <= RuntimeOption::GzipMaxCompressionLevel) { - compressionLevel = level; - } + if (!m_headerSent) { + if (m_compressionEnabled[CompressionType::BrotliChunked] == -1) { + String compression; + IniSetting::Get("brotli.chunked_compression", compression); + m_compressionEnabled[CompressionType::BrotliChunked] = + compression.size() == 2 && bstrcaseeq(compression.data(), "on", 2); } - if (m_compressor == nullptr) { - m_compressor = new StreamCompressor(compressionLevel, - CODING_GZIP, true); + + // If PHP disables a particular compression, then it is the same as if + // encoding is not accepted. + for (int i = 0; i < CompressionType::Max; i++) { + if (!m_compressionEnabled[i]) { + m_acceptedEncodings[i] = false; + } } - int len = size; - char *compressedData = - m_compressor->compress((const char*)data, len, last); - if (compressedData) { - StringHolder deleter(compressedData, len, true); - if (m_chunkedEncoding || len < size || - m_compressionDecision == CompressionDecision::HasTo) { - response = std::move(deleter); - compressed = true; + + // Gzip has 20 bytes header, so anything smaller than a few bytes probably + // wouldn't benefit much from compression + if (m_chunkedEncoding || size > 50 || + m_compressionDecision == CompressionDecision::HasTo) { + if (m_chunkedEncoding && + m_acceptedEncodings[CompressionType::BrotliChunked]) { + m_encodingType = CompressionType::Brotli; + } else if (!m_chunkedEncoding && + m_acceptedEncodings[CompressionType::Brotli]) { + m_encodingType = CompressionType::Brotli; + } else if (m_acceptedEncodings[CompressionType::Gzip]) { + m_encodingType = CompressionType::Gzip; } - } else { - Logger::Error("Unable to compress response: level=%d len=%d", - compressionLevel, len); } } + if (m_encodingType == CompressionType::Brotli) { + response = compressBrotli(data, size, compressed, last); + } else if (m_encodingType == CompressionType::Gzip) { + response = compressGzip(data, size, compressed, last); + } + return response; } +StringHolder Transport::compressGzip(const void *data, int size, + bool &compressed, bool last) { + StringHolder response((const char *)data, size); + + String compression; + int compressionLevel = RuntimeOption::GzipCompressionLevel; + IniSetting::Get("zlib.output_compression", compression); + if (compression.size() == 2 && bstrcaseeq(compression.data(), "on", 2)) { + String compressionLevelStr; + IniSetting::Get("zlib.output_compression_level", compressionLevelStr); + int level = compressionLevelStr.toInt64(); + if (level > compressionLevel && + level <= RuntimeOption::GzipMaxCompressionLevel) { + compressionLevel = level; + } + } + if (m_compressor == nullptr) { + m_compressor = folly::make_unique( + compressionLevel, CODING_GZIP, true); + } + int len = size; + char *compressedData = + m_compressor->compress((const char*)data, len, last); + if (compressedData) { + StringHolder deleter(compressedData, len, true); + if (m_chunkedEncoding || len < size || + m_compressionDecision == CompressionDecision::HasTo) { + response = std::move(deleter); + compressed = true; + } + } else { + Logger::Error("Unable to compress response: level=%d len=%d", + compressionLevel, len); + } + + return response; +} + +StringHolder Transport::compressBrotli(const void *data, int size, + bool &compressed, bool last) { + if (m_brotliCompressor == nullptr) { + brotli::BrotliParams params; + params.mode = + (brotli::BrotliParams::Mode)RuntimeOption::BrotliCompressionMode; + + Variant quality; + IniSetting::Get("brotli.compression_quality", quality); + params.quality = quality.asInt64Val(); + + Variant lgWindowSize; + IniSetting::Get("brotli.compression_lgwin", lgWindowSize); + params.lgwin = lgWindowSize.asInt64Val(); + + m_brotliCompressor = folly::make_unique(params); + } + + size_t len = size; + auto compressedData = + HPHP::compressBrotli(m_brotliCompressor.get(), data, len, last); + if (!compressedData) { + Logger::Error("Unable to compress response to brotli: size=%d", size); + return StringHolder((const char*)data, size); + } + + compressed = true; + return StringHolder(compressedData, len, true); +} + +void Transport::enableCompression() { + m_compressionEnabled[CompressionType::Brotli] = + RuntimeOption::BrotliCompressionEnabled; + m_compressionEnabled[CompressionType::BrotliChunked] = + RuntimeOption::BrotliChunkedCompressionEnabled; + m_compressionEnabled[CompressionType::Gzip] = + RuntimeOption::GzipCompressionLevel; +} + +void Transport::disableCompression() { + for (int i = 0; i < CompressionType::Max; ++i) { + m_compressionEnabled[i] = 0; + } +} + +bool Transport::isCompressionEnabled() const { + return m_compressionEnabled[CompressionType::Brotli] || + m_compressionEnabled[CompressionType::BrotliChunked] || + m_compressionEnabled[CompressionType::Gzip]; +} + bool Transport::setHeaderCallback(const Variant& callback) { if (cellAsVariant(m_headerCallback).toBoolean()) { // return false if a callback has already been set. @@ -944,7 +1074,7 @@ void Transport::sendRawInternal(const void *data, int size, void Transport::onSendEnd() { bool eomSent = false; - if (m_compressor && m_chunkedEncoding) { + if ((m_compressor || m_brotliCompressor) && m_chunkedEncoding) { assert(m_headerSent); bool compressed = false; StringHolder response = prepareResponse("", 0, compressed, true); diff --git a/hphp/runtime/server/transport.h b/hphp/runtime/server/transport.h index ea810dcad59..03f6817cf16 100644 --- a/hphp/runtime/server/transport.h +++ b/hphp/runtime/server/transport.h @@ -29,6 +29,10 @@ #include "hphp/runtime/base/string-holder.h" #include "hphp/runtime/base/type-string.h" +namespace brotli { + class BrotliCompressor; +}; + namespace HPHP { /////////////////////////////////////////////////////////////////////////////// @@ -202,13 +206,11 @@ public: void setUseDefaultContentType(bool send) { m_sendContentType = send;} /** - * Can we gzip response? + * Can we compress response? */ - void enableCompression() { m_compression = true;} - void disableCompression() { m_compression = false;} - bool isCompressionEnabled() const { - return m_compression && RuntimeOption::GzipCompressionLevel; - } + void enableCompression(); + void disableCompression(); + bool isCompressionEnabled() const; /** * Set cookie response header. @@ -470,10 +472,26 @@ protected: std::vector m_chunksSentSizes; + // Supported compression types. + enum CompressionType { + Brotli, + BrotliChunked, + Gzip, + Max, + }; + static const char* ENCODING_TYPE_TO_NAME[CompressionType::Max + 1]; + const char* compressionName(CompressionType type); + std::string m_mimeType; bool m_sendContentType; - bool m_compression; - StreamCompressor *m_compressor; + // 0 - disabled, -1 - ini_set dictates the setting, enabled otherwise + int8_t m_compressionEnabled[CompressionType::Max]; + // encodings accepted by the client, and enabled + bool m_acceptedEncodings[CompressionType::Max]; + // encoding we decided to use + CompressionType m_encodingType; + std::unique_ptr m_compressor; + std::unique_ptr m_brotliCompressor; bool m_isSSL; @@ -497,6 +515,10 @@ protected: StringHolder prepareResponse(const void *data, int size, bool &compressed, bool last); + StringHolder compressGzip(const void *data, int size, bool &compressed, + bool last); + StringHolder compressBrotli(const void *data, int size, bool &compressed, + bool last); private: void prepareHeaders(bool compressed, bool chunked, diff --git a/hphp/util/brotli.cpp b/hphp/util/brotli.cpp new file mode 100644 index 00000000000..bc2995aaae4 --- /dev/null +++ b/hphp/util/brotli.cpp @@ -0,0 +1,91 @@ +/* + +----------------------------------------------------------------------+ + | HipHop for PHP | + +----------------------------------------------------------------------+ + | Copyright (c) 2010-2016 Facebook, Inc. (http://www.facebook.com) | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ +*/ + +#include "hphp/util/brotli.h" +#include +#include + +// The version of brotli we are using does implement these helper +// methods that we need. However the methods are not declared in +// the core brotli libraries. Lets declare those here. +namespace brotli { +size_t CopyOneBlockToRingBuffer(BrotliIn* r, + BrotliCompressor* compressor); +bool BrotliInIsFinished(brotli::BrotliIn* r); +} + +using namespace brotli; + + +namespace HPHP { +/////////////////////////////////////////////////////////////////////////////// + +const char* compressBrotli(BrotliCompressor* compressor, + const void* data, + size_t& len, + bool last) { + // Brotli does not have a utility to compute max size of the buffer + // in case data is incompressible. Below link discusses some numbers + // and formula where 16MB block would use only 6 extra bytes. + // For all practical usage we should be fine with 20 bytes. + // https://github.com/google/brotli/issues/274 + // We should also allow 6 extra bytes for an empty meta-block at + // the end of each chunk to force "flush". + size_t availableBytes = len + 30; + auto available = (char *)malloc(len + availableBytes); + auto deleter = folly::makeGuard([&] { free(available); }); + + BrotliMemIn in(data, len); + BrotliMemOut out(available, availableBytes); + bool finalBlock = false; + while (!finalBlock) { + auto inBytes = CopyOneBlockToRingBuffer(&in, compressor); + finalBlock = inBytes == 0 || BrotliInIsFinished(&in); + size_t outBytes = 0; + uint8_t* output = nullptr; + if (!compressor->WriteBrotliData(last && finalBlock, + /* force_flush */ finalBlock, + &outBytes, + &output)) { + return nullptr; + } + + // This deserves an explanation as brotli's documentation is + // really incomplete on the topic. + // 'force_flush' is what they call a "soft flush" and it stops at the byte + // boundary of the compressed buffer. As such, there is a 7/8 chance that + // last few bytes will be held by the compressor. To force the compressor + // stop at the byte boundary one can write an empty meta-block. + if (!last && finalBlock) { + size_t bytes = 6; + compressor->WriteMetadata( + 0, nullptr, false, &bytes, output + outBytes); + outBytes += bytes; + } + + if (outBytes > 0 && !out.Write(output, outBytes)) { + return nullptr; + } + } + + deleter.dismiss(); + len = out.position(); + + return available; +} + +/////////////////////////////////////////////////////////////////////////////// +} diff --git a/hphp/util/brotli.h b/hphp/util/brotli.h new file mode 100644 index 00000000000..464d93a73ca --- /dev/null +++ b/hphp/util/brotli.h @@ -0,0 +1,37 @@ +/* + +----------------------------------------------------------------------+ + | HipHop for PHP | + +----------------------------------------------------------------------+ + | Copyright (c) 2010-2016 Facebook, Inc. (http://www.facebook.com) | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ +*/ + +#ifndef incl_HPHP_UTIL_BROTLI_HELPERS_H_ +#define incl_HPHP_UTIL_BROTLI_HELPERS_H_ + +#include + +namespace brotli { + class BrotliCompressor; +}; + +namespace HPHP { +/////////////////////////////////////////////////////////////////////////////// + +const char* compressBrotli(brotli::BrotliCompressor* compressor, + const void* data, + size_t& len, + bool last); + +/////////////////////////////////////////////////////////////////////////////// +} + +#endif diff --git a/hphp/util/test/brotli-test.cpp b/hphp/util/test/brotli-test.cpp new file mode 100644 index 00000000000..071127822fc --- /dev/null +++ b/hphp/util/test/brotli-test.cpp @@ -0,0 +1,101 @@ +/* + +----------------------------------------------------------------------+ + | HipHop for PHP | + +----------------------------------------------------------------------+ + | Copyright (c) 2010-2016 Facebook, Inc. (http://www.facebook.com) | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ +*/ + +#include "hphp/util/brotli.h" +#include +#include +#include +#include + +#include + +using namespace brotli; +using namespace std; +using namespace folly; + +namespace HPHP { + +TEST(BrotliTest, Chunks) { + + vector chunks = { + "\n" + "\n" + " \n" + " \n" + " Title\n" + " \n" + "\n" + "Sending data chunk 1 of 1000
\r\n", + "Sending data chunk 2 of 1000
\r\n", + "Sending data chunk 3 of 1000
\r\n", + "Sending data chunk 4 of 1000
\r\n", + "Sending data chunk 5 of 1000
\r\n" + " \n" + "\n", + }; + + BrotliState state; + BrotliStateInit(&state); + SCOPE_EXIT{ BrotliStateCleanup(&state); }; + + size_t total = 0; + brotli::BrotliCompressor compressor{BrotliParams()}; + + // generate a huge chunk + size_t hugeSize = compressor.input_block_size() + 20; + string hugeChunk; + hugeChunk.reserve(hugeSize); + string filler = "The quick brown fox jumps over the lazy dog."; + while (hugeChunk.size() != hugeSize) { + hugeChunk.append( + filler, 0, std::min(filler.size(), hugeSize - hugeChunk.size())); + } + chunks.emplace_back(std::move(hugeChunk)); + + + int i = 0; + for (auto& chunk : chunks) { + size_t size = chunk.size(); + bool last = ++i == chunks.size(); + + auto compressed = compressBrotli(&compressor, chunk.data(), size, last); + EXPECT_TRUE(compressed != nullptr); + SCOPE_EXIT { free((void*)compressed); }; + + size_t decompressedBufferSize = chunk.size(); + auto decompressedBuffer = new uint8_t[decompressedBufferSize]; + SCOPE_EXIT{ delete[] decompressedBuffer; }; + + uint8_t* decompressedPos = decompressedBuffer; + size_t decompressedAvailable = decompressedBufferSize; + + const uint8_t* compressedPos = (const uint8_t*)compressed; + BrotliDecompressBufferStreaming(&size, + &compressedPos, + 0, + &decompressedAvailable, + &decompressedPos, + &total, + &state); + EXPECT_EQ(size, 0); + EXPECT_EQ(chunk, + StringPiece((char*)decompressedBuffer, + decompressedPos - decompressedBuffer)); + } +} + +} -- 2.11.4.GIT