Clean up VectorEffects::init
[hiphop-php.git] / hphp / runtime / ext / ext_zlib.cpp
blobdd0ca8a5a478277de61affacb2bffa967b373f06
1 /*
2 +----------------------------------------------------------------------+
3 | HipHop for PHP |
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com) |
6 | Copyright (c) 1997-2010 The PHP Group |
7 +----------------------------------------------------------------------+
8 | This source file is subject to version 3.01 of the PHP license, |
9 | that is bundled with this package in the file LICENSE, and is |
10 | available through the world-wide-web at the following url: |
11 | http://www.php.net/license/3_01.txt |
12 | If you did not receive a copy of the PHP license and are unable to |
13 | obtain it through the world-wide-web, please send a note to |
14 | license@php.net so we can mail you a copy immediately. |
15 +----------------------------------------------------------------------+
18 #include "hphp/runtime/ext/ext_zlib.h"
19 #include "hphp/runtime/base/file/file.h"
20 #include "hphp/runtime/base/file/mem_file.h"
21 #include "hphp/runtime/base/file/zip_file.h"
22 #include "hphp/runtime/base/file/stream_wrapper.h"
23 #include "hphp/runtime/base/file/stream_wrapper_registry.h"
24 #include "hphp/runtime/base/file/file_stream_wrapper.h"
25 #include "hphp/util/compression.h"
26 #include "hphp/util/logger.h"
27 #ifdef HAVE_SNAPPY
28 #include <snappy.h>
29 #endif
30 #include <lz4.h>
31 #include <lz4hc.h>
32 #include <memory>
34 #define PHP_ZLIB_MODIFIER 1000
36 using namespace HPHP;
38 namespace {
40 ///////////////////////////////////////////////////////////////////////////////
41 // compress.zlib:// stream wrapper
43 static class ZlibStreamWrapper : public Stream::Wrapper {
44 public:
45 virtual File* open(CStrRef filename, CStrRef mode,
46 int options, CVarRef context) {
47 String fname;
48 static const char cz[] = "compress.zlib://";
50 if (!strncmp(filename.data(), "zlib:", sizeof("zlib:") - 1)) {
51 fname = filename.substr(sizeof("zlib:") - 1);
52 } else if (!strncmp(filename.data(), cz, sizeof(cz) - 1)) {
53 fname = filename.substr(sizeof(cz) - 1);
54 } else {
55 return NULL;
58 if (MemFile* file = FileStreamWrapper::openFromCache(fname, mode)) {
59 file->unzip();
60 return file;
63 std::unique_ptr<ZipFile> file(NEWOBJ(ZipFile)());
64 bool ret = file->open(File::TranslatePath(fname), mode);
65 if (!ret) {
66 raise_warning("%s", file->getLastError().c_str());
67 return NULL;
69 return file.release();
71 } s_zlib_stream_wrapper;
73 ///////////////////////////////////////////////////////////////////////////////
74 // Extension entry point
76 static class ZlibExtension : Extension {
77 public:
78 ZlibExtension() : Extension("zlib") {}
79 virtual void moduleLoad(Hdf hdf) {
80 s_zlib_stream_wrapper.registerAs("compress.zlib");
82 } s_zlib_extension;
84 } // nil namespace
86 namespace HPHP {
88 ///////////////////////////////////////////////////////////////////////////////
89 // zlib functions
91 static Variant gzcompress(const char *data, int len, int level /* = -1 */) {
92 if (level < -1 || level > 9) {
93 throw_invalid_argument("level: %d", level);
94 return false;
96 unsigned long l2 = len + (len / PHP_ZLIB_MODIFIER) + 15;
97 String str(l2, ReserveString);
98 char *s2 = str.mutableSlice().ptr;
100 int status;
101 if (level >= 0) {
102 status = compress2((Bytef*)s2, &l2, (const Bytef*)data, len, level);
103 } else {
104 status = compress((Bytef*)s2, &l2, (const Bytef*)data, len);
107 if (status == Z_OK) {
108 return str.shrink(l2);
111 Logger::Warning("%s", zError(status));
112 return false;
115 static Variant gzuncompress(const char *data, int len, int limit /* = 0 */) {
116 if (limit < 0) {
117 Logger::Warning("length (%d) must be greater or equal zero", limit);
118 return false;
121 unsigned long plength = limit;
122 unsigned long length;
123 unsigned int factor = 4, maxfactor = 16;
124 String str(std::max(plength, (unsigned long)StringData::MaxSmallSize),
125 ReserveString);
126 int status;
127 do {
128 length = plength ? plength : (unsigned long)len * (1 << factor++);
129 if (length > StringData::MaxSize) {
130 return false;
132 char* s2 = str.reserve(length).ptr;
133 status = uncompress((Bytef*)s2, &length, (const Bytef*)data, len);
134 } while ((status == Z_BUF_ERROR) && (!plength) && (factor < maxfactor));
136 if (status == Z_OK) {
137 return str.shrink(length);
139 Logger::Warning("%s", zError(status));
140 return false;
143 ///////////////////////////////////////////////////////////////////////////////
145 static Variant gzdeflate(const char *data, int len, int level /* = -1 */) {
146 if (level < -1 || level > 9) {
147 throw_invalid_argument("level: %d", level);
148 return false;
150 z_stream stream;
151 stream.data_type = Z_ASCII;
152 stream.zalloc = (alloc_func) Z_NULL;
153 stream.zfree = (free_func) Z_NULL;
154 stream.opaque = (voidpf) Z_NULL;
156 stream.next_in = (Bytef *)data;
157 stream.avail_in = len;
159 stream.avail_out = len + (len / PHP_ZLIB_MODIFIER) + 15 + 1; // room for \0
161 String str(stream.avail_out, ReserveString);
162 char* s2 = str.mutableSlice().ptr;
164 stream.next_out = (Bytef*)s2;
166 /* init with -MAX_WBITS disables the zlib internal headers */
167 int status = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
168 MAX_MEM_LEVEL, 0);
169 if (status == Z_OK) {
170 status = deflate(&stream, Z_FINISH);
171 if (status != Z_STREAM_END) {
172 deflateEnd(&stream);
173 if (status == Z_OK) {
174 status = Z_BUF_ERROR;
176 } else {
177 status = deflateEnd(&stream);
181 if (status == Z_OK) {
182 /* resize to buffer to the "right" size */
183 return str.shrink(stream.total_out);
185 Logger::Warning("%s", zError(status));
186 return false;
189 static Variant gzinflate(const char *data, int len, int limit /* = 0 */) {
190 if (len == 0) {
191 return false;
194 if (limit < 0) {
195 Logger::Warning("length (%d) must be greater or equal zero", limit);
196 return false;
198 unsigned long plength = limit;
200 z_stream stream;
201 stream.zalloc = (alloc_func) Z_NULL;
202 stream.zfree = (free_func) Z_NULL;
204 unsigned long length = 0;
205 int status;
207 // We reallocate with an expanding factor, but want to start with a
208 // smaller factor on larger strings to hope not to hit the max
209 // string size as fast.
210 unsigned int factor = len < 128 * 1024 * 1024 ? 4 : 2;
211 unsigned int maxfactor = 16;
213 String str(std::max(plength, (unsigned long)StringData::MaxSmallSize),
214 ReserveString);
215 do {
216 if (length >= StringData::MaxSize) {
217 return false;
220 length = plength ? plength : (unsigned long)len * (1 << factor++);
221 length = std::min<unsigned long>(length, StringData::MaxSize);
222 char* s2 = str.reserve(length).ptr;
224 stream.next_in = (Bytef *)data;
225 stream.avail_in = (uInt)len + 1; /* there is room for \0 */
227 stream.next_out = (Bytef*)s2;
228 stream.avail_out = (uInt)length;
230 /* init with -MAX_WBITS disables the zlib internal headers */
231 status = inflateInit2(&stream, -MAX_WBITS);
232 if (status == Z_OK) {
233 status = inflate(&stream, Z_FINISH);
234 if (status != Z_STREAM_END) {
235 inflateEnd(&stream);
236 if (status == Z_OK) {
237 status = Z_BUF_ERROR;
239 } else {
240 status = inflateEnd(&stream);
243 } while ((status == Z_BUF_ERROR) && (!plength) && (factor < maxfactor));
245 if (status == Z_OK) {
246 return str.shrink(stream.total_out);
248 Logger::Warning("%s", zError(status));
249 return false;
252 ///////////////////////////////////////////////////////////////////////////////
254 Variant f_readgzfile(CStrRef filename, bool use_include_path /* = false */) {
255 Resource stream = f_gzopen(filename, "rb", use_include_path);
256 if (stream.isNull()) {
257 return false;
259 return f_gzpassthru(stream);
262 Variant f_gzfile(CStrRef filename, bool use_include_path /* = false */) {
263 Resource stream = f_gzopen(filename, "rb", use_include_path);
264 if (stream.isNull()) {
265 return false;
268 Array ret;
269 Variant line;
270 while (!same(line = f_gzgets(stream), false)) {
271 ret.append(line);
273 return ret;
276 Variant f_gzcompress(CStrRef data, int level /* = -1 */) {
277 return gzcompress(data.data(), data.size(), level);
280 Variant f_gzuncompress(CStrRef data, int limit /* = 0 */) {
281 return gzuncompress(data.data(), data.size(), limit);
284 Variant f_gzdeflate(CStrRef data, int level /* = -1 */) {
285 return gzdeflate(data.data(), data.size(), level);
288 Variant f_gzinflate(CStrRef data, int limit /* = 0 */) {
289 return gzinflate(data.data(), data.size(), limit);
292 Variant f_gzencode(CStrRef data, int level /* = -1 */,
293 int encoding_mode /* = k_FORCE_GZIP */) {
294 int len = data.size();
295 char *ret = gzencode(data.data(), len, level, encoding_mode);
296 if (ret == NULL) {
297 return false;
299 return String(ret, len, AttachString);
302 Variant f_gzdecode(CStrRef data) {
303 int len = data.size();
304 char *ret = gzdecode(data.data(), len);
305 if (ret == NULL) {
306 return false;
308 return String(ret, len, AttachString);
311 String f_zlib_get_coding_type() {
312 throw NotSupportedException(__func__, "no use");
315 ///////////////////////////////////////////////////////////////////////////////
316 // stream functions
318 Resource f_gzopen(CStrRef filename, CStrRef mode,
319 bool use_include_path /* = false */) {
320 File *file = NEWOBJ(ZipFile)();
321 Resource handle(file);
322 bool ret = file->open(File::TranslatePath(filename), mode);
323 if (!ret) {
324 raise_warning("%s",Util::safe_strerror(errno).c_str());
325 return NULL;
327 return handle;
330 bool f_gzclose(CResRef zp) {
331 return f_fclose(zp);
333 Variant f_gzread(CResRef zp, int64_t length /* = 0 */) {
334 return f_fread(zp, length);
336 Variant f_gzseek(CResRef zp, int64_t offset, int64_t whence /* = k_SEEK_SET */) {
337 return f_fseek(zp, offset, whence);
339 Variant f_gztell(CResRef zp) {
340 return f_ftell(zp);
342 bool f_gzeof(CResRef zp) {
343 return f_feof(zp);
345 bool f_gzrewind(CResRef zp) {
346 return f_rewind(zp);
348 Variant f_gzgetc(CResRef zp) {
349 return f_fgetc(zp);
351 Variant f_gzgets(CResRef zp, int64_t length /* = 1024 */) {
352 return f_fgets(zp, length);
354 Variant f_gzgetss(CResRef zp, int64_t length /* = 0 */,
355 CStrRef allowable_tags /* = null_string */) {
356 return f_fgetss(zp, length, allowable_tags);
358 Variant f_gzpassthru(CResRef zp) {
359 return f_fpassthru(zp);
361 Variant f_gzputs(CResRef zp, CStrRef str, int64_t length /* = 0 */) {
362 return f_fwrite(zp, str, length);
364 Variant f_gzwrite(CResRef zp, CStrRef str, int64_t length /* = 0 */) {
365 return f_fwrite(zp, str, length);
369 ///////////////////////////////////////////////////////////////////////////////
370 // QuickLZ functions
372 #ifdef HAVE_QUICKLZ
374 namespace QuickLZ1 {
375 #ifdef QLZ_COMPRESSION_LEVEL
376 #undef QLZ_COMPRESSION_LEVEL
377 #endif
378 #ifdef QLZ_STREAMING_BUFFER
379 #undef QLZ_STREAMING_BUFFER
380 #endif
381 #define QLZ_COMPRESSION_LEVEL 1
382 #define QLZ_STREAMING_BUFFER 0
383 #include "hphp/runtime/ext/quicklz.inc"
386 namespace QuickLZ2 {
387 #ifdef QLZ_COMPRESSION_LEVEL
388 #undef QLZ_COMPRESSION_LEVEL
389 #endif
390 #ifdef QLZ_STREAMING_BUFFER
391 #undef QLZ_STREAMING_BUFFER
392 #endif
393 #define QLZ_COMPRESSION_LEVEL 2
394 #define QLZ_STREAMING_BUFFER 100000
395 #include "hphp/runtime/ext/quicklz.inc"
398 namespace QuickLZ3 {
399 #ifdef QLZ_COMPRESSION_LEVEL
400 #undef QLZ_COMPRESSION_LEVEL
401 #endif
402 #ifdef QLZ_STREAMING_BUFFER
403 #undef QLZ_STREAMING_BUFFER
404 #endif
405 #define QLZ_COMPRESSION_LEVEL 3
406 #define QLZ_STREAMING_BUFFER 1000000
407 #include "hphp/runtime/ext/quicklz.inc"
410 #endif // HAVE_QUICKLZ
412 Variant f_qlzcompress(CStrRef data, int level /* = 1 */) {
413 #ifndef HAVE_QUICKLZ
414 throw NotSupportedException(__func__, "QuickLZ library cannot be found");
415 #else
416 if (level < 1 || level > 3) {
417 throw_invalid_argument("level: %d", level);
418 return false;
421 String str(data.size() + 400, ReserveString);
422 char* compressed = str.mutableSlice().ptr;
423 size_t size = 0;
425 switch (level) {
426 case 1: {
427 QuickLZ1::qlz_state_compress state;
428 memset(&state, 0, sizeof(state));
429 size = QuickLZ1::qlz_compress(data.data(), compressed, data.size(),
430 &state);
431 break;
433 case 2: {
434 QuickLZ2::qlz_state_compress state;
435 memset(&state, 0, sizeof(state));
436 size = QuickLZ2::qlz_compress(data.data(), compressed, data.size(),
437 &state);
438 break;
440 case 3:
441 QuickLZ3::qlz_state_compress *state = new QuickLZ3::qlz_state_compress();
442 memset(state, 0, sizeof(*state));
443 size = QuickLZ3::qlz_compress(data.data(), compressed, data.size(),
444 state);
445 delete state;
446 break;
449 assert(size <= (size_t)data.size() + 400);
450 return str.shrink(size);
451 #endif
454 Variant f_qlzuncompress(CStrRef data, int level /* = 1 */) {
455 #ifndef HAVE_QUICKLZ
456 throw NotSupportedException(__func__, "QuickLZ library cannot be found");
457 #else
458 if (level < 1 || level > 3) {
459 throw_invalid_argument("level: %d", level);
460 return false;
463 if (data.size() < 9) {
464 raise_notice("passing invalid data to qlzuncompress()");
465 return false;
468 size_t size = QuickLZ1::qlz_size_decompressed(data.data());
469 if ((int64_t)size < 0 ||
470 (RuntimeOption::SerializationSizeLimit > 0 &&
471 (int64_t)size > RuntimeOption::SerializationSizeLimit)) {
472 raise_notice("invalid size in compressed header: %zd", size);
473 return false;
476 String s = String(size, ReserveString);
477 char *decompressed = s.mutableSlice().ptr;
478 size_t dsize = 0;
480 switch (level) {
481 case 1: {
482 QuickLZ1::qlz_state_decompress state;
483 memset(&state, 0, sizeof(state));
484 dsize = QuickLZ1::qlz_decompress(data.data(), decompressed, &state);
485 break;
487 case 2: {
488 QuickLZ2::qlz_state_decompress state;
489 memset(&state, 0, sizeof(state));
490 dsize = QuickLZ2::qlz_decompress(data.data(), decompressed, &state);
491 break;
493 case 3: {
494 QuickLZ3::qlz_state_decompress *state =
495 new QuickLZ3::qlz_state_decompress();
496 memset(state, 0, sizeof(*state));
497 dsize = QuickLZ3::qlz_decompress(data.data(), decompressed, state);
498 delete state;
499 break;
503 assert(dsize == size);
504 return s.setSize(dsize);
505 #endif
508 Variant f_sncompress(CStrRef data) {
509 #ifndef HAVE_SNAPPY
510 throw NotSupportedException(__func__, "Snappy library cannot be found");
511 #else
512 size_t size;
513 char *compressed =
514 (char *)malloc(snappy::MaxCompressedLength(data.size()) + 1);
516 snappy::RawCompress(data.data(), data.size(), compressed, &size);
517 compressed = (char *)realloc(compressed, size + 1);
518 compressed[size] = '\0';
519 return String(compressed, size, AttachString);
520 #endif
523 Variant f_snuncompress(CStrRef data) {
524 #ifndef HAVE_SNAPPY
525 throw NotSupportedException(__func__, "Snappy library cannot be found");
526 #else
527 size_t dsize;
529 snappy::GetUncompressedLength(data.data(), data.size(), &dsize);
530 String s = String(dsize, ReserveString);
531 char *uncompressed = s.mutableSlice().ptr;
533 if (!snappy::RawUncompress(data.data(), data.size(), uncompressed)) {
534 return false;
536 return s.setSize(dsize);
537 #endif
540 #define NZLIB_MAGIC 0x6e7a6c69 /* nzli */
541 /* The new compression format stores a magic number and the size
542 of the uncompressed object. The magic number is stored to make sure
543 bad values do not cause us to allocate bogus or extremely large amounts
544 of memory when encountering an object with the new format. */
545 typedef struct nzlib_format_s {
546 uint32_t magic;
547 uint32_t uncompressed_sz;
548 Bytef buf[0];
549 } nzlib_format_t;
551 Variant f_nzcompress(CStrRef uncompressed) {
552 size_t len = compressBound(uncompressed.size());
553 String str(sizeof(nzlib_format_t) + len, ReserveString);
554 nzlib_format_t* format = (nzlib_format_t*)str.mutableSlice().ptr;
556 format->magic = htonl(NZLIB_MAGIC);
557 format->uncompressed_sz = htonl(uncompressed.size());
559 int rc = compress(format->buf, &len, (uint8_t*)uncompressed.data(),
560 uncompressed.size());
561 if (rc == Z_OK) {
562 return str.shrink(len + sizeof(*format));
564 return false;
567 Variant f_nzuncompress(CStrRef compressed) {
568 if (compressed.size() < (ssize_t)sizeof(nzlib_format_t)) {
569 return false;
572 nzlib_format_t* format = (nzlib_format_t*)compressed.data();
573 if (ntohl(format->magic) != NZLIB_MAGIC) {
574 return false;
577 size_t len = ntohl(format->uncompressed_sz);
578 if (len == 0) {
579 return empty_string;
582 String str(len, ReserveString);
583 char* uncompressed = str.mutableSlice().ptr;
584 int rc = uncompress((Bytef*)uncompressed, &len, format->buf,
585 compressed.size() - sizeof(*format));
586 if (rc != Z_OK) {
587 return false;
590 if (format->uncompressed_sz == 0) {
591 return str.shrink(len);
593 return str.setSize(len);
596 // Varint helper functions for lz4
597 int VarintSize(int val) {
598 int s = 1;
599 while (val >= 128) {
600 ++s;
601 val >>= 7;
603 return s;
606 void VarintEncode(int val, char** dest) {
607 char* p = *dest;
608 while (val >= 128) {
609 *p++ = 0x80 | (static_cast<char>(val) & 0x7f);
610 val >>= 7;
612 *p++ = static_cast<char>(val);
613 *dest = p;
616 int VarintDecode(const char** src, int max_size) {
617 const char* p = *src;
618 int val = 0;
619 int shift = 0;
620 while (*p & 0x80) {
621 if (max_size <= 1) { return -1; }
622 --max_size;
623 val |= static_cast<int>(*p++ & 0x7f) << shift;
624 shift += 7;
626 val |= static_cast<int>(*p++) << shift;
627 *src = p;
628 return val;
631 Variant f_lz4compress(CStrRef uncompressed) {
632 int bufsize = LZ4_compressBound(uncompressed.size());
633 if (bufsize < 0) {
634 return false;
636 int headerSize = VarintSize(uncompressed.size());
637 bufsize += headerSize; // for the header
638 String s = String(bufsize, ReserveString);
639 char *compressed = s.mutableSlice().ptr;
641 VarintEncode(uncompressed.size(), &compressed); // write the header
643 int csize = LZ4_compress(uncompressed.data(), compressed,
644 uncompressed.size());
645 if (csize < 0) {
646 return false;
648 bufsize = csize + headerSize;
649 s.shrink(bufsize);
650 return s.setSize(bufsize);
653 Variant f_lz4hccompress(CStrRef uncompressed) {
654 int bufsize = LZ4_compressBound(uncompressed.size());
655 if (bufsize < 0) {
656 return false;
658 int headerSize = VarintSize(uncompressed.size());
659 bufsize += headerSize; // for the header
660 String s = String(bufsize, ReserveString);
661 char *compressed = s.mutableSlice().ptr;
663 VarintEncode(uncompressed.size(), &compressed); // write the header
665 int csize = LZ4_compressHC(uncompressed.data(),
666 compressed, uncompressed.size());
667 if (csize < 0) {
668 return false;
670 bufsize = csize + headerSize;
671 return s.shrink(bufsize);
674 Variant f_lz4uncompress(CStrRef compressed) {
675 const char* compressed_ptr = compressed.data();
676 int dsize = VarintDecode(&compressed_ptr, compressed.size());
677 if (dsize < 0) {
678 return false;
681 String s = String(dsize, ReserveString);
682 char *uncompressed = s.mutableSlice().ptr;
683 int ret = LZ4_uncompress(compressed_ptr, uncompressed, dsize);
685 if (ret <= 0) {
686 return false;
688 return s.setSize(dsize);
691 ///////////////////////////////////////////////////////////////////////////////