2 +----------------------------------------------------------------------+
4 +----------------------------------------------------------------------+
5 | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com) |
6 | Copyright (c) 1997-2010 The PHP Group |
7 +----------------------------------------------------------------------+
8 | This source file is subject to version 3.01 of the PHP license, |
9 | that is bundled with this package in the file LICENSE, and is |
10 | available through the world-wide-web at the following url: |
11 | http://www.php.net/license/3_01.txt |
12 | If you did not receive a copy of the PHP license and are unable to |
13 | obtain it through the world-wide-web, please send a note to |
14 | license@php.net so we can mail you a copy immediately. |
15 +----------------------------------------------------------------------+
18 #include "hphp/runtime/ext/ext_zlib.h"
19 #include "hphp/runtime/base/file/file.h"
20 #include "hphp/runtime/base/file/mem_file.h"
21 #include "hphp/runtime/base/file/zip_file.h"
22 #include "hphp/runtime/base/file/stream_wrapper.h"
23 #include "hphp/runtime/base/file/stream_wrapper_registry.h"
24 #include "hphp/runtime/base/file/file_stream_wrapper.h"
25 #include "hphp/util/compression.h"
26 #include "hphp/util/logger.h"
34 #define PHP_ZLIB_MODIFIER 1000
40 ///////////////////////////////////////////////////////////////////////////////
41 // compress.zlib:// stream wrapper
43 static class ZlibStreamWrapper
: public Stream::Wrapper
{
45 virtual File
* open(CStrRef filename
, CStrRef mode
,
46 int options
, CVarRef context
) {
48 static const char cz
[] = "compress.zlib://";
50 if (!strncmp(filename
.data(), "zlib:", sizeof("zlib:") - 1)) {
51 fname
= filename
.substr(sizeof("zlib:") - 1);
52 } else if (!strncmp(filename
.data(), cz
, sizeof(cz
) - 1)) {
53 fname
= filename
.substr(sizeof(cz
) - 1);
58 if (MemFile
* file
= FileStreamWrapper::openFromCache(fname
, mode
)) {
63 std::unique_ptr
<ZipFile
> file(NEWOBJ(ZipFile
)());
64 bool ret
= file
->open(File::TranslatePath(fname
), mode
);
66 raise_warning("%s", file
->getLastError().c_str());
69 return file
.release();
71 } s_zlib_stream_wrapper
;
73 ///////////////////////////////////////////////////////////////////////////////
74 // Extension entry point
76 static class ZlibExtension
: Extension
{
78 ZlibExtension() : Extension("zlib") {}
79 virtual void moduleLoad(Hdf hdf
) {
80 s_zlib_stream_wrapper
.registerAs("compress.zlib");
88 ///////////////////////////////////////////////////////////////////////////////
91 static Variant
gzcompress(const char *data
, int len
, int level
/* = -1 */) {
92 if (level
< -1 || level
> 9) {
93 throw_invalid_argument("level: %d", level
);
96 unsigned long l2
= len
+ (len
/ PHP_ZLIB_MODIFIER
) + 15;
97 String
str(l2
, ReserveString
);
98 char *s2
= str
.mutableSlice().ptr
;
102 status
= compress2((Bytef
*)s2
, &l2
, (const Bytef
*)data
, len
, level
);
104 status
= compress((Bytef
*)s2
, &l2
, (const Bytef
*)data
, len
);
107 if (status
== Z_OK
) {
108 return str
.shrink(l2
);
111 Logger::Warning("%s", zError(status
));
115 static Variant
gzuncompress(const char *data
, int len
, int limit
/* = 0 */) {
117 Logger::Warning("length (%d) must be greater or equal zero", limit
);
121 unsigned long plength
= limit
;
122 unsigned long length
;
123 unsigned int factor
= 4, maxfactor
= 16;
124 String
str(std::max(plength
, (unsigned long)StringData::MaxSmallSize
),
128 length
= plength
? plength
: (unsigned long)len
* (1 << factor
++);
129 if (length
> StringData::MaxSize
) {
132 char* s2
= str
.reserve(length
).ptr
;
133 status
= uncompress((Bytef
*)s2
, &length
, (const Bytef
*)data
, len
);
134 } while ((status
== Z_BUF_ERROR
) && (!plength
) && (factor
< maxfactor
));
136 if (status
== Z_OK
) {
137 return str
.shrink(length
);
139 Logger::Warning("%s", zError(status
));
143 ///////////////////////////////////////////////////////////////////////////////
145 static Variant
gzdeflate(const char *data
, int len
, int level
/* = -1 */) {
146 if (level
< -1 || level
> 9) {
147 throw_invalid_argument("level: %d", level
);
151 stream
.data_type
= Z_ASCII
;
152 stream
.zalloc
= (alloc_func
) Z_NULL
;
153 stream
.zfree
= (free_func
) Z_NULL
;
154 stream
.opaque
= (voidpf
) Z_NULL
;
156 stream
.next_in
= (Bytef
*)data
;
157 stream
.avail_in
= len
;
159 stream
.avail_out
= len
+ (len
/ PHP_ZLIB_MODIFIER
) + 15 + 1; // room for \0
161 String
str(stream
.avail_out
, ReserveString
);
162 char* s2
= str
.mutableSlice().ptr
;
164 stream
.next_out
= (Bytef
*)s2
;
166 /* init with -MAX_WBITS disables the zlib internal headers */
167 int status
= deflateInit2(&stream
, level
, Z_DEFLATED
, -MAX_WBITS
,
169 if (status
== Z_OK
) {
170 status
= deflate(&stream
, Z_FINISH
);
171 if (status
!= Z_STREAM_END
) {
173 if (status
== Z_OK
) {
174 status
= Z_BUF_ERROR
;
177 status
= deflateEnd(&stream
);
181 if (status
== Z_OK
) {
182 /* resize to buffer to the "right" size */
183 return str
.shrink(stream
.total_out
);
185 Logger::Warning("%s", zError(status
));
189 static Variant
gzinflate(const char *data
, int len
, int limit
/* = 0 */) {
195 Logger::Warning("length (%d) must be greater or equal zero", limit
);
198 unsigned long plength
= limit
;
201 stream
.zalloc
= (alloc_func
) Z_NULL
;
202 stream
.zfree
= (free_func
) Z_NULL
;
204 unsigned long length
= 0;
207 // We reallocate with an expanding factor, but want to start with a
208 // smaller factor on larger strings to hope not to hit the max
209 // string size as fast.
210 unsigned int factor
= len
< 128 * 1024 * 1024 ? 4 : 2;
211 unsigned int maxfactor
= 16;
213 String
str(std::max(plength
, (unsigned long)StringData::MaxSmallSize
),
216 if (length
>= StringData::MaxSize
) {
220 length
= plength
? plength
: (unsigned long)len
* (1 << factor
++);
221 length
= std::min
<unsigned long>(length
, StringData::MaxSize
);
222 char* s2
= str
.reserve(length
).ptr
;
224 stream
.next_in
= (Bytef
*)data
;
225 stream
.avail_in
= (uInt
)len
+ 1; /* there is room for \0 */
227 stream
.next_out
= (Bytef
*)s2
;
228 stream
.avail_out
= (uInt
)length
;
230 /* init with -MAX_WBITS disables the zlib internal headers */
231 status
= inflateInit2(&stream
, -MAX_WBITS
);
232 if (status
== Z_OK
) {
233 status
= inflate(&stream
, Z_FINISH
);
234 if (status
!= Z_STREAM_END
) {
236 if (status
== Z_OK
) {
237 status
= Z_BUF_ERROR
;
240 status
= inflateEnd(&stream
);
243 } while ((status
== Z_BUF_ERROR
) && (!plength
) && (factor
< maxfactor
));
245 if (status
== Z_OK
) {
246 return str
.shrink(stream
.total_out
);
248 Logger::Warning("%s", zError(status
));
252 ///////////////////////////////////////////////////////////////////////////////
254 Variant
f_readgzfile(CStrRef filename
, bool use_include_path
/* = false */) {
255 Resource stream
= f_gzopen(filename
, "rb", use_include_path
);
256 if (stream
.isNull()) {
259 return f_gzpassthru(stream
);
262 Variant
f_gzfile(CStrRef filename
, bool use_include_path
/* = false */) {
263 Resource stream
= f_gzopen(filename
, "rb", use_include_path
);
264 if (stream
.isNull()) {
270 while (!same(line
= f_gzgets(stream
), false)) {
276 Variant
f_gzcompress(CStrRef data
, int level
/* = -1 */) {
277 return gzcompress(data
.data(), data
.size(), level
);
280 Variant
f_gzuncompress(CStrRef data
, int limit
/* = 0 */) {
281 return gzuncompress(data
.data(), data
.size(), limit
);
284 Variant
f_gzdeflate(CStrRef data
, int level
/* = -1 */) {
285 return gzdeflate(data
.data(), data
.size(), level
);
288 Variant
f_gzinflate(CStrRef data
, int limit
/* = 0 */) {
289 return gzinflate(data
.data(), data
.size(), limit
);
292 Variant
f_gzencode(CStrRef data
, int level
/* = -1 */,
293 int encoding_mode
/* = k_FORCE_GZIP */) {
294 int len
= data
.size();
295 char *ret
= gzencode(data
.data(), len
, level
, encoding_mode
);
299 return String(ret
, len
, AttachString
);
302 Variant
f_gzdecode(CStrRef data
) {
303 int len
= data
.size();
304 char *ret
= gzdecode(data
.data(), len
);
308 return String(ret
, len
, AttachString
);
311 String
f_zlib_get_coding_type() {
312 throw NotSupportedException(__func__
, "no use");
315 ///////////////////////////////////////////////////////////////////////////////
318 Resource
f_gzopen(CStrRef filename
, CStrRef mode
,
319 bool use_include_path
/* = false */) {
320 File
*file
= NEWOBJ(ZipFile
)();
321 Resource
handle(file
);
322 bool ret
= file
->open(File::TranslatePath(filename
), mode
);
324 raise_warning("%s",Util::safe_strerror(errno
).c_str());
330 bool f_gzclose(CResRef zp
) {
333 Variant
f_gzread(CResRef zp
, int64_t length
/* = 0 */) {
334 return f_fread(zp
, length
);
336 Variant
f_gzseek(CResRef zp
, int64_t offset
, int64_t whence
/* = k_SEEK_SET */) {
337 return f_fseek(zp
, offset
, whence
);
339 Variant
f_gztell(CResRef zp
) {
342 bool f_gzeof(CResRef zp
) {
345 bool f_gzrewind(CResRef zp
) {
348 Variant
f_gzgetc(CResRef zp
) {
351 Variant
f_gzgets(CResRef zp
, int64_t length
/* = 1024 */) {
352 return f_fgets(zp
, length
);
354 Variant
f_gzgetss(CResRef zp
, int64_t length
/* = 0 */,
355 CStrRef allowable_tags
/* = null_string */) {
356 return f_fgetss(zp
, length
, allowable_tags
);
358 Variant
f_gzpassthru(CResRef zp
) {
359 return f_fpassthru(zp
);
361 Variant
f_gzputs(CResRef zp
, CStrRef str
, int64_t length
/* = 0 */) {
362 return f_fwrite(zp
, str
, length
);
364 Variant
f_gzwrite(CResRef zp
, CStrRef str
, int64_t length
/* = 0 */) {
365 return f_fwrite(zp
, str
, length
);
369 ///////////////////////////////////////////////////////////////////////////////
375 #ifdef QLZ_COMPRESSION_LEVEL
376 #undef QLZ_COMPRESSION_LEVEL
378 #ifdef QLZ_STREAMING_BUFFER
379 #undef QLZ_STREAMING_BUFFER
381 #define QLZ_COMPRESSION_LEVEL 1
382 #define QLZ_STREAMING_BUFFER 0
383 #include "hphp/runtime/ext/quicklz.inc"
387 #ifdef QLZ_COMPRESSION_LEVEL
388 #undef QLZ_COMPRESSION_LEVEL
390 #ifdef QLZ_STREAMING_BUFFER
391 #undef QLZ_STREAMING_BUFFER
393 #define QLZ_COMPRESSION_LEVEL 2
394 #define QLZ_STREAMING_BUFFER 100000
395 #include "hphp/runtime/ext/quicklz.inc"
399 #ifdef QLZ_COMPRESSION_LEVEL
400 #undef QLZ_COMPRESSION_LEVEL
402 #ifdef QLZ_STREAMING_BUFFER
403 #undef QLZ_STREAMING_BUFFER
405 #define QLZ_COMPRESSION_LEVEL 3
406 #define QLZ_STREAMING_BUFFER 1000000
407 #include "hphp/runtime/ext/quicklz.inc"
410 #endif // HAVE_QUICKLZ
412 Variant
f_qlzcompress(CStrRef data
, int level
/* = 1 */) {
414 throw NotSupportedException(__func__
, "QuickLZ library cannot be found");
416 if (level
< 1 || level
> 3) {
417 throw_invalid_argument("level: %d", level
);
421 String
str(data
.size() + 400, ReserveString
);
422 char* compressed
= str
.mutableSlice().ptr
;
427 QuickLZ1::qlz_state_compress state
;
428 memset(&state
, 0, sizeof(state
));
429 size
= QuickLZ1::qlz_compress(data
.data(), compressed
, data
.size(),
434 QuickLZ2::qlz_state_compress state
;
435 memset(&state
, 0, sizeof(state
));
436 size
= QuickLZ2::qlz_compress(data
.data(), compressed
, data
.size(),
441 QuickLZ3::qlz_state_compress
*state
= new QuickLZ3::qlz_state_compress();
442 memset(state
, 0, sizeof(*state
));
443 size
= QuickLZ3::qlz_compress(data
.data(), compressed
, data
.size(),
449 assert(size
<= (size_t)data
.size() + 400);
450 return str
.shrink(size
);
454 Variant
f_qlzuncompress(CStrRef data
, int level
/* = 1 */) {
456 throw NotSupportedException(__func__
, "QuickLZ library cannot be found");
458 if (level
< 1 || level
> 3) {
459 throw_invalid_argument("level: %d", level
);
463 if (data
.size() < 9) {
464 raise_notice("passing invalid data to qlzuncompress()");
468 size_t size
= QuickLZ1::qlz_size_decompressed(data
.data());
469 if ((int64_t)size
< 0 ||
470 (RuntimeOption::SerializationSizeLimit
> 0 &&
471 (int64_t)size
> RuntimeOption::SerializationSizeLimit
)) {
472 raise_notice("invalid size in compressed header: %zd", size
);
476 String s
= String(size
, ReserveString
);
477 char *decompressed
= s
.mutableSlice().ptr
;
482 QuickLZ1::qlz_state_decompress state
;
483 memset(&state
, 0, sizeof(state
));
484 dsize
= QuickLZ1::qlz_decompress(data
.data(), decompressed
, &state
);
488 QuickLZ2::qlz_state_decompress state
;
489 memset(&state
, 0, sizeof(state
));
490 dsize
= QuickLZ2::qlz_decompress(data
.data(), decompressed
, &state
);
494 QuickLZ3::qlz_state_decompress
*state
=
495 new QuickLZ3::qlz_state_decompress();
496 memset(state
, 0, sizeof(*state
));
497 dsize
= QuickLZ3::qlz_decompress(data
.data(), decompressed
, state
);
503 assert(dsize
== size
);
504 return s
.setSize(dsize
);
508 Variant
f_sncompress(CStrRef data
) {
510 throw NotSupportedException(__func__
, "Snappy library cannot be found");
514 (char *)malloc(snappy::MaxCompressedLength(data
.size()) + 1);
516 snappy::RawCompress(data
.data(), data
.size(), compressed
, &size
);
517 compressed
= (char *)realloc(compressed
, size
+ 1);
518 compressed
[size
] = '\0';
519 return String(compressed
, size
, AttachString
);
523 Variant
f_snuncompress(CStrRef data
) {
525 throw NotSupportedException(__func__
, "Snappy library cannot be found");
529 snappy::GetUncompressedLength(data
.data(), data
.size(), &dsize
);
530 String s
= String(dsize
, ReserveString
);
531 char *uncompressed
= s
.mutableSlice().ptr
;
533 if (!snappy::RawUncompress(data
.data(), data
.size(), uncompressed
)) {
536 return s
.setSize(dsize
);
540 #define NZLIB_MAGIC 0x6e7a6c69 /* nzli */
541 /* The new compression format stores a magic number and the size
542 of the uncompressed object. The magic number is stored to make sure
543 bad values do not cause us to allocate bogus or extremely large amounts
544 of memory when encountering an object with the new format. */
545 typedef struct nzlib_format_s
{
547 uint32_t uncompressed_sz
;
551 Variant
f_nzcompress(CStrRef uncompressed
) {
552 size_t len
= compressBound(uncompressed
.size());
553 String
str(sizeof(nzlib_format_t
) + len
, ReserveString
);
554 nzlib_format_t
* format
= (nzlib_format_t
*)str
.mutableSlice().ptr
;
556 format
->magic
= htonl(NZLIB_MAGIC
);
557 format
->uncompressed_sz
= htonl(uncompressed
.size());
559 int rc
= compress(format
->buf
, &len
, (uint8_t*)uncompressed
.data(),
560 uncompressed
.size());
562 return str
.shrink(len
+ sizeof(*format
));
567 Variant
f_nzuncompress(CStrRef compressed
) {
568 if (compressed
.size() < (ssize_t
)sizeof(nzlib_format_t
)) {
572 nzlib_format_t
* format
= (nzlib_format_t
*)compressed
.data();
573 if (ntohl(format
->magic
) != NZLIB_MAGIC
) {
577 size_t len
= ntohl(format
->uncompressed_sz
);
582 String
str(len
, ReserveString
);
583 char* uncompressed
= str
.mutableSlice().ptr
;
584 int rc
= uncompress((Bytef
*)uncompressed
, &len
, format
->buf
,
585 compressed
.size() - sizeof(*format
));
590 if (format
->uncompressed_sz
== 0) {
591 return str
.shrink(len
);
593 return str
.setSize(len
);
596 // Varint helper functions for lz4
597 int VarintSize(int val
) {
606 void VarintEncode(int val
, char** dest
) {
609 *p
++ = 0x80 | (static_cast<char>(val
) & 0x7f);
612 *p
++ = static_cast<char>(val
);
616 int VarintDecode(const char** src
, int max_size
) {
617 const char* p
= *src
;
621 if (max_size
<= 1) { return -1; }
623 val
|= static_cast<int>(*p
++ & 0x7f) << shift
;
626 val
|= static_cast<int>(*p
++) << shift
;
631 Variant
f_lz4compress(CStrRef uncompressed
) {
632 int bufsize
= LZ4_compressBound(uncompressed
.size());
636 int headerSize
= VarintSize(uncompressed
.size());
637 bufsize
+= headerSize
; // for the header
638 String s
= String(bufsize
, ReserveString
);
639 char *compressed
= s
.mutableSlice().ptr
;
641 VarintEncode(uncompressed
.size(), &compressed
); // write the header
643 int csize
= LZ4_compress(uncompressed
.data(), compressed
,
644 uncompressed
.size());
648 bufsize
= csize
+ headerSize
;
650 return s
.setSize(bufsize
);
653 Variant
f_lz4hccompress(CStrRef uncompressed
) {
654 int bufsize
= LZ4_compressBound(uncompressed
.size());
658 int headerSize
= VarintSize(uncompressed
.size());
659 bufsize
+= headerSize
; // for the header
660 String s
= String(bufsize
, ReserveString
);
661 char *compressed
= s
.mutableSlice().ptr
;
663 VarintEncode(uncompressed
.size(), &compressed
); // write the header
665 int csize
= LZ4_compressHC(uncompressed
.data(),
666 compressed
, uncompressed
.size());
670 bufsize
= csize
+ headerSize
;
671 return s
.shrink(bufsize
);
674 Variant
f_lz4uncompress(CStrRef compressed
) {
675 const char* compressed_ptr
= compressed
.data();
676 int dsize
= VarintDecode(&compressed_ptr
, compressed
.size());
681 String s
= String(dsize
, ReserveString
);
682 char *uncompressed
= s
.mutableSlice().ptr
;
683 int ret
= LZ4_uncompress(compressed_ptr
, uncompressed
, dsize
);
688 return s
.setSize(dsize
);
691 ///////////////////////////////////////////////////////////////////////////////