2 * Copyright (c) 2014 Michihiro NAKAJIMA
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #include "archive_platform.h"
28 __FBSDID("$FreeBSD$");
48 #include "archive_endian.h"
49 #include "archive_private.h"
50 #include "archive_write_private.h"
51 #include "archive_xxhash.h"
53 #define LZ4_MAGICNUMBER 0x184d2204
56 int compression_level
;
57 unsigned header_written
:1;
58 unsigned version_number
:1;
59 unsigned block_independence
:1;
60 unsigned block_checksum
:1;
61 unsigned stream_size
:1;
62 unsigned stream_checksum
:1;
63 unsigned preset_dictionary
:1;
64 unsigned block_maximum_size
:3;
65 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
69 size_t out_buffer_size
;
70 size_t out_block_size
;
72 char *in_buffer_allocated
;
74 size_t in_buffer_size
;
80 struct archive_write_program_data
*pdata
;
84 static int archive_filter_lz4_close(struct archive_write_filter
*);
85 static int archive_filter_lz4_free(struct archive_write_filter
*);
86 static int archive_filter_lz4_open(struct archive_write_filter
*);
87 static int archive_filter_lz4_options(struct archive_write_filter
*,
88 const char *, const char *);
89 static int archive_filter_lz4_write(struct archive_write_filter
*,
90 const void *, size_t);
93 * Add a lz4 compression filter to this write handle.
96 archive_write_add_filter_lz4(struct archive
*_a
)
98 struct archive_write
*a
= (struct archive_write
*)_a
;
99 struct archive_write_filter
*f
= __archive_write_allocate_filter(_a
);
100 struct private_data
*data
;
102 archive_check_magic(&a
->archive
, ARCHIVE_WRITE_MAGIC
,
103 ARCHIVE_STATE_NEW
, "archive_write_add_filter_lz4");
105 data
= calloc(1, sizeof(*data
));
107 archive_set_error(&a
->archive
, ENOMEM
, "Out of memory");
108 return (ARCHIVE_FATAL
);
112 * Setup default settings.
114 data
->compression_level
= 1;
115 data
->version_number
= 0x01;
116 data
->block_independence
= 1;
117 data
->block_checksum
= 0;
118 data
->stream_size
= 0;
119 data
->stream_checksum
= 1;
120 data
->preset_dictionary
= 0;
121 data
->block_maximum_size
= 7;
124 * Setup a filter setting.
127 f
->options
= &archive_filter_lz4_options
;
128 f
->close
= &archive_filter_lz4_close
;
129 f
->free
= &archive_filter_lz4_free
;
130 f
->open
= &archive_filter_lz4_open
;
131 f
->code
= ARCHIVE_FILTER_LZ4
;
133 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
137 * We don't have lz4 library, and execute external lz4 program
140 data
->pdata
= __archive_write_program_allocate("lz4");
141 if (data
->pdata
== NULL
) {
143 archive_set_error(&a
->archive
, ENOMEM
, "Out of memory");
144 return (ARCHIVE_FATAL
);
146 data
->compression_level
= 0;
147 archive_set_error(&a
->archive
, ARCHIVE_ERRNO_MISC
,
148 "Using external lz4 program");
149 return (ARCHIVE_WARN
);
157 archive_filter_lz4_options(struct archive_write_filter
*f
,
158 const char *key
, const char *value
)
160 struct private_data
*data
= (struct private_data
*)f
->data
;
162 if (strcmp(key
, "compression-level") == 0) {
164 if (value
== NULL
|| !((val
= value
[0] - '0') >= 1 && val
<= 9) ||
166 return (ARCHIVE_WARN
);
171 archive_set_error(f
->archive
, ARCHIVE_ERRNO_PROGRAMMER
,
172 "High compression not included in this build");
173 return (ARCHIVE_FATAL
);
176 data
->compression_level
= val
;
179 if (strcmp(key
, "stream-checksum") == 0) {
180 data
->stream_checksum
= value
!= NULL
;
183 if (strcmp(key
, "block-checksum") == 0) {
184 data
->block_checksum
= value
!= NULL
;
187 if (strcmp(key
, "block-size") == 0) {
188 if (value
== NULL
|| !(value
[0] >= '4' && value
[0] <= '7') ||
190 return (ARCHIVE_WARN
);
191 data
->block_maximum_size
= value
[0] - '0';
194 if (strcmp(key
, "block-dependence") == 0) {
195 data
->block_independence
= value
== NULL
;
199 /* Note: The "warn" return is just to inform the options
200 * supervisor that we didn't handle it. It will generate
201 * a suitable error if no one used this option. */
202 return (ARCHIVE_WARN
);
205 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
206 /* Don't compile this if we don't have liblz4. */
208 static int drive_compressor(struct archive_write_filter
*, const char *,
210 static int drive_compressor_independence(struct archive_write_filter
*,
211 const char *, size_t);
212 static int drive_compressor_dependence(struct archive_write_filter
*,
213 const char *, size_t);
214 static int lz4_write_stream_descriptor(struct archive_write_filter
*);
215 static ssize_t
lz4_write_one_block(struct archive_write_filter
*, const char *,
223 archive_filter_lz4_open(struct archive_write_filter
*f
)
225 struct private_data
*data
= (struct private_data
*)f
->data
;
227 size_t required_size
;
228 static size_t bkmap
[] = { 64 * 1024, 256 * 1024, 1 * 1024 * 1024,
230 size_t pre_block_size
;
232 ret
= __archive_write_open_filter(f
->next_filter
);
236 if (data
->block_maximum_size
< 4)
237 data
->block_size
= bkmap
[0];
239 data
->block_size
= bkmap
[data
->block_maximum_size
- 4];
241 required_size
= 4 + 15 + 4 + data
->block_size
+ 4 + 4;
242 if (data
->out_buffer_size
< required_size
) {
243 size_t bs
= required_size
, bpb
;
244 free(data
->out_buffer
);
245 if (f
->archive
->magic
== ARCHIVE_WRITE_MAGIC
) {
246 /* Buffer size should be a multiple number of
247 * the of bytes per block for performance. */
248 bpb
= archive_write_get_bytes_per_block(f
->archive
);
256 data
->out_block_size
= bs
;
258 data
->out_buffer
= malloc(bs
);
259 data
->out
= data
->out_buffer
;
260 data
->out_buffer_size
= bs
;
263 pre_block_size
= (data
->block_independence
)? 0: 64 * 1024;
264 if (data
->in_buffer_size
< data
->block_size
+ pre_block_size
) {
265 free(data
->in_buffer_allocated
);
266 data
->in_buffer_size
= data
->block_size
;
267 data
->in_buffer_allocated
=
268 malloc(data
->in_buffer_size
+ pre_block_size
);
269 data
->in_buffer
= data
->in_buffer_allocated
+ pre_block_size
;
270 if (!data
->block_independence
&& data
->compression_level
>= 3)
271 data
->in_buffer
= data
->in_buffer_allocated
;
272 data
->in
= data
->in_buffer
;
273 data
->in_buffer_size
= data
->block_size
;
276 if (data
->out_buffer
== NULL
|| data
->in_buffer_allocated
== NULL
) {
277 archive_set_error(f
->archive
, ENOMEM
,
278 "Can't allocate data for compression buffer");
279 return (ARCHIVE_FATAL
);
282 f
->write
= archive_filter_lz4_write
;
288 * Write data to the out stream.
290 * Returns ARCHIVE_OK if all data written, error otherwise.
293 archive_filter_lz4_write(struct archive_write_filter
*f
,
294 const void *buff
, size_t length
)
296 struct private_data
*data
= (struct private_data
*)f
->data
;
297 int ret
= ARCHIVE_OK
;
302 /* If we haven't written a stream descriptor, we have to do it first. */
303 if (!data
->header_written
) {
304 ret
= lz4_write_stream_descriptor(f
);
305 if (ret
!= ARCHIVE_OK
)
307 data
->header_written
= 1;
310 /* Update statistics */
311 data
->total_in
+= length
;
313 p
= (const char *)buff
;
317 /* Compress input data to output buffer */
318 size
= lz4_write_one_block(f
, p
, remaining
);
319 if (size
< ARCHIVE_OK
)
320 return (ARCHIVE_FATAL
);
321 l
= data
->out
- data
->out_buffer
;
322 if (l
>= data
->out_block_size
) {
323 ret
= __archive_write_filter(f
->next_filter
,
324 data
->out_buffer
, data
->out_block_size
);
325 l
-= data
->out_block_size
;
326 memcpy(data
->out_buffer
,
327 data
->out_buffer
+ data
->out_block_size
, l
);
328 data
->out
= data
->out_buffer
+ l
;
329 if (ret
< ARCHIVE_WARN
)
340 * Finish the compression.
343 archive_filter_lz4_close(struct archive_write_filter
*f
)
345 struct private_data
*data
= (struct private_data
*)f
->data
;
348 /* Finish compression cycle. */
349 ret
= (int)lz4_write_one_block(f
, NULL
, 0);
352 * Write the last block and the end of the stream data.
355 /* Write End Of Stream. */
356 memset(data
->out
, 0, 4); data
->out
+= 4;
357 /* Write Stream checksum if needed. */
358 if (data
->stream_checksum
) {
359 unsigned int checksum
;
360 checksum
= __archive_xxhash
.XXH32_digest(
362 data
->xxh32_state
= NULL
;
363 archive_le32enc(data
->out
, checksum
);
366 ret
= __archive_write_filter(f
->next_filter
,
367 data
->out_buffer
, data
->out
- data
->out_buffer
);
370 r1
= __archive_write_close_filter(f
->next_filter
);
371 return (r1
< ret
? r1
: ret
);
375 archive_filter_lz4_free(struct archive_write_filter
*f
)
377 struct private_data
*data
= (struct private_data
*)f
->data
;
379 if (data
->lz4_stream
!= NULL
) {
381 if (data
->compression_level
>= 3)
382 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
383 LZ4_freeStreamHC(data
->lz4_stream
);
385 LZ4_freeHC(data
->lz4_stream
);
389 #if LZ4_VERSION_MINOR >= 3
390 LZ4_freeStream(data
->lz4_stream
);
392 LZ4_free(data
->lz4_stream
);
395 free(data
->out_buffer
);
396 free(data
->in_buffer_allocated
);
397 free(data
->xxh32_state
);
404 lz4_write_stream_descriptor(struct archive_write_filter
*f
)
406 struct private_data
*data
= (struct private_data
*)f
->data
;
409 sd
= (uint8_t *)data
->out
;
410 /* Write Magic Number. */
411 archive_le32enc(&sd
[0], LZ4_MAGICNUMBER
);
413 sd
[4] = (data
->version_number
<< 6)
414 | (data
->block_independence
<< 5)
415 | (data
->block_checksum
<< 4)
416 | (data
->stream_size
<< 3)
417 | (data
->stream_checksum
<< 2)
418 | (data
->preset_dictionary
<< 0);
420 sd
[5] = (data
->block_maximum_size
<< 4);
421 sd
[6] = (__archive_xxhash
.XXH32(&sd
[4], 2, 0) >> 8) & 0xff;
423 if (data
->stream_checksum
)
424 data
->xxh32_state
= __archive_xxhash
.XXH32_init(0);
426 data
->xxh32_state
= NULL
;
431 lz4_write_one_block(struct archive_write_filter
*f
, const char *p
,
434 struct private_data
*data
= (struct private_data
*)f
->data
;
438 /* Compress remaining uncompressed data. */
439 if (data
->in_buffer
== data
->in
)
442 size_t l
= data
->in
- data
->in_buffer
;
443 r
= drive_compressor(f
, data
->in_buffer
, l
);
447 } else if ((data
->block_independence
|| data
->compression_level
< 3) &&
448 data
->in_buffer
== data
->in
&& length
>= data
->block_size
) {
449 r
= drive_compressor(f
, p
, data
->block_size
);
451 r
= (ssize_t
)data
->block_size
;
453 size_t remaining_size
= data
->in_buffer_size
-
454 (data
->in
- data
->in_buffer
);
455 size_t l
= (remaining_size
> length
)? length
: remaining_size
;
456 memcpy(data
->in
, p
, l
);
458 if (l
== remaining_size
) {
459 r
= drive_compressor(f
, data
->in_buffer
,
463 data
->in
= data
->in_buffer
;
473 * Utility function to push input data through compressor, writing
474 * full output blocks as necessary.
476 * Note that this handles both the regular write case (finishing ==
477 * false) and the end-of-archive case (finishing == true).
480 drive_compressor(struct archive_write_filter
*f
, const char *p
, size_t length
)
482 struct private_data
*data
= (struct private_data
*)f
->data
;
484 if (data
->stream_checksum
)
485 __archive_xxhash
.XXH32_update(data
->xxh32_state
,
487 if (data
->block_independence
)
488 return drive_compressor_independence(f
, p
, length
);
490 return drive_compressor_dependence(f
, p
, length
);
494 drive_compressor_independence(struct archive_write_filter
*f
, const char *p
,
497 struct private_data
*data
= (struct private_data
*)f
->data
;
498 unsigned int outsize
;
501 if (data
->compression_level
>= 3)
502 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
503 outsize
= LZ4_compress_HC(p
, data
->out
+ 4,
504 (int)length
, (int)data
->block_size
,
505 data
->compression_level
);
507 outsize
= LZ4_compressHC2_limitedOutput(p
, data
->out
+ 4,
508 (int)length
, (int)data
->block_size
,
509 data
->compression_level
);
513 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
514 outsize
= LZ4_compress_default(p
, data
->out
+ 4,
515 (int)length
, (int)data
->block_size
);
517 outsize
= LZ4_compress_limitedOutput(p
, data
->out
+ 4,
518 (int)length
, (int)data
->block_size
);
522 /* The buffer is compressed. */
523 archive_le32enc(data
->out
, outsize
);
526 /* The buffer is not compressed. The commpressed size was
527 * bigger than its uncompressed size. */
528 archive_le32enc(data
->out
, length
| 0x80000000);
530 memcpy(data
->out
, p
, length
);
533 data
->out
+= outsize
;
534 if (data
->block_checksum
) {
535 unsigned int checksum
=
536 __archive_xxhash
.XXH32(data
->out
- outsize
, outsize
, 0);
537 archive_le32enc(data
->out
, checksum
);
544 drive_compressor_dependence(struct archive_write_filter
*f
, const char *p
,
547 struct private_data
*data
= (struct private_data
*)f
->data
;
550 #define DICT_SIZE (64 * 1024)
552 if (data
->compression_level
>= 3) {
553 if (data
->lz4_stream
== NULL
) {
554 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
555 data
->lz4_stream
= LZ4_createStreamHC();
556 LZ4_resetStreamHC(data
->lz4_stream
, data
->compression_level
);
559 LZ4_createHC(data
->in_buffer_allocated
);
561 if (data
->lz4_stream
== NULL
) {
562 archive_set_error(f
->archive
, ENOMEM
,
563 "Can't allocate data for compression"
565 return (ARCHIVE_FATAL
);
569 LZ4_loadDictHC(data
->lz4_stream
, data
->in_buffer_allocated
, DICT_SIZE
);
571 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
572 outsize
= LZ4_compress_HC_continue(
573 data
->lz4_stream
, p
, data
->out
+ 4, (int)length
,
574 (int)data
->block_size
);
576 outsize
= LZ4_compressHC2_limitedOutput_continue(
577 data
->lz4_stream
, p
, data
->out
+ 4, (int)length
,
578 (int)data
->block_size
, data
->compression_level
);
583 if (data
->lz4_stream
== NULL
) {
584 data
->lz4_stream
= LZ4_createStream();
585 if (data
->lz4_stream
== NULL
) {
586 archive_set_error(f
->archive
, ENOMEM
,
587 "Can't allocate data for compression"
589 return (ARCHIVE_FATAL
);
593 LZ4_loadDict(data
->lz4_stream
, data
->in_buffer_allocated
, DICT_SIZE
);
595 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
596 outsize
= LZ4_compress_fast_continue(
597 data
->lz4_stream
, p
, data
->out
+ 4, (int)length
,
598 (int)data
->block_size
, 1);
600 outsize
= LZ4_compress_limitedOutput_continue(
601 data
->lz4_stream
, p
, data
->out
+ 4, (int)length
,
602 (int)data
->block_size
);
607 /* The buffer is compressed. */
608 archive_le32enc(data
->out
, outsize
);
611 /* The buffer is not compressed. The commpressed size was
612 * bigger than its uncompressed size. */
613 archive_le32enc(data
->out
, length
| 0x80000000);
615 memcpy(data
->out
, p
, length
);
618 data
->out
+= outsize
;
619 if (data
->block_checksum
) {
620 unsigned int checksum
=
621 __archive_xxhash
.XXH32(data
->out
- outsize
, outsize
, 0);
622 archive_le32enc(data
->out
, checksum
);
626 if (length
== data
->block_size
) {
628 if (data
->compression_level
>= 3) {
629 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
630 LZ4_saveDictHC(data
->lz4_stream
, data
->in_buffer_allocated
, DICT_SIZE
);
632 LZ4_slideInputBufferHC(data
->lz4_stream
);
634 data
->in_buffer
= data
->in_buffer_allocated
+ DICT_SIZE
;
638 LZ4_saveDict(data
->lz4_stream
,
639 data
->in_buffer_allocated
, DICT_SIZE
);
645 #else /* HAVE_LIBLZ4 */
648 archive_filter_lz4_open(struct archive_write_filter
*f
)
650 struct private_data
*data
= (struct private_data
*)f
->data
;
651 struct archive_string as
;
654 archive_string_init(&as
);
655 archive_strcpy(&as
, "lz4 -z -q -q");
657 /* Specify a compression level. */
658 if (data
->compression_level
> 0) {
659 archive_strcat(&as
, " -");
660 archive_strappend_char(&as
, '0' + data
->compression_level
);
662 /* Specify a block size. */
663 archive_strcat(&as
, " -B");
664 archive_strappend_char(&as
, '0' + data
->block_maximum_size
);
666 if (data
->block_checksum
)
667 archive_strcat(&as
, " -BX");
668 if (data
->stream_checksum
== 0)
669 archive_strcat(&as
, " --no-frame-crc");
670 if (data
->block_independence
== 0)
671 archive_strcat(&as
, " -BD");
673 f
->write
= archive_filter_lz4_write
;
675 r
= __archive_write_program_open(f
, data
->pdata
, as
.s
);
676 archive_string_free(&as
);
681 archive_filter_lz4_write(struct archive_write_filter
*f
, const void *buff
,
684 struct private_data
*data
= (struct private_data
*)f
->data
;
686 return __archive_write_program_write(f
, data
->pdata
, buff
, length
);
690 archive_filter_lz4_close(struct archive_write_filter
*f
)
692 struct private_data
*data
= (struct private_data
*)f
->data
;
694 return __archive_write_program_close(f
, data
->pdata
);
698 archive_filter_lz4_free(struct archive_write_filter
*f
)
700 struct private_data
*data
= (struct private_data
*)f
->data
;
702 __archive_write_program_free(data
->pdata
);
707 #endif /* HAVE_LIBLZ4 */