hier.7: Document some recently added directories.
[dragonfly.git] / contrib / libarchive / libarchive / archive_write_add_filter_lz4.c
blob1d0ab8c56a16dc7cc673fa73c6c19ab16521b62f
1 /*-
2 * Copyright (c) 2014 Michihiro NAKAJIMA
3 * All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #include "archive_platform.h"
28 __FBSDID("$FreeBSD$");
30 #ifdef HAVE_ERRNO_H
31 #include <errno.h>
32 #endif
33 #include <stdio.h>
34 #ifdef HAVE_STDLIB_H
35 #include <stdlib.h>
36 #endif
37 #ifdef HAVE_STRING_H
38 #include <string.h>
39 #endif
40 #ifdef HAVE_LZ4_H
41 #include <lz4.h>
42 #endif
43 #ifdef HAVE_LZ4HC_H
44 #include <lz4hc.h>
45 #endif
47 #include "archive.h"
48 #include "archive_endian.h"
49 #include "archive_private.h"
50 #include "archive_write_private.h"
51 #include "archive_xxhash.h"
53 #define LZ4_MAGICNUMBER 0x184d2204
55 struct private_data {
56 int compression_level;
57 unsigned header_written:1;
58 unsigned version_number:1;
59 unsigned block_independence:1;
60 unsigned block_checksum:1;
61 unsigned stream_size:1;
62 unsigned stream_checksum:1;
63 unsigned preset_dictionary:1;
64 unsigned block_maximum_size:3;
65 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
66 int64_t total_in;
67 char *out;
68 char *out_buffer;
69 size_t out_buffer_size;
70 size_t out_block_size;
71 char *in;
72 char *in_buffer_allocated;
73 char *in_buffer;
74 size_t in_buffer_size;
75 size_t block_size;
77 void *xxh32_state;
78 void *lz4_stream;
79 #else
80 struct archive_write_program_data *pdata;
81 #endif
84 static int archive_filter_lz4_close(struct archive_write_filter *);
85 static int archive_filter_lz4_free(struct archive_write_filter *);
86 static int archive_filter_lz4_open(struct archive_write_filter *);
87 static int archive_filter_lz4_options(struct archive_write_filter *,
88 const char *, const char *);
89 static int archive_filter_lz4_write(struct archive_write_filter *,
90 const void *, size_t);
93 * Add a lz4 compression filter to this write handle.
95 int
96 archive_write_add_filter_lz4(struct archive *_a)
98 struct archive_write *a = (struct archive_write *)_a;
99 struct archive_write_filter *f = __archive_write_allocate_filter(_a);
100 struct private_data *data;
102 archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
103 ARCHIVE_STATE_NEW, "archive_write_add_filter_lz4");
105 data = calloc(1, sizeof(*data));
106 if (data == NULL) {
107 archive_set_error(&a->archive, ENOMEM, "Out of memory");
108 return (ARCHIVE_FATAL);
112 * Setup default settings.
114 data->compression_level = 1;
115 data->version_number = 0x01;
116 data->block_independence = 1;
117 data->block_checksum = 0;
118 data->stream_size = 0;
119 data->stream_checksum = 1;
120 data->preset_dictionary = 0;
121 data->block_maximum_size = 7;
124 * Setup a filter setting.
126 f->data = data;
127 f->options = &archive_filter_lz4_options;
128 f->close = &archive_filter_lz4_close;
129 f->free = &archive_filter_lz4_free;
130 f->open = &archive_filter_lz4_open;
131 f->code = ARCHIVE_FILTER_LZ4;
132 f->name = "lz4";
133 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
134 return (ARCHIVE_OK);
135 #else
137 * We don't have lz4 library, and execute external lz4 program
138 * instead.
140 data->pdata = __archive_write_program_allocate("lz4");
141 if (data->pdata == NULL) {
142 free(data);
143 archive_set_error(&a->archive, ENOMEM, "Out of memory");
144 return (ARCHIVE_FATAL);
146 data->compression_level = 0;
147 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
148 "Using external lz4 program");
149 return (ARCHIVE_WARN);
150 #endif
154 * Set write options.
156 static int
157 archive_filter_lz4_options(struct archive_write_filter *f,
158 const char *key, const char *value)
160 struct private_data *data = (struct private_data *)f->data;
162 if (strcmp(key, "compression-level") == 0) {
163 int val;
164 if (value == NULL || !((val = value[0] - '0') >= 1 && val <= 9) ||
165 value[1] != '\0')
166 return (ARCHIVE_WARN);
168 #ifndef HAVE_LZ4HC_H
169 if(val >= 3)
171 archive_set_error(f->archive, ARCHIVE_ERRNO_PROGRAMMER,
172 "High compression not included in this build");
173 return (ARCHIVE_FATAL);
175 #endif
176 data->compression_level = val;
177 return (ARCHIVE_OK);
179 if (strcmp(key, "stream-checksum") == 0) {
180 data->stream_checksum = value != NULL;
181 return (ARCHIVE_OK);
183 if (strcmp(key, "block-checksum") == 0) {
184 data->block_checksum = value != NULL;
185 return (ARCHIVE_OK);
187 if (strcmp(key, "block-size") == 0) {
188 if (value == NULL || !(value[0] >= '4' && value[0] <= '7') ||
189 value[1] != '\0')
190 return (ARCHIVE_WARN);
191 data->block_maximum_size = value[0] - '0';
192 return (ARCHIVE_OK);
194 if (strcmp(key, "block-dependence") == 0) {
195 data->block_independence = value == NULL;
196 return (ARCHIVE_OK);
199 /* Note: The "warn" return is just to inform the options
200 * supervisor that we didn't handle it. It will generate
201 * a suitable error if no one used this option. */
202 return (ARCHIVE_WARN);
205 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
206 /* Don't compile this if we don't have liblz4. */
208 static int drive_compressor(struct archive_write_filter *, const char *,
209 size_t);
210 static int drive_compressor_independence(struct archive_write_filter *,
211 const char *, size_t);
212 static int drive_compressor_dependence(struct archive_write_filter *,
213 const char *, size_t);
214 static int lz4_write_stream_descriptor(struct archive_write_filter *);
215 static ssize_t lz4_write_one_block(struct archive_write_filter *, const char *,
216 size_t);
220 * Setup callback.
222 static int
223 archive_filter_lz4_open(struct archive_write_filter *f)
225 struct private_data *data = (struct private_data *)f->data;
226 int ret;
227 size_t required_size;
228 static size_t bkmap[] = { 64 * 1024, 256 * 1024, 1 * 1024 * 1024,
229 4 * 1024 * 1024 };
230 size_t pre_block_size;
232 ret = __archive_write_open_filter(f->next_filter);
233 if (ret != 0)
234 return (ret);
236 if (data->block_maximum_size < 4)
237 data->block_size = bkmap[0];
238 else
239 data->block_size = bkmap[data->block_maximum_size - 4];
241 required_size = 4 + 15 + 4 + data->block_size + 4 + 4;
242 if (data->out_buffer_size < required_size) {
243 size_t bs = required_size, bpb;
244 free(data->out_buffer);
245 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
246 /* Buffer size should be a multiple number of
247 * the of bytes per block for performance. */
248 bpb = archive_write_get_bytes_per_block(f->archive);
249 if (bpb > bs)
250 bs = bpb;
251 else if (bpb != 0) {
252 bs += bpb;
253 bs -= bs % bpb;
256 data->out_block_size = bs;
257 bs += required_size;
258 data->out_buffer = malloc(bs);
259 data->out = data->out_buffer;
260 data->out_buffer_size = bs;
263 pre_block_size = (data->block_independence)? 0: 64 * 1024;
264 if (data->in_buffer_size < data->block_size + pre_block_size) {
265 free(data->in_buffer_allocated);
266 data->in_buffer_size = data->block_size;
267 data->in_buffer_allocated =
268 malloc(data->in_buffer_size + pre_block_size);
269 data->in_buffer = data->in_buffer_allocated + pre_block_size;
270 if (!data->block_independence && data->compression_level >= 3)
271 data->in_buffer = data->in_buffer_allocated;
272 data->in = data->in_buffer;
273 data->in_buffer_size = data->block_size;
276 if (data->out_buffer == NULL || data->in_buffer_allocated == NULL) {
277 archive_set_error(f->archive, ENOMEM,
278 "Can't allocate data for compression buffer");
279 return (ARCHIVE_FATAL);
282 f->write = archive_filter_lz4_write;
284 return (ARCHIVE_OK);
288 * Write data to the out stream.
290 * Returns ARCHIVE_OK if all data written, error otherwise.
292 static int
293 archive_filter_lz4_write(struct archive_write_filter *f,
294 const void *buff, size_t length)
296 struct private_data *data = (struct private_data *)f->data;
297 int ret = ARCHIVE_OK;
298 const char *p;
299 size_t remaining;
300 ssize_t size;
302 /* If we haven't written a stream descriptor, we have to do it first. */
303 if (!data->header_written) {
304 ret = lz4_write_stream_descriptor(f);
305 if (ret != ARCHIVE_OK)
306 return (ret);
307 data->header_written = 1;
310 /* Update statistics */
311 data->total_in += length;
313 p = (const char *)buff;
314 remaining = length;
315 while (remaining) {
316 size_t l;
317 /* Compress input data to output buffer */
318 size = lz4_write_one_block(f, p, remaining);
319 if (size < ARCHIVE_OK)
320 return (ARCHIVE_FATAL);
321 l = data->out - data->out_buffer;
322 if (l >= data->out_block_size) {
323 ret = __archive_write_filter(f->next_filter,
324 data->out_buffer, data->out_block_size);
325 l -= data->out_block_size;
326 memcpy(data->out_buffer,
327 data->out_buffer + data->out_block_size, l);
328 data->out = data->out_buffer + l;
329 if (ret < ARCHIVE_WARN)
330 break;
332 p += size;
333 remaining -= size;
336 return (ret);
340 * Finish the compression.
342 static int
343 archive_filter_lz4_close(struct archive_write_filter *f)
345 struct private_data *data = (struct private_data *)f->data;
346 int ret, r1;
348 /* Finish compression cycle. */
349 ret = (int)lz4_write_one_block(f, NULL, 0);
350 if (ret >= 0) {
352 * Write the last block and the end of the stream data.
355 /* Write End Of Stream. */
356 memset(data->out, 0, 4); data->out += 4;
357 /* Write Stream checksum if needed. */
358 if (data->stream_checksum) {
359 unsigned int checksum;
360 checksum = __archive_xxhash.XXH32_digest(
361 data->xxh32_state);
362 data->xxh32_state = NULL;
363 archive_le32enc(data->out, checksum);
364 data->out += 4;
366 ret = __archive_write_filter(f->next_filter,
367 data->out_buffer, data->out - data->out_buffer);
370 r1 = __archive_write_close_filter(f->next_filter);
371 return (r1 < ret ? r1 : ret);
374 static int
375 archive_filter_lz4_free(struct archive_write_filter *f)
377 struct private_data *data = (struct private_data *)f->data;
379 if (data->lz4_stream != NULL) {
380 #ifdef HAVE_LZ4HC_H
381 if (data->compression_level >= 3)
382 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
383 LZ4_freeStreamHC(data->lz4_stream);
384 #else
385 LZ4_freeHC(data->lz4_stream);
386 #endif
387 else
388 #endif
389 #if LZ4_VERSION_MINOR >= 3
390 LZ4_freeStream(data->lz4_stream);
391 #else
392 LZ4_free(data->lz4_stream);
393 #endif
395 free(data->out_buffer);
396 free(data->in_buffer_allocated);
397 free(data->xxh32_state);
398 free(data);
399 f->data = NULL;
400 return (ARCHIVE_OK);
403 static int
404 lz4_write_stream_descriptor(struct archive_write_filter *f)
406 struct private_data *data = (struct private_data *)f->data;
407 uint8_t *sd;
409 sd = (uint8_t *)data->out;
410 /* Write Magic Number. */
411 archive_le32enc(&sd[0], LZ4_MAGICNUMBER);
412 /* FLG */
413 sd[4] = (data->version_number << 6)
414 | (data->block_independence << 5)
415 | (data->block_checksum << 4)
416 | (data->stream_size << 3)
417 | (data->stream_checksum << 2)
418 | (data->preset_dictionary << 0);
419 /* BD */
420 sd[5] = (data->block_maximum_size << 4);
421 sd[6] = (__archive_xxhash.XXH32(&sd[4], 2, 0) >> 8) & 0xff;
422 data->out += 7;
423 if (data->stream_checksum)
424 data->xxh32_state = __archive_xxhash.XXH32_init(0);
425 else
426 data->xxh32_state = NULL;
427 return (ARCHIVE_OK);
430 static ssize_t
431 lz4_write_one_block(struct archive_write_filter *f, const char *p,
432 size_t length)
434 struct private_data *data = (struct private_data *)f->data;
435 ssize_t r;
437 if (p == NULL) {
438 /* Compress remaining uncompressed data. */
439 if (data->in_buffer == data->in)
440 return 0;
441 else {
442 size_t l = data->in - data->in_buffer;
443 r = drive_compressor(f, data->in_buffer, l);
444 if (r == ARCHIVE_OK)
445 r = (ssize_t)l;
447 } else if ((data->block_independence || data->compression_level < 3) &&
448 data->in_buffer == data->in && length >= data->block_size) {
449 r = drive_compressor(f, p, data->block_size);
450 if (r == ARCHIVE_OK)
451 r = (ssize_t)data->block_size;
452 } else {
453 size_t remaining_size = data->in_buffer_size -
454 (data->in - data->in_buffer);
455 size_t l = (remaining_size > length)? length: remaining_size;
456 memcpy(data->in, p, l);
457 data->in += l;
458 if (l == remaining_size) {
459 r = drive_compressor(f, data->in_buffer,
460 data->block_size);
461 if (r == ARCHIVE_OK)
462 r = (ssize_t)l;
463 data->in = data->in_buffer;
464 } else
465 r = (ssize_t)l;
468 return (r);
473 * Utility function to push input data through compressor, writing
474 * full output blocks as necessary.
476 * Note that this handles both the regular write case (finishing ==
477 * false) and the end-of-archive case (finishing == true).
479 static int
480 drive_compressor(struct archive_write_filter *f, const char *p, size_t length)
482 struct private_data *data = (struct private_data *)f->data;
484 if (data->stream_checksum)
485 __archive_xxhash.XXH32_update(data->xxh32_state,
486 p, (int)length);
487 if (data->block_independence)
488 return drive_compressor_independence(f, p, length);
489 else
490 return drive_compressor_dependence(f, p, length);
493 static int
494 drive_compressor_independence(struct archive_write_filter *f, const char *p,
495 size_t length)
497 struct private_data *data = (struct private_data *)f->data;
498 unsigned int outsize;
500 #ifdef HAVE_LZ4HC_H
501 if (data->compression_level >= 3)
502 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
503 outsize = LZ4_compress_HC(p, data->out + 4,
504 (int)length, (int)data->block_size,
505 data->compression_level);
506 #else
507 outsize = LZ4_compressHC2_limitedOutput(p, data->out + 4,
508 (int)length, (int)data->block_size,
509 data->compression_level);
510 #endif
511 else
512 #endif
513 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
514 outsize = LZ4_compress_default(p, data->out + 4,
515 (int)length, (int)data->block_size);
516 #else
517 outsize = LZ4_compress_limitedOutput(p, data->out + 4,
518 (int)length, (int)data->block_size);
519 #endif
521 if (outsize) {
522 /* The buffer is compressed. */
523 archive_le32enc(data->out, outsize);
524 data->out += 4;
525 } else {
526 /* The buffer is not compressed. The commpressed size was
527 * bigger than its uncompressed size. */
528 archive_le32enc(data->out, length | 0x80000000);
529 data->out += 4;
530 memcpy(data->out, p, length);
531 outsize = length;
533 data->out += outsize;
534 if (data->block_checksum) {
535 unsigned int checksum =
536 __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
537 archive_le32enc(data->out, checksum);
538 data->out += 4;
540 return (ARCHIVE_OK);
543 static int
544 drive_compressor_dependence(struct archive_write_filter *f, const char *p,
545 size_t length)
547 struct private_data *data = (struct private_data *)f->data;
548 int outsize;
550 #define DICT_SIZE (64 * 1024)
551 #ifdef HAVE_LZ4HC_H
552 if (data->compression_level >= 3) {
553 if (data->lz4_stream == NULL) {
554 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
555 data->lz4_stream = LZ4_createStreamHC();
556 LZ4_resetStreamHC(data->lz4_stream, data->compression_level);
557 #else
558 data->lz4_stream =
559 LZ4_createHC(data->in_buffer_allocated);
560 #endif
561 if (data->lz4_stream == NULL) {
562 archive_set_error(f->archive, ENOMEM,
563 "Can't allocate data for compression"
564 " buffer");
565 return (ARCHIVE_FATAL);
568 else
569 LZ4_loadDictHC(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
571 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
572 outsize = LZ4_compress_HC_continue(
573 data->lz4_stream, p, data->out + 4, (int)length,
574 (int)data->block_size);
575 #else
576 outsize = LZ4_compressHC2_limitedOutput_continue(
577 data->lz4_stream, p, data->out + 4, (int)length,
578 (int)data->block_size, data->compression_level);
579 #endif
580 } else
581 #endif
583 if (data->lz4_stream == NULL) {
584 data->lz4_stream = LZ4_createStream();
585 if (data->lz4_stream == NULL) {
586 archive_set_error(f->archive, ENOMEM,
587 "Can't allocate data for compression"
588 " buffer");
589 return (ARCHIVE_FATAL);
592 else
593 LZ4_loadDict(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
595 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
596 outsize = LZ4_compress_fast_continue(
597 data->lz4_stream, p, data->out + 4, (int)length,
598 (int)data->block_size, 1);
599 #else
600 outsize = LZ4_compress_limitedOutput_continue(
601 data->lz4_stream, p, data->out + 4, (int)length,
602 (int)data->block_size);
603 #endif
606 if (outsize) {
607 /* The buffer is compressed. */
608 archive_le32enc(data->out, outsize);
609 data->out += 4;
610 } else {
611 /* The buffer is not compressed. The commpressed size was
612 * bigger than its uncompressed size. */
613 archive_le32enc(data->out, length | 0x80000000);
614 data->out += 4;
615 memcpy(data->out, p, length);
616 outsize = length;
618 data->out += outsize;
619 if (data->block_checksum) {
620 unsigned int checksum =
621 __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
622 archive_le32enc(data->out, checksum);
623 data->out += 4;
626 if (length == data->block_size) {
627 #ifdef HAVE_LZ4HC_H
628 if (data->compression_level >= 3) {
629 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
630 LZ4_saveDictHC(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
631 #else
632 LZ4_slideInputBufferHC(data->lz4_stream);
633 #endif
634 data->in_buffer = data->in_buffer_allocated + DICT_SIZE;
636 else
637 #endif
638 LZ4_saveDict(data->lz4_stream,
639 data->in_buffer_allocated, DICT_SIZE);
640 #undef DICT_SIZE
642 return (ARCHIVE_OK);
645 #else /* HAVE_LIBLZ4 */
647 static int
648 archive_filter_lz4_open(struct archive_write_filter *f)
650 struct private_data *data = (struct private_data *)f->data;
651 struct archive_string as;
652 int r;
654 archive_string_init(&as);
655 archive_strcpy(&as, "lz4 -z -q -q");
657 /* Specify a compression level. */
658 if (data->compression_level > 0) {
659 archive_strcat(&as, " -");
660 archive_strappend_char(&as, '0' + data->compression_level);
662 /* Specify a block size. */
663 archive_strcat(&as, " -B");
664 archive_strappend_char(&as, '0' + data->block_maximum_size);
666 if (data->block_checksum)
667 archive_strcat(&as, " -BX");
668 if (data->stream_checksum == 0)
669 archive_strcat(&as, " --no-frame-crc");
670 if (data->block_independence == 0)
671 archive_strcat(&as, " -BD");
673 f->write = archive_filter_lz4_write;
675 r = __archive_write_program_open(f, data->pdata, as.s);
676 archive_string_free(&as);
677 return (r);
680 static int
681 archive_filter_lz4_write(struct archive_write_filter *f, const void *buff,
682 size_t length)
684 struct private_data *data = (struct private_data *)f->data;
686 return __archive_write_program_write(f, data->pdata, buff, length);
689 static int
690 archive_filter_lz4_close(struct archive_write_filter *f)
692 struct private_data *data = (struct private_data *)f->data;
694 return __archive_write_program_close(f, data->pdata);
697 static int
698 archive_filter_lz4_free(struct archive_write_filter *f)
700 struct private_data *data = (struct private_data *)f->data;
702 __archive_write_program_free(data->pdata);
703 free(data);
704 return (ARCHIVE_OK);
707 #endif /* HAVE_LIBLZ4 */