2 * Copyright (c) 2003-2010 Tim Kientzle
3 * Copyright (c) 2009-2012 Michihiro NAKAJIMA
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 #include "archive_platform.h"
29 __FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_compression_xz.c 201108 2009-12-28 03:28:21Z kientzle $");
46 #include "archive_endian.h"
47 #include "archive_private.h"
48 #include "archive_write_private.h"
50 #if ARCHIVE_VERSION_NUMBER < 4000000
52 archive_write_set_compression_lzip(struct archive
*a
)
54 __archive_write_filters_free(a
);
55 return (archive_write_add_filter_lzip(a
));
59 archive_write_set_compression_lzma(struct archive
*a
)
61 __archive_write_filters_free(a
);
62 return (archive_write_add_filter_lzma(a
));
66 archive_write_set_compression_xz(struct archive
*a
)
68 __archive_write_filters_free(a
);
69 return (archive_write_add_filter_xz(a
));
76 archive_write_add_filter_xz(struct archive
*a
)
78 archive_set_error(a
, ARCHIVE_ERRNO_MISC
,
79 "xz compression not supported on this platform");
80 return (ARCHIVE_FATAL
);
84 archive_write_add_filter_lzma(struct archive
*a
)
86 archive_set_error(a
, ARCHIVE_ERRNO_MISC
,
87 "lzma compression not supported on this platform");
88 return (ARCHIVE_FATAL
);
92 archive_write_add_filter_lzip(struct archive
*a
)
94 archive_set_error(a
, ARCHIVE_ERRNO_MISC
,
95 "lzma compression not supported on this platform");
96 return (ARCHIVE_FATAL
);
99 /* Don't compile this if we don't have liblzma. */
101 struct private_data
{
102 int compression_level
;
105 lzma_filter lzmafilters
[2];
106 lzma_options_lzma lzma_opt
;
108 unsigned char *compressed
;
109 size_t compressed_buffer_size
;
111 /* the CRC32 value of uncompressed data for lzip */
115 static int archive_compressor_xz_options(struct archive_write_filter
*,
116 const char *, const char *);
117 static int archive_compressor_xz_open(struct archive_write_filter
*);
118 static int archive_compressor_xz_write(struct archive_write_filter
*,
119 const void *, size_t);
120 static int archive_compressor_xz_close(struct archive_write_filter
*);
121 static int archive_compressor_xz_free(struct archive_write_filter
*);
122 static int drive_compressor(struct archive_write_filter
*,
123 struct private_data
*, int finishing
);
125 struct option_value
{
128 lzma_match_finder mf
;
130 static const struct option_value option_values
[] = {
131 { 1 << 16, 32, LZMA_MF_HC3
},
132 { 1 << 20, 32, LZMA_MF_HC3
},
133 { 3 << 19, 32, LZMA_MF_HC4
},
134 { 1 << 21, 32, LZMA_MF_BT4
},
135 { 3 << 20, 32, LZMA_MF_BT4
},
136 { 1 << 22, 32, LZMA_MF_BT4
},
137 { 1 << 23, 64, LZMA_MF_BT4
},
138 { 1 << 24, 64, LZMA_MF_BT4
},
139 { 3 << 23, 64, LZMA_MF_BT4
},
140 { 1 << 25, 64, LZMA_MF_BT4
}
144 common_setup(struct archive_write_filter
*f
)
146 struct private_data
*data
;
147 struct archive_write
*a
= (struct archive_write
*)f
->archive
;
148 data
= calloc(1, sizeof(*data
));
150 archive_set_error(&a
->archive
, ENOMEM
, "Out of memory");
151 return (ARCHIVE_FATAL
);
154 data
->compression_level
= LZMA_PRESET_DEFAULT
;
156 f
->open
= &archive_compressor_xz_open
;
157 f
->close
= archive_compressor_xz_close
;
158 f
->free
= archive_compressor_xz_free
;
159 f
->options
= &archive_compressor_xz_options
;
164 * Add an xz compression filter to this write handle.
167 archive_write_add_filter_xz(struct archive
*_a
)
169 struct archive_write_filter
*f
;
172 archive_check_magic(_a
, ARCHIVE_WRITE_MAGIC
,
173 ARCHIVE_STATE_NEW
, "archive_write_add_filter_xz");
174 f
= __archive_write_allocate_filter(_a
);
176 if (r
== ARCHIVE_OK
) {
177 f
->code
= ARCHIVE_FILTER_XZ
;
183 /* LZMA is handled identically, we just need a different compression
184 * code set. (The liblzma setup looks at the code to determine
185 * the one place that XZ and LZMA require different handling.) */
187 archive_write_add_filter_lzma(struct archive
*_a
)
189 struct archive_write_filter
*f
;
192 archive_check_magic(_a
, ARCHIVE_WRITE_MAGIC
,
193 ARCHIVE_STATE_NEW
, "archive_write_add_filter_lzma");
194 f
= __archive_write_allocate_filter(_a
);
196 if (r
== ARCHIVE_OK
) {
197 f
->code
= ARCHIVE_FILTER_LZMA
;
204 archive_write_add_filter_lzip(struct archive
*_a
)
206 struct archive_write_filter
*f
;
209 archive_check_magic(_a
, ARCHIVE_WRITE_MAGIC
,
210 ARCHIVE_STATE_NEW
, "archive_write_add_filter_lzip");
211 f
= __archive_write_allocate_filter(_a
);
213 if (r
== ARCHIVE_OK
) {
214 f
->code
= ARCHIVE_FILTER_LZIP
;
221 archive_compressor_xz_init_stream(struct archive_write_filter
*f
,
222 struct private_data
*data
)
224 static const lzma_stream lzma_stream_init_data
= LZMA_STREAM_INIT
;
226 #ifdef HAVE_LZMA_STREAM_ENCODER_MT
230 data
->stream
= lzma_stream_init_data
;
231 data
->stream
.next_out
= data
->compressed
;
232 data
->stream
.avail_out
= data
->compressed_buffer_size
;
233 if (f
->code
== ARCHIVE_FILTER_XZ
) {
234 #ifdef HAVE_LZMA_STREAM_ENCODER_MT
235 if (data
->threads
!= 1) {
236 bzero(&mt_options
, sizeof(mt_options
));
237 mt_options
.threads
= data
->threads
;
238 mt_options
.timeout
= 300;
239 mt_options
.filters
= data
->lzmafilters
;
240 mt_options
.check
= LZMA_CHECK_CRC64
;
241 ret
= lzma_stream_encoder_mt(&(data
->stream
),
245 ret
= lzma_stream_encoder(&(data
->stream
),
246 data
->lzmafilters
, LZMA_CHECK_CRC64
);
247 } else if (f
->code
== ARCHIVE_FILTER_LZMA
) {
248 ret
= lzma_alone_encoder(&(data
->stream
), &data
->lzma_opt
);
249 } else { /* ARCHIVE_FILTER_LZIP */
250 int dict_size
= data
->lzma_opt
.dict_size
;
251 int ds
, log2dic
, wedges
;
253 /* Calculate a coded dictionary size */
254 if (dict_size
< (1 << 12) || dict_size
> (1 << 27)) {
255 archive_set_error(f
->archive
, ARCHIVE_ERRNO_MISC
,
256 "Unacceptable dictionary size for lzip: %d",
258 return (ARCHIVE_FATAL
);
260 for (log2dic
= 27; log2dic
>= 12; log2dic
--) {
261 if (dict_size
& (1 << log2dic
))
264 if (dict_size
> (1 << log2dic
)) {
267 ((1 << log2dic
) - dict_size
) / (1 << (log2dic
- 4));
270 ds
= ((wedges
<< 5) & 0xe0) | (log2dic
& 0x1f);
274 data
->compressed
[0] = 0x4C;
275 data
->compressed
[1] = 0x5A;
276 data
->compressed
[2] = 0x49;
277 data
->compressed
[3] = 0x50;
278 data
->compressed
[4] = 1;/* Version */
279 data
->compressed
[5] = (unsigned char)ds
;
280 data
->stream
.next_out
+= 6;
281 data
->stream
.avail_out
-= 6;
283 ret
= lzma_raw_encoder(&(data
->stream
), data
->lzmafilters
);
290 archive_set_error(f
->archive
, ENOMEM
,
291 "Internal error initializing compression library: "
292 "Cannot allocate memory");
295 archive_set_error(f
->archive
, ARCHIVE_ERRNO_MISC
,
296 "Internal error initializing compression library: "
297 "It's a bug in liblzma");
300 return (ARCHIVE_FATAL
);
307 archive_compressor_xz_open(struct archive_write_filter
*f
)
309 struct private_data
*data
= f
->data
;
312 ret
= __archive_write_open_filter(f
->next_filter
);
313 if (ret
!= ARCHIVE_OK
)
316 if (data
->compressed
== NULL
) {
317 size_t bs
= 65536, bpb
;
318 if (f
->archive
->magic
== ARCHIVE_WRITE_MAGIC
) {
319 /* Buffer size should be a multiple number of the of bytes
320 * per block for performance. */
321 bpb
= archive_write_get_bytes_per_block(f
->archive
);
327 data
->compressed_buffer_size
= bs
;
329 = (unsigned char *)malloc(data
->compressed_buffer_size
);
330 if (data
->compressed
== NULL
) {
331 archive_set_error(f
->archive
, ENOMEM
,
332 "Can't allocate data for compression buffer");
333 return (ARCHIVE_FATAL
);
337 f
->write
= archive_compressor_xz_write
;
339 /* Initialize compression library. */
340 if (f
->code
== ARCHIVE_FILTER_LZIP
) {
341 const struct option_value
*val
=
342 &option_values
[data
->compression_level
];
344 data
->lzma_opt
.dict_size
= val
->dict_size
;
345 data
->lzma_opt
.preset_dict
= NULL
;
346 data
->lzma_opt
.preset_dict_size
= 0;
347 data
->lzma_opt
.lc
= LZMA_LC_DEFAULT
;
348 data
->lzma_opt
.lp
= LZMA_LP_DEFAULT
;
349 data
->lzma_opt
.pb
= LZMA_PB_DEFAULT
;
350 data
->lzma_opt
.mode
=
351 data
->compression_level
<= 2? LZMA_MODE_FAST
:LZMA_MODE_NORMAL
;
352 data
->lzma_opt
.nice_len
= val
->nice_len
;
353 data
->lzma_opt
.mf
= val
->mf
;
354 data
->lzma_opt
.depth
= 0;
355 data
->lzmafilters
[0].id
= LZMA_FILTER_LZMA1
;
356 data
->lzmafilters
[0].options
= &data
->lzma_opt
;
357 data
->lzmafilters
[1].id
= LZMA_VLI_UNKNOWN
;/* Terminate */
359 if (lzma_lzma_preset(&data
->lzma_opt
, data
->compression_level
)) {
360 archive_set_error(f
->archive
, ARCHIVE_ERRNO_MISC
,
361 "Internal error initializing compression library");
363 data
->lzmafilters
[0].id
= LZMA_FILTER_LZMA2
;
364 data
->lzmafilters
[0].options
= &data
->lzma_opt
;
365 data
->lzmafilters
[1].id
= LZMA_VLI_UNKNOWN
;/* Terminate */
367 ret
= archive_compressor_xz_init_stream(f
, data
);
368 if (ret
== LZMA_OK
) {
372 return (ARCHIVE_FATAL
);
379 archive_compressor_xz_options(struct archive_write_filter
*f
,
380 const char *key
, const char *value
)
382 struct private_data
*data
= (struct private_data
*)f
->data
;
384 if (strcmp(key
, "compression-level") == 0) {
385 if (value
== NULL
|| !(value
[0] >= '0' && value
[0] <= '9') ||
387 return (ARCHIVE_WARN
);
388 data
->compression_level
= value
[0] - '0';
389 if (data
->compression_level
> 6)
390 data
->compression_level
= 6;
392 } else if (strcmp(key
, "threads") == 0) {
394 return (ARCHIVE_WARN
);
395 data
->threads
= (int)strtoul(value
, NULL
, 10);
396 if (data
->threads
== 0 && errno
!= 0) {
398 return (ARCHIVE_WARN
);
400 if (data
->threads
== 0) {
401 #ifdef HAVE_LZMA_STREAM_ENCODER_MT
402 data
->threads
= lzma_cputhreads();
410 /* Note: The "warn" return is just to inform the options
411 * supervisor that we didn't handle it. It will generate
412 * a suitable error if no one used this option. */
413 return (ARCHIVE_WARN
);
417 * Write data to the compressed stream.
420 archive_compressor_xz_write(struct archive_write_filter
*f
,
421 const void *buff
, size_t length
)
423 struct private_data
*data
= (struct private_data
*)f
->data
;
426 /* Update statistics */
427 data
->total_in
+= length
;
428 if (f
->code
== ARCHIVE_FILTER_LZIP
)
429 data
->crc32
= lzma_crc32(buff
, length
, data
->crc32
);
431 /* Compress input data to output buffer */
432 data
->stream
.next_in
= buff
;
433 data
->stream
.avail_in
= length
;
434 if ((ret
= drive_compressor(f
, data
, 0)) != ARCHIVE_OK
)
442 * Finish the compression...
445 archive_compressor_xz_close(struct archive_write_filter
*f
)
447 struct private_data
*data
= (struct private_data
*)f
->data
;
450 ret
= drive_compressor(f
, data
, 1);
451 if (ret
== ARCHIVE_OK
) {
453 data
->compressed_buffer_size
- data
->stream
.avail_out
;
454 ret
= __archive_write_filter(f
->next_filter
,
456 data
->compressed_buffer_size
- data
->stream
.avail_out
);
457 if (f
->code
== ARCHIVE_FILTER_LZIP
&& ret
== ARCHIVE_OK
) {
458 archive_le32enc(data
->compressed
, data
->crc32
);
459 archive_le64enc(data
->compressed
+4, data
->total_in
);
460 archive_le64enc(data
->compressed
+12, data
->total_out
+ 20);
461 ret
= __archive_write_filter(f
->next_filter
,
462 data
->compressed
, 20);
465 lzma_end(&(data
->stream
));
466 r1
= __archive_write_close_filter(f
->next_filter
);
467 return (r1
< ret
? r1
: ret
);
471 archive_compressor_xz_free(struct archive_write_filter
*f
)
473 struct private_data
*data
= (struct private_data
*)f
->data
;
474 free(data
->compressed
);
481 * Utility function to push input data through compressor,
482 * writing full output blocks as necessary.
484 * Note that this handles both the regular write case (finishing ==
485 * false) and the end-of-archive case (finishing == true).
488 drive_compressor(struct archive_write_filter
*f
,
489 struct private_data
*data
, int finishing
)
494 if (data
->stream
.avail_out
== 0) {
495 data
->total_out
+= data
->compressed_buffer_size
;
496 ret
= __archive_write_filter(f
->next_filter
,
498 data
->compressed_buffer_size
);
499 if (ret
!= ARCHIVE_OK
)
500 return (ARCHIVE_FATAL
);
501 data
->stream
.next_out
= data
->compressed
;
502 data
->stream
.avail_out
= data
->compressed_buffer_size
;
505 /* If there's nothing to do, we're done. */
506 if (!finishing
&& data
->stream
.avail_in
== 0)
509 ret
= lzma_code(&(data
->stream
),
510 finishing
? LZMA_FINISH
: LZMA_RUN
);
514 /* In non-finishing case, check if compressor
515 * consumed everything */
516 if (!finishing
&& data
->stream
.avail_in
== 0)
518 /* In finishing case, this return always means
519 * there's more work */
521 case LZMA_STREAM_END
:
522 /* This return can only occur in finishing case. */
525 archive_set_error(f
->archive
, ARCHIVE_ERRNO_MISC
,
526 "lzma compression data error");
527 return (ARCHIVE_FATAL
);
528 case LZMA_MEMLIMIT_ERROR
:
529 archive_set_error(f
->archive
, ENOMEM
,
530 "lzma compression error: "
531 "%ju MiB would have been needed",
532 (uintmax_t)((lzma_memusage(&(data
->stream
))
535 return (ARCHIVE_FATAL
);
537 /* Any other return value indicates an error. */
538 archive_set_error(f
->archive
, ARCHIVE_ERRNO_MISC
,
539 "lzma compression failed:"
540 " lzma_code() call returned status %d",
542 return (ARCHIVE_FATAL
);
547 #endif /* HAVE_LZMA_H */