1 // SPDX-License-Identifier: 0BSD
3 ///////////////////////////////////////////////////////////////////////////////
6 /// \brief Compresses or uncompresses a file
8 // Authors: Lasse Collin
11 ///////////////////////////////////////////////////////////////////////////////
14 #include "tuklib_integer.h"
17 /// Return value type for coder_init().
25 enum operation_mode opt_mode
= MODE_COMPRESS
;
26 enum format_type opt_format
= FORMAT_AUTO
;
27 bool opt_auto_adjust
= true;
28 bool opt_single_stream
= false;
29 uint64_t opt_block_size
= 0;
30 block_list_entry
*opt_block_list
= NULL
;
31 uint64_t block_list_largest
;
32 uint32_t block_list_chain_mask
;
34 /// Stream used to communicate with liblzma
35 static lzma_stream strm
= LZMA_STREAM_INIT
;
37 /// Maximum number of filter chains. The first filter chain is the default,
38 /// and 9 other filter chains can be specified with --filtersX.
39 #define NUM_FILTER_CHAIN_MAX 10
41 /// The default filter chain is in chains[0]. It is used for encoding
42 /// in all supported formats and also for decdoing raw streams. The other
43 /// filter chains are set by --filtersX to support changing filters with
44 /// the --block-list option.
45 static lzma_filter chains
[NUM_FILTER_CHAIN_MAX
][LZMA_FILTERS_MAX
+ 1];
47 /// Bitmask indicating which filter chains are actually used when encoding
48 /// in the .xz format. This is needed since the filter chains specified using
49 /// --filtersX (or the default filter chain) might in reality be unneeded
50 /// if they are never used in --block-list. When --block-list isn't
51 /// specified, only the default filter chain is used, thus the initial
52 /// value of this variable is 1U << 0 (the number of the default chain is 0).
53 static uint32_t chains_used_mask
= 1U << 0;
55 /// Input and output buffers
57 static io_buf out_buf
;
59 /// Number of filters in the default filter chain. Zero indicates that
60 /// we are using a preset.
61 static uint32_t filters_count
= 0;
63 /// Number of the preset (0-9)
64 static uint32_t preset_number
= LZMA_PRESET_DEFAULT
;
66 /// True if the current default filter chain was set using the --filters
67 /// option. The filter chain is reset if a preset option (like -9) or an
68 /// old-style filter option (like --lzma2) is used after a --filters option.
69 static bool string_to_filter_used
= false;
71 /// Integrity check type
72 static lzma_check check
;
74 /// This becomes false if the --check=CHECK option is used.
75 static bool check_default
= true;
77 /// Indicates if unconsumed input is allowed to remain after
78 /// decoding has successfully finished. This is set for each file
80 static bool allow_trailing_input
;
82 #ifdef MYTHREAD_ENABLED
83 static lzma_mt mt_options
= {
91 coder_set_check(lzma_check new_check
)
94 check_default
= false;
100 forget_filter_chain(void)
102 // Setting a preset or using --filters makes us forget
103 // the earlier custom filter chain (if any).
104 if (filters_count
> 0) {
105 lzma_filters_free(chains
[0], NULL
);
109 string_to_filter_used
= false;
115 coder_set_preset(uint32_t new_preset
)
117 preset_number
&= ~LZMA_PRESET_LEVEL_MASK
;
118 preset_number
|= new_preset
;
119 forget_filter_chain();
125 coder_set_extreme(void)
127 preset_number
|= LZMA_PRESET_EXTREME
;
128 forget_filter_chain();
134 coder_add_filter(lzma_vli id
, void *options
)
136 if (filters_count
== LZMA_FILTERS_MAX
)
137 message_fatal(_("Maximum number of filters is four"));
139 if (string_to_filter_used
)
140 forget_filter_chain();
142 chains
[0][filters_count
].id
= id
;
143 chains
[0][filters_count
].options
= options
;
145 // Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify
146 // implementation of forget_filter_chain().
147 chains
[0][++filters_count
].id
= LZMA_VLI_UNKNOWN
;
149 // Setting a custom filter chain makes us forget the preset options.
150 // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
151 // where the custom filter chain resets the preset level back to
152 // the default 6, making the example equivalent to "xz -6e".
153 preset_number
= LZMA_PRESET_DEFAULT
;
160 str_to_filters(const char *str
, uint32_t index
, uint32_t flags
)
163 const char *err
= lzma_str_to_filters(str
, &error_pos
,
164 chains
[index
], flags
, NULL
);
167 char filter_num
[2] = "";
169 filter_num
[0] = '0' + index
;
171 // FIXME? The message in err isn't translated.
172 // Including the translations in the xz translations is
173 // slightly ugly but possible. Creating a new domain for
174 // liblzma might not be worth it especially since on some
175 // OSes it adds extra dependencies to translation libraries.
176 message(V_ERROR
, _("Error in --filters%s=FILTERS option:"),
178 message(V_ERROR
, "%s", str
);
179 message(V_ERROR
, "%*s^", error_pos
, "");
180 message_fatal("%s", err
);
186 coder_add_filters_from_str(const char *filter_str
)
188 // Forget presets and previously defined filter chain. See
189 // coder_add_filter() above for why preset_number must be reset too.
190 forget_filter_chain();
191 preset_number
= LZMA_PRESET_DEFAULT
;
193 string_to_filter_used
= true;
195 // Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw.
196 str_to_filters(filter_str
, 0, LZMA_STR_ALL_FILTERS
);
198 // Set the filters_count to be the number of filters converted from
200 for (filters_count
= 0; chains
[0][filters_count
].id
204 assert(filters_count
> 0);
210 coder_add_block_filters(const char *str
, size_t slot
)
212 // Free old filters first, if they were previously allocated.
213 if (chains_used_mask
& (1U << slot
))
214 lzma_filters_free(chains
[slot
], NULL
);
216 str_to_filters(str
, slot
, 0);
218 chains_used_mask
|= 1U << slot
;
224 memlimit_too_small(uint64_t memory_usage
)
226 message(V_ERROR
, _("Memory usage limit is too low for the given "
228 message_mem_needed(V_ERROR
, memory_usage
);
229 tuklib_exit(E_ERROR
, E_ERROR
, false);
234 /// \brief Calculate the memory usage of each filter chain.
236 /// \param chains_memusages If non-NULL, the memusage of the encoder
237 /// or decoder for each chain is stored in
239 /// \param mt If non-NULL, calculate memory usage of
240 /// multithreaded encoder.
241 /// \param encode Whether to calculate encoder or decoder
242 /// memory usage. This must be true if
245 /// \return Return the highest memory usage of all of the filter chains.
247 get_chains_memusage(uint64_t *chains_memusages
, const lzma_mt
*mt
, bool encode
)
249 uint64_t max_memusage
= 0;
251 #ifdef MYTHREAD_ENABLED
252 // Copy multithreading options to a temporary struct since the
253 // "filters" member needs to be changed.
261 for (uint32_t i
= 0; i
< ARRAY_SIZE(chains
); i
++) {
262 if (!(chains_used_mask
& (1U << i
)))
265 uint64_t memusage
= UINT64_MAX
;
266 #ifdef MYTHREAD_ENABLED
269 mt_local
.filters
= chains
[i
];
270 memusage
= lzma_stream_encoder_mt_memusage(&mt_local
);
274 memusage
= lzma_raw_encoder_memusage(chains
[i
]);
278 memusage
= lzma_raw_decoder_memusage(chains
[i
]);
282 if (chains_memusages
!= NULL
)
283 chains_memusages
[i
] = memusage
;
285 if (memusage
> max_memusage
)
286 max_memusage
= memusage
;
295 coder_set_compression_settings(void)
297 #ifdef HAVE_LZIP_DECODER
298 // .lz compression isn't supported.
299 assert(opt_format
!= FORMAT_LZIP
);
302 // The default check type is CRC64, but fallback to CRC32
303 // if CRC64 isn't supported by the copy of liblzma we are
304 // using. CRC32 is always supported.
306 check
= LZMA_CHECK_CRC64
;
307 if (!lzma_check_is_supported(check
))
308 check
= LZMA_CHECK_CRC32
;
312 if (opt_block_list
!= NULL
) {
313 // args.c ensures these.
314 assert(opt_mode
== MODE_COMPRESS
);
315 assert(opt_format
== FORMAT_XZ
);
317 // Find out if block_list_chain_mask has a bit set that
318 // isn't set in chains_used_mask.
319 const uint32_t missing_chains_mask
320 = (block_list_chain_mask
^ chains_used_mask
)
321 & block_list_chain_mask
;
323 // If a filter chain was specified in --block-list but no
324 // matching --filtersX option was used, exit with an error.
325 if (missing_chains_mask
!= 0) {
326 // Get the number of the first missing filter chain
327 // and show it in the error message.
328 const unsigned first_missing
329 = (unsigned)ctz32(missing_chains_mask
);
331 message_fatal(_("filter chain %u used by "
332 "--block-list but not specified "
333 "with --filters%u="),
334 first_missing
, first_missing
);
337 // Omit the unused filter chains from mask of used chains.
339 // (FIXME? When built with debugging, coder_free() will free()
340 // the filter chains (except the default chain) which makes
341 // Valgrind show fewer reachable allocations. But coder_free()
342 // uses this mask to determine which chains to free. Thus it
343 // won't free the ones that are cleared here from the mask.
344 // In practice this doesn't matter.)
345 chains_used_mask
&= block_list_chain_mask
;
347 // Reset filters used mask in case --block-list is not
348 // used, but --filtersX is used.
349 chains_used_mask
= 1U << 0;
353 // Options for LZMA1 or LZMA2 in case we are using a preset.
354 static lzma_options_lzma opt_lzma
;
356 // The first filter in the chains[] array is for the default
358 lzma_filter
*default_filters
= chains
[0];
360 if (filters_count
== 0 && chains_used_mask
& 1) {
361 // We are using a preset. This is not a good idea in raw mode
362 // except when playing around with things. Different versions
363 // of this software may use different options in presets, and
364 // thus make uncompressing the raw data difficult.
365 if (opt_format
== FORMAT_RAW
) {
366 // The message is shown only if warnings are allowed
367 // but the exit status isn't changed.
368 message(V_WARNING
, _("Using a preset in raw mode "
370 message(V_WARNING
, _("The exact options of the "
371 "presets may vary between software "
375 // Get the preset for LZMA1 or LZMA2.
376 if (lzma_lzma_preset(&opt_lzma
, preset_number
))
379 // Use LZMA2 except with --format=lzma we use LZMA1.
380 default_filters
[0].id
= opt_format
== FORMAT_LZMA
381 ? LZMA_FILTER_LZMA1
: LZMA_FILTER_LZMA2
;
382 default_filters
[0].options
= &opt_lzma
;
386 // Terminate the filter options array.
387 default_filters
[1].id
= LZMA_VLI_UNKNOWN
;
390 // If we are using the .lzma format, allow exactly one filter
391 // which has to be LZMA1. There is no need to check if the default
392 // filter chain is being used since it can only be disabled if
393 // --block-list is used, which is incompatible with FORMAT_LZMA.
394 if (opt_format
== FORMAT_LZMA
&& (filters_count
!= 1
395 || default_filters
[0].id
!= LZMA_FILTER_LZMA1
))
396 message_fatal(_("The .lzma format supports only "
397 "the LZMA1 filter"));
399 // If we are using the .xz format, make sure that there is no LZMA1
400 // filter to prevent LZMA_PROG_ERROR. With the chains from --filtersX
401 // we have already ensured this by calling lzma_str_to_filters()
402 // without setting the flags that would allow non-.xz filters.
403 if (opt_format
== FORMAT_XZ
&& chains_used_mask
& 1)
404 for (size_t i
= 0; i
< filters_count
; ++i
)
405 if (default_filters
[i
].id
== LZMA_FILTER_LZMA1
)
406 message_fatal(_("LZMA1 cannot be used "
407 "with the .xz format"));
409 if (chains_used_mask
& 1) {
410 // Print the selected default filter chain.
411 message_filters_show(V_DEBUG
, default_filters
);
414 // The --flush-timeout option requires LZMA_SYNC_FLUSH support
415 // from the filter chain. Currently the threaded encoder doesn't
416 // support LZMA_SYNC_FLUSH so single-threaded mode must be used.
417 if (opt_mode
== MODE_COMPRESS
&& opt_flush_timeout
!= 0) {
418 for (unsigned i
= 0; i
< ARRAY_SIZE(chains
); ++i
) {
419 if (!(chains_used_mask
& (1U << i
)))
422 const lzma_filter
*fc
= chains
[i
];
423 for (size_t j
= 0; fc
[j
].id
!= LZMA_VLI_UNKNOWN
; j
++) {
425 case LZMA_FILTER_LZMA2
:
426 case LZMA_FILTER_DELTA
:
430 message_fatal(_("Filter chain %u is "
438 if (hardware_threads_is_mt()) {
439 message(V_WARNING
, _("Switching to single-threaded "
440 "mode due to --flush-timeout"));
441 hardware_threads_set(1);
445 // Get memory limit and the memory usage of the used filter chains.
446 // Note that if --format=raw was used, we can be decompressing
447 // using the default filter chain.
449 // If multithreaded .xz compression is done, the memory limit
451 uint64_t memory_limit
= hardware_memlimit_get(opt_mode
);
452 uint64_t memory_usage
= UINT64_MAX
;
455 // Memory usage for each encoder filter chain (default
456 // or --filtersX). The encoder options may need to be
457 // scaled down depending on the memory usage limit.
458 uint64_t encoder_memusages
[ARRAY_SIZE(chains
)];
461 if (opt_mode
== MODE_COMPRESS
) {
463 # ifdef MYTHREAD_ENABLED
464 if (opt_format
== FORMAT_XZ
&& hardware_threads_is_mt()) {
465 memory_limit
= hardware_memlimit_mtenc_get();
466 mt_options
.threads
= hardware_threads_get();
468 uint64_t block_size
= opt_block_size
;
470 // If opt_block_size is not set, find the maximum
471 // recommended Block size based on the filter chains
472 if (block_size
== 0) {
473 for (unsigned i
= 0; i
< ARRAY_SIZE(chains
);
475 if (!(chains_used_mask
& (1U << i
)))
478 uint64_t size
= lzma_mt_block_size(
481 // If this returns an error, then one
482 // of the filter chains in use is
483 // invalid, so there is no point in
484 // progressing further.
485 if (size
== UINT64_MAX
)
486 message_fatal(_("Unsupported "
490 if (size
> block_size
)
494 // If --block-list was used and our current
495 // Block size exceeds the largest size
496 // in --block-list, reduce the Block size of
497 // the multithreaded encoder. The extra size
498 // would only be a waste of RAM. With a
499 // smaller Block size we might even be able
500 // to use more threads in some cases.
501 if (block_list_largest
> 0 && block_size
502 > block_list_largest
)
503 block_size
= block_list_largest
;
506 mt_options
.block_size
= block_size
;
507 mt_options
.check
= check
;
509 memory_usage
= get_chains_memusage(encoder_memusages
,
511 if (memory_usage
!= UINT64_MAX
)
512 message(V_DEBUG
, _("Using up to %" PRIu32
518 memory_usage
= get_chains_memusage(encoder_memusages
,
524 memory_usage
= lzma_raw_decoder_memusage(default_filters
);
528 if (memory_usage
== UINT64_MAX
)
529 message_fatal(_("Unsupported filter chain or filter options"));
531 // Print memory usage info before possible dictionary
532 // size auto-adjusting.
534 // NOTE: If only encoder support was built, we cannot show
535 // what the decoder memory usage will be.
536 message_mem_needed(V_DEBUG
, memory_usage
);
538 #if defined(HAVE_ENCODERS) && defined(HAVE_DECODERS)
539 if (opt_mode
== MODE_COMPRESS
&& message_verbosity_get() >= V_DEBUG
) {
540 const uint64_t decmem
= get_chains_memusage(NULL
, NULL
, false);
541 if (decmem
!= UINT64_MAX
)
542 message(V_DEBUG
, _("Decompression will need "
543 "%s MiB of memory."), uint64_to_str(
544 round_up_to_mib(decmem
), 0));
548 if (memory_usage
<= memory_limit
)
551 // With --format=raw settings are never adjusted to meet
552 // the memory usage limit.
553 if (opt_format
== FORMAT_RAW
)
554 memlimit_too_small(memory_usage
);
556 assert(opt_mode
== MODE_COMPRESS
);
559 # ifdef MYTHREAD_ENABLED
560 if (opt_format
== FORMAT_XZ
&& hardware_threads_is_mt()) {
561 // Try to reduce the number of threads before
562 // adjusting the compression settings down.
563 while (mt_options
.threads
> 1) {
564 // Reduce the number of threads by one and check
566 --mt_options
.threads
;
567 memory_usage
= get_chains_memusage(encoder_memusages
,
569 if (memory_usage
== UINT64_MAX
)
572 if (memory_usage
<= memory_limit
) {
573 // The memory usage is now low enough.
575 // Since 5.6.1: This is only shown at
576 // V_DEBUG instead of V_WARNING because
577 // changing the number of threads doesn't
578 // affect the output. On some systems this
579 // message would be too common now that
580 // multithreaded compression is the default.
581 message(V_DEBUG
, _("Reduced the number of "
582 "threads from %s to %s to not exceed "
583 "the memory usage limit of %s MiB"),
585 hardware_threads_get(), 0),
586 uint64_to_str(mt_options
.threads
, 1),
587 uint64_to_str(round_up_to_mib(
593 // If the memory usage limit is only a soft limit (automatic
594 // number of threads and no --memlimit-compress), the limit
595 // is only used to reduce the number of threads and once at
596 // just one thread, the limit is completely ignored. This
597 // way -T0 won't use insane amount of memory but at the same
598 // time the soft limit will never make xz fail and never make
599 // xz change settings that would affect the compressed output.
601 // Since 5.6.1: Like above, this is now shown at V_DEBUG
602 // instead of V_WARNING.
603 if (hardware_memlimit_mtenc_is_default()) {
604 message(V_DEBUG
, _("Reduced the number of threads "
605 "from %s to one. The automatic memory usage "
606 "limit of %s MiB is still being exceeded. "
607 "%s MiB of memory is required. "
608 "Continuing anyway."),
609 uint64_to_str(hardware_threads_get(), 0),
611 round_up_to_mib(memory_limit
), 1),
613 round_up_to_mib(memory_usage
), 2));
617 // If --no-adjust was used, we cannot drop to single-threaded
618 // mode since it produces different compressed output.
620 // NOTE: In xz 5.2.x, --no-adjust also prevented reducing
621 // the number of threads. This changed in 5.3.3alpha.
622 if (!opt_auto_adjust
)
623 memlimit_too_small(memory_usage
);
625 // Switch to single-threaded mode. It uses
626 // less memory than using one thread in
627 // the multithreaded mode but the output
628 // is also different.
629 hardware_threads_set(1);
630 memory_usage
= get_chains_memusage(encoder_memusages
,
632 message(V_WARNING
, _("Switching to single-threaded mode "
633 "to not exceed the memory usage limit of %s MiB"),
634 uint64_to_str(round_up_to_mib(memory_limit
), 0));
638 if (memory_usage
<= memory_limit
)
641 // Don't adjust LZMA2 or LZMA1 dictionary size if --no-adjust
642 // was specified as that would change the compressed output.
643 if (!opt_auto_adjust
)
644 memlimit_too_small(memory_usage
);
646 // Adjust each filter chain that is exceeding the memory usage limit.
647 for (unsigned i
= 0; i
< ARRAY_SIZE(chains
); i
++) {
648 // Skip unused chains.
649 if (!(chains_used_mask
& (1U << i
)))
652 // Skip chains that already meet the memory usage limit.
653 if (encoder_memusages
[i
] <= memory_limit
)
656 // Look for the last filter if it is LZMA2 or LZMA1, so we
657 // can make it use less RAM. We cannot adjust other filters.
659 while (chains
[i
][j
].id
!= LZMA_FILTER_LZMA2
660 && chains
[i
][j
].id
!= LZMA_FILTER_LZMA1
) {
661 // NOTE: This displays the too high limit of this
662 // particular filter chain. If multiple chains are
663 // specified and another one would need more then
664 // this message could be confusing. As long as LZMA2
665 // is the only memory hungry filter in .xz this
666 // doesn't matter at all in practice.
668 // FIXME? However, it's sort of odd still if we had
669 // switched from multithreaded mode to single-threaded
670 // mode because single-threaded produces different
671 // output. So the messages could perhaps be clearer.
672 // Another case of this is a few lines below.
673 if (chains
[i
][j
].id
== LZMA_VLI_UNKNOWN
)
674 memlimit_too_small(encoder_memusages
[i
]);
679 // Decrease the dictionary size until we meet the memory
680 // usage limit. First round down to full mebibytes.
681 lzma_options_lzma
*opt
= chains
[i
][j
].options
;
682 const uint32_t orig_dict_size
= opt
->dict_size
;
683 opt
->dict_size
&= ~((UINT32_C(1) << 20) - 1);
686 // If it is below 1 MiB, auto-adjusting failed.
688 // FIXME? See the FIXME a few lines above.
689 if (opt
->dict_size
< (UINT32_C(1) << 20))
690 memlimit_too_small(encoder_memusages
[i
]);
693 = lzma_raw_encoder_memusage(chains
[i
]);
694 if (encoder_memusages
[i
] == UINT64_MAX
)
697 // Accept it if it is low enough.
698 if (encoder_memusages
[i
] <= memory_limit
)
701 // Otherwise adjust it 1 MiB down and try again.
702 opt
->dict_size
-= UINT32_C(1) << 20;
705 // Tell the user that we decreased the dictionary size.
706 // The message is slightly different between the default
707 // filter chain (0) or and chains from --filtersX.
708 const char lzma_num
= chains
[i
][j
].id
== LZMA_FILTER_LZMA2
710 const char *from_size
= uint64_to_str(orig_dict_size
>> 20, 0);
711 const char *to_size
= uint64_to_str(opt
->dict_size
>> 20, 1);
712 const char *limit_size
= uint64_to_str(round_up_to_mib(
715 message(V_WARNING
, _("Adjusted LZMA%c dictionary "
716 "size from %s MiB to %s MiB to not exceed the "
717 "memory usage limit of %s MiB"),
718 lzma_num
, from_size
, to_size
, limit_size
);
720 message(V_WARNING
, _("Adjusted LZMA%c dictionary size "
721 "for --filters%u from %s MiB to %s MiB to not "
722 "exceed the memory usage limit of %s MiB"),
723 lzma_num
, i
, from_size
, to_size
, limit_size
);
732 /// Return true if the data in in_buf seems to be in the .xz format.
736 // Specify the magic as hex to be compatible with EBCDIC systems.
737 static const uint8_t magic
[6] = { 0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00 };
738 return strm
.avail_in
>= sizeof(magic
)
739 && memcmp(in_buf
.u8
, magic
, sizeof(magic
)) == 0;
743 /// Return true if the data in in_buf seems to be in the .lzma format.
747 // The .lzma header is 13 bytes.
748 if (strm
.avail_in
< 13)
751 // Decode the LZMA1 properties.
752 lzma_filter filter
= { .id
= LZMA_FILTER_LZMA1
};
753 if (lzma_properties_decode(&filter
, NULL
, in_buf
.u8
, 5) != LZMA_OK
)
756 // A hack to ditch tons of false positives: We allow only dictionary
757 // sizes that are 2^n or 2^n + 2^(n-1) or UINT32_MAX. LZMA_Alone
758 // created only files with 2^n, but accepts any dictionary size.
759 // If someone complains, this will be reconsidered.
760 lzma_options_lzma
*opt
= filter
.options
;
761 const uint32_t dict_size
= opt
->dict_size
;
764 if (dict_size
!= UINT32_MAX
) {
765 uint32_t d
= dict_size
- 1;
772 if (d
!= dict_size
|| dict_size
== 0)
776 // Another hack to ditch false positives: Assume that if the
777 // uncompressed size is known, it must be less than 256 GiB.
778 // Again, if someone complains, this will be reconsidered.
779 uint64_t uncompressed_size
= 0;
780 for (size_t i
= 0; i
< 8; ++i
)
781 uncompressed_size
|= (uint64_t)(in_buf
.u8
[5 + i
]) << (i
* 8);
783 if (uncompressed_size
!= UINT64_MAX
784 && uncompressed_size
> (UINT64_C(1) << 38))
791 #ifdef HAVE_LZIP_DECODER
792 /// Return true if the data in in_buf seems to be in the .lz format.
796 static const uint8_t magic
[4] = { 0x4C, 0x5A, 0x49, 0x50 };
797 return strm
.avail_in
>= sizeof(magic
)
798 && memcmp(in_buf
.u8
, magic
, sizeof(magic
)) == 0;
804 /// Detect the input file type (for now, this done only when decompressing),
805 /// and initialize an appropriate coder. Return value indicates if a normal
806 /// liblzma-based coder was initialized (CODER_INIT_NORMAL), if passthru
807 /// mode should be used (CODER_INIT_PASSTHRU), or if an error occurred
808 /// (CODER_INIT_ERROR).
809 static enum coder_init_ret
810 coder_init(file_pair
*pair
)
812 lzma_ret ret
= LZMA_PROG_ERROR
;
814 // In most cases if there is input left when coding finishes,
815 // something has gone wrong. Exceptions are --single-stream
816 // and decoding .lz files which can contain trailing non-.lz data.
817 // These will be handled later in this function.
818 allow_trailing_input
= false;
820 // Set the first filter chain. If the --block-list option is not
821 // used then use the default filter chain (chains[0]).
822 // Otherwise, use first filter chain from the block list.
823 lzma_filter
*active_filters
= opt_block_list
== NULL
825 : chains
[opt_block_list
[0].chain_num
];
827 if (opt_mode
== MODE_COMPRESS
) {
829 switch (opt_format
) {
831 // args.c ensures this.
836 # ifdef MYTHREAD_ENABLED
837 mt_options
.filters
= active_filters
;
838 if (hardware_threads_is_mt())
839 ret
= lzma_stream_encoder_mt(
843 ret
= lzma_stream_encoder(
844 &strm
, active_filters
, check
);
848 ret
= lzma_alone_encoder(&strm
,
849 active_filters
[0].options
);
852 # ifdef HAVE_LZIP_DECODER
854 // args.c should disallow this.
856 ret
= LZMA_PROG_ERROR
;
861 ret
= lzma_raw_encoder(&strm
, active_filters
);
869 // It seems silly to warn about unsupported check if the
870 // check won't be verified anyway due to --ignore-check.
871 if (opt_ignore_check
)
872 flags
|= LZMA_IGNORE_CHECK
;
874 flags
|= LZMA_TELL_UNSUPPORTED_CHECK
;
876 if (opt_single_stream
)
877 allow_trailing_input
= true;
879 flags
|= LZMA_CONCATENATED
;
881 // We abuse FORMAT_AUTO to indicate unknown file format,
882 // for which we may consider passthru mode.
883 enum format_type init_format
= FORMAT_AUTO
;
885 switch (opt_format
) {
887 // .lz is checked before .lzma since .lzma detection
888 // is more complicated (no magic bytes).
890 init_format
= FORMAT_XZ
;
891 # ifdef HAVE_LZIP_DECODER
892 else if (is_format_lzip())
893 init_format
= FORMAT_LZIP
;
895 else if (is_format_lzma())
896 init_format
= FORMAT_LZMA
;
901 init_format
= FORMAT_XZ
;
905 if (is_format_lzma())
906 init_format
= FORMAT_LZMA
;
909 # ifdef HAVE_LZIP_DECODER
911 if (is_format_lzip())
912 init_format
= FORMAT_LZIP
;
917 init_format
= FORMAT_RAW
;
921 switch (init_format
) {
923 // Unknown file format. If --decompress --stdout
924 // --force have been given, then we copy the input
925 // as is to stdout. Checking for MODE_DECOMPRESS
926 // is needed, because we don't want to do use
927 // passthru mode with --test.
928 if (opt_mode
== MODE_DECOMPRESS
929 && opt_stdout
&& opt_force
) {
930 // These are needed for progress info.
933 return CODER_INIT_PASSTHRU
;
936 ret
= LZMA_FORMAT_ERROR
;
940 # ifdef MYTHREAD_ENABLED
941 mt_options
.flags
= flags
;
943 mt_options
.threads
= hardware_threads_get();
944 mt_options
.memlimit_stop
945 = hardware_memlimit_get(MODE_DECOMPRESS
);
947 // If single-threaded mode was requested, set the
948 // memlimit for threading to zero. This forces the
949 // decoder to use single-threaded mode which matches
950 // the behavior of lzma_stream_decoder().
952 // Otherwise use the limit for threaded decompression
953 // which has a sane default (users are still free to
954 // make it insanely high though).
955 mt_options
.memlimit_threading
956 = mt_options
.threads
== 1
957 ? 0 : hardware_memlimit_mtdec_get();
959 ret
= lzma_stream_decoder_mt(&strm
, &mt_options
);
961 ret
= lzma_stream_decoder(&strm
,
962 hardware_memlimit_get(
963 MODE_DECOMPRESS
), flags
);
968 ret
= lzma_alone_decoder(&strm
,
969 hardware_memlimit_get(
973 # ifdef HAVE_LZIP_DECODER
975 allow_trailing_input
= true;
976 ret
= lzma_lzip_decoder(&strm
,
977 hardware_memlimit_get(
978 MODE_DECOMPRESS
), flags
);
983 // Memory usage has already been checked in
984 // coder_set_compression_settings().
985 ret
= lzma_raw_decoder(&strm
, active_filters
);
989 // Try to decode the headers. This will catch too low
990 // memory usage limit in case it happens in the first
991 // Block of the first Stream, which is where it very
992 // probably will happen if it is going to happen.
994 // This will also catch unsupported check type which
995 // we treat as a warning only. If there are empty
996 // concatenated Streams with unsupported check type then
997 // the message can be shown more than once here. The loop
998 // is used in case there is first a warning about
999 // unsupported check type and then the first Block
1000 // would exceed the memlimit.
1001 if (ret
== LZMA_OK
&& init_format
!= FORMAT_RAW
) {
1002 strm
.next_out
= NULL
;
1004 while ((ret
= lzma_code(&strm
, LZMA_RUN
))
1005 == LZMA_UNSUPPORTED_CHECK
)
1006 message_warning(_("%s: %s"), pair
->src_name
,
1009 // With --single-stream lzma_code won't wait for
1010 // LZMA_FINISH and thus it can return LZMA_STREAM_END
1011 // if the file has no uncompressed data inside.
1012 // So treat LZMA_STREAM_END as LZMA_OK here.
1013 // When lzma_code() is called again in coder_normal()
1014 // it will return LZMA_STREAM_END again.
1015 if (ret
== LZMA_STREAM_END
)
1021 if (ret
!= LZMA_OK
) {
1022 message_error(_("%s: %s"), pair
->src_name
, message_strm(ret
));
1023 if (ret
== LZMA_MEMLIMIT_ERROR
)
1024 message_mem_needed(V_ERROR
, lzma_memusage(&strm
));
1026 return CODER_INIT_ERROR
;
1029 return CODER_INIT_NORMAL
;
1033 #ifdef HAVE_ENCODERS
1034 /// Resolve conflicts between opt_block_size and opt_block_list in single
1035 /// threaded mode. We want to default to opt_block_list, except when it is
1036 /// larger than opt_block_size. If this is the case for the current Block
1037 /// at *list_pos, then we break into smaller Blocks. Otherwise advance
1038 /// to the next Block in opt_block_list, and break apart if needed.
1040 split_block(uint64_t *block_remaining
,
1041 uint64_t *next_block_remaining
,
1044 if (*next_block_remaining
> 0) {
1045 // The Block at *list_pos has previously been split up.
1046 assert(!hardware_threads_is_mt());
1047 assert(opt_block_size
> 0);
1048 assert(opt_block_list
!= NULL
);
1050 if (*next_block_remaining
> opt_block_size
) {
1051 // We have to split the current Block at *list_pos
1052 // into another opt_block_size length Block.
1053 *block_remaining
= opt_block_size
;
1055 // This is the last remaining split Block for the
1056 // Block at *list_pos.
1057 *block_remaining
= *next_block_remaining
;
1060 *next_block_remaining
-= *block_remaining
;
1063 // The Block at *list_pos has been finished. Go to the next
1064 // entry in the list. If the end of the list has been
1065 // reached, reuse the size and filters of the last Block.
1066 if (opt_block_list
[*list_pos
+ 1].size
!= 0) {
1069 // Update the filters if needed.
1070 if (opt_block_list
[*list_pos
- 1].chain_num
1071 != opt_block_list
[*list_pos
].chain_num
) {
1072 const unsigned chain_num
1073 = opt_block_list
[*list_pos
].chain_num
;
1074 const lzma_filter
*next
= chains
[chain_num
];
1075 const lzma_ret ret
= lzma_filters_update(
1078 if (ret
!= LZMA_OK
) {
1079 // This message is only possible if
1080 // the filter chain has unsupported
1081 // options since the filter chain is
1083 // lzma_raw_encoder_memusage() or
1084 // lzma_stream_encoder_mt_memusage().
1085 // Some options are not validated until
1086 // the encoders are initialized.
1088 _("Error changing to "
1089 "filter chain %u: %s"),
1096 *block_remaining
= opt_block_list
[*list_pos
].size
;
1098 // If in single-threaded mode, split up the Block if needed.
1099 // This is not needed in multi-threaded mode because liblzma
1100 // will do this due to how threaded encoding works.
1101 if (!hardware_threads_is_mt() && opt_block_size
> 0
1102 && *block_remaining
> opt_block_size
) {
1103 *next_block_remaining
1104 = *block_remaining
- opt_block_size
;
1105 *block_remaining
= opt_block_size
;
1113 coder_write_output(file_pair
*pair
)
1115 if (opt_mode
!= MODE_TEST
) {
1116 if (io_write(pair
, &out_buf
, IO_BUFFER_SIZE
- strm
.avail_out
))
1120 strm
.next_out
= out_buf
.u8
;
1121 strm
.avail_out
= IO_BUFFER_SIZE
;
1126 /// Compress or decompress using liblzma.
1128 coder_normal(file_pair
*pair
)
1130 // Encoder needs to know when we have given all the input to it.
1131 // The decoders need to know it too when we are using
1132 // LZMA_CONCATENATED. We need to check for src_eof here, because
1133 // the first input chunk has been already read if decompressing,
1134 // and that may have been the only chunk we will read.
1135 lzma_action action
= pair
->src_eof
? LZMA_FINISH
: LZMA_RUN
;
1139 // Assume that something goes wrong.
1140 bool success
= false;
1142 #ifdef HAVE_ENCODERS
1143 // block_remaining indicates how many input bytes to encode before
1144 // finishing the current .xz Block. The Block size is set with
1145 // --block-size=SIZE and --block-list. They have an effect only when
1146 // compressing to the .xz format. If block_remaining == UINT64_MAX,
1147 // only a single block is created.
1148 uint64_t block_remaining
= UINT64_MAX
;
1150 // next_block_remaining for when we are in single-threaded mode and
1151 // the Block in --block-list is larger than the --block-size=SIZE.
1152 uint64_t next_block_remaining
= 0;
1154 // Position in opt_block_list. Unused if --block-list wasn't used.
1155 size_t list_pos
= 0;
1157 // Handle --block-size for single-threaded mode and the first step
1159 if (opt_mode
== MODE_COMPRESS
&& opt_format
== FORMAT_XZ
) {
1160 // --block-size doesn't do anything here in threaded mode,
1161 // because the threaded encoder will take care of splitting
1162 // to fixed-sized Blocks.
1163 if (!hardware_threads_is_mt() && opt_block_size
> 0)
1164 block_remaining
= opt_block_size
;
1166 // If --block-list was used, start with the first size.
1168 // For threaded case, --block-size specifies how big Blocks
1169 // the encoder needs to be prepared to create at maximum
1170 // and --block-list will simultaneously cause new Blocks
1171 // to be started at specified intervals. To keep things
1172 // logical, the same is done in single-threaded mode. The
1173 // output is still not identical because in single-threaded
1174 // mode the size info isn't written into Block Headers.
1175 if (opt_block_list
!= NULL
) {
1176 if (block_remaining
< opt_block_list
[list_pos
].size
) {
1177 assert(!hardware_threads_is_mt());
1178 next_block_remaining
=
1179 opt_block_list
[list_pos
].size
1183 opt_block_list
[list_pos
].size
;
1189 strm
.next_out
= out_buf
.u8
;
1190 strm
.avail_out
= IO_BUFFER_SIZE
;
1192 while (!user_abort
) {
1193 // Fill the input buffer if it is empty and we aren't
1194 // flushing or finishing.
1195 if (strm
.avail_in
== 0 && action
== LZMA_RUN
) {
1196 strm
.next_in
= in_buf
.u8
;
1197 #ifdef HAVE_ENCODERS
1198 const size_t read_size
= my_min(block_remaining
,
1201 const size_t read_size
= IO_BUFFER_SIZE
;
1203 strm
.avail_in
= io_read(pair
, &in_buf
, read_size
);
1205 if (strm
.avail_in
== SIZE_MAX
)
1208 if (pair
->src_eof
) {
1209 action
= LZMA_FINISH
;
1211 #ifdef HAVE_ENCODERS
1212 else if (block_remaining
!= UINT64_MAX
) {
1213 // Start a new Block after every
1214 // opt_block_size bytes of input.
1215 block_remaining
-= strm
.avail_in
;
1216 if (block_remaining
== 0)
1217 action
= LZMA_FULL_BARRIER
;
1220 if (action
== LZMA_RUN
&& pair
->flush_needed
)
1221 action
= LZMA_SYNC_FLUSH
;
1225 // Let liblzma do the actual work.
1226 ret
= lzma_code(&strm
, action
);
1228 // Write out if the output buffer became full.
1229 if (strm
.avail_out
== 0) {
1230 if (coder_write_output(pair
))
1234 #ifdef HAVE_ENCODERS
1235 if (ret
== LZMA_STREAM_END
&& (action
== LZMA_SYNC_FLUSH
1236 || action
== LZMA_FULL_BARRIER
)) {
1237 if (action
== LZMA_SYNC_FLUSH
) {
1238 // Flushing completed. Write the pending data
1239 // out immediately so that the reading side
1240 // can decompress everything compressed so far.
1241 if (coder_write_output(pair
))
1244 // Mark that we haven't seen any new input
1245 // since the previous flush.
1246 pair
->src_has_seen_input
= false;
1247 pair
->flush_needed
= false;
1249 // Start a new Block after LZMA_FULL_BARRIER.
1250 if (opt_block_list
== NULL
) {
1251 assert(!hardware_threads_is_mt());
1252 assert(opt_block_size
> 0);
1253 block_remaining
= opt_block_size
;
1255 split_block(&block_remaining
,
1256 &next_block_remaining
,
1261 // Start a new Block after LZMA_FULL_FLUSH or continue
1262 // the same block after LZMA_SYNC_FLUSH.
1266 if (ret
!= LZMA_OK
) {
1267 // Determine if the return value indicates that we
1268 // won't continue coding. LZMA_NO_CHECK would be
1269 // here too if LZMA_TELL_ANY_CHECK was used.
1270 const bool stop
= ret
!= LZMA_UNSUPPORTED_CHECK
;
1273 // Write the remaining bytes even if something
1274 // went wrong, because that way the user gets
1275 // as much data as possible, which can be good
1276 // when trying to get at least some useful
1277 // data out of damaged files.
1278 if (coder_write_output(pair
))
1282 if (ret
== LZMA_STREAM_END
) {
1283 if (allow_trailing_input
) {
1284 io_fix_src_pos(pair
, strm
.avail_in
);
1289 // Check that there is no trailing garbage.
1290 // This is needed for LZMA_Alone and raw
1291 // streams. This is *not* done with .lz files
1292 // as that format specifically requires
1293 // allowing trailing garbage.
1294 if (strm
.avail_in
== 0 && !pair
->src_eof
) {
1295 // Try reading one more byte.
1296 // Hopefully we don't get any more
1297 // input, and thus pair->src_eof
1299 strm
.avail_in
= io_read(
1301 if (strm
.avail_in
== SIZE_MAX
)
1304 assert(strm
.avail_in
== 0
1305 || strm
.avail_in
== 1);
1308 if (strm
.avail_in
== 0) {
1309 assert(pair
->src_eof
);
1314 // We hadn't reached the end of the file.
1315 ret
= LZMA_DATA_ERROR
;
1319 // If we get here and stop is true, something went
1320 // wrong and we print an error. Otherwise it's just
1321 // a warning and coding can continue.
1323 message_error(_("%s: %s"), pair
->src_name
,
1326 message_warning(_("%s: %s"), pair
->src_name
,
1329 // When compressing, all possible errors set
1331 assert(opt_mode
!= MODE_COMPRESS
);
1334 if (ret
== LZMA_MEMLIMIT_ERROR
) {
1335 // Display how much memory it would have
1337 message_mem_needed(V_ERROR
,
1338 lzma_memusage(&strm
));
1345 // Show progress information under certain conditions.
1346 message_progress_update();
1353 /// Copy from input file to output file without processing the data in any
1354 /// way. This is used only when trying to decompress unrecognized files
1355 /// with --decompress --stdout --force, so the output is always stdout.
1357 coder_passthru(file_pair
*pair
)
1359 while (strm
.avail_in
!= 0) {
1363 if (io_write(pair
, &in_buf
, strm
.avail_in
))
1366 strm
.total_in
+= strm
.avail_in
;
1367 strm
.total_out
= strm
.total_in
;
1368 message_progress_update();
1370 strm
.avail_in
= io_read(pair
, &in_buf
, IO_BUFFER_SIZE
);
1371 if (strm
.avail_in
== SIZE_MAX
)
1380 coder_run(const char *filename
)
1382 // Set and possibly print the filename for the progress message.
1383 message_filename(filename
);
1385 // Try to open the input file.
1386 file_pair
*pair
= io_open_src(filename
);
1390 // Assume that something goes wrong.
1391 bool success
= false;
1393 if (opt_mode
== MODE_COMPRESS
) {
1394 strm
.next_in
= NULL
;
1397 // Read the first chunk of input data. This is needed
1398 // to detect the input file type.
1399 strm
.next_in
= in_buf
.u8
;
1400 strm
.avail_in
= io_read(pair
, &in_buf
, IO_BUFFER_SIZE
);
1403 if (strm
.avail_in
!= SIZE_MAX
) {
1404 // Initialize the coder. This will detect the file format
1405 // and, in decompression or testing mode, check the memory
1406 // usage of the first Block too. This way we don't try to
1407 // open the destination file if we see that coding wouldn't
1408 // work at all anyway. This also avoids deleting the old
1409 // "target" file if --force was used.
1410 const enum coder_init_ret init_ret
= coder_init(pair
);
1412 if (init_ret
!= CODER_INIT_ERROR
&& !user_abort
) {
1413 // Don't open the destination file when --test
1415 if (opt_mode
== MODE_TEST
|| !io_open_dest(pair
)) {
1416 // Remember the current time. It is needed
1417 // for progress indicator.
1418 mytime_set_start_time();
1420 // Initialize the progress indicator.
1422 // NOTE: When reading from stdin, fstat()
1423 // isn't called on it and thus src_st.st_size
1424 // is zero. If stdin pointed to a regular
1425 // file, it would still be possible to know
1426 // the file size but then we would also need
1427 // to take into account the current reading
1428 // position since with stdin it isn't
1429 // necessarily at the beginning of the file.
1430 const bool is_passthru
= init_ret
1431 == CODER_INIT_PASSTHRU
;
1432 const uint64_t in_size
1433 = pair
->src_st
.st_size
<= 0
1434 ? 0 : (uint64_t)(pair
->src_st
.st_size
);
1435 message_progress_start(&strm
,
1436 is_passthru
, in_size
);
1438 // Do the actual coding or passthru.
1440 success
= coder_passthru(pair
);
1442 success
= coder_normal(pair
);
1444 message_progress_end(success
);
1449 // Close the file pair. It needs to know if coding was successful to
1450 // know if the source or target file should be unlinked.
1451 io_close(pair
, success
);
1461 // Free starting from the second filter chain since the default
1462 // filter chain may have its options set from a static variable
1463 // in coder_set_compression_settings(). Since this is only run in
1464 // debug mode and will be freed when the process ends anyway, we
1465 // don't worry about freeing it.
1466 for (uint32_t i
= 1; i
< ARRAY_SIZE(chains
); i
++) {
1467 if (chains_used_mask
& (1U << i
))
1468 lzma_filters_free(chains
[i
], NULL
);