1 ///////////////////////////////////////////////////////////////////////////////
4 /// \brief Compresses or uncompresses a file
6 // Authors: Lasse Collin
9 // This file has been put into the public domain.
10 // You can do whatever you want with this file.
12 ///////////////////////////////////////////////////////////////////////////////
17 /// Return value type for coder_init().
25 enum operation_mode opt_mode
= MODE_COMPRESS
;
26 enum format_type opt_format
= FORMAT_AUTO
;
27 bool opt_auto_adjust
= true;
28 bool opt_single_stream
= false;
29 uint64_t opt_block_size
= 0;
30 block_list_entry
*opt_block_list
= NULL
;
32 /// Stream used to communicate with liblzma
33 static lzma_stream strm
= LZMA_STREAM_INIT
;
35 /// Maximum number of filter chains. The first filter chain is the default,
36 /// and 9 other filter chains can be specified with --filtersX.
37 #define NUM_FILTER_CHAIN_MAX 10
39 /// The default filter chain is in filters[0]. It is used for encoding
40 /// in all supported formats and also for decdoing raw streams. The other
41 /// filter chains are set by --filtersX to support changing filters with
42 /// the --block-list option.
43 static lzma_filter filters
[NUM_FILTER_CHAIN_MAX
][LZMA_FILTERS_MAX
+ 1];
45 /// Bit mask representing the filters that are actually used when encoding
46 /// in the xz format. This is needed since a filter chain could be
47 /// specified in --filtersX (or the default filter chain), but never used
48 /// in --block-list. The default filter chain is always assumed to be used,
49 /// unless --block-list is specified and does not have a block using the
50 /// default filter chain.
51 static uint32_t filters_used_mask
= 1;
54 /// Track the memory usage for all filter chains (default or --filtersX).
55 /// The memory usage may need to be scaled down depending on the memory limit.
56 static uint64_t filter_memusages
[ARRAY_SIZE(filters
)];
59 /// Input and output buffers
61 static io_buf out_buf
;
63 /// Number of filters in the default filter chain. Zero indicates that
64 /// we are using a preset.
65 static uint32_t filters_count
= 0;
67 /// Number of the preset (0-9)
68 static uint32_t preset_number
= LZMA_PRESET_DEFAULT
;
70 /// True if the current default filter chain was set using the --filters
71 /// option. The filter chain is reset if a preset option (like -9) or an
72 /// old-style filter option (like --lzma2) is used after a --filters option.
73 static bool string_to_filter_used
= false;
75 /// Integrity check type
76 static lzma_check check
;
78 /// This becomes false if the --check=CHECK option is used.
79 static bool check_default
= true;
81 /// Indicates if unconsumed input is allowed to remain after
82 /// decoding has successfully finished. This is set for each file
84 static bool allow_trailing_input
;
86 #ifdef MYTHREAD_ENABLED
87 static lzma_mt mt_options
= {
95 coder_set_check(lzma_check new_check
)
98 check_default
= false;
104 forget_filter_chain(void)
106 // Setting a preset or using --filters makes us forget
107 // the earlier custom filter chain (if any).
108 if (filters_count
> 0) {
109 lzma_filters_free(filters
[0], NULL
);
113 string_to_filter_used
= false;
119 coder_set_preset(uint32_t new_preset
)
121 preset_number
&= ~LZMA_PRESET_LEVEL_MASK
;
122 preset_number
|= new_preset
;
123 forget_filter_chain();
129 coder_set_extreme(void)
131 preset_number
|= LZMA_PRESET_EXTREME
;
132 forget_filter_chain();
138 coder_add_filter(lzma_vli id
, void *options
)
140 if (filters_count
== LZMA_FILTERS_MAX
)
141 message_fatal(_("Maximum number of filters is four"));
143 if (string_to_filter_used
)
144 forget_filter_chain();
146 filters
[0][filters_count
].id
= id
;
147 filters
[0][filters_count
].options
= options
;
148 // Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify
149 // implementation of forget_filter_chain().
150 filters
[0][++filters_count
].id
= LZMA_VLI_UNKNOWN
;
152 // Setting a custom filter chain makes us forget the preset options.
153 // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
154 // where the custom filter chain resets the preset level back to
155 // the default 6, making the example equivalent to "xz -6e".
156 preset_number
= LZMA_PRESET_DEFAULT
;
163 str_to_filters(const char *str
, uint32_t index
, uint32_t flags
)
166 const char *err
= lzma_str_to_filters(str
, &error_pos
,
167 filters
[index
], flags
, NULL
);
170 char filter_num
[2] = "";
172 filter_num
[0] = '0' + index
;
174 // FIXME? The message in err isn't translated.
175 // Including the translations in the xz translations is
176 // slightly ugly but possible. Creating a new domain for
177 // liblzma might not be worth it especially since on some
178 // OSes it adds extra dependencies to translation libraries.
179 message(V_ERROR
, _("Error in --filters%s=FILTERS option:"),
181 message(V_ERROR
, "%s", str
);
182 message(V_ERROR
, "%*s^", error_pos
, "");
183 message_fatal("%s", err
);
189 coder_add_filters_from_str(const char *filter_str
)
191 // Forget presets and previously defined filter chain. See
192 // coder_add_filter() above for why preset_number must be reset too.
193 forget_filter_chain();
194 preset_number
= LZMA_PRESET_DEFAULT
;
196 string_to_filter_used
= true;
198 // Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw.
199 str_to_filters(filter_str
, 0, LZMA_STR_ALL_FILTERS
);
201 // Set the filters_count to be the number of filters converted from
203 for (filters_count
= 0; filters
[0][filters_count
].id
207 assert(filters_count
> 0);
213 coder_add_block_filters(const char *str
, size_t slot
)
215 // Free old filters first, if they were previously allocated.
216 if (filters_used_mask
& (1U << slot
))
217 lzma_filters_free(filters
[slot
], NULL
);
219 str_to_filters(str
, slot
, 0);
221 filters_used_mask
|= 1U << slot
;
225 static void lzma_attribute((__noreturn__
))
226 memlimit_too_small(uint64_t memory_usage
)
228 message(V_ERROR
, _("Memory usage limit is too low for the given "
230 message_mem_needed(V_ERROR
, memory_usage
);
231 tuklib_exit(E_ERROR
, E_ERROR
, false);
236 // For a given opt_block_list index, validate that the filter has been
237 // set. If it has not been set, we must exit with error to avoid using
238 // an uninitialized filter chain.
240 validate_block_list_filter(const uint32_t filter_num
)
242 if (!(filters_used_mask
& (1U << filter_num
)))
243 message_fatal(_("filter chain %u used by --block-list but "
244 "not specified with --filters%u="),
245 (unsigned)filter_num
, (unsigned)filter_num
);
249 // Sets the memory usage for each filter chain. It will return the maximum
250 // memory usage of all of the filter chains.
252 filters_memusage_max(const lzma_mt
*mt
, bool encode
)
254 uint64_t max_memusage
= 0;
256 #ifdef MYTHREAD_ENABLED
257 // Copy multithreaded options to a temporary struct since the
258 // filters member needs to be changed
266 for (uint32_t i
= 0; i
< ARRAY_SIZE(filters
); i
++) {
267 if (!(filters_used_mask
& (1U << i
)))
270 uint64_t memusage
= UINT64_MAX
;
271 #ifdef MYTHREAD_ENABLED
273 mt_local
.filters
= filters
[i
];
274 memusage
= lzma_stream_encoder_mt_memusage(&mt_local
);
275 filter_memusages
[i
] = memusage
;
281 memusage
= lzma_raw_encoder_memusage(filters
[i
]);
282 filter_memusages
[i
] = memusage
;
287 memusage
= lzma_raw_decoder_memusage(filters
[i
]);
291 if (memusage
> max_memusage
)
292 max_memusage
= memusage
;
301 coder_set_compression_settings(void)
303 #ifdef HAVE_LZIP_DECODER
304 // .lz compression isn't supported.
305 assert(opt_format
!= FORMAT_LZIP
);
309 # ifdef MYTHREAD_ENABLED
310 // Represents the largest Block size specified with --block-list. This
311 // is needed to help reduce the Block size in the multithreaded encoder
312 // so memory is not wasted.
313 uint64_t max_block_list_size
= 0;
316 if (opt_block_list
!= NULL
) {
317 // This mask tracks the filters actually referenced in
318 // --block-list. It is used to help remove bits from
319 // filters_used_mask when a filter chain was specified
320 // but never actually used.
321 uint32_t filters_ref_mask
= 0;
323 for (uint32_t i
= 0; opt_block_list
[i
].size
!= 0; i
++) {
324 validate_block_list_filter(
325 opt_block_list
[i
].filters_index
);
327 // Mark the current filter as referenced.
328 filters_ref_mask
|= 1U <<
329 opt_block_list
[i
].filters_index
;
331 # ifdef MYTHREAD_ENABLED
332 if (opt_block_list
[i
].size
> max_block_list_size
)
333 max_block_list_size
= opt_block_list
[i
].size
;
337 assert(filters_ref_mask
!= 0);
338 // Note: The filters that were initialized but not used do
339 // not free their options and do not have the filter
340 // IDs set to LZMA_VLI_UNKNOWN. Filter chains are not
341 // freed outside of debug mode and the default filter
342 // chain is never freed.
343 filters_used_mask
= filters_ref_mask
;
345 // Reset filters used mask in case --block-list is not
346 // used, but --filtersX is used.
347 filters_used_mask
= 1;
350 // The default check type is CRC64, but fallback to CRC32
351 // if CRC64 isn't supported by the copy of liblzma we are
352 // using. CRC32 is always supported.
354 check
= LZMA_CHECK_CRC64
;
355 if (!lzma_check_is_supported(check
))
356 check
= LZMA_CHECK_CRC32
;
359 // Options for LZMA1 or LZMA2 in case we are using a preset.
360 static lzma_options_lzma opt_lzma
;
362 // The first filter in the filters[] array is for the default
364 lzma_filter
*default_filters
= filters
[0];
366 if (filters_count
== 0 && filters_used_mask
& 1) {
367 // We are using a preset. This is not a good idea in raw mode
368 // except when playing around with things. Different versions
369 // of this software may use different options in presets, and
370 // thus make uncompressing the raw data difficult.
371 if (opt_format
== FORMAT_RAW
) {
372 // The message is shown only if warnings are allowed
373 // but the exit status isn't changed.
374 message(V_WARNING
, _("Using a preset in raw mode "
376 message(V_WARNING
, _("The exact options of the "
377 "presets may vary between software "
381 // Get the preset for LZMA1 or LZMA2.
382 if (lzma_lzma_preset(&opt_lzma
, preset_number
))
385 // Use LZMA2 except with --format=lzma we use LZMA1.
386 default_filters
[0].id
= opt_format
== FORMAT_LZMA
387 ? LZMA_FILTER_LZMA1
: LZMA_FILTER_LZMA2
;
388 default_filters
[0].options
= &opt_lzma
;
392 // Terminate the filter options array.
393 default_filters
[1].id
= LZMA_VLI_UNKNOWN
;
396 // If we are using the .lzma format, allow exactly one filter
397 // which has to be LZMA1. There is no need to check if the default
398 // filter chain is being used since it can only be disabled if
399 // --block-list is used, which is incompatible with FORMAT_LZMA.
400 if (opt_format
== FORMAT_LZMA
&& (filters_count
!= 1
401 || default_filters
[0].id
!= LZMA_FILTER_LZMA1
))
402 message_fatal(_("The .lzma format supports only "
403 "the LZMA1 filter"));
405 // If we are using the .xz format, make sure that there is no LZMA1
406 // filter to prevent LZMA_PROG_ERROR.
407 if (opt_format
== FORMAT_XZ
&& filters_used_mask
& 1)
408 for (size_t i
= 0; i
< filters_count
; ++i
)
409 if (default_filters
[i
].id
== LZMA_FILTER_LZMA1
)
410 message_fatal(_("LZMA1 cannot be used "
411 "with the .xz format"));
413 if (filters_used_mask
& 1) {
414 // Print the selected default filter chain.
415 message_filters_show(V_DEBUG
, default_filters
);
418 // The --flush-timeout option requires LZMA_SYNC_FLUSH support
419 // from the filter chain. Currently the threaded encoder doesn't
420 // support LZMA_SYNC_FLUSH so single-threaded mode must be used.
421 if (opt_mode
== MODE_COMPRESS
&& opt_flush_timeout
!= 0) {
422 for (uint32_t i
= 0; i
< ARRAY_SIZE(filters
); ++i
) {
423 if (!(filters_used_mask
& (1U << i
)))
426 const lzma_filter
*fc
= filters
[i
];
427 for (size_t j
= 0; fc
[j
].id
!= LZMA_VLI_UNKNOWN
; j
++) {
429 case LZMA_FILTER_LZMA2
:
430 case LZMA_FILTER_DELTA
:
434 message_fatal(_("Filter chain %u is "
442 if (hardware_threads_is_mt()) {
443 message(V_WARNING
, _("Switching to single-threaded "
444 "mode due to --flush-timeout"));
445 hardware_threads_set(1);
449 // Get the memory usage and memory limit. The memory usage is the
450 // maximum of the default filters[] and any filters specified by
452 // Note that if --format=raw was used, we can be decompressing and
453 // do not need to account for any filter chains created
456 // If multithreaded .xz compression is done, the memory limit
458 uint64_t memory_limit
= hardware_memlimit_get(opt_mode
);
459 uint64_t memory_usage
= UINT64_MAX
;
460 if (opt_mode
== MODE_COMPRESS
) {
462 # ifdef MYTHREAD_ENABLED
463 if (opt_format
== FORMAT_XZ
&& hardware_threads_is_mt()) {
464 memory_limit
= hardware_memlimit_mtenc_get();
465 mt_options
.threads
= hardware_threads_get();
467 uint64_t block_size
= opt_block_size
;
468 // If opt_block_size is not set, find the maximum
469 // recommended Block size based on the filter chains
470 if (block_size
== 0) {
471 for (uint32_t i
= 0; i
< ARRAY_SIZE(filters
);
473 if (!(filters_used_mask
& (1U << i
)))
476 uint64_t size
= lzma_mt_block_size(
479 // If this returns an error, then one
480 // of the filter chains in use is
481 // invalid, so there is no point in
482 // progressing further.
483 if (size
== UINT64_MAX
)
484 message_fatal(_("Unsupported "
489 if (size
> block_size
)
493 // If the largest block size specified
494 // with --block-list is less than the
495 // recommended Block size, then it is a waste
496 // of RAM to use a larger Block size. It may
497 // even allow more threads to be used in some
498 // situations. If the special 0 Block size is
499 // used (encode all remaining data in 1 Block)
500 // then max_block_list_size will be set to
501 // UINT64_MAX, so the recommended Block size
502 // will always be used in this case.
503 if (max_block_list_size
> 0
504 && max_block_list_size
506 block_size
= max_block_list_size
;
509 mt_options
.block_size
= block_size
;
510 mt_options
.check
= check
;
512 memory_usage
= filters_memusage_max(
514 if (memory_usage
!= UINT64_MAX
)
515 message(V_DEBUG
, _("Using up to %" PRIu32
521 memory_usage
= filters_memusage_max(NULL
, true);
526 memory_usage
= lzma_raw_decoder_memusage(default_filters
);
530 if (memory_usage
== UINT64_MAX
)
531 message_fatal(_("Unsupported filter chain or filter options"));
533 // Print memory usage info before possible dictionary
534 // size auto-adjusting.
536 // NOTE: If only encoder support was built, we cannot show the
537 // what the decoder memory usage will be.
538 message_mem_needed(V_DEBUG
, memory_usage
);
540 if (opt_mode
== MODE_COMPRESS
) {
542 const uint64_t decmem
=
543 filters_memusage_max(NULL
, false);
545 // If encoders are not enabled, then --block-list is never
546 // usable, so the other filter chains 1-9 can never be used.
547 // So there is no need to find the maximum decoder memory
548 // required in this case.
549 const uint64_t decmem
= lzma_raw_decoder_memusage(filters
[0]);
551 if (decmem
!= UINT64_MAX
)
552 message(V_DEBUG
, _("Decompression will need "
553 "%s MiB of memory."), uint64_to_str(
554 round_up_to_mib(decmem
), 0));
558 if (memory_usage
<= memory_limit
)
561 // With --format=raw settings are never adjusted to meet
562 // the memory usage limit.
563 if (opt_format
== FORMAT_RAW
)
564 memlimit_too_small(memory_usage
);
566 assert(opt_mode
== MODE_COMPRESS
);
569 # ifdef MYTHREAD_ENABLED
570 if (opt_format
== FORMAT_XZ
&& hardware_threads_is_mt()) {
571 // Try to reduce the number of threads before
572 // adjusting the compression settings down.
573 while (mt_options
.threads
> 1) {
574 // Reduce the number of threads by one and check
576 --mt_options
.threads
;
577 memory_usage
= filters_memusage_max(
579 if (memory_usage
== UINT64_MAX
)
582 if (memory_usage
<= memory_limit
) {
583 // The memory usage is now low enough.
584 message(V_WARNING
, _("Reduced the number of "
585 "threads from %s to %s to not exceed "
586 "the memory usage limit of %s MiB"),
588 hardware_threads_get(), 0),
589 uint64_to_str(mt_options
.threads
, 1),
590 uint64_to_str(round_up_to_mib(
596 // If the memory usage limit is only a soft limit (automatic
597 // number of threads and no --memlimit-compress), the limit
598 // is only used to reduce the number of threads and once at
599 // just one thread, the limit is completely ignored. This
600 // way -T0 won't use insane amount of memory but at the same
601 // time the soft limit will never make xz fail and never make
602 // xz change settings that would affect the compressed output.
603 if (hardware_memlimit_mtenc_is_default()) {
604 message(V_WARNING
, _("Reduced the number of threads "
605 "from %s to one. The automatic memory usage "
606 "limit of %s MiB is still being exceeded. "
607 "%s MiB of memory is required. "
608 "Continuing anyway."),
609 uint64_to_str(hardware_threads_get(), 0),
611 round_up_to_mib(memory_limit
), 1),
613 round_up_to_mib(memory_usage
), 2));
617 // If --no-adjust was used, we cannot drop to single-threaded
618 // mode since it produces different compressed output.
620 // NOTE: In xz 5.2.x, --no-adjust also prevented reducing
621 // the number of threads. This changed in 5.3.3alpha.
622 if (!opt_auto_adjust
)
623 memlimit_too_small(memory_usage
);
625 // Switch to single-threaded mode. It uses
626 // less memory than using one thread in
627 // the multithreaded mode but the output
628 // is also different.
629 hardware_threads_set(1);
630 memory_usage
= filters_memusage_max(NULL
, true);
631 message(V_WARNING
, _("Switching to single-threaded mode "
632 "to not exceed the memory usage limit of %s MiB"),
633 uint64_to_str(round_up_to_mib(memory_limit
), 0));
637 if (memory_usage
<= memory_limit
)
640 // Don't adjust LZMA2 or LZMA1 dictionary size if --no-adjust
641 // was specified as that would change the compressed output.
642 if (!opt_auto_adjust
)
643 memlimit_too_small(memory_usage
);
645 // Decrease the dictionary size until we meet the memory usage limit.
646 // The struct is used to track data needed to correctly reduce the
647 // memory usage and report which filters were adjusted.
649 // Pointer to the filter chain that needs to be reduced.
650 // NULL indicates that this filter chain was either never
651 // set or was never above the memory limit.
652 lzma_filter
*filters
;
654 // Original dictionary sizes are used to show how each
655 // filter's dictionary was reduced.
656 uint64_t orig_dict_size
;
658 // Index of the LZMA filter in the filters member. We only
659 // adjust this filter's memusage because we don't know how
660 // to reduce the memory usage of the other filters.
663 // Indicates if the filter's dictionary size needs to be
664 // reduced to fit under the memory limit (true) or if the
665 // filter chain is unused or is already under the memory
667 bool reduce_dict_size
;
668 } memusage_reduction_data
;
670 memusage_reduction_data memusage_reduction
[ARRAY_SIZE(filters
)];
672 // Counter represents how many filter chains are above the memory
676 for (uint32_t i
= 0; i
< ARRAY_SIZE(filters
); i
++) {
677 // The short var name "r" will reduce the number of lines
678 // of code needed since less lines will stretch past 80
680 memusage_reduction_data
*r
= &memusage_reduction
[i
];
682 r
->reduce_dict_size
= false;
684 if (!(filters_used_mask
& (1U << i
)))
687 for (uint32_t j
= 0; filters
[i
][j
].id
!= LZMA_VLI_UNKNOWN
;
689 if ((filters
[i
][j
].id
== LZMA_FILTER_LZMA2
691 == LZMA_FILTER_LZMA1
)
692 && filter_memusages
[i
]
695 r
->filters
= filters
[i
];
697 r
->reduce_dict_size
= true;
699 lzma_options_lzma
*opt
= r
->filters
700 [r
->lzma_idx
].options
;
701 r
->orig_dict_size
= opt
->dict_size
;
702 opt
->dict_size
&= ~((UINT32_C(1) << 20) - 1);
706 // Loop until all filters use <= memory_limit, or exit.
708 for (uint32_t i
= 0; i
< ARRAY_SIZE(memusage_reduction
); i
++) {
709 memusage_reduction_data
*r
= &memusage_reduction
[i
];
711 if (!r
->reduce_dict_size
)
714 lzma_options_lzma
*opt
=
715 r
->filters
[r
->lzma_idx
].options
;
717 // If it is below 1 MiB, auto-adjusting failed.
718 // We could be more sophisticated and scale it
719 // down even more, but nobody has complained so far.
720 if (opt
->dict_size
< (UINT32_C(1) << 20))
721 memlimit_too_small(memory_usage
);
723 uint64_t filt_mem_usage
=
724 lzma_raw_encoder_memusage(r
->filters
);
726 if (filt_mem_usage
== UINT64_MAX
)
729 if (filt_mem_usage
< memory_limit
) {
730 r
->reduce_dict_size
= false;
734 opt
->dict_size
-= UINT32_C(1) << 20;
739 // Tell the user that we decreased the dictionary size for
740 // each filter that was adjusted.
741 for (uint32_t i
= 0; i
< ARRAY_SIZE(memusage_reduction
); i
++) {
742 memusage_reduction_data
*r
= &memusage_reduction
[i
];
744 // If the filters were never set, then the memory usage
745 // was never adjusted.
746 if (r
->filters
== NULL
)
749 lzma_filter
*filter_lzma
= &(r
->filters
[r
->lzma_idx
]);
750 lzma_options_lzma
*opt
= filter_lzma
->options
;
752 // The first index is the default filter chain. The message
753 // should be slightly different if the default filter chain
754 // or if --filtersX was adjusted.
756 message(V_WARNING
, _("Adjusted LZMA%c dictionary "
757 "size from %s MiB to %s MiB to not exceed the "
758 "memory usage limit of %s MiB"),
759 filter_lzma
->id
== LZMA_FILTER_LZMA2
761 uint64_to_str(r
->orig_dict_size
>> 20, 0),
762 uint64_to_str(opt
->dict_size
>> 20, 1),
763 uint64_to_str(round_up_to_mib(
766 message(V_WARNING
, _("Adjusted LZMA%c dictionary size "
767 "for --filters%u from %s MiB to %s MiB to not "
768 "exceed the memory usage limit of %s MiB"),
769 filter_lzma
->id
== LZMA_FILTER_LZMA2
772 uint64_to_str(r
->orig_dict_size
>> 20, 0),
773 uint64_to_str(opt
->dict_size
>> 20, 1),
774 uint64_to_str(round_up_to_mib(
784 /// Return true if the data in in_buf seems to be in the .xz format.
788 // Specify the magic as hex to be compatible with EBCDIC systems.
789 static const uint8_t magic
[6] = { 0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00 };
790 return strm
.avail_in
>= sizeof(magic
)
791 && memcmp(in_buf
.u8
, magic
, sizeof(magic
)) == 0;
795 /// Return true if the data in in_buf seems to be in the .lzma format.
799 // The .lzma header is 13 bytes.
800 if (strm
.avail_in
< 13)
803 // Decode the LZMA1 properties.
804 lzma_filter filter
= { .id
= LZMA_FILTER_LZMA1
};
805 if (lzma_properties_decode(&filter
, NULL
, in_buf
.u8
, 5) != LZMA_OK
)
808 // A hack to ditch tons of false positives: We allow only dictionary
809 // sizes that are 2^n or 2^n + 2^(n-1) or UINT32_MAX. LZMA_Alone
810 // created only files with 2^n, but accepts any dictionary size.
811 // If someone complains, this will be reconsidered.
812 lzma_options_lzma
*opt
= filter
.options
;
813 const uint32_t dict_size
= opt
->dict_size
;
816 if (dict_size
!= UINT32_MAX
) {
817 uint32_t d
= dict_size
- 1;
824 if (d
!= dict_size
|| dict_size
== 0)
828 // Another hack to ditch false positives: Assume that if the
829 // uncompressed size is known, it must be less than 256 GiB.
830 // Again, if someone complains, this will be reconsidered.
831 uint64_t uncompressed_size
= 0;
832 for (size_t i
= 0; i
< 8; ++i
)
833 uncompressed_size
|= (uint64_t)(in_buf
.u8
[5 + i
]) << (i
* 8);
835 if (uncompressed_size
!= UINT64_MAX
836 && uncompressed_size
> (UINT64_C(1) << 38))
843 #ifdef HAVE_LZIP_DECODER
844 /// Return true if the data in in_buf seems to be in the .lz format.
848 static const uint8_t magic
[4] = { 0x4C, 0x5A, 0x49, 0x50 };
849 return strm
.avail_in
>= sizeof(magic
)
850 && memcmp(in_buf
.u8
, magic
, sizeof(magic
)) == 0;
856 /// Detect the input file type (for now, this done only when decompressing),
857 /// and initialize an appropriate coder. Return value indicates if a normal
858 /// liblzma-based coder was initialized (CODER_INIT_NORMAL), if passthru
859 /// mode should be used (CODER_INIT_PASSTHRU), or if an error occurred
860 /// (CODER_INIT_ERROR).
861 static enum coder_init_ret
862 coder_init(file_pair
*pair
)
864 lzma_ret ret
= LZMA_PROG_ERROR
;
866 // In most cases if there is input left when coding finishes,
867 // something has gone wrong. Exceptions are --single-stream
868 // and decoding .lz files which can contain trailing non-.lz data.
869 // These will be handled later in this function.
870 allow_trailing_input
= false;
872 // Set the first filter chain. If the --block-list option is not
873 // used then use the default filter chain (filters[0]).
874 // Otherwise, use first filter chain from the block list.
875 lzma_filter
*active_filters
= opt_block_list
== NULL
877 : filters
[opt_block_list
[0].filters_index
];
879 if (opt_mode
== MODE_COMPRESS
) {
881 switch (opt_format
) {
883 // args.c ensures this.
888 # ifdef MYTHREAD_ENABLED
889 mt_options
.filters
= active_filters
;
890 if (hardware_threads_is_mt())
891 ret
= lzma_stream_encoder_mt(
895 ret
= lzma_stream_encoder(
896 &strm
, active_filters
, check
);
900 ret
= lzma_alone_encoder(&strm
,
901 active_filters
[0].options
);
904 # ifdef HAVE_LZIP_DECODER
906 // args.c should disallow this.
908 ret
= LZMA_PROG_ERROR
;
913 ret
= lzma_raw_encoder(&strm
, active_filters
);
921 // It seems silly to warn about unsupported check if the
922 // check won't be verified anyway due to --ignore-check.
923 if (opt_ignore_check
)
924 flags
|= LZMA_IGNORE_CHECK
;
926 flags
|= LZMA_TELL_UNSUPPORTED_CHECK
;
928 if (opt_single_stream
)
929 allow_trailing_input
= true;
931 flags
|= LZMA_CONCATENATED
;
933 // We abuse FORMAT_AUTO to indicate unknown file format,
934 // for which we may consider passthru mode.
935 enum format_type init_format
= FORMAT_AUTO
;
937 switch (opt_format
) {
939 // .lz is checked before .lzma since .lzma detection
940 // is more complicated (no magic bytes).
942 init_format
= FORMAT_XZ
;
943 # ifdef HAVE_LZIP_DECODER
944 else if (is_format_lzip())
945 init_format
= FORMAT_LZIP
;
947 else if (is_format_lzma())
948 init_format
= FORMAT_LZMA
;
953 init_format
= FORMAT_XZ
;
957 if (is_format_lzma())
958 init_format
= FORMAT_LZMA
;
961 # ifdef HAVE_LZIP_DECODER
963 if (is_format_lzip())
964 init_format
= FORMAT_LZIP
;
969 init_format
= FORMAT_RAW
;
973 switch (init_format
) {
975 // Unknown file format. If --decompress --stdout
976 // --force have been given, then we copy the input
977 // as is to stdout. Checking for MODE_DECOMPRESS
978 // is needed, because we don't want to do use
979 // passthru mode with --test.
980 if (opt_mode
== MODE_DECOMPRESS
981 && opt_stdout
&& opt_force
) {
982 // These are needed for progress info.
985 return CODER_INIT_PASSTHRU
;
988 ret
= LZMA_FORMAT_ERROR
;
992 # ifdef MYTHREAD_ENABLED
993 mt_options
.flags
= flags
;
995 mt_options
.threads
= hardware_threads_get();
996 mt_options
.memlimit_stop
997 = hardware_memlimit_get(MODE_DECOMPRESS
);
999 // If single-threaded mode was requested, set the
1000 // memlimit for threading to zero. This forces the
1001 // decoder to use single-threaded mode which matches
1002 // the behavior of lzma_stream_decoder().
1004 // Otherwise use the limit for threaded decompression
1005 // which has a sane default (users are still free to
1006 // make it insanely high though).
1007 mt_options
.memlimit_threading
1008 = mt_options
.threads
== 1
1009 ? 0 : hardware_memlimit_mtdec_get();
1011 ret
= lzma_stream_decoder_mt(&strm
, &mt_options
);
1013 ret
= lzma_stream_decoder(&strm
,
1014 hardware_memlimit_get(
1015 MODE_DECOMPRESS
), flags
);
1020 ret
= lzma_alone_decoder(&strm
,
1021 hardware_memlimit_get(
1025 # ifdef HAVE_LZIP_DECODER
1027 allow_trailing_input
= true;
1028 ret
= lzma_lzip_decoder(&strm
,
1029 hardware_memlimit_get(
1030 MODE_DECOMPRESS
), flags
);
1035 // Memory usage has already been checked in
1036 // coder_set_compression_settings().
1037 ret
= lzma_raw_decoder(&strm
, active_filters
);
1041 // Try to decode the headers. This will catch too low
1042 // memory usage limit in case it happens in the first
1043 // Block of the first Stream, which is where it very
1044 // probably will happen if it is going to happen.
1046 // This will also catch unsupported check type which
1047 // we treat as a warning only. If there are empty
1048 // concatenated Streams with unsupported check type then
1049 // the message can be shown more than once here. The loop
1050 // is used in case there is first a warning about
1051 // unsupported check type and then the first Block
1052 // would exceed the memlimit.
1053 if (ret
== LZMA_OK
&& init_format
!= FORMAT_RAW
) {
1054 strm
.next_out
= NULL
;
1056 while ((ret
= lzma_code(&strm
, LZMA_RUN
))
1057 == LZMA_UNSUPPORTED_CHECK
)
1058 message_warning(_("%s: %s"), pair
->src_name
,
1061 // With --single-stream lzma_code won't wait for
1062 // LZMA_FINISH and thus it can return LZMA_STREAM_END
1063 // if the file has no uncompressed data inside.
1064 // So treat LZMA_STREAM_END as LZMA_OK here.
1065 // When lzma_code() is called again in coder_normal()
1066 // it will return LZMA_STREAM_END again.
1067 if (ret
== LZMA_STREAM_END
)
1073 if (ret
!= LZMA_OK
) {
1074 message_error(_("%s: %s"), pair
->src_name
, message_strm(ret
));
1075 if (ret
== LZMA_MEMLIMIT_ERROR
)
1076 message_mem_needed(V_ERROR
, lzma_memusage(&strm
));
1078 return CODER_INIT_ERROR
;
1081 return CODER_INIT_NORMAL
;
1085 #ifdef HAVE_ENCODERS
1086 /// Resolve conflicts between opt_block_size and opt_block_list in single
1087 /// threaded mode. We want to default to opt_block_list, except when it is
1088 /// larger than opt_block_size. If this is the case for the current Block
1089 /// at *list_pos, then we break into smaller Blocks. Otherwise advance
1090 /// to the next Block in opt_block_list, and break apart if needed.
1092 split_block(uint64_t *block_remaining
,
1093 uint64_t *next_block_remaining
,
1096 if (*next_block_remaining
> 0) {
1097 // The Block at *list_pos has previously been split up.
1098 assert(!hardware_threads_is_mt());
1099 assert(opt_block_size
> 0);
1100 assert(opt_block_list
!= NULL
);
1102 if (*next_block_remaining
> opt_block_size
) {
1103 // We have to split the current Block at *list_pos
1104 // into another opt_block_size length Block.
1105 *block_remaining
= opt_block_size
;
1107 // This is the last remaining split Block for the
1108 // Block at *list_pos.
1109 *block_remaining
= *next_block_remaining
;
1112 *next_block_remaining
-= *block_remaining
;
1115 // The Block at *list_pos has been finished. Go to the next
1116 // entry in the list. If the end of the list has been
1117 // reached, reuse the size and filters of the last Block.
1118 if (opt_block_list
[*list_pos
+ 1].size
!= 0) {
1121 // Update the filters if needed.
1122 if (opt_block_list
[*list_pos
- 1].filters_index
1123 != opt_block_list
[*list_pos
].filters_index
) {
1124 const uint32_t filter_idx
= opt_block_list
1125 [*list_pos
].filters_index
;
1126 const lzma_filter
*next
= filters
[filter_idx
];
1127 const lzma_ret ret
= lzma_filters_update(
1130 if (ret
!= LZMA_OK
) {
1131 // This message is only possible if
1132 // the filter chain has unsupported
1133 // options since the filter chain is
1135 // lzma_raw_encoder_memusage() or
1136 // lzma_stream_encoder_mt_memusage().
1137 // Some options are not validated until
1138 // the encoders are initialized.
1140 _("Error changing to "
1141 "filter chain %u: %s"),
1142 (unsigned)filter_idx
,
1148 *block_remaining
= opt_block_list
[*list_pos
].size
;
1150 // If in single-threaded mode, split up the Block if needed.
1151 // This is not needed in multi-threaded mode because liblzma
1152 // will do this due to how threaded encoding works.
1153 if (!hardware_threads_is_mt() && opt_block_size
> 0
1154 && *block_remaining
> opt_block_size
) {
1155 *next_block_remaining
1156 = *block_remaining
- opt_block_size
;
1157 *block_remaining
= opt_block_size
;
1165 coder_write_output(file_pair
*pair
)
1167 if (opt_mode
!= MODE_TEST
) {
1168 if (io_write(pair
, &out_buf
, IO_BUFFER_SIZE
- strm
.avail_out
))
1172 strm
.next_out
= out_buf
.u8
;
1173 strm
.avail_out
= IO_BUFFER_SIZE
;
1178 /// Compress or decompress using liblzma.
1180 coder_normal(file_pair
*pair
)
1182 // Encoder needs to know when we have given all the input to it.
1183 // The decoders need to know it too when we are using
1184 // LZMA_CONCATENATED. We need to check for src_eof here, because
1185 // the first input chunk has been already read if decompressing,
1186 // and that may have been the only chunk we will read.
1187 lzma_action action
= pair
->src_eof
? LZMA_FINISH
: LZMA_RUN
;
1191 // Assume that something goes wrong.
1192 bool success
= false;
1194 #ifdef HAVE_ENCODERS
1195 // block_remaining indicates how many input bytes to encode before
1196 // finishing the current .xz Block. The Block size is set with
1197 // --block-size=SIZE and --block-list. They have an effect only when
1198 // compressing to the .xz format. If block_remaining == UINT64_MAX,
1199 // only a single block is created.
1200 uint64_t block_remaining
= UINT64_MAX
;
1202 // next_block_remaining for when we are in single-threaded mode and
1203 // the Block in --block-list is larger than the --block-size=SIZE.
1204 uint64_t next_block_remaining
= 0;
1206 // Position in opt_block_list. Unused if --block-list wasn't used.
1207 size_t list_pos
= 0;
1209 // Handle --block-size for single-threaded mode and the first step
1211 if (opt_mode
== MODE_COMPRESS
&& opt_format
== FORMAT_XZ
) {
1212 // --block-size doesn't do anything here in threaded mode,
1213 // because the threaded encoder will take care of splitting
1214 // to fixed-sized Blocks.
1215 if (!hardware_threads_is_mt() && opt_block_size
> 0)
1216 block_remaining
= opt_block_size
;
1218 // If --block-list was used, start with the first size.
1220 // For threaded case, --block-size specifies how big Blocks
1221 // the encoder needs to be prepared to create at maximum
1222 // and --block-list will simultaneously cause new Blocks
1223 // to be started at specified intervals. To keep things
1224 // logical, the same is done in single-threaded mode. The
1225 // output is still not identical because in single-threaded
1226 // mode the size info isn't written into Block Headers.
1227 if (opt_block_list
!= NULL
) {
1228 if (block_remaining
< opt_block_list
[list_pos
].size
) {
1229 assert(!hardware_threads_is_mt());
1230 next_block_remaining
=
1231 opt_block_list
[list_pos
].size
1235 opt_block_list
[list_pos
].size
;
1241 strm
.next_out
= out_buf
.u8
;
1242 strm
.avail_out
= IO_BUFFER_SIZE
;
1244 while (!user_abort
) {
1245 // Fill the input buffer if it is empty and we aren't
1246 // flushing or finishing.
1247 if (strm
.avail_in
== 0 && action
== LZMA_RUN
) {
1248 strm
.next_in
= in_buf
.u8
;
1249 #ifdef HAVE_ENCODERS
1250 const size_t read_size
= my_min(block_remaining
,
1253 const size_t read_size
= IO_BUFFER_SIZE
;
1255 strm
.avail_in
= io_read(pair
, &in_buf
, read_size
);
1257 if (strm
.avail_in
== SIZE_MAX
)
1260 if (pair
->src_eof
) {
1261 action
= LZMA_FINISH
;
1263 #ifdef HAVE_ENCODERS
1264 else if (block_remaining
!= UINT64_MAX
) {
1265 // Start a new Block after every
1266 // opt_block_size bytes of input.
1267 block_remaining
-= strm
.avail_in
;
1268 if (block_remaining
== 0)
1269 action
= LZMA_FULL_BARRIER
;
1272 if (action
== LZMA_RUN
&& pair
->flush_needed
)
1273 action
= LZMA_SYNC_FLUSH
;
1277 // Let liblzma do the actual work.
1278 ret
= lzma_code(&strm
, action
);
1280 // Write out if the output buffer became full.
1281 if (strm
.avail_out
== 0)
1282 if (coder_write_output(pair
))
1285 #ifdef HAVE_ENCODERS
1286 if (ret
== LZMA_STREAM_END
&& (action
== LZMA_SYNC_FLUSH
1287 || action
== LZMA_FULL_BARRIER
)) {
1288 if (action
== LZMA_SYNC_FLUSH
) {
1289 // Flushing completed. Write the pending data
1290 // out immediately so that the reading side
1291 // can decompress everything compressed so far.
1292 if (coder_write_output(pair
))
1295 // Mark that we haven't seen any new input
1296 // since the previous flush.
1297 pair
->src_has_seen_input
= false;
1298 pair
->flush_needed
= false;
1300 // Start a new Block after LZMA_FULL_BARRIER.
1301 if (opt_block_list
== NULL
) {
1302 assert(!hardware_threads_is_mt());
1303 assert(opt_block_size
> 0);
1304 block_remaining
= opt_block_size
;
1306 split_block(&block_remaining
,
1307 &next_block_remaining
,
1312 // Start a new Block after LZMA_FULL_FLUSH or continue
1313 // the same block after LZMA_SYNC_FLUSH.
1317 if (ret
!= LZMA_OK
) {
1318 // Determine if the return value indicates that we
1319 // won't continue coding. LZMA_NO_CHECK would be
1320 // here too if LZMA_TELL_ANY_CHECK was used.
1321 const bool stop
= ret
!= LZMA_UNSUPPORTED_CHECK
;
1324 // Write the remaining bytes even if something
1325 // went wrong, because that way the user gets
1326 // as much data as possible, which can be good
1327 // when trying to get at least some useful
1328 // data out of damaged files.
1329 if (coder_write_output(pair
))
1333 if (ret
== LZMA_STREAM_END
) {
1334 if (allow_trailing_input
) {
1335 io_fix_src_pos(pair
, strm
.avail_in
);
1340 // Check that there is no trailing garbage.
1341 // This is needed for LZMA_Alone and raw
1342 // streams. This is *not* done with .lz files
1343 // as that format specifically requires
1344 // allowing trailing garbage.
1345 if (strm
.avail_in
== 0 && !pair
->src_eof
) {
1346 // Try reading one more byte.
1347 // Hopefully we don't get any more
1348 // input, and thus pair->src_eof
1350 strm
.avail_in
= io_read(
1352 if (strm
.avail_in
== SIZE_MAX
)
1355 assert(strm
.avail_in
== 0
1356 || strm
.avail_in
== 1);
1359 if (strm
.avail_in
== 0) {
1360 assert(pair
->src_eof
);
1365 // We hadn't reached the end of the file.
1366 ret
= LZMA_DATA_ERROR
;
1370 // If we get here and stop is true, something went
1371 // wrong and we print an error. Otherwise it's just
1372 // a warning and coding can continue.
1374 message_error(_("%s: %s"), pair
->src_name
,
1377 message_warning(_("%s: %s"), pair
->src_name
,
1380 // When compressing, all possible errors set
1382 assert(opt_mode
!= MODE_COMPRESS
);
1385 if (ret
== LZMA_MEMLIMIT_ERROR
) {
1386 // Display how much memory it would have
1388 message_mem_needed(V_ERROR
,
1389 lzma_memusage(&strm
));
1396 // Show progress information under certain conditions.
1397 message_progress_update();
1404 /// Copy from input file to output file without processing the data in any
1405 /// way. This is used only when trying to decompress unrecognized files
1406 /// with --decompress --stdout --force, so the output is always stdout.
1408 coder_passthru(file_pair
*pair
)
1410 while (strm
.avail_in
!= 0) {
1414 if (io_write(pair
, &in_buf
, strm
.avail_in
))
1417 strm
.total_in
+= strm
.avail_in
;
1418 strm
.total_out
= strm
.total_in
;
1419 message_progress_update();
1421 strm
.avail_in
= io_read(pair
, &in_buf
, IO_BUFFER_SIZE
);
1422 if (strm
.avail_in
== SIZE_MAX
)
1431 coder_run(const char *filename
)
1433 // Set and possibly print the filename for the progress message.
1434 message_filename(filename
);
1436 // Try to open the input file.
1437 file_pair
*pair
= io_open_src(filename
);
1441 // Assume that something goes wrong.
1442 bool success
= false;
1444 if (opt_mode
== MODE_COMPRESS
) {
1445 strm
.next_in
= NULL
;
1448 // Read the first chunk of input data. This is needed
1449 // to detect the input file type.
1450 strm
.next_in
= in_buf
.u8
;
1451 strm
.avail_in
= io_read(pair
, &in_buf
, IO_BUFFER_SIZE
);
1454 if (strm
.avail_in
!= SIZE_MAX
) {
1455 // Initialize the coder. This will detect the file format
1456 // and, in decompression or testing mode, check the memory
1457 // usage of the first Block too. This way we don't try to
1458 // open the destination file if we see that coding wouldn't
1459 // work at all anyway. This also avoids deleting the old
1460 // "target" file if --force was used.
1461 const enum coder_init_ret init_ret
= coder_init(pair
);
1463 if (init_ret
!= CODER_INIT_ERROR
&& !user_abort
) {
1464 // Don't open the destination file when --test
1466 if (opt_mode
== MODE_TEST
|| !io_open_dest(pair
)) {
1467 // Remember the current time. It is needed
1468 // for progress indicator.
1469 mytime_set_start_time();
1471 // Initialize the progress indicator.
1473 // NOTE: When reading from stdin, fstat()
1474 // isn't called on it and thus src_st.st_size
1475 // is zero. If stdin pointed to a regular
1476 // file, it would still be possible to know
1477 // the file size but then we would also need
1478 // to take into account the current reading
1479 // position since with stdin it isn't
1480 // necessarily at the beginning of the file.
1481 const bool is_passthru
= init_ret
1482 == CODER_INIT_PASSTHRU
;
1483 const uint64_t in_size
1484 = pair
->src_st
.st_size
<= 0
1485 ? 0 : (uint64_t)(pair
->src_st
.st_size
);
1486 message_progress_start(&strm
,
1487 is_passthru
, in_size
);
1489 // Do the actual coding or passthru.
1491 success
= coder_passthru(pair
);
1493 success
= coder_normal(pair
);
1495 message_progress_end(success
);
1500 // Close the file pair. It needs to know if coding was successful to
1501 // know if the source or target file should be unlinked.
1502 io_close(pair
, success
);
1512 // Free starting from the second filter chain since the default
1513 // filter chain may have its options set from a static variable
1514 // in coder_set_compression_settings(). Since this is only run in
1515 // debug mode and will be freed when the process ends anyway, we
1516 // don't worry about freeing it.
1517 for (uint32_t i
= 1; i
< ARRAY_SIZE(filters
); i
++) {
1518 if (filters_used_mask
& (1U << i
))
1519 lzma_filters_free(filters
[i
], NULL
);