4 * Author: Lasse Collin <lasse.collin@tukaani.org>
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
10 #include "xz_private.h"
11 #include "xz_stream.h"
14 # define IS_CRC64(check_type) ((check_type) == XZ_CHECK_CRC64)
16 # define IS_CRC64(check_type) false
19 /* Hash used to validate the Index field */
22 vli_type uncompressed
;
27 /* Position in dec_main() */
41 /* Position in variable-length integers and Check fields */
44 /* Variable-length integer decoded by dec_vli() */
47 /* Saved in_pos and out_pos */
52 /* CRC32 or CRC64 value in Block or CRC32 value in Index */
55 /* CRC32 value in Block or Index */
59 /* Type of the integrity check calculated from uncompressed data */
60 enum xz_check check_type
;
66 * True if the next call to xz_dec_run() is allowed to return
71 /* Information stored in Block Header */
74 * Value stored in the Compressed Size field, or
75 * VLI_UNKNOWN if Compressed Size is not present.
80 * Value stored in the Uncompressed Size field, or
81 * VLI_UNKNOWN if Uncompressed Size is not present.
83 vli_type uncompressed
;
85 /* Size of the Block Header field */
89 /* Information collected when decoding Blocks */
91 /* Observed compressed size of the current Block */
94 /* Observed uncompressed size of the current Block */
95 vli_type uncompressed
;
97 /* Number of Blocks decoded so far */
101 * Hash calculated from the Block sizes. This is used to
102 * validate the Index field.
104 struct xz_dec_hash hash
;
107 /* Variables needed when verifying the Index field */
109 /* Position in dec_index() */
113 SEQ_INDEX_UNCOMPRESSED
116 /* Size of the Index in bytes */
119 /* Number of Records (matches block.count in valid files) */
123 * Hash calculated from the Records (matches block.hash in
126 struct xz_dec_hash hash
;
130 * Temporary buffer needed to hold Stream Header, Block Header,
131 * and Stream Footer. The Block Header is the biggest (1 KiB)
132 * so we reserve space according to that. buf[] has to be aligned
133 * to a multiple of four bytes; the size_t variables before it
134 * should guarantee this.
142 struct xz_dec_lzma2
*lzma2
;
145 struct xz_dec_bcj
*bcj
;
150 #ifdef XZ_DEC_ANY_CHECK
151 /* Sizes of the Check field with different Check IDs */
152 static const uint8_t check_sizes
[16] = {
163 * Fill s->temp by copying data starting from b->in[b->in_pos]. Caller
164 * must have set s->temp.pos to indicate how much data we are supposed
165 * to copy into s->temp.buf. Return true once s->temp.pos has reached
168 static bool fill_temp(struct xz_dec
*s
, struct xz_buf
*b
)
170 size_t copy_size
= min_t(size_t,
171 b
->in_size
- b
->in_pos
, s
->temp
.size
- s
->temp
.pos
);
173 memcpy(s
->temp
.buf
+ s
->temp
.pos
, b
->in
+ b
->in_pos
, copy_size
);
174 b
->in_pos
+= copy_size
;
175 s
->temp
.pos
+= copy_size
;
177 if (s
->temp
.pos
== s
->temp
.size
) {
185 /* Decode a variable-length integer (little-endian base-128 encoding) */
186 static enum xz_ret
dec_vli(struct xz_dec
*s
, const uint8_t *in
,
187 size_t *in_pos
, size_t in_size
)
194 while (*in_pos
< in_size
) {
198 s
->vli
|= (vli_type
)(byte
& 0x7F) << s
->pos
;
200 if ((byte
& 0x80) == 0) {
201 /* Don't allow non-minimal encodings. */
202 if (byte
== 0 && s
->pos
!= 0)
203 return XZ_DATA_ERROR
;
206 return XZ_STREAM_END
;
210 if (s
->pos
== 7 * VLI_BYTES_MAX
)
211 return XZ_DATA_ERROR
;
218 * Decode the Compressed Data field from a Block. Update and validate
219 * the observed compressed and uncompressed sizes of the Block so that
220 * they don't exceed the values possibly stored in the Block Header
221 * (validation assumes that no integer overflow occurs, since vli_type
222 * is normally uint64_t). Update the CRC32 or CRC64 value if presence of
223 * the CRC32 or CRC64 field was indicated in Stream Header.
225 * Once the decoding is finished, validate that the observed sizes match
226 * the sizes possibly stored in the Block Header. Update the hash and
227 * Block count, which are later used to validate the Index field.
229 static enum xz_ret
dec_block(struct xz_dec
*s
, struct xz_buf
*b
)
233 s
->in_start
= b
->in_pos
;
234 s
->out_start
= b
->out_pos
;
238 ret
= xz_dec_bcj_run(s
->bcj
, s
->lzma2
, b
);
241 ret
= xz_dec_lzma2_run(s
->lzma2
, b
);
243 s
->block
.compressed
+= b
->in_pos
- s
->in_start
;
244 s
->block
.uncompressed
+= b
->out_pos
- s
->out_start
;
247 * There is no need to separately check for VLI_UNKNOWN, since
248 * the observed sizes are always smaller than VLI_UNKNOWN.
250 if (s
->block
.compressed
> s
->block_header
.compressed
251 || s
->block
.uncompressed
252 > s
->block_header
.uncompressed
)
253 return XZ_DATA_ERROR
;
255 if (s
->check_type
== XZ_CHECK_CRC32
)
256 s
->crc
= xz_crc32(b
->out
+ s
->out_start
,
257 b
->out_pos
- s
->out_start
, s
->crc
);
259 else if (s
->check_type
== XZ_CHECK_CRC64
)
260 s
->crc
= xz_crc64(b
->out
+ s
->out_start
,
261 b
->out_pos
- s
->out_start
, s
->crc
);
264 if (ret
== XZ_STREAM_END
) {
265 if (s
->block_header
.compressed
!= VLI_UNKNOWN
266 && s
->block_header
.compressed
267 != s
->block
.compressed
)
268 return XZ_DATA_ERROR
;
270 if (s
->block_header
.uncompressed
!= VLI_UNKNOWN
271 && s
->block_header
.uncompressed
272 != s
->block
.uncompressed
)
273 return XZ_DATA_ERROR
;
275 s
->block
.hash
.unpadded
+= s
->block_header
.size
276 + s
->block
.compressed
;
278 #ifdef XZ_DEC_ANY_CHECK
279 s
->block
.hash
.unpadded
+= check_sizes
[s
->check_type
];
281 if (s
->check_type
== XZ_CHECK_CRC32
)
282 s
->block
.hash
.unpadded
+= 4;
283 else if (IS_CRC64(s
->check_type
))
284 s
->block
.hash
.unpadded
+= 8;
287 s
->block
.hash
.uncompressed
+= s
->block
.uncompressed
;
288 s
->block
.hash
.crc32
= xz_crc32(
289 (const uint8_t *)&s
->block
.hash
,
290 sizeof(s
->block
.hash
), s
->block
.hash
.crc32
);
298 /* Update the Index size and the CRC32 value. */
299 static void index_update(struct xz_dec
*s
, const struct xz_buf
*b
)
301 size_t in_used
= b
->in_pos
- s
->in_start
;
302 s
->index
.size
+= in_used
;
303 s
->crc
= xz_crc32(b
->in
+ s
->in_start
, in_used
, s
->crc
);
307 * Decode the Number of Records, Unpadded Size, and Uncompressed Size
308 * fields from the Index field. That is, Index Padding and CRC32 are not
309 * decoded by this function.
311 * This can return XZ_OK (more input needed), XZ_STREAM_END (everything
312 * successfully decoded), or XZ_DATA_ERROR (input is corrupt).
314 static enum xz_ret
dec_index(struct xz_dec
*s
, struct xz_buf
*b
)
319 ret
= dec_vli(s
, b
->in
, &b
->in_pos
, b
->in_size
);
320 if (ret
!= XZ_STREAM_END
) {
325 switch (s
->index
.sequence
) {
326 case SEQ_INDEX_COUNT
:
327 s
->index
.count
= s
->vli
;
330 * Validate that the Number of Records field
331 * indicates the same number of Records as
332 * there were Blocks in the Stream.
334 if (s
->index
.count
!= s
->block
.count
)
335 return XZ_DATA_ERROR
;
337 s
->index
.sequence
= SEQ_INDEX_UNPADDED
;
340 case SEQ_INDEX_UNPADDED
:
341 s
->index
.hash
.unpadded
+= s
->vli
;
342 s
->index
.sequence
= SEQ_INDEX_UNCOMPRESSED
;
345 case SEQ_INDEX_UNCOMPRESSED
:
346 s
->index
.hash
.uncompressed
+= s
->vli
;
347 s
->index
.hash
.crc32
= xz_crc32(
348 (const uint8_t *)&s
->index
.hash
,
349 sizeof(s
->index
.hash
),
350 s
->index
.hash
.crc32
);
352 s
->index
.sequence
= SEQ_INDEX_UNPADDED
;
355 } while (s
->index
.count
> 0);
357 return XZ_STREAM_END
;
361 * Validate that the next four or eight input bytes match the value
362 * of s->crc. s->pos must be zero when starting to validate the first byte.
363 * The "bits" argument allows using the same code for both CRC32 and CRC64.
365 static enum xz_ret
crc_validate(struct xz_dec
*s
, struct xz_buf
*b
,
369 if (b
->in_pos
== b
->in_size
)
372 if (((s
->crc
>> s
->pos
) & 0xFF) != b
->in
[b
->in_pos
++])
373 return XZ_DATA_ERROR
;
377 } while (s
->pos
< bits
);
382 return XZ_STREAM_END
;
385 #ifdef XZ_DEC_ANY_CHECK
387 * Skip over the Check field when the Check ID is not supported.
388 * Returns true once the whole Check field has been skipped over.
390 static bool check_skip(struct xz_dec
*s
, struct xz_buf
*b
)
392 while (s
->pos
< check_sizes
[s
->check_type
]) {
393 if (b
->in_pos
== b
->in_size
)
406 /* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
407 static enum xz_ret
dec_stream_header(struct xz_dec
*s
)
409 if (!memeq(s
->temp
.buf
, HEADER_MAGIC
, HEADER_MAGIC_SIZE
))
410 return XZ_FORMAT_ERROR
;
412 if (xz_crc32(s
->temp
.buf
+ HEADER_MAGIC_SIZE
, 2, 0)
413 != get_le32(s
->temp
.buf
+ HEADER_MAGIC_SIZE
+ 2))
414 return XZ_DATA_ERROR
;
416 if (s
->temp
.buf
[HEADER_MAGIC_SIZE
] != 0)
417 return XZ_OPTIONS_ERROR
;
420 * Of integrity checks, we support none (Check ID = 0),
421 * CRC32 (Check ID = 1), and optionally CRC64 (Check ID = 4).
422 * However, if XZ_DEC_ANY_CHECK is defined, we will accept other
423 * check types too, but then the check won't be verified and
424 * a warning (XZ_UNSUPPORTED_CHECK) will be given.
426 s
->check_type
= s
->temp
.buf
[HEADER_MAGIC_SIZE
+ 1];
428 #ifdef XZ_DEC_ANY_CHECK
429 if (s
->check_type
> XZ_CHECK_MAX
)
430 return XZ_OPTIONS_ERROR
;
432 if (s
->check_type
> XZ_CHECK_CRC32
&& !IS_CRC64(s
->check_type
))
433 return XZ_UNSUPPORTED_CHECK
;
435 if (s
->check_type
> XZ_CHECK_CRC32
&& !IS_CRC64(s
->check_type
))
436 return XZ_OPTIONS_ERROR
;
442 /* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
443 static enum xz_ret
dec_stream_footer(struct xz_dec
*s
)
445 if (!memeq(s
->temp
.buf
+ 10, FOOTER_MAGIC
, FOOTER_MAGIC_SIZE
))
446 return XZ_DATA_ERROR
;
448 if (xz_crc32(s
->temp
.buf
+ 4, 6, 0) != get_le32(s
->temp
.buf
))
449 return XZ_DATA_ERROR
;
452 * Validate Backward Size. Note that we never added the size of the
453 * Index CRC32 field to s->index.size, thus we use s->index.size / 4
454 * instead of s->index.size / 4 - 1.
456 if ((s
->index
.size
>> 2) != get_le32(s
->temp
.buf
+ 4))
457 return XZ_DATA_ERROR
;
459 if (s
->temp
.buf
[8] != 0 || s
->temp
.buf
[9] != s
->check_type
)
460 return XZ_DATA_ERROR
;
463 * Use XZ_STREAM_END instead of XZ_OK to be more convenient
466 return XZ_STREAM_END
;
469 /* Decode the Block Header and initialize the filter chain. */
470 static enum xz_ret
dec_block_header(struct xz_dec
*s
)
475 * Validate the CRC32. We know that the temp buffer is at least
476 * eight bytes so this is safe.
479 if (xz_crc32(s
->temp
.buf
, s
->temp
.size
, 0)
480 != get_le32(s
->temp
.buf
+ s
->temp
.size
))
481 return XZ_DATA_ERROR
;
486 * Catch unsupported Block Flags. We support only one or two filters
487 * in the chain, so we catch that with the same test.
490 if (s
->temp
.buf
[1] & 0x3E)
492 if (s
->temp
.buf
[1] & 0x3F)
494 return XZ_OPTIONS_ERROR
;
496 /* Compressed Size */
497 if (s
->temp
.buf
[1] & 0x40) {
498 if (dec_vli(s
, s
->temp
.buf
, &s
->temp
.pos
, s
->temp
.size
)
500 return XZ_DATA_ERROR
;
502 s
->block_header
.compressed
= s
->vli
;
504 s
->block_header
.compressed
= VLI_UNKNOWN
;
507 /* Uncompressed Size */
508 if (s
->temp
.buf
[1] & 0x80) {
509 if (dec_vli(s
, s
->temp
.buf
, &s
->temp
.pos
, s
->temp
.size
)
511 return XZ_DATA_ERROR
;
513 s
->block_header
.uncompressed
= s
->vli
;
515 s
->block_header
.uncompressed
= VLI_UNKNOWN
;
519 /* If there are two filters, the first one must be a BCJ filter. */
520 s
->bcj_active
= s
->temp
.buf
[1] & 0x01;
522 if (s
->temp
.size
- s
->temp
.pos
< 2)
523 return XZ_OPTIONS_ERROR
;
525 ret
= xz_dec_bcj_reset(s
->bcj
, s
->temp
.buf
[s
->temp
.pos
++]);
530 * We don't support custom start offset,
531 * so Size of Properties must be zero.
533 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x00)
534 return XZ_OPTIONS_ERROR
;
538 /* Valid Filter Flags always take at least two bytes. */
539 if (s
->temp
.size
- s
->temp
.pos
< 2)
540 return XZ_DATA_ERROR
;
542 /* Filter ID = LZMA2 */
543 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x21)
544 return XZ_OPTIONS_ERROR
;
546 /* Size of Properties = 1-byte Filter Properties */
547 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x01)
548 return XZ_OPTIONS_ERROR
;
550 /* Filter Properties contains LZMA2 dictionary size. */
551 if (s
->temp
.size
- s
->temp
.pos
< 1)
552 return XZ_DATA_ERROR
;
554 ret
= xz_dec_lzma2_reset(s
->lzma2
, s
->temp
.buf
[s
->temp
.pos
++]);
558 /* The rest must be Header Padding. */
559 while (s
->temp
.pos
< s
->temp
.size
)
560 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x00)
561 return XZ_OPTIONS_ERROR
;
564 s
->block
.compressed
= 0;
565 s
->block
.uncompressed
= 0;
570 static enum xz_ret
dec_main(struct xz_dec
*s
, struct xz_buf
*b
)
575 * Store the start position for the case when we are in the middle
576 * of the Index field.
578 s
->in_start
= b
->in_pos
;
581 switch (s
->sequence
) {
582 case SEQ_STREAM_HEADER
:
584 * Stream Header is copied to s->temp, and then
585 * decoded from there. This way if the caller
586 * gives us only little input at a time, we can
587 * still keep the Stream Header decoding code
588 * simple. Similar approach is used in many places
591 if (!fill_temp(s
, b
))
595 * If dec_stream_header() returns
596 * XZ_UNSUPPORTED_CHECK, it is still possible
597 * to continue decoding if working in multi-call
598 * mode. Thus, update s->sequence before calling
599 * dec_stream_header().
601 s
->sequence
= SEQ_BLOCK_START
;
603 ret
= dec_stream_header(s
);
607 case SEQ_BLOCK_START
:
608 /* We need one byte of input to continue. */
609 if (b
->in_pos
== b
->in_size
)
612 /* See if this is the beginning of the Index field. */
613 if (b
->in
[b
->in_pos
] == 0) {
614 s
->in_start
= b
->in_pos
++;
615 s
->sequence
= SEQ_INDEX
;
620 * Calculate the size of the Block Header and
621 * prepare to decode it.
624 = ((uint32_t)b
->in
[b
->in_pos
] + 1) * 4;
626 s
->temp
.size
= s
->block_header
.size
;
628 s
->sequence
= SEQ_BLOCK_HEADER
;
630 case SEQ_BLOCK_HEADER
:
631 if (!fill_temp(s
, b
))
634 ret
= dec_block_header(s
);
638 s
->sequence
= SEQ_BLOCK_UNCOMPRESS
;
640 case SEQ_BLOCK_UNCOMPRESS
:
641 ret
= dec_block(s
, b
);
642 if (ret
!= XZ_STREAM_END
)
645 s
->sequence
= SEQ_BLOCK_PADDING
;
647 case SEQ_BLOCK_PADDING
:
649 * Size of Compressed Data + Block Padding
650 * must be a multiple of four. We don't need
651 * s->block.compressed for anything else
652 * anymore, so we use it here to test the size
653 * of the Block Padding field.
655 while (s
->block
.compressed
& 3) {
656 if (b
->in_pos
== b
->in_size
)
659 if (b
->in
[b
->in_pos
++] != 0)
660 return XZ_DATA_ERROR
;
662 ++s
->block
.compressed
;
665 s
->sequence
= SEQ_BLOCK_CHECK
;
667 case SEQ_BLOCK_CHECK
:
668 if (s
->check_type
== XZ_CHECK_CRC32
) {
669 ret
= crc_validate(s
, b
, 32);
670 if (ret
!= XZ_STREAM_END
)
673 else if (IS_CRC64(s
->check_type
)) {
674 ret
= crc_validate(s
, b
, 64);
675 if (ret
!= XZ_STREAM_END
)
678 #ifdef XZ_DEC_ANY_CHECK
679 else if (!check_skip(s
, b
)) {
684 s
->sequence
= SEQ_BLOCK_START
;
688 ret
= dec_index(s
, b
);
689 if (ret
!= XZ_STREAM_END
)
692 s
->sequence
= SEQ_INDEX_PADDING
;
694 case SEQ_INDEX_PADDING
:
695 while ((s
->index
.size
+ (b
->in_pos
- s
->in_start
))
697 if (b
->in_pos
== b
->in_size
) {
702 if (b
->in
[b
->in_pos
++] != 0)
703 return XZ_DATA_ERROR
;
706 /* Finish the CRC32 value and Index size. */
709 /* Compare the hashes to validate the Index field. */
710 if (!memeq(&s
->block
.hash
, &s
->index
.hash
,
711 sizeof(s
->block
.hash
)))
712 return XZ_DATA_ERROR
;
714 s
->sequence
= SEQ_INDEX_CRC32
;
716 case SEQ_INDEX_CRC32
:
717 ret
= crc_validate(s
, b
, 32);
718 if (ret
!= XZ_STREAM_END
)
721 s
->temp
.size
= STREAM_HEADER_SIZE
;
722 s
->sequence
= SEQ_STREAM_FOOTER
;
724 case SEQ_STREAM_FOOTER
:
725 if (!fill_temp(s
, b
))
728 return dec_stream_footer(s
);
736 * xz_dec_run() is a wrapper for dec_main() to handle some special cases in
737 * multi-call and single-call decoding.
739 * In multi-call mode, we must return XZ_BUF_ERROR when it seems clear that we
740 * are not going to make any progress anymore. This is to prevent the caller
741 * from calling us infinitely when the input file is truncated or otherwise
742 * corrupt. Since zlib-style API allows that the caller fills the input buffer
743 * only when the decoder doesn't produce any new output, we have to be careful
744 * to avoid returning XZ_BUF_ERROR too easily: XZ_BUF_ERROR is returned only
745 * after the second consecutive call to xz_dec_run() that makes no progress.
747 * In single-call mode, if we couldn't decode everything and no error
748 * occurred, either the input is truncated or the output buffer is too small.
749 * Since we know that the last input byte never produces any output, we know
750 * that if all the input was consumed and decoding wasn't finished, the file
751 * must be corrupt. Otherwise the output buffer has to be too small or the
752 * file is corrupt in a way that decoding it produces too big output.
754 * If single-call decoding fails, we reset b->in_pos and b->out_pos back to
755 * their original values. This is because with some filter chains there won't
756 * be any valid uncompressed data in the output buffer unless the decoding
757 * actually succeeds (that's the price to pay of using the output buffer as
760 XZ_EXTERN
enum xz_ret
xz_dec_run(struct xz_dec
*s
, struct xz_buf
*b
)
766 if (DEC_IS_SINGLE(s
->mode
))
769 in_start
= b
->in_pos
;
770 out_start
= b
->out_pos
;
771 ret
= dec_main(s
, b
);
773 if (DEC_IS_SINGLE(s
->mode
)) {
775 ret
= b
->in_pos
== b
->in_size
776 ? XZ_DATA_ERROR
: XZ_BUF_ERROR
;
778 if (ret
!= XZ_STREAM_END
) {
779 b
->in_pos
= in_start
;
780 b
->out_pos
= out_start
;
783 } else if (ret
== XZ_OK
&& in_start
== b
->in_pos
784 && out_start
== b
->out_pos
) {
785 if (s
->allow_buf_error
)
788 s
->allow_buf_error
= true;
790 s
->allow_buf_error
= false;
796 XZ_EXTERN
struct xz_dec
*xz_dec_init(enum xz_mode mode
, uint32_t dict_max
)
798 struct xz_dec
*s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
805 s
->bcj
= xz_dec_bcj_create(DEC_IS_SINGLE(mode
));
810 s
->lzma2
= xz_dec_lzma2_create(mode
, dict_max
);
811 if (s
->lzma2
== NULL
)
819 xz_dec_bcj_end(s
->bcj
);
826 XZ_EXTERN
void xz_dec_reset(struct xz_dec
*s
)
828 s
->sequence
= SEQ_STREAM_HEADER
;
829 s
->allow_buf_error
= false;
832 memzero(&s
->block
, sizeof(s
->block
));
833 memzero(&s
->index
, sizeof(s
->index
));
835 s
->temp
.size
= STREAM_HEADER_SIZE
;
838 XZ_EXTERN
void xz_dec_end(struct xz_dec
*s
)
841 xz_dec_lzma2_end(s
->lzma2
);
843 xz_dec_bcj_end(s
->bcj
);