4 * Author: Lasse Collin <lasse.collin@tukaani.org>
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
10 #include "xz_private.h"
11 #include "xz_stream.h"
13 /* Hash used to validate the Index field */
16 vli_type uncompressed
;
21 /* Position in dec_main() */
35 /* Position in variable-length integers and Check fields */
38 /* Variable-length integer decoded by dec_vli() */
41 /* Saved in_pos and out_pos */
45 /* CRC32 value in Block or Index */
48 /* Type of the integrity check calculated from uncompressed data */
49 enum xz_check check_type
;
55 * True if the next call to xz_dec_run() is allowed to return
60 /* Information stored in Block Header */
63 * Value stored in the Compressed Size field, or
64 * VLI_UNKNOWN if Compressed Size is not present.
69 * Value stored in the Uncompressed Size field, or
70 * VLI_UNKNOWN if Uncompressed Size is not present.
72 vli_type uncompressed
;
74 /* Size of the Block Header field */
78 /* Information collected when decoding Blocks */
80 /* Observed compressed size of the current Block */
83 /* Observed uncompressed size of the current Block */
84 vli_type uncompressed
;
86 /* Number of Blocks decoded so far */
90 * Hash calculated from the Block sizes. This is used to
91 * validate the Index field.
93 struct xz_dec_hash hash
;
96 /* Variables needed when verifying the Index field */
98 /* Position in dec_index() */
102 SEQ_INDEX_UNCOMPRESSED
105 /* Size of the Index in bytes */
108 /* Number of Records (matches block.count in valid files) */
112 * Hash calculated from the Records (matches block.hash in
115 struct xz_dec_hash hash
;
119 * Temporary buffer needed to hold Stream Header, Block Header,
120 * and Stream Footer. The Block Header is the biggest (1 KiB)
121 * so we reserve space according to that. buf[] has to be aligned
122 * to a multiple of four bytes; the size_t variables before it
123 * should guarantee this.
131 struct xz_dec_lzma2
*lzma2
;
134 struct xz_dec_bcj
*bcj
;
139 #ifdef XZ_DEC_ANY_CHECK
140 /* Sizes of the Check field with different Check IDs */
141 static const uint8_t check_sizes
[16] = {
152 * Fill s->temp by copying data starting from b->in[b->in_pos]. Caller
153 * must have set s->temp.pos to indicate how much data we are supposed
154 * to copy into s->temp.buf. Return true once s->temp.pos has reached
157 static bool XZ_FUNC
fill_temp(struct xz_dec
*s
, struct xz_buf
*b
)
159 size_t copy_size
= min_t(size_t,
160 b
->in_size
- b
->in_pos
, s
->temp
.size
- s
->temp
.pos
);
162 memcpy(s
->temp
.buf
+ s
->temp
.pos
, b
->in
+ b
->in_pos
, copy_size
);
163 b
->in_pos
+= copy_size
;
164 s
->temp
.pos
+= copy_size
;
166 if (s
->temp
.pos
== s
->temp
.size
) {
174 /* Decode a variable-length integer (little-endian base-128 encoding) */
175 static enum xz_ret XZ_FUNC
dec_vli(struct xz_dec
*s
,
176 const uint8_t *in
, size_t *in_pos
, size_t in_size
)
183 while (*in_pos
< in_size
) {
187 s
->vli
|= (vli_type
)(byte
& 0x7F) << s
->pos
;
189 if ((byte
& 0x80) == 0) {
190 /* Don't allow non-minimal encodings. */
191 if (byte
== 0 && s
->pos
!= 0)
192 return XZ_DATA_ERROR
;
195 return XZ_STREAM_END
;
199 if (s
->pos
== 7 * VLI_BYTES_MAX
)
200 return XZ_DATA_ERROR
;
207 * Decode the Compressed Data field from a Block. Update and validate
208 * the observed compressed and uncompressed sizes of the Block so that
209 * they don't exceed the values possibly stored in the Block Header
210 * (validation assumes that no integer overflow occurs, since vli_type
211 * is normally uint64_t). Update the CRC32 if presence of the CRC32
212 * field was indicated in Stream Header.
214 * Once the decoding is finished, validate that the observed sizes match
215 * the sizes possibly stored in the Block Header. Update the hash and
216 * Block count, which are later used to validate the Index field.
218 static enum xz_ret XZ_FUNC
dec_block(struct xz_dec
*s
, struct xz_buf
*b
)
222 s
->in_start
= b
->in_pos
;
223 s
->out_start
= b
->out_pos
;
227 ret
= xz_dec_bcj_run(s
->bcj
, s
->lzma2
, b
);
230 ret
= xz_dec_lzma2_run(s
->lzma2
, b
);
232 s
->block
.compressed
+= b
->in_pos
- s
->in_start
;
233 s
->block
.uncompressed
+= b
->out_pos
- s
->out_start
;
236 * There is no need to separately check for VLI_UNKNOWN, since
237 * the observed sizes are always smaller than VLI_UNKNOWN.
239 if (s
->block
.compressed
> s
->block_header
.compressed
240 || s
->block
.uncompressed
241 > s
->block_header
.uncompressed
)
242 return XZ_DATA_ERROR
;
244 if (s
->check_type
== XZ_CHECK_CRC32
)
245 s
->crc32
= xz_crc32(b
->out
+ s
->out_start
,
246 b
->out_pos
- s
->out_start
, s
->crc32
);
248 if (ret
== XZ_STREAM_END
) {
249 if (s
->block_header
.compressed
!= VLI_UNKNOWN
250 && s
->block_header
.compressed
251 != s
->block
.compressed
)
252 return XZ_DATA_ERROR
;
254 if (s
->block_header
.uncompressed
!= VLI_UNKNOWN
255 && s
->block_header
.uncompressed
256 != s
->block
.uncompressed
)
257 return XZ_DATA_ERROR
;
259 s
->block
.hash
.unpadded
+= s
->block_header
.size
260 + s
->block
.compressed
;
262 #ifdef XZ_DEC_ANY_CHECK
263 s
->block
.hash
.unpadded
+= check_sizes
[s
->check_type
];
265 if (s
->check_type
== XZ_CHECK_CRC32
)
266 s
->block
.hash
.unpadded
+= 4;
269 s
->block
.hash
.uncompressed
+= s
->block
.uncompressed
;
270 s
->block
.hash
.crc32
= xz_crc32(
271 (const uint8_t *)&s
->block
.hash
,
272 sizeof(s
->block
.hash
), s
->block
.hash
.crc32
);
280 /* Update the Index size and the CRC32 value. */
281 static void XZ_FUNC
index_update(struct xz_dec
*s
, const struct xz_buf
*b
)
283 size_t in_used
= b
->in_pos
- s
->in_start
;
284 s
->index
.size
+= in_used
;
285 s
->crc32
= xz_crc32(b
->in
+ s
->in_start
, in_used
, s
->crc32
);
289 * Decode the Number of Records, Unpadded Size, and Uncompressed Size
290 * fields from the Index field. That is, Index Padding and CRC32 are not
291 * decoded by this function.
293 * This can return XZ_OK (more input needed), XZ_STREAM_END (everything
294 * successfully decoded), or XZ_DATA_ERROR (input is corrupt).
296 static enum xz_ret XZ_FUNC
dec_index(struct xz_dec
*s
, struct xz_buf
*b
)
301 ret
= dec_vli(s
, b
->in
, &b
->in_pos
, b
->in_size
);
302 if (ret
!= XZ_STREAM_END
) {
307 switch (s
->index
.sequence
) {
308 case SEQ_INDEX_COUNT
:
309 s
->index
.count
= s
->vli
;
312 * Validate that the Number of Records field
313 * indicates the same number of Records as
314 * there were Blocks in the Stream.
316 if (s
->index
.count
!= s
->block
.count
)
317 return XZ_DATA_ERROR
;
319 s
->index
.sequence
= SEQ_INDEX_UNPADDED
;
322 case SEQ_INDEX_UNPADDED
:
323 s
->index
.hash
.unpadded
+= s
->vli
;
324 s
->index
.sequence
= SEQ_INDEX_UNCOMPRESSED
;
327 case SEQ_INDEX_UNCOMPRESSED
:
328 s
->index
.hash
.uncompressed
+= s
->vli
;
329 s
->index
.hash
.crc32
= xz_crc32(
330 (const uint8_t *)&s
->index
.hash
,
331 sizeof(s
->index
.hash
),
332 s
->index
.hash
.crc32
);
334 s
->index
.sequence
= SEQ_INDEX_UNPADDED
;
337 } while (s
->index
.count
> 0);
339 return XZ_STREAM_END
;
343 * Validate that the next four input bytes match the value of s->crc32.
344 * s->pos must be zero when starting to validate the first byte.
346 static enum xz_ret XZ_FUNC
crc32_validate(struct xz_dec
*s
, struct xz_buf
*b
)
349 if (b
->in_pos
== b
->in_size
)
352 if (((s
->crc32
>> s
->pos
) & 0xFF) != b
->in
[b
->in_pos
++])
353 return XZ_DATA_ERROR
;
356 } while (s
->pos
< 32);
361 return XZ_STREAM_END
;
364 #ifdef XZ_DEC_ANY_CHECK
366 * Skip over the Check field when the Check ID is not supported.
367 * Returns true once the whole Check field has been skipped over.
369 static bool XZ_FUNC
check_skip(struct xz_dec
*s
, struct xz_buf
*b
)
371 while (s
->pos
< check_sizes
[s
->check_type
]) {
372 if (b
->in_pos
== b
->in_size
)
385 /* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
386 static enum xz_ret XZ_FUNC
dec_stream_header(struct xz_dec
*s
)
388 if (!memeq(s
->temp
.buf
, HEADER_MAGIC
, HEADER_MAGIC_SIZE
))
389 return XZ_FORMAT_ERROR
;
391 if (xz_crc32(s
->temp
.buf
+ HEADER_MAGIC_SIZE
, 2, 0)
392 != get_le32(s
->temp
.buf
+ HEADER_MAGIC_SIZE
+ 2))
393 return XZ_DATA_ERROR
;
395 if (s
->temp
.buf
[HEADER_MAGIC_SIZE
] != 0)
396 return XZ_OPTIONS_ERROR
;
399 * Of integrity checks, we support only none (Check ID = 0) and
400 * CRC32 (Check ID = 1). However, if XZ_DEC_ANY_CHECK is defined,
401 * we will accept other check types too, but then the check won't
402 * be verified and a warning (XZ_UNSUPPORTED_CHECK) will be given.
404 s
->check_type
= s
->temp
.buf
[HEADER_MAGIC_SIZE
+ 1];
406 #ifdef XZ_DEC_ANY_CHECK
407 if (s
->check_type
> XZ_CHECK_MAX
)
408 return XZ_OPTIONS_ERROR
;
410 if (s
->check_type
> XZ_CHECK_CRC32
)
411 return XZ_UNSUPPORTED_CHECK
;
413 if (s
->check_type
> XZ_CHECK_CRC32
)
414 return XZ_OPTIONS_ERROR
;
420 /* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
421 static enum xz_ret XZ_FUNC
dec_stream_footer(struct xz_dec
*s
)
423 if (!memeq(s
->temp
.buf
+ 10, FOOTER_MAGIC
, FOOTER_MAGIC_SIZE
))
424 return XZ_DATA_ERROR
;
426 if (xz_crc32(s
->temp
.buf
+ 4, 6, 0) != get_le32(s
->temp
.buf
))
427 return XZ_DATA_ERROR
;
430 * Validate Backward Size. Note that we never added the size of the
431 * Index CRC32 field to s->index.size, thus we use s->index.size / 4
432 * instead of s->index.size / 4 - 1.
434 if ((s
->index
.size
>> 2) != get_le32(s
->temp
.buf
+ 4))
435 return XZ_DATA_ERROR
;
437 if (s
->temp
.buf
[8] != 0 || s
->temp
.buf
[9] != s
->check_type
)
438 return XZ_DATA_ERROR
;
441 * Use XZ_STREAM_END instead of XZ_OK to be more convenient
444 return XZ_STREAM_END
;
447 /* Decode the Block Header and initialize the filter chain. */
448 static enum xz_ret XZ_FUNC
dec_block_header(struct xz_dec
*s
)
453 * Validate the CRC32. We know that the temp buffer is at least
454 * eight bytes so this is safe.
457 if (xz_crc32(s
->temp
.buf
, s
->temp
.size
, 0)
458 != get_le32(s
->temp
.buf
+ s
->temp
.size
))
459 return XZ_DATA_ERROR
;
464 * Catch unsupported Block Flags. We support only one or two filters
465 * in the chain, so we catch that with the same test.
468 if (s
->temp
.buf
[1] & 0x3E)
470 if (s
->temp
.buf
[1] & 0x3F)
472 return XZ_OPTIONS_ERROR
;
474 /* Compressed Size */
475 if (s
->temp
.buf
[1] & 0x40) {
476 if (dec_vli(s
, s
->temp
.buf
, &s
->temp
.pos
, s
->temp
.size
)
478 return XZ_DATA_ERROR
;
480 s
->block_header
.compressed
= s
->vli
;
482 s
->block_header
.compressed
= VLI_UNKNOWN
;
485 /* Uncompressed Size */
486 if (s
->temp
.buf
[1] & 0x80) {
487 if (dec_vli(s
, s
->temp
.buf
, &s
->temp
.pos
, s
->temp
.size
)
489 return XZ_DATA_ERROR
;
491 s
->block_header
.uncompressed
= s
->vli
;
493 s
->block_header
.uncompressed
= VLI_UNKNOWN
;
497 /* If there are two filters, the first one must be a BCJ filter. */
498 s
->bcj_active
= s
->temp
.buf
[1] & 0x01;
500 if (s
->temp
.size
- s
->temp
.pos
< 2)
501 return XZ_OPTIONS_ERROR
;
503 ret
= xz_dec_bcj_reset(s
->bcj
, s
->temp
.buf
[s
->temp
.pos
++]);
508 * We don't support custom start offset,
509 * so Size of Properties must be zero.
511 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x00)
512 return XZ_OPTIONS_ERROR
;
516 /* Valid Filter Flags always take at least two bytes. */
517 if (s
->temp
.size
- s
->temp
.pos
< 2)
518 return XZ_DATA_ERROR
;
520 /* Filter ID = LZMA2 */
521 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x21)
522 return XZ_OPTIONS_ERROR
;
524 /* Size of Properties = 1-byte Filter Properties */
525 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x01)
526 return XZ_OPTIONS_ERROR
;
528 /* Filter Properties contains LZMA2 dictionary size. */
529 if (s
->temp
.size
- s
->temp
.pos
< 1)
530 return XZ_DATA_ERROR
;
532 ret
= xz_dec_lzma2_reset(s
->lzma2
, s
->temp
.buf
[s
->temp
.pos
++]);
536 /* The rest must be Header Padding. */
537 while (s
->temp
.pos
< s
->temp
.size
)
538 if (s
->temp
.buf
[s
->temp
.pos
++] != 0x00)
539 return XZ_OPTIONS_ERROR
;
542 s
->block
.compressed
= 0;
543 s
->block
.uncompressed
= 0;
548 static enum xz_ret XZ_FUNC
dec_main(struct xz_dec
*s
, struct xz_buf
*b
)
553 * Store the start position for the case when we are in the middle
554 * of the Index field.
556 s
->in_start
= b
->in_pos
;
559 switch (s
->sequence
) {
560 case SEQ_STREAM_HEADER
:
562 * Stream Header is copied to s->temp, and then
563 * decoded from there. This way if the caller
564 * gives us only little input at a time, we can
565 * still keep the Stream Header decoding code
566 * simple. Similar approach is used in many places
569 if (!fill_temp(s
, b
))
573 * If dec_stream_header() returns
574 * XZ_UNSUPPORTED_CHECK, it is still possible
575 * to continue decoding if working in multi-call
576 * mode. Thus, update s->sequence before calling
577 * dec_stream_header().
579 s
->sequence
= SEQ_BLOCK_START
;
581 ret
= dec_stream_header(s
);
585 case SEQ_BLOCK_START
:
586 /* We need one byte of input to continue. */
587 if (b
->in_pos
== b
->in_size
)
590 /* See if this is the beginning of the Index field. */
591 if (b
->in
[b
->in_pos
] == 0) {
592 s
->in_start
= b
->in_pos
++;
593 s
->sequence
= SEQ_INDEX
;
598 * Calculate the size of the Block Header and
599 * prepare to decode it.
602 = ((uint32_t)b
->in
[b
->in_pos
] + 1) * 4;
604 s
->temp
.size
= s
->block_header
.size
;
606 s
->sequence
= SEQ_BLOCK_HEADER
;
608 case SEQ_BLOCK_HEADER
:
609 if (!fill_temp(s
, b
))
612 ret
= dec_block_header(s
);
616 s
->sequence
= SEQ_BLOCK_UNCOMPRESS
;
618 case SEQ_BLOCK_UNCOMPRESS
:
619 ret
= dec_block(s
, b
);
620 if (ret
!= XZ_STREAM_END
)
623 s
->sequence
= SEQ_BLOCK_PADDING
;
625 case SEQ_BLOCK_PADDING
:
627 * Size of Compressed Data + Block Padding
628 * must be a multiple of four. We don't need
629 * s->block.compressed for anything else
630 * anymore, so we use it here to test the size
631 * of the Block Padding field.
633 while (s
->block
.compressed
& 3) {
634 if (b
->in_pos
== b
->in_size
)
637 if (b
->in
[b
->in_pos
++] != 0)
638 return XZ_DATA_ERROR
;
640 ++s
->block
.compressed
;
643 s
->sequence
= SEQ_BLOCK_CHECK
;
645 case SEQ_BLOCK_CHECK
:
646 if (s
->check_type
== XZ_CHECK_CRC32
) {
647 ret
= crc32_validate(s
, b
);
648 if (ret
!= XZ_STREAM_END
)
651 #ifdef XZ_DEC_ANY_CHECK
652 else if (!check_skip(s
, b
)) {
657 s
->sequence
= SEQ_BLOCK_START
;
661 ret
= dec_index(s
, b
);
662 if (ret
!= XZ_STREAM_END
)
665 s
->sequence
= SEQ_INDEX_PADDING
;
667 case SEQ_INDEX_PADDING
:
668 while ((s
->index
.size
+ (b
->in_pos
- s
->in_start
))
670 if (b
->in_pos
== b
->in_size
) {
675 if (b
->in
[b
->in_pos
++] != 0)
676 return XZ_DATA_ERROR
;
679 /* Finish the CRC32 value and Index size. */
682 /* Compare the hashes to validate the Index field. */
683 if (!memeq(&s
->block
.hash
, &s
->index
.hash
,
684 sizeof(s
->block
.hash
)))
685 return XZ_DATA_ERROR
;
687 s
->sequence
= SEQ_INDEX_CRC32
;
689 case SEQ_INDEX_CRC32
:
690 ret
= crc32_validate(s
, b
);
691 if (ret
!= XZ_STREAM_END
)
694 s
->temp
.size
= STREAM_HEADER_SIZE
;
695 s
->sequence
= SEQ_STREAM_FOOTER
;
697 case SEQ_STREAM_FOOTER
:
698 if (!fill_temp(s
, b
))
701 return dec_stream_footer(s
);
709 * xz_dec_run() is a wrapper for dec_main() to handle some special cases in
710 * multi-call and single-call decoding.
712 * In multi-call mode, we must return XZ_BUF_ERROR when it seems clear that we
713 * are not going to make any progress anymore. This is to prevent the caller
714 * from calling us infinitely when the input file is truncated or otherwise
715 * corrupt. Since zlib-style API allows that the caller fills the input buffer
716 * only when the decoder doesn't produce any new output, we have to be careful
717 * to avoid returning XZ_BUF_ERROR too easily: XZ_BUF_ERROR is returned only
718 * after the second consecutive call to xz_dec_run() that makes no progress.
720 * In single-call mode, if we couldn't decode everything and no error
721 * occurred, either the input is truncated or the output buffer is too small.
722 * Since we know that the last input byte never produces any output, we know
723 * that if all the input was consumed and decoding wasn't finished, the file
724 * must be corrupt. Otherwise the output buffer has to be too small or the
725 * file is corrupt in a way that decoding it produces too big output.
727 * If single-call decoding fails, we reset b->in_pos and b->out_pos back to
728 * their original values. This is because with some filter chains there won't
729 * be any valid uncompressed data in the output buffer unless the decoding
730 * actually succeeds (that's the price to pay of using the output buffer as
733 XZ_EXTERN
enum xz_ret XZ_FUNC
xz_dec_run(struct xz_dec
*s
, struct xz_buf
*b
)
739 if (DEC_IS_SINGLE(s
->mode
))
742 in_start
= b
->in_pos
;
743 out_start
= b
->out_pos
;
744 ret
= dec_main(s
, b
);
746 if (DEC_IS_SINGLE(s
->mode
)) {
748 ret
= b
->in_pos
== b
->in_size
749 ? XZ_DATA_ERROR
: XZ_BUF_ERROR
;
751 if (ret
!= XZ_STREAM_END
) {
752 b
->in_pos
= in_start
;
753 b
->out_pos
= out_start
;
755 } else if (ret
== XZ_OK
&& in_start
== b
->in_pos
756 && out_start
== b
->out_pos
) {
757 if (s
->allow_buf_error
)
760 s
->allow_buf_error
= true;
762 s
->allow_buf_error
= false;
768 XZ_EXTERN
struct xz_dec
* XZ_FUNC
xz_dec_init(
769 enum xz_mode mode
, uint32_t dict_max
)
771 struct xz_dec
*s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
778 s
->bcj
= xz_dec_bcj_create(DEC_IS_SINGLE(mode
));
783 s
->lzma2
= xz_dec_lzma2_create(mode
, dict_max
);
784 if (s
->lzma2
== NULL
)
792 xz_dec_bcj_end(s
->bcj
);
799 XZ_EXTERN
void XZ_FUNC
xz_dec_reset(struct xz_dec
*s
)
801 s
->sequence
= SEQ_STREAM_HEADER
;
802 s
->allow_buf_error
= false;
805 memzero(&s
->block
, sizeof(s
->block
));
806 memzero(&s
->index
, sizeof(s
->index
));
808 s
->temp
.size
= STREAM_HEADER_SIZE
;
811 XZ_EXTERN
void XZ_FUNC
xz_dec_end(struct xz_dec
*s
)
814 xz_dec_lzma2_end(s
->lzma2
);
816 xz_dec_bcj_end(s
->bcj
);