1 /* lzo1x_9x.c -- implementation of the LZO1X-999 compression algorithm
3 This file is part of the LZO real-time data compression library.
5 Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer
6 Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer
7 Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer
8 Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer
9 Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer
10 Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer
11 Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer
12 Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer
13 Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer
14 Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer
15 Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer
16 Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer
17 Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer
18 Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer
19 Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer
20 Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer
23 The LZO library is free software; you can redistribute it and/or
24 modify it under the terms of the GNU General Public License as
25 published by the Free Software Foundation; either version 2 of
26 the License, or (at your option) any later version.
28 The LZO library is distributed in the hope that it will be useful,
29 but WITHOUT ANY WARRANTY; without even the implied warranty of
30 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 GNU General Public License for more details.
33 You should have received a copy of the GNU General Public License
34 along with the LZO library; see the file COPYING.
35 If not, write to the Free Software Foundation, Inc.,
36 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
38 Markus F.X.J. Oberhumer
39 <markus@oberhumer.com>
40 http://www.oberhumer.com/opensource/lzo/
44 #if !defined(LZO1X) && !defined(LZO1Y) && !defined(LZO1Z)
49 # include "config1x.h"
51 # include "config1y.h"
53 # include "config1z.h"
59 /***********************************************************************
61 ************************************************************************/
63 #define SWD_N M4_MAX_OFFSET /* size of ring buffer */
64 #define SWD_THRESHOLD 1 /* lower limit for match length */
65 #define SWD_F 2048 /* upper limit for match length */
67 #define SWD_BEST_OFF (LZO_MAX3( M2_MAX_LEN, M3_MAX_LEN, M4_MAX_LEN ) + 1)
70 # define LZO_COMPRESS_T lzo1x_999_t
71 # define lzo_swd_t lzo1x_999_swd_t
73 # define LZO_COMPRESS_T lzo1y_999_t
74 # define lzo_swd_t lzo1y_999_swd_t
75 # define lzo1x_999_compress_internal lzo1y_999_compress_internal
76 # define lzo1x_999_compress_dict lzo1y_999_compress_dict
77 # define lzo1x_999_compress_level lzo1y_999_compress_level
78 # define lzo1x_999_compress lzo1y_999_compress
80 # define LZO_COMPRESS_T lzo1z_999_t
81 # define lzo_swd_t lzo1z_999_swd_t
82 # define lzo1x_999_compress_internal lzo1z_999_compress_internal
83 # define lzo1x_999_compress_dict lzo1z_999_compress_dict
84 # define lzo1x_999_compress_level lzo1z_999_compress_level
85 # define lzo1x_999_compress lzo1z_999_compress
92 ((((((lzo_xint)b[p]<<3)^b[p+1])<<3)^b[p+2]) & (SWD_HSIZE-1))
94 #if 0 && defined(LZO_UNALIGNED_OK_4) && defined(LZO_ABI_LITTLE_ENDIAN)
96 (((* (lzo_uint32p) &b[p]) ^ ((* (lzo_uint32p) &b[p])>>10)) & (SWD_HSIZE-1))
99 #include "lzo_mchw.ch"
102 /* this is a public functions, but there is no prototype in a header file */
104 lzo1x_999_compress_internal ( const lzo_bytep in
, lzo_uint in_len
,
105 lzo_bytep out
, lzo_uintp out_len
,
107 const lzo_bytep dict
, lzo_uint dict_len
,
110 lzo_uint good_length
,
112 lzo_uint nice_length
,
117 /***********************************************************************
119 ************************************************************************/
122 code_match ( LZO_COMPRESS_T
*c
, lzo_bytep op
, lzo_uint m_len
, lzo_uint m_off
)
124 lzo_uint x_len
= m_len
;
125 lzo_uint x_off
= m_off
;
127 c
->match_bytes
+= (unsigned long) m_len
;
131 static lzo_uint last_m_len = 0, last_m_off = 0;
132 static lzo_uint prev_m_off[4];
133 static unsigned prev_m_off_ptr = 0;
136 //if (m_len >= 3 && m_len <= M2_MAX_LEN && m_off <= M2_MAX_OFFSET)
137 if (m_len >= 3 && m_len <= M2_MAX_LEN)
139 //if (m_len == last_m_len && m_off == last_m_off)
140 //printf("last_m_len + last_m_off\n");
142 if (m_off == last_m_off)
143 printf("last_m_off\n");
146 for (i = 0; i < 4; i++)
147 if (m_off == prev_m_off[i])
148 printf("prev_m_off %u: %5ld\n",i,(long)m_off);
152 last_m_off = prev_m_off[prev_m_off_ptr] = m_off;
153 prev_m_off_ptr = (prev_m_off_ptr + 1) & 3;
160 assert(m_off
<= M1_MAX_OFFSET
);
161 assert(c
->r1_lit
> 0); assert(c
->r1_lit
< 4);
164 *op
++ = LZO_BYTE(M1_MARKER
| (m_off
>> 6));
165 *op
++ = LZO_BYTE(m_off
<< 2);
167 *op
++ = LZO_BYTE(M1_MARKER
| ((m_off
& 3) << 2));
168 *op
++ = LZO_BYTE(m_off
>> 2);
173 else if (m_len
<= M2_MAX_LEN
&& (m_off
<= M2_MAX_OFFSET
|| m_off
== c
->last_m_off
))
175 else if (m_len
<= M2_MAX_LEN
&& m_off
<= M2_MAX_OFFSET
)
181 *op
++ = LZO_BYTE(((m_len
- 1) << 5) | ((m_off
& 7) << 2));
182 *op
++ = LZO_BYTE(m_off
>> 3);
183 assert(op
[-2] >= M2_MARKER
);
186 *op
++ = LZO_BYTE(((m_len
+ 1) << 4) | ((m_off
& 3) << 2));
187 *op
++ = LZO_BYTE(m_off
>> 2);
188 assert(op
[-2] >= M2_MARKER
);
190 if (m_off
== c
->last_m_off
)
191 *op
++ = LZO_BYTE(((m_len
- 1) << 5) | (0x700 >> 6));
195 *op
++ = LZO_BYTE(((m_len
- 1) << 5) | (m_off
>> 6));
196 *op
++ = LZO_BYTE(m_off
<< 2);
201 else if (m_len
== M2_MIN_LEN
&& m_off
<= MX_MAX_OFFSET
&& c
->r1_lit
>= 4)
204 assert(m_off
> M2_MAX_OFFSET
);
205 m_off
-= 1 + M2_MAX_OFFSET
;
207 *op
++ = LZO_BYTE(M1_MARKER
| (m_off
>> 6));
208 *op
++ = LZO_BYTE(m_off
<< 2);
210 *op
++ = LZO_BYTE(M1_MARKER
| ((m_off
& 3) << 2));
211 *op
++ = LZO_BYTE(m_off
>> 2);
215 else if (m_off
<= M3_MAX_OFFSET
)
219 if (m_len
<= M3_MAX_LEN
)
220 *op
++ = LZO_BYTE(M3_MARKER
| (m_len
- 2));
224 *op
++ = M3_MARKER
| 0;
231 *op
++ = LZO_BYTE(m_len
);
234 *op
++ = LZO_BYTE(m_off
>> 6);
235 *op
++ = LZO_BYTE(m_off
<< 2);
237 *op
++ = LZO_BYTE(m_off
<< 2);
238 *op
++ = LZO_BYTE(m_off
>> 6);
247 assert(m_off
> 0x4000); assert(m_off
<= 0xbfff);
249 k
= (m_off
& 0x4000) >> 11;
250 if (m_len
<= M4_MAX_LEN
)
251 *op
++ = LZO_BYTE(M4_MARKER
| k
| (m_len
- 2));
255 *op
++ = LZO_BYTE(M4_MARKER
| k
| 0);
262 *op
++ = LZO_BYTE(m_len
);
265 *op
++ = LZO_BYTE(m_off
>> 6);
266 *op
++ = LZO_BYTE(m_off
<< 2);
268 *op
++ = LZO_BYTE(m_off
<< 2);
269 *op
++ = LZO_BYTE(m_off
>> 6);
274 c
->last_m_len
= x_len
;
275 c
->last_m_off
= x_off
;
281 STORE_RUN ( LZO_COMPRESS_T
*c
, lzo_bytep op
, const lzo_bytep ii
, lzo_uint t
)
283 c
->lit_bytes
+= (unsigned long) t
;
285 if (op
== c
->out
&& t
<= 238)
287 *op
++ = LZO_BYTE(17 + t
);
292 op
[-1] |= LZO_BYTE(t
);
294 op
[-2] |= LZO_BYTE(t
);
300 *op
++ = LZO_BYTE(t
- 3);
305 lzo_uint tt
= t
- 18;
314 *op
++ = LZO_BYTE(tt
);
317 do *op
++ = *ii
++; while (--t
> 0);
324 code_run ( LZO_COMPRESS_T
*c
, lzo_bytep op
, const lzo_bytep ii
,
325 lzo_uint lit
, lzo_uint m_len
)
330 op
= STORE_RUN(c
,op
,ii
,lit
);
345 /***********************************************************************
347 ************************************************************************/
350 len_of_coded_match ( lzo_uint m_len
, lzo_uint m_off
, lzo_uint lit
)
357 return (m_off
<= M1_MAX_OFFSET
&& lit
> 0 && lit
< 4) ? 2 : 0;
358 if (m_len
<= M2_MAX_LEN
&& m_off
<= M2_MAX_OFFSET
)
360 if (m_len
== M2_MIN_LEN
&& m_off
<= MX_MAX_OFFSET
&& lit
>= 4)
362 if (m_off
<= M3_MAX_OFFSET
)
364 if (m_len
<= M3_MAX_LEN
)
374 if (m_off
<= M4_MAX_OFFSET
)
376 if (m_len
<= M4_MAX_LEN
)
391 min_gain(lzo_uint ahead
, lzo_uint lit1
, lzo_uint lit2
, lzo_uint l1
, lzo_uint l2
, lzo_uint l3
)
393 lzo_uint lazy_match_min_gain
;
396 lazy_match_min_gain
= ahead
;
404 lazy_match_min_gain
+= (lit2
<= 3) ? 0 : 2;
406 lazy_match_min_gain
+= (lit2
<= 18) ? 0 : 1;
408 lazy_match_min_gain
+= (l2
- l1
) * 2;
410 lazy_match_min_gain
-= (ahead
- l3
) * 2;
412 if ((lzo_int
) lazy_match_min_gain
< 0)
413 lazy_match_min_gain
= 0;
417 if (lazy_match_min_gain
== 0)
418 lazy_match_min_gain
= 1;
421 return lazy_match_min_gain
;
425 /***********************************************************************
427 ************************************************************************/
431 void assert_match( const lzo_swd_p swd
, lzo_uint m_len
, lzo_uint m_off
)
433 const LZO_COMPRESS_T
*c
= swd
->c
;
437 if (m_off
<= (lzo_uint
) (c
->bp
- c
->in
))
439 assert(c
->bp
- m_off
+ m_len
< c
->ip
);
440 assert(lzo_memcmp(c
->bp
, c
->bp
- m_off
, m_len
) == 0);
444 assert(swd
->dict
!= NULL
);
445 d_off
= m_off
- (lzo_uint
) (c
->bp
- c
->in
);
446 assert(d_off
<= swd
->dict_len
);
449 assert(lzo_memcmp(c
->bp
, swd
->dict_end
- d_off
, d_off
) == 0);
450 assert(c
->in
+ m_len
- d_off
< c
->ip
);
451 assert(lzo_memcmp(c
->bp
+ d_off
, c
->in
, m_len
- d_off
) == 0);
455 assert(lzo_memcmp(c
->bp
, swd
->dict_end
- d_off
, m_len
) == 0);
460 # define assert_match(a,b,c) ((void)0)
464 #if defined(SWD_BEST_OFF)
467 better_match ( const lzo_swd_p swd
, lzo_uint
*m_len
, lzo_uint
*m_off
)
470 const LZO_COMPRESS_T
*c
= swd
->c
;
473 if (*m_len
<= M2_MIN_LEN
)
476 if (*m_off
== c
->last_m_off
&& *m_len
<= M2_MAX_LEN
)
479 if (*m_len
>= M2_MIN_LEN
+ 1 && *m_len
<= M2_MAX_LEN
+ 1 &&
480 c
->last_m_off
&& swd
->best_off
[*m_len
-1] == c
->last_m_off
)
483 *m_off
= swd
->best_off
[*m_len
];
489 if (*m_off
<= M2_MAX_OFFSET
)
494 if (*m_off
> M2_MAX_OFFSET
&&
495 *m_len
>= M2_MIN_LEN
+ 1 && *m_len
<= M2_MAX_LEN
+ 1 &&
496 swd
->best_off
[*m_len
-1] && swd
->best_off
[*m_len
-1] <= M2_MAX_OFFSET
)
499 *m_off
= swd
->best_off
[*m_len
];
506 if (*m_off
> M3_MAX_OFFSET
&&
507 *m_len
>= M4_MAX_LEN
+ 1 && *m_len
<= M2_MAX_LEN
+ 2 &&
508 swd
->best_off
[*m_len
-2] && swd
->best_off
[*m_len
-2] <= M2_MAX_OFFSET
)
511 *m_off
= swd
->best_off
[*m_len
];
518 if (*m_off
> M3_MAX_OFFSET
&&
519 *m_len
>= M4_MAX_LEN
+ 1 && *m_len
<= M3_MAX_LEN
+ 1 &&
520 swd
->best_off
[*m_len
-1] && swd
->best_off
[*m_len
-1] <= M3_MAX_OFFSET
)
523 *m_off
= swd
->best_off
[*m_len
];
531 /***********************************************************************
533 ************************************************************************/
536 lzo1x_999_compress_internal ( const lzo_bytep in
, lzo_uint in_len
,
537 lzo_bytep out
, lzo_uintp out_len
,
539 const lzo_bytep dict
, lzo_uint dict_len
,
542 lzo_uint good_length
,
544 lzo_uint nice_length
,
551 lzo_uint m_len
, m_off
;
553 LZO_COMPRESS_T
* const c
= &cc
;
554 lzo_swd_p
const swd
= (lzo_swd_p
) wrkmem
;
560 LZO_COMPILE_TIME_ASSERT(LZO1X_999_MEM_COMPRESS
>= SIZEOF_LZO_SWD_T
)
562 LZO_COMPILE_TIME_ASSERT(LZO1Y_999_MEM_COMPRESS
>= SIZEOF_LZO_SWD_T
)
564 LZO_COMPILE_TIME_ASSERT(LZO1Z_999_MEM_COMPRESS
>= SIZEOF_LZO_SWD_T
)
569 /* setup parameter defaults */
570 /* number of lazy match tries */
571 try_lazy
= (lzo_uint
) try_lazy_parm
;
572 if (try_lazy_parm
< 0)
574 /* reduce lazy match search if we already have a match with this length */
575 if (good_length
== 0)
577 /* do not try a lazy match if we already have a match with this length */
580 /* stop searching for longer matches than this one */
581 if (nice_length
== 0)
583 /* don't search more positions than this */
585 max_chain
= SWD_MAX_CHAIN
;
589 c
->in_end
= in
+ in_len
;
592 c
->m1a_m
= c
->m1b_m
= c
->m2_m
= c
->m3_m
= c
->m4_m
= 0;
593 c
->lit1_r
= c
->lit2_r
= c
->lit3_r
= 0;
596 ii
= c
->ip
; /* point to start of literal run */
598 c
->r1_lit
= c
->r1_m_len
= 0;
600 r
= init_match(c
,swd
,dict
,dict_len
,flags
);
604 swd
->max_chain
= max_chain
;
606 swd
->nice_length
= nice_length
;
608 r
= find_match(c
,swd
,0,0);
617 c
->codesize
= pd(op
, out
);
622 assert(c
->bp
== c
->ip
- c
->look
);
626 assert(ii
+ lit
== c
->bp
);
627 assert(swd
->b_char
== *(c
->bp
));
630 (m_len
== 2 && (m_off
> M1_MAX_OFFSET
|| lit
== 0 || lit
>= 4)) ||
632 /* Do not accept this match for compressed-data compatibility
633 * with LZO v1.01 and before
634 * [ might be a problem for decompress() and optimize() ]
636 (m_len
== 2 && op
== out
) ||
638 (op
== out
&& lit
== 0))
643 else if (m_len
== M2_MIN_LEN
)
645 /* compression ratio improves if we code a literal in some cases */
646 if (m_off
> MX_MAX_OFFSET
&& lit
>= 4)
654 swd
->max_chain
= max_chain
;
655 r
= find_match(c
,swd
,1,0);
656 assert(r
== 0); LZO_UNUSED(r
);
661 #if defined(SWD_BEST_OFF)
662 if (swd
->use_best_off
)
663 better_match(swd
,&m_len
,&m_off
);
665 assert_match(swd
,m_len
,m_off
);
668 /* shall we try a lazy match ? */
670 if (try_lazy
== 0 || m_len
>= max_lazy
)
678 /* yes, try a lazy match */
679 l1
= len_of_coded_match(m_len
,m_off
,lit
);
682 max_ahead
= LZO_MIN(try_lazy
, l1
- 1);
684 max_ahead
= LZO_MIN3(try_lazy
, l1
, m_len
- 1);
689 while (ahead
< max_ahead
&& c
->look
> m_len
)
691 lzo_uint lazy_match_min_gain
;
693 if (m_len
>= good_length
)
694 swd
->max_chain
= max_chain
>> 2;
696 swd
->max_chain
= max_chain
;
697 r
= find_match(c
,swd
,1,0);
700 assert(r
== 0); LZO_UNUSED(r
);
702 assert(ii
+ lit
+ ahead
== c
->bp
);
705 if (m_off
== c
->last_m_off
&& c
->m_off
!= c
->last_m_off
)
706 if (m_len
>= M2_MIN_LEN
&& m_len
<= M2_MAX_LEN
)
709 if (c
->m_len
< m_len
)
712 if (c
->m_len
== m_len
&& c
->m_off
>= m_off
)
715 #if defined(SWD_BEST_OFF)
716 if (swd
->use_best_off
)
717 better_match(swd
,&c
->m_len
,&c
->m_off
);
719 l2
= len_of_coded_match(c
->m_len
,c
->m_off
,lit
+ahead
);
723 if (c
->m_len
== m_len
&& l2
>= l1
)
729 /* compressed-data compatibility [see above] */
730 l3
= (op
== out
) ? 0 : len_of_coded_match(ahead
,m_off
,lit
);
732 l3
= len_of_coded_match(ahead
,m_off
,lit
);
735 lazy_match_min_gain
= min_gain(ahead
,lit
,lit
+ahead
,l1
,l2
,l3
);
736 if (c
->m_len
>= m_len
+ lazy_match_min_gain
)
739 assert_match(swd
,c
->m_len
,c
->m_off
);
743 /* code previous run */
744 op
= code_run(c
,op
,ii
,lit
,ahead
);
746 /* code shortened match */
747 op
= code_match(c
,op
,ahead
,m_off
);
752 assert(ii
+ lit
== c
->bp
);
754 goto lazy_match_done
;
759 assert(ii
+ lit
+ ahead
== c
->bp
);
762 op
= code_run(c
,op
,ii
,lit
,m_len
);
766 op
= code_match(c
,op
,m_len
,m_off
);
767 swd
->max_chain
= max_chain
;
768 r
= find_match(c
,swd
,m_len
,1+ahead
);
769 assert(r
== 0); LZO_UNUSED(r
);
775 /* store final run */
777 op
= STORE_RUN(c
,op
,ii
,lit
);
779 #if defined(LZO_EOF_CODE)
780 *op
++ = M4_MARKER
| 1;
785 c
->codesize
= pd(op
, out
);
786 assert(c
->textsize
== in_len
);
788 *out_len
= pd(op
, out
);
790 if (c
->cb
&& c
->cb
->nprogress
)
791 (*c
->cb
->nprogress
)(c
->cb
, c
->textsize
, c
->codesize
, 0);
794 printf("%ld %ld -> %ld %ld: %ld %ld %ld %ld %ld %ld: %ld %ld %ld %ld\n",
795 (long) c
->textsize
, (long) in_len
, (long) c
->codesize
,
796 c
->match_bytes
, c
->m1a_m
, c
->m1b_m
, c
->m2_m
, c
->m3_m
, c
->m4_m
,
797 c
->lit_bytes
, c
->lit1_r
, c
->lit2_r
, c
->lit3_r
, c
->lazy
);
799 assert(c
->lit_bytes
+ c
->match_bytes
== in_len
);
805 /***********************************************************************
807 ************************************************************************/
810 lzo1x_999_compress_level ( const lzo_bytep in
, lzo_uint in_len
,
811 lzo_bytep out
, lzo_uintp out_len
,
813 const lzo_bytep dict
, lzo_uint dict_len
,
815 int compression_level
)
820 lzo_uint good_length
;
822 lzo_uint nice_length
;
826 /* faster compression */
827 { 0, 0, 0, 8, 4, 0 },
828 { 0, 0, 0, 16, 8, 0 },
829 { 0, 0, 0, 32, 16, 0 },
830 { 1, 4, 4, 16, 16, 0 },
831 { 1, 8, 16, 32, 32, 0 },
832 { 1, 8, 16, 128, 128, 0 },
833 { 2, 8, 32, 128, 256, 0 },
834 { 2, 32, 128, SWD_F
, 2048, 1 },
835 { 2, SWD_F
, SWD_F
, SWD_F
, 4096, 1 }
836 /* max. compression */
839 if (compression_level
< 1 || compression_level
> 9)
842 compression_level
-= 1;
843 return lzo1x_999_compress_internal(in
, in_len
, out
, out_len
, wrkmem
,
845 c
[compression_level
].try_lazy_parm
,
846 c
[compression_level
].good_length
,
847 c
[compression_level
].max_lazy
,
849 c
[compression_level
].nice_length
,
853 c
[compression_level
].max_chain
,
854 c
[compression_level
].flags
);
858 /***********************************************************************
860 ************************************************************************/
863 lzo1x_999_compress_dict ( const lzo_bytep in
, lzo_uint in_len
,
864 lzo_bytep out
, lzo_uintp out_len
,
866 const lzo_bytep dict
, lzo_uint dict_len
)
868 return lzo1x_999_compress_level(in
, in_len
, out
, out_len
, wrkmem
,
869 dict
, dict_len
, 0, 8);
873 lzo1x_999_compress ( const lzo_bytep in
, lzo_uint in_len
,
874 lzo_bytep out
, lzo_uintp out_len
,
877 return lzo1x_999_compress_level(in
, in_len
, out
, out_len
, wrkmem
,
878 NULL
, 0, (lzo_callback_p
) 0, 8);