agssim: stop execution on invalid jump
[rofl0r-agsutils.git] / rle.c
blob063dadb46924ebc73a737e86a29be8ffe20e4440
1 #include <stdlib.h>
3 #define rle_read_col(OUT, IDX) \
4 for(OUT=0, i=0; i<bpp;++i) {OUT |= (q[bpp*(IDX)+i] << (i*8));}
5 unsigned rle_encode(
6 unsigned char *data, unsigned data_size,
7 unsigned bpp, unsigned char** result)
9 /* worst case length: entire file consisting of sequences of 2
10 identical, and one different pixel, resulting in
11 1 byte flag + 1 pixel + 1 byte flag + 1 pixel. in the case of
12 8 bit, that's 1 byte overhead every 3 pixels. */
13 unsigned char *out = malloc(data_size + 1 + (data_size/bpp/3));
14 if(!out) return 0;
15 unsigned char *p = out, *q = data;
16 unsigned i, count = 0, togo = data_size/bpp, repcol = 0;
17 unsigned mode = 0; /* 0: stateless, 1: series: 2: repetition */
18 unsigned jump_flag;
19 while(1) {
20 jump_flag = 0;
21 unsigned col[2] = {0};
22 if(togo) {
23 rle_read_col(col[0], count);
24 if(togo>1 && mode < 2) rle_read_col(col[1], count+1);
25 } else {
26 if(count) goto write_series;
27 else break;
29 switch(mode) {
30 case 0:
31 if(togo>1) {
32 if(col[0] == col[1]) {
33 start_rep:
34 mode = 2;
35 repcol = col[0];
36 } else {
37 start_series:
38 mode = 1;
40 count = 1;
41 } else {
42 goto start_series;
44 break;
45 case 1:
46 if(togo>1) {
47 if(col[0] == col[1]) {
48 jump_flag = 1;
49 goto write_series;
50 } else {
51 advance:
52 if(++count == 128) {
53 write_series:
54 *(p++) = ((mode - 1) << 7) | (count - 1);
55 if(mode == 1) for(i=0;i<count*bpp;++i)
56 *(p++) = *(q++);
57 else {
58 for(i=0;i<bpp*8;i+=8)
59 *(p++) = (repcol & (0xff << i)) >> i;
60 q += count * bpp;
62 if(!togo) goto done;
63 if(jump_flag == 1) goto start_rep;
64 if(jump_flag == 2) goto start_series;
65 mode = 0;
66 count = 0;
69 } else goto advance;
70 break;
71 case 2:
72 if(col[0] == repcol) goto advance;
73 else {
74 jump_flag = 2;
75 goto write_series;
78 togo--;
80 done:
81 *result = out;
82 return p-out;
85 /* caller needs to provide result buffer of w*h*bpp size. */
86 void rle_decode(
87 unsigned char *data, unsigned data_size,
88 unsigned bpp, unsigned char* result, unsigned result_size) {
89 unsigned char
90 *p = result, *p_e = p + result_size,
91 *q = data, *q_e = q + data_size;
92 while(q+1+bpp <= q_e) {
93 unsigned count = (*q & 127) + 1;
94 unsigned rep = *q & 128;
95 unsigned color = 0, i, j;
96 ++q;
97 if(rep) {
98 for(i = 0; i < bpp; ++i)
99 color = (color << 8) | *(q++);
100 for(i = 0; i < count && p+bpp <= p_e; ++i)
101 for(j=0; j<bpp; j++)
102 *(p++) = (color >> ((bpp-j-1)*8)) & 0xff;
103 } else {
104 for(i = 0; i < count && p+bpp <= p_e && q+bpp <= q_e; ++i)
105 for(j=0; j<bpp; j++) *(p++) = *(q++);