rtl8139: fix possible out of bound access
[qemu/ar7.git] / ui / vnc-enc-zywrle-template.c
blobb446380a7a3fcbcc5b2b11ad70f2f2a00658bb48
2 /********************************************************************
3 * *
4 * THIS FILE IS PART OF THE 'ZYWRLE' VNC CODEC SOURCE CODE. *
5 * *
6 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
7 * GOVERNED BY A FOLLOWING BSD-STYLE SOURCE LICENSE. *
8 * PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
9 * *
10 * THE 'ZYWRLE' VNC CODEC SOURCE CODE IS (C) COPYRIGHT 2006 *
11 * BY Hitachi Systems & Services, Ltd. *
12 * (Noriaki Yamazaki, Research & Development Center) *
13 * *
14 * *
15 ********************************************************************
16 Redistribution and use in source and binary forms, with or without
17 modification, are permitted provided that the following conditions
18 are met:
20 - Redistributions of source code must retain the above copyright
21 notice, this list of conditions and the following disclaimer.
23 - Redistributions in binary form must reproduce the above copyright
24 notice, this list of conditions and the following disclaimer in the
25 documentation and/or other materials provided with the distribution.
27 - Neither the name of the Hitachi Systems & Services, Ltd. nor
28 the names of its contributors may be used to endorse or promote
29 products derived from this software without specific prior written
30 permission.
32 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
33 ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
34 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
35 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION
36 OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
37 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
38 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
39 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
40 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
41 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
42 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 ********************************************************************/
45 /* Change Log:
46 V0.02 : 2008/02/04 : Fix mis encode/decode when width != scanline
47 (Thanks Johannes Schindelin, author of LibVNC
48 Server/Client)
49 V0.01 : 2007/02/06 : Initial release
53 [References]
54 PLHarr:
55 Senecal, J. G., P. Lindstrom, M. A. Duchaineau, and K. I. Joy,
56 "An Improved N-Bit to N-Bit Reversible Haar-Like Transform,"
57 Pacific Graphics 2004, October 2004, pp. 371-380.
58 EZW:
59 Shapiro, JM: Embedded Image Coding Using Zerotrees of Wavelet Coefficients,
60 IEEE Trans. Signal. Process., Vol.41, pp.3445-3462 (1993).
64 /* Template Macro stuffs. */
65 #undef ZYWRLE_ANALYZE
66 #undef ZYWRLE_SYNTHESIZE
68 #define ZYWRLE_SUFFIX ZRLE_CONCAT2(ZRLE_BPP,ZRLE_ENDIAN_SUFFIX)
70 #define ZYWRLE_ANALYZE ZRLE_CONCAT2(zywrle_analyze_, ZYWRLE_SUFFIX)
71 #define ZYWRLE_SYNTHESIZE ZRLE_CONCAT2(zywrle_synthesize_,ZYWRLE_SUFFIX)
73 #define ZYWRLE_RGBYUV ZRLE_CONCAT2(zywrle_rgbyuv_, ZYWRLE_SUFFIX)
74 #define ZYWRLE_YUVRGB ZRLE_CONCAT2(zywrle_yuvrgb_, ZYWRLE_SUFFIX)
75 #define ZYWRLE_YMASK ZRLE_CONCAT2(ZYWRLE_YMASK, ZRLE_BPP)
76 #define ZYWRLE_UVMASK ZRLE_CONCAT2(ZYWRLE_UVMASK, ZRLE_BPP)
77 #define ZYWRLE_LOAD_PIXEL ZRLE_CONCAT2(ZYWRLE_LOAD_PIXEL, ZRLE_BPP)
78 #define ZYWRLE_SAVE_PIXEL ZRLE_CONCAT2(ZYWRLE_SAVE_PIXEL, ZRLE_BPP)
80 /* Packing/Unpacking pixel stuffs.
81 Endian conversion stuffs. */
82 #undef S_0
83 #undef S_1
84 #undef L_0
85 #undef L_1
86 #undef L_2
88 #if ZYWRLE_ENDIAN == ENDIAN_BIG
89 # define S_0 1
90 # define S_1 0
91 # define L_0 3
92 # define L_1 2
93 # define L_2 1
94 #else
95 # define S_0 0
96 # define S_1 1
97 # define L_0 0
98 # define L_1 1
99 # define L_2 2
100 #endif
102 #define ZYWRLE_QUANTIZE
103 #include "qemu/osdep.h"
104 #include "vnc-enc-zywrle.h"
106 #ifndef ZRLE_COMPACT_PIXEL
107 static inline void ZYWRLE_RGBYUV(int *buf, ZRLE_PIXEL *data,
108 int width, int height, int scanline)
110 int r, g, b;
111 int y, u, v;
112 int *line;
113 int *end;
115 end = buf + height * width;
116 while (buf < end) {
117 line = buf + width;
118 while (buf < line) {
119 ZYWRLE_LOAD_PIXEL(data, r, g, b);
120 ZYWRLE_RGBYUV_(r, g, b, y, u, v, ZYWRLE_YMASK, ZYWRLE_UVMASK);
121 ZYWRLE_SAVE_COEFF(buf, v, y, u);
122 buf++;
123 data++;
125 data += scanline - width;
129 static ZRLE_PIXEL *ZYWRLE_ANALYZE(ZRLE_PIXEL *dst, ZRLE_PIXEL *src,
130 int w, int h, int scanline, int level,
131 int *buf) {
132 int l;
133 int uw = w;
134 int uh = h;
135 int *top;
136 int *end;
137 int *line;
138 ZRLE_PIXEL *p;
139 int r, g, b;
140 int s;
141 int *ph;
143 zywrle_calc_size(&w, &h, level);
145 if (w == 0 || h == 0) {
146 return NULL;
148 uw -= w;
149 uh -= h;
151 p = dst;
152 ZYWRLE_LOAD_UNALIGN(src,*(ZRLE_PIXEL*)top = *p;);
153 ZYWRLE_RGBYUV(buf, src, w, h, scanline);
154 wavelet(buf, w, h, level);
155 for (l = 0; l < level; l++) {
156 ZYWRLE_PACK_COEFF(buf, dst, 3, w, h, scanline, l);
157 ZYWRLE_PACK_COEFF(buf, dst, 2, w, h, scanline, l);
158 ZYWRLE_PACK_COEFF(buf, dst, 1, w, h, scanline, l);
159 if (l == level - 1) {
160 ZYWRLE_PACK_COEFF(buf, dst, 0, w, h, scanline, l);
163 ZYWRLE_SAVE_UNALIGN(dst,*dst = *(ZRLE_PIXEL*)top;);
164 return dst;
166 #endif /* ZRLE_COMPACT_PIXEL */
168 #undef ZYWRLE_RGBYUV
169 #undef ZYWRLE_YUVRGB
170 #undef ZYWRLE_LOAD_PIXEL
171 #undef ZYWRLE_SAVE_PIXEL