2 * QEMU VNC display driver: Zlib Run-length Encoding (ZRLE)
4 * From libvncserver/libvncserver/zrle.c
5 * Copyright (C) 2002 RealVNC Ltd. All Rights Reserved.
6 * Copyright (C) 2003 Sun Microsystems, Inc.
8 * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
30 #include "vnc-enc-zrle.h"
32 static const int bits_per_packed_pixel
[] = {
33 0, 1, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
37 static void vnc_zrle_start(VncState
*vs
)
39 buffer_reset(&vs
->zrle
.zrle
);
41 /* make the output buffer be the zlib buffer, so we can compress it later */
42 vs
->zrle
.tmp
= vs
->output
;
43 vs
->output
= vs
->zrle
.zrle
;
46 static void vnc_zrle_stop(VncState
*vs
)
48 /* switch back to normal output/zlib buffers */
49 vs
->zrle
.zrle
= vs
->output
;
50 vs
->output
= vs
->zrle
.tmp
;
53 static void *zrle_convert_fb(VncState
*vs
, int x
, int y
, int w
, int h
,
58 buffer_reset(&vs
->zrle
.fb
);
59 buffer_reserve(&vs
->zrle
.fb
, w
* h
* bpp
+ bpp
);
62 vs
->output
= vs
->zrle
.fb
;
64 vnc_raw_send_framebuffer_update(vs
, x
, y
, w
, h
);
66 vs
->zrle
.fb
= vs
->output
;
68 return vs
->zrle
.fb
.buffer
;
71 static int zrle_compress_data(VncState
*vs
, int level
)
73 z_streamp zstream
= &vs
->zrle
.stream
;
75 buffer_reset(&vs
->zrle
.zlib
);
77 if (zstream
->opaque
!= vs
) {
80 zstream
->zalloc
= vnc_zlib_zalloc
;
81 zstream
->zfree
= vnc_zlib_zfree
;
83 err
= deflateInit2(zstream
, level
, Z_DEFLATED
, MAX_WBITS
,
84 MAX_MEM_LEVEL
, Z_DEFAULT_STRATEGY
);
87 fprintf(stderr
, "VNC: error initializing zlib\n");
94 /* reserve memory in output buffer */
95 buffer_reserve(&vs
->zrle
.zlib
, vs
->zrle
.zrle
.offset
+ 64);
98 zstream
->next_in
= vs
->zrle
.zrle
.buffer
;
99 zstream
->avail_in
= vs
->zrle
.zrle
.offset
;
100 zstream
->next_out
= vs
->zrle
.zlib
.buffer
+ vs
->zrle
.zlib
.offset
;
101 zstream
->avail_out
= vs
->zrle
.zlib
.capacity
- vs
->zrle
.zlib
.offset
;
102 zstream
->data_type
= Z_BINARY
;
105 if (deflate(zstream
, Z_SYNC_FLUSH
) != Z_OK
) {
106 fprintf(stderr
, "VNC: error during zrle compression\n");
110 vs
->zrle
.zlib
.offset
= vs
->zrle
.zlib
.capacity
- zstream
->avail_out
;
111 return vs
->zrle
.zlib
.offset
;
114 /* Try to work out whether to use RLE and/or a palette. We do this by
115 * estimating the number of bytes which will be generated and picking the
116 * method which results in the fewest bytes. Of course this may not result
117 * in the fewest bytes after compression... */
118 static void zrle_choose_palette_rle(VncState
*vs
, int w
, int h
,
119 VncPalette
*palette
, int bpp_out
,
120 int runs
, int single_pixels
,
122 bool *use_rle
, bool *use_palette
)
124 size_t estimated_bytes
;
125 size_t plain_rle_bytes
;
127 *use_palette
= *use_rle
= false;
129 estimated_bytes
= w
* h
* (bpp_out
/ 8); /* start assuming raw */
132 if (zywrle_level
> 0 && !(zywrle_level
& 0x80))
133 estimated_bytes
>>= zywrle_level
;
136 plain_rle_bytes
= ((bpp_out
/ 8) + 1) * (runs
+ single_pixels
);
138 if (plain_rle_bytes
< estimated_bytes
) {
140 estimated_bytes
= plain_rle_bytes
;
143 if (palette_size(palette
) < 128) {
144 int palette_rle_bytes
;
146 palette_rle_bytes
= (bpp_out
/ 8) * palette_size(palette
);
147 palette_rle_bytes
+= 2 * runs
+ single_pixels
;
149 if (palette_rle_bytes
< estimated_bytes
) {
152 estimated_bytes
= palette_rle_bytes
;
155 if (palette_size(palette
) < 17) {
158 packed_bytes
= (bpp_out
/ 8) * palette_size(palette
);
159 packed_bytes
+= w
* h
*
160 bits_per_packed_pixel
[palette_size(palette
)-1] / 8;
162 if (packed_bytes
< estimated_bytes
) {
165 estimated_bytes
= packed_bytes
;
171 static void zrle_write_u32(VncState
*vs
, uint32_t value
)
173 vnc_write(vs
, (uint8_t *)&value
, 4);
176 static void zrle_write_u24a(VncState
*vs
, uint32_t value
)
178 vnc_write(vs
, (uint8_t *)&value
, 3);
181 static void zrle_write_u24b(VncState
*vs
, uint32_t value
)
183 vnc_write(vs
, ((uint8_t *)&value
) + 1, 3);
186 static void zrle_write_u16(VncState
*vs
, uint16_t value
)
188 vnc_write(vs
, (uint8_t *)&value
, 2);
191 static void zrle_write_u8(VncState
*vs
, uint8_t value
)
193 vnc_write_u8(vs
, value
);
196 #define ENDIAN_LITTLE 0
201 #define ZYWRLE_ENDIAN ENDIAN_NO
202 #include "vnc-enc-zrle-template.c"
207 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
208 #include "vnc-enc-zrle-template.c"
211 #define ZYWRLE_ENDIAN ENDIAN_BIG
212 #include "vnc-enc-zrle-template.c"
217 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
218 #include "vnc-enc-zrle-template.c"
221 #define ZYWRLE_ENDIAN ENDIAN_BIG
222 #include "vnc-enc-zrle-template.c"
227 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
228 #include "vnc-enc-zrle-template.c"
231 #define ZYWRLE_ENDIAN ENDIAN_BIG
232 #include "vnc-enc-zrle-template.c"
234 #define ZRLE_COMPACT_PIXEL 24a
236 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
237 #include "vnc-enc-zrle-template.c"
240 #define ZYWRLE_ENDIAN ENDIAN_BIG
241 #include "vnc-enc-zrle-template.c"
243 #undef ZRLE_COMPACT_PIXEL
244 #define ZRLE_COMPACT_PIXEL 24b
246 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
247 #include "vnc-enc-zrle-template.c"
250 #define ZYWRLE_ENDIAN ENDIAN_BIG
251 #include "vnc-enc-zrle-template.c"
252 #undef ZRLE_COMPACT_PIXEL
255 static int zrle_send_framebuffer_update(VncState
*vs
, int x
, int y
,
258 bool be
= !!(vs
->clientds
.flags
& QEMU_BIG_ENDIAN_FLAG
);
262 if (vs
->zrle
.type
== VNC_ENCODING_ZYWRLE
) {
263 if (!vs
->vd
->lossy
|| vs
->tight
.quality
< 0 || vs
->tight
.quality
== 9) {
265 vs
->zrle
.type
= VNC_ENCODING_ZRLE
;
266 } else if (vs
->tight
.quality
< 3) {
268 } else if (vs
->tight
.quality
< 6) {
279 switch(vs
->clientds
.pf
.bytes_per_pixel
) {
281 zrle_encode_8ne(vs
, x
, y
, w
, h
, zywrle_level
);
285 if (vs
->clientds
.pf
.gmax
> 0x1F) {
287 zrle_encode_16be(vs
, x
, y
, w
, h
, zywrle_level
);
289 zrle_encode_16le(vs
, x
, y
, w
, h
, zywrle_level
);
293 zrle_encode_15be(vs
, x
, y
, w
, h
, zywrle_level
);
295 zrle_encode_15le(vs
, x
, y
, w
, h
, zywrle_level
);
302 bool fits_in_ls3bytes
;
303 bool fits_in_ms3bytes
;
306 ((vs
->clientds
.pf
.rmax
<< vs
->clientds
.pf
.rshift
) < (1 << 24) &&
307 (vs
->clientds
.pf
.gmax
<< vs
->clientds
.pf
.gshift
) < (1 << 24) &&
308 (vs
->clientds
.pf
.bmax
<< vs
->clientds
.pf
.bshift
) < (1 << 24));
310 fits_in_ms3bytes
= (vs
->clientds
.pf
.rshift
> 7 &&
311 vs
->clientds
.pf
.gshift
> 7 &&
312 vs
->clientds
.pf
.bshift
> 7);
314 if ((fits_in_ls3bytes
&& !be
) || (fits_in_ms3bytes
&& be
)) {
316 zrle_encode_24abe(vs
, x
, y
, w
, h
, zywrle_level
);
318 zrle_encode_24ale(vs
, x
, y
, w
, h
, zywrle_level
);
320 } else if ((fits_in_ls3bytes
&& be
) || (fits_in_ms3bytes
&& !be
)) {
322 zrle_encode_24bbe(vs
, x
, y
, w
, h
, zywrle_level
);
324 zrle_encode_24ble(vs
, x
, y
, w
, h
, zywrle_level
);
328 zrle_encode_32be(vs
, x
, y
, w
, h
, zywrle_level
);
330 zrle_encode_32le(vs
, x
, y
, w
, h
, zywrle_level
);
338 bytes
= zrle_compress_data(vs
, Z_DEFAULT_COMPRESSION
);
339 vnc_framebuffer_update(vs
, x
, y
, w
, h
, vs
->zrle
.type
);
340 vnc_write_u32(vs
, bytes
);
341 vnc_write(vs
, vs
->zrle
.zlib
.buffer
, vs
->zrle
.zlib
.offset
);
345 int vnc_zrle_send_framebuffer_update(VncState
*vs
, int x
, int y
, int w
, int h
)
347 vs
->zrle
.type
= VNC_ENCODING_ZRLE
;
348 return zrle_send_framebuffer_update(vs
, x
, y
, w
, h
);
351 int vnc_zywrle_send_framebuffer_update(VncState
*vs
, int x
, int y
, int w
, int h
)
353 vs
->zrle
.type
= VNC_ENCODING_ZYWRLE
;
354 return zrle_send_framebuffer_update(vs
, x
, y
, w
, h
);
357 void vnc_zrle_clear(VncState
*vs
)
359 if (vs
->zrle
.stream
.opaque
) {
360 deflateEnd(&vs
->zrle
.stream
);
362 buffer_free(&vs
->zrle
.zrle
);
363 buffer_free(&vs
->zrle
.fb
);
364 buffer_free(&vs
->zrle
.zlib
);