2 * QEMU VNC display driver: tight encoding
4 * From libvncserver/libvncserver/tight.c
5 * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved.
6 * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
8 * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29 #include "qemu-common.h"
31 #ifdef CONFIG_VNC_JPEG
40 #include "vnc-enc-tight.h"
42 /* Compression level stuff. The following array contains various
43 encoder parameters for each of 10 compression levels (0..9).
44 Last three parameters correspond to JPEG quality levels (0..9). */
47 int max_rect_size
, max_rect_width
;
48 int mono_min_rect_size
, gradient_min_rect_size
;
49 int idx_zlib_level
, mono_zlib_level
, raw_zlib_level
, gradient_zlib_level
;
50 int gradient_threshold
, gradient_threshold24
;
51 int idx_max_colors_divisor
;
52 int jpeg_quality
, jpeg_threshold
, jpeg_threshold24
;
54 { 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 5, 10000, 23000 },
55 { 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 10, 8000, 18000 },
56 { 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 15, 6500, 15000 },
57 { 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 25, 5000, 12000 },
58 { 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 37, 4000, 10000 },
59 { 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 50, 3000, 8000 },
60 { 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 60, 2000, 5000 },
61 { 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 },
62 { 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 },
63 { 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 }
67 * Code to guess if given rectangle is suitable for smooth image
68 * compression (by applying "gradient" filter or JPEG coder).
72 tight_detect_smooth_image24(VncState
*vs
, int w
, int h
)
81 unsigned char *buf
= vs
->tight
.buffer
;
84 * If client is big-endian, color samples begin from the second
85 * byte (offset 1) of a 32-bit pixel value.
87 off
= !!(vs
->clientds
.flags
& QEMU_BIG_ENDIAN_FLAG
);
89 memset(stats
, 0, sizeof (stats
));
91 for (y
= 0, x
= 0; y
< h
&& x
< w
;) {
92 for (d
= 0; d
< h
- y
&& d
< w
- x
- VNC_TIGHT_DETECT_SUBROW_WIDTH
;
94 for (c
= 0; c
< 3; c
++) {
95 left
[c
] = buf
[((y
+d
)*w
+x
+d
)*4+off
+c
] & 0xFF;
97 for (dx
= 1; dx
<= VNC_TIGHT_DETECT_SUBROW_WIDTH
; dx
++) {
98 for (c
= 0; c
< 3; c
++) {
99 pix
= buf
[((y
+d
)*w
+x
+d
+dx
)*4+off
+c
] & 0xFF;
100 stats
[abs(pix
- left
[c
])]++;
115 /* 95% smooth or more ... */
116 if (stats
[0] * 33 / pixels
>= 95) {
121 for (c
= 1; c
< 8; c
++) {
122 errors
+= stats
[c
] * (c
* c
);
123 if (stats
[c
] == 0 || stats
[c
] > stats
[c
-1] * 2) {
127 for (; c
< 256; c
++) {
128 errors
+= stats
[c
] * (c
* c
);
130 errors
/= (pixels
* 3 - stats
[0]);
135 #define DEFINE_DETECT_FUNCTION(bpp) \
138 tight_detect_smooth_image##bpp(VncState *vs, int w, int h) { \
141 int max[3], shift[3]; \
146 int sample, sum, left[3]; \
148 unsigned char *buf = vs->tight.buffer; \
150 endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
151 (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); \
154 max[0] = vs->clientds.pf.rmax; \
155 max[1] = vs->clientds.pf.gmax; \
156 max[2] = vs->clientds.pf.bmax; \
157 shift[0] = vs->clientds.pf.rshift; \
158 shift[1] = vs->clientds.pf.gshift; \
159 shift[2] = vs->clientds.pf.bshift; \
161 memset(stats, 0, sizeof(stats)); \
164 while (y < h && x < w) { \
165 for (d = 0; d < h - y && \
166 d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; d++) { \
167 pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d]; \
169 pix = bswap_##bpp(pix); \
171 for (c = 0; c < 3; c++) { \
172 left[c] = (int)(pix >> shift[c] & max[c]); \
174 for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; \
176 pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d+dx]; \
178 pix = bswap_##bpp(pix); \
181 for (c = 0; c < 3; c++) { \
182 sample = (int)(pix >> shift[c] & max[c]); \
183 sum += abs(sample - left[c]); \
202 if ((stats[0] + stats[1]) * 100 / pixels >= 90) { \
207 for (c = 1; c < 8; c++) { \
208 errors += stats[c] * (c * c); \
209 if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { \
213 for (; c < 256; c++) { \
214 errors += stats[c] * (c * c); \
216 errors /= (pixels - stats[0]); \
221 DEFINE_DETECT_FUNCTION(16)
222 DEFINE_DETECT_FUNCTION(32)
225 tight_detect_smooth_image(VncState
*vs
, int w
, int h
)
228 int compression
= vs
->tight_compression
;
229 int quality
= vs
->tight_quality
;
231 if (!vs
->vd
->lossy
) {
235 if (ds_get_bytes_per_pixel(vs
->ds
) == 1 ||
236 vs
->clientds
.pf
.bytes_per_pixel
== 1 ||
237 w
< VNC_TIGHT_DETECT_MIN_WIDTH
|| h
< VNC_TIGHT_DETECT_MIN_HEIGHT
) {
241 if (vs
->tight_quality
!= -1) {
242 if (w
* h
< VNC_TIGHT_JPEG_MIN_RECT_SIZE
) {
246 if (w
* h
< tight_conf
[compression
].gradient_min_rect_size
) {
251 if (vs
->clientds
.pf
.bytes_per_pixel
== 4) {
252 if (vs
->tight_pixel24
) {
253 errors
= tight_detect_smooth_image24(vs
, w
, h
);
254 if (vs
->tight_quality
!= -1) {
255 return (errors
< tight_conf
[quality
].jpeg_threshold24
);
257 return (errors
< tight_conf
[compression
].gradient_threshold24
);
259 errors
= tight_detect_smooth_image32(vs
, w
, h
);
262 errors
= tight_detect_smooth_image16(vs
, w
, h
);
265 return (errors
< tight_conf
[quality
].jpeg_threshold
);
267 return (errors
< tight_conf
[compression
].gradient_threshold
);
271 * Code to determine how many different colors used in rectangle.
274 static void tight_palette_rgb2buf(uint32_t rgb
, int bpp
, uint8_t buf
[6])
279 buf
[0] = ((rgb
>> 24) & 0xFF);
280 buf
[1] = ((rgb
>> 16) & 0xFF);
281 buf
[2] = ((rgb
>> 8) & 0xFF);
282 buf
[3] = ((rgb
>> 0) & 0xFF);
283 buf
[4] = ((buf
[0] & 1) == 0) << 3 | ((buf
[1] & 1) == 0) << 2;
284 buf
[4]|= ((buf
[2] & 1) == 0) << 1 | ((buf
[3] & 1) == 0) << 0;
291 buf
[0] = ((rgb
>> 8) & 0xFF);
292 buf
[1] = ((rgb
>> 0) & 0xFF);
293 buf
[2] = ((buf
[0] & 1) == 0) << 1 | ((buf
[1] & 1) == 0) << 0;
299 static uint32_t tight_palette_buf2rgb(int bpp
, const uint8_t *buf
)
304 rgb
|= ((buf
[0] & ~1) | !((buf
[4] >> 3) & 1)) << 24;
305 rgb
|= ((buf
[1] & ~1) | !((buf
[4] >> 2) & 1)) << 16;
306 rgb
|= ((buf
[2] & ~1) | !((buf
[4] >> 1) & 1)) << 8;
307 rgb
|= ((buf
[3] & ~1) | !((buf
[4] >> 0) & 1)) << 0;
310 rgb
|= ((buf
[0] & ~1) | !((buf
[2] >> 1) & 1)) << 8;
311 rgb
|= ((buf
[1] & ~1) | !((buf
[2] >> 0) & 1)) << 0;
317 static int tight_palette_insert(QDict
*palette
, uint32_t rgb
, int bpp
, int max
)
320 int idx
= qdict_size(palette
);
323 tight_palette_rgb2buf(rgb
, bpp
, key
);
324 present
= qdict_haskey(palette
, (char *)key
);
325 if (idx
>= max
&& !present
) {
329 qdict_put(palette
, (char *)key
, qint_from_int(idx
));
331 return qdict_size(palette
);
334 #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \
337 tight_fill_palette##bpp(VncState *vs, int x, int y, \
338 int max, size_t count, \
339 uint32_t *bg, uint32_t *fg, \
340 struct QDict **palette) { \
341 uint##bpp##_t *data; \
342 uint##bpp##_t c0, c1, ci; \
345 data = (uint##bpp##_t *)vs->tight.buffer; \
349 while (i < count && data[i] == c0) \
363 for (i++; i < count; i++) { \
367 } else if (ci == c1) { \
374 *bg = (uint32_t)c0; \
375 *fg = (uint32_t)c1; \
377 *bg = (uint32_t)c1; \
378 *fg = (uint32_t)c0; \
387 *palette = qdict_new(); \
388 tight_palette_insert(*palette, c0, bpp, max); \
389 tight_palette_insert(*palette, c1, bpp, max); \
390 tight_palette_insert(*palette, ci, bpp, max); \
392 for (i++; i < count; i++) { \
393 if (data[i] == ci) { \
397 if (!tight_palette_insert(*palette, (uint32_t)ci, \
404 return qdict_size(*palette); \
407 DEFINE_FILL_PALETTE_FUNCTION(8)
408 DEFINE_FILL_PALETTE_FUNCTION(16)
409 DEFINE_FILL_PALETTE_FUNCTION(32)
411 static int tight_fill_palette(VncState
*vs
, int x
, int y
,
412 size_t count
, uint32_t *bg
, uint32_t *fg
,
413 struct QDict
**palette
)
417 max
= count
/ tight_conf
[vs
->tight_compression
].idx_max_colors_divisor
;
419 count
>= tight_conf
[vs
->tight_compression
].mono_min_rect_size
) {
426 switch(vs
->clientds
.pf
.bytes_per_pixel
) {
428 return tight_fill_palette32(vs
, x
, y
, max
, count
, bg
, fg
, palette
);
430 return tight_fill_palette16(vs
, x
, y
, max
, count
, bg
, fg
, palette
);
433 return tight_fill_palette8(vs
, x
, y
, max
, count
, bg
, fg
, palette
);
438 /* Callback to dump a palette with qdict_iter
439 static void print_palette(const char *key, QObject *obj, void *opaque)
441 uint8_t idx = qint_get_int(qobject_to_qint(obj));
442 uint32_t rgb = tight_palette_buf2rgb(32, (uint8_t *)key);
444 fprintf(stderr, "%.2x ", (unsigned char)*key);
446 fprintf(stderr, "%.2x ", (unsigned char)*key);
448 fprintf(stderr, ": idx: %x rgb: %x\n", idx, rgb);
453 * Converting truecolor samples into palette indices.
455 #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \
458 tight_encode_indexed_rect##bpp(uint8_t *buf, int count, \
459 struct QDict *palette) { \
460 uint##bpp##_t *src; \
466 src = (uint##bpp##_t *) buf; \
468 for (i = 0; i < count; i++) { \
471 while (i < count && *src == rgb) { \
474 tight_palette_rgb2buf(rgb, bpp, key); \
475 if (!qdict_haskey(palette, (char *)key)) { \
477 * Should never happen, but don't break everything \
478 * if it does, use the first color instead \
482 idx = qdict_get_int(palette, (char *)key); \
491 DEFINE_IDX_ENCODE_FUNCTION(16)
492 DEFINE_IDX_ENCODE_FUNCTION(32)
494 #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \
497 tight_encode_mono_rect##bpp(uint8_t *buf, int w, int h, \
498 uint##bpp##_t bg, uint##bpp##_t fg) { \
499 uint##bpp##_t *ptr; \
500 unsigned int value, mask; \
504 ptr = (uint##bpp##_t *) buf; \
505 aligned_width = w - w % 8; \
507 for (y = 0; y < h; y++) { \
508 for (x = 0; x < aligned_width; x += 8) { \
509 for (bg_bits = 0; bg_bits < 8; bg_bits++) { \
510 if (*ptr++ != bg) { \
514 if (bg_bits == 8) { \
518 mask = 0x80 >> bg_bits; \
520 for (bg_bits++; bg_bits < 8; bg_bits++) { \
522 if (*ptr++ != bg) { \
526 *buf++ = (uint8_t)value; \
535 for (; x < w; x++) { \
536 if (*ptr++ != bg) { \
541 *buf++ = (uint8_t)value; \
545 DEFINE_MONO_ENCODE_FUNCTION(8)
546 DEFINE_MONO_ENCODE_FUNCTION(16)
547 DEFINE_MONO_ENCODE_FUNCTION(32)
550 * ``Gradient'' filter for 24-bit color samples.
551 * Should be called only when redMax, greenMax and blueMax are 255.
552 * Color components assumed to be byte-aligned.
556 tight_filter_gradient24(VncState
*vs
, uint8_t *buf
, int w
, int h
)
562 int here
[3], upper
[3], left
[3], upperleft
[3];
566 buf32
= (uint32_t *)buf
;
567 memset(vs
->tight_gradient
.buffer
, 0, w
* 3 * sizeof(int));
569 if ((vs
->clientds
.flags
& QEMU_BIG_ENDIAN_FLAG
) ==
570 (vs
->ds
->surface
->flags
& QEMU_BIG_ENDIAN_FLAG
)) {
571 shift
[0] = vs
->clientds
.pf
.rshift
;
572 shift
[1] = vs
->clientds
.pf
.gshift
;
573 shift
[2] = vs
->clientds
.pf
.bshift
;
575 shift
[0] = 24 - vs
->clientds
.pf
.rshift
;
576 shift
[1] = 24 - vs
->clientds
.pf
.gshift
;
577 shift
[2] = 24 - vs
->clientds
.pf
.bshift
;
580 for (y
= 0; y
< h
; y
++) {
581 for (c
= 0; c
< 3; c
++) {
585 prev
= (int *)vs
->tight_gradient
.buffer
;
586 for (x
= 0; x
< w
; x
++) {
588 for (c
= 0; c
< 3; c
++) {
589 upperleft
[c
] = upper
[c
];
592 here
[c
] = (int)(pix32
>> shift
[c
] & 0xFF);
595 prediction
= left
[c
] + upper
[c
] - upperleft
[c
];
596 if (prediction
< 0) {
598 } else if (prediction
> 0xFF) {
601 *buf
++ = (char)(here
[c
] - prediction
);
609 * ``Gradient'' filter for other color depths.
612 #define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \
615 tight_filter_gradient##bpp(VncState *vs, uint##bpp##_t *buf, \
617 uint##bpp##_t pix, diff; \
620 int max[3], shift[3]; \
621 int here[3], upper[3], left[3], upperleft[3]; \
625 memset (vs->tight_gradient.buffer, 0, w * 3 * sizeof(int)); \
627 endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
628 (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); \
630 max[0] = vs->clientds.pf.rmax; \
631 max[1] = vs->clientds.pf.gmax; \
632 max[2] = vs->clientds.pf.bmax; \
633 shift[0] = vs->clientds.pf.rshift; \
634 shift[1] = vs->clientds.pf.gshift; \
635 shift[2] = vs->clientds.pf.bshift; \
637 for (y = 0; y < h; y++) { \
638 for (c = 0; c < 3; c++) { \
642 prev = (int *)vs->tight_gradient.buffer; \
643 for (x = 0; x < w; x++) { \
646 pix = bswap_##bpp(pix); \
649 for (c = 0; c < 3; c++) { \
650 upperleft[c] = upper[c]; \
653 here[c] = (int)(pix >> shift[c] & max[c]); \
656 prediction = left[c] + upper[c] - upperleft[c]; \
657 if (prediction < 0) { \
659 } else if (prediction > max[c]) { \
660 prediction = max[c]; \
662 diff |= ((here[c] - prediction) & max[c]) \
666 diff = bswap_##bpp(diff); \
673 DEFINE_GRADIENT_FILTER_FUNCTION(16)
674 DEFINE_GRADIENT_FILTER_FUNCTION(32)
677 * Check if a rectangle is all of the same color. If needSameColor is
678 * set to non-zero, then also check that its color equals to the
679 * *colorPtr value. The result is 1 if the test is successfull, and in
680 * that case new color will be stored in *colorPtr.
683 #define DEFINE_CHECK_SOLID_FUNCTION(bpp) \
686 check_solid_tile##bpp(VncState *vs, int x, int y, int w, int h, \
687 uint32_t* color, bool samecolor) \
689 VncDisplay *vd = vs->vd; \
690 uint##bpp##_t *fbptr; \
694 fbptr = (uint##bpp##_t *) \
695 (vd->server->data + y * ds_get_linesize(vs->ds) + \
696 x * ds_get_bytes_per_pixel(vs->ds)); \
699 if (samecolor && (uint32_t)c != *color) { \
703 for (dy = 0; dy < h; dy++) { \
704 for (dx = 0; dx < w; dx++) { \
705 if (c != fbptr[dx]) { \
709 fbptr = (uint##bpp##_t *) \
710 ((uint8_t *)fbptr + ds_get_linesize(vs->ds)); \
713 *color = (uint32_t)c; \
717 DEFINE_CHECK_SOLID_FUNCTION(32)
718 DEFINE_CHECK_SOLID_FUNCTION(16)
719 DEFINE_CHECK_SOLID_FUNCTION(8)
721 static bool check_solid_tile(VncState
*vs
, int x
, int y
, int w
, int h
,
722 uint32_t* color
, bool samecolor
)
724 VncDisplay
*vd
= vs
->vd
;
726 switch(vd
->server
->pf
.bytes_per_pixel
) {
728 return check_solid_tile32(vs
, x
, y
, w
, h
, color
, samecolor
);
730 return check_solid_tile16(vs
, x
, y
, w
, h
, color
, samecolor
);
732 return check_solid_tile8(vs
, x
, y
, w
, h
, color
, samecolor
);
736 static void find_best_solid_area(VncState
*vs
, int x
, int y
, int w
, int h
,
737 uint32_t color
, int *w_ptr
, int *h_ptr
)
741 int w_best
= 0, h_best
= 0;
745 for (dy
= y
; dy
< y
+ h
; dy
+= VNC_TIGHT_MAX_SPLIT_TILE_SIZE
) {
747 dh
= MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE
, y
+ h
- dy
);
748 dw
= MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE
, w_prev
);
750 if (!check_solid_tile(vs
, x
, dy
, dw
, dh
, &color
, true)) {
754 for (dx
= x
+ dw
; dx
< x
+ w_prev
;) {
755 dw
= MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE
, x
+ w_prev
- dx
);
757 if (!check_solid_tile(vs
, dx
, dy
, dw
, dh
, &color
, true)) {
764 if (w_prev
* (dy
+ dh
- y
) > w_best
* h_best
) {
766 h_best
= dy
+ dh
- y
;
774 static void extend_solid_area(VncState
*vs
, int x
, int y
, int w
, int h
,
775 uint32_t color
, int *x_ptr
, int *y_ptr
,
776 int *w_ptr
, int *h_ptr
)
780 /* Try to extend the area upwards. */
781 for ( cy
= *y_ptr
- 1;
782 cy
>= y
&& check_solid_tile(vs
, *x_ptr
, cy
, *w_ptr
, 1, &color
, true);
784 *h_ptr
+= *y_ptr
- (cy
+ 1);
788 for ( cy
= *y_ptr
+ *h_ptr
;
790 check_solid_tile(vs
, *x_ptr
, cy
, *w_ptr
, 1, &color
, true);
792 *h_ptr
+= cy
- (*y_ptr
+ *h_ptr
);
794 /* ... to the left. */
795 for ( cx
= *x_ptr
- 1;
796 cx
>= x
&& check_solid_tile(vs
, cx
, *y_ptr
, 1, *h_ptr
, &color
, true);
798 *w_ptr
+= *x_ptr
- (cx
+ 1);
801 /* ... to the right. */
802 for ( cx
= *x_ptr
+ *w_ptr
;
804 check_solid_tile(vs
, cx
, *y_ptr
, 1, *h_ptr
, &color
, true);
806 *w_ptr
+= cx
- (*x_ptr
+ *w_ptr
);
809 static int tight_init_stream(VncState
*vs
, int stream_id
,
810 int level
, int strategy
)
812 z_streamp zstream
= &vs
->tight_stream
[stream_id
];
814 if (zstream
->opaque
== NULL
) {
817 VNC_DEBUG("VNC: TIGHT: initializing zlib stream %d\n", stream_id
);
818 VNC_DEBUG("VNC: TIGHT: opaque = %p | vs = %p\n", zstream
->opaque
, vs
);
819 zstream
->zalloc
= vnc_zlib_zalloc
;
820 zstream
->zfree
= vnc_zlib_zfree
;
822 err
= deflateInit2(zstream
, level
, Z_DEFLATED
, MAX_WBITS
,
823 MAX_MEM_LEVEL
, strategy
);
826 fprintf(stderr
, "VNC: error initializing zlib\n");
830 vs
->tight_levels
[stream_id
] = level
;
831 zstream
->opaque
= vs
;
834 if (vs
->tight_levels
[stream_id
] != level
) {
835 if (deflateParams(zstream
, level
, strategy
) != Z_OK
) {
838 vs
->tight_levels
[stream_id
] = level
;
843 static void tight_send_compact_size(VncState
*vs
, size_t len
)
847 char buf
[3] = {0, 0, 0};
849 buf
[bytes
++] = len
& 0x7F;
851 buf
[bytes
-1] |= 0x80;
852 buf
[bytes
++] = (len
>> 7) & 0x7F;
854 buf
[bytes
-1] |= 0x80;
855 buf
[bytes
++] = (len
>> 14) & 0xFF;
858 for (lpc
= 0; lpc
< bytes
; lpc
++) {
859 vnc_write_u8(vs
, buf
[lpc
]);
863 static int tight_compress_data(VncState
*vs
, int stream_id
, size_t bytes
,
864 int level
, int strategy
)
866 z_streamp zstream
= &vs
->tight_stream
[stream_id
];
869 if (bytes
< VNC_TIGHT_MIN_TO_COMPRESS
) {
870 vnc_write(vs
, vs
->tight
.buffer
, vs
->tight
.offset
);
874 if (tight_init_stream(vs
, stream_id
, level
, strategy
)) {
878 /* reserve memory in output buffer */
879 buffer_reserve(&vs
->tight_zlib
, bytes
+ 64);
882 zstream
->next_in
= vs
->tight
.buffer
;
883 zstream
->avail_in
= vs
->tight
.offset
;
884 zstream
->next_out
= vs
->tight_zlib
.buffer
+ vs
->tight_zlib
.offset
;
885 zstream
->avail_out
= vs
->tight_zlib
.capacity
- vs
->tight_zlib
.offset
;
886 zstream
->data_type
= Z_BINARY
;
887 previous_out
= zstream
->total_out
;
890 if (deflate(zstream
, Z_SYNC_FLUSH
) != Z_OK
) {
891 fprintf(stderr
, "VNC: error during tight compression\n");
895 vs
->tight_zlib
.offset
= vs
->tight_zlib
.capacity
- zstream
->avail_out
;
896 bytes
= zstream
->total_out
- previous_out
;
898 tight_send_compact_size(vs
, bytes
);
899 vnc_write(vs
, vs
->tight_zlib
.buffer
, bytes
);
901 buffer_reset(&vs
->tight_zlib
);
907 * Subencoding implementations.
909 static void tight_pack24(VncState
*vs
, uint8_t *buf
, size_t count
, size_t *ret
)
913 int rshift
, gshift
, bshift
;
915 buf32
= (uint32_t *)buf
;
917 if ((vs
->clientds
.flags
& QEMU_BIG_ENDIAN_FLAG
) ==
918 (vs
->ds
->surface
->flags
& QEMU_BIG_ENDIAN_FLAG
)) {
919 rshift
= vs
->clientds
.pf
.rshift
;
920 gshift
= vs
->clientds
.pf
.gshift
;
921 bshift
= vs
->clientds
.pf
.bshift
;
923 rshift
= 24 - vs
->clientds
.pf
.rshift
;
924 gshift
= 24 - vs
->clientds
.pf
.gshift
;
925 bshift
= 24 - vs
->clientds
.pf
.bshift
;
934 *buf
++ = (char)(pix
>> rshift
);
935 *buf
++ = (char)(pix
>> gshift
);
936 *buf
++ = (char)(pix
>> bshift
);
940 static int send_full_color_rect(VncState
*vs
, int w
, int h
)
945 vnc_write_u8(vs
, stream
<< 4); /* no flushing, no filter */
947 if (vs
->tight_pixel24
) {
948 tight_pack24(vs
, vs
->tight
.buffer
, w
* h
, &vs
->tight
.offset
);
951 bytes
= vs
->clientds
.pf
.bytes_per_pixel
;
954 bytes
= tight_compress_data(vs
, stream
, w
* h
* bytes
,
955 tight_conf
[vs
->tight_compression
].raw_zlib_level
,
961 static int send_solid_rect(VncState
*vs
)
965 vnc_write_u8(vs
, VNC_TIGHT_FILL
<< 4); /* no flushing, no filter */
967 if (vs
->tight_pixel24
) {
968 tight_pack24(vs
, vs
->tight
.buffer
, 1, &vs
->tight
.offset
);
971 bytes
= vs
->clientds
.pf
.bytes_per_pixel
;
974 vnc_write(vs
, vs
->tight
.buffer
, bytes
);
978 static int send_mono_rect(VncState
*vs
, int w
, int h
, uint32_t bg
, uint32_t fg
)
982 int level
= tight_conf
[vs
->tight_compression
].mono_zlib_level
;
984 bytes
= ((w
+ 7) / 8) * h
;
986 vnc_write_u8(vs
, (stream
| VNC_TIGHT_EXPLICIT_FILTER
) << 4);
987 vnc_write_u8(vs
, VNC_TIGHT_FILTER_PALETTE
);
990 switch(vs
->clientds
.pf
.bytes_per_pixel
) {
993 uint32_t buf
[2] = {bg
, fg
};
994 size_t ret
= sizeof (buf
);
996 if (vs
->tight_pixel24
) {
997 tight_pack24(vs
, (unsigned char*)buf
, 2, &ret
);
999 vnc_write(vs
, buf
, ret
);
1001 tight_encode_mono_rect32(vs
->tight
.buffer
, w
, h
, bg
, fg
);
1005 vnc_write(vs
, &bg
, 2);
1006 vnc_write(vs
, &fg
, 2);
1007 tight_encode_mono_rect16(vs
->tight
.buffer
, w
, h
, bg
, fg
);
1010 vnc_write_u8(vs
, bg
);
1011 vnc_write_u8(vs
, fg
);
1012 tight_encode_mono_rect8(vs
->tight
.buffer
, w
, h
, bg
, fg
);
1015 vs
->tight
.offset
= bytes
;
1017 bytes
= tight_compress_data(vs
, stream
, bytes
, level
, Z_DEFAULT_STRATEGY
);
1018 return (bytes
>= 0);
1021 struct palette_cb_priv
{
1026 static void write_palette(const char *key
, QObject
*obj
, void *opaque
)
1028 struct palette_cb_priv
*priv
= opaque
;
1029 VncState
*vs
= priv
->vs
;
1030 uint32_t bytes
= vs
->clientds
.pf
.bytes_per_pixel
;
1031 uint8_t idx
= qint_get_int(qobject_to_qint(obj
));
1034 uint32_t color
= tight_palette_buf2rgb(32, (uint8_t *)key
);
1036 ((uint32_t*)priv
->header
)[idx
] = color
;
1038 uint16_t color
= tight_palette_buf2rgb(16, (uint8_t *)key
);
1040 ((uint16_t*)priv
->header
)[idx
] = color
;
1044 static bool send_gradient_rect(VncState
*vs
, int w
, int h
)
1047 int level
= tight_conf
[vs
->tight_compression
].gradient_zlib_level
;
1050 if (vs
->clientds
.pf
.bytes_per_pixel
== 1)
1051 return send_full_color_rect(vs
, w
, h
);
1053 vnc_write_u8(vs
, (stream
| VNC_TIGHT_EXPLICIT_FILTER
) << 4);
1054 vnc_write_u8(vs
, VNC_TIGHT_FILTER_GRADIENT
);
1056 buffer_reserve(&vs
->tight_gradient
, w
* 3 * sizeof (int));
1058 if (vs
->tight_pixel24
) {
1059 tight_filter_gradient24(vs
, vs
->tight
.buffer
, w
, h
);
1061 } else if (vs
->clientds
.pf
.bytes_per_pixel
== 4) {
1062 tight_filter_gradient32(vs
, (uint32_t *)vs
->tight
.buffer
, w
, h
);
1065 tight_filter_gradient16(vs
, (uint16_t *)vs
->tight
.buffer
, w
, h
);
1069 buffer_reset(&vs
->tight_gradient
);
1071 bytes
= w
* h
* bytes
;
1072 vs
->tight
.offset
= bytes
;
1074 bytes
= tight_compress_data(vs
, stream
, bytes
,
1076 return (bytes
>= 0);
1079 static int send_palette_rect(VncState
*vs
, int w
, int h
, struct QDict
*palette
)
1082 int level
= tight_conf
[vs
->tight_compression
].idx_zlib_level
;
1086 colors
= qdict_size(palette
);
1088 vnc_write_u8(vs
, (stream
| VNC_TIGHT_EXPLICIT_FILTER
) << 4);
1089 vnc_write_u8(vs
, VNC_TIGHT_FILTER_PALETTE
);
1090 vnc_write_u8(vs
, colors
- 1);
1092 switch(vs
->clientds
.pf
.bytes_per_pixel
) {
1095 size_t old_offset
, offset
;
1096 uint32_t header
[qdict_size(palette
)];
1097 struct palette_cb_priv priv
= { vs
, (uint8_t *)header
};
1099 old_offset
= vs
->output
.offset
;
1100 qdict_iter(palette
, write_palette
, &priv
);
1101 vnc_write(vs
, header
, sizeof(header
));
1103 if (vs
->tight_pixel24
) {
1104 tight_pack24(vs
, vs
->output
.buffer
+ old_offset
, colors
, &offset
);
1105 vs
->output
.offset
= old_offset
+ offset
;
1108 tight_encode_indexed_rect32(vs
->tight
.buffer
, w
* h
, palette
);
1113 uint16_t header
[qdict_size(palette
)];
1114 struct palette_cb_priv priv
= { vs
, (uint8_t *)header
};
1116 qdict_iter(palette
, write_palette
, &priv
);
1117 vnc_write(vs
, header
, sizeof(header
));
1118 tight_encode_indexed_rect16(vs
->tight
.buffer
, w
* h
, palette
);
1122 return -1; /* No palette for 8bits colors */
1126 vs
->tight
.offset
= bytes
;
1128 bytes
= tight_compress_data(vs
, stream
, bytes
,
1129 level
, Z_DEFAULT_STRATEGY
);
1130 return (bytes
>= 0);
1134 * JPEG compression stuff.
1136 #ifdef CONFIG_VNC_JPEG
1137 static void jpeg_prepare_row24(VncState
*vs
, uint8_t *dst
, int x
, int y
,
1140 VncDisplay
*vd
= vs
->vd
;
1144 fbptr
= (uint32_t *)(vd
->server
->data
+ y
* ds_get_linesize(vs
->ds
) +
1145 x
* ds_get_bytes_per_pixel(vs
->ds
));
1149 *dst
++ = (uint8_t)(pix
>> vs
->ds
->surface
->pf
.rshift
);
1150 *dst
++ = (uint8_t)(pix
>> vs
->ds
->surface
->pf
.gshift
);
1151 *dst
++ = (uint8_t)(pix
>> vs
->ds
->surface
->pf
.bshift
);
1155 #define DEFINE_JPEG_GET_ROW_FUNCTION(bpp) \
1158 jpeg_prepare_row##bpp(VncState *vs, uint8_t *dst, \
1159 int x, int y, int count) \
1161 VncDisplay *vd = vs->vd; \
1162 uint##bpp##_t *fbptr; \
1163 uint##bpp##_t pix; \
1166 fbptr = (uint##bpp##_t *) \
1167 (vd->server->data + y * ds_get_linesize(vs->ds) + \
1168 x * ds_get_bytes_per_pixel(vs->ds)); \
1173 r = (int)((pix >> vs->ds->surface->pf.rshift) \
1174 & vs->ds->surface->pf.rmax); \
1175 g = (int)((pix >> vs->ds->surface->pf.gshift) \
1176 & vs->ds->surface->pf.gmax); \
1177 b = (int)((pix >> vs->ds->surface->pf.bshift) \
1178 & vs->ds->surface->pf.bmax); \
1180 *dst++ = (uint8_t)((r * 255 + vs->ds->surface->pf.rmax / 2) \
1181 / vs->ds->surface->pf.rmax); \
1182 *dst++ = (uint8_t)((g * 255 + vs->ds->surface->pf.gmax / 2) \
1183 / vs->ds->surface->pf.gmax); \
1184 *dst++ = (uint8_t)((b * 255 + vs->ds->surface->pf.bmax / 2) \
1185 / vs->ds->surface->pf.bmax); \
1189 DEFINE_JPEG_GET_ROW_FUNCTION(16)
1190 DEFINE_JPEG_GET_ROW_FUNCTION(32)
1192 static void jpeg_prepare_row(VncState
*vs
, uint8_t *dst
, int x
, int y
,
1195 if (vs
->tight_pixel24
)
1196 jpeg_prepare_row24(vs
, dst
, x
, y
, count
);
1197 else if (ds_get_bytes_per_pixel(vs
->ds
) == 4)
1198 jpeg_prepare_row32(vs
, dst
, x
, y
, count
);
1200 jpeg_prepare_row16(vs
, dst
, x
, y
, count
);
1204 * Destination manager implementation for JPEG library.
1207 /* This is called once per encoding */
1208 static void jpeg_init_destination(j_compress_ptr cinfo
)
1210 VncState
*vs
= cinfo
->client_data
;
1211 Buffer
*buffer
= &vs
->tight_jpeg
;
1213 cinfo
->dest
->next_output_byte
= (JOCTET
*)buffer
->buffer
+ buffer
->offset
;
1214 cinfo
->dest
->free_in_buffer
= (size_t)(buffer
->capacity
- buffer
->offset
);
1217 /* This is called when we ran out of buffer (shouldn't happen!) */
1218 static boolean
jpeg_empty_output_buffer(j_compress_ptr cinfo
)
1220 VncState
*vs
= cinfo
->client_data
;
1221 Buffer
*buffer
= &vs
->tight_jpeg
;
1223 buffer
->offset
= buffer
->capacity
;
1224 buffer_reserve(buffer
, 2048);
1225 jpeg_init_destination(cinfo
);
1229 /* This is called when we are done processing data */
1230 static void jpeg_term_destination(j_compress_ptr cinfo
)
1232 VncState
*vs
= cinfo
->client_data
;
1233 Buffer
*buffer
= &vs
->tight_jpeg
;
1235 buffer
->offset
= buffer
->capacity
- cinfo
->dest
->free_in_buffer
;
1238 static int send_jpeg_rect(VncState
*vs
, int x
, int y
, int w
, int h
, int quality
)
1240 struct jpeg_compress_struct cinfo
;
1241 struct jpeg_error_mgr jerr
;
1242 struct jpeg_destination_mgr manager
;
1247 if (ds_get_bytes_per_pixel(vs
->ds
) == 1)
1248 return send_full_color_rect(vs
, w
, h
);
1250 buf
= qemu_malloc(w
* 3);
1252 buffer_reserve(&vs
->tight_jpeg
, 2048);
1254 cinfo
.err
= jpeg_std_error(&jerr
);
1255 jpeg_create_compress(&cinfo
);
1257 cinfo
.client_data
= vs
;
1258 cinfo
.image_width
= w
;
1259 cinfo
.image_height
= h
;
1260 cinfo
.input_components
= 3;
1261 cinfo
.in_color_space
= JCS_RGB
;
1263 jpeg_set_defaults(&cinfo
);
1264 jpeg_set_quality(&cinfo
, quality
, true);
1266 manager
.init_destination
= jpeg_init_destination
;
1267 manager
.empty_output_buffer
= jpeg_empty_output_buffer
;
1268 manager
.term_destination
= jpeg_term_destination
;
1269 cinfo
.dest
= &manager
;
1271 jpeg_start_compress(&cinfo
, true);
1273 for (dy
= 0; dy
< h
; dy
++) {
1274 jpeg_prepare_row(vs
, buf
, x
, y
+ dy
, w
);
1275 jpeg_write_scanlines(&cinfo
, row
, 1);
1278 jpeg_finish_compress(&cinfo
);
1279 jpeg_destroy_compress(&cinfo
);
1281 vnc_write_u8(vs
, VNC_TIGHT_JPEG
<< 4);
1283 tight_send_compact_size(vs
, vs
->tight_jpeg
.offset
);
1284 vnc_write(vs
, vs
->tight_jpeg
.buffer
, vs
->tight_jpeg
.offset
);
1285 buffer_reset(&vs
->tight_jpeg
);
1289 #endif /* CONFIG_VNC_JPEG */
1291 static void vnc_tight_start(VncState
*vs
)
1293 buffer_reset(&vs
->tight
);
1295 // make the output buffer be the zlib buffer, so we can compress it later
1296 vs
->tight_tmp
= vs
->output
;
1297 vs
->output
= vs
->tight
;
1300 static void vnc_tight_stop(VncState
*vs
)
1302 // switch back to normal output/zlib buffers
1303 vs
->tight
= vs
->output
;
1304 vs
->output
= vs
->tight_tmp
;
1307 static int send_sub_rect(VncState
*vs
, int x
, int y
, int w
, int h
)
1309 struct QDict
*palette
= NULL
;
1310 uint32_t bg
= 0, fg
= 0;
1314 vnc_framebuffer_update(vs
, x
, y
, w
, h
, VNC_ENCODING_TIGHT
);
1316 vnc_tight_start(vs
);
1317 vnc_raw_send_framebuffer_update(vs
, x
, y
, w
, h
);
1320 colors
= tight_fill_palette(vs
, x
, y
, w
* h
, &fg
, &bg
, &palette
);
1323 if (tight_detect_smooth_image(vs
, w
, h
)) {
1324 if (vs
->tight_quality
== -1) {
1325 ret
= send_gradient_rect(vs
, w
, h
);
1327 #ifdef CONFIG_VNC_JPEG
1328 int quality
= tight_conf
[vs
->tight_quality
].jpeg_quality
;
1330 ret
= send_jpeg_rect(vs
, x
, y
, w
, h
, quality
);
1332 ret
= send_full_color_rect(vs
, w
, h
);
1336 ret
= send_full_color_rect(vs
, w
, h
);
1338 } else if (colors
== 1) {
1339 ret
= send_solid_rect(vs
);
1340 } else if (colors
== 2) {
1341 ret
= send_mono_rect(vs
, w
, h
, bg
, fg
);
1342 } else if (colors
<= 256) {
1343 #ifdef CONFIG_VNC_JPEG
1344 if (colors
> 96 && vs
->tight_quality
!= -1 && vs
->tight_quality
<= 3 &&
1345 tight_detect_smooth_image(vs
, w
, h
)) {
1346 int quality
= tight_conf
[vs
->tight_quality
].jpeg_quality
;
1348 ret
= send_jpeg_rect(vs
, x
, y
, w
, h
, quality
);
1350 ret
= send_palette_rect(vs
, w
, h
, palette
);
1353 ret
= send_palette_rect(vs
, w
, h
, palette
);
1360 static int send_sub_rect_solid(VncState
*vs
, int x
, int y
, int w
, int h
)
1362 vnc_framebuffer_update(vs
, x
, y
, w
, h
, VNC_ENCODING_TIGHT
);
1364 vnc_tight_start(vs
);
1365 vnc_raw_send_framebuffer_update(vs
, x
, y
, w
, h
);
1368 return send_solid_rect(vs
);
1371 static int send_rect_simple(VncState
*vs
, int x
, int y
, int w
, int h
)
1373 int max_size
, max_width
;
1374 int max_sub_width
, max_sub_height
;
1379 max_size
= tight_conf
[vs
->tight_compression
].max_rect_size
;
1380 max_width
= tight_conf
[vs
->tight_compression
].max_rect_width
;
1382 if (w
> max_width
|| w
* h
> max_size
) {
1383 max_sub_width
= (w
> max_width
) ? max_width
: w
;
1384 max_sub_height
= max_size
/ max_sub_width
;
1386 for (dy
= 0; dy
< h
; dy
+= max_sub_height
) {
1387 for (dx
= 0; dx
< w
; dx
+= max_width
) {
1388 rw
= MIN(max_sub_width
, w
- dx
);
1389 rh
= MIN(max_sub_height
, h
- dy
);
1390 n
+= send_sub_rect(vs
, x
+dx
, y
+dy
, rw
, rh
);
1394 n
+= send_sub_rect(vs
, x
, y
, w
, h
);
1400 static int find_large_solid_color_rect(VncState
*vs
, int x
, int y
,
1401 int w
, int h
, int max_rows
)
1406 /* Try to find large solid-color areas and send them separately. */
1408 for (dy
= y
; dy
< y
+ h
; dy
+= VNC_TIGHT_MAX_SPLIT_TILE_SIZE
) {
1410 /* If a rectangle becomes too large, send its upper part now. */
1412 if (dy
- y
>= max_rows
) {
1413 n
+= send_rect_simple(vs
, x
, y
, w
, max_rows
);
1418 dh
= MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE
, (y
+ h
- dy
));
1420 for (dx
= x
; dx
< x
+ w
; dx
+= VNC_TIGHT_MAX_SPLIT_TILE_SIZE
) {
1421 uint32_t color_value
;
1422 int x_best
, y_best
, w_best
, h_best
;
1424 dw
= MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE
, (x
+ w
- dx
));
1426 if (!check_solid_tile(vs
, dx
, dy
, dw
, dh
, &color_value
, false)) {
1430 /* Get dimensions of solid-color area. */
1432 find_best_solid_area(vs
, dx
, dy
, w
- (dx
- x
), h
- (dy
- y
),
1433 color_value
, &w_best
, &h_best
);
1435 /* Make sure a solid rectangle is large enough
1436 (or the whole rectangle is of the same color). */
1438 if (w_best
* h_best
!= w
* h
&&
1439 w_best
* h_best
< VNC_TIGHT_MIN_SOLID_SUBRECT_SIZE
) {
1443 /* Try to extend solid rectangle to maximum size. */
1445 x_best
= dx
; y_best
= dy
;
1446 extend_solid_area(vs
, x
, y
, w
, h
, color_value
,
1447 &x_best
, &y_best
, &w_best
, &h_best
);
1449 /* Send rectangles at top and left to solid-color area. */
1452 n
+= send_rect_simple(vs
, x
, y
, w
, y_best
-y
);
1455 n
+= vnc_tight_send_framebuffer_update(vs
, x
, y_best
,
1459 /* Send solid-color rectangle. */
1460 n
+= send_sub_rect_solid(vs
, x_best
, y_best
, w_best
, h_best
);
1462 /* Send remaining rectangles (at right and bottom). */
1464 if (x_best
+ w_best
!= x
+ w
) {
1465 n
+= vnc_tight_send_framebuffer_update(vs
, x_best
+w_best
,
1467 w
-(x_best
-x
)-w_best
,
1470 if (y_best
+ h_best
!= y
+ h
) {
1471 n
+= vnc_tight_send_framebuffer_update(vs
, x
, y_best
+h_best
,
1472 w
, h
-(y_best
-y
)-h_best
);
1475 /* Return after all recursive calls are done. */
1479 return n
+ send_rect_simple(vs
, x
, y
, w
, h
);
1482 int vnc_tight_send_framebuffer_update(VncState
*vs
, int x
, int y
,
1487 if (vs
->clientds
.pf
.bytes_per_pixel
== 4 && vs
->clientds
.pf
.rmax
== 0xFF &&
1488 vs
->clientds
.pf
.bmax
== 0xFF && vs
->clientds
.pf
.gmax
== 0xFF) {
1489 vs
->tight_pixel24
= true;
1491 vs
->tight_pixel24
= false;
1494 if (w
* h
< VNC_TIGHT_MIN_SPLIT_RECT_SIZE
)
1495 return send_rect_simple(vs
, x
, y
, w
, h
);
1497 /* Calculate maximum number of rows in one non-solid rectangle. */
1499 max_rows
= tight_conf
[vs
->tight_compression
].max_rect_size
;
1500 max_rows
/= MIN(tight_conf
[vs
->tight_compression
].max_rect_width
, w
);
1502 return find_large_solid_color_rect(vs
, x
, y
, w
, h
, max_rows
);
1505 void vnc_tight_clear(VncState
*vs
)
1508 for (i
=0; i
<ARRAY_SIZE(vs
->tight_stream
); i
++) {
1509 if (vs
->tight_stream
[i
].opaque
) {
1510 deflateEnd(&vs
->tight_stream
[i
]);
1514 buffer_free(&vs
->tight
);
1515 buffer_free(&vs
->tight_zlib
);
1516 buffer_free(&vs
->tight_gradient
);
1517 #ifdef CONFIG_VNC_JPEG
1518 buffer_free(&vs
->tight_jpeg
);