?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
monitor: fix use after free
[qemu/ar7.git] / ui / vnc-enc-tight.c
blobf02352cc466b5acf65f06318022f328ac3d2a018
1 /*
2 * QEMU VNC display driver: tight encoding
3 *
4 * From libvncserver/libvncserver/tight.c
5 * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved.
6 * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
7 *
8 * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com>
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
27 */
28
29 #include "config-host.h"
30
31 /* This needs to be before jpeglib.h line because of conflict with
32 INT32 definitions between jmorecfg.h (included by jpeglib.h) and
33 Win32 basetsd.h (included by windows.h). */
34 #include "qemu-common.h"
35
36 #ifdef CONFIG_VNC_PNG
37 /* The following define is needed by pngconf.h. Otherwise it won't compile,
38 because setjmp.h was already included by qemu-common.h. */
39 #define PNG_SKIP_SETJMP_CHECK
40 #include <png.h>
41 #endif
42 #ifdef CONFIG_VNC_JPEG
43 #include <stdio.h>
44 #include <jpeglib.h>
45 #endif
46
47 #include "qemu/bswap.h"
48 #include "qapi/qmp/qint.h"
49 #include "vnc.h"
50 #include "vnc-enc-tight.h"
51 #include "vnc-palette.h"
52
53 /* Compression level stuff. The following array contains various
54 encoder parameters for each of 10 compression levels (0..9).
55 Last three parameters correspond to JPEG quality levels (0..9). */
56
57 static const struct {
58 int max_rect_size, max_rect_width;
59 int mono_min_rect_size, gradient_min_rect_size;
60 int idx_zlib_level, mono_zlib_level, raw_zlib_level, gradient_zlib_level;
61 int gradient_threshold, gradient_threshold24;
62 int idx_max_colors_divisor;
63 int jpeg_quality, jpeg_threshold, jpeg_threshold24;
64 } tight_conf[] = {
65 { 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 5, 10000, 23000 },
66 { 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 10, 8000, 18000 },
67 { 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 15, 6500, 15000 },
68 { 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 25, 5000, 12000 },
69 { 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 37, 4000, 10000 },
70 { 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 50, 3000, 8000 },
71 { 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 60, 2000, 5000 },
72 { 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 },
73 { 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 },
74 { 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 }
75 };
76
77
78 static int tight_send_framebuffer_update(VncState *vs, int x, int y,
79 int w, int h);
80
81 #ifdef CONFIG_VNC_JPEG
82 static const struct {
83 double jpeg_freq_min; /* Don't send JPEG if the freq is bellow */
84 double jpeg_freq_threshold; /* Always send JPEG if the freq is above */
85 int jpeg_idx; /* Allow indexed JPEG */
86 int jpeg_full; /* Allow full color JPEG */
87 } tight_jpeg_conf[] = {
88 { 0, 8, 1, 1 },
89 { 0, 8, 1, 1 },
90 { 0, 8, 1, 1 },
91 { 0, 8, 1, 1 },
92 { 0, 10, 1, 1 },
93 { 0.1, 10, 1, 1 },
94 { 0.2, 10, 1, 1 },
95 { 0.3, 12, 0, 0 },
96 { 0.4, 14, 0, 0 },
97 { 0.5, 16, 0, 0 },
98 };
99 #endif
100
101 #ifdef CONFIG_VNC_PNG
102 static const struct {
103 int png_zlib_level, png_filters;
104 } tight_png_conf[] = {
105 { 0, PNG_NO_FILTERS },
106 { 1, PNG_NO_FILTERS },
107 { 2, PNG_NO_FILTERS },
108 { 3, PNG_NO_FILTERS },
109 { 4, PNG_NO_FILTERS },
110 { 5, PNG_ALL_FILTERS },
111 { 6, PNG_ALL_FILTERS },
112 { 7, PNG_ALL_FILTERS },
113 { 8, PNG_ALL_FILTERS },
114 { 9, PNG_ALL_FILTERS },
115 };
116
117 static int send_png_rect(VncState *vs, int x, int y, int w, int h,
118 VncPalette *palette);
119
120 static bool tight_can_send_png_rect(VncState *vs, int w, int h)
121 {
122 if (vs->tight.type != VNC_ENCODING_TIGHT_PNG) {
123 return false;
124 }
125
126 if (surface_bytes_per_pixel(vs->vd->ds) == 1 ||
127 vs->client_pf.bytes_per_pixel == 1) {
128 return false;
129 }
130
131 return true;
132 }
133 #endif
134
135 /*
136 * Code to guess if given rectangle is suitable for smooth image
137 * compression (by applying "gradient" filter or JPEG coder).
138 */
139
140 static unsigned int
141 tight_detect_smooth_image24(VncState *vs, int w, int h)
142 {
143 int off;
144 int x, y, d, dx;
145 unsigned int c;
146 unsigned int stats[256];
147 int pixels = 0;
148 int pix, left[3];
149 unsigned int errors;
150 unsigned char *buf = vs->tight.tight.buffer;
151
152 /*
153 * If client is big-endian, color samples begin from the second
154 * byte (offset 1) of a 32-bit pixel value.
155 */
156 off = vs->client_be;
157
158 memset(stats, 0, sizeof (stats));
159
160 for (y = 0, x = 0; y < h && x < w;) {
161 for (d = 0; d < h - y && d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH;
162 d++) {
163 for (c = 0; c < 3; c++) {
164 left[c] = buf[((y+d)*w+x+d)*4+off+c] & 0xFF;
165 }
166 for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; dx++) {
167 for (c = 0; c < 3; c++) {
168 pix = buf[((y+d)*w+x+d+dx)*4+off+c] & 0xFF;
169 stats[abs(pix - left[c])]++;
170 left[c] = pix;
171 }
172 pixels++;
173 }
174 }
175 if (w > h) {
176 x += h;
177 y = 0;
178 } else {
179 x = 0;
180 y += w;
181 }
182 }
183
184 if (pixels == 0) {
185 return 0;
186 }
187
188 /* 95% smooth or more ... */
189 if (stats[0] * 33 / pixels >= 95) {
190 return 0;
191 }
192
193 errors = 0;
194 for (c = 1; c < 8; c++) {
195 errors += stats[c] * (c * c);
196 if (stats[c] == 0 || stats[c] > stats[c-1] * 2) {
197 return 0;
198 }
199 }
200 for (; c < 256; c++) {
201 errors += stats[c] * (c * c);
202 }
203 errors /= (pixels * 3 - stats[0]);
204
205 return errors;
206 }
207
208 #define DEFINE_DETECT_FUNCTION(bpp) \
209 \
210 static unsigned int \
211 tight_detect_smooth_image##bpp(VncState *vs, int w, int h) { \
212 bool endian; \
213 uint##bpp##_t pix; \
214 int max[3], shift[3]; \
215 int x, y, d, dx; \
216 unsigned int c; \
217 unsigned int stats[256]; \
218 int pixels = 0; \
219 int sample, sum, left[3]; \
220 unsigned int errors; \
221 unsigned char *buf = vs->tight.tight.buffer; \
222 \
223 endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
224 (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
225 \
226 \
227 max[0] = vs->client_pf.rmax; \
228 max[1] = vs->client_pf.gmax; \
229 max[2] = vs->client_pf.bmax; \
230 shift[0] = vs->client_pf.rshift; \
231 shift[1] = vs->client_pf.gshift; \
232 shift[2] = vs->client_pf.bshift; \
233 \
234 memset(stats, 0, sizeof(stats)); \
235 \
236 y = 0, x = 0; \
237 while (y < h && x < w) { \
238 for (d = 0; d < h - y && \
239 d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; d++) { \
240 pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d]; \
241 if (endian) { \
242 pix = bswap##bpp(pix); \
243 } \
244 for (c = 0; c < 3; c++) { \
245 left[c] = (int)(pix >> shift[c] & max[c]); \
246 } \
247 for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; \
248 dx++) { \
249 pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d+dx]; \
250 if (endian) { \
251 pix = bswap##bpp(pix); \
252 } \
253 sum = 0; \
254 for (c = 0; c < 3; c++) { \
255 sample = (int)(pix >> shift[c] & max[c]); \
256 sum += abs(sample - left[c]); \
257 left[c] = sample; \
258 } \
259 if (sum > 255) { \
260 sum = 255; \
261 } \
262 stats[sum]++; \
263 pixels++; \
264 } \
265 } \
266 if (w > h) { \
267 x += h; \
268 y = 0; \
269 } else { \
270 x = 0; \
271 y += w; \
272 } \
273 } \
274 if (pixels == 0) { \
275 return 0; \
276 } \
277 if ((stats[0] + stats[1]) * 100 / pixels >= 90) { \
278 return 0; \
279 } \
280 \
281 errors = 0; \
282 for (c = 1; c < 8; c++) { \
283 errors += stats[c] * (c * c); \
284 if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { \
285 return 0; \
286 } \
287 } \
288 for (; c < 256; c++) { \
289 errors += stats[c] * (c * c); \
290 } \
291 errors /= (pixels - stats[0]); \
292 \
293 return errors; \
294 }
295
296 DEFINE_DETECT_FUNCTION(16)
297 DEFINE_DETECT_FUNCTION(32)
298
299 static int
300 tight_detect_smooth_image(VncState *vs, int w, int h)
301 {
302 unsigned int errors;
303 int compression = vs->tight.compression;
304 int quality = vs->tight.quality;
305
306 if (!vs->vd->lossy) {
307 return 0;
308 }
309
310 if (surface_bytes_per_pixel(vs->vd->ds) == 1 ||
311 vs->client_pf.bytes_per_pixel == 1 ||
312 w < VNC_TIGHT_DETECT_MIN_WIDTH || h < VNC_TIGHT_DETECT_MIN_HEIGHT) {
313 return 0;
314 }
315
316 if (vs->tight.quality != (uint8_t)-1) {
317 if (w * h < VNC_TIGHT_JPEG_MIN_RECT_SIZE) {
318 return 0;
319 }
320 } else {
321 if (w * h < tight_conf[compression].gradient_min_rect_size) {
322 return 0;
323 }
324 }
325
326 if (vs->client_pf.bytes_per_pixel == 4) {
327 if (vs->tight.pixel24) {
328 errors = tight_detect_smooth_image24(vs, w, h);
329 if (vs->tight.quality != (uint8_t)-1) {
330 return (errors < tight_conf[quality].jpeg_threshold24);
331 }
332 return (errors < tight_conf[compression].gradient_threshold24);
333 } else {
334 errors = tight_detect_smooth_image32(vs, w, h);
335 }
336 } else {
337 errors = tight_detect_smooth_image16(vs, w, h);
338 }
339 if (quality != (uint8_t)-1) {
340 return (errors < tight_conf[quality].jpeg_threshold);
341 }
342 return (errors < tight_conf[compression].gradient_threshold);
343 }
344
345 /*
346 * Code to determine how many different colors used in rectangle.
347 */
348 #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \
349 \
350 static int \
351 tight_fill_palette##bpp(VncState *vs, int x, int y, \
352 int max, size_t count, \
353 uint32_t *bg, uint32_t *fg, \
354 VncPalette **palette) { \
355 uint##bpp##_t *data; \
356 uint##bpp##_t c0, c1, ci; \
357 int i, n0, n1; \
358 \
359 data = (uint##bpp##_t *)vs->tight.tight.buffer; \
360 \
361 c0 = data[0]; \
362 i = 1; \
363 while (i < count && data[i] == c0) \
364 i++; \
365 if (i >= count) { \
366 *bg = *fg = c0; \
367 return 1; \
368 } \
369 \
370 if (max < 2) { \
371 return 0; \
372 } \
373 \
374 n0 = i; \
375 c1 = data[i]; \
376 n1 = 0; \
377 for (i++; i < count; i++) { \
378 ci = data[i]; \
379 if (ci == c0) { \
380 n0++; \
381 } else if (ci == c1) { \
382 n1++; \
383 } else \
384 break; \
385 } \
386 if (i >= count) { \
387 if (n0 > n1) { \
388 *bg = (uint32_t)c0; \
389 *fg = (uint32_t)c1; \
390 } else { \
391 *bg = (uint32_t)c1; \
392 *fg = (uint32_t)c0; \
393 } \
394 return 2; \
395 } \
396 \
397 if (max == 2) { \
398 return 0; \
399 } \
400 \
401 *palette = palette_new(max, bpp); \
402 palette_put(*palette, c0); \
403 palette_put(*palette, c1); \
404 palette_put(*palette, ci); \
405 \
406 for (i++; i < count; i++) { \
407 if (data[i] == ci) { \
408 continue; \
409 } else { \
410 ci = data[i]; \
411 if (!palette_put(*palette, (uint32_t)ci)) { \
412 return 0; \
413 } \
414 } \
415 } \
416 \
417 return palette_size(*palette); \
418 }
419
420 DEFINE_FILL_PALETTE_FUNCTION(8)
421 DEFINE_FILL_PALETTE_FUNCTION(16)
422 DEFINE_FILL_PALETTE_FUNCTION(32)
423
424 static int tight_fill_palette(VncState *vs, int x, int y,
425 size_t count, uint32_t *bg, uint32_t *fg,
426 VncPalette **palette)
427 {
428 int max;
429
430 max = count / tight_conf[vs->tight.compression].idx_max_colors_divisor;
431 if (max < 2 &&
432 count >= tight_conf[vs->tight.compression].mono_min_rect_size) {
433 max = 2;
434 }
435 if (max >= 256) {
436 max = 256;
437 }
438
439 switch (vs->client_pf.bytes_per_pixel) {
440 case 4:
441 return tight_fill_palette32(vs, x, y, max, count, bg, fg, palette);
442 case 2:
443 return tight_fill_palette16(vs, x, y, max, count, bg, fg, palette);
444 default:
445 max = 2;
446 return tight_fill_palette8(vs, x, y, max, count, bg, fg, palette);
447 }
448 return 0;
449 }
450
451 /*
452 * Converting truecolor samples into palette indices.
453 */
454 #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \
455 \
456 static void \
457 tight_encode_indexed_rect##bpp(uint8_t *buf, int count, \
458 VncPalette *palette) { \
459 uint##bpp##_t *src; \
460 uint##bpp##_t rgb; \
461 int i, rep; \
462 uint8_t idx; \
463 \
464 src = (uint##bpp##_t *) buf; \
465 \
466 for (i = 0; i < count; i++) { \
467 \
468 rgb = *src++; \
469 rep = 0; \
470 while (i < count && *src == rgb) { \
471 rep++, src++, i++; \
472 } \
473 idx = palette_idx(palette, rgb); \
474 /* \
475 * Should never happen, but don't break everything \
476 * if it does, use the first color instead \
477 */ \
478 if (idx == (uint8_t)-1) { \
479 idx = 0; \
480 } \
481 while (rep >= 0) { \
482 *buf++ = idx; \
483 rep--; \
484 } \
485 } \
486 }
487
488 DEFINE_IDX_ENCODE_FUNCTION(16)
489 DEFINE_IDX_ENCODE_FUNCTION(32)
490
491 #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \
492 \
493 static void \
494 tight_encode_mono_rect##bpp(uint8_t *buf, int w, int h, \
495 uint##bpp##_t bg, uint##bpp##_t fg) { \
496 uint##bpp##_t *ptr; \
497 unsigned int value, mask; \
498 int aligned_width; \
499 int x, y, bg_bits; \
500 \
501 ptr = (uint##bpp##_t *) buf; \
502 aligned_width = w - w % 8; \
503 \
504 for (y = 0; y < h; y++) { \
505 for (x = 0; x < aligned_width; x += 8) { \
506 for (bg_bits = 0; bg_bits < 8; bg_bits++) { \
507 if (*ptr++ != bg) { \
508 break; \
509 } \
510 } \
511 if (bg_bits == 8) { \
512 *buf++ = 0; \
513 continue; \
514 } \
515 mask = 0x80 >> bg_bits; \
516 value = mask; \
517 for (bg_bits++; bg_bits < 8; bg_bits++) { \
518 mask >>= 1; \
519 if (*ptr++ != bg) { \
520 value |= mask; \
521 } \
522 } \
523 *buf++ = (uint8_t)value; \
524 } \
525 \
526 mask = 0x80; \
527 value = 0; \
528 if (x >= w) { \
529 continue; \
530 } \
531 \
532 for (; x < w; x++) { \
533 if (*ptr++ != bg) { \
534 value |= mask; \
535 } \
536 mask >>= 1; \
537 } \
538 *buf++ = (uint8_t)value; \
539 } \
540 }
541
542 DEFINE_MONO_ENCODE_FUNCTION(8)
543 DEFINE_MONO_ENCODE_FUNCTION(16)
544 DEFINE_MONO_ENCODE_FUNCTION(32)
545
546 /*
547 * ``Gradient'' filter for 24-bit color samples.
548 * Should be called only when redMax, greenMax and blueMax are 255.
549 * Color components assumed to be byte-aligned.
550 */
551
552 static void
553 tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
554 {
555 uint32_t *buf32;
556 uint32_t pix32;
557 int shift[3];
558 int *prev;
559 int here[3], upper[3], left[3], upperleft[3];
560 int prediction;
561 int x, y, c;
562
563 buf32 = (uint32_t *)buf;
564 memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));
565
566 if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
567 (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
568 shift[0] = vs->client_pf.rshift;
569 shift[1] = vs->client_pf.gshift;
570 shift[2] = vs->client_pf.bshift;
571 } else {
572 shift[0] = 24 - vs->client_pf.rshift;
573 shift[1] = 24 - vs->client_pf.gshift;
574 shift[2] = 24 - vs->client_pf.bshift;
575 }
576
577 for (y = 0; y < h; y++) {
578 for (c = 0; c < 3; c++) {
579 upper[c] = 0;
580 here[c] = 0;
581 }
582 prev = (int *)vs->tight.gradient.buffer;
583 for (x = 0; x < w; x++) {
584 pix32 = *buf32++;
585 for (c = 0; c < 3; c++) {
586 upperleft[c] = upper[c];
587 left[c] = here[c];
588 upper[c] = *prev;
589 here[c] = (int)(pix32 >> shift[c] & 0xFF);
590 *prev++ = here[c];
591
592 prediction = left[c] + upper[c] - upperleft[c];
593 if (prediction < 0) {
594 prediction = 0;
595 } else if (prediction > 0xFF) {
596 prediction = 0xFF;
597 }
598 *buf++ = (char)(here[c] - prediction);
599 }
600 }
601 }
602 }
603
604
605 /*
606 * ``Gradient'' filter for other color depths.
607 */
608
609 #define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \
610 \
611 static void \
612 tight_filter_gradient##bpp(VncState *vs, uint##bpp##_t *buf, \
613 int w, int h) { \
614 uint##bpp##_t pix, diff; \
615 bool endian; \
616 int *prev; \
617 int max[3], shift[3]; \
618 int here[3], upper[3], left[3], upperleft[3]; \
619 int prediction; \
620 int x, y, c; \
621 \
622 memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int)); \
623 \
624 endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
625 (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
626 \
627 max[0] = vs->client_pf.rmax; \
628 max[1] = vs->client_pf.gmax; \
629 max[2] = vs->client_pf.bmax; \
630 shift[0] = vs->client_pf.rshift; \
631 shift[1] = vs->client_pf.gshift; \
632 shift[2] = vs->client_pf.bshift; \
633 \
634 for (y = 0; y < h; y++) { \
635 for (c = 0; c < 3; c++) { \
636 upper[c] = 0; \
637 here[c] = 0; \
638 } \
639 prev = (int *)vs->tight.gradient.buffer; \
640 for (x = 0; x < w; x++) { \
641 pix = *buf; \
642 if (endian) { \
643 pix = bswap##bpp(pix); \
644 } \
645 diff = 0; \
646 for (c = 0; c < 3; c++) { \
647 upperleft[c] = upper[c]; \
648 left[c] = here[c]; \
649 upper[c] = *prev; \
650 here[c] = (int)(pix >> shift[c] & max[c]); \
651 *prev++ = here[c]; \
652 \
653 prediction = left[c] + upper[c] - upperleft[c]; \
654 if (prediction < 0) { \
655 prediction = 0; \
656 } else if (prediction > max[c]) { \
657 prediction = max[c]; \
658 } \
659 diff |= ((here[c] - prediction) & max[c]) \
660 << shift[c]; \
661 } \
662 if (endian) { \
663 diff = bswap##bpp(diff); \
664 } \
665 *buf++ = diff; \
666 } \
667 } \
668 }
669
670 DEFINE_GRADIENT_FILTER_FUNCTION(16)
671 DEFINE_GRADIENT_FILTER_FUNCTION(32)
672
673 /*
674 * Check if a rectangle is all of the same color. If needSameColor is
675 * set to non-zero, then also check that its color equals to the
676 * *colorPtr value. The result is 1 if the test is successful, and in
677 * that case new color will be stored in *colorPtr.
678 */
679
680 static bool
681 check_solid_tile32(VncState *vs, int x, int y, int w, int h,
682 uint32_t *color, bool samecolor)
683 {
684 VncDisplay *vd = vs->vd;
685 uint32_t *fbptr;
686 uint32_t c;
687 int dx, dy;
688
689 fbptr = vnc_server_fb_ptr(vd, x, y);
690
691 c = *fbptr;
692 if (samecolor && (uint32_t)c != *color) {
693 return false;
694 }
695
696 for (dy = 0; dy < h; dy++) {
697 for (dx = 0; dx < w; dx++) {
698 if (c != fbptr[dx]) {
699 return false;
700 }
701 }
702 fbptr = (uint32_t *)
703 ((uint8_t *)fbptr + vnc_server_fb_stride(vd));
704 }
705
706 *color = (uint32_t)c;
707 return true;
708 }
709
710 static bool check_solid_tile(VncState *vs, int x, int y, int w, int h,
711 uint32_t* color, bool samecolor)
712 {
713 switch (VNC_SERVER_FB_BYTES) {
714 case 4:
715 return check_solid_tile32(vs, x, y, w, h, color, samecolor);
716 }
717 }
718
719 static void find_best_solid_area(VncState *vs, int x, int y, int w, int h,
720 uint32_t color, int *w_ptr, int *h_ptr)
721 {
722 int dx, dy, dw, dh;
723 int w_prev;
724 int w_best = 0, h_best = 0;
725
726 w_prev = w;
727
728 for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) {
729
730 dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, y + h - dy);
731 dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, w_prev);
732
733 if (!check_solid_tile(vs, x, dy, dw, dh, &color, true)) {
734 break;
735 }
736
737 for (dx = x + dw; dx < x + w_prev;) {
738 dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, x + w_prev - dx);
739
740 if (!check_solid_tile(vs, dx, dy, dw, dh, &color, true)) {
741 break;
742 }
743 dx += dw;
744 }
745
746 w_prev = dx - x;
747 if (w_prev * (dy + dh - y) > w_best * h_best) {
748 w_best = w_prev;
749 h_best = dy + dh - y;
750 }
751 }
752
753 *w_ptr = w_best;
754 *h_ptr = h_best;
755 }
756
757 static void extend_solid_area(VncState *vs, int x, int y, int w, int h,
758 uint32_t color, int *x_ptr, int *y_ptr,
759 int *w_ptr, int *h_ptr)
760 {
761 int cx, cy;
762
763 /* Try to extend the area upwards. */
764 for ( cy = *y_ptr - 1;
765 cy >= y && check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true);
766 cy-- );
767 *h_ptr += *y_ptr - (cy + 1);
768 *y_ptr = cy + 1;
769
770 /* ... downwards. */
771 for ( cy = *y_ptr + *h_ptr;
772 cy < y + h &&
773 check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true);
774 cy++ );
775 *h_ptr += cy - (*y_ptr + *h_ptr);
776
777 /* ... to the left. */
778 for ( cx = *x_ptr - 1;
779 cx >= x && check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true);
780 cx-- );
781 *w_ptr += *x_ptr - (cx + 1);
782 *x_ptr = cx + 1;
783
784 /* ... to the right. */
785 for ( cx = *x_ptr + *w_ptr;
786 cx < x + w &&
787 check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true);
788 cx++ );
789 *w_ptr += cx - (*x_ptr + *w_ptr);
790 }
791
792 static int tight_init_stream(VncState *vs, int stream_id,
793 int level, int strategy)
794 {
795 z_streamp zstream = &vs->tight.stream[stream_id];
796
797 if (zstream->opaque == NULL) {
798 int err;
799
800 VNC_DEBUG("VNC: TIGHT: initializing zlib stream %d\n", stream_id);
801 VNC_DEBUG("VNC: TIGHT: opaque = %p | vs = %p\n", zstream->opaque, vs);
802 zstream->zalloc = vnc_zlib_zalloc;
803 zstream->zfree = vnc_zlib_zfree;
804
805 err = deflateInit2(zstream, level, Z_DEFLATED, MAX_WBITS,
806 MAX_MEM_LEVEL, strategy);
807
808 if (err != Z_OK) {
809 fprintf(stderr, "VNC: error initializing zlib\n");
810 return -1;
811 }
812
813 vs->tight.levels[stream_id] = level;
814 zstream->opaque = vs;
815 }
816
817 if (vs->tight.levels[stream_id] != level) {
818 if (deflateParams(zstream, level, strategy) != Z_OK) {
819 return -1;
820 }
821 vs->tight.levels[stream_id] = level;
822 }
823 return 0;
824 }
825
826 static void tight_send_compact_size(VncState *vs, size_t len)
827 {
828 int lpc = 0;
829 int bytes = 0;
830 char buf[3] = {0, 0, 0};
831
832 buf[bytes++] = len & 0x7F;
833 if (len > 0x7F) {
834 buf[bytes-1] |= 0x80;
835 buf[bytes++] = (len >> 7) & 0x7F;
836 if (len > 0x3FFF) {
837 buf[bytes-1] |= 0x80;
838 buf[bytes++] = (len >> 14) & 0xFF;
839 }
840 }
841 for (lpc = 0; lpc < bytes; lpc++) {
842 vnc_write_u8(vs, buf[lpc]);
843 }
844 }
845
846 static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,
847 int level, int strategy)
848 {
849 z_streamp zstream = &vs->tight.stream[stream_id];
850 int previous_out;
851
852 if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) {
853 vnc_write(vs, vs->tight.tight.buffer, vs->tight.tight.offset);
854 return bytes;
855 }
856
857 if (tight_init_stream(vs, stream_id, level, strategy)) {
858 return -1;
859 }
860
861 /* reserve memory in output buffer */
862 buffer_reserve(&vs->tight.zlib, bytes + 64);
863
864 /* set pointers */
865 zstream->next_in = vs->tight.tight.buffer;
866 zstream->avail_in = vs->tight.tight.offset;
867 zstream->next_out = vs->tight.zlib.buffer + vs->tight.zlib.offset;
868 zstream->avail_out = vs->tight.zlib.capacity - vs->tight.zlib.offset;
869 previous_out = zstream->avail_out;
870 zstream->data_type = Z_BINARY;
871
872 /* start encoding */
873 if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) {
874 fprintf(stderr, "VNC: error during tight compression\n");
875 return -1;
876 }
877
878 vs->tight.zlib.offset = vs->tight.zlib.capacity - zstream->avail_out;
879 /* ...how much data has actually been produced by deflate() */
880 bytes = previous_out - zstream->avail_out;
881
882 tight_send_compact_size(vs, bytes);
883 vnc_write(vs, vs->tight.zlib.buffer, bytes);
884
885 buffer_reset(&vs->tight.zlib);
886
887 return bytes;
888 }
889
890 /*
891 * Subencoding implementations.
892 */
893 static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret)
894 {
895 uint32_t *buf32;
896 uint32_t pix;
897 int rshift, gshift, bshift;
898
899 buf32 = (uint32_t *)buf;
900
901 if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
902 (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
903 rshift = vs->client_pf.rshift;
904 gshift = vs->client_pf.gshift;
905 bshift = vs->client_pf.bshift;
906 } else {
907 rshift = 24 - vs->client_pf.rshift;
908 gshift = 24 - vs->client_pf.gshift;
909 bshift = 24 - vs->client_pf.bshift;
910 }
911
912 if (ret) {
913 *ret = count * 3;
914 }
915
916 while (count--) {
917 pix = *buf32++;
918 *buf++ = (char)(pix >> rshift);
919 *buf++ = (char)(pix >> gshift);
920 *buf++ = (char)(pix >> bshift);
921 }
922 }
923
924 static int send_full_color_rect(VncState *vs, int x, int y, int w, int h)
925 {
926 int stream = 0;
927 ssize_t bytes;
928
929 #ifdef CONFIG_VNC_PNG
930 if (tight_can_send_png_rect(vs, w, h)) {
931 return send_png_rect(vs, x, y, w, h, NULL);
932 }
933 #endif
934
935 vnc_write_u8(vs, stream << 4); /* no flushing, no filter */
936
937 if (vs->tight.pixel24) {
938 tight_pack24(vs, vs->tight.tight.buffer, w * h, &vs->tight.tight.offset);
939 bytes = 3;
940 } else {
941 bytes = vs->client_pf.bytes_per_pixel;
942 }
943
944 bytes = tight_compress_data(vs, stream, w * h * bytes,
945 tight_conf[vs->tight.compression].raw_zlib_level,
946 Z_DEFAULT_STRATEGY);
947
948 return (bytes >= 0);
949 }
950
951 static int send_solid_rect(VncState *vs)
952 {
953 size_t bytes;
954
955 vnc_write_u8(vs, VNC_TIGHT_FILL << 4); /* no flushing, no filter */
956
957 if (vs->tight.pixel24) {
958 tight_pack24(vs, vs->tight.tight.buffer, 1, &vs->tight.tight.offset);
959 bytes = 3;
960 } else {
961 bytes = vs->client_pf.bytes_per_pixel;
962 }
963
964 vnc_write(vs, vs->tight.tight.buffer, bytes);
965 return 1;
966 }
967
968 static int send_mono_rect(VncState *vs, int x, int y,
969 int w, int h, uint32_t bg, uint32_t fg)
970 {
971 ssize_t bytes;
972 int stream = 1;
973 int level = tight_conf[vs->tight.compression].mono_zlib_level;
974
975 #ifdef CONFIG_VNC_PNG
976 if (tight_can_send_png_rect(vs, w, h)) {
977 int ret;
978 int bpp = vs->client_pf.bytes_per_pixel * 8;
979 VncPalette *palette = palette_new(2, bpp);
980
981 palette_put(palette, bg);
982 palette_put(palette, fg);
983 ret = send_png_rect(vs, x, y, w, h, palette);
984 palette_destroy(palette);
985 return ret;
986 }
987 #endif
988
989 bytes = ((w + 7) / 8) * h;
990
991 vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
992 vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
993 vnc_write_u8(vs, 1);
994
995 switch (vs->client_pf.bytes_per_pixel) {
996 case 4:
997 {
998 uint32_t buf[2] = {bg, fg};
999 size_t ret = sizeof (buf);
1000
1001 if (vs->tight.pixel24) {
1002 tight_pack24(vs, (unsigned char*)buf, 2, &ret);
1003 }
1004 vnc_write(vs, buf, ret);
1005
1006 tight_encode_mono_rect32(vs->tight.tight.buffer, w, h, bg, fg);
1007 break;
1008 }
1009 case 2:
1010 vnc_write(vs, &bg, 2);
1011 vnc_write(vs, &fg, 2);
1012 tight_encode_mono_rect16(vs->tight.tight.buffer, w, h, bg, fg);
1013 break;
1014 default:
1015 vnc_write_u8(vs, bg);
1016 vnc_write_u8(vs, fg);
1017 tight_encode_mono_rect8(vs->tight.tight.buffer, w, h, bg, fg);
1018 break;
1019 }
1020 vs->tight.tight.offset = bytes;
1021
1022 bytes = tight_compress_data(vs, stream, bytes, level, Z_DEFAULT_STRATEGY);
1023 return (bytes >= 0);
1024 }
1025
1026 struct palette_cb_priv {
1027 VncState *vs;
1028 uint8_t *header;
1029 #ifdef CONFIG_VNC_PNG
1030 png_colorp png_palette;
1031 #endif
1032 };
1033
1034 static void write_palette(int idx, uint32_t color, void *opaque)
1035 {
1036 struct palette_cb_priv *priv = opaque;
1037 VncState *vs = priv->vs;
1038 uint32_t bytes = vs->client_pf.bytes_per_pixel;
1039
1040 if (bytes == 4) {
1041 ((uint32_t*)priv->header)[idx] = color;
1042 } else {
1043 ((uint16_t*)priv->header)[idx] = color;
1044 }
1045 }
1046
1047 static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
1048 {
1049 int stream = 3;
1050 int level = tight_conf[vs->tight.compression].gradient_zlib_level;
1051 ssize_t bytes;
1052
1053 if (vs->client_pf.bytes_per_pixel == 1) {
1054 return send_full_color_rect(vs, x, y, w, h);
1055 }
1056
1057 vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
1058 vnc_write_u8(vs, VNC_TIGHT_FILTER_GRADIENT);
1059
1060 buffer_reserve(&vs->tight.gradient, w * 3 * sizeof (int));
1061
1062 if (vs->tight.pixel24) {
1063 tight_filter_gradient24(vs, vs->tight.tight.buffer, w, h);
1064 bytes = 3;
1065 } else if (vs->client_pf.bytes_per_pixel == 4) {
1066 tight_filter_gradient32(vs, (uint32_t *)vs->tight.tight.buffer, w, h);
1067 bytes = 4;
1068 } else {
1069 tight_filter_gradient16(vs, (uint16_t *)vs->tight.tight.buffer, w, h);
1070 bytes = 2;
1071 }
1072
1073 buffer_reset(&vs->tight.gradient);
1074
1075 bytes = w * h * bytes;
1076 vs->tight.tight.offset = bytes;
1077
1078 bytes = tight_compress_data(vs, stream, bytes,
1079 level, Z_FILTERED);
1080 return (bytes >= 0);
1081 }
1082
1083 static int send_palette_rect(VncState *vs, int x, int y,
1084 int w, int h, VncPalette *palette)
1085 {
1086 int stream = 2;
1087 int level = tight_conf[vs->tight.compression].idx_zlib_level;
1088 int colors;
1089 ssize_t bytes;
1090
1091 #ifdef CONFIG_VNC_PNG
1092 if (tight_can_send_png_rect(vs, w, h)) {
1093 return send_png_rect(vs, x, y, w, h, palette);
1094 }
1095 #endif
1096
1097 colors = palette_size(palette);
1098
1099 vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
1100 vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
1101 vnc_write_u8(vs, colors - 1);
1102
1103 switch (vs->client_pf.bytes_per_pixel) {
1104 case 4:
1105 {
1106 size_t old_offset, offset;
1107 uint32_t header[palette_size(palette)];
1108 struct palette_cb_priv priv = { vs, (uint8_t *)header };
1109
1110 old_offset = vs->output.offset;
1111 palette_iter(palette, write_palette, &priv);
1112 vnc_write(vs, header, sizeof(header));
1113
1114 if (vs->tight.pixel24) {
1115 tight_pack24(vs, vs->output.buffer + old_offset, colors, &offset);
1116 vs->output.offset = old_offset + offset;
1117 }
1118
1119 tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette);
1120 break;
1121 }
1122 case 2:
1123 {
1124 uint16_t header[palette_size(palette)];
1125 struct palette_cb_priv priv = { vs, (uint8_t *)header };
1126
1127 palette_iter(palette, write_palette, &priv);
1128 vnc_write(vs, header, sizeof(header));
1129 tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette);
1130 break;
1131 }
1132 default:
1133 return -1; /* No palette for 8bits colors */
1134 break;
1135 }
1136 bytes = w * h;
1137 vs->tight.tight.offset = bytes;
1138
1139 bytes = tight_compress_data(vs, stream, bytes,
1140 level, Z_DEFAULT_STRATEGY);
1141 return (bytes >= 0);
1142 }
1143
1144 /*
1145 * JPEG compression stuff.
1146 */
1147 #ifdef CONFIG_VNC_JPEG
1148 /*
1149 * Destination manager implementation for JPEG library.
1150 */
1151
1152 /* This is called once per encoding */
1153 static void jpeg_init_destination(j_compress_ptr cinfo)
1154 {
1155 VncState *vs = cinfo->client_data;
1156 Buffer *buffer = &vs->tight.jpeg;
1157
1158 cinfo->dest->next_output_byte = (JOCTET *)buffer->buffer + buffer->offset;
1159 cinfo->dest->free_in_buffer = (size_t)(buffer->capacity - buffer->offset);
1160 }
1161
1162 /* This is called when we ran out of buffer (shouldn't happen!) */
1163 static boolean jpeg_empty_output_buffer(j_compress_ptr cinfo)
1164 {
1165 VncState *vs = cinfo->client_data;
1166 Buffer *buffer = &vs->tight.jpeg;
1167
1168 buffer->offset = buffer->capacity;
1169 buffer_reserve(buffer, 2048);
1170 jpeg_init_destination(cinfo);
1171 return TRUE;
1172 }
1173
1174 /* This is called when we are done processing data */
1175 static void jpeg_term_destination(j_compress_ptr cinfo)
1176 {
1177 VncState *vs = cinfo->client_data;
1178 Buffer *buffer = &vs->tight.jpeg;
1179
1180 buffer->offset = buffer->capacity - cinfo->dest->free_in_buffer;
1181 }
1182
1183 static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
1184 {
1185 struct jpeg_compress_struct cinfo;
1186 struct jpeg_error_mgr jerr;
1187 struct jpeg_destination_mgr manager;
1188 pixman_image_t *linebuf;
1189 JSAMPROW row[1];
1190 uint8_t *buf;
1191 int dy;
1192
1193 if (surface_bytes_per_pixel(vs->vd->ds) == 1) {
1194 return send_full_color_rect(vs, x, y, w, h);
1195 }
1196
1197 buffer_reserve(&vs->tight.jpeg, 2048);
1198
1199 cinfo.err = jpeg_std_error(&jerr);
1200 jpeg_create_compress(&cinfo);
1201
1202 cinfo.client_data = vs;
1203 cinfo.image_width = w;
1204 cinfo.image_height = h;
1205 cinfo.input_components = 3;
1206 cinfo.in_color_space = JCS_RGB;
1207
1208 jpeg_set_defaults(&cinfo);
1209 jpeg_set_quality(&cinfo, quality, true);
1210
1211 manager.init_destination = jpeg_init_destination;
1212 manager.empty_output_buffer = jpeg_empty_output_buffer;
1213 manager.term_destination = jpeg_term_destination;
1214 cinfo.dest = &manager;
1215
1216 jpeg_start_compress(&cinfo, true);
1217
1218 linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
1219 buf = (uint8_t *)pixman_image_get_data(linebuf);
1220 row[0] = buf;
1221 for (dy = 0; dy < h; dy++) {
1222 qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
1223 jpeg_write_scanlines(&cinfo, row, 1);
1224 }
1225 qemu_pixman_image_unref(linebuf);
1226
1227 jpeg_finish_compress(&cinfo);
1228 jpeg_destroy_compress(&cinfo);
1229
1230 vnc_write_u8(vs, VNC_TIGHT_JPEG << 4);
1231
1232 tight_send_compact_size(vs, vs->tight.jpeg.offset);
1233 vnc_write(vs, vs->tight.jpeg.buffer, vs->tight.jpeg.offset);
1234 buffer_reset(&vs->tight.jpeg);
1235
1236 return 1;
1237 }
1238 #endif /* CONFIG_VNC_JPEG */
1239
1240 /*
1241 * PNG compression stuff.
1242 */
1243 #ifdef CONFIG_VNC_PNG
1244 static void write_png_palette(int idx, uint32_t pix, void *opaque)
1245 {
1246 struct palette_cb_priv *priv = opaque;
1247 VncState *vs = priv->vs;
1248 png_colorp color = &priv->png_palette[idx];
1249
1250 if (vs->tight.pixel24)
1251 {
1252 color->red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
1253 color->green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
1254 color->blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
1255 }
1256 else
1257 {
1258 int red, green, blue;
1259
1260 red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
1261 green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
1262 blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
1263 color->red = ((red * 255 + vs->client_pf.rmax / 2) /
1264 vs->client_pf.rmax);
1265 color->green = ((green * 255 + vs->client_pf.gmax / 2) /
1266 vs->client_pf.gmax);
1267 color->blue = ((blue * 255 + vs->client_pf.bmax / 2) /
1268 vs->client_pf.bmax);
1269 }
1270 }
1271
1272 static void png_write_data(png_structp png_ptr, png_bytep data,
1273 png_size_t length)
1274 {
1275 VncState *vs = png_get_io_ptr(png_ptr);
1276
1277 buffer_reserve(&vs->tight.png, vs->tight.png.offset + length);
1278 memcpy(vs->tight.png.buffer + vs->tight.png.offset, data, length);
1279
1280 vs->tight.png.offset += length;
1281 }
1282
1283 static void png_flush_data(png_structp png_ptr)
1284 {
1285 }
1286
1287 static void *vnc_png_malloc(png_structp png_ptr, png_size_t size)
1288 {
1289 return g_malloc(size);
1290 }
1291
1292 static void vnc_png_free(png_structp png_ptr, png_voidp ptr)
1293 {
1294 g_free(ptr);
1295 }
1296
1297 static int send_png_rect(VncState *vs, int x, int y, int w, int h,
1298 VncPalette *palette)
1299 {
1300 png_byte color_type;
1301 png_structp png_ptr;
1302 png_infop info_ptr;
1303 png_colorp png_palette = NULL;
1304 pixman_image_t *linebuf;
1305 int level = tight_png_conf[vs->tight.compression].png_zlib_level;
1306 int filters = tight_png_conf[vs->tight.compression].png_filters;
1307 uint8_t *buf;
1308 int dy;
1309
1310 png_ptr = png_create_write_struct_2(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL,
1311 NULL, vnc_png_malloc, vnc_png_free);
1312
1313 if (png_ptr == NULL)
1314 return -1;
1315
1316 info_ptr = png_create_info_struct(png_ptr);
1317
1318 if (info_ptr == NULL) {
1319 png_destroy_write_struct(&png_ptr, NULL);
1320 return -1;
1321 }
1322
1323 png_set_write_fn(png_ptr, (void *) vs, png_write_data, png_flush_data);
1324 png_set_compression_level(png_ptr, level);
1325 png_set_filter(png_ptr, PNG_FILTER_TYPE_DEFAULT, filters);
1326
1327 if (palette) {
1328 color_type = PNG_COLOR_TYPE_PALETTE;
1329 } else {
1330 color_type = PNG_COLOR_TYPE_RGB;
1331 }
1332
1333 png_set_IHDR(png_ptr, info_ptr, w, h,
1334 8, color_type, PNG_INTERLACE_NONE,
1335 PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
1336
1337 if (color_type == PNG_COLOR_TYPE_PALETTE) {
1338 struct palette_cb_priv priv;
1339
1340 png_palette = png_malloc(png_ptr, sizeof(*png_palette) *
1341 palette_size(palette));
1342
1343 priv.vs = vs;
1344 priv.png_palette = png_palette;
1345 palette_iter(palette, write_png_palette, &priv);
1346
1347 png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette));
1348
1349 if (vs->client_pf.bytes_per_pixel == 4) {
1350 tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette);
1351 } else {
1352 tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette);
1353 }
1354 }
1355
1356 png_write_info(png_ptr, info_ptr);
1357
1358 buffer_reserve(&vs->tight.png, 2048);
1359 linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
1360 buf = (uint8_t *)pixman_image_get_data(linebuf);
1361 for (dy = 0; dy < h; dy++)
1362 {
1363 if (color_type == PNG_COLOR_TYPE_PALETTE) {
1364 memcpy(buf, vs->tight.tight.buffer + (dy * w), w);
1365 } else {
1366 qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy);
1367 }
1368 png_write_row(png_ptr, buf);
1369 }
1370 qemu_pixman_image_unref(linebuf);
1371
1372 png_write_end(png_ptr, NULL);
1373
1374 if (color_type == PNG_COLOR_TYPE_PALETTE) {
1375 png_free(png_ptr, png_palette);
1376 }
1377
1378 png_destroy_write_struct(&png_ptr, &info_ptr);
1379
1380 vnc_write_u8(vs, VNC_TIGHT_PNG << 4);
1381
1382 tight_send_compact_size(vs, vs->tight.png.offset);
1383 vnc_write(vs, vs->tight.png.buffer, vs->tight.png.offset);
1384 buffer_reset(&vs->tight.png);
1385 return 1;
1386 }
1387 #endif /* CONFIG_VNC_PNG */
1388
1389 static void vnc_tight_start(VncState *vs)
1390 {
1391 buffer_reset(&vs->tight.tight);
1392
1393 // make the output buffer be the zlib buffer, so we can compress it later
1394 vs->tight.tmp = vs->output;
1395 vs->output = vs->tight.tight;
1396 }
1397
1398 static void vnc_tight_stop(VncState *vs)
1399 {
1400 // switch back to normal output/zlib buffers
1401 vs->tight.tight = vs->output;
1402 vs->output = vs->tight.tmp;
1403 }
1404
1405 static int send_sub_rect_nojpeg(VncState *vs, int x, int y, int w, int h,
1406 int bg, int fg, int colors, VncPalette *palette)
1407 {
1408 int ret;
1409
1410 if (colors == 0) {
1411 if (tight_detect_smooth_image(vs, w, h)) {
1412 ret = send_gradient_rect(vs, x, y, w, h);
1413 } else {
1414 ret = send_full_color_rect(vs, x, y, w, h);
1415 }
1416 } else if (colors == 1) {
1417 ret = send_solid_rect(vs);
1418 } else if (colors == 2) {
1419 ret = send_mono_rect(vs, x, y, w, h, bg, fg);
1420 } else if (colors <= 256) {
1421 ret = send_palette_rect(vs, x, y, w, h, palette);
1422 } else {
1423 ret = 0;
1424 }
1425 return ret;
1426 }
1427
1428 #ifdef CONFIG_VNC_JPEG
1429 static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h,
1430 int bg, int fg, int colors,
1431 VncPalette *palette, bool force)
1432 {
1433 int ret;
1434
1435 if (colors == 0) {
1436 if (force || (tight_jpeg_conf[vs->tight.quality].jpeg_full &&
1437 tight_detect_smooth_image(vs, w, h))) {
1438 int quality = tight_conf[vs->tight.quality].jpeg_quality;
1439
1440 ret = send_jpeg_rect(vs, x, y, w, h, quality);
1441 } else {
1442 ret = send_full_color_rect(vs, x, y, w, h);
1443 }
1444 } else if (colors == 1) {
1445 ret = send_solid_rect(vs);
1446 } else if (colors == 2) {
1447 ret = send_mono_rect(vs, x, y, w, h, bg, fg);
1448 } else if (colors <= 256) {
1449 if (force || (colors > 96 &&
1450 tight_jpeg_conf[vs->tight.quality].jpeg_idx &&
1451 tight_detect_smooth_image(vs, w, h))) {
1452 int quality = tight_conf[vs->tight.quality].jpeg_quality;
1453
1454 ret = send_jpeg_rect(vs, x, y, w, h, quality);
1455 } else {
1456 ret = send_palette_rect(vs, x, y, w, h, palette);
1457 }
1458 } else {
1459 ret = 0;
1460 }
1461 return ret;
1462 }
1463 #endif
1464
1465 static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
1466 {
1467 VncPalette *palette = NULL;
1468 uint32_t bg = 0, fg = 0;
1469 int colors;
1470 int ret = 0;
1471 #ifdef CONFIG_VNC_JPEG
1472 bool force_jpeg = false;
1473 bool allow_jpeg = true;
1474 #endif
1475
1476 vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
1477
1478 vnc_tight_start(vs);
1479 vnc_raw_send_framebuffer_update(vs, x, y, w, h);
1480 vnc_tight_stop(vs);
1481
1482 #ifdef CONFIG_VNC_JPEG
1483 if (!vs->vd->non_adaptive && vs->tight.quality != (uint8_t)-1) {
1484 double freq = vnc_update_freq(vs, x, y, w, h);
1485
1486 if (freq < tight_jpeg_conf[vs->tight.quality].jpeg_freq_min) {
1487 allow_jpeg = false;
1488 }
1489 if (freq >= tight_jpeg_conf[vs->tight.quality].jpeg_freq_threshold) {
1490 force_jpeg = true;
1491 vnc_sent_lossy_rect(vs, x, y, w, h);
1492 }
1493 }
1494 #endif
1495
1496 colors = tight_fill_palette(vs, x, y, w * h, &fg, &bg, &palette);
1497
1498 #ifdef CONFIG_VNC_JPEG
1499 if (allow_jpeg && vs->tight.quality != (uint8_t)-1) {
1500 ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette,
1501 force_jpeg);
1502 } else {
1503 ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
1504 }
1505 #else
1506 ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
1507 #endif
1508
1509 palette_destroy(palette);
1510 return ret;
1511 }
1512
1513 static int send_sub_rect_solid(VncState *vs, int x, int y, int w, int h)
1514 {
1515 vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
1516
1517 vnc_tight_start(vs);
1518 vnc_raw_send_framebuffer_update(vs, x, y, w, h);
1519 vnc_tight_stop(vs);
1520
1521 return send_solid_rect(vs);
1522 }
1523
1524 static int send_rect_simple(VncState *vs, int x, int y, int w, int h,
1525 bool split)
1526 {
1527 int max_size, max_width;
1528 int max_sub_width, max_sub_height;
1529 int dx, dy;
1530 int rw, rh;
1531 int n = 0;
1532
1533 max_size = tight_conf[vs->tight.compression].max_rect_size;
1534 max_width = tight_conf[vs->tight.compression].max_rect_width;
1535
1536 if (split && (w > max_width || w * h > max_size)) {
1537 max_sub_width = (w > max_width) ? max_width : w;
1538 max_sub_height = max_size / max_sub_width;
1539
1540 for (dy = 0; dy < h; dy += max_sub_height) {
1541 for (dx = 0; dx < w; dx += max_width) {
1542 rw = MIN(max_sub_width, w - dx);
1543 rh = MIN(max_sub_height, h - dy);
1544 n += send_sub_rect(vs, x+dx, y+dy, rw, rh);
1545 }
1546 }
1547 } else {
1548 n += send_sub_rect(vs, x, y, w, h);
1549 }
1550
1551 return n;
1552 }
1553
1554 static int find_large_solid_color_rect(VncState *vs, int x, int y,
1555 int w, int h, int max_rows)
1556 {
1557 int dx, dy, dw, dh;
1558 int n = 0;
1559
1560 /* Try to find large solid-color areas and send them separately. */
1561
1562 for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) {
1563
1564 /* If a rectangle becomes too large, send its upper part now. */
1565
1566 if (dy - y >= max_rows) {
1567 n += send_rect_simple(vs, x, y, w, max_rows, true);
1568 y += max_rows;
1569 h -= max_rows;
1570 }
1571
1572 dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (y + h - dy));
1573
1574 for (dx = x; dx < x + w; dx += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) {
1575 uint32_t color_value;
1576 int x_best, y_best, w_best, h_best;
1577
1578 dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (x + w - dx));
1579
1580 if (!check_solid_tile(vs, dx, dy, dw, dh, &color_value, false)) {
1581 continue ;
1582 }
1583
1584 /* Get dimensions of solid-color area. */
1585
1586 find_best_solid_area(vs, dx, dy, w - (dx - x), h - (dy - y),
1587 color_value, &w_best, &h_best);
1588
1589 /* Make sure a solid rectangle is large enough
1590 (or the whole rectangle is of the same color). */
1591
1592 if (w_best * h_best != w * h &&
1593 w_best * h_best < VNC_TIGHT_MIN_SOLID_SUBRECT_SIZE) {
1594 continue;
1595 }
1596
1597 /* Try to extend solid rectangle to maximum size. */
1598
1599 x_best = dx; y_best = dy;
1600 extend_solid_area(vs, x, y, w, h, color_value,
1601 &x_best, &y_best, &w_best, &h_best);
1602
1603 /* Send rectangles at top and left to solid-color area. */
1604
1605 if (y_best != y) {
1606 n += send_rect_simple(vs, x, y, w, y_best-y, true);
1607 }
1608 if (x_best != x) {
1609 n += tight_send_framebuffer_update(vs, x, y_best,
1610 x_best-x, h_best);
1611 }
1612
1613 /* Send solid-color rectangle. */
1614 n += send_sub_rect_solid(vs, x_best, y_best, w_best, h_best);
1615
1616 /* Send remaining rectangles (at right and bottom). */
1617
1618 if (x_best + w_best != x + w) {
1619 n += tight_send_framebuffer_update(vs, x_best+w_best,
1620 y_best,
1621 w-(x_best-x)-w_best,
1622 h_best);
1623 }
1624 if (y_best + h_best != y + h) {
1625 n += tight_send_framebuffer_update(vs, x, y_best+h_best,
1626 w, h-(y_best-y)-h_best);
1627 }
1628
1629 /* Return after all recursive calls are done. */
1630 return n;
1631 }
1632 }
1633 return n + send_rect_simple(vs, x, y, w, h, true);
1634 }
1635
1636 static int tight_send_framebuffer_update(VncState *vs, int x, int y,
1637 int w, int h)
1638 {
1639 int max_rows;
1640
1641 if (vs->client_pf.bytes_per_pixel == 4 && vs->client_pf.rmax == 0xFF &&
1642 vs->client_pf.bmax == 0xFF && vs->client_pf.gmax == 0xFF) {
1643 vs->tight.pixel24 = true;
1644 } else {
1645 vs->tight.pixel24 = false;
1646 }
1647
1648 #ifdef CONFIG_VNC_JPEG
1649 if (vs->tight.quality != (uint8_t)-1) {
1650 double freq = vnc_update_freq(vs, x, y, w, h);
1651
1652 if (freq > tight_jpeg_conf[vs->tight.quality].jpeg_freq_threshold) {
1653 return send_rect_simple(vs, x, y, w, h, false);
1654 }
1655 }
1656 #endif
1657
1658 if (w * h < VNC_TIGHT_MIN_SPLIT_RECT_SIZE) {
1659 return send_rect_simple(vs, x, y, w, h, true);
1660 }
1661
1662 /* Calculate maximum number of rows in one non-solid rectangle. */
1663
1664 max_rows = tight_conf[vs->tight.compression].max_rect_size;
1665 max_rows /= MIN(tight_conf[vs->tight.compression].max_rect_width, w);
1666
1667 return find_large_solid_color_rect(vs, x, y, w, h, max_rows);
1668 }
1669
1670 int vnc_tight_send_framebuffer_update(VncState *vs, int x, int y,
1671 int w, int h)
1672 {
1673 vs->tight.type = VNC_ENCODING_TIGHT;
1674 return tight_send_framebuffer_update(vs, x, y, w, h);
1675 }
1676
1677 int vnc_tight_png_send_framebuffer_update(VncState *vs, int x, int y,
1678 int w, int h)
1679 {
1680 vs->tight.type = VNC_ENCODING_TIGHT_PNG;
1681 return tight_send_framebuffer_update(vs, x, y, w, h);
1682 }
1683
1684 void vnc_tight_clear(VncState *vs)
1685 {
1686 int i;
1687 for (i=0; i<ARRAY_SIZE(vs->tight.stream); i++) {
1688 if (vs->tight.stream[i].opaque) {
1689 deflateEnd(&vs->tight.stream[i]);
1690 }
1691 }
1692
1693 buffer_free(&vs->tight.tight);
1694 buffer_free(&vs->tight.zlib);
1695 buffer_free(&vs->tight.gradient);
1696 #ifdef CONFIG_VNC_JPEG
1697 buffer_free(&vs->tight.jpeg);
1698 #endif
1699 #ifdef CONFIG_VNC_PNG
1700 buffer_free(&vs->tight.png);
1701 #endif
1702 }
AR7 emulation for QEMU