?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add feature 85117: [liboscar/icq] password changing support for ICQ.
[kdenetwork.git] / krfb / libvncserver / tight.c
blobd54f22999a5d8734e75075b8f0fd14172f6048b6
1 /*
2 * tight.c
3 *
4 * Routines to implement Tight Encoding
5 */
6
7 /*
8 * Copyright 2000, 2001 Const Kaplinsky <const@ce.cctpu.edu.ru> All Rights Reserved.
9 * Copyright 1999 AT&T Laboratories Cambridge. All Rights Reserved.
10 *
11 * This is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This software is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this software; if not, write to the Free Software
23 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
24 * USA.
25 */
26
27 /*#include <stdio.h>*/
28 #include "rfb.h"
29
30 #ifdef WIN32
31 #define XMD_H
32 #undef FAR
33 #define NEEDFAR_POINTERS
34 #endif
35
36 #include <jpeglib.h>
37
38 /* Note: The following constant should not be changed. */
39 #define TIGHT_MIN_TO_COMPRESS 12
40
41 /* The parameters below may be adjusted. */
42 #define MIN_SPLIT_RECT_SIZE 4096
43 #define MIN_SOLID_SUBRECT_SIZE 2048
44 #define MAX_SPLIT_TILE_SIZE 16
45
46 /* May be set to TRUE with "-lazytight" Xvnc option. */
47 Bool rfbTightDisableGradient = FALSE;
48
49 /* This variable is set on every rfbSendRectEncodingTight() call. */
50 static Bool usePixelFormat24;
51
52
53 /* Compression level stuff. The following array contains various
54 encoder parameters for each of 10 compression levels (0..9).
55 Last three parameters correspond to JPEG quality levels (0..9). */
56
57 typedef struct TIGHT_CONF_s {
58 int maxRectSize, maxRectWidth;
59 int monoMinRectSize, gradientMinRectSize;
60 int idxZlibLevel, monoZlibLevel, rawZlibLevel, gradientZlibLevel;
61 int gradientThreshold, gradientThreshold24;
62 int idxMaxColorsDivisor;
63 int jpegQuality, jpegThreshold, jpegThreshold24;
64 } TIGHT_CONF;
65
66 static TIGHT_CONF tightConf[10] = {
67 { 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 20, 10000, 23000 },
68 { 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 30, 8000, 18000 },
69 { 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 40, 6500, 15000 },
70 { 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 50, 5000, 12000 },
71 { 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 55, 4000, 10000 },
72 { 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 60, 3000, 8000 },
73 { 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 65, 2000, 5000 },
74 { 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 },
75 { 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 },
76 { 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 }
77 };
78
79 static int compressLevel;
80 static int qualityLevel;
81
82 /* Stuff dealing with palettes. */
83
84 typedef struct COLOR_LIST_s {
85 struct COLOR_LIST_s *next;
86 int idx;
87 CARD32 rgb;
88 } COLOR_LIST;
89
90 typedef struct PALETTE_ENTRY_s {
91 COLOR_LIST *listNode;
92 int numPixels;
93 } PALETTE_ENTRY;
94
95 typedef struct PALETTE_s {
96 PALETTE_ENTRY entry[256];
97 COLOR_LIST *hash[256];
98 COLOR_LIST list[256];
99 } PALETTE;
100
101 static int paletteNumColors, paletteMaxColors;
102 static CARD32 monoBackground, monoForeground;
103 static PALETTE palette;
104
105 /* Pointers to dynamically-allocated buffers. */
106
107 static int tightBeforeBufSize = 0;
108 static char *tightBeforeBuf = NULL;
109
110 static int tightAfterBufSize = 0;
111 static char *tightAfterBuf = NULL;
112
113 static int *prevRowBuf = NULL;
114
115
116 /* Prototypes for static functions. */
117
118 static void FindBestSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
119 CARD32 colorValue, int *w_ptr, int *h_ptr);
120 static void ExtendSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
121 CARD32 colorValue,
122 int *x_ptr, int *y_ptr, int *w_ptr, int *h_ptr);
123 static Bool CheckSolidTile (rfbClientPtr cl, int x, int y, int w, int h,
124 CARD32 *colorPtr, Bool needSameColor);
125 static Bool CheckSolidTile8 (rfbClientPtr cl, int x, int y, int w, int h,
126 CARD32 *colorPtr, Bool needSameColor);
127 static Bool CheckSolidTile16 (rfbClientPtr cl, int x, int y, int w, int h,
128 CARD32 *colorPtr, Bool needSameColor);
129 static Bool CheckSolidTile32 (rfbClientPtr cl, int x, int y, int w, int h,
130 CARD32 *colorPtr, Bool needSameColor);
131
132 static Bool SendRectSimple (rfbClientPtr cl, int x, int y, int w, int h);
133 static Bool SendSubrect (rfbClientPtr cl, int x, int y, int w, int h);
134 static Bool SendTightHeader (rfbClientPtr cl, int x, int y, int w, int h);
135
136 static Bool SendSolidRect (rfbClientPtr cl);
137 static Bool SendMonoRect (rfbClientPtr cl, int w, int h);
138 static Bool SendIndexedRect (rfbClientPtr cl, int w, int h);
139 static Bool SendFullColorRect (rfbClientPtr cl, int w, int h);
140 static Bool SendGradientRect (rfbClientPtr cl, int w, int h);
141
142 static Bool CompressData(rfbClientPtr cl, int streamId, int dataLen,
143 int zlibLevel, int zlibStrategy);
144 static Bool SendCompressedData(rfbClientPtr cl, int compressedLen);
145
146 static void FillPalette8(int count);
147 static void FillPalette16(int count);
148 static void FillPalette32(int count);
149
150 static void PaletteReset(void);
151 static int PaletteInsert(CARD32 rgb, int numPixels, int bpp);
152
153 static void Pack24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int count);
154
155 static void EncodeIndexedRect16(CARD8 *buf, int count);
156 static void EncodeIndexedRect32(CARD8 *buf, int count);
157
158 static void EncodeMonoRect8(CARD8 *buf, int w, int h);
159 static void EncodeMonoRect16(CARD8 *buf, int w, int h);
160 static void EncodeMonoRect32(CARD8 *buf, int w, int h);
161
162 static void FilterGradient24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int w, int h);
163 static void FilterGradient16(rfbClientPtr cl, CARD16 *buf, rfbPixelFormat *fmt, int w, int h);
164 static void FilterGradient32(rfbClientPtr cl, CARD32 *buf, rfbPixelFormat *fmt, int w, int h);
165
166 static int DetectSmoothImage(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
167 static unsigned long DetectSmoothImage24(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
168 static unsigned long DetectSmoothImage16(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
169 static unsigned long DetectSmoothImage32(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
170
171 static Bool SendJpegRect(rfbClientPtr cl, int x, int y, int w, int h,
172 int quality);
173 static void PrepareRowForJpeg(rfbClientPtr cl, CARD8 *dst, int x, int y, int count);
174 static void PrepareRowForJpeg24(rfbClientPtr cl, CARD8 *dst, int x, int y, int count);
175 static void PrepareRowForJpeg16(rfbClientPtr cl, CARD8 *dst, int x, int y, int count);
176 static void PrepareRowForJpeg32(rfbClientPtr cl, CARD8 *dst, int x, int y, int count);
177
178 static void JpegInitDestination(j_compress_ptr cinfo);
179 static boolean JpegEmptyOutputBuffer(j_compress_ptr cinfo);
180 static void JpegTermDestination(j_compress_ptr cinfo);
181 static void JpegSetDstManager(j_compress_ptr cinfo);
182
183
184 /*
185 * Tight encoding implementation.
186 */
187
188 int
189 rfbNumCodedRectsTight(cl, x, y, w, h)
190 rfbClientPtr cl;
191 int x, y, w, h;
192 {
193 int maxRectSize, maxRectWidth;
194 int subrectMaxWidth, subrectMaxHeight;
195
196 /* No matter how many rectangles we will send if LastRect markers
197 are used to terminate rectangle stream. */
198 if (cl->enableLastRectEncoding && w * h >= MIN_SPLIT_RECT_SIZE)
199 return 0;
200
201 maxRectSize = tightConf[cl->tightCompressLevel].maxRectSize;
202 maxRectWidth = tightConf[cl->tightCompressLevel].maxRectWidth;
203
204 if (w > maxRectWidth || w * h > maxRectSize) {
205 subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
206 subrectMaxHeight = maxRectSize / subrectMaxWidth;
207 return (((w - 1) / maxRectWidth + 1) *
208 ((h - 1) / subrectMaxHeight + 1));
209 } else {
210 return 1;
211 }
212 }
213
214 Bool
215 rfbSendRectEncodingTight(cl, x, y, w, h)
216 rfbClientPtr cl;
217 int x, y, w, h;
218 {
219 int nMaxRows;
220 CARD32 colorValue;
221 int dx, dy, dw, dh;
222 int x_best, y_best, w_best, h_best;
223 char *fbptr;
224
225 compressLevel = cl->tightCompressLevel;
226 qualityLevel = cl->tightQualityLevel;
227
228 if ( cl->format.depth == 24 && cl->format.redMax == 0xFF &&
229 cl->format.greenMax == 0xFF && cl->format.blueMax == 0xFF ) {
230 usePixelFormat24 = TRUE;
231 } else {
232 usePixelFormat24 = FALSE;
233 }
234
235 if (!cl->enableLastRectEncoding || w * h < MIN_SPLIT_RECT_SIZE)
236 return SendRectSimple(cl, x, y, w, h);
237
238 /* Make sure we can write at least one pixel into tightBeforeBuf. */
239
240 if (tightBeforeBufSize < 4) {
241 tightBeforeBufSize = 4;
242 if (tightBeforeBuf == NULL)
243 tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
244 else
245 tightBeforeBuf = (char *)realloc(tightBeforeBuf,
246 tightBeforeBufSize);
247 }
248
249 /* Calculate maximum number of rows in one non-solid rectangle. */
250
251 {
252 int maxRectSize, maxRectWidth, nMaxWidth;
253
254 maxRectSize = tightConf[compressLevel].maxRectSize;
255 maxRectWidth = tightConf[compressLevel].maxRectWidth;
256 nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
257 nMaxRows = maxRectSize / nMaxWidth;
258 }
259
260 /* Try to find large solid-color areas and send them separately. */
261
262 for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
263
264 /* If a rectangle becomes too large, send its upper part now. */
265
266 if (dy - y >= nMaxRows) {
267 if (!SendRectSimple(cl, x, y, w, nMaxRows))
268 return 0;
269 y += nMaxRows;
270 h -= nMaxRows;
271 }
272
273 dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
274 MAX_SPLIT_TILE_SIZE : (y + h - dy);
275
276 for (dx = x; dx < x + w; dx += MAX_SPLIT_TILE_SIZE) {
277
278 dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w) ?
279 MAX_SPLIT_TILE_SIZE : (x + w - dx);
280
281 if (CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, FALSE)) {
282
283 /* Get dimensions of solid-color area. */
284
285 FindBestSolidArea(cl, dx, dy, w - (dx - x), h - (dy - y),
286 colorValue, &w_best, &h_best);
287
288 /* Make sure a solid rectangle is large enough
289 (or the whole rectangle is of the same color). */
290
291 if ( w_best * h_best != w * h &&
292 w_best * h_best < MIN_SOLID_SUBRECT_SIZE )
293 continue;
294
295 /* Try to extend solid rectangle to maximum size. */
296
297 x_best = dx; y_best = dy;
298 ExtendSolidArea(cl, x, y, w, h, colorValue,
299 &x_best, &y_best, &w_best, &h_best);
300
301 /* Send rectangles at top and left to solid-color area. */
302
303 if ( y_best != y &&
304 !SendRectSimple(cl, x, y, w, y_best-y) )
305 return FALSE;
306 if ( x_best != x &&
307 !rfbSendRectEncodingTight(cl, x, y_best,
308 x_best-x, h_best) )
309 return FALSE;
310
311 /* Send solid-color rectangle. */
312
313 if (!SendTightHeader(cl, x_best, y_best, w_best, h_best))
314 return FALSE;
315
316 fbptr = (cl->screen->frameBuffer +
317 (cl->screen->paddedWidthInBytes * y_best) +
318 (x_best * (cl->screen->bitsPerPixel / 8)));
319
320 (*cl->translateFn)(cl->translateLookupTable, &cl->screen->rfbServerFormat,
321 &cl->format, fbptr, tightBeforeBuf,
322 cl->screen->paddedWidthInBytes, 1, 1);
323
324 if (!SendSolidRect(cl))
325 return FALSE;
326
327 /* Send remaining rectangles (at right and bottom). */
328
329 if ( x_best + w_best != x + w &&
330 !rfbSendRectEncodingTight(cl, x_best+w_best, y_best,
331 w-(x_best-x)-w_best, h_best) )
332 return FALSE;
333 if ( y_best + h_best != y + h &&
334 !rfbSendRectEncodingTight(cl, x, y_best+h_best,
335 w, h-(y_best-y)-h_best) )
336 return FALSE;
337
338 /* Return after all recursive calls are done. */
339
340 return TRUE;
341 }
342
343 }
344
345 }
346
347 /* No suitable solid-color rectangles found. */
348
349 return SendRectSimple(cl, x, y, w, h);
350 }
351
352 static void
353 FindBestSolidArea(cl, x, y, w, h, colorValue, w_ptr, h_ptr)
354 rfbClientPtr cl;
355 int x, y, w, h;
356 CARD32 colorValue;
357 int *w_ptr, *h_ptr;
358 {
359 int dx, dy, dw, dh;
360 int w_prev;
361 int w_best = 0, h_best = 0;
362
363 w_prev = w;
364
365 for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
366
367 dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
368 MAX_SPLIT_TILE_SIZE : (y + h - dy);
369 dw = (w_prev > MAX_SPLIT_TILE_SIZE) ?
370 MAX_SPLIT_TILE_SIZE : w_prev;
371
372 if (!CheckSolidTile(cl, x, dy, dw, dh, &colorValue, TRUE))
373 break;
374
375 for (dx = x + dw; dx < x + w_prev;) {
376 dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w_prev) ?
377 MAX_SPLIT_TILE_SIZE : (x + w_prev - dx);
378 if (!CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, TRUE))
379 break;
380 dx += dw;
381 }
382
383 w_prev = dx - x;
384 if (w_prev * (dy + dh - y) > w_best * h_best) {
385 w_best = w_prev;
386 h_best = dy + dh - y;
387 }
388 }
389
390 *w_ptr = w_best;
391 *h_ptr = h_best;
392 }
393
394 static void
395 ExtendSolidArea(cl, x, y, w, h, colorValue, x_ptr, y_ptr, w_ptr, h_ptr)
396 rfbClientPtr cl;
397 int x, y, w, h;
398 CARD32 colorValue;
399 int *x_ptr, *y_ptr, *w_ptr, *h_ptr;
400 {
401 int cx, cy;
402
403 /* Try to extend the area upwards. */
404 for ( cy = *y_ptr - 1;
405 cy >= y && CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
406 cy-- );
407 *h_ptr += *y_ptr - (cy + 1);
408 *y_ptr = cy + 1;
409
410 /* ... downwards. */
411 for ( cy = *y_ptr + *h_ptr;
412 cy < y + h &&
413 CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
414 cy++ );
415 *h_ptr += cy - (*y_ptr + *h_ptr);
416
417 /* ... to the left. */
418 for ( cx = *x_ptr - 1;
419 cx >= x && CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
420 cx-- );
421 *w_ptr += *x_ptr - (cx + 1);
422 *x_ptr = cx + 1;
423
424 /* ... to the right. */
425 for ( cx = *x_ptr + *w_ptr;
426 cx < x + w &&
427 CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
428 cx++ );
429 *w_ptr += cx - (*x_ptr + *w_ptr);
430 }
431
432 static Bool
433 CheckSolidTile(cl, x, y, w, h, colorPtr, needSameColor)
434 rfbClientPtr cl;
435 int x, y, w, h;
436 CARD32 *colorPtr;
437 Bool needSameColor;
438 {
439 switch(cl->screen->rfbServerFormat.bitsPerPixel) {
440 case 32:
441 return CheckSolidTile32(cl, x, y, w, h, colorPtr, needSameColor);
442 case 16:
443 return CheckSolidTile16(cl, x, y, w, h, colorPtr, needSameColor);
444 default:
445 return CheckSolidTile8(cl, x, y, w, h, colorPtr, needSameColor);
446 }
447 }
448
449 #define DEFINE_CHECK_SOLID_FUNCTION(bpp) \
450 \
451 static Bool \
452 CheckSolidTile##bpp(cl, x, y, w, h, colorPtr, needSameColor) \
453 rfbClientPtr cl; \
454 int x, y, w, h; \
455 CARD32 *colorPtr; \
456 Bool needSameColor; \
457 { \
458 CARD##bpp *fbptr; \
459 CARD##bpp colorValue; \
460 int dx, dy; \
461 \
462 fbptr = (CARD##bpp *) \
463 &cl->screen->frameBuffer[y * cl->screen->paddedWidthInBytes + x * (bpp/8)]; \
464 \
465 colorValue = *fbptr; \
466 if (needSameColor && (CARD32)colorValue != *colorPtr) \
467 return FALSE; \
468 \
469 for (dy = 0; dy < h; dy++) { \
470 for (dx = 0; dx < w; dx++) { \
471 if (colorValue != fbptr[dx]) \
472 return FALSE; \
473 } \
474 fbptr = (CARD##bpp *)((CARD8 *)fbptr + cl->screen->paddedWidthInBytes); \
475 } \
476 \
477 *colorPtr = (CARD32)colorValue; \
478 return TRUE; \
479 }
480
481 DEFINE_CHECK_SOLID_FUNCTION(8)
482 DEFINE_CHECK_SOLID_FUNCTION(16)
483 DEFINE_CHECK_SOLID_FUNCTION(32)
484
485 static Bool
486 SendRectSimple(cl, x, y, w, h)
487 rfbClientPtr cl;
488 int x, y, w, h;
489 {
490 int maxBeforeSize, maxAfterSize;
491 int maxRectSize, maxRectWidth;
492 int subrectMaxWidth, subrectMaxHeight;
493 int dx, dy;
494 int rw, rh;
495
496 maxRectSize = tightConf[compressLevel].maxRectSize;
497 maxRectWidth = tightConf[compressLevel].maxRectWidth;
498
499 maxBeforeSize = maxRectSize * (cl->format.bitsPerPixel / 8);
500 maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
501
502 if (tightBeforeBufSize < maxBeforeSize) {
503 tightBeforeBufSize = maxBeforeSize;
504 if (tightBeforeBuf == NULL)
505 tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
506 else
507 tightBeforeBuf = (char *)realloc(tightBeforeBuf,
508 tightBeforeBufSize);
509 }
510
511 if (tightAfterBufSize < maxAfterSize) {
512 tightAfterBufSize = maxAfterSize;
513 if (tightAfterBuf == NULL)
514 tightAfterBuf = (char *)malloc(tightAfterBufSize);
515 else
516 tightAfterBuf = (char *)realloc(tightAfterBuf,
517 tightAfterBufSize);
518 }
519
520 if (w > maxRectWidth || w * h > maxRectSize) {
521 subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
522 subrectMaxHeight = maxRectSize / subrectMaxWidth;
523
524 for (dy = 0; dy < h; dy += subrectMaxHeight) {
525 for (dx = 0; dx < w; dx += maxRectWidth) {
526 rw = (dx + maxRectWidth < w) ? maxRectWidth : w - dx;
527 rh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;
528 if (!SendSubrect(cl, x+dx, y+dy, rw, rh))
529 return FALSE;
530 }
531 }
532 } else {
533 if (!SendSubrect(cl, x, y, w, h))
534 return FALSE;
535 }
536
537 return TRUE;
538 }
539
540 static Bool
541 SendSubrect(cl, x, y, w, h)
542 rfbClientPtr cl;
543 int x, y, w, h;
544 {
545 char *fbptr;
546 Bool success = FALSE;
547
548 /* Send pending data if there is more than 128 bytes. */
549 if (cl->ublen > 128) {
550 if (!rfbSendUpdateBuf(cl))
551 return FALSE;
552 }
553
554 if (!SendTightHeader(cl, x, y, w, h))
555 return FALSE;
556
557 fbptr = (cl->screen->frameBuffer + (cl->screen->paddedWidthInBytes * y)
558 + (x * (cl->screen->bitsPerPixel / 8)));
559
560 (*cl->translateFn)(cl->translateLookupTable, &cl->screen->rfbServerFormat,
561 &cl->format, fbptr, tightBeforeBuf,
562 cl->screen->paddedWidthInBytes, w, h);
563
564 paletteMaxColors = w * h / tightConf[compressLevel].idxMaxColorsDivisor;
565 if ( paletteMaxColors < 2 &&
566 w * h >= tightConf[compressLevel].monoMinRectSize ) {
567 paletteMaxColors = 2;
568 }
569 switch (cl->format.bitsPerPixel) {
570 case 8:
571 FillPalette8(w * h);
572 break;
573 case 16:
574 FillPalette16(w * h);
575 break;
576 default:
577 FillPalette32(w * h);
578 }
579
580 switch (paletteNumColors) {
581 case 0:
582 /* Truecolor image */
583 if (DetectSmoothImage(cl, &cl->format, w, h)) {
584 if (qualityLevel != -1) {
585 success = SendJpegRect(cl, x, y, w, h,
586 tightConf[qualityLevel].jpegQuality);
587 } else {
588 success = SendGradientRect(cl, w, h);
589 }
590 } else {
591 success = SendFullColorRect(cl, w, h);
592 }
593 break;
594 case 1:
595 /* Solid rectangle */
596 success = SendSolidRect(cl);
597 break;
598 case 2:
599 /* Two-color rectangle */
600 success = SendMonoRect(cl, w, h);
601 break;
602 default:
603 /* Up to 256 different colors */
604 if ( paletteNumColors > 96 &&
605 qualityLevel != -1 && qualityLevel <= 3 &&
606 DetectSmoothImage(cl, &cl->format, w, h) ) {
607 success = SendJpegRect(cl, x, y, w, h,
608 tightConf[qualityLevel].jpegQuality);
609 } else {
610 success = SendIndexedRect(cl, w, h);
611 }
612 }
613 return success;
614 }
615
616 static Bool
617 SendTightHeader(cl, x, y, w, h)
618 rfbClientPtr cl;
619 int x, y, w, h;
620 {
621 rfbFramebufferUpdateRectHeader rect;
622
623 if (cl->ublen + sz_rfbFramebufferUpdateRectHeader > UPDATE_BUF_SIZE) {
624 if (!rfbSendUpdateBuf(cl))
625 return FALSE;
626 }
627
628 rect.r.x = Swap16IfLE(x);
629 rect.r.y = Swap16IfLE(y);
630 rect.r.w = Swap16IfLE(w);
631 rect.r.h = Swap16IfLE(h);
632 rect.encoding = Swap32IfLE(rfbEncodingTight);
633
634 memcpy(&cl->updateBuf[cl->ublen], (char *)&rect,
635 sz_rfbFramebufferUpdateRectHeader);
636 cl->ublen += sz_rfbFramebufferUpdateRectHeader;
637
638 cl->rfbRectanglesSent[rfbEncodingTight]++;
639 cl->rfbBytesSent[rfbEncodingTight] += sz_rfbFramebufferUpdateRectHeader;
640
641 return TRUE;
642 }
643
644 /*
645 * Subencoding implementations.
646 */
647
648 static Bool
649 SendSolidRect(cl)
650 rfbClientPtr cl;
651 {
652 int len;
653
654 if (usePixelFormat24) {
655 Pack24(cl, tightBeforeBuf, &cl->format, 1);
656 len = 3;
657 } else
658 len = cl->format.bitsPerPixel / 8;
659
660 if (cl->ublen + 1 + len > UPDATE_BUF_SIZE) {
661 if (!rfbSendUpdateBuf(cl))
662 return FALSE;
663 }
664
665 cl->updateBuf[cl->ublen++] = (char)(rfbTightFill << 4);
666 memcpy (&cl->updateBuf[cl->ublen], tightBeforeBuf, len);
667 cl->ublen += len;
668
669 cl->rfbBytesSent[rfbEncodingTight] += len + 1;
670
671 return TRUE;
672 }
673
674 static Bool
675 SendMonoRect(cl, w, h)
676 rfbClientPtr cl;
677 int w, h;
678 {
679 int streamId = 1;
680 int paletteLen, dataLen;
681
682 if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
683 2 * cl->format.bitsPerPixel / 8 > UPDATE_BUF_SIZE ) {
684 if (!rfbSendUpdateBuf(cl))
685 return FALSE;
686 }
687
688 /* Prepare tight encoding header. */
689 dataLen = (w + 7) / 8;
690 dataLen *= h;
691
692 cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
693 cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
694 cl->updateBuf[cl->ublen++] = 1;
695
696 /* Prepare palette, convert image. */
697 switch (cl->format.bitsPerPixel) {
698
699 case 32:
700 EncodeMonoRect32((CARD8 *)tightBeforeBuf, w, h);
701
702 ((CARD32 *)tightAfterBuf)[0] = monoBackground;
703 ((CARD32 *)tightAfterBuf)[1] = monoForeground;
704 if (usePixelFormat24) {
705 Pack24(cl, tightAfterBuf, &cl->format, 2);
706 paletteLen = 6;
707 } else
708 paletteLen = 8;
709
710 memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteLen);
711 cl->ublen += paletteLen;
712 cl->rfbBytesSent[rfbEncodingTight] += 3 + paletteLen;
713 break;
714
715 case 16:
716 EncodeMonoRect16((CARD8 *)tightBeforeBuf, w, h);
717
718 ((CARD16 *)tightAfterBuf)[0] = (CARD16)monoBackground;
719 ((CARD16 *)tightAfterBuf)[1] = (CARD16)monoForeground;
720
721 memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, 4);
722 cl->ublen += 4;
723 cl->rfbBytesSent[rfbEncodingTight] += 7;
724 break;
725
726 default:
727 EncodeMonoRect8((CARD8 *)tightBeforeBuf, w, h);
728
729 cl->updateBuf[cl->ublen++] = (char)monoBackground;
730 cl->updateBuf[cl->ublen++] = (char)monoForeground;
731 cl->rfbBytesSent[rfbEncodingTight] += 5;
732 }
733
734 return CompressData(cl, streamId, dataLen,
735 tightConf[compressLevel].monoZlibLevel,
736 Z_DEFAULT_STRATEGY);
737 }
738
739 static Bool
740 SendIndexedRect(cl, w, h)
741 rfbClientPtr cl;
742 int w, h;
743 {
744 int streamId = 2;
745 int i, entryLen;
746
747 if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
748 paletteNumColors * cl->format.bitsPerPixel / 8 >
749 UPDATE_BUF_SIZE ) {
750 if (!rfbSendUpdateBuf(cl))
751 return FALSE;
752 }
753
754 /* Prepare tight encoding header. */
755 cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
756 cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
757 cl->updateBuf[cl->ublen++] = (char)(paletteNumColors - 1);
758
759 /* Prepare palette, convert image. */
760 switch (cl->format.bitsPerPixel) {
761
762 case 32:
763 EncodeIndexedRect32((CARD8 *)tightBeforeBuf, w * h);
764
765 for (i = 0; i < paletteNumColors; i++) {
766 ((CARD32 *)tightAfterBuf)[i] =
767 palette.entry[i].listNode->rgb;
768 }
769 if (usePixelFormat24) {
770 Pack24(cl, tightAfterBuf, &cl->format, paletteNumColors);
771 entryLen = 3;
772 } else
773 entryLen = 4;
774
775 memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteNumColors * entryLen);
776 cl->ublen += paletteNumColors * entryLen;
777 cl->rfbBytesSent[rfbEncodingTight] += 3 + paletteNumColors * entryLen;
778 break;
779
780 case 16:
781 EncodeIndexedRect16((CARD8 *)tightBeforeBuf, w * h);
782
783 for (i = 0; i < paletteNumColors; i++) {
784 ((CARD16 *)tightAfterBuf)[i] =
785 (CARD16)palette.entry[i].listNode->rgb;
786 }
787
788 memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteNumColors * 2);
789 cl->ublen += paletteNumColors * 2;
790 cl->rfbBytesSent[rfbEncodingTight] += 3 + paletteNumColors * 2;
791 break;
792
793 default:
794 return FALSE; /* Should never happen. */
795 }
796
797 return CompressData(cl, streamId, w * h,
798 tightConf[compressLevel].idxZlibLevel,
799 Z_DEFAULT_STRATEGY);
800 }
801
802 static Bool
803 SendFullColorRect(cl, w, h)
804 rfbClientPtr cl;
805 int w, h;
806 {
807 int streamId = 0;
808 int len;
809
810 if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
811 if (!rfbSendUpdateBuf(cl))
812 return FALSE;
813 }
814
815 cl->updateBuf[cl->ublen++] = 0x00; /* stream id = 0, no flushing, no filter */
816 cl->rfbBytesSent[rfbEncodingTight]++;
817
818 if (usePixelFormat24) {
819 Pack24(cl, tightBeforeBuf, &cl->format, w * h);
820 len = 3;
821 } else
822 len = cl->format.bitsPerPixel / 8;
823
824 return CompressData(cl, streamId, w * h * len,
825 tightConf[compressLevel].rawZlibLevel,
826 Z_DEFAULT_STRATEGY);
827 }
828
829 static Bool
830 SendGradientRect(cl, w, h)
831 rfbClientPtr cl;
832 int w, h;
833 {
834 int streamId = 3;
835 int len;
836
837 if (cl->format.bitsPerPixel == 8)
838 return SendFullColorRect(cl, w, h);
839
840 if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 2 > UPDATE_BUF_SIZE) {
841 if (!rfbSendUpdateBuf(cl))
842 return FALSE;
843 }
844
845 if (prevRowBuf == NULL)
846 prevRowBuf = (int *)malloc(2048 * 3 * sizeof(int));
847
848 cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
849 cl->updateBuf[cl->ublen++] = rfbTightFilterGradient;
850 cl->rfbBytesSent[rfbEncodingTight] += 2;
851
852 if (usePixelFormat24) {
853 FilterGradient24(cl, tightBeforeBuf, &cl->format, w, h);
854 len = 3;
855 } else if (cl->format.bitsPerPixel == 32) {
856 FilterGradient32(cl, (CARD32 *)tightBeforeBuf, &cl->format, w, h);
857 len = 4;
858 } else {
859 FilterGradient16(cl, (CARD16 *)tightBeforeBuf, &cl->format, w, h);
860 len = 2;
861 }
862
863 return CompressData(cl, streamId, w * h * len,
864 tightConf[compressLevel].gradientZlibLevel,
865 Z_FILTERED);
866 }
867
868 static Bool
869 CompressData(cl, streamId, dataLen, zlibLevel, zlibStrategy)
870 rfbClientPtr cl;
871 int streamId, dataLen, zlibLevel, zlibStrategy;
872 {
873 z_streamp pz;
874 int err;
875
876 if (dataLen < TIGHT_MIN_TO_COMPRESS) {
877 memcpy(&cl->updateBuf[cl->ublen], tightBeforeBuf, dataLen);
878 cl->ublen += dataLen;
879 cl->rfbBytesSent[rfbEncodingTight] += dataLen;
880 return TRUE;
881 }
882
883 pz = &cl->zsStruct[streamId];
884
885 /* Initialize compression stream if needed. */
886 if (!cl->zsActive[streamId]) {
887 pz->zalloc = Z_NULL;
888 pz->zfree = Z_NULL;
889 pz->opaque = Z_NULL;
890
891 err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
892 MAX_MEM_LEVEL, zlibStrategy);
893 if (err != Z_OK)
894 return FALSE;
895
896 cl->zsActive[streamId] = TRUE;
897 cl->zsLevel[streamId] = zlibLevel;
898 }
899
900 /* Prepare buffer pointers. */
901 pz->next_in = (Bytef *)tightBeforeBuf;
902 pz->avail_in = dataLen;
903 pz->next_out = (Bytef *)tightAfterBuf;
904 pz->avail_out = tightAfterBufSize;
905
906 /* Change compression parameters if needed. */
907 if (zlibLevel != cl->zsLevel[streamId]) {
908 if (deflateParams (pz, zlibLevel, zlibStrategy) != Z_OK) {
909 return FALSE;
910 }
911 cl->zsLevel[streamId] = zlibLevel;
912 }
913
914 /* Actual compression. */
915 if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK ||
916 pz->avail_in != 0 || pz->avail_out == 0 ) {
917 return FALSE;
918 }
919
920 return SendCompressedData(cl, tightAfterBufSize - pz->avail_out);
921 }
922
923 static Bool SendCompressedData(cl, compressedLen)
924 rfbClientPtr cl;
925 int compressedLen;
926 {
927 int i, portionLen;
928
929 cl->updateBuf[cl->ublen++] = compressedLen & 0x7F;
930 cl->rfbBytesSent[rfbEncodingTight]++;
931 if (compressedLen > 0x7F) {
932 cl->updateBuf[cl->ublen-1] |= 0x80;
933 cl->updateBuf[cl->ublen++] = compressedLen >> 7 & 0x7F;
934 cl->rfbBytesSent[rfbEncodingTight]++;
935 if (compressedLen > 0x3FFF) {
936 cl->updateBuf[cl->ublen-1] |= 0x80;
937 cl->updateBuf[cl->ublen++] = compressedLen >> 14 & 0xFF;
938 cl->rfbBytesSent[rfbEncodingTight]++;
939 }
940 }
941
942 portionLen = UPDATE_BUF_SIZE;
943 for (i = 0; i < compressedLen; i += portionLen) {
944 if (i + portionLen > compressedLen) {
945 portionLen = compressedLen - i;
946 }
947 if (cl->ublen + portionLen > UPDATE_BUF_SIZE) {
948 if (!rfbSendUpdateBuf(cl))
949 return FALSE;
950 }
951 memcpy(&cl->updateBuf[cl->ublen], &tightAfterBuf[i], portionLen);
952 cl->ublen += portionLen;
953 }
954 portionLen = UPDATE_BUF_SIZE;
955 cl->rfbBytesSent[rfbEncodingTight] += compressedLen;
956 return TRUE;
957 }
958
959 /*
960 * Code to determine how many different colors used in rectangle.
961 */
962
963 static void
964 FillPalette8(count)
965 int count;
966 {
967 CARD8 *data = (CARD8 *)tightBeforeBuf;
968 CARD8 c0, c1;
969 int i, n0, n1;
970
971 paletteNumColors = 0;
972
973 c0 = data[0];
974 for (i = 1; i < count && data[i] == c0; i++);
975 if (i == count) {
976 paletteNumColors = 1;
977 return; /* Solid rectangle */
978 }
979
980 if (paletteMaxColors < 2)
981 return;
982
983 n0 = i;
984 c1 = data[i];
985 n1 = 0;
986 for (i++; i < count; i++) {
987 if (data[i] == c0) {
988 n0++;
989 } else if (data[i] == c1) {
990 n1++;
991 } else
992 break;
993 }
994 if (i == count) {
995 if (n0 > n1) {
996 monoBackground = (CARD32)c0;
997 monoForeground = (CARD32)c1;
998 } else {
999 monoBackground = (CARD32)c1;
1000 monoForeground = (CARD32)c0;
1001 }
1002 paletteNumColors = 2; /* Two colors */
1003 }
1004 }
1005
1006 #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \
1007 \
1008 static void \
1009 FillPalette##bpp(count) \
1010 int count; \
1011 { \
1012 CARD##bpp *data = (CARD##bpp *)tightBeforeBuf; \
1013 CARD##bpp c0, c1, ci; \
1014 int i, n0, n1, ni; \
1015 \
1016 c0 = data[0]; \
1017 for (i = 1; i < count && data[i] == c0; i++); \
1018 if (i >= count) { \
1019 paletteNumColors = 1; /* Solid rectangle */ \
1020 return; \
1021 } \
1022 \
1023 if (paletteMaxColors < 2) { \
1024 paletteNumColors = 0; /* Full-color encoding preferred */ \
1025 return; \
1026 } \
1027 \
1028 n0 = i; \
1029 c1 = data[i]; \
1030 n1 = 0; \
1031 for (i++; i < count; i++) { \
1032 ci = data[i]; \
1033 if (ci == c0) { \
1034 n0++; \
1035 } else if (ci == c1) { \
1036 n1++; \
1037 } else \
1038 break; \
1039 } \
1040 if (i >= count) { \
1041 if (n0 > n1) { \
1042 monoBackground = (CARD32)c0; \
1043 monoForeground = (CARD32)c1; \
1044 } else { \
1045 monoBackground = (CARD32)c1; \
1046 monoForeground = (CARD32)c0; \
1047 } \
1048 paletteNumColors = 2; /* Two colors */ \
1049 return; \
1050 } \
1051 \
1052 PaletteReset(); \
1053 PaletteInsert (c0, (CARD32)n0, bpp); \
1054 PaletteInsert (c1, (CARD32)n1, bpp); \
1055 \
1056 ni = 1; \
1057 for (i++; i < count; i++) { \
1058 if (data[i] == ci) { \
1059 ni++; \
1060 } else { \
1061 if (!PaletteInsert (ci, (CARD32)ni, bpp)) \
1062 return; \
1063 ci = data[i]; \
1064 ni = 1; \
1065 } \
1066 } \
1067 PaletteInsert (ci, (CARD32)ni, bpp); \
1068 }
1069
1070 DEFINE_FILL_PALETTE_FUNCTION(16)
1071 DEFINE_FILL_PALETTE_FUNCTION(32)
1072
1073
1074 /*
1075 * Functions to operate with palette structures.
1076 */
1077
1078 #define HASH_FUNC16(rgb) ((int)(((rgb >> 8) + rgb) & 0xFF))
1079 #define HASH_FUNC32(rgb) ((int)(((rgb >> 16) + (rgb >> 8)) & 0xFF))
1080
1081 static void
1082 PaletteReset(void)
1083 {
1084 paletteNumColors = 0;
1085 memset(palette.hash, 0, 256 * sizeof(COLOR_LIST *));
1086 }
1087
1088 static int
1089 PaletteInsert(rgb, numPixels, bpp)
1090 CARD32 rgb;
1091 int numPixels;
1092 int bpp;
1093 {
1094 COLOR_LIST *pnode;
1095 COLOR_LIST *prev_pnode = NULL;
1096 int hash_key, idx, new_idx, count;
1097
1098 hash_key = (bpp == 16) ? HASH_FUNC16(rgb) : HASH_FUNC32(rgb);
1099
1100 pnode = palette.hash[hash_key];
1101
1102 while (pnode != NULL) {
1103 if (pnode->rgb == rgb) {
1104 /* Such palette entry already exists. */
1105 new_idx = idx = pnode->idx;
1106 count = palette.entry[idx].numPixels + numPixels;
1107 if (new_idx && palette.entry[new_idx-1].numPixels < count) {
1108 do {
1109 palette.entry[new_idx] = palette.entry[new_idx-1];
1110 palette.entry[new_idx].listNode->idx = new_idx;
1111 new_idx--;
1112 }
1113 while (new_idx && palette.entry[new_idx-1].numPixels < count);
1114 palette.entry[new_idx].listNode = pnode;
1115 pnode->idx = new_idx;
1116 }
1117 palette.entry[new_idx].numPixels = count;
1118 return paletteNumColors;
1119 }
1120 prev_pnode = pnode;
1121 pnode = pnode->next;
1122 }
1123
1124 /* Check if palette is full. */
1125 if (paletteNumColors == 256 || paletteNumColors == paletteMaxColors) {
1126 paletteNumColors = 0;
1127 return 0;
1128 }
1129
1130 /* Move palette entries with lesser pixel counts. */
1131 for ( idx = paletteNumColors;
1132 idx > 0 && palette.entry[idx-1].numPixels < numPixels;
1133 idx-- ) {
1134 palette.entry[idx] = palette.entry[idx-1];
1135 palette.entry[idx].listNode->idx = idx;
1136 }
1137
1138 /* Add new palette entry into the freed slot. */
1139 pnode = &palette.list[paletteNumColors];
1140 if (prev_pnode != NULL) {
1141 prev_pnode->next = pnode;
1142 } else {
1143 palette.hash[hash_key] = pnode;
1144 }
1145 pnode->next = NULL;
1146 pnode->idx = idx;
1147 pnode->rgb = rgb;
1148 palette.entry[idx].listNode = pnode;
1149 palette.entry[idx].numPixels = numPixels;
1150
1151 return (++paletteNumColors);
1152 }
1153
1154
1155 /*
1156 * Converting 32-bit color samples into 24-bit colors.
1157 * Should be called only when redMax, greenMax and blueMax are 255.
1158 * Color components assumed to be byte-aligned.
1159 */
1160
1161 static void Pack24(cl, buf, fmt, count)
1162 rfbClientPtr cl;
1163 char *buf;
1164 rfbPixelFormat *fmt;
1165 int count;
1166 {
1167 CARD32 *buf32;
1168 CARD32 pix;
1169 int r_shift, g_shift, b_shift;
1170
1171 buf32 = (CARD32 *)buf;
1172
1173 if (!cl->screen->rfbServerFormat.bigEndian == !fmt->bigEndian) {
1174 r_shift = fmt->redShift;
1175 g_shift = fmt->greenShift;
1176 b_shift = fmt->blueShift;
1177 } else {
1178 r_shift = 24 - fmt->redShift;
1179 g_shift = 24 - fmt->greenShift;
1180 b_shift = 24 - fmt->blueShift;
1181 }
1182
1183 while (count--) {
1184 pix = *buf32++;
1185 *buf++ = (char)(pix >> r_shift);
1186 *buf++ = (char)(pix >> g_shift);
1187 *buf++ = (char)(pix >> b_shift);
1188 }
1189 }
1190
1191
1192 /*
1193 * Converting truecolor samples into palette indices.
1194 */
1195
1196 #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \
1197 \
1198 static void \
1199 EncodeIndexedRect##bpp(buf, count) \
1200 CARD8 *buf; \
1201 int count; \
1202 { \
1203 COLOR_LIST *pnode; \
1204 CARD##bpp *src; \
1205 CARD##bpp rgb; \
1206 int rep = 0; \
1207 \
1208 src = (CARD##bpp *) buf; \
1209 \
1210 while (count--) { \
1211 rgb = *src++; \
1212 while (count && *src == rgb) { \
1213 rep++, src++, count--; \
1214 } \
1215 pnode = palette.hash[HASH_FUNC##bpp(rgb)]; \
1216 while (pnode != NULL) { \
1217 if ((CARD##bpp)pnode->rgb == rgb) { \
1218 *buf++ = (CARD8)pnode->idx; \
1219 while (rep) { \
1220 *buf++ = (CARD8)pnode->idx; \
1221 rep--; \
1222 } \
1223 break; \
1224 } \
1225 pnode = pnode->next; \
1226 } \
1227 } \
1228 }
1229
1230 DEFINE_IDX_ENCODE_FUNCTION(16)
1231 DEFINE_IDX_ENCODE_FUNCTION(32)
1232
1233 #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \
1234 \
1235 static void \
1236 EncodeMonoRect##bpp(buf, w, h) \
1237 CARD8 *buf; \
1238 int w, h; \
1239 { \
1240 CARD##bpp *ptr; \
1241 CARD##bpp bg; \
1242 unsigned int value, mask; \
1243 int aligned_width; \
1244 int x, y, bg_bits; \
1245 \
1246 ptr = (CARD##bpp *) buf; \
1247 bg = (CARD##bpp) monoBackground; \
1248 aligned_width = w - w % 8; \
1249 \
1250 for (y = 0; y < h; y++) { \
1251 for (x = 0; x < aligned_width; x += 8) { \
1252 for (bg_bits = 0; bg_bits < 8; bg_bits++) { \
1253 if (*ptr++ != bg) \
1254 break; \
1255 } \
1256 if (bg_bits == 8) { \
1257 *buf++ = 0; \
1258 continue; \
1259 } \
1260 mask = 0x80 >> bg_bits; \
1261 value = mask; \
1262 for (bg_bits++; bg_bits < 8; bg_bits++) { \
1263 mask >>= 1; \
1264 if (*ptr++ != bg) { \
1265 value |= mask; \
1266 } \
1267 } \
1268 *buf++ = (CARD8)value; \
1269 } \
1270 \
1271 mask = 0x80; \
1272 value = 0; \
1273 if (x >= w) \
1274 continue; \
1275 \
1276 for (; x < w; x++) { \
1277 if (*ptr++ != bg) { \
1278 value |= mask; \
1279 } \
1280 mask >>= 1; \
1281 } \
1282 *buf++ = (CARD8)value; \
1283 } \
1284 }
1285
1286 DEFINE_MONO_ENCODE_FUNCTION(8)
1287 DEFINE_MONO_ENCODE_FUNCTION(16)
1288 DEFINE_MONO_ENCODE_FUNCTION(32)
1289
1290
1291 /*
1292 * ``Gradient'' filter for 24-bit color samples.
1293 * Should be called only when redMax, greenMax and blueMax are 255.
1294 * Color components assumed to be byte-aligned.
1295 */
1296
1297 static void
1298 FilterGradient24(cl, buf, fmt, w, h)
1299 rfbClientPtr cl;
1300 char *buf;
1301 rfbPixelFormat *fmt;
1302 int w, h;
1303 {
1304 CARD32 *buf32;
1305 CARD32 pix32;
1306 int *prevRowPtr;
1307 int shiftBits[3];
1308 int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3];
1309 int prediction;
1310 int x, y, c;
1311
1312 buf32 = (CARD32 *)buf;
1313 memset (prevRowBuf, 0, w * 3 * sizeof(int));
1314
1315 if (!cl->screen->rfbServerFormat.bigEndian == !fmt->bigEndian) {
1316 shiftBits[0] = fmt->redShift;
1317 shiftBits[1] = fmt->greenShift;
1318 shiftBits[2] = fmt->blueShift;
1319 } else {
1320 shiftBits[0] = 24 - fmt->redShift;
1321 shiftBits[1] = 24 - fmt->greenShift;
1322 shiftBits[2] = 24 - fmt->blueShift;
1323 }
1324
1325 for (y = 0; y < h; y++) {
1326 for (c = 0; c < 3; c++) {
1327 pixUpper[c] = 0;
1328 pixHere[c] = 0;
1329 }
1330 prevRowPtr = prevRowBuf;
1331 for (x = 0; x < w; x++) {
1332 pix32 = *buf32++;
1333 for (c = 0; c < 3; c++) {
1334 pixUpperLeft[c] = pixUpper[c];
1335 pixLeft[c] = pixHere[c];
1336 pixUpper[c] = *prevRowPtr;
1337 pixHere[c] = (int)(pix32 >> shiftBits[c] & 0xFF);
1338 *prevRowPtr++ = pixHere[c];
1339
1340 prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c];
1341 if (prediction < 0) {
1342 prediction = 0;
1343 } else if (prediction > 0xFF) {
1344 prediction = 0xFF;
1345 }
1346 *buf++ = (char)(pixHere[c] - prediction);
1347 }
1348 }
1349 }
1350 }
1351
1352
1353 /*
1354 * ``Gradient'' filter for other color depths.
1355 */
1356
1357 #define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \
1358 \
1359 static void \
1360 FilterGradient##bpp(cl, buf, fmt, w, h) \
1361 rfbClientPtr cl; \
1362 CARD##bpp *buf; \
1363 rfbPixelFormat *fmt; \
1364 int w, h; \
1365 { \
1366 CARD##bpp pix, diff; \
1367 Bool endianMismatch; \
1368 int *prevRowPtr; \
1369 int maxColor[3], shiftBits[3]; \
1370 int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3]; \
1371 int prediction; \
1372 int x, y, c; \
1373 \
1374 memset (prevRowBuf, 0, w * 3 * sizeof(int)); \
1375 \
1376 endianMismatch = (!cl->screen->rfbServerFormat.bigEndian != !fmt->bigEndian); \
1377 \
1378 maxColor[0] = fmt->redMax; \
1379 maxColor[1] = fmt->greenMax; \
1380 maxColor[2] = fmt->blueMax; \
1381 shiftBits[0] = fmt->redShift; \
1382 shiftBits[1] = fmt->greenShift; \
1383 shiftBits[2] = fmt->blueShift; \
1384 \
1385 for (y = 0; y < h; y++) { \
1386 for (c = 0; c < 3; c++) { \
1387 pixUpper[c] = 0; \
1388 pixHere[c] = 0; \
1389 } \
1390 prevRowPtr = prevRowBuf; \
1391 for (x = 0; x < w; x++) { \
1392 pix = *buf; \
1393 if (endianMismatch) { \
1394 pix = Swap##bpp(pix); \
1395 } \
1396 diff = 0; \
1397 for (c = 0; c < 3; c++) { \
1398 pixUpperLeft[c] = pixUpper[c]; \
1399 pixLeft[c] = pixHere[c]; \
1400 pixUpper[c] = *prevRowPtr; \
1401 pixHere[c] = (int)(pix >> shiftBits[c] & maxColor[c]); \
1402 *prevRowPtr++ = pixHere[c]; \
1403 \
1404 prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c]; \
1405 if (prediction < 0) { \
1406 prediction = 0; \
1407 } else if (prediction > maxColor[c]) { \
1408 prediction = maxColor[c]; \
1409 } \
1410 diff |= ((pixHere[c] - prediction) & maxColor[c]) \
1411 << shiftBits[c]; \
1412 } \
1413 if (endianMismatch) { \
1414 diff = Swap##bpp(diff); \
1415 } \
1416 *buf++ = diff; \
1417 } \
1418 } \
1419 }
1420
1421 DEFINE_GRADIENT_FILTER_FUNCTION(16)
1422 DEFINE_GRADIENT_FILTER_FUNCTION(32)
1423
1424
1425 /*
1426 * Code to guess if given rectangle is suitable for smooth image
1427 * compression (by applying "gradient" filter or JPEG coder).
1428 */
1429
1430 #define JPEG_MIN_RECT_SIZE 4096
1431
1432 #define DETECT_SUBROW_WIDTH 7
1433 #define DETECT_MIN_WIDTH 8
1434 #define DETECT_MIN_HEIGHT 8
1435
1436 static int
1437 DetectSmoothImage (cl, fmt, w, h)
1438 rfbClientPtr cl;
1439 rfbPixelFormat *fmt;
1440 int w, h;
1441 {
1442 long avgError;
1443
1444 if ( cl->screen->rfbServerFormat.bitsPerPixel == 8 || fmt->bitsPerPixel == 8 ||
1445 w < DETECT_MIN_WIDTH || h < DETECT_MIN_HEIGHT ) {
1446 return 0;
1447 }
1448
1449 if (qualityLevel != -1) {
1450 if (w * h < JPEG_MIN_RECT_SIZE) {
1451 return 0;
1452 }
1453 } else {
1454 if ( rfbTightDisableGradient ||
1455 w * h < tightConf[compressLevel].gradientMinRectSize ) {
1456 return 0;
1457 }
1458 }
1459
1460 if (fmt->bitsPerPixel == 32) {
1461 if (usePixelFormat24) {
1462 avgError = DetectSmoothImage24(cl, fmt, w, h);
1463 if (qualityLevel != -1) {
1464 return (avgError < tightConf[qualityLevel].jpegThreshold24);
1465 }
1466 return (avgError < tightConf[compressLevel].gradientThreshold24);
1467 } else {
1468 avgError = DetectSmoothImage32(cl, fmt, w, h);
1469 }
1470 } else {
1471 avgError = DetectSmoothImage16(cl, fmt, w, h);
1472 }
1473 if (qualityLevel != -1) {
1474 return (avgError < tightConf[qualityLevel].jpegThreshold);
1475 }
1476 return (avgError < tightConf[compressLevel].gradientThreshold);
1477 }
1478
1479 static unsigned long
1480 DetectSmoothImage24 (cl, fmt, w, h)
1481 rfbClientPtr cl;
1482 rfbPixelFormat *fmt;
1483 int w, h;
1484 {
1485 int off;
1486 int x, y, d, dx, c;
1487 int diffStat[256];
1488 int pixelCount = 0;
1489 int pix, left[3];
1490 unsigned long avgError;
1491
1492 /* If client is big-endian, color samples begin from the second
1493 byte (offset 1) of a 32-bit pixel value. */
1494 off = (fmt->bigEndian != 0);
1495
1496 memset(diffStat, 0, 256*sizeof(int));
1497
1498 y = 0, x = 0;
1499 while (y < h && x < w) {
1500 for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) {
1501 for (c = 0; c < 3; c++) {
1502 left[c] = (int)tightBeforeBuf[((y+d)*w+x+d)*4+off+c] & 0xFF;
1503 }
1504 for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) {
1505 for (c = 0; c < 3; c++) {
1506 pix = (int)tightBeforeBuf[((y+d)*w+x+d+dx)*4+off+c] & 0xFF;
1507 diffStat[abs(pix - left[c])]++;
1508 left[c] = pix;
1509 }
1510 pixelCount++;
1511 }
1512 }
1513 if (w > h) {
1514 x += h;
1515 y = 0;
1516 } else {
1517 x = 0;
1518 y += w;
1519 }
1520 }
1521
1522 if (diffStat[0] * 33 / pixelCount >= 95)
1523 return 0;
1524
1525 avgError = 0;
1526 for (c = 1; c < 8; c++) {
1527 avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
1528 if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2)
1529 return 0;
1530 }
1531 for (; c < 256; c++) {
1532 avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
1533 }
1534 avgError /= (pixelCount * 3 - diffStat[0]);
1535
1536 return avgError;
1537 }
1538
1539 #define DEFINE_DETECT_FUNCTION(bpp) \
1540 \
1541 static unsigned long \
1542 DetectSmoothImage##bpp (cl, fmt, w, h) \
1543 rfbClientPtr cl; \
1544 rfbPixelFormat *fmt; \
1545 int w, h; \
1546 { \
1547 Bool endianMismatch; \
1548 CARD##bpp pix; \
1549 int maxColor[3], shiftBits[3]; \
1550 int x, y, d, dx, c; \
1551 int diffStat[256]; \
1552 int pixelCount = 0; \
1553 int sample, sum, left[3]; \
1554 unsigned long avgError; \
1555 \
1556 endianMismatch = (!cl->screen->rfbServerFormat.bigEndian != !fmt->bigEndian); \
1557 \
1558 maxColor[0] = fmt->redMax; \
1559 maxColor[1] = fmt->greenMax; \
1560 maxColor[2] = fmt->blueMax; \
1561 shiftBits[0] = fmt->redShift; \
1562 shiftBits[1] = fmt->greenShift; \
1563 shiftBits[2] = fmt->blueShift; \
1564 \
1565 memset(diffStat, 0, 256*sizeof(int)); \
1566 \
1567 y = 0, x = 0; \
1568 while (y < h && x < w) { \
1569 for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) { \
1570 pix = ((CARD##bpp *)tightBeforeBuf)[(y+d)*w+x+d]; \
1571 if (endianMismatch) { \
1572 pix = Swap##bpp(pix); \
1573 } \
1574 for (c = 0; c < 3; c++) { \
1575 left[c] = (int)(pix >> shiftBits[c] & maxColor[c]); \
1576 } \
1577 for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) { \
1578 pix = ((CARD##bpp *)tightBeforeBuf)[(y+d)*w+x+d+dx]; \
1579 if (endianMismatch) { \
1580 pix = Swap##bpp(pix); \
1581 } \
1582 sum = 0; \
1583 for (c = 0; c < 3; c++) { \
1584 sample = (int)(pix >> shiftBits[c] & maxColor[c]); \
1585 sum += abs(sample - left[c]); \
1586 left[c] = sample; \
1587 } \
1588 if (sum > 255) \
1589 sum = 255; \
1590 diffStat[sum]++; \
1591 pixelCount++; \
1592 } \
1593 } \
1594 if (w > h) { \
1595 x += h; \
1596 y = 0; \
1597 } else { \
1598 x = 0; \
1599 y += w; \
1600 } \
1601 } \
1602 \
1603 if ((diffStat[0] + diffStat[1]) * 100 / pixelCount >= 90) \
1604 return 0; \
1605 \
1606 avgError = 0; \
1607 for (c = 1; c < 8; c++) { \
1608 avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c); \
1609 if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2) \
1610 return 0; \
1611 } \
1612 for (; c < 256; c++) { \
1613 avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c); \
1614 } \
1615 avgError /= (pixelCount - diffStat[0]); \
1616 \
1617 return avgError; \
1618 }
1619
1620 DEFINE_DETECT_FUNCTION(16)
1621 DEFINE_DETECT_FUNCTION(32)
1622
1623
1624 /*
1625 * JPEG compression stuff.
1626 */
1627
1628 static struct jpeg_destination_mgr jpegDstManager;
1629 static Bool jpegError;
1630 static int jpegDstDataLen;
1631
1632 static Bool
1633 SendJpegRect(cl, x, y, w, h, quality)
1634 rfbClientPtr cl;
1635 int x, y, w, h;
1636 int quality;
1637 {
1638 struct jpeg_compress_struct cinfo;
1639 struct jpeg_error_mgr jerr;
1640 CARD8 *srcBuf;
1641 JSAMPROW rowPointer[1];
1642 int dy;
1643
1644 if (cl->screen->rfbServerFormat.bitsPerPixel == 8)
1645 return SendFullColorRect(cl, w, h);
1646
1647 srcBuf = (CARD8 *)malloc(w * 3);
1648 if (srcBuf == NULL) {
1649 return SendFullColorRect(cl, w, h);
1650 }
1651 rowPointer[0] = srcBuf;
1652
1653 cinfo.err = jpeg_std_error(&jerr);
1654 jpeg_create_compress(&cinfo);
1655
1656 cinfo.image_width = w;
1657 cinfo.image_height = h;
1658 cinfo.input_components = 3;
1659 cinfo.in_color_space = JCS_RGB;
1660
1661 jpeg_set_defaults(&cinfo);
1662 jpeg_set_quality(&cinfo, quality, TRUE);
1663
1664 JpegSetDstManager (&cinfo);
1665
1666 jpeg_start_compress(&cinfo, TRUE);
1667
1668 for (dy = 0; dy < h; dy++) {
1669 PrepareRowForJpeg(cl, srcBuf, x, y + dy, w);
1670 jpeg_write_scanlines(&cinfo, rowPointer, 1);
1671 if (jpegError)
1672 break;
1673 }
1674
1675 if (!jpegError)
1676 jpeg_finish_compress(&cinfo);
1677
1678 jpeg_destroy_compress(&cinfo);
1679 free(srcBuf);
1680
1681 if (jpegError)
1682 return SendFullColorRect(cl, w, h);
1683
1684 if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
1685 if (!rfbSendUpdateBuf(cl))
1686 return FALSE;
1687 }
1688
1689 cl->updateBuf[cl->ublen++] = (char)(rfbTightJpeg << 4);
1690 cl->rfbBytesSent[rfbEncodingTight]++;
1691
1692 return SendCompressedData(cl, jpegDstDataLen);
1693 }
1694
1695 static void
1696 PrepareRowForJpeg(cl, dst, x, y, count)
1697 rfbClientPtr cl;
1698 CARD8 *dst;
1699 int x, y, count;
1700 {
1701 if (cl->screen->rfbServerFormat.bitsPerPixel == 32) {
1702 if ( cl->screen->rfbServerFormat.redMax == 0xFF &&
1703 cl->screen->rfbServerFormat.greenMax == 0xFF &&
1704 cl->screen->rfbServerFormat.blueMax == 0xFF ) {
1705 PrepareRowForJpeg24(cl, dst, x, y, count);
1706 } else {
1707 PrepareRowForJpeg32(cl, dst, x, y, count);
1708 }
1709 } else {
1710 /* 16 bpp assumed. */
1711 PrepareRowForJpeg16(cl, dst, x, y, count);
1712 }
1713 }
1714
1715 static void
1716 PrepareRowForJpeg24(cl, dst, x, y, count)
1717 rfbClientPtr cl;
1718 CARD8 *dst;
1719 int x, y, count;
1720 {
1721 CARD32 *fbptr;
1722 CARD32 pix;
1723
1724 fbptr = (CARD32 *)
1725 &cl->screen->frameBuffer[y * cl->screen->paddedWidthInBytes + x * 4];
1726
1727 while (count--) {
1728 pix = *fbptr++;
1729 *dst++ = (CARD8)(pix >> cl->screen->rfbServerFormat.redShift);
1730 *dst++ = (CARD8)(pix >> cl->screen->rfbServerFormat.greenShift);
1731 *dst++ = (CARD8)(pix >> cl->screen->rfbServerFormat.blueShift);
1732 }
1733 }
1734
1735 #define DEFINE_JPEG_GET_ROW_FUNCTION(bpp) \
1736 \
1737 static void \
1738 PrepareRowForJpeg##bpp(cl, dst, x, y, count) \
1739 rfbClientPtr cl; \
1740 CARD8 *dst; \
1741 int x, y, count; \
1742 { \
1743 CARD##bpp *fbptr; \
1744 CARD##bpp pix; \
1745 int inRed, inGreen, inBlue; \
1746 \
1747 fbptr = (CARD##bpp *) \
1748 &cl->screen->frameBuffer[y * cl->screen->paddedWidthInBytes + \
1749 x * (bpp / 8)]; \
1750 \
1751 while (count--) { \
1752 pix = *fbptr++; \
1753 \
1754 inRed = (int) \
1755 (pix >> cl->screen->rfbServerFormat.redShift & cl->screen->rfbServerFormat.redMax); \
1756 inGreen = (int) \
1757 (pix >> cl->screen->rfbServerFormat.greenShift & cl->screen->rfbServerFormat.greenMax); \
1758 inBlue = (int) \
1759 (pix >> cl->screen->rfbServerFormat.blueShift & cl->screen->rfbServerFormat.blueMax); \
1760 \
1761 *dst++ = (CARD8)((inRed * 255 + cl->screen->rfbServerFormat.redMax / 2) / \
1762 cl->screen->rfbServerFormat.redMax); \
1763 *dst++ = (CARD8)((inGreen * 255 + cl->screen->rfbServerFormat.greenMax / 2) / \
1764 cl->screen->rfbServerFormat.greenMax); \
1765 *dst++ = (CARD8)((inBlue * 255 + cl->screen->rfbServerFormat.blueMax / 2) / \
1766 cl->screen->rfbServerFormat.blueMax); \
1767 } \
1768 }
1769
1770 DEFINE_JPEG_GET_ROW_FUNCTION(16)
1771 DEFINE_JPEG_GET_ROW_FUNCTION(32)
1772
1773 /*
1774 * Destination manager implementation for JPEG library.
1775 */
1776
1777 static void
1778 JpegInitDestination(j_compress_ptr cinfo)
1779 {
1780 jpegError = FALSE;
1781 jpegDstManager.next_output_byte = (JOCTET *)tightAfterBuf;
1782 jpegDstManager.free_in_buffer = (size_t)tightAfterBufSize;
1783 }
1784
1785 static boolean
1786 JpegEmptyOutputBuffer(j_compress_ptr cinfo)
1787 {
1788 jpegError = TRUE;
1789 jpegDstManager.next_output_byte = (JOCTET *)tightAfterBuf;
1790 jpegDstManager.free_in_buffer = (size_t)tightAfterBufSize;
1791
1792 return TRUE;
1793 }
1794
1795 static void
1796 JpegTermDestination(j_compress_ptr cinfo)
1797 {
1798 jpegDstDataLen = tightAfterBufSize - jpegDstManager.free_in_buffer;
1799 }
1800
1801 static void
1802 JpegSetDstManager(j_compress_ptr cinfo)
1803 {
1804 jpegDstManager.init_destination = JpegInitDestination;
1805 jpegDstManager.empty_output_buffer = JpegEmptyOutputBuffer;
1806 jpegDstManager.term_destination = JpegTermDestination;
1807 cinfo->dest = &jpegDstManager;
1808 }
1809
kdenetwork experimental stuff