krdc/vnc/tight.c

   1 /*
   2  *  Copyright 2000, 2001 Const Kaplinsky <const@ce.cctpu.edu.ru>
   3  *
   4  *  This is free software; you can redistribute it and/or modify
   5  *  it under the terms of the GNU General Public License as published by
   6  *  the Free Software Foundation; either version 2 of the License, or
   7  *  (at your option) any later version.
   8  *
   9  *  This software is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13  *
  14  *  You should have received a copy of the GNU General Public License
  15  *  along with this software; if not, write to the Free Software
  16  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  17  *  USA.
  18  */
  19
  20 /*
  21  * tight.c - handle ``tight'' encoding.
  22  *
  23  * This file shouldn't be compiled directly. It is included multiple
  24  * times by rfbproto.c, each time with a different definition of the
  25  * macro BPP. For each value of BPP, this file defines a function
  26  * which handles a tight-encoded rectangle with BPP bits per pixel.
  27  *
  28  */
  29
  30 #define TIGHT_MIN_TO_COMPRESS 12
  31
  32 #define CARDBPP CONCAT2E(CARD,BPP)
  33 #define filterPtrBPP CONCAT2E(filterPtr,BPP)
  34
  35 #define HandleTightBPP CONCAT2E(HandleTight,BPP)
  36 #define InitFilterCopyBPP CONCAT2E(InitFilterCopy,BPP)
  37 #define InitFilterPaletteBPP CONCAT2E(InitFilterPalette,BPP)
  38 #define InitFilterGradientBPP CONCAT2E(InitFilterGradient,BPP)
  39 #define FilterCopyBPP CONCAT2E(FilterCopy,BPP)
  40 #define FilterPaletteBPP CONCAT2E(FilterPalette,BPP)
  41 #define FilterGradientBPP CONCAT2E(FilterGradient,BPP)
  42 #define FillRectangleBPP CONCAT2E(FillRectangle,BPP)
  43
  44 #if BPP != 8
  45 #define DecompressJpegRectBPP CONCAT2E(DecompressJpegRect,BPP)
  46 #endif
  47
  48 #ifndef RGB_TO_PIXEL
  49
  50 #define RGB_TO_PIXEL(bpp,r,g,b)                                         \
  51   (((CARD##bpp)(r) & myFormat.redMax) << myFormat.redShift |            \
  52    ((CARD##bpp)(g) & myFormat.greenMax) << myFormat.greenShift |        \
  53    ((CARD##bpp)(b) & myFormat.blueMax) << myFormat.blueShift)
  54
  55 #define RGB24_TO_PIXEL(bpp,r,g,b)                                       \
  56    ((((CARD##bpp)(r) & 0xFF) * myFormat.redMax + 127) / 255             \
  57     << myFormat.redShift |                                              \
  58     (((CARD##bpp)(g) & 0xFF) * myFormat.greenMax + 127) / 255           \
  59     << myFormat.greenShift |                                            \
  60     (((CARD##bpp)(b) & 0xFF) * myFormat.blueMax + 127) / 255            \
  61     << myFormat.blueShift)
  62
  63 #define RGB24_TO_PIXEL32(r,g,b)                                         \
  64   (((CARD32)(r) & 0xFF) << myFormat.redShift |                          \
  65    ((CARD32)(g) & 0xFF) << myFormat.greenShift |                        \
  66    ((CARD32)(b) & 0xFF) << myFormat.blueShift)
  67
  68 #endif
  69
  70 /* Type declarations */
  71
  72 typedef void (*filterPtrBPP)(int, CARDBPP *);
  73
  74 /* Prototypes */
  75
  76 static int InitFilterCopyBPP (int rw, int rh);
  77 static int InitFilterPaletteBPP (int rw, int rh);
  78 static int InitFilterGradientBPP (int rw, int rh);
  79 static void FilterCopyBPP (int numRows, CARDBPP *destBuffer);
  80 static void FilterPaletteBPP (int numRows, CARDBPP *destBuffer);
  81 static void FilterGradientBPP (int numRows, CARDBPP *destBuffer);
  82
  83 static Bool DecompressJpegRectBPP(int x, int y, int w, int h);
  84
  85 /* Definitions */
  86
  87 static Bool
  88 HandleTightBPP (int rx, int ry, int rw, int rh)
  89 {
  90   CARDBPP fill_colour;
  91   XGCValues gcv;
  92   CARD8 comp_ctl;
  93   CARD8 filter_id;
  94   filterPtrBPP filterFn;
  95   z_streamp zs;
  96   char *buffer2;
  97   int err, stream_id, compressedLen, bitsPixel;
  98   int bufferSize, rowSize, numRows, portionLen, rowsProcessed, extraBytes;
  99   CARDBPP *rawData;
 100
 101   if (!ReadFromRFBServer((char *)&comp_ctl, 1))
 102     return False;
 103
 104   /* Flush zlib streams if we are told by the server to do so. */
 105   for (stream_id = 0; stream_id < 4; stream_id++) {
 106     if ((comp_ctl & 1) && zlibStreamActive[stream_id]) {
 107       if (inflateEnd (&zlibStream[stream_id]) != Z_OK &&
 108           zlibStream[stream_id].msg != NULL)
 109         fprintf(stderr, "inflateEnd: %s\n", zlibStream[stream_id].msg);
 110       zlibStreamActive[stream_id] = False;
 111     }
 112     comp_ctl >>= 1;
 113   }
 114
 115   /* Handle solid rectangles. */
 116   if (comp_ctl == rfbTightFill) {
 117 #if BPP == 32
 118     if (myFormat.depth == 24 && myFormat.redMax == 0xFF &&
 119         myFormat.greenMax == 0xFF && myFormat.blueMax == 0xFF) {
 120       if (!ReadFromRFBServer(buffer, 3))
 121         return False;
 122       fill_colour = RGB24_TO_PIXEL32(buffer[0], buffer[1], buffer[2]);
 123     } else {
 124       if (!ReadFromRFBServer((char*)&fill_colour, sizeof(fill_colour)))
 125         return False;
 126     }
 127 #else
 128     if (!ReadFromRFBServer((char*)&fill_colour, sizeof(fill_colour)))
 129         return False;
 130 #endif
 131
 132     LockFramebuffer();
 133     FillRectangleBPP(fill_colour, rx, ry, rw, rh);
 134     UnlockFramebuffer();
 135     SyncScreenRegion(rx, ry, rw, rh);
 136     return True;
 137   }
 138
 139 #if BPP == 8
 140   if (comp_ctl == rfbTightJpeg) {
 141     fprintf(stderr, "Tight encoding: JPEG is not supported in 8 bpp mode.\n");
 142     return False;
 143   }
 144 #else
 145   if (comp_ctl == rfbTightJpeg) {
 146     return DecompressJpegRectBPP(rx, ry, rw, rh);
 147   }
 148 #endif
 149
 150   /* Quit on unsupported subencoding value. */
 151   if (comp_ctl > rfbTightMaxSubencoding) {
 152     fprintf(stderr, "Tight encoding: bad subencoding value received.\n");
 153     return False;
 154   }
 155
 156   /*
 157    * Here primary compression mode handling begins.
 158    * Data was processed with optional filter + zlib compression.
 159    */
 160
 161   /* First, we should identify a filter to use. */
 162   if ((comp_ctl & rfbTightExplicitFilter) != 0) {
 163     if (!ReadFromRFBServer((char*)&filter_id, 1))
 164       return False;
 165
 166     switch (filter_id) {
 167     case rfbTightFilterCopy:
 168       filterFn = FilterCopyBPP;
 169       bitsPixel = InitFilterCopyBPP(rw, rh);
 170       break;
 171     case rfbTightFilterPalette:
 172       filterFn = FilterPaletteBPP;
 173       bitsPixel = InitFilterPaletteBPP(rw, rh);
 174       break;
 175     case rfbTightFilterGradient:
 176       filterFn = FilterGradientBPP;
 177       bitsPixel = InitFilterGradientBPP(rw, rh);
 178       break;
 179     default:
 180       fprintf(stderr, "Tight encoding: unknown filter code received.\n");
 181       return False;
 182     }
 183   } else {
 184     filterFn = FilterCopyBPP;
 185     bitsPixel = InitFilterCopyBPP(rw, rh);
 186   }
 187   if (bitsPixel == 0) {
 188     fprintf(stderr, "Tight encoding: error receiving palette.\n");
 189     return False;
 190   }
 191
 192   /* Determine if the data should be decompressed or just copied. */
 193   rowSize = (rw * bitsPixel + 7) / 8;
 194   if (rh * rowSize < TIGHT_MIN_TO_COMPRESS) {
 195     if (!ReadFromRFBServer((char*)buffer, rh * rowSize))
 196       return False;
 197
 198     buffer2 = &buffer[TIGHT_MIN_TO_COMPRESS * 4];
 199     filterFn(rh, (CARDBPP *)buffer2);
 200     CopyDataToScreen(buffer2, rx, ry, rw, rh);
 201
 202     return True;
 203   }
 204
 205   /* Read the length (1..3 bytes) of compressed data following. */
 206   compressedLen = (int)ReadCompactLen();
 207   if (compressedLen <= 0) {
 208     fprintf(stderr, "Incorrect data received from the server.\n");
 209     return False;
 210   }
 211
 212   /* Now let's initialize compression stream if needed. */
 213   stream_id = comp_ctl & 0x03;
 214   zs = &zlibStream[stream_id];
 215   if (!zlibStreamActive[stream_id]) {
 216     zs->zalloc = Z_NULL;
 217     zs->zfree = Z_NULL;
 218     zs->opaque = Z_NULL;
 219     err = inflateInit(zs);
 220     if (err != Z_OK) {
 221       if (zs->msg != NULL)
 222         fprintf(stderr, "InflateInit error: %s.\n", zs->msg);
 223       return False;
 224     }
 225     zlibStreamActive[stream_id] = True;
 226   }
 227
 228   /* Read, decode and draw actual pixel data in a loop. */
 229
 230   bufferSize = BUFFER_SIZE * bitsPixel / (bitsPixel + BPP) & 0xFFFFFFFC;
 231   buffer2 = &buffer[bufferSize];
 232   if (rowSize > bufferSize) {
 233     /* Should be impossible when BUFFER_SIZE >= 16384 */
 234     fprintf(stderr, "Internal error: incorrect buffer size.\n");
 235     return False;
 236   }
 237
 238   rowsProcessed = 0;
 239   extraBytes = 0;
 240
 241   while (compressedLen > 0) {
 242     if (compressedLen > ZLIB_BUFFER_SIZE)
 243       portionLen = ZLIB_BUFFER_SIZE;
 244     else
 245       portionLen = compressedLen;
 246
 247     if (!ReadFromRFBServer((char*)zlib_buffer, portionLen))
 248       return False;
 249
 250     compressedLen -= portionLen;
 251
 252     zs->next_in = (Bytef *)zlib_buffer;
 253     zs->avail_in = portionLen;
 254
 255     do {
 256       zs->next_out = (Bytef *)&buffer[extraBytes];
 257       zs->avail_out = bufferSize - extraBytes;
 258
 259       err = inflate(zs, Z_SYNC_FLUSH);
 260       if (err == Z_BUF_ERROR)   /* Input exhausted -- no problem. */
 261         break;
 262       if (err != Z_OK && err != Z_STREAM_END) {
 263         if (zs->msg != NULL) {
 264           fprintf(stderr, "Inflate error: %s.\n", zs->msg);
 265         } else {
 266           fprintf(stderr, "Inflate error: %d.\n", err);
 267         }
 268         return False;
 269       }
 270
 271       numRows = (bufferSize - zs->avail_out) / rowSize;
 272
 273       filterFn(numRows, (CARDBPP *)buffer2);
 274
 275       extraBytes = bufferSize - zs->avail_out - numRows * rowSize;
 276       if (extraBytes > 0)
 277         memcpy(buffer, &buffer[numRows * rowSize], extraBytes);
 278
 279       CopyDataToScreen(buffer2, rx, ry + rowsProcessed, rw, numRows);
 280       rowsProcessed += numRows;
 281     }
 282     while (zs->avail_out == 0);
 283   }
 284
 285   if (rowsProcessed != rh) {
 286     fprintf(stderr, "Incorrect number of scan lines after decompression.\n");
 287     return False;
 288   }
 289
 290   return True;
 291 }
 292
 293 /*----------------------------------------------------------------------------
 294  *
 295  * Filter stuff.
 296  *
 297  */
 298
 299 /*
 300    The following variables are defined in rfbproto.c:
 301      static Bool cutZeros;
 302      static int rectWidth, rectColors;
 303      static CARD8 tightPalette[256*4];
 304      static CARD8 tightPrevRow[2048*3*sizeof(CARD16)];
 305 */
 306
 307 static int
 308 InitFilterCopyBPP (int rw, int rh)
 309 {
 310   rectWidth = rw;
 311
 312 #if BPP == 32
 313   if (myFormat.depth == 24 && myFormat.redMax == 0xFF &&
 314       myFormat.greenMax == 0xFF && myFormat.blueMax == 0xFF) {
 315     cutZeros = True;
 316     return 24;
 317   } else {
 318     cutZeros = False;
 319   }
 320 #endif
 321
 322   return BPP;
 323 }
 324
 325 static void
 326 FilterCopyBPP (int numRows, CARDBPP *dst)
 327 {
 328
 329 #if BPP == 32
 330   int x, y;
 331
 332   if (cutZeros) {
 333     for (y = 0; y < numRows; y++) {
 334       for (x = 0; x < rectWidth; x++) {
 335         dst[y*rectWidth+x] =
 336           RGB24_TO_PIXEL32(buffer[(y*rectWidth+x)*3],
 337                            buffer[(y*rectWidth+x)*3+1],
 338                            buffer[(y*rectWidth+x)*3+2]);
 339       }
 340     }
 341     return;
 342   }
 343 #endif
 344
 345   memcpy (dst, buffer, numRows * rectWidth * (BPP / 8));
 346 }
 347
 348 static int
 349 InitFilterGradientBPP (int rw, int rh)
 350 {
 351   int bits;
 352
 353   bits = InitFilterCopyBPP(rw, rh);
 354   if (cutZeros)
 355     memset(tightPrevRow, 0, rw * 3);
 356   else
 357     memset(tightPrevRow, 0, rw * 3 * sizeof(CARD16));
 358
 359   return bits;
 360 }
 361
 362 #if BPP == 32
 363
 364 static void
 365 FilterGradient24 (int numRows, CARD32 *dst)
 366 {
 367   int x, y, c;
 368   CARD8 thisRow[2048*3];
 369   CARD8 pix[3];
 370   int est[3];
 371
 372   for (y = 0; y < numRows; y++) {
 373
 374     /* First pixel in a row */
 375     for (c = 0; c < 3; c++) {
 376       pix[c] = tightPrevRow[c] + buffer[y*rectWidth*3+c];
 377       thisRow[c] = pix[c];
 378     }
 379     dst[y*rectWidth] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);
 380
 381     /* Remaining pixels of a row */
 382     for (x = 1; x < rectWidth; x++) {
 383       for (c = 0; c < 3; c++) {
 384         est[c] = (int)tightPrevRow[x*3+c] + (int)pix[c] -
 385                  (int)tightPrevRow[(x-1)*3+c];
 386         if (est[c] > 0xFF) {
 387           est[c] = 0xFF;
 388         } else if (est[c] < 0x00) {
 389           est[c] = 0x00;
 390         }
 391         pix[c] = (CARD8)est[c] + buffer[(y*rectWidth+x)*3+c];
 392         thisRow[x*3+c] = pix[c];
 393       }
 394       dst[y*rectWidth+x] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);
 395     }
 396
 397     memcpy(tightPrevRow, thisRow, rectWidth * 3);
 398   }
 399 }
 400
 401 #endif
 402
 403 static void
 404 FilterGradientBPP (int numRows, CARDBPP *dst)
 405 {
 406   int x, y, c;
 407   CARDBPP *src = (CARDBPP *)buffer;
 408   CARD16 *thatRow = (CARD16 *)tightPrevRow;
 409   CARD16 thisRow[2048*3];
 410   CARD16 pix[3];
 411   CARD16 max[3];
 412   int shift[3];
 413   int est[3];
 414
 415 #if BPP == 32
 416   if (cutZeros) {
 417     FilterGradient24(numRows, dst);
 418     return;
 419   }
 420 #endif
 421
 422   max[0] = myFormat.redMax;
 423   max[1] = myFormat.greenMax;
 424   max[2] = myFormat.blueMax;
 425
 426   shift[0] = myFormat.redShift;
 427   shift[1] = myFormat.greenShift;
 428   shift[2] = myFormat.blueShift;
 429
 430   for (y = 0; y < numRows; y++) {
 431
 432     /* First pixel in a row */
 433     for (c = 0; c < 3; c++) {
 434       pix[c] = (CARD16)((src[y*rectWidth] >> shift[c]) + thatRow[c] & max[c]);
 435       thisRow[c] = pix[c];
 436     }
 437     dst[y*rectWidth] = RGB_TO_PIXEL(BPP, pix[0], pix[1], pix[2]);
 438
 439     /* Remaining pixels of a row */
 440     for (x = 1; x < rectWidth; x++) {
 441       for (c = 0; c < 3; c++) {
 442         est[c] = (int)thatRow[x*3+c] + (int)pix[c] - (int)thatRow[(x-1)*3+c];
 443         if (est[c] > (int)max[c]) {
 444           est[c] = (int)max[c];
 445         } else if (est[c] < 0) {
 446           est[c] = 0;
 447         }
 448         pix[c] = (CARD16)((src[y*rectWidth+x] >> shift[c]) + est[c] & max[c]);
 449         thisRow[x*3+c] = pix[c];
 450       }
 451       dst[y*rectWidth+x] = RGB_TO_PIXEL(BPP, pix[0], pix[1], pix[2]);
 452     }
 453     memcpy(thatRow, thisRow, rectWidth * 3 * sizeof(CARD16));
 454   }
 455 }
 456
 457 static int
 458 InitFilterPaletteBPP (int rw, int rh)
 459 {
 460   int i;
 461   CARD8 numColors;
 462   CARDBPP *palette = (CARDBPP *)tightPalette;
 463
 464   rectWidth = rw;
 465
 466   if (!ReadFromRFBServer((char*)&numColors, 1))
 467     return 0;
 468
 469   rectColors = (int)numColors;
 470   if (++rectColors < 2)
 471     return 0;
 472
 473 #if BPP == 32
 474   if (myFormat.depth == 24 && myFormat.redMax == 0xFF &&
 475       myFormat.greenMax == 0xFF && myFormat.blueMax == 0xFF) {
 476     if (!ReadFromRFBServer((char*)&tightPalette, rectColors * 3))
 477       return 0;
 478     for (i = rectColors - 1; i >= 0; i--) {
 479       palette[i] = RGB24_TO_PIXEL32(tightPalette[i*3],
 480                                     tightPalette[i*3+1],
 481                                     tightPalette[i*3+2]);
 482     }
 483     return (rectColors == 2) ? 1 : 8;
 484   }
 485 #endif
 486
 487   if (!ReadFromRFBServer((char*)&tightPalette, rectColors * (BPP / 8)))
 488     return 0;
 489
 490   return (rectColors == 2) ? 1 : 8;
 491 }
 492
 493 static void
 494 FilterPaletteBPP (int numRows, CARDBPP *dst)
 495 {
 496   int x, y, b, w;
 497   CARD8 *src = (CARD8 *)buffer;
 498   CARDBPP *palette = (CARDBPP *)tightPalette;
 499
 500   if (rectColors == 2) {
 501     w = (rectWidth + 7) / 8;
 502     for (y = 0; y < numRows; y++) {
 503       for (x = 0; x < rectWidth / 8; x++) {
 504         for (b = 7; b >= 0; b--)
 505           dst[y*rectWidth+x*8+7-b] = palette[src[y*w+x] >> b & 1];
 506       }
 507       for (b = 7; b >= 8 - rectWidth % 8; b--) {
 508         dst[y*rectWidth+x*8+7-b] = palette[src[y*w+x] >> b & 1];
 509       }
 510     }
 511   } else {
 512     for (y = 0; y < numRows; y++)
 513       for (x = 0; x < rectWidth; x++)
 514         dst[y*rectWidth+x] = palette[(int)src[y*rectWidth+x]];
 515   }
 516 }
 517
 518 #if BPP != 8
 519
 520 /*----------------------------------------------------------------------------
 521  *
 522  * JPEG decompression.
 523  *
 524  */
 525
 526 /*
 527    The following variables are defined in rfbproto.c:
 528      static Bool jpegError;
 529      static struct jpeg_source_mgr jpegSrcManager;
 530      static JOCTET *jpegBufferPtr;
 531      static size_t *jpegBufferLen;
 532 */
 533
 534 static Bool
 535 DecompressJpegRectBPP(int x, int y, int w, int h)
 536 {
 537   struct jpeg_decompress_struct cinfo;
 538   struct jpeg_error_mgr jerr;
 539   int compressedLen;
 540   CARD8 *compressedData;
 541   CARDBPP *pixelPtr;
 542   JSAMPROW rowPointer[1];
 543   int dx, dy;
 544
 545   compressedLen = (int)ReadCompactLen();
 546   if (compressedLen <= 0) {
 547     fprintf(stderr, "Incorrect data received from the server.\n");
 548     return False;
 549   }
 550
 551   if (compressedLen > MAX_JPEG_SIZE) {
 552     fprintf(stderr, "To large data announced by the server.\n");
 553     return False;
 554   }
 555
 556   compressedData = malloc(compressedLen);
 557   if (compressedData == NULL) {
 558     fprintf(stderr, "Memory allocation error.\n");
 559     return False;
 560   }
 561
 562   if (!ReadFromRFBServer((char*)compressedData, compressedLen)) {
 563     free(compressedData);
 564     return False;
 565   }
 566
 567   cinfo.err = jpeg_std_error(&jerr);
 568   jpeg_create_decompress(&cinfo);
 569
 570   JpegSetSrcManager(&cinfo, compressedData, compressedLen);
 571
 572   jpeg_read_header(&cinfo, TRUE);
 573   cinfo.out_color_space = JCS_RGB;
 574
 575   jpeg_start_decompress(&cinfo);
 576   if (cinfo.output_width != w || cinfo.output_height != h ||
 577       cinfo.output_components != 3) {
 578     fprintf(stderr, "Tight Encoding: Wrong JPEG data received.\n");
 579     jpeg_destroy_decompress(&cinfo);
 580     free(compressedData);
 581     return False;
 582   }
 583
 584   rowPointer[0] = (JSAMPROW)buffer;
 585   dy = 0;
 586   while (cinfo.output_scanline < cinfo.output_height) {
 587     jpeg_read_scanlines(&cinfo, rowPointer, 1);
 588     if (jpegError) {
 589       break;
 590     }
 591     pixelPtr = (CARDBPP *)&buffer[BUFFER_SIZE / 2];
 592     for (dx = 0; dx < w; dx++) {
 593       *pixelPtr++ =
 594         RGB24_TO_PIXEL(BPP, buffer[dx*3], buffer[dx*3+1], buffer[dx*3+2]);
 595     }
 596     CopyDataToScreen(&buffer[BUFFER_SIZE / 2], x, y + dy, w, 1);
 597     dy++;
 598   }
 599
 600   if (!jpegError)
 601     jpeg_finish_decompress(&cinfo);
 602
 603   jpeg_destroy_decompress(&cinfo);
 604   free(compressedData);
 605
 606   return !jpegError;
 607 }
 608
 609 #endif
 610