libs/tiff/libtiff/tif_predict.c

   1 /*
   2  * Copyright (c) 1988-1997 Sam Leffler
   3  * Copyright (c) 1991-1997 Silicon Graphics, Inc.
   4  *
   5  * Permission to use, copy, modify, distribute, and sell this software and
   6  * its documentation for any purpose is hereby granted without fee, provided
   7  * that (i) the above copyright notices and this permission notice appear in
   8  * all copies of the software and related documentation, and (ii) the names of
   9  * Sam Leffler and Silicon Graphics may not be used in any advertising or
  10  * publicity relating to the software without the specific, prior written
  11  * permission of Sam Leffler and Silicon Graphics.
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND,
  14  * EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY
  15  * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
  16  *
  17  * IN NO EVENT SHALL SAM LEFFLER OR SILICON GRAPHICS BE LIABLE FOR
  18  * ANY SPECIAL, INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND,
  19  * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20  * WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF
  21  * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22  * OF THIS SOFTWARE.
  23  */
  24
  25 /*
  26  * TIFF Library.
  27  *
  28  * Predictor Tag Support (used by multiple codecs).
  29  */
  30 #include "tif_predict.h"
  31 #include "tiffiop.h"
  32
  33 #define PredictorState(tif) ((TIFFPredictorState *)(tif)->tif_data)
  34
  35 static int horAcc8(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  36 static int horAcc16(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  37 static int horAcc32(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  38 static int horAcc64(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  39 static int swabHorAcc16(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  40 static int swabHorAcc32(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  41 static int swabHorAcc64(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  42 static int horDiff8(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  43 static int horDiff16(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  44 static int horDiff32(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  45 static int horDiff64(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  46 static int swabHorDiff16(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  47 static int swabHorDiff32(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  48 static int swabHorDiff64(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  49 static int fpAcc(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  50 static int fpDiff(TIFF *tif, uint8_t *cp0, tmsize_t cc);
  51 static int PredictorDecodeRow(TIFF *tif, uint8_t *op0, tmsize_t occ0,
  52                               uint16_t s);
  53 static int PredictorDecodeTile(TIFF *tif, uint8_t *op0, tmsize_t occ0,
  54                                uint16_t s);
  55 static int PredictorEncodeRow(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s);
  56 static int PredictorEncodeTile(TIFF *tif, uint8_t *bp0, tmsize_t cc0,
  57                                uint16_t s);
  58
  59 static int PredictorSetup(TIFF *tif)
  60 {
  61     static const char module[] = "PredictorSetup";
  62
  63     TIFFPredictorState *sp = PredictorState(tif);
  64     TIFFDirectory *td = &tif->tif_dir;
  65
  66     switch (sp->predictor) /* no differencing */
  67     {
  68         case PREDICTOR_NONE:
  69             return 1;
  70         case PREDICTOR_HORIZONTAL:
  71             if (td->td_bitspersample != 8 && td->td_bitspersample != 16 &&
  72                 td->td_bitspersample != 32 && td->td_bitspersample != 64)
  73             {
  74                 TIFFErrorExtR(tif, module,
  75                               "Horizontal differencing \"Predictor\" not "
  76                               "supported with %" PRIu16 "-bit samples",
  77                               td->td_bitspersample);
  78                 return 0;
  79             }
  80             break;
  81         case PREDICTOR_FLOATINGPOINT:
  82             if (td->td_sampleformat != SAMPLEFORMAT_IEEEFP)
  83             {
  84                 TIFFErrorExtR(
  85                     tif, module,
  86                     "Floating point \"Predictor\" not supported with %" PRIu16
  87                     " data format",
  88                     td->td_sampleformat);
  89                 return 0;
  90             }
  91             if (td->td_bitspersample != 16 && td->td_bitspersample != 24 &&
  92                 td->td_bitspersample != 32 && td->td_bitspersample != 64)
  93             { /* Should 64 be allowed? */
  94                 TIFFErrorExtR(
  95                     tif, module,
  96                     "Floating point \"Predictor\" not supported with %" PRIu16
  97                     "-bit samples",
  98                     td->td_bitspersample);
  99                 return 0;
 100             }
 101             break;
 102         default:
 103             TIFFErrorExtR(tif, module, "\"Predictor\" value %d not supported",
 104                           sp->predictor);
 105             return 0;
 106     }
 107     sp->stride =
 108         (td->td_planarconfig == PLANARCONFIG_CONTIG ? td->td_samplesperpixel
 109                                                     : 1);
 110     /*
 111      * Calculate the scanline/tile-width size in bytes.
 112      */
 113     if (isTiled(tif))
 114         sp->rowsize = TIFFTileRowSize(tif);
 115     else
 116         sp->rowsize = TIFFScanlineSize(tif);
 117     if (sp->rowsize == 0)
 118         return 0;
 119
 120     return 1;
 121 }
 122
 123 static int PredictorSetupDecode(TIFF *tif)
 124 {
 125     TIFFPredictorState *sp = PredictorState(tif);
 126     TIFFDirectory *td = &tif->tif_dir;
 127
 128     /* Note: when PredictorSetup() fails, the effets of setupdecode() */
 129     /* will not be "canceled" so setupdecode() might be robust to */
 130     /* be called several times. */
 131     if (!(*sp->setupdecode)(tif) || !PredictorSetup(tif))
 132         return 0;
 133
 134     if (sp->predictor == 2)
 135     {
 136         switch (td->td_bitspersample)
 137         {
 138             case 8:
 139                 sp->decodepfunc = horAcc8;
 140                 break;
 141             case 16:
 142                 sp->decodepfunc = horAcc16;
 143                 break;
 144             case 32:
 145                 sp->decodepfunc = horAcc32;
 146                 break;
 147             case 64:
 148                 sp->decodepfunc = horAcc64;
 149                 break;
 150         }
 151         /*
 152          * Override default decoding method with one that does the
 153          * predictor stuff.
 154          */
 155         if (tif->tif_decoderow != PredictorDecodeRow)
 156         {
 157             sp->decoderow = tif->tif_decoderow;
 158             tif->tif_decoderow = PredictorDecodeRow;
 159             sp->decodestrip = tif->tif_decodestrip;
 160             tif->tif_decodestrip = PredictorDecodeTile;
 161             sp->decodetile = tif->tif_decodetile;
 162             tif->tif_decodetile = PredictorDecodeTile;
 163         }
 164
 165         /*
 166          * If the data is horizontally differenced 16-bit data that
 167          * requires byte-swapping, then it must be byte swapped before
 168          * the accumulation step.  We do this with a special-purpose
 169          * routine and override the normal post decoding logic that
 170          * the library setup when the directory was read.
 171          */
 172         if (tif->tif_flags & TIFF_SWAB)
 173         {
 174             if (sp->decodepfunc == horAcc16)
 175             {
 176                 sp->decodepfunc = swabHorAcc16;
 177                 tif->tif_postdecode = _TIFFNoPostDecode;
 178             }
 179             else if (sp->decodepfunc == horAcc32)
 180             {
 181                 sp->decodepfunc = swabHorAcc32;
 182                 tif->tif_postdecode = _TIFFNoPostDecode;
 183             }
 184             else if (sp->decodepfunc == horAcc64)
 185             {
 186                 sp->decodepfunc = swabHorAcc64;
 187                 tif->tif_postdecode = _TIFFNoPostDecode;
 188             }
 189         }
 190     }
 191
 192     else if (sp->predictor == 3)
 193     {
 194         sp->decodepfunc = fpAcc;
 195         /*
 196          * Override default decoding method with one that does the
 197          * predictor stuff.
 198          */
 199         if (tif->tif_decoderow != PredictorDecodeRow)
 200         {
 201             sp->decoderow = tif->tif_decoderow;
 202             tif->tif_decoderow = PredictorDecodeRow;
 203             sp->decodestrip = tif->tif_decodestrip;
 204             tif->tif_decodestrip = PredictorDecodeTile;
 205             sp->decodetile = tif->tif_decodetile;
 206             tif->tif_decodetile = PredictorDecodeTile;
 207         }
 208         /*
 209          * The data should not be swapped outside of the floating
 210          * point predictor, the accumulation routine should return
 211          * byres in the native order.
 212          */
 213         if (tif->tif_flags & TIFF_SWAB)
 214         {
 215             tif->tif_postdecode = _TIFFNoPostDecode;
 216         }
 217         /*
 218          * Allocate buffer to keep the decoded bytes before
 219          * rearranging in the right order
 220          */
 221     }
 222
 223     return 1;
 224 }
 225
 226 static int PredictorSetupEncode(TIFF *tif)
 227 {
 228     TIFFPredictorState *sp = PredictorState(tif);
 229     TIFFDirectory *td = &tif->tif_dir;
 230
 231     if (!(*sp->setupencode)(tif) || !PredictorSetup(tif))
 232         return 0;
 233
 234     if (sp->predictor == 2)
 235     {
 236         switch (td->td_bitspersample)
 237         {
 238             case 8:
 239                 sp->encodepfunc = horDiff8;
 240                 break;
 241             case 16:
 242                 sp->encodepfunc = horDiff16;
 243                 break;
 244             case 32:
 245                 sp->encodepfunc = horDiff32;
 246                 break;
 247             case 64:
 248                 sp->encodepfunc = horDiff64;
 249                 break;
 250         }
 251         /*
 252          * Override default encoding method with one that does the
 253          * predictor stuff.
 254          */
 255         if (tif->tif_encoderow != PredictorEncodeRow)
 256         {
 257             sp->encoderow = tif->tif_encoderow;
 258             tif->tif_encoderow = PredictorEncodeRow;
 259             sp->encodestrip = tif->tif_encodestrip;
 260             tif->tif_encodestrip = PredictorEncodeTile;
 261             sp->encodetile = tif->tif_encodetile;
 262             tif->tif_encodetile = PredictorEncodeTile;
 263         }
 264
 265         /*
 266          * If the data is horizontally differenced 16-bit data that
 267          * requires byte-swapping, then it must be byte swapped after
 268          * the differentiation step.  We do this with a special-purpose
 269          * routine and override the normal post decoding logic that
 270          * the library setup when the directory was read.
 271          */
 272         if (tif->tif_flags & TIFF_SWAB)
 273         {
 274             if (sp->encodepfunc == horDiff16)
 275             {
 276                 sp->encodepfunc = swabHorDiff16;
 277                 tif->tif_postdecode = _TIFFNoPostDecode;
 278             }
 279             else if (sp->encodepfunc == horDiff32)
 280             {
 281                 sp->encodepfunc = swabHorDiff32;
 282                 tif->tif_postdecode = _TIFFNoPostDecode;
 283             }
 284             else if (sp->encodepfunc == horDiff64)
 285             {
 286                 sp->encodepfunc = swabHorDiff64;
 287                 tif->tif_postdecode = _TIFFNoPostDecode;
 288             }
 289         }
 290     }
 291
 292     else if (sp->predictor == 3)
 293     {
 294         sp->encodepfunc = fpDiff;
 295         /*
 296          * Override default encoding method with one that does the
 297          * predictor stuff.
 298          */
 299         if (tif->tif_encoderow != PredictorEncodeRow)
 300         {
 301             sp->encoderow = tif->tif_encoderow;
 302             tif->tif_encoderow = PredictorEncodeRow;
 303             sp->encodestrip = tif->tif_encodestrip;
 304             tif->tif_encodestrip = PredictorEncodeTile;
 305             sp->encodetile = tif->tif_encodetile;
 306             tif->tif_encodetile = PredictorEncodeTile;
 307         }
 308     }
 309
 310     return 1;
 311 }
 312
 313 #define REPEAT4(n, op)                                                         \
 314     switch (n)                                                                 \
 315     {                                                                          \
 316         default:                                                               \
 317         {                                                                      \
 318             tmsize_t i;                                                        \
 319             for (i = n - 4; i > 0; i--)                                        \
 320             {                                                                  \
 321                 op;                                                            \
 322             }                                                                  \
 323         } /*-fallthrough*/                                                     \
 324         case 4:                                                                \
 325             op; /*-fallthrough*/                                               \
 326         case 3:                                                                \
 327             op; /*-fallthrough*/                                               \
 328         case 2:                                                                \
 329             op; /*-fallthrough*/                                               \
 330         case 1:                                                                \
 331             op; /*-fallthrough*/                                               \
 332         case 0:;                                                               \
 333     }
 334
 335 /* Remarks related to C standard compliance in all below functions : */
 336 /* - to avoid any undefined behavior, we only operate on unsigned types */
 337 /*   since the behavior of "overflows" is defined (wrap over) */
 338 /* - when storing into the byte stream, we explicitly mask with 0xff so */
 339 /*   as to make icc -check=conversions happy (not necessary by the standard) */
 340
 341 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 342 static int horAcc8(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 343 {
 344     tmsize_t stride = PredictorState(tif)->stride;
 345
 346     unsigned char *cp = (unsigned char *)cp0;
 347     if ((cc % stride) != 0)
 348     {
 349         TIFFErrorExtR(tif, "horAcc8", "%s", "(cc%stride)!=0");
 350         return 0;
 351     }
 352
 353     if (cc > stride)
 354     {
 355         /*
 356          * Pipeline the most common cases.
 357          */
 358         if (stride == 3)
 359         {
 360             unsigned int cr = cp[0];
 361             unsigned int cg = cp[1];
 362             unsigned int cb = cp[2];
 363             tmsize_t i = stride;
 364             for (; i < cc; i += stride)
 365             {
 366                 cp[i + 0] = (unsigned char)((cr += cp[i + 0]) & 0xff);
 367                 cp[i + 1] = (unsigned char)((cg += cp[i + 1]) & 0xff);
 368                 cp[i + 2] = (unsigned char)((cb += cp[i + 2]) & 0xff);
 369             }
 370         }
 371         else if (stride == 4)
 372         {
 373             unsigned int cr = cp[0];
 374             unsigned int cg = cp[1];
 375             unsigned int cb = cp[2];
 376             unsigned int ca = cp[3];
 377             tmsize_t i = stride;
 378             for (; i < cc; i += stride)
 379             {
 380                 cp[i + 0] = (unsigned char)((cr += cp[i + 0]) & 0xff);
 381                 cp[i + 1] = (unsigned char)((cg += cp[i + 1]) & 0xff);
 382                 cp[i + 2] = (unsigned char)((cb += cp[i + 2]) & 0xff);
 383                 cp[i + 3] = (unsigned char)((ca += cp[i + 3]) & 0xff);
 384             }
 385         }
 386         else
 387         {
 388             cc -= stride;
 389             do
 390             {
 391                 REPEAT4(stride,
 392                         cp[stride] = (unsigned char)((cp[stride] + *cp) & 0xff);
 393                         cp++)
 394                 cc -= stride;
 395             } while (cc > 0);
 396         }
 397     }
 398     return 1;
 399 }
 400
 401 static int swabHorAcc16(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 402 {
 403     uint16_t *wp = (uint16_t *)cp0;
 404     tmsize_t wc = cc / 2;
 405
 406     TIFFSwabArrayOfShort(wp, wc);
 407     return horAcc16(tif, cp0, cc);
 408 }
 409
 410 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 411 static int horAcc16(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 412 {
 413     tmsize_t stride = PredictorState(tif)->stride;
 414     uint16_t *wp = (uint16_t *)cp0;
 415     tmsize_t wc = cc / 2;
 416
 417     if ((cc % (2 * stride)) != 0)
 418     {
 419         TIFFErrorExtR(tif, "horAcc16", "%s", "cc%(2*stride))!=0");
 420         return 0;
 421     }
 422
 423     if (wc > stride)
 424     {
 425         wc -= stride;
 426         do
 427         {
 428             REPEAT4(stride, wp[stride] = (uint16_t)(((unsigned int)wp[stride] +
 429                                                      (unsigned int)wp[0]) &
 430                                                     0xffff);
 431                     wp++)
 432             wc -= stride;
 433         } while (wc > 0);
 434     }
 435     return 1;
 436 }
 437
 438 static int swabHorAcc32(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 439 {
 440     uint32_t *wp = (uint32_t *)cp0;
 441     tmsize_t wc = cc / 4;
 442
 443     TIFFSwabArrayOfLong(wp, wc);
 444     return horAcc32(tif, cp0, cc);
 445 }
 446
 447 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 448 static int horAcc32(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 449 {
 450     tmsize_t stride = PredictorState(tif)->stride;
 451     uint32_t *wp = (uint32_t *)cp0;
 452     tmsize_t wc = cc / 4;
 453
 454     if ((cc % (4 * stride)) != 0)
 455     {
 456         TIFFErrorExtR(tif, "horAcc32", "%s", "cc%(4*stride))!=0");
 457         return 0;
 458     }
 459
 460     if (wc > stride)
 461     {
 462         wc -= stride;
 463         do
 464         {
 465             REPEAT4(stride, wp[stride] += wp[0]; wp++)
 466             wc -= stride;
 467         } while (wc > 0);
 468     }
 469     return 1;
 470 }
 471
 472 static int swabHorAcc64(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 473 {
 474     uint64_t *wp = (uint64_t *)cp0;
 475     tmsize_t wc = cc / 8;
 476
 477     TIFFSwabArrayOfLong8(wp, wc);
 478     return horAcc64(tif, cp0, cc);
 479 }
 480
 481 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 482 static int horAcc64(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 483 {
 484     tmsize_t stride = PredictorState(tif)->stride;
 485     uint64_t *wp = (uint64_t *)cp0;
 486     tmsize_t wc = cc / 8;
 487
 488     if ((cc % (8 * stride)) != 0)
 489     {
 490         TIFFErrorExtR(tif, "horAcc64", "%s", "cc%(8*stride))!=0");
 491         return 0;
 492     }
 493
 494     if (wc > stride)
 495     {
 496         wc -= stride;
 497         do
 498         {
 499             REPEAT4(stride, wp[stride] += wp[0]; wp++)
 500             wc -= stride;
 501         } while (wc > 0);
 502     }
 503     return 1;
 504 }
 505
 506 /*
 507  * Floating point predictor accumulation routine.
 508  */
 509 static int fpAcc(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 510 {
 511     tmsize_t stride = PredictorState(tif)->stride;
 512     uint32_t bps = tif->tif_dir.td_bitspersample / 8;
 513     tmsize_t wc = cc / bps;
 514     tmsize_t count = cc;
 515     uint8_t *cp = (uint8_t *)cp0;
 516     uint8_t *tmp;
 517
 518     if (cc % (bps * stride) != 0)
 519     {
 520         TIFFErrorExtR(tif, "fpAcc", "%s", "cc%(bps*stride))!=0");
 521         return 0;
 522     }
 523
 524     tmp = (uint8_t *)_TIFFmallocExt(tif, cc);
 525     if (!tmp)
 526         return 0;
 527
 528     while (count > stride)
 529     {
 530         REPEAT4(stride,
 531                 cp[stride] = (unsigned char)((cp[stride] + cp[0]) & 0xff);
 532                 cp++)
 533         count -= stride;
 534     }
 535
 536     _TIFFmemcpy(tmp, cp0, cc);
 537     cp = (uint8_t *)cp0;
 538     for (count = 0; count < wc; count++)
 539     {
 540         uint32_t byte;
 541         for (byte = 0; byte < bps; byte++)
 542         {
 543 #if WORDS_BIGENDIAN
 544             cp[bps * count + byte] = tmp[byte * wc + count];
 545 #else
 546             cp[bps * count + byte] = tmp[(bps - byte - 1) * wc + count];
 547 #endif
 548         }
 549     }
 550     _TIFFfreeExt(tif, tmp);
 551     return 1;
 552 }
 553
 554 /*
 555  * Decode a scanline and apply the predictor routine.
 556  */
 557 static int PredictorDecodeRow(TIFF *tif, uint8_t *op0, tmsize_t occ0,
 558                               uint16_t s)
 559 {
 560     TIFFPredictorState *sp = PredictorState(tif);
 561
 562     assert(sp != NULL);
 563     assert(sp->decoderow != NULL);
 564     assert(sp->decodepfunc != NULL);
 565
 566     if ((*sp->decoderow)(tif, op0, occ0, s))
 567     {
 568         return (*sp->decodepfunc)(tif, op0, occ0);
 569     }
 570     else
 571         return 0;
 572 }
 573
 574 /*
 575  * Decode a tile/strip and apply the predictor routine.
 576  * Note that horizontal differencing must be done on a
 577  * row-by-row basis.  The width of a "row" has already
 578  * been calculated at pre-decode time according to the
 579  * strip/tile dimensions.
 580  */
 581 static int PredictorDecodeTile(TIFF *tif, uint8_t *op0, tmsize_t occ0,
 582                                uint16_t s)
 583 {
 584     TIFFPredictorState *sp = PredictorState(tif);
 585
 586     assert(sp != NULL);
 587     assert(sp->decodetile != NULL);
 588
 589     if ((*sp->decodetile)(tif, op0, occ0, s))
 590     {
 591         tmsize_t rowsize = sp->rowsize;
 592         assert(rowsize > 0);
 593         if ((occ0 % rowsize) != 0)
 594         {
 595             TIFFErrorExtR(tif, "PredictorDecodeTile", "%s",
 596                           "occ0%rowsize != 0");
 597             return 0;
 598         }
 599         assert(sp->decodepfunc != NULL);
 600         while (occ0 > 0)
 601         {
 602             if (!(*sp->decodepfunc)(tif, op0, rowsize))
 603                 return 0;
 604             occ0 -= rowsize;
 605             op0 += rowsize;
 606         }
 607         return 1;
 608     }
 609     else
 610         return 0;
 611 }
 612
 613 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 614 static int horDiff8(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 615 {
 616     TIFFPredictorState *sp = PredictorState(tif);
 617     tmsize_t stride = sp->stride;
 618     unsigned char *cp = (unsigned char *)cp0;
 619
 620     if ((cc % stride) != 0)
 621     {
 622         TIFFErrorExtR(tif, "horDiff8", "%s", "(cc%stride)!=0");
 623         return 0;
 624     }
 625
 626     if (cc > stride)
 627     {
 628         cc -= stride;
 629         /*
 630          * Pipeline the most common cases.
 631          */
 632         if (stride == 3)
 633         {
 634             unsigned int r1, g1, b1;
 635             unsigned int r2 = cp[0];
 636             unsigned int g2 = cp[1];
 637             unsigned int b2 = cp[2];
 638             do
 639             {
 640                 r1 = cp[3];
 641                 cp[3] = (unsigned char)((r1 - r2) & 0xff);
 642                 r2 = r1;
 643                 g1 = cp[4];
 644                 cp[4] = (unsigned char)((g1 - g2) & 0xff);
 645                 g2 = g1;
 646                 b1 = cp[5];
 647                 cp[5] = (unsigned char)((b1 - b2) & 0xff);
 648                 b2 = b1;
 649                 cp += 3;
 650             } while ((cc -= 3) > 0);
 651         }
 652         else if (stride == 4)
 653         {
 654             unsigned int r1, g1, b1, a1;
 655             unsigned int r2 = cp[0];
 656             unsigned int g2 = cp[1];
 657             unsigned int b2 = cp[2];
 658             unsigned int a2 = cp[3];
 659             do
 660             {
 661                 r1 = cp[4];
 662                 cp[4] = (unsigned char)((r1 - r2) & 0xff);
 663                 r2 = r1;
 664                 g1 = cp[5];
 665                 cp[5] = (unsigned char)((g1 - g2) & 0xff);
 666                 g2 = g1;
 667                 b1 = cp[6];
 668                 cp[6] = (unsigned char)((b1 - b2) & 0xff);
 669                 b2 = b1;
 670                 a1 = cp[7];
 671                 cp[7] = (unsigned char)((a1 - a2) & 0xff);
 672                 a2 = a1;
 673                 cp += 4;
 674             } while ((cc -= 4) > 0);
 675         }
 676         else
 677         {
 678             cp += cc - 1;
 679             do
 680             {
 681                 REPEAT4(stride,
 682                         cp[stride] =
 683                             (unsigned char)((cp[stride] - cp[0]) & 0xff);
 684                         cp--)
 685             } while ((cc -= stride) > 0);
 686         }
 687     }
 688     return 1;
 689 }
 690
 691 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 692 static int horDiff16(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 693 {
 694     TIFFPredictorState *sp = PredictorState(tif);
 695     tmsize_t stride = sp->stride;
 696     uint16_t *wp = (uint16_t *)cp0;
 697     tmsize_t wc = cc / 2;
 698
 699     if ((cc % (2 * stride)) != 0)
 700     {
 701         TIFFErrorExtR(tif, "horDiff8", "%s", "(cc%(2*stride))!=0");
 702         return 0;
 703     }
 704
 705     if (wc > stride)
 706     {
 707         wc -= stride;
 708         wp += wc - 1;
 709         do
 710         {
 711             REPEAT4(stride, wp[stride] = (uint16_t)(((unsigned int)wp[stride] -
 712                                                      (unsigned int)wp[0]) &
 713                                                     0xffff);
 714                     wp--)
 715             wc -= stride;
 716         } while (wc > 0);
 717     }
 718     return 1;
 719 }
 720
 721 static int swabHorDiff16(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 722 {
 723     uint16_t *wp = (uint16_t *)cp0;
 724     tmsize_t wc = cc / 2;
 725
 726     if (!horDiff16(tif, cp0, cc))
 727         return 0;
 728
 729     TIFFSwabArrayOfShort(wp, wc);
 730     return 1;
 731 }
 732
 733 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 734 static int horDiff32(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 735 {
 736     TIFFPredictorState *sp = PredictorState(tif);
 737     tmsize_t stride = sp->stride;
 738     uint32_t *wp = (uint32_t *)cp0;
 739     tmsize_t wc = cc / 4;
 740
 741     if ((cc % (4 * stride)) != 0)
 742     {
 743         TIFFErrorExtR(tif, "horDiff32", "%s", "(cc%(4*stride))!=0");
 744         return 0;
 745     }
 746
 747     if (wc > stride)
 748     {
 749         wc -= stride;
 750         wp += wc - 1;
 751         do
 752         {
 753             REPEAT4(stride, wp[stride] -= wp[0]; wp--)
 754             wc -= stride;
 755         } while (wc > 0);
 756     }
 757     return 1;
 758 }
 759
 760 static int swabHorDiff32(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 761 {
 762     uint32_t *wp = (uint32_t *)cp0;
 763     tmsize_t wc = cc / 4;
 764
 765     if (!horDiff32(tif, cp0, cc))
 766         return 0;
 767
 768     TIFFSwabArrayOfLong(wp, wc);
 769     return 1;
 770 }
 771
 772 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 773 static int horDiff64(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 774 {
 775     TIFFPredictorState *sp = PredictorState(tif);
 776     tmsize_t stride = sp->stride;
 777     uint64_t *wp = (uint64_t *)cp0;
 778     tmsize_t wc = cc / 8;
 779
 780     if ((cc % (8 * stride)) != 0)
 781     {
 782         TIFFErrorExtR(tif, "horDiff64", "%s", "(cc%(8*stride))!=0");
 783         return 0;
 784     }
 785
 786     if (wc > stride)
 787     {
 788         wc -= stride;
 789         wp += wc - 1;
 790         do
 791         {
 792             REPEAT4(stride, wp[stride] -= wp[0]; wp--)
 793             wc -= stride;
 794         } while (wc > 0);
 795     }
 796     return 1;
 797 }
 798
 799 static int swabHorDiff64(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 800 {
 801     uint64_t *wp = (uint64_t *)cp0;
 802     tmsize_t wc = cc / 8;
 803
 804     if (!horDiff64(tif, cp0, cc))
 805         return 0;
 806
 807     TIFFSwabArrayOfLong8(wp, wc);
 808     return 1;
 809 }
 810
 811 /*
 812  * Floating point predictor differencing routine.
 813  */
 814 TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW
 815 static int fpDiff(TIFF *tif, uint8_t *cp0, tmsize_t cc)
 816 {
 817     tmsize_t stride = PredictorState(tif)->stride;
 818     uint32_t bps = tif->tif_dir.td_bitspersample / 8;
 819     tmsize_t wc = cc / bps;
 820     tmsize_t count;
 821     uint8_t *cp = (uint8_t *)cp0;
 822     uint8_t *tmp;
 823
 824     if ((cc % (bps * stride)) != 0)
 825     {
 826         TIFFErrorExtR(tif, "fpDiff", "%s", "(cc%(bps*stride))!=0");
 827         return 0;
 828     }
 829
 830     tmp = (uint8_t *)_TIFFmallocExt(tif, cc);
 831     if (!tmp)
 832         return 0;
 833
 834     _TIFFmemcpy(tmp, cp0, cc);
 835     for (count = 0; count < wc; count++)
 836     {
 837         uint32_t byte;
 838         for (byte = 0; byte < bps; byte++)
 839         {
 840 #if WORDS_BIGENDIAN
 841             cp[byte * wc + count] = tmp[bps * count + byte];
 842 #else
 843             cp[(bps - byte - 1) * wc + count] = tmp[bps * count + byte];
 844 #endif
 845         }
 846     }
 847     _TIFFfreeExt(tif, tmp);
 848
 849     cp = (uint8_t *)cp0;
 850     cp += cc - stride - 1;
 851     for (count = cc; count > stride; count -= stride)
 852         REPEAT4(stride,
 853                 cp[stride] = (unsigned char)((cp[stride] - cp[0]) & 0xff);
 854                 cp--)
 855     return 1;
 856 }
 857
 858 static int PredictorEncodeRow(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s)
 859 {
 860     TIFFPredictorState *sp = PredictorState(tif);
 861
 862     assert(sp != NULL);
 863     assert(sp->encodepfunc != NULL);
 864     assert(sp->encoderow != NULL);
 865
 866     /* XXX horizontal differencing alters user's data XXX */
 867     if (!(*sp->encodepfunc)(tif, bp, cc))
 868         return 0;
 869     return (*sp->encoderow)(tif, bp, cc, s);
 870 }
 871
 872 static int PredictorEncodeTile(TIFF *tif, uint8_t *bp0, tmsize_t cc0,
 873                                uint16_t s)
 874 {
 875     static const char module[] = "PredictorEncodeTile";
 876     TIFFPredictorState *sp = PredictorState(tif);
 877     uint8_t *working_copy;
 878     tmsize_t cc = cc0, rowsize;
 879     unsigned char *bp;
 880     int result_code;
 881
 882     assert(sp != NULL);
 883     assert(sp->encodepfunc != NULL);
 884     assert(sp->encodetile != NULL);
 885
 886     /*
 887      * Do predictor manipulation in a working buffer to avoid altering
 888      * the callers buffer. http://trac.osgeo.org/gdal/ticket/1965
 889      */
 890     working_copy = (uint8_t *)_TIFFmallocExt(tif, cc0);
 891     if (working_copy == NULL)
 892     {
 893         TIFFErrorExtR(tif, module,
 894                       "Out of memory allocating %" PRId64 " byte temp buffer.",
 895                       (int64_t)cc0);
 896         return 0;
 897     }
 898     memcpy(working_copy, bp0, cc0);
 899     bp = working_copy;
 900
 901     rowsize = sp->rowsize;
 902     assert(rowsize > 0);
 903     if ((cc0 % rowsize) != 0)
 904     {
 905         TIFFErrorExtR(tif, "PredictorEncodeTile", "%s", "(cc0%rowsize)!=0");
 906         _TIFFfreeExt(tif, working_copy);
 907         return 0;
 908     }
 909     while (cc > 0)
 910     {
 911         (*sp->encodepfunc)(tif, bp, rowsize);
 912         cc -= rowsize;
 913         bp += rowsize;
 914     }
 915     result_code = (*sp->encodetile)(tif, working_copy, cc0, s);
 916
 917     _TIFFfreeExt(tif, working_copy);
 918
 919     return result_code;
 920 }
 921
 922 #define FIELD_PREDICTOR (FIELD_CODEC + 0) /* XXX */
 923
 924 static const TIFFField predictFields[] = {
 925     {TIFFTAG_PREDICTOR, 1, 1, TIFF_SHORT, 0, TIFF_SETGET_UINT16,
 926      TIFF_SETGET_UINT16, FIELD_PREDICTOR, FALSE, FALSE, "Predictor", NULL},
 927 };
 928
 929 static int PredictorVSetField(TIFF *tif, uint32_t tag, va_list ap)
 930 {
 931     TIFFPredictorState *sp = PredictorState(tif);
 932
 933     assert(sp != NULL);
 934     assert(sp->vsetparent != NULL);
 935
 936     switch (tag)
 937     {
 938         case TIFFTAG_PREDICTOR:
 939             sp->predictor = (uint16_t)va_arg(ap, uint16_vap);
 940             TIFFSetFieldBit(tif, FIELD_PREDICTOR);
 941             break;
 942         default:
 943             return (*sp->vsetparent)(tif, tag, ap);
 944     }
 945     tif->tif_flags |= TIFF_DIRTYDIRECT;
 946     return 1;
 947 }
 948
 949 static int PredictorVGetField(TIFF *tif, uint32_t tag, va_list ap)
 950 {
 951     TIFFPredictorState *sp = PredictorState(tif);
 952
 953     assert(sp != NULL);
 954     assert(sp->vgetparent != NULL);
 955
 956     switch (tag)
 957     {
 958         case TIFFTAG_PREDICTOR:
 959             *va_arg(ap, uint16_t *) = (uint16_t)sp->predictor;
 960             break;
 961         default:
 962             return (*sp->vgetparent)(tif, tag, ap);
 963     }
 964     return 1;
 965 }
 966
 967 static void PredictorPrintDir(TIFF *tif, FILE *fd, long flags)
 968 {
 969     TIFFPredictorState *sp = PredictorState(tif);
 970
 971     (void)flags;
 972     if (TIFFFieldSet(tif, FIELD_PREDICTOR))
 973     {
 974         fprintf(fd, "  Predictor: ");
 975         switch (sp->predictor)
 976         {
 977             case 1:
 978                 fprintf(fd, "none ");
 979                 break;
 980             case 2:
 981                 fprintf(fd, "horizontal differencing ");
 982                 break;
 983             case 3:
 984                 fprintf(fd, "floating point predictor ");
 985                 break;
 986         }
 987         fprintf(fd, "%d (0x%x)\n", sp->predictor, sp->predictor);
 988     }
 989     if (sp->printdir)
 990         (*sp->printdir)(tif, fd, flags);
 991 }
 992
 993 int TIFFPredictorInit(TIFF *tif)
 994 {
 995     TIFFPredictorState *sp = PredictorState(tif);
 996
 997     assert(sp != 0);
 998
 999     /*
1000      * Merge codec-specific tag information.
1001      */
1002     if (!_TIFFMergeFields(tif, predictFields, TIFFArrayCount(predictFields)))
1003     {
1004         TIFFErrorExtR(tif, "TIFFPredictorInit",
1005                       "Merging Predictor codec-specific tags failed");
1006         return 0;
1007     }
1008
1009     /*
1010      * Override parent get/set field methods.
1011      */
1012     sp->vgetparent = tif->tif_tagmethods.vgetfield;
1013     tif->tif_tagmethods.vgetfield =
1014         PredictorVGetField; /* hook for predictor tag */
1015     sp->vsetparent = tif->tif_tagmethods.vsetfield;
1016     tif->tif_tagmethods.vsetfield =
1017         PredictorVSetField; /* hook for predictor tag */
1018     sp->printdir = tif->tif_tagmethods.printdir;
1019     tif->tif_tagmethods.printdir =
1020         PredictorPrintDir; /* hook for predictor tag */
1021
1022     sp->setupdecode = tif->tif_setupdecode;
1023     tif->tif_setupdecode = PredictorSetupDecode;
1024     sp->setupencode = tif->tif_setupencode;
1025     tif->tif_setupencode = PredictorSetupEncode;
1026
1027     sp->predictor = 1;      /* default value */
1028     sp->encodepfunc = NULL; /* no predictor routine */
1029     sp->decodepfunc = NULL; /* no predictor routine */
1030     return 1;
1031 }
1032
1033 int TIFFPredictorCleanup(TIFF *tif)
1034 {
1035     TIFFPredictorState *sp = PredictorState(tif);
1036
1037     assert(sp != 0);
1038
1039     tif->tif_tagmethods.vgetfield = sp->vgetparent;
1040     tif->tif_tagmethods.vsetfield = sp->vsetparent;
1041     tif->tif_tagmethods.printdir = sp->printdir;
1042     tif->tif_setupdecode = sp->setupdecode;
1043     tif->tif_setupencode = sp->setupencode;
1044
1045     return 1;
1046 }