tools/ucl/uclpack.c

   1 /* uclpack.c -- example program: a simple file packer
   2
   3    This file is part of the UCL data compression library.
   4
   5    Copyright (C) 1996-2002 Markus Franz Xaver Johannes Oberhumer
   6    All Rights Reserved.
   7
   8    The UCL library is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public License as
  10    published by the Free Software Foundation; either version 2 of
  11    the License, or (at your option) any later version.
  12
  13    The UCL library is distributed in the hope that it will be useful,
  14    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16    GNU General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with the UCL library; see the file COPYING.
  20    If not, write to the Free Software Foundation, Inc.,
  21    59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  22
  23    Markus F.X.J. Oberhumer
  24    <markus@oberhumer.com>
  25  */
  26
  27
  28 /*************************************************************************
  29 // NOTE: this is an example program, so do not use to backup your data
  30 //
  31 // This program lacks things like sophisticated file handling but is
  32 // pretty complete regarding compression - it should provide a good
  33 // starting point for adaption for you applications.
  34 **************************************************************************/
  35
  36 #include <ucl/ucl.h>
  37 #include "lutil.h"
  38
  39 static const char *progname = NULL;
  40
  41 static unsigned long total_in = 0;
  42 static unsigned long total_out = 0;
  43 static ucl_bool opt_debug = 0;
  44
  45 /* don't compute or verify checksum, always use fast decompressor */
  46 static ucl_bool opt_fast = 0;
  47
  48 /* magic file header for compressed files */
  49 static const unsigned char magic[8] =
  50     { 0x00, 0xe9, 0x55, 0x43, 0x4c, 0xff, 0x01, 0x1a };
  51
  52
  53 /*************************************************************************
  54 // file IO
  55 **************************************************************************/
  56
  57 ucl_uint xread(FILE *f, ucl_voidp buf, ucl_uint len, ucl_bool allow_eof)
  58 {
  59     ucl_uint l;
  60
  61     l = ucl_fread(f,buf,len);
  62     if (l > len)
  63     {
  64         fprintf(stderr,"\nsomething's wrong with your C library !!!\n");
  65         exit(1);
  66     }
  67     if (l != len && !allow_eof)
  68     {
  69         fprintf(stderr,"\nread error - premature end of file\n");
  70         exit(1);
  71     }
  72     total_in += l;
  73     return l;
  74 }
  75
  76 ucl_uint xwrite(FILE *f, const ucl_voidp buf, ucl_uint len)
  77 {
  78     ucl_uint l;
  79
  80     if (f != NULL)
  81     {
  82         l = ucl_fwrite(f,buf,len);
  83         if (l != len)
  84         {
  85             fprintf(stderr,"\nwrite error [%ld %ld]  (disk full ?)\n",
  86                    (long)len, (long)l);
  87             exit(1);
  88         }
  89     }
  90     total_out += len;
  91     return len;
  92 }
  93
  94
  95 int xgetc(FILE *f)
  96 {
  97     unsigned char c;
  98     xread(f,(ucl_voidp) &c,1,0);
  99     return c;
 100 }
 101
 102 void xputc(FILE *f, int c)
 103 {
 104     unsigned char cc = (unsigned char) c;
 105     xwrite(f,(const ucl_voidp) &cc,1);
 106 }
 107
 108 /* read and write portable 32-bit integers */
 109
 110 ucl_uint32 xread32(FILE *f)
 111 {
 112     unsigned char b[4];
 113     ucl_uint32 v;
 114
 115     xread(f,b,4,0);
 116     v  = (ucl_uint32) b[3] <<  0;
 117     v |= (ucl_uint32) b[2] <<  8;
 118     v |= (ucl_uint32) b[1] << 16;
 119     v |= (ucl_uint32) b[0] << 24;
 120     return v;
 121 }
 122
 123 void xwrite32(FILE *f, ucl_uint32 v)
 124 {
 125     unsigned char b[4];
 126
 127     b[3] = (unsigned char) (v >>  0);
 128     b[2] = (unsigned char) (v >>  8);
 129     b[1] = (unsigned char) (v >> 16);
 130     b[0] = (unsigned char) (v >> 24);
 131     xwrite(f,b,4);
 132 }
 133
 134
 135 /*************************************************************************
 136 // util
 137 **************************************************************************/
 138
 139 static ucl_uint get_overhead(int method, ucl_uint size)
 140 {
 141     if (method == 0x2b || method == 0x2d || method == 0x2e)
 142         return size / 8 + 256;
 143     return 0;
 144 }
 145
 146
 147 static char method_name[64];
 148
 149 static ucl_bool set_method_name(int method, int level)
 150 {
 151     method_name[0] = 0;
 152     if (level < 0 || level > 10)
 153         return 0;
 154     if (level == 0)
 155         sprintf(method_name,"uncompressed/%d", level);
 156     else if (method == 0x2b)
 157         sprintf(method_name,"NRV2B-99/%d", level);
 158     else if (method == 0x2d)
 159         sprintf(method_name,"NRV2D-99/%d", level);
 160     else if (method == 0x2e)
 161         sprintf(method_name,"NRV2E-99/%d", level);
 162     else
 163         return 0;
 164     return 1;
 165 }
 166
 167
 168 /*************************************************************************
 169 // compress
 170 **************************************************************************/
 171
 172 int do_compress(FILE *fi, FILE *fo, int method, int level, ucl_uint block_size)
 173 {
 174     int r = 0;
 175     ucl_byte *in = NULL;
 176     ucl_byte *out = NULL;
 177     ucl_uint in_len;
 178     ucl_uint out_len;
 179     ucl_uint32 flags = opt_fast ? 0 : 1;
 180     ucl_uint32 checksum;
 181     ucl_uint overhead = 0;
 182
 183     total_in = total_out = 0;
 184
 185 /*
 186  * Step 1: write magic header, flags & block size, init checksum
 187  */
 188     xwrite(fo,magic,sizeof(magic));
 189     xwrite32(fo,flags);
 190     xputc(fo,method);           /* compression method */
 191     xputc(fo,level);            /* compression level */
 192     xwrite32(fo,block_size);
 193     checksum = ucl_adler32(0,NULL,0);
 194
 195 /*
 196  * Step 2: allocate compression buffers and work-memory
 197  */
 198     overhead = get_overhead(method,block_size);
 199     in = (ucl_byte *) ucl_malloc(block_size);
 200     out = (ucl_byte *) ucl_malloc(block_size + overhead);
 201     if (in == NULL || out == NULL)
 202     {
 203         printf("%s: out of memory\n", progname);
 204         r = 1;
 205         goto err;
 206     }
 207
 208 /*
 209  * Step 3: process blocks
 210  */
 211     for (;;)
 212     {
 213         /* read block */
 214         in_len = xread(fi,in,block_size,1);
 215         if (in_len <= 0)
 216             break;
 217
 218         /* update checksum */
 219         if (flags & 1)
 220             checksum = ucl_adler32(checksum,in,in_len);
 221
 222         /* compress block */
 223         r = UCL_E_ERROR;
 224         if (level == 0)
 225         {
 226             out_len = in_len; /* uncompressed */
 227             r = UCL_E_OK;
 228         }
 229         else if (method == 0x2b)
 230             r = ucl_nrv2b_99_compress(in,in_len,out,&out_len,0,level,NULL,NULL);
 231         else if (method == 0x2d)
 232             r = ucl_nrv2d_99_compress(in,in_len,out,&out_len,0,level,NULL,NULL);
 233         else if (method == 0x2e)
 234             r = ucl_nrv2e_99_compress(in,in_len,out,&out_len,0,level,NULL,NULL);
 235         if (r != UCL_E_OK || out_len > in_len + get_overhead(method,in_len))
 236         {
 237             /* this should NEVER happen */
 238             printf("internal error - compression failed: %d\n", r);
 239             r = 2;
 240             goto err;
 241         }
 242
 243         /* write uncompressed block size */
 244         xwrite32(fo,in_len);
 245
 246         if (out_len < in_len)
 247         {
 248             /* write compressed block */
 249             xwrite32(fo,out_len);
 250             xwrite(fo,out,out_len);
 251         }
 252         else
 253         {
 254             /* not compressible - write uncompressed block */
 255             xwrite32(fo,in_len);
 256             xwrite(fo,in,in_len);
 257         }
 258     }
 259
 260     /* write EOF marker */
 261     xwrite32(fo,0);
 262
 263     /* write checksum */
 264     if (flags & 1)
 265         xwrite32(fo,checksum);
 266
 267     r = 0;
 268 err:
 269     ucl_free(out);
 270     ucl_free(in);
 271     return r;
 272 }
 273
 274
 275 /*************************************************************************
 276 // decompress / test
 277 //
 278 // We are using overlapping (in-place) decompression to save some
 279 // memory - see overlap.c.
 280 **************************************************************************/
 281
 282 int do_decompress(FILE *fi, FILE *fo)
 283 {
 284     int r = 0;
 285     ucl_byte *buf = NULL;
 286     ucl_uint buf_len;
 287     unsigned char m [ sizeof(magic) ];
 288     ucl_uint32 flags;
 289     int method;
 290     int level;
 291     ucl_uint block_size;
 292     ucl_uint32 checksum;
 293     ucl_uint overhead = 0;
 294
 295     total_in = total_out = 0;
 296
 297 /*
 298  * Step 1: check magic header, read flags & block size, init checksum
 299  */
 300     if (xread(fi,m,sizeof(magic),1) != sizeof(magic) ||
 301         memcmp(m,magic,sizeof(magic)) != 0)
 302     {
 303         printf("%s: header error - this file is not compressed by uclpack\n", progname);
 304         r = 1;
 305         goto err;
 306     }
 307     flags = xread32(fi);
 308     method = xgetc(fi);
 309     level = xgetc(fi);
 310     block_size = xread32(fi);
 311     overhead = get_overhead(method,block_size);
 312     if (overhead == 0 || !set_method_name(method, level))
 313     {
 314         printf("%s: header error - invalid method %d (level %d)\n",
 315                 progname, method, level);
 316         r = 2;
 317         goto err;
 318     }
 319     if (block_size < 1024 || block_size > 8*1024*1024L)
 320     {
 321         printf("%s: header error - invalid block size %ld\n",
 322                 progname, (long) block_size);
 323         r = 3;
 324         goto err;
 325     }
 326     checksum = ucl_adler32(0,NULL,0);
 327
 328 /*
 329  * Step 2: allocate buffer for in-place decompression
 330  */
 331     buf_len = block_size + overhead;
 332     buf = (ucl_byte *) ucl_malloc(buf_len);
 333     if (buf == NULL)
 334     {
 335         printf("%s: out of memory\n", progname);
 336         r = 4;
 337         goto err;
 338     }
 339
 340 /*
 341  * Step 3: process blocks
 342  */
 343     for (;;)
 344     {
 345         ucl_byte *in;
 346         ucl_byte *out;
 347         ucl_uint in_len;
 348         ucl_uint out_len;
 349
 350         /* read uncompressed size */
 351         out_len = xread32(fi);
 352
 353         /* exit if last block (EOF marker) */
 354         if (out_len == 0)
 355             break;
 356
 357         /* read compressed size */
 358         in_len = xread32(fi);
 359
 360         /* sanity check of the size values */
 361         if (in_len > block_size || out_len > block_size ||
 362             in_len == 0 || in_len > out_len)
 363         {
 364             printf("%s: block size error - data corrupted\n", progname);
 365             r = 5;
 366             goto err;
 367         }
 368
 369         /* place compressed block at the top of the buffer */
 370         in = buf + buf_len - in_len;
 371         out = buf;
 372
 373         /* read compressed block data */
 374         xread(fi,in,in_len,0);
 375
 376         if (in_len < out_len)
 377         {
 378             /* decompress - use safe decompressor as data might be corrupted */
 379             ucl_uint new_len = out_len;
 380
 381             if (method == 0x2b)
 382             {
 383                 if (opt_fast)
 384                     r = ucl_nrv2b_decompress_8(in,in_len,out,&new_len,NULL);
 385                 else
 386                     r = ucl_nrv2b_decompress_safe_8(in,in_len,out,&new_len,NULL);
 387             }
 388             else if (method == 0x2d)
 389             {
 390                 if (opt_fast)
 391                     r = ucl_nrv2d_decompress_8(in,in_len,out,&new_len,NULL);
 392                 else
 393                     r = ucl_nrv2d_decompress_safe_8(in,in_len,out,&new_len,NULL);
 394             }
 395             else if (method == 0x2e)
 396             {
 397                 if (opt_fast)
 398                     r = ucl_nrv2e_decompress_8(in,in_len,out,&new_len,NULL);
 399                 else
 400                     r = ucl_nrv2e_decompress_safe_8(in,in_len,out,&new_len,NULL);
 401             }
 402             if (r != UCL_E_OK || new_len != out_len)
 403             {
 404                 printf("%s: compressed data violation: error %d (0x%x: %ld/%ld/%ld)\n", progname, r, method, (long) in_len, (long) out_len, (long) new_len);
 405                 r = 6;
 406                 goto err;
 407             }
 408             /* write decompressed block */
 409             xwrite(fo,out,out_len);
 410             /* update checksum */
 411             if ((flags & 1) && !opt_fast)
 412                 checksum = ucl_adler32(checksum,out,out_len);
 413         }
 414         else
 415         {
 416             /* write original (incompressible) block */
 417             xwrite(fo,in,in_len);
 418             /* update checksum */
 419             if ((flags & 1) && !opt_fast)
 420                 checksum = ucl_adler32(checksum,in,in_len);
 421         }
 422     }
 423
 424     /* read and verify checksum */
 425     if (flags & 1)
 426     {
 427         ucl_uint32 c = xread32(fi);
 428         if (!opt_fast && c != checksum)
 429         {
 430             printf("%s: checksum error - data corrupted\n", progname);
 431             r = 7;
 432             goto err;
 433         }
 434     }
 435
 436     r = 0;
 437 err:
 438     ucl_free(buf);
 439     return r;
 440 }
 441
 442
 443 /*************************************************************************
 444 //
 445 **************************************************************************/
 446
 447 static void usage(void)
 448 {
 449     printf("usage: %s [-0123456789] input-file output-file  (compress)\n", progname);
 450     printf("usage: %s -d input-file output-file             (decompress)\n", progname);
 451     printf("usage: %s -t input-file...                      (test)\n", progname);
 452     exit(1);
 453 }
 454
 455
 456 /* open input file */
 457 static FILE *xopen_fi(const char *name)
 458 {
 459     FILE *f;
 460
 461     f = fopen(name,"rb");
 462     if (f == NULL)
 463     {
 464         printf("%s: cannot open input file %s\n", progname, name);
 465         exit(1);
 466     }
 467 #if defined(HAVE_STAT) && defined(S_ISREG)
 468     {
 469         struct stat st;
 470 #if defined(HAVE_LSTAT)
 471         if (lstat(name,&st) != 0 || !S_ISREG(st.st_mode))
 472 #else
 473         if (stat(name,&st) != 0 || !S_ISREG(st.st_mode))
 474 #endif
 475         {
 476             printf("%s: %s is not a regular file\n", progname, name);
 477             fclose(f);
 478             exit(1);
 479         }
 480     }
 481 #endif
 482     return f;
 483 }
 484
 485
 486 /* open output file */
 487 static FILE *xopen_fo(const char *name)
 488 {
 489     FILE *f;
 490
 491 #if 0
 492     /* this is an example program, so make sure we don't overwrite a file */
 493     f = fopen(name,"rb");
 494     if (f != NULL)
 495     {
 496         printf("%s: file %s already exists -- not overwritten\n", progname, name);
 497         fclose(f);
 498         exit(1);
 499     }
 500 #endif
 501     f = fopen(name,"wb");
 502     if (f == NULL)
 503     {
 504         printf("%s: cannot open output file %s\n", progname, name);
 505         exit(1);
 506     }
 507     return f;
 508 }
 509
 510
 511 /*************************************************************************
 512 //
 513 **************************************************************************/
 514
 515 int main(int argc, char *argv[])
 516 {
 517     int i = 1;
 518     int r = 0;
 519     FILE *fi = NULL;
 520     FILE *fo = NULL;
 521     const char *in_name = NULL;
 522     const char *out_name = NULL;
 523     ucl_bool opt_decompress = 0;
 524     ucl_bool opt_test = 0;
 525     int opt_method = 0x2b;
 526     int opt_level = 7;
 527 #if defined(MAINT)
 528     ucl_uint opt_block_size = (2*1024*1024L);
 529 #else
 530     ucl_uint opt_block_size = (256*1024L);
 531 #endif
 532     const char *s;
 533
 534 #if defined(__EMX__)
 535     _response(&argc,&argv);
 536     _wildcard(&argc,&argv);
 537 #endif
 538     progname = argv[0];
 539     for (s = progname; *s; s++)
 540         if (*s == '/' || *s == '\\')
 541             progname = s + 1;
 542
 543     printf("\nUCL real-time data compression library (v%s, %s).\n",
 544             ucl_version_string(), ucl_version_date());
 545     printf("Copyright (C) 1996-2002 Markus Franz Xaver Johannes Oberhumer\n\n");
 546
 547     printf(
 548 "*** WARNING ***\n"
 549 "   This is an example program, do not use to backup your data !\n"
 550 "\n");
 551
 552 /*
 553  * Step 1: initialize the UCL library
 554  */
 555     if (ucl_init() != UCL_E_OK)
 556     {
 557         printf("ucl_init() failed !!!\n");
 558         exit(1);
 559     }
 560
 561 /*
 562  * Step 2: get options
 563  */
 564
 565     while (i < argc && argv[i][0] == '-')
 566     {
 567         if (strcmp(argv[i],"-d") == 0)
 568             opt_decompress = 1;
 569         else if (strcmp(argv[i],"-t") == 0)
 570             opt_test = 1;
 571         else if (strcmp(argv[i],"-F") == 0)
 572             opt_fast = 1;
 573         else if (strcmp(argv[i],"--2b") == 0)
 574             opt_method = 0x2b;
 575         else if (strcmp(argv[i],"--nrv2b") == 0)
 576             opt_method = 0x2b;
 577         else if (strcmp(argv[i],"--2d") == 0)
 578             opt_method = 0x2d;
 579         else if (strcmp(argv[i],"--nrv2d") == 0)
 580             opt_method = 0x2d;
 581         else if (strcmp(argv[i],"--2e") == 0)
 582             opt_method = 0x2e;
 583         else if (strcmp(argv[i],"--nrv2e") == 0)
 584             opt_method = 0x2e;
 585         else if ((argv[i][1] >= '0' && argv[i][1] <= '9') && !argv[i][2])
 586             opt_level = argv[i][1] - '0';
 587         else if (strcmp(argv[i],"--10") == 0)
 588             opt_level = 10;
 589         else if (strcmp(argv[i],"--best") == 0)
 590             opt_level = 10;
 591         else if (strcmp(argv[i],"--none") == 0)
 592             opt_level = 0;
 593         else if (argv[i][1] == 'b' && argv[i][2])
 594         {
 595 #if (UCL_UINT_MAX > UINT_MAX) && defined(HAVE_ATOL)
 596             ucl_int b = (ucl_int) atol(&argv[i][2]);
 597 #else
 598             ucl_int b = (ucl_int) atoi(&argv[i][2]);
 599 #endif
 600             if (b >= 1024L && b <= 8*1024*1024L)
 601                 opt_block_size = b;
 602         }
 603         else if (strcmp(argv[i],"--debug") == 0)
 604             opt_debug = 1;
 605         else
 606             usage();
 607         i++;
 608     }
 609     if (opt_test && i >= argc)
 610         usage();
 611     if (!opt_test && i + 2 != argc)
 612         usage();
 613
 614 /*
 615  * Step 3: process file(s)
 616  */
 617     if (opt_test)
 618     {
 619         while (i < argc && r == 0)
 620         {
 621             in_name = argv[i++];
 622             fi = xopen_fi(in_name);
 623             r = do_decompress(fi,NULL);
 624             if (r == 0)
 625                 printf("%s: tested ok: %-10s %-11s: %6ld -> %6ld bytes\n",
 626                         progname, in_name, method_name, total_in, total_out);
 627             fclose(fi);
 628             fi = NULL;
 629         }
 630     }
 631     else if (opt_decompress)
 632     {
 633         in_name = argv[i++];
 634         out_name = argv[i++];
 635         fi = xopen_fi(in_name);
 636         fo = xopen_fo(out_name);
 637         r = do_decompress(fi,fo);
 638         if (r == 0)
 639             printf("%s: decompressed %ld into %ld bytes\n",
 640                     progname, total_in, total_out);
 641     }
 642     else /* compress */
 643     {
 644         if (!set_method_name(opt_method, opt_level))
 645         {
 646             printf("%s: internal error - invalid method %d (level %d)\n",
 647                    progname, opt_method, opt_level);
 648             goto quit;
 649         }
 650         in_name = argv[i++];
 651         out_name = argv[i++];
 652         fi = xopen_fi(in_name);
 653         fo = xopen_fo(out_name);
 654         r = do_compress(fi,fo,opt_method,opt_level,opt_block_size);
 655         if (r == 0)
 656             printf("%s: algorithm %s, compressed %ld into %ld bytes\n",
 657                     progname, method_name, total_in, total_out);
 658     }
 659
 660 quit:
 661     if (fi) fclose(fi);
 662     if (fo) fclose(fo);
 663     return r;
 664 }
 665
 666 /*
 667 vi:ts=4:et
 668 */
 669