dlzma/test/dlzmatest.d

   1 /* converted by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   2  * Understanding is not required. Only obedience.
   3  *
   4  * This program is free software. It comes without any warranty, to
   5  * the extent permitted by applicable law. You can redistribute it
   6  * and/or modify it under the terms of the Do What The Fuck You Want
   7  * To Public License, Version 2, as published by Sam Hocevar. See
   8  * http://www.wtfpl.net/txt/copying/ for more details.
   9  */
  10 module dlzmatest;
  11
  12 import std.digest.ripemd;
  13
  14 import iv.pxclock;
  15 import iv.strex;
  16 import iv.vfs;
  17 import iv.vfs.io;
  18
  19 import iv.dlzma;
  20
  21
  22 // ////////////////////////////////////////////////////////////////////////// //
  23 void compressFile (ref VFile fi, ref VFile fo) {
  24   fi.seek(0);
  25   ulong insize = fi.size;
  26   //fi.rawReadExact(inbuf[0..insize]);
  27   //fi.close();
  28
  29   CLzmaEncProps props;
  30   LzmaEncProps_Init(&props);
  31   props.level = 9;
  32   //props.dictSize = 1;
  33   //while (props.dictSize < insize) props.dictSize <<= 1;
  34   props.dictSize = 1<<27; //128MB
  35   //props.dictSize = 1<<22; //4MB
  36   props.reduceSize = insize;
  37
  38   ubyte[LZMA_PROPS_SIZE+8] header;
  39   uint headerSize = cast(uint)header.sizeof;
  40
  41   CLzmaEncHandle enc = LzmaEnc_Create(&lzmaDefAllocator);
  42   scope(exit) LzmaEnc_Destroy(enc, &lzmaDefAllocator, &lzmaDefAllocator);
  43
  44   if (LzmaEnc_SetProps(enc, &props) != SZ_OK) throw new Exception("cannot set encoder properties");
  45   LzmaEnc_SetDataSize(enc, insize); // just in case
  46
  47   if (LzmaEnc_WriteProperties(enc, header.ptr, &headerSize) != SZ_OK) throw new Exception("cannot encode encoder properties");
  48   assert(headerSize > 0 && headerSize < 256);
  49
  50   writeln("compressing...");
  51   fo.writeNum!ushort(cast(ushort)1); // version and endianness check
  52   fo.writeNum!ulong(insize); // unpacked file size
  53   fo.writeNum!ubyte(cast(ubyte)headerSize); // properties size
  54   fo.rawWriteExact(header[0..headerSize]);
  55
  56   ISeqInStream inStream;
  57   ISeqOutStream outStream;
  58   ICompressProgress progress;
  59
  60   immutable origInSize = insize;
  61   ulong destSize = 0;
  62
  63   auto csum = makeDigest!RIPEMD160;
  64
  65   /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
  66      (output(*size) < input(*size)) is allowed */
  67   inStream.Read = delegate SRes (ISeqInStream* p, void* buf, usize* size) nothrow {
  68     if (*size > insize) *size = cast(usize)insize;
  69     if (*size) {
  70       try {
  71         fi.rawReadExact(buf[0..*size]);
  72       } catch (Exception e) {
  73         return SZ_ERROR_READ;
  74       }
  75       csum.put((cast(const(ubyte)*)buf)[0..*size]);
  76       insize -= *size;
  77     }
  78     return SZ_OK;
  79   };
  80
  81   /* Returns: result - the number of actually written bytes.
  82      (result < size) means error */
  83   outStream.Write = delegate usize (ISeqOutStream* p, const(void)* buf, usize size) nothrow {
  84     try {
  85       fo.rawWriteExact(buf[0..size]);
  86       destSize += size;
  87     } catch (Exception e) {
  88       return 0;
  89     }
  90     return size;
  91   };
  92
  93   uint prevPrc = uint.max;
  94   ulong prevReport = clockMilli();
  95
  96   progress.Progress = delegate SRes (ICompressProgress* p, ulong inSize, ulong outSize) nothrow {
  97     if (origInSize == 0) return SZ_OK; // just in case
  98     immutable uint prc = cast(uint)(inSize*100U/origInSize);
  99     if (prc == prevPrc) {
 100       ulong rtt = clockMilli();
 101       if (rtt-prevReport < 500) return SZ_OK;
 102     }
 103     prevPrc = prc;
 104     char[128] i0 = void;
 105     char[128] i1 = void;
 106     auto num0 = intWithCommas(i0[], inSize);
 107     auto num1 = intWithCommas(i1[], origInSize);
 108     try {
 109       write(" [", num0[], "/", num1[], "]  ", prc, "%\x1b[K\r");
 110       //writeln;
 111     } catch (Exception) {}
 112     prevReport = clockMilli();
 113     return SZ_OK;
 114   };
 115
 116   progress.Progress(&progress, 0, 0);
 117
 118   SRes res = LzmaEnc_Encode(enc, &outStream, &inStream, &progress, &lzmaDefAllocator, &lzmaDefAllocator);
 119
 120   switch (res) {
 121     case SZ_OK: break;
 122     case SZ_ERROR_MEM: throw new Exception("FUCK: memory");
 123     case SZ_ERROR_PARAM: throw new Exception("FUCK: param");
 124     case SZ_ERROR_OUTPUT_EOF: throw new Exception("FUCK: compressed is bigger");
 125     default: throw new Exception("FUCK: something else");
 126   }
 127
 128   ubyte[20] hash = csum.finish()[];
 129   fo.rawWriteExact(hash[]);
 130
 131   writeln("\rcompressed ", intWithCommas(origInSize), " to ", intWithCommas(destSize), "; ratio: ", destSize*100U/(origInSize ? origInSize : 1), "%\x1b[K");
 132 }
 133
 134
 135 // ////////////////////////////////////////////////////////////////////////// //
 136 void decompressFile (ref VFile fi, ref VFile fo) {
 137   ubyte[LZMA_PROPS_SIZE+8] header;
 138
 139   fi.seek(0);
 140   ulong pksize = fi.size;
 141   if (fi.readNum!ushort != 1) throw new Exception("invalid archive version");
 142   ulong unsize = fi.readNum!ulong; // unpacked size
 143   ubyte hdrSize = fi.readNum!ubyte;
 144   if (hdrSize == 0 || hdrSize > header.sizeof) throw new Exception("invalid properties size");
 145   fi.rawReadExact(header[0..hdrSize]);
 146   pksize -= fi.tell;
 147
 148   if (pksize < 20) throw new Exception("invalid archive size");
 149   if (pksize == 0) {
 150     if (unsize != 0) throw new Exception("invalid archive size");
 151     return; // nothing to do
 152   }
 153   pksize -= 20; // digest size
 154
 155   auto csum = makeDigest!RIPEMD160;
 156
 157   enum InBufSize = 1024*1024;
 158   enum OutBufSize = 1024*1024;
 159
 160   ubyte *inbuf = cast(ubyte*)ISzAlloc_Alloc(&lzmaDefAllocator, InBufSize);
 161   ubyte *outbuf = cast(ubyte*)ISzAlloc_Alloc(&lzmaDefAllocator, OutBufSize);
 162   scope(exit) {
 163     ISzAlloc_Free(&lzmaDefAllocator, inbuf);
 164     ISzAlloc_Free(&lzmaDefAllocator, outbuf);
 165   }
 166
 167   CLzmaDec dec;
 168   LzmaDec_Init(&dec);
 169
 170   SRes res = LzmaDec_Allocate(&dec, header.ptr, hdrSize, &lzmaDefAllocator);
 171   if (res != SZ_OK) throw new Exception("cannot initialize decoder");
 172   scope(exit) LzmaDec_Free(&dec, &lzmaDefAllocator);
 173
 174   ulong unpackedTotal = 0;
 175   ulong readleft = pksize;
 176   usize inused = 0;
 177
 178   uint prevPrc = uint.max;
 179   ulong prevReport = clockMilli();
 180   immutable ulong sttime = prevReport;
 181
 182   void showProgress () nothrow {
 183     immutable rds = unpackedTotal; //pksize-readleft;
 184     immutable uint prc = cast(uint)(rds*100U/unsize/*pksize*/);
 185     if (prc == prevPrc && unpackedTotal != unsize) {
 186       ulong rtt = clockMilli();
 187       if (rtt-prevReport < 500) return;
 188     }
 189     prevPrc = prc;
 190     char[128] i0 = void;
 191     char[128] i1 = void;
 192     auto num0 = intWithCommas(i0[], rds);
 193     auto num1 = intWithCommas(i1[], unsize/*pksize*/);
 194     try {
 195       write(" [", num0[], "/", num1[], "]  ", prc, "%\x1b[K\r");
 196       //writeln;
 197     } catch (Exception) {}
 198     prevReport = clockMilli();
 199   }
 200
 201   while (readleft || inused) {
 202     // read more
 203     if (readleft && inused < InBufSize) {
 204       uint rd = InBufSize-cast(uint)inused;
 205       if (rd > readleft) rd = cast(usize)readleft;
 206       fi.rawReadExact(inbuf[inused..inused+rd]);
 207       inused += rd;
 208       readleft -= rd;
 209       //showProgress();
 210     }
 211     usize outSize = OutBufSize;
 212     usize inSize = inused;
 213     ELzmaStatus status;
 214     // as we don't have a proper EOF mark, make sure to not unpack extra data
 215     if (unsize-unpackedTotal < outSize) outSize = cast(usize)(unsize-unpackedTotal);
 216     //writeln("\nunsize=", unsize, "; unpackedTotal=", unpackedTotal, "; outSize=", outSize);
 217     res = LzmaDec_DecodeToBuf(&dec, outbuf, &outSize, inbuf, &inSize, LZMA_FINISH_ANY, &status);
 218     if (res != SZ_OK) {
 219       writeln;
 220       writeln("ERROR: readleft=", readleft, "; inused=", inused, "; written=", unpackedTotal, " of ", unsize);
 221       switch (res) {
 222         case SZ_ERROR_DATA: throw new Exception("corrupted data");
 223         case SZ_ERROR_MEM: throw new Exception("out of memory");
 224         case SZ_ERROR_UNSUPPORTED: throw new Exception("unsupported properties");
 225         case SZ_ERROR_INPUT_EOF: throw new Exception("need bigger input buffer, but we don't have any");
 226         default: throw new Exception("some other error");
 227       }
 228     }
 229     if (outSize) {
 230       fo.rawWriteExact(outbuf[0..outSize]);
 231       unpackedTotal += outSize;
 232       csum.put((cast(const(ubyte)*)outbuf)[0..outSize]);
 233       showProgress();
 234       if (unpackedTotal == unsize) break; // we're done (we don't have EOF mark, so...)
 235     }
 236     if (inSize < inused) {
 237       import core.stdc.string : memmove;
 238       memmove(inbuf, inbuf+inSize, inused-inSize);
 239     }
 240     inused -= inSize;
 241     switch (status) {
 242       case LZMA_STATUS_FINISHED_WITH_MARK: throw new Exception("found EOF mark, but there should not be one");
 243       case LZMA_STATUS_NOT_FINISHED: break; // it is ok
 244       case LZMA_STATUS_NEEDS_MORE_INPUT: break; // it is ok
 245       case LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK: break; // ok ;-)
 246       default: break; // ignore others
 247     }
 248   }
 249
 250   if (unpackedTotal != unsize) {
 251     //import std.conv : to;
 252     throw new Exception("invalid unpacked file size; expected "~intWithCommas(unsize).idup~" but got "~intWithCommas(unpackedTotal).idup);
 253   }
 254
 255   // check hash
 256   ubyte[20] origcsum;
 257   fi.rawReadExact(origcsum[]);
 258   ubyte[20] hash = csum.finish()[];
 259   if (origcsum[] != hash[]) throw new Exception("invalid unpacked file hash");
 260
 261   ulong etime = clockMilli()-sttime;
 262   if (!etime) etime = 1;
 263   writeln("\rsuccesfully unpacked ", intWithCommas(unpackedTotal), " bytes (", intWithCommas(unpackedTotal*1000U/etime), " MB/sec).\x1b[K");
 264 }
 265
 266
 267 // ////////////////////////////////////////////////////////////////////////// //
 268 void main (string[] args) {
 269   if (args.length < 2) {
 270     writeln("usage: fuckme <c|x> infile outfile");
 271     return;
 272   }
 273
 274   if (args[1] == "c") {
 275     if (args.length < 4) throw new Exception("out of args");
 276     if (args[2] == args[3]) throw new Exception("cannot compress in-place");
 277     auto fi = VFile(args[2]);
 278     auto fo = VFile(args[3], "w");
 279     compressFile(fi, fo);
 280   } else if (args[1] == "x") {
 281     if (args.length < 4) throw new Exception("out of args");
 282     if (args[2] == args[3]) throw new Exception("cannot decompress in-place");
 283     auto fi = VFile(args[2]);
 284     auto fo = VFile(args[3], "w");
 285     decompressFile(fi, fo);
 286   } else if (args[1] == "t") {
 287     if (args.length < 3) throw new Exception("out of args");
 288     auto fi = VFile(args[2]);
 289     auto fo = VFile("/dev/null", "w");
 290     decompressFile(fi, fo);
 291   }
 292 }