vfs/arc/arcz.d

   1 /* Invisible Vector Library
   2  * coded by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   3  * Understanding is not required. Only obedience.
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, version 3 of the License ONLY.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 module iv.vfs.arc.arcz /*is aliced*/;
  18
  19 import iv.alice;
  20 import iv.vfs.types : Seek;
  21 import iv.vfs.error;
  22 import iv.vfs.main;
  23 import iv.vfs.util;
  24 import iv.vfs.vfile;
  25 import iv.vfs.arc.internal;
  26
  27
  28 // ////////////////////////////////////////////////////////////////////////// //
  29 mixin(VFSSimpleArchiveDetectorMixin!"ArcZ");
  30
  31 // use Balz compressor if available
  32 static if (__traits(compiles, { import iv.balz; })) enum iv_vfs_arcz_has_balz = true; else enum iv_vfs_arcz_has_balz = false;
  33 static if (__traits(compiles, { import iv.dlzma; })) enum iv_vfs_arcz_has_dlzma = true; else enum iv_vfs_arcz_has_dlzma = false;
  34 static if (iv_vfs_arcz_has_balz) import iv.balz;
  35 static if (iv_vfs_arcz_has_dlzma) import iv.dlzma;
  36
  37
  38 // ////////////////////////////////////////////////////////////////////////// //
  39 // ARZ archive accessor. Use this to open ARZ archives, and open packed files from ARZ archives.
  40 private struct ArzArchive {
  41 private import core.stdc.stdio : SEEK_SET, SEEK_CUR, SEEK_END;
  42 private:
  43   static assert(usize.sizeof >= (void*).sizeof);
  44   private import etc.c.zlib;
  45
  46   enum Packer {
  47     Zlib,
  48     Balz,
  49     Lzma,
  50   }
  51
  52   static align(1) struct ChunkInfo {
  53   align(1):
  54     uint ofs; // offset in file
  55     uint pksize; // packed chunk size (same as chunk size: chunk is unpacked)
  56   }
  57
  58   static align(1) struct FileInfo {
  59   align(1):
  60     //string name;
  61     uint nameofs; // in index
  62     uint namelen;
  63     const(char)[] name (const(void)* index) const pure nothrow @trusted @nogc { pragma(inline, true); return (cast(const(char)*)index)[nameofs..nameofs+namelen]; }
  64     // this is copied from index
  65     uint chunk;
  66     uint chunkofs; // offset of first file byte in unpacked chunk
  67     uint size; // unpacked file size
  68   }
  69
  70   static struct Nfo {
  71     uint rc = 1; // refcounter
  72     ubyte* index; // unpacked archive index
  73     ChunkInfo* chunks; // in index, not allocated
  74     FileInfo* files; // in index, not allocated
  75     uint fcount; // number of valid entries in files
  76     uint ccount; // number of valid entries in chunks
  77     uint chunkSize;
  78     uint lastChunkSize;
  79     Packer packer;
  80     VFile afl; // archive file, we'll keep it opened
  81
  82     @disable this (this); // no copies!
  83
  84     static void decRef (usize me) {
  85       if (me) {
  86         auto nfo = cast(Nfo*)me;
  87         assert(nfo.rc);
  88         if (--nfo.rc == 0) {
  89           import core.memory : GC;
  90           import core.stdc.stdlib : free;
  91           if (nfo.afl.isOpen) nfo.afl.close();
  92           if (nfo.index !is null) free(nfo.index);
  93           //GC.removeRange(cast(void*)nfo/*, Nfo.sizeof*/);
  94           free(nfo);
  95           debug(iv_vfs_arcz_rc) { import core.stdc.stdio : printf; printf("Nfo %p freed\n", nfo); }
  96         }
  97       }
  98     }
  99   }
 100
 101   usize nfop; // hide it from GC
 102
 103   private @property Nfo* nfo () { pragma(inline, true); return cast(Nfo*)nfop; }
 104   void decRef () { pragma(inline, true); Nfo.decRef(nfop); nfop = 0; }
 105
 106   static uint readUint (VFile fl) {
 107     if (!fl.isOpen) throw new Exception("cannot read from closed file");
 108     uint v;
 109     fl.rawReadExact((&v)[0..1]);
 110     version(BigEndian) {
 111       import core.bitop : bswap;
 112       v = bswap(v);
 113     } else version(LittleEndian) {
 114       // nothing to do
 115     } else {
 116       static assert(0, "wtf?!");
 117     }
 118     return v;
 119   }
 120
 121   static uint readUbyte (VFile fl) {
 122     if (!fl.isOpen) throw new Exception("cannot read from closed file");
 123     ubyte v;
 124     fl.rawReadExact((&v)[0..1]);
 125     return v;
 126   }
 127
 128   static void readBuf (VFile fl, void[] buf) {
 129     if (buf.length > 0) {
 130       fl.rawReadExact(buf[]);
 131     }
 132   }
 133
 134   static T* xalloc(T, bool clear=true) (uint mem) if (T.sizeof > 0) {
 135     import core.exception : onOutOfMemoryError;
 136     assert(mem != 0);
 137     static if (clear) {
 138       import core.stdc.stdlib : calloc;
 139       auto res = calloc(mem, T.sizeof);
 140       if (res is null) onOutOfMemoryError();
 141       static if (is(T == struct)) {
 142         import core.stdc.string : memcpy;
 143         static immutable T i = T.init;
 144         foreach (immutable idx; 0..mem) memcpy(res+idx, &i, T.sizeof);
 145       }
 146       debug(iv_vfs_arcz_alloc) { import core.stdc.stdio : printf; printf("allocated %u bytes at %p\n", cast(uint)(mem*T.sizeof), res); }
 147       return cast(T*)res;
 148     } else {
 149       import core.stdc.stdlib : malloc;
 150       auto res = malloc(mem*T.sizeof);
 151       if (res is null) onOutOfMemoryError();
 152       static if (is(T == struct)) {
 153         import core.stdc.string : memcpy;
 154         static immutable T i = T.init;
 155         foreach (immutable idx; 0..mem) memcpy(res+idx, &i, T.sizeof);
 156       }
 157       debug(iv_vfs_arcz_alloc) { import core.stdc.stdio : printf; printf("allocated %u bytes at %p\n", cast(uint)(mem*T.sizeof), res); }
 158       return cast(T*)res;
 159     }
 160   }
 161
 162   static void xfree(T) (T* ptr) {
 163     if (ptr !is null) {
 164       import core.stdc.stdlib : free;
 165       debug(iv_vfs_arcz_alloc) { import core.stdc.stdio : printf; printf("freing at %p\n", ptr); }
 166       free(ptr);
 167     }
 168   }
 169
 170   static if (iv_vfs_arcz_has_balz) static ubyte balzDictSize (uint blockSize) {
 171     foreach (ubyte bits; Balz.MinDictBits..Balz.MaxDictBits+1) {
 172       if ((1U<<bits) >= blockSize) return bits;
 173     }
 174     return Balz.MaxDictBits;
 175   }
 176
 177   // unpack exactly `destlen` bytes
 178   static if (iv_vfs_arcz_has_balz) static void unpackBlockBalz (void* dest, uint destlen, const(void)* src, uint srclen, uint blocksize) {
 179     Unbalz bz;
 180     bz.reinit(balzDictSize(blocksize));
 181     int ipos, opos;
 182     auto dc = bz.decompress(
 183       // reader
 184       (buf) {
 185         import core.stdc.string : memcpy;
 186         if (ipos >= srclen) return 0;
 187         uint rd = destlen-ipos;
 188         if (rd > buf.length) rd = cast(uint)buf.length;
 189         memcpy(buf.ptr, src+ipos, rd);
 190         ipos += rd;
 191         return rd;
 192       },
 193       // writer
 194       (buf) {
 195         //if (opos+buf.length > destlen) throw new Exception("error unpacking archive");
 196         uint wr = destlen-opos;
 197         if (wr > buf.length) wr = cast(uint)buf.length;
 198         if (wr > 0) {
 199           import core.stdc.string : memcpy;
 200           memcpy(dest+opos, buf.ptr, wr);
 201           opos += wr;
 202         }
 203       },
 204       // unpack length
 205       destlen
 206     );
 207     if (opos != destlen) throw new Exception("error unpacking archive");
 208   }
 209
 210   // unpack exactly `destlen` bytes
 211   static if (iv_vfs_arcz_has_dlzma) static void unpackBlockLzma (void* dest, uint destlen, const(void)* src, uint srclen, uint blocksize) {
 212     if (srclen < LZMA_PROPS_SIZE) throw new Exception("error unpacking LZMA data");
 213     if (srclen == LZMA_PROPS_SIZE) {
 214       if (destlen == 0) return;
 215       throw new Exception("error unpacking LZMA data");
 216     }
 217     const(ubyte)* srcBytes = cast(const(ubyte)*)src;
 218     ubyte* destBytes = cast(ubyte*)dest;
 219     CLzmaDec lzdec;
 220     LzmaDec_Init(&lzdec);
 221     if (LzmaDec_Allocate(&lzdec, srcBytes, LZMA_PROPS_SIZE, &lzmaDefAllocator) != SZ_OK) {
 222       throw new Exception("error allocating LZMA decoder");
 223     }
 224     scope(exit) LzmaDec_Free(&lzdec, &lzmaDefAllocator);
 225     srcBytes += LZMA_PROPS_SIZE;
 226     srclen -= LZMA_PROPS_SIZE;
 227
 228     usize dlen = destlen;
 229     usize slen = srclen;
 230     ELzmaStatus status;
 231     immutable dres = LzmaDec_DecodeToBuf(&lzdec, destBytes, &dlen, srcBytes, &slen, LZMA_FINISH_ANY, &status);
 232     if (dres != SZ_OK || dlen != destlen) throw new Exception("error unpacking LZMA data");
 233   }
 234
 235   static void unpackBlockZLib (void* dest, uint destlen, const(void)* src, uint srclen, uint blocksize) {
 236     z_stream zs;
 237     zs.avail_in = 0;
 238     zs.avail_out = 0;
 239     // initialize unpacker
 240     if (inflateInit2(&zs, 15) != Z_OK) throw new Exception("can't initialize zlib");
 241     scope(exit) inflateEnd(&zs);
 242     zs.next_in = cast(typeof(zs.next_in))src;
 243     zs.avail_in = srclen;
 244     zs.next_out = cast(typeof(zs.next_out))dest;
 245     zs.avail_out = destlen;
 246     while (zs.avail_out > 0) {
 247       auto err = inflate(&zs, Z_SYNC_FLUSH);
 248       if (err != Z_STREAM_END && err != Z_OK) throw new Exception("error unpacking archive");
 249       if (err == Z_STREAM_END) break;
 250     }
 251     if (zs.avail_out != 0) throw new Exception("error unpacking archive");
 252   }
 253
 254   static void unpackBlock (void* dest, uint destlen, const(void)* src, uint srclen, uint blocksize, in Packer packer) {
 255     final switch (packer) {
 256       case Packer.Zlib:
 257         unpackBlockZLib(dest, destlen, src, srclen, blocksize);
 258         break;
 259       case Packer.Balz:
 260         static if (iv_vfs_arcz_has_balz) {
 261           unpackBlockBalz(dest, destlen, src, srclen, blocksize);
 262           break;
 263         } else {
 264           throw new Exception("no Balz support was compiled in ArcZ");
 265         }
 266       case Packer.Lzma:
 267         static if (iv_vfs_arcz_has_dlzma) {
 268           unpackBlockLzma(dest, destlen, src, srclen, blocksize);
 269           break;
 270         } else {
 271           throw new Exception("no LZMA support was compiled in ArcZ");
 272         }
 273     }
 274   }
 275
 276 public:
 277   this (in ArzArchive arc) {
 278     assert(nfop == 0);
 279     nfop = arc.nfop;
 280     if (nfop) ++nfo.rc;
 281   }
 282
 283   this (this) {
 284     if (nfop) ++nfo.rc;
 285   }
 286
 287   ~this () { close(); }
 288
 289   void opAssign (in ArzArchive arc) {
 290     if (arc.nfop) {
 291       auto n = cast(Nfo*)arc.nfop;
 292       ++n.rc;
 293     }
 294     decRef();
 295     nfop = arc.nfop;
 296   }
 297
 298   void close () { decRef(); }
 299
 300   //@property FileInfo[string] files () { return (nfop ? nfo.files : null); }
 301
 302   void openArchive (VFile fl) {
 303     debug/*(arcz)*/ import core.stdc.stdio : printf;
 304     char[4] sign;
 305     Packer packer;
 306     readBuf(fl, sign[]);
 307     if (sign != "CZA2") throw new Exception("invalid arcz archive file");
 308     switch (readUbyte(fl)) {
 309       case 0: packer = Packer.Zlib; break;
 310       case 1: packer = Packer.Balz; break;
 311       case 2: packer = Packer.Lzma; break;
 312       default: throw new Exception("invalid version of arcz archive file");
 313     }
 314     uint indexofs = readUint(fl); // index offset in file
 315     uint pkidxsize = readUint(fl); // packed index size
 316     uint idxsize = readUint(fl); // unpacked index size
 317     if (pkidxsize == 0 || idxsize == 0 || indexofs == 0) throw new Exception("invalid arcz archive file");
 318     // now read index
 319     ubyte* idxbuf = null;
 320     scope(failure) xfree(idxbuf);
 321     {
 322       auto pib = xalloc!ubyte(pkidxsize);
 323       scope(exit) xfree(pib);
 324       fl.seek(indexofs);
 325       readBuf(fl, pib[0..pkidxsize]);
 326       idxbuf = xalloc!ubyte(idxsize);
 327       unpackBlock(idxbuf, idxsize, pib, pkidxsize, idxsize, packer);
 328     }
 329
 330     // parse index and build structures
 331     uint idxbufpos = 0;
 332
 333     ubyte getUbyte () {
 334       if (idxsize-idxbufpos < ubyte.sizeof) throw new Exception("invalid index for arcz archive file");
 335       return idxbuf[idxbufpos++];
 336     }
 337
 338     uint getUint () {
 339       if (idxsize-idxbufpos < uint.sizeof) throw new Exception("invalid index for arcz archive file");
 340       version(BigEndian) {
 341         import core.bitop : bswap;
 342         uint v = *cast(uint*)(idxbuf+idxbufpos);
 343         idxbufpos += 4;
 344         return bswap(v);
 345       } else version(LittleEndian) {
 346         uint v = *cast(uint*)(idxbuf+idxbufpos);
 347         idxbufpos += 4;
 348         return v;
 349       } else {
 350         static assert(0, "wtf?!");
 351       }
 352     }
 353
 354     void getBuf (void[] buf) {
 355       if (buf.length > 0) {
 356         import core.stdc.string : memcpy;
 357         if (idxsize-idxbufpos < buf.length) throw new Exception("invalid index for arcz archive file");
 358         memcpy(buf.ptr, idxbuf+idxbufpos, buf.length);
 359         idxbufpos += buf.length;
 360       }
 361     }
 362
 363     void skipBuf (uint len) {
 364       if (len > 0) {
 365         if (idxsize-idxbufpos < len) throw new Exception("invalid index for arcz archive file");
 366         idxbufpos += len;
 367       }
 368     }
 369
 370     // allocate shared info struct
 371     Nfo* nfo = xalloc!Nfo(1);
 372     assert(nfo.rc == 1);
 373     debug(iv_vfs_arcz_rc) { import core.stdc.stdio : printf; printf("Nfo %p allocated\n", nfo); }
 374     scope(failure) decRef();
 375     nfop = cast(usize)nfo;
 376     /* no need to, there is nothing that needs GC there
 377     {
 378       import core.memory : GC;
 379       GC.addRange(nfo, Nfo.sizeof);
 380     }
 381     */
 382
 383     // read chunk info and data
 384     nfo.packer = packer;
 385     nfo.chunkSize = getUint;
 386     auto ccount = getUint; // chunk count
 387     nfo.lastChunkSize = getUint;
 388     debug(iv_vfs_arcz_dirread) printf("chunk size: %u\nchunk count: %u\nlast chunk size:%u\n", nfo.chunkSize, ccount, nfo.lastChunkSize);
 389     if (ccount == 0 || nfo.chunkSize < 1 || nfo.lastChunkSize < 1 || nfo.lastChunkSize > nfo.chunkSize) throw new Exception("invalid arcz archive file");
 390     /*
 391     nfo.chunks.length = ccount;
 392     // chunk offsets and sizes
 393     foreach (ref ci; nfo.chunks) {
 394       ci.ofs = getUint;
 395       ci.pksize = getUint;
 396     }
 397     */
 398     nfo.chunks = cast(ChunkInfo*)(idxbuf+idxbufpos);
 399     nfo.ccount = ccount;
 400     skipBuf(ccount*8);
 401     // fix endianness
 402     version(BigEndian) {
 403       foreach (ref ChunkInfo ci; nfo.chunks[0..ccount]) {
 404         import core.bitop : bswap;
 405         ci.ofs = bswap(ci.ofs);
 406         ci.pksize = bswap(ci.pksize);
 407       }
 408     }
 409     // read file count and info
 410     auto fcount = getUint;
 411     if (fcount == 0) throw new Exception("empty arcz archive");
 412     debug(iv_vfs_arcz_dirread) printf("file count: %u\n", fcount);
 413     if (fcount >= uint.max/(5*4)) throw new Exception("too many files in arcz archive");
 414     if (fcount*(5*4) > idxsize-idxbufpos) throw new Exception("invalid index in arcz archive");
 415     nfo.files = cast(FileInfo*)(idxbuf+idxbufpos);
 416     nfo.fcount = fcount;
 417     skipBuf(fcount*(5*4));
 418     //immutable uint ubofs = idxbufpos;
 419     //immutable uint ubsize = idxsize-ubofs;
 420     foreach (ref FileInfo fi; nfo.files[0..fcount]) {
 421       // fix endianness
 422       version(BigEndian) {
 423         import core.bitop : bswap;
 424         fi.nameofs = bswap(fi.nameofs);
 425         fi.namelen = bswap(fi.namelen);
 426         fi.chunk = bswap(fi.chunk);
 427         fi.chunkofs = bswap(fi.chunkofs);
 428         fi.size = bswap(fi.size);
 429       }
 430       if (fi.namelen > idxsize || fi.nameofs > idxsize || fi.nameofs+fi.namelen > idxsize) throw new Exception("invalid index in arcz archive");
 431     }
 432     // transfer achive file ownership
 433     nfo.afl = fl;
 434     nfo.index = idxbuf;
 435   }
 436
 437   //bool exists (const(char)[] name) { if (nfop) return ((name in nfo.files) !is null); else return false; }
 438
 439   AZFile openByIndex (uint idx) {
 440     if (!nfop) throw new Exception("can't open file from non-opened archive");
 441     if (idx >= nfo.fcount) throw new Exception("can't open non-existing file");
 442     auto fi = nfo.files+idx;
 443     auto zl = xalloc!LowLevelPackedRO(1);
 444     scope(failure) xfree(zl);
 445     debug(iv_vfs_arcz_rc) { import core.stdc.stdio : printf; printf("Zl %p allocated\n", zl); }
 446     zl.setup(nfo, fi.chunk, fi.chunkofs, fi.size);
 447     AZFile fl;
 448     fl.zlp = cast(usize)zl;
 449     return fl;
 450   }
 451
 452 private:
 453   static struct LowLevelPackedRO {
 454     private import etc.c.zlib;
 455
 456     uint rc = 1;
 457     usize nfop; // hide it from GC
 458
 459     private @property inout(Nfo*) nfo () inout pure nothrow @trusted @nogc { pragma(inline, true); return cast(typeof(return))nfop; }
 460     static void decRef (usize me) {
 461       if (me) {
 462         auto zl = cast(LowLevelPackedRO*)me;
 463         assert(zl.rc);
 464         if (--zl.rc == 0) {
 465           import core.stdc.stdlib : free;
 466           if (zl.chunkData !is null) free(zl.chunkData);
 467           version(iv_vfs_arcz_use_more_memory) if (zl.pkdata !is null) free(zl.pkdata);
 468           Nfo.decRef(zl.nfop);
 469           free(zl);
 470           debug(iv_vfs_arcz_rc) { import core.stdc.stdio : printf; printf("Zl %p freed\n", zl); }
 471         } else {
 472           //debug(iv_vfs_arcz_rc) { import core.stdc.stdio : printf; printf("Zl %p; rc after decRef is %u\n", zl, zl.rc); }
 473         }
 474       }
 475     }
 476
 477     uint nextchunk; // next chunk to read
 478     uint curcpos; // position in current chunk
 479     uint curcsize; // number of valid bytes in `chunkData`
 480     uint stchunk; // starting chunk
 481     uint stofs; // offset in starting chunk
 482     uint totalsize; // total file size
 483     uint pos; // current file position
 484     uint lastrdpos; // last actual read position
 485     z_stream zs;
 486     ubyte* chunkData; // can be null
 487     version(iv_vfs_arcz_use_more_memory) {
 488       ubyte* pkdata;
 489       uint pkdatasize;
 490     }
 491
 492     @disable this (this);
 493
 494     void setup (Nfo* anfo, uint astchunk, uint astofs, uint asize) {
 495       assert(anfo !is null);
 496       assert(rc == 1);
 497       nfop = cast(usize)anfo;
 498       ++anfo.rc;
 499       nextchunk = stchunk = astchunk;
 500       //curcpos = 0;
 501       stofs = astofs;
 502       totalsize = asize;
 503     }
 504
 505     @property bool eof () { pragma(inline, true); return (pos >= totalsize); }
 506
 507     // return less than chunk size if our file fits in one non-full chunk completely
 508     uint justEnoughMemory () pure const nothrow @safe @nogc {
 509       pragma(inline, true);
 510       version(none) {
 511         return nfo.chunkSize;
 512       } else {
 513         return (totalsize < nfo.chunkSize && stofs+totalsize < nfo.chunkSize ? stofs+totalsize : nfo.chunkSize);
 514       }
 515     }
 516
 517     void unpackNextChunk () {
 518       if (nfop == 0) assert(0, "wtf?!");
 519       //scope(failure) if (chunkData !is null) { xfree(chunkData); chunkData = null; }
 520       debug(iv_vfs_arcz_unp) { import core.stdc.stdio : printf; printf("unpacking chunk %u\n", nextchunk); }
 521       // allocate buffer for unpacked data
 522       if (chunkData is null) {
 523         // optimize things a little: if our file fits in less then one chunk, allocate "just enough" memory
 524         chunkData = xalloc!(ubyte, false)(justEnoughMemory);
 525       }
 526       auto chunk = &nfo.chunks[nextchunk];
 527       if (chunk.pksize == nfo.chunkSize) {
 528         // unpacked chunk, just read it
 529         debug(iv_vfs_arcz_unp) { import core.stdc.stdio : printf; printf(" chunk is not packed\n"); }
 530         nfo.afl.seek(chunk.ofs, Seek.Set);
 531         nfo.afl.rawReadExact(chunkData[0..nfo.chunkSize]);
 532         curcsize = nfo.chunkSize;
 533       } else {
 534         // packed chunk, unpack it
 535         // allocate buffer for packed data
 536         version(iv_vfs_arcz_use_more_memory) {
 537           import core.stdc.stdlib : realloc;
 538           if (pkdatasize < chunk.pksize) {
 539             import core.exception : onOutOfMemoryError;
 540             auto newpk = realloc(pkdata, chunk.pksize);
 541             if (newpk is null) onOutOfMemoryError();
 542             debug(iv_vfs_arcz_alloc) { import core.stdc.stdio : printf; printf("reallocated from %u to %u bytes; %p -> %p\n", cast(uint)pkdatasize, cast(uint)chunk.pksize, pkdata, newpk); }
 543             pkdata = cast(ubyte*)newpk;
 544             pkdatasize = chunk.pksize;
 545           }
 546           alias pkd = pkdata;
 547         } else {
 548           auto pkd = xalloc!(ubyte, false)(chunk.pksize);
 549           scope(exit) xfree(pkd);
 550         }
 551         nfo.afl.seek(chunk.ofs, Seek.Set);
 552         nfo.afl.rawReadExact(pkd[0..chunk.pksize]);
 553         uint upsize = (nextchunk == nfo.ccount-1 ? nfo.lastChunkSize : nfo.chunkSize); // unpacked chunk size
 554         immutable uint cksz = upsize;
 555         immutable uint jem = justEnoughMemory;
 556         if (upsize > jem) upsize = jem;
 557         debug(iv_vfs_arcz_unp) { import core.stdc.stdio : printf; printf(" unpacking %u bytes to %u bytes\n", chunk.pksize, upsize); }
 558         ArzArchive.unpackBlock(chunkData, upsize, pkd, chunk.pksize, cksz, nfo.packer);
 559         curcsize = upsize;
 560       }
 561       curcpos = 0;
 562       // fix first chunk offset if necessary
 563       if (nextchunk == stchunk && stofs > 0) {
 564         // it's easier to just memmove it
 565         import core.stdc.string : memmove;
 566         assert(stofs < curcsize);
 567         memmove(chunkData, chunkData+stofs, curcsize-stofs);
 568         curcsize -= stofs;
 569       }
 570       ++nextchunk; // advance to next chunk
 571     }
 572
 573     void syncReadPos () {
 574       if (pos >= totalsize || pos == lastrdpos) return;
 575       immutable uint fcdata = nfo.chunkSize-stofs; // number of our bytes in the first chunk
 576       // does our pos lie in the first chunk?
 577       if (pos < fcdata) {
 578         // yep, just read it
 579         if (nextchunk != stchunk+1) {
 580           nextchunk = stchunk;
 581           unpackNextChunk(); // we'll need it anyway
 582         } else {
 583           // just rewind
 584           curcpos = 0;
 585         }
 586         curcpos += pos;
 587         lastrdpos = pos;
 588         return;
 589       }
 590       // find the chunk we want
 591       uint npos = pos-fcdata;
 592       uint xblock = stchunk+1+npos/nfo.chunkSize;
 593       uint curcstart = (xblock-(stchunk+1))*nfo.chunkSize+fcdata;
 594       if (xblock != nextchunk-1) {
 595         // read and unpack this chunk
 596         nextchunk = xblock;
 597         unpackNextChunk();
 598       } else {
 599         // just rewind
 600         curcpos = 0;
 601       }
 602       assert(pos >= curcstart && pos < curcstart+nfo.chunkSize);
 603       uint skip = pos-curcstart;
 604       lastrdpos = pos;
 605       curcpos += skip;
 606     }
 607
 608     ssize read (void* buf, usize count) {
 609       if (buf is null) return -1;
 610       if (count == 0 || totalsize == 0) return 0;
 611       if (totalsize >= 0 && pos >= totalsize) return 0; // EOF
 612       syncReadPos();
 613       assert(lastrdpos == pos);
 614       if (cast(long)pos+count > totalsize) count = totalsize-pos;
 615       auto res = count;
 616       while (count > 0) {
 617         debug(iv_vfs_arcz_read) { import core.stdc.stdio : printf; printf("reading %u bytes; pos=%u; lastrdpos=%u; curcpos=%u; curcsize=%u\n", count, pos, lastrdpos, curcpos, curcsize); }
 618         import core.stdc.string : memcpy;
 619         if (curcpos >= curcsize) {
 620           unpackNextChunk(); // we want next chunk!
 621           debug(iv_vfs_arcz_read) { import core.stdc.stdio : printf; printf(" *reading %u bytes; pos=%u; lastrdpos=%u; curcpos=%u; curcsize=%u\n", count, pos, lastrdpos, curcpos, curcsize); }
 622         }
 623         assert(curcpos < curcsize && curcsize != 0);
 624         ssize rd = (curcsize-curcpos >= count ? count : curcsize-curcpos);
 625         assert(rd > 0);
 626         memcpy(buf, chunkData+curcpos, rd);
 627         curcpos += rd;
 628         pos += rd;
 629         lastrdpos += rd;
 630         buf += rd;
 631         count -= rd;
 632       }
 633       assert(pos == lastrdpos);
 634       return res;
 635     }
 636
 637     long lseek (long ofs, int origin) {
 638       //TODO: overflow checks
 639       switch (origin) {
 640         case SEEK_SET: break;
 641         case SEEK_CUR: ofs += pos; break;
 642         case SEEK_END:
 643           if (ofs > 0) ofs = 0;
 644           if (-ofs > totalsize) ofs = -cast(long)totalsize;
 645           ofs += totalsize;
 646           break;
 647         default:
 648           return -1;
 649       }
 650       if (ofs < 0) return -1;
 651       if (totalsize >= 0 && ofs > totalsize) ofs = totalsize;
 652       pos = cast(uint)ofs;
 653       return pos;
 654     }
 655   }
 656 }
 657
 658
 659 // ////////////////////////////////////////////////////////////////////////// //
 660 // Opened file.
 661 private struct AZFile {
 662 private import core.stdc.stdio : SEEK_SET, SEEK_CUR, SEEK_END;
 663 private:
 664   usize zlp;
 665
 666   private @property inout(ArzArchive.LowLevelPackedRO)* zl () inout pure nothrow @trusted @nogc { pragma(inline, true); return cast(typeof(return))zlp; }
 667   private void decRef () { pragma(inline, true); ArzArchive.LowLevelPackedRO.decRef(zlp); zlp = 0; }
 668
 669 public:
 670   this (in AZFile afl) {
 671     assert(zlp == 0);
 672     zlp = afl.zlp;
 673     if (zlp) ++zl.rc;
 674   }
 675
 676   this (this) {
 677     if (zlp) ++zl.rc;
 678   }
 679
 680   ~this () { close(); }
 681
 682   void opAssign (in AZFile afl) {
 683     if (afl.zlp) {
 684       auto n = cast(ArzArchive.LowLevelPackedRO*)afl.zlp;
 685       ++n.rc;
 686     }
 687     decRef();
 688     zlp = afl.zlp;
 689   }
 690
 691   void close () { decRef(); }
 692
 693   @property bool isOpen () const pure nothrow @safe @nogc { pragma(inline, true); return (zlp != 0); }
 694   @property uint size () const pure nothrow @safe @nogc { pragma(inline, true); return (zlp ? zl.totalsize : 0); }
 695   @property uint tell () const pure nothrow @safe @nogc { pragma(inline, true); return (zlp ? zl.pos : 0); }
 696
 697   void seek (long ofs, int origin=SEEK_SET) {
 698     if (!zlp) throw new Exception("can't seek in closed file");
 699     auto res = zl.lseek(ofs, origin);
 700     if (res < 0) throw new Exception("seek error");
 701   }
 702
 703   //TODO: overflow check
 704   T[] rawRead(T) (T[] buf) if (!is(T == const) && !is(T == immutable)) {
 705     if (!zlp) throw new Exception("can't read from closed file");
 706     if (buf.length > 0) {
 707       auto res = zl.read(buf.ptr, buf.length*T.sizeof);
 708       if (res == -1 || res%T.sizeof != 0) throw new Exception("read error");
 709       return buf[0..res/T.sizeof];
 710     } else {
 711       return buf[0..0];
 712     }
 713   }
 714 }
 715
 716
 717 // ////////////////////////////////////////////////////////////////////////// //
 718 public final class VFSDriverArcZ : VFSDriver {
 719   mixin VFSSimpleArchiveDriverMixin;
 720
 721 private:
 722   ArzArchive arc;
 723
 724   static struct FileInfo {
 725     long size;
 726     string name; // with path
 727     uint aidx;
 728   }
 729
 730   VFile wrap (usize idx) { return wrapStream(arc.openByIndex(dir[idx].aidx), dir[idx].name); }
 731
 732   void open (VFile fl, const(char)[] prefixpath) {
 733     arc.openArchive(fl);
 734     debug(iv_vfs_arcz_dirread) { import core.stdc.stdio; printf("files: %u\n", cast(uint)arc.nfo.fcount); }
 735     foreach (immutable aidx, ref afi; arc.nfo.files[0..arc.nfo.fcount]) {
 736       auto xname = afi.name(arc.nfo.index);
 737       if (xname.length == 0) continue; // just in case
 738       FileInfo fi;
 739       fi.size = afi.size;
 740       fi.aidx = cast(uint)aidx;
 741       auto name = new char[](prefixpath.length+xname.length);
 742       if (prefixpath.length) name[0..prefixpath.length] = prefixpath;
 743       name[prefixpath.length..$] = xname[];
 744       fi.name = cast(string)name; // it is safe to cast here
 745       dir.arrayAppendUnsafe(fi);
 746     }
 747   }
 748 }
 749
 750
 751 // ////////////////////////////////////////////////////////////////////////// //
 752 /* arcz file format:
 753 header
 754 ======
 755 db 'CZA2'     ; signature
 756 db version    ; 0: zlib; 1: balz
 757 dd indexofs   ; offset to packed index
 758 dd pkindexsz  ; size of packed index
 759 dd upindexsz  ; size of unpacked index
 760
 761
 762 index
 763 =====
 764 dd chunksize    ; unpacked chunk size in bytes
 765 dd chunkcount   ; number of chunks in file
 766 dd lastchunksz  ; size of last chunk (it may be incomplete); 0: last chunk is completely used (all `chunksize` bytes)
 767
 768 then chunk offsets and sizes follows:
 769   dd chunkofs   ; from file start
 770   dd pkchunksz  ; size of (possibly packed) chunk data; if it equals to `chunksize`, this chunk is not packed
 771
 772 then file list follows:
 773 dd filecount  ; number of files in archive
 774
 775 then file info follows:
 776   dd nameofs     ; (in index)
 777   dd namelen     ; length of name (can't be 0)
 778   dd firstchunk  ; chunk where file starts
 779   dd firstofs    ; offset in first chunk (unpacked) where file starts
 780   dd filesize    ; unpacked file size
 781
 782 then name buffer follows -- just bytes
 783 */