Code/Tools/CryCommonTools/ZipDir/ZipDirCacheRW.cpp

   1 // Copyright 2001-2018 Crytek GmbH / Crytek Group. All rights reserved.
   2
   3 #include "StdAfx.h"
   4 #include <CryCore/smartptr.h>
   5 #include "Util.h"
   6 #include "ZipFileFormat.h"
   7 #include "zipdirstructures.h"
   8 #include "ZipDirTree.h"
   9 #include "ZipDirList.h"
  10 #include "ZipDirCache.h"
  11 #include "ZipDirCacheRW.h"
  12 #include "ZipDirCacheFactory.h"
  13 #include "ZipDirFindRW.h"
  14
  15 #include "ThreadUtils.h"
  16
  17 #include <zlib.h>  // declaration of Z_OK for ZipRawDecompress
  18
  19 using namespace ZipFile;
  20
  21 enum PackFileStatus
  22 {
  23         PACKFILE_COMPRESSED,
  24
  25         PACKFILE_ADDED,
  26         PACKFILE_UPTODATE,
  27         PACKFILE_SKIPPED,
  28         PACKFILE_MISSING,
  29         PACKFILE_FAILED
  30 };
  31
  32 class PackFilePool;
  33 struct PackFileBatch
  34 {
  35         PackFilePool* pool;
  36
  37         int zipMaxSize;
  38         int sourceMinSize;
  39         int sourceMaxSize;
  40         int compressionMethod;
  41         int compressionLevel;
  42
  43         PackFileBatch()
  44         : pool(0)
  45         , sourceMinSize(0)
  46         , sourceMaxSize(0)
  47         , zipMaxSize(0)
  48         , compressionMethod(0)
  49         , compressionLevel(0)
  50         {
  51         }
  52 };
  53
  54 class PackFilePool;
  55 struct PackFileJob
  56 {
  57         int index;
  58         int key;
  59         PackFileBatch* batch;
  60         const char* relativePathSrc;
  61         const char* realFilename;
  62
  63         unsigned int existingCRC;
  64
  65         void* compressedData;
  66         unsigned long compressedSize;
  67         unsigned long compressedSizePreviously;
  68
  69         void* uncompressedData;
  70         unsigned long uncompressedSize;
  71         unsigned long uncompressedSizePreviously;
  72
  73         __int64 modTime;
  74         ZipDir::ErrorEnum zdError;
  75         PackFileStatus status;
  76
  77         PackFileJob()
  78         : index(0)
  79         , key(0)
  80         , batch(0)
  81         , realFilename(0)
  82         , relativePathSrc(0)
  83         , existingCRC(0)
  84         , compressedData(0)
  85         , compressedSize(0)
  86         , compressedSizePreviously(0)
  87         , uncompressedData(0)
  88         , uncompressedSize(0)
  89         , uncompressedSizePreviously(0)
  90         , modTime(0)
  91         , zdError(ZipDir::ZD_ERROR_NOT_IMPLEMENTED)
  92         , status(PACKFILE_FAILED)
  93         {
  94         }
  95
  96         void DetachUncompressedData()
  97         {
  98                 if (uncompressedData && uncompressedData == compressedData)
  99                 {
 100                         compressedData = 0;
 101                         compressedSize = 0;
 102                 }
 103
 104                 uncompressedData = 0;
 105                 uncompressedSize = 0;
 106         }
 107
 108         ~PackFileJob()
 109         {
 110                 if (compressedData && compressedData != uncompressedData)
 111                 {
 112                         free(compressedData);
 113                         compressedData = 0;
 114                 }
 115
 116                 if (uncompressedData)
 117                 {
 118                         free(uncompressedData);
 119                         uncompressedData = 0;
 120                 }
 121         }
 122 };
 123
 124
 125 // ---------------------------------------------------------------------------
 126
 127 static void PackFileFromDisc(PackFileJob* job);
 128
 129 class PackFilePool
 130 {
 131 public:
 132         PackFilePool(int numFiles, size_t memoryLimit)
 133         : m_pool(false)
 134         , m_skip(false)
 135         , m_awaitedFile(0)
 136         , m_memoryLimit(memoryLimit)
 137         , m_allocatedMemory(0)
 138         {
 139                 m_files.reserve(numFiles);
 140         }
 141
 142         ~PackFilePool()
 143         {
 144         }
 145
 146         void Submit(int key, const PackFileJob& job)
 147         {
 148                 PackFileJob* newJob = new PackFileJob(job);
 149
 150                 // index in queue, and custom key for identification
 151                 newJob->index = int(m_files.size());
 152                 newJob->key = key;
 153
 154                 m_files.push_back(newJob);
 155         }
 156
 157         PackFileJob* WaitForFile(int index)
 158         {
 159                 while(true)
 160                 {
 161                         {
 162                                 ThreadUtils::AutoLock lock(m_filesLock);
 163                                 m_awaitedFile = index;
 164                                 if (size_t(index) >= m_files.size())
 165                                         return 0;
 166                                 if (m_files[index])
 167                                         return m_files[index];
 168                         }
 169                         Sleep(0);
 170                 }
 171
 172                 assert(0);
 173                 return 0;
 174         }
 175
 176         void Start(int numThreads)
 177         {
 178                 for (size_t i = 0; i < m_files.size(); ++i)
 179                 {
 180                         PackFileJob* job = m_files[i];
 181                         m_files[i] = 0;
 182                         m_pool.Submit(&ProcessFile, job);
 183                 }
 184
 185                 m_pool.Start(numThreads);
 186         }
 187
 188         size_t GetJobCount() const
 189         {
 190                 return m_files.size();
 191         }
 192
 193         void SkipPendingFiles()
 194         {
 195                 m_skip = true;
 196         }
 197
 198         void ReleaseFile(int index)
 199         {
 200                 assert(m_files[index] != 0);
 201                 if (m_files[index])
 202                 {
 203                         if (m_memoryLimit != 0)
 204                         {
 205                                 ThreadUtils::AutoLock lock(m_filesLock);
 206
 207                                 m_allocatedMemory -= m_files[index]->uncompressedSize;
 208                                 m_allocatedMemory -= m_files[index]->compressedSize;
 209                         }
 210
 211                         delete m_files[index];
 212                         m_files[index] = 0;
 213                 }
 214         }
 215
 216 private:
 217
 218         // called from non-main thread
 219         static void ProcessFile(PackFileJob* job)
 220         {
 221                 PackFilePool* self = job->batch->pool;
 222
 223                 if (!self->m_skip)
 224                 {
 225                         if (self->m_memoryLimit != 0)
 226                         {
 227                                 while (true)
 228                                 {
 229                                         size_t allocatedMemory = 0;
 230                                         int awaitedFile = 0;
 231                                         {
 232                                                 ThreadUtils::AutoLock lock(self->m_filesLock);
 233                                                 allocatedMemory = self->m_allocatedMemory;
 234                                                 awaitedFile = self->m_awaitedFile;
 235                                         }
 236
 237                                         if (allocatedMemory > self->m_memoryLimit && job->index > awaitedFile + 1)
 238                                                 Sleep(10); // give time to main thread to write data to file
 239                                         else
 240                                                 break;
 241                                 }
 242                         }
 243
 244                         PackFileFromDisc(job);
 245                 }
 246
 247                 self->FileCompleted(job);
 248         }
 249
 250         // called from non-main thread
 251         void FileCompleted(PackFileJob* job)
 252         {
 253                 ThreadUtils::AutoLock lock(m_filesLock);
 254
 255                 assert(job);
 256                 assert(job->index < m_files.size());
 257                 assert(m_files[job->index] == 0);
 258                 m_files[job->index] = job;
 259
 260                 if (m_memoryLimit != 0)
 261                 {
 262                         m_allocatedMemory += job->uncompressedSize;
 263                         m_allocatedMemory += job->compressedSize;
 264                 }
 265         }
 266
 267         size_t m_memoryLimit;
 268
 269         ThreadUtils::CriticalSection m_filesLock;
 270         std::vector<PackFileJob*> m_files;
 271         int m_awaitedFile;
 272         size_t m_allocatedMemory;
 273         bool m_skip;
 274
 275         ThreadUtils::SimpleThreadPool m_pool;
 276 };
 277
 278 //////////////////////////////////////////////////////////////////////////
 279 static size_t AlignTo(size_t offset, size_t alignment)
 280 {
 281         const size_t remainder = offset % alignment;
 282         return remainder ? offset + alignment - remainder : offset;
 283 }
 284 //////////////////////////////////////////////////////////////////////////
 285 // Calculates new offset of the header to make sure that following data are
 286 // aligned properly
 287 static size_t CalculateAlignedHeaderOffset(const char* fileName, size_t currentOffset, size_t alignment)
 288 {
 289         // Since file should start from header
 290         if (currentOffset == 0)
 291         {
 292                 return 0;
 293         }
 294
 295         // Local header is followed by filename
 296         const size_t totalHeaderSize = sizeof(LocalFileHeader) + strlen(fileName);
 297
 298         // Align end of the header
 299         const size_t dataOffset = AlignTo(currentOffset + totalHeaderSize, alignment);
 300
 301         return dataOffset - totalHeaderSize;
 302 }
 303
 304 //////////////////////////////////////////////////////////////////////////
 305 ZipDir::CacheRW::CacheRW(bool encryptHeaders, const EncryptionKey& encryptionKey)
 306 : m_pFile (NULL)
 307 , m_nFlags (0)
 308 , m_lCDROffset (0)
 309 , m_fileAlignment (1)
 310 , m_bEncryptedHeaders(encryptHeaders)
 311 , m_bHeadersEncryptedOnClose(encryptHeaders)
 312 , m_encryptionKey(encryptionKey)
 313 {
 314
 315         m_nRefCount = 0;
 316 }
 317 //////////////////////////////////////////////////////////////////////////
 318 ZipDir::CacheRW::~CacheRW()
 319 {
 320         Close();
 321 }
 322 //////////////////////////////////////////////////////////////////////////
 323 void ZipDir::CacheRW::AddRef()
 324 {
 325         ++m_nRefCount;
 326 }
 327
 328 //////////////////////////////////////////////////////////////////////////
 329 void ZipDir::CacheRW::Release()
 330 {
 331         if (--m_nRefCount <= 0)
 332                 delete this;
 333 }
 334
 335 void ZipDir::CacheRW::Close()
 336 {
 337         if (m_pFile)
 338         {
 339                 if (!(m_nFlags & FLAGS_READ_ONLY))
 340                 {
 341                         if ((m_nFlags & FLAGS_UNCOMPACTED) && !(m_nFlags&FLAGS_DONT_COMPACT))
 342                         {
 343                                 if (!RelinkZip())
 344                                         WriteCDR();
 345                         }
 346                         else
 347                         if (m_nFlags & FLAGS_CDR_DIRTY)
 348                                 WriteCDR();
 349                 }
 350
 351                 if(m_pFile)                                             // RelinkZip() might have closed the file
 352                         fclose (m_pFile);
 353
 354                 m_pFile = NULL;
 355         }
 356         m_treeDir.Clear();
 357 }
 358
 359 //////////////////////////////////////////////////////////////////////////
 360 char* ZipDir::CacheRW::UnifyPath( char *const str, const char *pPath )
 361 {
 362         assert(str);
 363         const char *src = pPath;
 364         char *trg = str;
 365         while (*src)
 366         {
 367                 if (*src != '/')
 368                 {
 369                         *trg++ = ::tolower(*src++);
 370                 }
 371                 else
 372                 {
 373                         *trg++ = '\\';
 374                         src++;
 375                 }
 376         }
 377         *trg = 0;
 378         return str;
 379 }
 380
 381 //////////////////////////////////////////////////////////////////////////
 382 char* ZipDir::CacheRW::ToUnixPath( char *const str, const char *pPath )
 383 {
 384         assert(str);
 385         const char *src = pPath;
 386         char *trg = str;
 387         while (*src)
 388         {
 389                 if (*src != '/')
 390                 {
 391                         *trg++ = *src++;
 392                 }
 393                 else
 394                 {
 395                         *trg++ = '\\';
 396                         src++;
 397                 }
 398         }
 399         *trg = 0;
 400         return str;
 401 }
 402
 403 //////////////////////////////////////////////////////////////////////////
 404 char* ZipDir::CacheRW::AllocPath( const char *pPath )
 405 {
 406         char str[_MAX_PATH];
 407         char *temp = ToUnixPath(str, pPath);
 408         temp = m_tempStringPool.Append( temp,strlen(temp) );
 409         return temp;
 410 }
 411
 412 static void PackFileFromMemory(PackFileJob* job)
 413 {
 414         if (job->existingCRC != 0)
 415         {
 416                 unsigned int crcCode = (unsigned int)crc32(0, (unsigned char*)job->uncompressedData, job->uncompressedSize);
 417                 if (crcCode == job->existingCRC)
 418                 {
 419                         job->compressedData = 0;
 420                         job->compressedSize = 0;
 421                         job->status = PACKFILE_UPTODATE;
 422                         job->zdError = ZipDir::ZD_ERROR_SUCCESS;
 423                         // This file with same data already in pak, skip it.
 424                         return;
 425                 }
 426         }
 427
 428         switch (job->batch->compressionMethod)
 429         {
 430         case METHOD_DEFLATE_AND_ENCRYPT:
 431         case METHOD_DEFLATE:
 432         {
 433                 // allocate memory for compression. Min is nSize * 1.001 + 12
 434                 if (job->uncompressedSize > 0)
 435                 {
 436                         job->compressedSize = job->uncompressedSize + (job->uncompressedSize >> 3) + 32;
 437                         job->compressedData = malloc(job->compressedSize);
 438                         int error = ZipDir::ZipRawCompress(job->uncompressedData, &job->compressedSize, job->compressedData, job->uncompressedSize, job->batch->compressionLevel);
 439                         if (error == Z_OK)
 440                         {
 441                                 job->status = PACKFILE_COMPRESSED;
 442                                 job->zdError = ZipDir::ZD_ERROR_SUCCESS;
 443                         }
 444                         else
 445                         {
 446                                 job->status = PACKFILE_FAILED;
 447                                 job->zdError = ZipDir::ZD_ERROR_ZLIB_FAILED;
 448                         }
 449                 }
 450                 else
 451                 {
 452                         job->status = PACKFILE_COMPRESSED;
 453                         job->zdError = ZipDir::ZD_ERROR_SUCCESS;
 454
 455                         job->compressedSize = 0;
 456                         job->compressedData = 0;
 457                 }
 458                 break;
 459         }
 460         case METHOD_STORE:
 461                 job->compressedData = job->uncompressedData;
 462                 job->compressedSize = job->uncompressedSize;
 463                 job->status = PACKFILE_COMPRESSED;
 464                 job->zdError = ZipDir::ZD_ERROR_SUCCESS;
 465                 break;
 466
 467         default:
 468                 job->status = PACKFILE_FAILED;
 469                 job->zdError = ZipDir::ZD_ERROR_UNSUPPORTED;
 470                 break;
 471         }
 472 }
 473
 474 bool ZipDir::CacheRW::WriteCompressedData(const char* data, size_t size, bool encrypt, FILE* file)
 475 {
 476         if (size == 0)
 477         {
 478                 return true;
 479         }
 480
 481         std::vector<char> buffer;
 482         if (encrypt)
 483         {
 484                 buffer.resize(size);
 485                 memcpy(&buffer[0], data, size);
 486                 ZipDir::Encrypt(&buffer[0], size, m_encryptionKey);
 487                 data = &buffer[0];
 488         }
 489
 490         // Danny - writing a single large chunk (more than 6MB?) causes
 491         // Windows fwrite to (silently?!) fail. So we're writing data
 492         // in small chunks.
 493         while (size > 0)
 494         {
 495                 const size_t sizeToWrite = Util::getMin(size, size_t(1024 * 1024));
 496                 if (fwrite(data, sizeToWrite, 1, file) != 1)
 497                 {
 498                         return false;
 499                 }
 500                 data += sizeToWrite;
 501                 size -= sizeToWrite;
 502         }
 503
 504         return true;
 505 }
 506
 507 static bool WriteRandomData(FILE* file, size_t size)
 508 {
 509         if (size == 0)
 510         {
 511                 return true;
 512         }
 513
 514         const size_t bufferSize = Util::getMin(size, size_t(1024 * 1024));
 515         std::vector<char> buffer(bufferSize);
 516
 517         while (size > 0)
 518         {
 519                 const size_t sizeToWrite = Util::getMin(size, bufferSize);
 520
 521                 for (size_t i = 0; i < sizeToWrite; ++i)
 522                 {
 523                         buffer[i] = rand() & 0xff;
 524                 }
 525
 526                 if (fwrite(&buffer[0], sizeToWrite, 1, file) != 1)
 527                 {
 528                         return false;
 529                 }
 530
 531                 size -= sizeToWrite;
 532         }
 533
 534         return true;
 535 }
 536
 537 bool ZipDir::CacheRW::WriteNullData(size_t size)
 538 {
 539         if (size == 0)
 540         {
 541                 return true;
 542         }
 543
 544         const size_t bufferSize = Util::getMin(size, size_t(1024 * 1024));
 545         std::vector<char> buffer(bufferSize, 0);
 546
 547         while (size > 0)
 548         {
 549                 const size_t sizeToWrite = Util::getMin(size, bufferSize);
 550
 551                 if (fwrite(&buffer[0], sizeToWrite, 1, m_pFile) != 1)
 552                 {
 553                         return false;
 554                 }
 555
 556                 size -= sizeToWrite;
 557         }
 558
 559         return true;
 560 }
 561
 562 void ZipDir::CacheRW::StorePackedFile(PackFileJob* job)
 563 {
 564         if (job->batch->zipMaxSize > 0 && GetTotalFileSize() > job->batch->zipMaxSize)
 565         {
 566                 job->status = PACKFILE_SKIPPED;
 567                 job->zdError = ZipDir::ZD_ERROR_SUCCESS;
 568                 return;
 569         }
 570
 571         job->status = PACKFILE_FAILED;
 572
 573         char str[_MAX_PATH];
 574         char* relativePath = UnifyPath(str, job->relativePathSrc);
 575
 576         // create or find the file entry.. this object will rollback (delete the object
 577         // if the operation fails) if needed.
 578         FileEntryTransactionAdd pFileEntry(this, AllocPath(job->relativePathSrc), AllocPath(relativePath));
 579
 580         if (!pFileEntry)
 581         {
 582                 job->zdError = ZipDir::ZD_ERROR_INVALID_PATH;
 583                 return;
 584         }
 585
 586         pFileEntry->OnNewFileData(job->uncompressedData, job->uncompressedSize,
 587                                                                                                                 job->compressedSize, job->batch->compressionMethod, false);
 588         pFileEntry->SetFromFileTimeNTFS(job->modTime);
 589
 590         // since we changed the time, we'll have to update CDR
 591         m_nFlags |= FLAGS_CDR_DIRTY;
 592
 593         // the new CDR position, if the operation completes successfully
 594         unsigned lNewCDROffset = m_lCDROffset;
 595
 596         if (pFileEntry->IsInitialized())
 597         {
 598                 // this file entry is already allocated in CDR
 599
 600                 // check if the new compressed data fits into the old place
 601                 unsigned nFreeSpace = pFileEntry->nEOFOffset - pFileEntry->nFileHeaderOffset - (unsigned)sizeof(ZipFile::LocalFileHeader) - (unsigned)strlen(relativePath);
 602
 603                 if (nFreeSpace != job->compressedSize)
 604                         m_nFlags |= FLAGS_UNCOMPACTED;
 605
 606                 if (nFreeSpace >= job->compressedSize)
 607                 {
 608                         // and we can just override the compressed data in the file
 609                         ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, job->relativePathSrc, m_bEncryptedHeaders);
 610                         if (e != ZipDir::ZD_ERROR_SUCCESS)
 611                         {
 612                                 job->zdError = e;
 613                                 return;
 614                         }
 615                 }
 616                 else
 617                 {
 618                         // we need to write the file anew - in place of current CDR
 619                         pFileEntry->nFileHeaderOffset = CalculateAlignedHeaderOffset(job->relativePathSrc, m_lCDROffset, m_fileAlignment);
 620                         ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, job->relativePathSrc, m_bEncryptedHeaders);
 621                         lNewCDROffset = pFileEntry->nEOFOffset;
 622                         if (e != ZipDir::ZD_ERROR_SUCCESS)
 623                         {
 624                                 job->zdError = e;
 625                                 return;
 626                         }
 627                 }
 628         }
 629         else
 630         {
 631                 pFileEntry->nFileHeaderOffset = CalculateAlignedHeaderOffset(job->relativePathSrc, m_lCDROffset, m_fileAlignment);
 632                 ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, job->relativePathSrc, m_bEncryptedHeaders);
 633                 if (e != ZipDir::ZD_ERROR_SUCCESS)
 634                 {
 635                         job->zdError = e;
 636                         return;
 637                 }
 638
 639                 lNewCDROffset = pFileEntry->nFileDataOffset + job->compressedSize;
 640
 641                 m_nFlags |= FLAGS_CDR_DIRTY;
 642         }
 643
 644         // now we have the fresh local header and data offset
 645
 646 #if CRY_PLATFORM_WINDOWS
 647         if (_fseeki64 (m_pFile, (__int64)pFileEntry->nFileDataOffset, SEEK_SET)!=0)
 648 #else
 649   if (fseek (m_pFile, pFileEntry->nFileDataOffset, SEEK_SET)!=0)
 650 #endif
 651         {
 652                 job->zdError = ZD_ERROR_IO_FAILED;
 653                 return;
 654         }
 655
 656         const bool encrypt = pFileEntry->nMethod == METHOD_DEFLATE_AND_ENCRYPT;
 657
 658         if (!WriteCompressedData((char*)job->compressedData, job->compressedSize, encrypt, m_pFile))
 659         {
 660                 job->zdError = ZD_ERROR_IO_FAILED;
 661                 return;
 662         }
 663
 664         // since we wrote the file successfully, update the new CDR position
 665         m_lCDROffset = lNewCDROffset;
 666         pFileEntry.Commit();
 667
 668         job->status = PACKFILE_ADDED;
 669         job->zdError = ZD_ERROR_SUCCESS;
 670 }
 671
 672 // Adds a new file to the zip or update an existing one
 673 // adds a directory (creates several nested directories if needed)
 674 ZipDir::ErrorEnum ZipDir::CacheRW::UpdateFile (const char* szRelativePathSrc, void* pUncompressed, unsigned nSize,
 675                                                                                                                                                                                          unsigned nCompressionMethod, int nCompressionLevel, __time64_t modTime)
 676 {
 677         char str[_MAX_PATH];
 678         char *szRelativePath = UnifyPath(str, szRelativePathSrc);
 679
 680
 681         PackFileBatch batch;
 682         batch.compressionMethod = nCompressionMethod;
 683         batch.compressionLevel = nCompressionLevel;
 684
 685         PackFileJob job;
 686         job.relativePathSrc = szRelativePathSrc;
 687         job.modTime = modTime;
 688         job.uncompressedData = pUncompressed;
 689         job.uncompressedSize = nSize;
 690         job.batch = &batch;
 691
 692         // crc will be used to check if this file need to be updated at all
 693         ZipDir::FileEntry* entry = FindFile(szRelativePath);
 694         if (entry)
 695                 job.existingCRC = entry->desc.lCRC32;
 696
 697         PackFileFromMemory(&job);
 698
 699         switch (job.status)
 700         {
 701     case PACKFILE_SKIPPED :
 702     case PACKFILE_MISSING :
 703     case PACKFILE_FAILED :
 704       return ZD_ERROR_IO_FAILED;
 705         }
 706
 707         StorePackedFile(&job);
 708         job.DetachUncompressedData();
 709         return job.zdError;
 710 }
 711
 712 static FILETIME GetFileWriteTimeAndSize( uint64* fileSize, const char *filename )
 713 {
 714         // Warning: FindFirstFile on NTFS may report file size that
 715         // is not up-to-date with the actual file content.
 716         // http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
 717
 718         FILETIME fileTime;
 719
 720         WIN32_FIND_DATAA FindFileData;
 721         HANDLE hFind = FindFirstFileA( filename, &FindFileData );
 722
 723         if (hFind == INVALID_HANDLE_VALUE)
 724         {
 725                 fileTime.dwLowDateTime = 0;
 726                 fileTime.dwHighDateTime = 0;
 727                 if (fileSize)
 728                 {
 729                         *fileSize = 0;
 730                 }
 731         }
 732         else
 733         {
 734                 fileTime.dwLowDateTime = FindFileData.ftLastWriteTime.dwLowDateTime;
 735                 fileTime.dwHighDateTime = FindFileData.ftLastWriteTime.dwHighDateTime;
 736                 if (fileSize)
 737                 {
 738                         *fileSize = (uint64(FindFileData.nFileSizeHigh) << 32) + FindFileData.nFileSizeLow;
 739                 }
 740                 FindClose(hFind);
 741         }
 742
 743         return fileTime;
 744 }
 745
 746 static void PackFileFromDisc(PackFileJob* job)
 747 {
 748         const FILETIME ft = GetFileWriteTimeAndSize(0, job->realFilename);
 749         LARGE_INTEGER lt;
 750         lt.HighPart = ft.dwHighDateTime;
 751         lt.LowPart = ft.dwLowDateTime;
 752         job->modTime = lt.QuadPart;
 753
 754                 FILE *f = fopen(job->realFilename, "rb");
 755                 if (!f)
 756                 {
 757                         job->status = PACKFILE_FAILED;
 758                         job->zdError = ZipDir::ZD_ERROR_FILE_NOT_FOUND;
 759                         return;
 760                 }
 761
 762                 fseek(f, 0, SEEK_END);
 763                 size_t fileSize = (size_t)ftell(f);
 764
 765                 if (fileSize < job->batch->sourceMinSize || (job->batch->sourceMaxSize > 0 && fileSize > job->batch->sourceMaxSize))
 766                 {
 767                         fclose(f);
 768
 769                         job->status = PACKFILE_SKIPPED;
 770                         job->zdError = ZipDir::ZD_ERROR_SUCCESS;
 771                         return;
 772                 }
 773
 774                 job->uncompressedData = malloc(fileSize);
 775
 776                 fseek(f, 0, SEEK_SET);
 777                 if (fread(job->uncompressedData, 1, fileSize, f) != fileSize)
 778                 {
 779                         free(job->uncompressedData);
 780                         job->uncompressedData = 0;
 781                         fclose(f);
 782
 783                         job->status = PACKFILE_FAILED;
 784                         job->zdError = ZipDir::ZD_ERROR_IO_FAILED;
 785                         return;
 786                 }
 787                 fclose(f);
 788                 job->uncompressedSize = fileSize;
 789
 790         PackFileFromMemory(job);
 791 }
 792
 793 bool ZipDir::CacheRW::UpdateMultipleFiles(const char** realFilenames, const char** filenamesInZip, size_t fileCount,
 794                                                                                                                                                                         int compressionLevel, bool encryptContent, size_t zipMaxSize, int sourceMinSize, int sourceMaxSize,
 795                                                                                                                                                                         int numExtraThreads, ZipDir::IReporter* reporter, ZipDir::ISplitter* splitter)
 796 {
 797         int compressionMethod = METHOD_DEFLATE;
 798         if (encryptContent)
 799                 compressionMethod = METHOD_DEFLATE_AND_ENCRYPT;
 800         else if (compressionLevel == 0)
 801                 compressionMethod = METHOD_STORE;
 802
 803         uint64 totalSize = 0;
 804
 805         clock_t startTime = clock();
 806
 807         PackFileBatch batch;
 808         batch.compressionLevel = compressionLevel;
 809         batch.compressionMethod = compressionMethod;
 810         batch.pool = 0;
 811         batch.sourceMinSize = sourceMinSize;
 812         batch.sourceMaxSize = sourceMaxSize;
 813         batch.zipMaxSize = zipMaxSize;
 814
 815         if (numExtraThreads == 0)
 816         {
 817                 for (int i = 0; i < fileCount; ++i)
 818                 {
 819                         const char* realFilename = realFilenames[i];
 820                         const char* filenameInZip = filenamesInZip[i];
 821
 822                         PackFileJob job;
 823
 824                         job.key = i;
 825                         job.relativePathSrc = filenameInZip;
 826                         job.realFilename = realFilename;
 827                         job.batch = &batch;
 828
 829                         {
 830                                 // crc will be used to check if this file need to be updated at all
 831                                 ZipDir::FileEntry* entry = FindFile(filenameInZip);
 832                                 if (entry)
 833                                 {
 834                                         uint64 fileSize = 0;
 835
 836                                         const FILETIME ft = GetFileWriteTimeAndSize(&fileSize, realFilename);
 837                                         LARGE_INTEGER lt;
 838
 839                                         lt.HighPart = ft.dwHighDateTime;
 840                                         lt.LowPart = ft.dwLowDateTime;
 841                                         job.modTime = lt.QuadPart;
 842                                         job.existingCRC = entry->desc.lCRC32;
 843                                         job.compressedSizePreviously = entry->desc.lSizeCompressed;
 844                                         job.uncompressedSizePreviously = entry->desc.lSizeUncompressed;
 845
 846                                         // Check if file with the same name, timestamp and size already exists in pak.
 847                                         if (entry->CompareFileTimeNTFS(job.modTime) && fileSize == entry->desc.lSizeUncompressed)
 848                                         {
 849                                                 if (reporter)
 850                                                         reporter->ReportUpToDate(filenameInZip);
 851                                                 continue;
 852                                         }
 853                                 }
 854                         }
 855
 856                         PackFileFromDisc(&job);
 857
 858                         if (job.status == PACKFILE_COMPRESSED)
 859                         {
 860                                 if (splitter != NULL)
 861                                 {
 862                                         size_t dsk = GetTotalFileSizeOnDiskSoFar();
 863                                         size_t bse = 0;
 864                                         size_t add = 0;
 865                                         size_t sub = 0;
 866
 867                                         bse += sizeof(ZipFile::CDRFileHeader)   + strlen(job.relativePathSrc);
 868                                         bse += sizeof(ZipFile::LocalFileHeader) + strlen(job.relativePathSrc);
 869
 870                                         if (job.compressedSize)
 871                                                 add += bse + job.compressedSize;
 872                                         if (job.compressedSizePreviously)
 873                                                 sub += bse + job.compressedSizePreviously;
 874
 875                                         if (splitter->CheckWriteLimit(dsk, add, sub))
 876                                         {
 877                                                 splitter->SetLastFile(dsk, add, sub, job.key - 1);
 878                                                 break;
 879                                         }
 880                                 }
 881
 882                                 StorePackedFile(&job);
 883                         }
 884
 885                         switch (job.status)
 886                         {
 887                         case PACKFILE_ADDED:
 888                                 if (reporter)
 889                                         reporter->ReportAdded(filenameInZip);
 890
 891                                 totalSize += job.uncompressedSize;
 892                                 break;
 893                         case PACKFILE_MISSING:
 894                                 if (reporter)
 895                                         reporter->ReportMissing(realFilename);
 896                                 break;
 897                         case PACKFILE_UPTODATE:
 898                                 if (reporter)
 899                                         reporter->ReportUpToDate(realFilename);
 900                                 break;
 901                         case PACKFILE_SKIPPED:
 902                                 if (reporter)
 903                                         reporter->ReportSkipped(realFilename);
 904                                 break;
 905                         default:
 906                                 if (reporter)
 907                                         reporter->ReportFailed(realFilename, ""); // TODO reason
 908                                 continue;
 909                         };
 910                 }
 911
 912         }
 913         else
 914         {
 915                 const size_t memoryLimit = 1024 * 1024 * 1024; // prevents threads from generating more than 1GB of data
 916                 PackFilePool pool(fileCount, memoryLimit);
 917                 batch.pool = &pool;
 918
 919                 for (int i = 0; i < fileCount; ++i)
 920                 {
 921                         const char* realFilename = realFilenames[i];
 922                         const char* filenameInZip = filenamesInZip[i];
 923
 924                         PackFileJob job;
 925
 926                         job.relativePathSrc = filenameInZip;
 927                         job.realFilename = realFilename;
 928                         job.batch = &batch;
 929
 930                         {
 931                                 // crc will be used to check if this file need to be updated at all
 932                                 ZipDir::FileEntry* entry = FindFile(filenameInZip);
 933                                 if (entry)
 934                                 {
 935                                         uint64 fileSize = 0;
 936                                         const FILETIME ft = GetFileWriteTimeAndSize(&fileSize, realFilename);
 937                                         LARGE_INTEGER lt;
 938                                         lt.HighPart = ft.dwHighDateTime;
 939                                         lt.LowPart = ft.dwLowDateTime;
 940                                         job.modTime = lt.QuadPart;
 941                                         job.existingCRC = entry->desc.lCRC32;
 942                                         job.compressedSizePreviously = entry->desc.lSizeCompressed;
 943                                         job.uncompressedSizePreviously = entry->desc.lSizeUncompressed;
 944
 945                                         // Check if file with the same name, timestamp and size already exists in pak.
 946                                         if (entry->CompareFileTimeNTFS(job.modTime) && fileSize == entry->desc.lSizeUncompressed)
 947                                         {
 948                                                 if (reporter)
 949                                                         reporter->ReportUpToDate(filenameInZip);
 950                                                 continue;
 951                                         }
 952                                 }
 953                         }
 954
 955                         pool.Submit(i, job);
 956                 }
 957
 958                 // Get the number of submitted jobs, which is at most
 959                 // as large as the largest successfully submitted file-index.
 960                 // Any number of files can be skipped for submittion.
 961                 const int jobCount = pool.GetJobCount();
 962                 if (jobCount == 0)
 963                 {
 964                         return true;
 965                 }
 966
 967                 pool.Start(numExtraThreads);
 968
 969                 for (int i = 0; i < jobCount; ++i)
 970                 {
 971                         PackFileJob* job = pool.WaitForFile(i);
 972                         if (!job)
 973                         {
 974                                 assert(job);
 975                                 continue;
 976                         }
 977
 978                         if (job->status == PACKFILE_COMPRESSED)
 979                         {
 980                                 if (splitter != NULL)
 981                                 {
 982                                         size_t dsk = GetTotalFileSizeOnDiskSoFar();
 983                                         size_t bse = 0;
 984                                         size_t add = 0;
 985                                         size_t sub = 0;
 986
 987                                         bse += sizeof(ZipFile::CDRFileHeader)   + strlen(job->relativePathSrc);
 988                                         bse += sizeof(ZipFile::LocalFileHeader) + strlen(job->relativePathSrc);
 989
 990                                         if (job->compressedSize)
 991                                                 add += bse + job->compressedSize;
 992                                         if (job->compressedSizePreviously)
 993                                                 sub += bse + job->compressedSizePreviously;
 994
 995                                         if (splitter->CheckWriteLimit(dsk, add, sub))
 996                                         {
 997                                                 splitter->SetLastFile(dsk, add, sub, job->key - 1);
 998
 999                                                 // deplete the pool before leaving the loop
1000                                                 pool.SkipPendingFiles();
1001                                                 for (; i < jobCount; ++i)
1002                                                 {
1003                                                         pool.WaitForFile(i);
1004                                                         pool.ReleaseFile(i);
1005                                                 }
1006
1007                                                 break;
1008                                         }
1009                                 }
1010
1011                                 StorePackedFile(job);
1012                         }
1013
1014                         switch (job->status)
1015                         {
1016                         case PACKFILE_ADDED:
1017                                 if (reporter)
1018                                         reporter->ReportAdded(job->relativePathSrc);
1019
1020                                 totalSize += job->uncompressedSize;
1021                                 break;
1022                         case PACKFILE_MISSING:
1023                                 if (reporter)
1024                                         reporter->ReportMissing(job->realFilename);
1025                                 break;
1026                         case PACKFILE_UPTODATE:
1027                                 if (reporter)
1028                                         reporter->ReportUpToDate(job->realFilename);
1029                                 break;
1030                         case PACKFILE_SKIPPED:
1031                                 if (reporter)
1032                                         reporter->ReportSkipped(job->realFilename);
1033                                 break;
1034                         default:
1035                                 if (reporter)
1036                                         reporter->ReportFailed(job->realFilename, ""); // TODO reason
1037                                 continue;
1038                         };
1039
1040                         pool.ReleaseFile(i);
1041                 }
1042         }
1043
1044         clock_t endTime = clock();
1045         double timeSeconds = double(endTime - startTime) / CLOCKS_PER_SEC;
1046         double speed = (endTime - startTime) == 0 ? 0.0 : double(totalSize) / timeSeconds;
1047
1048         if (reporter)
1049                 reporter->ReportSpeed(speed);
1050
1051         return true;
1052 }
1053
1054
1055 //   Adds a new file to the zip or update an existing one if it is not compressed - just stored  - start a big file
1056 ZipDir::ErrorEnum ZipDir::CacheRW::StartContinuousFileUpdate( const char* szRelativePathSrc, unsigned nSize )
1057 {
1058         char str[_MAX_PATH];
1059         char *szRelativePath = UnifyPath(str, szRelativePathSrc);
1060
1061         SmartPtr pBufferDestroyer;
1062
1063         // create or find the file entry.. this object will rollback (delete the object
1064         // if the operation fails) if needed.
1065         FileEntryTransactionAdd pFileEntry(this, AllocPath(szRelativePathSrc), AllocPath(szRelativePath));
1066
1067         if (!pFileEntry)
1068                 return ZD_ERROR_INVALID_PATH;
1069
1070         pFileEntry->OnNewFileData (NULL, nSize, nSize, METHOD_STORE, false );
1071         // since we changed the time, we'll have to update CDR
1072         m_nFlags |= FLAGS_CDR_DIRTY;
1073
1074         // the new CDR position, if the operation completes successfully
1075         unsigned lNewCDROffset = m_lCDROffset;
1076         if (pFileEntry->IsInitialized())
1077         {
1078                 // check if the new compressed data fits into the old place
1079                 unsigned nFreeSpace = pFileEntry->nEOFOffset - pFileEntry->nFileHeaderOffset - (unsigned)sizeof(ZipFile::LocalFileHeader) - (unsigned)strlen(szRelativePath);
1080
1081                 if (nFreeSpace != nSize)
1082                         m_nFlags |= FLAGS_UNCOMPACTED;
1083
1084                 if (nFreeSpace >= nSize)
1085                 {
1086                         // and we can just override the compressed data in the file
1087                         ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePathSrc, m_bEncryptedHeaders);
1088                         if (e != ZD_ERROR_SUCCESS)
1089                                 return e;
1090                 }
1091                 else
1092                 {
1093                         // we need to write the file anew - in place of current CDR
1094                         pFileEntry->nFileHeaderOffset = CalculateAlignedHeaderOffset(szRelativePathSrc, m_lCDROffset, m_fileAlignment);
1095                         ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePathSrc, m_bEncryptedHeaders);
1096                         lNewCDROffset = pFileEntry->nEOFOffset;
1097                         if (e != ZD_ERROR_SUCCESS)
1098                                 return e;
1099                 }
1100         }
1101         else
1102         {
1103                 pFileEntry->nFileHeaderOffset = CalculateAlignedHeaderOffset(szRelativePathSrc, m_lCDROffset, m_fileAlignment);
1104                 ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePathSrc, m_bEncryptedHeaders);
1105                 if (e != ZD_ERROR_SUCCESS)
1106                         return e;
1107
1108                 lNewCDROffset = pFileEntry->nFileDataOffset + nSize;
1109
1110                 m_nFlags |= FLAGS_CDR_DIRTY;
1111         }
1112
1113 #if CRY_PLATFORM_WINDOWS
1114         if (_fseeki64 (m_pFile, (__int64)pFileEntry->nFileDataOffset, SEEK_SET)!=0)
1115 #else
1116   if (fseek (m_pFile, pFileEntry->nFileDataOffset, SEEK_SET)!=0)
1117 #endif
1118         {
1119                 return ZD_ERROR_IO_FAILED;
1120         }
1121
1122         if (!WriteNullData(nSize))
1123         {
1124                 return ZD_ERROR_IO_FAILED;
1125         }
1126
1127         pFileEntry->nEOFOffset = pFileEntry->nFileDataOffset;
1128
1129         // since we wrote the file successfully, update the new CDR position
1130         m_lCDROffset = lNewCDROffset;
1131         pFileEntry.Commit();
1132
1133         return ZD_ERROR_SUCCESS;
1134 }
1135
1136 // Adds a new file to the zip or update an existing's segment if it is not compressed - just stored
1137 // adds a directory (creates several nested directories if needed)
1138 ZipDir::ErrorEnum ZipDir::CacheRW::UpdateFileContinuousSegment (const char* szRelativePathSrc, unsigned nSize, void* pUncompressed, unsigned nSegmentSize, unsigned nOverwriteSeekPos )
1139 {
1140         char str[_MAX_PATH];
1141         char *szRelativePath = UnifyPath(str, szRelativePathSrc);
1142
1143         SmartPtr pBufferDestroyer;
1144
1145         // create or find the file entry.. this object will rollback (delete the object
1146         // if the operation fails) if needed.
1147         FileEntryTransactionAdd pFileEntry(this, AllocPath(szRelativePathSrc), AllocPath(szRelativePath));
1148
1149         if (!pFileEntry)
1150                 return ZD_ERROR_INVALID_PATH;
1151
1152         pFileEntry->OnNewFileData (pUncompressed, nSegmentSize, nSegmentSize, METHOD_STORE, true);
1153         // since we changed the time, we'll have to update CDR
1154         m_nFlags |= FLAGS_CDR_DIRTY;
1155
1156         // this file entry is already allocated in CDR
1157         unsigned lSegmentOffset = pFileEntry->nEOFOffset;
1158
1159 #if CRY_PLATFORM_WINDOWS
1160         if (_fseeki64 (m_pFile, (__int64)pFileEntry->nFileHeaderOffset, SEEK_SET)!=0)
1161 #else
1162   if (fseek (m_pFile, pFileEntry->nFileHeaderOffset, SEEK_SET)!=0)
1163 #endif
1164                 return ZD_ERROR_IO_FAILED;
1165
1166         // and we can just override the compressed data in the file
1167         ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePath, m_bEncryptedHeaders);
1168         if (e != ZD_ERROR_SUCCESS)
1169                 return e;
1170
1171         if(nOverwriteSeekPos!=0xffffffff)
1172                 lSegmentOffset = pFileEntry->nFileDataOffset + nOverwriteSeekPos;
1173
1174         // now we have the fresh local header and data offset
1175 #if CRY_PLATFORM_WINDOWS
1176         if (_fseeki64 (m_pFile, (__int64)lSegmentOffset, SEEK_SET)!=0)
1177 #else
1178   if (fseek (m_pFile, lSegmentOffset, SEEK_SET)!=0)
1179 #endif
1180                 return ZD_ERROR_IO_FAILED;
1181
1182         const bool encrypt = false; // encryption is not supported for continous updates
1183         if (!WriteCompressedData((char*)pUncompressed, nSegmentSize, encrypt, m_pFile))
1184         {
1185                 return ZD_ERROR_IO_FAILED;
1186         }
1187
1188         if(nOverwriteSeekPos==0xffffffff)
1189                 pFileEntry->nEOFOffset = lSegmentOffset + nSegmentSize;
1190
1191         // since we wrote the file successfully, update CDR
1192         pFileEntry.Commit();
1193         return ZD_ERROR_SUCCESS;
1194 }
1195
1196
1197 ZipDir::ErrorEnum ZipDir::CacheRW::UpdateFileCRC (const char* szRelativePathSrc, unsigned dwCRC32 )
1198 {
1199         char str[_MAX_PATH];
1200         char *szRelativePath = UnifyPath(str, szRelativePathSrc);
1201
1202         SmartPtr pBufferDestroyer;
1203
1204         // create or find the file entry.. this object will rollback (delete the object
1205         // if the operation fails) if needed.
1206         FileEntryTransactionAdd pFileEntry(this, AllocPath(szRelativePathSrc), AllocPath(szRelativePath));
1207
1208         if (!pFileEntry)
1209                 return ZD_ERROR_INVALID_PATH;
1210
1211         // since we changed the time, we'll have to update CDR
1212         m_nFlags |= FLAGS_CDR_DIRTY;
1213
1214         pFileEntry->desc.lCRC32=dwCRC32;
1215
1216 #if CRY_PLATFORM_WINDOWS
1217         if (_fseeki64 (m_pFile, (__int64)pFileEntry->nFileHeaderOffset, SEEK_SET)!=0)
1218 #else
1219   if (fseek (m_pFile, pFileEntry->nFileHeaderOffset, SEEK_SET)!=0)
1220 #endif
1221                 return ZD_ERROR_IO_FAILED;
1222
1223         // and we can just override the compressed data in the file
1224         ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePath, m_bEncryptedHeaders);
1225         if (e != ZD_ERROR_SUCCESS)
1226                 return e;
1227
1228         // since we wrote the file successfully, update
1229         pFileEntry.Commit();
1230         return ZD_ERROR_SUCCESS;
1231 }
1232
1233
1234 // deletes the file from the archive
1235 ZipDir::ErrorEnum ZipDir::CacheRW::RemoveFile (const char* szRelativePathSrc)
1236 {
1237         char str[_MAX_PATH];
1238         char *szRelativePath = UnifyPath(str, szRelativePathSrc);
1239
1240         // find the last slash in the path
1241         const char* pSlash = (std::max)(strrchr(szRelativePath, '/'), strrchr(szRelativePath, '\\'));
1242
1243         const char* pFileName; // the name of the file to delete
1244
1245         FileEntryTree* pDir; // the dir from which the subdir will be deleted
1246
1247         if (pSlash)
1248         {
1249                 FindDirRW fd (GetRoot());
1250                 // the directory to remove
1251                 pDir = fd.FindExact(string (szRelativePath, pSlash-szRelativePath).c_str());
1252                 if (!pDir)
1253                         return ZD_ERROR_DIR_NOT_FOUND;// there is no such directory
1254                 pFileName = pSlash+1;
1255         }
1256         else
1257         {
1258                 pDir = GetRoot();
1259                 pFileName = szRelativePath;
1260         }
1261
1262         ErrorEnum e = pDir->RemoveFile (pFileName);
1263         if (e == ZD_ERROR_SUCCESS)
1264                 m_nFlags |= FLAGS_UNCOMPACTED|FLAGS_CDR_DIRTY;
1265         return e;
1266 }
1267
1268
1269 // deletes the directory, with all its descendants (files and subdirs)
1270 ZipDir::ErrorEnum ZipDir::CacheRW::RemoveDir (const char* szRelativePathSrc)
1271 {
1272         char str[_MAX_PATH];
1273         char *szRelativePath = UnifyPath(str, szRelativePathSrc);
1274
1275         // find the last slash in the path
1276         const char* pSlash = (std::max)(strrchr(szRelativePath, '/'), strrchr(szRelativePath, '\\'));
1277
1278         const char* pDirName; // the name of the dir to delete
1279
1280         FileEntryTree* pDir; // the dir from which the subdir will be deleted
1281
1282         if (pSlash)
1283         {
1284                 FindDirRW fd (GetRoot());
1285                 // the directory to remove
1286                 pDir = fd.FindExact(string (szRelativePath, pSlash-szRelativePath).c_str());
1287                 if (!pDir)
1288                         return ZD_ERROR_DIR_NOT_FOUND;// there is no such directory
1289                 pDirName = pSlash+1;
1290         }
1291         else
1292         {
1293                 pDir = GetRoot();
1294                 pDirName = szRelativePath;
1295         }
1296
1297         ErrorEnum e = pDir->RemoveDir (pDirName);
1298         if (e == ZD_ERROR_SUCCESS)
1299                 m_nFlags |= FLAGS_UNCOMPACTED|FLAGS_CDR_DIRTY;
1300         return e;
1301 }
1302
1303 // deletes all files and directories in this archive
1304 ZipDir::ErrorEnum ZipDir::CacheRW::RemoveAll()
1305 {
1306         ErrorEnum e = m_treeDir.RemoveAll();
1307         if (e == ZD_ERROR_SUCCESS)
1308                 m_nFlags |= FLAGS_UNCOMPACTED|FLAGS_CDR_DIRTY;
1309         return e;
1310 }
1311
1312 ZipDir::ErrorEnum ZipDir::CacheRW::ReadFile (FileEntry* pFileEntry, void* pCompressed, void* pUncompressed)
1313 {
1314         if (!pFileEntry)
1315                 return ZD_ERROR_INVALID_CALL;
1316
1317         if (pFileEntry->desc.lSizeUncompressed == 0)
1318         {
1319                 assert (pFileEntry->desc.lSizeCompressed == 0);
1320                 return ZD_ERROR_SUCCESS;
1321         }
1322
1323         assert (pFileEntry->desc.lSizeCompressed > 0);
1324
1325         ErrorEnum nError = Refresh(pFileEntry);
1326         if (nError != ZD_ERROR_SUCCESS)
1327                 return nError;
1328
1329 #if CRY_PLATFORM_WINDOWS
1330         if (_fseeki64 (m_pFile, (__int64)pFileEntry->nFileDataOffset, SEEK_SET))
1331 #else
1332   if (fseek (m_pFile, pFileEntry->nFileDataOffset, SEEK_SET))
1333 #endif
1334                 return ZD_ERROR_IO_FAILED;
1335
1336         SmartPtr pBufferDestroyer;
1337
1338         void* pBuffer = pCompressed; // the buffer where the compressed data will go
1339
1340         if (pFileEntry->nMethod == 0 && pUncompressed)
1341         {
1342                 // we can directly read into the uncompress buffer
1343                 pBuffer = pUncompressed;
1344         }
1345
1346         if (!pBuffer)
1347         {
1348                 if (!pUncompressed)
1349                         // what's the sense of it - no buffers at all?
1350                         return ZD_ERROR_INVALID_CALL;
1351
1352                 pBuffer = malloc(pFileEntry->desc.lSizeCompressed);
1353                 pBufferDestroyer.Attach(pBuffer); // we want it auto-freed once we return
1354         }
1355
1356         if (fread((char*)pBuffer, pFileEntry->desc.lSizeCompressed, 1, m_pFile) != 1)
1357         {
1358                 return ZD_ERROR_IO_FAILED;
1359         }
1360
1361         if (pFileEntry->nMethod == METHOD_DEFLATE_AND_ENCRYPT)
1362         {
1363                 ZipDir::Decrypt((char*)pBuffer, pFileEntry->desc.lSizeCompressed, m_encryptionKey);
1364         }
1365
1366         // if there's a buffer for uncompressed data, uncompress it to that buffer
1367         if (pUncompressed)
1368         {
1369                 if (pFileEntry->nMethod == 0)
1370                 {
1371                         assert (pBuffer == pUncompressed);
1372                         //assert (pFileEntry->nSizeCompressed == pFileEntry->nSizeUncompressed);
1373                         //memcpy (pUncompressed, pBuffer, pFileEntry->nSizeCompressed);
1374                 }
1375                 else
1376                 {
1377                         unsigned long nSizeUncompressed = pFileEntry->desc.lSizeUncompressed;
1378                         if (nSizeUncompressed > 0)
1379                         {
1380                                 if (Z_OK != ZipRawUncompress(pUncompressed, &nSizeUncompressed, pBuffer, pFileEntry->desc.lSizeCompressed))
1381                                         return ZD_ERROR_CORRUPTED_DATA;
1382                         }
1383                 }
1384         }
1385
1386         return ZD_ERROR_SUCCESS;
1387 }
1388
1389
1390 //////////////////////////////////////////////////////////////////////////
1391 // finds the file by exact path
1392 ZipDir::FileEntry* ZipDir::CacheRW::FindFile (const char* szPathSrc, bool bFullInfo)
1393 {
1394         char str[_MAX_PATH];
1395         char *szPath = UnifyPath(str, szPathSrc);
1396
1397         if (!this)
1398                 return NULL;
1399         ZipDir::FindFileRW fd (GetRoot());
1400         if (!fd.FindExact(szPath))
1401         {
1402                 assert (!fd.GetFileEntry());
1403                 return NULL;
1404         }
1405         ZipDir::FileEntry* const p = fd.GetFileEntry();
1406         assert (p);
1407         return p;
1408 }
1409
1410 // returns the size of memory occupied by the instance referred to by this cache
1411 size_t ZipDir::CacheRW::GetSize() const
1412 {
1413         return sizeof(*this) + m_strFilePath.capacity() + m_treeDir.GetSize() - sizeof(m_treeDir);
1414 }
1415
1416 // returns the compressed size of all the entries
1417 size_t ZipDir::CacheRW::GetCompressedSize() const
1418 {
1419         return m_treeDir.GetCompressedFileSize();
1420 }
1421
1422 // returns the total size of memory occupied by the instance of this cache and all the compressed files
1423 size_t ZipDir::CacheRW::GetTotalFileSize() const
1424 {
1425         return GetSize() + GetCompressedSize();
1426 }
1427
1428 // returns the total size of space occupied on disk by the instance of this cache and all the compressed files
1429 size_t ZipDir::CacheRW::GetTotalFileSizeOnDiskSoFar()
1430 {
1431         FileRecordList arrFiles(GetRoot());
1432         FileRecordList::ZipStats statFiles = arrFiles.GetStats();
1433
1434         return m_lCDROffset + statFiles.nSizeCDR;
1435 }
1436
1437 // refreshes information about the given file entry into this file entry
1438 ZipDir::ErrorEnum ZipDir::CacheRW::Refresh (FileEntry* pFileEntry)
1439 {
1440         if (!pFileEntry)
1441                 return ZD_ERROR_INVALID_CALL;
1442
1443         if (pFileEntry->nFileDataOffset != pFileEntry->INVALID_DATA_OFFSET)
1444                 return ZD_ERROR_SUCCESS; // the data offset has been successfully read..
1445
1446         if (!this)
1447                 return ZD_ERROR_INVALID_CALL; // from which cache is this file entry???
1448
1449         return ZipDir::Refresh(m_pFile, pFileEntry, m_bEncryptedHeaders);
1450 }
1451
1452
1453 // writes the CDR to the disk
1454 bool ZipDir::CacheRW::WriteCDR(FILE* fTarget, bool encryptCDR)
1455 {
1456         if (!fTarget)
1457                 return false;
1458
1459 #if CRY_PLATFORM_WINDOWS
1460         if (_fseeki64(fTarget, (__int64)m_lCDROffset, SEEK_SET))
1461 #else
1462   if (fseek(fTarget, m_lCDROffset, SEEK_SET))
1463 #endif
1464                 return false;
1465
1466         FileRecordList arrFiles(GetRoot());
1467         //arrFiles.SortByFileOffset();
1468         size_t nSizeCDR = arrFiles.GetStats().nSizeCDR;
1469         void* pCDR = malloc(nSizeCDR);
1470         size_t nSizeCDRSerialized = arrFiles.MakeZipCDR(m_lCDROffset, pCDR, encryptCDR);
1471         assert (nSizeCDRSerialized == nSizeCDR);
1472
1473         if (encryptCDR)
1474         {
1475                 // We do not encrypt CDREnd, so we could find it by signature
1476                 ZipDir::Encrypt((char*)pCDR, nSizeCDR - sizeof(ZipFile::CDREnd), m_encryptionKey);
1477         }
1478
1479         size_t nWriteRes = fwrite (pCDR, nSizeCDR, 1, fTarget);
1480         free(pCDR);
1481         return nWriteRes == 1;
1482 }
1483
1484 // generates random file name
1485 string ZipDir::CacheRW::GetRandomName(int nAttempt)
1486 {
1487         if (nAttempt)
1488         {
1489                 char szBuf[8];
1490                 int i;
1491                 for (i = 0; i < sizeof(szBuf)-1;++i)
1492                 {
1493                         int r = rand()%(10 + 'z' - 'a' + 1);
1494                         szBuf[i] = r > 9 ? (r-10)+'a' : '0' + r;
1495                 }
1496                 szBuf[i] = '\0';
1497                 return szBuf;
1498         }
1499         else
1500                 return string();
1501 }
1502
1503 bool ZipDir::CacheRW::RelinkZip()
1504 {
1505         for (int nAttempt = 0; nAttempt < 32; ++nAttempt)
1506         {
1507                 string strNewFilePath = m_strFilePath + "$" + GetRandomName(nAttempt);
1508
1509                 FILE* f = fopen (strNewFilePath.c_str(), "wb");
1510                 if (f)
1511                 {
1512                         bool bOk = RelinkZip(f);
1513                         fclose (f); // we don't need the temporary file handle anyway
1514
1515                         if (!bOk)
1516                         {
1517                                 // we don't need the temporary file
1518                                 DeleteFileA(strNewFilePath.c_str());
1519                                 return false;
1520                         }
1521
1522                         // we successfully relinked, now copy the temporary file to the original file
1523                         fclose (m_pFile);
1524                         m_pFile = NULL;
1525
1526                         DeleteFileA(m_strFilePath.c_str());
1527                         if (MoveFileA(strNewFilePath.c_str(), m_strFilePath.c_str()) == 0)
1528                         {
1529                                 // successfully renamed - reopen
1530                                 m_pFile = fopen (m_strFilePath.c_str(), "r+b");
1531                                 return m_pFile == NULL;
1532                         }
1533                         else
1534                         {
1535                                 // could not rename
1536
1537                                 //m_pFile = fopen (strNewFilePath.c_str(), "r+b");
1538                                 return false;
1539                         }
1540                 }
1541         }
1542
1543         // couldn't open temp file
1544         return false;
1545 }
1546
1547 bool ZipDir::CacheRW::RelinkZip(FILE* fTmp)
1548 {
1549         FileRecordList arrFiles(GetRoot());
1550         arrFiles.SortByFileOffset();
1551
1552         // we back up our file entries, because we'll need to restore them
1553         // in case the operation fails
1554         std::vector<FileEntry> arrFileEntryBackup;
1555         arrFiles.Backup (arrFileEntryBackup);
1556
1557         // this is the set of files that are to be written out - compressed data and the file record iterator
1558         std::vector<FileDataRecordPtr> queFiles;
1559         queFiles.reserve (g_nMaxItemsRelinkBuffer);
1560
1561         // the total size of data in the queue
1562         unsigned nQueueSize = 0;
1563
1564         for (FileRecordList::iterator it = arrFiles.begin(); it != arrFiles.end(); ++it)
1565         {
1566                 FileEntry* entry = it->pFileEntry;
1567                 // find the file data offset
1568                 if (ZD_ERROR_SUCCESS != Refresh(entry))
1569                         return false;
1570
1571                 // go to the file data
1572 #if CRY_PLATFORM_WINDOWS
1573                 if (_fseeki64 (m_pFile, (__int64)entry->nFileDataOffset, SEEK_SET) != 0)
1574 #else
1575     if (fseek (m_pFile, entry->nFileDataOffset, SEEK_SET) != 0)
1576 #endif
1577                         return false;
1578
1579                 // allocate memory for the file compressed data
1580                 FileDataRecordPtr pFile = FileDataRecord::New (*it);
1581
1582                 if(!pFile)
1583                         return false;
1584
1585                 // read the compressed data
1586                 if (entry->desc.lSizeCompressed && fread (pFile->GetData(), entry->desc.lSizeCompressed, 1, m_pFile) != 1)
1587                         return false;
1588
1589                 if (entry->nMethod == METHOD_DEFLATE_AND_ENCRYPT)
1590                 {
1591                         ZipDir::Decrypt((char*)pFile->GetData(), entry->desc.lSizeCompressed, m_encryptionKey);
1592                 }
1593
1594                 // put the file into the queue for copying (writing)
1595                 queFiles.push_back(pFile);
1596                 nQueueSize += entry->desc.lSizeCompressed;
1597
1598                 // if the queue is big enough, write it out
1599                 if(nQueueSize > g_nSizeRelinkBuffer || queFiles.size() >= g_nMaxItemsRelinkBuffer)
1600                 {
1601                         nQueueSize = 0;
1602                         if (!WriteZipFiles(queFiles, fTmp))
1603                                 return false;
1604                 }
1605         }
1606
1607         if (!WriteZipFiles(queFiles, fTmp))
1608                 return false;
1609
1610         ZipFile::ulong lOldCDROffset = m_lCDROffset;
1611         // the file data has now been written out. Now write the CDR
1612 #if CRY_PLATFORM_WINDOWS
1613         m_lCDROffset = (ZipFile::ulong)_ftelli64(fTmp);
1614 #else
1615         m_lCDROffset = ftell(fTmp);
1616 #endif
1617         if (m_lCDROffset != (ZipFile::ulong)-1L && WriteCDR(fTmp, m_bHeadersEncryptedOnClose) && 0 == fflush (fTmp))
1618         {
1619                 // the new file positions are already there - just discard the backup and return
1620                 return true;
1621         }
1622         // recover from backup
1623         arrFiles.Restore (arrFileEntryBackup);
1624         m_lCDROffset = lOldCDROffset;
1625         m_bEncryptedHeaders = m_bHeadersEncryptedOnClose;
1626         return false;
1627 }
1628
1629 // writes out the file data in the queue into the given file. Empties the queue
1630 bool ZipDir::CacheRW::WriteZipFiles(std::vector<FileDataRecordPtr>& queFiles, FILE* fTmp)
1631 {
1632         for (std::vector<FileDataRecordPtr>::iterator it = queFiles.begin(); it != queFiles.end(); ++it)
1633         {
1634                 // set the new header offset to the file entry - we won't need it
1635 #if CRY_PLATFORM_WINDOWS
1636                 const unsigned long currentPos = (unsigned long)_ftelli64 (fTmp);
1637 #else
1638                 const unsigned long currentPos = ftell (fTmp);
1639 #endif
1640                 (*it)->pFileEntry->nFileHeaderOffset = CalculateAlignedHeaderOffset((*it)->strPath.c_str(), currentPos, m_fileAlignment);
1641
1642                 // while writing the local header, the data offset will also be calculated
1643                 if (ZD_ERROR_SUCCESS != WriteLocalHeader(fTmp, (*it)->pFileEntry, (*it)->strPath.c_str(), m_bHeadersEncryptedOnClose))
1644                         return false;;
1645
1646                 // write the compressed file data
1647                 const bool encrypt = (*it)->pFileEntry->nMethod == METHOD_DEFLATE_AND_ENCRYPT;
1648                 if (!WriteCompressedData((char*)(*it)->GetData(), (*it)->pFileEntry->desc.lSizeCompressed, encrypt, fTmp))
1649                         return false;
1650
1651 #if CRY_PLATFORM_WINDOWS
1652     assert ((*it)->pFileEntry->nEOFOffset == (unsigned long)_ftelli64 (fTmp));
1653 #else
1654     assert ((*it)->pFileEntry->nEOFOffset == ftell (fTmp));
1655 #endif
1656         }
1657         queFiles.clear();
1658         queFiles.reserve (g_nMaxItemsRelinkBuffer);
1659         return true;
1660 }
1661
1662 void TruncateFile(FILE* file, size_t newLength)
1663 {
1664         int filedes = _fileno(file);
1665         _chsize_s(filedes, newLength);
1666 }
1667
1668 bool ZipDir::CacheRW::EncryptArchive(EncryptionChange change, IEncryptPredicate* encryptContentPredicate, int* numChanged, int* numSkipped)
1669 {
1670         FileRecordList arrFiles(GetRoot());
1671         arrFiles.SortByFileOffset();
1672
1673         size_t unusedSpace = 0;
1674         size_t lastDataEnd = 0;
1675
1676         for (FileRecordList::iterator it = arrFiles.begin(); it != arrFiles.end(); ++it)
1677         {
1678                 FileEntry* entry = it->pFileEntry;
1679
1680                 if (entry->nFileHeaderOffset > lastDataEnd)
1681                 {
1682                         fseek(m_pFile, lastDataEnd, SEEK_SET);
1683                         size_t gapLength = entry->nFileHeaderOffset - lastDataEnd;
1684                         unusedSpace += gapLength;
1685                         if (change == ENCRYPT)
1686                         {
1687                                 if (!WriteRandomData(m_pFile, gapLength))
1688                                 {
1689                                         return false;
1690                                 }
1691                         }
1692                         else
1693                         {
1694                                 if (!WriteNullData(gapLength))
1695                                 {
1696                                         return false;
1697                                 }
1698                         }
1699                 }
1700                 lastDataEnd = entry->nEOFOffset;
1701
1702                 if (numSkipped)
1703                 {
1704                         ++(*numSkipped);
1705                 }
1706
1707                 // find the file data offset
1708                 if (ZD_ERROR_SUCCESS != Refresh (entry))
1709                 {
1710                         return false;
1711                 }
1712
1713                 ZipFile::ushort oldMethod = entry->nMethod;
1714                 ZipFile::ushort newMethod = oldMethod;
1715                 if (change == ENCRYPT)
1716                 {
1717                         if (entry->nMethod == METHOD_DEFLATE)
1718                         {
1719                                         newMethod = METHOD_DEFLATE_AND_ENCRYPT;
1720                         }
1721                 }
1722                 else
1723                 {
1724                         if (entry->nMethod == METHOD_DEFLATE_AND_ENCRYPT)
1725                         {
1726                                 newMethod = METHOD_DEFLATE;
1727                         }
1728                 }
1729
1730                 // allow encryption only for matching files
1731                 if (newMethod == METHOD_DEFLATE_AND_ENCRYPT &&
1732                           (!encryptContentPredicate || !encryptContentPredicate->Match(it->strPath.c_str())))
1733                 {
1734                         newMethod = METHOD_DEFLATE;
1735                 }
1736
1737                 entry->nMethod = newMethod;
1738
1739                 const bool encryptHeaders = change == ENCRYPT;
1740                 // encryption is toggled or compression method changed...
1741                 if (newMethod != oldMethod || encryptHeaders != m_bEncryptedHeaders)
1742                 {
1743                         // ... update header
1744                         if (ZipDir::WriteLocalHeader(m_pFile, entry, it->strPath.c_str(), encryptHeaders) != ZD_ERROR_SUCCESS)
1745                         {
1746                                 return false;
1747                         }
1748                 }
1749
1750                 if (newMethod == oldMethod)
1751                 {
1752                         // no need to update file content
1753                         continue;
1754                 }
1755
1756                 // go to the file data
1757 #if CRY_PLATFORM_WINDOWS
1758                 if (_fseeki64 (m_pFile, (__int64)entry->nFileDataOffset, SEEK_SET) != 0)
1759 #else
1760     if (fseek (m_pFile, entry->nFileDataOffset, SEEK_SET) != 0)
1761 #endif
1762                         return false;
1763
1764                 // allocate memory for the file compressed data
1765                 FileDataRecordPtr pFile = FileDataRecord::New(*it);
1766                 if (!pFile)
1767                 {
1768                         return false;
1769                 }
1770
1771                 // read the compressed data
1772                 if (entry->desc.lSizeCompressed && fread (pFile->GetData(), entry->desc.lSizeCompressed, 1, m_pFile) != 1)
1773                 {
1774                         return false;
1775                 }
1776
1777                 if (oldMethod == METHOD_DEFLATE_AND_ENCRYPT)
1778                 {
1779                         ZipDir::Decrypt((char*)pFile->GetData(), entry->desc.lSizeCompressed, m_encryptionKey);
1780                 }
1781
1782 #if CRY_PLATFORM_WINDOWS
1783                 if (_fseeki64 (m_pFile, (__int64)entry->nFileDataOffset, SEEK_SET) != 0)
1784 #else
1785     if (fseek (m_pFile, entry->nFileDataOffset, SEEK_SET) != 0)
1786 #endif
1787                 {
1788                         return false;
1789                 }
1790
1791                 const bool encryptContent = newMethod == METHOD_DEFLATE_AND_ENCRYPT;
1792                 if (!WriteCompressedData((const char*)pFile->GetData(), entry->desc.lSizeCompressed, encryptContent, m_pFile))
1793                 {
1794                         return false;
1795                 }
1796
1797                 if (numSkipped)
1798                 {
1799                         --(*numSkipped);
1800                 }
1801                 if (numChanged)
1802                 {
1803                         ++(*numChanged);
1804                 }
1805         }
1806
1807         m_bEncryptedHeaders = change == ENCRYPT;
1808         m_bHeadersEncryptedOnClose = m_bEncryptedHeaders;
1809
1810         if (!WriteCDR(m_pFile, m_bEncryptedHeaders))
1811         {
1812                 return false;
1813         }
1814
1815         if (fflush (m_pFile) != 0)
1816         {
1817                 return false;
1818         }
1819
1820         size_t endOfCDR = (size_t)ftell(m_pFile);
1821
1822         fseek(m_pFile, 0, SEEK_END);
1823         size_t fileSize = (size_t)ftell(m_pFile);
1824
1825         if (fileSize != endOfCDR)
1826         {
1827                 TruncateFile(m_pFile, endOfCDR);
1828         }
1829
1830
1831         if (unusedSpace > 0)
1832         {
1833 //              RCLog("Archive contains %i bytes of uncompacted space.", (int)unusedSpace);
1834         }
1835
1836         fclose(m_pFile);
1837         m_pFile = 0;
1838         m_treeDir.Clear();
1839         return true;
1840 }