src/library/fileimage.cpp

   1 #include "fileimage.hpp"
   2 #include "sha256.hpp"
   3 #include "fileimage-patch.hpp"
   4 #include "string.hpp"
   5 #include "minmax.hpp"
   6 #include "zip.hpp"
   7 #include "directory.hpp"
   8 #include <sstream>
   9
  10 namespace fileimage
  11 {
  12 namespace
  13 {
  14         std::map<std::string, std::pair<time_t, std::string>> cached_entries;
  15
  16         mutex_class& global_queue_mutex()
  17         {
  18                 static mutex_class m;
  19                 return m;
  20         }
  21
  22         uint64_t calculate_headersize(uint64_t f, uint64_t h)
  23         {
  24                 if(!h) return 0;
  25                 if(f % (2 * h) == h) return h;
  26                 return 0;
  27         }
  28
  29         void* thread_trampoline(hash* h)
  30         {
  31                 h->entrypoint();
  32                 return NULL;
  33         }
  34
  35         std::string lookup_cache(const std::string& filename, uint64_t prefixlen)
  36         {
  37                 std::string cache = filename + ".sha256";
  38                 if(prefixlen) cache += (stringfmt() << "-" << prefixlen).str();
  39                 time_t filetime = file_get_mtime(filename);
  40                 if(cached_entries.count(cache)) {
  41                         //Found the cache entry...
  42                         if(cached_entries[cache].first == filetime)
  43                                 return cached_entries[cache].second;
  44                         else {
  45                                 //Stale.
  46                                 unlink(cache.c_str());
  47                                 cached_entries.erase(cache);
  48                                 return "";
  49                         }
  50                 }
  51
  52                 std::string cached_hash;
  53                 time_t rfiletime;
  54
  55                 std::ifstream in(cache);
  56                 if(!in)
  57                         return "";      //Failed.
  58
  59                 std::string tmp;
  60                 std::getline(in, tmp);
  61                 std::istringstream _in(tmp);
  62                 _in >> rfiletime;
  63                 std::getline(in, cached_hash);
  64
  65                 if(rfiletime == filetime) {
  66                         cached_entries[cache] = std::make_pair(rfiletime, cached_hash);
  67                 } else  {
  68                         //Stale.
  69                         unlink(cache.c_str());
  70                         cached_entries.erase(cache);
  71                         return "";
  72                 }
  73                 return cached_hash;
  74         }
  75
  76         void store_cache(const std::string& filename, uint64_t prefixlen, const std::string& value)
  77         {
  78                 std::string cache = filename + ".sha256";
  79                 if(prefixlen) cache += (stringfmt() << "-" << prefixlen).str();
  80                 time_t filetime = file_get_mtime(filename);
  81                 std::ofstream out(cache);
  82                 cached_entries[cache] = std::make_pair(filetime, value);
  83                 if(!out)
  84                         return;         //Failed!
  85                 out << filetime << std::endl;
  86                 out << value << std::endl;
  87                 out.close();
  88         }
  89
  90         uint64_t get_file_size(const std::string& filename)
  91         {
  92                 uintmax_t size = file_get_size(filename);
  93                 if(size == static_cast<uintmax_t>(-1))
  94                         return 0;
  95                 return size;
  96         }
  97 }
  98
  99 hashval::hashval()
 100 {
 101         is_ready = false;
 102         cbid = 0;
 103         prev = next = NULL;
 104         hasher = NULL;
 105 }
 106
 107 hashval::hashval(const std::string& _value, uint64_t _prefix)
 108 {
 109         is_ready = true;
 110         value = _value;
 111         cbid = 0;
 112         prefixv = _prefix;
 113         prev = next = NULL;
 114         hasher = NULL;
 115 }
 116
 117 hashval::hashval(hash& h, unsigned id)
 118 {
 119         umutex_class h2(global_queue_mutex());
 120         is_ready = false;
 121         cbid = id;
 122         prev = next = NULL;
 123         hasher = &h;
 124         hasher->link(*this);
 125 }
 126
 127 hashval::~hashval()
 128 {
 129         umutex_class h2(global_queue_mutex());
 130         umutex_class h(mutex);
 131         if(hasher)
 132                 hasher->unlink(*this);
 133 }
 134
 135 bool hashval::ready() const
 136 {
 137         umutex_class h(mutex);
 138         return is_ready;
 139 }
 140
 141 std::string hashval::read() const
 142 {
 143         umutex_class h(mutex);
 144         while(!is_ready)
 145                 condition.wait(h);
 146         if(error != "")
 147                 throw std::runtime_error(error);
 148         return value;
 149 }
 150
 151 uint64_t hashval::prefix() const
 152 {
 153         umutex_class h(mutex);
 154         while(!is_ready)
 155                 condition.wait(h);
 156         if(error != "")
 157                 throw std::runtime_error(error);
 158         return prefixv;
 159 }
 160
 161 hashval::hashval(const hashval& f)
 162 {
 163         umutex_class h2(global_queue_mutex());
 164         umutex_class h(f.mutex);
 165         is_ready = f.is_ready;
 166         cbid = f.cbid;
 167         value = f.value;
 168         error = f.error;
 169         prefixv = f.prefixv;
 170         prev = next = NULL;
 171         hasher = f.hasher;
 172         if(!is_ready && hasher)
 173                 hasher->link(*this);
 174 }
 175
 176 hashval& hashval::operator=(const hashval& f)
 177 {
 178         if(this == &f)
 179                 return *this;
 180         umutex_class h2(global_queue_mutex());
 181         if((size_t)this < (size_t)&f) {
 182                 mutex.lock();
 183                 f.mutex.lock();
 184         } else {
 185                 f.mutex.lock();
 186                 mutex.lock();
 187         }
 188
 189         if(!is_ready && hasher)
 190                 hasher->unlink(*this);
 191         is_ready = f.is_ready;
 192         cbid = f.cbid;
 193         value = f.value;
 194         error = f.error;
 195         prefixv = f.prefixv;
 196         prev = next = NULL;
 197         hasher = f.hasher;
 198         if(!is_ready && hasher)
 199                 hasher->link(*this);
 200         mutex.unlock();
 201         f.mutex.unlock();
 202 }
 203
 204 void hashval::resolve(unsigned id, const std::string& hash, uint64_t _prefix)
 205 {
 206         umutex_class h(mutex);
 207         hasher->unlink(*this);
 208         if(id != cbid)
 209                 return;
 210         is_ready = true;
 211         value = hash;
 212         prefixv = _prefix;
 213         condition.notify_all();
 214 }
 215
 216 void hashval::resolve_error(unsigned id, const std::string& err)
 217 {
 218         umutex_class h(mutex);
 219         hasher->unlink(*this);
 220         if(id != cbid)
 221                 return;
 222         is_ready = true;
 223         error = err;
 224         prefixv = 0;
 225         condition.notify_all();
 226 }
 227
 228 void hash::link(hashval& future)
 229 {
 230         //We assume caller holds global queue lock.
 231         {
 232                 umutex_class h(mutex);
 233                 unsigned cbid = future.cbid;
 234                 for(auto& i : queue)
 235                         if(i.cbid == cbid)
 236                                 i.interested--;
 237         }
 238         future.prev = last_future;
 239         future.next = NULL;
 240         if(last_future)
 241                 last_future->next = &future;
 242         last_future = &future;
 243         if(!first_future)
 244                 first_future = &future;
 245 }
 246
 247 void hash::unlink(hashval& future)
 248 {
 249         //We assume caller holds global queue lock.
 250         {
 251                 umutex_class h(mutex);
 252                 unsigned cbid = future.cbid;
 253                 for(auto& i : queue)
 254                         if(i.cbid == cbid)
 255                                 i.interested++;
 256         }
 257         if(&future == first_future)
 258                 first_future = future.next;
 259         if(&future == last_future)
 260                 last_future = future.prev;
 261         if(future.prev)
 262                 future.prev->next = future.next;
 263         if(future.next)
 264                 future.next->prev = future.prev;
 265 }
 266
 267 hashval hash::operator()(const std::string& filename, uint64_t prefixlen)
 268 {
 269         queue_job j;
 270         j.filename = filename;
 271         j.prefix = prefixlen;
 272         j.size = get_file_size(filename);
 273         j.cbid = next_cbid++;
 274         j.interested = 1;
 275         hashval future(*this, j.cbid);
 276         queue.push_back(j);
 277         umutex_class h(mutex);
 278         total_work += j.size;
 279         work_size += j.size;
 280         condition.notify_all();
 281         return future;
 282 }
 283
 284 hashval hash::operator()(const std::string& filename, std::function<uint64_t(uint64_t)> prefixlen)
 285 {
 286         queue_job j;
 287         j.filename = filename;
 288         j.size = get_file_size(filename);
 289         j.prefix = prefixlen(j.size);
 290         j.cbid = next_cbid++;
 291         j.interested = 1;
 292         hashval future(*this, j.cbid);
 293         queue.push_back(j);
 294         umutex_class h(mutex);
 295         total_work += j.size;
 296         work_size += j.size;
 297         condition.notify_all();
 298         return future;
 299 }
 300
 301 void hash::set_callback(std::function<void(uint64_t, uint64_t)> cb)
 302 {
 303         umutex_class h(mutex);
 304         progresscb = cb;
 305 }
 306
 307 hash::hash()
 308 {
 309         quitting = false;
 310         first_future = NULL;
 311         last_future = NULL;
 312         next_cbid = 0;
 313         total_work = 0;
 314         work_size = 0;
 315         progresscb = [](uint64_t x, uint64_t y) -> void {};
 316         hash_thread = new thread_class(thread_trampoline, this);
 317 }
 318
 319 hash::~hash()
 320 {
 321         {
 322                 umutex_class h(mutex);
 323                 quitting = true;
 324                 condition.notify_all();
 325         }
 326         hash_thread->join();
 327         delete hash_thread;
 328         umutex_class h2(global_queue_mutex());
 329         while(first_future)
 330                 first_future->resolve_error(first_future->cbid, "Hasher deleted");
 331 }
 332
 333 void hash::entrypoint()
 334 {
 335         FILE* fp;
 336         while(true) {
 337                 //Wait for work or quit signal.
 338                 {
 339                         umutex_class h(mutex);
 340                         while(!quitting && queue.empty()) {
 341                                 send_idle();
 342                                 condition.wait(h);
 343                         }
 344                         if(quitting)
 345                                 return;
 346                         //We hawe work.
 347                         current_job = queue.begin();
 348                 }
 349
 350                 //Hash this item.
 351                 uint64_t progress = 0;
 352                 std::string cached_hash;
 353                 fp = NULL;
 354                 cached_hash = lookup_cache(current_job->filename, current_job->prefix);
 355                 if(cached_hash != "") {
 356                         umutex_class h2(global_queue_mutex());
 357                         for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 358                                 fut->resolve(current_job->cbid, cached_hash, current_job->prefix);
 359                         goto finished;
 360                 }
 361                 fp = fopen(current_job->filename.c_str(), "rb");
 362                 if(!fp) {
 363                         umutex_class h2(global_queue_mutex());
 364                         for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 365                                 fut->resolve_error(current_job->cbid, "Can't open file");
 366                 } else {
 367                         sha256 hash;
 368                         uint64_t toskip = current_job->prefix;
 369                         while(!feof(fp) && !ferror(fp)) {
 370                                 {
 371                                         umutex_class h(mutex);
 372                                         if(!current_job->interested)
 373                                                 goto finished; //Aborted.
 374                                 }
 375                                 unsigned char buf[16384];
 376                                 uint64_t offset = 0;
 377                                 size_t s = fread(buf, 1, sizeof(buf), fp);
 378                                 progress += s;
 379                                 //The first current_job->prefix bytes need to be skipped.
 380                                 offset = min(toskip, (uint64_t)s);
 381                                 toskip -= offset;
 382                                 if(s > offset) hash.write(buf + offset, s - offset);
 383                                 send_callback(progress);
 384                         }
 385                         if(ferror(fp)) {
 386                                 umutex_class h2(global_queue_mutex());
 387                                 for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 388                                         fut->resolve_error(current_job->cbid, "Can't read file");
 389                         } else {
 390                                 std::string hval = hash.read();
 391                                 umutex_class h2(global_queue_mutex());
 392                                 for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 393                                         fut->resolve(current_job->cbid, hval, current_job->prefix);
 394                                 store_cache(current_job->filename, current_job->prefix, hval);
 395                         }
 396                 }
 397 finished:
 398                 if(fp) fclose(fp);
 399                 //Okay, this work item is complete.
 400                 {
 401                         umutex_class h(mutex);
 402                         total_work -= current_job->size;
 403                         queue.erase(current_job);
 404                 }
 405                 send_callback(0);
 406         }
 407 }
 408
 409 void hash::send_callback(uint64_t this_completed)
 410 {
 411         uint64_t amount;
 412         {
 413                 umutex_class h(mutex);
 414                 if(this_completed > total_work)
 415                         amount = 0;
 416                 else
 417                         amount = total_work - this_completed;
 418         }
 419         progresscb(amount, work_size);
 420 }
 421
 422 void hash::send_idle()
 423 {
 424         work_size = 0;  //Delete work when idle.
 425         progresscb(0xFFFFFFFFFFFFFFFFULL, 0);
 426 }
 427
 428 image::image() throw(std::bad_alloc)
 429 {
 430         type = info::IT_NONE;
 431         sha_256 = hashval("");
 432         filename = "";
 433 }
 434
 435 image::image(hash& h, const std::string& _filename, const std::string& base,
 436         const struct image::info& info) throw(std::bad_alloc, std::runtime_error)
 437 {
 438         if(info.type == info::IT_NONE && _filename != "")
 439                 throw std::runtime_error("Tried to load NULL image");
 440         if(_filename == "") {
 441                 //NULL.
 442                 type = info::IT_NONE;
 443                 sha_256 = hashval("");
 444                 stripped = 0;
 445                 return;
 446         }
 447
 448         std::string xfilename = _filename;
 449 #if defined(_WIN32) || defined(_WIN64)
 450         const char* split = "/\\";
 451 #else
 452         const char* split = "/";
 453 #endif
 454         size_t s1 = xfilename.find_last_of(split);
 455         size_t s2 = xfilename.find_last_of(".");
 456         if(s1 < xfilename.length()) s1 = s1 + 1; else s1 = 0;
 457         if(s2 <= s1 || s2 >= xfilename.length()) s2 = xfilename.length();
 458         namehint = xfilename.substr(s1, s2 - s1);
 459
 460         //Load markups and memory images.
 461         if(info.type == info::IT_MEMORY || info.type == info::IT_MARKUP) {
 462                 unsigned headered = 0;
 463                 filename = zip::resolverel(_filename, base);
 464                 type = info.type;
 465
 466                 data.reset(new std::vector<char>(zip::readrel(_filename, base)));
 467                 headered = (info.type == info::IT_MEMORY) ? calculate_headersize(data->size(), info.headersize) : 0;
 468                 if(data->size() >= headered) {
 469                         if(headered) {
 470                                 memmove(&(*data)[0], &(*data)[headered], data->size() - headered);
 471                                 data->resize(data->size() - headered);
 472                         }
 473                 } else {
 474                         data->resize(0);
 475                 }
 476                 stripped = headered;
 477                 sha_256 = hashval(sha256::hash(*data), headered);
 478                 if(info.type == info::IT_MARKUP) {
 479                         size_t osize = data->size();
 480                         data->resize(osize + 1);
 481                         (*data)[osize] = 0;
 482                 }
 483                 return;
 484         }
 485
 486         if(info.type == info::IT_FILE) {
 487                 filename = zip::resolverel(_filename, base);
 488                 filename = get_absolute_path(filename);
 489                 type = info::IT_FILE;
 490                 data.reset(new std::vector<char>(filename.begin(), filename.end()));
 491                 stripped = 0;
 492                 sha_256 = h(filename);
 493                 return;
 494         }
 495         throw std::runtime_error("Unknown image type");
 496 }
 497
 498 void image::patch(const std::vector<char>& patch, int32_t offset) throw(std::bad_alloc, std::runtime_error)
 499 {
 500         if(type == info::IT_NONE)
 501                 throw std::runtime_error("Not an image");
 502         if(type != info::IT_MEMORY && type != info::IT_MARKUP)
 503                 throw std::runtime_error("File images can't be patched on the fly");
 504         try {
 505                 std::vector<char> data2 = *data;
 506                 if(type == info::IT_MARKUP)
 507                         data2.resize(data2.size() - 1);
 508                 data2 = ::fileimage::patch(data2, patch, offset);
 509                 //Mark the slot as valid and update hash.
 510                 std::string new_sha256 = sha256::hash(data2);
 511                 if(type == info::IT_MARKUP) {
 512                         size_t osize = data2.size();
 513                         data2.resize(osize + 1);
 514                         data2[osize] = 0;
 515                 }
 516                 data.reset(new std::vector<char>(data2));
 517                 sha_256 = hashval(new_sha256);
 518         } catch(...) {
 519                 throw;
 520         }
 521 }
 522
 523 std::function<uint64_t(uint64_t)> std_headersize_fn(uint64_t hdrsize)
 524 {
 525         uint64_t h = hdrsize;
 526         return ([h](uint64_t x) -> uint64_t { return calculate_headersize(x, h); });
 527 }
 528 }