src/library/fileimage.cpp

   1 #include "fileimage.hpp"
   2 #include "sha256.hpp"
   3 #include "fileimage-patch.hpp"
   4 #include "string.hpp"
   5 #include "minmax.hpp"
   6 #include "zip.hpp"
   7 #include "directory.hpp"
   8 #include <functional>
   9 #include <sstream>
  10
  11 namespace fileimage
  12 {
  13 namespace
  14 {
  15         std::map<std::string, std::pair<time_t, std::string>> cached_entries;
  16
  17         threads::lock& global_queue_mutex()
  18         {
  19                 static bool init = false;
  20                 static threads::lock* m;
  21                 if(!init)
  22                         m = new threads::lock();
  23                 init = true;
  24                 return *m;
  25         }
  26
  27         uint64_t calculate_headersize(uint64_t f, uint64_t h)
  28         {
  29                 if(!h) return 0;
  30                 return f % (2 * h);
  31         }
  32
  33         void* thread_trampoline(hash* h)
  34         {
  35                 h->entrypoint();
  36                 return NULL;
  37         }
  38
  39         std::string lookup_cache(const std::string& filename, uint64_t prefixlen)
  40         {
  41                 std::string cache = filename + ".sha256";
  42                 if(prefixlen) cache += (stringfmt() << "-" << prefixlen).str();
  43                 time_t filetime = directory::mtime(filename);
  44                 if(cached_entries.count(cache)) {
  45                         //Found the cache entry...
  46                         if(cached_entries[cache].first == filetime)
  47                                 return cached_entries[cache].second;
  48                         else {
  49                                 //Stale.
  50                                 unlink(cache.c_str());
  51                                 cached_entries.erase(cache);
  52                                 return "";
  53                         }
  54                 }
  55
  56                 std::string cached_hash;
  57                 time_t rfiletime;
  58
  59                 std::ifstream in(cache);
  60                 if(!in)
  61                         return "";      //Failed.
  62
  63                 std::string tmp;
  64                 std::getline(in, tmp);
  65                 std::istringstream _in(tmp);
  66                 _in >> rfiletime;
  67                 std::getline(in, cached_hash);
  68
  69                 if(rfiletime == filetime) {
  70                         cached_entries[cache] = std::make_pair(rfiletime, cached_hash);
  71                 } else  {
  72                         //Stale.
  73                         unlink(cache.c_str());
  74                         cached_entries.erase(cache);
  75                         return "";
  76                 }
  77                 return cached_hash;
  78         }
  79
  80         void store_cache(const std::string& filename, uint64_t prefixlen, const std::string& value)
  81         {
  82                 std::string cache = filename + ".sha256";
  83                 if(prefixlen) cache += (stringfmt() << "-" << prefixlen).str();
  84                 time_t filetime = directory::mtime(filename);
  85                 std::ofstream out(cache);
  86                 cached_entries[cache] = std::make_pair(filetime, value);
  87                 if(!out)
  88                         return;         //Failed!
  89                 out << filetime << std::endl;
  90                 out << value << std::endl;
  91                 out.close();
  92         }
  93
  94         uint64_t get_file_size(const std::string& filename)
  95         {
  96                 uintmax_t size = directory::size(filename);
  97                 if(size == static_cast<uintmax_t>(-1))
  98                         return 0;
  99                 return size;
 100         }
 101 }
 102
 103 hashval::hashval()
 104 {
 105         is_ready = false;
 106         cbid = 0;
 107         prev = next = NULL;
 108         hasher = NULL;
 109 }
 110
 111 hashval::hashval(const std::string& _value, uint64_t _prefix)
 112 {
 113         is_ready = true;
 114         value = _value;
 115         cbid = 0;
 116         prefixv = _prefix;
 117         prev = next = NULL;
 118         hasher = NULL;
 119 }
 120
 121 hashval::hashval(hash& h, unsigned id)
 122 {
 123         threads::alock h2(global_queue_mutex());
 124         is_ready = false;
 125         cbid = id;
 126         prev = next = NULL;
 127         hasher = &h;
 128         hasher->link(*this);
 129 }
 130
 131 hashval::~hashval()
 132 {
 133         threads::alock h2(global_queue_mutex());
 134         threads::alock h(mlock);
 135         if(hasher)
 136                 hasher->unlink(*this);
 137 }
 138
 139 bool hashval::ready() const
 140 {
 141         threads::alock h(mlock);
 142         return is_ready;
 143 }
 144
 145 std::string hashval::read() const
 146 {
 147         threads::alock h(mlock);
 148         while(!is_ready)
 149                 condition.wait(h);
 150         if(error != "")
 151                 throw std::runtime_error(error);
 152         return value;
 153 }
 154
 155 uint64_t hashval::prefix() const
 156 {
 157         threads::alock h(mlock);
 158         while(!is_ready)
 159                 condition.wait(h);
 160         if(error != "")
 161                 throw std::runtime_error(error);
 162         return prefixv;
 163 }
 164
 165 hashval::hashval(const hashval& f)
 166 {
 167         threads::alock h2(global_queue_mutex());
 168         threads::alock h(f.mlock);
 169         is_ready = f.is_ready;
 170         cbid = f.cbid;
 171         value = f.value;
 172         error = f.error;
 173         prefixv = f.prefixv;
 174         prev = next = NULL;
 175         hasher = f.hasher;
 176         if(!is_ready && hasher)
 177                 hasher->link(*this);
 178 }
 179
 180 hashval& hashval::operator=(const hashval& f)
 181 {
 182         if(this == &f)
 183                 return *this;
 184         threads::alock h2(global_queue_mutex());
 185         threads::alock_multiple({&mlock, &f.mlock});
 186
 187         if(!is_ready && hasher)
 188                 hasher->unlink(*this);
 189         is_ready = f.is_ready;
 190         cbid = f.cbid;
 191         value = f.value;
 192         error = f.error;
 193         prefixv = f.prefixv;
 194         prev = next = NULL;
 195         hasher = f.hasher;
 196         if(!is_ready && hasher)
 197                 hasher->link(*this);
 198         return *this;
 199 }
 200
 201 void hashval::resolve(unsigned id, const std::string& hash, uint64_t _prefix)
 202 {
 203         threads::alock h(mlock);
 204         hasher->unlink(*this);
 205         if(id != cbid)
 206                 return;
 207         is_ready = true;
 208         value = hash;
 209         prefixv = _prefix;
 210         condition.notify_all();
 211 }
 212
 213 void hashval::resolve_error(unsigned id, const std::string& err)
 214 {
 215         threads::alock h(mlock);
 216         hasher->unlink(*this);
 217         if(id != cbid)
 218                 return;
 219         is_ready = true;
 220         error = err;
 221         prefixv = 0;
 222         condition.notify_all();
 223 }
 224
 225 void hash::link(hashval& future)
 226 {
 227         //We assume caller holds global queue lock.
 228         {
 229                 threads::alock h(mlock);
 230                 unsigned cbid = future.cbid;
 231                 for(auto& i : queue)
 232                         if(i.cbid == cbid)
 233                                 i.interested--;
 234         }
 235         future.prev = last_future;
 236         future.next = NULL;
 237         if(last_future)
 238                 last_future->next = &future;
 239         last_future = &future;
 240         if(!first_future)
 241                 first_future = &future;
 242 }
 243
 244 void hash::unlink(hashval& future)
 245 {
 246         //We assume caller holds global queue lock.
 247         {
 248                 threads::alock h(mlock);
 249                 unsigned cbid = future.cbid;
 250                 for(auto& i : queue)
 251                         if(i.cbid == cbid)
 252                                 i.interested++;
 253         }
 254         if(&future == first_future)
 255                 first_future = future.next;
 256         if(&future == last_future)
 257                 last_future = future.prev;
 258         if(future.prev)
 259                 future.prev->next = future.next;
 260         if(future.next)
 261                 future.next->prev = future.prev;
 262 }
 263
 264 hashval hash::operator()(const std::string& filename, uint64_t prefixlen)
 265 {
 266         queue_job j;
 267         j.filename = filename;
 268         j.prefix = prefixlen;
 269         j.size = get_file_size(filename);
 270         j.cbid = next_cbid++;
 271         j.interested = 1;
 272         hashval future(*this, j.cbid);
 273         queue.push_back(j);
 274         threads::alock h(mlock);
 275         total_work += j.size;
 276         work_size += j.size;
 277         condition.notify_all();
 278         return future;
 279 }
 280
 281 hashval hash::operator()(const std::string& filename, std::function<uint64_t(uint64_t)> prefixlen)
 282 {
 283         queue_job j;
 284         j.filename = filename;
 285         j.size = get_file_size(filename);
 286         j.prefix = prefixlen(j.size);
 287         j.cbid = next_cbid++;
 288         j.interested = 1;
 289         hashval future(*this, j.cbid);
 290         queue.push_back(j);
 291         threads::alock h(mlock);
 292         total_work += j.size;
 293         work_size += j.size;
 294         condition.notify_all();
 295         return future;
 296 }
 297
 298 void hash::set_callback(std::function<void(uint64_t, uint64_t)> cb)
 299 {
 300         threads::alock h(mlock);
 301         progresscb = cb;
 302 }
 303
 304 hash::hash()
 305 {
 306         quitting = false;
 307         first_future = NULL;
 308         last_future = NULL;
 309         next_cbid = 0;
 310         total_work = 0;
 311         work_size = 0;
 312         progresscb = [](uint64_t x, uint64_t y) -> void {};
 313         hash_thread = new threads::thread(thread_trampoline, this);
 314 }
 315
 316 hash::~hash()
 317 {
 318         {
 319                 threads::alock h(mlock);
 320                 quitting = true;
 321                 condition.notify_all();
 322         }
 323         hash_thread->join();
 324         delete hash_thread;
 325         threads::alock h2(global_queue_mutex());
 326         while(first_future)
 327                 first_future->resolve_error(first_future->cbid, "Hasher deleted");
 328 }
 329
 330 void hash::entrypoint()
 331 {
 332         FILE* fp;
 333         while(true) {
 334                 //Wait for work or quit signal.
 335                 {
 336                         threads::alock h(mlock);
 337                         while(!quitting && queue.empty()) {
 338                                 send_idle();
 339                                 condition.wait(h);
 340                         }
 341                         if(quitting)
 342                                 return;
 343                         //We hawe work.
 344                         current_job = queue.begin();
 345                 }
 346
 347                 //Hash this item.
 348                 uint64_t progress = 0;
 349                 std::string cached_hash;
 350                 fp = NULL;
 351                 cached_hash = lookup_cache(current_job->filename, current_job->prefix);
 352                 if(cached_hash != "") {
 353                         threads::alock h2(global_queue_mutex());
 354                         for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 355                                 fut->resolve(current_job->cbid, cached_hash, current_job->prefix);
 356                         goto finished;
 357                 }
 358                 fp = fopen(current_job->filename.c_str(), "rb");
 359                 if(!fp) {
 360                         threads::alock h2(global_queue_mutex());
 361                         for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 362                                 fut->resolve_error(current_job->cbid, "Can't open file");
 363                 } else {
 364                         sha256 hash;
 365                         uint64_t toskip = current_job->prefix;
 366                         while(!feof(fp) && !ferror(fp)) {
 367                                 {
 368                                         threads::alock h(mlock);
 369                                         if(!current_job->interested)
 370                                                 goto finished; //Aborted.
 371                                 }
 372                                 unsigned char buf[16384];
 373                                 uint64_t offset = 0;
 374                                 size_t s = fread(buf, 1, sizeof(buf), fp);
 375                                 progress += s;
 376                                 //The first current_job->prefix bytes need to be skipped.
 377                                 offset = min(toskip, (uint64_t)s);
 378                                 toskip -= offset;
 379                                 if(s > offset) hash.write(buf + offset, s - offset);
 380                                 send_callback(progress);
 381                         }
 382                         if(ferror(fp)) {
 383                                 threads::alock h2(global_queue_mutex());
 384                                 for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 385                                         fut->resolve_error(current_job->cbid, "Can't read file");
 386                         } else {
 387                                 std::string hval = hash.read();
 388                                 threads::alock h2(global_queue_mutex());
 389                                 for(hashval* fut = first_future; fut != NULL; fut = fut->next)
 390                                         fut->resolve(current_job->cbid, hval, current_job->prefix);
 391                                 store_cache(current_job->filename, current_job->prefix, hval);
 392                         }
 393                 }
 394 finished:
 395                 if(fp) fclose(fp);
 396                 //Okay, this work item is complete.
 397                 {
 398                         threads::alock h(mlock);
 399                         total_work -= current_job->size;
 400                         queue.erase(current_job);
 401                 }
 402                 send_callback(0);
 403         }
 404 }
 405
 406 void hash::send_callback(uint64_t this_completed)
 407 {
 408         uint64_t amount;
 409         {
 410                 threads::alock h(mlock);
 411                 if(this_completed > total_work)
 412                         amount = 0;
 413                 else
 414                         amount = total_work - this_completed;
 415         }
 416         progresscb(amount, work_size);
 417 }
 418
 419 void hash::send_idle()
 420 {
 421         work_size = 0;  //Delete work when idle.
 422         progresscb(0xFFFFFFFFFFFFFFFFULL, 0);
 423 }
 424
 425 image::image() throw(std::bad_alloc)
 426 {
 427         type = info::IT_NONE;
 428         sha_256 = hashval("");
 429         filename = "";
 430 }
 431
 432 image::image(hash& h, const std::string& _filename, const std::string& base,
 433         const struct image::info& info) throw(std::bad_alloc, std::runtime_error)
 434 {
 435         if(info.type == info::IT_NONE && _filename != "")
 436                 throw std::runtime_error("Tried to load NULL image");
 437         if(_filename == "") {
 438                 //NULL.
 439                 type = info::IT_NONE;
 440                 sha_256 = hashval("");
 441                 stripped = 0;
 442                 return;
 443         }
 444
 445         std::string xfilename = _filename;
 446 #if defined(_WIN32) || defined(_WIN64)
 447         const char* split = "/\\";
 448 #else
 449         const char* split = "/";
 450 #endif
 451         size_t s1 = xfilename.find_last_of(split);
 452         size_t s2 = xfilename.find_last_of(".");
 453         if(s1 < xfilename.length()) s1 = s1 + 1; else s1 = 0;
 454         if(s2 <= s1 || s2 >= xfilename.length()) s2 = xfilename.length();
 455         namehint = xfilename.substr(s1, s2 - s1);
 456
 457         //Load markups and memory images.
 458         if(info.type == info::IT_MEMORY || info.type == info::IT_MARKUP) {
 459                 unsigned headered = 0;
 460                 filename = zip::resolverel(_filename, base);
 461                 type = info.type;
 462
 463                 data.reset(new std::vector<char>(zip::readrel(_filename, base)));
 464                 headered = (info.type == info::IT_MEMORY) ? calculate_headersize(data->size(), info.headersize) : 0;
 465                 if(data->size() >= headered) {
 466                         if(headered) {
 467                                 memmove(&(*data)[0], &(*data)[headered], data->size() - headered);
 468                                 data->resize(data->size() - headered);
 469                         }
 470                 } else {
 471                         data->resize(0);
 472                 }
 473                 stripped = headered;
 474                 sha_256 = hashval(sha256::hash(*data), headered);
 475                 if(info.type == info::IT_MARKUP) {
 476                         size_t osize = data->size();
 477                         data->resize(osize + 1);
 478                         (*data)[osize] = 0;
 479                 }
 480                 return;
 481         }
 482
 483         if(info.type == info::IT_FILE) {
 484                 filename = zip::resolverel(_filename, base);
 485                 filename = directory::absolute_path(filename);
 486                 type = info::IT_FILE;
 487                 data.reset(new std::vector<char>(filename.begin(), filename.end()));
 488                 stripped = 0;
 489                 sha_256 = h(filename);
 490                 return;
 491         }
 492         throw std::runtime_error("Unknown image type");
 493 }
 494
 495 void image::patch(const std::vector<char>& patch, int32_t offset) throw(std::bad_alloc, std::runtime_error)
 496 {
 497         if(type == info::IT_NONE)
 498                 throw std::runtime_error("Not an image");
 499         if(type != info::IT_MEMORY && type != info::IT_MARKUP)
 500                 throw std::runtime_error("File images can't be patched on the fly");
 501         try {
 502                 std::vector<char> data2 = *data;
 503                 if(type == info::IT_MARKUP)
 504                         data2.resize(data2.size() - 1);
 505                 data2 = ::fileimage::patch(data2, patch, offset);
 506                 //Mark the slot as valid and update hash.
 507                 std::string new_sha256 = sha256::hash(data2);
 508                 if(type == info::IT_MARKUP) {
 509                         size_t osize = data2.size();
 510                         data2.resize(osize + 1);
 511                         data2[osize] = 0;
 512                 }
 513                 data.reset(new std::vector<char>(data2));
 514                 sha_256 = hashval(new_sha256);
 515         } catch(...) {
 516                 throw;
 517         }
 518 }
 519
 520 std::function<uint64_t(uint64_t)> std_headersize_fn(uint64_t hdrsize)
 521 {
 522         uint64_t h = hdrsize;
 523         return ([h](uint64_t x) -> uint64_t { return calculate_headersize(x, h); });
 524 }
 525 }