1 #include "fileimage.hpp"
3 #include "fileimage-patch.hpp"
7 #include "directory.hpp"
15 std::map
<std::string
, std::pair
<time_t, std::string
>> cached_entries
;
17 threads::lock
& global_queue_mutex()
19 static bool init
= false;
20 static threads::lock
* m
;
22 m
= new threads::lock();
27 uint64_t calculate_headersize(uint64_t f
, uint64_t h
)
33 void* thread_trampoline(hash
* h
)
39 std::string
lookup_cache(const std::string
& filename
, uint64_t prefixlen
)
41 std::string cache
= filename
+ ".sha256";
42 if(prefixlen
) cache
+= (stringfmt() << "-" << prefixlen
).str();
43 time_t filetime
= directory::mtime(filename
);
44 if(cached_entries
.count(cache
)) {
45 //Found the cache entry...
46 if(cached_entries
[cache
].first
== filetime
)
47 return cached_entries
[cache
].second
;
50 unlink(cache
.c_str());
51 cached_entries
.erase(cache
);
56 std::string cached_hash
;
59 std::ifstream
in(cache
);
64 std::getline(in
, tmp
);
65 std::istringstream
_in(tmp
);
67 std::getline(in
, cached_hash
);
69 if(rfiletime
== filetime
) {
70 cached_entries
[cache
] = std::make_pair(rfiletime
, cached_hash
);
73 unlink(cache
.c_str());
74 cached_entries
.erase(cache
);
80 void store_cache(const std::string
& filename
, uint64_t prefixlen
, const std::string
& value
)
82 std::string cache
= filename
+ ".sha256";
83 if(prefixlen
) cache
+= (stringfmt() << "-" << prefixlen
).str();
84 time_t filetime
= directory::mtime(filename
);
85 std::ofstream
out(cache
);
86 cached_entries
[cache
] = std::make_pair(filetime
, value
);
89 out
<< filetime
<< std::endl
;
90 out
<< value
<< std::endl
;
94 uint64_t get_file_size(const std::string
& filename
)
96 uintmax_t size
= directory::size(filename
);
97 if(size
== static_cast<uintmax_t>(-1))
111 hashval::hashval(const std::string
& _value
, uint64_t _prefix
)
121 hashval::hashval(hash
& h
, unsigned id
)
123 threads::alock
h2(global_queue_mutex());
133 threads::alock
h2(global_queue_mutex());
134 threads::alock
h(mlock
);
136 hasher
->unlink(*this);
139 bool hashval::ready() const
141 threads::alock
h(mlock
);
145 std::string
hashval::read() const
147 threads::alock
h(mlock
);
151 throw std::runtime_error(error
);
155 uint64_t hashval::prefix() const
157 threads::alock
h(mlock
);
161 throw std::runtime_error(error
);
165 hashval::hashval(const hashval
& f
)
167 threads::alock
h2(global_queue_mutex());
168 threads::alock
h(f
.mlock
);
169 is_ready
= f
.is_ready
;
176 if(!is_ready
&& hasher
)
180 hashval
& hashval::operator=(const hashval
& f
)
184 threads::alock
h2(global_queue_mutex());
185 threads::alock_multiple({&mlock
, &f
.mlock
});
187 if(!is_ready
&& hasher
)
188 hasher
->unlink(*this);
189 is_ready
= f
.is_ready
;
196 if(!is_ready
&& hasher
)
201 void hashval::resolve(unsigned id
, const std::string
& hash
, uint64_t _prefix
)
203 threads::alock
h(mlock
);
204 hasher
->unlink(*this);
210 condition
.notify_all();
213 void hashval::resolve_error(unsigned id
, const std::string
& err
)
215 threads::alock
h(mlock
);
216 hasher
->unlink(*this);
222 condition
.notify_all();
225 void hash::link(hashval
& future
)
227 //We assume caller holds global queue lock.
229 threads::alock
h(mlock
);
230 unsigned cbid
= future
.cbid
;
235 future
.prev
= last_future
;
238 last_future
->next
= &future
;
239 last_future
= &future
;
241 first_future
= &future
;
244 void hash::unlink(hashval
& future
)
246 //We assume caller holds global queue lock.
248 threads::alock
h(mlock
);
249 unsigned cbid
= future
.cbid
;
254 if(&future
== first_future
)
255 first_future
= future
.next
;
256 if(&future
== last_future
)
257 last_future
= future
.prev
;
259 future
.prev
->next
= future
.next
;
261 future
.next
->prev
= future
.prev
;
264 hashval
hash::operator()(const std::string
& filename
, uint64_t prefixlen
)
267 j
.filename
= filename
;
268 j
.prefix
= prefixlen
;
269 j
.size
= get_file_size(filename
);
270 j
.cbid
= next_cbid
++;
272 hashval
future(*this, j
.cbid
);
274 threads::alock
h(mlock
);
275 total_work
+= j
.size
;
277 condition
.notify_all();
281 hashval
hash::operator()(const std::string
& filename
, std::function
<uint64_t(uint64_t)> prefixlen
)
284 j
.filename
= filename
;
285 j
.size
= get_file_size(filename
);
286 j
.prefix
= prefixlen(j
.size
);
287 j
.cbid
= next_cbid
++;
289 hashval
future(*this, j
.cbid
);
291 threads::alock
h(mlock
);
292 total_work
+= j
.size
;
294 condition
.notify_all();
298 void hash::set_callback(std::function
<void(uint64_t, uint64_t)> cb
)
300 threads::alock
h(mlock
);
312 progresscb
= [](uint64_t x
, uint64_t y
) -> void {};
313 hash_thread
= new threads::thread(thread_trampoline
, this);
319 threads::alock
h(mlock
);
321 condition
.notify_all();
325 threads::alock
h2(global_queue_mutex());
327 first_future
->resolve_error(first_future
->cbid
, "Hasher deleted");
330 void hash::entrypoint()
334 //Wait for work or quit signal.
336 threads::alock
h(mlock
);
337 while(!quitting
&& queue
.empty()) {
344 current_job
= queue
.begin();
348 uint64_t progress
= 0;
349 std::string cached_hash
;
351 cached_hash
= lookup_cache(current_job
->filename
, current_job
->prefix
);
352 if(cached_hash
!= "") {
353 threads::alock
h2(global_queue_mutex());
354 for(hashval
* fut
= first_future
; fut
!= NULL
; fut
= fut
->next
)
355 fut
->resolve(current_job
->cbid
, cached_hash
, current_job
->prefix
);
358 fp
= fopen(current_job
->filename
.c_str(), "rb");
360 threads::alock
h2(global_queue_mutex());
361 for(hashval
* fut
= first_future
; fut
!= NULL
; fut
= fut
->next
)
362 fut
->resolve_error(current_job
->cbid
, "Can't open file");
365 uint64_t toskip
= current_job
->prefix
;
366 while(!feof(fp
) && !ferror(fp
)) {
368 threads::alock
h(mlock
);
369 if(!current_job
->interested
)
370 goto finished
; //Aborted.
372 unsigned char buf
[16384];
374 size_t s
= fread(buf
, 1, sizeof(buf
), fp
);
376 //The first current_job->prefix bytes need to be skipped.
377 offset
= min(toskip
, (uint64_t)s
);
379 if(s
> offset
) hash
.write(buf
+ offset
, s
- offset
);
380 send_callback(progress
);
383 threads::alock
h2(global_queue_mutex());
384 for(hashval
* fut
= first_future
; fut
!= NULL
; fut
= fut
->next
)
385 fut
->resolve_error(current_job
->cbid
, "Can't read file");
387 std::string hval
= hash
.read();
388 threads::alock
h2(global_queue_mutex());
389 for(hashval
* fut
= first_future
; fut
!= NULL
; fut
= fut
->next
)
390 fut
->resolve(current_job
->cbid
, hval
, current_job
->prefix
);
391 store_cache(current_job
->filename
, current_job
->prefix
, hval
);
396 //Okay, this work item is complete.
398 threads::alock
h(mlock
);
399 total_work
-= current_job
->size
;
400 queue
.erase(current_job
);
406 void hash::send_callback(uint64_t this_completed
)
410 threads::alock
h(mlock
);
411 if(this_completed
> total_work
)
414 amount
= total_work
- this_completed
;
416 progresscb(amount
, work_size
);
419 void hash::send_idle()
421 work_size
= 0; //Delete work when idle.
422 progresscb(0xFFFFFFFFFFFFFFFFULL
, 0);
425 image::image() throw(std::bad_alloc
)
427 type
= info::IT_NONE
;
428 sha_256
= hashval("");
432 image::image(hash
& h
, const std::string
& _filename
, const std::string
& base
,
433 const struct image::info
& info
) throw(std::bad_alloc
, std::runtime_error
)
435 if(info
.type
== info::IT_NONE
&& _filename
!= "")
436 throw std::runtime_error("Tried to load NULL image");
437 if(_filename
== "") {
439 type
= info::IT_NONE
;
440 sha_256
= hashval("");
445 std::string xfilename
= _filename
;
446 #if defined(_WIN32) || defined(_WIN64)
447 const char* split
= "/\\";
449 const char* split
= "/";
451 size_t s1
= xfilename
.find_last_of(split
);
452 size_t s2
= xfilename
.find_last_of(".");
453 if(s1
< xfilename
.length()) s1
= s1
+ 1; else s1
= 0;
454 if(s2
<= s1
|| s2
>= xfilename
.length()) s2
= xfilename
.length();
455 namehint
= xfilename
.substr(s1
, s2
- s1
);
457 //Load markups and memory images.
458 if(info
.type
== info::IT_MEMORY
|| info
.type
== info::IT_MARKUP
) {
459 unsigned headered
= 0;
460 filename
= zip::resolverel(_filename
, base
);
463 data
.reset(new std::vector
<char>(zip::readrel(_filename
, base
)));
464 headered
= (info
.type
== info::IT_MEMORY
) ? calculate_headersize(data
->size(), info
.headersize
) : 0;
465 if(data
->size() >= headered
) {
467 memmove(&(*data
)[0], &(*data
)[headered
], data
->size() - headered
);
468 data
->resize(data
->size() - headered
);
474 sha_256
= hashval(sha256::hash(*data
), headered
);
475 if(info
.type
== info::IT_MARKUP
) {
476 size_t osize
= data
->size();
477 data
->resize(osize
+ 1);
483 if(info
.type
== info::IT_FILE
) {
484 filename
= zip::resolverel(_filename
, base
);
485 filename
= directory::absolute_path(filename
);
486 type
= info::IT_FILE
;
487 data
.reset(new std::vector
<char>(filename
.begin(), filename
.end()));
489 sha_256
= h(filename
);
492 throw std::runtime_error("Unknown image type");
495 void image::patch(const std::vector
<char>& patch
, int32_t offset
) throw(std::bad_alloc
, std::runtime_error
)
497 if(type
== info::IT_NONE
)
498 throw std::runtime_error("Not an image");
499 if(type
!= info::IT_MEMORY
&& type
!= info::IT_MARKUP
)
500 throw std::runtime_error("File images can't be patched on the fly");
502 std::vector
<char> data2
= *data
;
503 if(type
== info::IT_MARKUP
)
504 data2
.resize(data2
.size() - 1);
505 data2
= ::fileimage::patch(data2
, patch
, offset
);
506 //Mark the slot as valid and update hash.
507 std::string new_sha256
= sha256::hash(data2
);
508 if(type
== info::IT_MARKUP
) {
509 size_t osize
= data2
.size();
510 data2
.resize(osize
+ 1);
513 data
.reset(new std::vector
<char>(data2
));
514 sha_256
= hashval(new_sha256
);
520 std::function
<uint64_t(uint64_t)> std_headersize_fn(uint64_t hdrsize
)
522 uint64_t h
= hdrsize
;
523 return ([h
](uint64_t x
) -> uint64_t { return calculate_headersize(x
, h
); });