Upload UI
[lsnes.git] / src / library / zip.cpp
blobc040f0854e24a7d10dcd8227e8445c4189128090
1 #include "zip.hpp"
3 #include <cstdint>
4 #include <cstring>
5 #include <iostream>
6 #include <stdexcept>
7 #include <boost/filesystem.hpp>
8 #include <boost/iostreams/categories.hpp>
9 #include <boost/iostreams/copy.hpp>
10 #include <boost/iostreams/stream.hpp>
11 #include <boost/iostreams/stream_buffer.hpp>
12 #include <boost/iostreams/filter/symmetric.hpp>
13 #include <boost/iostreams/filter/zlib.hpp>
14 #include <boost/iostreams/filtering_stream.hpp>
15 #include <boost/iostreams/device/back_inserter.hpp>
16 #if defined(_WIN32) || defined(_WIN64) || defined(TEST_WIN32_CODE)
17 #include <windows.h>
18 #endif
20 #ifdef BOOST_FILESYSTEM3
21 namespace boost_fs = boost::filesystem3;
22 #else
23 namespace boost_fs = boost::filesystem;
24 #endif
26 int rename_file_overwrite(const char* oldname, const char* newname)
28 #if defined(_WIN32) || defined(_WIN64) || defined(TEST_WIN32_CODE)
29 return MoveFileEx(oldname, newname, MOVEFILE_REPLACE_EXISTING);
30 #else
31 return rename(oldname, newname);
32 #endif
35 namespace
37 bool is_regular_file(const std::string& filename)
39 boost::system::error_code ec;
40 boost_fs::file_status stat = status(boost_fs::path(filename), ec);
41 bool e = is_regular_file(stat);
42 return e;
45 uint32_t read32(const unsigned char* buf, unsigned offset = 0, unsigned modulo = 4) throw()
47 return (uint32_t)buf[offset % modulo] |
48 ((uint32_t)buf[(offset + 1) % modulo] << 8) |
49 ((uint32_t)buf[(offset + 2) % modulo] << 16) |
50 ((uint32_t)buf[(offset + 3) % modulo] << 24);
53 uint16_t read16(const unsigned char* buf) throw()
55 return (uint16_t)buf[0] | ((uint16_t)buf[1] << 8);
58 void write16(unsigned char* buf, uint16_t value) throw()
60 buf[0] = (value) & 0xFF;
61 buf[1] = (value >> 8) & 0xFF;
64 void write32(unsigned char* buf, uint32_t value) throw()
66 buf[0] = (value) & 0xFF;
67 buf[1] = (value >> 8) & 0xFF;
68 buf[2] = (value >> 16) & 0xFF;
69 buf[3] = (value >> 24) & 0xFF;
72 class file_input
74 public:
75 typedef char char_type;
76 typedef boost::iostreams::source_tag category;
77 file_input(std::ifstream& _stream, size_t* _refcnt)
78 : stream(_stream), stream_refcnt(*_refcnt)
80 stream_refcnt++;
81 position = stream.tellg();
82 left_unlimited = true;
85 file_input(std::ifstream& _stream, uint32_t size, size_t* _refcnt)
86 : stream(_stream), stream_refcnt(*_refcnt)
88 stream_refcnt++;
89 position = stream.tellg();
90 left_unlimited = false;
91 left = size;
94 void close()
98 std::streamsize read(char* s, std::streamsize n)
100 stream.clear();
101 stream.seekg(position, std::ios_base::beg);
102 if(stream.fail())
103 throw std::runtime_error("Can't seek ZIP file");
104 if(!left_unlimited && left == 0)
105 return -1;
106 if(!left_unlimited && n > left)
107 n = left;
108 stream.read(s, n);
109 std::streamsize r = stream.gcount();
110 if(r == 0 && stream.fail())
111 throw std::runtime_error("Can't read compressed data from ZIP file");
112 if(!stream && r == 0)
113 return -1;
114 position += r;
115 left -= r;
116 return r;
119 ~file_input()
121 if(!--stream_refcnt) {
122 delete &stream;
123 delete &stream_refcnt;
127 file_input(const file_input& f)
128 : stream(f.stream), stream_refcnt(f.stream_refcnt)
130 stream_refcnt++;
131 position = f.position;
132 left_unlimited = f.left_unlimited;
133 left = f.left;
135 protected:
136 std::ifstream& stream;
137 size_t& stream_refcnt;
138 std::streamoff position;
139 bool left_unlimited;
140 uint32_t left;
141 private:
142 file_input& operator=(const file_input& f);
145 class vector_output
147 public:
148 typedef char char_type;
149 typedef boost::iostreams::sink_tag category;
150 vector_output(std::vector<char>& _stream)
151 : stream(_stream)
155 void close()
159 std::streamsize write(const char* s, std::streamsize n)
161 size_t oldsize = stream.size();
162 stream.resize(oldsize + n);
163 memcpy(&stream[oldsize], s, n);
164 return n;
166 protected:
167 std::vector<char>& stream;
170 class size_and_crc_filter_impl
172 public:
173 typedef char char_type;
175 size_and_crc_filter_impl()
177 dsize = 0;
178 crc = ::crc32(0, NULL, 0);
181 void close()
185 bool filter(const char*& src_begin, const char* src_end, char*& dest_begin, char* dest_end,
186 bool flush)
188 ptrdiff_t amount = src_end - src_begin;
189 if(flush && amount == 0)
190 return false;
191 if(amount > dest_end - dest_begin)
192 amount = dest_end - dest_begin;
193 dsize += amount;
194 crc = ::crc32(crc, reinterpret_cast<const unsigned char*>(src_begin), amount);
195 memcpy(dest_begin, src_begin, amount);
196 src_begin += amount;
197 dest_begin += amount;
198 return true;
201 uint32_t size()
203 return dsize;
206 uint32_t crc32()
208 return crc;
210 private:
211 uint32_t dsize;
212 uint32_t crc;
215 class size_and_crc_filter : public boost::iostreams::symmetric_filter<size_and_crc_filter_impl,
216 std::allocator<char>>
218 typedef symmetric_filter<size_and_crc_filter_impl, std::allocator<char>> base_type;
219 public:
220 typedef typename base_type::char_type char_type;
221 typedef typename base_type::category category;
222 size_and_crc_filter(int bsize)
223 : base_type(bsize)
227 uint32_t size()
229 return filter().size();
232 uint32_t crc32()
234 return filter().crc32();
238 struct zipfile_member_info
240 bool central_directory_special; //Central directory, not real member.
241 uint16_t version_needed;
242 uint16_t flags;
243 uint16_t compression;
244 uint16_t mtime_time;
245 uint16_t mtime_day;
246 uint32_t crc;
247 uint32_t compressed_size;
248 uint32_t uncompressed_size;
249 std::string filename;
250 uint32_t header_offset;
251 uint32_t data_offset;
252 uint32_t next_offset;
255 //Parse member starting from current offset.
256 zipfile_member_info parse_member(std::ifstream& file)
258 zipfile_member_info info;
259 info.central_directory_special = false;
260 info.header_offset = file.tellg();
261 //The file header is 30 bytes (this could also hit central header, but that's even larger).
262 unsigned char buffer[30];
263 if(!(file.read(reinterpret_cast<char*>(buffer), 30)))
264 throw std::runtime_error("Can't read file header from ZIP file");
265 uint32_t magic = read32(buffer);
266 if(magic == 0x02014b50) {
267 info.central_directory_special = true;
268 return info;
270 if(magic != 0x04034b50)
271 throw std::runtime_error("ZIP archive corrupt: Expected file or central directory magic");
272 info.version_needed = read16(buffer + 4);
273 info.flags = read16(buffer + 6);
274 info.compression = read16(buffer + 8);
275 info.mtime_time = read16(buffer + 10);
276 info.mtime_day = read16(buffer + 12);
277 info.crc = read32(buffer + 14);
278 info.compressed_size = read32(buffer + 18);
279 info.uncompressed_size = read32(buffer + 22);
280 uint16_t filename_len = read16(buffer + 26);
281 uint16_t extra_len = read16(buffer + 28);
282 if(!filename_len)
283 throw std::runtime_error("Unsupported ZIP feature: Empty filename not allowed");
284 if(info.version_needed > 20) {
285 throw std::runtime_error("Unsupported ZIP feature: Only ZIP versions up to 2.0 supported");
287 if(info.flags & 0x2001)
288 throw std::runtime_error("Unsupported ZIP feature: Encryption is not supported");
289 if(info.flags & 0x8)
290 throw std::runtime_error("Unsupported ZIP feature: Indeterminate length not supported");
291 if(info.flags & 0x20)
292 throw std::runtime_error("Unsupported ZIP feature: Binary patching is not supported");
293 if(info.compression != 0 && info.compression != 8)
294 throw std::runtime_error("Unsupported ZIP feature: Unsupported compression method");
295 if(info.compression == 0 && info.compressed_size != info.uncompressed_size)
296 throw std::runtime_error("ZIP archive corrupt: csize ≠ usize for stored member");
297 std::vector<unsigned char> filename_storage;
298 filename_storage.resize(filename_len);
299 if(!(file.read(reinterpret_cast<char*>(&filename_storage[0]), filename_len)))
300 throw std::runtime_error("Can't read file name from zip file");
301 info.filename = std::string(reinterpret_cast<char*>(&filename_storage[0]), filename_len);
302 info.data_offset = info.header_offset + 30 + filename_len + extra_len;
303 info.next_offset = info.data_offset + info.compressed_size;
304 return info;
308 bool zip_reader::has_member(const std::string& name) throw()
310 return (offsets.count(name) > 0);
313 std::string zip_reader::find_first() throw(std::bad_alloc)
315 if(offsets.empty())
316 return "";
317 else
318 return offsets.begin()->first;
321 std::string zip_reader::find_next(const std::string& name) throw(std::bad_alloc)
323 auto i = offsets.upper_bound(name);
324 if(i == offsets.end())
325 return "";
326 else
327 return i->first;
330 std::istream& zip_reader::operator[](const std::string& name) throw(std::bad_alloc, std::runtime_error)
332 if(!offsets.count(name))
333 throw std::runtime_error("No such file '" + name + "' in zip archive");
334 zipstream->clear();
335 zipstream->seekg(offsets[name], std::ios::beg);
336 zipfile_member_info info = parse_member(*zipstream);
337 zipstream->clear();
338 zipstream->seekg(info.data_offset, std::ios::beg);
339 if(info.compression == 0) {
340 return *new boost::iostreams::stream<file_input>(*zipstream, info.uncompressed_size, refcnt);
341 } else if(info.compression == 8) {
342 boost::iostreams::filtering_istream* s = new boost::iostreams::filtering_istream();
343 boost::iostreams::zlib_params params;
344 params.noheader = true;
345 s->push(boost::iostreams::zlib_decompressor(params));
346 s->push(file_input(*zipstream, info.compressed_size, refcnt));
347 return *s;
348 } else
349 throw std::runtime_error("Unsupported ZIP feature: Unsupported compression method");
352 zip_reader::iterator zip_reader::begin() throw(std::bad_alloc)
354 return iterator(offsets.begin());
357 zip_reader::iterator zip_reader::end() throw(std::bad_alloc)
359 return iterator(offsets.end());
362 zip_reader::riterator zip_reader::rbegin() throw(std::bad_alloc)
364 return riterator(offsets.rbegin());
367 zip_reader::riterator zip_reader::rend() throw(std::bad_alloc)
369 return riterator(offsets.rend());
372 zip_reader::~zip_reader() throw()
374 if(!--*refcnt) {
375 delete zipstream;
376 delete refcnt;
380 zip_reader::zip_reader(const std::string& zipfile) throw(std::bad_alloc, std::runtime_error)
382 if(!is_regular_file(zipfile))
383 throw std::runtime_error("Zipfile '" + zipfile + "' is not regular file");
384 zipstream = NULL;
385 refcnt = NULL;
386 try {
387 zipfile_member_info info;
388 info.next_offset = 0;
389 zipstream = new std::ifstream;
390 zipstream->open(zipfile.c_str(), std::ios::binary);
391 refcnt = new size_t;
392 *refcnt = 1;
393 if(!*zipstream)
394 throw std::runtime_error("Can't open zipfile '" + zipfile + "' for reading");
395 do {
396 zipstream->clear();
397 zipstream->seekg(info.next_offset);
398 if(zipstream->fail())
399 throw std::runtime_error("Can't seek ZIP file");
400 info = parse_member(*zipstream);
401 if(info.central_directory_special)
402 break;
403 offsets[info.filename] = info.header_offset;
404 } while(1);
405 } catch(...) {
406 delete zipstream;
407 delete refcnt;
408 throw;
412 zip_writer::zip_writer(const std::string& zipfile, unsigned _compression) throw(std::bad_alloc, std::runtime_error)
414 compression = _compression;
415 zipfile_path = zipfile;
416 temp_path = zipfile + ".tmp";
417 zipstream = new std::ofstream(temp_path.c_str(), std::ios::binary);
418 if(!*zipstream)
419 throw std::runtime_error("Can't open zipfile '" + temp_path + "' for writing");
420 committed = false;
421 system_stream = true;
424 zip_writer::zip_writer(std::ostream& stream, unsigned _compression) throw(std::bad_alloc, std::runtime_error)
426 compression = _compression;
427 zipstream = &stream;
428 committed = false;
429 system_stream = false;
432 zip_writer::~zip_writer() throw()
434 if(!committed && system_stream)
435 remove(temp_path.c_str());
436 if(system_stream)
437 delete zipstream;
440 void zip_writer::commit() throw(std::bad_alloc, std::logic_error, std::runtime_error)
442 if(committed)
443 throw std::logic_error("Can't commit twice");
444 if(open_file != "")
445 throw std::logic_error("Can't commit with file open");
446 std::vector<unsigned char> directory_entry;
447 uint32_t cdirsize = 0;
448 uint32_t cdiroff = zipstream->tellp();
449 if(cdiroff == (uint32_t)-1)
450 throw std::runtime_error("Can't read current ZIP stream position");
451 for(auto i : files) {
452 cdirsize += (46 + i.first.length());
453 directory_entry.resize(46 + i.first.length());
454 write32(&directory_entry[0], 0x02014b50);
455 write16(&directory_entry[4], 3);
456 write16(&directory_entry[6], 20);
457 write16(&directory_entry[8], 0);
458 write16(&directory_entry[10], compression ? 8 : 0);
459 write16(&directory_entry[12], 0);
460 write16(&directory_entry[14], 10273);
461 write32(&directory_entry[16], i.second.crc);
462 write32(&directory_entry[20], i.second.compressed_size);
463 write32(&directory_entry[24], i.second.uncompressed_size);
464 write16(&directory_entry[28], i.first.length());
465 write16(&directory_entry[30], 0);
466 write16(&directory_entry[32], 0);
467 write16(&directory_entry[34], 0);
468 write16(&directory_entry[36], 0);
469 write32(&directory_entry[38], 0);
470 write32(&directory_entry[42], i.second.offset);
471 memcpy(&directory_entry[46], i.first.c_str(), i.first.length());
472 zipstream->write(reinterpret_cast<char*>(&directory_entry[0]), directory_entry.size());
473 if(!*zipstream)
474 throw std::runtime_error("Failed to write central directory entry to output file");
476 directory_entry.resize(22);
477 write32(&directory_entry[0], 0x06054b50);
478 write16(&directory_entry[4], 0);
479 write16(&directory_entry[6], 0);
480 write16(&directory_entry[8], files.size());
481 write16(&directory_entry[10], files.size());
482 write32(&directory_entry[12], cdirsize);
483 write32(&directory_entry[16], cdiroff);
484 write16(&directory_entry[20], 0);
485 zipstream->write(reinterpret_cast<char*>(&directory_entry[0]), directory_entry.size());
486 if(!*zipstream)
487 throw std::runtime_error("Failed to write central directory end marker to output file");
488 if(system_stream) {
489 dynamic_cast<std::ofstream*>(zipstream)->close();
490 std::string backup = zipfile_path + ".backup";
491 rename_file_overwrite(zipfile_path.c_str(), backup.c_str());
492 if(rename_file_overwrite(temp_path.c_str(), zipfile_path.c_str()) < 0)
493 throw std::runtime_error("Can't rename '" + temp_path + "' -> '" + zipfile_path + "'");
495 committed = true;
498 std::ostream& zip_writer::create_file(const std::string& name) throw(std::bad_alloc, std::logic_error,
499 std::runtime_error)
501 if(open_file != "")
502 throw std::logic_error("Can't open file with file open");
503 if(name == "")
504 throw std::runtime_error("Bad member name");
505 current_compressed_file.resize(0);
506 s = new boost::iostreams::filtering_ostream();
507 s->push(size_and_crc_filter(4096));
508 if(compression) {
509 boost::iostreams::zlib_params params;
510 params.noheader = true;
511 s->push(boost::iostreams::zlib_compressor(params));
513 s->push(vector_output(current_compressed_file));
514 open_file = name;
515 return *s;
518 void zip_writer::close_file() throw(std::bad_alloc, std::logic_error, std::runtime_error)
520 if(open_file == "")
521 throw std::logic_error("Can't close file with no file open");
522 uint32_t ucs, cs, crc32;
523 boost::iostreams::close(*s);
524 size_and_crc_filter& f = *s->component<size_and_crc_filter>(0);
525 cs = current_compressed_file.size();
526 ucs = f.size();
527 crc32 = f.crc32();
528 delete s;
530 base_offset = zipstream->tellp();
531 if(base_offset == (uint32_t)-1)
532 throw std::runtime_error("Can't read current ZIP stream position");
533 unsigned char header[30];
534 memset(header, 0, 30);
535 write32(header, 0x04034b50);
536 header[4] = 20;
537 header[6] = 0;
538 header[8] = compression ? 8 : 0;
539 header[12] = 33;
540 header[13] = 40;
541 write32(header + 14, crc32);
542 write32(header + 18, cs);
543 write32(header + 22, ucs);
544 write16(header + 26, open_file.length());
545 zipstream->write(reinterpret_cast<char*>(header), 30);
546 zipstream->write(open_file.c_str(), open_file.length());
547 zipstream->write(&current_compressed_file[0], current_compressed_file.size());
548 if(!*zipstream)
549 throw std::runtime_error("Can't write member to ZIP file");
550 current_compressed_file.resize(0);
551 zip_file_info info;
552 info.crc = crc32;
553 info.uncompressed_size = ucs;
554 info.compressed_size = cs;
555 info.offset = base_offset;
556 files[open_file] = info;
557 open_file = "";
560 namespace
562 #if defined(_WIN32) || defined(_WIN64) || defined(TEST_WIN32_CODE)
563 const char* path_splitters = "\\/";
564 bool drives_allowed = true;
565 #else
566 //Assume Unix(-like) system.
567 const char* path_splitters = "/";
568 bool drives_allowed = false;
569 #endif
571 const char* str_index(const char* str, int ch)
573 for(size_t i = 0; str[i]; i++)
574 if(str[i] == ch)
575 return str + i;
576 return NULL;
579 bool ispathsep(char ch)
581 return (str_index(path_splitters, static_cast<int>(static_cast<unsigned char>(ch))) != NULL);
584 bool isroot(const std::string& path)
586 if(path.length() == 1 && ispathsep(path[0]))
587 return true;
588 if(!drives_allowed)
589 //NO more cases for this.
590 return false;
591 if(path.length() == 3 && ((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] < '<')) &&
592 path[1] == ':' && ispathsep(path[2]))
593 return true;
594 //UNC.
595 if(path.length() <= 3 || !ispathsep(path[0]) || !ispathsep(path[1]) ||
596 !ispathsep(path[path.length() - 1]))
597 return false;
598 return (path.find_first_of(path_splitters, 2) == path.length() - 1);
601 std::string walk(const std::string& path, const std::string& component)
603 if(component == "" || component == ".")
604 //Current directory.
605 return path;
606 else if(component == "..") {
607 //Parent directory.
608 if(path == "" || isroot(path))
609 throw std::runtime_error("Can't rise to containing directory");
610 std::string _path = path;
611 size_t split = _path.find_last_of(path_splitters);
612 if(split < _path.length())
613 return _path.substr(0, split);
614 else
615 return "";
616 } else if(path == "" || ispathsep(path[path.length() - 1]))
617 return path + component;
618 else
619 return path + "/" + component;
622 std::string combine_path(const std::string& _name, const std::string& _referencing_path)
624 std::string name = _name;
625 std::string referencing_path = _referencing_path;
626 size_t x = referencing_path.find_last_of(path_splitters);
627 if(x < referencing_path.length())
628 referencing_path = referencing_path.substr(0, x);
629 else
630 return name;
631 //Check if name is absolute.
632 if(ispathsep(name[0]))
633 return name;
634 if(drives_allowed && name.length() >= 3 && ((name[0] >= 'A' && name[0] <= 'Z') || (name[0] >= 'a' &&
635 name[0] <= 'z')) && name[1] == ':' && ispathsep(name[2]))
636 return name;
637 //It is not absolute.
638 std::string path = referencing_path;
639 size_t pindex = 0;
640 while(true) {
641 size_t split = name.find_first_of(path_splitters, pindex);
642 std::string c;
643 if(split < name.length())
644 c = name.substr(pindex, split - pindex);
645 else
646 c = name.substr(pindex);
647 path = walk(path, c);
648 if(split < name.length())
649 pindex = split + 1;
650 else
651 break;
653 //If path becomes empty, assume it means current directory.
654 if(path == "")
655 path = ".";
656 return path;
660 std::string resolve_file_relative(const std::string& name, const std::string& referencing_path) throw(std::bad_alloc,
661 std::runtime_error)
663 return combine_path(name, referencing_path);
666 std::istream& open_file_relative(const std::string& name, const std::string& referencing_path) throw(std::bad_alloc,
667 std::runtime_error)
669 std::string path_to_open = combine_path(name, referencing_path);
670 std::string final_path = path_to_open;
671 //Try to open this from the main OS filesystem.
672 if(is_regular_file(path_to_open)) {
673 std::ifstream* i = new std::ifstream(path_to_open.c_str(), std::ios::binary);
674 if(i->is_open()) {
675 return *i;
677 delete i;
679 //Didn't succeed. Try to open as ZIP archive.
680 std::string membername;
681 while(true) {
682 size_t split = path_to_open.find_last_of("/");
683 if(split >= path_to_open.length())
684 throw std::runtime_error("Can't open '" + final_path + "'");
685 //Move a component to member name.
686 if(membername != "")
687 membername = path_to_open.substr(split + 1) + "/" + membername;
688 else
689 membername = path_to_open.substr(split + 1);
690 path_to_open = path_to_open.substr(0, split);
691 if(is_regular_file(path_to_open))
692 try {
693 zip_reader r(path_to_open);
694 return r[membername];
695 } catch(std::bad_alloc& e) {
696 throw;
697 } catch(std::runtime_error& e) {
702 std::vector<char> read_file_relative(const std::string& name, const std::string& referencing_path) throw(std::bad_alloc,
703 std::runtime_error)
705 std::vector<char> out;
706 std::istream& s = open_file_relative(name, referencing_path);
707 boost::iostreams::back_insert_device<std::vector<char>> rd(out);
708 boost::iostreams::copy(s, rd);
709 delete &s;
710 return out;
713 bool file_exists_zip(const std::string& name) throw(std::bad_alloc)
715 std::string path_to_open = name;
716 std::string final_path = path_to_open;
717 if(is_regular_file(path_to_open))
718 return true;
719 //Didn't succeed. Try to open as ZIP archive.
720 std::string membername;
721 while(true) {
722 size_t split = path_to_open.find_last_of("/");
723 if(split >= path_to_open.length())
724 return false;
725 //Move a component to member name.
726 if(membername != "")
727 membername = path_to_open.substr(split + 1) + "/" + membername;
728 else
729 membername = path_to_open.substr(split + 1);
730 path_to_open = path_to_open.substr(0, split);
731 if(is_regular_file(path_to_open))
732 try {
733 zip_reader r(path_to_open);
734 return r.has_member(membername);
735 } catch(std::bad_alloc& e) {
736 throw;
737 } catch(std::runtime_error& e) {
740 return false;