src/streams.h

   1 // Copyright (c) 2009-2010 Satoshi Nakamoto
   2 // Copyright (c) 2009-2016 The Bitcoin Core developers
   3 // Distributed under the MIT software license, see the accompanying
   4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #ifndef BITCOIN_STREAMS_H
   7 #define BITCOIN_STREAMS_H
   8
   9 #include <support/allocators/zeroafterfree.h>
  10 #include <serialize.h>
  11
  12 #include <algorithm>
  13 #include <assert.h>
  14 #include <ios>
  15 #include <limits>
  16 #include <map>
  17 #include <set>
  18 #include <stdint.h>
  19 #include <stdio.h>
  20 #include <string>
  21 #include <string.h>
  22 #include <utility>
  23 #include <vector>
  24
  25 template<typename Stream>
  26 class OverrideStream
  27 {
  28     Stream* stream;
  29
  30     const int nType;
  31     const int nVersion;
  32
  33 public:
  34     OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {}
  35
  36     template<typename T>
  37     OverrideStream<Stream>& operator<<(const T& obj)
  38     {
  39         // Serialize to this stream
  40         ::Serialize(*this, obj);
  41         return (*this);
  42     }
  43
  44     template<typename T>
  45     OverrideStream<Stream>& operator>>(T& obj)
  46     {
  47         // Unserialize from this stream
  48         ::Unserialize(*this, obj);
  49         return (*this);
  50     }
  51
  52     void write(const char* pch, size_t nSize)
  53     {
  54         stream->write(pch, nSize);
  55     }
  56
  57     void read(char* pch, size_t nSize)
  58     {
  59         stream->read(pch, nSize);
  60     }
  61
  62     int GetVersion() const { return nVersion; }
  63     int GetType() const { return nType; }
  64 };
  65
  66 template<typename S>
  67 OverrideStream<S> WithOrVersion(S* s, int nVersionFlag)
  68 {
  69     return OverrideStream<S>(s, s->GetType(), s->GetVersion() | nVersionFlag);
  70 }
  71
  72 /* Minimal stream for overwriting and/or appending to an existing byte vector
  73  *
  74  * The referenced vector will grow as necessary
  75  */
  76 class CVectorWriter
  77 {
  78  public:
  79
  80 /*
  81  * @param[in]  nTypeIn Serialization Type
  82  * @param[in]  nVersionIn Serialization Version (including any flags)
  83  * @param[in]  vchDataIn  Referenced byte vector to overwrite/append
  84  * @param[in]  nPosIn Starting position. Vector index where writes should start. The vector will initially
  85  *                    grow as necessary to  max(nPosIn, vec.size()). So to append, use vec.size().
  86 */
  87     CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn) : nType(nTypeIn), nVersion(nVersionIn), vchData(vchDataIn), nPos(nPosIn)
  88     {
  89         if(nPos > vchData.size())
  90             vchData.resize(nPos);
  91     }
  92 /*
  93  * (other params same as above)
  94  * @param[in]  args  A list of items to serialize starting at nPosIn.
  95 */
  96     template <typename... Args>
  97     CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter(nTypeIn, nVersionIn, vchDataIn, nPosIn)
  98     {
  99         ::SerializeMany(*this, std::forward<Args>(args)...);
 100     }
 101     void write(const char* pch, size_t nSize)
 102     {
 103         assert(nPos <= vchData.size());
 104         size_t nOverwrite = std::min(nSize, vchData.size() - nPos);
 105         if (nOverwrite) {
 106             memcpy(vchData.data() + nPos, reinterpret_cast<const unsigned char*>(pch), nOverwrite);
 107         }
 108         if (nOverwrite < nSize) {
 109             vchData.insert(vchData.end(), reinterpret_cast<const unsigned char*>(pch) + nOverwrite, reinterpret_cast<const unsigned char*>(pch) + nSize);
 110         }
 111         nPos += nSize;
 112     }
 113     template<typename T>
 114     CVectorWriter& operator<<(const T& obj)
 115     {
 116         // Serialize to this stream
 117         ::Serialize(*this, obj);
 118         return (*this);
 119     }
 120     int GetVersion() const
 121     {
 122         return nVersion;
 123     }
 124     int GetType() const
 125     {
 126         return nType;
 127     }
 128     void seek(size_t nSize)
 129     {
 130         nPos += nSize;
 131         if(nPos > vchData.size())
 132             vchData.resize(nPos);
 133     }
 134 private:
 135     const int nType;
 136     const int nVersion;
 137     std::vector<unsigned char>& vchData;
 138     size_t nPos;
 139 };
 140
 141 /** Double ended buffer combining vector and stream-like interfaces.
 142  *
 143  * >> and << read and write unformatted data using the above serialization templates.
 144  * Fills with data in linear time; some stringstream implementations take N^2 time.
 145  */
 146 class CDataStream
 147 {
 148 protected:
 149     typedef CSerializeData vector_type;
 150     vector_type vch;
 151     unsigned int nReadPos;
 152
 153     int nType;
 154     int nVersion;
 155 public:
 156
 157     typedef vector_type::allocator_type   allocator_type;
 158     typedef vector_type::size_type        size_type;
 159     typedef vector_type::difference_type  difference_type;
 160     typedef vector_type::reference        reference;
 161     typedef vector_type::const_reference  const_reference;
 162     typedef vector_type::value_type       value_type;
 163     typedef vector_type::iterator         iterator;
 164     typedef vector_type::const_iterator   const_iterator;
 165     typedef vector_type::reverse_iterator reverse_iterator;
 166
 167     explicit CDataStream(int nTypeIn, int nVersionIn)
 168     {
 169         Init(nTypeIn, nVersionIn);
 170     }
 171
 172     CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 173     {
 174         Init(nTypeIn, nVersionIn);
 175     }
 176
 177     CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 178     {
 179         Init(nTypeIn, nVersionIn);
 180     }
 181
 182     CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 183     {
 184         Init(nTypeIn, nVersionIn);
 185     }
 186
 187     CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 188     {
 189         Init(nTypeIn, nVersionIn);
 190     }
 191
 192     CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 193     {
 194         Init(nTypeIn, nVersionIn);
 195     }
 196
 197     template <typename... Args>
 198     CDataStream(int nTypeIn, int nVersionIn, Args&&... args)
 199     {
 200         Init(nTypeIn, nVersionIn);
 201         ::SerializeMany(*this, std::forward<Args>(args)...);
 202     }
 203
 204     void Init(int nTypeIn, int nVersionIn)
 205     {
 206         nReadPos = 0;
 207         nType = nTypeIn;
 208         nVersion = nVersionIn;
 209     }
 210
 211     CDataStream& operator+=(const CDataStream& b)
 212     {
 213         vch.insert(vch.end(), b.begin(), b.end());
 214         return *this;
 215     }
 216
 217     friend CDataStream operator+(const CDataStream& a, const CDataStream& b)
 218     {
 219         CDataStream ret = a;
 220         ret += b;
 221         return (ret);
 222     }
 223
 224     std::string str() const
 225     {
 226         return (std::string(begin(), end()));
 227     }
 228
 229
 230     //
 231     // Vector subset
 232     //
 233     const_iterator begin() const                     { return vch.begin() + nReadPos; }
 234     iterator begin()                                 { return vch.begin() + nReadPos; }
 235     const_iterator end() const                       { return vch.end(); }
 236     iterator end()                                   { return vch.end(); }
 237     size_type size() const                           { return vch.size() - nReadPos; }
 238     bool empty() const                               { return vch.size() == nReadPos; }
 239     void resize(size_type n, value_type c=0)         { vch.resize(n + nReadPos, c); }
 240     void reserve(size_type n)                        { vch.reserve(n + nReadPos); }
 241     const_reference operator[](size_type pos) const  { return vch[pos + nReadPos]; }
 242     reference operator[](size_type pos)              { return vch[pos + nReadPos]; }
 243     void clear()                                     { vch.clear(); nReadPos = 0; }
 244     iterator insert(iterator it, const char x=char()) { return vch.insert(it, x); }
 245     void insert(iterator it, size_type n, const char x) { vch.insert(it, n, x); }
 246     value_type* data()                               { return vch.data() + nReadPos; }
 247     const value_type* data() const                   { return vch.data() + nReadPos; }
 248
 249     void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last)
 250     {
 251         if (last == first) return;
 252         assert(last - first > 0);
 253         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 254         {
 255             // special case for inserting at the front when there's room
 256             nReadPos -= (last - first);
 257             memcpy(&vch[nReadPos], &first[0], last - first);
 258         }
 259         else
 260             vch.insert(it, first, last);
 261     }
 262
 263     void insert(iterator it, const char* first, const char* last)
 264     {
 265         if (last == first) return;
 266         assert(last - first > 0);
 267         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 268         {
 269             // special case for inserting at the front when there's room
 270             nReadPos -= (last - first);
 271             memcpy(&vch[nReadPos], &first[0], last - first);
 272         }
 273         else
 274             vch.insert(it, first, last);
 275     }
 276
 277     iterator erase(iterator it)
 278     {
 279         if (it == vch.begin() + nReadPos)
 280         {
 281             // special case for erasing from the front
 282             if (++nReadPos >= vch.size())
 283             {
 284                 // whenever we reach the end, we take the opportunity to clear the buffer
 285                 nReadPos = 0;
 286                 return vch.erase(vch.begin(), vch.end());
 287             }
 288             return vch.begin() + nReadPos;
 289         }
 290         else
 291             return vch.erase(it);
 292     }
 293
 294     iterator erase(iterator first, iterator last)
 295     {
 296         if (first == vch.begin() + nReadPos)
 297         {
 298             // special case for erasing from the front
 299             if (last == vch.end())
 300             {
 301                 nReadPos = 0;
 302                 return vch.erase(vch.begin(), vch.end());
 303             }
 304             else
 305             {
 306                 nReadPos = (last - vch.begin());
 307                 return last;
 308             }
 309         }
 310         else
 311             return vch.erase(first, last);
 312     }
 313
 314     inline void Compact()
 315     {
 316         vch.erase(vch.begin(), vch.begin() + nReadPos);
 317         nReadPos = 0;
 318     }
 319
 320     bool Rewind(size_type n)
 321     {
 322         // Rewind by n characters if the buffer hasn't been compacted yet
 323         if (n > nReadPos)
 324             return false;
 325         nReadPos -= n;
 326         return true;
 327     }
 328
 329
 330     //
 331     // Stream subset
 332     //
 333     bool eof() const             { return size() == 0; }
 334     CDataStream* rdbuf()         { return this; }
 335     int in_avail() const         { return size(); }
 336
 337     void SetType(int n)          { nType = n; }
 338     int GetType() const          { return nType; }
 339     void SetVersion(int n)       { nVersion = n; }
 340     int GetVersion() const       { return nVersion; }
 341
 342     void read(char* pch, size_t nSize)
 343     {
 344         if (nSize == 0) return;
 345
 346         // Read from the beginning of the buffer
 347         unsigned int nReadPosNext = nReadPos + nSize;
 348         if (nReadPosNext > vch.size()) {
 349             throw std::ios_base::failure("CDataStream::read(): end of data");
 350         }
 351         memcpy(pch, &vch[nReadPos], nSize);
 352         if (nReadPosNext == vch.size())
 353         {
 354             nReadPos = 0;
 355             vch.clear();
 356             return;
 357         }
 358         nReadPos = nReadPosNext;
 359     }
 360
 361     void ignore(int nSize)
 362     {
 363         // Ignore from the beginning of the buffer
 364         if (nSize < 0) {
 365             throw std::ios_base::failure("CDataStream::ignore(): nSize negative");
 366         }
 367         unsigned int nReadPosNext = nReadPos + nSize;
 368         if (nReadPosNext >= vch.size())
 369         {
 370             if (nReadPosNext > vch.size())
 371                 throw std::ios_base::failure("CDataStream::ignore(): end of data");
 372             nReadPos = 0;
 373             vch.clear();
 374             return;
 375         }
 376         nReadPos = nReadPosNext;
 377     }
 378
 379     void write(const char* pch, size_t nSize)
 380     {
 381         // Write to the end of the buffer
 382         vch.insert(vch.end(), pch, pch + nSize);
 383     }
 384
 385     template<typename Stream>
 386     void Serialize(Stream& s) const
 387     {
 388         // Special case: stream << stream concatenates like stream += stream
 389         if (!vch.empty())
 390             s.write((char*)vch.data(), vch.size() * sizeof(value_type));
 391     }
 392
 393     template<typename T>
 394     CDataStream& operator<<(const T& obj)
 395     {
 396         // Serialize to this stream
 397         ::Serialize(*this, obj);
 398         return (*this);
 399     }
 400
 401     template<typename T>
 402     CDataStream& operator>>(T& obj)
 403     {
 404         // Unserialize from this stream
 405         ::Unserialize(*this, obj);
 406         return (*this);
 407     }
 408
 409     void GetAndClear(CSerializeData &d) {
 410         d.insert(d.end(), begin(), end());
 411         clear();
 412     }
 413
 414     /**
 415      * XOR the contents of this stream with a certain key.
 416      *
 417      * @param[in] key    The key used to XOR the data in this stream.
 418      */
 419     void Xor(const std::vector<unsigned char>& key)
 420     {
 421         if (key.size() == 0) {
 422             return;
 423         }
 424
 425         for (size_type i = 0, j = 0; i != size(); i++) {
 426             vch[i] ^= key[j++];
 427
 428             // This potentially acts on very many bytes of data, so it's
 429             // important that we calculate `j`, i.e. the `key` index in this
 430             // way instead of doing a %, which would effectively be a division
 431             // for each byte Xor'd -- much slower than need be.
 432             if (j == key.size())
 433                 j = 0;
 434         }
 435     }
 436 };
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447 /** Non-refcounted RAII wrapper for FILE*
 448  *
 449  * Will automatically close the file when it goes out of scope if not null.
 450  * If you're returning the file pointer, return file.release().
 451  * If you need to close the file early, use file.fclose() instead of fclose(file).
 452  */
 453 class CAutoFile
 454 {
 455 private:
 456     const int nType;
 457     const int nVersion;
 458
 459     FILE* file;
 460
 461 public:
 462     CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn)
 463     {
 464         file = filenew;
 465     }
 466
 467     ~CAutoFile()
 468     {
 469         fclose();
 470     }
 471
 472     // Disallow copies
 473     CAutoFile(const CAutoFile&) = delete;
 474     CAutoFile& operator=(const CAutoFile&) = delete;
 475
 476     void fclose()
 477     {
 478         if (file) {
 479             ::fclose(file);
 480             file = nullptr;
 481         }
 482     }
 483
 484     /** Get wrapped FILE* with transfer of ownership.
 485      * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller
 486      * of this function to clean up the returned FILE*.
 487      */
 488     FILE* release()             { FILE* ret = file; file = nullptr; return ret; }
 489
 490     /** Get wrapped FILE* without transfer of ownership.
 491      * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the
 492      * CAutoFile outlives use of the passed pointer.
 493      */
 494     FILE* Get() const           { return file; }
 495
 496     /** Return true if the wrapped FILE* is nullptr, false otherwise.
 497      */
 498     bool IsNull() const         { return (file == nullptr); }
 499
 500     //
 501     // Stream subset
 502     //
 503     int GetType() const          { return nType; }
 504     int GetVersion() const       { return nVersion; }
 505
 506     void read(char* pch, size_t nSize)
 507     {
 508         if (!file)
 509             throw std::ios_base::failure("CAutoFile::read: file handle is nullptr");
 510         if (fread(pch, 1, nSize, file) != nSize)
 511             throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed");
 512     }
 513
 514     void ignore(size_t nSize)
 515     {
 516         if (!file)
 517             throw std::ios_base::failure("CAutoFile::ignore: file handle is nullptr");
 518         unsigned char data[4096];
 519         while (nSize > 0) {
 520             size_t nNow = std::min<size_t>(nSize, sizeof(data));
 521             if (fread(data, 1, nNow, file) != nNow)
 522                 throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed");
 523             nSize -= nNow;
 524         }
 525     }
 526
 527     void write(const char* pch, size_t nSize)
 528     {
 529         if (!file)
 530             throw std::ios_base::failure("CAutoFile::write: file handle is nullptr");
 531         if (fwrite(pch, 1, nSize, file) != nSize)
 532             throw std::ios_base::failure("CAutoFile::write: write failed");
 533     }
 534
 535     template<typename T>
 536     CAutoFile& operator<<(const T& obj)
 537     {
 538         // Serialize to this stream
 539         if (!file)
 540             throw std::ios_base::failure("CAutoFile::operator<<: file handle is nullptr");
 541         ::Serialize(*this, obj);
 542         return (*this);
 543     }
 544
 545     template<typename T>
 546     CAutoFile& operator>>(T& obj)
 547     {
 548         // Unserialize from this stream
 549         if (!file)
 550             throw std::ios_base::failure("CAutoFile::operator>>: file handle is nullptr");
 551         ::Unserialize(*this, obj);
 552         return (*this);
 553     }
 554 };
 555
 556 /** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to
 557  *  deserialize from. It guarantees the ability to rewind a given number of bytes.
 558  *
 559  *  Will automatically close the file when it goes out of scope if not null.
 560  *  If you need to close the file early, use file.fclose() instead of fclose(file).
 561  */
 562 class CBufferedFile
 563 {
 564 private:
 565     const int nType;
 566     const int nVersion;
 567
 568     FILE *src;            // source file
 569     uint64_t nSrcPos;     // how many bytes have been read from source
 570     uint64_t nReadPos;    // how many bytes have been read from this
 571     uint64_t nReadLimit;  // up to which position we're allowed to read
 572     uint64_t nRewind;     // how many bytes we guarantee to rewind
 573     std::vector<char> vchBuf; // the buffer
 574
 575 protected:
 576     // read data from the source to fill the buffer
 577     bool Fill() {
 578         unsigned int pos = nSrcPos % vchBuf.size();
 579         unsigned int readNow = vchBuf.size() - pos;
 580         unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
 581         if (nAvail < readNow)
 582             readNow = nAvail;
 583         if (readNow == 0)
 584             return false;
 585         size_t nBytes = fread((void*)&vchBuf[pos], 1, readNow, src);
 586         if (nBytes == 0) {
 587             throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed");
 588         } else {
 589             nSrcPos += nBytes;
 590             return true;
 591         }
 592     }
 593
 594 public:
 595     CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
 596         nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
 597     {
 598         src = fileIn;
 599     }
 600
 601     ~CBufferedFile()
 602     {
 603         fclose();
 604     }
 605
 606     // Disallow copies
 607     CBufferedFile(const CBufferedFile&) = delete;
 608     CBufferedFile& operator=(const CBufferedFile&) = delete;
 609
 610     int GetVersion() const { return nVersion; }
 611     int GetType() const { return nType; }
 612
 613     void fclose()
 614     {
 615         if (src) {
 616             ::fclose(src);
 617             src = nullptr;
 618         }
 619     }
 620
 621     // check whether we're at the end of the source file
 622     bool eof() const {
 623         return nReadPos == nSrcPos && feof(src);
 624     }
 625
 626     // read a number of bytes
 627     void read(char *pch, size_t nSize) {
 628         if (nSize + nReadPos > nReadLimit)
 629             throw std::ios_base::failure("Read attempted past buffer limit");
 630         if (nSize + nRewind > vchBuf.size())
 631             throw std::ios_base::failure("Read larger than buffer size");
 632         while (nSize > 0) {
 633             if (nReadPos == nSrcPos)
 634                 Fill();
 635             unsigned int pos = nReadPos % vchBuf.size();
 636             size_t nNow = nSize;
 637             if (nNow + pos > vchBuf.size())
 638                 nNow = vchBuf.size() - pos;
 639             if (nNow + nReadPos > nSrcPos)
 640                 nNow = nSrcPos - nReadPos;
 641             memcpy(pch, &vchBuf[pos], nNow);
 642             nReadPos += nNow;
 643             pch += nNow;
 644             nSize -= nNow;
 645         }
 646     }
 647
 648     // return the current reading position
 649     uint64_t GetPos() const {
 650         return nReadPos;
 651     }
 652
 653     // rewind to a given reading position
 654     bool SetPos(uint64_t nPos) {
 655         nReadPos = nPos;
 656         if (nReadPos + nRewind < nSrcPos) {
 657             nReadPos = nSrcPos - nRewind;
 658             return false;
 659         } else if (nReadPos > nSrcPos) {
 660             nReadPos = nSrcPos;
 661             return false;
 662         } else {
 663             return true;
 664         }
 665     }
 666
 667     bool Seek(uint64_t nPos) {
 668         long nLongPos = nPos;
 669         if (nPos != (uint64_t)nLongPos)
 670             return false;
 671         if (fseek(src, nLongPos, SEEK_SET))
 672             return false;
 673         nLongPos = ftell(src);
 674         nSrcPos = nLongPos;
 675         nReadPos = nLongPos;
 676         return true;
 677     }
 678
 679     // prevent reading beyond a certain position
 680     // no argument removes the limit
 681     bool SetLimit(uint64_t nPos = (uint64_t)(-1)) {
 682         if (nPos < nReadPos)
 683             return false;
 684         nReadLimit = nPos;
 685         return true;
 686     }
 687
 688     template<typename T>
 689     CBufferedFile& operator>>(T& obj) {
 690         // Unserialize from this stream
 691         ::Unserialize(*this, obj);
 692         return (*this);
 693     }
 694
 695     // search for a given byte in the stream, and remain positioned on it
 696     void FindByte(char ch) {
 697         while (true) {
 698             if (nReadPos == nSrcPos)
 699                 Fill();
 700             if (vchBuf[nReadPos % vchBuf.size()] == ch)
 701                 break;
 702             nReadPos++;
 703         }
 704     }
 705 };
 706
 707 #endif // BITCOIN_STREAMS_H