src/streams.h

   1 // Copyright (c) 2009-2010 Satoshi Nakamoto
   2 // Copyright (c) 2009-2016 The Bitcoin Core developers
   3 // Distributed under the MIT software license, see the accompanying
   4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #ifndef BITCOIN_STREAMS_H
   7 #define BITCOIN_STREAMS_H
   8
   9 #include "support/allocators/zeroafterfree.h"
  10 #include "serialize.h"
  11
  12 #include <algorithm>
  13 #include <assert.h>
  14 #include <ios>
  15 #include <limits>
  16 #include <map>
  17 #include <set>
  18 #include <stdint.h>
  19 #include <stdio.h>
  20 #include <string>
  21 #include <string.h>
  22 #include <utility>
  23 #include <vector>
  24
  25 template<typename Stream>
  26 class OverrideStream
  27 {
  28     Stream* stream;
  29
  30     const int nType;
  31     const int nVersion;
  32
  33 public:
  34     OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {}
  35
  36     template<typename T>
  37     OverrideStream<Stream>& operator<<(const T& obj)
  38     {
  39         // Serialize to this stream
  40         ::Serialize(*this, obj);
  41         return (*this);
  42     }
  43
  44     template<typename T>
  45     OverrideStream<Stream>& operator>>(T& obj)
  46     {
  47         // Unserialize from this stream
  48         ::Unserialize(*this, obj);
  49         return (*this);
  50     }
  51
  52     void write(const char* pch, size_t nSize)
  53     {
  54         stream->write(pch, nSize);
  55     }
  56
  57     void read(char* pch, size_t nSize)
  58     {
  59         stream->read(pch, nSize);
  60     }
  61
  62     int GetVersion() const { return nVersion; }
  63     int GetType() const { return nType; }
  64 };
  65
  66 template<typename S>
  67 OverrideStream<S> WithOrVersion(S* s, int nVersionFlag)
  68 {
  69     return OverrideStream<S>(s, s->GetType(), s->GetVersion() | nVersionFlag);
  70 }
  71
  72 /* Minimal stream for overwriting and/or appending to an existing byte vector
  73  *
  74  * The referenced vector will grow as necessary
  75  */
  76 class CVectorWriter
  77 {
  78  public:
  79
  80 /*
  81  * @param[in]  nTypeIn Serialization Type
  82  * @param[in]  nVersionIn Serialization Version (including any flags)
  83  * @param[in]  vchDataIn  Referenced byte vector to overwrite/append
  84  * @param[in]  nPosIn Starting position. Vector index where writes should start. The vector will initially
  85  *                    grow as necessary to  max(index, vec.size()). So to append, use vec.size().
  86 */
  87     CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn) : nType(nTypeIn), nVersion(nVersionIn), vchData(vchDataIn), nPos(nPosIn)
  88     {
  89         if(nPos > vchData.size())
  90             vchData.resize(nPos);
  91     }
  92 /*
  93  * (other params same as above)
  94  * @param[in]  args  A list of items to serialize starting at nPos.
  95 */
  96     template <typename... Args>
  97     CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter(nTypeIn, nVersionIn, vchDataIn, nPosIn)
  98     {
  99         ::SerializeMany(*this, std::forward<Args>(args)...);
 100     }
 101     void write(const char* pch, size_t nSize)
 102     {
 103         assert(nPos <= vchData.size());
 104         size_t nOverwrite = std::min(nSize, vchData.size() - nPos);
 105         if (nOverwrite) {
 106             memcpy(vchData.data() + nPos, reinterpret_cast<const unsigned char*>(pch), nOverwrite);
 107         }
 108         if (nOverwrite < nSize) {
 109             vchData.insert(vchData.end(), reinterpret_cast<const unsigned char*>(pch) + nOverwrite, reinterpret_cast<const unsigned char*>(pch) + nSize);
 110         }
 111         nPos += nSize;
 112     }
 113     template<typename T>
 114     CVectorWriter& operator<<(const T& obj)
 115     {
 116         // Serialize to this stream
 117         ::Serialize(*this, obj);
 118         return (*this);
 119     }
 120     int GetVersion() const
 121     {
 122         return nVersion;
 123     }
 124     int GetType() const
 125     {
 126         return nType;
 127     }
 128     void seek(size_t nSize)
 129     {
 130         nPos += nSize;
 131         if(nPos > vchData.size())
 132             vchData.resize(nPos);
 133     }
 134 private:
 135     const int nType;
 136     const int nVersion;
 137     std::vector<unsigned char>& vchData;
 138     size_t nPos;
 139 };
 140
 141 /** Double ended buffer combining vector and stream-like interfaces.
 142  *
 143  * >> and << read and write unformatted data using the above serialization templates.
 144  * Fills with data in linear time; some stringstream implementations take N^2 time.
 145  */
 146 class CDataStream
 147 {
 148 protected:
 149     typedef CSerializeData vector_type;
 150     vector_type vch;
 151     unsigned int nReadPos;
 152
 153     int nType;
 154     int nVersion;
 155 public:
 156
 157     typedef vector_type::allocator_type   allocator_type;
 158     typedef vector_type::size_type        size_type;
 159     typedef vector_type::difference_type  difference_type;
 160     typedef vector_type::reference        reference;
 161     typedef vector_type::const_reference  const_reference;
 162     typedef vector_type::value_type       value_type;
 163     typedef vector_type::iterator         iterator;
 164     typedef vector_type::const_iterator   const_iterator;
 165     typedef vector_type::reverse_iterator reverse_iterator;
 166
 167     explicit CDataStream(int nTypeIn, int nVersionIn)
 168     {
 169         Init(nTypeIn, nVersionIn);
 170     }
 171
 172     CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 173     {
 174         Init(nTypeIn, nVersionIn);
 175     }
 176
 177     CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 178     {
 179         Init(nTypeIn, nVersionIn);
 180     }
 181
 182     CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 183     {
 184         Init(nTypeIn, nVersionIn);
 185     }
 186
 187     CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 188     {
 189         Init(nTypeIn, nVersionIn);
 190     }
 191
 192     CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 193     {
 194         Init(nTypeIn, nVersionIn);
 195     }
 196
 197     template <typename... Args>
 198     CDataStream(int nTypeIn, int nVersionIn, Args&&... args)
 199     {
 200         Init(nTypeIn, nVersionIn);
 201         ::SerializeMany(*this, std::forward<Args>(args)...);
 202     }
 203
 204     void Init(int nTypeIn, int nVersionIn)
 205     {
 206         nReadPos = 0;
 207         nType = nTypeIn;
 208         nVersion = nVersionIn;
 209     }
 210
 211     CDataStream& operator+=(const CDataStream& b)
 212     {
 213         vch.insert(vch.end(), b.begin(), b.end());
 214         return *this;
 215     }
 216
 217     friend CDataStream operator+(const CDataStream& a, const CDataStream& b)
 218     {
 219         CDataStream ret = a;
 220         ret += b;
 221         return (ret);
 222     }
 223
 224     std::string str() const
 225     {
 226         return (std::string(begin(), end()));
 227     }
 228
 229
 230     //
 231     // Vector subset
 232     //
 233     const_iterator begin() const                     { return vch.begin() + nReadPos; }
 234     iterator begin()                                 { return vch.begin() + nReadPos; }
 235     const_iterator end() const                       { return vch.end(); }
 236     iterator end()                                   { return vch.end(); }
 237     size_type size() const                           { return vch.size() - nReadPos; }
 238     bool empty() const                               { return vch.size() == nReadPos; }
 239     void resize(size_type n, value_type c=0)         { vch.resize(n + nReadPos, c); }
 240     void reserve(size_type n)                        { vch.reserve(n + nReadPos); }
 241     const_reference operator[](size_type pos) const  { return vch[pos + nReadPos]; }
 242     reference operator[](size_type pos)              { return vch[pos + nReadPos]; }
 243     void clear()                                     { vch.clear(); nReadPos = 0; }
 244     iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); }
 245     void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); }
 246     value_type* data()                               { return vch.data() + nReadPos; }
 247     const value_type* data() const                   { return vch.data() + nReadPos; }
 248
 249     void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last)
 250     {
 251         if (last == first) return;
 252         assert(last - first > 0);
 253         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 254         {
 255             // special case for inserting at the front when there's room
 256             nReadPos -= (last - first);
 257             memcpy(&vch[nReadPos], &first[0], last - first);
 258         }
 259         else
 260             vch.insert(it, first, last);
 261     }
 262
 263     void insert(iterator it, const char* first, const char* last)
 264     {
 265         if (last == first) return;
 266         assert(last - first > 0);
 267         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 268         {
 269             // special case for inserting at the front when there's room
 270             nReadPos -= (last - first);
 271             memcpy(&vch[nReadPos], &first[0], last - first);
 272         }
 273         else
 274             vch.insert(it, first, last);
 275     }
 276
 277     iterator erase(iterator it)
 278     {
 279         if (it == vch.begin() + nReadPos)
 280         {
 281             // special case for erasing from the front
 282             if (++nReadPos >= vch.size())
 283             {
 284                 // whenever we reach the end, we take the opportunity to clear the buffer
 285                 nReadPos = 0;
 286                 return vch.erase(vch.begin(), vch.end());
 287             }
 288             return vch.begin() + nReadPos;
 289         }
 290         else
 291             return vch.erase(it);
 292     }
 293
 294     iterator erase(iterator first, iterator last)
 295     {
 296         if (first == vch.begin() + nReadPos)
 297         {
 298             // special case for erasing from the front
 299             if (last == vch.end())
 300             {
 301                 nReadPos = 0;
 302                 return vch.erase(vch.begin(), vch.end());
 303             }
 304             else
 305             {
 306                 nReadPos = (last - vch.begin());
 307                 return last;
 308             }
 309         }
 310         else
 311             return vch.erase(first, last);
 312     }
 313
 314     inline void Compact()
 315     {
 316         vch.erase(vch.begin(), vch.begin() + nReadPos);
 317         nReadPos = 0;
 318     }
 319
 320     bool Rewind(size_type n)
 321     {
 322         // Rewind by n characters if the buffer hasn't been compacted yet
 323         if (n > nReadPos)
 324             return false;
 325         nReadPos -= n;
 326         return true;
 327     }
 328
 329
 330     //
 331     // Stream subset
 332     //
 333     bool eof() const             { return size() == 0; }
 334     CDataStream* rdbuf()         { return this; }
 335     int in_avail()               { return size(); }
 336
 337     void SetType(int n)          { nType = n; }
 338     int GetType() const          { return nType; }
 339     void SetVersion(int n)       { nVersion = n; }
 340     int GetVersion() const       { return nVersion; }
 341
 342     void read(char* pch, size_t nSize)
 343     {
 344         if (nSize == 0) return;
 345
 346         // Read from the beginning of the buffer
 347         unsigned int nReadPosNext = nReadPos + nSize;
 348         if (nReadPosNext >= vch.size())
 349         {
 350             if (nReadPosNext > vch.size())
 351             {
 352                 throw std::ios_base::failure("CDataStream::read(): end of data");
 353             }
 354             memcpy(pch, &vch[nReadPos], nSize);
 355             nReadPos = 0;
 356             vch.clear();
 357             return;
 358         }
 359         memcpy(pch, &vch[nReadPos], nSize);
 360         nReadPos = nReadPosNext;
 361     }
 362
 363     void ignore(int nSize)
 364     {
 365         // Ignore from the beginning of the buffer
 366         if (nSize < 0) {
 367             throw std::ios_base::failure("CDataStream::ignore(): nSize negative");
 368         }
 369         unsigned int nReadPosNext = nReadPos + nSize;
 370         if (nReadPosNext >= vch.size())
 371         {
 372             if (nReadPosNext > vch.size())
 373                 throw std::ios_base::failure("CDataStream::ignore(): end of data");
 374             nReadPos = 0;
 375             vch.clear();
 376             return;
 377         }
 378         nReadPos = nReadPosNext;
 379     }
 380
 381     void write(const char* pch, size_t nSize)
 382     {
 383         // Write to the end of the buffer
 384         vch.insert(vch.end(), pch, pch + nSize);
 385     }
 386
 387     template<typename Stream>
 388     void Serialize(Stream& s) const
 389     {
 390         // Special case: stream << stream concatenates like stream += stream
 391         if (!vch.empty())
 392             s.write((char*)&vch[0], vch.size() * sizeof(vch[0]));
 393     }
 394
 395     template<typename T>
 396     CDataStream& operator<<(const T& obj)
 397     {
 398         // Serialize to this stream
 399         ::Serialize(*this, obj);
 400         return (*this);
 401     }
 402
 403     template<typename T>
 404     CDataStream& operator>>(T& obj)
 405     {
 406         // Unserialize from this stream
 407         ::Unserialize(*this, obj);
 408         return (*this);
 409     }
 410
 411     void GetAndClear(CSerializeData &d) {
 412         d.insert(d.end(), begin(), end());
 413         clear();
 414     }
 415
 416     /**
 417      * XOR the contents of this stream with a certain key.
 418      *
 419      * @param[in] key    The key used to XOR the data in this stream.
 420      */
 421     void Xor(const std::vector<unsigned char>& key)
 422     {
 423         if (key.size() == 0) {
 424             return;
 425         }
 426
 427         for (size_type i = 0, j = 0; i != size(); i++) {
 428             vch[i] ^= key[j++];
 429
 430             // This potentially acts on very many bytes of data, so it's
 431             // important that we calculate `j`, i.e. the `key` index in this
 432             // way instead of doing a %, which would effectively be a division
 433             // for each byte Xor'd -- much slower than need be.
 434             if (j == key.size())
 435                 j = 0;
 436         }
 437     }
 438 };
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449 /** Non-refcounted RAII wrapper for FILE*
 450  *
 451  * Will automatically close the file when it goes out of scope if not null.
 452  * If you're returning the file pointer, return file.release().
 453  * If you need to close the file early, use file.fclose() instead of fclose(file).
 454  */
 455 class CAutoFile
 456 {
 457 private:
 458     // Disallow copies
 459     CAutoFile(const CAutoFile&);
 460     CAutoFile& operator=(const CAutoFile&);
 461
 462     const int nType;
 463     const int nVersion;
 464
 465     FILE* file;
 466
 467 public:
 468     CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn)
 469     {
 470         file = filenew;
 471     }
 472
 473     ~CAutoFile()
 474     {
 475         fclose();
 476     }
 477
 478     void fclose()
 479     {
 480         if (file) {
 481             ::fclose(file);
 482             file = NULL;
 483         }
 484     }
 485
 486     /** Get wrapped FILE* with transfer of ownership.
 487      * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller
 488      * of this function to clean up the returned FILE*.
 489      */
 490     FILE* release()             { FILE* ret = file; file = NULL; return ret; }
 491
 492     /** Get wrapped FILE* without transfer of ownership.
 493      * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the
 494      * CAutoFile outlives use of the passed pointer.
 495      */
 496     FILE* Get() const           { return file; }
 497
 498     /** Return true if the wrapped FILE* is NULL, false otherwise.
 499      */
 500     bool IsNull() const         { return (file == NULL); }
 501
 502     //
 503     // Stream subset
 504     //
 505     int GetType() const          { return nType; }
 506     int GetVersion() const       { return nVersion; }
 507
 508     void read(char* pch, size_t nSize)
 509     {
 510         if (!file)
 511             throw std::ios_base::failure("CAutoFile::read: file handle is NULL");
 512         if (fread(pch, 1, nSize, file) != nSize)
 513             throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed");
 514     }
 515
 516     void ignore(size_t nSize)
 517     {
 518         if (!file)
 519             throw std::ios_base::failure("CAutoFile::ignore: file handle is NULL");
 520         unsigned char data[4096];
 521         while (nSize > 0) {
 522             size_t nNow = std::min<size_t>(nSize, sizeof(data));
 523             if (fread(data, 1, nNow, file) != nNow)
 524                 throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed");
 525             nSize -= nNow;
 526         }
 527     }
 528
 529     void write(const char* pch, size_t nSize)
 530     {
 531         if (!file)
 532             throw std::ios_base::failure("CAutoFile::write: file handle is NULL");
 533         if (fwrite(pch, 1, nSize, file) != nSize)
 534             throw std::ios_base::failure("CAutoFile::write: write failed");
 535     }
 536
 537     template<typename T>
 538     CAutoFile& operator<<(const T& obj)
 539     {
 540         // Serialize to this stream
 541         if (!file)
 542             throw std::ios_base::failure("CAutoFile::operator<<: file handle is NULL");
 543         ::Serialize(*this, obj);
 544         return (*this);
 545     }
 546
 547     template<typename T>
 548     CAutoFile& operator>>(T& obj)
 549     {
 550         // Unserialize from this stream
 551         if (!file)
 552             throw std::ios_base::failure("CAutoFile::operator>>: file handle is NULL");
 553         ::Unserialize(*this, obj);
 554         return (*this);
 555     }
 556 };
 557
 558 /** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to
 559  *  deserialize from. It guarantees the ability to rewind a given number of bytes.
 560  *
 561  *  Will automatically close the file when it goes out of scope if not null.
 562  *  If you need to close the file early, use file.fclose() instead of fclose(file).
 563  */
 564 class CBufferedFile
 565 {
 566 private:
 567     // Disallow copies
 568     CBufferedFile(const CBufferedFile&);
 569     CBufferedFile& operator=(const CBufferedFile&);
 570
 571     const int nType;
 572     const int nVersion;
 573
 574     FILE *src;            // source file
 575     uint64_t nSrcPos;     // how many bytes have been read from source
 576     uint64_t nReadPos;    // how many bytes have been read from this
 577     uint64_t nReadLimit;  // up to which position we're allowed to read
 578     uint64_t nRewind;     // how many bytes we guarantee to rewind
 579     std::vector<char> vchBuf; // the buffer
 580
 581 protected:
 582     // read data from the source to fill the buffer
 583     bool Fill() {
 584         unsigned int pos = nSrcPos % vchBuf.size();
 585         unsigned int readNow = vchBuf.size() - pos;
 586         unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
 587         if (nAvail < readNow)
 588             readNow = nAvail;
 589         if (readNow == 0)
 590             return false;
 591         size_t nBytes = fread((void*)&vchBuf[pos], 1, readNow, src);
 592         if (nBytes == 0) {
 593             throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed");
 594         } else {
 595             nSrcPos += nBytes;
 596             return true;
 597         }
 598     }
 599
 600 public:
 601     CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
 602         nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
 603     {
 604         src = fileIn;
 605     }
 606
 607     ~CBufferedFile()
 608     {
 609         fclose();
 610     }
 611
 612     int GetVersion() const { return nVersion; }
 613     int GetType() const { return nType; }
 614
 615     void fclose()
 616     {
 617         if (src) {
 618             ::fclose(src);
 619             src = NULL;
 620         }
 621     }
 622
 623     // check whether we're at the end of the source file
 624     bool eof() const {
 625         return nReadPos == nSrcPos && feof(src);
 626     }
 627
 628     // read a number of bytes
 629     void read(char *pch, size_t nSize) {
 630         if (nSize + nReadPos > nReadLimit)
 631             throw std::ios_base::failure("Read attempted past buffer limit");
 632         if (nSize + nRewind > vchBuf.size())
 633             throw std::ios_base::failure("Read larger than buffer size");
 634         while (nSize > 0) {
 635             if (nReadPos == nSrcPos)
 636                 Fill();
 637             unsigned int pos = nReadPos % vchBuf.size();
 638             size_t nNow = nSize;
 639             if (nNow + pos > vchBuf.size())
 640                 nNow = vchBuf.size() - pos;
 641             if (nNow + nReadPos > nSrcPos)
 642                 nNow = nSrcPos - nReadPos;
 643             memcpy(pch, &vchBuf[pos], nNow);
 644             nReadPos += nNow;
 645             pch += nNow;
 646             nSize -= nNow;
 647         }
 648     }
 649
 650     // return the current reading position
 651     uint64_t GetPos() {
 652         return nReadPos;
 653     }
 654
 655     // rewind to a given reading position
 656     bool SetPos(uint64_t nPos) {
 657         nReadPos = nPos;
 658         if (nReadPos + nRewind < nSrcPos) {
 659             nReadPos = nSrcPos - nRewind;
 660             return false;
 661         } else if (nReadPos > nSrcPos) {
 662             nReadPos = nSrcPos;
 663             return false;
 664         } else {
 665             return true;
 666         }
 667     }
 668
 669     bool Seek(uint64_t nPos) {
 670         long nLongPos = nPos;
 671         if (nPos != (uint64_t)nLongPos)
 672             return false;
 673         if (fseek(src, nLongPos, SEEK_SET))
 674             return false;
 675         nLongPos = ftell(src);
 676         nSrcPos = nLongPos;
 677         nReadPos = nLongPos;
 678         return true;
 679     }
 680
 681     // prevent reading beyond a certain position
 682     // no argument removes the limit
 683     bool SetLimit(uint64_t nPos = (uint64_t)(-1)) {
 684         if (nPos < nReadPos)
 685             return false;
 686         nReadLimit = nPos;
 687         return true;
 688     }
 689
 690     template<typename T>
 691     CBufferedFile& operator>>(T& obj) {
 692         // Unserialize from this stream
 693         ::Unserialize(*this, obj);
 694         return (*this);
 695     }
 696
 697     // search for a given byte in the stream, and remain positioned on it
 698     void FindByte(char ch) {
 699         while (true) {
 700             if (nReadPos == nSrcPos)
 701                 Fill();
 702             if (vchBuf[nReadPos % vchBuf.size()] == ch)
 703                 break;
 704             nReadPos++;
 705         }
 706     }
 707 };
 708
 709 #endif // BITCOIN_STREAMS_H