src/streams.h

   1 // Copyright (c) 2009-2010 Satoshi Nakamoto
   2 // Copyright (c) 2009-2016 The Bitcoin Core developers
   3 // Distributed under the MIT software license, see the accompanying
   4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #ifndef BITCOIN_STREAMS_H
   7 #define BITCOIN_STREAMS_H
   8
   9 #include "support/allocators/zeroafterfree.h"
  10 #include "serialize.h"
  11
  12 #include <algorithm>
  13 #include <assert.h>
  14 #include <ios>
  15 #include <limits>
  16 #include <map>
  17 #include <set>
  18 #include <stdint.h>
  19 #include <stdio.h>
  20 #include <string>
  21 #include <string.h>
  22 #include <utility>
  23 #include <vector>
  24
  25 template<typename Stream>
  26 class OverrideStream
  27 {
  28     Stream* stream;
  29
  30     const int nType;
  31     const int nVersion;
  32
  33 public:
  34     OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {}
  35
  36     template<typename T>
  37     OverrideStream<Stream>& operator<<(const T& obj)
  38     {
  39         // Serialize to this stream
  40         ::Serialize(*this, obj);
  41         return (*this);
  42     }
  43
  44     template<typename T>
  45     OverrideStream<Stream>& operator>>(T& obj)
  46     {
  47         // Unserialize from this stream
  48         ::Unserialize(*this, obj);
  49         return (*this);
  50     }
  51
  52     void write(const char* pch, size_t nSize)
  53     {
  54         stream->write(pch, nSize);
  55     }
  56
  57     void read(char* pch, size_t nSize)
  58     {
  59         stream->read(pch, nSize);
  60     }
  61
  62     int GetVersion() const { return nVersion; }
  63     int GetType() const { return nType; }
  64 };
  65
  66 template<typename S>
  67 OverrideStream<S> WithOrVersion(S* s, int nVersionFlag)
  68 {
  69     return OverrideStream<S>(s, s->GetType(), s->GetVersion() | nVersionFlag);
  70 }
  71
  72 /* Minimal stream for overwriting and/or appending to an existing byte vector
  73  *
  74  * The referenced vector will grow as necessary
  75  */
  76 class CVectorWriter
  77 {
  78  public:
  79
  80 /*
  81  * @param[in]  nTypeIn Serialization Type
  82  * @param[in]  nVersionIn Serialization Version (including any flags)
  83  * @param[in]  vchDataIn  Referenced byte vector to overwrite/append
  84  * @param[in]  nPosIn Starting position. Vector index where writes should start. The vector will initially
  85  *                    grow as necessary to  max(index, vec.size()). So to append, use vec.size().
  86 */
  87     CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn) : nType(nTypeIn), nVersion(nVersionIn), vchData(vchDataIn), nPos(nPosIn)
  88     {
  89         if(nPos > vchData.size())
  90             vchData.resize(nPos);
  91     }
  92 /*
  93  * (other params same as above)
  94  * @param[in]  args  A list of items to serialize starting at nPos.
  95 */
  96     template <typename... Args>
  97     CVectorWriter(int nTypeIn, int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter(nTypeIn, nVersionIn, vchDataIn, nPosIn)
  98     {
  99         ::SerializeMany(*this, std::forward<Args>(args)...);
 100     }
 101     void write(const char* pch, size_t nSize)
 102     {
 103         assert(nPos <= vchData.size());
 104         size_t nOverwrite = std::min(nSize, vchData.size() - nPos);
 105         if (nOverwrite) {
 106             memcpy(vchData.data() + nPos, reinterpret_cast<const unsigned char*>(pch), nOverwrite);
 107         }
 108         if (nOverwrite < nSize) {
 109             vchData.insert(vchData.end(), reinterpret_cast<const unsigned char*>(pch) + nOverwrite, reinterpret_cast<const unsigned char*>(pch) + nSize);
 110         }
 111         nPos += nSize;
 112     }
 113     template<typename T>
 114     CVectorWriter& operator<<(const T& obj)
 115     {
 116         // Serialize to this stream
 117         ::Serialize(*this, obj);
 118         return (*this);
 119     }
 120     int GetVersion() const
 121     {
 122         return nVersion;
 123     }
 124     int GetType() const
 125     {
 126         return nType;
 127     }
 128     void seek(size_t nSize)
 129     {
 130         nPos += nSize;
 131         if(nPos > vchData.size())
 132             vchData.resize(nPos);
 133     }
 134 private:
 135     const int nType;
 136     const int nVersion;
 137     std::vector<unsigned char>& vchData;
 138     size_t nPos;
 139 };
 140
 141 /** Double ended buffer combining vector and stream-like interfaces.
 142  *
 143  * >> and << read and write unformatted data using the above serialization templates.
 144  * Fills with data in linear time; some stringstream implementations take N^2 time.
 145  */
 146 class CDataStream
 147 {
 148 protected:
 149     typedef CSerializeData vector_type;
 150     vector_type vch;
 151     unsigned int nReadPos;
 152
 153     int nType;
 154     int nVersion;
 155 public:
 156
 157     typedef vector_type::allocator_type   allocator_type;
 158     typedef vector_type::size_type        size_type;
 159     typedef vector_type::difference_type  difference_type;
 160     typedef vector_type::reference        reference;
 161     typedef vector_type::const_reference  const_reference;
 162     typedef vector_type::value_type       value_type;
 163     typedef vector_type::iterator         iterator;
 164     typedef vector_type::const_iterator   const_iterator;
 165     typedef vector_type::reverse_iterator reverse_iterator;
 166
 167     explicit CDataStream(int nTypeIn, int nVersionIn)
 168     {
 169         Init(nTypeIn, nVersionIn);
 170     }
 171
 172     CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 173     {
 174         Init(nTypeIn, nVersionIn);
 175     }
 176
 177     CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 178     {
 179         Init(nTypeIn, nVersionIn);
 180     }
 181
 182     CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 183     {
 184         Init(nTypeIn, nVersionIn);
 185     }
 186
 187     CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 188     {
 189         Init(nTypeIn, nVersionIn);
 190     }
 191
 192     CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 193     {
 194         Init(nTypeIn, nVersionIn);
 195     }
 196
 197     template <typename... Args>
 198     CDataStream(int nTypeIn, int nVersionIn, Args&&... args)
 199     {
 200         Init(nTypeIn, nVersionIn);
 201         ::SerializeMany(*this, std::forward<Args>(args)...);
 202     }
 203
 204     void Init(int nTypeIn, int nVersionIn)
 205     {
 206         nReadPos = 0;
 207         nType = nTypeIn;
 208         nVersion = nVersionIn;
 209     }
 210
 211     CDataStream& operator+=(const CDataStream& b)
 212     {
 213         vch.insert(vch.end(), b.begin(), b.end());
 214         return *this;
 215     }
 216
 217     friend CDataStream operator+(const CDataStream& a, const CDataStream& b)
 218     {
 219         CDataStream ret = a;
 220         ret += b;
 221         return (ret);
 222     }
 223
 224     std::string str() const
 225     {
 226         return (std::string(begin(), end()));
 227     }
 228
 229
 230     //
 231     // Vector subset
 232     //
 233     const_iterator begin() const                     { return vch.begin() + nReadPos; }
 234     iterator begin()                                 { return vch.begin() + nReadPos; }
 235     const_iterator end() const                       { return vch.end(); }
 236     iterator end()                                   { return vch.end(); }
 237     size_type size() const                           { return vch.size() - nReadPos; }
 238     bool empty() const                               { return vch.size() == nReadPos; }
 239     void resize(size_type n, value_type c=0)         { vch.resize(n + nReadPos, c); }
 240     void reserve(size_type n)                        { vch.reserve(n + nReadPos); }
 241     const_reference operator[](size_type pos) const  { return vch[pos + nReadPos]; }
 242     reference operator[](size_type pos)              { return vch[pos + nReadPos]; }
 243     void clear()                                     { vch.clear(); nReadPos = 0; }
 244     iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); }
 245     void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); }
 246     value_type* data()                               { return vch.data() + nReadPos; }
 247     const value_type* data() const                   { return vch.data() + nReadPos; }
 248
 249     void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last)
 250     {
 251         assert(last - first >= 0);
 252         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 253         {
 254             // special case for inserting at the front when there's room
 255             nReadPos -= (last - first);
 256             memcpy(&vch[nReadPos], &first[0], last - first);
 257         }
 258         else
 259             vch.insert(it, first, last);
 260     }
 261
 262     void insert(iterator it, const char* first, const char* last)
 263     {
 264         assert(last - first >= 0);
 265         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 266         {
 267             // special case for inserting at the front when there's room
 268             nReadPos -= (last - first);
 269             memcpy(&vch[nReadPos], &first[0], last - first);
 270         }
 271         else
 272             vch.insert(it, first, last);
 273     }
 274
 275     iterator erase(iterator it)
 276     {
 277         if (it == vch.begin() + nReadPos)
 278         {
 279             // special case for erasing from the front
 280             if (++nReadPos >= vch.size())
 281             {
 282                 // whenever we reach the end, we take the opportunity to clear the buffer
 283                 nReadPos = 0;
 284                 return vch.erase(vch.begin(), vch.end());
 285             }
 286             return vch.begin() + nReadPos;
 287         }
 288         else
 289             return vch.erase(it);
 290     }
 291
 292     iterator erase(iterator first, iterator last)
 293     {
 294         if (first == vch.begin() + nReadPos)
 295         {
 296             // special case for erasing from the front
 297             if (last == vch.end())
 298             {
 299                 nReadPos = 0;
 300                 return vch.erase(vch.begin(), vch.end());
 301             }
 302             else
 303             {
 304                 nReadPos = (last - vch.begin());
 305                 return last;
 306             }
 307         }
 308         else
 309             return vch.erase(first, last);
 310     }
 311
 312     inline void Compact()
 313     {
 314         vch.erase(vch.begin(), vch.begin() + nReadPos);
 315         nReadPos = 0;
 316     }
 317
 318     bool Rewind(size_type n)
 319     {
 320         // Rewind by n characters if the buffer hasn't been compacted yet
 321         if (n > nReadPos)
 322             return false;
 323         nReadPos -= n;
 324         return true;
 325     }
 326
 327
 328     //
 329     // Stream subset
 330     //
 331     bool eof() const             { return size() == 0; }
 332     CDataStream* rdbuf()         { return this; }
 333     int in_avail()               { return size(); }
 334
 335     void SetType(int n)          { nType = n; }
 336     int GetType() const          { return nType; }
 337     void SetVersion(int n)       { nVersion = n; }
 338     int GetVersion() const       { return nVersion; }
 339
 340     void read(char* pch, size_t nSize)
 341     {
 342         // Read from the beginning of the buffer
 343         unsigned int nReadPosNext = nReadPos + nSize;
 344         if (nReadPosNext >= vch.size())
 345         {
 346             if (nReadPosNext > vch.size())
 347             {
 348                 throw std::ios_base::failure("CDataStream::read(): end of data");
 349             }
 350             memcpy(pch, &vch[nReadPos], nSize);
 351             nReadPos = 0;
 352             vch.clear();
 353             return;
 354         }
 355         memcpy(pch, &vch[nReadPos], nSize);
 356         nReadPos = nReadPosNext;
 357     }
 358
 359     void ignore(int nSize)
 360     {
 361         // Ignore from the beginning of the buffer
 362         if (nSize < 0) {
 363             throw std::ios_base::failure("CDataStream::ignore(): nSize negative");
 364         }
 365         unsigned int nReadPosNext = nReadPos + nSize;
 366         if (nReadPosNext >= vch.size())
 367         {
 368             if (nReadPosNext > vch.size())
 369                 throw std::ios_base::failure("CDataStream::ignore(): end of data");
 370             nReadPos = 0;
 371             vch.clear();
 372             return;
 373         }
 374         nReadPos = nReadPosNext;
 375     }
 376
 377     void write(const char* pch, size_t nSize)
 378     {
 379         // Write to the end of the buffer
 380         vch.insert(vch.end(), pch, pch + nSize);
 381     }
 382
 383     template<typename Stream>
 384     void Serialize(Stream& s) const
 385     {
 386         // Special case: stream << stream concatenates like stream += stream
 387         if (!vch.empty())
 388             s.write((char*)&vch[0], vch.size() * sizeof(vch[0]));
 389     }
 390
 391     template<typename T>
 392     CDataStream& operator<<(const T& obj)
 393     {
 394         // Serialize to this stream
 395         ::Serialize(*this, obj);
 396         return (*this);
 397     }
 398
 399     template<typename T>
 400     CDataStream& operator>>(T& obj)
 401     {
 402         // Unserialize from this stream
 403         ::Unserialize(*this, obj);
 404         return (*this);
 405     }
 406
 407     void GetAndClear(CSerializeData &data) {
 408         data.insert(data.end(), begin(), end());
 409         clear();
 410     }
 411
 412     /**
 413      * XOR the contents of this stream with a certain key.
 414      *
 415      * @param[in] key    The key used to XOR the data in this stream.
 416      */
 417     void Xor(const std::vector<unsigned char>& key)
 418     {
 419         if (key.size() == 0) {
 420             return;
 421         }
 422
 423         for (size_type i = 0, j = 0; i != size(); i++) {
 424             vch[i] ^= key[j++];
 425
 426             // This potentially acts on very many bytes of data, so it's
 427             // important that we calculate `j`, i.e. the `key` index in this
 428             // way instead of doing a %, which would effectively be a division
 429             // for each byte Xor'd -- much slower than need be.
 430             if (j == key.size())
 431                 j = 0;
 432         }
 433     }
 434 };
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445 /** Non-refcounted RAII wrapper for FILE*
 446  *
 447  * Will automatically close the file when it goes out of scope if not null.
 448  * If you're returning the file pointer, return file.release().
 449  * If you need to close the file early, use file.fclose() instead of fclose(file).
 450  */
 451 class CAutoFile
 452 {
 453 private:
 454     // Disallow copies
 455     CAutoFile(const CAutoFile&);
 456     CAutoFile& operator=(const CAutoFile&);
 457
 458     const int nType;
 459     const int nVersion;
 460
 461     FILE* file;
 462
 463 public:
 464     CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn)
 465     {
 466         file = filenew;
 467     }
 468
 469     ~CAutoFile()
 470     {
 471         fclose();
 472     }
 473
 474     void fclose()
 475     {
 476         if (file) {
 477             ::fclose(file);
 478             file = NULL;
 479         }
 480     }
 481
 482     /** Get wrapped FILE* with transfer of ownership.
 483      * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller
 484      * of this function to clean up the returned FILE*.
 485      */
 486     FILE* release()             { FILE* ret = file; file = NULL; return ret; }
 487
 488     /** Get wrapped FILE* without transfer of ownership.
 489      * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the
 490      * CAutoFile outlives use of the passed pointer.
 491      */
 492     FILE* Get() const           { return file; }
 493
 494     /** Return true if the wrapped FILE* is NULL, false otherwise.
 495      */
 496     bool IsNull() const         { return (file == NULL); }
 497
 498     //
 499     // Stream subset
 500     //
 501     int GetType() const          { return nType; }
 502     int GetVersion() const       { return nVersion; }
 503
 504     void read(char* pch, size_t nSize)
 505     {
 506         if (!file)
 507             throw std::ios_base::failure("CAutoFile::read: file handle is NULL");
 508         if (fread(pch, 1, nSize, file) != nSize)
 509             throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed");
 510     }
 511
 512     void ignore(size_t nSize)
 513     {
 514         if (!file)
 515             throw std::ios_base::failure("CAutoFile::ignore: file handle is NULL");
 516         unsigned char data[4096];
 517         while (nSize > 0) {
 518             size_t nNow = std::min<size_t>(nSize, sizeof(data));
 519             if (fread(data, 1, nNow, file) != nNow)
 520                 throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed");
 521             nSize -= nNow;
 522         }
 523     }
 524
 525     void write(const char* pch, size_t nSize)
 526     {
 527         if (!file)
 528             throw std::ios_base::failure("CAutoFile::write: file handle is NULL");
 529         if (fwrite(pch, 1, nSize, file) != nSize)
 530             throw std::ios_base::failure("CAutoFile::write: write failed");
 531     }
 532
 533     template<typename T>
 534     CAutoFile& operator<<(const T& obj)
 535     {
 536         // Serialize to this stream
 537         if (!file)
 538             throw std::ios_base::failure("CAutoFile::operator<<: file handle is NULL");
 539         ::Serialize(*this, obj);
 540         return (*this);
 541     }
 542
 543     template<typename T>
 544     CAutoFile& operator>>(T& obj)
 545     {
 546         // Unserialize from this stream
 547         if (!file)
 548             throw std::ios_base::failure("CAutoFile::operator>>: file handle is NULL");
 549         ::Unserialize(*this, obj);
 550         return (*this);
 551     }
 552 };
 553
 554 /** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to
 555  *  deserialize from. It guarantees the ability to rewind a given number of bytes.
 556  *
 557  *  Will automatically close the file when it goes out of scope if not null.
 558  *  If you need to close the file early, use file.fclose() instead of fclose(file).
 559  */
 560 class CBufferedFile
 561 {
 562 private:
 563     // Disallow copies
 564     CBufferedFile(const CBufferedFile&);
 565     CBufferedFile& operator=(const CBufferedFile&);
 566
 567     const int nType;
 568     const int nVersion;
 569
 570     FILE *src;            // source file
 571     uint64_t nSrcPos;     // how many bytes have been read from source
 572     uint64_t nReadPos;    // how many bytes have been read from this
 573     uint64_t nReadLimit;  // up to which position we're allowed to read
 574     uint64_t nRewind;     // how many bytes we guarantee to rewind
 575     std::vector<char> vchBuf; // the buffer
 576
 577 protected:
 578     // read data from the source to fill the buffer
 579     bool Fill() {
 580         unsigned int pos = nSrcPos % vchBuf.size();
 581         unsigned int readNow = vchBuf.size() - pos;
 582         unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
 583         if (nAvail < readNow)
 584             readNow = nAvail;
 585         if (readNow == 0)
 586             return false;
 587         size_t read = fread((void*)&vchBuf[pos], 1, readNow, src);
 588         if (read == 0) {
 589             throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed");
 590         } else {
 591             nSrcPos += read;
 592             return true;
 593         }
 594     }
 595
 596 public:
 597     CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
 598         nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
 599     {
 600         src = fileIn;
 601     }
 602
 603     ~CBufferedFile()
 604     {
 605         fclose();
 606     }
 607
 608     int GetVersion() const { return nVersion; }
 609     int GetType() const { return nType; }
 610
 611     void fclose()
 612     {
 613         if (src) {
 614             ::fclose(src);
 615             src = NULL;
 616         }
 617     }
 618
 619     // check whether we're at the end of the source file
 620     bool eof() const {
 621         return nReadPos == nSrcPos && feof(src);
 622     }
 623
 624     // read a number of bytes
 625     void read(char *pch, size_t nSize) {
 626         if (nSize + nReadPos > nReadLimit)
 627             throw std::ios_base::failure("Read attempted past buffer limit");
 628         if (nSize + nRewind > vchBuf.size())
 629             throw std::ios_base::failure("Read larger than buffer size");
 630         while (nSize > 0) {
 631             if (nReadPos == nSrcPos)
 632                 Fill();
 633             unsigned int pos = nReadPos % vchBuf.size();
 634             size_t nNow = nSize;
 635             if (nNow + pos > vchBuf.size())
 636                 nNow = vchBuf.size() - pos;
 637             if (nNow + nReadPos > nSrcPos)
 638                 nNow = nSrcPos - nReadPos;
 639             memcpy(pch, &vchBuf[pos], nNow);
 640             nReadPos += nNow;
 641             pch += nNow;
 642             nSize -= nNow;
 643         }
 644     }
 645
 646     // return the current reading position
 647     uint64_t GetPos() {
 648         return nReadPos;
 649     }
 650
 651     // rewind to a given reading position
 652     bool SetPos(uint64_t nPos) {
 653         nReadPos = nPos;
 654         if (nReadPos + nRewind < nSrcPos) {
 655             nReadPos = nSrcPos - nRewind;
 656             return false;
 657         } else if (nReadPos > nSrcPos) {
 658             nReadPos = nSrcPos;
 659             return false;
 660         } else {
 661             return true;
 662         }
 663     }
 664
 665     bool Seek(uint64_t nPos) {
 666         long nLongPos = nPos;
 667         if (nPos != (uint64_t)nLongPos)
 668             return false;
 669         if (fseek(src, nLongPos, SEEK_SET))
 670             return false;
 671         nLongPos = ftell(src);
 672         nSrcPos = nLongPos;
 673         nReadPos = nLongPos;
 674         return true;
 675     }
 676
 677     // prevent reading beyond a certain position
 678     // no argument removes the limit
 679     bool SetLimit(uint64_t nPos = (uint64_t)(-1)) {
 680         if (nPos < nReadPos)
 681             return false;
 682         nReadLimit = nPos;
 683         return true;
 684     }
 685
 686     template<typename T>
 687     CBufferedFile& operator>>(T& obj) {
 688         // Unserialize from this stream
 689         ::Unserialize(*this, obj);
 690         return (*this);
 691     }
 692
 693     // search for a given byte in the stream, and remain positioned on it
 694     void FindByte(char ch) {
 695         while (true) {
 696             if (nReadPos == nSrcPos)
 697                 Fill();
 698             if (vchBuf[nReadPos % vchBuf.size()] == ch)
 699                 break;
 700             nReadPos++;
 701         }
 702     }
 703 };
 704
 705 #endif // BITCOIN_STREAMS_H