src/streams.h

   1 // Copyright (c) 2009-2010 Satoshi Nakamoto
   2 // Copyright (c) 2009-2015 The Bitcoin Core developers
   3 // Distributed under the MIT software license, see the accompanying
   4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #ifndef BITCOIN_STREAMS_H
   7 #define BITCOIN_STREAMS_H
   8
   9 #include "support/allocators/zeroafterfree.h"
  10 #include "serialize.h"
  11
  12 #include <algorithm>
  13 #include <assert.h>
  14 #include <ios>
  15 #include <limits>
  16 #include <map>
  17 #include <set>
  18 #include <stdint.h>
  19 #include <stdio.h>
  20 #include <string>
  21 #include <string.h>
  22 #include <utility>
  23 #include <vector>
  24
  25 template<typename Stream>
  26 class OverrideStream
  27 {
  28     Stream* stream;
  29
  30     const int nType;
  31     const int nVersion;
  32
  33 public:
  34     OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {}
  35
  36     template<typename T>
  37     OverrideStream<Stream>& operator<<(const T& obj)
  38     {
  39         // Serialize to this stream
  40         ::Serialize(*this, obj);
  41         return (*this);
  42     }
  43
  44     template<typename T>
  45     OverrideStream<Stream>& operator>>(T& obj)
  46     {
  47         // Unserialize from this stream
  48         ::Unserialize(*this, obj);
  49         return (*this);
  50     }
  51
  52     void write(const char* pch, size_t nSize)
  53     {
  54         stream->write(pch, nSize);
  55     }
  56
  57     void read(char* pch, size_t nSize)
  58     {
  59         stream->read(pch, nSize);
  60     }
  61
  62     int GetVersion() const { return nVersion; }
  63     int GetType() const { return nType; }
  64 };
  65
  66 template<typename S>
  67 OverrideStream<S> WithOrVersion(S* s, int nVersionFlag)
  68 {
  69     return OverrideStream<S>(s, s->GetType(), s->GetVersion() | nVersionFlag);
  70 }
  71
  72 /** Double ended buffer combining vector and stream-like interfaces.
  73  *
  74  * >> and << read and write unformatted data using the above serialization templates.
  75  * Fills with data in linear time; some stringstream implementations take N^2 time.
  76  */
  77 class CDataStream
  78 {
  79 protected:
  80     typedef CSerializeData vector_type;
  81     vector_type vch;
  82     unsigned int nReadPos;
  83
  84     int nType;
  85     int nVersion;
  86 public:
  87
  88     typedef vector_type::allocator_type   allocator_type;
  89     typedef vector_type::size_type        size_type;
  90     typedef vector_type::difference_type  difference_type;
  91     typedef vector_type::reference        reference;
  92     typedef vector_type::const_reference  const_reference;
  93     typedef vector_type::value_type       value_type;
  94     typedef vector_type::iterator         iterator;
  95     typedef vector_type::const_iterator   const_iterator;
  96     typedef vector_type::reverse_iterator reverse_iterator;
  97
  98     explicit CDataStream(int nTypeIn, int nVersionIn)
  99     {
 100         Init(nTypeIn, nVersionIn);
 101     }
 102
 103     CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 104     {
 105         Init(nTypeIn, nVersionIn);
 106     }
 107
 108 #if !defined(_MSC_VER) || _MSC_VER >= 1300
 109     CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
 110     {
 111         Init(nTypeIn, nVersionIn);
 112     }
 113 #endif
 114
 115     CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 116     {
 117         Init(nTypeIn, nVersionIn);
 118     }
 119
 120     CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 121     {
 122         Init(nTypeIn, nVersionIn);
 123     }
 124
 125     CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 126     {
 127         Init(nTypeIn, nVersionIn);
 128     }
 129
 130     template <typename... Args>
 131     CDataStream(int nTypeIn, int nVersionIn, Args&&... args)
 132     {
 133         Init(nTypeIn, nVersionIn);
 134         ::SerializeMany(*this, std::forward<Args>(args)...);
 135     }
 136
 137     void Init(int nTypeIn, int nVersionIn)
 138     {
 139         nReadPos = 0;
 140         nType = nTypeIn;
 141         nVersion = nVersionIn;
 142     }
 143
 144     CDataStream& operator+=(const CDataStream& b)
 145     {
 146         vch.insert(vch.end(), b.begin(), b.end());
 147         return *this;
 148     }
 149
 150     friend CDataStream operator+(const CDataStream& a, const CDataStream& b)
 151     {
 152         CDataStream ret = a;
 153         ret += b;
 154         return (ret);
 155     }
 156
 157     std::string str() const
 158     {
 159         return (std::string(begin(), end()));
 160     }
 161
 162
 163     //
 164     // Vector subset
 165     //
 166     const_iterator begin() const                     { return vch.begin() + nReadPos; }
 167     iterator begin()                                 { return vch.begin() + nReadPos; }
 168     const_iterator end() const                       { return vch.end(); }
 169     iterator end()                                   { return vch.end(); }
 170     size_type size() const                           { return vch.size() - nReadPos; }
 171     bool empty() const                               { return vch.size() == nReadPos; }
 172     void resize(size_type n, value_type c=0)         { vch.resize(n + nReadPos, c); }
 173     void reserve(size_type n)                        { vch.reserve(n + nReadPos); }
 174     const_reference operator[](size_type pos) const  { return vch[pos + nReadPos]; }
 175     reference operator[](size_type pos)              { return vch[pos + nReadPos]; }
 176     void clear()                                     { vch.clear(); nReadPos = 0; }
 177     iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); }
 178     void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); }
 179
 180     void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last)
 181     {
 182         assert(last - first >= 0);
 183         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 184         {
 185             // special case for inserting at the front when there's room
 186             nReadPos -= (last - first);
 187             memcpy(&vch[nReadPos], &first[0], last - first);
 188         }
 189         else
 190             vch.insert(it, first, last);
 191     }
 192
 193 #if !defined(_MSC_VER) || _MSC_VER >= 1300
 194     void insert(iterator it, const char* first, const char* last)
 195     {
 196         assert(last - first >= 0);
 197         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 198         {
 199             // special case for inserting at the front when there's room
 200             nReadPos -= (last - first);
 201             memcpy(&vch[nReadPos], &first[0], last - first);
 202         }
 203         else
 204             vch.insert(it, first, last);
 205     }
 206 #endif
 207
 208     iterator erase(iterator it)
 209     {
 210         if (it == vch.begin() + nReadPos)
 211         {
 212             // special case for erasing from the front
 213             if (++nReadPos >= vch.size())
 214             {
 215                 // whenever we reach the end, we take the opportunity to clear the buffer
 216                 nReadPos = 0;
 217                 return vch.erase(vch.begin(), vch.end());
 218             }
 219             return vch.begin() + nReadPos;
 220         }
 221         else
 222             return vch.erase(it);
 223     }
 224
 225     iterator erase(iterator first, iterator last)
 226     {
 227         if (first == vch.begin() + nReadPos)
 228         {
 229             // special case for erasing from the front
 230             if (last == vch.end())
 231             {
 232                 nReadPos = 0;
 233                 return vch.erase(vch.begin(), vch.end());
 234             }
 235             else
 236             {
 237                 nReadPos = (last - vch.begin());
 238                 return last;
 239             }
 240         }
 241         else
 242             return vch.erase(first, last);
 243     }
 244
 245     inline void Compact()
 246     {
 247         vch.erase(vch.begin(), vch.begin() + nReadPos);
 248         nReadPos = 0;
 249     }
 250
 251     bool Rewind(size_type n)
 252     {
 253         // Rewind by n characters if the buffer hasn't been compacted yet
 254         if (n > nReadPos)
 255             return false;
 256         nReadPos -= n;
 257         return true;
 258     }
 259
 260
 261     //
 262     // Stream subset
 263     //
 264     bool eof() const             { return size() == 0; }
 265     CDataStream* rdbuf()         { return this; }
 266     int in_avail()               { return size(); }
 267
 268     void SetType(int n)          { nType = n; }
 269     int GetType() const          { return nType; }
 270     void SetVersion(int n)       { nVersion = n; }
 271     int GetVersion() const       { return nVersion; }
 272
 273     void read(char* pch, size_t nSize)
 274     {
 275         // Read from the beginning of the buffer
 276         unsigned int nReadPosNext = nReadPos + nSize;
 277         if (nReadPosNext >= vch.size())
 278         {
 279             if (nReadPosNext > vch.size())
 280             {
 281                 throw std::ios_base::failure("CDataStream::read(): end of data");
 282             }
 283             memcpy(pch, &vch[nReadPos], nSize);
 284             nReadPos = 0;
 285             vch.clear();
 286             return;
 287         }
 288         memcpy(pch, &vch[nReadPos], nSize);
 289         nReadPos = nReadPosNext;
 290     }
 291
 292     void ignore(int nSize)
 293     {
 294         // Ignore from the beginning of the buffer
 295         if (nSize < 0) {
 296             throw std::ios_base::failure("CDataStream::ignore(): nSize negative");
 297         }
 298         unsigned int nReadPosNext = nReadPos + nSize;
 299         if (nReadPosNext >= vch.size())
 300         {
 301             if (nReadPosNext > vch.size())
 302                 throw std::ios_base::failure("CDataStream::ignore(): end of data");
 303             nReadPos = 0;
 304             vch.clear();
 305             return;
 306         }
 307         nReadPos = nReadPosNext;
 308     }
 309
 310     void write(const char* pch, size_t nSize)
 311     {
 312         // Write to the end of the buffer
 313         vch.insert(vch.end(), pch, pch + nSize);
 314     }
 315
 316     template<typename Stream>
 317     void Serialize(Stream& s) const
 318     {
 319         // Special case: stream << stream concatenates like stream += stream
 320         if (!vch.empty())
 321             s.write((char*)&vch[0], vch.size() * sizeof(vch[0]));
 322     }
 323
 324     template<typename T>
 325     CDataStream& operator<<(const T& obj)
 326     {
 327         // Serialize to this stream
 328         ::Serialize(*this, obj);
 329         return (*this);
 330     }
 331
 332     template<typename T>
 333     CDataStream& operator>>(T& obj)
 334     {
 335         // Unserialize from this stream
 336         ::Unserialize(*this, obj);
 337         return (*this);
 338     }
 339
 340     void GetAndClear(CSerializeData &data) {
 341         data.insert(data.end(), begin(), end());
 342         clear();
 343     }
 344
 345     /**
 346      * XOR the contents of this stream with a certain key.
 347      *
 348      * @param[in] key    The key used to XOR the data in this stream.
 349      */
 350     void Xor(const std::vector<unsigned char>& key)
 351     {
 352         if (key.size() == 0) {
 353             return;
 354         }
 355
 356         for (size_type i = 0, j = 0; i != size(); i++) {
 357             vch[i] ^= key[j++];
 358
 359             // This potentially acts on very many bytes of data, so it's
 360             // important that we calculate `j`, i.e. the `key` index in this
 361             // way instead of doing a %, which would effectively be a division
 362             // for each byte Xor'd -- much slower than need be.
 363             if (j == key.size())
 364                 j = 0;
 365         }
 366     }
 367 };
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378 /** Non-refcounted RAII wrapper for FILE*
 379  *
 380  * Will automatically close the file when it goes out of scope if not null.
 381  * If you're returning the file pointer, return file.release().
 382  * If you need to close the file early, use file.fclose() instead of fclose(file).
 383  */
 384 class CAutoFile
 385 {
 386 private:
 387     // Disallow copies
 388     CAutoFile(const CAutoFile&);
 389     CAutoFile& operator=(const CAutoFile&);
 390
 391     const int nType;
 392     const int nVersion;
 393
 394     FILE* file;
 395
 396 public:
 397     CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn)
 398     {
 399         file = filenew;
 400     }
 401
 402     ~CAutoFile()
 403     {
 404         fclose();
 405     }
 406
 407     void fclose()
 408     {
 409         if (file) {
 410             ::fclose(file);
 411             file = NULL;
 412         }
 413     }
 414
 415     /** Get wrapped FILE* with transfer of ownership.
 416      * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller
 417      * of this function to clean up the returned FILE*.
 418      */
 419     FILE* release()             { FILE* ret = file; file = NULL; return ret; }
 420
 421     /** Get wrapped FILE* without transfer of ownership.
 422      * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the
 423      * CAutoFile outlives use of the passed pointer.
 424      */
 425     FILE* Get() const           { return file; }
 426
 427     /** Return true if the wrapped FILE* is NULL, false otherwise.
 428      */
 429     bool IsNull() const         { return (file == NULL); }
 430
 431     //
 432     // Stream subset
 433     //
 434     int GetType() const          { return nType; }
 435     int GetVersion() const       { return nVersion; }
 436
 437     void read(char* pch, size_t nSize)
 438     {
 439         if (!file)
 440             throw std::ios_base::failure("CAutoFile::read: file handle is NULL");
 441         if (fread(pch, 1, nSize, file) != nSize)
 442             throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed");
 443     }
 444
 445     void ignore(size_t nSize)
 446     {
 447         if (!file)
 448             throw std::ios_base::failure("CAutoFile::ignore: file handle is NULL");
 449         unsigned char data[4096];
 450         while (nSize > 0) {
 451             size_t nNow = std::min<size_t>(nSize, sizeof(data));
 452             if (fread(data, 1, nNow, file) != nNow)
 453                 throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed");
 454             nSize -= nNow;
 455         }
 456     }
 457
 458     void write(const char* pch, size_t nSize)
 459     {
 460         if (!file)
 461             throw std::ios_base::failure("CAutoFile::write: file handle is NULL");
 462         if (fwrite(pch, 1, nSize, file) != nSize)
 463             throw std::ios_base::failure("CAutoFile::write: write failed");
 464     }
 465
 466     template<typename T>
 467     CAutoFile& operator<<(const T& obj)
 468     {
 469         // Serialize to this stream
 470         if (!file)
 471             throw std::ios_base::failure("CAutoFile::operator<<: file handle is NULL");
 472         ::Serialize(*this, obj);
 473         return (*this);
 474     }
 475
 476     template<typename T>
 477     CAutoFile& operator>>(T& obj)
 478     {
 479         // Unserialize from this stream
 480         if (!file)
 481             throw std::ios_base::failure("CAutoFile::operator>>: file handle is NULL");
 482         ::Unserialize(*this, obj);
 483         return (*this);
 484     }
 485 };
 486
 487 /** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to
 488  *  deserialize from. It guarantees the ability to rewind a given number of bytes.
 489  *
 490  *  Will automatically close the file when it goes out of scope if not null.
 491  *  If you need to close the file early, use file.fclose() instead of fclose(file).
 492  */
 493 class CBufferedFile
 494 {
 495 private:
 496     // Disallow copies
 497     CBufferedFile(const CBufferedFile&);
 498     CBufferedFile& operator=(const CBufferedFile&);
 499
 500     const int nType;
 501     const int nVersion;
 502
 503     FILE *src;            // source file
 504     uint64_t nSrcPos;     // how many bytes have been read from source
 505     uint64_t nReadPos;    // how many bytes have been read from this
 506     uint64_t nReadLimit;  // up to which position we're allowed to read
 507     uint64_t nRewind;     // how many bytes we guarantee to rewind
 508     std::vector<char> vchBuf; // the buffer
 509
 510 protected:
 511     // read data from the source to fill the buffer
 512     bool Fill() {
 513         unsigned int pos = nSrcPos % vchBuf.size();
 514         unsigned int readNow = vchBuf.size() - pos;
 515         unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
 516         if (nAvail < readNow)
 517             readNow = nAvail;
 518         if (readNow == 0)
 519             return false;
 520         size_t read = fread((void*)&vchBuf[pos], 1, readNow, src);
 521         if (read == 0) {
 522             throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed");
 523         } else {
 524             nSrcPos += read;
 525             return true;
 526         }
 527     }
 528
 529 public:
 530     CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
 531         nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
 532     {
 533         src = fileIn;
 534     }
 535
 536     ~CBufferedFile()
 537     {
 538         fclose();
 539     }
 540
 541     int GetVersion() const { return nVersion; }
 542     int GetType() const { return nType; }
 543
 544     void fclose()
 545     {
 546         if (src) {
 547             ::fclose(src);
 548             src = NULL;
 549         }
 550     }
 551
 552     // check whether we're at the end of the source file
 553     bool eof() const {
 554         return nReadPos == nSrcPos && feof(src);
 555     }
 556
 557     // read a number of bytes
 558     void read(char *pch, size_t nSize) {
 559         if (nSize + nReadPos > nReadLimit)
 560             throw std::ios_base::failure("Read attempted past buffer limit");
 561         if (nSize + nRewind > vchBuf.size())
 562             throw std::ios_base::failure("Read larger than buffer size");
 563         while (nSize > 0) {
 564             if (nReadPos == nSrcPos)
 565                 Fill();
 566             unsigned int pos = nReadPos % vchBuf.size();
 567             size_t nNow = nSize;
 568             if (nNow + pos > vchBuf.size())
 569                 nNow = vchBuf.size() - pos;
 570             if (nNow + nReadPos > nSrcPos)
 571                 nNow = nSrcPos - nReadPos;
 572             memcpy(pch, &vchBuf[pos], nNow);
 573             nReadPos += nNow;
 574             pch += nNow;
 575             nSize -= nNow;
 576         }
 577     }
 578
 579     // return the current reading position
 580     uint64_t GetPos() {
 581         return nReadPos;
 582     }
 583
 584     // rewind to a given reading position
 585     bool SetPos(uint64_t nPos) {
 586         nReadPos = nPos;
 587         if (nReadPos + nRewind < nSrcPos) {
 588             nReadPos = nSrcPos - nRewind;
 589             return false;
 590         } else if (nReadPos > nSrcPos) {
 591             nReadPos = nSrcPos;
 592             return false;
 593         } else {
 594             return true;
 595         }
 596     }
 597
 598     bool Seek(uint64_t nPos) {
 599         long nLongPos = nPos;
 600         if (nPos != (uint64_t)nLongPos)
 601             return false;
 602         if (fseek(src, nLongPos, SEEK_SET))
 603             return false;
 604         nLongPos = ftell(src);
 605         nSrcPos = nLongPos;
 606         nReadPos = nLongPos;
 607         return true;
 608     }
 609
 610     // prevent reading beyond a certain position
 611     // no argument removes the limit
 612     bool SetLimit(uint64_t nPos = (uint64_t)(-1)) {
 613         if (nPos < nReadPos)
 614             return false;
 615         nReadLimit = nPos;
 616         return true;
 617     }
 618
 619     template<typename T>
 620     CBufferedFile& operator>>(T& obj) {
 621         // Unserialize from this stream
 622         ::Unserialize(*this, obj);
 623         return (*this);
 624     }
 625
 626     // search for a given byte in the stream, and remain positioned on it
 627     void FindByte(char ch) {
 628         while (true) {
 629             if (nReadPos == nSrcPos)
 630                 Fill();
 631             if (vchBuf[nReadPos % vchBuf.size()] == ch)
 632                 break;
 633             nReadPos++;
 634         }
 635     }
 636 };
 637
 638 #endif // BITCOIN_STREAMS_H