src/streams.h

   1 // Copyright (c) 2009-2010 Satoshi Nakamoto
   2 // Copyright (c) 2009-2015 The Bitcoin Core developers
   3 // Distributed under the MIT software license, see the accompanying
   4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #ifndef BITCOIN_STREAMS_H
   7 #define BITCOIN_STREAMS_H
   8
   9 #include "support/allocators/zeroafterfree.h"
  10 #include "serialize.h"
  11
  12 #include <algorithm>
  13 #include <assert.h>
  14 #include <ios>
  15 #include <limits>
  16 #include <map>
  17 #include <set>
  18 #include <stdint.h>
  19 #include <stdio.h>
  20 #include <string>
  21 #include <string.h>
  22 #include <utility>
  23 #include <vector>
  24
  25 template<typename Stream>
  26 class OverrideStream
  27 {
  28     Stream* stream;
  29 public:
  30     const int nType;
  31     const int nVersion;
  32
  33     OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {}
  34
  35     template<typename T>
  36     OverrideStream<Stream>& operator<<(const T& obj)
  37     {
  38         // Serialize to this stream
  39         ::Serialize(*this->stream, obj, nType, nVersion);
  40         return (*this);
  41     }
  42
  43     template<typename T>
  44     OverrideStream<Stream>& operator>>(T& obj)
  45     {
  46         // Unserialize from this stream
  47         ::Unserialize(*this->stream, obj, nType, nVersion);
  48         return (*this);
  49     }
  50 };
  51
  52 template<typename S>
  53 OverrideStream<S> WithOrVersion(S* s, int nVersionFlag)
  54 {
  55     return OverrideStream<S>(s, s->GetType(), s->GetVersion() | nVersionFlag);
  56 }
  57
  58 /** Double ended buffer combining vector and stream-like interfaces.
  59  *
  60  * >> and << read and write unformatted data using the above serialization templates.
  61  * Fills with data in linear time; some stringstream implementations take N^2 time.
  62  */
  63 class CDataStream
  64 {
  65 protected:
  66     typedef CSerializeData vector_type;
  67     vector_type vch;
  68     unsigned int nReadPos;
  69 public:
  70     int nType;
  71     int nVersion;
  72
  73     typedef vector_type::allocator_type   allocator_type;
  74     typedef vector_type::size_type        size_type;
  75     typedef vector_type::difference_type  difference_type;
  76     typedef vector_type::reference        reference;
  77     typedef vector_type::const_reference  const_reference;
  78     typedef vector_type::value_type       value_type;
  79     typedef vector_type::iterator         iterator;
  80     typedef vector_type::const_iterator   const_iterator;
  81     typedef vector_type::reverse_iterator reverse_iterator;
  82
  83     explicit CDataStream(int nTypeIn, int nVersionIn)
  84     {
  85         Init(nTypeIn, nVersionIn);
  86     }
  87
  88     CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
  89     {
  90         Init(nTypeIn, nVersionIn);
  91     }
  92
  93 #if !defined(_MSC_VER) || _MSC_VER >= 1300
  94     CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend)
  95     {
  96         Init(nTypeIn, nVersionIn);
  97     }
  98 #endif
  99
 100     CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 101     {
 102         Init(nTypeIn, nVersionIn);
 103     }
 104
 105     CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 106     {
 107         Init(nTypeIn, nVersionIn);
 108     }
 109
 110     CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end())
 111     {
 112         Init(nTypeIn, nVersionIn);
 113     }
 114
 115     template <typename... Args>
 116     CDataStream(int nTypeIn, int nVersionIn, Args&&... args)
 117     {
 118         Init(nTypeIn, nVersionIn);
 119         ::SerializeMany(*this, nType, nVersion, std::forward<Args>(args)...);
 120     }
 121
 122     void Init(int nTypeIn, int nVersionIn)
 123     {
 124         nReadPos = 0;
 125         nType = nTypeIn;
 126         nVersion = nVersionIn;
 127     }
 128
 129     CDataStream& operator+=(const CDataStream& b)
 130     {
 131         vch.insert(vch.end(), b.begin(), b.end());
 132         return *this;
 133     }
 134
 135     friend CDataStream operator+(const CDataStream& a, const CDataStream& b)
 136     {
 137         CDataStream ret = a;
 138         ret += b;
 139         return (ret);
 140     }
 141
 142     std::string str() const
 143     {
 144         return (std::string(begin(), end()));
 145     }
 146
 147
 148     //
 149     // Vector subset
 150     //
 151     const_iterator begin() const                     { return vch.begin() + nReadPos; }
 152     iterator begin()                                 { return vch.begin() + nReadPos; }
 153     const_iterator end() const                       { return vch.end(); }
 154     iterator end()                                   { return vch.end(); }
 155     size_type size() const                           { return vch.size() - nReadPos; }
 156     bool empty() const                               { return vch.size() == nReadPos; }
 157     void resize(size_type n, value_type c=0)         { vch.resize(n + nReadPos, c); }
 158     void reserve(size_type n)                        { vch.reserve(n + nReadPos); }
 159     const_reference operator[](size_type pos) const  { return vch[pos + nReadPos]; }
 160     reference operator[](size_type pos)              { return vch[pos + nReadPos]; }
 161     void clear()                                     { vch.clear(); nReadPos = 0; }
 162     iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); }
 163     void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); }
 164
 165     void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last)
 166     {
 167         assert(last - first >= 0);
 168         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 169         {
 170             // special case for inserting at the front when there's room
 171             nReadPos -= (last - first);
 172             memcpy(&vch[nReadPos], &first[0], last - first);
 173         }
 174         else
 175             vch.insert(it, first, last);
 176     }
 177
 178 #if !defined(_MSC_VER) || _MSC_VER >= 1300
 179     void insert(iterator it, const char* first, const char* last)
 180     {
 181         assert(last - first >= 0);
 182         if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos)
 183         {
 184             // special case for inserting at the front when there's room
 185             nReadPos -= (last - first);
 186             memcpy(&vch[nReadPos], &first[0], last - first);
 187         }
 188         else
 189             vch.insert(it, first, last);
 190     }
 191 #endif
 192
 193     iterator erase(iterator it)
 194     {
 195         if (it == vch.begin() + nReadPos)
 196         {
 197             // special case for erasing from the front
 198             if (++nReadPos >= vch.size())
 199             {
 200                 // whenever we reach the end, we take the opportunity to clear the buffer
 201                 nReadPos = 0;
 202                 return vch.erase(vch.begin(), vch.end());
 203             }
 204             return vch.begin() + nReadPos;
 205         }
 206         else
 207             return vch.erase(it);
 208     }
 209
 210     iterator erase(iterator first, iterator last)
 211     {
 212         if (first == vch.begin() + nReadPos)
 213         {
 214             // special case for erasing from the front
 215             if (last == vch.end())
 216             {
 217                 nReadPos = 0;
 218                 return vch.erase(vch.begin(), vch.end());
 219             }
 220             else
 221             {
 222                 nReadPos = (last - vch.begin());
 223                 return last;
 224             }
 225         }
 226         else
 227             return vch.erase(first, last);
 228     }
 229
 230     inline void Compact()
 231     {
 232         vch.erase(vch.begin(), vch.begin() + nReadPos);
 233         nReadPos = 0;
 234     }
 235
 236     bool Rewind(size_type n)
 237     {
 238         // Rewind by n characters if the buffer hasn't been compacted yet
 239         if (n > nReadPos)
 240             return false;
 241         nReadPos -= n;
 242         return true;
 243     }
 244
 245
 246     //
 247     // Stream subset
 248     //
 249     bool eof() const             { return size() == 0; }
 250     CDataStream* rdbuf()         { return this; }
 251     int in_avail()               { return size(); }
 252
 253     void SetType(int n)          { nType = n; }
 254     int GetType()                { return nType; }
 255     void SetVersion(int n)       { nVersion = n; }
 256     int GetVersion()             { return nVersion; }
 257
 258     CDataStream& read(char* pch, size_t nSize)
 259     {
 260         // Read from the beginning of the buffer
 261         unsigned int nReadPosNext = nReadPos + nSize;
 262         if (nReadPosNext >= vch.size())
 263         {
 264             if (nReadPosNext > vch.size())
 265             {
 266                 throw std::ios_base::failure("CDataStream::read(): end of data");
 267             }
 268             memcpy(pch, &vch[nReadPos], nSize);
 269             nReadPos = 0;
 270             vch.clear();
 271             return (*this);
 272         }
 273         memcpy(pch, &vch[nReadPos], nSize);
 274         nReadPos = nReadPosNext;
 275         return (*this);
 276     }
 277
 278     CDataStream& ignore(int nSize)
 279     {
 280         // Ignore from the beginning of the buffer
 281         if (nSize < 0) {
 282             throw std::ios_base::failure("CDataStream::ignore(): nSize negative");
 283         }
 284         unsigned int nReadPosNext = nReadPos + nSize;
 285         if (nReadPosNext >= vch.size())
 286         {
 287             if (nReadPosNext > vch.size())
 288                 throw std::ios_base::failure("CDataStream::ignore(): end of data");
 289             nReadPos = 0;
 290             vch.clear();
 291             return (*this);
 292         }
 293         nReadPos = nReadPosNext;
 294         return (*this);
 295     }
 296
 297     CDataStream& write(const char* pch, size_t nSize)
 298     {
 299         // Write to the end of the buffer
 300         vch.insert(vch.end(), pch, pch + nSize);
 301         return (*this);
 302     }
 303
 304     template<typename Stream>
 305     void Serialize(Stream& s, int nType, int nVersion) const
 306     {
 307         // Special case: stream << stream concatenates like stream += stream
 308         if (!vch.empty())
 309             s.write((char*)&vch[0], vch.size() * sizeof(vch[0]));
 310     }
 311
 312     template<typename T>
 313     unsigned int GetSerializeSize(const T& obj)
 314     {
 315         // Tells the size of the object if serialized to this stream
 316         return ::GetSerializeSize(obj, nType, nVersion);
 317     }
 318
 319     template<typename T>
 320     CDataStream& operator<<(const T& obj)
 321     {
 322         // Serialize to this stream
 323         ::Serialize(*this, obj, nType, nVersion);
 324         return (*this);
 325     }
 326
 327     template<typename T>
 328     CDataStream& operator>>(T& obj)
 329     {
 330         // Unserialize from this stream
 331         ::Unserialize(*this, obj, nType, nVersion);
 332         return (*this);
 333     }
 334
 335     void GetAndClear(CSerializeData &data) {
 336         data.insert(data.end(), begin(), end());
 337         clear();
 338     }
 339
 340     /**
 341      * XOR the contents of this stream with a certain key.
 342      *
 343      * @param[in] key    The key used to XOR the data in this stream.
 344      */
 345     void Xor(const std::vector<unsigned char>& key)
 346     {
 347         if (key.size() == 0) {
 348             return;
 349         }
 350
 351         for (size_type i = 0, j = 0; i != size(); i++) {
 352             vch[i] ^= key[j++];
 353
 354             // This potentially acts on very many bytes of data, so it's
 355             // important that we calculate `j`, i.e. the `key` index in this
 356             // way instead of doing a %, which would effectively be a division
 357             // for each byte Xor'd -- much slower than need be.
 358             if (j == key.size())
 359                 j = 0;
 360         }
 361     }
 362 };
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373 /** Non-refcounted RAII wrapper for FILE*
 374  *
 375  * Will automatically close the file when it goes out of scope if not null.
 376  * If you're returning the file pointer, return file.release().
 377  * If you need to close the file early, use file.fclose() instead of fclose(file).
 378  */
 379 class CAutoFile
 380 {
 381 private:
 382     // Disallow copies
 383     CAutoFile(const CAutoFile&);
 384     CAutoFile& operator=(const CAutoFile&);
 385
 386     int nType;
 387     int nVersion;
 388
 389     FILE* file;
 390
 391 public:
 392     CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn)
 393     {
 394         file = filenew;
 395         nType = nTypeIn;
 396         nVersion = nVersionIn;
 397     }
 398
 399     ~CAutoFile()
 400     {
 401         fclose();
 402     }
 403
 404     void fclose()
 405     {
 406         if (file) {
 407             ::fclose(file);
 408             file = NULL;
 409         }
 410     }
 411
 412     /** Get wrapped FILE* with transfer of ownership.
 413      * @note This will invalidate the CAutoFile object, and makes it the responsibility of the caller
 414      * of this function to clean up the returned FILE*.
 415      */
 416     FILE* release()             { FILE* ret = file; file = NULL; return ret; }
 417
 418     /** Get wrapped FILE* without transfer of ownership.
 419      * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the
 420      * CAutoFile outlives use of the passed pointer.
 421      */
 422     FILE* Get() const           { return file; }
 423
 424     /** Return true if the wrapped FILE* is NULL, false otherwise.
 425      */
 426     bool IsNull() const         { return (file == NULL); }
 427
 428     //
 429     // Stream subset
 430     //
 431     void SetType(int n)          { nType = n; }
 432     int GetType()                { return nType; }
 433     void SetVersion(int n)       { nVersion = n; }
 434     int GetVersion()             { return nVersion; }
 435
 436     CAutoFile& read(char* pch, size_t nSize)
 437     {
 438         if (!file)
 439             throw std::ios_base::failure("CAutoFile::read: file handle is NULL");
 440         if (fread(pch, 1, nSize, file) != nSize)
 441             throw std::ios_base::failure(feof(file) ? "CAutoFile::read: end of file" : "CAutoFile::read: fread failed");
 442         return (*this);
 443     }
 444
 445     CAutoFile& ignore(size_t nSize)
 446     {
 447         if (!file)
 448             throw std::ios_base::failure("CAutoFile::ignore: file handle is NULL");
 449         unsigned char data[4096];
 450         while (nSize > 0) {
 451             size_t nNow = std::min<size_t>(nSize, sizeof(data));
 452             if (fread(data, 1, nNow, file) != nNow)
 453                 throw std::ios_base::failure(feof(file) ? "CAutoFile::ignore: end of file" : "CAutoFile::read: fread failed");
 454             nSize -= nNow;
 455         }
 456         return (*this);
 457     }
 458
 459     CAutoFile& write(const char* pch, size_t nSize)
 460     {
 461         if (!file)
 462             throw std::ios_base::failure("CAutoFile::write: file handle is NULL");
 463         if (fwrite(pch, 1, nSize, file) != nSize)
 464             throw std::ios_base::failure("CAutoFile::write: write failed");
 465         return (*this);
 466     }
 467
 468     template<typename T>
 469     unsigned int GetSerializeSize(const T& obj)
 470     {
 471         // Tells the size of the object if serialized to this stream
 472         return ::GetSerializeSize(obj, nType, nVersion);
 473     }
 474
 475     template<typename T>
 476     CAutoFile& operator<<(const T& obj)
 477     {
 478         // Serialize to this stream
 479         if (!file)
 480             throw std::ios_base::failure("CAutoFile::operator<<: file handle is NULL");
 481         ::Serialize(*this, obj, nType, nVersion);
 482         return (*this);
 483     }
 484
 485     template<typename T>
 486     CAutoFile& operator>>(T& obj)
 487     {
 488         // Unserialize from this stream
 489         if (!file)
 490             throw std::ios_base::failure("CAutoFile::operator>>: file handle is NULL");
 491         ::Unserialize(*this, obj, nType, nVersion);
 492         return (*this);
 493     }
 494 };
 495
 496 /** Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to
 497  *  deserialize from. It guarantees the ability to rewind a given number of bytes.
 498  *
 499  *  Will automatically close the file when it goes out of scope if not null.
 500  *  If you need to close the file early, use file.fclose() instead of fclose(file).
 501  */
 502 class CBufferedFile
 503 {
 504 private:
 505     // Disallow copies
 506     CBufferedFile(const CBufferedFile&);
 507     CBufferedFile& operator=(const CBufferedFile&);
 508
 509     int nType;
 510     int nVersion;
 511
 512     FILE *src;            // source file
 513     uint64_t nSrcPos;     // how many bytes have been read from source
 514     uint64_t nReadPos;    // how many bytes have been read from this
 515     uint64_t nReadLimit;  // up to which position we're allowed to read
 516     uint64_t nRewind;     // how many bytes we guarantee to rewind
 517     std::vector<char> vchBuf; // the buffer
 518
 519 protected:
 520     // read data from the source to fill the buffer
 521     bool Fill() {
 522         unsigned int pos = nSrcPos % vchBuf.size();
 523         unsigned int readNow = vchBuf.size() - pos;
 524         unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind;
 525         if (nAvail < readNow)
 526             readNow = nAvail;
 527         if (readNow == 0)
 528             return false;
 529         size_t read = fread((void*)&vchBuf[pos], 1, readNow, src);
 530         if (read == 0) {
 531             throw std::ios_base::failure(feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed");
 532         } else {
 533             nSrcPos += read;
 534             return true;
 535         }
 536     }
 537
 538 public:
 539     CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) :
 540         nSrcPos(0), nReadPos(0), nReadLimit((uint64_t)(-1)), nRewind(nRewindIn), vchBuf(nBufSize, 0)
 541     {
 542         src = fileIn;
 543         nType = nTypeIn;
 544         nVersion = nVersionIn;
 545     }
 546
 547     ~CBufferedFile()
 548     {
 549         fclose();
 550     }
 551
 552     void fclose()
 553     {
 554         if (src) {
 555             ::fclose(src);
 556             src = NULL;
 557         }
 558     }
 559
 560     // check whether we're at the end of the source file
 561     bool eof() const {
 562         return nReadPos == nSrcPos && feof(src);
 563     }
 564
 565     // read a number of bytes
 566     CBufferedFile& read(char *pch, size_t nSize) {
 567         if (nSize + nReadPos > nReadLimit)
 568             throw std::ios_base::failure("Read attempted past buffer limit");
 569         if (nSize + nRewind > vchBuf.size())
 570             throw std::ios_base::failure("Read larger than buffer size");
 571         while (nSize > 0) {
 572             if (nReadPos == nSrcPos)
 573                 Fill();
 574             unsigned int pos = nReadPos % vchBuf.size();
 575             size_t nNow = nSize;
 576             if (nNow + pos > vchBuf.size())
 577                 nNow = vchBuf.size() - pos;
 578             if (nNow + nReadPos > nSrcPos)
 579                 nNow = nSrcPos - nReadPos;
 580             memcpy(pch, &vchBuf[pos], nNow);
 581             nReadPos += nNow;
 582             pch += nNow;
 583             nSize -= nNow;
 584         }
 585         return (*this);
 586     }
 587
 588     // return the current reading position
 589     uint64_t GetPos() {
 590         return nReadPos;
 591     }
 592
 593     // rewind to a given reading position
 594     bool SetPos(uint64_t nPos) {
 595         nReadPos = nPos;
 596         if (nReadPos + nRewind < nSrcPos) {
 597             nReadPos = nSrcPos - nRewind;
 598             return false;
 599         } else if (nReadPos > nSrcPos) {
 600             nReadPos = nSrcPos;
 601             return false;
 602         } else {
 603             return true;
 604         }
 605     }
 606
 607     bool Seek(uint64_t nPos) {
 608         long nLongPos = nPos;
 609         if (nPos != (uint64_t)nLongPos)
 610             return false;
 611         if (fseek(src, nLongPos, SEEK_SET))
 612             return false;
 613         nLongPos = ftell(src);
 614         nSrcPos = nLongPos;
 615         nReadPos = nLongPos;
 616         return true;
 617     }
 618
 619     // prevent reading beyond a certain position
 620     // no argument removes the limit
 621     bool SetLimit(uint64_t nPos = (uint64_t)(-1)) {
 622         if (nPos < nReadPos)
 623             return false;
 624         nReadLimit = nPos;
 625         return true;
 626     }
 627
 628     template<typename T>
 629     CBufferedFile& operator>>(T& obj) {
 630         // Unserialize from this stream
 631         ::Unserialize(*this, obj, nType, nVersion);
 632         return (*this);
 633     }
 634
 635     // search for a given byte in the stream, and remain positioned on it
 636     void FindByte(char ch) {
 637         while (true) {
 638             if (nReadPos == nSrcPos)
 639                 Fill();
 640             if (vchBuf[nReadPos % vchBuf.size()] == ch)
 641                 break;
 642             nReadPos++;
 643         }
 644     }
 645 };
 646
 647 #endif // BITCOIN_STREAMS_H