hphp/util/blob-encoder.h

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #pragma once
  18
  19 #include "hphp/util/compact-vector.h"
  20 #include "hphp/util/optional.h"
  21
  22 #include <folly/sorted_vector_types.h>
  23 #include <folly/Varint.h>
  24 #include <folly/container/F14Map.h>
  25 #include <folly/container/F14Set.h>
  26
  27 #include <filesystem>
  28 #include <set>
  29 #include <type_traits>
  30 #include <unordered_map>
  31 #include <vector>
  32
  33 /*
  34  * This module contains helpers for serializing and deserializing
  35  * metadata into binary blobs.
  36  *
  37  * Types may provide their own serialization logic by implementing a
  38  * single-argument member function template called `serde' (as in
  39  * "serialization/deserialization"), and pushing data it wants to
  40  * serialize into the parameter.  The function may take a set of
  41  * optional arguments but if your function require those arguments
  42  * they also have to be passed in when calling sd. Those members may
  43  * in turn have customized serialization behavior, or they may "bottom
  44  * out" to these helpers if they are basic-enough types.
  45  *
  46  * If you cannot add a serde member function to the type (for example,
  47  * for pointers), you can get the same effect by defining a
  48  * specialization of BlobEncoderHelper with the appropriate member(s).
  49  *
  50  * For example:
  51  *
  52  *    struct MyBlobbableStuff {
  53  *      int m_foo;
  54  *      std::vector<const StringData*> m_strList;
  55  *      OtherBlobbableData m_other;
  56  *
  57  *      template<class SerDe> void serde(SerDe& sd) {
  58  *        sd(m_foo)(m_strList)(m_other);
  59  *      }
  60  *    };
  61  *
  62  * The argument to SerDe will have a static const member called
  63  * `deserializing' which can be used to do custom behavior depending
  64  * on whether you're deserializing.  But generally the goal here
  65  * should be to eliminate a class of bugs by having the same (textual)
  66  * code implement both serialization and deserialization.
  67  */
  68
  69 namespace HPHP {
  70
  71 //////////////////////////////////////////////////////////////////////
  72
  73 /*
  74  * This defines a trait that determines whether or not a type T has a
  75  * member function template called `serde' that specializes for the
  76  * type `SerDe'.  The point here is that calling one of the blob
  77  * helpers on an object that knows how to do its own serialization
  78  * will just reflect back to it.
  79  */
  80
  81 template<typename T, typename SerDe, typename... F>
  82 struct IsNontrivialSerializable {
  83 private:
  84   using yes = std::true_type;
  85   using no = std::false_type;
  86   template<typename U> static auto test(int) ->
  87     decltype(
  88       std::declval<U>().serde(
  89         std::declval<SerDe&>(),
  90         std::declval<F>()...),
  91       yes()
  92     );
  93   template<typename> static no test(...);
  94 public:
  95   static constexpr bool value = std::is_same<decltype(test<T>(0)),yes>::value;
  96 };
  97
  98 //////////////////////////////////////////////////////////////////////
  99
 100 /*
 101  * Like IsNontrivialSerializable, but checks for the presence of a
 102  * static makeForSerde function (with a matching signature) in the
 103  * type T.
 104  */
 105 template<typename T, typename SerDe>
 106 struct HasMakeForSerde {
 107 private:
 108   using yes = std::true_type;
 109   using no = std::false_type;
 110   template<typename U> static auto test(int) ->
 111     decltype(U::makeForSerde(std::declval<SerDe&>()), yes());
 112   template<typename> static no test(...);
 113 public:
 114   static constexpr bool value = std::is_same<decltype(test<T>(0)),yes>::value;
 115 };
 116
 117 //////////////////////////////////////////////////////////////////////
 118
 119 template <typename T> struct BlobEncoderHelper {};
 120
 121 //////////////////////////////////////////////////////////////////////
 122
 123 struct BlobEncoder {
 124   static const bool deserializing = false;
 125   BlobEncoder() = default;
 126
 127   void writeRaw(const char* ptr, size_t size) {
 128     auto const start = m_blob.size();
 129     m_blob.resize(start + size);
 130     std::copy(ptr, ptr + size, m_blob.data() + start);
 131   }
 132
 133   /*
 134    * Currently the most basic encoder/decode only works for integral
 135    * types.  (We don't want this to accidentally get used for things
 136    * like pointers or aggregates.)
 137    */
 138    template<class T>
 139    typename std::enable_if<
 140      std::is_integral<T>::value && std::is_signed<T>::value
 141    >::type
 142    encode(const T& t) {
 143      encode(folly::encodeZigZag(static_cast<int64_t>(t)));
 144    }
 145
 146   template<class T>
 147   typename std::enable_if<
 148     (std::is_integral<T>::value && !std::is_signed<T>::value) ||
 149     std::is_enum<T>::value
 150   >::type
 151   encode(const T& t) {
 152     const size_t start = m_blob.size();
 153     unsigned char buf[folly::kMaxVarintLength64];
 154     uint64_t value = uint64_t{
 155       static_cast<typename std::make_unsigned<T>::type>(t)};
 156     size_t buf_size = folly::encodeVarint(value, buf);
 157
 158     m_blob.resize(start + buf_size);
 159
 160     std::copy(buf, buf + buf_size, &m_blob[start]);
 161   }
 162
 163   template<class T, class... F>
 164   typename std::enable_if<
 165     IsNontrivialSerializable<T,BlobEncoder, F...>::value
 166   >::type encode(const T& t, F... lambdas) {
 167     const_cast<T&>(t).serde(*this, lambdas...);
 168   }
 169
 170   template<typename T, typename... F>
 171   typename std::enable_if<
 172     IsNontrivialSerializable<BlobEncoderHelper<T>, BlobEncoder, T&, F...>::value
 173   >::type encode(const T& t, F... lambdas) {
 174     BlobEncoderHelper<T>::serde(*this, const_cast<T&>(t), lambdas...);
 175   }
 176
 177   // Avoid pointer to integer conversions
 178   template<typename T, typename... F>
 179   typename std::enable_if<
 180     !IsNontrivialSerializable<BlobEncoderHelper<T>, BlobEncoder,
 181                               T&, F...>::value &&
 182     std::is_pointer<T>::value
 183   >::type encode(const T&, F...) = delete;
 184
 185   void encode(bool b) {
 186     encode(b ? 1 : 0);
 187   }
 188
 189   void encode(double d) {
 190     writeRaw(reinterpret_cast<const char*>(&d), sizeof(double));
 191   }
 192
 193   void encode(const std::string& s) {
 194     uint32_t sz = s.size();
 195     encode(sz);
 196     if (!sz) return;
 197     const size_t start = m_blob.size();
 198     m_blob.resize(start + sz);
 199     std::copy(s.data(), s.data() + sz, &m_blob[start]);
 200   }
 201
 202   void encode(const std::filesystem::path& p) {
 203     encode(p.string());
 204   }
 205
 206   template<class T>
 207   void encode(const Optional<T>& opt) {
 208     const bool some = opt.has_value();
 209     encode(some);
 210     if (some) encode(*opt);
 211   }
 212
 213   template <size_t I = 0, typename... Ts>
 214   typename std::enable_if<I == sizeof...(Ts), void>::type
 215   encode(const std::tuple<Ts...>& /*val*/) {}
 216
 217   template<size_t I = 0, typename ...Ts>
 218   typename std::enable_if<I < sizeof...(Ts), void>::type
 219   encode(const std::tuple<Ts...>& val) {
 220     encode(std::get<I>(val));
 221     encode<I + 1, Ts...>(val);
 222   }
 223
 224   template<class K, class V>
 225   void encode(const std::pair<K,V>& kv) {
 226     encode(kv.first);
 227     encode(kv.second);
 228   }
 229
 230   template<typename T, typename A, typename... Extra>
 231   void encode(const std::vector<T, A>& vec, const Extra&... extra) {
 232     encodeOrderedContainer(vec, extra...);
 233   }
 234
 235   template<typename T, typename A, typename... Extra>
 236   void encode(const CompactVector<T, A>& vec, const Extra&... extra) {
 237     encodeOrderedContainer(vec, extra...);
 238   }
 239
 240   template<typename K, typename C, typename A>
 241   void encode(const std::set<K, C, A>& set) {
 242     encodeOrderedContainer(set);
 243   }
 244
 245   template<typename T, typename H, typename E, typename A, typename C,
 246            typename... Extra>
 247   void encode(const folly::F14NodeSet<T, H, E, A>& set, const C& c,
 248               const Extra&... extra) {
 249     encodeUnorderedSet(set, c, extra...);
 250   }
 251
 252   template<typename T, typename H, typename E, typename A, typename C,
 253            typename... Extra>
 254   void encode(const folly::F14VectorSet<T, H, E, A>& set, const C& c,
 255               const Extra&... extra) {
 256     encodeUnorderedSet(set, c, extra...);
 257   }
 258
 259   template<typename T, typename H, typename E, typename A, typename C,
 260            typename... Extra>
 261   void encode(const folly::F14ValueSet<T, H, E, A>& set, const C& c,
 262               const Extra&... extra) {
 263     encodeUnorderedSet(set, c, extra...);
 264   }
 265
 266   template<typename T, typename H, typename E, typename A, typename C,
 267            typename... Extra>
 268   void encode(const folly::F14FastSet<T, H, E, A>& set, const C& c,
 269               const Extra&... extra) {
 270     encodeUnorderedSet(set, c, extra...);
 271   }
 272
 273   template<typename K, typename V,
 274            typename C, typename A,
 275            typename G, typename C2>
 276   void encode(const folly::sorted_vector_map<K, V, C, A, G, C2>& map) {
 277     encodeOrderedContainer(map);
 278   }
 279
 280   template<typename K, typename V, typename C, typename A>
 281   void encode(const std::map<K, V, C, A>& map) {
 282     encodeOrderedContainer(map);
 283   }
 284
 285   template<typename K, typename V, typename H, typename E, typename A,
 286            typename C, typename... Extra>
 287   void encode(const std::unordered_map<K, V, H, E, A>& map, const C& c,
 288               const Extra&... extra) {
 289     encodeUnorderedMap(map, c, extra...);
 290   }
 291
 292   template<typename K, typename V, typename H, typename E, typename A,
 293            typename C, typename... Extra>
 294   void encode(const folly::F14NodeMap<K, V, H, E, A>& map, const C& c,
 295               const Extra&... extra) {
 296     encodeUnorderedMap(map, c, extra...);
 297   }
 298
 299   template<typename K, typename V, typename H, typename E, typename A,
 300            typename C, typename... Extra>
 301   void encode(const folly::F14VectorMap<K, V, H, E, A>& map, const C& c,
 302               const Extra&... extra) {
 303     encodeUnorderedMap(map, c, extra...);
 304   }
 305
 306   template<typename K, typename V, typename H, typename E, typename A,
 307            typename C, typename... Extra>
 308   void encode(const folly::F14ValueMap<K, V, H, E, A>& map, const C& c,
 309               const Extra&... extra) {
 310     encodeUnorderedMap(map, c, extra...);
 311   }
 312
 313   template<typename K, typename V, typename H, typename E, typename A,
 314            typename C, typename... Extra>
 315   void encode(const folly::F14FastMap<K, V, H, E, A>& map, const C& c,
 316               const Extra&... extra) {
 317     encodeUnorderedMap(map, c, extra...);
 318   }
 319
 320   template<class T, class... F>
 321   BlobEncoder& operator()(const T& t, F... lambdas) {
 322     encode(t, lambdas...);
 323     return *this;
 324   }
 325
 326   template<typename T, typename FDeltaEncode>
 327   BlobEncoder& delta(const std::vector<T>& cont, FDeltaEncode deltaEncode) {
 328     if (cont.size() >= 0xffffffffu) {
 329       throw std::runtime_error("maximum vector size exceeded in BlobEncoder");
 330     }
 331
 332     auto prev = T{};
 333     encode(uint32_t(cont.size()));
 334     for (auto it = cont.begin(); it != cont.end(); ++it) {
 335       encode(deltaEncode(prev, *it));
 336       prev = *it;
 337     }
 338     return *this;
 339   }
 340
 341   // Encode a type which has a conversion to nullptr, skipping the
 342   // encoding if it is nullptr. This is mainly meant for smart pointer
 343   // types where the pointer may not be set.
 344   template <typename T>
 345   BlobEncoder& nullable(const T& t) {
 346     if (t) {
 347       (*this)(true);
 348       (*this)(t);
 349     } else {
 350       (*this)(false);
 351     }
 352     return *this;
 353   }
 354
 355   /*
 356    * Record the size of the data emitted during f(), which
 357    * BlobDecoder::skipSize or BlobDecoder::peekSize can later read.
 358    */
 359   template <typename F>
 360   BlobEncoder& withSize(F f) {
 361     uint64_t start = m_blob.size();
 362     m_blob.resize(start + sizeof(uint64_t));
 363     // The size is stored before the data, but we don't know it yet,
 364     // so store 0 and then patch it afterwards (this means we have to
 365     // used a fixed size encoding).
 366     std::memset(&m_blob[start], 0, sizeof(uint64_t));
 367     f();
 368     uint64_t size = m_blob.size() - start;
 369     assertx(size >= sizeof(uint64_t));
 370     size -= sizeof(uint64_t);
 371     std::copy((char*)&size, (char*)&size + sizeof(uint64_t), &m_blob[start]);
 372     return *this;
 373   }
 374
 375   // Run f1 to encode data, then run f2 to encode more data. The data
 376   // encoded by f1 is encoded with a size prefix (using withSize), so
 377   // that it can be skipped. This is meant to be paired with
 378   // BlobDecoder::alternate, where the data will be decoded in
 379   // opposite order.
 380   template <typename F1, typename F2>
 381   BlobEncoder& alternate(F1 f1, F2 f2) {
 382     withSize(std::move(f1));
 383     f2();
 384     return *this;
 385   }
 386
 387   size_t size() const { return m_blob.size(); }
 388   const void* data() const { return m_blob.data(); }
 389
 390   std::vector<char>&& take() { return std::move(m_blob); }
 391
 392 private:
 393   template<typename Cont, typename... Extra>
 394   void encodeOrderedContainer(const Cont& cont, Extra... extra) {
 395     if (cont.size() >= 0xffffffffu) {
 396       throw std::runtime_error("maximum size exceeded in BlobEncoder");
 397     }
 398     encode(uint32_t(cont.size()));
 399     for (auto const& e : cont) encode(e, extra...);
 400   }
 401
 402   // Unordered containers need to be sorted first, to ensure
 403   // deterministic output.
 404   template<typename Cont, typename Cmp, typename... Extra>
 405   void encodeUnorderedMap(const Cont& cont, const Cmp& cmp,
 406                          const Extra&... extra) {
 407     if (cont.size() >= 0xffffffffu) {
 408       throw std::runtime_error("maximum size exceeded in BlobEncoder");
 409     }
 410
 411     std::vector<typename Cont::key_type> keys;
 412     keys.reserve(cont.size());
 413     for (auto const& e : cont) keys.emplace_back(e.first);
 414     std::sort(keys.begin(), keys.end(), cmp);
 415
 416     encode(uint32_t(keys.size()));
 417     for (auto const& k : keys) {
 418       auto const it = cont.find(k);
 419       assertx(it != cont.end());
 420       encode(it->first);
 421       encode(it->second, extra...);
 422     }
 423   }
 424
 425   template<typename Cont, typename Cmp, typename... Extra>
 426   void encodeUnorderedSet(const Cont& cont, const Cmp& cmp,
 427                           const Extra&... extra) {
 428     if (cont.size() >= 0xffffffffu) {
 429       throw std::runtime_error("maximum size exceeded in BlobEncoder");
 430     }
 431
 432     std::vector<typename Cont::key_type> keys;
 433     keys.reserve(cont.size());
 434     for (auto const& e : cont) keys.emplace_back(e);
 435     std::sort(keys.begin(), keys.end(), cmp);
 436
 437     encode(uint32_t(keys.size()));
 438     for (auto const& k : keys) encode(k, extra...);
 439   }
 440
 441   std::vector<char> m_blob;
 442 };
 443
 444 //////////////////////////////////////////////////////////////////////
 445
 446 struct BlobDecoder {
 447   static const bool deserializing = true;
 448
 449   BlobDecoder(const void* vp, size_t sz)
 450     : m_start{static_cast<const unsigned char*>(vp)}
 451     , m_p{m_start}
 452     , m_last{m_p + sz}
 453   {}
 454
 455   void assertDone() {
 456     assertx(m_p >= m_last);
 457   }
 458
 459   const unsigned char* data() const { return m_p; }
 460
 461   void advance(size_t s) {
 462     assertx(remaining() >= s);
 463     m_p += s;
 464   }
 465
 466   void retreat(size_t s) {
 467     assertx(advanced() >= s);
 468     m_p -= s;
 469   }
 470
 471   size_t remaining() const {
 472     return m_p <= m_last ? (m_last - m_p) : 0;
 473   }
 474
 475   size_t advanced() const { return m_p - m_start; }
 476
 477   // Produce a value of type T from the decoder. Uses a specialized
 478   // creation function if available, otherwise just default constructs
 479   // the value and calls the decoder on it.
 480   template <typename T> T make() {
 481     if constexpr (HasMakeForSerde<T, BlobDecoder>::value) {
 482       return T::makeForSerde(*this);
 483     } else {
 484       T t;
 485       (*this)(t);
 486       return t;
 487     }
 488   }
 489
 490   // Like make(), except asserts the decoder is done afterwards.
 491   template <typename T> T makeWhole() {
 492     if constexpr (HasMakeForSerde<T, BlobDecoder>::value) {
 493       auto t = T::makeForSerde(*this);
 494       assertDone();
 495       return t;
 496     } else {
 497       T t;
 498       (*this)(t);
 499       assertDone();
 500       return t;
 501     }
 502   }
 503
 504   // See encode() in BlobEncoder for why this only allows integral
 505   // types.
 506   template<class T>
 507   typename std::enable_if<
 508     std::is_integral<T>::value && std::is_signed<T>::value
 509   >::type
 510   decode(T& t) {
 511     uint64_t value;
 512     decode(value);
 513     t = static_cast<T>(folly::decodeZigZag(value));
 514   }
 515
 516   template<class T>
 517   typename std::enable_if<
 518     (std::is_integral<T>::value && !std::is_signed<T>::value) ||
 519     std::is_enum<T>::value
 520   >::type
 521   decode(T& t) {
 522     folly::ByteRange range(m_p, m_last);
 523     t = static_cast<T>(folly::decodeVarint(range));
 524     m_p = range.begin();
 525   }
 526
 527   template<class T, class... F>
 528   typename std::enable_if<
 529     IsNontrivialSerializable<T,BlobDecoder, F...>::value
 530   >::type decode(T& t, F... lambdas) {
 531     t.serde(*this, lambdas...);
 532   }
 533
 534   template<typename T, typename... F>
 535   typename std::enable_if<
 536     IsNontrivialSerializable<BlobEncoderHelper<T>, BlobDecoder, T&, F...>::value
 537   >::type decode(T& t, F... lambdas) {
 538     BlobEncoderHelper<T>::serde(*this, t, lambdas...);
 539   }
 540
 541   // Avoid pointer to integer conversions
 542   template<typename T, typename... F>
 543   typename std::enable_if<
 544     !IsNontrivialSerializable<BlobEncoderHelper<T>, BlobDecoder,
 545                               T&, F...>::value &&
 546     std::is_pointer<T>::value
 547   >::type decode(T&, F...) = delete;
 548
 549   void decode(double& d) {
 550     assertx(remaining() >= sizeof(double));
 551     std::memcpy(&d, data(), sizeof(double));
 552     advance(sizeof(double));
 553   }
 554
 555   void decode(std::string& s) {
 556     uint32_t sz;
 557     decode(sz);
 558     assertx(m_last - m_p >= sz);
 559     s = std::string{m_p, m_p + sz};
 560     m_p += sz;
 561   }
 562
 563   void decode(std::filesystem::path& p) {
 564     std::string s;
 565     decode(s);
 566     p = std::filesystem::path(std::move(s));
 567   }
 568
 569   size_t peekStdStringSize() {
 570     auto const before = advanced();
 571     uint32_t sz;
 572     decode(sz);
 573     auto const sizeBytes = advanced() - before;
 574     m_p -= sizeBytes;
 575     return sz + sizeBytes;
 576   }
 577
 578   template<class T>
 579   void decode(Optional<T>& opt) {
 580     bool some;
 581     decode(some);
 582
 583     if (!some) {
 584       opt = std::nullopt;
 585     } else {
 586       T value;
 587       decode(value);
 588       opt = value;
 589     }
 590   }
 591
 592   template <size_t I = 0, typename... Ts>
 593   typename std::enable_if<I == sizeof...(Ts), void>::type
 594   decode(std::tuple<Ts...>& /*val*/) {}
 595
 596   template<size_t I = 0, typename ...Ts>
 597   typename std::enable_if<I < sizeof...(Ts), void>::type
 598   decode(std::tuple<Ts...>& val) {
 599     decode(std::get<I>(val));
 600     decode<I + 1, Ts...>(val);
 601   }
 602
 603   template<class K, class V>
 604   void decode(std::pair<K,V>& val) {
 605     decode(val.first);
 606     decode(val.second);
 607   }
 608
 609   template<typename T, typename A, typename... Extra>
 610   void decode(std::vector<T, A>& vec, Extra... extra) {
 611     decodeVecContainer(vec, extra...);
 612     vec.shrink_to_fit();
 613   }
 614
 615   template<typename T, typename A, typename... Extra>
 616   void decode(CompactVector<T, A>& vec, Extra... extra) {
 617     decodeVecContainer(vec, extra...);
 618     vec.shrink_to_fit();
 619   }
 620
 621   template<typename K, typename C, typename A>
 622   void decode(std::set<K, C, A>& set) {
 623     decodeSetContainer(set);
 624   }
 625
 626   template<typename T, typename H, typename E, typename A, typename C,
 627            typename... Extra>
 628   void decode(folly::F14NodeSet<T, H, E, A>& set, const C&,
 629               const Extra&... extra) {
 630     decodeSetContainer(set, extra...);
 631   }
 632
 633   template<typename T, typename H, typename E, typename A, typename C,
 634            typename... Extra>
 635   void decode(folly::F14VectorSet<T, H, E, A>& set, const C&,
 636               const Extra&... extra) {
 637     decodeSetContainer(set, extra...);
 638   }
 639
 640   template<typename T, typename H, typename E, typename A, typename C,
 641            typename... Extra>
 642   void decode(folly::F14ValueSet<T, H, E, A>& set, const C&,
 643               const Extra&... extra) {
 644     decodeSetContainer(set, extra...);
 645   }
 646
 647   template<typename T, typename H, typename E, typename A, typename C,
 648            typename... Extra>
 649   void decode(folly::F14FastSet<T, H, E, A>& set, const C&,
 650               const Extra&... extra) {
 651     decodeSetContainer(set, extra...);
 652   }
 653
 654   template<typename K, typename V,
 655            typename C, typename A,
 656            typename G, typename C2>
 657   void decode(folly::sorted_vector_map<K, V, C, A, G, C2>& map) {
 658     decodeMapContainer(map);
 659   }
 660
 661   template<typename K, typename V, typename C, typename A>
 662   void decode(std::map<K, V, C, A>& map) {
 663     decodeMapContainer(map);
 664   }
 665
 666   template<typename K, typename V, typename H, typename E, typename A,
 667            typename C, typename... Extra>
 668   void decode(std::unordered_map<K, V, H, E, A>& map, const C&,
 669               const Extra&... extra) {
 670     decodeMapContainer(map, extra...);
 671   }
 672
 673   template<typename K, typename V, typename H, typename E, typename A,
 674            typename C, typename... Extra>
 675   void decode(folly::F14NodeMap<K, V, H, E, A>& map, const C&,
 676               const Extra&... extra) {
 677     decodeMapContainer(map, extra...);
 678   }
 679
 680   template<typename K, typename V, typename H, typename E, typename A,
 681            typename C, typename... Extra>
 682   void decode(folly::F14VectorMap<K, V, H, E, A>& map, const C&,
 683               const Extra&... extra) {
 684     decodeMapContainer(map, extra...);
 685   }
 686
 687   template<typename K, typename V, typename H, typename E, typename A,
 688            typename C, typename... Extra>
 689   void decode(folly::F14ValueMap<K, V, H, E, A>& map, const C&,
 690               const Extra&... extra) {
 691     decodeMapContainer(map, extra...);
 692   }
 693
 694   template<typename K, typename V, typename H, typename E, typename A,
 695            typename C, typename... Extra>
 696   void decode(folly::F14FastMap<K, V, H, E, A>& map, const C&,
 697               const Extra&... extra) {
 698     decodeMapContainer(map, extra...);
 699   }
 700
 701   template<class T, class... F>
 702   BlobDecoder& operator()(T& t, F... lambdas) {
 703     decode(t, lambdas...);
 704     return *this;
 705   }
 706
 707   template<class T, typename FDeltaDecode>
 708   BlobDecoder& delta(std::vector<T>& vec, FDeltaDecode deltaDecode) {
 709     uint32_t size;
 710     decode(size);
 711     vec.reserve(size);
 712
 713     auto prev = T{};
 714     auto delta = T{};
 715     for (uint32_t i = 0; i < size; ++i) {
 716       decode(delta);
 717       prev = deltaDecode(prev, delta);
 718       vec.push_back(prev);
 719     }
 720     vec.shrink_to_fit();
 721     return *this;
 722   }
 723
 724   // Decode a type encoded by BlobEncoder::nullable. If the value was
 725   // not encoded, it will be set to nullptr.
 726   template <typename T>
 727   BlobDecoder& nullable(T& t) {
 728     bool present;
 729     (*this)(present);
 730     if (present) {
 731       (*this)(t);
 732     } else {
 733       t = nullptr;
 734     }
 735     return *this;
 736   }
 737
 738   /*
 739    * Read a block of data (using f()) which is previously encoded with
 740    * BlobEncoder::withSize.
 741    */
 742   template <typename F>
 743   BlobDecoder& withSize(F f) {
 744     // Since we're going to read the data anyways, we don't actually
 745     // need the size, but we'll assert if it doesn't match what we
 746     // decode.
 747     assertx(remaining() >= sizeof(uint64_t));
 748     uint64_t size;
 749     std::copy(m_p, m_p + sizeof(uint64_t), (unsigned char*)&size);
 750     advance(sizeof(uint64_t));
 751     auto const DEBUG_ONLY before = remaining();
 752     assertx(before >= size);
 753     f();
 754     assertx(before - remaining() == size);
 755     return *this;
 756   }
 757
 758   /*
 759    * Skip over a block of data encoded with BlobEncoder::withSize.
 760    */
 761   BlobDecoder& skipWithSize() {
 762     assertx(remaining() >= sizeof(uint64_t));
 763     uint64_t size;
 764     std::copy(m_p, m_p + sizeof(uint64_t), (unsigned char*)&size);
 765     assertx(remaining() >= size + sizeof(uint64_t));
 766     advance(sizeof(uint64_t) + size);
 767     return *this;
 768   }
 769
 770   /*
 771    * Read the size encoded by BlobEncoder::withSize, without advancing
 772    * the decoder state.
 773    */
 774   size_t peekSize() const {
 775     assertx(remaining() >= sizeof(uint64_t));
 776     uint64_t size;
 777     std::copy(m_p, m_p + sizeof(uint64_t), (unsigned char*)&size);
 778     return size + sizeof(uint64_t);
 779   }
 780
 781   // Decode data encoded by BlobEncoder::alternate. First the data
 782   // encoded by f2 is decoded (this involves skipping over the f1
 783   // data), then the data cursor is rewound to point at the data
 784   // encoded by f1. Once decoded, the data cursor is then set to point
 785   // to after f2. The end result is that the data blocks are decoded
 786   // in the opposite order as they were encoded.
 787   template <typename F1, typename F2>
 788   BlobDecoder& alternate(F1 f1, F2 f2) {
 789     auto const start = advanced();
 790     // Skip over f1
 791     skipWithSize();
 792     // Decode f2
 793     f2();
 794     auto const end = advanced();
 795     assertx(end >= start);
 796     // Move back to f1
 797     retreat(end - start);
 798     // Decode f1
 799     withSize(std::move(f1));
 800     auto const middle = advanced();
 801     assertx(end >= middle);
 802     // Advance past f2
 803     advance(end - middle);
 804     return *this;
 805   }
 806
 807   /*
 808    * Skip over an encoded std::string
 809    */
 810   BlobDecoder& skipStdString() {
 811     uint32_t sz;
 812     decode(sz);
 813     if (sz < 1) return *this;
 814     assertx(remaining() >= sz);
 815     advance(sz);
 816     return *this;
 817   }
 818
 819 private:
 820   template<typename Cont, typename... Extra>
 821   void decodeMapContainer(Cont& cont, const Extra&... extra) {
 822     cont.clear();
 823     uint32_t size;
 824     decode(size);
 825     for (uint32_t i = 0; i < size; ++i) {
 826       // Cont::value_type typically has a const key, so we cannot use
 827       // it directly
 828       typename Cont::key_type key;
 829       typename Cont::mapped_type val;
 830       decode(key);
 831       decode(val, extra...);
 832       cont.emplace(std::move(key), std::move(val));
 833     }
 834   }
 835
 836   template<typename Cont, typename... Extra>
 837   void decodeSetContainer(Cont& cont, const Extra&... extra) {
 838     cont.clear();
 839     uint32_t size;
 840     decode(size);
 841     cont.reserve(size);
 842     for (uint32_t i = 0; i < size; ++i) {
 843       typename Cont::value_type val;
 844       decode(val, extra...);
 845       cont.emplace(std::move(val));
 846     }
 847   }
 848
 849   template<typename Cont, typename... Extra>
 850   void decodeVecContainer(Cont& cont, const Extra&... extra) {
 851     cont.clear();
 852     uint32_t size;
 853     decode(size);
 854     cont.reserve(size);
 855     for (uint32_t i = 0; i < size; ++i) {
 856       typename Cont::value_type val;
 857       decode(val, extra...);
 858       cont.emplace_back(std::move(val));
 859     }
 860   }
 861
 862   const unsigned char* m_start;
 863   const unsigned char* m_p;
 864   const unsigned char* const m_last;
 865 };
 866
 867 //////////////////////////////////////////////////////////////////////
 868
 869 }