hphp/runtime/vm/blob-helper.h

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #ifndef incl_HPHP_RUNTIME_VM_BLOB_HELPER_H_
  18 #define incl_HPHP_RUNTIME_VM_BLOB_HELPER_H_
  19
  20 #include <algorithm>
  21 #include <cstdlib>
  22 #include <limits>
  23 #include <type_traits>
  24 #include <unordered_set>
  25 #include <vector>
  26
  27 #include <folly/Optional.h>
  28 #include <folly/Varint.h>
  29
  30 #include "hphp/compiler/option.h"
  31 #include "hphp/runtime/base/builtin-functions.h"
  32 #include "hphp/runtime/base/tv-mutate.h"
  33 #include "hphp/runtime/base/tv-variant.h"
  34 #include "hphp/runtime/base/type-string.h"
  35 #include "hphp/runtime/base/variable-serializer.h"
  36 #include "hphp/runtime/vm/litstr-table.h"
  37 #include "hphp/runtime/vm/repo.h"
  38 #include "hphp/runtime/vm/repo-global-data.h"
  39
  40 /*
  41  * This module contains helpers for serializing and deserializing
  42  * metadata into blobs suitable for insertion into the hhbc repo.
  43  *
  44  * Types may provide their own serialization logic by implementing a
  45  * single-argument member function template called `serde' (as in
  46  * "serialization/deserialization"), and pushing data it wants to
  47  * serialize into the parameter.  Those members may in turn have
  48  * customized serialization behavior, or they may "bottom out" to
  49  * these helpers if they are basic-enough types.
  50  *
  51  * For example:
  52  *
  53  *    struct MyBlobableStuff {
  54  *      int m_foo;
  55  *      std::vector<const StringData*> m_strList;
  56  *      OtherBlobbableData m_other;
  57  *
  58  *      template<class SerDe> void serde(SerDe& sd) {
  59  *        sd(m_foo)(m_strList)(m_other);
  60  *      }
  61  *    };
  62  *
  63  * The argument to SerDe will have a static const member called
  64  * `deserializing' which can be used to do custom behavior depending
  65  * on whether you're deserializing.  But generally the goal here
  66  * should be to eliminate a class of bugs by having the same (textual)
  67  * code implement both serialization and deserialization.
  68  */
  69
  70 namespace HPHP {
  71
  72 //////////////////////////////////////////////////////////////////////
  73
  74 /*
  75  * Warning: some language abuse below.
  76  *
  77  * This defines a trait that determines whether or not a type T has a
  78  * member function template called `serde' that specializes for the
  79  * type `SerDe'.  The point here is that calling one of the blob
  80  * helpers on an object that knows how to do its own serialization
  81  * will just reflect back to it.
  82  *
  83  * If you are unlucky enough to need to know how this works, look up
  84  * "SFINAE".
  85  */
  86
  87 template<class T, class SerDe>
  88 struct IsNontrivialSerializable {
  89 private:
  90   template<class U, void (U::*)(SerDe&)> struct Checker;
  91   template<class U> static char test(Checker<U,&U::template serde<SerDe> >*);
  92   template<class>   static long test(...);
  93 public:
  94   enum { value = sizeof(test<T>(0)) == 1 };
  95 };
  96
  97 //////////////////////////////////////////////////////////////////////
  98
  99 struct BlobEncoder {
 100   static const bool deserializing = false;
 101
 102   /*
 103    * Currently the most basic encoder/decode only works for integral
 104    * types.  (We don't want this to accidentally get used for things
 105    * like pointers or aggregates.)
 106    *
 107    * Floating point support could be added later if we need it ...
 108    */
 109    template<class T>
 110    typename std::enable_if<
 111      std::is_integral<T>::value && std::is_signed<T>::value
 112    >::type
 113    encode(const T& t) {
 114      encode(folly::encodeZigZag(static_cast<int64_t>(t)));
 115    }
 116
 117   template<class T>
 118   typename std::enable_if<
 119     (std::is_integral<T>::value && !std::is_signed<T>::value) ||
 120     std::is_enum<T>::value
 121   >::type
 122   encode(const T& t) {
 123     const size_t start = m_blob.size();
 124     unsigned char buf[folly::kMaxVarintLength64];
 125     uint64_t value = uint64_t{
 126       static_cast<typename std::make_unsigned<T>::type>(t)};
 127     size_t buf_size = folly::encodeVarint(value, buf);
 128
 129     m_blob.resize(start + buf_size);
 130
 131     std::copy(buf, buf + buf_size, &m_blob[start]);
 132   }
 133
 134   template<class T>
 135   typename std::enable_if<
 136     IsNontrivialSerializable<T,BlobEncoder>::value
 137   >::type encode(const T& t) {
 138     const_cast<T&>(t).serde(*this);
 139   }
 140
 141   void encode(bool b) {
 142     encode(b ? 1 : 0);
 143   }
 144
 145   void encode(DataType t) {
 146     // always encode DataType as int8 even if it's a bigger size.
 147     assertx(DataType(int8_t(t)) == t);
 148     encode(int8_t(t));
 149   }
 150
 151   void encode(const LowStringPtr& s) {
 152     const StringData* sd = s;
 153     if (Option::WholeProgram) {
 154       Id id = LitstrTable::get().mergeLitstr(sd);
 155       encode(id);
 156       return;
 157     }
 158
 159     encode(sd);
 160   }
 161
 162   void encode(const StringData* sd) {
 163     if (!sd) { return encode(uint32_t(0)); }
 164     uint32_t sz = sd->size();
 165     encode(sz + 1);
 166     if (!sz) { return; }
 167
 168     const size_t start = m_blob.size();
 169     m_blob.resize(start + sz);
 170     std::copy(sd->data(), sd->data() + sz, &m_blob[start]);
 171   }
 172
 173   void encode(const std::string& s) {
 174     uint32_t sz = s.size();
 175     encode(sz + 1);
 176     if (!sz) { return; }
 177
 178     const size_t start = m_blob.size();
 179     m_blob.resize(start + sz);
 180     std::copy(s.data(), s.data() + sz, &m_blob[start]);
 181   }
 182
 183   void encode(const TypedValue& tv) {
 184     if (tv.m_type == KindOfUninit) {
 185       return encode(staticEmptyString());
 186     }
 187     auto s = internal_serialize(tvAsCVarRef(&tv));
 188     encode(s.get());
 189   }
 190
 191   void encode(const TypedValueAux& tv) = delete;
 192
 193   template<class T>
 194   void encode(const folly::Optional<T>& opt) {
 195     const bool some = opt.hasValue();
 196     encode(some);
 197     if (some) encode(*opt);
 198   }
 199
 200   template <size_t I = 0, typename... Ts>
 201   typename std::enable_if<I == sizeof...(Ts), void>::type
 202   encode(const std::tuple<Ts...>& /*val*/) {}
 203
 204   template<size_t I = 0, typename ...Ts>
 205   typename std::enable_if<I < sizeof...(Ts), void>::type
 206   encode(const std::tuple<Ts...>& val) {
 207     encode(std::get<I>(val));
 208     encode<I + 1, Ts...>(val);
 209   }
 210
 211   template<class K, class V>
 212   void encode(const std::pair<K,V>& kv) {
 213     encode(kv.first);
 214     encode(kv.second);
 215   }
 216
 217   template<class T, typename FDeltaEncode>
 218   void encode(const std::vector<T>& cont, FDeltaEncode deltaEncode) {
 219     if (cont.size() >= 0xffffffffu) {
 220       throw std::runtime_error("maximum vector size exceeded in BlobEncoder");
 221     }
 222
 223     auto prev = T{};
 224     encode(uint32_t(cont.size()));
 225     for (auto it = cont.begin(); it != cont.end(); ++it) {
 226       encode(deltaEncode(prev, *it));
 227       prev = *it;
 228     }
 229   }
 230
 231   template<class T>
 232   void encode(const std::vector<T>& vec) {
 233     encodeContainer(vec, "vector");
 234   }
 235
 236   template<class T>
 237   typename std::enable_if<
 238     std::is_same<typename T::value_type,
 239                  std::pair<typename T::key_type const,
 240                            typename T::mapped_type>>::value &&
 241     !IsNontrivialSerializable<T,BlobEncoder>::value
 242   >::type encode(const T& map) {
 243     encodeContainer(map, "map");
 244   }
 245
 246   template<class T>
 247   typename std::enable_if<
 248     std::is_same<typename T::key_type,
 249                  typename T::value_type>::value &&
 250     !IsNontrivialSerializable<T,BlobEncoder>::value
 251   >::type encode(const T& set) {
 252     encodeContainer(set, "set");
 253   }
 254
 255   template<class T>
 256   BlobEncoder& operator()(const T& t) {
 257     encode(t);
 258     return *this;
 259   }
 260
 261   template<class T, class DeltaEncode>
 262   BlobEncoder& operator()(T& t, DeltaEncode e) {
 263     encode(t, e);
 264     return *this;
 265   }
 266
 267   template<class T, class DeltaEncode, class DeltaDecode>
 268   BlobEncoder& operator()(T& t, DeltaEncode e, DeltaDecode d) {
 269     encode(t, e);
 270     return *this;
 271   }
 272
 273   size_t size() const { return m_blob.size(); }
 274   const void* data() const { return &m_blob[0]; }
 275
 276 private:
 277   template<class Cont>
 278   void encodeContainer(Cont& cont, const char* desc) {
 279     if (cont.size() >= 0xffffffffu) {
 280       throw std::runtime_error("maximum " + std::string(desc) +
 281         " size exceeded in BlobEncoder");
 282     }
 283     encode(uint32_t(cont.size()));
 284     typedef typename Cont::const_iterator iter;
 285     for (iter it = cont.begin(); it != cont.end(); ++it) {
 286       encode(*it);
 287     }
 288   }
 289
 290 private:
 291   std::vector<char> m_blob;
 292 };
 293
 294 //////////////////////////////////////////////////////////////////////
 295
 296 struct BlobDecoder {
 297   static const bool deserializing = true;
 298
 299   explicit BlobDecoder(const void* vp, size_t sz)
 300     : m_p(static_cast<const unsigned char*>(vp))
 301     , m_last(m_p + sz)
 302   {}
 303
 304   void assertDone() {
 305     assertx(m_p >= m_last);
 306   }
 307
 308   // See encode() in BlobEncoder for why this only allows integral
 309   // types.
 310   template<class T>
 311   typename std::enable_if<
 312     std::is_integral<T>::value && std::is_signed<T>::value
 313   >::type
 314   decode(T& t) {
 315     uint64_t value;
 316     decode(value);
 317     t = static_cast<T>(folly::decodeZigZag(value));
 318   }
 319
 320   template<class T>
 321   typename std::enable_if<
 322     (std::is_integral<T>::value && !std::is_signed<T>::value) ||
 323     std::is_enum<T>::value
 324   >::type
 325   decode(T& t) {
 326     folly::ByteRange range(m_p, m_last);
 327     t = static_cast<T>(folly::decodeVarint(range));
 328     m_p = range.begin();
 329   }
 330
 331   template<class T>
 332   typename std::enable_if<
 333     IsNontrivialSerializable<T,BlobDecoder>::value
 334   >::type decode(T& t) {
 335     t.serde(*this);
 336   }
 337
 338   void decode(DataType& t) {
 339     // always decode DataType as int8 even if it's a bigger size.
 340     int8_t t2;
 341     decode(t2);
 342     t = DataType(t2);
 343   }
 344
 345   void decode(const StringData*& sd) {
 346     String s(decodeString());
 347     sd = s.get() ? makeStaticString(s) : 0;
 348   }
 349
 350   void decode(LowStringPtr& s) {
 351     if (RuntimeOption::RepoAuthoritative) {
 352       Id id;
 353       decode(id);
 354       s = LitstrTable::get().lookupLitstrId(id);
 355       return;
 356     }
 357
 358     String st(decodeString());
 359     s = st.get() ? makeStaticString(st) : 0;
 360   }
 361
 362   void decode(std::string& s) {
 363     String str(decodeString());
 364     s = str.toCppString();
 365   }
 366
 367   void decode(TypedValue& tv) {
 368     tvWriteUninit(tv);
 369
 370     String s = decodeString();
 371     assertx(!!s);
 372     if (s.empty()) return;
 373
 374     tvAsVariant(&tv) =
 375       unserialize_from_string(s, VariableUnserializer::Type::Internal);
 376     tvAsVariant(&tv).setEvalScalar();
 377   }
 378
 379   void decode(TypedValueAux& tv) = delete;
 380
 381   template<class T>
 382   void decode(folly::Optional<T>& opt) {
 383     bool some;
 384     decode(some);
 385
 386     if (!some) {
 387       opt = folly::none;
 388     } else {
 389       T value;
 390       decode(value);
 391       opt = value;
 392     }
 393   }
 394
 395   template <size_t I = 0, typename... Ts>
 396   typename std::enable_if<I == sizeof...(Ts), void>::type
 397   decode(std::tuple<Ts...>& /*val*/) {}
 398
 399   template<size_t I = 0, typename ...Ts>
 400   typename std::enable_if<I < sizeof...(Ts), void>::type
 401   decode(std::tuple<Ts...>& val) {
 402     decode(std::get<I>(val));
 403     decode<I + 1, Ts...>(val);
 404   }
 405
 406   template<class K, class V>
 407   void decode(std::pair<K,V>& val) {
 408     decode(val.first);
 409     decode(val.second);
 410   }
 411
 412   template<class T>
 413   void decode(std::vector<T>& vec) {
 414     uint32_t size;
 415     decode(size);
 416     for (uint32_t i = 0; i < size; ++i) {
 417       vec.push_back(T());
 418       decode(vec.back());
 419     }
 420   }
 421
 422   template<class T, typename FDeltaDecode>
 423   void decode(std::vector<T>& vec, FDeltaDecode deltaDecode) {
 424     uint32_t size;
 425     decode(size);
 426     vec.reserve(size);
 427
 428     auto prev = T{};
 429     auto delta = T{};
 430     for (uint32_t i = 0; i < size; ++i) {
 431       decode(delta);
 432       prev = deltaDecode(prev, delta);
 433       vec.push_back(prev);
 434     }
 435   }
 436
 437   template<class T>
 438   typename std::enable_if<
 439     std::is_same<typename T::value_type,
 440                  std::pair<typename T::key_type const,
 441                            typename T::mapped_type>>::value &&
 442     !IsNontrivialSerializable<T,BlobDecoder>::value
 443   >::type decode(T& map) {
 444     uint32_t size;
 445     decode(size);
 446     for (uint32_t i = 0; i < size; ++i) {
 447       typename T::key_type key;
 448       decode(key);
 449       typename T::mapped_type val;
 450       decode(val);
 451       map.emplace(key, val);
 452     }
 453   }
 454
 455   template<class T>
 456   typename std::enable_if<
 457     std::is_same<typename T::key_type,
 458                  typename T::value_type>::value &&
 459     !IsNontrivialSerializable<T,BlobDecoder>::value
 460   >::type decode(T& set) {
 461     uint32_t size;
 462     decode(size);
 463     for (uint32_t i = 0; i < size; ++i) {
 464       typename T::value_type val;
 465       decode(val);
 466       set.insert(val);
 467     }
 468   }
 469
 470   template<class T>
 471   BlobDecoder& operator()(T& t) {
 472     decode(t);
 473     return *this;
 474   }
 475
 476   template<class T, class DeltaDecode>
 477   BlobDecoder& operator()(T& t, DeltaDecode d) {
 478     decode(t, d);
 479     return *this;
 480   }
 481
 482   template<class T, class DeltaEncode, class DeltaDecode>
 483   BlobDecoder& operator()(T& t, DeltaEncode e, DeltaDecode d) {
 484     decode(t, d);
 485     return *this;
 486   }
 487
 488 private:
 489   String decodeString() {
 490     uint32_t sz;
 491     decode(sz);
 492     if (sz == 0) return String();
 493     sz--;
 494     if (sz == 0) return empty_string();
 495
 496     String s = String(sz, ReserveString);
 497     char* pch = s.mutableData();
 498     assertx(m_last - m_p >= sz);
 499     std::copy(m_p, m_p + sz, pch);
 500     m_p += sz;
 501     s.setSize(sz);
 502     return s;
 503   }
 504
 505 private:
 506   const unsigned char* m_p;
 507   const unsigned char* const m_last;
 508 };
 509
 510 //////////////////////////////////////////////////////////////////////
 511
 512 }
 513
 514 #endif