hphp/runtime/base/bespoke/layout.cpp

   1 /*
   2   +----------------------------------------------------------------------+
   3   | HipHop for PHP                                                       |
   4   +----------------------------------------------------------------------+
   5   | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6   +----------------------------------------------------------------------+
   7   | This source file is subject to version 3.01 of the PHP license,      |
   8   | that is bundled with this package in the file LICENSE, and is        |
   9   | available through the world-wide-web at the following url:           |
  10   | http://www.php.net/license/3_01.txt                                  |
  11   | If you did not receive a copy of the PHP license and are unable to   |
  12   | obtain it through the world-wide-web, please send a note to          |
  13   | license@php.net so we can mail you a copy immediately.               |
  14   +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/base/bespoke/layout.h"
  18
  19 #include "hphp/runtime/base/bespoke/logging-array.h"
  20 #include "hphp/runtime/base/bespoke/logging-profile.h"
  21 #include "hphp/runtime/base/bespoke/monotype-dict.h"
  22 #include "hphp/runtime/base/bespoke/monotype-vec.h"
  23 #include "hphp/runtime/base/bespoke/struct-dict.h"
  24 #include "hphp/runtime/base/bespoke/type-structure.h"
  25 #include "hphp/runtime/base/vanilla-vec-defs.h"
  26 #include "hphp/runtime/vm/jit/irgen.h"
  27 #include "hphp/runtime/vm/jit/irgen-internal.h"
  28 #include "hphp/runtime/vm/jit/mcgen-translate.h"
  29 #include "hphp/runtime/vm/jit/type.h"
  30 #include "hphp/runtime/vm/jit/punt.h"
  31 #include "hphp/util/trace.h"
  32
  33 #include <atomic>
  34 #include <array>
  35 #include <folly/lang/Bits.h>
  36 #include <vector>
  37 #include <sstream>
  38
  39 namespace HPHP::bespoke {
  40
  41 //////////////////////////////////////////////////////////////////////////////
  42
  43 using namespace jit;
  44
  45 namespace {
  46
  47 auto constexpr kMaxNumLayouts = (1 << 16) - 1;
  48
  49 std::atomic<size_t> s_topoIndex;
  50 std::array<Layout*, kMaxNumLayouts> s_layoutTable;
  51 size_t s_numStructLayouts; // For logging.
  52 std::atomic<bool> s_hierarchyFinal = false;
  53 std::mutex s_layoutCreationMutex;
  54
  55 LayoutFunctions s_emptyVtable;
  56
  57 constexpr LayoutIndex kBespokeTopIndex = {0};
  58
  59 bool checkLayoutTest(
  60     const std::vector<uint16_t>& liveVec,
  61     const std::vector<uint16_t>& deadVec,
  62     LayoutTest test) {
  63   for (auto const live : liveVec) {
  64     if (!test.accepts(live)) return false;
  65   }
  66   for (auto const dead : deadVec) {
  67     if (test.accepts(dead)) return false;
  68   }
  69   return true;
  70 }
  71
  72 Optional<LayoutTest> compute2ByteTest(
  73     const std::vector<uint16_t>& liveVec,
  74     const std::vector<uint16_t>& deadVec) {
  75   assertx(!liveVec.empty());
  76   auto const val = liveVec[0];
  77   auto const cmp = LayoutTest { val, LayoutTest::Cmp2Byte };
  78   if (checkLayoutTest(liveVec, deadVec, cmp)) return cmp;
  79
  80   auto imm = uint16_t(-1);
  81   for (auto const live : liveVec) {
  82     imm &= ~live;
  83   }
  84   auto const and = LayoutTest { imm, LayoutTest::And2Byte };
  85   if (checkLayoutTest(liveVec, deadVec, and)) return and;
  86
  87   return std::nullopt;
  88 }
  89
  90 std::string show(LayoutTest test) {
  91   switch (test.mode) {
  92     case LayoutTest::And1Byte:
  93       return folly::sformat("AND {:02x}xx", test.imm & 0xff);
  94     case LayoutTest::And2Byte:
  95       return folly::sformat("AND {:04x}", test.imm);
  96     case LayoutTest::Cmp1Byte:
  97       return folly::sformat("CMP {:02x}xx", test.imm & 0xff);
  98     case LayoutTest::Cmp2Byte:
  99       return folly::sformat("CMP {:04x}", test.imm);
 100   }
 101   always_assert(false);
 102 }
 103
 104 }
 105
 106 LayoutFunctionsDispatch g_layout_funcs;
 107
 108 Layout::Layout(LayoutIndex index, std::string description, LayoutSet parents,
 109                const LayoutFunctions* vtable)
 110   : m_index(index)
 111   , m_description(std::move(description))
 112   , m_parents(std::move(parents))
 113   , m_vtable(vtable ? vtable : &s_emptyVtable)
 114 {
 115   std::lock_guard<std::mutex> lock(s_layoutCreationMutex);
 116   assertx(!s_hierarchyFinal.load(std::memory_order_acquire));
 117   assertx(m_index.raw < kMaxNumLayouts);
 118   assertx(s_layoutTable[m_index.raw] == nullptr);
 119   s_layoutTable[m_index.raw] = this;
 120   m_topoIndex = s_topoIndex++;
 121   assertx(checkInvariants());
 122 }
 123
 124 BespokeArray* Layout::coerce(ArrayData* ad) const {
 125   assertx(ad->isVanilla());
 126   auto const layout = ArrayLayout(this);
 127
 128   if (layout.monotype()) {
 129     return maybeMonoify(ad);
 130   } else if (layout.is_struct()) {
 131     return StructDict::MakeFromVanilla(ad, StructLayout::As(this));
 132   } else if (layout.is_type_structure()) {
 133     return TypeStructure::MakeFromVanilla(ad);
 134   }
 135   always_assert(false);
 136 }
 137
 138 /*
 139  * A BFS implementation used to traverse the bespoke type lattice.
 140  */
 141 struct Layout::BFSWalker {
 142   BFSWalker(bool upward, LayoutIndex initIdx)
 143     : m_workQ({initIdx})
 144     , m_upward(upward)
 145   {}
 146
 147   template <typename C>
 148   BFSWalker(bool upward, C&& col)
 149     : m_workQ(col.cbegin(), col.cend())
 150     , m_upward(upward)
 151   {}
 152
 153   /*
 154    * Return the next new node in the graph discovered by BFS, or std::nullopt if
 155    * the BFS has terminated.
 156    */
 157   Optional<LayoutIndex> next() {
 158     while (!m_workQ.empty()) {
 159       auto const index = m_workQ.front();
 160       m_workQ.pop_front();
 161       if (m_processed.find(index) != m_processed.end()) continue;
 162       m_processed.insert(index);
 163       auto const layout = FromIndex(index);
 164       auto const nextSet = m_upward ? layout->m_parents : layout->m_children;
 165       for (auto const next : nextSet) {
 166         m_workQ.push_back(next);
 167       }
 168       return index;
 169     }
 170     return std::nullopt;
 171   }
 172
 173   /*
 174    * Checks if the BFS has encountered a given node.
 175    */
 176   bool hasSeen(LayoutIndex index) const {
 177     return m_processed.find(index) != m_processed.end();
 178   }
 179
 180   /*
 181    * Returns the full set of nodes seen during the BFS.
 182    */
 183   LayoutSet allSeen() const { return m_processed; }
 184
 185 private:
 186   LayoutSet m_processed;
 187   std::deque<LayoutIndex> m_workQ;
 188   bool m_upward;
 189 };
 190
 191 /*
 192  * Computes whether the layout is a descendant of another layout. To do so, we
 193  * simply perform an upward BFS from the current layout until we encounter the
 194  * other layout or the BFS terminates.
 195  */
 196 bool Layout::isDescendantOfDebug(const Layout* other) const {
 197   BFSWalker walker(true, index());
 198   while (auto const idx = walker.next()) {
 199     if (idx == other->index()) return true;
 200   }
 201   return false;
 202 }
 203
 204 /*
 205  * Compute the set of ancestors by running upward DFS until termination.
 206  */
 207 Layout::LayoutSet Layout::computeAncestors() const {
 208   BFSWalker walker(true, index());
 209   while (walker.next()) {}
 210   return walker.allSeen();
 211 }
 212
 213 /*
 214  * Compute the set of descendants by running downward DFS until termination.
 215  */
 216 Layout::LayoutSet Layout::computeDescendants() const {
 217   BFSWalker walker(false, index());
 218   while (walker.next()) {}
 219   return walker.allSeen();
 220 }
 221
 222 bool Layout::checkInvariants() const {
 223   if (!allowBespokeArrayLikes()) return true;
 224
 225   for (auto const DEBUG_ONLY parent : m_parents) {
 226     auto const DEBUG_ONLY layout = FromIndex(parent);
 227     assertx(layout);
 228     assertx(layout->m_topoIndex < m_topoIndex);
 229   }
 230
 231   return true;
 232 }
 233
 234 struct Layout::DescendantOrdering {
 235   bool operator()(const Layout* a, const Layout* b) const {
 236     return a->m_topoIndex < b->m_topoIndex;
 237   }
 238 };
 239
 240 struct Layout::AncestorOrdering {
 241   bool operator()(const Layout* a, const Layout* b) const {
 242     return a->m_topoIndex > b->m_topoIndex;
 243   }
 244 };
 245
 246 void Layout::ClearHierarchy() {
 247   for (size_t i = 0; i < kMaxNumLayouts; i++) {
 248     if (i != kBespokeTopIndex.raw) s_layoutTable[i] = nullptr;
 249   }
 250   s_numStructLayouts = 0;
 251 }
 252
 253 void Layout::FinalizeHierarchy() {
 254   assertx(allowBespokeArrayLikes());
 255   assertx(!s_hierarchyFinal.load(std::memory_order_acquire));
 256   s_numStructLayouts = 0;
 257   std::vector<Layout*> allLayouts;
 258   for (size_t i = 0; i < kMaxNumLayouts; i++) {
 259     auto const layout = s_layoutTable[i];
 260     if (!layout) continue;
 261     if (ArrayLayout(layout).is_struct()) s_numStructLayouts++;
 262     allLayouts.push_back(layout);
 263     assertx(layout->checkInvariants());
 264     for (auto const pIdx : layout->m_parents) {
 265       auto const parent = s_layoutTable[pIdx.raw];
 266       assertx(parent);
 267       parent->m_children.insert(layout->index());
 268     }
 269   }
 270
 271   for (size_t i = 0; i < kMaxNumLayouts; i++) {
 272     auto layout = s_layoutTable[i];
 273     if (!layout) continue;
 274     auto const descendants = layout->computeDescendants();
 275     std::transform(
 276       descendants.begin(), descendants.end(),
 277       std::back_inserter(layout->m_descendants),
 278       [&] (LayoutIndex i) { return s_layoutTable[i.raw]; }
 279     );
 280     auto const ancestors = layout->computeAncestors();
 281     std::transform(
 282       ancestors.begin(), ancestors.end(),
 283       std::back_inserter(layout->m_ancestors),
 284       [&] (LayoutIndex i) { return s_layoutTable[i.raw]; }
 285     );
 286
 287     std::sort(layout->m_descendants.begin(), layout->m_descendants.end(),
 288               DescendantOrdering{});
 289     std::sort(layout->m_ancestors.begin(), layout->m_ancestors.end(),
 290               AncestorOrdering{});
 291   }
 292
 293   // Compute mask and compare sets in topological order so that they can use
 294   // their children's masks if necessary.
 295   std::sort(allLayouts.begin(), allLayouts.end(), AncestorOrdering{});
 296   for (auto const layout : allLayouts) {
 297     if (layout->index() == kBespokeTopIndex) continue;
 298     layout->m_layout_test = layout->computeLayoutTest();
 299   }
 300
 301   s_hierarchyFinal.store(true, std::memory_order_release);
 302 }
 303
 304 bool Layout::HierarchyFinalized() {
 305   return s_hierarchyFinal.load(std::memory_order_acquire);
 306 }
 307
 308 bool Layout::operator<=(const Layout& other) const {
 309   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 310   auto const res = std::binary_search(m_ancestors.begin(), m_ancestors.end(),
 311                                       &other, AncestorOrdering{});
 312   assertx(isDescendantOfDebug(&other) == res);
 313   return res;
 314 }
 315
 316 const Layout* Layout::operator|(const Layout& other) const {
 317   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 318   auto lIter = m_ancestors.cbegin();
 319   auto rIter = other.m_ancestors.cbegin();
 320   auto const lt = AncestorOrdering{};
 321   while (lIter != m_ancestors.cend() && rIter != other.m_ancestors.cend()) {
 322     if (*lIter == *rIter) {
 323       return *lIter;
 324     }
 325     if (lt(*lIter, *rIter)) {
 326       lIter++;
 327     } else {
 328       rIter++;
 329     }
 330   }
 331   not_reached();
 332 }
 333
 334 const Layout* Layout::operator&(const Layout& other) const {
 335   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 336   auto lIter = m_descendants.cbegin();
 337   auto rIter = other.m_descendants.cbegin();
 338   auto const lt = DescendantOrdering{};
 339   while (lIter != m_descendants.cend() && rIter != other.m_descendants.cend()) {
 340     if (*lIter == *rIter) {
 341       return *lIter;
 342     }
 343     if (lt(*lIter, *rIter)) {
 344       lIter++;
 345     } else {
 346       rIter++;
 347     }
 348   }
 349   return nullptr;
 350 }
 351
 352 const Layout* Layout::FromIndex(LayoutIndex index) {
 353   auto const layout = s_layoutTable[index.raw];
 354   assertx(layout != nullptr);
 355   assertx(layout->index() == index);
 356   return layout;
 357 }
 358
 359 std::string Layout::dumpAllLayouts() {
 360   std::ostringstream ss;
 361   for (size_t i = 0; i < kMaxNumLayouts; i++) {
 362     auto const layout = s_layoutTable[i];
 363     if (!layout) continue;
 364     ss << layout->dumpInformation();
 365   }
 366   return ss.str();
 367 }
 368
 369 std::string Layout::dumpInformation() const {
 370   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 371
 372   auto const concreteDesc = [&](const Layout* layout) {
 373     return layout->isConcrete() ? "Concrete" : "Abstract";
 374   };
 375
 376   std::ostringstream ss;
 377   ss << folly::format("{:04x}: {} [{}]\n", m_index.raw, describe(),
 378                        concreteDesc(this));
 379
 380   auto const top = index() == kBespokeTopIndex;
 381   auto const type_test = top ? "kind() & 1" : show(m_layout_test);
 382   ss << folly::format("  Type test: {}\n", type_test);
 383
 384   ss << folly::format("  Ancestors:\n");
 385   for (auto const ancestor : m_ancestors) {
 386     ss << folly::format("  - {:04x}: {} [{}]\n",
 387                         ancestor->m_index.raw, ancestor->describe(),
 388                         concreteDesc(ancestor));
 389   }
 390
 391   ss << folly::format("  Descendants:\n");
 392   for (auto const descendant : m_descendants) {
 393     ss << folly::format("  - {:04x}: {} [{}]\n",
 394                         descendant->m_index.raw, descendant->describe(),
 395                         concreteDesc(descendant));
 396   }
 397
 398   ss << "\n";
 399
 400   return ss.str();
 401 }
 402
 403 LayoutTest Layout::computeLayoutTest() const {
 404   // The set of all concrete layouts that descend from the layout.
 405   std::vector<uint16_t> liveVec;
 406   for(auto const layout : m_descendants) {
 407     if (!layout->isConcrete()) continue;
 408     liveVec.push_back(layout->m_index.raw);
 409   }
 410   assertx(!liveVec.empty());
 411   std::sort(liveVec.begin(), liveVec.end());
 412
 413   // The set of all possible concrete layouts.
 414   std::vector<uint16_t> allConcrete;
 415   for (size_t i = 0; i < kMaxNumLayouts; i++) {
 416     auto const layout = s_layoutTable[i];
 417     if (!layout) continue;
 418     if (!layout->isConcrete()) continue;
 419     allConcrete.push_back(layout->m_index.raw);
 420   }
 421
 422   // The set of all concrete layouts that do *not* descend from this layout.
 423   // This set includes the vanilla index, so that our test will exclude it.
 424   std::vector<uint16_t> deadVec;
 425   std::set_difference(
 426     allConcrete.cbegin(), allConcrete.cend(),
 427     liveVec.cbegin(), liveVec.cend(),
 428     std::back_inserter(deadVec)
 429   );
 430   static_assert(ArrayData::kVanillaLayoutIndex.raw == uint16_t(-1));
 431   deadVec.push_back(ArrayData::kVanillaLayoutIndex.raw);
 432
 433   SCOPE_ASSERT_DETAIL("bespoke::Layout::computeLayoutTest") {
 434     std::string ret = folly::sformat("{:04x}: {}\n", m_index.raw, describe());
 435     ret += folly::sformat("  Live:\n");
 436     for (auto const live : liveVec) {
 437       auto const layout = s_layoutTable[live]->describe();
 438       ret += folly::sformat("  - {:04x}: {}\n", live, layout);
 439     }
 440     ret += folly::sformat("  Dead:\n");
 441     for (auto const dead : deadVec) {
 442       auto const layout = dead == ArrayData::kVanillaLayoutIndex.raw
 443         ? "Vanilla" : s_layoutTable[dead]->describe();
 444       ret += folly::sformat("  - {:04x}: {}\n", dead, layout);
 445     }
 446     return ret;
 447   };
 448
 449   // Try a 1-byte test on the layout's high byte. Fall back to a 2-byte test.
 450   auto const result = [&]{
 451     std::vector<uint16_t> live1ByteVec;
 452     for (auto const live : liveVec) {
 453       live1ByteVec.push_back(live >> 8);
 454     }
 455     std::vector<uint16_t> dead1ByteVec;
 456     for (auto const dead : deadVec) {
 457       dead1ByteVec.push_back(dead >> 8);
 458     }
 459
 460     if (auto const test = compute2ByteTest(live1ByteVec, dead1ByteVec)) {
 461       auto one_byte = *test;
 462       one_byte.mode = [&]{
 463         switch (one_byte.mode) {
 464           case LayoutTest::And1Byte: always_assert(false);
 465           case LayoutTest::And2Byte: return LayoutTest::And1Byte;
 466           case LayoutTest::Cmp1Byte: always_assert(false);
 467           case LayoutTest::Cmp2Byte: return LayoutTest::Cmp1Byte;
 468         }
 469         always_assert(false);
 470       }();
 471       one_byte.imm = uint16_t(int8_t(one_byte.imm & 0xff));
 472       return one_byte;
 473     }
 474
 475     if (auto const test = compute2ByteTest(liveVec, deadVec)) return *test;
 476
 477     always_assert(false);
 478   }();
 479
 480   SCOPE_ASSERT_DETAIL("selected layout test") { return show(result); };
 481
 482   always_assert(checkLayoutTest(liveVec, deadVec, result));
 483
 484   return result;
 485 }
 486
 487 LayoutTest Layout::getLayoutTest() const {
 488   assertx(index() != kBespokeTopIndex);
 489   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 490   return m_layout_test;
 491 }
 492
 493 struct Layout::Initializer {
 494   Initializer() {
 495     AbstractLayout::InitializeLayouts();
 496     LoggingArray::InitializeLayouts();
 497     MonotypeVec::InitializeLayouts();
 498     EmptyMonotypeDict::InitializeLayouts();
 499     TypeStructure::InitializeLayouts();
 500   }
 501 };
 502 Layout::Initializer Layout::s_initializer;
 503
 504 //////////////////////////////////////////////////////////////////////////////
 505
 506 ArrayLayout Layout::appendType(Type val) const {
 507   return ArrayLayout::Top();
 508 }
 509
 510 ArrayLayout Layout::removeType(Type key) const {
 511   return ArrayLayout::Top();
 512 }
 513
 514 ArrayLayout Layout::setType(Type key, Type val) const {
 515   return ArrayLayout::Top();
 516 }
 517
 518 std::pair<Type, bool> Layout::elemType(Type key) const {
 519   return {TInitCell, false};
 520 }
 521
 522 bool Layout::slotAlwaysPresent(const Type&) const {
 523   return false;
 524 }
 525
 526 std::pair<Type, bool> Layout::firstLastType(bool isFirst, bool isKey) const {
 527   return {isKey ? (TInt | TStr) : TInitCell, false};
 528 }
 529
 530 Type Layout::iterPosType(Type pos, bool isKey) const {
 531   return isKey ? (TInt | TStr) : TInitCell;
 532 }
 533
 534 Type Layout::getTypeBound(Type) const {
 535   return TCell;
 536 }
 537
 538 Optional<int64_t> Layout::numElements() const {
 539   return std::nullopt;
 540 }
 541
 542 //////////////////////////////////////////////////////////////////////////////
 543
 544 AbstractLayout::AbstractLayout(LayoutIndex index,
 545                                std::string description,
 546                                LayoutSet parents,
 547                                const LayoutFunctions* vtable /*=nullptr*/)
 548   : Layout(index, std::move(description), std::move(parents), vtable)
 549 {}
 550
 551 void AbstractLayout::InitializeLayouts() {
 552   new AbstractLayout(kBespokeTopIndex, "BespokeTop", {});
 553 }
 554
 555 LayoutIndex AbstractLayout::GetBespokeTopIndex() {
 556   return kBespokeTopIndex;
 557 }
 558
 559 ConcreteLayout::ConcreteLayout(LayoutIndex index,
 560                                std::string description,
 561                                LayoutSet parents,
 562                                const LayoutFunctions* vtable)
 563   : Layout(index, std::move(description), std::move(parents), vtable)
 564 {
 565   assertx(vtable);
 566   auto const vindex = index.byte() & kBespokeVtableMask;
 567 #define X(Return, Name, Args...) {                          \
 568     assertx(vtable->fn##Name);                              \
 569     auto& entry = g_layout_funcs.fn##Name[vindex];          \
 570     assertx(entry == nullptr || entry == vtable->fn##Name); \
 571     entry = vtable->fn##Name;                               \
 572   }
 573   BESPOKE_LAYOUT_FUNCTIONS(ArrayData)
 574   BESPOKE_SYNTHESIZED_LAYOUT_FUNCTIONS(ArrayData)
 575 #undef X
 576 }
 577
 578 const ConcreteLayout* ConcreteLayout::FromConcreteIndex(LayoutIndex index) {
 579   auto const layout = s_layoutTable[index.raw];
 580   assertx(layout != nullptr);
 581   assertx(layout->index() == index);
 582   assertx(layout->isConcrete());
 583   return reinterpret_cast<ConcreteLayout*>(layout);
 584 }
 585
 586 //////////////////////////////////////////////////////////////////////////////
 587
 588 namespace {
 589
 590 template<typename Bespokify>
 591 ArrayData* maybeBespokifyForTesting(ArrayData* ad,
 592                                     LoggingProfile* profile,
 593                                     std::atomic<ArrayData*>& cache,
 594                                     Bespokify bespokify) {
 595   if (!profile->data) return ad;
 596   auto const bad = cache.load(std::memory_order_acquire);
 597   if (bad) return bad;
 598
 599   auto const result = bespokify(ad, profile);
 600   if (!result) return ad;
 601   ad->decRefAndRelease();
 602
 603   // We should cache a static bespoke array iff this profile is for a static
 604   // array constructor or static prop - i.e. iff staticLoggingArray is set.
 605   if (!profile->data->staticLoggingArray) return result;
 606
 607   ArrayData* current = nullptr;
 608   if (cache.compare_exchange_strong(current, result)) return result;
 609   RO::EvalLowStaticArrays ? low_free(result) : uncounted_free(result);
 610   return current;
 611 }
 612
 613 }
 614
 615 void logBespokeDispatch(const BespokeArray* bad, const char* fn) {
 616   if (Trace::moduleEnabled(Trace::bespoke, 6)) {
 617     DEBUG_ONLY auto const sk = getSrcKey();
 618     DEBUG_ONLY auto const layout = Layout::FromIndex(bad->layoutIndex());
 619     TRACE_MOD(Trace::bespoke, 6, "Bespoke dispatch: %s: %s::%s\n",
 620               sk.valid() ? sk.getSymbol().data() : "(unknown)",
 621               layout->describe().data(), fn);
 622   }
 623 }
 624
 625 BespokeArray* maybeMonoify(ArrayData* ad) {
 626   if (!ad->isVanilla() || ad->isKeysetType()) return nullptr;
 627
 628   auto const et = EntryTypes::ForArray(ad);
 629   if (!et.isMonotypeState()) return nullptr;
 630
 631   auto const legacy = ad->isLegacyArray();
 632
 633   if (et.valueTypes == ValueTypes::Empty) {
 634     switch (ad->toDataType()) {
 635       case KindOfVec:  return EmptyMonotypeVec::GetVec(legacy);
 636       case KindOfDict: return EmptyMonotypeDict::GetDict(legacy);
 637       default: always_assert(false);
 638     }
 639   }
 640
 641   auto const dt = et.valueDatatype;
 642   return ad->isDictType()
 643     ? MakeMonotypeDictFromVanilla(ad, dt, et.keyTypes)
 644     : MonotypeVec::MakeFromVanilla(ad, dt);
 645 }
 646
 647 StructLayout::FieldVector collectFieldVector(const KeyOrder& ko) {
 648   if (!ko.valid()) return {};
 649   StructLayout::FieldVector result;
 650   for (auto const key : ko) result.push_back({key, false, 0});
 651   return result;
 652 }
 653
 654 BespokeArray* maybeStructify(ArrayData* ad, const LoggingProfile* profile) {
 655   if (!ad->isVanilla() || ad->isKeysetType()) return nullptr;
 656
 657   assertx(profile->data);
 658   auto const& koMap = profile->data->keyOrders;
 659   if (koMap.empty()) return nullptr;
 660
 661   auto const ko = collectKeyOrder(koMap);
 662   auto const create = !s_hierarchyFinal.load(std::memory_order_acquire);
 663   auto const fields = collectFieldVector(ko);
 664   auto const layout = StructLayout::GetLayout(fields, create);
 665   return layout ? StructDict::MakeFromVanilla(ad, layout) : nullptr;
 666 }
 667
 668 ArrayData* makeBespokeForTesting(ArrayData* ad, LoggingProfile* profile) {
 669   if (!jit::mcgen::retranslateAllEnabled()) {
 670     return bespoke::maybeMakeLoggingArray(ad, profile);
 671   }
 672   auto const mod = requestCount() % 4;
 673   if (mod == 1) return bespoke::maybeMakeLoggingArray(ad, profile);
 674   if (mod == 2) {
 675     return bespoke::maybeBespokifyForTesting(
 676       ad, profile, profile->data->staticMonotypeArray,
 677       [](auto ad, auto /*profile*/) { return maybeMonoify(ad); }
 678     );
 679   }
 680   return ad;
 681 }
 682
 683 void eachLayout(std::function<void(Layout& layout)> fn) {
 684   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 685   for (size_t i = 0; i < kMaxNumLayouts; i++) {
 686     auto const layout = s_layoutTable[i];
 687     if (layout) fn(*layout);
 688   }
 689 }
 690
 691 Layout** layoutsForJIT() {
 692   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 693   return s_layoutTable.data();
 694 }
 695
 696 size_t numStructLayouts() {
 697   if (!s_hierarchyFinal.load(std::memory_order_acquire)) return 0;
 698   return s_numStructLayouts;
 699 }
 700
 701 //////////////////////////////////////////////////////////////////////////////
 702
 703 }