hphp/runtime/base/bespoke/layout.cpp

   1 /*
   2   +----------------------------------------------------------------------+
   3   | HipHop for PHP                                                       |
   4   +----------------------------------------------------------------------+
   5   | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6   +----------------------------------------------------------------------+
   7   | This source file is subject to version 3.01 of the PHP license,      |
   8   | that is bundled with this package in the file LICENSE, and is        |
   9   | available through the world-wide-web at the following url:           |
  10   | http://www.php.net/license/3_01.txt                                  |
  11   | If you did not receive a copy of the PHP license and are unable to   |
  12   | obtain it through the world-wide-web, please send a note to          |
  13   | license@php.net so we can mail you a copy immediately.               |
  14   +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/base/bespoke/layout.h"
  18
  19 #include "hphp/runtime/base/bespoke/logging-array.h"
  20 #include "hphp/runtime/base/bespoke/logging-profile.h"
  21 #include "hphp/runtime/base/bespoke/monotype-dict.h"
  22 #include "hphp/runtime/base/bespoke/monotype-vec.h"
  23 #include "hphp/runtime/base/packed-array-defs.h"
  24 #include "hphp/runtime/vm/jit/irgen.h"
  25 #include "hphp/runtime/vm/jit/irgen-internal.h"
  26 #include "hphp/runtime/vm/jit/mcgen-translate.h"
  27 #include "hphp/runtime/vm/jit/type.h"
  28 #include "hphp/runtime/vm/jit/punt.h"
  29 #include "hphp/util/trace.h"
  30
  31 #include <atomic>
  32 #include <array>
  33 #include <folly/lang/Bits.h>
  34 #include <folly/Optional.h>
  35 #include <vector>
  36 #include <sstream>
  37
  38 namespace HPHP { namespace bespoke {
  39
  40 //////////////////////////////////////////////////////////////////////////////
  41
  42 using namespace jit;
  43
  44 namespace {
  45
  46 std::atomic<size_t> s_topoIndex;
  47 std::atomic<size_t> s_layoutTableIndex;
  48 std::array<Layout*, Layout::kMaxIndex.raw + 1> s_layoutTable;
  49 std::atomic<bool> s_hierarchyFinal = false;
  50 std::mutex s_layoutCreationMutex;
  51
  52 LayoutFunctions s_emptyVtable;
  53
  54 constexpr LayoutIndex kBespokeTopIndex = {0};
  55
  56 bool isSingletonLayout(LayoutIndex index) {
  57   return index == EmptyMonotypeVecLayout::Index() ||
  58          index == EmptyMonotypeDictLayout::Index();
  59 }
  60
  61 using Split = std::pair<uint16_t, std::vector<uint16_t>>;
  62
  63 folly::Optional<MaskAndCompare> computeSimpleMaskAndCompare(
  64     const std::vector<uint16_t>& liveVec,
  65     const std::vector<uint16_t>& deadVec) {
  66   uint16_t liveAgree = 0xffff;
  67   for (auto const live : liveVec) {
  68     auto const agree = ~(live ^ liveVec[0]);
  69     liveAgree &= agree;
  70   }
  71
  72   const uint16_t base = liveVec[0] & liveAgree;
  73   for (auto const dead : deadVec) {
  74     if ((dead & liveAgree) == base) {
  75       return folly::none;
  76     }
  77   }
  78
  79   return MaskAndCompare{base, liveAgree, 0};
  80 }
  81
  82 folly::Optional<MaskAndCompare> computeXORMaskAndCompare(
  83     const std::vector<uint16_t>& liveVec,
  84     const std::vector<uint16_t>& deadVec) {
  85   std::vector<uint16_t> remainingLive(liveVec);
  86   std::vector<uint16_t> remainingDead(deadVec);
  87
  88   // We process each bit from most significant to least significant,
  89   // determining the proper assignment for that bit in the XOR and CMP
  90   // values.
  91   //
  92   // For each bit, there three relevant scenarios:
  93   //
  94   // 1. All live layouts agree on this bit having value x. We should
  95   // XOR with x and CMP with 0.
  96   //
  97   // 2. Live layouts do not agree on the bit, but dead layouts agree on
  98   // it having value y. We should XOR with ~y and CMP with 1.
  99   //
 100   // 3. Live layouts do not agree on the bit, and neither do dead
 101   // layouts.  We must exclude this bit in the mask.
 102   //
 103   // After each action, we remove all dead layouts that are guaranteed
 104   // the fail our test and all live layouts that are guaranteed to pass
 105   // it.
 106   auto const recomputeLive = [&] () -> std::pair<uint16_t, uint16_t> {
 107     uint16_t liveAgree = 0xffff;
 108     for (auto const live : remainingLive) {
 109       auto const agree = ~(live ^ remainingLive[0]);
 110       liveAgree &= agree;
 111     }
 112     uint16_t deadAgree = 0xffff;
 113     for (auto const dead : remainingDead) {
 114       auto const agree = ~(dead ^ remainingDead[0]);
 115       deadAgree &= agree;
 116     }
 117     return {liveAgree, deadAgree};
 118   };
 119
 120   uint16_t xorVal = 0;
 121   uint16_t cmpVal = 0;
 122   uint16_t andVal = 0xffff;
 123   auto [liveAgree, deadAgree] = recomputeLive();
 124   for (int i = 0; i < 16; i++) {
 125     // The bit currently being examined.
 126     const uint16_t mask = (1 << (15 - i));
 127
 128     if (remainingDead.empty() && remainingLive.empty()) break;
 129
 130     if (!remainingLive.empty() && (liveAgree & mask)) {
 131       xorVal |= remainingLive[0] & mask;
 132     } else if (!remainingDead.empty() && (deadAgree & mask)) {
 133       xorVal |= (~remainingDead[0]) & mask;
 134       cmpVal |= mask;
 135     } else if (remainingDead.empty()) {
 136       cmpVal |= mask;
 137     } else if (!remainingLive.empty()) {
 138       andVal ^= mask;
 139       // Nothing will be filtered out as we have not constrained our mask, so
 140       // reuse the agreement sets from the prior round.
 141       continue;
 142     }
 143
 144     // The set of bits that have already been fixed and will not be masked
 145     // away. This is a prefix of the set of bits that will be fixed and is used
 146     // for determining which live and dead layouts must pass or fail the test,
 147     // regardless of the decisions made for future (less significant) bits.
 148     const uint16_t cmpMask = andVal & ~(mask - 1);
 149
 150     remainingDead.erase(
 151       std::remove_if(remainingDead.begin(), remainingDead.end(),
 152         [&](auto v) { return ((v ^ xorVal) & cmpMask) > cmpVal; }),
 153       remainingDead.end());
 154     remainingLive.erase(
 155       std::remove_if(remainingLive.begin(), remainingLive.end(),
 156         [&](auto v) { return ((v ^ xorVal) & cmpMask) < cmpVal; }),
 157       remainingLive.end());
 158
 159     auto const [liveAgreeNew, deadAgreeNew] = recomputeLive();
 160     liveAgree = liveAgreeNew;
 161     deadAgree = deadAgreeNew;
 162   }
 163
 164   if (!remainingDead.empty() && !remainingLive.empty()) {
 165     return folly::none;
 166   }
 167
 168   // If it came down to the last bit, adjust the comparison accordingly.
 169   if (!remainingDead.empty()) cmpVal--;
 170
 171   return MaskAndCompare{xorVal, andVal, cmpVal};
 172 }
 173
 174 }
 175
 176 Layout::Layout(LayoutIndex index, std::string description, LayoutSet parents,
 177                const LayoutFunctions* vtable)
 178   : m_index(index)
 179   , m_description(std::move(description))
 180   , m_parents(std::move(parents))
 181   , m_vtable(vtable ? vtable : &s_emptyVtable)
 182 {
 183   std::lock_guard<std::mutex> lock(s_layoutCreationMutex);
 184   assertx(!s_hierarchyFinal.load(std::memory_order_acquire));
 185   assertx(s_layoutTable[m_index.raw] == nullptr);
 186   s_layoutTable[m_index.raw] = this;
 187   m_topoIndex = s_topoIndex++;
 188   assertx(checkInvariants());
 189 }
 190
 191 /*
 192  * A BFS implementation used to traverse the bespoke type lattice.
 193  */
 194 struct Layout::BFSWalker {
 195   BFSWalker(bool upward, LayoutIndex initIdx)
 196     : m_workQ({initIdx})
 197     , m_upward(upward)
 198   {}
 199
 200   template <typename C>
 201   BFSWalker(bool upward, C&& col)
 202     : m_workQ(col.cbegin(), col.cend())
 203     , m_upward(upward)
 204   {}
 205
 206   /*
 207    * Return the next new node in the graph discovered by BFS, or folly::none if
 208    * the BFS has terminated.
 209    */
 210   folly::Optional<LayoutIndex> next() {
 211     while (!m_workQ.empty()) {
 212       auto const index = m_workQ.front();
 213       m_workQ.pop_front();
 214       if (m_processed.find(index) != m_processed.end()) continue;
 215       m_processed.insert(index);
 216       auto const layout = FromIndex(index);
 217       auto const nextSet = m_upward ? layout->m_parents : layout->m_children;
 218       for (auto const next : nextSet) {
 219         m_workQ.push_back(next);
 220       }
 221       return index;
 222     }
 223     return folly::none;
 224   }
 225
 226   /*
 227    * Checks if the BFS has encountered a given node.
 228    */
 229   bool hasSeen(LayoutIndex index) const {
 230     return m_processed.find(index) != m_processed.end();
 231   }
 232
 233   /*
 234    * Returns the full set of nodes seen during the BFS.
 235    */
 236   LayoutSet allSeen() const { return m_processed; }
 237
 238 private:
 239   LayoutSet m_processed;
 240   std::deque<LayoutIndex> m_workQ;
 241   bool m_upward;
 242 };
 243
 244 /*
 245  * Computes whether the layout is a descendant of another layout. To do so, we
 246  * simply perform an upward BFS from the current layout until we encounter the
 247  * other layout or the BFS terminates.
 248  */
 249 bool Layout::isDescendantOfDebug(const Layout* other) const {
 250   BFSWalker walker(true, index());
 251   while (auto const idx = walker.next()) {
 252     if (idx == other->index()) return true;
 253   }
 254   return false;
 255 }
 256
 257 /*
 258  * Compute the set of ancestors by running upward DFS until termination.
 259  */
 260 Layout::LayoutSet Layout::computeAncestors() const {
 261   BFSWalker walker(true, index());
 262   while (walker.next()) {}
 263   return walker.allSeen();
 264 }
 265
 266 /*
 267  * Compute the set of descendants by running downward DFS until termination.
 268  */
 269 Layout::LayoutSet Layout::computeDescendants() const {
 270   BFSWalker walker(false, index());
 271   while (walker.next()) {}
 272   return walker.allSeen();
 273 }
 274
 275 bool Layout::checkInvariants() const {
 276   if (!allowBespokeArrayLikes()) return true;
 277
 278   for (auto const DEBUG_ONLY parent : m_parents) {
 279     auto const DEBUG_ONLY layout = FromIndex(parent);
 280     assertx(layout);
 281     assertx(layout->m_topoIndex < m_topoIndex);
 282   }
 283
 284   return true;
 285 }
 286
 287 struct Layout::DescendantOrdering {
 288   bool operator()(const Layout* a, const Layout* b) const {
 289     return a->m_topoIndex < b->m_topoIndex;
 290   }
 291 };
 292
 293 struct Layout::AncestorOrdering {
 294   bool operator()(const Layout* a, const Layout* b) const {
 295     return a->m_topoIndex > b->m_topoIndex;
 296   }
 297 };
 298
 299 void Layout::FinalizeHierarchy() {
 300   assertx(allowBespokeArrayLikes());
 301   assertx(!s_hierarchyFinal.load(std::memory_order_acquire));
 302   std::vector<Layout*> allLayouts;
 303   for (size_t i = 0; i < s_layoutTableIndex; i++) {
 304     auto const layout = s_layoutTable[i];
 305     if (!layout) continue;
 306     allLayouts.push_back(layout);
 307     assertx(layout->checkInvariants());
 308     for (auto const pIdx : layout->m_parents) {
 309       auto const parent = s_layoutTable[pIdx.raw];
 310       assertx(parent);
 311       parent->m_children.insert(layout->index());
 312     }
 313   }
 314
 315   // TODO(mcolavita): implement these by merging in topological order instead
 316   // of repeated BFS
 317   for (size_t i = 0; i < s_layoutTableIndex; i++) {
 318     auto layout = s_layoutTable[i];
 319     if (!layout) continue;
 320     auto const descendants = layout->computeDescendants();
 321     std::transform(
 322       descendants.begin(), descendants.end(),
 323       std::back_inserter(layout->m_descendants),
 324       [&] (LayoutIndex i) { return s_layoutTable[i.raw]; }
 325     );
 326     auto const ancestors = layout->computeAncestors();
 327     std::transform(
 328       ancestors.begin(), ancestors.end(),
 329       std::back_inserter(layout->m_ancestors),
 330       [&] (LayoutIndex i) { return s_layoutTable[i.raw]; }
 331     );
 332
 333     std::sort(layout->m_descendants.begin(), layout->m_descendants.end(),
 334               DescendantOrdering{});
 335     std::sort(layout->m_ancestors.begin(), layout->m_ancestors.end(),
 336               AncestorOrdering{});
 337   }
 338
 339   // Compute mask and compare sets in topological order so that they can use
 340   // their children's masks if necessary.
 341   std::sort(allLayouts.begin(), allLayouts.end(), AncestorOrdering{});
 342   for (auto const layout : allLayouts) {
 343     layout->m_maskAndCompare = layout->computeMaskAndCompare();
 344   }
 345
 346   s_hierarchyFinal.store(true, std::memory_order_release);
 347 }
 348
 349 bool Layout::operator<=(const Layout& other) const {
 350   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 351   auto const res = std::binary_search(m_ancestors.begin(), m_ancestors.end(),
 352                                       &other, AncestorOrdering{});
 353   assertx(isDescendantOfDebug(&other) == res);
 354   return res;
 355 }
 356
 357 const Layout* Layout::operator|(const Layout& other) const {
 358   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 359   auto lIter = m_ancestors.cbegin();
 360   auto rIter = other.m_ancestors.cbegin();
 361   auto const lt = AncestorOrdering{};
 362   while (lIter != m_ancestors.cend() && rIter != other.m_ancestors.cend()) {
 363     if (*lIter == *rIter) {
 364       return *lIter;
 365     }
 366     if (lt(*lIter, *rIter)) {
 367       lIter++;
 368     } else {
 369       rIter++;
 370     }
 371   }
 372   not_reached();
 373 }
 374
 375 const Layout* Layout::operator&(const Layout& other) const {
 376   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 377   auto lIter = m_descendants.cbegin();
 378   auto rIter = other.m_descendants.cbegin();
 379   auto const lt = DescendantOrdering{};
 380   while (lIter != m_descendants.cend() && rIter != other.m_descendants.cend()) {
 381     if (*lIter == *rIter) {
 382       return *lIter;
 383     }
 384     if (lt(*lIter, *rIter)) {
 385       lIter++;
 386     } else {
 387       rIter++;
 388     }
 389   }
 390   return nullptr;
 391 }
 392
 393 LayoutIndex Layout::ReserveIndices(size_t count) {
 394   assertx(folly::isPowTwo(count));
 395   auto const padded_count = 2 * count - 1;
 396   auto const base = s_layoutTableIndex.fetch_add(padded_count);
 397   always_assert(base + padded_count <= kMaxIndex.raw + 1);
 398   for (auto i = 0; i < padded_count; i++) {
 399     s_layoutTable[base + i] = nullptr;
 400   }
 401   auto const result = (base + count - 1) & ~(count - 1);
 402   assertx(result % count == 0);
 403   return {safe_cast<uint16_t>(result)};
 404 }
 405
 406 const Layout* Layout::FromIndex(LayoutIndex index) {
 407   auto const layout = s_layoutTable[index.raw];
 408   assertx(layout != nullptr);
 409   assertx(layout->index() == index);
 410   return layout;
 411 }
 412
 413 std::string Layout::dumpAllLayouts() {
 414   std::ostringstream ss;
 415
 416   for (size_t i = 0; i < s_layoutTableIndex; i++) {
 417     auto const layout = s_layoutTable[i];
 418     if (!layout) continue;
 419
 420     ss << layout->dumpInformation();
 421   }
 422
 423   return ss.str();
 424 }
 425
 426 std::string Layout::dumpInformation() const {
 427   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 428
 429   auto const concreteDesc = [&](const Layout* layout) {
 430     return layout->isConcrete() ? "Concrete" : "Abstract";
 431   };
 432
 433   std::ostringstream ss;
 434   ss << folly::format("{:04x}: {} [{}]\n", m_index.raw, describe(),
 435                        concreteDesc(this));
 436
 437   ss << folly::format("  Type test:\n");
 438   ss << folly::format("    XOR {:04x}\n", m_maskAndCompare.xorVal);
 439   ss << folly::format("    AND {:04x}\n", m_maskAndCompare.andVal);
 440   ss << folly::format("    CMP {:04x}\n", m_maskAndCompare.cmpVal);
 441
 442   ss << folly::format("  Ancestors:\n");
 443   for (auto const ancestor : m_ancestors) {
 444     ss << folly::format("  - {:04x}: {} [{}]\n",
 445                         ancestor->m_index.raw, ancestor->describe(),
 446                         concreteDesc(ancestor));
 447   }
 448
 449   ss << folly::format("  Descendants:\n");
 450   for (auto const descendant : m_descendants) {
 451     ss << folly::format("  - {:04x}: {} [{}]\n",
 452                         descendant->m_index.raw, descendant->describe(),
 453                         concreteDesc(descendant));
 454   }
 455
 456   ss << "\n";
 457
 458   return ss.str();
 459 }
 460
 461 MaskAndCompare Layout::computeMaskAndCompare() const {
 462   FTRACE_MOD(Trace::bespoke, 1, "Try: {}\n", describe());
 463
 464   // The set of all concrete layouts that descend from the layout.
 465   std::vector<uint16_t> liveVec;
 466   for(auto const layout : m_descendants) {
 467     if (!layout->isConcrete()) continue;
 468     liveVec.push_back(layout->m_index.raw);
 469   }
 470
 471   assertx(!liveVec.empty());
 472   if (liveVec.size() == 1) return {MaskAndCompare::fullCompare(liveVec[0])};
 473   std::sort(liveVec.begin(), liveVec.end());
 474
 475   // The set of all possible concrete layouts.
 476   std::vector<uint16_t> allConcrete;
 477   for (size_t i = 0; i < s_layoutTableIndex; i++) {
 478     auto const layout = s_layoutTable[i];
 479     if (!layout) continue;
 480     if (!layout->isConcrete()) continue;
 481     allConcrete.push_back(layout->m_index.raw);
 482   }
 483
 484   // The set of all concrete layouts that do *not* descend from this layout.
 485   // This set includes the vanilla index, so that our test will exclude it.
 486   std::vector<uint16_t> deadVec;
 487   std::set_difference(
 488     allConcrete.cbegin(), allConcrete.cend(),
 489     liveVec.cbegin(), liveVec.cend(),
 490     std::back_inserter(deadVec)
 491   );
 492   static_assert(ArrayData::kDefaultVanillaArrayExtra == uint32_t(-1));
 493   auto constexpr kVanillaLayoutIndex = uint16_t(-1);
 494   deadVec.push_back(kVanillaLayoutIndex);
 495
 496   auto const check = [&] () -> MaskAndCompare {
 497     // 1. Attempt to find a trivial mask to cover.
 498     if (auto const result = computeSimpleMaskAndCompare(liveVec, deadVec)) {
 499       return *result;
 500     }
 501
 502     // 2. Attempt to find a single mask to cover.
 503     if (auto const result = computeXORMaskAndCompare(liveVec, deadVec)) {
 504       return *result;
 505     }
 506
 507     // 3. Give up.
 508     SCOPE_ASSERT_DETAIL("bespoke::Layout::computeMaskAndCompare") {
 509       std::string ret = folly::sformat("{:04x}: {}\n", m_index.raw, describe());
 510       ret += folly::sformat("  Live:\n");
 511       for (auto const live : liveVec) {
 512         auto const layout = s_layoutTable[live];
 513         ret += folly::sformat("  - {:04x}: {}\n", live, layout->describe());
 514       }
 515       ret += folly::sformat("  Dead:\n");
 516       for (auto const dead : deadVec) {
 517         auto const layout = s_layoutTable[dead];
 518         ret += folly::sformat("  - {:04x}: {}\n", dead, layout->describe());
 519       }
 520       return ret;
 521     };
 522     always_assert(false);
 523   }();
 524
 525   if (debug) {
 526     // The check should pass on live values and fail on dead values.
 527     for (auto const live : liveVec) always_assert(check.accepts(live));
 528     for (auto const dead : deadVec) always_assert(!check.accepts(dead));
 529   }
 530
 531   return check;
 532 }
 533
 534 MaskAndCompare Layout::maskAndCompare() const {
 535   assertx(s_hierarchyFinal.load(std::memory_order_acquire));
 536   return m_maskAndCompare;
 537 }
 538
 539 struct Layout::Initializer {
 540   Initializer() {
 541     AbstractLayout::InitializeLayouts();
 542     LoggingArray::InitializeLayouts();
 543     MonotypeVec::InitializeLayouts();
 544     EmptyMonotypeDict::InitializeLayouts();
 545   }
 546 };
 547 Layout::Initializer Layout::s_initializer;
 548
 549 //////////////////////////////////////////////////////////////////////////////
 550
 551 ArrayLayout Layout::appendType(Type val) const {
 552   return ArrayLayout::Top();
 553 }
 554
 555 ArrayLayout Layout::removeType(Type key) const {
 556   return ArrayLayout::Top();
 557 }
 558
 559 ArrayLayout Layout::setType(Type key, Type val) const {
 560   return ArrayLayout::Top();
 561 }
 562
 563 std::pair<Type, bool> Layout::elemType(Type key) const {
 564   return {TInitCell, false};
 565 }
 566
 567 std::pair<Type, bool> Layout::firstLastType(bool isFirst, bool isKey) const {
 568   return {isKey ? (TInt | TStr) : TInitCell, false};
 569 }
 570
 571 Type Layout::iterPosType(Type pos, bool isKey) const {
 572   return isKey ? (TInt | TStr) : TInitCell;
 573 }
 574
 575 //////////////////////////////////////////////////////////////////////////////
 576
 577 AbstractLayout::AbstractLayout(LayoutIndex index,
 578                                std::string description,
 579                                LayoutSet parents,
 580                                const LayoutFunctions* vtable /*=nullptr*/)
 581   : Layout(index, std::move(description), std::move(parents), vtable)
 582 {}
 583
 584 void AbstractLayout::InitializeLayouts() {
 585   auto const index = Layout::ReserveIndices(1);
 586   always_assert(index == kBespokeTopIndex);
 587   new AbstractLayout(index, "BespokeTop", {});
 588 }
 589
 590 LayoutIndex AbstractLayout::GetBespokeTopIndex() {
 591   return kBespokeTopIndex;
 592 }
 593
 594 ConcreteLayout::ConcreteLayout(LayoutIndex index,
 595                                std::string description,
 596                                LayoutSet parents,
 597                                const LayoutFunctions* vtable)
 598   : Layout(index, std::move(description), std::move(parents), vtable)
 599 {
 600   assertx(vtable);
 601 #define X(Return, Name, Args...) \
 602   assertx(vtable->fn##Name);
 603   BESPOKE_LAYOUT_FUNCTIONS(ArrayData)
 604 #undef X
 605 }
 606
 607 const ConcreteLayout* ConcreteLayout::FromConcreteIndex(LayoutIndex index) {
 608   auto const layout = s_layoutTable[index.raw];
 609   assertx(layout != nullptr);
 610   assertx(layout->index() == index);
 611   assertx(layout->isConcrete());
 612   return reinterpret_cast<ConcreteLayout*>(layout);
 613 }
 614
 615 //////////////////////////////////////////////////////////////////////////////
 616
 617 namespace {
 618 ArrayData* maybeMonoifyForTesting(ArrayData* ad, LoggingProfile* profile) {
 619   assertx(profile->data);
 620   auto& profileMonotype = profile->data->staticMonotypeArray;
 621   auto const mad = profileMonotype.load(std::memory_order_relaxed);
 622   if (mad) return mad;
 623
 624   auto const result = maybeMonoify(ad);
 625   if (!result) return ad;
 626   ad->decRefAndRelease();
 627
 628   // We should cache a staticMonotypeArray iff this profile is for a static
 629   // array constructor or static prop - i.e. iff staticLoggingArray is set.
 630   if (!profile->data->staticLoggingArray) return result;
 631
 632   ArrayData* current = nullptr;
 633   if (profileMonotype.compare_exchange_strong(current, result)) {
 634     return result;
 635   }
 636   RO::EvalLowStaticArrays ? low_free(result) : uncounted_free(result);
 637   return current;
 638 }
 639 }
 640
 641 void logBespokeDispatch(const ArrayData* ad, const char* fn) {
 642   DEBUG_ONLY auto const sk = getSrcKey();
 643   DEBUG_ONLY auto const index = BespokeArray::asBespoke(ad)->layoutIndex();
 644   DEBUG_ONLY auto const layout = Layout::FromIndex(index);
 645   TRACE_MOD(Trace::bespoke, 6, "Bespoke dispatch: %s: %s::%s\n",
 646             sk.valid() ? sk.getSymbol().data() : "(unknown)",
 647             layout->describe().data(), fn);
 648 }
 649
 650 BespokeArray* maybeMonoify(ArrayData* ad) {
 651   if (!ad->isVanilla() || ad->isKeysetType()) return nullptr;
 652
 653   auto const et = EntryTypes::ForArray(ad);
 654   if (!et.isMonotypeState()) return nullptr;
 655
 656   auto const legacy = ad->isLegacyArray();
 657
 658   if (et.valueTypes == ValueTypes::Empty) {
 659     switch (ad->toDataType()) {
 660       case KindOfVArray: return EmptyMonotypeVec::GetVArray(legacy);
 661       case KindOfVec:    return EmptyMonotypeVec::GetVec(legacy);
 662       case KindOfDArray: return EmptyMonotypeDict::GetDArray(legacy);
 663       case KindOfDict:   return EmptyMonotypeDict::GetDict(legacy);
 664       default: always_assert(false);
 665     }
 666   }
 667
 668   auto const dt = et.valueDatatype;
 669   return ad->isDArray() || ad->isDictType()
 670     ? MakeMonotypeDictFromVanilla(ad, dt, et.keyTypes)
 671     : MonotypeVec::MakeFromVanilla(ad, dt);
 672 }
 673
 674 ArrayData* makeBespokeForTesting(ArrayData* ad, LoggingProfile* profile) {
 675   if (!jit::mcgen::retranslateAllEnabled()) {
 676     return bespoke::maybeMakeLoggingArray(ad, profile);
 677   }
 678   auto const mod = requestCount() % 3;
 679   if (mod == 1) return bespoke::maybeMakeLoggingArray(ad, profile);
 680   if (mod == 2) return bespoke::maybeMonoifyForTesting(ad, profile);
 681   return ad;
 682 }
 683
 684 //////////////////////////////////////////////////////////////////////////////
 685
 686 }}