From 6615b2273b4e7490dce1f54706733e2b50163f1d Mon Sep 17 00:00:00 2001 From: Arnab De Date: Wed, 21 Apr 2021 08:52:15 -0700 Subject: [PATCH] Optimize struct element initialization Summary: For elements <= HHIRMaxInlineInitStructElements, JIT moving the elements from the execution stack to the slots. This avoids copying the elements in registers to and from the stack. Differential Revision: D27884363 fbshipit-source-id: 0476dab9521b4a0ded3ac4bf74b09498e8b0dfb4 --- hphp/doc/ir.specification | 14 +++++++++- hphp/runtime/base/bespoke/struct-dict.cpp | 20 +++++++++++--- hphp/runtime/base/bespoke/struct-dict.h | 3 +++ hphp/runtime/base/runtime-option.h | 1 + hphp/runtime/vm/jit/dce.cpp | 4 ++- hphp/runtime/vm/jit/extra-data.cpp | 12 +++++++++ hphp/runtime/vm/jit/extra-data.h | 35 ++++++++++++++++++++++++ hphp/runtime/vm/jit/ir-instruction.cpp | 13 ++++++--- hphp/runtime/vm/jit/ir-opcode.cpp | 2 ++ hphp/runtime/vm/jit/irgen-bespoke.cpp | 44 ++++++++++++++++++++++--------- hphp/runtime/vm/jit/irlower-bespoke.cpp | 33 +++++++++++++++++++++++ hphp/runtime/vm/jit/memory-effects.cpp | 8 ++++++ 12 files changed, 167 insertions(+), 22 deletions(-) diff --git a/hphp/doc/ir.specification b/hphp/doc/ir.specification index 06a2cb2db04..d0155421f32 100644 --- a/hphp/doc/ir.specification +++ b/hphp/doc/ir.specification @@ -1503,10 +1503,22 @@ To string conversions: and N stack elements at `offset` on stack S0 for values. This op assumes it can move values from the stack without inc-reffing them. -| AllocBespokeStructDict, DStructDict, NA, CRc +| AllocBespokeStructDict, DStructDict, NA, PRc Allocate a new, empty dict with the bespoke StructLayout `layout`. +| AllocUninitBespokeStructDict, DStructDict, NA, PRc + + Allocate a new dict with the bespoke StructLayout `layout` and insertion order + specified by `slots`. This instruction does not set the elements of the dict; + use InitStructElem for that. + +| InitStructElem, ND, SStructDict S(Cell), CRc + + Initialize the element at slot `slot` corresponding to `key` in StructDict S0 + with S1. This instruction assumes that S1 has already been inc-reffed. + Used with the result of AllocUninitBespokeStructDict. + | NewBespokeStructDict, DStructDict, S(StkPtr), PRc|CRc Allocate a new dict with the bespoke StructLayout `layout`. Then, init the diff --git a/hphp/runtime/base/bespoke/struct-dict.cpp b/hphp/runtime/base/bespoke/struct-dict.cpp index 29b553feb54..6f21077bd0e 100644 --- a/hphp/runtime/base/bespoke/struct-dict.cpp +++ b/hphp/runtime/base/bespoke/struct-dict.cpp @@ -268,19 +268,31 @@ StructDict* StructDict::AllocStructDict(const StructLayout* layout) { return MakeReserve(HeaderKind::BespokeDict, false, layout); } -StructDict* StructDict::MakeStructDict( - const StructLayout* layout, uint32_t size, - const uint8_t* slots, const TypedValue* tvs) { +StructDict* StructDict::AllocUninitStructDict(const StructLayout* layout, + uint32_t size, + const uint8_t* slots) { auto const result = MakeReserve( HeaderKind::BespokeDict, false, layout); - result->m_size = size; + result->m_size = size; auto const positions = result->rawPositions(); assertx(uintptr_t(positions) % 8 == 0); assertx(uintptr_t(slots) % 8 == 0); memcpy8(positions, slots, size); + assertx(result->checkInvariants()); + assertx(result->layout() == layout); + assertx(result->size() == size); + + return result; +} + +StructDict* StructDict::MakeStructDict( + const StructLayout* layout, uint32_t size, + const uint8_t* slots, const TypedValue* tvs) { + auto const result = AllocUninitStructDict(layout, size, slots); + auto const types = result->rawTypes(); auto const vals = result->rawValues(); diff --git a/hphp/runtime/base/bespoke/struct-dict.h b/hphp/runtime/base/bespoke/struct-dict.h index f719dac4224..54045dba8f1 100644 --- a/hphp/runtime/base/bespoke/struct-dict.h +++ b/hphp/runtime/base/bespoke/struct-dict.h @@ -44,6 +44,9 @@ struct StructDict : public BespokeArray { // The `slots` array must be aligned to 8 bytes, and any leftover bytes must // be padded with KindOfUninit, because type bytes follow position bytes. + static StructDict* AllocUninitStructDict(const StructLayout* layout, + uint32_t size, + const uint8_t* slots); static StructDict* MakeStructDict( const StructLayout* layout, uint32_t size, const uint8_t* slots, const TypedValue* vals); diff --git a/hphp/runtime/base/runtime-option.h b/hphp/runtime/base/runtime-option.h index 579fc60dda6..75668282615 100644 --- a/hphp/runtime/base/runtime-option.h +++ b/hphp/runtime/base/runtime-option.h @@ -930,6 +930,7 @@ struct RuntimeOption { /* How many elements to inline for packed- or mixed-array inits. */ \ F(uint32_t, HHIRMaxInlineInitPackedElements, 8) \ F(uint32_t, HHIRMaxInlineInitMixedElements, 4) \ + F(uint32_t, HHIRMaxInlineInitStructElements, 8) \ F(double, HHIROffsetArrayProfileThreshold, 0.85) \ F(double, HHIRSmallArrayProfileThreshold, 0.8) \ F(double, HHIRMissingArrayProfileThreshold, 0.8) \ diff --git a/hphp/runtime/vm/jit/dce.cpp b/hphp/runtime/vm/jit/dce.cpp index e6d41a86306..b3f6253ef7c 100644 --- a/hphp/runtime/vm/jit/dce.cpp +++ b/hphp/runtime/vm/jit/dce.cpp @@ -280,6 +280,7 @@ bool canDCE(IRInstruction* inst) { case ConstructClosure: case AllocBespokeStructDict: case AllocStructDict: + case AllocUninitBespokeStructDict: case AllocVec: case GetDictPtrIter: case AdvanceDictPtrIter: @@ -443,6 +444,7 @@ bool canDCE(IRInstruction* inst) { case InitThrowableFileAndLine: case ConstructInstance: case InitDictElem: + case InitStructElem: case InitVecElem: case InitVecElemLoop: case NewKeysetArray: @@ -1097,7 +1099,7 @@ void fullDCE(IRUnit& unit) { if (inst->is(DecRef)) { rcInsts[srcInst].decs.emplace_back(inst); } - if (inst->is(InitVecElem, StClosureArg)) { + if (inst->is(InitVecElem, InitStructElem, StClosureArg)) { if (ix == 0) rcInsts[srcInst].aux.emplace_back(inst); } } diff --git a/hphp/runtime/vm/jit/extra-data.cpp b/hphp/runtime/vm/jit/extra-data.cpp index 9d7edf183e7..98c3eb4031f 100644 --- a/hphp/runtime/vm/jit/extra-data.cpp +++ b/hphp/runtime/vm/jit/extra-data.cpp @@ -58,6 +58,18 @@ std::string NewBespokeStructData::show() const { return os.str(); } +std::string AllocUninitBespokeStructData::show() const { + std::ostringstream os; + os << layout.describe() << ",("; + auto delimiter = ""; + for (auto i = 0; i < numSlots; i++) { + os << delimiter << slots[i]; + delimiter = ","; + } + os << ')'; + return os.str(); +} + size_t LoggingProfileData::stableHash() const { return profile ? profile->key.stableHash() : 0; } diff --git a/hphp/runtime/vm/jit/extra-data.h b/hphp/runtime/vm/jit/extra-data.h index 46067103e6f..17201b81055 100644 --- a/hphp/runtime/vm/jit/extra-data.h +++ b/hphp/runtime/vm/jit/extra-data.h @@ -1592,6 +1592,39 @@ struct NewBespokeStructData : IRExtraData { Slot* slots; }; +struct AllocUninitBespokeStructData : IRExtraData { + AllocUninitBespokeStructData(ArrayLayout layout, + uint32_t numSlots, + Slot* slots) + : layout(layout), numSlots(numSlots), slots(slots) {} + + std::string show() const; + + size_t stableHash() const { + auto hash = folly::hash::hash_combine( + std::hash()(layout.toUint16()), + std::hash()(numSlots) + ); + for (auto i = 0; i < numSlots; i++) { + hash = folly::hash::hash_combine(hash, slots[i]); + } + return hash; + } + + bool equals(const AllocUninitBespokeStructData& o) const { + if (layout != o.layout) return false; + if (numSlots != o.numSlots) return false; + for (auto i = 0; i < numSlots; i++) { + if (slots[i] != o.slots[i]) return false; + } + return true; + } + + ArrayLayout layout; + uint32_t numSlots; + Slot* slots; +}; + struct PackedArrayData : IRExtraData { explicit PackedArrayData(uint32_t size) : size(size) {} std::string show() const { return folly::format("{}", size).str(); } @@ -2706,12 +2739,14 @@ X(NewStructDict, NewStructData); X(NewRecord, NewStructData); X(AllocStructDict, NewStructData); X(AllocBespokeStructDict, ArrayLayoutData); +X(AllocUninitBespokeStructDict, AllocUninitBespokeStructData); X(NewBespokeStructDict, NewBespokeStructData); X(AllocVec, PackedArrayData); X(NewKeysetArray, NewKeysetArrayData); X(InitVecElemLoop, InitPackedArrayLoopData); X(InitVecElem, IndexData); X(InitDictElem, KeyedIndexData); +X(InitStructElem, KeyedIndexData); X(CreateAAWH, CreateAAWHData); X(CountWHNotDone, CountWHNotDoneData); X(CheckDictOffset, IndexData); diff --git a/hphp/runtime/vm/jit/ir-instruction.cpp b/hphp/runtime/vm/jit/ir-instruction.cpp index b628e3ac286..1d8d41581bb 100644 --- a/hphp/runtime/vm/jit/ir-instruction.cpp +++ b/hphp/runtime/vm/jit/ir-instruction.cpp @@ -209,6 +209,7 @@ bool consumesRefImpl(const IRInstruction* inst, int srcNo) { return srcNo == 0; case InitVecElem: + case InitStructElem: return srcNo == 1; case InitVecElemLoop: @@ -528,10 +529,14 @@ Type loggingArrLikeReturn(const IRInstruction* inst) { } Type structDictReturn(const IRInstruction* inst) { - assertx(inst->is(AllocBespokeStructDict, NewBespokeStructDict)); - auto const layout = inst->is(AllocBespokeStructDict) - ? inst->extra()->layout - : inst->extra()->layout; + assertx(inst->is(AllocBespokeStructDict, + AllocUninitBespokeStructDict, + NewBespokeStructDict)); + auto const layout = inst->is(AllocBespokeStructDict) ? + inst->extra()->layout : + inst->is(AllocUninitBespokeStructDict) ? + inst->extra()->layout : + inst->extra()->layout; return TDict.narrowToLayout(layout); } diff --git a/hphp/runtime/vm/jit/ir-opcode.cpp b/hphp/runtime/vm/jit/ir-opcode.cpp index e84100c5e58..f11dc568fe7 100644 --- a/hphp/runtime/vm/jit/ir-opcode.cpp +++ b/hphp/runtime/vm/jit/ir-opcode.cpp @@ -496,6 +496,7 @@ bool opcodeMayRaise(Opcode opc) { case AKExistsKeyset: case AllocBespokeStructDict: case AllocStructDict: + case AllocUninitBespokeStructDict: case AllocVec: case AndInt: case AssertLoc: @@ -670,6 +671,7 @@ bool opcodeMayRaise(Opcode opc) { case InitDictElem: case InitObjMemoSlots: case InitObjProps: + case InitStructElem: case InitThrowableFileAndLine: case InitVecElem: case InitVecElemLoop: diff --git a/hphp/runtime/vm/jit/irgen-bespoke.cpp b/hphp/runtime/vm/jit/irgen-bespoke.cpp index 95e44ba285f..9c87c175af5 100644 --- a/hphp/runtime/vm/jit/irgen-bespoke.cpp +++ b/hphp/runtime/vm/jit/irgen-bespoke.cpp @@ -1071,21 +1071,41 @@ bool specializeStructSource(IRGS& env, SrcKey sk, ArrayLayout layout) { auto const imms = getImmVector(sk.pc()); auto const size = safe_cast(imms.size()); - auto const data = [&]() -> NewBespokeStructData { - auto const slayout = bespoke::StructLayout::As(layout.bespokeLayout()); - auto const slots = size ? new (env.unit.arena()) Slot[size] : nullptr; - for (auto i = 0; i < size; i++) { - auto const key = curUnit(env)->lookupLitstrId(imms.vec32()[i]); - slots[i] = slayout->keySlot(key); - assertx(slots[i] != kInvalidSlot); - } - return {layout, spOffBCFromIRSP(env), safe_cast(size), slots}; - }(); + auto const slayout = bespoke::StructLayout::As(layout.bespokeLayout()); + auto const slots = size ? new (env.unit.arena()) Slot[size] : nullptr; + for (auto i = 0; i < size; i++) { + auto const key = curUnit(env)->lookupLitstrId(imms.vec32()[i]); + slots[i] = slayout->keySlot(key); + assertx(slots[i] != kInvalidSlot); + } - auto const arr = gen(env, NewBespokeStructDict, data, sp(env)); - discard(env, size); + if (size > RuntimeOption::EvalHHIRMaxInlineInitStructElements) { + auto const data = NewBespokeStructData {layout, + spOffBCFromIRSP(env), + safe_cast(size), + slots}; + auto const arr = gen(env, NewBespokeStructDict, data, sp(env)); + discard(env, size); + push(env, arr); + return true; + } + auto const data = + AllocUninitBespokeStructData {layout, safe_cast(size), slots}; + auto const arr = gen(env, AllocUninitBespokeStructDict, data); + for (auto i = 0; i < size; ++i) { + auto idx = size - i - 1; + auto const key = curUnit(env)->lookupLitstrId(imms.vec32()[idx]); + gen( + env, + InitStructElem, + KeyedIndexData {slots[idx], key}, + arr, + popC(env, DataTypeGeneric) + ); + } push(env, arr); return true; + } bool specializeSource(IRGS& env, SrcKey sk) { diff --git a/hphp/runtime/vm/jit/irlower-bespoke.cpp b/hphp/runtime/vm/jit/irlower-bespoke.cpp index a5e4c05aa61..314890e8497 100644 --- a/hphp/runtime/vm/jit/irlower-bespoke.cpp +++ b/hphp/runtime/vm/jit/irlower-bespoke.cpp @@ -460,6 +460,39 @@ void cgAllocBespokeStructDict(IRLS& env, const IRInstruction* inst) { cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::None, args); } +void cgAllocUninitBespokeStructDict(IRLS& env, const IRInstruction* inst) { + auto const extra = inst->extra(); + auto& v = vmain(env); + + auto const n = static_cast((extra->numSlots + 7) & ~7); + auto const slots = reinterpret_cast(v.allocData(n / 8)); + for (auto i = 0; i < extra->numSlots; i++) { + slots[i] = safe_cast(extra->slots[i]); + } + for (auto i = extra->numSlots; i < n; i++) { + slots[i] = static_cast(KindOfUninit); + } + + auto const target = CallSpec::direct(StructDict::AllocUninitStructDict); + auto const args = argGroup(env, inst) + .imm(StructLayout::As(extra->layout.bespokeLayout())) + .imm(extra->numSlots) + .dataPtr(slots); + + cgCallHelper(v, env, target, callDest(env, inst), SyncOptions::None, args); +} + +void cgInitStructElem(IRLS& env, const IRInstruction* inst) { + auto const arr = inst->src(0); + auto const layout = arr->type().arrSpec().layout(); + auto const slayout = bespoke::StructLayout::As(layout.bespokeLayout()); + auto const rarr = srcLoc(env, inst, 0).reg(); + auto const slot = inst->extra()->index; + auto const type = rarr[slayout->typeOffsetForSlot(slot)]; + auto const data = rarr[slayout->valueOffsetForSlot(slot)]; + storeTV(vmain(env), inst->src(1)->type(), srcLoc(env, inst, 1), type, data); +} + void cgNewBespokeStructDict(IRLS& env, const IRInstruction* inst) { auto const sp = srcLoc(env, inst, 0).reg(); auto const extra = inst->extra(); diff --git a/hphp/runtime/vm/jit/memory-effects.cpp b/hphp/runtime/vm/jit/memory-effects.cpp index 3af6000efaa..9bd2274554c 100644 --- a/hphp/runtime/vm/jit/memory-effects.cpp +++ b/hphp/runtime/vm/jit/memory-effects.cpp @@ -1058,6 +1058,13 @@ MemEffects memory_effects_impl(const IRInstruction& inst) { return PureStore { AElemS { arr, key }, val, arr }; } + case InitStructElem: { + auto const arr = inst.src(0); + auto const val = inst.src(1); + auto const key = inst.extra()->key; + return PureStore { AElemS { arr, key }, val, arr }; + } + case LdMonotypeDictVal: { // TODO(mcolavita): When we have a type-array-elem method to get the key // of an arbitrary array-like type, use that to narrow this load. @@ -1419,6 +1426,7 @@ MemEffects memory_effects_impl(const IRInstruction& inst) { case AllocVec: case AllocStructDict: case AllocBespokeStructDict: + case AllocUninitBespokeStructDict: case ConvDblToStr: case ConvIntToStr: return IrrelevantEffects {}; -- 2.11.4.GIT