1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
9 #include "mozilla/ScopeExit.h"
11 #include <type_traits>
13 #include "jit/JitSpewer.h"
14 #include "jit/MIR-wasm.h"
16 #include "jit/MIRGenerator.h"
17 #include "js/Printf.h"
18 #include "util/Unicode.h"
21 using namespace js::jit
;
23 const char* const js::jit::LIROpNames
[] = {
24 #define OPNAME(op, ...) #op,
25 LIR_OPCODE_LIST(OPNAME
)
29 LIRGraph::LIRGraph(MIRGraph
* mir
)
30 : constantPool_(mir
->alloc()),
31 constantPoolMap_(mir
->alloc()),
32 safepoints_(mir
->alloc()),
33 nonCallSafepoints_(mir
->alloc()),
34 numVirtualRegisters_(0),
35 numInstructions_(1), // First id is 1.
37 argumentSlotCount_(0),
40 bool LIRGraph::addConstantToPool(const Value
& v
, uint32_t* index
) {
41 ConstantPoolMap::AddPtr p
= constantPoolMap_
.lookupForAdd(v
);
46 *index
= constantPool_
.length();
47 return constantPool_
.append(v
) && constantPoolMap_
.add(p
, v
, *index
);
50 bool LIRGraph::noteNeedsSafepoint(LInstruction
* ins
) {
51 // Instructions with safepoints must be in linear order.
52 MOZ_ASSERT_IF(!safepoints_
.empty(), safepoints_
.back()->id() < ins
->id());
53 if (!ins
->isCall() && !nonCallSafepoints_
.append(ins
)) {
56 return safepoints_
.append(ins
);
60 void LIRGraph::dump(GenericPrinter
& out
) {
61 for (size_t i
= 0; i
< numBlocks(); i
++) {
62 getBlock(i
)->dump(out
);
67 void LIRGraph::dump() {
74 LBlock::LBlock(MBasicBlock
* from
)
75 : block_(from
), entryMoveGroup_(nullptr), exitMoveGroup_(nullptr) {
76 from
->assignLir(this);
79 bool LBlock::init(TempAllocator
& alloc
) {
80 // Count the number of LPhis we'll need.
82 for (MPhiIterator
i(block_
->phisBegin()), e(block_
->phisEnd()); i
!= e
; ++i
) {
84 switch (phi
->type()) {
86 numLPhis
+= BOX_PIECES
;
89 numLPhis
+= INT64_PIECES
;
97 // Allocate space for the LPhis.
98 if (!phis_
.init(alloc
, numLPhis
)) {
102 // For each MIR phi, set up LIR phis as appropriate. We'll fill in their
103 // operands on each incoming edge, and set their definitions at the start of
104 // their defining block.
106 size_t numPreds
= block_
->numPredecessors();
107 for (MPhiIterator
i(block_
->phisBegin()), e(block_
->phisEnd()); i
!= e
; ++i
) {
109 MOZ_ASSERT(phi
->numOperands() == numPreds
);
112 switch (phi
->type()) {
114 numPhis
= BOX_PIECES
;
117 numPhis
= INT64_PIECES
;
123 for (int i
= 0; i
< numPhis
; i
++) {
124 LAllocation
* inputs
= alloc
.allocateArray
<LAllocation
>(numPreds
);
129 void* addr
= &phis_
[phiIndex
++];
130 LPhi
* lphi
= new (addr
) LPhi(phi
, inputs
);
131 lphi
->setBlock(this);
137 const LInstruction
* LBlock::firstInstructionWithId() const {
138 for (LInstructionIterator
i(instructions_
.begin()); i
!= instructions_
.end();
147 LMoveGroup
* LBlock::getEntryMoveGroup(TempAllocator
& alloc
) {
148 if (entryMoveGroup_
) {
149 return entryMoveGroup_
;
151 entryMoveGroup_
= LMoveGroup::New(alloc
);
152 insertBefore(*begin(), entryMoveGroup_
);
153 return entryMoveGroup_
;
156 LMoveGroup
* LBlock::getExitMoveGroup(TempAllocator
& alloc
) {
157 if (exitMoveGroup_
) {
158 return exitMoveGroup_
;
160 exitMoveGroup_
= LMoveGroup::New(alloc
);
161 insertBefore(*rbegin(), exitMoveGroup_
);
162 return exitMoveGroup_
;
166 void LBlock::dump(GenericPrinter
& out
) {
167 out
.printf("block%u:\n", mir()->id());
168 for (size_t i
= 0; i
< numPhis(); ++i
) {
169 getPhi(i
)->dump(out
);
172 for (LInstructionIterator iter
= begin(); iter
!= end(); iter
++) {
174 if (iter
->safepoint()) {
175 out
.printf(" SAFEPOINT(0x%p) ", iter
->safepoint());
181 void LBlock::dump() {
182 Fprinter
out(stderr
);
188 static size_t TotalOperandCount(LRecoverInfo
* recoverInfo
) {
190 for (LRecoverInfo::OperandIter
it(recoverInfo
); !it
; ++it
) {
191 if (!it
->isRecoveredOnBailout()) {
198 LRecoverInfo::LRecoverInfo(TempAllocator
& alloc
)
199 : instructions_(alloc
), recoverOffset_(INVALID_RECOVER_OFFSET
) {}
201 LRecoverInfo
* LRecoverInfo::New(MIRGenerator
* gen
, MResumePoint
* mir
) {
202 LRecoverInfo
* recoverInfo
= new (gen
->alloc()) LRecoverInfo(gen
->alloc());
203 if (!recoverInfo
|| !recoverInfo
->init(mir
)) {
207 JitSpew(JitSpew_IonSnapshots
, "Generating LIR recover info %p from MIR (%p)",
208 (void*)recoverInfo
, (void*)mir
);
213 // de-virtualise MResumePoint::getOperand calls.
214 template <typename Node
>
215 bool LRecoverInfo::appendOperands(Node
* ins
) {
216 for (size_t i
= 0, end
= ins
->numOperands(); i
< end
; i
++) {
217 MDefinition
* def
= ins
->getOperand(i
);
219 // As there is no cycle in the data-flow (without MPhi), checking for
220 // isInWorkList implies that the definition is already in the
221 // instruction vector, and not processed by a caller of the current
223 if (def
->isRecoveredOnBailout() && !def
->isInWorklist()) {
224 if (!appendDefinition(def
)) {
233 bool LRecoverInfo::appendDefinition(MDefinition
* def
) {
234 MOZ_ASSERT(def
->isRecoveredOnBailout());
235 def
->setInWorklist();
236 auto clearWorklistFlagOnFailure
=
237 mozilla::MakeScopeExit([&] { def
->setNotInWorklist(); });
239 if (!appendOperands(def
)) {
243 if (!instructions_
.append(def
)) {
247 clearWorklistFlagOnFailure
.release();
251 bool LRecoverInfo::appendResumePoint(MResumePoint
* rp
) {
252 // Stores should be recovered first.
253 if (!rp
->storesEmpty()) {
254 hasSideEffects_
= true;
256 for (auto iter(rp
->storesBegin()), end(rp
->storesEnd()); iter
!= end
;
258 if (!appendDefinition(iter
->operand
)) {
263 if (rp
->caller() && !appendResumePoint(rp
->caller())) {
267 if (!appendOperands(rp
)) {
271 return instructions_
.append(rp
);
274 bool LRecoverInfo::init(MResumePoint
* rp
) {
275 // Before exiting this function, remove temporary flags from all definitions
276 // added in the vector.
277 auto clearWorklistFlags
= mozilla::MakeScopeExit([&] {
278 for (MNode
** it
= begin(); it
!= end(); it
++) {
279 if (!(*it
)->isDefinition()) {
282 (*it
)->toDefinition()->setNotInWorklist();
286 // Sort operations in the order in which we need to restore the stack. This
287 // implies that outer frames, as well as operations needed to recover the
288 // current frame, are located before the current frame. The inner-most
289 // resume point should be the last element in the list.
290 if (!appendResumePoint(rp
)) {
294 MOZ_ASSERT(mir() == rp
);
298 LSnapshot::LSnapshot(LRecoverInfo
* recoverInfo
, BailoutKind kind
)
300 recoverInfo_(recoverInfo
),
301 snapshotOffset_(INVALID_SNAPSHOT_OFFSET
),
302 numSlots_(TotalOperandCount(recoverInfo
) * BOX_PIECES
),
303 bailoutKind_(kind
) {}
305 bool LSnapshot::init(MIRGenerator
* gen
) {
306 slots_
= gen
->allocate
<LAllocation
>(numSlots_
);
310 LSnapshot
* LSnapshot::New(MIRGenerator
* gen
, LRecoverInfo
* recover
,
312 LSnapshot
* snapshot
= new (gen
->alloc()) LSnapshot(recover
, kind
);
313 if (!snapshot
|| !snapshot
->init(gen
)) {
317 JitSpew(JitSpew_IonSnapshots
, "Generating LIR snapshot %p from recover (%p)",
318 (void*)snapshot
, (void*)recover
);
323 void LSnapshot::rewriteRecoveredInput(LUse input
) {
324 // Mark any operands to this snapshot with the same value as input as being
325 // equal to the instruction's result.
326 for (size_t i
= 0; i
< numEntries(); i
++) {
327 if (getEntry(i
)->isUse() &&
328 getEntry(i
)->toUse()->virtualRegister() == input
.virtualRegister()) {
329 setEntry(i
, LUse(input
.virtualRegister(), LUse::RECOVERED_INPUT
));
335 void LNode::printName(GenericPrinter
& out
, Opcode op
) {
336 static const char* const names
[] = {
337 # define LIROP(x) #x,
338 LIR_OPCODE_LIST(LIROP
)
341 const char* name
= names
[uint32_t(op
)];
342 size_t len
= strlen(name
);
343 for (size_t i
= 0; i
< len
; i
++) {
344 out
.printf("%c", unicode::ToLowerCase(name
[i
]));
348 void LNode::printName(GenericPrinter
& out
) { printName(out
, op()); }
351 bool LAllocation::aliases(const LAllocation
& other
) const {
352 if (isFloatReg() && other
.isFloatReg()) {
353 return toFloatReg()->reg().aliases(other
.toFloatReg()->reg());
355 return *this == other
;
359 static const char* DefTypeName(LDefinition::Type type
) {
361 case LDefinition::GENERAL
:
363 case LDefinition::INT32
:
365 case LDefinition::OBJECT
:
367 case LDefinition::SLOTS
:
369 case LDefinition::WASM_ANYREF
:
371 case LDefinition::FLOAT32
:
373 case LDefinition::DOUBLE
:
375 case LDefinition::SIMD128
:
377 case LDefinition::STACKRESULTS
:
378 return "stackresults";
380 case LDefinition::TYPE
:
382 case LDefinition::PAYLOAD
:
385 case LDefinition::BOX
:
389 MOZ_CRASH("Invalid type");
392 UniqueChars
LDefinition::toString() const {
393 AutoEnterOOMUnsafeRegion oomUnsafe
;
397 buf
= JS_smprintf("bogus");
399 buf
= JS_smprintf("v%u<%s>", virtualRegister(), DefTypeName(type()));
401 if (policy() == LDefinition::FIXED
) {
402 buf
= JS_sprintf_append(std::move(buf
), ":%s",
403 output()->toString().get());
404 } else if (policy() == LDefinition::MUST_REUSE_INPUT
) {
405 buf
= JS_sprintf_append(std::move(buf
), ":tied(%u)", getReusedInput());
411 oomUnsafe
.crash("LDefinition::toString()");
417 static UniqueChars
PrintUse(const LUse
* use
) {
418 switch (use
->policy()) {
420 return JS_smprintf("v%u:R", use
->virtualRegister());
422 return JS_smprintf("v%u:F:%s", use
->virtualRegister(),
423 AnyRegister::FromCode(use
->registerCode()).name());
425 return JS_smprintf("v%u:A", use
->virtualRegister());
426 case LUse::KEEPALIVE
:
427 return JS_smprintf("v%u:KA", use
->virtualRegister());
429 return JS_smprintf("v%u:S", use
->virtualRegister());
430 case LUse::RECOVERED_INPUT
:
431 return JS_smprintf("v%u:RI", use
->virtualRegister());
433 MOZ_CRASH("invalid use policy");
437 UniqueChars
LAllocation::toString() const {
438 AutoEnterOOMUnsafeRegion oomUnsafe
;
442 buf
= JS_smprintf("bogus");
445 case LAllocation::CONSTANT_VALUE
:
446 case LAllocation::CONSTANT_INDEX
: {
447 const MConstant
* c
= toConstant();
450 buf
= JS_smprintf("%d", c
->toInt32());
453 buf
= JS_smprintf("%" PRId64
, c
->toInt64());
455 case MIRType::IntPtr
:
456 buf
= JS_smprintf("%" PRIxPTR
, c
->toIntPtr());
458 case MIRType::String
:
459 // If a JSContext is a available, output the actual string
460 if (JSContext
* cx
= TlsContext
.get()) {
463 oomUnsafe
.crash("LAllocation::toString()");
465 spr
.putString(cx
, c
->toString());
468 buf
= JS_smprintf("string");
471 case MIRType::Symbol
:
472 buf
= JS_smprintf("sym");
474 case MIRType::Object
:
476 buf
= JS_smprintf("obj %p", c
->toObjectOrNull());
479 buf
= JS_smprintf("shape");
482 if (c
->isTypeRepresentableAsDouble()) {
483 buf
= JS_smprintf("%g", c
->numberToDouble());
485 buf
= JS_smprintf("const");
489 case LAllocation::GPR
:
490 buf
= JS_smprintf("%s", toGeneralReg()->reg().name());
492 case LAllocation::FPU
:
493 buf
= JS_smprintf("%s", toFloatReg()->reg().name());
495 case LAllocation::STACK_SLOT
:
496 buf
= JS_smprintf("stack:%u", toStackSlot()->slot());
498 case LAllocation::ARGUMENT_SLOT
:
499 buf
= JS_smprintf("arg:%u", toArgument()->index());
501 case LAllocation::STACK_AREA
:
502 buf
= JS_smprintf("stackarea:%u+%u", toStackArea()->base(),
503 toStackArea()->size());
505 case LAllocation::USE
:
506 buf
= PrintUse(toUse());
514 oomUnsafe
.crash("LAllocation::toString()");
520 void LAllocation::dump() const { fprintf(stderr
, "%s\n", toString().get()); }
522 void LDefinition::dump() const { fprintf(stderr
, "%s\n", toString().get()); }
524 template <typename T
>
525 static void PrintOperands(GenericPrinter
& out
, T
* node
) {
526 size_t numOperands
= node
->numOperands();
528 for (size_t i
= 0; i
< numOperands
; i
++) {
529 out
.printf(" (%s)", node
->getOperand(i
)->toString().get());
530 if (i
!= numOperands
- 1) {
536 void LNode::printOperands(GenericPrinter
& out
) {
538 toMoveGroup()->printOperands(out
);
542 out
.printf(" (%d)", toInteger()->i32());
546 out
.printf(" (%" PRId64
")", toInteger64()->i64());
551 PrintOperands(out
, toPhi());
553 PrintOperands(out
, toInstruction());
558 void LInstruction::assignSnapshot(LSnapshot
* snapshot
) {
559 MOZ_ASSERT(!snapshot_
);
560 snapshot_
= snapshot
;
563 if (JitSpewEnabled(JitSpew_IonSnapshots
)) {
564 JitSpewHeader(JitSpew_IonSnapshots
);
565 Fprinter
& out
= JitSpewPrinter();
566 out
.printf("Assigning snapshot %p to instruction %p (", (void*)snapshot
,
575 static size_t NumSuccessorsHelper(const LNode
* ins
) { return 0; }
577 template <size_t Succs
, size_t Operands
, size_t Temps
>
578 static size_t NumSuccessorsHelper(
579 const LControlInstructionHelper
<Succs
, Operands
, Temps
>* ins
) {
583 static size_t NumSuccessors(const LInstruction
* ins
) {
586 MOZ_CRASH("Unexpected LIR op");
588 case LNode::Opcode::x: \
589 return NumSuccessorsHelper(ins->to##x());
590 LIR_OPCODE_LIST(LIROP
)
595 static MBasicBlock
* GetSuccessorHelper(const LNode
* ins
, size_t i
) {
596 MOZ_CRASH("Unexpected instruction with successors");
599 template <size_t Succs
, size_t Operands
, size_t Temps
>
600 static MBasicBlock
* GetSuccessorHelper(
601 const LControlInstructionHelper
<Succs
, Operands
, Temps
>* ins
, size_t i
) {
602 return ins
->getSuccessor(i
);
605 static MBasicBlock
* GetSuccessor(const LInstruction
* ins
, size_t i
) {
606 MOZ_ASSERT(i
< NumSuccessors(ins
));
610 MOZ_CRASH("Unexpected LIR op");
612 case LNode::Opcode::x: \
613 return GetSuccessorHelper(ins->to##x(), i);
614 LIR_OPCODE_LIST(LIROP
)
621 void LNode::dump(GenericPrinter
& out
) {
622 if (numDefs() != 0) {
624 for (size_t i
= 0; i
< numDefs(); i
++) {
625 const LDefinition
* def
=
626 isPhi() ? toPhi()->getDef(i
) : toInstruction()->getDef(i
);
627 out
.printf("%s", def
->toString().get());
628 if (i
!= numDefs() - 1) {
638 if (isInstruction()) {
639 LInstruction
* ins
= toInstruction();
640 size_t numTemps
= ins
->numTemps();
643 for (size_t i
= 0; i
< numTemps
; i
++) {
644 out
.printf("%s", ins
->getTemp(i
)->toString().get());
645 if (i
!= numTemps
- 1) {
652 size_t numSuccessors
= NumSuccessors(ins
);
653 if (numSuccessors
> 0) {
655 for (size_t i
= 0; i
< numSuccessors
; i
++) {
656 MBasicBlock
* succ
= GetSuccessor(ins
, i
);
657 out
.printf("block%u", succ
->id());
658 if (i
!= numSuccessors
- 1) {
668 Fprinter
out(stderr
);
674 const char* LNode::getExtraName() const {
677 MOZ_CRASH("Unexpected LIR op");
679 case LNode::Opcode::x: \
680 return to##x()->extraName();
681 LIR_OPCODE_LIST(LIROP
)
687 void LInstruction::initSafepoint(TempAllocator
& alloc
) {
688 MOZ_ASSERT(!safepoint_
);
689 safepoint_
= new (alloc
) LSafepoint(alloc
);
690 MOZ_ASSERT(safepoint_
);
693 bool LMoveGroup::add(LAllocation from
, LAllocation to
, LDefinition::Type type
) {
695 MOZ_ASSERT(from
!= to
);
696 for (size_t i
= 0; i
< moves_
.length(); i
++) {
697 MOZ_ASSERT(to
!= moves_
[i
].to());
700 // Check that SIMD moves are aligned according to ABI requirements.
702 # ifdef ENABLE_WASM_SIMD
703 // Alignment is not currently required for SIMD on x86/x64/arm64. See also
704 // CodeGeneratorShared::CodeGeneratorShared and in general everywhere
705 // SimdMemoryAignment is used. Likely, alignment requirements will return.
706 # if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) || \
707 defined(JS_CODEGEN_ARM64)
708 // No need for any check on x86/x64/arm64.
710 # error "Need to consider SIMD alignment on this target."
711 // The following code may be of use if we need alignment checks on
712 // some future target.
713 //if (LDefinition(type).type() == LDefinition::SIMD128) {
714 // MOZ_ASSERT(from.isMemory() || from.isFloatReg());
715 // if (from.isMemory()) {
716 // if (from.isArgument()) {
717 // MOZ_ASSERT(from.toArgument()->index() % SimdMemoryAlignment == 0);
719 // MOZ_ASSERT(from.toStackSlot()->slot() % SimdMemoryAlignment == 0);
722 // MOZ_ASSERT(to.isMemory() || to.isFloatReg());
723 // if (to.isMemory()) {
724 // if (to.isArgument()) {
725 // MOZ_ASSERT(to.toArgument()->index() % SimdMemoryAlignment == 0);
727 // MOZ_ASSERT(to.toStackSlot()->slot() % SimdMemoryAlignment == 0);
736 return moves_
.append(LMove(from
, to
, type
));
739 bool LMoveGroup::addAfter(LAllocation from
, LAllocation to
,
740 LDefinition::Type type
) {
741 // Transform the operands to this move so that performing the result
742 // simultaneously with existing moves in the group will have the same
743 // effect as if the original move took place after the existing moves.
745 for (size_t i
= 0; i
< moves_
.length(); i
++) {
746 if (moves_
[i
].to() == from
) {
747 from
= moves_
[i
].from();
756 for (size_t i
= 0; i
< moves_
.length(); i
++) {
757 if (to
== moves_
[i
].to()) {
758 moves_
[i
] = LMove(from
, to
, type
);
763 return add(from
, to
, type
);
767 void LMoveGroup::printOperands(GenericPrinter
& out
) {
768 for (size_t i
= 0; i
< numMoves(); i
++) {
769 const LMove
& move
= getMove(i
);
770 out
.printf(" [%s -> %s", move
.from().toString().get(),
771 move
.to().toString().get());
772 out
.printf(", %s", DefTypeName(move
.type()));
774 if (i
!= numMoves() - 1) {
782 static_assert(!std::is_polymorphic_v<L##x>, \
783 "LIR instructions should not have virtual methods");
784 LIR_OPCODE_LIST(LIROP
)