1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "jit/RegisterAllocator.h"
10 using namespace js::jit
;
13 bool AllocationIntegrityState::record() {
14 // Ignore repeated record() calls.
15 if (!instructions
.empty()) {
19 if (!instructions
.appendN(InstructionInfo(), graph
.numInstructions())) {
23 if (!virtualRegisters
.appendN((LDefinition
*)nullptr,
24 graph
.numVirtualRegisters())) {
28 if (!blocks
.reserve(graph
.numBlocks())) {
31 for (size_t i
= 0; i
< graph
.numBlocks(); i
++) {
32 blocks
.infallibleAppend(BlockInfo());
33 LBlock
* block
= graph
.getBlock(i
);
34 MOZ_ASSERT(block
->mir()->id() == i
);
36 BlockInfo
& blockInfo
= blocks
[i
];
37 if (!blockInfo
.phis
.reserve(block
->numPhis())) {
41 for (size_t j
= 0; j
< block
->numPhis(); j
++) {
42 blockInfo
.phis
.infallibleAppend(InstructionInfo());
43 InstructionInfo
& info
= blockInfo
.phis
[j
];
44 LPhi
* phi
= block
->getPhi(j
);
45 MOZ_ASSERT(phi
->numDefs() == 1);
46 uint32_t vreg
= phi
->getDef(0)->virtualRegister();
47 virtualRegisters
[vreg
] = phi
->getDef(0);
48 if (!info
.outputs
.append(*phi
->getDef(0))) {
51 for (size_t k
= 0, kend
= phi
->numOperands(); k
< kend
; k
++) {
52 if (!info
.inputs
.append(*phi
->getOperand(k
))) {
58 for (LInstructionIterator iter
= block
->begin(); iter
!= block
->end();
60 LInstruction
* ins
= *iter
;
61 InstructionInfo
& info
= instructions
[ins
->id()];
63 for (size_t k
= 0; k
< ins
->numTemps(); k
++) {
64 if (!ins
->getTemp(k
)->isBogusTemp()) {
65 uint32_t vreg
= ins
->getTemp(k
)->virtualRegister();
66 virtualRegisters
[vreg
] = ins
->getTemp(k
);
68 if (!info
.temps
.append(*ins
->getTemp(k
))) {
72 for (size_t k
= 0; k
< ins
->numDefs(); k
++) {
73 if (!ins
->getDef(k
)->isBogusTemp()) {
74 uint32_t vreg
= ins
->getDef(k
)->virtualRegister();
75 virtualRegisters
[vreg
] = ins
->getDef(k
);
77 if (!info
.outputs
.append(*ins
->getDef(k
))) {
81 for (LInstruction::InputIterator
alloc(*ins
); alloc
.more();
83 if (!info
.inputs
.append(**alloc
)) {
93 bool AllocationIntegrityState::check() {
94 MOZ_ASSERT(!instructions
.empty());
97 if (JitSpewEnabled(JitSpew_RegAlloc
)) {
101 for (size_t blockIndex
= 0; blockIndex
< graph
.numBlocks(); blockIndex
++) {
102 LBlock
* block
= graph
.getBlock(blockIndex
);
104 // Check that all instruction inputs and outputs have been assigned an
106 for (LInstructionIterator iter
= block
->begin(); iter
!= block
->end();
108 LInstruction
* ins
= *iter
;
110 for (LInstruction::InputIterator
alloc(*ins
); alloc
.more();
112 MOZ_ASSERT(!alloc
->isUse());
115 for (size_t i
= 0; i
< ins
->numDefs(); i
++) {
116 LDefinition
* def
= ins
->getDef(i
);
117 MOZ_ASSERT(!def
->output()->isUse());
119 LDefinition oldDef
= instructions
[ins
->id()].outputs
[i
];
121 oldDef
.policy() == LDefinition::MUST_REUSE_INPUT
,
122 *def
->output() == *ins
->getOperand(oldDef
.getReusedInput()));
125 for (size_t i
= 0; i
< ins
->numTemps(); i
++) {
126 LDefinition
* temp
= ins
->getTemp(i
);
127 MOZ_ASSERT_IF(!temp
->isBogusTemp(), temp
->output()->isRegister());
129 LDefinition oldTemp
= instructions
[ins
->id()].temps
[i
];
131 oldTemp
.policy() == LDefinition::MUST_REUSE_INPUT
,
132 *temp
->output() == *ins
->getOperand(oldTemp
.getReusedInput()));
137 // Check that the register assignment and move groups preserve the original
138 // semantics of the virtual registers. Each virtual register has a single
139 // write (owing to the SSA representation), but the allocation may move the
140 // written value around between registers and memory locations along
141 // different paths through the script.
143 // For each use of an allocation, follow the physical value which is read
144 // backward through the script, along all paths to the value's virtual
145 // register's definition.
146 for (size_t blockIndex
= 0; blockIndex
< graph
.numBlocks(); blockIndex
++) {
147 LBlock
* block
= graph
.getBlock(blockIndex
);
148 for (LInstructionIterator iter
= block
->begin(); iter
!= block
->end();
150 LInstruction
* ins
= *iter
;
151 const InstructionInfo
& info
= instructions
[ins
->id()];
153 LSafepoint
* safepoint
= ins
->safepoint();
155 for (size_t i
= 0; i
< ins
->numTemps(); i
++) {
156 if (ins
->getTemp(i
)->isBogusTemp()) {
159 uint32_t vreg
= info
.temps
[i
].virtualRegister();
160 LAllocation
* alloc
= ins
->getTemp(i
)->output();
161 checkSafepointAllocation(ins
, vreg
, *alloc
);
163 MOZ_ASSERT_IF(ins
->isCall(), safepoint
->liveRegs().emptyFloat() &&
164 safepoint
->liveRegs().emptyGeneral());
167 size_t inputIndex
= 0;
168 for (LInstruction::InputIterator
alloc(*ins
); alloc
.more();
169 inputIndex
++, alloc
.next()) {
170 LAllocation oldInput
= info
.inputs
[inputIndex
];
171 if (!oldInput
.isUse()) {
175 uint32_t vreg
= oldInput
.toUse()->virtualRegister();
177 if (safepoint
&& !oldInput
.toUse()->usedAtStart()) {
178 checkSafepointAllocation(ins
, vreg
, **alloc
);
181 // Temps must never alias inputs (even at-start uses) unless explicitly
183 for (size_t i
= 0; i
< ins
->numTemps(); i
++) {
184 if (ins
->getTemp(i
)->isBogusTemp()) {
187 LAllocation
* tempAlloc
= ins
->getTemp(i
)->output();
189 // Fixed uses and fixed temps are allowed to alias.
190 if (oldInput
.toUse()->isFixedRegister() && info
.temps
[i
].isFixed()) {
194 // MUST_REUSE_INPUT temps will alias their input.
195 if (info
.temps
[i
].policy() == LDefinition::MUST_REUSE_INPUT
&&
196 info
.temps
[i
].getReusedInput() == inputIndex
) {
200 MOZ_ASSERT(!tempAlloc
->aliases(**alloc
));
203 // Start checking at the previous instruction, in case this
204 // instruction reuses its input register for an output.
205 LInstructionReverseIterator riter
= block
->rbegin(ins
);
207 if (!checkIntegrity(block
, *riter
, vreg
, **alloc
)) {
211 while (!worklist
.empty()) {
212 IntegrityItem item
= worklist
.popCopy();
213 if (!checkIntegrity(item
.block
, *item
.block
->rbegin(), item
.vreg
,
225 bool AllocationIntegrityState::checkIntegrity(LBlock
* block
, LInstruction
* ins
,
228 for (LInstructionReverseIterator
iter(block
->rbegin(ins
));
229 iter
!= block
->rend(); iter
++) {
232 // Follow values through assignments in move groups. All assignments in
233 // a move group are considered to happen simultaneously, so stop after
234 // the first matching move is found.
235 if (ins
->isMoveGroup()) {
236 LMoveGroup
* group
= ins
->toMoveGroup();
237 for (int i
= group
->numMoves() - 1; i
>= 0; i
--) {
238 if (group
->getMove(i
).to() == alloc
) {
239 alloc
= group
->getMove(i
).from();
245 const InstructionInfo
& info
= instructions
[ins
->id()];
247 // Make sure the physical location being tracked is not clobbered by
248 // another instruction, and that if the originating vreg definition is
249 // found that it is writing to the tracked location.
251 for (size_t i
= 0; i
< ins
->numDefs(); i
++) {
252 LDefinition
* def
= ins
->getDef(i
);
253 if (def
->isBogusTemp()) {
256 if (info
.outputs
[i
].virtualRegister() == vreg
) {
258 // If the following assertion is about to fail, print some useful info.
259 if (!(*def
->output() == alloc
) && JitSpewEnabled(JitSpew_RegAlloc
)) {
260 CodePosition
input(ins
->id(), CodePosition::INPUT
);
261 CodePosition
output(ins
->id(), CodePosition::OUTPUT
);
262 JitSpew(JitSpew_RegAlloc
,
263 "Instruction at %u-%u, output number %u:", input
.bits(),
264 output
.bits(), unsigned(i
));
265 JitSpew(JitSpew_RegAlloc
,
266 " Error: conflicting allocations: %s vs %s",
267 (*def
->output()).toString().get(), alloc
.toString().get());
270 MOZ_ASSERT(*def
->output() == alloc
);
272 // Found the original definition, done scanning.
275 MOZ_ASSERT(*def
->output() != alloc
);
279 for (size_t i
= 0; i
< ins
->numTemps(); i
++) {
280 LDefinition
* temp
= ins
->getTemp(i
);
281 if (!temp
->isBogusTemp()) {
282 MOZ_ASSERT(*temp
->output() != alloc
);
286 if (ins
->safepoint()) {
287 checkSafepointAllocation(ins
, vreg
, alloc
);
291 // Phis are effectless, but change the vreg we are tracking. Check if there
292 // is one which produced this vreg. We need to follow back through the phi
293 // inputs as it is not guaranteed the register allocator filled in physical
294 // allocations for the inputs and outputs of the phis.
295 for (size_t i
= 0; i
< block
->numPhis(); i
++) {
296 const InstructionInfo
& info
= blocks
[block
->mir()->id()].phis
[i
];
297 LPhi
* phi
= block
->getPhi(i
);
298 if (info
.outputs
[0].virtualRegister() == vreg
) {
299 for (size_t j
= 0, jend
= phi
->numOperands(); j
< jend
; j
++) {
300 uint32_t newvreg
= info
.inputs
[j
].toUse()->virtualRegister();
301 LBlock
* predecessor
= block
->mir()->getPredecessor(j
)->lir();
302 if (!addPredecessor(predecessor
, newvreg
, alloc
)) {
310 // No phi which defined the vreg we are tracking, follow back through all
311 // predecessors with the existing vreg.
312 for (size_t i
= 0, iend
= block
->mir()->numPredecessors(); i
< iend
; i
++) {
313 LBlock
* predecessor
= block
->mir()->getPredecessor(i
)->lir();
314 if (!addPredecessor(predecessor
, vreg
, alloc
)) {
322 void AllocationIntegrityState::checkSafepointAllocation(LInstruction
* ins
,
325 LSafepoint
* safepoint
= ins
->safepoint();
326 MOZ_ASSERT(safepoint
);
328 if (ins
->isCall() && alloc
.isRegister()) {
332 if (alloc
.isRegister()) {
333 MOZ_ASSERT(safepoint
->liveRegs().has(alloc
.toRegister()));
336 // The |this| argument slot is implicitly included in all safepoints.
337 if (alloc
.isArgument() &&
338 alloc
.toArgument()->index() < THIS_FRAME_ARGSLOT
+ sizeof(Value
)) {
342 LDefinition::Type type
= virtualRegisters
[vreg
]
343 ? virtualRegisters
[vreg
]->type()
344 : LDefinition::GENERAL
;
347 case LDefinition::OBJECT
:
348 MOZ_ASSERT(safepoint
->hasGcPointer(alloc
));
350 case LDefinition::STACKRESULTS
:
351 MOZ_ASSERT(safepoint
->hasAllWasmAnyRefsFromStackArea(alloc
));
353 case LDefinition::SLOTS
:
354 MOZ_ASSERT(safepoint
->hasSlotsOrElementsPointer(alloc
));
356 case LDefinition::WASM_ANYREF
:
357 MOZ_ASSERT(safepoint
->hasWasmAnyRef(alloc
));
360 // Do not assert that safepoint information for nunbox types is complete,
361 // as if a vreg for a value's components are copied in multiple places
362 // then the safepoint information may not reflect all copies. All copies
363 // of payloads must be reflected, however, for generational GC.
364 case LDefinition::TYPE
:
366 case LDefinition::PAYLOAD
:
367 MOZ_ASSERT(safepoint
->hasNunboxPayload(alloc
));
370 case LDefinition::BOX
:
371 MOZ_ASSERT(safepoint
->hasBoxedValue(alloc
));
379 bool AllocationIntegrityState::addPredecessor(LBlock
* block
, uint32_t vreg
,
381 // There is no need to reanalyze if we have already seen this predecessor.
382 // We share the seen allocations across analysis of each use, as there will
383 // likely be common ground between different uses of the same vreg.
388 item
.index
= seen
.count();
390 IntegrityItemSet::AddPtr p
= seen
.lookupForAdd(item
);
394 if (!seen
.add(p
, item
)) {
398 return worklist
.append(item
);
401 void AllocationIntegrityState::dump() {
403 JitSpewCont(JitSpew_RegAlloc
, "\n");
404 JitSpew(JitSpew_RegAlloc
, "Register Allocation Integrity State:");
406 for (size_t blockIndex
= 0; blockIndex
< graph
.numBlocks(); blockIndex
++) {
407 LBlock
* block
= graph
.getBlock(blockIndex
);
408 MBasicBlock
* mir
= block
->mir();
410 JitSpewHeader(JitSpew_RegAlloc
);
411 JitSpewCont(JitSpew_RegAlloc
, " Block %lu",
412 static_cast<unsigned long>(blockIndex
));
413 for (size_t i
= 0; i
< mir
->numSuccessors(); i
++) {
414 JitSpewCont(JitSpew_RegAlloc
, " [successor %u]",
415 mir
->getSuccessor(i
)->id());
417 JitSpewCont(JitSpew_RegAlloc
, "\n");
419 for (size_t i
= 0; i
< block
->numPhis(); i
++) {
420 const InstructionInfo
& info
= blocks
[blockIndex
].phis
[i
];
421 LPhi
* phi
= block
->getPhi(i
);
422 CodePosition
input(block
->getPhi(0)->id(), CodePosition::INPUT
);
423 CodePosition
output(block
->getPhi(block
->numPhis() - 1)->id(),
424 CodePosition::OUTPUT
);
426 JitSpewHeader(JitSpew_RegAlloc
);
427 JitSpewCont(JitSpew_RegAlloc
, " %u-%u Phi [def %s] ", input
.bits(),
428 output
.bits(), phi
->getDef(0)->toString().get());
429 for (size_t j
= 0; j
< phi
->numOperands(); j
++) {
430 JitSpewCont(JitSpew_RegAlloc
, " [use %s]",
431 info
.inputs
[j
].toString().get());
433 JitSpewCont(JitSpew_RegAlloc
, "\n");
436 for (LInstructionIterator iter
= block
->begin(); iter
!= block
->end();
438 LInstruction
* ins
= *iter
;
439 const InstructionInfo
& info
= instructions
[ins
->id()];
441 CodePosition
input(ins
->id(), CodePosition::INPUT
);
442 CodePosition
output(ins
->id(), CodePosition::OUTPUT
);
444 JitSpewHeader(JitSpew_RegAlloc
);
445 JitSpewCont(JitSpew_RegAlloc
, " ");
446 if (input
!= CodePosition::MIN
) {
447 JitSpewCont(JitSpew_RegAlloc
, "%u-%u ", input
.bits(), output
.bits());
449 JitSpewCont(JitSpew_RegAlloc
, "%s", ins
->opName());
451 if (ins
->isMoveGroup()) {
452 LMoveGroup
* group
= ins
->toMoveGroup();
453 for (int i
= group
->numMoves() - 1; i
>= 0; i
--) {
454 JitSpewCont(JitSpew_RegAlloc
, " [%s <- %s]",
455 group
->getMove(i
).to().toString().get(),
456 group
->getMove(i
).from().toString().get());
458 JitSpewCont(JitSpew_RegAlloc
, "\n");
462 for (size_t i
= 0; i
< ins
->numDefs(); i
++) {
463 JitSpewCont(JitSpew_RegAlloc
, " [def %s]",
464 ins
->getDef(i
)->toString().get());
467 for (size_t i
= 0; i
< ins
->numTemps(); i
++) {
468 LDefinition
* temp
= ins
->getTemp(i
);
469 if (!temp
->isBogusTemp()) {
470 JitSpewCont(JitSpew_RegAlloc
, " [temp v%u %s]",
471 info
.temps
[i
].virtualRegister(), temp
->toString().get());
476 for (LInstruction::InputIterator
alloc(*ins
); alloc
.more();
478 JitSpewCont(JitSpew_RegAlloc
, " [use %s",
479 info
.inputs
[index
++].toString().get());
480 if (!alloc
->isConstant()) {
481 JitSpewCont(JitSpew_RegAlloc
, " %s", alloc
->toString().get());
483 JitSpewCont(JitSpew_RegAlloc
, "]");
486 JitSpewCont(JitSpew_RegAlloc
, "\n");
490 // Print discovered allocations at the ends of blocks, in the order they
493 Vector
<IntegrityItem
, 20, SystemAllocPolicy
> seenOrdered
;
494 if (!seenOrdered
.appendN(IntegrityItem(), seen
.count())) {
495 fprintf(stderr
, "OOM while dumping allocations\n");
499 for (IntegrityItemSet::Enum
iter(seen
); !iter
.empty(); iter
.popFront()) {
500 IntegrityItem item
= iter
.front();
501 seenOrdered
[item
.index
] = item
;
504 if (!seenOrdered
.empty()) {
505 fprintf(stderr
, "Intermediate Allocations:\n");
507 for (size_t i
= 0; i
< seenOrdered
.length(); i
++) {
508 IntegrityItem item
= seenOrdered
[i
];
509 fprintf(stderr
, " block %u reg v%u alloc %s\n", item
.block
->mir()->id(),
510 item
.vreg
, item
.alloc
.toString().get());
514 fprintf(stderr
, "\n");
519 const CodePosition
CodePosition::MAX(UINT_MAX
);
520 const CodePosition
CodePosition::MIN(0);
522 bool RegisterAllocator::init() {
523 if (!insData
.init(mir
, graph
.numInstructions())) {
527 if (!entryPositions
.reserve(graph
.numBlocks()) ||
528 !exitPositions
.reserve(graph
.numBlocks())) {
532 for (size_t i
= 0; i
< graph
.numBlocks(); i
++) {
533 LBlock
* block
= graph
.getBlock(i
);
534 for (LInstructionIterator ins
= block
->begin(); ins
!= block
->end();
536 insData
[ins
->id()] = *ins
;
538 for (size_t j
= 0; j
< block
->numPhis(); j
++) {
539 LPhi
* phi
= block
->getPhi(j
);
540 insData
[phi
->id()] = phi
;
544 block
->numPhis() != 0
545 ? CodePosition(block
->getPhi(0)->id(), CodePosition::INPUT
)
546 : inputOf(block
->firstInstructionWithId());
547 CodePosition exit
= outputOf(block
->lastInstructionWithId());
549 MOZ_ASSERT(block
->mir()->id() == i
);
550 entryPositions
.infallibleAppend(entry
);
551 exitPositions
.infallibleAppend(exit
);
557 LMoveGroup
* RegisterAllocator::getInputMoveGroup(LInstruction
* ins
) {
558 MOZ_ASSERT(!ins
->fixReuseMoves());
559 if (ins
->inputMoves()) {
560 return ins
->inputMoves();
563 LMoveGroup
* moves
= LMoveGroup::New(alloc());
564 ins
->setInputMoves(moves
);
565 ins
->block()->insertBefore(ins
, moves
);
569 LMoveGroup
* RegisterAllocator::getFixReuseMoveGroup(LInstruction
* ins
) {
570 if (ins
->fixReuseMoves()) {
571 return ins
->fixReuseMoves();
574 LMoveGroup
* moves
= LMoveGroup::New(alloc());
575 ins
->setFixReuseMoves(moves
);
576 ins
->block()->insertBefore(ins
, moves
);
580 LMoveGroup
* RegisterAllocator::getMoveGroupAfter(LInstruction
* ins
) {
581 if (ins
->movesAfter()) {
582 return ins
->movesAfter();
585 LMoveGroup
* moves
= LMoveGroup::New(alloc());
586 ins
->setMovesAfter(moves
);
588 ins
->block()->insertAfter(ins
, moves
);
592 void RegisterAllocator::dumpInstructions(const char* who
) {
594 JitSpew(JitSpew_RegAlloc
, "LIR instructions %s", who
);
596 for (size_t blockIndex
= 0; blockIndex
< graph
.numBlocks(); blockIndex
++) {
597 LBlock
* block
= graph
.getBlock(blockIndex
);
598 MBasicBlock
* mir
= block
->mir();
600 JitSpewHeader(JitSpew_RegAlloc
);
601 JitSpewCont(JitSpew_RegAlloc
, " Block %lu",
602 static_cast<unsigned long>(blockIndex
));
603 for (size_t i
= 0; i
< mir
->numSuccessors(); i
++) {
604 JitSpewCont(JitSpew_RegAlloc
, " [successor %u]",
605 mir
->getSuccessor(i
)->id());
607 JitSpewCont(JitSpew_RegAlloc
, "\n");
609 for (size_t i
= 0; i
< block
->numPhis(); i
++) {
610 LPhi
* phi
= block
->getPhi(i
);
612 JitSpewHeader(JitSpew_RegAlloc
);
613 JitSpewCont(JitSpew_RegAlloc
, " %u-%u Phi [def %s]",
614 inputOf(phi
).bits(), outputOf(phi
).bits(),
615 phi
->getDef(0)->toString().get());
616 for (size_t j
= 0; j
< phi
->numOperands(); j
++) {
617 JitSpewCont(JitSpew_RegAlloc
, " [use %s]",
618 phi
->getOperand(j
)->toString().get());
620 JitSpewCont(JitSpew_RegAlloc
, "\n");
623 for (LInstructionIterator iter
= block
->begin(); iter
!= block
->end();
625 LInstruction
* ins
= *iter
;
627 JitSpewHeader(JitSpew_RegAlloc
);
628 JitSpewCont(JitSpew_RegAlloc
, " ");
629 if (ins
->id() != 0) {
630 JitSpewCont(JitSpew_RegAlloc
, "%u-%u ", inputOf(ins
).bits(),
631 outputOf(ins
).bits());
633 JitSpewCont(JitSpew_RegAlloc
, "%s", ins
->opName());
635 if (ins
->isMoveGroup()) {
636 LMoveGroup
* group
= ins
->toMoveGroup();
637 for (int i
= group
->numMoves() - 1; i
>= 0; i
--) {
638 // Use two printfs, as LAllocation::toString is not reentant.
639 JitSpewCont(JitSpew_RegAlloc
, " [%s",
640 group
->getMove(i
).to().toString().get());
641 JitSpewCont(JitSpew_RegAlloc
, " <- %s]",
642 group
->getMove(i
).from().toString().get());
644 JitSpewCont(JitSpew_RegAlloc
, "\n");
648 for (size_t i
= 0; i
< ins
->numDefs(); i
++) {
649 JitSpewCont(JitSpew_RegAlloc
, " [def %s]",
650 ins
->getDef(i
)->toString().get());
653 for (size_t i
= 0; i
< ins
->numTemps(); i
++) {
654 LDefinition
* temp
= ins
->getTemp(i
);
655 if (!temp
->isBogusTemp()) {
656 JitSpewCont(JitSpew_RegAlloc
, " [temp %s]", temp
->toString().get());
660 for (LInstruction::InputIterator
alloc(*ins
); alloc
.more();
662 if (!alloc
->isBogus()) {
663 JitSpewCont(JitSpew_RegAlloc
, " [use %s]", alloc
->toString().get());
667 JitSpewCont(JitSpew_RegAlloc
, "\n");
670 JitSpewCont(JitSpew_RegAlloc
, "\n");