Bug 1885489 - Part 5: Add SnapshotIterator::readInt32(). r=iain
[gecko.git] / js / src / jit / RegisterAllocator.cpp
blob085ea1a19086137de606b659dff41d379a508b88
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #include "jit/RegisterAllocator.h"
9 using namespace js;
10 using namespace js::jit;
12 #ifdef DEBUG
13 bool AllocationIntegrityState::record() {
14 // Ignore repeated record() calls.
15 if (!instructions.empty()) {
16 return true;
19 if (!instructions.appendN(InstructionInfo(), graph.numInstructions())) {
20 return false;
23 if (!virtualRegisters.appendN((LDefinition*)nullptr,
24 graph.numVirtualRegisters())) {
25 return false;
28 if (!blocks.reserve(graph.numBlocks())) {
29 return false;
31 for (size_t i = 0; i < graph.numBlocks(); i++) {
32 blocks.infallibleAppend(BlockInfo());
33 LBlock* block = graph.getBlock(i);
34 MOZ_ASSERT(block->mir()->id() == i);
36 BlockInfo& blockInfo = blocks[i];
37 if (!blockInfo.phis.reserve(block->numPhis())) {
38 return false;
41 for (size_t j = 0; j < block->numPhis(); j++) {
42 blockInfo.phis.infallibleAppend(InstructionInfo());
43 InstructionInfo& info = blockInfo.phis[j];
44 LPhi* phi = block->getPhi(j);
45 MOZ_ASSERT(phi->numDefs() == 1);
46 uint32_t vreg = phi->getDef(0)->virtualRegister();
47 virtualRegisters[vreg] = phi->getDef(0);
48 if (!info.outputs.append(*phi->getDef(0))) {
49 return false;
51 for (size_t k = 0, kend = phi->numOperands(); k < kend; k++) {
52 if (!info.inputs.append(*phi->getOperand(k))) {
53 return false;
58 for (LInstructionIterator iter = block->begin(); iter != block->end();
59 iter++) {
60 LInstruction* ins = *iter;
61 InstructionInfo& info = instructions[ins->id()];
63 for (size_t k = 0; k < ins->numTemps(); k++) {
64 if (!ins->getTemp(k)->isBogusTemp()) {
65 uint32_t vreg = ins->getTemp(k)->virtualRegister();
66 virtualRegisters[vreg] = ins->getTemp(k);
68 if (!info.temps.append(*ins->getTemp(k))) {
69 return false;
72 for (size_t k = 0; k < ins->numDefs(); k++) {
73 if (!ins->getDef(k)->isBogusTemp()) {
74 uint32_t vreg = ins->getDef(k)->virtualRegister();
75 virtualRegisters[vreg] = ins->getDef(k);
77 if (!info.outputs.append(*ins->getDef(k))) {
78 return false;
81 for (LInstruction::InputIterator alloc(*ins); alloc.more();
82 alloc.next()) {
83 if (!info.inputs.append(**alloc)) {
84 return false;
90 return true;
93 bool AllocationIntegrityState::check() {
94 MOZ_ASSERT(!instructions.empty());
96 # ifdef JS_JITSPEW
97 if (JitSpewEnabled(JitSpew_RegAlloc)) {
98 dump();
100 # endif
101 for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
102 LBlock* block = graph.getBlock(blockIndex);
104 // Check that all instruction inputs and outputs have been assigned an
105 // allocation.
106 for (LInstructionIterator iter = block->begin(); iter != block->end();
107 iter++) {
108 LInstruction* ins = *iter;
110 for (LInstruction::InputIterator alloc(*ins); alloc.more();
111 alloc.next()) {
112 MOZ_ASSERT(!alloc->isUse());
115 for (size_t i = 0; i < ins->numDefs(); i++) {
116 LDefinition* def = ins->getDef(i);
117 MOZ_ASSERT(!def->output()->isUse());
119 LDefinition oldDef = instructions[ins->id()].outputs[i];
120 MOZ_ASSERT_IF(
121 oldDef.policy() == LDefinition::MUST_REUSE_INPUT,
122 *def->output() == *ins->getOperand(oldDef.getReusedInput()));
125 for (size_t i = 0; i < ins->numTemps(); i++) {
126 LDefinition* temp = ins->getTemp(i);
127 MOZ_ASSERT_IF(!temp->isBogusTemp(), temp->output()->isRegister());
129 LDefinition oldTemp = instructions[ins->id()].temps[i];
130 MOZ_ASSERT_IF(
131 oldTemp.policy() == LDefinition::MUST_REUSE_INPUT,
132 *temp->output() == *ins->getOperand(oldTemp.getReusedInput()));
137 // Check that the register assignment and move groups preserve the original
138 // semantics of the virtual registers. Each virtual register has a single
139 // write (owing to the SSA representation), but the allocation may move the
140 // written value around between registers and memory locations along
141 // different paths through the script.
143 // For each use of an allocation, follow the physical value which is read
144 // backward through the script, along all paths to the value's virtual
145 // register's definition.
146 for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
147 LBlock* block = graph.getBlock(blockIndex);
148 for (LInstructionIterator iter = block->begin(); iter != block->end();
149 iter++) {
150 LInstruction* ins = *iter;
151 const InstructionInfo& info = instructions[ins->id()];
153 LSafepoint* safepoint = ins->safepoint();
154 if (safepoint) {
155 for (size_t i = 0; i < ins->numTemps(); i++) {
156 if (ins->getTemp(i)->isBogusTemp()) {
157 continue;
159 uint32_t vreg = info.temps[i].virtualRegister();
160 LAllocation* alloc = ins->getTemp(i)->output();
161 checkSafepointAllocation(ins, vreg, *alloc);
163 MOZ_ASSERT_IF(ins->isCall(), safepoint->liveRegs().emptyFloat() &&
164 safepoint->liveRegs().emptyGeneral());
167 size_t inputIndex = 0;
168 for (LInstruction::InputIterator alloc(*ins); alloc.more();
169 inputIndex++, alloc.next()) {
170 LAllocation oldInput = info.inputs[inputIndex];
171 if (!oldInput.isUse()) {
172 continue;
175 uint32_t vreg = oldInput.toUse()->virtualRegister();
177 if (safepoint && !oldInput.toUse()->usedAtStart()) {
178 checkSafepointAllocation(ins, vreg, **alloc);
181 // Temps must never alias inputs (even at-start uses) unless explicitly
182 // requested.
183 for (size_t i = 0; i < ins->numTemps(); i++) {
184 if (ins->getTemp(i)->isBogusTemp()) {
185 continue;
187 LAllocation* tempAlloc = ins->getTemp(i)->output();
189 // Fixed uses and fixed temps are allowed to alias.
190 if (oldInput.toUse()->isFixedRegister() && info.temps[i].isFixed()) {
191 continue;
194 // MUST_REUSE_INPUT temps will alias their input.
195 if (info.temps[i].policy() == LDefinition::MUST_REUSE_INPUT &&
196 info.temps[i].getReusedInput() == inputIndex) {
197 continue;
200 MOZ_ASSERT(!tempAlloc->aliases(**alloc));
203 // Start checking at the previous instruction, in case this
204 // instruction reuses its input register for an output.
205 LInstructionReverseIterator riter = block->rbegin(ins);
206 riter++;
207 if (!checkIntegrity(block, *riter, vreg, **alloc)) {
208 return false;
211 while (!worklist.empty()) {
212 IntegrityItem item = worklist.popCopy();
213 if (!checkIntegrity(item.block, *item.block->rbegin(), item.vreg,
214 item.alloc)) {
215 return false;
222 return true;
225 bool AllocationIntegrityState::checkIntegrity(LBlock* block, LInstruction* ins,
226 uint32_t vreg,
227 LAllocation alloc) {
228 for (LInstructionReverseIterator iter(block->rbegin(ins));
229 iter != block->rend(); iter++) {
230 ins = *iter;
232 // Follow values through assignments in move groups. All assignments in
233 // a move group are considered to happen simultaneously, so stop after
234 // the first matching move is found.
235 if (ins->isMoveGroup()) {
236 LMoveGroup* group = ins->toMoveGroup();
237 for (int i = group->numMoves() - 1; i >= 0; i--) {
238 if (group->getMove(i).to() == alloc) {
239 alloc = group->getMove(i).from();
240 break;
245 const InstructionInfo& info = instructions[ins->id()];
247 // Make sure the physical location being tracked is not clobbered by
248 // another instruction, and that if the originating vreg definition is
249 // found that it is writing to the tracked location.
251 for (size_t i = 0; i < ins->numDefs(); i++) {
252 LDefinition* def = ins->getDef(i);
253 if (def->isBogusTemp()) {
254 continue;
256 if (info.outputs[i].virtualRegister() == vreg) {
257 # ifdef JS_JITSPEW
258 // If the following assertion is about to fail, print some useful info.
259 if (!(*def->output() == alloc) && JitSpewEnabled(JitSpew_RegAlloc)) {
260 CodePosition input(ins->id(), CodePosition::INPUT);
261 CodePosition output(ins->id(), CodePosition::OUTPUT);
262 JitSpew(JitSpew_RegAlloc,
263 "Instruction at %u-%u, output number %u:", input.bits(),
264 output.bits(), unsigned(i));
265 JitSpew(JitSpew_RegAlloc,
266 " Error: conflicting allocations: %s vs %s",
267 (*def->output()).toString().get(), alloc.toString().get());
269 # endif
270 MOZ_ASSERT(*def->output() == alloc);
272 // Found the original definition, done scanning.
273 return true;
274 } else {
275 MOZ_ASSERT(*def->output() != alloc);
279 for (size_t i = 0; i < ins->numTemps(); i++) {
280 LDefinition* temp = ins->getTemp(i);
281 if (!temp->isBogusTemp()) {
282 MOZ_ASSERT(*temp->output() != alloc);
286 if (ins->safepoint()) {
287 checkSafepointAllocation(ins, vreg, alloc);
291 // Phis are effectless, but change the vreg we are tracking. Check if there
292 // is one which produced this vreg. We need to follow back through the phi
293 // inputs as it is not guaranteed the register allocator filled in physical
294 // allocations for the inputs and outputs of the phis.
295 for (size_t i = 0; i < block->numPhis(); i++) {
296 const InstructionInfo& info = blocks[block->mir()->id()].phis[i];
297 LPhi* phi = block->getPhi(i);
298 if (info.outputs[0].virtualRegister() == vreg) {
299 for (size_t j = 0, jend = phi->numOperands(); j < jend; j++) {
300 uint32_t newvreg = info.inputs[j].toUse()->virtualRegister();
301 LBlock* predecessor = block->mir()->getPredecessor(j)->lir();
302 if (!addPredecessor(predecessor, newvreg, alloc)) {
303 return false;
306 return true;
310 // No phi which defined the vreg we are tracking, follow back through all
311 // predecessors with the existing vreg.
312 for (size_t i = 0, iend = block->mir()->numPredecessors(); i < iend; i++) {
313 LBlock* predecessor = block->mir()->getPredecessor(i)->lir();
314 if (!addPredecessor(predecessor, vreg, alloc)) {
315 return false;
319 return true;
322 void AllocationIntegrityState::checkSafepointAllocation(LInstruction* ins,
323 uint32_t vreg,
324 LAllocation alloc) {
325 LSafepoint* safepoint = ins->safepoint();
326 MOZ_ASSERT(safepoint);
328 if (ins->isCall() && alloc.isRegister()) {
329 return;
332 if (alloc.isRegister()) {
333 MOZ_ASSERT(safepoint->liveRegs().has(alloc.toRegister()));
336 // The |this| argument slot is implicitly included in all safepoints.
337 if (alloc.isArgument() &&
338 alloc.toArgument()->index() < THIS_FRAME_ARGSLOT + sizeof(Value)) {
339 return;
342 LDefinition::Type type = virtualRegisters[vreg]
343 ? virtualRegisters[vreg]->type()
344 : LDefinition::GENERAL;
346 switch (type) {
347 case LDefinition::OBJECT:
348 MOZ_ASSERT(safepoint->hasGcPointer(alloc));
349 break;
350 case LDefinition::STACKRESULTS:
351 MOZ_ASSERT(safepoint->hasAllWasmAnyRefsFromStackArea(alloc));
352 break;
353 case LDefinition::SLOTS:
354 MOZ_ASSERT(safepoint->hasSlotsOrElementsPointer(alloc));
355 break;
356 case LDefinition::WASM_ANYREF:
357 MOZ_ASSERT(safepoint->hasWasmAnyRef(alloc));
358 break;
359 # ifdef JS_NUNBOX32
360 // Do not assert that safepoint information for nunbox types is complete,
361 // as if a vreg for a value's components are copied in multiple places
362 // then the safepoint information may not reflect all copies. All copies
363 // of payloads must be reflected, however, for generational GC.
364 case LDefinition::TYPE:
365 break;
366 case LDefinition::PAYLOAD:
367 MOZ_ASSERT(safepoint->hasNunboxPayload(alloc));
368 break;
369 # else
370 case LDefinition::BOX:
371 MOZ_ASSERT(safepoint->hasBoxedValue(alloc));
372 break;
373 # endif
374 default:
375 break;
379 bool AllocationIntegrityState::addPredecessor(LBlock* block, uint32_t vreg,
380 LAllocation alloc) {
381 // There is no need to reanalyze if we have already seen this predecessor.
382 // We share the seen allocations across analysis of each use, as there will
383 // likely be common ground between different uses of the same vreg.
384 IntegrityItem item;
385 item.block = block;
386 item.vreg = vreg;
387 item.alloc = alloc;
388 item.index = seen.count();
390 IntegrityItemSet::AddPtr p = seen.lookupForAdd(item);
391 if (p) {
392 return true;
394 if (!seen.add(p, item)) {
395 return false;
398 return worklist.append(item);
401 void AllocationIntegrityState::dump() {
402 # ifdef JS_JITSPEW
403 JitSpewCont(JitSpew_RegAlloc, "\n");
404 JitSpew(JitSpew_RegAlloc, "Register Allocation Integrity State:");
406 for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
407 LBlock* block = graph.getBlock(blockIndex);
408 MBasicBlock* mir = block->mir();
410 JitSpewHeader(JitSpew_RegAlloc);
411 JitSpewCont(JitSpew_RegAlloc, " Block %lu",
412 static_cast<unsigned long>(blockIndex));
413 for (size_t i = 0; i < mir->numSuccessors(); i++) {
414 JitSpewCont(JitSpew_RegAlloc, " [successor %u]",
415 mir->getSuccessor(i)->id());
417 JitSpewCont(JitSpew_RegAlloc, "\n");
419 for (size_t i = 0; i < block->numPhis(); i++) {
420 const InstructionInfo& info = blocks[blockIndex].phis[i];
421 LPhi* phi = block->getPhi(i);
422 CodePosition input(block->getPhi(0)->id(), CodePosition::INPUT);
423 CodePosition output(block->getPhi(block->numPhis() - 1)->id(),
424 CodePosition::OUTPUT);
426 JitSpewHeader(JitSpew_RegAlloc);
427 JitSpewCont(JitSpew_RegAlloc, " %u-%u Phi [def %s] ", input.bits(),
428 output.bits(), phi->getDef(0)->toString().get());
429 for (size_t j = 0; j < phi->numOperands(); j++) {
430 JitSpewCont(JitSpew_RegAlloc, " [use %s]",
431 info.inputs[j].toString().get());
433 JitSpewCont(JitSpew_RegAlloc, "\n");
436 for (LInstructionIterator iter = block->begin(); iter != block->end();
437 iter++) {
438 LInstruction* ins = *iter;
439 const InstructionInfo& info = instructions[ins->id()];
441 CodePosition input(ins->id(), CodePosition::INPUT);
442 CodePosition output(ins->id(), CodePosition::OUTPUT);
444 JitSpewHeader(JitSpew_RegAlloc);
445 JitSpewCont(JitSpew_RegAlloc, " ");
446 if (input != CodePosition::MIN) {
447 JitSpewCont(JitSpew_RegAlloc, "%u-%u ", input.bits(), output.bits());
449 JitSpewCont(JitSpew_RegAlloc, "%s", ins->opName());
451 if (ins->isMoveGroup()) {
452 LMoveGroup* group = ins->toMoveGroup();
453 for (int i = group->numMoves() - 1; i >= 0; i--) {
454 JitSpewCont(JitSpew_RegAlloc, " [%s <- %s]",
455 group->getMove(i).to().toString().get(),
456 group->getMove(i).from().toString().get());
458 JitSpewCont(JitSpew_RegAlloc, "\n");
459 continue;
462 for (size_t i = 0; i < ins->numDefs(); i++) {
463 JitSpewCont(JitSpew_RegAlloc, " [def %s]",
464 ins->getDef(i)->toString().get());
467 for (size_t i = 0; i < ins->numTemps(); i++) {
468 LDefinition* temp = ins->getTemp(i);
469 if (!temp->isBogusTemp()) {
470 JitSpewCont(JitSpew_RegAlloc, " [temp v%u %s]",
471 info.temps[i].virtualRegister(), temp->toString().get());
475 size_t index = 0;
476 for (LInstruction::InputIterator alloc(*ins); alloc.more();
477 alloc.next()) {
478 JitSpewCont(JitSpew_RegAlloc, " [use %s",
479 info.inputs[index++].toString().get());
480 if (!alloc->isConstant()) {
481 JitSpewCont(JitSpew_RegAlloc, " %s", alloc->toString().get());
483 JitSpewCont(JitSpew_RegAlloc, "]");
486 JitSpewCont(JitSpew_RegAlloc, "\n");
490 // Print discovered allocations at the ends of blocks, in the order they
491 // were discovered.
493 Vector<IntegrityItem, 20, SystemAllocPolicy> seenOrdered;
494 if (!seenOrdered.appendN(IntegrityItem(), seen.count())) {
495 fprintf(stderr, "OOM while dumping allocations\n");
496 return;
499 for (IntegrityItemSet::Enum iter(seen); !iter.empty(); iter.popFront()) {
500 IntegrityItem item = iter.front();
501 seenOrdered[item.index] = item;
504 if (!seenOrdered.empty()) {
505 fprintf(stderr, "Intermediate Allocations:\n");
507 for (size_t i = 0; i < seenOrdered.length(); i++) {
508 IntegrityItem item = seenOrdered[i];
509 fprintf(stderr, " block %u reg v%u alloc %s\n", item.block->mir()->id(),
510 item.vreg, item.alloc.toString().get());
514 fprintf(stderr, "\n");
515 # endif
517 #endif // DEBUG
519 const CodePosition CodePosition::MAX(UINT_MAX);
520 const CodePosition CodePosition::MIN(0);
522 bool RegisterAllocator::init() {
523 if (!insData.init(mir, graph.numInstructions())) {
524 return false;
527 if (!entryPositions.reserve(graph.numBlocks()) ||
528 !exitPositions.reserve(graph.numBlocks())) {
529 return false;
532 for (size_t i = 0; i < graph.numBlocks(); i++) {
533 LBlock* block = graph.getBlock(i);
534 for (LInstructionIterator ins = block->begin(); ins != block->end();
535 ins++) {
536 insData[ins->id()] = *ins;
538 for (size_t j = 0; j < block->numPhis(); j++) {
539 LPhi* phi = block->getPhi(j);
540 insData[phi->id()] = phi;
543 CodePosition entry =
544 block->numPhis() != 0
545 ? CodePosition(block->getPhi(0)->id(), CodePosition::INPUT)
546 : inputOf(block->firstInstructionWithId());
547 CodePosition exit = outputOf(block->lastInstructionWithId());
549 MOZ_ASSERT(block->mir()->id() == i);
550 entryPositions.infallibleAppend(entry);
551 exitPositions.infallibleAppend(exit);
554 return true;
557 LMoveGroup* RegisterAllocator::getInputMoveGroup(LInstruction* ins) {
558 MOZ_ASSERT(!ins->fixReuseMoves());
559 if (ins->inputMoves()) {
560 return ins->inputMoves();
563 LMoveGroup* moves = LMoveGroup::New(alloc());
564 ins->setInputMoves(moves);
565 ins->block()->insertBefore(ins, moves);
566 return moves;
569 LMoveGroup* RegisterAllocator::getFixReuseMoveGroup(LInstruction* ins) {
570 if (ins->fixReuseMoves()) {
571 return ins->fixReuseMoves();
574 LMoveGroup* moves = LMoveGroup::New(alloc());
575 ins->setFixReuseMoves(moves);
576 ins->block()->insertBefore(ins, moves);
577 return moves;
580 LMoveGroup* RegisterAllocator::getMoveGroupAfter(LInstruction* ins) {
581 if (ins->movesAfter()) {
582 return ins->movesAfter();
585 LMoveGroup* moves = LMoveGroup::New(alloc());
586 ins->setMovesAfter(moves);
588 ins->block()->insertAfter(ins, moves);
589 return moves;
592 void RegisterAllocator::dumpInstructions(const char* who) {
593 #ifdef JS_JITSPEW
594 JitSpew(JitSpew_RegAlloc, "LIR instructions %s", who);
596 for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
597 LBlock* block = graph.getBlock(blockIndex);
598 MBasicBlock* mir = block->mir();
600 JitSpewHeader(JitSpew_RegAlloc);
601 JitSpewCont(JitSpew_RegAlloc, " Block %lu",
602 static_cast<unsigned long>(blockIndex));
603 for (size_t i = 0; i < mir->numSuccessors(); i++) {
604 JitSpewCont(JitSpew_RegAlloc, " [successor %u]",
605 mir->getSuccessor(i)->id());
607 JitSpewCont(JitSpew_RegAlloc, "\n");
609 for (size_t i = 0; i < block->numPhis(); i++) {
610 LPhi* phi = block->getPhi(i);
612 JitSpewHeader(JitSpew_RegAlloc);
613 JitSpewCont(JitSpew_RegAlloc, " %u-%u Phi [def %s]",
614 inputOf(phi).bits(), outputOf(phi).bits(),
615 phi->getDef(0)->toString().get());
616 for (size_t j = 0; j < phi->numOperands(); j++) {
617 JitSpewCont(JitSpew_RegAlloc, " [use %s]",
618 phi->getOperand(j)->toString().get());
620 JitSpewCont(JitSpew_RegAlloc, "\n");
623 for (LInstructionIterator iter = block->begin(); iter != block->end();
624 iter++) {
625 LInstruction* ins = *iter;
627 JitSpewHeader(JitSpew_RegAlloc);
628 JitSpewCont(JitSpew_RegAlloc, " ");
629 if (ins->id() != 0) {
630 JitSpewCont(JitSpew_RegAlloc, "%u-%u ", inputOf(ins).bits(),
631 outputOf(ins).bits());
633 JitSpewCont(JitSpew_RegAlloc, "%s", ins->opName());
635 if (ins->isMoveGroup()) {
636 LMoveGroup* group = ins->toMoveGroup();
637 for (int i = group->numMoves() - 1; i >= 0; i--) {
638 // Use two printfs, as LAllocation::toString is not reentant.
639 JitSpewCont(JitSpew_RegAlloc, " [%s",
640 group->getMove(i).to().toString().get());
641 JitSpewCont(JitSpew_RegAlloc, " <- %s]",
642 group->getMove(i).from().toString().get());
644 JitSpewCont(JitSpew_RegAlloc, "\n");
645 continue;
648 for (size_t i = 0; i < ins->numDefs(); i++) {
649 JitSpewCont(JitSpew_RegAlloc, " [def %s]",
650 ins->getDef(i)->toString().get());
653 for (size_t i = 0; i < ins->numTemps(); i++) {
654 LDefinition* temp = ins->getTemp(i);
655 if (!temp->isBogusTemp()) {
656 JitSpewCont(JitSpew_RegAlloc, " [temp %s]", temp->toString().get());
660 for (LInstruction::InputIterator alloc(*ins); alloc.more();
661 alloc.next()) {
662 if (!alloc->isBogus()) {
663 JitSpewCont(JitSpew_RegAlloc, " [use %s]", alloc->toString().get());
667 JitSpewCont(JitSpew_RegAlloc, "\n");
670 JitSpewCont(JitSpew_RegAlloc, "\n");
671 #endif // JS_JITSPEW