js/src/jit/x86/Lowering-x86.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/x86/Lowering-x86.h"
   8
   9 #include "jit/Lowering.h"
  10 #include "jit/MIR.h"
  11 #include "jit/x86/Assembler-x86.h"
  12
  13 #include "jit/shared/Lowering-shared-inl.h"
  14
  15 using namespace js;
  16 using namespace js::jit;
  17
  18 LBoxAllocation LIRGeneratorX86::useBoxFixed(MDefinition* mir, Register reg1,
  19                                             Register reg2, bool useAtStart) {
  20   MOZ_ASSERT(mir->type() == MIRType::Value);
  21   MOZ_ASSERT(reg1 != reg2);
  22
  23   ensureDefined(mir);
  24   return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart),
  25                         LUse(reg2, VirtualRegisterOfPayload(mir), useAtStart));
  26 }
  27
  28 LAllocation LIRGeneratorX86::useByteOpRegister(MDefinition* mir) {
  29   return useFixed(mir, eax);
  30 }
  31
  32 LAllocation LIRGeneratorX86::useByteOpRegisterAtStart(MDefinition* mir) {
  33   return useFixedAtStart(mir, eax);
  34 }
  35
  36 LAllocation LIRGeneratorX86::useByteOpRegisterOrNonDoubleConstant(
  37     MDefinition* mir) {
  38   return useFixed(mir, eax);
  39 }
  40
  41 LDefinition LIRGeneratorX86::tempByteOpRegister() { return tempFixed(eax); }
  42
  43 void LIRGenerator::visitBox(MBox* box) {
  44   MDefinition* inner = box->getOperand(0);
  45
  46   // If the box wrapped a double, it needs a new register.
  47   if (IsFloatingPointType(inner->type())) {
  48     LDefinition spectreTemp =
  49         JitOptions.spectreValueMasking ? temp() : LDefinition::BogusTemp();
  50     defineBox(new (alloc()) LBoxFloatingPoint(useRegisterAtStart(inner),
  51                                               tempCopy(inner, 0), spectreTemp,
  52                                               inner->type()),
  53               box);
  54     return;
  55   }
  56
  57   if (box->canEmitAtUses()) {
  58     emitAtUses(box);
  59     return;
  60   }
  61
  62   if (inner->isConstant()) {
  63     defineBox(new (alloc()) LValue(inner->toConstant()->toJSValue()), box);
  64     return;
  65   }
  66
  67   LBox* lir = new (alloc()) LBox(use(inner), inner->type());
  68
  69   // Otherwise, we should not define a new register for the payload portion
  70   // of the output, so bypass defineBox().
  71   uint32_t vreg = getVirtualRegister();
  72
  73   // Note that because we're using BogusTemp(), we do not change the type of
  74   // the definition. We also do not define the first output as "TYPE",
  75   // because it has no corresponding payload at (vreg + 1). Also note that
  76   // although we copy the input's original type for the payload half of the
  77   // definition, this is only for clarity. BogusTemp() definitions are
  78   // ignored.
  79   lir->setDef(0, LDefinition(vreg, LDefinition::GENERAL));
  80   lir->setDef(1, LDefinition::BogusTemp());
  81   box->setVirtualRegister(vreg);
  82   add(lir);
  83 }
  84
  85 void LIRGenerator::visitUnbox(MUnbox* unbox) {
  86   MDefinition* inner = unbox->getOperand(0);
  87
  88   // An unbox on x86 reads in a type tag (either in memory or a register) and
  89   // a payload. Unlike most instructions consuming a box, we ask for the type
  90   // second, so that the result can re-use the first input.
  91   MOZ_ASSERT(inner->type() == MIRType::Value);
  92
  93   ensureDefined(inner);
  94
  95   if (IsFloatingPointType(unbox->type())) {
  96     LUnboxFloatingPoint* lir =
  97         new (alloc()) LUnboxFloatingPoint(useBox(inner), unbox->type());
  98     if (unbox->fallible()) {
  99       assignSnapshot(lir, unbox->bailoutKind());
 100     }
 101     define(lir, unbox);
 102     return;
 103   }
 104
 105   // Swap the order we use the box pieces so we can re-use the payload register.
 106   LUnbox* lir = new (alloc()) LUnbox;
 107   bool reusePayloadReg = !JitOptions.spectreValueMasking ||
 108                          unbox->type() == MIRType::Int32 ||
 109                          unbox->type() == MIRType::Boolean;
 110   if (reusePayloadReg) {
 111     lir->setOperand(0, usePayloadInRegisterAtStart(inner));
 112     lir->setOperand(1, useType(inner, LUse::ANY));
 113   } else {
 114     lir->setOperand(0, usePayload(inner, LUse::REGISTER));
 115     lir->setOperand(1, useType(inner, LUse::ANY));
 116   }
 117
 118   if (unbox->fallible()) {
 119     assignSnapshot(lir, unbox->bailoutKind());
 120   }
 121
 122   // Types and payloads form two separate intervals. If the type becomes dead
 123   // before the payload, it could be used as a Value without the type being
 124   // recoverable. Unbox's purpose is to eagerly kill the definition of a type
 125   // tag, so keeping both alive (for the purpose of gcmaps) is unappealing.
 126   // Instead, we create a new virtual register.
 127   if (reusePayloadReg) {
 128     defineReuseInput(lir, unbox, 0);
 129   } else {
 130     define(lir, unbox);
 131   }
 132 }
 133
 134 void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
 135   MOZ_ASSERT(opd->type() == MIRType::Value);
 136
 137   LReturn* ins = new (alloc()) LReturn(isGenerator);
 138   ins->setOperand(0, LUse(JSReturnReg_Type));
 139   ins->setOperand(1, LUse(JSReturnReg_Data));
 140   fillBoxUses(ins, 0, opd);
 141   add(ins);
 142 }
 143
 144 void LIRGeneratorX86::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
 145                                            LBlock* block, size_t lirIndex) {
 146   MDefinition* operand = phi->getOperand(inputPosition);
 147   LPhi* type = block->getPhi(lirIndex + VREG_TYPE_OFFSET);
 148   LPhi* payload = block->getPhi(lirIndex + VREG_DATA_OFFSET);
 149   type->setOperand(
 150       inputPosition,
 151       LUse(operand->virtualRegister() + VREG_TYPE_OFFSET, LUse::ANY));
 152   payload->setOperand(inputPosition,
 153                       LUse(VirtualRegisterOfPayload(operand), LUse::ANY));
 154 }
 155
 156 void LIRGeneratorX86::defineInt64Phi(MPhi* phi, size_t lirIndex) {
 157   LPhi* low = current->getPhi(lirIndex + INT64LOW_INDEX);
 158   LPhi* high = current->getPhi(lirIndex + INT64HIGH_INDEX);
 159
 160   uint32_t lowVreg = getVirtualRegister();
 161
 162   phi->setVirtualRegister(lowVreg);
 163
 164   uint32_t highVreg = getVirtualRegister();
 165   MOZ_ASSERT(lowVreg + INT64HIGH_INDEX == highVreg + INT64LOW_INDEX);
 166
 167   low->setDef(0, LDefinition(lowVreg, LDefinition::INT32));
 168   high->setDef(0, LDefinition(highVreg, LDefinition::INT32));
 169   annotate(high);
 170   annotate(low);
 171 }
 172
 173 void LIRGeneratorX86::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
 174                                          LBlock* block, size_t lirIndex) {
 175   MDefinition* operand = phi->getOperand(inputPosition);
 176   LPhi* low = block->getPhi(lirIndex + INT64LOW_INDEX);
 177   LPhi* high = block->getPhi(lirIndex + INT64HIGH_INDEX);
 178   low->setOperand(inputPosition,
 179                   LUse(operand->virtualRegister() + INT64LOW_INDEX, LUse::ANY));
 180   high->setOperand(
 181       inputPosition,
 182       LUse(operand->virtualRegister() + INT64HIGH_INDEX, LUse::ANY));
 183 }
 184
 185 void LIRGeneratorX86::lowerForALUInt64(
 186     LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
 187     MDefinition* input) {
 188   ins->setInt64Operand(0, useInt64RegisterAtStart(input));
 189   defineInt64ReuseInput(ins, mir, 0);
 190 }
 191
 192 void LIRGeneratorX86::lowerForALUInt64(
 193     LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
 194     MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
 195   ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
 196   ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs));
 197   defineInt64ReuseInput(ins, mir, 0);
 198 }
 199
 200 void LIRGeneratorX86::lowerForMulInt64(LMulI64* ins, MMul* mir,
 201                                        MDefinition* lhs, MDefinition* rhs) {
 202   bool needsTemp = true;
 203
 204   if (rhs->isConstant()) {
 205     int64_t constant = rhs->toConstant()->toInt64();
 206     int32_t shift = mozilla::FloorLog2(constant);
 207     // See special cases in CodeGeneratorX86Shared::visitMulI64.
 208     if (constant >= -1 && constant <= 2) {
 209       needsTemp = false;
 210     }
 211     if (constant > 0 && int64_t(1) << shift == constant) {
 212       needsTemp = false;
 213     }
 214   }
 215
 216   // MulI64 on x86 needs output to be in edx, eax;
 217   ins->setInt64Operand(
 218       0, useInt64Fixed(lhs, Register64(edx, eax), /*useAtStart = */ true));
 219   ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs));
 220   if (needsTemp) {
 221     ins->setTemp(0, temp());
 222   }
 223
 224   defineInt64Fixed(ins, mir,
 225                    LInt64Allocation(LAllocation(AnyRegister(edx)),
 226                                     LAllocation(AnyRegister(eax))));
 227 }
 228
 229 void LIRGenerator::visitCompareExchangeTypedArrayElement(
 230     MCompareExchangeTypedArrayElement* ins) {
 231   MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
 232   MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
 233
 234   if (Scalar::isBigIntType(ins->arrayType())) {
 235     LUse elements = useFixed(ins->elements(), esi);
 236     LAllocation index =
 237         useRegisterOrIndexConstant(ins->index(), ins->arrayType());
 238     LUse oldval = useFixed(ins->oldval(), eax);
 239     LUse newval = useFixed(ins->newval(), edx);
 240     LDefinition temp = tempFixed(ebx);
 241
 242     auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
 243         elements, index, oldval, newval, temp);
 244     defineFixed(lir, ins, LAllocation(AnyRegister(ecx)));
 245     assignSafepoint(lir, ins);
 246     return;
 247   }
 248
 249   lowerCompareExchangeTypedArrayElement(ins, /* useI386ByteRegisters = */ true);
 250 }
 251
 252 void LIRGenerator::visitAtomicExchangeTypedArrayElement(
 253     MAtomicExchangeTypedArrayElement* ins) {
 254   MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
 255   MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
 256
 257   if (Scalar::isBigIntType(ins->arrayType())) {
 258     LUse elements = useRegister(ins->elements());
 259     LAllocation index =
 260         useRegisterOrIndexConstant(ins->index(), ins->arrayType());
 261     LAllocation value = useFixed(ins->value(), edx);
 262     LInt64Definition temp = tempInt64Fixed(Register64(ecx, ebx));
 263
 264     auto* lir = new (alloc())
 265         LAtomicExchangeTypedArrayElement64(elements, index, value, temp);
 266     defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
 267     assignSafepoint(lir, ins);
 268     return;
 269   }
 270
 271   lowerAtomicExchangeTypedArrayElement(ins, /*useI386ByteRegisters=*/true);
 272 }
 273
 274 void LIRGenerator::visitAtomicTypedArrayElementBinop(
 275     MAtomicTypedArrayElementBinop* ins) {
 276   MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
 277   MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
 278
 279   if (Scalar::isBigIntType(ins->arrayType())) {
 280     LUse elements = useRegister(ins->elements());
 281     LAllocation index =
 282         useRegisterOrIndexConstant(ins->index(), ins->arrayType());
 283     LAllocation value = useFixed(ins->value(), edx);
 284     LInt64Definition temp = tempInt64Fixed(Register64(ecx, ebx));
 285
 286     // Case 1: the result of the operation is not used.
 287     //
 288     // We can omit allocating the result BigInt.
 289
 290     if (ins->isForEffect()) {
 291       LDefinition tempLow = tempFixed(eax);
 292
 293       auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
 294           elements, index, value, temp, tempLow);
 295       add(lir, ins);
 296       return;
 297     }
 298
 299     // Case 2: the result of the operation is used.
 300
 301     auto* lir = new (alloc())
 302         LAtomicTypedArrayElementBinop64(elements, index, value, temp);
 303     defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
 304     assignSafepoint(lir, ins);
 305     return;
 306   }
 307
 308   lowerAtomicTypedArrayElementBinop(ins, /* useI386ByteRegisters = */ true);
 309 }
 310
 311 void LIRGeneratorX86::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
 312   const LUse elements = useRegister(ins->elements());
 313   const LAllocation index =
 314       useRegisterOrIndexConstant(ins->index(), ins->storageType());
 315
 316   auto* lir = new (alloc()) LAtomicLoad64(elements, index, tempFixed(ebx),
 317                                           tempInt64Fixed(Register64(edx, eax)));
 318   defineFixed(lir, ins, LAllocation(AnyRegister(ecx)));
 319   assignSafepoint(lir, ins);
 320 }
 321
 322 void LIRGeneratorX86::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
 323   LUse elements = useRegister(ins->elements());
 324   LAllocation index =
 325       useRegisterOrIndexConstant(ins->index(), ins->writeType());
 326   LAllocation value = useFixed(ins->value(), edx);
 327   LInt64Definition temp1 = tempInt64Fixed(Register64(ecx, ebx));
 328   LDefinition temp2 = tempFixed(eax);
 329
 330   add(new (alloc()) LAtomicStore64(elements, index, value, temp1, temp2), ins);
 331 }
 332
 333 void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
 334   MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
 335   LWasmUint32ToDouble* lir = new (alloc())
 336       LWasmUint32ToDouble(useRegisterAtStart(ins->input()), temp());
 337   define(lir, ins);
 338 }
 339
 340 void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
 341   MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
 342   LWasmUint32ToFloat32* lir = new (alloc())
 343       LWasmUint32ToFloat32(useRegisterAtStart(ins->input()), temp());
 344   define(lir, ins);
 345 }
 346
 347 // If the base is a constant, and it is zero or its offset is zero, then
 348 // code generation will fold the values into the access.  Allocate the
 349 // pointer to a register only if that can't happen.
 350
 351 static bool OptimizableConstantAccess(MDefinition* base,
 352                                       const wasm::MemoryAccessDesc& access) {
 353   MOZ_ASSERT(base->isConstant());
 354   MOZ_ASSERT(base->type() == MIRType::Int32);
 355
 356   if (!(base->toConstant()->isInt32(0) || access.offset() == 0)) {
 357     return false;
 358   }
 359   if (access.type() == Scalar::Int64) {
 360     // For int64 accesses on 32-bit systems we will need to add another offset
 361     // of 4 to access the high part of the value; make sure this does not
 362     // overflow the value.
 363     int32_t v;
 364     if (base->toConstant()->isInt32(0)) {
 365       v = access.offset();
 366     } else {
 367       v = base->toConstant()->toInt32();
 368     }
 369     return v <= int32_t(INT32_MAX - INT64HIGH_OFFSET);
 370   }
 371   return true;
 372 }
 373
 374 void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
 375   MDefinition* base = ins->base();
 376   MOZ_ASSERT(base->type() == MIRType::Int32);
 377
 378   MDefinition* memoryBase = ins->memoryBase();
 379   MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
 380
 381   if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) {
 382     auto* lir = new (alloc())
 383         LWasmAtomicLoadI64(useRegister(memoryBase), useRegister(base),
 384                            tempFixed(ecx), tempFixed(ebx));
 385     defineInt64Fixed(lir, ins,
 386                      LInt64Allocation(LAllocation(AnyRegister(edx)),
 387                                       LAllocation(AnyRegister(eax))));
 388     return;
 389   }
 390
 391   LAllocation baseAlloc;
 392   if (!base->isConstant() || !OptimizableConstantAccess(base, ins->access())) {
 393     baseAlloc = ins->type() == MIRType::Int64 ? useRegister(base)
 394                                               : useRegisterAtStart(base);
 395   }
 396
 397   if (ins->type() != MIRType::Int64) {
 398     auto* lir =
 399         new (alloc()) LWasmLoad(baseAlloc, useRegisterAtStart(memoryBase));
 400     define(lir, ins);
 401     return;
 402   }
 403
 404   // "AtStart" register usage does not work for the 64-bit case because we
 405   // clobber two registers for the result and may need two registers for a
 406   // scaled address; we can't guarantee non-interference.
 407
 408   auto* lir = new (alloc()) LWasmLoadI64(baseAlloc, useRegister(memoryBase));
 409
 410   Scalar::Type accessType = ins->access().type();
 411   if (accessType == Scalar::Int8 || accessType == Scalar::Int16 ||
 412       accessType == Scalar::Int32) {
 413     // We use cdq to sign-extend the result and cdq demands these registers.
 414     defineInt64Fixed(lir, ins,
 415                      LInt64Allocation(LAllocation(AnyRegister(edx)),
 416                                       LAllocation(AnyRegister(eax))));
 417     return;
 418   }
 419
 420   defineInt64(lir, ins);
 421 }
 422
 423 void LIRGenerator::visitWasmStore(MWasmStore* ins) {
 424   MDefinition* base = ins->base();
 425   MOZ_ASSERT(base->type() == MIRType::Int32);
 426
 427   MDefinition* memoryBase = ins->memoryBase();
 428   MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
 429
 430   if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) {
 431     auto* lir = new (alloc())
 432         LWasmAtomicStoreI64(useRegister(memoryBase), useRegister(base),
 433                             useInt64Fixed(ins->value(), Register64(ecx, ebx)),
 434                             tempFixed(edx), tempFixed(eax));
 435     add(lir, ins);
 436     return;
 437   }
 438
 439   LAllocation baseAlloc;
 440   if (!base->isConstant() || !OptimizableConstantAccess(base, ins->access())) {
 441     baseAlloc = useRegisterAtStart(base);
 442   }
 443
 444   LAllocation valueAlloc;
 445   switch (ins->access().type()) {
 446     case Scalar::Int8:
 447     case Scalar::Uint8:
 448       // See comment for LIRGeneratorX86::useByteOpRegister.
 449       valueAlloc = useFixed(ins->value(), eax);
 450       break;
 451     case Scalar::Int16:
 452     case Scalar::Uint16:
 453     case Scalar::Int32:
 454     case Scalar::Uint32:
 455     case Scalar::Float32:
 456     case Scalar::Float64:
 457       // For now, don't allow constant values. The immediate operand affects
 458       // instruction layout which affects patching.
 459       valueAlloc = useRegisterAtStart(ins->value());
 460       break;
 461     case Scalar::Simd128:
 462 #ifdef ENABLE_WASM_SIMD
 463       valueAlloc = useRegisterAtStart(ins->value());
 464       break;
 465 #else
 466       MOZ_CRASH("unexpected array type");
 467 #endif
 468     case Scalar::Int64: {
 469       LInt64Allocation valueAlloc = useInt64RegisterAtStart(ins->value());
 470       auto* lir = new (alloc())
 471           LWasmStoreI64(baseAlloc, valueAlloc, useRegisterAtStart(memoryBase));
 472       add(lir, ins);
 473       return;
 474     }
 475     case Scalar::Uint8Clamped:
 476     case Scalar::BigInt64:
 477     case Scalar::BigUint64:
 478     case Scalar::MaxTypedArrayViewType:
 479       MOZ_CRASH("unexpected array type");
 480   }
 481
 482   auto* lir = new (alloc())
 483       LWasmStore(baseAlloc, valueAlloc, useRegisterAtStart(memoryBase));
 484   add(lir, ins);
 485 }
 486
 487 void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
 488   MDefinition* base = ins->base();
 489   MOZ_ASSERT(base->type() == MIRType::Int32);
 490
 491   MDefinition* memoryBase = ins->memoryBase();
 492   MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
 493
 494   if (ins->access().type() == Scalar::Int64) {
 495     auto* lir = new (alloc()) LWasmCompareExchangeI64(
 496         useRegisterAtStart(memoryBase), useRegisterAtStart(base),
 497         useInt64FixedAtStart(ins->oldValue(), Register64(edx, eax)),
 498         useInt64FixedAtStart(ins->newValue(), Register64(ecx, ebx)));
 499     defineInt64Fixed(lir, ins,
 500                      LInt64Allocation(LAllocation(AnyRegister(edx)),
 501                                       LAllocation(AnyRegister(eax))));
 502     return;
 503   }
 504
 505   MOZ_ASSERT(ins->access().type() < Scalar::Float32);
 506
 507   bool byteArray = byteSize(ins->access().type()) == 1;
 508
 509   // Register allocation:
 510   //
 511   // The output may not be used, but eax will be clobbered regardless
 512   // so pin the output to eax.
 513   //
 514   // oldval must be in a register.
 515   //
 516   // newval must be in a register.  If the source is a byte array
 517   // then newval must be a register that has a byte size: this must
 518   // be ebx, ecx, or edx (eax is taken).
 519   //
 520   // Bug #1077036 describes some optimization opportunities.
 521
 522   const LAllocation oldval = useRegister(ins->oldValue());
 523   const LAllocation newval =
 524       byteArray ? useFixed(ins->newValue(), ebx) : useRegister(ins->newValue());
 525
 526   LWasmCompareExchangeHeap* lir = new (alloc()) LWasmCompareExchangeHeap(
 527       useRegister(base), oldval, newval, useRegister(memoryBase));
 528
 529   lir->setAddrTemp(temp());
 530   defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
 531 }
 532
 533 void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
 534   MDefinition* memoryBase = ins->memoryBase();
 535   MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
 536
 537   if (ins->access().type() == Scalar::Int64) {
 538     MDefinition* base = ins->base();
 539     auto* lir = new (alloc()) LWasmAtomicExchangeI64(
 540         useRegister(memoryBase), useRegister(base),
 541         useInt64Fixed(ins->value(), Register64(ecx, ebx)), ins->access());
 542     defineInt64Fixed(lir, ins,
 543                      LInt64Allocation(LAllocation(AnyRegister(edx)),
 544                                       LAllocation(AnyRegister(eax))));
 545     return;
 546   }
 547
 548   const LAllocation base = useRegister(ins->base());
 549   const LAllocation value = useRegister(ins->value());
 550
 551   LWasmAtomicExchangeHeap* lir = new (alloc())
 552       LWasmAtomicExchangeHeap(base, value, useRegister(memoryBase));
 553
 554   lir->setAddrTemp(temp());
 555   if (byteSize(ins->access().type()) == 1) {
 556     defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
 557   } else {
 558     define(lir, ins);
 559   }
 560 }
 561
 562 void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
 563   MDefinition* base = ins->base();
 564   MOZ_ASSERT(base->type() == MIRType::Int32);
 565
 566   MDefinition* memoryBase = ins->memoryBase();
 567   MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
 568
 569   if (ins->access().type() == Scalar::Int64) {
 570     auto* lir = new (alloc())
 571         LWasmAtomicBinopI64(useRegister(memoryBase), useRegister(base),
 572                             useInt64Fixed(ins->value(), Register64(ecx, ebx)),
 573                             ins->access(), ins->operation());
 574     defineInt64Fixed(lir, ins,
 575                      LInt64Allocation(LAllocation(AnyRegister(edx)),
 576                                       LAllocation(AnyRegister(eax))));
 577     return;
 578   }
 579
 580   MOZ_ASSERT(ins->access().type() < Scalar::Float32);
 581
 582   bool byteArray = byteSize(ins->access().type()) == 1;
 583
 584   // Case 1: the result of the operation is not used.
 585   //
 586   // We'll emit a single instruction: LOCK ADD, LOCK SUB, LOCK AND,
 587   // LOCK OR, or LOCK XOR.  These can all take an immediate.
 588
 589   if (!ins->hasUses()) {
 590     LAllocation value;
 591     if (byteArray && !ins->value()->isConstant()) {
 592       value = useFixed(ins->value(), ebx);
 593     } else {
 594       value = useRegisterOrConstant(ins->value());
 595     }
 596     LWasmAtomicBinopHeapForEffect* lir =
 597         new (alloc()) LWasmAtomicBinopHeapForEffect(useRegister(base), value,
 598                                                     LDefinition::BogusTemp(),
 599                                                     useRegister(memoryBase));
 600     lir->setAddrTemp(temp());
 601     add(lir, ins);
 602     return;
 603   }
 604
 605   // Case 2: the result of the operation is used.
 606   //
 607   // For ADD and SUB we'll use XADD:
 608   //
 609   //    movl       value, output
 610   //    lock xaddl output, mem
 611   //
 612   // For the 8-bit variants XADD needs a byte register for the
 613   // output only, we can still set up with movl; just pin the output
 614   // to eax (or ebx / ecx / edx).
 615   //
 616   // For AND/OR/XOR we need to use a CMPXCHG loop:
 617   //
 618   //    movl          *mem, eax
 619   // L: mov           eax, temp
 620   //    andl          value, temp
 621   //    lock cmpxchg  temp, mem  ; reads eax also
 622   //    jnz           L
 623   //    ; result in eax
 624   //
 625   // Note the placement of L, cmpxchg will update eax with *mem if
 626   // *mem does not have the expected value, so reloading it at the
 627   // top of the loop would be redundant.
 628   //
 629   // We want to fix eax as the output.  We also need a temp for
 630   // the intermediate value.
 631   //
 632   // For the 8-bit variants the temp must have a byte register.
 633   //
 634   // There are optimization opportunities:
 635   //  - better 8-bit register allocation and instruction selection, Bug
 636   //  #1077036.
 637
 638   bool bitOp =
 639       !(ins->operation() == AtomicOp::Add || ins->operation() == AtomicOp::Sub);
 640   LDefinition tempDef = LDefinition::BogusTemp();
 641   LAllocation value;
 642
 643   if (byteArray) {
 644     value = useFixed(ins->value(), ebx);
 645     if (bitOp) {
 646       tempDef = tempFixed(ecx);
 647     }
 648   } else if (bitOp || ins->value()->isConstant()) {
 649     value = useRegisterOrConstant(ins->value());
 650     if (bitOp) {
 651       tempDef = temp();
 652     }
 653   } else {
 654     value = useRegisterAtStart(ins->value());
 655   }
 656
 657   LWasmAtomicBinopHeap* lir = new (alloc())
 658       LWasmAtomicBinopHeap(useRegister(base), value, tempDef,
 659                            LDefinition::BogusTemp(), useRegister(memoryBase));
 660
 661   lir->setAddrTemp(temp());
 662   if (byteArray || bitOp) {
 663     defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
 664   } else if (ins->value()->isConstant()) {
 665     define(lir, ins);
 666   } else {
 667     defineReuseInput(lir, ins, LWasmAtomicBinopHeap::valueOp);
 668   }
 669 }
 670
 671 void LIRGeneratorX86::lowerDivI64(MDiv* div) {
 672   MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
 673 }
 674
 675 void LIRGeneratorX86::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
 676   MOZ_ASSERT(div->lhs()->type() == div->rhs()->type());
 677   MOZ_ASSERT(IsNumberType(div->type()));
 678
 679   MOZ_ASSERT(div->type() == MIRType::Int64);
 680
 681   if (div->isUnsigned()) {
 682     LUDivOrModI64* lir = new (alloc())
 683         LUDivOrModI64(useInt64FixedAtStart(div->lhs(), Register64(eax, ebx)),
 684                       useInt64FixedAtStart(div->rhs(), Register64(ecx, edx)),
 685                       useFixedAtStart(div->instance(), InstanceReg));
 686     defineReturn(lir, div);
 687     return;
 688   }
 689
 690   LDivOrModI64* lir = new (alloc())
 691       LDivOrModI64(useInt64FixedAtStart(div->lhs(), Register64(eax, ebx)),
 692                    useInt64FixedAtStart(div->rhs(), Register64(ecx, edx)),
 693                    useFixedAtStart(div->instance(), InstanceReg));
 694   defineReturn(lir, div);
 695 }
 696
 697 void LIRGeneratorX86::lowerModI64(MMod* mod) {
 698   MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
 699 }
 700
 701 void LIRGeneratorX86::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
 702   MDefinition* lhs = mod->lhs();
 703   MDefinition* rhs = mod->rhs();
 704   MOZ_ASSERT(lhs->type() == rhs->type());
 705   MOZ_ASSERT(IsNumberType(mod->type()));
 706
 707   MOZ_ASSERT(mod->type() == MIRType::Int64);
 708   MOZ_ASSERT(mod->type() == MIRType::Int64);
 709
 710   if (mod->isUnsigned()) {
 711     LUDivOrModI64* lir = new (alloc())
 712         LUDivOrModI64(useInt64FixedAtStart(lhs, Register64(eax, ebx)),
 713                       useInt64FixedAtStart(rhs, Register64(ecx, edx)),
 714                       useFixedAtStart(mod->instance(), InstanceReg));
 715     defineReturn(lir, mod);
 716     return;
 717   }
 718
 719   LDivOrModI64* lir = new (alloc())
 720       LDivOrModI64(useInt64FixedAtStart(lhs, Register64(eax, ebx)),
 721                    useInt64FixedAtStart(rhs, Register64(ecx, edx)),
 722                    useFixedAtStart(mod->instance(), InstanceReg));
 723   defineReturn(lir, mod);
 724 }
 725
 726 void LIRGeneratorX86::lowerUDivI64(MDiv* div) {
 727   MOZ_CRASH("We use MWasmBuiltinDivI64 instead.");
 728 }
 729
 730 void LIRGeneratorX86::lowerUModI64(MMod* mod) {
 731   MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
 732 }
 733
 734 void LIRGeneratorX86::lowerBigIntDiv(MBigIntDiv* ins) {
 735   auto* lir = new (alloc()) LBigIntDiv(
 736       useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(eax), temp());
 737   defineFixed(lir, ins, LAllocation(AnyRegister(edx)));
 738   assignSafepoint(lir, ins);
 739 }
 740
 741 void LIRGeneratorX86::lowerBigIntMod(MBigIntMod* ins) {
 742   auto* lir = new (alloc()) LBigIntMod(
 743       useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(eax), temp());
 744   defineFixed(lir, ins, LAllocation(AnyRegister(edx)));
 745   assignSafepoint(lir, ins);
 746 }
 747
 748 void LIRGenerator::visitSubstr(MSubstr* ins) {
 749   // Due to lack of registers on x86, we reuse the string register as
 750   // temporary. As a result we only need two temporary registers and take a
 751   // bogus temporary as fifth argument.
 752   LSubstr* lir = new (alloc())
 753       LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
 754               useRegister(ins->length()), temp(), LDefinition::BogusTemp(),
 755               tempByteOpRegister());
 756   define(lir, ins);
 757   assignSafepoint(lir, ins);
 758 }
 759
 760 void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
 761   MDefinition* opd = ins->input();
 762   MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
 763
 764   LDefinition temp = tempDouble();
 765   defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd), temp), ins);
 766 }
 767
 768 void LIRGeneratorX86::lowerWasmBuiltinTruncateToInt64(
 769     MWasmBuiltinTruncateToInt64* ins) {
 770   MOZ_CRASH("We don't use it for this architecture");
 771 }
 772
 773 void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
 774   MDefinition* opd = ins->input();
 775   MOZ_ASSERT(opd->type() == MIRType::Int64);
 776   MOZ_ASSERT(IsFloatingPointType(ins->type()));
 777
 778   LDefinition maybeTemp =
 779       (ins->isUnsigned() &&
 780        ((ins->type() == MIRType::Double && AssemblerX86Shared::HasSSE3()) ||
 781         ins->type() == MIRType::Float32))
 782           ? temp()
 783           : LDefinition::BogusTemp();
 784
 785   define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd), maybeTemp),
 786          ins);
 787 }
 788
 789 void LIRGeneratorX86::lowerBuiltinInt64ToFloatingPoint(
 790     MBuiltinInt64ToFloatingPoint* ins) {
 791   MOZ_CRASH("We don't use it for this architecture");
 792 }
 793
 794 void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
 795   if (ins->isUnsigned()) {
 796     defineInt64(new (alloc())
 797                     LExtendInt32ToInt64(useRegisterAtStart(ins->input())),
 798                 ins);
 799   } else {
 800     LExtendInt32ToInt64* lir =
 801         new (alloc()) LExtendInt32ToInt64(useFixedAtStart(ins->input(), eax));
 802     defineInt64Fixed(lir, ins,
 803                      LInt64Allocation(LAllocation(AnyRegister(edx)),
 804                                       LAllocation(AnyRegister(eax))));
 805   }
 806 }
 807
 808 void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
 809   // Here we'll end up using cdq which requires input and output in (edx,eax).
 810   LSignExtendInt64* lir = new (alloc()) LSignExtendInt64(
 811       useInt64FixedAtStart(ins->input(), Register64(edx, eax)));
 812   defineInt64Fixed(lir, ins,
 813                    LInt64Allocation(LAllocation(AnyRegister(edx)),
 814                                     LAllocation(AnyRegister(eax))));
 815 }
 816
 817 // On x86 we specialize the only cases where compare is {U,}Int32 and select
 818 // is {U,}Int32.
 819 bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
 820     MCompare::CompareType compTy, MIRType insTy) {
 821   return insTy == MIRType::Int32 && (compTy == MCompare::Compare_Int32 ||
 822                                      compTy == MCompare::Compare_UInt32);
 823 }
 824
 825 void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
 826                                                    MDefinition* lhs,
 827                                                    MDefinition* rhs,
 828                                                    MCompare::CompareType compTy,
 829                                                    JSOp jsop) {
 830   MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
 831   auto* lir = new (alloc()) LWasmCompareAndSelect(
 832       useRegister(lhs), useAny(rhs), compTy, jsop,
 833       useRegisterAtStart(ins->trueExpr()), useAny(ins->falseExpr()));
 834   defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
 835 }