hphp/vixl/a64/assembler-a64.h

   1 // Copyright 2013, ARM Limited
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are met:
   6 //
   7 //   * Redistributions of source code must retain the above copyright notice,
   8 //     this list of conditions and the following disclaimer.
   9 //   * Redistributions in binary form must reproduce the above copyright notice,
  10 //     this list of conditions and the following disclaimer in the documentation
  11 //     and/or other materials provided with the distribution.
  12 //   * Neither the name of ARM Limited nor the names of its contributors may be
  13 //     used to endorse or promote products derived from this software without
  14 //     specific prior written permission.
  15 //
  16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
  17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26
  27 #ifndef VIXL_A64_ASSEMBLER_A64_H_
  28 #define VIXL_A64_ASSEMBLER_A64_H_
  29
  30 #include <list>
  31
  32 #include "hphp/util/data-block.h"
  33 #include "hphp/runtime/vm/jit/types.h"
  34
  35 #include "hphp/vixl/globals.h"
  36 #include "hphp/vixl/utils.h"
  37 #include "hphp/vixl/a64/instructions-a64.h"
  38
  39 namespace vixl {
  40
  41 typedef uint64_t RegList;
  42 constexpr int kRegListSizeInBits = sizeof(RegList) * 8;
  43
  44 // Registers.
  45
  46 // Some CPURegister methods can return Register and FPRegister types, so we
  47 // need to declare them in advance.
  48 class Register;
  49 class FPRegister;
  50 class MemOperand;
  51
  52
  53 class CPURegister {
  54  public:
  55   enum RegisterType {
  56     // The kInvalid value is used to detect uninitialized static instances,
  57     // which are always zero-initialized before any constructors are called.
  58     kInvalid = 0,
  59     kRegister,
  60     kFPRegister,
  61     kNoRegister
  62   };
  63
  64   CPURegister() : code_(0), size_(0), type_(kNoRegister) {
  65     assert(!IsValid());
  66     assert(IsNone());
  67   }
  68
  69   CPURegister(unsigned code, unsigned size, RegisterType type)
  70       : code_(code), size_(size), type_(type) {
  71     assert(IsValidOrNone());
  72   }
  73
  74   unsigned code() const {
  75     assert(IsValid());
  76     return code_;
  77   }
  78
  79   RegisterType type() const {
  80     assert(IsValidOrNone());
  81     return type_;
  82   }
  83
  84   RegList Bit() const {
  85     assert(code_ < (sizeof(RegList) * 8));
  86     return IsValid() ? (static_cast<RegList>(1) << code_) : 0;
  87   }
  88
  89   unsigned size() const {
  90     assert(IsValid());
  91     return size_;
  92   }
  93
  94   int SizeInBytes() const {
  95     assert(IsValid());
  96     assert(size() % 8 == 0);
  97     return size_ / 8;
  98   }
  99
 100   int SizeInBits() const {
 101     assert(IsValid());
 102     return size_;
 103   }
 104
 105   bool Is32Bits() const {
 106     assert(IsValid());
 107     return size_ == 32;
 108   }
 109
 110   bool Is64Bits() const {
 111     assert(IsValid());
 112     return size_ == 64;
 113   }
 114
 115   bool IsValid() const {
 116     if (IsValidRegister() || IsValidFPRegister()) {
 117       assert(!IsNone());
 118       return true;
 119     } else {
 120       assert(IsNone());
 121       return false;
 122     }
 123   }
 124
 125   bool IsValidRegister() const {
 126     return IsRegister() &&
 127            ((size_ == kWRegSize) || (size_ == kXRegSize)) &&
 128            ((code_ < kNumberOfRegisters) || (code_ == kSPRegInternalCode));
 129   }
 130
 131   bool IsValidFPRegister() const {
 132     return IsFPRegister() &&
 133            ((size_ == kSRegSize) || (size_ == kDRegSize)) &&
 134            (code_ < kNumberOfFPRegisters);
 135   }
 136
 137   bool IsNone() const {
 138     // kNoRegister types should always have size 0 and code 0.
 139     assert((type_ != kNoRegister) || (code_ == 0));
 140     assert((type_ != kNoRegister) || (size_ == 0));
 141
 142     return type_ == kNoRegister;
 143   }
 144
 145   bool Is(const CPURegister& other) const {
 146     assert(IsValidOrNone() && other.IsValidOrNone());
 147     return (code_ == other.code_) && (size_ == other.size_) &&
 148            (type_ == other.type_);
 149   }
 150
 151   inline bool IsZero() const {
 152     assert(IsValid());
 153     return IsRegister() && (code_ == kZeroRegCode);
 154   }
 155
 156   inline bool IsSP() const {
 157     assert(IsValid());
 158     return IsRegister() && (code_ == kSPRegInternalCode);
 159   }
 160
 161   inline bool IsRegister() const {
 162     return type_ == kRegister;
 163   }
 164
 165   inline bool IsFPRegister() const {
 166     return type_ == kFPRegister;
 167   }
 168
 169   const Register& W() const;
 170   const Register& X() const;
 171   const FPRegister& S() const;
 172   const FPRegister& D() const;
 173
 174   inline bool IsSameSizeAndType(const CPURegister& other) const {
 175     return (size_ == other.size_) && (type_ == other.type_);
 176   }
 177
 178  protected:
 179   unsigned code_;
 180   unsigned size_;
 181   RegisterType type_;
 182
 183  private:
 184   bool IsValidOrNone() const {
 185     return IsValid() || IsNone();
 186   }
 187 };
 188
 189
 190 class Register : public CPURegister {
 191  public:
 192   explicit Register() : CPURegister() {}
 193   inline explicit Register(const CPURegister& other)
 194       : CPURegister(other.code(), other.size(), other.type()) {
 195     assert(IsValidRegister());
 196   }
 197   explicit Register(unsigned code, unsigned size)
 198       : CPURegister(code, size, kRegister) {}
 199
 200   bool IsValid() const {
 201     assert(IsRegister() || IsNone());
 202     return IsValidRegister();
 203   }
 204
 205   MemOperand operator[](const ptrdiff_t offset) const;
 206   MemOperand operator[](const Register& offset) const;
 207
 208   static const Register& WRegFromCode(unsigned code);
 209   static const Register& XRegFromCode(unsigned code);
 210
 211   // V8 compatibility.
 212   static const int kNumRegisters = kNumberOfRegisters;
 213   static const int kNumAllocatableRegisters = kNumberOfRegisters - 1;
 214
 215  private:
 216   static const Register wregisters[];
 217   static const Register xregisters[];
 218 };
 219
 220
 221 class FPRegister : public CPURegister {
 222  public:
 223   inline FPRegister() : CPURegister() {}
 224   inline explicit FPRegister(const CPURegister& other)
 225       : CPURegister(other.code(), other.size(), other.type()) {
 226     assert(IsValidFPRegister());
 227   }
 228   inline FPRegister(unsigned code, unsigned size)
 229       : CPURegister(code, size, kFPRegister) {}
 230
 231   bool IsValid() const {
 232     assert(IsFPRegister() || IsNone());
 233     return IsValidFPRegister();
 234   }
 235
 236   static const FPRegister& SRegFromCode(unsigned code);
 237   static const FPRegister& DRegFromCode(unsigned code);
 238
 239   // V8 compatibility.
 240   static const int kNumRegisters = kNumberOfFPRegisters;
 241   static const int kNumAllocatableRegisters = kNumberOfFPRegisters - 1;
 242
 243  private:
 244   static const FPRegister sregisters[];
 245   static const FPRegister dregisters[];
 246 };
 247
 248
 249 // No*Reg is used to indicate an unused argument, or an error case. Note that
 250 // these all compare equal (using the Is() method). The Register and FPRegister
 251 // variants are provided for convenience.
 252 const Register NoReg;
 253 const FPRegister NoFPReg;
 254 const CPURegister NoCPUReg;
 255
 256
 257 #define DEFINE_REGISTERS(N)  \
 258 const Register w##N(N, kWRegSize);  \
 259 const Register x##N(N, kXRegSize);
 260 REGISTER_CODE_LIST(DEFINE_REGISTERS)
 261 #undef DEFINE_REGISTERS
 262 const Register wsp(kSPRegInternalCode, kWRegSize);
 263 const Register sp(kSPRegInternalCode, kXRegSize);
 264
 265
 266 #define DEFINE_FPREGISTERS(N)  \
 267 const FPRegister s##N(N, kSRegSize);  \
 268 const FPRegister d##N(N, kDRegSize);
 269 REGISTER_CODE_LIST(DEFINE_FPREGISTERS)
 270 #undef DEFINE_FPREGISTERS
 271
 272
 273 // Registers aliases.
 274 const Register ip0 = x16;
 275 const Register ip1 = x17;
 276 const Register lr = x30;
 277 const Register xzr = x31;
 278 const Register wzr = w31;
 279
 280
 281 // AreAliased returns true if any of the named registers overlap. Arguments
 282 // set to NoReg are ignored. The system stack pointer may be specified.
 283 bool AreAliased(const CPURegister& reg1,
 284                 const CPURegister& reg2,
 285                 const CPURegister& reg3 = NoReg,
 286                 const CPURegister& reg4 = NoReg,
 287                 const CPURegister& reg5 = NoReg,
 288                 const CPURegister& reg6 = NoReg,
 289                 const CPURegister& reg7 = NoReg,
 290                 const CPURegister& reg8 = NoReg);
 291
 292
 293 // AreSameSizeAndType returns true if all of the specified registers have the
 294 // same size, and are of the same type. The system stack pointer may be
 295 // specified. Arguments set to NoReg are ignored, as are any subsequent
 296 // arguments. At least one argument (reg1) must be valid (not NoCPUReg).
 297 bool AreSameSizeAndType(const CPURegister& reg1,
 298                         const CPURegister& reg2,
 299                         const CPURegister& reg3 = NoCPUReg,
 300                         const CPURegister& reg4 = NoCPUReg,
 301                         const CPURegister& reg5 = NoCPUReg,
 302                         const CPURegister& reg6 = NoCPUReg,
 303                         const CPURegister& reg7 = NoCPUReg,
 304                         const CPURegister& reg8 = NoCPUReg);
 305
 306
 307 // Lists of registers.
 308 class CPURegList {
 309  public:
 310   inline explicit CPURegList(CPURegister reg1,
 311                              CPURegister reg2 = NoCPUReg,
 312                              CPURegister reg3 = NoCPUReg,
 313                              CPURegister reg4 = NoCPUReg)
 314       : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()),
 315         size_(reg1.size()), type_(reg1.type()) {
 316     assert(AreSameSizeAndType(reg1, reg2, reg3, reg4));
 317     assert(IsValid());
 318   }
 319
 320   inline CPURegList(CPURegister::RegisterType type, unsigned size, RegList list)
 321       : list_(list), size_(size), type_(type) {
 322     assert(IsValid());
 323   }
 324
 325   inline CPURegList(CPURegister::RegisterType type, unsigned size,
 326                     unsigned first_reg, unsigned last_reg)
 327       : size_(size), type_(type) {
 328     assert(((type == CPURegister::kRegister) &&
 329             (last_reg < kNumberOfRegisters)) ||
 330            ((type == CPURegister::kFPRegister) &&
 331             (last_reg < kNumberOfFPRegisters)));
 332     assert(last_reg >= first_reg);
 333     list_ = (1UL << (last_reg + 1)) - 1;
 334     list_ &= ~((1UL << first_reg) - 1);
 335     assert(IsValid());
 336   }
 337
 338   inline CPURegister::RegisterType type() const {
 339     assert(IsValid());
 340     return type_;
 341   }
 342
 343   // Combine another CPURegList into this one. Registers that already exist in
 344   // this list are left unchanged. The type and size of the registers in the
 345   // 'other' list must match those in this list.
 346   void Combine(const CPURegList& other) {
 347     assert(IsValid());
 348     assert(other.type() == type_);
 349     assert(other.RegisterSizeInBits() == size_);
 350     list_ |= other.list();
 351   }
 352
 353   // Remove every register in the other CPURegList from this one. Registers that
 354   // do not exist in this list are ignored. The type and size of the registers
 355   // in the 'other' list must match those in this list.
 356   void Remove(const CPURegList& other) {
 357     assert(IsValid());
 358     assert(other.type() == type_);
 359     assert(other.RegisterSizeInBits() == size_);
 360     list_ &= ~other.list();
 361   }
 362
 363   // Variants of Combine and Remove which take a single register.
 364   inline void Combine(const CPURegister& other) {
 365     assert(other.type() == type_);
 366     assert(other.size() == size_);
 367     Combine(other.code());
 368   }
 369
 370   inline void Remove(const CPURegister& other) {
 371     assert(other.type() == type_);
 372     assert(other.size() == size_);
 373     Remove(other.code());
 374   }
 375
 376   // Variants of Combine and Remove which take a single register by its code;
 377   // the type and size of the register is inferred from this list.
 378   inline void Combine(int code) {
 379     assert(IsValid());
 380     assert(CPURegister(code, size_, type_).IsValid());
 381     list_ |= (1UL << code);
 382   }
 383
 384   inline void Remove(int code) {
 385     assert(IsValid());
 386     assert(CPURegister(code, size_, type_).IsValid());
 387     list_ &= ~(1UL << code);
 388   }
 389
 390   inline RegList list() const {
 391     assert(IsValid());
 392     return list_;
 393   }
 394
 395   // Remove all callee-saved registers from the list. This can be useful when
 396   // preparing registers for an AAPCS64 function call, for example.
 397   void RemoveCalleeSaved();
 398
 399   CPURegister PopLowestIndex();
 400   CPURegister PopHighestIndex();
 401
 402   // AAPCS64 callee-saved registers.
 403   static CPURegList GetCalleeSaved(unsigned size = kXRegSize);
 404   static CPURegList GetCalleeSavedFP(unsigned size = kDRegSize);
 405
 406   // AAPCS64 caller-saved registers. Note that this includes lr.
 407   static CPURegList GetCallerSaved(unsigned size = kXRegSize);
 408   static CPURegList GetCallerSavedFP(unsigned size = kDRegSize);
 409
 410   inline bool IsEmpty() const {
 411     assert(IsValid());
 412     return list_ == 0;
 413   }
 414
 415   inline bool IncludesAliasOf(const CPURegister& other) const {
 416     assert(IsValid());
 417     return (type_ == other.type()) && (other.Bit() & list_);
 418   }
 419
 420   inline int Count() const {
 421     assert(IsValid());
 422     return CountSetBits(list_, kRegListSizeInBits);
 423   }
 424
 425   inline unsigned RegisterSizeInBits() const {
 426     assert(IsValid());
 427     return size_;
 428   }
 429
 430   inline unsigned RegisterSizeInBytes() const {
 431     int size_in_bits = RegisterSizeInBits();
 432     assert((size_in_bits % 8) == 0);
 433     return size_in_bits / 8;
 434   }
 435
 436  private:
 437   RegList list_;
 438   unsigned size_;
 439   CPURegister::RegisterType type_;
 440
 441   bool IsValid() const;
 442 };
 443
 444
 445 // AAPCS64 callee-saved registers.
 446 extern const CPURegList kCalleeSaved;
 447 extern const CPURegList kCalleeSavedFP;
 448
 449
 450 // AAPCS64 caller-saved registers. Note that this includes lr.
 451 extern const CPURegList kCallerSaved;
 452 extern const CPURegList kCallerSavedFP;
 453
 454
 455 // Operand.
 456 class Operand {
 457  public:
 458   // #<immediate>
 459   // where <immediate> is int64_t.
 460   // This is allowed to be an implicit constructor because Operand is
 461   // a wrapper class that doesn't normally perform any type conversion.
 462   /* implicit */ Operand(int64_t immediate);
 463
 464   // rm, {<shift> #<shift_amount>}
 465   // where <shift> is one of {LSL, LSR, ASR, ROR}.
 466   //       <shift_amount> is uint6_t.
 467   // This is allowed to be an implicit constructor because Operand is
 468   // a wrapper class that doesn't normally perform any type conversion.
 469   /* implicit */ Operand(Register reg,
 470                          Shift shift = LSL,
 471                          unsigned shift_amount = 0);
 472
 473   // rm, {<extend> {#<shift_amount>}}
 474   // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
 475   //       <shift_amount> is uint2_t.
 476   explicit Operand(Register reg, Extend extend, unsigned shift_amount = 0);
 477
 478   bool IsImmediate() const;
 479   bool IsShiftedRegister() const;
 480   bool IsExtendedRegister() const;
 481
 482   // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
 483   // which helps in the encoding of instructions that use the stack pointer.
 484   Operand ToExtendedRegister() const;
 485
 486   int64_t immediate() const {
 487     assert(IsImmediate());
 488     return immediate_;
 489   }
 490
 491   Register reg() const {
 492     assert(IsShiftedRegister() || IsExtendedRegister());
 493     return reg_;
 494   }
 495
 496   Shift shift() const {
 497     assert(IsShiftedRegister());
 498     return shift_;
 499   }
 500
 501   Extend extend() const {
 502     assert(IsExtendedRegister());
 503     return extend_;
 504   }
 505
 506   unsigned shift_amount() const {
 507     assert(IsShiftedRegister() || IsExtendedRegister());
 508     return shift_amount_;
 509   }
 510
 511  private:
 512   int64_t immediate_;
 513   Register reg_;
 514   Shift shift_;
 515   Extend extend_;
 516   unsigned shift_amount_;
 517 };
 518
 519
 520 // MemOperand represents the addressing mode of a load or store instruction.
 521 class MemOperand {
 522  public:
 523   explicit MemOperand(Register base,
 524                       ptrdiff_t offset = 0,
 525                       AddrMode addrmode = Offset);
 526   explicit MemOperand(Register base,
 527                       Register regoffset,
 528                       Shift shift = LSL,
 529                       unsigned shift_amount = 0);
 530   explicit MemOperand(Register base,
 531                       Register regoffset,
 532                       Extend extend,
 533                       unsigned shift_amount = 0);
 534   explicit MemOperand(Register base,
 535                       const Operand& offset,
 536                       AddrMode addrmode = Offset);
 537
 538   const Register& base() const { return base_; }
 539   const Register& regoffset() const { return regoffset_; }
 540   ptrdiff_t offset() const { return offset_; }
 541   AddrMode addrmode() const { return addrmode_; }
 542   Shift shift() const { return shift_; }
 543   Extend extend() const { return extend_; }
 544   unsigned shift_amount() const { return shift_amount_; }
 545   bool IsImmediateOffset() const;
 546   bool IsRegisterOffset() const;
 547   bool IsPreIndex() const;
 548   bool IsPostIndex() const;
 549
 550  private:
 551   Register base_;
 552   Register regoffset_;
 553   ptrdiff_t offset_;
 554   AddrMode addrmode_;
 555   Shift shift_;
 556   Extend extend_;
 557   unsigned shift_amount_;
 558 };
 559
 560
 561 class Label {
 562  public:
 563   Label() : is_bound_(false), link_(nullptr), target_(nullptr) {}
 564   ~Label() {
 565     // If the label has been linked to, it needs to be bound to a target.
 566     assert(!IsLinked() || IsBound());
 567   }
 568
 569   inline HPHP::CodeAddress link() const { return link_; }
 570   inline HPHP::CodeAddress target() const { return target_; }
 571
 572   inline bool IsBound() const { return is_bound_; }
 573   inline bool IsLinked() const { return link_ != nullptr; }
 574
 575   inline void set_link(HPHP::CodeAddress new_link) { link_ = new_link; }
 576
 577   static const int kEndOfChain = 0;
 578
 579  private:
 580   // Indicates if the label has been bound, ie its location is fixed.
 581   bool is_bound_;
 582   // Branches instructions branching to this label form a chained list, with
 583   // their offset indicating where the next instruction is located.
 584   // link_ points to the latest branch instruction generated branching to this
 585   // branch.
 586   // If link_ is not nullptr, the label has been linked to.
 587   HPHP::CodeAddress link_;
 588   // The label location.
 589   HPHP::CodeAddress target_;
 590
 591   friend class Assembler;
 592 };
 593
 594
 595 // TODO: Obtain better values for these, based on real-world data.
 596 const int kLiteralPoolCheckInterval = 4 * KBytes;
 597 const int kRecommendedLiteralPoolRange = 2 * kLiteralPoolCheckInterval;
 598
 599
 600 // Control whether a branch over the literal pool should also be emitted. This
 601 // is needed if the literal pool has to be emitted in the middle of the JITted
 602 // code.
 603 enum LiteralPoolEmitOption {
 604   JumpRequired,
 605   NoJumpRequired
 606 };
 607
 608
 609 // Literal pool entry.
 610 class Literal {
 611  public:
 612   Literal(HPHP::CodeAddress pc, uint64_t imm, unsigned size)
 613       : pc_(pc), value_(imm), size_(size) {}
 614
 615  private:
 616   HPHP::CodeAddress pc_;
 617   int64_t value_;
 618   unsigned size_;
 619
 620   friend class Assembler;
 621 };
 622
 623
 624 // Assembler.
 625 class Assembler {
 626  public:
 627   explicit Assembler(HPHP::CodeBlock& cb);
 628
 629   // The destructor asserts that one of the following is true:
 630   //  * The Assembler object has not been used.
 631   //  * Nothing has been emitted since the last Reset() call.
 632   //  * Nothing has been emitted since the last FinalizeCode() call.
 633   ~Assembler();
 634
 635   // System functions.
 636
 637   // Start generating code from the beginning of the buffer, discarding any code
 638   // and data that has already been emitted into the buffer.
 639   //
 640   // In order to avoid any accidental transfer of state, Reset ASSERTs that the
 641   // constant pool is not blocked.
 642   void Reset();
 643
 644   // Finalize a code buffer of generated instructions. This function must be
 645   // called before executing or copying code from the buffer.
 646   void FinalizeCode();
 647
 648   // Label.
 649   // Bind a label to the current PC.
 650   void bind(Label* label);
 651   int UpdateAndGetByteOffsetTo(Label* label);
 652   inline int UpdateAndGetInstructionOffsetTo(Label* label) {
 653     assert(Label::kEndOfChain == 0);
 654     return UpdateAndGetByteOffsetTo(label) >> kInstructionSizeLog2;
 655   }
 656
 657   HPHP::Transl::TCA frontier() const {
 658     return cb_.frontier();
 659   }
 660
 661   bool isFrontierAligned(size_t align) const {
 662     return cb_.isFrontierAligned(align);
 663   }
 664
 665   // Instruction set functions.
 666
 667   // Branch / Jump instructions.
 668   // Branch to register.
 669   void br(const Register& xn);
 670
 671   // Branch with link to register.
 672   void blr(const Register& xn);
 673
 674   // Branch to register with return hint.
 675   void ret(const Register& xn = lr);
 676
 677   // Unconditional branch to label.
 678   void b(Label* label);
 679
 680   // Conditional branch to label.
 681   void b(Label* label, Condition cond);
 682
 683   // Unconditional branch to PC offset.
 684   void b(int imm26);
 685
 686   // Conditional branch to PC offset.
 687   void b(int imm19, Condition cond);
 688
 689   // Branch with link to label.
 690   void bl(Label* label);
 691
 692   // Branch with link to PC offset.
 693   void bl(int imm26);
 694
 695   // Compare and branch to label if zero.
 696   void cbz(const Register& rt, Label* label);
 697
 698   // Compare and branch to PC offset if zero.
 699   void cbz(const Register& rt, int imm19);
 700
 701   // Compare and branch to label if not zero.
 702   void cbnz(const Register& rt, Label* label);
 703
 704   // Compare and branch to PC offset if not zero.
 705   void cbnz(const Register& rt, int imm19);
 706
 707   // Test bit and branch to label if zero.
 708   void tbz(const Register& rt, unsigned bit_pos, Label* label);
 709
 710   // Test bit and branch to PC offset if zero.
 711   void tbz(const Register& rt, unsigned bit_pos, int imm14);
 712
 713   // Test bit and branch to label if not zero.
 714   void tbnz(const Register& rt, unsigned bit_pos, Label* label);
 715
 716   // Test bit and branch to PC offset if not zero.
 717   void tbnz(const Register& rt, unsigned bit_pos, int imm14);
 718
 719   // Address calculation instructions.
 720   // Calculate a PC-relative address. Unlike for branches the offset in adr is
 721   // unscaled (i.e. the result can be unaligned).
 722
 723   // Calculate the address of a label.
 724   void adr(const Register& rd, Label* label);
 725
 726   // Calculate the address of a PC offset.
 727   void adr(const Register& rd, int imm21);
 728
 729   // Data Processing instructions.
 730   // Add.
 731   void add(const Register& rd,
 732            const Register& rn,
 733            const Operand& operand,
 734            FlagsUpdate S = LeaveFlags);
 735
 736   // Compare negative.
 737   void cmn(const Register& rn, const Operand& operand);
 738
 739   // Subtract.
 740   void sub(const Register& rd,
 741            const Register& rn,
 742            const Operand& operand,
 743            FlagsUpdate S = LeaveFlags);
 744
 745   // Compare.
 746   void cmp(const Register& rn, const Operand& operand);
 747
 748   // Negate.
 749   void neg(const Register& rd,
 750            const Operand& operand,
 751            FlagsUpdate S = LeaveFlags);
 752
 753   // Add with carry bit.
 754   void adc(const Register& rd,
 755            const Register& rn,
 756            const Operand& operand,
 757            FlagsUpdate S = LeaveFlags);
 758
 759   // Subtract with carry bit.
 760   void sbc(const Register& rd,
 761            const Register& rn,
 762            const Operand& operand,
 763            FlagsUpdate S = LeaveFlags);
 764
 765   // Negate with carry bit.
 766   void ngc(const Register& rd,
 767            const Operand& operand,
 768            FlagsUpdate S = LeaveFlags);
 769
 770   // Logical instructions.
 771   // Bitwise and (A & B).
 772   void and_(const Register& rd,
 773             const Register& rn,
 774             const Operand& operand,
 775             FlagsUpdate S = LeaveFlags);
 776
 777   // Bit test and set flags.
 778   void tst(const Register& rn, const Operand& operand);
 779
 780   // Bit clear (A & ~B).
 781   void bic(const Register& rd,
 782            const Register& rn,
 783            const Operand& operand,
 784            FlagsUpdate S = LeaveFlags);
 785
 786   // Bitwise or (A | B).
 787   void orr(const Register& rd, const Register& rn, const Operand& operand);
 788
 789   // Bitwise nor (A | ~B).
 790   void orn(const Register& rd, const Register& rn, const Operand& operand);
 791
 792   // Bitwise eor/xor (A ^ B).
 793   void eor(const Register& rd, const Register& rn, const Operand& operand);
 794
 795   // Bitwise enor/xnor (A ^ ~B).
 796   void eon(const Register& rd, const Register& rn, const Operand& operand);
 797
 798   // Logical shift left by variable.
 799   void lslv(const Register& rd, const Register& rn, const Register& rm);
 800
 801   // Logical shift right by variable.
 802   void lsrv(const Register& rd, const Register& rn, const Register& rm);
 803
 804   // Arithmetic shift right by variable.
 805   void asrv(const Register& rd, const Register& rn, const Register& rm);
 806
 807   // Rotate right by variable.
 808   void rorv(const Register& rd, const Register& rn, const Register& rm);
 809
 810   // Bitfield instructions.
 811   // Bitfield move.
 812   void bfm(const Register& rd,
 813            const Register& rn,
 814            unsigned immr,
 815            unsigned imms);
 816
 817   // Signed bitfield move.
 818   void sbfm(const Register& rd,
 819             const Register& rn,
 820             unsigned immr,
 821             unsigned imms);
 822
 823   // Unsigned bitfield move.
 824   void ubfm(const Register& rd,
 825             const Register& rn,
 826             unsigned immr,
 827             unsigned imms);
 828
 829   // Bfm aliases.
 830   // Bitfield insert.
 831   inline void bfi(const Register& rd,
 832                   const Register& rn,
 833                   unsigned lsb,
 834                   unsigned width) {
 835     assert(width >= 1);
 836     assert(lsb + width <= rn.size());
 837     bfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
 838   }
 839
 840   // Bitfield extract and insert low.
 841   inline void bfxil(const Register& rd,
 842                     const Register& rn,
 843                     unsigned lsb,
 844                     unsigned width) {
 845     assert(width >= 1);
 846     assert(lsb + width <= rn.size());
 847     bfm(rd, rn, lsb, lsb + width - 1);
 848   }
 849
 850   // Sbfm aliases.
 851   // Arithmetic shift right.
 852   inline void asr(const Register& rd, const Register& rn, unsigned shift) {
 853     assert(shift < rd.size());
 854     sbfm(rd, rn, shift, rd.size() - 1);
 855   }
 856
 857   // Signed bitfield insert with zero at right.
 858   inline void sbfiz(const Register& rd,
 859                     const Register& rn,
 860                     unsigned lsb,
 861                     unsigned width) {
 862     assert(width >= 1);
 863     assert(lsb + width <= rn.size());
 864     sbfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
 865   }
 866
 867   // Signed bitfield extract.
 868   inline void sbfx(const Register& rd,
 869                    const Register& rn,
 870                    unsigned lsb,
 871                    unsigned width) {
 872     assert(width >= 1);
 873     assert(lsb + width <= rn.size());
 874     sbfm(rd, rn, lsb, lsb + width - 1);
 875   }
 876
 877   // Signed extend byte.
 878   inline void sxtb(const Register& rd, const Register& rn) {
 879     sbfm(rd, rn, 0, 7);
 880   }
 881
 882   // Signed extend halfword.
 883   inline void sxth(const Register& rd, const Register& rn) {
 884     sbfm(rd, rn, 0, 15);
 885   }
 886
 887   // Signed extend word.
 888   inline void sxtw(const Register& rd, const Register& rn) {
 889     sbfm(rd, rn, 0, 31);
 890   }
 891
 892   // Ubfm aliases.
 893   // Logical shift left.
 894   inline void lsl(const Register& rd, const Register& rn, unsigned shift) {
 895     unsigned reg_size = rd.size();
 896     assert(shift < reg_size);
 897     ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
 898   }
 899
 900   // Logical shift right.
 901   inline void lsr(const Register& rd, const Register& rn, unsigned shift) {
 902     assert(shift < rd.size());
 903     ubfm(rd, rn, shift, rd.size() - 1);
 904   }
 905
 906   // Unsigned bitfield insert with zero at right.
 907   inline void ubfiz(const Register& rd,
 908                     const Register& rn,
 909                     unsigned lsb,
 910                     unsigned width) {
 911     assert(width >= 1);
 912     assert(lsb + width <= rn.size());
 913     ubfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
 914   }
 915
 916   // Unsigned bitfield extract.
 917   inline void ubfx(const Register& rd,
 918                    const Register& rn,
 919                    unsigned lsb,
 920                    unsigned width) {
 921     assert(width >= 1);
 922     assert(lsb + width <= rn.size());
 923     ubfm(rd, rn, lsb, lsb + width - 1);
 924   }
 925
 926   // Unsigned extend byte.
 927   inline void uxtb(const Register& rd, const Register& rn) {
 928     ubfm(rd, rn, 0, 7);
 929   }
 930
 931   // Unsigned extend halfword.
 932   inline void uxth(const Register& rd, const Register& rn) {
 933     ubfm(rd, rn, 0, 15);
 934   }
 935
 936   // Unsigned extend word.
 937   inline void uxtw(const Register& rd, const Register& rn) {
 938     ubfm(rd, rn, 0, 31);
 939   }
 940
 941   // Extract.
 942   void extr(const Register& rd,
 943             const Register& rn,
 944             const Register& rm,
 945             unsigned lsb);
 946
 947   // Conditional select: rd = cond ? rn : rm.
 948   void csel(const Register& rd,
 949             const Register& rn,
 950             const Register& rm,
 951             Condition cond);
 952
 953   // Conditional select increment: rd = cond ? rn : rm + 1.
 954   void csinc(const Register& rd,
 955              const Register& rn,
 956              const Register& rm,
 957              Condition cond);
 958
 959   // Conditional select inversion: rd = cond ? rn : ~rm.
 960   void csinv(const Register& rd,
 961              const Register& rn,
 962              const Register& rm,
 963              Condition cond);
 964
 965   // Conditional select negation: rd = cond ? rn : -rm.
 966   void csneg(const Register& rd,
 967              const Register& rn,
 968              const Register& rm,
 969              Condition cond);
 970
 971   // Conditional set: rd = cond ? 1 : 0.
 972   void cset(const Register& rd, Condition cond);
 973
 974   // Conditional set mask: rd = cond ? -1 : 0.
 975   void csetm(const Register& rd, Condition cond);
 976
 977   // Conditional increment: rd = cond ? rn + 1 : rn.
 978   void cinc(const Register& rd, const Register& rn, Condition cond);
 979
 980   // Conditional invert: rd = cond ? ~rn : rn.
 981   void cinv(const Register& rd, const Register& rn, Condition cond);
 982
 983   // Conditional negate: rd = cond ? -rn : rn.
 984   void cneg(const Register& rd, const Register& rn, Condition cond);
 985
 986   // Rotate right.
 987   inline void ror(const Register& rd, const Register& rs, unsigned shift) {
 988     extr(rd, rs, rs, shift);
 989   }
 990
 991   // Conditional comparison.
 992   // Conditional compare negative.
 993   void ccmn(const Register& rn,
 994             const Operand& operand,
 995             StatusFlags nzcv,
 996             Condition cond);
 997
 998   // Conditional compare.
 999   void ccmp(const Register& rn,
1000             const Operand& operand,
1001             StatusFlags nzcv,
1002             Condition cond);
1003
1004   // Multiply.
1005   void mul(const Register& rd, const Register& rn, const Register& rm);
1006
1007   // Negated multiply.
1008   void mneg(const Register& rd, const Register& rn, const Register& rm);
1009
1010   // Signed long multiply: 32 x 32 -> 64-bit.
1011   void smull(const Register& rd, const Register& rn, const Register& rm);
1012
1013   // Signed multiply high: 64 x 64 -> 64-bit <127:64>.
1014   void smulh(const Register& xd, const Register& xn, const Register& xm);
1015
1016   // Multiply and accumulate.
1017   void madd(const Register& rd,
1018             const Register& rn,
1019             const Register& rm,
1020             const Register& ra);
1021
1022   // Multiply and subtract.
1023   void msub(const Register& rd,
1024             const Register& rn,
1025             const Register& rm,
1026             const Register& ra);
1027
1028   // Signed long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
1029   void smaddl(const Register& rd,
1030               const Register& rn,
1031               const Register& rm,
1032               const Register& ra);
1033
1034   // Unsigned long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
1035   void umaddl(const Register& rd,
1036               const Register& rn,
1037               const Register& rm,
1038               const Register& ra);
1039
1040   // Signed long multiply and subtract: 64 - (32 x 32) -> 64-bit.
1041   void smsubl(const Register& rd,
1042               const Register& rn,
1043               const Register& rm,
1044               const Register& ra);
1045
1046   // Unsigned long multiply and subtract: 64 - (32 x 32) -> 64-bit.
1047   void umsubl(const Register& rd,
1048               const Register& rn,
1049               const Register& rm,
1050               const Register& ra);
1051
1052   // Signed integer divide.
1053   void sdiv(const Register& rd, const Register& rn, const Register& rm);
1054
1055   // Unsigned integer divide.
1056   void udiv(const Register& rd, const Register& rn, const Register& rm);
1057
1058   // Bit reverse.
1059   void rbit(const Register& rd, const Register& rn);
1060
1061   // Reverse bytes in 16-bit half words.
1062   void rev16(const Register& rd, const Register& rn);
1063
1064   // Reverse bytes in 32-bit words.
1065   void rev32(const Register& rd, const Register& rn);
1066
1067   // Reverse bytes.
1068   void rev(const Register& rd, const Register& rn);
1069
1070   // Count leading zeroes.
1071   void clz(const Register& rd, const Register& rn);
1072
1073   // Count leading sign bits.
1074   void cls(const Register& rd, const Register& rn);
1075
1076   // Memory instructions.
1077   // Load integer or FP register.
1078   void ldr(const CPURegister& rt, const MemOperand& src);
1079
1080   // Store integer or FP register.
1081   void str(const CPURegister& rt, const MemOperand& dst);
1082
1083   // PC-relative load.
1084   void ldr(const Register& rt, Label* label);
1085
1086   // Load word with sign extension.
1087   void ldrsw(const Register& rt, const MemOperand& src);
1088
1089   // Load byte.
1090   void ldrb(const Register& rt, const MemOperand& src);
1091
1092   // Store byte.
1093   void strb(const Register& rt, const MemOperand& dst);
1094
1095   // Load byte with sign extension.
1096   void ldrsb(const Register& rt, const MemOperand& src);
1097
1098   // Load half-word.
1099   void ldrh(const Register& rt, const MemOperand& src);
1100
1101   // Store half-word.
1102   void strh(const Register& rt, const MemOperand& dst);
1103
1104   // Load half-word with sign extension.
1105   void ldrsh(const Register& rt, const MemOperand& src);
1106
1107   // Load integer or FP register pair.
1108   void ldp(const CPURegister& rt, const CPURegister& rt2,
1109            const MemOperand& src);
1110
1111   // Store integer or FP register pair.
1112   void stp(const CPURegister& rt, const CPURegister& rt2,
1113            const MemOperand& dst);
1114
1115   // Load word pair with sign extension.
1116   void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
1117
1118   // Load integer or FP register pair, non-temporal.
1119   void ldnp(const CPURegister& rt, const CPURegister& rt2,
1120             const MemOperand& src);
1121
1122   // Store integer or FP register pair, non-temporal.
1123   void stnp(const CPURegister& rt, const CPURegister& rt2,
1124             const MemOperand& dst);
1125
1126   // Load literal to register.
1127   void ldr(const Register& rt, uint64_t imm);
1128
1129   // Load literal to FP register.
1130   void ldr(const FPRegister& ft, double imm);
1131
1132   // Move instructions. The default shift of -1 indicates that the move
1133   // instruction will calculate an appropriate 16-bit immediate and left shift
1134   // that is equal to the 64-bit immediate argument. If an explicit left shift
1135   // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
1136   //
1137   // For movk, an explicit shift can be used to indicate which half word should
1138   // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
1139   // half word with zero, whereas movk(x0, 0, 48) will overwrite the
1140   // most-significant.
1141
1142   // Move immediate and keep.
1143   void movk(const Register& rd, uint64_t imm, int shift = -1) {
1144     MoveWide(rd, imm, shift, MOVK);
1145   }
1146
1147   // Move inverted immediate.
1148   void movn(const Register& rd, uint64_t imm, int shift = -1) {
1149     MoveWide(rd, imm, shift, MOVN);
1150   }
1151
1152   // Move immediate.
1153   void movz(const Register& rd, uint64_t imm, int shift = -1) {
1154     MoveWide(rd, imm, shift, MOVZ);
1155   }
1156
1157   // Misc instructions.
1158   // Monitor debug-mode breakpoint.
1159   void brk(int code);
1160
1161   // Halting debug-mode breakpoint.
1162   void hlt(int code);
1163
1164   // Move register to register.
1165   void mov(const Register& rd, const Register& rn);
1166
1167   // Move inverted operand to register.
1168   void mvn(const Register& rd, const Operand& operand);
1169
1170   // System instructions.
1171   // Move to register from system register.
1172   void mrs(const Register& rt, SystemRegister sysreg);
1173
1174   // Move from register to system register.
1175   void msr(SystemRegister sysreg, const Register& rt);
1176
1177   // System hint.
1178   void hint(SystemHint code);
1179
1180   // Alias for system instructions.
1181   // No-op.
1182   void nop() {
1183     hint(NOP);
1184   }
1185
1186   // FP instructions.
1187   // Move immediate to FP register.
1188   void fmov(FPRegister fd, double imm);
1189
1190   // Move FP register to register.
1191   void fmov(Register rd, FPRegister fn);
1192
1193   // Move register to FP register.
1194   void fmov(FPRegister fd, Register rn);
1195
1196   // Move FP register to FP register.
1197   void fmov(FPRegister fd, FPRegister fn);
1198
1199   // FP add.
1200   void fadd(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1201
1202   // FP subtract.
1203   void fsub(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1204
1205   // FP multiply.
1206   void fmul(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1207
1208   // FP multiply and subtract.
1209   void fmsub(const FPRegister& fd,
1210              const FPRegister& fn,
1211              const FPRegister& fm,
1212              const FPRegister& fa);
1213
1214   // FP divide.
1215   void fdiv(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1216
1217   // FP maximum.
1218   void fmax(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1219
1220   // FP minimum.
1221   void fmin(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
1222
1223   // FP absolute.
1224   void fabs(const FPRegister& fd, const FPRegister& fn);
1225
1226   // FP negate.
1227   void fneg(const FPRegister& fd, const FPRegister& fn);
1228
1229   // FP square root.
1230   void fsqrt(const FPRegister& fd, const FPRegister& fn);
1231
1232   // FP round to integer (nearest with ties to even).
1233   void frintn(const FPRegister& fd, const FPRegister& fn);
1234
1235   // FP round to integer (towards zero).
1236   void frintz(const FPRegister& fd, const FPRegister& fn);
1237
1238   // FP compare registers.
1239   void fcmp(const FPRegister& fn, const FPRegister& fm);
1240
1241   // FP compare immediate.
1242   void fcmp(const FPRegister& fn, double value);
1243
1244   // FP conditional compare.
1245   void fccmp(const FPRegister& fn,
1246              const FPRegister& fm,
1247              StatusFlags nzcv,
1248              Condition cond);
1249
1250   // FP conditional select.
1251   void fcsel(const FPRegister& fd,
1252              const FPRegister& fn,
1253              const FPRegister& fm,
1254              Condition cond);
1255
1256   // Common FP Convert function.
1257   void FPConvertToInt(const Register& rd,
1258                       const FPRegister& fn,
1259                       FPIntegerConvertOp op);
1260
1261   // FP convert between single and double precision.
1262   void fcvt(const FPRegister& fd, const FPRegister& fn);
1263
1264   // Convert FP to unsigned integer (round towards -infinity).
1265   void fcvtmu(const Register& rd, const FPRegister& fn);
1266
1267   // Convert FP to signed integer (round towards -infinity).
1268   void fcvtms(const Register& rd, const FPRegister& fn);
1269
1270   // Convert FP to unsigned integer (nearest with ties to even).
1271   void fcvtnu(const Register& rd, const FPRegister& fn);
1272
1273   // Convert FP to signed integer (nearest with ties to even).
1274   void fcvtns(const Register& rd, const FPRegister& fn);
1275
1276   // Convert FP to unsigned integer (round towards zero).
1277   void fcvtzu(const Register& rd, const FPRegister& fn);
1278
1279   // Convert FP to signed integer (round towards zero).
1280   void fcvtzs(const Register& rd, const FPRegister& fn);
1281
1282   // Convert signed integer or fixed point to FP.
1283   void scvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
1284
1285   // Convert unsigned integer or fixed point to FP.
1286   void ucvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
1287
1288   // Emit generic instructions.
1289   // Emit raw instructions into the instruction stream.
1290   inline void dci(Instr raw_inst) { Emit(raw_inst); }
1291
1292   // Emit 32 bits of data into the instruction stream.
1293   inline void dc32(uint32_t data) { EmitData(&data, sizeof(data)); }
1294
1295   // Emit 64 bits of data into the instruction stream.
1296   inline void dc64(uint64_t data) { EmitData(&data, sizeof(data)); }
1297
1298   template<typename T>
1299   inline void dc64(T* data) { dc64(reinterpret_cast<uint64_t>(data)); }
1300
1301   // Copy a string into the instruction stream, including the terminating NUL
1302   // character. The instruction pointer (pc_) is then aligned correctly for
1303   // subsequent instructions.
1304   void EmitStringData(const char * string) {
1305     assert(string != nullptr);
1306
1307     size_t len = strlen(string) + 1;
1308     EmitData(string, len);
1309
1310     // Pad with NUL characters until pc_ is aligned.
1311     const char pad[] = {'\0', '\0', '\0', '\0'};
1312     assert(sizeof(pad) == kInstructionSize);
1313     auto next_pc = AlignUp(cb_.frontier(), kInstructionSize);
1314     EmitData(&pad, next_pc - cb_.frontier());
1315   }
1316
1317   // Code generation helpers.
1318
1319   // Register encoding.
1320   static Instr Rd(CPURegister rd) {
1321     assert(rd.code() != kSPRegInternalCode);
1322     return rd.code() << Rd_offset;
1323   }
1324
1325   static Instr Rn(CPURegister rn) {
1326     assert(rn.code() != kSPRegInternalCode);
1327     return rn.code() << Rn_offset;
1328   }
1329
1330   static Instr Rm(CPURegister rm) {
1331     assert(rm.code() != kSPRegInternalCode);
1332     return rm.code() << Rm_offset;
1333   }
1334
1335   static Instr Ra(CPURegister ra) {
1336     assert(ra.code() != kSPRegInternalCode);
1337     return ra.code() << Ra_offset;
1338   }
1339
1340   static Instr Rt(CPURegister rt) {
1341     assert(rt.code() != kSPRegInternalCode);
1342     return rt.code() << Rt_offset;
1343   }
1344
1345   static Instr Rt2(CPURegister rt2) {
1346     assert(rt2.code() != kSPRegInternalCode);
1347     return rt2.code() << Rt2_offset;
1348   }
1349
1350   // These encoding functions allow the stack pointer to be encoded, and
1351   // disallow the zero register.
1352   static Instr RdSP(Register rd) {
1353     assert(!rd.IsZero());
1354     return (rd.code() & kRegCodeMask) << Rd_offset;
1355   }
1356
1357   static Instr RnSP(Register rn) {
1358     assert(!rn.IsZero());
1359     return (rn.code() & kRegCodeMask) << Rn_offset;
1360   }
1361
1362   // Flags encoding.
1363   static Instr Flags(FlagsUpdate S) {
1364     if (S == SetFlags) {
1365       return 1 << FlagsUpdate_offset;
1366     } else if (S == LeaveFlags) {
1367       return 0 << FlagsUpdate_offset;
1368     }
1369     not_reached();
1370     return 0;
1371   }
1372
1373   static Instr Cond(Condition cond) {
1374     return cond << Condition_offset;
1375   }
1376
1377   // PC-relative address encoding.
1378   static Instr ImmPCRelAddress(int imm21) {
1379     assert(is_int21(imm21));
1380     Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
1381     Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
1382     Instr immlo = imm << ImmPCRelLo_offset;
1383     return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
1384   }
1385
1386   static Instr ImmPCRelLoadStoreAddress(int imm19) {
1387     assert(is_int19(imm19));
1388     return truncate_to_int19(imm19) << ImmLLiteral_offset;
1389   }
1390
1391   // Branch encoding.
1392   static Instr ImmUncondBranch(int imm26) {
1393     assert(is_int26(imm26));
1394     return truncate_to_int26(imm26) << ImmUncondBranch_offset;
1395   }
1396
1397   static Instr ImmCondBranch(int imm19) {
1398     assert(is_int19(imm19));
1399     return truncate_to_int19(imm19) << ImmCondBranch_offset;
1400   }
1401
1402   static Instr ImmCmpBranch(int imm19) {
1403     assert(is_int19(imm19));
1404     return truncate_to_int19(imm19) << ImmCmpBranch_offset;
1405   }
1406
1407   static Instr ImmTestBranch(int imm14) {
1408     assert(is_int14(imm14));
1409     return truncate_to_int14(imm14) << ImmTestBranch_offset;
1410   }
1411
1412   static Instr ImmTestBranchBit(unsigned bit_pos) {
1413     assert(is_uint6(bit_pos));
1414     // Subtract five from the shift offset, as we need bit 5 from bit_pos.
1415     unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
1416     unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
1417     b5 &= ImmTestBranchBit5_mask;
1418     b40 &= ImmTestBranchBit40_mask;
1419     return b5 | b40;
1420   }
1421
1422   // Data Processing encoding.
1423   static Instr SF(Register rd) {
1424       return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
1425   }
1426
1427   static Instr ImmAddSub(int64_t imm) {
1428     assert(IsImmAddSub(imm));
1429     if (is_uint12(imm)) {  // No shift required.
1430       return imm << ImmAddSub_offset;
1431     } else {
1432       return ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
1433     }
1434   }
1435
1436   static inline Instr ImmS(unsigned imms, unsigned reg_size) {
1437     assert(((reg_size == kXRegSize) && is_uint6(imms)) ||
1438            ((reg_size == kWRegSize) && is_uint5(imms)));
1439     USE(reg_size);
1440     return imms << ImmS_offset;
1441   }
1442
1443   static inline Instr ImmR(unsigned immr, unsigned reg_size) {
1444     assert(((reg_size == kXRegSize) && is_uint6(immr)) ||
1445            ((reg_size == kWRegSize) && is_uint5(immr)));
1446     USE(reg_size);
1447     assert(is_uint6(immr));
1448     return immr << ImmR_offset;
1449   }
1450
1451   static inline Instr ImmSetBits(unsigned imms, unsigned reg_size) {
1452     assert((reg_size == kWRegSize) || (reg_size == kXRegSize));
1453     assert(is_uint6(imms));
1454     assert((reg_size == kXRegSize) || is_uint6(imms + 3));
1455     USE(reg_size);
1456     return imms << ImmSetBits_offset;
1457   }
1458
1459   static inline Instr ImmRotate(unsigned immr, unsigned reg_size) {
1460     assert((reg_size == kWRegSize) || (reg_size == kXRegSize));
1461     assert(((reg_size == kXRegSize) && is_uint6(immr)) ||
1462            ((reg_size == kWRegSize) && is_uint5(immr)));
1463     USE(reg_size);
1464     return immr << ImmRotate_offset;
1465   }
1466
1467   static inline Instr ImmLLiteral(int imm19) {
1468     assert(is_int19(imm19));
1469     return truncate_to_int19(imm19) << ImmLLiteral_offset;
1470   }
1471
1472   static inline Instr BitN(unsigned bitn, unsigned reg_size) {
1473     assert((reg_size == kWRegSize) || (reg_size == kXRegSize));
1474     assert((reg_size == kXRegSize) || (bitn == 0));
1475     USE(reg_size);
1476     return bitn << BitN_offset;
1477   }
1478
1479   static Instr ShiftDP(Shift shift) {
1480     assert(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
1481     return shift << ShiftDP_offset;
1482   }
1483
1484   static Instr ImmDPShift(unsigned amount) {
1485     assert(is_uint6(amount));
1486     return amount << ImmDPShift_offset;
1487   }
1488
1489   static Instr ExtendMode(Extend extend) {
1490     return extend << ExtendMode_offset;
1491   }
1492
1493   static Instr ImmExtendShift(unsigned left_shift) {
1494     assert(left_shift <= 4);
1495     return left_shift << ImmExtendShift_offset;
1496   }
1497
1498   static Instr ImmCondCmp(unsigned imm) {
1499     assert(is_uint5(imm));
1500     return imm << ImmCondCmp_offset;
1501   }
1502
1503   static Instr Nzcv(StatusFlags nzcv) {
1504     return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
1505   }
1506
1507   // MemOperand offset encoding.
1508   static Instr ImmLSUnsigned(int imm12) {
1509     assert(is_uint12(imm12));
1510     return imm12 << ImmLSUnsigned_offset;
1511   }
1512
1513   static Instr ImmLS(int imm9) {
1514     assert(is_int9(imm9));
1515     return truncate_to_int9(imm9) << ImmLS_offset;
1516   }
1517
1518   static Instr ImmLSPair(int imm7, LSDataSize size) {
1519     assert(((imm7 >> size) << size) == imm7);
1520     int scaled_imm7 = imm7 >> size;
1521     assert(is_int7(scaled_imm7));
1522     return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
1523   }
1524
1525   static Instr ImmShiftLS(unsigned shift_amount) {
1526     assert(is_uint1(shift_amount));
1527     return shift_amount << ImmShiftLS_offset;
1528   }
1529
1530   static Instr ImmException(int imm16) {
1531     assert(is_uint16(imm16));
1532     return imm16 << ImmException_offset;
1533   }
1534
1535   static Instr ImmSystemRegister(int imm15) {
1536     assert(is_uint15(imm15));
1537     return imm15 << ImmSystemRegister_offset;
1538   }
1539
1540   static Instr ImmHint(int imm7) {
1541     assert(is_uint7(imm7));
1542     return imm7 << ImmHint_offset;
1543   }
1544
1545   static LSDataSize CalcLSDataSize(LoadStoreOp op) {
1546     assert((SizeLS_offset + SizeLS_width) == (kInstructionSize * 8));
1547     return static_cast<LSDataSize>(op >> SizeLS_offset);
1548   }
1549
1550   // Move immediates encoding.
1551   static Instr ImmMoveWide(uint64_t imm) {
1552     assert(is_uint16(imm));
1553     return imm << ImmMoveWide_offset;
1554   }
1555
1556   static Instr ShiftMoveWide(int64_t shift) {
1557     assert(is_uint2(shift));
1558     return shift << ShiftMoveWide_offset;
1559   }
1560
1561   // FP Immediates.
1562   static Instr ImmFP32(float imm);
1563   static Instr ImmFP64(double imm);
1564
1565   // FP register type.
1566   static Instr FPType(FPRegister fd) {
1567     return fd.Is64Bits() ? FP64 : FP32;
1568   }
1569
1570   static Instr FPScale(unsigned scale) {
1571     assert(is_uint6(scale));
1572     return scale << FPScale_offset;
1573   }
1574
1575   // Size of the code generated in bytes
1576   uint64_t SizeOfCodeGenerated() const {
1577     assert(cb_.available() > 0);
1578     return cb_.used();
1579   }
1580
1581   // Size of the code generated since label to the current position.
1582   uint64_t SizeOfCodeGeneratedSince(Label* label) const {
1583     assert(label->IsBound());
1584     auto addr = label->target();
1585     assert(cb_.frontier() >= addr);
1586     return cb_.frontier() - addr;
1587   }
1588
1589
1590   inline void BlockLiteralPool() {
1591     literal_pool_monitor_++;
1592   }
1593
1594   inline void ReleaseLiteralPool() {
1595     if (--literal_pool_monitor_ == 0) {
1596       // Has the literal pool been blocked for too long?
1597       assert(literals_.empty() ||
1598              (cb_.frontier() < (literals_.back()->pc_ + kMaxLoadLiteralRange)));
1599     }
1600   }
1601
1602   inline bool IsLiteralPoolBlocked() {
1603     return literal_pool_monitor_ != 0;
1604   }
1605
1606   void CheckLiteralPool(LiteralPoolEmitOption option = JumpRequired);
1607   void EmitLiteralPool(LiteralPoolEmitOption option = NoJumpRequired);
1608   size_t LiteralPoolSize();
1609
1610   static bool IsImmLogical(uint64_t value, unsigned width) {
1611     unsigned ignored;
1612     return IsImmLogical(value, width, &ignored, &ignored, &ignored);
1613   }
1614
1615   static bool IsImmArithmetic(int64_t value) {
1616     return IsImmAddSub(value);
1617   }
1618
1619  protected:
1620   inline const Register& AppropriateZeroRegFor(const CPURegister& reg) const {
1621     return reg.Is64Bits() ? xzr : wzr;
1622   }
1623
1624
1625   void LoadStore(const CPURegister& rt,
1626                  const MemOperand& addr,
1627                  LoadStoreOp op);
1628   static bool IsImmLSUnscaled(ptrdiff_t offset);
1629   static bool IsImmLSScaled(ptrdiff_t offset, LSDataSize size);
1630
1631   void Logical(const Register& rd,
1632                const Register& rn,
1633                const Operand& operand,
1634                LogicalOp op);
1635   void LogicalImmediate(const Register& rd,
1636                         const Register& rn,
1637                         unsigned n,
1638                         unsigned imm_s,
1639                         unsigned imm_r,
1640                         LogicalOp op);
1641   static bool IsImmLogical(uint64_t value,
1642                            unsigned width,
1643                            unsigned* n,
1644                            unsigned* imm_s,
1645                            unsigned* imm_r);
1646
1647   void ConditionalCompare(const Register& rn,
1648                           const Operand& operand,
1649                           StatusFlags nzcv,
1650                           Condition cond,
1651                           ConditionalCompareOp op);
1652   static bool IsImmConditionalCompare(int64_t immediate);
1653
1654   void AddSubWithCarry(const Register& rd,
1655                        const Register& rn,
1656                        const Operand& operand,
1657                        FlagsUpdate S,
1658                        AddSubWithCarryOp op);
1659
1660   // Functions for emulating operands not directly supported by the instruction
1661   // set.
1662   void EmitShift(const Register& rd,
1663                  const Register& rn,
1664                  Shift shift,
1665                  unsigned amount);
1666   void EmitExtendShift(const Register& rd,
1667                        const Register& rn,
1668                        Extend extend,
1669                        unsigned left_shift);
1670
1671   void AddSub(const Register& rd,
1672               const Register& rn,
1673               const Operand& operand,
1674               FlagsUpdate S,
1675               AddSubOp op);
1676   static bool IsImmAddSub(int64_t immediate);
1677
1678   // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
1679   // registers. Only simple loads are supported; sign- and zero-extension (such
1680   // as in LDPSW_x or LDRB_w) are not supported.
1681   static LoadStoreOp LoadOpFor(const CPURegister& rt);
1682   static LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
1683                                        const CPURegister& rt2);
1684   static LoadStoreOp StoreOpFor(const CPURegister& rt);
1685   static LoadStorePairOp StorePairOpFor(const CPURegister& rt,
1686                                         const CPURegister& rt2);
1687   static LoadStorePairNonTemporalOp LoadPairNonTemporalOpFor(
1688     const CPURegister& rt, const CPURegister& rt2);
1689   static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
1690     const CPURegister& rt, const CPURegister& rt2);
1691
1692
1693  private:
1694   // Instruction helpers.
1695   void MoveWide(const Register& rd,
1696                 uint64_t imm,
1697                 int shift,
1698                 MoveWideImmediateOp mov_op);
1699   void DataProcShiftedRegister(const Register& rd,
1700                                const Register& rn,
1701                                const Operand& operand,
1702                                FlagsUpdate S,
1703                                Instr op);
1704   void DataProcExtendedRegister(const Register& rd,
1705                                 const Register& rn,
1706                                 const Operand& operand,
1707                                 FlagsUpdate S,
1708                                 Instr op);
1709   void LoadStorePair(const CPURegister& rt,
1710                      const CPURegister& rt2,
1711                      const MemOperand& addr,
1712                      LoadStorePairOp op);
1713   void LoadStorePairNonTemporal(const CPURegister& rt,
1714                                 const CPURegister& rt2,
1715                                 const MemOperand& addr,
1716                                 LoadStorePairNonTemporalOp op);
1717   void LoadLiteral(const CPURegister& rt, uint64_t imm, LoadLiteralOp op);
1718   void ConditionalSelect(const Register& rd,
1719                          const Register& rn,
1720                          const Register& rm,
1721                          Condition cond,
1722                          ConditionalSelectOp op);
1723   void DataProcessing1Source(const Register& rd,
1724                              const Register& rn,
1725                              DataProcessing1SourceOp op);
1726   void DataProcessing3Source(const Register& rd,
1727                              const Register& rn,
1728                              const Register& rm,
1729                              const Register& ra,
1730                              DataProcessing3SourceOp op);
1731   void FPDataProcessing1Source(const FPRegister& fd,
1732                                const FPRegister& fn,
1733                                FPDataProcessing1SourceOp op);
1734   void FPDataProcessing2Source(const FPRegister& fd,
1735                                const FPRegister& fn,
1736                                const FPRegister& fm,
1737                                FPDataProcessing2SourceOp op);
1738   void FPDataProcessing3Source(const FPRegister& fd,
1739                                const FPRegister& fn,
1740                                const FPRegister& fm,
1741                                const FPRegister& fa,
1742                                FPDataProcessing3SourceOp op);
1743
1744   // Encoding helpers.
1745   static bool IsImmFP32(float imm);
1746   static bool IsImmFP64(double imm);
1747
1748   void RecordLiteral(int64_t imm, unsigned size);
1749
1750   // Emit the instruction at pc_.
1751   void Emit(Instr instruction) {
1752     assert(cb_.canEmit(sizeof(instruction)));
1753     assert(sizeof(instruction) == sizeof(uint32_t));
1754
1755 #ifdef DEBUG
1756     finalized_ = false;
1757 #endif
1758
1759     cb_.dword(instruction);
1760     CheckBufferSpace();
1761   }
1762
1763   // Emit data inline in the instruction stream.
1764   void EmitData(void const * data, unsigned size) {
1765     assert(cb_.canEmit(size));
1766
1767 #ifdef DEBUG
1768     finalized_ = false;
1769 #endif
1770
1771     // TODO: Record this 'instruction' as data, so that it can be disassembled
1772     // correctly.
1773     cb_.bytes(size, reinterpret_cast<const uint8_t*>(data));
1774     CheckBufferSpace();
1775   }
1776
1777   inline void CheckBufferSpace() {
1778     assert(cb_.available() > 0);
1779     if (cb_.frontier() > next_literal_pool_check_) {
1780       CheckLiteralPool();
1781     }
1782   }
1783
1784   // The buffer into which code and relocation info are generated.
1785   HPHP::CodeBlock& cb_;
1786
1787   std::list<Literal*> literals_;
1788   HPHP::CodeAddress next_literal_pool_check_;
1789   unsigned literal_pool_monitor_;
1790
1791   friend class BlockLiteralPoolScope;
1792
1793 #ifdef DEBUG
1794   bool finalized_;
1795 #endif
1796 };
1797
1798 class BlockLiteralPoolScope {
1799  public:
1800   explicit BlockLiteralPoolScope(Assembler* assm) : assm_(assm) {
1801     assm_->BlockLiteralPool();
1802   }
1803
1804   ~BlockLiteralPoolScope() {
1805     assm_->ReleaseLiteralPool();
1806   }
1807
1808  private:
1809   Assembler* assm_;
1810 };
1811 }  // namespace vixl
1812
1813 #endif  // VIXL_A64_ASSEMBLER_A64_H_