yjit_asm.c

   1 // This file is a fragment of the yjit.o compilation unit. See yjit.c.
   2 //
   3 // Note that the definition for some of these functions don't specify
   4 // static inline, but their declaration in yjit_asm.h do. The resulting
   5 // linkage is the same as if they both specify. The relevant sections in
   6 // N1256 is 6.2.2p4, 6.2.2p5, and 6.7.4p5.
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <stdarg.h>
  11 #include <stdint.h>
  12 #include <assert.h>
  13 #include <errno.h>
  14
  15 // For mmapp(), sysconf()
  16 #ifndef _WIN32
  17 #include <unistd.h>
  18 #include <sys/mman.h>
  19 #endif
  20
  21 #include "yjit_asm.h"
  22
  23 // Compute the number of bits needed to encode a signed value
  24 uint32_t sig_imm_size(int64_t imm)
  25 {
  26     // Compute the smallest size this immediate fits in
  27     if (imm >= INT8_MIN && imm <= INT8_MAX)
  28         return 8;
  29     if (imm >= INT16_MIN && imm <= INT16_MAX)
  30         return 16;
  31     if (imm >= INT32_MIN && imm <= INT32_MAX)
  32         return 32;
  33
  34     return 64;
  35 }
  36
  37 // Compute the number of bits needed to encode an unsigned value
  38 uint32_t unsig_imm_size(uint64_t imm)
  39 {
  40     // Compute the smallest size this immediate fits in
  41     if (imm <= UINT8_MAX)
  42         return 8;
  43     else if (imm <= UINT16_MAX)
  44         return 16;
  45     else if (imm <= UINT32_MAX)
  46         return 32;
  47
  48     return 64;
  49 }
  50
  51 x86opnd_t mem_opnd(uint32_t num_bits, x86opnd_t base_reg, int32_t disp)
  52 {
  53     bool is_iprel = base_reg.as.reg.reg_type == REG_IP;
  54
  55     x86opnd_t opnd = {
  56         OPND_MEM,
  57         num_bits,
  58         .as.mem = { base_reg.as.reg.reg_no, 0, 0, false, is_iprel, disp }
  59     };
  60
  61     return opnd;
  62 }
  63
  64 x86opnd_t mem_opnd_sib(uint32_t num_bits, x86opnd_t base_reg, x86opnd_t index_reg, int32_t scale, int32_t disp)
  65 {
  66     uint8_t scale_exp;
  67     switch (scale) {
  68       case 8:
  69         scale_exp = 3;
  70         break;
  71       case 4:
  72         scale_exp = 2;
  73         break;
  74       case 2:
  75         scale_exp = 1;
  76         break;
  77       case 1:
  78         scale_exp = 0;
  79         break;
  80       default:
  81         rb_bug("yjit: scale not one of 1,2,4,8");
  82         break;
  83     }
  84
  85     bool is_iprel = base_reg.as.reg.reg_type == REG_IP;
  86
  87     x86opnd_t opnd = {
  88         OPND_MEM,
  89         num_bits,
  90         .as.mem = {
  91             .base_reg_no = base_reg.as.reg.reg_no,
  92             .idx_reg_no = index_reg.as.reg.reg_no,
  93             .has_idx = 1,
  94             .scale_exp = scale_exp,
  95             .is_iprel = is_iprel,
  96             .disp = disp
  97         }
  98     };
  99
 100     return opnd;
 101 }
 102
 103 static x86opnd_t resize_opnd(x86opnd_t opnd, uint32_t num_bits)
 104 {
 105     assert (num_bits % 8 == 0);
 106     x86opnd_t sub = opnd;
 107     sub.num_bits = num_bits;
 108     return sub;
 109 }
 110
 111 x86opnd_t imm_opnd(int64_t imm)
 112 {
 113     x86opnd_t opnd = {
 114         OPND_IMM,
 115         sig_imm_size(imm),
 116         .as.imm = imm
 117     };
 118
 119     return opnd;
 120 }
 121
 122 x86opnd_t const_ptr_opnd(const void *ptr)
 123 {
 124     x86opnd_t opnd = {
 125         OPND_IMM,
 126         64,
 127         .as.unsig_imm = (uint64_t)ptr
 128     };
 129
 130     return opnd;
 131 }
 132
 133 // Align the current write position to a multiple of bytes
 134 static uint8_t *align_ptr(uint8_t *ptr, uint32_t multiple)
 135 {
 136     // Compute the pointer modulo the given alignment boundary
 137     uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
 138
 139     // If the pointer is already aligned, stop
 140     if (rem == 0)
 141         return ptr;
 142
 143     // Pad the pointer by the necessary amount to align it
 144     uint32_t pad = multiple - rem;
 145
 146     return ptr + pad;
 147 }
 148
 149 // Allocate a block of executable memory
 150 static uint8_t *alloc_exec_mem(uint32_t mem_size)
 151 {
 152 #ifndef _WIN32
 153     uint8_t *mem_block;
 154
 155     // On Linux
 156     #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
 157         // Align the requested address to page size
 158         uint32_t page_size = (uint32_t)sysconf(_SC_PAGESIZE);
 159         uint8_t *req_addr = align_ptr((uint8_t*)&alloc_exec_mem, page_size);
 160
 161         do {
 162             // Try to map a chunk of memory as executable
 163             mem_block = (uint8_t*)mmap(
 164                 (void*)req_addr,
 165                 mem_size,
 166                 PROT_READ | PROT_EXEC,
 167                 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
 168                 -1,
 169                 0
 170             );
 171
 172             // If we succeeded, stop
 173             if (mem_block != MAP_FAILED) {
 174                 break;
 175             }
 176
 177             // +4MB
 178             req_addr += 4 * 1024 * 1024;
 179         } while (req_addr < (uint8_t*)&alloc_exec_mem + INT32_MAX);
 180
 181     // On MacOS and other platforms
 182     #else
 183         // Try to map a chunk of memory as executable
 184         mem_block = (uint8_t*)mmap(
 185             (void*)alloc_exec_mem,
 186             mem_size,
 187             PROT_READ | PROT_EXEC,
 188             MAP_PRIVATE | MAP_ANONYMOUS,
 189             -1,
 190             0
 191         );
 192     #endif
 193
 194     // Fallback
 195     if (mem_block == MAP_FAILED) {
 196         // Try again without the address hint (e.g., valgrind)
 197         mem_block = (uint8_t*)mmap(
 198             NULL,
 199             mem_size,
 200             PROT_READ | PROT_EXEC,
 201             MAP_PRIVATE | MAP_ANONYMOUS,
 202             -1,
 203             0
 204             );
 205     }
 206
 207     // Check that the memory mapping was successful
 208     if (mem_block == MAP_FAILED) {
 209         perror("mmap call failed");
 210         exit(-1);
 211     }
 212
 213     codeblock_t block;
 214     codeblock_t *cb = &block;
 215
 216     cb_init(cb, mem_block, mem_size);
 217
 218     // Fill the executable memory with PUSH DS (0x1E) so that
 219     // executing uninitialized memory will fault with #UD in
 220     // 64-bit mode.
 221     cb_mark_all_writeable(cb);
 222     memset(mem_block, 0x1E, mem_size);
 223     cb_mark_all_executable(cb);
 224
 225     return mem_block;
 226 #else
 227     // Windows not supported for now
 228     return NULL;
 229 #endif
 230 }
 231
 232 // Initialize a code block object
 233 void cb_init(codeblock_t *cb, uint8_t *mem_block, uint32_t mem_size)
 234 {
 235     assert (mem_block);
 236     cb->mem_block_ = mem_block;
 237     cb->mem_size = mem_size;
 238     cb->write_pos = 0;
 239     cb->num_labels = 0;
 240     cb->num_refs = 0;
 241     cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
 242 }
 243
 244 // Set the current write position
 245 void cb_set_pos(codeblock_t *cb, uint32_t pos)
 246 {
 247     // Assert here since while assembler functions do bounds checking, there is
 248     // nothing stopping users from taking out an out-of-bounds pointer and
 249     // doing bad accesses with it.
 250     assert (pos < cb->mem_size);
 251     cb->write_pos = pos;
 252 }
 253
 254 // Align the current write position to a multiple of bytes
 255 void cb_align_pos(codeblock_t *cb, uint32_t multiple)
 256 {
 257     // Compute the pointer modulo the given alignment boundary
 258     uint8_t *ptr = cb_get_write_ptr(cb);
 259     uint8_t *aligned_ptr = align_ptr(ptr, multiple);
 260     const uint32_t write_pos = cb->write_pos;
 261
 262     // Pad the pointer by the necessary amount to align it
 263     ptrdiff_t pad = aligned_ptr - ptr;
 264     cb_set_pos(cb, write_pos + (int32_t)pad);
 265 }
 266
 267 // Set the current write position from a pointer
 268 void cb_set_write_ptr(codeblock_t *cb, uint8_t *code_ptr)
 269 {
 270     intptr_t pos = code_ptr - cb->mem_block_;
 271     assert (pos < cb->mem_size);
 272     cb_set_pos(cb, (uint32_t)pos);
 273 }
 274
 275 // Get a direct pointer into the executable memory block
 276 uint8_t *cb_get_ptr(const codeblock_t *cb, uint32_t index)
 277 {
 278     if (index < cb->mem_size) {
 279         return &cb->mem_block_[index];
 280     }
 281     else {
 282         return NULL;
 283     }
 284 }
 285
 286 // Get a direct pointer to the current write position
 287 uint8_t *cb_get_write_ptr(const codeblock_t *cb)
 288 {
 289     return cb_get_ptr(cb, cb->write_pos);
 290 }
 291
 292 // Write a byte at the current position
 293 void cb_write_byte(codeblock_t *cb, uint8_t byte)
 294 {
 295     assert (cb->mem_block_);
 296     if (cb->write_pos < cb->mem_size) {
 297         cb_mark_position_writeable(cb, cb->write_pos);
 298         cb->mem_block_[cb->write_pos] = byte;
 299         cb->write_pos++;
 300     }
 301     else {
 302         cb->dropped_bytes = true;
 303     }
 304 }
 305
 306 // Write multiple bytes starting from the current position
 307 void cb_write_bytes(codeblock_t *cb, uint32_t num_bytes, ...)
 308 {
 309     va_list va;
 310     va_start(va, num_bytes);
 311
 312     for (uint32_t i = 0; i < num_bytes; ++i)
 313     {
 314         uint8_t byte = va_arg(va, int);
 315         cb_write_byte(cb, byte);
 316     }
 317
 318     va_end(va);
 319 }
 320
 321 // Write a signed integer over a given number of bits at the current position
 322 void cb_write_int(codeblock_t *cb, uint64_t val, uint32_t num_bits)
 323 {
 324     assert (num_bits > 0);
 325     assert (num_bits % 8 == 0);
 326
 327     // Switch on the number of bits
 328     switch (num_bits) {
 329       case 8:
 330         cb_write_byte(cb, (uint8_t)val);
 331         break;
 332
 333       case 16:
 334         cb_write_bytes(
 335             cb,
 336             2,
 337             (uint8_t)((val >> 0) & 0xFF),
 338             (uint8_t)((val >> 8) & 0xFF)
 339         );
 340         break;
 341
 342       case 32:
 343         cb_write_bytes(
 344             cb,
 345             4,
 346             (uint8_t)((val >>  0) & 0xFF),
 347             (uint8_t)((val >>  8) & 0xFF),
 348             (uint8_t)((val >> 16) & 0xFF),
 349             (uint8_t)((val >> 24) & 0xFF)
 350         );
 351         break;
 352
 353       default:
 354         {
 355             // Compute the size in bytes
 356             uint32_t num_bytes = num_bits / 8;
 357
 358             // Write out the bytes
 359             for (uint32_t i = 0; i < num_bytes; ++i)
 360             {
 361                 uint8_t byte_val = (uint8_t)(val & 0xFF);
 362                 cb_write_byte(cb, byte_val);
 363                 val >>= 8;
 364             }
 365         }
 366     }
 367 }
 368
 369 // Allocate a new label with a given name
 370 uint32_t cb_new_label(codeblock_t *cb, const char *name)
 371 {
 372     //if (hasASM)
 373     //    writeString(to!string(label) ~ ":");
 374
 375     assert (cb->num_labels < MAX_LABELS);
 376
 377     // Allocate the new label
 378     uint32_t label_idx = cb->num_labels++;
 379
 380     // This label doesn't have an address yet
 381     cb->label_addrs[label_idx] = 0;
 382     cb->label_names[label_idx] = name;
 383
 384     return label_idx;
 385 }
 386
 387 // Write a label at the current address
 388 void cb_write_label(codeblock_t *cb, uint32_t label_idx)
 389 {
 390     assert (label_idx < MAX_LABELS);
 391     cb->label_addrs[label_idx] = cb->write_pos;
 392 }
 393
 394 // Add a label reference at the current write position
 395 void cb_label_ref(codeblock_t *cb, uint32_t label_idx)
 396 {
 397     assert (label_idx < MAX_LABELS);
 398     assert (cb->num_refs < MAX_LABEL_REFS);
 399
 400     // Keep track of the reference
 401     cb->label_refs[cb->num_refs] = (labelref_t){ cb->write_pos, label_idx };
 402     cb->num_refs++;
 403 }
 404
 405 // Link internal label references
 406 void cb_link_labels(codeblock_t *cb)
 407 {
 408     uint32_t orig_pos = cb->write_pos;
 409
 410     // For each label reference
 411     for (uint32_t i = 0; i < cb->num_refs; ++i)
 412     {
 413         uint32_t ref_pos = cb->label_refs[i].pos;
 414         uint32_t label_idx = cb->label_refs[i].label_idx;
 415         assert (ref_pos < cb->mem_size);
 416         assert (label_idx < MAX_LABELS);
 417
 418         uint32_t label_addr = cb->label_addrs[label_idx];
 419         assert (label_addr < cb->mem_size);
 420
 421         // Compute the offset from the reference's end to the label
 422         int64_t offset = (int64_t)label_addr - (int64_t)(ref_pos + 4);
 423
 424         cb_set_pos(cb, ref_pos);
 425         cb_write_int(cb, offset, 32);
 426     }
 427
 428     cb->write_pos = orig_pos;
 429
 430     // Clear the label positions and references
 431     cb->num_labels = 0;
 432     cb->num_refs = 0;
 433 }
 434
 435 // Check if an operand needs a REX byte to be encoded
 436 static bool rex_needed(x86opnd_t opnd)
 437 {
 438     if (opnd.type == OPND_NONE || opnd.type == OPND_IMM)
 439     {
 440         return false;
 441     }
 442
 443     if (opnd.type == OPND_REG)
 444     {
 445         return (
 446             opnd.as.reg.reg_no > 7 ||
 447             (opnd.num_bits == 8 && opnd.as.reg.reg_no >= 4 && opnd.as.reg.reg_no <= 7)
 448         );
 449     }
 450
 451     if (opnd.type == OPND_MEM)
 452     {
 453         return (opnd.as.mem.base_reg_no > 7) || (opnd.as.mem.has_idx && opnd.as.mem.idx_reg_no > 7);
 454     }
 455
 456     rb_bug("unreachable");
 457 }
 458
 459 // Check if an SIB byte is needed to encode this operand
 460 static bool sib_needed(x86opnd_t opnd)
 461 {
 462     if (opnd.type != OPND_MEM)
 463         return false;
 464
 465     return (
 466         opnd.as.mem.has_idx ||
 467         opnd.as.mem.base_reg_no == RSP.as.reg.reg_no ||
 468         opnd.as.mem.base_reg_no == R12.as.reg.reg_no
 469     );
 470 }
 471
 472 // Compute the size of the displacement field needed for a memory operand
 473 static uint32_t disp_size(x86opnd_t opnd)
 474 {
 475     assert (opnd.type == OPND_MEM);
 476
 477     // If using RIP as the base, use disp32
 478     if (opnd.as.mem.is_iprel)
 479     {
 480         return 32;
 481     }
 482
 483     // Compute the required displacement size
 484     if (opnd.as.mem.disp != 0)
 485     {
 486         uint32_t num_bits = sig_imm_size(opnd.as.mem.disp);
 487         assert (num_bits <= 32 && "displacement does not fit in 32 bits");
 488
 489         // x86 can only encode 8-bit and 32-bit displacements
 490         if (num_bits == 16)
 491             num_bits = 32;;
 492
 493         return num_bits;
 494     }
 495
 496     // If EBP or RBP or R13 is used as the base, displacement must be encoded
 497     if (opnd.as.mem.base_reg_no == RBP.as.reg.reg_no ||
 498         opnd.as.mem.base_reg_no == R13.as.reg.reg_no)
 499     {
 500         return 8;
 501     }
 502
 503     return 0;
 504 }
 505
 506 // Write the REX byte
 507 static void cb_write_rex(
 508     codeblock_t *cb,
 509     bool w_flag,
 510     uint8_t reg_no,
 511     uint8_t idx_reg_no,
 512     uint8_t rm_reg_no
 513 )
 514 {
 515     // 0 1 0 0 w r x b
 516     // w - 64-bit operand size flag
 517     // r - MODRM.reg extension
 518     // x - SIB.index extension
 519     // b - MODRM.rm or SIB.base extension
 520     uint8_t w = w_flag? 1:0;
 521     uint8_t r = (reg_no & 8)? 1:0;
 522     uint8_t x = (idx_reg_no & 8)? 1:0;
 523     uint8_t b = (rm_reg_no & 8)? 1:0;
 524
 525     // Encode and write the REX byte
 526     uint8_t rexByte = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b);
 527     cb_write_byte(cb, rexByte);
 528 }
 529
 530 // Write an opcode byte with an embedded register operand
 531 static void cb_write_opcode(codeblock_t *cb, uint8_t opcode, x86opnd_t reg)
 532 {
 533     // Write the reg field into the opcode byte
 534     uint8_t op_byte = opcode | (reg.as.reg.reg_no & 7);
 535     cb_write_byte(cb, op_byte);
 536 }
 537
 538 // Encode an RM instruction
 539 static void cb_write_rm(
 540     codeblock_t *cb,
 541     bool szPref,
 542     bool rexW,
 543     x86opnd_t r_opnd,
 544     x86opnd_t rm_opnd,
 545     uint8_t opExt,
 546     uint32_t op_len,
 547     ...)
 548 {
 549     assert (op_len > 0 && op_len <= 3);
 550     assert (r_opnd.type == OPND_REG || r_opnd.type == OPND_NONE);
 551
 552     // Flag to indicate the REX prefix is needed
 553     bool need_rex = rexW || rex_needed(r_opnd) || rex_needed(rm_opnd);
 554
 555     // Flag to indicate SIB byte is needed
 556     bool need_sib = sib_needed(r_opnd) || sib_needed(rm_opnd);
 557
 558     // Add the operand-size prefix, if needed
 559     if (szPref == true)
 560         cb_write_byte(cb, 0x66);
 561
 562     // Add the REX prefix, if needed
 563     if (need_rex)
 564     {
 565         // 0 1 0 0 w r x b
 566         // w - 64-bit operand size flag
 567         // r - MODRM.reg extension
 568         // x - SIB.index extension
 569         // b - MODRM.rm or SIB.base extension
 570
 571         uint8_t w = rexW? 1:0;
 572
 573         uint8_t r;
 574         if (r_opnd.type != OPND_NONE)
 575             r = (r_opnd.as.reg.reg_no & 8)? 1:0;
 576         else
 577             r = 0;
 578
 579         uint8_t x;
 580         if (need_sib && rm_opnd.as.mem.has_idx)
 581             x = (rm_opnd.as.mem.idx_reg_no & 8)? 1:0;
 582         else
 583             x = 0;
 584
 585         uint8_t b;
 586         if (rm_opnd.type == OPND_REG)
 587             b = (rm_opnd.as.reg.reg_no & 8)? 1:0;
 588         else if (rm_opnd.type == OPND_MEM)
 589             b = (rm_opnd.as.mem.base_reg_no & 8)? 1:0;
 590         else
 591             b = 0;
 592
 593         // Encode and write the REX byte
 594         uint8_t rex_byte = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b);
 595         cb_write_byte(cb, rex_byte);
 596     }
 597
 598     // Write the opcode bytes to the code block
 599     va_list va;
 600     va_start(va, op_len);
 601     for (uint32_t i = 0; i < op_len; ++i)
 602     {
 603         uint8_t byte = va_arg(va, int);
 604         cb_write_byte(cb, byte);
 605     }
 606     va_end(va);
 607
 608     // MODRM.mod (2 bits)
 609     // MODRM.reg (3 bits)
 610     // MODRM.rm  (3 bits)
 611
 612     assert (
 613         !(opExt != 0xFF && r_opnd.type != OPND_NONE) &&
 614         "opcode extension and register operand present"
 615     );
 616
 617     // Encode the mod field
 618     uint8_t mod;
 619     if (rm_opnd.type == OPND_REG)
 620     {
 621         mod = 3;
 622     }
 623     else
 624     {
 625         uint32_t dsize = disp_size(rm_opnd);
 626         if (dsize == 0 || rm_opnd.as.mem.is_iprel)
 627             mod = 0;
 628         else if (dsize == 8)
 629             mod = 1;
 630         else if (dsize == 32)
 631             mod = 2;
 632         else
 633             rb_bug("unreachable");
 634     }
 635
 636     // Encode the reg field
 637     uint8_t reg;
 638     if (opExt != 0xFF)
 639         reg = opExt;
 640     else if (r_opnd.type == OPND_REG)
 641         reg = r_opnd.as.reg.reg_no & 7;
 642     else
 643         reg = 0;
 644
 645     // Encode the rm field
 646     uint8_t rm;
 647     if (rm_opnd.type == OPND_REG)
 648     {
 649         rm = rm_opnd.as.reg.reg_no & 7;
 650     }
 651     else
 652     {
 653         if (need_sib)
 654             rm = 4;
 655         else
 656             rm = rm_opnd.as.mem.base_reg_no & 7;
 657     }
 658
 659     // Encode and write the ModR/M byte
 660     uint8_t rm_byte = (mod << 6) + (reg << 3) + (rm);
 661     cb_write_byte(cb, rm_byte);
 662
 663     // Add the SIB byte, if needed
 664     if (need_sib)
 665     {
 666         // SIB.scale (2 bits)
 667         // SIB.index (3 bits)
 668         // SIB.base  (3 bits)
 669
 670         assert (rm_opnd.type == OPND_MEM);
 671
 672         // Encode the scale value
 673         uint8_t scale = rm_opnd.as.mem.scale_exp;
 674
 675         // Encode the index value
 676         uint8_t index;
 677         if (!rm_opnd.as.mem.has_idx)
 678             index = 4;
 679         else
 680             index = rm_opnd.as.mem.idx_reg_no & 7;
 681
 682         // Encode the base register
 683         uint8_t base = rm_opnd.as.mem.base_reg_no & 7;
 684
 685         // Encode and write the SIB byte
 686         uint8_t sib_byte = (scale << 6) + (index << 3) + (base);
 687         cb_write_byte(cb, sib_byte);
 688     }
 689
 690     // Add the displacement
 691     if (rm_opnd.type == OPND_MEM)
 692     {
 693         uint32_t dsize = disp_size(rm_opnd);
 694         if (dsize > 0)
 695             cb_write_int(cb, rm_opnd.as.mem.disp, dsize);
 696     }
 697 }
 698
 699 // Encode a mul-like single-operand RM instruction
 700 static void write_rm_unary(
 701     codeblock_t *cb,
 702     const char *mnem,
 703     uint8_t opMemReg8,
 704     uint8_t opMemRegPref,
 705     uint8_t opExt,
 706     x86opnd_t opnd)
 707 {
 708     // Write a disassembly string
 709     //cb.writeASM(mnem, opnd);
 710
 711     // Check the size of opnd0
 712     uint32_t opndSize;
 713     if (opnd.type == OPND_REG || opnd.type == OPND_MEM)
 714         opndSize = opnd.num_bits;
 715     else
 716         rb_bug("yjit: invalid operand");
 717
 718     assert (opndSize == 8 || opndSize == 16 || opndSize == 32 || opndSize == 64);
 719     bool szPref = opndSize == 16;
 720     bool rexW = opndSize == 64;
 721
 722     if (opndSize == 8)
 723         cb_write_rm(cb, false, false, NO_OPND, opnd, opExt, 1, opMemReg8);
 724     else
 725         cb_write_rm(cb, szPref, rexW, NO_OPND, opnd, opExt, 1, opMemRegPref);
 726 }
 727
 728 // Encode an add-like RM instruction with multiple possible encodings
 729 static void cb_write_rm_multi(
 730     codeblock_t *cb,
 731     const char *mnem,
 732     uint8_t opMemReg8,
 733     uint8_t opMemRegPref,
 734     uint8_t opRegMem8,
 735     uint8_t opRegMemPref,
 736     uint8_t opMemImm8,
 737     uint8_t opMemImmSml,
 738     uint8_t opMemImmLrg,
 739     uint8_t opExtImm,
 740     x86opnd_t opnd0,
 741     x86opnd_t opnd1)
 742 {
 743     assert (opnd0.type == OPND_REG || opnd0.type == OPND_MEM);
 744
 745     /*
 746     // Write disassembly string
 747     if (!opnd1.isNone)
 748         cb.writeASM(mnem, opnd0, opnd1);
 749     else
 750         cb.writeASM(mnem, opnd0);
 751     */
 752
 753     // Check the size of opnd0
 754     uint32_t opndSize = opnd0.num_bits;
 755
 756     // Check the size of opnd1
 757     if (opnd1.type == OPND_REG || opnd1.type == OPND_MEM)
 758     {
 759         assert (opnd1.num_bits == opndSize && "operand size mismatch");
 760     }
 761     else if (opnd1.type == OPND_IMM)
 762     {
 763         assert (opnd1.num_bits <= opndSize);
 764     }
 765
 766     assert (opndSize == 8 || opndSize == 16 || opndSize == 32 || opndSize == 64);
 767     bool szPref = opndSize == 16;
 768     bool rexW = opndSize == 64;
 769
 770     // R/M + Reg
 771     if ((opnd0.type == OPND_MEM && opnd1.type == OPND_REG) ||
 772         (opnd0.type == OPND_REG && opnd1.type == OPND_REG))
 773     {
 774         // R/M is opnd0
 775         if (opndSize == 8)
 776             cb_write_rm(cb, false, false, opnd1, opnd0, 0xFF, 1, opMemReg8);
 777         else
 778             cb_write_rm(cb, szPref, rexW, opnd1, opnd0, 0xFF, 1, opMemRegPref);
 779     }
 780
 781     // Reg + R/M
 782     else if (opnd0.type == OPND_REG && opnd1.type == OPND_MEM)
 783     {
 784         // R/M is opnd1
 785         if (opndSize == 8)
 786             cb_write_rm(cb, false, false, opnd0, opnd1, 0xFF, 1, opRegMem8);
 787         else
 788             cb_write_rm(cb, szPref, rexW, opnd0, opnd1, 0xFF, 1, opRegMemPref);
 789     }
 790
 791     // R/M + Imm
 792     else if (opnd1.type == OPND_IMM)
 793     {
 794         // 8-bit immediate
 795         if (opnd1.num_bits <= 8)
 796         {
 797             if (opndSize == 8)
 798                 cb_write_rm(cb, false, false, NO_OPND, opnd0, opExtImm, 1, opMemImm8);
 799             else
 800                 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExtImm, 1, opMemImmSml);
 801
 802             cb_write_int(cb, opnd1.as.imm, 8);
 803         }
 804
 805         // 32-bit immediate
 806         else if (opnd1.num_bits <= 32)
 807         {
 808             assert (opnd1.num_bits <= opndSize && "immediate too large for dst");
 809             cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExtImm, 1, opMemImmLrg);
 810             cb_write_int(cb, opnd1.as.imm, (opndSize > 32)? 32:opndSize);
 811         }
 812
 813         // Immediate too large
 814         else
 815         {
 816             assert (false && "immediate value too large");
 817         }
 818     }
 819
 820     // Invalid operands
 821     else
 822     {
 823         assert (false && "invalid operand combination");
 824     }
 825 }
 826
 827 // Encode a single-operand shift instruction
 828 static void cb_write_shift(
 829     codeblock_t *cb,
 830     const char *mnem,
 831     uint8_t opMemOnePref,
 832     uint8_t opMemClPref,
 833     uint8_t opMemImmPref,
 834     uint8_t opExt,
 835     x86opnd_t opnd0,
 836     x86opnd_t opnd1)
 837 {
 838     // Write a disassembly string
 839     //cb.writeASM(mnem, opnd0, opnd1);
 840
 841     // Check the size of opnd0
 842     uint32_t opndSize;
 843     if (opnd0.type == OPND_REG || opnd0.type == OPND_MEM)
 844         opndSize = opnd0.num_bits;
 845     else
 846         rb_bug("yjit: shift: invalid first operand");
 847
 848     assert (opndSize == 16 || opndSize == 32 || opndSize == 64);
 849     bool szPref = opndSize == 16;
 850     bool rexW = opndSize == 64;
 851
 852     if (opnd1.type == OPND_IMM)
 853     {
 854         if (opnd1.as.imm == 1)
 855         {
 856             cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExt, 1, opMemOnePref);
 857         }
 858         else
 859         {
 860             assert (opnd1.num_bits <= 8);
 861             cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExt, 1, opMemImmPref);
 862             cb_write_byte(cb, (uint8_t)opnd1.as.imm);
 863         }
 864     }
 865     /*
 866     else if (opnd1.isReg && opnd1.reg == CL)
 867     {
 868         cb.writeRMInstr!('l', opExt, opMemClPref)(szPref, rexW, opnd0, X86Opnd.NONE);
 869     }
 870     */
 871     else
 872     {
 873         assert (false);
 874     }
 875 }
 876
 877 // Encode a relative jump to a label (direct or conditional)
 878 // Note: this always encodes a 32-bit offset
 879 static void cb_write_jcc(codeblock_t *cb, const char *mnem, uint8_t op0, uint8_t op1, uint32_t label_idx)
 880 {
 881     //cb.writeASM(mnem, label);
 882
 883     // Write the opcode
 884     if (op0 != 0xFF)
 885         cb_write_byte(cb, op0);
 886     cb_write_byte(cb, op1);
 887
 888     // Add a reference to the label
 889     cb_label_ref(cb, label_idx);
 890
 891     // Relative 32-bit offset to be patched
 892     cb_write_int(cb, 0, 32);
 893 }
 894
 895 // Encode a relative jump to a pointer at a 32-bit offset (direct or conditional)
 896 static void cb_write_jcc_ptr(codeblock_t *cb, const char *mnem, uint8_t op0, uint8_t op1, uint8_t *dst_ptr)
 897 {
 898     //cb.writeASM(mnem, label);
 899
 900     // Write the opcode
 901     if (op0 != 0xFF)
 902         cb_write_byte(cb, op0);
 903     cb_write_byte(cb, op1);
 904
 905     // Pointer to the end of this jump instruction
 906     uint8_t *end_ptr = cb_get_ptr(cb, cb->write_pos + 4);
 907
 908     // Compute the jump offset
 909     int64_t rel64 = (int64_t)(dst_ptr - end_ptr);
 910     if (rel64 >= INT32_MIN && rel64 <= INT32_MAX) {
 911         // Write the relative 32-bit jump offset
 912         cb_write_int(cb, (int32_t)rel64, 32);
 913     }
 914     else {
 915         // Offset doesn't fit in 4 bytes. Report error.
 916         cb->dropped_bytes = true;
 917     }
 918 }
 919
 920 // Encode a conditional move instruction
 921 static void cb_write_cmov(codeblock_t *cb, const char *mnem, uint8_t opcode1, x86opnd_t dst, x86opnd_t src)
 922 {
 923     //cb.writeASM(mnem, dst, src);
 924
 925     assert (dst.type == OPND_REG);
 926     assert (src.type == OPND_REG || src.type == OPND_MEM);
 927     assert (dst.num_bits >= 16 && "invalid dst reg size in cmov");
 928
 929     bool szPref = dst.num_bits == 16;
 930     bool rexW = dst.num_bits == 64;
 931
 932     cb_write_rm(cb, szPref, rexW, dst, src, 0xFF, 2, 0x0F, opcode1);
 933 }
 934
 935 // add - Integer addition
 936 void add(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
 937 {
 938     cb_write_rm_multi(
 939         cb,
 940         "add",
 941         0x00, // opMemReg8
 942         0x01, // opMemRegPref
 943         0x02, // opRegMem8
 944         0x03, // opRegMemPref
 945         0x80, // opMemImm8
 946         0x83, // opMemImmSml
 947         0x81, // opMemImmLrg
 948         0x00, // opExtImm
 949         opnd0,
 950         opnd1
 951     );
 952 }
 953
 954 /// and - Bitwise AND
 955 void and(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
 956 {
 957     cb_write_rm_multi(
 958         cb,
 959         "and",
 960         0x20, // opMemReg8
 961         0x21, // opMemRegPref
 962         0x22, // opRegMem8
 963         0x23, // opRegMemPref
 964         0x80, // opMemImm8
 965         0x83, // opMemImmSml
 966         0x81, // opMemImmLrg
 967         0x04, // opExtImm
 968         opnd0,
 969         opnd1
 970     );
 971 }
 972
 973 // call - Call to a pointer with a 32-bit displacement offset
 974 static void call_rel32(codeblock_t *cb, int32_t rel32)
 975 {
 976     //cb.writeASM("call", rel32);
 977
 978     // Write the opcode
 979     cb_write_byte(cb, 0xE8);
 980
 981     // Write the relative 32-bit jump offset
 982     cb_write_int(cb, (int32_t)rel32, 32);
 983 }
 984
 985 // call - Call a pointer, encode with a 32-bit offset if possible
 986 void call_ptr(codeblock_t *cb, x86opnd_t scratch_reg, uint8_t *dst_ptr)
 987 {
 988     assert (scratch_reg.type == OPND_REG);
 989
 990     // Pointer to the end of this call instruction
 991     uint8_t *end_ptr = cb_get_ptr(cb, cb->write_pos + 5);
 992
 993     // Compute the jump offset
 994     int64_t rel64 = (int64_t)(dst_ptr - end_ptr);
 995
 996     // If the offset fits in 32-bit
 997     if (rel64 >= INT32_MIN && rel64 <= INT32_MAX) {
 998         call_rel32(cb, (int32_t)rel64);
 999         return;
1000     }
1001
1002     // Move the pointer into the scratch register and call
1003     mov(cb, scratch_reg, const_ptr_opnd(dst_ptr));
1004     call(cb, scratch_reg);
1005 }
1006
1007 /// call - Call to label with 32-bit offset
1008 void call_label(codeblock_t *cb, uint32_t label_idx)
1009 {
1010     //cb.writeASM("call", label);
1011
1012     // Write the opcode
1013     cb_write_byte(cb, 0xE8);
1014
1015     // Add a reference to the label
1016     cb_label_ref(cb, label_idx);
1017
1018     // Relative 32-bit offset to be patched
1019     cb_write_int(cb, 0, 32);
1020 }
1021
1022 /// call - Indirect call with an R/M operand
1023 void call(codeblock_t *cb, x86opnd_t opnd)
1024 {
1025     //cb.writeASM("call", opnd);
1026     cb_write_rm(cb, false, false, NO_OPND, opnd, 2, 1, 0xFF);
1027 }
1028
1029 /// cmovcc - Conditional move
1030 void cmova(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmova", 0x47, dst, src); }
1031 void cmovae(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovae", 0x43, dst, src); }
1032 void cmovb(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovb", 0x42, dst, src); }
1033 void cmovbe(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovbe", 0x46, dst, src); }
1034 void cmovc(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovc", 0x42, dst, src); }
1035 void cmove(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmove", 0x44, dst, src); }
1036 void cmovg(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovg", 0x4F, dst, src); }
1037 void cmovge(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovge", 0x4D, dst, src); }
1038 void cmovl(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovl", 0x4C, dst, src); }
1039 void cmovle(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovle", 0x4E, dst, src); }
1040 void cmovna(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovna", 0x46, dst, src); }
1041 void cmovnae(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnae", 0x42, dst, src); }
1042 void cmovnb(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnb", 0x43, dst, src); }
1043 void cmovnbe(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnbe", 0x47, dst, src); }
1044 void cmovnc(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnc", 0x43, dst, src); }
1045 void cmovne(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovne", 0x45, dst, src); }
1046 void cmovng(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovng", 0x4E, dst, src); }
1047 void cmovnge(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnge", 0x4C, dst, src); }
1048 void cmovnl(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnl" , 0x4D, dst, src); }
1049 void cmovnle(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnle", 0x4F, dst, src); }
1050 void cmovno(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovno", 0x41, dst, src); }
1051 void cmovnp(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnp", 0x4B, dst, src); }
1052 void cmovns(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovns", 0x49, dst, src); }
1053 void cmovnz(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnz", 0x45, dst, src); }
1054 void cmovo(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovo", 0x40, dst, src); }
1055 void cmovp(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovp", 0x4A, dst, src); }
1056 void cmovpe(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovpe", 0x4A, dst, src); }
1057 void cmovpo(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovpo", 0x4B, dst, src); }
1058 void cmovs(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovs", 0x48, dst, src); }
1059 void cmovz(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovz", 0x44, dst, src); }
1060
1061 /// cmp - Compare and set flags
1062 void cmp(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1063 {
1064     cb_write_rm_multi(
1065         cb,
1066         "cmp",
1067         0x38, // opMemReg8
1068         0x39, // opMemRegPref
1069         0x3A, // opRegMem8
1070         0x3B, // opRegMemPref
1071         0x80, // opMemImm8
1072         0x83, // opMemImmSml
1073         0x81, // opMemImmLrg
1074         0x07, // opExtImm
1075         opnd0,
1076         opnd1
1077     );
1078 }
1079
1080 /// cdq - Convert doubleword to quadword
1081 void cdq(codeblock_t *cb)
1082 {
1083     //cb.writeASM("cdq");
1084     cb_write_byte(cb, 0x99);
1085 }
1086
1087 /// cqo - Convert quadword to octaword
1088 void cqo(codeblock_t *cb)
1089 {
1090     //cb.writeASM("cqo");
1091     cb_write_bytes(cb, 2, 0x48, 0x99);
1092 }
1093
1094 /// Interrupt 3 - trap to debugger
1095 void int3(codeblock_t *cb)
1096 {
1097     //cb.writeASM("INT 3");
1098     cb_write_byte(cb, 0xCC);
1099 }
1100
1101 /*
1102 // div - Unsigned integer division
1103 alias div = writeRMUnary!(
1104     "div",
1105     0xF6, // opMemReg8
1106     0xF7, // opMemRegPref
1107     0x06  // opExt
1108 );
1109 */
1110
1111 /*
1112 /// divsd - Divide scalar double
1113 alias divsd = writeXMM64!(
1114     "divsd",
1115     0xF2, // prefix
1116     0x0F, // opRegMem0
1117     0x5E  // opRegMem1
1118 );
1119 */
1120
1121 /*
1122 // idiv - Signed integer division
1123 alias idiv = writeRMUnary!(
1124     "idiv",
1125     0xF6, // opMemReg8
1126     0xF7, // opMemRegPref
1127     0x07  // opExt
1128 );
1129 */
1130
1131 /*
1132 /// imul - Signed integer multiplication with two operands
1133 void imul(CodeBlock cb, X86Opnd opnd0, X86Opnd opnd1)
1134 {
1135     cb.writeASM("imul", opnd0, opnd1);
1136
1137     assert (opnd0.isReg, "invalid first operand");
1138     auto opndSize = opnd0.reg.size;
1139
1140     // Check the size of opnd1
1141     if (opnd1.isReg)
1142         assert (opnd1.reg.size is opndSize, "operand size mismatch");
1143     else if (opnd1.isMem)
1144         assert (opnd1.mem.size is opndSize, "operand size mismatch");
1145
1146     assert (opndSize is 16 || opndSize is 32 || opndSize is 64);
1147     auto szPref = opndSize is 16;
1148     auto rexW = opndSize is 64;
1149
1150     cb.writeRMInstr!('r', 0xFF, 0x0F, 0xAF)(szPref, rexW, opnd0, opnd1);
1151 }
1152 */
1153
1154 /*
1155 /// imul - Signed integer multiplication with three operands (one immediate)
1156 void imul(CodeBlock cb, X86Opnd opnd0, X86Opnd opnd1, X86Opnd opnd2)
1157 {
1158     cb.writeASM("imul", opnd0, opnd1, opnd2);
1159
1160     assert (opnd0.isReg, "invalid first operand");
1161     auto opndSize = opnd0.reg.size;
1162
1163     // Check the size of opnd1
1164     if (opnd1.isReg)
1165         assert (opnd1.reg.size is opndSize, "operand size mismatch");
1166     else if (opnd1.isMem)
1167         assert (opnd1.mem.size is opndSize, "operand size mismatch");
1168
1169     assert (opndSize is 16 || opndSize is 32 || opndSize is 64);
1170     auto szPref = opndSize is 16;
1171     auto rexW = opndSize is 64;
1172
1173     assert (opnd2.isImm, "invalid third operand");
1174     auto imm = opnd2.imm;
1175
1176     // 8-bit immediate
1177     if (imm.immSize <= 8)
1178     {
1179         cb.writeRMInstr!('r', 0xFF, 0x6B)(szPref, rexW, opnd0, opnd1);
1180         cb.writeInt(imm.imm, 8);
1181     }
1182
1183     // 32-bit immediate
1184     else if (imm.immSize <= 32)
1185     {
1186         assert (imm.immSize <= opndSize, "immediate too large for dst");
1187         cb.writeRMInstr!('r', 0xFF, 0x69)(szPref, rexW, opnd0, opnd1);
1188         cb.writeInt(imm.imm, min(opndSize, 32));
1189     }
1190
1191     // Immediate too large
1192     else
1193     {
1194         assert (false, "immediate value too large");
1195     }
1196 }
1197 */
1198
1199 /// jcc - relative jumps to a label
1200 void ja_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "ja"  , 0x0F, 0x87, label_idx); }
1201 void jae_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jae" , 0x0F, 0x83, label_idx); }
1202 void jb_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jb"  , 0x0F, 0x82, label_idx); }
1203 void jbe_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jbe" , 0x0F, 0x86, label_idx); }
1204 void jc_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jc"  , 0x0F, 0x82, label_idx); }
1205 void je_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "je"  , 0x0F, 0x84, label_idx); }
1206 void jg_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jg"  , 0x0F, 0x8F, label_idx); }
1207 void jge_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jge" , 0x0F, 0x8D, label_idx); }
1208 void jl_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jl"  , 0x0F, 0x8C, label_idx); }
1209 void jle_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jle" , 0x0F, 0x8E, label_idx); }
1210 void jna_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jna" , 0x0F, 0x86, label_idx); }
1211 void jnae_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnae", 0x0F, 0x82, label_idx); }
1212 void jnb_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnb" , 0x0F, 0x83, label_idx); }
1213 void jnbe_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnbe", 0x0F, 0x87, label_idx); }
1214 void jnc_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnc" , 0x0F, 0x83, label_idx); }
1215 void jne_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jne" , 0x0F, 0x85, label_idx); }
1216 void jng_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jng" , 0x0F, 0x8E, label_idx); }
1217 void jnge_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnge", 0x0F, 0x8C, label_idx); }
1218 void jnl_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnl" , 0x0F, 0x8D, label_idx); }
1219 void jnle_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnle", 0x0F, 0x8F, label_idx); }
1220 void jno_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jno" , 0x0F, 0x81, label_idx); }
1221 void jnp_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnp" , 0x0F, 0x8b, label_idx); }
1222 void jns_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jns" , 0x0F, 0x89, label_idx); }
1223 void jnz_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnz" , 0x0F, 0x85, label_idx); }
1224 void jo_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jo"  , 0x0F, 0x80, label_idx); }
1225 void jp_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jp"  , 0x0F, 0x8A, label_idx); }
1226 void jpe_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jpe" , 0x0F, 0x8A, label_idx); }
1227 void jpo_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jpo" , 0x0F, 0x8B, label_idx); }
1228 void js_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "js"  , 0x0F, 0x88, label_idx); }
1229 void jz_label  (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jz"  , 0x0F, 0x84, label_idx); }
1230 void jmp_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jmp" , 0xFF, 0xE9, label_idx); }
1231
1232 /// jcc - relative jumps to a pointer (32-bit offset)
1233 void ja_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "ja"  , 0x0F, 0x87, ptr); }
1234 void jae_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jae" , 0x0F, 0x83, ptr); }
1235 void jb_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jb"  , 0x0F, 0x82, ptr); }
1236 void jbe_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jbe" , 0x0F, 0x86, ptr); }
1237 void jc_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jc"  , 0x0F, 0x82, ptr); }
1238 void je_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "je"  , 0x0F, 0x84, ptr); }
1239 void jg_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jg"  , 0x0F, 0x8F, ptr); }
1240 void jge_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jge" , 0x0F, 0x8D, ptr); }
1241 void jl_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jl"  , 0x0F, 0x8C, ptr); }
1242 void jle_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jle" , 0x0F, 0x8E, ptr); }
1243 void jna_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jna" , 0x0F, 0x86, ptr); }
1244 void jnae_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnae", 0x0F, 0x82, ptr); }
1245 void jnb_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnb" , 0x0F, 0x83, ptr); }
1246 void jnbe_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnbe", 0x0F, 0x87, ptr); }
1247 void jnc_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnc" , 0x0F, 0x83, ptr); }
1248 void jne_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jne" , 0x0F, 0x85, ptr); }
1249 void jng_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jng" , 0x0F, 0x8E, ptr); }
1250 void jnge_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnge", 0x0F, 0x8C, ptr); }
1251 void jnl_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnl" , 0x0F, 0x8D, ptr); }
1252 void jnle_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnle", 0x0F, 0x8F, ptr); }
1253 void jno_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jno" , 0x0F, 0x81, ptr); }
1254 void jnp_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnp" , 0x0F, 0x8b, ptr); }
1255 void jns_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jns" , 0x0F, 0x89, ptr); }
1256 void jnz_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnz" , 0x0F, 0x85, ptr); }
1257 void jo_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jo"  , 0x0F, 0x80, ptr); }
1258 void jp_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jp"  , 0x0F, 0x8A, ptr); }
1259 void jpe_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jpe" , 0x0F, 0x8A, ptr); }
1260 void jpo_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jpo" , 0x0F, 0x8B, ptr); }
1261 void js_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "js"  , 0x0F, 0x88, ptr); }
1262 void jz_ptr  (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jz"  , 0x0F, 0x84, ptr); }
1263 void jmp_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jmp" , 0xFF, 0xE9, ptr); }
1264
1265 /// jmp - Indirect jump near to an R/M operand
1266 void jmp_rm(codeblock_t *cb, x86opnd_t opnd)
1267 {
1268     //cb.writeASM("jmp", opnd);
1269     cb_write_rm(cb, false, false, NO_OPND, opnd, 4, 1, 0xFF);
1270 }
1271
1272 // jmp - Jump with relative 32-bit offset
1273 void jmp32(codeblock_t *cb, int32_t offset)
1274 {
1275     //cb.writeASM("jmp", ((offset > 0)? "+":"-") ~ to!string(offset));
1276     cb_write_byte(cb, 0xE9);
1277     cb_write_int(cb, offset, 32);
1278 }
1279
1280 /// lea - Load Effective Address
1281 void lea(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1282 {
1283     //cb.writeASM("lea", dst, src);
1284     assert (dst.num_bits == 64);
1285     cb_write_rm(cb, false, true, dst, src, 0xFF, 1, 0x8D);
1286 }
1287
1288 // Does this number fit in 32 bits and stays the same if you zero extend it to 64 bit?
1289 // If the sign bit is clear, sign extension and zero extension yield the same
1290 // result.
1291 static bool
1292 zero_extendable_32bit(uint64_t number)
1293 {
1294     return number <= UINT32_MAX && (number & (1ull << 31ull)) == 0;
1295 }
1296
1297 /// mov - Data move operation
1298 void mov(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1299 {
1300     // R/M + Imm
1301     if (src.type == OPND_IMM)
1302     {
1303         //cb.writeASM("mov", dst, src);
1304
1305         // R + Imm
1306         if (dst.type == OPND_REG)
1307         {
1308             assert (
1309                 src.num_bits <= dst.num_bits ||
1310                 unsig_imm_size(src.as.imm) <= dst.num_bits
1311             );
1312
1313             // In case the source immediate could be zero extended to be 64
1314             // bit, we can use the 32-bit operands version of the instruction.
1315             // For example, we can turn mov(rax, 0x34) into the equivalent
1316             // mov(eax, 0x34).
1317             if (dst.num_bits == 64 && zero_extendable_32bit(src.as.unsig_imm)) {
1318                 if (rex_needed(dst))
1319                     cb_write_rex(cb, false, 0, 0, dst.as.reg.reg_no);
1320                 cb_write_opcode(cb, 0xB8, dst);
1321                 cb_write_int(cb, src.as.imm, 32);
1322             }
1323             else {
1324                 if (dst.num_bits == 16)
1325                     cb_write_byte(cb, 0x66);
1326
1327                 if (rex_needed(dst) || dst.num_bits == 64)
1328                     cb_write_rex(cb, dst.num_bits == 64, 0, 0, dst.as.reg.reg_no);
1329
1330                 cb_write_opcode(cb, (dst.num_bits == 8)? 0xB0:0xB8, dst);
1331
1332                 cb_write_int(cb, src.as.imm, dst.num_bits);
1333             }
1334         }
1335
1336         // M + Imm
1337         else if (dst.type == OPND_MEM)
1338         {
1339             assert (src.num_bits <= dst.num_bits);
1340
1341             if (dst.num_bits == 8)
1342                 cb_write_rm(cb, false, false, NO_OPND, dst, 0xFF, 1, 0xC6);
1343             else
1344                 cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, NO_OPND, dst, 0, 1, 0xC7);
1345
1346             const uint32_t output_num_bits = (dst.num_bits > 32u) ? 32u : dst.num_bits;
1347             // assert that we can write whole immediate without loss of information
1348             assert (sig_imm_size(src.as.imm) <= output_num_bits);
1349             cb_write_int(cb, src.as.imm, output_num_bits);
1350         }
1351
1352         else
1353         {
1354             assert (false);
1355         }
1356     }
1357     else
1358     {
1359         cb_write_rm_multi(
1360             cb,
1361             "mov",
1362             0x88, // opMemReg8
1363             0x89, // opMemRegPref
1364             0x8A, // opRegMem8
1365             0x8B, // opRegMemPref
1366             0xC6, // opMemImm8
1367             0xFF, // opMemImmSml (not available)
1368             0xFF, // opMemImmLrg
1369             0xFF, // opExtImm
1370             dst,
1371             src
1372         );
1373     }
1374 }
1375
1376 /// movsx - Move with sign extension (signed integers)
1377 void movsx(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1378 {
1379     assert (dst.type == OPND_REG);
1380     assert (src.type == OPND_REG || src.type == OPND_MEM);
1381     assert (src.num_bits < dst.num_bits);
1382
1383     //cb.writeASM("movsx", dst, src);
1384
1385     if (src.num_bits == 8)
1386     {
1387         cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, dst, src, 0xFF, 2, 0x0F, 0xBE);
1388     }
1389     else if (src.num_bits == 16)
1390     {
1391         cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, dst, src, 0xFF, 2, 0x0F, 0xBF);
1392     }
1393     else if (src.num_bits == 32)
1394     {
1395         cb_write_rm(cb, false, true, dst, src, 0xFF, 1, 0x63);
1396     }
1397     else
1398     {
1399         assert (false);
1400     }
1401 }
1402
1403 /*
1404 /// movzx - Move with zero extension (unsigned values)
1405 void movzx(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1406 {
1407     cb.writeASM("movzx", dst, src);
1408
1409     uint32_t dstSize;
1410     if (dst.isReg)
1411         dstSize = dst.reg.size;
1412     else
1413         assert (false, "movzx dst must be a register");
1414
1415     uint32_t srcSize;
1416     if (src.isReg)
1417         srcSize = src.reg.size;
1418     else if (src.isMem)
1419         srcSize = src.mem.size;
1420     else
1421         assert (false);
1422
1423     assert (
1424         srcSize < dstSize,
1425         "movzx: srcSize >= dstSize"
1426     );
1427
1428     if (srcSize is 8)
1429     {
1430         cb.writeRMInstr!('r', 0xFF, 0x0F, 0xB6)(dstSize is 16, dstSize is 64, dst, src);
1431     }
1432     else if (srcSize is 16)
1433     {
1434         cb.writeRMInstr!('r', 0xFF, 0x0F, 0xB7)(dstSize is 16, dstSize is 64, dst, src);
1435     }
1436     else
1437     {
1438         assert (false, "invalid src operand size for movxz");
1439     }
1440 }
1441 */
1442
1443 // neg - Integer negation (multiplication by -1)
1444 void neg(codeblock_t *cb, x86opnd_t opnd)
1445 {
1446     write_rm_unary(
1447         cb,
1448         "neg",
1449         0xF6, // opMemReg8
1450         0xF7, // opMemRegPref
1451         0x03,  // opExt
1452         opnd
1453     );
1454 }
1455
1456 // nop - Noop, one or multiple bytes long
1457 void nop(codeblock_t *cb, uint32_t length)
1458 {
1459     switch (length) {
1460       case 0:
1461         break;
1462
1463       case 1:
1464         //cb.writeASM("nop1");
1465         cb_write_byte(cb, 0x90);
1466         break;
1467
1468       case 2:
1469         //cb.writeASM("nop2");
1470         cb_write_bytes(cb, 2, 0x66,0x90);
1471         break;
1472
1473       case 3:
1474         //cb.writeASM("nop3");
1475         cb_write_bytes(cb, 3, 0x0F,0x1F,0x00);
1476         break;
1477
1478       case 4:
1479         //cb.writeASM("nop4");
1480         cb_write_bytes(cb, 4, 0x0F,0x1F,0x40,0x00);
1481         break;
1482
1483       case 5:
1484         //cb.writeASM("nop5");
1485         cb_write_bytes(cb, 5, 0x0F,0x1F,0x44,0x00,0x00);
1486         break;
1487
1488       case 6:
1489         //cb.writeASM("nop6");
1490         cb_write_bytes(cb, 6, 0x66,0x0F,0x1F,0x44,0x00,0x00);
1491         break;
1492
1493       case 7:
1494         //cb.writeASM("nop7");
1495         cb_write_bytes(cb, 7, 0x0F,0x1F,0x80,0x00,0x00,0x00,0x00);
1496         break;
1497
1498       case 8:
1499         //cb.writeASM("nop8");
1500         cb_write_bytes(cb, 8, 0x0F,0x1F,0x84,0x00,0x00,0x00,0x00,0x00);
1501         break;
1502
1503       case 9:
1504         //cb.writeASM("nop9");
1505         cb_write_bytes(cb, 9, 0x66,0x0F,0x1F,0x84,0x00,0x00,0x00,0x00,0x00);
1506         break;
1507
1508       default:
1509         {
1510             uint32_t written = 0;
1511             while (written + 9 <= length)
1512             {
1513                 nop(cb, 9);
1514                 written += 9;
1515             }
1516             nop(cb, length - written);
1517         }
1518         break;
1519     }
1520 }
1521
1522 // not - Bitwise NOT
1523 void not(codeblock_t *cb, x86opnd_t opnd)
1524 {
1525     write_rm_unary(
1526         cb,
1527         "not",
1528         0xF6, // opMemReg8
1529         0xF7, // opMemRegPref
1530         0x02, // opExt
1531         opnd
1532     );
1533 }
1534
1535 /// or - Bitwise OR
1536 void or(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1537 {
1538     cb_write_rm_multi(
1539         cb,
1540         "or",
1541         0x08, // opMemReg8
1542         0x09, // opMemRegPref
1543         0x0A, // opRegMem8
1544         0x0B, // opRegMemPref
1545         0x80, // opMemImm8
1546         0x83, // opMemImmSml
1547         0x81, // opMemImmLrg
1548         0x01, // opExtImm
1549         opnd0,
1550         opnd1
1551     );
1552 }
1553
1554 /// pop - Pop a register off the stack
1555 void pop(codeblock_t *cb, x86opnd_t opnd)
1556 {
1557     assert (opnd.num_bits == 64);
1558
1559     //cb.writeASM("pop", opnd);
1560
1561     if (opnd.type == OPND_REG) {
1562         if (rex_needed(opnd))
1563             cb_write_rex(cb, false, 0, 0, opnd.as.reg.reg_no);
1564         cb_write_opcode(cb, 0x58, opnd);
1565     }
1566     else if (opnd.type == OPND_MEM) {
1567         cb_write_rm(cb, false, false, NO_OPND, opnd, 0, 1, 0x8F);
1568     }
1569     else {
1570         assert(false && "unexpected operand type");
1571     }
1572 }
1573
1574 /// popfq - Pop the flags register (64-bit)
1575 void popfq(codeblock_t *cb)
1576 {
1577     //cb.writeASM("popfq");
1578
1579     // REX.W + 0x9D
1580     cb_write_bytes(cb, 2, 0x48, 0x9D);
1581 }
1582
1583 /// push - Push an operand on the stack
1584 void push(codeblock_t *cb, x86opnd_t opnd)
1585 {
1586     assert (opnd.num_bits == 64);
1587
1588     //cb.writeASM("push", opnd);
1589
1590     if (opnd.type == OPND_REG) {
1591       if (rex_needed(opnd))
1592           cb_write_rex(cb, false, 0, 0, opnd.as.reg.reg_no);
1593       cb_write_opcode(cb, 0x50, opnd);
1594     }
1595     else if (opnd.type == OPND_MEM) {
1596       cb_write_rm(cb, false, false, NO_OPND, opnd, 6, 1, 0xFF);
1597     }
1598     else {
1599       assert(false && "unexpected operand type");
1600     }
1601 }
1602
1603 /// pushfq - Push the flags register (64-bit)
1604 void pushfq(codeblock_t *cb)
1605 {
1606     //cb.writeASM("pushfq");
1607     cb_write_byte(cb, 0x9C);
1608 }
1609
1610 /// ret - Return from call, popping only the return address
1611 void ret(codeblock_t *cb)
1612 {
1613     //cb.writeASM("ret");
1614     cb_write_byte(cb, 0xC3);
1615 }
1616
1617 // sal - Shift arithmetic left
1618 void sal(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1619 {
1620     cb_write_shift(
1621         cb,
1622         "sal",
1623         0xD1, // opMemOnePref,
1624         0xD3, // opMemClPref,
1625         0xC1, // opMemImmPref,
1626         0x04,
1627         opnd0,
1628         opnd1
1629     );
1630 }
1631
1632 /// sar - Shift arithmetic right (signed)
1633 void sar(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1634 {
1635     cb_write_shift(
1636         cb,
1637         "sar",
1638         0xD1, // opMemOnePref,
1639         0xD3, // opMemClPref,
1640         0xC1, // opMemImmPref,
1641         0x07,
1642         opnd0,
1643         opnd1
1644     );
1645 }
1646 // shl - Shift logical left
1647 void shl(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1648 {
1649     cb_write_shift(
1650         cb,
1651         "shl",
1652         0xD1, // opMemOnePref,
1653         0xD3, // opMemClPref,
1654         0xC1, // opMemImmPref,
1655         0x04,
1656         opnd0,
1657         opnd1
1658     );
1659 }
1660
1661 /// shr - Shift logical right (unsigned)
1662 void shr(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1663 {
1664     cb_write_shift(
1665         cb,
1666         "shr",
1667         0xD1, // opMemOnePref,
1668         0xD3, // opMemClPref,
1669         0xC1, // opMemImmPref,
1670         0x05,
1671         opnd0,
1672         opnd1
1673     );
1674 }
1675
1676 /// sub - Integer subtraction
1677 void sub(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1678 {
1679     cb_write_rm_multi(
1680         cb,
1681         "sub",
1682         0x28, // opMemReg8
1683         0x29, // opMemRegPref
1684         0x2A, // opRegMem8
1685         0x2B, // opRegMemPref
1686         0x80, // opMemImm8
1687         0x83, // opMemImmSml
1688         0x81, // opMemImmLrg
1689         0x05, // opExtImm
1690         opnd0,
1691         opnd1
1692     );
1693 }
1694
1695 /// test - Logical Compare
1696 void test(codeblock_t *cb, x86opnd_t rm_opnd, x86opnd_t test_opnd)
1697 {
1698     assert (rm_opnd.type == OPND_REG || rm_opnd.type == OPND_MEM);
1699     assert (test_opnd.type == OPND_REG || test_opnd.type == OPND_IMM);
1700
1701     // If the second operand is an immediate
1702     if (test_opnd.type == OPND_IMM)
1703     {
1704         x86opnd_t imm_opnd = test_opnd;
1705
1706         if (imm_opnd.as.imm >= 0)
1707         {
1708             assert (unsig_imm_size(imm_opnd.as.unsig_imm) <= 32);
1709             assert (unsig_imm_size(imm_opnd.as.unsig_imm) <= rm_opnd.num_bits);
1710
1711             // Use the smallest operand size possible
1712             rm_opnd = resize_opnd(rm_opnd, unsig_imm_size(imm_opnd.as.unsig_imm));
1713
1714             if (rm_opnd.num_bits == 8)
1715             {
1716                 cb_write_rm(cb, false, false, NO_OPND, rm_opnd, 0x00, 1, 0xF6);
1717                 cb_write_int(cb, imm_opnd.as.imm, rm_opnd.num_bits);
1718             }
1719             else
1720             {
1721                 cb_write_rm(cb, rm_opnd.num_bits == 16, false, NO_OPND, rm_opnd, 0x00, 1, 0xF7);
1722                 cb_write_int(cb, imm_opnd.as.imm, rm_opnd.num_bits);
1723             }
1724         }
1725         else
1726         {
1727             // This mode only applies to 64-bit R/M operands with 32-bit signed immediates
1728             assert (imm_opnd.as.imm < 0);
1729             assert (sig_imm_size(imm_opnd.as.imm) <= 32);
1730             assert (rm_opnd.num_bits == 64);
1731             cb_write_rm(cb, false, true, NO_OPND, rm_opnd, 0x00, 1, 0xF7);
1732             cb_write_int(cb, imm_opnd.as.imm, 32);
1733         }
1734     }
1735     else
1736     {
1737         assert (test_opnd.num_bits == rm_opnd.num_bits);
1738
1739         if (rm_opnd.num_bits == 8)
1740         {
1741             cb_write_rm(cb, false, false, test_opnd, rm_opnd, 0xFF, 1, 0x84);
1742         }
1743         else
1744         {
1745             cb_write_rm(cb, rm_opnd.num_bits == 16, rm_opnd.num_bits == 64, test_opnd, rm_opnd, 0xFF, 1, 0x85);
1746         }
1747     }
1748 }
1749
1750 /// Undefined opcode
1751 void ud2(codeblock_t *cb)
1752 {
1753     cb_write_bytes(cb, 2, 0x0F, 0x0B);
1754 }
1755
1756 /// xchg - Exchange Register/Memory with Register
1757 void xchg(codeblock_t *cb, x86opnd_t rm_opnd, x86opnd_t r_opnd)
1758 {
1759     assert (rm_opnd.num_bits == 64);
1760     assert (r_opnd.num_bits == 64);
1761     assert (rm_opnd.type == OPND_REG);
1762     assert (r_opnd.type == OPND_REG);
1763
1764     // If we're exchanging with RAX
1765     if (rm_opnd.type == OPND_REG && rm_opnd.as.reg.reg_no == RAX.as.reg.reg_no)
1766     {
1767         // Write the REX byte
1768         cb_write_rex(cb, rm_opnd.num_bits == 64, 0, 0, r_opnd.as.reg.reg_no);
1769
1770         // Write the opcode and register number
1771         cb_write_byte(cb, 0x90 + (r_opnd.as.reg.reg_no & 7));
1772     }
1773     else
1774     {
1775         cb_write_rm(cb, rm_opnd.num_bits == 16, rm_opnd.num_bits == 64, r_opnd, rm_opnd, 0xFF, 1, 0x87);
1776     }
1777 }
1778
1779 /// xor - Exclusive bitwise OR
1780 void xor(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1781 {
1782     cb_write_rm_multi(
1783         cb,
1784         "xor",
1785         0x30, // opMemReg8
1786         0x31, // opMemRegPref
1787         0x32, // opRegMem8
1788         0x33, // opRegMemPref
1789         0x80, // opMemImm8
1790         0x83, // opMemImmSml
1791         0x81, // opMemImmLrg
1792         0x06, // opExtImm
1793         opnd0,
1794         opnd1
1795     );
1796 }
1797
1798 // LOCK - lock prefix for atomic shared memory operations
1799 void cb_write_lock_prefix(codeblock_t *cb)
1800 {
1801     cb_write_byte(cb, 0xF0);
1802 }
1803
1804 void cb_mark_all_writeable(codeblock_t * cb)
1805 {
1806     if (mprotect(cb->mem_block_, cb->mem_size, PROT_READ | PROT_WRITE)) {
1807         fprintf(stderr, "Couldn't make JIT page (%p) writeable, errno: %s", (void *)cb->mem_block_, strerror(errno));
1808         abort();
1809     }
1810 }
1811
1812 void cb_mark_position_writeable(codeblock_t * cb, uint32_t write_pos)
1813 {
1814     uint32_t pagesize = (uint32_t)sysconf(_SC_PAGESIZE);
1815     uint32_t aligned_position = (write_pos / pagesize) * pagesize;
1816
1817     if (cb->current_aligned_write_pos != aligned_position) {
1818         cb->current_aligned_write_pos = aligned_position;
1819         void *const page_addr = cb_get_ptr(cb, aligned_position);
1820         if (mprotect(page_addr, pagesize, PROT_READ | PROT_WRITE)) {
1821             fprintf(stderr, "Couldn't make JIT page (%p) writeable, errno: %s", page_addr, strerror(errno));
1822             abort();
1823         }
1824     }
1825 }
1826
1827 void cb_mark_all_executable(codeblock_t * cb)
1828 {
1829     cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
1830     if (mprotect(cb->mem_block_, cb->mem_size, PROT_READ | PROT_EXEC)) {
1831         fprintf(stderr, "Couldn't make JIT page (%p) executable, errno: %s", (void *)cb->mem_block_, strerror(errno));
1832         abort();
1833     }
1834 }
1835