x64.c

   1 #include "tok.h"
   2 #include "gen.h"
   3 #include "out.h"
   4
   5 /* registers */
   6 #define R_RAX           0x00
   7 #define R_RCX           0x01
   8 #define R_RDX           0x02
   9 #define R_RBX           0x03
  10 #define R_RSP           0x04
  11 #define R_RBP           0x05
  12 #define R_RSI           0x06
  13 #define R_RDI           0x07
  14 /* x86_64 registers */
  15 #define R_R8            0x08
  16 #define R_R9            0x09
  17 #define R_R10           0x0a
  18 #define R_R11           0x0b
  19 #define R_R12           0x0c
  20 #define R_R13           0x0d
  21 #define R_R14           0x0e
  22 #define R_R15           0x0f
  23
  24 /* x86 opcodes */
  25 #define I_MOV           0x89
  26 #define I_MOVI          0xc7
  27 #define I_MOVIR         0xb8
  28 #define I_MOVR          0x8b
  29 #define I_MOVSXD        0x63
  30 #define I_SHX           0xd3
  31 #define I_CMP           0x3b
  32 #define I_TST           0x85
  33 #define I_LEA           0x8d
  34 #define I_NOT           0xf7
  35 #define I_CALL          0xff
  36 #define I_MUL           0xf7
  37 #define I_XOR           0x33
  38 #define I_TEST          0x85
  39 #define I_CQO           0x99
  40 #define I_PUSH          0x50
  41 #define I_POP           0x58
  42
  43 #define MIN(a, b)               ((a) < (b) ? (a) : (b))
  44 #define ALIGN(x, a)             (((x) + (a) - 1) & ~((a) - 1))
  45
  46 int tmpregs[] = {0, 7, 6, 2, 1, 8, 9, 10, 11, 3, 12, 13, 14, 15};
  47 int argregs[] = {7, 6, 2, 1, 8, 9};
  48
  49 #define OP2(o2, o1)             (0x010000 | ((o2) << 8) | (o1))
  50 #define O2(op)                  (((op) >> 8) & 0xff)
  51 #define O1(op)                  ((op) & 0xff)
  52 #define MODRM(m, r1, r2)        ((m) << 6 | (r1) << 3 | (r2))
  53 #define REX(r1, r2)             (0x48 | (((r1) & 8) >> 1) | (((r2) & 8) >> 3))
  54
  55 static void putint(char *s, long n, int l)
  56 {
  57         while (l--) {
  58                 *s++ = n;
  59                 n >>= 8;
  60         }
  61 }
  62
  63 static void op_x(int op, int r1, int r2, int bt)
  64 {
  65         int sz = BT_SZ(bt);
  66         int rex = 0;
  67         if (sz == 8)
  68                 rex |= 8;
  69         if (sz == 1)
  70                 rex |= 0x40;
  71         if (r1 & 0x8)
  72                 rex |= 4;
  73         if (r2 & 0x8)
  74                 rex |= 1;
  75         if (sz == 2)
  76                 oi(0x66, 1);
  77         if (rex)
  78                 oi(rex | 0x40, 1);
  79         if (op & 0x10000)
  80                 oi(O2(op), 1);
  81         oi(sz == 1 ? O1(op) & ~0x1 : O1(op), 1);
  82 }
  83
  84 #define op_mr           op_rm
  85
  86 /* op_*(): r=reg, m=mem, i=imm, s=sym */
  87 static void op_rm(int op, int src, int base, int off, int bt)
  88 {
  89         int dis = off == (char) off ? 1 : 4;
  90         int mod = dis == 4 ? 2 : 1;
  91         if (!off && (base & 7) != R_RBP)
  92                 mod = 0;
  93         op_x(op, src, base, bt);
  94         oi(MODRM(mod, src & 0x07, base & 0x07), 1);
  95         if ((base & 7) == R_RSP)
  96                 oi(0x24, 1);
  97         if (mod)
  98                 oi(off, dis);
  99 }
 100
 101 static void op_rr(int op, int src, int dst, int bt)
 102 {
 103         op_x(op, src, dst, bt);
 104         oi(MODRM(3, src & 0x07, dst & 0x07), 1);
 105 }
 106
 107 #define movrx_bt(bt)            (((bt) == 4) ? 4 : LONGSZ)
 108
 109 static int movrx_op(int bt, int mov)
 110 {
 111         int sz = BT_SZ(bt);
 112         if (sz == 4)
 113                 return bt & BT_SIGNED ? I_MOVSXD : mov;
 114         if (sz == 2)
 115                 return OP2(0x0f, bt & BT_SIGNED ? 0xbf : 0xb7);
 116         if (sz == 1)
 117                 return OP2(0x0f, bt & BT_SIGNED ? 0xbe : 0xb6);
 118         return mov;
 119 }
 120
 121 static void mov_r2r(int rd, int r1, unsigned bt)
 122 {
 123         if (rd != r1 || BT_SZ(bt) != LONGSZ)
 124                 op_rr(movrx_op(bt, I_MOVR), rd, r1, movrx_bt(bt));
 125 }
 126
 127 static void mov_m2r(int dst, int base, int off, int bt)
 128 {
 129         op_rm(movrx_op(bt, I_MOVR), dst, base, off, movrx_bt(bt));
 130 }
 131
 132 int i_imm(int op, long imm)
 133 {
 134         if ((op & 0xf0) == 0x20)
 135                 return 0;
 136         return imm <= 127 && imm >= -128;
 137 }
 138
 139 static void i_push(int reg)
 140 {
 141         op_x(I_PUSH | (reg & 0x7), 0, reg, 4);
 142 }
 143
 144 static void i_pop(int reg)
 145 {
 146         op_x(I_POP | (reg & 0x7), 0, reg, 4);
 147 }
 148
 149 void i_mov(int rd, int rn)
 150 {
 151         op_rr(movrx_op(LONGSZ, I_MOVR), rd, rn, movrx_bt(LONGSZ));
 152 }
 153
 154 void i_load(int rd, int rn, int off, int bt)
 155 {
 156         mov_m2r(rd, rn, off, bt);
 157 }
 158
 159 void i_save(int rd, int rn, int off, int bt)
 160 {
 161         op_rm(I_MOV, rd, rn, off, bt);
 162 }
 163
 164 void i_reg(int op, int *rd, int *r1, int *r2, int *tmp)
 165 {
 166         *rd = 0;
 167         *r1 = R_TMPS;
 168         *r2 = op & O_IMM ? 0 : R_TMPS;
 169         *tmp = 0;
 170         if ((op & 0xf0) == 0x00)        /* add */
 171                 return;
 172         if ((op & 0xf0) == 0x10) {      /* shl */
 173                 if (~op & O_IMM) {
 174                         *r2 = 1 << R_RCX;
 175                         *r1 = R_TMPS & ~*r2;
 176                 }
 177                 return;
 178         }
 179         if ((op & 0xf0) == 0x20) {      /* mul */
 180                 *rd = (op & 0xff) == O_MOD ? (1 << R_RDX) : (1 << R_RAX);
 181                 *r1 = (1 << R_RAX);
 182                 *r2 = R_TMPS & ~*rd & ~*r1;
 183                 if ((op & 0xff) == O_DIV)
 184                         *r2 &= ~(1 << R_RDX);
 185                 *tmp = (1 << R_RDX) | (1 << R_RAX);
 186                 return;
 187         }
 188         if ((op & 0xf0) == 0x30) {      /* cmp */
 189                 *rd = 1 << R_RAX;
 190                 return;
 191         }
 192         if ((op & 0xf0) == 0x40) {      /* uop */
 193                 *r2 = 0;
 194                 if ((op & 0xff) == O_LNOT)
 195                         *r1 = 1 << R_RAX;
 196                 return;
 197         }
 198         if ((op & 0xf0) == 0x50) {      /* etc */
 199                 if (op == O_MSET) {
 200                         *rd = 1 << R_RDI;
 201                         *r1 = 1 << R_RAX;
 202                         *r2 = 1 << R_RCX;
 203                 }
 204                 if (op == O_MCPY) {
 205                         *rd = 1 << R_RDI;
 206                         *r1 = 1 << R_RSI;
 207                         *r2 = 1 << R_RCX;
 208                 }
 209                 if (op == O_SX || op == O_ZX || op == O_MOV) {
 210                         *rd = R_TMPS;
 211                         *r2 = 0;
 212                 }
 213                 return;
 214         }
 215 }
 216
 217 static void i_add(int op, int rd, int r1, int r2)
 218 {
 219         /* opcode for O_ADD, O_SUB, O_AND, O_OR, O_XOR */
 220         static int rx[] = {0003, 0053, 0043, 0013, 0063};
 221         op_rr(rx[op & 0x0f], rd, r2, LONGSZ);
 222 }
 223
 224 static void i_add_imm(int op, int rd, int rn, long n)
 225 {
 226         /* opcode for O_ADD, O_SUB, O_AND, O_OR, O_XOR */
 227         static int rx[] = {0xc0, 0xe8, 0xe0, 0xc8, 0xf0};
 228         unsigned char s[4] = {REX(0, rd), 0x83, rx[op & 0x0f] | (rd & 7), n & 0xff};
 229         os((void *) s, 4);
 230 }
 231
 232 void i_num(int rd, long n)
 233 {
 234         if (!n) {
 235                 op_rr(I_XOR, rd, rd, 4);
 236                 return;
 237         }
 238         if (n < 0 && -n <= 0xffffffff) {
 239                 op_rr(I_MOVI, 0, rd, LONGSZ);
 240                 oi(n, 4);
 241         } else {
 242                 int len = 8;
 243                 if (n > 0 && n <= 0xffffffff)
 244                         len = 4;
 245                 op_x(I_MOVIR + (rd & 7), 0, rd, len);
 246                 oi(n, len);
 247         }
 248 }
 249
 250 static void i_mul(int rd, int r1, int r2)
 251 {
 252         if (r2 != R_RDX)
 253                 i_num(R_RDX, 0);
 254         op_rr(I_MUL, 4, r2, LONGSZ);
 255 }
 256
 257 static void i_div(int op, int rd, int r1, int r2)
 258 {
 259         if (r2 != R_RDX) {
 260                 if (op & O_SIGNED)
 261                         op_x(I_CQO, R_RAX, R_RDX, LONGSZ);
 262                 else
 263                         i_num(R_RDX, 0);
 264         }
 265         op_rr(I_MUL, op & O_SIGNED ? 7 : 6, r2, LONGSZ);
 266 }
 267
 268 static void i_tst(int rn, int rm)
 269 {
 270         op_rr(I_TST, rn, rm, LONGSZ);
 271 }
 272
 273 static void i_cmp(int rn, int rm)
 274 {
 275         op_rr(I_CMP, rn, rm, LONGSZ);
 276 }
 277
 278 static void i_cmp_imm(int rn, long n)
 279 {
 280         unsigned char s[4] = {REX(0, rn), 0x83, 0xf8 | rn, n & 0xff};
 281         os(s, 4);
 282 }
 283
 284 static void i_shl(int op, int rd, int r1, int rs)
 285 {
 286         int sm = 4;
 287         if ((op & 0x0f) == 1)
 288                 sm = op & O_SIGNED ? 7 : 5;
 289         op_rr(I_SHX, sm, rd, LONGSZ);
 290 }
 291
 292 static void i_shl_imm(int op, int rd, int rn, long n)
 293 {
 294         int sm = (op & 0x1) ? (op & O_SIGNED ? 0xf8 : 0xe8) : 0xe0 ;
 295         char s[4] = {REX(0, rn), 0xc1, sm | (rn & 7), n & 0xff};
 296         os(s, 4);
 297 }
 298
 299 void i_sym(int rd, char *sym, int off)
 300 {
 301         int sz = X64_ABS_RL & OUT_RL32 ? 4 : LONGSZ;
 302         if (X64_ABS_RL & OUT_RLSX)
 303                 op_rr(I_MOVI, 0, rd, sz);
 304         else
 305                 op_x(I_MOVIR + (rd & 7), 0, rd, sz);
 306         if (!pass1)
 307                 out_rel(sym, OUT_CS | X64_ABS_RL, cslen);
 308         oi(off, sz);
 309 }
 310
 311 static void i_neg(int rd)
 312 {
 313         op_rr(I_NOT, 3, rd, LONGSZ);
 314 }
 315
 316 static void i_not(int rd)
 317 {
 318         op_rr(I_NOT, 2, rd, LONGSZ);
 319 }
 320
 321 /* for optimizing cmp + tst + jmp to cmp + jmp */
 322 #define OPT_ISCMP()             (last_set >= 0 && last_set + 7 == cslen)
 323 #define OPT_CCOND()             (cs[last_set + 1])
 324
 325 static long last_set = -1;
 326
 327 static void i_set(int op, int rd)
 328 {
 329         /* lt, gt, le, ge, eq, neq */
 330         static int ucond[] = {0x92, 0x97, 0x96, 0x93, 0x94, 0x95};
 331         static int scond[] = {0x9c, 0x9f, 0x9e, 0x9d, 0x94, 0x95};
 332         int cond = op & O_SIGNED ? scond[op & 0x0f] : ucond[op & 0x0f];
 333         char set[] = "\x0f\x00\xc0";
 334         set[1] = cond;
 335         last_set = cslen;
 336         os(set, 3);                     /* setl al */
 337         os("\x48\x0f\xb6\xc0", 4);      /* movzx rax, al */
 338 }
 339
 340 static void i_lnot(int rd)
 341 {
 342         if (OPT_ISCMP()) {
 343                 cs[last_set + 1] ^= 0x01;
 344         } else {
 345                 char cmp[] = "\x00\x83\xf8\x00";
 346                 cmp[0] = REX(0, rd);
 347                 cmp[2] |= rd & 7;
 348                 os(cmp, 4);             /* cmp rax, 0 */
 349                 i_set(O_EQ, rd);
 350         }
 351 }
 352
 353 static void jx(int x, int nbytes)
 354 {
 355         char op[2] = {0x0f};
 356         if (nbytes == 1) {
 357                 op[0] = 0x70 | (x & 0x0f);
 358                 os(op, 1);              /* jx $addr */
 359         } else {
 360                 op[1] = x;
 361                 os(op, 2);              /* jx $addr */
 362         }
 363         oi(0, nbytes);
 364 }
 365
 366 void i_jmp(int rn, int z, int nbytes)
 367 {
 368         if (!nbytes)
 369                 return;
 370         if (nbytes > 1)
 371                 nbytes = 4;
 372         if (rn >= 0) {
 373                 if (OPT_ISCMP()) {
 374                         int cond = OPT_CCOND();
 375                         cslen = last_set;
 376                         jx((!z ? cond : cond ^ 0x01) & ~0x10, nbytes);
 377                         last_set = -1;
 378                 } else {
 379                         i_tst(rn, rn);
 380                         jx(z ? 0x84 : 0x85, nbytes);
 381                 }
 382         } else {
 383                 os(nbytes == 1 ? "\xeb" : "\xe9", 1);           /* jmp $addr */
 384                 oi(0, nbytes);
 385         }
 386 }
 387
 388 long i_fill(long src, long dst, int nbytes)
 389 {
 390         if (!nbytes)
 391                 return 0;
 392         if (nbytes > 1)
 393                 nbytes = 4;
 394         putint((void *) (cs + src - nbytes), dst - src, nbytes);
 395         return dst - src;
 396 }
 397
 398 static void i_zx(int rd, int r1, int bits)
 399 {
 400         if (bits & 0x07) {
 401                 i_shl_imm(O_SHL, rd, rd, LONGSZ * 8 - bits);
 402                 i_shl_imm(O_SHR, rd, rd, LONGSZ * 8 - bits);
 403         } else {
 404                 mov_r2r(rd, r1, bits >> 3);
 405         }
 406 }
 407
 408 static void i_sx(int rd, int r1, int bits)
 409 {
 410         mov_r2r(rd, r1, BT_SIGNED | (bits >> 3));
 411 }
 412
 413 void i_op(int op, int rd, int r1, int r2)
 414 {
 415         if ((op & 0xf0) == 0x00)
 416                 i_add(op, r1, r1, r2);
 417         if ((op & 0xf0) == 0x10)
 418                 i_shl(op, r1, r1, r2);
 419         if ((op & 0xf0) == 0x20) {
 420                 if ((op & 0xff) == O_MUL)
 421                         i_mul(R_RAX, r1, r2);
 422                 if ((op & 0xff) == O_DIV)
 423                         i_div(op, R_RAX, r1, r2);
 424                 if ((op & 0xff) == O_MOD)
 425                         i_div(op, R_RDX, r1, r2);
 426                 return;
 427         }
 428         if ((op & 0xf0) == 0x30) {
 429                 i_cmp(r1, r2);
 430                 i_set(op, rd);
 431                 return;
 432         }
 433         if ((op & 0xf0) == 0x40) {      /* uop */
 434                 if ((op & 0xff) == O_NEG)
 435                         i_neg(r1);
 436                 if ((op & 0xff) == O_NOT)
 437                         i_not(r1);
 438                 if ((op & 0xff) == O_LNOT)
 439                         i_lnot(r1);
 440                 return;
 441         }
 442 }
 443
 444 static void i_add_anyimm(int rd, int rn, long n)
 445 {
 446         op_rm(I_LEA, rd, rn, n, LONGSZ);
 447 }
 448
 449 void i_op_imm(int op, int rd, int r1, long n)
 450 {
 451         if ((op & 0xf0) == 0x00) {      /* add */
 452                 if (rd == r1 && i_imm(O_ADD, n))
 453                         i_add_imm(op, rd, r1, n);
 454                 else
 455                         i_add_anyimm(rd, r1, n);
 456         }
 457         if ((op & 0xf0) == 0x10)        /* shl */
 458                 i_shl_imm(op, rd, r1, n);
 459         if ((op & 0xf0) == 0x20)        /* mul */
 460                 die("mul/imm not implemented");
 461         if ((op & 0xf0) == 0x30) {      /* imm */
 462                 i_cmp_imm(r1, n);
 463                 i_set(op, rd);
 464         }
 465         if ((op & 0xf0) == 0x50) {      /* etc */
 466                 if ((op & 0xff) == O_ZX)
 467                         i_zx(rd, r1, n);
 468                 if ((op & 0xff) == O_SX)
 469                         i_sx(rd, r1, n);
 470                 if ((op & 0xff) == O_MOV)
 471                         i_mov(rd, r1);
 472         }
 473 }
 474
 475 void i_memcpy(int r0, int r1, int r2)
 476 {
 477         os("\xfc\xf3\xa4", 3);          /* cld; rep movs */
 478 }
 479
 480 void i_memset(int r0, int r1, int r2)
 481 {
 482         os("\xfc\xf3\xaa", 3);          /* cld; rep stosb */
 483 }
 484
 485 void i_call_reg(int rd)
 486 {
 487         op_rr(I_CALL, 2, rd, LONGSZ);
 488 }
 489
 490 void i_call(char *sym, int off)
 491 {
 492         os("\xe8", 1);          /* call $x */
 493         if (!pass1)
 494                 out_rel(sym, OUT_CS | OUT_RLREL, cslen);
 495         oi(-4 + off, 4);
 496 }
 497
 498 static int func_argc;
 499 static int func_varg;
 500 static int func_spsub;
 501 static int func_sargs;
 502 static int func_sregs;
 503 static int func_initfp;
 504 static int spsub_addr;
 505
 506 int i_args(void)
 507 {
 508         return 16;
 509 }
 510
 511 int i_sp(void)
 512 {
 513         int i;
 514         int n = 0;
 515         for (i = 0; i < N_TMPS; i++)
 516                 if ((1 << tmpregs[i]) & func_sregs)
 517                         n += 8;
 518         return -n;
 519 }
 520
 521 static void i_saveargs(void)
 522 {
 523         int i;
 524         os("\x58", 1);          /* pop rax */
 525         for (i = N_ARGS - 1; i >= 0; i--)
 526                 if ((1 << argregs[i]) & func_sargs)
 527                         i_push(argregs[i]);
 528         os("\x50", 1);          /* push rax */
 529 }
 530
 531 void i_prolog(int argc, int varg, int sargs, int sregs, int initfp, int subsp)
 532 {
 533         int i;
 534         last_set = -1;
 535         func_argc = argc;
 536         func_varg = varg;
 537         func_sargs = sargs;
 538         func_sregs = sregs;
 539         func_initfp = initfp;
 540         func_spsub = subsp;
 541         if (func_sargs)
 542                 i_saveargs();
 543         if (initfp) {
 544                 os("\x55", 1);                  /* push rbp */
 545                 os("\x48\x89\xe5", 3);          /* mov rbp, rsp */
 546         }
 547         if (func_sregs) {
 548                 for (i = N_TMPS - 1; i >= 0; i--)
 549                         if ((1 << tmpregs[i]) & func_sregs)
 550                                 i_push(tmpregs[i]);
 551         }
 552         if (func_spsub) {
 553                 os("\x48\x81\xec", 3);          /* sub rsp, $xxx */
 554                 spsub_addr = cslen;
 555                 oi(0, 4);
 556         }
 557 }
 558
 559 void i_epilog(int sp_max)
 560 {
 561         int diff;
 562         int nsregs = 0;
 563         int nsargs = 0;
 564         int i;
 565         for (i = 0; i < N_TMPS; i++)
 566                 if ((1 << tmpregs[i]) & func_sregs)
 567                         nsregs++;
 568         for (i = 0; i < N_ARGS; i++)
 569                 if ((1 << argregs[i]) & func_sargs)
 570                         nsargs++;
 571         diff = ALIGN(-sp_max - nsregs * LONGSZ, 16);
 572         /* forcing 16-byte alignment */
 573         diff = (nsregs + nsargs) & 1 ? diff + LONGSZ : diff;
 574         if (func_spsub && diff) {
 575                 i_add_anyimm(R_RSP, R_RBP, -nsregs * LONGSZ);
 576                 putint(cs + spsub_addr, diff, 4);
 577         }
 578         if (func_sregs) {
 579                 for (i = 0; i < N_TMPS; i++)
 580                         if ((1 << tmpregs[i]) & func_sregs)
 581                                 i_pop(tmpregs[i]);
 582         }
 583         if (func_initfp)
 584                 os("\xc9", 1);          /* leave */
 585         if (func_sargs) {
 586                 os("\xc2", 1);          /* ret n */
 587                 oi(nsargs * LONGSZ, 2);
 588         } else {
 589                 os("\xc3", 1);          /* ret */
 590         }
 591 }
 592
 593 void i_done(void)
 594 {
 595 }