gen.c

   1 #include <stdlib.h>
   2 #include <string.h>
   3 #include "gen.h"
   4 #include "tok.h"
   5
   6 #define TMP_ADDR        0x0001
   7 #define LOC_REG         0x0100
   8 #define LOC_MEM         0x0200
   9 #define LOC_NUM         0x0400
  10 #define LOC_SYM         0x0800
  11 #define LOC_LOCAL       0x1000
  12 #define LOC_MASK        0xff00
  13
  14 #define R_RAX           0x00
  15 #define R_RCX           0x01
  16 #define R_RDX           0x02
  17 #define R_RBX           0x03
  18 #define R_RSP           0x04
  19 #define R_RBP           0x05
  20 #define R_RSI           0x06
  21 #define R_RDI           0x07
  22 #define R_R8            0x08
  23 #define R_R9            0x09
  24 #define R_R10           0x0a
  25 #define R_R11           0x0b
  26 #define R_R12           0x0c
  27 #define R_R13           0x0d
  28 #define R_R14           0x0e
  29 #define R_R15           0x0f
  30 #define NREGS           0x10
  31
  32 #define MOV_M2R         0x8b
  33 #define MOV_R2X         0x89
  34 #define MOV_I2X         0xc7
  35 #define MOV_I2R         0xb8
  36 #define ADD_R2X         0x03
  37 #define SUB_R2X         0x2b
  38 #define SHX_REG         0xd3
  39 #define CMP_R2X         0x3b
  40 #define LEA_M2R         0x8d
  41 #define NEG_REG         0xf7
  42 #define NOT_REG         0xf7
  43 #define CALL_REG        0xff
  44 #define MUL_A2X         0xf7
  45 #define XOR_R2X         0x33
  46 #define AND_R2X         0x23
  47 #define OR_R2X          0x0b
  48 #define TEST_R2R        0x85
  49 #define INC_X           0xff
  50
  51 #define MIN(a, b)               ((a) < (b) ? (a) : (b))
  52
  53 #define TMP_BT(t)               ((t)->flags & TMP_ADDR ? 8 : (t)->bt)
  54 #define TMP_REG(t)              ((t)->flags & LOC_REG ? (t)->addr : reg_get(~0))
  55 #define TMP_REG2(t, r)          ((t)->flags & LOC_REG && (t)->addr != r ? \
  56                                         (t)->addr : reg_get(~(1 << r)))
  57 #define BT_TMPBT(bt)            (BT_SZ(bt) >= 4 ? (bt) : (bt) & BT_SIGNED | 4)
  58
  59 static char buf[SECSIZE];
  60 static char *cur;
  61 static int nogen;
  62 static long sp;
  63 static long spsub_addr;
  64 static long maxsp;
  65
  66 #define TMP(i)          (&tmps[ntmp - 1 - (i)])
  67
  68 static struct tmp {
  69         long addr;
  70         long off;       /* used for LOC_SYM; offset from a symbol address */
  71         unsigned flags;
  72         unsigned bt;
  73 } tmps[MAXTMP];
  74 static int ntmp;
  75
  76 static int tmpsp;
  77
  78 static struct tmp *regs[NREGS];
  79 static int tmpregs[] = {R_RAX, R_RDI, R_RSI, R_RDX, R_RCX, R_R8, R_R9};
  80
  81 #define MAXRET                  (1 << 8)
  82
  83 static long ret[MAXRET];
  84 static int nret;
  85
  86 static long cmp_last;
  87 static long cmp_setl;
  88
  89 static void putint(char *s, long n, int l)
  90 {
  91         while (l--) {
  92                 *s++ = n;
  93                 n >>= 8;
  94         }
  95 }
  96
  97 static void os(char *s, int n)
  98 {
  99         if (nogen)
 100                 return;
 101         while (n--)
 102                 *cur++ = *s++;
 103 }
 104
 105 static void oi(long n, int l)
 106 {
 107         if (nogen)
 108                 return;
 109         while (l--) {
 110                 *cur++ = n;
 111                 n >>= 8;
 112         }
 113 }
 114
 115 static long codeaddr(void)
 116 {
 117         return cur - buf;
 118 }
 119
 120 #define OP2(o2, o1)             (0x010000 | ((o2) << 8) | (o1))
 121 #define O2(op)                  (((op) >> 8) & 0xff)
 122 #define O1(op)                  ((op) & 0xff)
 123 #define MODRM(m, r1, r2)        ((m) << 6 | (r1) << 3 | (r2))
 124
 125 static void o_op(int op, int r1, int r2, unsigned bt)
 126 {
 127         int rex = 0;
 128         int sz = BT_SZ(bt);
 129         if (r1 & 0x8)
 130                 rex |= 4;
 131         if (r2 & 0x8)
 132                 rex |= 1;
 133         if (sz == 8)
 134                 rex |= 8;
 135         if (sz == 1 && (r1 == R_RSI || r1 == R_RDI ||
 136                         r2 == R_RSI || r2 == R_RDI))
 137                 rex |= 0x40;
 138         /* hack: for movxx ops, bt does not represent the second arg */
 139         if (op & 0x10000 && O2(op) == 0x0f && (O1(op) & 0xf7) == 0xb6 &&
 140                         (r2 == R_RSI || r2 == R_RDI))
 141                 rex |= 0x40;
 142         if (rex || sz == 8)
 143                 oi(0x40 | rex, 1);
 144         if (sz == 2)
 145                 oi(0x66, 1);
 146         if (op & 0x10000)
 147                 oi(O2(op), 1);
 148         oi(sz == 1 ? O1(op) & ~0x1 : O1(op), 1);
 149 }
 150
 151 static void memop(int op, int src, int base, int off, unsigned bt)
 152 {
 153         int dis = off == (char) off ? 1 : 4;
 154         int mod = dis == 4 ? 2 : 1;
 155         o_op(op, src, base, bt);
 156         if (!off)
 157                 mod = 0;
 158         oi(MODRM(mod, src & 0x07, base & 0x07), 1);
 159         if (off)
 160                 oi(off, dis);
 161 }
 162
 163 static void regop(int op, int src, int dst, unsigned bt)
 164 {
 165         o_op(op, src, dst, bt);
 166         oi(MODRM(3, src & 0x07, dst & 0x07), 1);
 167 }
 168
 169 static long sp_push(int size)
 170 {
 171         sp += size;
 172         if (sp > maxsp)
 173                 maxsp = sp;
 174         return sp;
 175 }
 176
 177 #define LOC_NEW(f, l)           (((f) & ~LOC_MASK) | (l))
 178
 179 static void tmp_mem(struct tmp *tmp)
 180 {
 181         int src = tmp->addr;
 182         if (!(tmp->flags & LOC_REG))
 183                 return;
 184         if (tmpsp == -1)
 185                 tmpsp = sp;
 186         tmp->addr = -sp_push(8);
 187         memop(MOV_R2X, src, R_RBP, tmp->addr, BT_TMPBT(TMP_BT(tmp)));
 188         regs[src] = NULL;
 189         tmp->flags = LOC_NEW(tmp->flags, LOC_MEM);
 190 }
 191
 192 static int movxx_x2r(int bt)
 193 {
 194         int o2;
 195         if (BT_SZ(bt) == 1)
 196                 o2 = bt & BT_SIGNED ? 0xbe : 0xb6;
 197         else
 198                 o2 = bt & BT_SIGNED ? 0xbf : 0xb7;
 199         return OP2(0x0f, o2);
 200 }
 201
 202 #define MOVSXD          0x63
 203
 204 static void mov_r2r(int r1, int r2, unsigned bt1, unsigned bt2)
 205 {
 206         int s1 = bt1 & BT_SIGNED;
 207         int s2 = bt2 & BT_SIGNED;
 208         int sz1 = BT_SZ(bt1);
 209         int sz2 = BT_SZ(bt2);
 210         if (sz2 < 4 && (sz1 > sz2 || s1 != s2)) {
 211                 regop(movxx_x2r(bt2), r1, r2, 4);
 212                 return;
 213         }
 214         if (sz1 == 4 && sz2 == 8 && s1) {
 215                 regop(MOVSXD, r2, r1, sz2);
 216                 return;
 217         }
 218         if (r1 != r2 || sz1 > sz2)
 219                 regop(MOV_R2X, r1, r2, BT_TMPBT(bt2));
 220 }
 221
 222 static void mov_m2r(int dst, int base, int off, int bt1, int bt2)
 223 {
 224         if (BT_SZ(bt1) < 4) {
 225                 memop(movxx_x2r(bt1), dst, base, off,
 226                         bt1 & BT_SIGNED && BT_SZ(bt2) == 8 ? 8 : 4);
 227                 mov_r2r(dst, dst, bt1, bt2);
 228         } else {
 229                 memop(MOV_M2R, dst, base, off, bt1);
 230                 mov_r2r(dst, dst, bt1, bt2);
 231         }
 232 }
 233
 234 static void num_cast(struct tmp *t, unsigned bt)
 235 {
 236         if (!(bt & BT_SIGNED) && BT_SZ(bt) != 8)
 237                 t->addr &= ((1l << (long) (BT_SZ(bt) * 8)) - 1);
 238         if (bt & BT_SIGNED && BT_SZ(bt) != 8 &&
 239                                 t->addr > (1l << (BT_SZ(bt) * 8 - 1)))
 240                 t->addr = -((1l << (BT_SZ(bt) * 8)) - t->addr);
 241         t->bt = bt;
 242 }
 243
 244 static void num_reg(int reg, unsigned bt, long num)
 245 {
 246         int op = MOV_I2R + (reg & 7);
 247         if (BT_SZ(bt) == 8 && num >= 0 && num == (unsigned) num)
 248                 bt = 4;
 249         o_op(op, 0, reg, bt);
 250         oi(num, BT_SZ(bt));
 251 }
 252
 253 static void tmp_reg(struct tmp *tmp, int dst, unsigned bt, int deref)
 254 {
 255         if (deref && tmp->flags & TMP_ADDR)
 256                 tmp->flags &= ~TMP_ADDR;
 257         else
 258                 deref = 0;
 259         if (tmp->flags & LOC_NUM) {
 260                 num_cast(tmp, bt);
 261                 tmp->bt = BT_TMPBT(bt);
 262                 num_reg(dst, tmp->bt, tmp->addr);
 263                 tmp->addr = dst;
 264                 regs[dst] = tmp;
 265                 tmp->flags = LOC_NEW(tmp->flags, LOC_REG);
 266         }
 267         if (tmp->flags & LOC_SYM) {
 268                 regop(MOV_I2X, 0, dst, 4);
 269                 if (!nogen)
 270                         out_rela(tmp->addr, codeaddr(), 0);
 271                 oi(tmp->off, 4);
 272                 tmp->addr = dst;
 273                 regs[dst] = tmp;
 274                 tmp->flags = LOC_NEW(tmp->flags, LOC_REG);
 275         }
 276         if (tmp->flags & LOC_REG) {
 277                 if (deref)
 278                         mov_m2r(dst, tmp->addr, 0, tmp->bt, bt);
 279                 else
 280                         mov_r2r(tmp->addr, dst, TMP_BT(tmp),
 281                                 tmp->flags & TMP_ADDR ? 8 : bt);
 282                 regs[tmp->addr] = NULL;
 283         }
 284         if (tmp->flags & LOC_LOCAL) {
 285                 if (deref)
 286                         mov_m2r(dst, R_RBP, tmp->addr, tmp->bt, bt);
 287                 else
 288                         memop(LEA_M2R, dst, R_RBP, tmp->addr, 8);
 289         }
 290         if (tmp->flags & LOC_MEM) {
 291                 int nbt = deref ? 8 : TMP_BT(tmp);
 292                 mov_m2r(dst, R_RBP, tmp->addr, nbt, nbt);
 293                 if (deref)
 294                         mov_m2r(dst, dst, 0, tmp->bt, bt);
 295         }
 296         tmp->addr = dst;
 297         tmp->bt = tmp->flags & TMP_ADDR ? bt : BT_TMPBT(bt);
 298         regs[dst] = tmp;
 299         tmp->flags = LOC_NEW(tmp->flags, LOC_REG);
 300 }
 301
 302 static void reg_free(int reg)
 303 {
 304         int i;
 305         if (!regs[reg])
 306                 return;
 307         for (i = 0; i < ARRAY_SIZE(tmpregs); i++)
 308                 if (!regs[tmpregs[i]]) {
 309                         tmp_reg(regs[reg], tmpregs[i], regs[reg]->bt, 0);
 310                         return;
 311                 }
 312         tmp_mem(regs[reg]);
 313 }
 314
 315 static void reg_for(int reg, struct tmp *t)
 316 {
 317         if (regs[reg] && regs[reg] != t)
 318                 reg_free(reg);
 319 }
 320
 321 static void tmp_mv(struct tmp *t, int reg)
 322 {
 323         reg_for(reg, t);
 324         tmp_reg(t, reg, t->bt, 0);
 325 }
 326
 327 static void tmp_to(struct tmp *t, int reg, int bt)
 328 {
 329         reg_for(reg, t);
 330         tmp_reg(t, reg, bt ? bt : t->bt, 1);
 331 }
 332
 333 static void tmp_drop(int n)
 334 {
 335         int i;
 336         for (i = ntmp - n; i < ntmp; i++)
 337                 if (tmps[i].flags & LOC_REG)
 338                         regs[tmps[i].addr] = NULL;
 339         cmp_last = -1;
 340         ntmp -= n;
 341 }
 342
 343 static int tmp_pop(int reg, int bt)
 344 {
 345         struct tmp *t = TMP(0);
 346         tmp_to(t, reg, bt);
 347         tmp_drop(1);
 348         return t->bt;
 349 }
 350
 351 static struct tmp *tmp_new(void)
 352 {
 353         cmp_last = -1;
 354         return &tmps[ntmp++];
 355 }
 356
 357 static void tmp_push(int reg, unsigned bt)
 358 {
 359         struct tmp *t = tmp_new();
 360         t->addr = reg;
 361         t->bt = bt;
 362         t->flags = LOC_REG;
 363         regs[reg] = t;
 364 }
 365
 366 void o_local(long addr, unsigned bt)
 367 {
 368         struct tmp *t = tmp_new();
 369         t->addr = -addr;
 370         t->bt = bt;
 371         t->flags = LOC_LOCAL | TMP_ADDR;
 372 }
 373
 374 void o_num(long num, unsigned bt)
 375 {
 376         struct tmp *t = tmp_new();
 377         t->addr = num;
 378         t->bt = bt;
 379         t->flags = LOC_NUM;
 380 }
 381
 382 void o_symaddr(long addr, unsigned bt)
 383 {
 384         struct tmp *t = tmp_new();
 385         t->bt = bt;
 386         t->addr = addr;
 387         t->flags = LOC_SYM | TMP_ADDR;
 388         t->off = 0;
 389 }
 390
 391 void o_tmpdrop(int n)
 392 {
 393         if (n == -1 || n > ntmp)
 394                 n = ntmp;
 395         tmp_drop(n);
 396         if (!ntmp) {
 397                 if (tmpsp != -1)
 398                         sp = tmpsp;
 399                 tmpsp = -1;
 400         }
 401 }
 402
 403 #define FORK_REG                R_RAX
 404
 405 /* make sure tmps remain intact after a conditional expression */
 406 void o_fork(void)
 407 {
 408         int i;
 409         for (i = 0; i < ntmp - 1; i++)
 410                 tmp_mem(&tmps[i]);
 411 }
 412
 413 void o_forkpush(void)
 414 {
 415         tmp_pop(R_RAX, 0);
 416 }
 417
 418 void o_forkjoin(void)
 419 {
 420         tmp_push(FORK_REG, 0);
 421 }
 422
 423 void o_tmpswap(void)
 424 {
 425         struct tmp *t1 = TMP(0);
 426         struct tmp *t2 = TMP(1);
 427         struct tmp t;
 428         memcpy(&t, t1, sizeof(t));
 429         memcpy(t1, t2, sizeof(t));
 430         memcpy(t2, &t, sizeof(t));
 431         if (t1->flags & LOC_REG)
 432                 regs[t1->addr] = t1;
 433         if (t2->flags & LOC_REG)
 434                 regs[t2->addr] = t2;
 435 }
 436
 437 static int reg_get(int mask)
 438 {
 439         int i;
 440         for (i = 0; i < ARRAY_SIZE(tmpregs); i++)
 441                 if ((1 << tmpregs[i]) & mask && !regs[tmpregs[i]])
 442                         return tmpregs[i];
 443         for (i = 0; i < ARRAY_SIZE(tmpregs); i++)
 444                 if ((1 << tmpregs[i]) & mask) {
 445                         reg_free(tmpregs[i]);
 446                         return tmpregs[i];
 447                 }
 448         return 0;
 449 }
 450
 451 void tmp_copy(struct tmp *t1)
 452 {
 453         struct tmp *t2 = tmp_new();
 454         memcpy(t2, t1, sizeof(*t1));
 455         if (!(t1->flags & (LOC_REG | LOC_MEM)))
 456                 return;
 457         if (t1->flags & LOC_MEM) {
 458                 tmp_reg(t2, reg_get(~0), t2->bt, 0);
 459         } else if (t1->flags & LOC_REG) {
 460                 t2->addr = reg_get(~(1 << t1->addr));
 461                 regop(MOV_R2X, t1->addr, t2->addr, BT_TMPBT(TMP_BT(t1)));
 462                 regs[t2->addr] = t2;
 463         }
 464 }
 465
 466 void o_tmpcopy(void)
 467 {
 468         tmp_copy(TMP(0));
 469 }
 470
 471 void o_cast(unsigned bt)
 472 {
 473         struct tmp *t = TMP(0);
 474         int reg;
 475         if (t->bt == bt)
 476                 return;
 477         if (t->flags & LOC_NUM) {
 478                 num_cast(t, bt);
 479                 return;
 480         }
 481         reg = TMP_REG(t);
 482         tmp_to(t, reg, bt);
 483 }
 484
 485 long o_func_beg(char *name, int global)
 486 {
 487         long addr = out_func_beg(name, global);
 488         cur = buf;
 489         os("\x55", 1);                  /* push %rbp */
 490         os("\x48\x89\xe5", 3);          /* mov %rsp, %rbp */
 491         sp = 0;
 492         maxsp = 0;
 493         ntmp = 0;
 494         tmpsp = -1;
 495         nret = 0;
 496         cmp_last = -1;
 497         memset(regs, 0, sizeof(regs));
 498         os("\x48\x81\xec", 3);          /* sub $xxx, %rsp */
 499         spsub_addr = codeaddr();
 500         oi(0, 4);
 501         return addr;
 502 }
 503
 504 void o_deref(unsigned bt)
 505 {
 506         struct tmp *t = TMP(0);
 507         if (t->flags & TMP_ADDR)
 508                 tmp_to(t, TMP_REG(t), t->bt);
 509         t->bt = bt;
 510         t->flags |= TMP_ADDR;
 511 }
 512
 513 void o_load(void)
 514 {
 515         struct tmp *t = TMP(0);
 516         tmp_to(t, TMP_REG(t), t->bt);
 517 }
 518
 519 static unsigned bt_op(unsigned bt1, unsigned bt2)
 520 {
 521         unsigned s1 = BT_SZ(bt1);
 522         unsigned s2 = BT_SZ(bt2);
 523         unsigned bt = (bt1 | bt2) & BT_SIGNED | (s1 > s2 ? s1 : s2);
 524         return BT_TMPBT(bt);
 525 }
 526
 527 #define TMP_CONST(t)    ((t)->flags & LOC_NUM && !((t)->flags & TMP_ADDR))
 528 #define LOCAL_PTR(t)    ((t)->flags & LOC_LOCAL && !((t)->flags & TMP_ADDR))
 529 #define SYM_PTR(t)      ((t)->flags & LOC_SYM && !((t)->flags & TMP_ADDR))
 530
 531 int o_popnum(long *c)
 532 {
 533         struct tmp *t = TMP(0);
 534         if (!TMP_CONST(t))
 535                 return 1;
 536         *c = t->addr;
 537         tmp_drop(1);
 538         return 0;
 539 }
 540
 541 static int c_op(long (*cop)(long a, unsigned bt), unsigned bt)
 542 {
 543         struct tmp *t1 = TMP(0);
 544         long ret;
 545         if (!TMP_CONST(t1))
 546                 return 1;
 547         if (!bt)
 548                 bt = t1->bt;
 549         ret = cop(t1->addr, bt);
 550         tmp_drop(1);
 551         o_num(ret, bt);
 552         return 0;
 553 }
 554
 555 static void shx(int uop, int sop)
 556 {
 557         struct tmp *t2 = TMP(1);
 558         int r2 = TMP_REG2(t2, R_RCX);
 559         int bt;
 560         tmp_pop(R_RCX, 0);
 561         bt = tmp_pop(r2, 0);
 562         regop(SHX_REG, bt & BT_SIGNED ? sop : uop, r2, BT_TMPBT(bt));
 563         tmp_push(r2, bt);
 564 }
 565
 566 static int mulop(int uop, int sop, int reg)
 567 {
 568         struct tmp *t1 = TMP(0);
 569         struct tmp *t2 = TMP(1);
 570         int bt = bt_op(t1->bt, t2->bt);
 571         if (t1->flags & LOC_REG && t1->addr != R_RAX && t1->addr != R_RDX)
 572                 reg = t1->addr;
 573         tmp_to(t1, reg, bt);
 574         tmp_to(t2, R_RAX, bt);
 575         if (reg != R_RDX) {
 576                 reg_free(R_RDX);
 577                 if (bt & BT_SIGNED)
 578                         o_op(0x99, R_RAX, R_RDX, bt);
 579                 else
 580                         regop(XOR_R2X, R_RDX, R_RDX, bt);
 581         }
 582         tmp_drop(2);
 583         regop(MUL_A2X, bt & BT_SIGNED ? sop : uop, reg, BT_TMPBT(t2->bt));
 584         return bt;
 585 }
 586
 587 void o_addr(void)
 588 {
 589         tmps[ntmp - 1].flags &= ~TMP_ADDR;
 590         tmps[ntmp - 1].bt = 8;
 591 }
 592
 593 void o_ret(unsigned bt)
 594 {
 595         if (bt)
 596                 tmp_pop(R_RAX, bt);
 597         else
 598                 os("\x31\xc0", 2);      /* xor %eax, %eax */
 599         ret[nret++] = o_jmp(0);
 600 }
 601
 602 static void inc(int op)
 603 {
 604         struct tmp *t = TMP(0);
 605         int reg;
 606         int off;
 607         if (t->flags & LOC_LOCAL) {
 608                 reg = R_RBP;
 609                 off = t->addr;
 610         } else {
 611                 reg = TMP_REG(t);
 612                 off = 0;
 613                 tmp_mv(t, reg);
 614         }
 615         memop(INC_X, op, reg, off, t->bt);
 616 }
 617
 618 void o_lnot(void)
 619 {
 620         if (cmp_last == codeaddr()) {
 621                 buf[cmp_setl + 1] ^= 0x01;
 622         } else {
 623                 o_num(0, 4 | BT_SIGNED);
 624                 o_bop(O_EQ);
 625         }
 626 }
 627
 628 void o_neg(int id)
 629 {
 630         struct tmp *t = TMP(0);
 631         int reg;
 632         unsigned bt = BT_TMPBT(t->bt);
 633         reg = TMP_REG(t);
 634         tmp_to(t, reg, bt);
 635         regop(NOT_REG, id, reg, bt);
 636 }
 637
 638 void o_func_end(void)
 639 {
 640         int i;
 641         for (i = 0; i < nret; i++)
 642                 o_filljmp(ret[i]);
 643         os("\xc9\xc3", 2);              /* leave; ret; */
 644         putint(buf + spsub_addr, (maxsp + 7) & ~0x07, 4);
 645         out_func_end(buf, cur - buf);
 646 }
 647
 648 long o_mklocal(int size)
 649 {
 650         return sp_push((size + 7) & ~0x07);
 651 }
 652
 653 void o_rmlocal(long addr, int sz)
 654 {
 655         sp = addr - sz;
 656 }
 657
 658 static int arg_regs[] = {R_RDI, R_RSI, R_RDX, R_RCX, R_R8, R_R9};
 659
 660 #define R_NARGS                 ARRAY_SIZE(arg_regs)
 661
 662 long o_arg(int i, unsigned bt)
 663 {
 664         long addr;
 665         if (i < R_NARGS) {
 666                 addr = o_mklocal(BT_SZ(bt));
 667                 memop(MOV_R2X, arg_regs[i], R_RBP, -addr, BT_TMPBT(bt));
 668         } else {
 669                 addr = -8 * (i - R_NARGS + 2);
 670         }
 671         return addr;
 672 }
 673
 674 void o_assign(unsigned bt)
 675 {
 676         struct tmp *t1 = TMP(0);
 677         struct tmp *t2 = TMP(1);
 678         int r1 = TMP_REG(t1);
 679         int reg;
 680         int off;
 681         tmp_to(t1, r1, BT_TMPBT(bt));
 682         if (t2->flags & LOC_LOCAL) {
 683                 reg = R_RBP;
 684                 off = t2->addr;
 685         } else {
 686                 reg = TMP_REG2(t2, r1);
 687                 off = 0;
 688                 tmp_mv(t2, reg);
 689         }
 690         tmp_drop(2);
 691         memop(MOV_R2X, r1, reg, off, bt);
 692         tmp_push(r1, bt);
 693 }
 694
 695 static long cu(int op, long i)
 696 {
 697         switch (op) {
 698         case O_NEG:
 699                 return -i;
 700         case O_NOT:
 701                 return ~i;
 702         case O_LNOT:
 703                 return !i;
 704         }
 705 }
 706
 707 static int c_uop(int op)
 708 {
 709         struct tmp *t1 = TMP(0);
 710         if (!TMP_CONST(t1))
 711                 return 1;
 712         tmp_drop(1);
 713         o_num(cu(op, t1->addr), t1->bt);
 714         return 0;
 715 }
 716
 717 static long cb(int op, long a, long b, int *bt)
 718 {
 719         switch (op) {
 720         case O_ADD:
 721                 return a + b;
 722         case O_SUB:
 723                 return a - b;
 724         case O_AND:
 725                 return a & b;
 726         case O_OR:
 727                 return a | b;
 728         case O_XOR:
 729                 return a ^ b;
 730         case O_MUL:
 731                 return a * b;
 732         case O_DIV:
 733                 return a / b;
 734         case O_MOD:
 735                 return a % b;
 736         case O_SHL:
 737                 return a << b;
 738         case O_SHR:
 739                 if (*bt & BT_SIGNED)
 740                         return a >> b;
 741                 else
 742                         return (unsigned long) a >> b;
 743         case O_LT:
 744                 *bt = 4;
 745                 return a < b;
 746         case O_GT:
 747                 *bt = 4;
 748                 return a > b;
 749         case O_LE:
 750                 *bt = 4;
 751                 return a <= b;
 752         case O_GE:
 753                 *bt = 4;
 754                 return a >= b;
 755         case O_EQ:
 756                 *bt = 4;
 757                 return a == b;
 758         case O_NEQ:
 759                 *bt = 4;
 760                 return a != b;
 761         }
 762 }
 763
 764 static int c_bop(int op)
 765 {
 766         struct tmp *t1 = TMP(0);
 767         struct tmp *t2 = TMP(1);
 768         int locals = LOCAL_PTR(t1) + LOCAL_PTR(t2);
 769         int syms = SYM_PTR(t1) + SYM_PTR(t2);
 770         int nums = TMP_CONST(t1) + TMP_CONST(t2);
 771         int bt;
 772         if (syms == 2 || syms && locals || syms + nums + locals != 2)
 773                 return 1;
 774         if (!locals)
 775                 bt = syms ? 8 : bt_op(t1->bt, t2->bt);
 776         if (locals == 1)
 777                 bt = 8;
 778         if (locals == 2)
 779                 bt = 4 | BT_SIGNED;
 780         if (syms) {
 781                 long o1 = SYM_PTR(t1) ? t1->off : t1->addr;
 782                 long o2 = SYM_PTR(t2) ? t2->off : t2->addr;
 783                 long ret = cb(op, o2, o1, &bt);
 784                 if (!SYM_PTR(t2))
 785                         o_tmpswap();
 786                 t2->off = ret;
 787                 tmp_drop(1);
 788         } else {
 789                 long ret = cb(op, t2->addr, t1->addr, &bt);
 790                 tmp_drop(2);
 791                 if (locals == 1) {
 792                         o_local(-ret, bt);
 793                         o_addr();
 794                 } else {
 795                         o_num(ret, bt);
 796                 }
 797         }
 798         return 0;
 799 }
 800
 801 void o_uop(int op)
 802 {
 803         if (!c_uop(op))
 804                 return;
 805         switch (op) {
 806         case O_NEG:
 807         case O_NOT:
 808                 o_neg(op == O_NEG ? 3 : 2);
 809                 break;
 810         case O_LNOT:
 811                 o_lnot();
 812                 break;
 813         case O_INC:
 814         case O_DEC:
 815                 inc(op == O_DEC);
 816                 break;
 817         }
 818 }
 819
 820 static int binop(int op, int *reg)
 821 {
 822         struct tmp *t1 = TMP(0);
 823         struct tmp *t2 = TMP(1);
 824         int r1, r2;
 825         unsigned bt = bt_op(t1->bt, t2->bt);
 826         r1 = TMP_REG(t1);
 827         r2 = TMP_REG2(t2, r1);
 828         tmp_pop(r1, bt);
 829         tmp_pop(r2, bt);
 830         regop(op, r2, r1, bt);
 831         *reg = r2;
 832         return bt;
 833 }
 834
 835 static void bin_add(int op)
 836 {
 837         /* opcode for O_ADD, O_SUB, O_AND, O_OR, O_XOR */
 838         static int rx[] = {0x03, 0x2b, 0x23, 0x0b, 0x33};
 839         int reg;
 840         int bt = binop(rx[op & 0x0f], &reg);
 841         tmp_push(reg, bt);
 842 }
 843
 844 static void bin_shx(int op)
 845 {
 846         if ((op & 0xff) == O_SHL)
 847                 shx(4, 4);
 848         else
 849                 shx(5, 7);
 850 }
 851
 852 static void bin_mul(int op)
 853 {
 854         if ((op & 0xff) == O_MUL)
 855                 tmp_push(R_RAX, mulop(4, 5, R_RDX));
 856         if ((op & 0xff) == O_DIV)
 857                 tmp_push(R_RAX, mulop(6, 7, R_RCX));
 858         if ((op & 0xff) == O_MOD)
 859                 tmp_push(R_RDX, mulop(6, 7, R_RCX));
 860 }
 861
 862 static void o_cmp(int uop, int sop)
 863 {
 864         struct tmp *t1 = TMP(0);
 865         struct tmp *t2 = TMP(1);
 866         char set[] = "\x0f\x00\xc0";
 867         int reg;
 868         int bt;
 869         if (regs[R_RAX] && regs[R_RAX] != t1 && regs[R_RAX] != t2)
 870                 reg_free(R_RAX);
 871         bt = binop(CMP_R2X, &reg);
 872         set[1] = bt & BT_SIGNED ? sop : uop;
 873         cmp_setl = codeaddr();
 874         os(set, 3);                     /* setl %al */
 875         os("\x0f\xb6\xc0", 3);          /* movzbl %al, %eax */
 876         tmp_push(R_RAX, 4 | BT_SIGNED);
 877         cmp_last = codeaddr();
 878 }
 879
 880 static void bin_cmp(int op)
 881 {
 882         switch (op & 0xff) {
 883         case O_LT:
 884                 o_cmp(0x92, 0x9c);
 885                 break;
 886         case O_GT:
 887                 o_cmp(0x97, 0x9f);
 888                 break;
 889         case O_LE:
 890                 o_cmp(0x96, 0x9e);
 891                 break;
 892         case O_GE:
 893                 o_cmp(0x93, 0x9d);
 894                 break;
 895         case O_EQ:
 896                 o_cmp(0x94, 0x94);
 897                 break;
 898         case O_NEQ:
 899                 o_cmp(0x95, 0x95);
 900                 break;
 901         }
 902 }
 903
 904 static void o_bopset(int op)
 905 {
 906         tmp_copy(TMP(1));
 907         o_tmpswap();
 908         o_bop(op & ~O_SET);
 909         o_assign(TMP(1)->bt);
 910 }
 911
 912 void o_bop(int op)
 913 {
 914         if (!(op & O_SET) && !c_bop(op))
 915                 return;
 916         if (op & O_SET) {
 917                 o_bopset(op);
 918                 return;
 919         }
 920         if ((op & 0xf0) == 0x00)
 921                 bin_add(op);
 922         if ((op & 0xf0) == 0x10)
 923                 bin_shx(op);
 924         if ((op & 0xf0) == 0x20)
 925                 bin_mul(op);
 926         if ((op & 0xf0) == 0x30)
 927                 bin_cmp(op);
 928 }
 929
 930 void o_memcpy(int sz)
 931 {
 932         struct tmp *t0 = TMP(-1);
 933         struct tmp *t1 = TMP(0);
 934         struct tmp *t2 = TMP(1);
 935         o_num(sz, 4);
 936         tmp_to(t0, R_RCX, 0);
 937         tmp_mv(t1, R_RSI);
 938         tmp_mv(t2, R_RDI);
 939         os("\xf3\xa4", 2);              /* rep movs */
 940         tmp_drop(2);
 941 }
 942
 943 void o_memset(int x, int sz)
 944 {
 945         struct tmp *t0 = TMP(-2);
 946         struct tmp *t1 = TMP(-1);
 947         struct tmp *t2 = TMP(0);
 948         o_num(sz, 4);
 949         o_num(x, 4);
 950         tmp_to(t0, R_RAX, 0);
 951         tmp_to(t1, R_RCX, 0);
 952         tmp_mv(t2, R_RDI);
 953         os("\xf3\xaa", 2);              /* rep stosb */
 954         tmp_drop(2);
 955 }
 956
 957 long o_mklabel(void)
 958 {
 959         return codeaddr();
 960 }
 961
 962 static long jx(int x, long addr)
 963 {
 964         char op[2] = {0x0f};
 965         op[1] = x;
 966         os(op, 2);              /* jx $addr */
 967         oi(addr - codeaddr() - 4, 4);
 968         return codeaddr() - 4;
 969 }
 970
 971 static long jxtest(int x, long addr)
 972 {
 973         int bt = tmp_pop(R_RAX, 0);
 974         regop(TEST_R2R, R_RAX, R_RAX, bt);
 975         return jx(x, addr);
 976 }
 977
 978 static long jxcmp(long addr, int inv)
 979 {
 980         int x;
 981         if (codeaddr() != cmp_last)
 982                 return -1;
 983         tmp_drop(1);
 984         cur = buf + cmp_setl;
 985         x = (unsigned char) buf[cmp_setl + 1];
 986         return jx((inv ? x : x ^ 0x01) & ~0x10, addr);
 987 }
 988
 989 long o_jz(long addr)
 990 {
 991         long ret = jxcmp(addr, 0);
 992         return ret != -1 ? ret : jxtest(0x84, addr);
 993 }
 994
 995 long o_jnz(long addr)
 996 {
 997         long ret = jxcmp(addr, 1);
 998         return ret != -1 ? ret : jxtest(0x85, addr);
 999 }
1000
1001 long o_jmp(long addr)
1002 {
1003         os("\xe9", 1);                  /* jmp $addr */
1004         oi(addr - codeaddr() - 4, 4);
1005         return codeaddr() - 4;
1006 }
1007
1008 void o_filljmp2(long addr, long jmpdst)
1009 {
1010         putint(buf + addr, jmpdst - addr - 4, 4);
1011 }
1012
1013 void o_filljmp(long addr)
1014 {
1015         o_filljmp2(addr, codeaddr());
1016 }
1017
1018 void o_call(int argc, unsigned *bt, unsigned ret_bt)
1019 {
1020         int i;
1021         struct tmp *t;
1022         for (i = 0; i < ARRAY_SIZE(tmpregs); i++)
1023                 if (regs[tmpregs[i]] && regs[tmpregs[i]] - tmps < ntmp - argc)
1024                         tmp_mem(regs[tmpregs[i]]);
1025         if (argc > R_NARGS) {
1026                 long addr = sp_push(8 * (argc - R_NARGS));
1027                 for (i = argc - 1; i >= R_NARGS; --i) {
1028                         int reg = TMP_REG(TMP(0));
1029                         tmp_pop(reg, bt[i]);
1030                         memop(MOV_R2X, reg, R_RBP,
1031                                 -(addr - (i - R_NARGS) * 8), BT_TMPBT(bt[i]));
1032                 }
1033         }
1034         for (i = MIN(argc, R_NARGS) - 1; i >= 0; i--)
1035                 tmp_pop(arg_regs[i], BT_TMPBT(bt[i]));
1036         t = TMP(0);
1037         if (t->flags & LOC_SYM) {
1038                 os("\x31\xc0", 2);      /* xor %eax, %eax */
1039                 os("\xe8", 1);          /* call $x */
1040                 if (!nogen)
1041                         out_rela(t->addr, codeaddr(), 1);
1042                 oi(-4 + t->off, 4);
1043                 tmp_drop(1);
1044         } else {
1045                 tmp_mv(TMP(0), R_RAX);
1046                 tmp_drop(1);
1047                 regop(CALL_REG, 2, R_RAX, 4);
1048         }
1049         if (ret_bt)
1050                 tmp_push(R_RAX, ret_bt);
1051 }
1052
1053 int o_nogen(void)
1054 {
1055         return nogen++;
1056 }
1057
1058 void o_dogen(void)
1059 {
1060         nogen = 0;
1061 }
1062
1063 void o_datset(long addr, int off, unsigned bt)
1064 {
1065         struct tmp *t = TMP(0);
1066         if (t->flags & LOC_NUM && !(t->flags & TMP_ADDR)) {
1067                 num_cast(t, bt);
1068                 out_datcpy(addr, off, (void *) &t->addr, BT_SZ(bt));
1069         }
1070         if (t->flags & LOC_SYM && !(t->flags & TMP_ADDR)) {
1071                 out_datrela(t->addr, addr, off);
1072                 out_datcpy(addr, off, (void *) &t->off, BT_SZ(bt));
1073         }
1074         tmp_drop(1);
1075 }