x86_64-gen.c

   1 /*
   2  *  x86-64 code generator for TCC
   3  *
   4  *  Copyright (c) 2008 Shinichiro Hamaji
   5  *
   6  *  Based on i386-gen.c by Fabrice Bellard
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  21  */
  22
  23 #ifdef TARGET_DEFS_ONLY
  24
  25 /* number of available registers */
  26 #define NB_REGS         10
  27 #define NB_ASM_REGS     8
  28
  29 /* a register can belong to several classes. The classes must be
  30    sorted from more general to more precise (see gv2() code which does
  31    assumptions on it). */
  32 #define RC_INT     0x0001 /* generic integer register */
  33 #define RC_FLOAT   0x0002 /* generic float register */
  34 #define RC_RAX     0x0004
  35 #define RC_RCX     0x0008
  36 #define RC_RDX     0x0010
  37 #define RC_R8      0x0100
  38 #define RC_R9      0x0200
  39 #define RC_XMM0    0x0020
  40 #define RC_ST0     0x0040 /* only for long double */
  41 #define RC_IRET    RC_RAX /* function return: integer register */
  42 #define RC_LRET    RC_RDX /* function return: second integer register */
  43 #define RC_FRET    RC_XMM0 /* function return: float register */
  44
  45 /* pretty names for the registers */
  46 enum {
  47     TREG_RAX = 0,
  48     TREG_RCX = 1,
  49     TREG_RDX = 2,
  50     TREG_XMM0 = 3,
  51     TREG_ST0 = 4,
  52
  53     TREG_RSI = 6,
  54     TREG_RDI = 7,
  55     TREG_R8  = 8,
  56     TREG_R9  = 9,
  57
  58     TREG_R10 = 10,
  59     TREG_R11 = 11,
  60
  61     TREG_MEM = 0x10,
  62 };
  63
  64 #define REX_BASE(reg) (((reg) >> 3) & 1)
  65 #define REG_VALUE(reg) ((reg) & 7)
  66
  67 /* return registers for function */
  68 #define REG_IRET TREG_RAX /* single word int return register */
  69 #define REG_LRET TREG_RDX /* second word return register (for long long) */
  70 #define REG_FRET TREG_XMM0 /* float return register */
  71
  72 /* defined if function parameters must be evaluated in reverse order */
  73 #define INVERT_FUNC_PARAMS
  74
  75 /* pointer size, in bytes */
  76 #define PTR_SIZE 8
  77
  78 /* long double size and alignment, in bytes */
  79 #define LDOUBLE_SIZE  16
  80 #define LDOUBLE_ALIGN 8
  81 /* maximum alignment (for aligned attribute support) */
  82 #define MAX_ALIGN     8
  83
  84 ST_FUNC void gen_opl(int op);
  85 ST_FUNC void gen_le64(int64_t c);
  86
  87 /******************************************************/
  88 /* ELF defines */
  89
  90 #define EM_TCC_TARGET EM_X86_64
  91
  92 /* relocation type for 32 bit data relocation */
  93 #define R_DATA_32   R_X86_64_32
  94 #define R_DATA_PTR  R_X86_64_64
  95 #define R_JMP_SLOT  R_X86_64_JUMP_SLOT
  96 #define R_COPY      R_X86_64_COPY
  97
  98 #define ELF_START_ADDR 0x08048000
  99 #define ELF_PAGE_SIZE  0x1000
 100
 101 /******************************************************/
 102 #else /* ! TARGET_DEFS_ONLY */
 103 /******************************************************/
 104 #include "tcc.h"
 105 #include <assert.h>
 106
 107 ST_DATA const int reg_classes[NB_REGS] = {
 108     /* eax */ RC_INT | RC_RAX,
 109     /* ecx */ RC_INT | RC_RCX,
 110     /* edx */ RC_INT | RC_RDX,
 111     /* xmm0 */ RC_FLOAT | RC_XMM0,
 112     /* st0 */ RC_ST0,
 113 #if NB_REGS == 10
 114     0,
 115     0,
 116     0,
 117     RC_R8,
 118     RC_R9,
 119 #endif
 120 };
 121
 122 static unsigned long func_sub_sp_offset;
 123 static int func_ret_sub;
 124
 125 /* XXX: make it faster ? */
 126 void g(int c)
 127 {
 128     int ind1;
 129     ind1 = ind + 1;
 130     if (ind1 > cur_text_section->data_allocated)
 131         section_realloc(cur_text_section, ind1);
 132     cur_text_section->data[ind] = c;
 133     ind = ind1;
 134 }
 135
 136 void o(unsigned int c)
 137 {
 138     while (c) {
 139         g(c);
 140         c = c >> 8;
 141     }
 142 }
 143
 144 void gen_le16(int v)
 145 {
 146     g(v);
 147     g(v >> 8);
 148 }
 149
 150 void gen_le32(int c)
 151 {
 152     g(c);
 153     g(c >> 8);
 154     g(c >> 16);
 155     g(c >> 24);
 156 }
 157
 158 void gen_le64(int64_t c)
 159 {
 160     g(c);
 161     g(c >> 8);
 162     g(c >> 16);
 163     g(c >> 24);
 164     g(c >> 32);
 165     g(c >> 40);
 166     g(c >> 48);
 167     g(c >> 56);
 168 }
 169
 170 /* output a symbol and patch all calls to it */
 171 void gsym_addr(int t, int a)
 172 {
 173     int n, *ptr;
 174     while (t) {
 175         ptr = (int *)(cur_text_section->data + t);
 176         n = *ptr; /* next value */
 177         *ptr = a - t - 4;
 178         t = n;
 179     }
 180 }
 181
 182 void gsym(int t)
 183 {
 184     gsym_addr(t, ind);
 185 }
 186
 187 /* psym is used to put an instruction with a data field which is a
 188    reference to a symbol. It is in fact the same as oad ! */
 189 #define psym oad
 190
 191 static int is64_type(int t)
 192 {
 193     return ((t & VT_BTYPE) == VT_PTR ||
 194             (t & VT_BTYPE) == VT_FUNC ||
 195             (t & VT_BTYPE) == VT_LLONG);
 196 }
 197
 198 static int is_sse_float(int t) {
 199     int bt;
 200     bt = t & VT_BTYPE;
 201     return bt == VT_DOUBLE || bt == VT_FLOAT;
 202 }
 203
 204 /* instruction + 4 bytes data. Return the address of the data */
 205 ST_FUNC int oad(int c, int s)
 206 {
 207     int ind1;
 208
 209     o(c);
 210     ind1 = ind + 4;
 211     if (ind1 > cur_text_section->data_allocated)
 212         section_realloc(cur_text_section, ind1);
 213     *(int *)(cur_text_section->data + ind) = s;
 214     s = ind;
 215     ind = ind1;
 216     return s;
 217 }
 218
 219 ST_FUNC void gen_addr32(int r, Sym *sym, int c)
 220 {
 221     if (r & VT_SYM)
 222         greloc(cur_text_section, sym, ind, R_X86_64_32);
 223     gen_le32(c);
 224 }
 225
 226 /* output constant with relocation if 'r & VT_SYM' is true */
 227 ST_FUNC void gen_addr64(int r, Sym *sym, int64_t c)
 228 {
 229     if (r & VT_SYM)
 230         greloc(cur_text_section, sym, ind, R_X86_64_64);
 231     gen_le64(c);
 232 }
 233
 234 /* output constant with relocation if 'r & VT_SYM' is true */
 235 ST_FUNC void gen_addrpc32(int r, Sym *sym, int c)
 236 {
 237     if (r & VT_SYM)
 238         greloc(cur_text_section, sym, ind, R_X86_64_PC32);
 239     gen_le32(c-4);
 240 }
 241
 242 /* output got address with relocation */
 243 static void gen_gotpcrel(int r, Sym *sym, int c)
 244 {
 245 #ifndef TCC_TARGET_PE
 246     Section *sr;
 247     ElfW(Rela) *rel;
 248     greloc(cur_text_section, sym, ind, R_X86_64_GOTPCREL);
 249     sr = cur_text_section->reloc;
 250     rel = (ElfW(Rela) *)(sr->data + sr->data_offset - sizeof(ElfW(Rela)));
 251     rel->r_addend = -4;
 252 #else
 253     printf("picpic: %s %x %x | %02x %02x %02x\n", get_tok_str(sym->v, NULL), c, r,
 254         cur_text_section->data[ind-3],
 255         cur_text_section->data[ind-2],
 256         cur_text_section->data[ind-1]
 257         );
 258     greloc(cur_text_section, sym, ind, R_X86_64_PC32);
 259 #endif
 260     gen_le32(0);
 261
 262     if (c) {
 263         /* we use add c, %xxx for displacement */
 264         o(0x48 + REX_BASE(r));
 265         o(0x81);
 266         o(0xc0 + REG_VALUE(r));
 267         gen_le32(c);
 268     }
 269 }
 270
 271 static void gen_modrm_impl(int op_reg, int r, Sym *sym, int c, int is_got)
 272 {
 273     op_reg = REG_VALUE(op_reg) << 3;
 274     if ((r & VT_VALMASK) == VT_CONST) {
 275         /* constant memory reference */
 276         o(0x05 | op_reg);
 277         if (is_got) {
 278             gen_gotpcrel(r, sym, c);
 279         } else {
 280             gen_addrpc32(r, sym, c);
 281         }
 282     } else if ((r & VT_VALMASK) == VT_LOCAL) {
 283         /* currently, we use only ebp as base */
 284         if (c == (char)c) {
 285             /* short reference */
 286             o(0x45 | op_reg);
 287             g(c);
 288         } else {
 289             oad(0x85 | op_reg, c);
 290         }
 291     } else if ((r & VT_VALMASK) >= TREG_MEM) {
 292         if (c) {
 293             g(0x80 | op_reg | REG_VALUE(r));
 294             gen_le32(c);
 295         } else {
 296             g(0x00 | op_reg | REG_VALUE(r));
 297         }
 298     } else {
 299         g(0x00 | op_reg | REG_VALUE(r));
 300     }
 301 }
 302
 303 /* generate a modrm reference. 'op_reg' contains the addtionnal 3
 304    opcode bits */
 305 static void gen_modrm(int op_reg, int r, Sym *sym, int c)
 306 {
 307     gen_modrm_impl(op_reg, r, sym, c, 0);
 308 }
 309
 310 /* generate a modrm reference. 'op_reg' contains the addtionnal 3
 311    opcode bits */
 312 static void gen_modrm64(int opcode, int op_reg, int r, Sym *sym, int c)
 313 {
 314     int is_got;
 315     int rex = 0x48 | (REX_BASE(op_reg) << 2);
 316     if ((r & VT_VALMASK) != VT_CONST &&
 317         (r & VT_VALMASK) != VT_LOCAL) {
 318         rex |= REX_BASE(VT_VALMASK & r);
 319     }
 320     o(rex);
 321     o(opcode);
 322     is_got = (op_reg & TREG_MEM) && !(sym->type.t & VT_STATIC);
 323     gen_modrm_impl(op_reg, r, sym, c, is_got);
 324 }
 325
 326
 327 /* load 'r' from value 'sv' */
 328 void load(int r, SValue *sv)
 329 {
 330     int v, t, ft, fc, fr;
 331     SValue v1;
 332
 333 #ifdef TCC_TARGET_PE
 334     if (pe_dllimport(r, sv, load))
 335         return;
 336 #endif
 337
 338     fr = sv->r;
 339     ft = sv->type.t;
 340     fc = sv->c.ul;
 341
 342 #ifndef TCC_TARGET_PE
 343     /* we use indirect access via got */
 344     if ((fr & VT_VALMASK) == VT_CONST && (fr & VT_SYM) &&
 345         (fr & VT_LVAL) && !(sv->sym->type.t & VT_STATIC)) {
 346         /* use the result register as a temporal register */
 347         int tr = r | TREG_MEM;
 348         if (is_float(ft)) {
 349             /* we cannot use float registers as a temporal register */
 350             tr = get_reg(RC_INT) | TREG_MEM;
 351         }
 352         gen_modrm64(0x8b, tr, fr, sv->sym, 0);
 353
 354         /* load from the temporal register */
 355         fr = tr | VT_LVAL;
 356     }
 357 #endif
 358
 359     v = fr & VT_VALMASK;
 360     if (fr & VT_LVAL) {
 361         if (v == VT_LLOCAL) {
 362             v1.type.t = VT_PTR;
 363             v1.r = VT_LOCAL | VT_LVAL;
 364             v1.c.ul = fc;
 365             load(r, &v1);
 366             fr = r;
 367         }
 368         if ((ft & VT_BTYPE) == VT_FLOAT) {
 369             o(0x6e0f66); /* movd */
 370             r = 0;
 371         } else if ((ft & VT_BTYPE) == VT_DOUBLE) {
 372             o(0x7e0ff3); /* movq */
 373             r = 0;
 374         } else if ((ft & VT_BTYPE) == VT_LDOUBLE) {
 375             o(0xdb); /* fldt */
 376             r = 5;
 377         } else {
 378             int x;
 379             if ((ft & VT_TYPE) == VT_BYTE) {
 380                 x = 0xbe0f;   /* movsbl */
 381             } else if ((ft & VT_TYPE) == (VT_BYTE | VT_UNSIGNED)) {
 382                 x = 0xb60f;   /* movzbl */
 383             } else if ((ft & VT_TYPE) == VT_SHORT) {
 384                 x = 0xbf0f;   /* movswl */
 385             } else if ((ft & VT_TYPE) == (VT_SHORT | VT_UNSIGNED)) {
 386                 x = 0xb70f;   /* movzwl */
 387             } else if (is64_type(ft)) {
 388                 gen_modrm64(0x8b, r, fr, sv->sym, fc);
 389                 return;
 390             } else {
 391                 x = 0x8b; /* movl */
 392             }
 393             if (REX_BASE(r) || (!(fr & ~0x0f) && REX_BASE(fr)))
 394                 o(0x40 + REX_BASE(fr) + (REX_BASE(r) << 2));
 395             o(x);
 396         }
 397         gen_modrm(r, fr, sv->sym, fc);
 398     } else {
 399         if (v == VT_CONST) {
 400             if (fr & VT_SYM) {
 401 #ifdef TCC_TARGET_PE
 402                 o(0x8d48 + (REX_BASE(r) << 2));
 403                 o(0x05 + REG_VALUE(r) * 8); /* lea xx(%rip), r */
 404                 gen_addrpc32(fr, sv->sym, fc);
 405 #else
 406                 if (sv->sym->type.t & VT_STATIC) {
 407                     o(0x8d48 + REX_BASE(r));
 408                     o(0x05 + REG_VALUE(r) * 8); /* lea xx(%rip), r */
 409                     gen_addrpc32(fr, sv->sym, fc);
 410                 } else {
 411                     o(0x8b48 + REX_BASE(r));
 412                     o(0x05 + REG_VALUE(r) * 8); /* mov xx(%rip), r */
 413                     gen_gotpcrel(r, sv->sym, fc);
 414                 }
 415 #endif
 416             } else if (is64_type(ft)) {
 417                 o(0x48 + REX_BASE(r));
 418                 o(0xb8 + REG_VALUE(r)); /* mov $xx, r */
 419                 gen_le64(sv->c.ull);
 420             } else {
 421                 if (REX_BASE(r))
 422                     o(0x41);
 423                 o(0xb8 + REG_VALUE(r)); /* mov $xx, r */
 424                 gen_le32(fc);
 425             }
 426         } else if (v == VT_LOCAL) {
 427             o(0x48 | (REX_BASE(r) << 2));
 428             o(0x8d); /* lea xxx(%ebp), r */
 429             gen_modrm(r, VT_LOCAL, sv->sym, fc);
 430         } else if (v == VT_CMP) {
 431             if (REX_BASE(r))
 432                 o(0x41);
 433             oad(0xb8 + REG_VALUE(r), 0); /* mov $0, r */
 434             if (REX_BASE(r))
 435                 o(0x41);
 436             o(0x0f); /* setxx %br */
 437             o(fc);
 438             o(0xc0 + REG_VALUE(r));
 439         } else if (v == VT_JMP || v == VT_JMPI) {
 440             t = v & 1;
 441             if (REX_BASE(r))
 442                 o(0x41);
 443             oad(0xb8 + REG_VALUE(r), t); /* mov $1, r */
 444             o(0x05eb + (REX_BASE(r) << 8)); /* jmp after */
 445             gsym(fc);
 446             if (REX_BASE(r))
 447                 o(0x41);
 448             oad(0xb8 + REG_VALUE(r), t ^ 1); /* mov $0, r */
 449         } else if (v != r) {
 450             if (r == TREG_XMM0) {
 451                 assert(v == TREG_ST0);
 452                 /* gen_cvt_ftof(VT_DOUBLE); */
 453                 o(0xf0245cdd); /* fstpl -0x10(%rsp) */
 454                 /* movsd -0x10(%rsp),%xmm0 */
 455                 o(0x44100ff2);
 456                 o(0xf024);
 457             } else if (r == TREG_ST0) {
 458                 assert(v == TREG_XMM0);
 459                 /* gen_cvt_ftof(VT_LDOUBLE); */
 460                 /* movsd %xmm0,-0x10(%rsp) */
 461                 o(0x44110ff2);
 462                 o(0xf024);
 463                 o(0xf02444dd); /* fldl -0x10(%rsp) */
 464             } else {
 465                 o(0x48 | REX_BASE(r) | (REX_BASE(v) << 2));
 466                 o(0x89);
 467                 o(0xc0 + REG_VALUE(r) + REG_VALUE(v) * 8); /* mov v, r */
 468             }
 469         }
 470     }
 471 }
 472
 473 /* store register 'r' in lvalue 'v' */
 474 void store(int r, SValue *v)
 475 {
 476     int fr, bt, ft, fc;
 477     int op64 = 0;
 478     /* store the REX prefix in this variable when PIC is enabled */
 479     int pic = 0;
 480
 481 #ifdef TCC_TARGET_PE
 482     if (pe_dllimport(r, v, store))
 483         return;
 484 #endif
 485
 486     ft = v->type.t;
 487     fc = v->c.ul;
 488     fr = v->r & VT_VALMASK;
 489     bt = ft & VT_BTYPE;
 490
 491 #ifndef TCC_TARGET_PE
 492     /* we need to access the variable via got */
 493     if (fr == VT_CONST && (v->r & VT_SYM)) {
 494         /* mov xx(%rip), %r11 */
 495         o(0x1d8b4c);
 496         gen_gotpcrel(TREG_R11, v->sym, v->c.ul);
 497         pic = is64_type(bt) ? 0x49 : 0x41;
 498     }
 499 #endif
 500
 501     /* XXX: incorrect if float reg to reg */
 502     if (bt == VT_FLOAT) {
 503         o(0x66);
 504         o(pic);
 505         o(0x7e0f); /* movd */
 506         r = 0;
 507     } else if (bt == VT_DOUBLE) {
 508         o(0x66);
 509         o(pic);
 510         o(0xd60f); /* movq */
 511         r = 0;
 512     } else if (bt == VT_LDOUBLE) {
 513         o(0xc0d9); /* fld %st(0) */
 514         o(pic);
 515         o(0xdb); /* fstpt */
 516         r = 7;
 517     } else {
 518         if (bt == VT_SHORT)
 519             o(0x66);
 520         o(pic);
 521         if (bt == VT_BYTE || bt == VT_BOOL)
 522             o(0x88);
 523         else if (is64_type(bt))
 524             op64 = 0x89;
 525         else
 526             o(0x89);
 527     }
 528     if (pic) {
 529         /* xxx r, (%r11) where xxx is mov, movq, fld, or etc */
 530         if (op64)
 531             o(op64);
 532         o(3 + (r << 3));
 533     } else if (op64) {
 534         if (fr == VT_CONST || fr == VT_LOCAL || (v->r & VT_LVAL)) {
 535             gen_modrm64(op64, r, v->r, v->sym, fc);
 536         } else if (fr != r) {
 537             /* XXX: don't we really come here? */
 538             abort();
 539             o(0xc0 + fr + r * 8); /* mov r, fr */
 540         }
 541     } else {
 542         if (fr == VT_CONST || fr == VT_LOCAL || (v->r & VT_LVAL)) {
 543             gen_modrm(r, v->r, v->sym, fc);
 544         } else if (fr != r) {
 545             /* XXX: don't we really come here? */
 546             abort();
 547             o(0xc0 + fr + r * 8); /* mov r, fr */
 548         }
 549     }
 550 }
 551
 552 /* 'is_jmp' is '1' if it is a jump */
 553 static void gcall_or_jmp(int is_jmp)
 554 {
 555     int r;
 556     if ((vtop->r & (VT_VALMASK | VT_LVAL)) == VT_CONST) {
 557         /* constant case */
 558         if (vtop->r & VT_SYM) {
 559             /* relocation case */
 560             greloc(cur_text_section, vtop->sym,
 561                    ind + 1, R_X86_64_PC32);
 562         } else {
 563             /* put an empty PC32 relocation */
 564             put_elf_reloc(symtab_section, cur_text_section,
 565                           ind + 1, R_X86_64_PC32, 0);
 566         }
 567         oad(0xe8 + is_jmp, vtop->c.ul - 4); /* call/jmp im */
 568     } else {
 569         /* otherwise, indirect call */
 570         r = TREG_R11;
 571         load(r, vtop);
 572         o(0x41); /* REX */
 573         o(0xff); /* call/jmp *r */
 574         o(0xd0 + REG_VALUE(r) + (is_jmp << 4));
 575     }
 576 }
 577
 578 #ifdef TCC_TARGET_PE
 579
 580 #define REGN 4
 581 static const uint8_t arg_regs[] = {
 582     TREG_RCX, TREG_RDX, TREG_R8, TREG_R9
 583 };
 584
 585 static int func_scratch;
 586
 587 /* Generate function call. The function address is pushed first, then
 588    all the parameters in call order. This functions pops all the
 589    parameters and the function address. */
 590
 591 void gen_offs_sp(int b, int r, int d)
 592 {
 593     if (d == (char)d) {
 594         o(b | 0x4000 | (r << 11));
 595         g(d);
 596     } else {
 597         o(b | 0x8000 | (r << 11));
 598         gen_le32(d);
 599     }
 600 }
 601
 602 void gfunc_call(int nb_args)
 603 {
 604     int size, align, r, args_size, i, d, j, bt;
 605     int nb_reg_args, gen_reg;
 606
 607     /* calculate the number of integer/float arguments */
 608     nb_reg_args = 0;
 609     for(i = 0; i < nb_args; i++) {
 610         bt = (vtop[-i].type.t & VT_BTYPE);
 611         if (bt != VT_STRUCT && bt != VT_LDOUBLE)
 612             nb_reg_args++;
 613     }
 614
 615     args_size = (nb_reg_args < REGN ? REGN : nb_reg_args) * PTR_SIZE;
 616
 617     /* for struct arguments, we need to call memcpy and the function
 618        call breaks register passing arguments we are preparing.
 619        So, we process arguments which will be passed by stack first. */
 620     for(i = 0; i < nb_args; i++) {
 621         SValue *sv = &vtop[-i];
 622         bt = (sv->type.t & VT_BTYPE);
 623         if (bt == VT_STRUCT) {
 624             size = type_size(&sv->type, &align);
 625             /* align to stack align size */
 626             size = (size + 15) & ~16;
 627             /* generate structure store */
 628             r = get_reg(RC_INT);
 629             o(0x48);
 630             gen_offs_sp(0x24048d, r, args_size);
 631             args_size += size;
 632
 633             /* generate memcpy call */
 634             vset(&sv->type, r | VT_LVAL, 0);
 635             vpushv(sv);
 636             vstore();
 637             --vtop;
 638
 639         } else if (bt == VT_LDOUBLE) {
 640
 641             gv(RC_ST0);
 642             gen_offs_sp(0x243cdb, 0, args_size);
 643             args_size += 16;
 644
 645         }
 646     }
 647
 648     if (func_scratch < args_size)
 649         func_scratch = args_size;
 650
 651     for (i = 0; i < REGN; ++i)
 652         save_reg(arg_regs[i]);
 653
 654     gen_reg = nb_reg_args;
 655     for(i = 0; i < nb_args; i++) {
 656         bt = (vtop->type.t & VT_BTYPE);
 657         if (bt == VT_STRUCT || bt == VT_LDOUBLE) {
 658             ; /* done */
 659         } else if (is_sse_float(vtop->type.t)) {
 660             gv(RC_FLOAT); /* only one float register */
 661             j = --gen_reg;
 662             if (j >= REGN) {
 663                 o(0x0f66),
 664                 /* movq %xmm0, j*8(%rsp) */
 665                 gen_offs_sp(0x2444d6, 0, j*8);
 666             } else {
 667                 /* movaps %xmm0, %xmmN */
 668                 o(0x280f);
 669                 o(0xc0 + (j << 3));
 670                 d = arg_regs[j];
 671                 /* mov %xmm0, %rxx */
 672                 o(0x66);
 673                 o(0x7e0f48 + REX_BASE(d));
 674                 o(0xc0 + REG_VALUE(d));
 675             }
 676         } else {
 677             j = --gen_reg;
 678             if (j >= REGN) {
 679                 r = gv(RC_INT);
 680                 o(0x48);
 681                 gen_offs_sp(0x244489, r, j*8);
 682             } else {
 683                 d = arg_regs[j];
 684                 if (d < NB_REGS) {
 685                     gv(reg_classes[d] & ~RC_INT);
 686                 } else {
 687                     r = gv(RC_INT);
 688                     if (d != r) {
 689                         o(0x8948 + REX_BASE(d));
 690                         o(0xc0 + r*8 + REG_VALUE(d));
 691                     }
 692                 }
 693
 694             }
 695         }
 696         vtop--;
 697     }
 698
 699     save_regs(0);
 700     gcall_or_jmp(0);
 701     vtop--;
 702 }
 703
 704
 705 #define FUNC_PROLOG_SIZE 11
 706
 707 /* generate function prolog of type 't' */
 708 void gfunc_prolog(CType *func_type)
 709 {
 710     int addr, align, size, reg_param_index, bt;
 711     Sym *sym;
 712     CType *type;
 713
 714     func_ret_sub = 0;
 715     func_scratch = 0;
 716     loc = 0;
 717
 718     addr = PTR_SIZE * 2;
 719     ind += FUNC_PROLOG_SIZE;
 720     func_sub_sp_offset = ind;
 721     reg_param_index = 0;
 722
 723     sym = func_type->ref;
 724
 725     /* if the function returns a structure, then add an
 726        implicit pointer parameter */
 727     func_vt = sym->type;
 728     if ((func_vt.t & VT_BTYPE) == VT_STRUCT) {
 729         gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 730         reg_param_index++;
 731         addr += PTR_SIZE;
 732     }
 733
 734     /* define parameters */
 735     while ((sym = sym->next) != NULL) {
 736         type = &sym->type;
 737         bt = type->t & VT_BTYPE;
 738         if (bt == VT_STRUCT || bt == VT_LDOUBLE)
 739             continue;
 740         if (reg_param_index < REGN) {
 741             /* save arguments passed by register */
 742             gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 743         }
 744         sym_push(sym->v & ~SYM_FIELD, type, VT_LOCAL | VT_LVAL, addr);
 745         reg_param_index++;
 746         addr += PTR_SIZE;
 747     }
 748
 749     while (reg_param_index < REGN) {
 750         if (func_type->ref->c == FUNC_ELLIPSIS)
 751             gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 752         reg_param_index++;
 753         addr += PTR_SIZE;
 754     }
 755
 756     sym = func_type->ref;
 757     while ((sym = sym->next) != NULL) {
 758         type = &sym->type;
 759         bt = type->t & VT_BTYPE;
 760         if (bt == VT_STRUCT || bt == VT_LDOUBLE) {
 761             size = type_size(type, &align);
 762             size = (size + 15) & -16;
 763             sym_push(sym->v & ~SYM_FIELD, type, VT_LOCAL | VT_LVAL, addr);
 764             addr += size;
 765         }
 766     }
 767 }
 768
 769 /* generate function epilog */
 770 void gfunc_epilog(void)
 771 {
 772     int v, saved_ind;
 773
 774     o(0xc9); /* leave */
 775     if (func_ret_sub == 0) {
 776         o(0xc3); /* ret */
 777     } else {
 778         o(0xc2); /* ret n */
 779         g(func_ret_sub);
 780         g(func_ret_sub >> 8);
 781     }
 782
 783     saved_ind = ind;
 784     ind = func_sub_sp_offset - FUNC_PROLOG_SIZE;
 785     /* align local size to word & save local variables */
 786     v = (func_scratch + -loc + 15) & -16;
 787
 788     pe_add_unwind_data(ind, saved_ind, v);
 789
 790     if (v >= 4096) {
 791         Sym *sym = external_global_sym(TOK___chkstk, &func_old_type, 0);
 792         oad(0xb8, v); /* mov stacksize, %eax */
 793         oad(0xe8, -4); /* call __chkstk, (does the stackframe too) */
 794         greloc(cur_text_section, sym, ind-4, R_X86_64_PC32);
 795         o(0x90); /* fill for FUNC_PROLOG_SIZE = 11 bytes */
 796     } else {
 797         o(0xe5894855);  /* push %rbp, mov %rsp, %rbp */
 798         o(0xec8148);  /* sub rsp, stacksize */
 799         gen_le32(v);
 800     }
 801     ind = saved_ind;
 802 }
 803
 804 #else
 805
 806 static void gadd_sp(int val)
 807 {
 808     if (val == (char)val) {
 809         o(0xc48348);
 810         g(val);
 811     } else {
 812         oad(0xc48148, val); /* add $xxx, %rsp */
 813     }
 814 }
 815
 816 #define REGN 6
 817 static const uint8_t arg_regs[REGN] = {
 818     TREG_RDI, TREG_RSI, TREG_RDX, TREG_RCX, TREG_R8, TREG_R9
 819 };
 820
 821 /* Generate function call. The function address is pushed first, then
 822    all the parameters in call order. This functions pops all the
 823    parameters and the function address. */
 824 void gfunc_call(int nb_args)
 825 {
 826     int size, align, r, args_size, i;
 827     SValue *orig_vtop;
 828     int nb_reg_args = 0;
 829     int nb_sse_args = 0;
 830     int sse_reg, gen_reg;
 831
 832     /* calculate the number of integer/float arguments */
 833     args_size = 0;
 834     for(i = 0; i < nb_args; i++) {
 835         if ((vtop[-i].type.t & VT_BTYPE) == VT_STRUCT) {
 836             args_size += type_size(&vtop->type, &align);
 837         } else if ((vtop[-i].type.t & VT_BTYPE) == VT_LDOUBLE) {
 838             args_size += 16;
 839         } else if (is_sse_float(vtop[-i].type.t)) {
 840             nb_sse_args++;
 841             if (nb_sse_args > 8) args_size += 8;
 842         } else {
 843             nb_reg_args++;
 844             if (nb_reg_args > REGN) args_size += 8;
 845         }
 846     }
 847
 848     /* for struct arguments, we need to call memcpy and the function
 849        call breaks register passing arguments we are preparing.
 850        So, we process arguments which will be passed by stack first. */
 851     orig_vtop = vtop;
 852     gen_reg = nb_reg_args;
 853     sse_reg = nb_sse_args;
 854
 855     /* adjust stack to align SSE boundary */
 856     if (args_size &= 8) {
 857         o(0x50); /* push $rax */
 858     }
 859     for(i = 0; i < nb_args; i++) {
 860         if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
 861             size = type_size(&vtop->type, &align);
 862             /* align to stack align size */
 863             size = (size + 3) & ~3;
 864             /* allocate the necessary size on stack */
 865             o(0x48);
 866             oad(0xec81, size); /* sub $xxx, %rsp */
 867             /* generate structure store */
 868             r = get_reg(RC_INT);
 869             o(0x48 + REX_BASE(r));
 870             o(0x89); /* mov %rsp, r */
 871             o(0xe0 + r);
 872             {
 873                 /* following code breaks vtop[1] */
 874                 SValue tmp = vtop[1];
 875                 vset(&vtop->type, r | VT_LVAL, 0);
 876                 vswap();
 877                 vstore();
 878                 vtop[1] = tmp;
 879             }
 880             args_size += size;
 881         } else if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
 882             gv(RC_ST0);
 883             size = LDOUBLE_SIZE;
 884             oad(0xec8148, size); /* sub $xxx, %rsp */
 885             o(0x7cdb); /* fstpt 0(%rsp) */
 886             g(0x24);
 887             g(0x00);
 888             args_size += size;
 889         } else if (is_sse_float(vtop->type.t)) {
 890             int j = --sse_reg;
 891             if (j >= 8) {
 892                 gv(RC_FLOAT);
 893                 o(0x50); /* push $rax */
 894                 /* movq %xmm0, (%rsp) */
 895                 o(0x04d60f66);
 896                 o(0x24);
 897                 args_size += 8;
 898             }
 899         } else {
 900             int j = --gen_reg;
 901             /* simple type */
 902             /* XXX: implicit cast ? */
 903             if (j >= REGN) {
 904                 r = gv(RC_INT);
 905                 o(0x50 + r); /* push r */
 906                 args_size += 8;
 907             }
 908         }
 909         vtop--;
 910     }
 911     vtop = orig_vtop;
 912
 913     /* then, we prepare register passing arguments.
 914        Note that we cannot set RDX and RCX in this loop because gv()
 915        may break these temporary registers. Let's use R10 and R11
 916        instead of them */
 917     gen_reg = nb_reg_args;
 918     sse_reg = nb_sse_args;
 919     for(i = 0; i < nb_args; i++) {
 920         if ((vtop->type.t & VT_BTYPE) == VT_STRUCT ||
 921             (vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
 922         } else if (is_sse_float(vtop->type.t)) {
 923             int j = --sse_reg;
 924             if (j < 8) {
 925                 gv(RC_FLOAT); /* only one float register */
 926                 /* movaps %xmm0, %xmmN */
 927                 o(0x280f);
 928                 o(0xc0 + (sse_reg << 3));
 929             }
 930         } else {
 931             int j = --gen_reg;
 932             /* simple type */
 933             /* XXX: implicit cast ? */
 934             if (j < REGN) {
 935                 r = gv(RC_INT);
 936                 if (j < 2) {
 937                     o(0x8948); /* mov */
 938                     o(0xc0 + r * 8 + arg_regs[j]);
 939                 } else if (j < 4) {
 940                     o(0x8949); /* mov */
 941                     /* j=2: r10, j=3: r11 */
 942                     o(0xc0 + r * 8 + j);
 943                 } else {
 944                     o(0x8949); /* mov */
 945                     /* j=4: r8, j=5: r9 */
 946                     o(0xc0 + r * 8 + j - 4);
 947                 }
 948             }
 949         }
 950         vtop--;
 951     }
 952
 953     save_regs(0); /* save used temporary registers */
 954
 955     /* Copy R10 and R11 into RDX and RCX, respectively */
 956     if (nb_reg_args > 2) {
 957         o(0xd2894c); /* mov %r10, %rdx */
 958         if (nb_reg_args > 3) {
 959             o(0xd9894c); /* mov %r11, %rcx */
 960         }
 961     }
 962
 963     oad(0xb8, nb_sse_args < 8 ? nb_sse_args : 8); /* mov nb_sse_args, %eax */
 964     gcall_or_jmp(0);
 965     if (args_size)
 966         gadd_sp(args_size);
 967     vtop--;
 968 }
 969
 970
 971 #define FUNC_PROLOG_SIZE 11
 972
 973 static void push_arg_reg(int i) {
 974     loc -= 8;
 975     gen_modrm64(0x89, arg_regs[i], VT_LOCAL, NULL, loc);
 976 }
 977
 978 /* generate function prolog of type 't' */
 979 void gfunc_prolog(CType *func_type)
 980 {
 981     int i, addr, align, size;
 982     int param_index, param_addr, reg_param_index, sse_param_index;
 983     Sym *sym;
 984     CType *type;
 985
 986     sym = func_type->ref;
 987     addr = PTR_SIZE * 2;
 988     loc = 0;
 989     ind += FUNC_PROLOG_SIZE;
 990     func_sub_sp_offset = ind;
 991     func_ret_sub = 0;
 992
 993     if (func_type->ref->c == FUNC_ELLIPSIS) {
 994         int seen_reg_num, seen_sse_num, seen_stack_size;
 995         seen_reg_num = seen_sse_num = 0;
 996         /* frame pointer and return address */
 997         seen_stack_size = PTR_SIZE * 2;
 998         /* count the number of seen parameters */
 999         sym = func_type->ref;
1000         while ((sym = sym->next) != NULL) {
1001             type = &sym->type;
1002             if (is_sse_float(type->t)) {
1003                 if (seen_sse_num < 8) {
1004                     seen_sse_num++;
1005                 } else {
1006                     seen_stack_size += 8;
1007                 }
1008             } else if ((type->t & VT_BTYPE) == VT_STRUCT) {
1009                 size = type_size(type, &align);
1010                 size = (size + 3) & ~3;
1011                 seen_stack_size += size;
1012             } else if ((type->t & VT_BTYPE) == VT_LDOUBLE) {
1013                 seen_stack_size += LDOUBLE_SIZE;
1014             } else {
1015                 if (seen_reg_num < REGN) {
1016                     seen_reg_num++;
1017                 } else {
1018                     seen_stack_size += 8;
1019                 }
1020             }
1021         }
1022
1023         loc -= 16;
1024         /* movl $0x????????, -0x10(%rbp) */
1025         o(0xf045c7);
1026         gen_le32(seen_reg_num * 8);
1027         /* movl $0x????????, -0xc(%rbp) */
1028         o(0xf445c7);
1029         gen_le32(seen_sse_num * 16 + 48);
1030         /* movl $0x????????, -0x8(%rbp) */
1031         o(0xf845c7);
1032         gen_le32(seen_stack_size);
1033
1034         /* save all register passing arguments */
1035         for (i = 0; i < 8; i++) {
1036             loc -= 16;
1037             o(0xd60f66); /* movq */
1038             gen_modrm(7 - i, VT_LOCAL, NULL, loc);
1039             /* movq $0, loc+8(%rbp) */
1040             o(0x85c748);
1041             gen_le32(loc + 8);
1042             gen_le32(0);
1043         }
1044         for (i = 0; i < REGN; i++) {
1045             push_arg_reg(REGN-1-i);
1046         }
1047     }
1048
1049     sym = func_type->ref;
1050     param_index = 0;
1051     reg_param_index = 0;
1052     sse_param_index = 0;
1053
1054     /* if the function returns a structure, then add an
1055        implicit pointer parameter */
1056     func_vt = sym->type;
1057     if ((func_vt.t & VT_BTYPE) == VT_STRUCT) {
1058         push_arg_reg(reg_param_index);
1059         param_addr = loc;
1060
1061         func_vc = loc;
1062         param_index++;
1063         reg_param_index++;
1064     }
1065     /* define parameters */
1066     while ((sym = sym->next) != NULL) {
1067         type = &sym->type;
1068         size = type_size(type, &align);
1069         size = (size + 3) & ~3;
1070         if (is_sse_float(type->t)) {
1071             if (sse_param_index < 8) {
1072                 /* save arguments passed by register */
1073                 loc -= 8;
1074                 o(0xd60f66); /* movq */
1075                 gen_modrm(sse_param_index, VT_LOCAL, NULL, loc);
1076                 param_addr = loc;
1077             } else {
1078                 param_addr = addr;
1079                 addr += size;
1080             }
1081             sse_param_index++;
1082
1083         } else if ((type->t & VT_BTYPE) == VT_STRUCT ||
1084                    (type->t & VT_BTYPE) == VT_LDOUBLE) {
1085             param_addr = addr;
1086             addr += size;
1087         } else {
1088             if (reg_param_index < REGN) {
1089                 /* save arguments passed by register */
1090                 push_arg_reg(reg_param_index);
1091                 param_addr = loc;
1092             } else {
1093                 param_addr = addr;
1094                 addr += 8;
1095             }
1096             reg_param_index++;
1097         }
1098         sym_push(sym->v & ~SYM_FIELD, type,
1099                  VT_LOCAL | VT_LVAL, param_addr);
1100         param_index++;
1101     }
1102 }
1103
1104 /* generate function epilog */
1105 void gfunc_epilog(void)
1106 {
1107     int v, saved_ind;
1108
1109     o(0xc9); /* leave */
1110     if (func_ret_sub == 0) {
1111         o(0xc3); /* ret */
1112     } else {
1113         o(0xc2); /* ret n */
1114         g(func_ret_sub);
1115         g(func_ret_sub >> 8);
1116     }
1117     /* align local size to word & save local variables */
1118     v = (-loc + 15) & -16;
1119     saved_ind = ind;
1120     ind = func_sub_sp_offset - FUNC_PROLOG_SIZE;
1121     o(0xe5894855);  /* push %rbp, mov %rsp, %rbp */
1122     o(0xec8148);  /* sub rsp, stacksize */
1123     gen_le32(v);
1124     ind = saved_ind;
1125 }
1126
1127 #endif /* not PE */
1128
1129 /* generate a jump to a label */
1130 int gjmp(int t)
1131 {
1132     return psym(0xe9, t);
1133 }
1134
1135 /* generate a jump to a fixed address */
1136 void gjmp_addr(int a)
1137 {
1138     int r;
1139     r = a - ind - 2;
1140     if (r == (char)r) {
1141         g(0xeb);
1142         g(r);
1143     } else {
1144         oad(0xe9, a - ind - 5);
1145     }
1146 }
1147
1148 /* generate a test. set 'inv' to invert test. Stack entry is popped */
1149 int gtst(int inv, int t)
1150 {
1151     int v, *p;
1152
1153     v = vtop->r & VT_VALMASK;
1154     if (v == VT_CMP) {
1155         /* fast case : can jump directly since flags are set */
1156         g(0x0f);
1157         t = psym((vtop->c.i - 16) ^ inv, t);
1158     } else if (v == VT_JMP || v == VT_JMPI) {
1159         /* && or || optimization */
1160         if ((v & 1) == inv) {
1161             /* insert vtop->c jump list in t */
1162             p = &vtop->c.i;
1163             while (*p != 0)
1164                 p = (int *)(cur_text_section->data + *p);
1165             *p = t;
1166             t = vtop->c.i;
1167         } else {
1168             t = gjmp(t);
1169             gsym(vtop->c.i);
1170         }
1171     } else {
1172         if (is_float(vtop->type.t) ||
1173             (vtop->type.t & VT_BTYPE) == VT_LLONG) {
1174             vpushi(0);
1175             gen_op(TOK_NE);
1176         }
1177         if ((vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST) {
1178             /* constant jmp optimization */
1179             if ((vtop->c.i != 0) != inv)
1180                 t = gjmp(t);
1181         } else {
1182             v = gv(RC_INT);
1183             o(0x85);
1184             o(0xc0 + v * 9);
1185             g(0x0f);
1186             t = psym(0x85 ^ inv, t);
1187         }
1188     }
1189     vtop--;
1190     return t;
1191 }
1192
1193 /* generate an integer binary operation */
1194 void gen_opi(int op)
1195 {
1196     int r, fr, opc, c;
1197
1198     switch(op) {
1199     case '+':
1200     case TOK_ADDC1: /* add with carry generation */
1201         opc = 0;
1202     gen_op8:
1203         if ((vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST &&
1204             !is64_type(vtop->type.t)) {
1205             /* constant case */
1206             vswap();
1207             r = gv(RC_INT);
1208             if (is64_type(vtop->type.t)) {
1209                 o(0x48 | REX_BASE(r));
1210             }
1211             vswap();
1212             c = vtop->c.i;
1213             if (c == (char)c) {
1214                 /* XXX: generate inc and dec for smaller code ? */
1215                 o(0x83);
1216                 o(0xc0 | (opc << 3) | REG_VALUE(r));
1217                 g(c);
1218             } else {
1219                 o(0x81);
1220                 oad(0xc0 | (opc << 3) | REG_VALUE(r), c);
1221             }
1222         } else {
1223             gv2(RC_INT, RC_INT);
1224             r = vtop[-1].r;
1225             fr = vtop[0].r;
1226             if (opc != 7 ||
1227                 is64_type(vtop[0].type.t) || (vtop[0].type.t & VT_UNSIGNED) ||
1228                 is64_type(vtop[-1].type.t) || (vtop[-1].type.t & VT_UNSIGNED)) {
1229                 o(0x48 | REX_BASE(r) | (REX_BASE(fr) << 2));
1230             }
1231             o((opc << 3) | 0x01);
1232             o(0xc0 + REG_VALUE(r) + REG_VALUE(fr) * 8);
1233         }
1234         vtop--;
1235         if (op >= TOK_ULT && op <= TOK_GT) {
1236             vtop->r = VT_CMP;
1237             vtop->c.i = op;
1238         }
1239         break;
1240     case '-':
1241     case TOK_SUBC1: /* sub with carry generation */
1242         opc = 5;
1243         goto gen_op8;
1244     case TOK_ADDC2: /* add with carry use */
1245         opc = 2;
1246         goto gen_op8;
1247     case TOK_SUBC2: /* sub with carry use */
1248         opc = 3;
1249         goto gen_op8;
1250     case '&':
1251         opc = 4;
1252         goto gen_op8;
1253     case '^':
1254         opc = 6;
1255         goto gen_op8;
1256     case '|':
1257         opc = 1;
1258         goto gen_op8;
1259     case '*':
1260         gv2(RC_INT, RC_INT);
1261         r = vtop[-1].r;
1262         fr = vtop[0].r;
1263         if (is64_type(vtop[0].type.t) || (vtop[0].type.t & VT_UNSIGNED) ||
1264             is64_type(vtop[-1].type.t) || (vtop[-1].type.t & VT_UNSIGNED)) {
1265             o(0x48 | REX_BASE(fr) | (REX_BASE(r) << 2));
1266         }
1267         vtop--;
1268         o(0xaf0f); /* imul fr, r */
1269         o(0xc0 + fr + r * 8);
1270         break;
1271     case TOK_SHL:
1272         opc = 4;
1273         goto gen_shift;
1274     case TOK_SHR:
1275         opc = 5;
1276         goto gen_shift;
1277     case TOK_SAR:
1278         opc = 7;
1279     gen_shift:
1280         opc = 0xc0 | (opc << 3);
1281         if ((vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST) {
1282             /* constant case */
1283             vswap();
1284             r = gv(RC_INT);
1285             if ((vtop->type.t & VT_BTYPE) == VT_LLONG) {
1286                 o(0x48 | REX_BASE(r));
1287                 c = 0x3f;
1288             } else {
1289                 c = 0x1f;
1290             }
1291             vswap();
1292             c &= vtop->c.i;
1293             o(0xc1); /* shl/shr/sar $xxx, r */
1294             o(opc | r);
1295             g(c);
1296         } else {
1297             /* we generate the shift in ecx */
1298             gv2(RC_INT, RC_RCX);
1299             r = vtop[-1].r;
1300             if ((vtop[-1].type.t & VT_BTYPE) == VT_LLONG) {
1301                 o(0x48 | REX_BASE(r));
1302             }
1303             o(0xd3); /* shl/shr/sar %cl, r */
1304             o(opc | r);
1305         }
1306         vtop--;
1307         break;
1308     case '/':
1309     case TOK_UDIV:
1310     case TOK_PDIV:
1311     case '%':
1312     case TOK_UMOD:
1313     case TOK_UMULL:
1314         /* first operand must be in eax */
1315         /* XXX: need better constraint for second operand */
1316         gv2(RC_RAX, RC_RCX);
1317         r = vtop[-1].r;
1318         fr = vtop[0].r;
1319         vtop--;
1320         save_reg(TREG_RDX);
1321         if (op == TOK_UMULL) {
1322             o(0xf7); /* mul fr */
1323             o(0xe0 + fr);
1324             vtop->r2 = TREG_RDX;
1325             r = TREG_RAX;
1326         } else {
1327             if (op == TOK_UDIV || op == TOK_UMOD) {
1328                 if ((vtop->type.t & VT_BTYPE) & VT_LLONG) {
1329                     o(0xd23148); /* xor %rdx, %rdx */
1330                     o(0x48 + REX_BASE(fr));
1331                 } else {
1332                     o(0xd231); /* xor %edx, %edx */
1333                 }
1334                 o(0xf7); /* div fr, %eax */
1335                 o(0xf0 + fr);
1336             } else {
1337                 if ((vtop->type.t & VT_BTYPE) & VT_LLONG) {
1338                     o(0x9948); /* cqto */
1339                     o(0x48 + REX_BASE(fr));
1340                 } else {
1341                     o(0x99); /* cltd */
1342                 }
1343                 o(0xf7); /* idiv fr, %eax */
1344                 o(0xf8 + fr);
1345             }
1346             if (op == '%' || op == TOK_UMOD)
1347                 r = TREG_RDX;
1348             else
1349                 r = TREG_RAX;
1350         }
1351         vtop->r = r;
1352         break;
1353     default:
1354         opc = 7;
1355         goto gen_op8;
1356     }
1357 }
1358
1359 void gen_opl(int op)
1360 {
1361     gen_opi(op);
1362 }
1363
1364 /* generate a floating point operation 'v = t1 op t2' instruction. The
1365    two operands are guaranted to have the same floating point type */
1366 /* XXX: need to use ST1 too */
1367 void gen_opf(int op)
1368 {
1369     int a, ft, fc, swapped, r;
1370     int float_type =
1371         (vtop->type.t & VT_BTYPE) == VT_LDOUBLE ? RC_ST0 : RC_FLOAT;
1372
1373     /* convert constants to memory references */
1374     if ((vtop[-1].r & (VT_VALMASK | VT_LVAL)) == VT_CONST) {
1375         vswap();
1376         gv(float_type);
1377         vswap();
1378     }
1379     if ((vtop[0].r & (VT_VALMASK | VT_LVAL)) == VT_CONST)
1380         gv(float_type);
1381
1382     /* must put at least one value in the floating point register */
1383     if ((vtop[-1].r & VT_LVAL) &&
1384         (vtop[0].r & VT_LVAL)) {
1385         vswap();
1386         gv(float_type);
1387         vswap();
1388     }
1389     swapped = 0;
1390     /* swap the stack if needed so that t1 is the register and t2 is
1391        the memory reference */
1392     if (vtop[-1].r & VT_LVAL) {
1393         vswap();
1394         swapped = 1;
1395     }
1396     if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
1397         if (op >= TOK_ULT && op <= TOK_GT) {
1398             /* load on stack second operand */
1399             load(TREG_ST0, vtop);
1400             save_reg(TREG_RAX); /* eax is used by FP comparison code */
1401             if (op == TOK_GE || op == TOK_GT)
1402                 swapped = !swapped;
1403             else if (op == TOK_EQ || op == TOK_NE)
1404                 swapped = 0;
1405             if (swapped)
1406                 o(0xc9d9); /* fxch %st(1) */
1407             o(0xe9da); /* fucompp */
1408             o(0xe0df); /* fnstsw %ax */
1409             if (op == TOK_EQ) {
1410                 o(0x45e480); /* and $0x45, %ah */
1411                 o(0x40fC80); /* cmp $0x40, %ah */
1412             } else if (op == TOK_NE) {
1413                 o(0x45e480); /* and $0x45, %ah */
1414                 o(0x40f480); /* xor $0x40, %ah */
1415                 op = TOK_NE;
1416             } else if (op == TOK_GE || op == TOK_LE) {
1417                 o(0x05c4f6); /* test $0x05, %ah */
1418                 op = TOK_EQ;
1419             } else {
1420                 o(0x45c4f6); /* test $0x45, %ah */
1421                 op = TOK_EQ;
1422             }
1423             vtop--;
1424             vtop->r = VT_CMP;
1425             vtop->c.i = op;
1426         } else {
1427             /* no memory reference possible for long double operations */
1428             load(TREG_ST0, vtop);
1429             swapped = !swapped;
1430
1431             switch(op) {
1432             default:
1433             case '+':
1434                 a = 0;
1435                 break;
1436             case '-':
1437                 a = 4;
1438                 if (swapped)
1439                     a++;
1440                 break;
1441             case '*':
1442                 a = 1;
1443                 break;
1444             case '/':
1445                 a = 6;
1446                 if (swapped)
1447                     a++;
1448                 break;
1449             }
1450             ft = vtop->type.t;
1451             fc = vtop->c.ul;
1452             o(0xde); /* fxxxp %st, %st(1) */
1453             o(0xc1 + (a << 3));
1454             vtop--;
1455         }
1456     } else {
1457         if (op >= TOK_ULT && op <= TOK_GT) {
1458             /* if saved lvalue, then we must reload it */
1459             r = vtop->r;
1460             fc = vtop->c.ul;
1461             if ((r & VT_VALMASK) == VT_LLOCAL) {
1462                 SValue v1;
1463                 r = get_reg(RC_INT);
1464                 v1.type.t = VT_INT;
1465                 v1.r = VT_LOCAL | VT_LVAL;
1466                 v1.c.ul = fc;
1467                 load(r, &v1);
1468                 fc = 0;
1469             }
1470
1471             if (op == TOK_EQ || op == TOK_NE) {
1472                 swapped = 0;
1473             } else {
1474                 if (op == TOK_LE || op == TOK_LT)
1475                     swapped = !swapped;
1476                 if (op == TOK_LE || op == TOK_GE) {
1477                     op = 0x93; /* setae */
1478                 } else {
1479                     op = 0x97; /* seta */
1480                 }
1481             }
1482
1483             if (swapped) {
1484                 o(0x7e0ff3); /* movq */
1485                 gen_modrm(1, r, vtop->sym, fc);
1486
1487                 if ((vtop->type.t & VT_BTYPE) == VT_DOUBLE) {
1488                     o(0x66);
1489                 }
1490                 o(0x2e0f); /* ucomisd %xmm0, %xmm1 */
1491                 o(0xc8);
1492             } else {
1493                 if ((vtop->type.t & VT_BTYPE) == VT_DOUBLE) {
1494                     o(0x66);
1495                 }
1496                 o(0x2e0f); /* ucomisd */
1497                 gen_modrm(0, r, vtop->sym, fc);
1498             }
1499
1500             vtop--;
1501             vtop->r = VT_CMP;
1502             vtop->c.i = op;
1503         } else {
1504             /* no memory reference possible for long double operations */
1505             if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
1506                 load(TREG_XMM0, vtop);
1507                 swapped = !swapped;
1508             }
1509             switch(op) {
1510             default:
1511             case '+':
1512                 a = 0;
1513                 break;
1514             case '-':
1515                 a = 4;
1516                 break;
1517             case '*':
1518                 a = 1;
1519                 break;
1520             case '/':
1521                 a = 6;
1522                 break;
1523             }
1524             ft = vtop->type.t;
1525             fc = vtop->c.ul;
1526             if ((ft & VT_BTYPE) == VT_LDOUBLE) {
1527                 o(0xde); /* fxxxp %st, %st(1) */
1528                 o(0xc1 + (a << 3));
1529             } else {
1530                 /* if saved lvalue, then we must reload it */
1531                 r = vtop->r;
1532                 if ((r & VT_VALMASK) == VT_LLOCAL) {
1533                     SValue v1;
1534                     r = get_reg(RC_INT);
1535                     v1.type.t = VT_INT;
1536                     v1.r = VT_LOCAL | VT_LVAL;
1537                     v1.c.ul = fc;
1538                     load(r, &v1);
1539                     fc = 0;
1540                 }
1541                 if (swapped) {
1542                     /* movq %xmm0,%xmm1 */
1543                     o(0x7e0ff3);
1544                     o(0xc8);
1545                     load(TREG_XMM0, vtop);
1546                     /* subsd  %xmm1,%xmm0 (f2 0f 5c c1) */
1547                     if ((ft & VT_BTYPE) == VT_DOUBLE) {
1548                         o(0xf2);
1549                     } else {
1550                         o(0xf3);
1551                     }
1552                     o(0x0f);
1553                     o(0x58 + a);
1554                     o(0xc1);
1555                 } else {
1556                     if ((ft & VT_BTYPE) == VT_DOUBLE) {
1557                         o(0xf2);
1558                     } else {
1559                         o(0xf3);
1560                     }
1561                     o(0x0f);
1562                     o(0x58 + a);
1563                     gen_modrm(0, r, vtop->sym, fc);
1564                 }
1565             }
1566             vtop--;
1567         }
1568     }
1569 }
1570
1571 /* convert integers to fp 't' type. Must handle 'int', 'unsigned int'
1572    and 'long long' cases. */
1573 void gen_cvt_itof(int t)
1574 {
1575     if ((t & VT_BTYPE) == VT_LDOUBLE) {
1576         save_reg(TREG_ST0);
1577         gv(RC_INT);
1578         if ((vtop->type.t & VT_BTYPE) == VT_LLONG) {
1579             /* signed long long to float/double/long double (unsigned case
1580                is handled generically) */
1581             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
1582             o(0x242cdf); /* fildll (%rsp) */
1583             o(0x08c48348); /* add $8, %rsp */
1584         } else if ((vtop->type.t & (VT_BTYPE | VT_UNSIGNED)) ==
1585                    (VT_INT | VT_UNSIGNED)) {
1586             /* unsigned int to float/double/long double */
1587             o(0x6a); /* push $0 */
1588             g(0x00);
1589             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
1590             o(0x242cdf); /* fildll (%rsp) */
1591             o(0x10c48348); /* add $16, %rsp */
1592         } else {
1593             /* int to float/double/long double */
1594             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
1595             o(0x2404db); /* fildl (%rsp) */
1596             o(0x08c48348); /* add $8, %rsp */
1597         }
1598         vtop->r = TREG_ST0;
1599     } else {
1600         save_reg(TREG_XMM0);
1601         gv(RC_INT);
1602         o(0xf2 + ((t & VT_BTYPE) == VT_FLOAT));
1603         if ((vtop->type.t & (VT_BTYPE | VT_UNSIGNED)) ==
1604             (VT_INT | VT_UNSIGNED) ||
1605             (vtop->type.t & VT_BTYPE) == VT_LLONG) {
1606             o(0x48); /* REX */
1607         }
1608         o(0x2a0f);
1609         o(0xc0 + (vtop->r & VT_VALMASK)); /* cvtsi2sd */
1610         vtop->r = TREG_XMM0;
1611     }
1612 }
1613
1614 /* convert from one floating point type to another */
1615 void gen_cvt_ftof(int t)
1616 {
1617     int ft, bt, tbt;
1618
1619     ft = vtop->type.t;
1620     bt = ft & VT_BTYPE;
1621     tbt = t & VT_BTYPE;
1622
1623     if (bt == VT_FLOAT) {
1624         gv(RC_FLOAT);
1625         if (tbt == VT_DOUBLE) {
1626             o(0xc0140f); /* unpcklps */
1627             o(0xc05a0f); /* cvtps2pd */
1628         } else if (tbt == VT_LDOUBLE) {
1629             /* movss %xmm0,-0x10(%rsp) */
1630             o(0x44110ff3);
1631             o(0xf024);
1632             o(0xf02444d9); /* flds -0x10(%rsp) */
1633             vtop->r = TREG_ST0;
1634         }
1635     } else if (bt == VT_DOUBLE) {
1636         gv(RC_FLOAT);
1637         if (tbt == VT_FLOAT) {
1638             o(0xc0140f66); /* unpcklpd */
1639             o(0xc05a0f66); /* cvtpd2ps */
1640         } else if (tbt == VT_LDOUBLE) {
1641             /* movsd %xmm0,-0x10(%rsp) */
1642             o(0x44110ff2);
1643             o(0xf024);
1644             o(0xf02444dd); /* fldl -0x10(%rsp) */
1645             vtop->r = TREG_ST0;
1646         }
1647     } else {
1648         gv(RC_ST0);
1649         if (tbt == VT_DOUBLE) {
1650             o(0xf0245cdd); /* fstpl -0x10(%rsp) */
1651             /* movsd -0x10(%rsp),%xmm0 */
1652             o(0x44100ff2);
1653             o(0xf024);
1654             vtop->r = TREG_XMM0;
1655         } else if (tbt == VT_FLOAT) {
1656             o(0xf0245cd9); /* fstps -0x10(%rsp) */
1657             /* movss -0x10(%rsp),%xmm0 */
1658             o(0x44100ff3);
1659             o(0xf024);
1660             vtop->r = TREG_XMM0;
1661         }
1662     }
1663 }
1664
1665 /* convert fp to int 't' type */
1666 void gen_cvt_ftoi(int t)
1667 {
1668     int ft, bt, size, r;
1669     ft = vtop->type.t;
1670     bt = ft & VT_BTYPE;
1671     if (bt == VT_LDOUBLE) {
1672         gen_cvt_ftof(VT_DOUBLE);
1673         bt = VT_DOUBLE;
1674     }
1675
1676     gv(RC_FLOAT);
1677     if (t != VT_INT)
1678         size = 8;
1679     else
1680         size = 4;
1681
1682     r = get_reg(RC_INT);
1683     if (bt == VT_FLOAT) {
1684         o(0xf3);
1685     } else if (bt == VT_DOUBLE) {
1686         o(0xf2);
1687     } else {
1688         assert(0);
1689     }
1690     if (size == 8) {
1691         o(0x48 + REX_BASE(r));
1692     }
1693     o(0x2c0f); /* cvttss2si or cvttsd2si */
1694     o(0xc0 + (REG_VALUE(r) << 3));
1695     vtop->r = r;
1696 }
1697
1698 /* computed goto support */
1699 void ggoto(void)
1700 {
1701     gcall_or_jmp(1);
1702     vtop--;
1703 }
1704
1705 /* end of x86-64 code generator */
1706 /*************************************************************/
1707 #endif /* ! TARGET_DEFS_ONLY */
1708 /******************************************************/