x86_64-gen.c

   1 /*
   2  *  x86-64 code generator for TCC
   3  *
   4  *  Copyright (c) 2008 Shinichiro Hamaji
   5  *
   6  *  Based on i386-gen.c by Fabrice Bellard
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  21  */
  22
  23 #ifdef TARGET_DEFS_ONLY
  24
  25 /* number of available registers */
  26 #define NB_REGS         5
  27 #define NB_ASM_REGS     8
  28
  29 /* a register can belong to several classes. The classes must be
  30    sorted from more general to more precise (see gv2() code which does
  31    assumptions on it). */
  32 #define RC_INT     0x0001 /* generic integer register */
  33 #define RC_FLOAT   0x0002 /* generic float register */
  34 #define RC_RAX     0x0004
  35 #define RC_RCX     0x0008
  36 #define RC_RDX     0x0010
  37 #define RC_R8      0x0100
  38 #define RC_R9      0x0200
  39 #define RC_XMM0    0x0020
  40 #define RC_ST0     0x0040 /* only for long double */
  41 #define RC_IRET    RC_RAX /* function return: integer register */
  42 #define RC_LRET    RC_RDX /* function return: second integer register */
  43 #define RC_FRET    RC_XMM0 /* function return: float register */
  44
  45 /* pretty names for the registers */
  46 enum {
  47     TREG_RAX = 0,
  48     TREG_RCX = 1,
  49     TREG_RDX = 2,
  50     TREG_XMM0 = 3,
  51     TREG_ST0 = 4,
  52
  53     TREG_RSI = 6,
  54     TREG_RDI = 7,
  55     TREG_R8  = 8,
  56     TREG_R9  = 9,
  57
  58     TREG_R10 = 10,
  59     TREG_R11 = 11,
  60
  61     TREG_MEM = 0x10,
  62 };
  63
  64 #define REX_BASE(reg) (((reg) >> 3) & 1)
  65 #define REG_VALUE(reg) ((reg) & 7)
  66
  67 /* return registers for function */
  68 #define REG_IRET TREG_RAX /* single word int return register */
  69 #define REG_LRET TREG_RDX /* second word return register (for long long) */
  70 #define REG_FRET TREG_XMM0 /* float return register */
  71
  72 /* defined if function parameters must be evaluated in reverse order */
  73 #define INVERT_FUNC_PARAMS
  74
  75 /* pointer size, in bytes */
  76 #define PTR_SIZE 8
  77
  78 /* long double size and alignment, in bytes */
  79 #define LDOUBLE_SIZE  16
  80 #define LDOUBLE_ALIGN 8
  81 /* maximum alignment (for aligned attribute support) */
  82 #define MAX_ALIGN     8
  83
  84 ST_FUNC void gen_opl(int op);
  85 ST_FUNC void gen_le64(int64_t c);
  86
  87 /******************************************************/
  88 /* ELF defines */
  89
  90 #define EM_TCC_TARGET EM_X86_64
  91
  92 /* relocation type for 32 bit data relocation */
  93 #define R_DATA_32   R_X86_64_32
  94 #define R_DATA_PTR  R_X86_64_64
  95 #define R_JMP_SLOT  R_X86_64_JUMP_SLOT
  96 #define R_COPY      R_X86_64_COPY
  97
  98 #define ELF_START_ADDR 0x08048000
  99 #define ELF_PAGE_SIZE  0x1000
 100
 101 /******************************************************/
 102 #else /* ! TARGET_DEFS_ONLY */
 103 /******************************************************/
 104 #include "tcc.h"
 105 #include <assert.h>
 106
 107 ST_DATA const int reg_classes[NB_REGS] = {
 108     /* eax */ RC_INT | RC_RAX,
 109     /* ecx */ RC_INT | RC_RCX,
 110     /* edx */ RC_INT | RC_RDX,
 111     /* xmm0 */ RC_FLOAT | RC_XMM0,
 112     /* st0 */ RC_ST0,
 113 #if NB_REGS == 10
 114     0,
 115     0,
 116     0,
 117     RC_INT | RC_R8,
 118     RC_INT | RC_R9,
 119 #endif
 120 };
 121
 122 static unsigned long func_sub_sp_offset;
 123 static int func_ret_sub;
 124
 125 /* XXX: make it faster ? */
 126 void g(int c)
 127 {
 128     int ind1;
 129     ind1 = ind + 1;
 130     if (ind1 > cur_text_section->data_allocated)
 131         section_realloc(cur_text_section, ind1);
 132     cur_text_section->data[ind] = c;
 133     ind = ind1;
 134 }
 135
 136 void o(unsigned int c)
 137 {
 138     while (c) {
 139         g(c);
 140         c = c >> 8;
 141     }
 142 }
 143
 144 void gen_le16(int v)
 145 {
 146     g(v);
 147     g(v >> 8);
 148 }
 149
 150 void gen_le32(int c)
 151 {
 152     g(c);
 153     g(c >> 8);
 154     g(c >> 16);
 155     g(c >> 24);
 156 }
 157
 158 void gen_le64(int64_t c)
 159 {
 160     g(c);
 161     g(c >> 8);
 162     g(c >> 16);
 163     g(c >> 24);
 164     g(c >> 32);
 165     g(c >> 40);
 166     g(c >> 48);
 167     g(c >> 56);
 168 }
 169
 170 void orex(int ll, int r, int r2, int b)
 171 {
 172     if ((r & VT_VALMASK) >= VT_CONST)
 173         r = 0;
 174     if ((r2 & VT_VALMASK) >= VT_CONST)
 175         r2 = 0;
 176     if (ll || REX_BASE(r) || REX_BASE(r2))
 177         o(0x40 | REX_BASE(r) | (REX_BASE(r2) << 2) | (ll << 3));
 178     o(b);
 179 }
 180
 181 /* output a symbol and patch all calls to it */
 182 void gsym_addr(int t, int a)
 183 {
 184     int n, *ptr;
 185     while (t) {
 186         ptr = (int *)(cur_text_section->data + t);
 187         n = *ptr; /* next value */
 188         *ptr = a - t - 4;
 189         t = n;
 190     }
 191 }
 192
 193 void gsym(int t)
 194 {
 195     gsym_addr(t, ind);
 196 }
 197
 198 /* psym is used to put an instruction with a data field which is a
 199    reference to a symbol. It is in fact the same as oad ! */
 200 #define psym oad
 201
 202 static int is64_type(int t)
 203 {
 204     return ((t & VT_BTYPE) == VT_PTR ||
 205             (t & VT_BTYPE) == VT_FUNC ||
 206             (t & VT_BTYPE) == VT_LLONG);
 207 }
 208
 209 static int is_sse_float(int t) {
 210     int bt;
 211     bt = t & VT_BTYPE;
 212     return bt == VT_DOUBLE || bt == VT_FLOAT;
 213 }
 214
 215
 216 /* instruction + 4 bytes data. Return the address of the data */
 217 ST_FUNC int oad(int c, int s)
 218 {
 219     int ind1;
 220
 221     o(c);
 222     ind1 = ind + 4;
 223     if (ind1 > cur_text_section->data_allocated)
 224         section_realloc(cur_text_section, ind1);
 225     *(int *)(cur_text_section->data + ind) = s;
 226     s = ind;
 227     ind = ind1;
 228     return s;
 229 }
 230
 231 ST_FUNC void gen_addr32(int r, Sym *sym, int c)
 232 {
 233     if (r & VT_SYM)
 234         greloc(cur_text_section, sym, ind, R_X86_64_32);
 235     gen_le32(c);
 236 }
 237
 238 /* output constant with relocation if 'r & VT_SYM' is true */
 239 ST_FUNC void gen_addr64(int r, Sym *sym, int64_t c)
 240 {
 241     if (r & VT_SYM)
 242         greloc(cur_text_section, sym, ind, R_X86_64_64);
 243     gen_le64(c);
 244 }
 245
 246 /* output constant with relocation if 'r & VT_SYM' is true */
 247 ST_FUNC void gen_addrpc32(int r, Sym *sym, int c)
 248 {
 249     if (r & VT_SYM)
 250         greloc(cur_text_section, sym, ind, R_X86_64_PC32);
 251     gen_le32(c-4);
 252 }
 253
 254 /* output got address with relocation */
 255 static void gen_gotpcrel(int r, Sym *sym, int c)
 256 {
 257 #ifndef TCC_TARGET_PE
 258     Section *sr;
 259     ElfW(Rela) *rel;
 260     greloc(cur_text_section, sym, ind, R_X86_64_GOTPCREL);
 261     sr = cur_text_section->reloc;
 262     rel = (ElfW(Rela) *)(sr->data + sr->data_offset - sizeof(ElfW(Rela)));
 263     rel->r_addend = -4;
 264 #else
 265     printf("picpic: %s %x %x | %02x %02x %02x\n", get_tok_str(sym->v, NULL), c, r,
 266         cur_text_section->data[ind-3],
 267         cur_text_section->data[ind-2],
 268         cur_text_section->data[ind-1]
 269         );
 270     greloc(cur_text_section, sym, ind, R_X86_64_PC32);
 271 #endif
 272     gen_le32(0);
 273     if (c) {
 274         /* we use add c, %xxx for displacement */
 275         orex(1, r, 0, 0x81);
 276         o(0xc0 + REG_VALUE(r));
 277         gen_le32(c);
 278     }
 279 }
 280
 281 static void gen_modrm_impl(int op_reg, int r, Sym *sym, int c, int is_got)
 282 {
 283     op_reg = REG_VALUE(op_reg) << 3;
 284     if ((r & VT_VALMASK) == VT_CONST) {
 285         /* constant memory reference */
 286         o(0x05 | op_reg);
 287         if (is_got) {
 288             gen_gotpcrel(r, sym, c);
 289         } else {
 290             gen_addrpc32(r, sym, c);
 291         }
 292     } else if ((r & VT_VALMASK) == VT_LOCAL) {
 293         /* currently, we use only ebp as base */
 294         if (c == (char)c) {
 295             /* short reference */
 296             o(0x45 | op_reg);
 297             g(c);
 298         } else {
 299             oad(0x85 | op_reg, c);
 300         }
 301     } else if ((r & VT_VALMASK) >= TREG_MEM) {
 302         if (c) {
 303             g(0x80 | op_reg | REG_VALUE(r));
 304             gen_le32(c);
 305         } else {
 306             g(0x00 | op_reg | REG_VALUE(r));
 307         }
 308     } else {
 309         g(0x00 | op_reg | REG_VALUE(r));
 310     }
 311 }
 312
 313 /* generate a modrm reference. 'op_reg' contains the addtionnal 3
 314    opcode bits */
 315 static void gen_modrm(int op_reg, int r, Sym *sym, int c)
 316 {
 317     gen_modrm_impl(op_reg, r, sym, c, 0);
 318 }
 319
 320 /* generate a modrm reference. 'op_reg' contains the addtionnal 3
 321    opcode bits */
 322 static void gen_modrm64(int opcode, int op_reg, int r, Sym *sym, int c)
 323 {
 324     int is_got;
 325     is_got = (op_reg & TREG_MEM) && !(sym->type.t & VT_STATIC);
 326     orex(1, r, op_reg, opcode);
 327     gen_modrm_impl(op_reg, r, sym, c, is_got);
 328 }
 329
 330
 331 /* load 'r' from value 'sv' */
 332 void load(int r, SValue *sv)
 333 {
 334     int v, t, ft, fc, fr;
 335     SValue v1;
 336
 337 #ifdef TCC_TARGET_PE
 338     SValue v2;
 339     sv = pe_getimport(sv, &v2);
 340 #endif
 341
 342     fr = sv->r;
 343     ft = sv->type.t;
 344     fc = sv->c.ul;
 345
 346 #ifndef TCC_TARGET_PE
 347     /* we use indirect access via got */
 348     if ((fr & VT_VALMASK) == VT_CONST && (fr & VT_SYM) &&
 349         (fr & VT_LVAL) && !(sv->sym->type.t & VT_STATIC)) {
 350         /* use the result register as a temporal register */
 351         int tr = r | TREG_MEM;
 352         if (is_float(ft)) {
 353             /* we cannot use float registers as a temporal register */
 354             tr = get_reg(RC_INT) | TREG_MEM;
 355         }
 356         gen_modrm64(0x8b, tr, fr, sv->sym, 0);
 357
 358         /* load from the temporal register */
 359         fr = tr | VT_LVAL;
 360     }
 361 #endif
 362
 363     v = fr & VT_VALMASK;
 364     if (fr & VT_LVAL) {
 365         int b, ll;
 366         if (v == VT_LLOCAL) {
 367             v1.type.t = VT_PTR;
 368             v1.r = VT_LOCAL | VT_LVAL;
 369             v1.c.ul = fc;
 370             load(r, &v1);
 371             fr = r;
 372         }
 373         ll = 0;
 374         if ((ft & VT_BTYPE) == VT_FLOAT) {
 375             b = 0x6e0f66, r = 0; /* movd */
 376         } else if ((ft & VT_BTYPE) == VT_DOUBLE) {
 377             b = 0x7e0ff3, r = 0; /* movq */
 378         } else if ((ft & VT_BTYPE) == VT_LDOUBLE) {
 379             b = 0xdb, r = 5; /* fldt */
 380         } else if ((ft & VT_TYPE) == VT_BYTE) {
 381             b = 0xbe0f;   /* movsbl */
 382         } else if ((ft & VT_TYPE) == (VT_BYTE | VT_UNSIGNED)) {
 383             b = 0xb60f;   /* movzbl */
 384         } else if ((ft & VT_TYPE) == VT_SHORT) {
 385             b = 0xbf0f;   /* movswl */
 386         } else if ((ft & VT_TYPE) == (VT_SHORT | VT_UNSIGNED)) {
 387             b = 0xb70f;   /* movzwl */
 388         } else {
 389             ll = is64_type(ft);
 390             b = 0x8b;
 391         }
 392         if (ll) {
 393             gen_modrm64(b, r, fr, sv->sym, fc);
 394         } else {
 395             orex(ll, fr, r, b);
 396             gen_modrm(r, fr, sv->sym, fc);
 397         }
 398     } else {
 399         if (v == VT_CONST) {
 400             if (fr & VT_SYM) {
 401 #ifdef TCC_TARGET_PE
 402                 orex(1,0,r,0x8d);
 403                 o(0x05 + REG_VALUE(r) * 8); /* lea xx(%rip), r */
 404                 gen_addrpc32(fr, sv->sym, fc);
 405 #else
 406                 if (sv->sym->type.t & VT_STATIC) {
 407                     orex(1,0,r,0x8d);
 408                     o(0x05 + REG_VALUE(r) * 8); /* lea xx(%rip), r */
 409                     gen_addrpc32(fr, sv->sym, fc);
 410                 } else {
 411                     orex(1,0,r,0x8b);
 412                     o(0x05 + REG_VALUE(r) * 8); /* mov xx(%rip), r */
 413                     gen_gotpcrel(fr, sv->sym, fc);
 414                 }
 415 #endif
 416             } else if (is64_type(ft)) {
 417                 orex(1,r,0, 0xb8 + REG_VALUE(r)); /* mov $xx, r */
 418                 gen_le64(sv->c.ull);
 419             } else {
 420                 orex(0,r,0, 0xb8 + REG_VALUE(r)); /* mov $xx, r */
 421                 gen_le32(fc);
 422             }
 423         } else if (v == VT_LOCAL) {
 424             orex(1,0,r,0x8d); /* lea xxx(%ebp), r */
 425             gen_modrm(r, VT_LOCAL, sv->sym, fc);
 426         } else if (v == VT_CMP) {
 427             orex(0,r,0,0);
 428             oad(0xb8 + REG_VALUE(r), 0); /* mov $0, r */
 429             orex(0,r,0, 0x0f); /* setxx %br */
 430             o(fc);
 431             o(0xc0 + REG_VALUE(r));
 432         } else if (v == VT_JMP || v == VT_JMPI) {
 433             t = v & 1;
 434             orex(0,r,0,0);
 435             oad(0xb8 + REG_VALUE(r), t); /* mov $1, r */
 436             o(0x05eb + (REX_BASE(r) << 8)); /* jmp after */
 437             gsym(fc);
 438             orex(0,r,0,0);
 439             oad(0xb8 + REG_VALUE(r), t ^ 1); /* mov $0, r */
 440         } else if (v != r) {
 441             if (r == TREG_XMM0) {
 442                 assert(v == TREG_ST0);
 443                 /* gen_cvt_ftof(VT_DOUBLE); */
 444                 o(0xf0245cdd); /* fstpl -0x10(%rsp) */
 445                 /* movsd -0x10(%rsp),%xmm0 */
 446                 o(0x44100ff2);
 447                 o(0xf024);
 448             } else if (r == TREG_ST0) {
 449                 assert(v == TREG_XMM0);
 450                 /* gen_cvt_ftof(VT_LDOUBLE); */
 451                 /* movsd %xmm0,-0x10(%rsp) */
 452                 o(0x44110ff2);
 453                 o(0xf024);
 454                 o(0xf02444dd); /* fldl -0x10(%rsp) */
 455             } else {
 456                 orex(1,r,v, 0x89);
 457                 o(0xc0 + REG_VALUE(r) + REG_VALUE(v) * 8); /* mov v, r */
 458             }
 459         }
 460     }
 461 }
 462
 463 /* store register 'r' in lvalue 'v' */
 464 void store(int r, SValue *v)
 465 {
 466     int fr, bt, ft, fc;
 467     int op64 = 0;
 468     /* store the REX prefix in this variable when PIC is enabled */
 469     int pic = 0;
 470
 471 #ifdef TCC_TARGET_PE
 472     SValue v2;
 473     v = pe_getimport(v, &v2);
 474 #endif
 475
 476     ft = v->type.t;
 477     fc = v->c.ul;
 478     fr = v->r & VT_VALMASK;
 479     bt = ft & VT_BTYPE;
 480
 481 #ifndef TCC_TARGET_PE
 482     /* we need to access the variable via got */
 483     if (fr == VT_CONST && (v->r & VT_SYM)) {
 484         /* mov xx(%rip), %r11 */
 485         o(0x1d8b4c);
 486         gen_gotpcrel(TREG_R11, v->sym, v->c.ul);
 487         pic = is64_type(bt) ? 0x49 : 0x41;
 488     }
 489 #endif
 490
 491     /* XXX: incorrect if float reg to reg */
 492     if (bt == VT_FLOAT) {
 493         o(0x66);
 494         o(pic);
 495         o(0x7e0f); /* movd */
 496         r = 0;
 497     } else if (bt == VT_DOUBLE) {
 498         o(0x66);
 499         o(pic);
 500         o(0xd60f); /* movq */
 501         r = 0;
 502     } else if (bt == VT_LDOUBLE) {
 503         o(0xc0d9); /* fld %st(0) */
 504         o(pic);
 505         o(0xdb); /* fstpt */
 506         r = 7;
 507     } else {
 508         if (bt == VT_SHORT)
 509             o(0x66);
 510         o(pic);
 511         if (bt == VT_BYTE || bt == VT_BOOL)
 512             orex(0, 0, r, 0x88);
 513         else if (is64_type(bt))
 514             op64 = 0x89;
 515         else
 516             orex(0, 0, r, 0x89);
 517     }
 518     if (pic) {
 519         /* xxx r, (%r11) where xxx is mov, movq, fld, or etc */
 520         if (op64)
 521             o(op64);
 522         o(3 + (r << 3));
 523     } else if (op64) {
 524         if (fr == VT_CONST || fr == VT_LOCAL || (v->r & VT_LVAL)) {
 525             gen_modrm64(op64, r, v->r, v->sym, fc);
 526         } else if (fr != r) {
 527             /* XXX: don't we really come here? */
 528             abort();
 529             o(0xc0 + fr + r * 8); /* mov r, fr */
 530         }
 531     } else {
 532         if (fr == VT_CONST || fr == VT_LOCAL || (v->r & VT_LVAL)) {
 533             gen_modrm(r, v->r, v->sym, fc);
 534         } else if (fr != r) {
 535             /* XXX: don't we really come here? */
 536             abort();
 537             o(0xc0 + fr + r * 8); /* mov r, fr */
 538         }
 539     }
 540 }
 541
 542 /* 'is_jmp' is '1' if it is a jump */
 543 static void gcall_or_jmp(int is_jmp)
 544 {
 545     int r;
 546     if ((vtop->r & (VT_VALMASK | VT_LVAL)) == VT_CONST) {
 547         /* constant case */
 548         if (vtop->r & VT_SYM) {
 549             /* relocation case */
 550             greloc(cur_text_section, vtop->sym,
 551                    ind + 1, R_X86_64_PC32);
 552         } else {
 553             /* put an empty PC32 relocation */
 554             put_elf_reloc(symtab_section, cur_text_section,
 555                           ind + 1, R_X86_64_PC32, 0);
 556         }
 557         oad(0xe8 + is_jmp, vtop->c.ul - 4); /* call/jmp im */
 558     } else {
 559         /* otherwise, indirect call */
 560         r = TREG_R11;
 561         load(r, vtop);
 562         o(0x41); /* REX */
 563         o(0xff); /* call/jmp *r */
 564         o(0xd0 + REG_VALUE(r) + (is_jmp << 4));
 565     }
 566 }
 567
 568 #ifdef TCC_TARGET_PE
 569
 570 #define REGN 4
 571 static const uint8_t arg_regs[] = {
 572     TREG_RCX, TREG_RDX, TREG_R8, TREG_R9
 573 };
 574
 575 static int func_scratch;
 576
 577 /* Generate function call. The function address is pushed first, then
 578    all the parameters in call order. This functions pops all the
 579    parameters and the function address. */
 580
 581 void gen_offs_sp(int b, int r, int d)
 582 {
 583     orex(1,0,r & 0x100 ? 0 : r, b);
 584     if (d == (char)d) {
 585         o(0x2444 | (REG_VALUE(r) << 3));
 586         g(d);
 587     } else {
 588         o(0x2484 | (REG_VALUE(r) << 3));
 589         gen_le32(d);
 590     }
 591 }
 592
 593 void gfunc_call(int nb_args)
 594 {
 595     int size, align, r, args_size, i, d, j, bt;
 596     int nb_reg_args, gen_reg;
 597
 598     /* calculate the number of integer/float arguments */
 599     nb_reg_args = 0;
 600     for(i = 0; i < nb_args; i++) {
 601         bt = (vtop[-i].type.t & VT_BTYPE);
 602         if (bt != VT_STRUCT && bt != VT_LDOUBLE)
 603             nb_reg_args++;
 604     }
 605
 606     args_size = (nb_reg_args < REGN ? REGN : nb_reg_args) * PTR_SIZE;
 607
 608     /* for struct arguments, we need to call memcpy and the function
 609        call breaks register passing arguments we are preparing.
 610        So, we process arguments which will be passed by stack first. */
 611     for(i = 0; i < nb_args; i++) {
 612         SValue *sv = &vtop[-i];
 613         bt = (sv->type.t & VT_BTYPE);
 614         if (bt == VT_STRUCT) {
 615             size = type_size(&sv->type, &align);
 616             /* align to stack align size */
 617             size = (size + 15) & ~16;
 618             /* generate structure store */
 619             r = get_reg(RC_INT);
 620             gen_offs_sp(0x8d, r, args_size);
 621             args_size += size;
 622
 623             /* generate memcpy call */
 624             vset(&sv->type, r | VT_LVAL, 0);
 625             vpushv(sv);
 626             vstore();
 627             --vtop;
 628
 629         } else if (bt == VT_LDOUBLE) {
 630
 631             gv(RC_ST0);
 632             gen_offs_sp(0xdb, 0x107, args_size);
 633             args_size += 16;
 634
 635         }
 636     }
 637
 638     if (func_scratch < args_size)
 639         func_scratch = args_size;
 640
 641     for (i = 0; i < REGN; ++i)
 642         save_reg(arg_regs[i]);
 643
 644     gen_reg = nb_reg_args;
 645     for(i = 0; i < nb_args; i++) {
 646         bt = (vtop->type.t & VT_BTYPE);
 647         if (bt == VT_STRUCT || bt == VT_LDOUBLE) {
 648             ; /* done */
 649         } else if (is_sse_float(vtop->type.t)) {
 650             gv(RC_FLOAT); /* only one float register */
 651             j = --gen_reg;
 652             if (j >= REGN) {
 653                 /* movq %xmm0, j*8(%rsp) */
 654                 gen_offs_sp(0xd60f66, 0x100, j*8);
 655             } else {
 656                 /* movaps %xmm0, %xmmN */
 657                 o(0x280f);
 658                 o(0xc0 + (j << 3));
 659                 d = arg_regs[j];
 660                 /* mov %xmm0, %rxx */
 661                 o(0x66);
 662                 orex(1,d,0, 0x7e0f);
 663                 o(0xc0 + REG_VALUE(d));
 664             }
 665         } else {
 666             j = --gen_reg;
 667             if (j >= REGN) {
 668                 r = gv(RC_INT);
 669                 gen_offs_sp(0x89, r, j*8);
 670             } else {
 671                 d = arg_regs[j];
 672                 if (d < NB_REGS) {
 673                     gv(reg_classes[d] & ~RC_INT);
 674                 } else {
 675                     r = gv(RC_INT);
 676                     if (d != r) {
 677                         orex(1,d,r, 0x89);
 678                         o(0xc0 + REG_VALUE(d) + REG_VALUE(r) * 8);
 679                     }
 680                 }
 681
 682             }
 683         }
 684         vtop--;
 685     }
 686     save_regs(0);
 687     gcall_or_jmp(0);
 688     vtop--;
 689 }
 690
 691
 692 #define FUNC_PROLOG_SIZE 11
 693
 694 /* generate function prolog of type 't' */
 695 void gfunc_prolog(CType *func_type)
 696 {
 697     int addr, align, size, reg_param_index, bt;
 698     Sym *sym;
 699     CType *type;
 700
 701     func_ret_sub = 0;
 702     func_scratch = 0;
 703     loc = 0;
 704
 705     addr = PTR_SIZE * 2;
 706     ind += FUNC_PROLOG_SIZE;
 707     func_sub_sp_offset = ind;
 708     reg_param_index = 0;
 709
 710     sym = func_type->ref;
 711
 712     /* if the function returns a structure, then add an
 713        implicit pointer parameter */
 714     func_vt = sym->type;
 715     if ((func_vt.t & VT_BTYPE) == VT_STRUCT) {
 716         gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 717         reg_param_index++;
 718         addr += PTR_SIZE;
 719     }
 720
 721     /* define parameters */
 722     while ((sym = sym->next) != NULL) {
 723         type = &sym->type;
 724         bt = type->t & VT_BTYPE;
 725         if (bt == VT_STRUCT || bt == VT_LDOUBLE)
 726             continue;
 727         if (reg_param_index < REGN) {
 728             /* save arguments passed by register */
 729             gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 730         }
 731         sym_push(sym->v & ~SYM_FIELD, type, VT_LOCAL | VT_LVAL, addr);
 732         reg_param_index++;
 733         addr += PTR_SIZE;
 734     }
 735
 736     while (reg_param_index < REGN) {
 737         if (func_type->ref->c == FUNC_ELLIPSIS)
 738             gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 739         reg_param_index++;
 740         addr += PTR_SIZE;
 741     }
 742
 743     sym = func_type->ref;
 744     while ((sym = sym->next) != NULL) {
 745         type = &sym->type;
 746         bt = type->t & VT_BTYPE;
 747         if (bt == VT_STRUCT || bt == VT_LDOUBLE) {
 748             size = type_size(type, &align);
 749             size = (size + 15) & -16;
 750             sym_push(sym->v & ~SYM_FIELD, type, VT_LOCAL | VT_LVAL, addr);
 751             addr += size;
 752         }
 753     }
 754 }
 755
 756 /* generate function epilog */
 757 void gfunc_epilog(void)
 758 {
 759     int v, saved_ind;
 760
 761     o(0xc9); /* leave */
 762     if (func_ret_sub == 0) {
 763         o(0xc3); /* ret */
 764     } else {
 765         o(0xc2); /* ret n */
 766         g(func_ret_sub);
 767         g(func_ret_sub >> 8);
 768     }
 769
 770     saved_ind = ind;
 771     ind = func_sub_sp_offset - FUNC_PROLOG_SIZE;
 772     /* align local size to word & save local variables */
 773     v = (func_scratch + -loc + 15) & -16;
 774
 775     pe_add_unwind_data(ind, saved_ind, v);
 776
 777     if (v >= 4096) {
 778         Sym *sym = external_global_sym(TOK___chkstk, &func_old_type, 0);
 779         oad(0xb8, v); /* mov stacksize, %eax */
 780         oad(0xe8, -4); /* call __chkstk, (does the stackframe too) */
 781         greloc(cur_text_section, sym, ind-4, R_X86_64_PC32);
 782         o(0x90); /* fill for FUNC_PROLOG_SIZE = 11 bytes */
 783     } else {
 784         o(0xe5894855);  /* push %rbp, mov %rsp, %rbp */
 785         o(0xec8148);  /* sub rsp, stacksize */
 786         gen_le32(v);
 787     }
 788     ind = saved_ind;
 789 }
 790
 791 #else
 792
 793 static void gadd_sp(int val)
 794 {
 795     if (val == (char)val) {
 796         o(0xc48348);
 797         g(val);
 798     } else {
 799         oad(0xc48148, val); /* add $xxx, %rsp */
 800     }
 801 }
 802
 803 #define REGN 6
 804 static const uint8_t arg_regs[REGN] = {
 805     TREG_RDI, TREG_RSI, TREG_RDX, TREG_RCX, TREG_R8, TREG_R9
 806 };
 807
 808 /* Generate function call. The function address is pushed first, then
 809    all the parameters in call order. This functions pops all the
 810    parameters and the function address. */
 811 void gfunc_call(int nb_args)
 812 {
 813     int size, align, r, args_size, i;
 814     SValue *orig_vtop;
 815     int nb_reg_args = 0;
 816     int nb_sse_args = 0;
 817     int sse_reg, gen_reg;
 818
 819     /* calculate the number of integer/float arguments */
 820     args_size = 0;
 821     for(i = 0; i < nb_args; i++) {
 822         if ((vtop[-i].type.t & VT_BTYPE) == VT_STRUCT) {
 823             args_size += type_size(&vtop->type, &align);
 824         } else if ((vtop[-i].type.t & VT_BTYPE) == VT_LDOUBLE) {
 825             args_size += 16;
 826         } else if (is_sse_float(vtop[-i].type.t)) {
 827             nb_sse_args++;
 828             if (nb_sse_args > 8) args_size += 8;
 829         } else {
 830             nb_reg_args++;
 831             if (nb_reg_args > REGN) args_size += 8;
 832         }
 833     }
 834
 835     /* for struct arguments, we need to call memcpy and the function
 836        call breaks register passing arguments we are preparing.
 837        So, we process arguments which will be passed by stack first. */
 838     orig_vtop = vtop;
 839     gen_reg = nb_reg_args;
 840     sse_reg = nb_sse_args;
 841
 842     /* adjust stack to align SSE boundary */
 843     if (args_size &= 8) {
 844         o(0x50); /* push $rax */
 845     }
 846     for(i = 0; i < nb_args; i++) {
 847         if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) {
 848             size = type_size(&vtop->type, &align);
 849             /* align to stack align size */
 850             size = (size + 3) & ~3;
 851             /* allocate the necessary size on stack */
 852             o(0x48);
 853             oad(0xec81, size); /* sub $xxx, %rsp */
 854             /* generate structure store */
 855             r = get_reg(RC_INT);
 856             orex(1, r, 0, 0x89); /* mov %rsp, r */
 857             o(0xe0 + REG_VALUE(r));
 858             {
 859                 /* following code breaks vtop[1] */
 860                 SValue tmp = vtop[1];
 861                 vset(&vtop->type, r | VT_LVAL, 0);
 862                 vswap();
 863                 vstore();
 864                 vtop[1] = tmp;
 865             }
 866             args_size += size;
 867         } else if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
 868             gv(RC_ST0);
 869             size = LDOUBLE_SIZE;
 870             oad(0xec8148, size); /* sub $xxx, %rsp */
 871             o(0x7cdb); /* fstpt 0(%rsp) */
 872             g(0x24);
 873             g(0x00);
 874             args_size += size;
 875         } else if (is_sse_float(vtop->type.t)) {
 876             int j = --sse_reg;
 877             if (j >= 8) {
 878                 gv(RC_FLOAT);
 879                 o(0x50); /* push $rax */
 880                 /* movq %xmm0, (%rsp) */
 881                 o(0x04d60f66);
 882                 o(0x24);
 883                 args_size += 8;
 884             }
 885         } else {
 886             int j = --gen_reg;
 887             /* simple type */
 888             /* XXX: implicit cast ? */
 889             if (j >= REGN) {
 890                 r = gv(RC_INT);
 891                 orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */
 892                 args_size += 8;
 893             }
 894         }
 895         vtop--;
 896     }
 897     vtop = orig_vtop;
 898
 899     save_regs(0); /* save used temporary registers */
 900
 901     /* then, we prepare register passing arguments.
 902        Note that we cannot set RDX and RCX in this loop because gv()
 903        may break these temporary registers. Let's use R10 and R11
 904        instead of them */
 905     gen_reg = nb_reg_args;
 906     sse_reg = nb_sse_args;
 907     for(i = 0; i < nb_args; i++) {
 908         if ((vtop->type.t & VT_BTYPE) == VT_STRUCT ||
 909             (vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
 910         } else if (is_sse_float(vtop->type.t)) {
 911             int j = --sse_reg;
 912             if (j < 8) {
 913                 gv(RC_FLOAT); /* only one float register */
 914                 /* movaps %xmm0, %xmmN */
 915                 o(0x280f);
 916                 o(0xc0 + (sse_reg << 3));
 917             }
 918         } else {
 919             int j = --gen_reg;
 920             /* simple type */
 921             /* XXX: implicit cast ? */
 922             if (j < REGN) {
 923                 int d = arg_regs[j];
 924                 r = gv(RC_INT);
 925                 if (j == 2 || j == 3)
 926                     /* j=2: r10, j=3: r11 */
 927                     d = j + 8;
 928                 orex(1,d,r,0x89); /* mov */
 929                 o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d));
 930             }
 931         }
 932         vtop--;
 933     }
 934
 935     /* Copy R10 and R11 into RDX and RCX, respectively */
 936     if (nb_reg_args > 2) {
 937         o(0xd2894c); /* mov %r10, %rdx */
 938         if (nb_reg_args > 3) {
 939             o(0xd9894c); /* mov %r11, %rcx */
 940         }
 941     }
 942
 943     oad(0xb8, nb_sse_args < 8 ? nb_sse_args : 8); /* mov nb_sse_args, %eax */
 944     gcall_or_jmp(0);
 945     if (args_size)
 946         gadd_sp(args_size);
 947     vtop--;
 948 }
 949
 950
 951 #define FUNC_PROLOG_SIZE 11
 952
 953 static void push_arg_reg(int i) {
 954     loc -= 8;
 955     gen_modrm64(0x89, arg_regs[i], VT_LOCAL, NULL, loc);
 956 }
 957
 958 /* generate function prolog of type 't' */
 959 void gfunc_prolog(CType *func_type)
 960 {
 961     int i, addr, align, size;
 962     int param_index, param_addr, reg_param_index, sse_param_index;
 963     Sym *sym;
 964     CType *type;
 965
 966     sym = func_type->ref;
 967     addr = PTR_SIZE * 2;
 968     loc = 0;
 969     ind += FUNC_PROLOG_SIZE;
 970     func_sub_sp_offset = ind;
 971     func_ret_sub = 0;
 972
 973     if (func_type->ref->c == FUNC_ELLIPSIS) {
 974         int seen_reg_num, seen_sse_num, seen_stack_size;
 975         seen_reg_num = seen_sse_num = 0;
 976         /* frame pointer and return address */
 977         seen_stack_size = PTR_SIZE * 2;
 978         /* count the number of seen parameters */
 979         sym = func_type->ref;
 980         while ((sym = sym->next) != NULL) {
 981             type = &sym->type;
 982             if (is_sse_float(type->t)) {
 983                 if (seen_sse_num < 8) {
 984                     seen_sse_num++;
 985                 } else {
 986                     seen_stack_size += 8;
 987                 }
 988             } else if ((type->t & VT_BTYPE) == VT_STRUCT) {
 989                 size = type_size(type, &align);
 990                 size = (size + 3) & ~3;
 991                 seen_stack_size += size;
 992             } else if ((type->t & VT_BTYPE) == VT_LDOUBLE) {
 993                 seen_stack_size += LDOUBLE_SIZE;
 994             } else {
 995                 if (seen_reg_num < REGN) {
 996                     seen_reg_num++;
 997                 } else {
 998                     seen_stack_size += 8;
 999                 }
1000             }
1001         }
1002
1003         loc -= 16;
1004         /* movl $0x????????, -0x10(%rbp) */
1005         o(0xf045c7);
1006         gen_le32(seen_reg_num * 8);
1007         /* movl $0x????????, -0xc(%rbp) */
1008         o(0xf445c7);
1009         gen_le32(seen_sse_num * 16 + 48);
1010         /* movl $0x????????, -0x8(%rbp) */
1011         o(0xf845c7);
1012         gen_le32(seen_stack_size);
1013
1014         /* save all register passing arguments */
1015         for (i = 0; i < 8; i++) {
1016             loc -= 16;
1017             o(0xd60f66); /* movq */
1018             gen_modrm(7 - i, VT_LOCAL, NULL, loc);
1019             /* movq $0, loc+8(%rbp) */
1020             o(0x85c748);
1021             gen_le32(loc + 8);
1022             gen_le32(0);
1023         }
1024         for (i = 0; i < REGN; i++) {
1025             push_arg_reg(REGN-1-i);
1026         }
1027     }
1028
1029     sym = func_type->ref;
1030     param_index = 0;
1031     reg_param_index = 0;
1032     sse_param_index = 0;
1033
1034     /* if the function returns a structure, then add an
1035        implicit pointer parameter */
1036     func_vt = sym->type;
1037     if ((func_vt.t & VT_BTYPE) == VT_STRUCT) {
1038         push_arg_reg(reg_param_index);
1039         param_addr = loc;
1040
1041         func_vc = loc;
1042         param_index++;
1043         reg_param_index++;
1044     }
1045     /* define parameters */
1046     while ((sym = sym->next) != NULL) {
1047         type = &sym->type;
1048         size = type_size(type, &align);
1049         size = (size + 3) & ~3;
1050         if (is_sse_float(type->t)) {
1051             if (sse_param_index < 8) {
1052                 /* save arguments passed by register */
1053                 loc -= 8;
1054                 o(0xd60f66); /* movq */
1055                 gen_modrm(sse_param_index, VT_LOCAL, NULL, loc);
1056                 param_addr = loc;
1057             } else {
1058                 param_addr = addr;
1059                 addr += size;
1060             }
1061             sse_param_index++;
1062
1063         } else if ((type->t & VT_BTYPE) == VT_STRUCT ||
1064                    (type->t & VT_BTYPE) == VT_LDOUBLE) {
1065             param_addr = addr;
1066             addr += size;
1067         } else {
1068             if (reg_param_index < REGN) {
1069                 /* save arguments passed by register */
1070                 push_arg_reg(reg_param_index);
1071                 param_addr = loc;
1072             } else {
1073                 param_addr = addr;
1074                 addr += 8;
1075             }
1076             reg_param_index++;
1077         }
1078         sym_push(sym->v & ~SYM_FIELD, type,
1079                  VT_LOCAL | VT_LVAL, param_addr);
1080         param_index++;
1081     }
1082 }
1083
1084 /* generate function epilog */
1085 void gfunc_epilog(void)
1086 {
1087     int v, saved_ind;
1088
1089     o(0xc9); /* leave */
1090     if (func_ret_sub == 0) {
1091         o(0xc3); /* ret */
1092     } else {
1093         o(0xc2); /* ret n */
1094         g(func_ret_sub);
1095         g(func_ret_sub >> 8);
1096     }
1097     /* align local size to word & save local variables */
1098     v = (-loc + 15) & -16;
1099     saved_ind = ind;
1100     ind = func_sub_sp_offset - FUNC_PROLOG_SIZE;
1101     o(0xe5894855);  /* push %rbp, mov %rsp, %rbp */
1102     o(0xec8148);  /* sub rsp, stacksize */
1103     gen_le32(v);
1104     ind = saved_ind;
1105 }
1106
1107 #endif /* not PE */
1108
1109 /* generate a jump to a label */
1110 int gjmp(int t)
1111 {
1112     return psym(0xe9, t);
1113 }
1114
1115 /* generate a jump to a fixed address */
1116 void gjmp_addr(int a)
1117 {
1118     int r;
1119     r = a - ind - 2;
1120     if (r == (char)r) {
1121         g(0xeb);
1122         g(r);
1123     } else {
1124         oad(0xe9, a - ind - 5);
1125     }
1126 }
1127
1128 /* generate a test. set 'inv' to invert test. Stack entry is popped */
1129 int gtst(int inv, int t)
1130 {
1131     int v, *p;
1132
1133     v = vtop->r & VT_VALMASK;
1134     if (v == VT_CMP) {
1135         /* fast case : can jump directly since flags are set */
1136         g(0x0f);
1137         t = psym((vtop->c.i - 16) ^ inv, t);
1138     } else if (v == VT_JMP || v == VT_JMPI) {
1139         /* && or || optimization */
1140         if ((v & 1) == inv) {
1141             /* insert vtop->c jump list in t */
1142             p = &vtop->c.i;
1143             while (*p != 0)
1144                 p = (int *)(cur_text_section->data + *p);
1145             *p = t;
1146             t = vtop->c.i;
1147         } else {
1148             t = gjmp(t);
1149             gsym(vtop->c.i);
1150         }
1151     } else {
1152         if (is_float(vtop->type.t) ||
1153             (vtop->type.t & VT_BTYPE) == VT_LLONG) {
1154             vpushi(0);
1155             gen_op(TOK_NE);
1156         }
1157         if ((vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST) {
1158             /* constant jmp optimization */
1159             if ((vtop->c.i != 0) != inv)
1160                 t = gjmp(t);
1161         } else {
1162             v = gv(RC_INT);
1163             orex(0,v,v,0x85);
1164             o(0xc0 + REG_VALUE(v) * 9);
1165             g(0x0f);
1166             t = psym(0x85 ^ inv, t);
1167         }
1168     }
1169     vtop--;
1170     return t;
1171 }
1172
1173 /* generate an integer binary operation */
1174 void gen_opi(int op)
1175 {
1176     int r, fr, opc, c;
1177     int ll, uu, cc;
1178
1179     ll = is64_type(vtop[-1].type.t);
1180     uu = (vtop[-1].type.t & VT_UNSIGNED) != 0;
1181     cc = (vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST;
1182
1183     switch(op) {
1184     case '+':
1185     case TOK_ADDC1: /* add with carry generation */
1186         opc = 0;
1187     gen_op8:
1188         if (cc && (!ll || (int)vtop->c.ll == vtop->c.ll)) {
1189             /* constant case */
1190             vswap();
1191             r = gv(RC_INT);
1192             vswap();
1193             c = vtop->c.i;
1194             if (c == (char)c) {
1195                 /* XXX: generate inc and dec for smaller code ? */
1196                 orex(ll, r, 0, 0x83);
1197                 o(0xc0 | (opc << 3) | REG_VALUE(r));
1198                 g(c);
1199             } else {
1200                 orex(ll, r, 0, 0x81);
1201                 oad(0xc0 | (opc << 3) | REG_VALUE(r), c);
1202             }
1203         } else {
1204             gv2(RC_INT, RC_INT);
1205             r = vtop[-1].r;
1206             fr = vtop[0].r;
1207             orex(ll, r, fr, (opc << 3) | 0x01);
1208             o(0xc0 + REG_VALUE(r) + REG_VALUE(fr) * 8);
1209         }
1210         vtop--;
1211         if (op >= TOK_ULT && op <= TOK_GT) {
1212             vtop->r = VT_CMP;
1213             vtop->c.i = op;
1214         }
1215         break;
1216     case '-':
1217     case TOK_SUBC1: /* sub with carry generation */
1218         opc = 5;
1219         goto gen_op8;
1220     case TOK_ADDC2: /* add with carry use */
1221         opc = 2;
1222         goto gen_op8;
1223     case TOK_SUBC2: /* sub with carry use */
1224         opc = 3;
1225         goto gen_op8;
1226     case '&':
1227         opc = 4;
1228         goto gen_op8;
1229     case '^':
1230         opc = 6;
1231         goto gen_op8;
1232     case '|':
1233         opc = 1;
1234         goto gen_op8;
1235     case '*':
1236         gv2(RC_INT, RC_INT);
1237         r = vtop[-1].r;
1238         fr = vtop[0].r;
1239         orex(ll, fr, r, 0xaf0f); /* imul fr, r */
1240         o(0xc0 + REG_VALUE(fr) + REG_VALUE(r) * 8);
1241         vtop--;
1242         break;
1243     case TOK_SHL:
1244         opc = 4;
1245         goto gen_shift;
1246     case TOK_SHR:
1247         opc = 5;
1248         goto gen_shift;
1249     case TOK_SAR:
1250         opc = 7;
1251     gen_shift:
1252         opc = 0xc0 | (opc << 3);
1253         if (cc) {
1254             /* constant case */
1255             vswap();
1256             r = gv(RC_INT);
1257             vswap();
1258             orex(ll, r, 0, 0xc1); /* shl/shr/sar $xxx, r */
1259             o(opc | REG_VALUE(r));
1260             g(vtop->c.i & (ll ? 63 : 31));
1261         } else {
1262             /* we generate the shift in ecx */
1263             gv2(RC_INT, RC_RCX);
1264             r = vtop[-1].r;
1265             orex(ll, r, 0, 0xd3); /* shl/shr/sar %cl, r */
1266             o(opc | REG_VALUE(r));
1267         }
1268         vtop--;
1269         break;
1270     case TOK_UDIV:
1271     case TOK_UMOD:
1272         uu = 1;
1273         goto divmod;
1274     case '/':
1275     case '%':
1276     case TOK_PDIV:
1277         uu = 0;
1278     divmod:
1279         /* first operand must be in eax */
1280         /* XXX: need better constraint for second operand */
1281         gv2(RC_RAX, RC_RCX);
1282         r = vtop[-1].r;
1283         fr = vtop[0].r;
1284         vtop--;
1285         save_reg(TREG_RDX);
1286         orex(ll, 0, 0, uu ? 0xd231 : 0x99); /* xor %edx,%edx : cqto */
1287         orex(ll, fr, 0, 0xf7); /* div fr, %eax */
1288         o((uu ? 0xf0 : 0xf8) + REG_VALUE(fr));
1289         if (op == '%' || op == TOK_UMOD)
1290             r = TREG_RDX;
1291         else
1292             r = TREG_RAX;
1293         vtop->r = r;
1294         break;
1295     default:
1296         opc = 7;
1297         goto gen_op8;
1298     }
1299 }
1300
1301 void gen_opl(int op)
1302 {
1303     gen_opi(op);
1304 }
1305
1306 /* generate a floating point operation 'v = t1 op t2' instruction. The
1307    two operands are guaranted to have the same floating point type */
1308 /* XXX: need to use ST1 too */
1309 void gen_opf(int op)
1310 {
1311     int a, ft, fc, swapped, r;
1312     int float_type =
1313         (vtop->type.t & VT_BTYPE) == VT_LDOUBLE ? RC_ST0 : RC_FLOAT;
1314
1315     /* convert constants to memory references */
1316     if ((vtop[-1].r & (VT_VALMASK | VT_LVAL)) == VT_CONST) {
1317         vswap();
1318         gv(float_type);
1319         vswap();
1320     }
1321     if ((vtop[0].r & (VT_VALMASK | VT_LVAL)) == VT_CONST)
1322         gv(float_type);
1323
1324     /* must put at least one value in the floating point register */
1325     if ((vtop[-1].r & VT_LVAL) &&
1326         (vtop[0].r & VT_LVAL)) {
1327         vswap();
1328         gv(float_type);
1329         vswap();
1330     }
1331     swapped = 0;
1332     /* swap the stack if needed so that t1 is the register and t2 is
1333        the memory reference */
1334     if (vtop[-1].r & VT_LVAL) {
1335         vswap();
1336         swapped = 1;
1337     }
1338     if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
1339         if (op >= TOK_ULT && op <= TOK_GT) {
1340             /* load on stack second operand */
1341             load(TREG_ST0, vtop);
1342             save_reg(TREG_RAX); /* eax is used by FP comparison code */
1343             if (op == TOK_GE || op == TOK_GT)
1344                 swapped = !swapped;
1345             else if (op == TOK_EQ || op == TOK_NE)
1346                 swapped = 0;
1347             if (swapped)
1348                 o(0xc9d9); /* fxch %st(1) */
1349             o(0xe9da); /* fucompp */
1350             o(0xe0df); /* fnstsw %ax */
1351             if (op == TOK_EQ) {
1352                 o(0x45e480); /* and $0x45, %ah */
1353                 o(0x40fC80); /* cmp $0x40, %ah */
1354             } else if (op == TOK_NE) {
1355                 o(0x45e480); /* and $0x45, %ah */
1356                 o(0x40f480); /* xor $0x40, %ah */
1357                 op = TOK_NE;
1358             } else if (op == TOK_GE || op == TOK_LE) {
1359                 o(0x05c4f6); /* test $0x05, %ah */
1360                 op = TOK_EQ;
1361             } else {
1362                 o(0x45c4f6); /* test $0x45, %ah */
1363                 op = TOK_EQ;
1364             }
1365             vtop--;
1366             vtop->r = VT_CMP;
1367             vtop->c.i = op;
1368         } else {
1369             /* no memory reference possible for long double operations */
1370             load(TREG_ST0, vtop);
1371             swapped = !swapped;
1372
1373             switch(op) {
1374             default:
1375             case '+':
1376                 a = 0;
1377                 break;
1378             case '-':
1379                 a = 4;
1380                 if (swapped)
1381                     a++;
1382                 break;
1383             case '*':
1384                 a = 1;
1385                 break;
1386             case '/':
1387                 a = 6;
1388                 if (swapped)
1389                     a++;
1390                 break;
1391             }
1392             ft = vtop->type.t;
1393             fc = vtop->c.ul;
1394             o(0xde); /* fxxxp %st, %st(1) */
1395             o(0xc1 + (a << 3));
1396             vtop--;
1397         }
1398     } else {
1399         if (op >= TOK_ULT && op <= TOK_GT) {
1400             /* if saved lvalue, then we must reload it */
1401             r = vtop->r;
1402             fc = vtop->c.ul;
1403             if ((r & VT_VALMASK) == VT_LLOCAL) {
1404                 SValue v1;
1405                 r = get_reg(RC_INT);
1406                 v1.type.t = VT_INT;
1407                 v1.r = VT_LOCAL | VT_LVAL;
1408                 v1.c.ul = fc;
1409                 load(r, &v1);
1410                 fc = 0;
1411             }
1412
1413             if (op == TOK_EQ || op == TOK_NE) {
1414                 swapped = 0;
1415             } else {
1416                 if (op == TOK_LE || op == TOK_LT)
1417                     swapped = !swapped;
1418                 if (op == TOK_LE || op == TOK_GE) {
1419                     op = 0x93; /* setae */
1420                 } else {
1421                     op = 0x97; /* seta */
1422                 }
1423             }
1424
1425             if (swapped) {
1426                 o(0x7e0ff3); /* movq */
1427                 gen_modrm(1, r, vtop->sym, fc);
1428
1429                 if ((vtop->type.t & VT_BTYPE) == VT_DOUBLE) {
1430                     o(0x66);
1431                 }
1432                 o(0x2e0f); /* ucomisd %xmm0, %xmm1 */
1433                 o(0xc8);
1434             } else {
1435                 if ((vtop->type.t & VT_BTYPE) == VT_DOUBLE) {
1436                     o(0x66);
1437                 }
1438                 o(0x2e0f); /* ucomisd */
1439                 gen_modrm(0, r, vtop->sym, fc);
1440             }
1441
1442             vtop--;
1443             vtop->r = VT_CMP;
1444             vtop->c.i = op;
1445         } else {
1446             /* no memory reference possible for long double operations */
1447             if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
1448                 load(TREG_XMM0, vtop);
1449                 swapped = !swapped;
1450             }
1451             switch(op) {
1452             default:
1453             case '+':
1454                 a = 0;
1455                 break;
1456             case '-':
1457                 a = 4;
1458                 break;
1459             case '*':
1460                 a = 1;
1461                 break;
1462             case '/':
1463                 a = 6;
1464                 break;
1465             }
1466             ft = vtop->type.t;
1467             fc = vtop->c.ul;
1468             if ((ft & VT_BTYPE) == VT_LDOUBLE) {
1469                 o(0xde); /* fxxxp %st, %st(1) */
1470                 o(0xc1 + (a << 3));
1471             } else {
1472                 /* if saved lvalue, then we must reload it */
1473                 r = vtop->r;
1474                 if ((r & VT_VALMASK) == VT_LLOCAL) {
1475                     SValue v1;
1476                     r = get_reg(RC_INT);
1477                     v1.type.t = VT_INT;
1478                     v1.r = VT_LOCAL | VT_LVAL;
1479                     v1.c.ul = fc;
1480                     load(r, &v1);
1481                     fc = 0;
1482                 }
1483                 if (swapped) {
1484                     /* movq %xmm0,%xmm1 */
1485                     o(0x7e0ff3);
1486                     o(0xc8);
1487                     load(TREG_XMM0, vtop);
1488                     /* subsd  %xmm1,%xmm0 (f2 0f 5c c1) */
1489                     if ((ft & VT_BTYPE) == VT_DOUBLE) {
1490                         o(0xf2);
1491                     } else {
1492                         o(0xf3);
1493                     }
1494                     o(0x0f);
1495                     o(0x58 + a);
1496                     o(0xc1);
1497                 } else {
1498                     if ((ft & VT_BTYPE) == VT_DOUBLE) {
1499                         o(0xf2);
1500                     } else {
1501                         o(0xf3);
1502                     }
1503                     o(0x0f);
1504                     o(0x58 + a);
1505                     gen_modrm(0, r, vtop->sym, fc);
1506                 }
1507             }
1508             vtop--;
1509         }
1510     }
1511 }
1512
1513 /* convert integers to fp 't' type. Must handle 'int', 'unsigned int'
1514    and 'long long' cases. */
1515 void gen_cvt_itof(int t)
1516 {
1517     if ((t & VT_BTYPE) == VT_LDOUBLE) {
1518         save_reg(TREG_ST0);
1519         gv(RC_INT);
1520         if ((vtop->type.t & VT_BTYPE) == VT_LLONG) {
1521             /* signed long long to float/double/long double (unsigned case
1522                is handled generically) */
1523             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
1524             o(0x242cdf); /* fildll (%rsp) */
1525             o(0x08c48348); /* add $8, %rsp */
1526         } else if ((vtop->type.t & (VT_BTYPE | VT_UNSIGNED)) ==
1527                    (VT_INT | VT_UNSIGNED)) {
1528             /* unsigned int to float/double/long double */
1529             o(0x6a); /* push $0 */
1530             g(0x00);
1531             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
1532             o(0x242cdf); /* fildll (%rsp) */
1533             o(0x10c48348); /* add $16, %rsp */
1534         } else {
1535             /* int to float/double/long double */
1536             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
1537             o(0x2404db); /* fildl (%rsp) */
1538             o(0x08c48348); /* add $8, %rsp */
1539         }
1540         vtop->r = TREG_ST0;
1541     } else {
1542         save_reg(TREG_XMM0);
1543         gv(RC_INT);
1544         o(0xf2 + ((t & VT_BTYPE) == VT_FLOAT));
1545         if ((vtop->type.t & (VT_BTYPE | VT_UNSIGNED)) ==
1546             (VT_INT | VT_UNSIGNED) ||
1547             (vtop->type.t & VT_BTYPE) == VT_LLONG) {
1548             o(0x48); /* REX */
1549         }
1550         o(0x2a0f);
1551         o(0xc0 + (vtop->r & VT_VALMASK)); /* cvtsi2sd */
1552         vtop->r = TREG_XMM0;
1553     }
1554 }
1555
1556 /* convert from one floating point type to another */
1557 void gen_cvt_ftof(int t)
1558 {
1559     int ft, bt, tbt;
1560
1561     ft = vtop->type.t;
1562     bt = ft & VT_BTYPE;
1563     tbt = t & VT_BTYPE;
1564
1565     if (bt == VT_FLOAT) {
1566         gv(RC_FLOAT);
1567         if (tbt == VT_DOUBLE) {
1568             o(0xc0140f); /* unpcklps */
1569             o(0xc05a0f); /* cvtps2pd */
1570         } else if (tbt == VT_LDOUBLE) {
1571             /* movss %xmm0,-0x10(%rsp) */
1572             o(0x44110ff3);
1573             o(0xf024);
1574             o(0xf02444d9); /* flds -0x10(%rsp) */
1575             vtop->r = TREG_ST0;
1576         }
1577     } else if (bt == VT_DOUBLE) {
1578         gv(RC_FLOAT);
1579         if (tbt == VT_FLOAT) {
1580             o(0xc0140f66); /* unpcklpd */
1581             o(0xc05a0f66); /* cvtpd2ps */
1582         } else if (tbt == VT_LDOUBLE) {
1583             /* movsd %xmm0,-0x10(%rsp) */
1584             o(0x44110ff2);
1585             o(0xf024);
1586             o(0xf02444dd); /* fldl -0x10(%rsp) */
1587             vtop->r = TREG_ST0;
1588         }
1589     } else {
1590         gv(RC_ST0);
1591         if (tbt == VT_DOUBLE) {
1592             o(0xf0245cdd); /* fstpl -0x10(%rsp) */
1593             /* movsd -0x10(%rsp),%xmm0 */
1594             o(0x44100ff2);
1595             o(0xf024);
1596             vtop->r = TREG_XMM0;
1597         } else if (tbt == VT_FLOAT) {
1598             o(0xf0245cd9); /* fstps -0x10(%rsp) */
1599             /* movss -0x10(%rsp),%xmm0 */
1600             o(0x44100ff3);
1601             o(0xf024);
1602             vtop->r = TREG_XMM0;
1603         }
1604     }
1605 }
1606
1607 /* convert fp to int 't' type */
1608 void gen_cvt_ftoi(int t)
1609 {
1610     int ft, bt, size, r;
1611     ft = vtop->type.t;
1612     bt = ft & VT_BTYPE;
1613     if (bt == VT_LDOUBLE) {
1614         gen_cvt_ftof(VT_DOUBLE);
1615         bt = VT_DOUBLE;
1616     }
1617
1618     gv(RC_FLOAT);
1619     if (t != VT_INT)
1620         size = 8;
1621     else
1622         size = 4;
1623
1624     r = get_reg(RC_INT);
1625     if (bt == VT_FLOAT) {
1626         o(0xf3);
1627     } else if (bt == VT_DOUBLE) {
1628         o(0xf2);
1629     } else {
1630         assert(0);
1631     }
1632     orex(size == 8, r, 0, 0x2c0f); /* cvttss2si or cvttsd2si */
1633     o(0xc0 + (REG_VALUE(r) << 3));
1634     vtop->r = r;
1635 }
1636
1637 /* computed goto support */
1638 void ggoto(void)
1639 {
1640     gcall_or_jmp(1);
1641     vtop--;
1642 }
1643
1644 /* end of x86-64 code generator */
1645 /*************************************************************/
1646 #endif /* ! TARGET_DEFS_ONLY */
1647 /******************************************************/