x86_64-gen.c

   1 /*
   2  *  x86-64 code generator for TCC
   3  *
   4  *  Copyright (c) 2008 Shinichiro Hamaji
   5  *
   6  *  Based on i386-gen.c by Fabrice Bellard
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  21  */
  22
  23 #ifdef TARGET_DEFS_ONLY
  24
  25 /* number of available registers */
  26 #define NB_REGS         25
  27 #define NB_ASM_REGS     16
  28 #define CONFIG_TCC_ASM
  29
  30 /* a register can belong to several classes. The classes must be
  31    sorted from more general to more precise (see gv2() code which does
  32    assumptions on it). */
  33 #define RC_INT     0x0001 /* generic integer register */
  34 #define RC_FLOAT   0x0002 /* generic float register */
  35 #define RC_RAX     0x0004
  36 #define RC_RCX     0x0008
  37 #define RC_RDX     0x0010
  38 #define RC_ST0     0x0080 /* only for long double */
  39 #define RC_R8      0x0100
  40 #define RC_R9      0x0200
  41 #define RC_R10     0x0400
  42 #define RC_R11     0x0800
  43 #define RC_XMM0    0x1000
  44 #define RC_XMM1    0x2000
  45 #define RC_XMM2    0x4000
  46 #define RC_XMM3    0x8000
  47 #define RC_XMM4    0x10000
  48 #define RC_XMM5    0x20000
  49 #define RC_XMM6    0x40000
  50 #define RC_XMM7    0x80000
  51 #define RC_IRET    RC_RAX /* function return: integer register */
  52 #define RC_IRE2    RC_RDX /* function return: second integer register */
  53 #define RC_FRET    RC_XMM0 /* function return: float register */
  54 #define RC_FRE2    RC_XMM1 /* function return: second float register */
  55
  56 /* pretty names for the registers */
  57 enum {
  58     TREG_RAX = 0,
  59     TREG_RCX = 1,
  60     TREG_RDX = 2,
  61     TREG_RSP = 4,
  62     TREG_RSI = 6,
  63     TREG_RDI = 7,
  64
  65     TREG_R8  = 8,
  66     TREG_R9  = 9,
  67     TREG_R10 = 10,
  68     TREG_R11 = 11,
  69
  70     TREG_XMM0 = 16,
  71     TREG_XMM1 = 17,
  72     TREG_XMM2 = 18,
  73     TREG_XMM3 = 19,
  74     TREG_XMM4 = 20,
  75     TREG_XMM5 = 21,
  76     TREG_XMM6 = 22,
  77     TREG_XMM7 = 23,
  78
  79     TREG_ST0 = 24,
  80
  81     TREG_MEM = 0x20
  82 };
  83
  84 #define REX_BASE(reg) (((reg) >> 3) & 1)
  85 #define REG_VALUE(reg) ((reg) & 7)
  86
  87 /* return registers for function */
  88 #define REG_IRET TREG_RAX /* single word int return register */
  89 #define REG_IRE2 TREG_RDX /* second word return register (for long long) */
  90 #define REG_FRET TREG_XMM0 /* float return register */
  91 #define REG_FRE2 TREG_XMM1 /* second float return register */
  92
  93 /* defined if function parameters must be evaluated in reverse order */
  94 #define INVERT_FUNC_PARAMS
  95
  96 /* pointer size, in bytes */
  97 #define PTR_SIZE 8
  98
  99 /* long double size and alignment, in bytes */
 100 #define LDOUBLE_SIZE  16
 101 #define LDOUBLE_ALIGN 16
 102 /* maximum alignment (for aligned attribute support) */
 103 #define MAX_ALIGN     16
 104
 105 /* define if return values need to be extended explicitely
 106    at caller side (for interfacing with non-TCC compilers) */
 107 #define PROMOTE_RET
 108 /******************************************************/
 109 #else /* ! TARGET_DEFS_ONLY */
 110 /******************************************************/
 111 #define USING_GLOBALS
 112 #include "tcc.h"
 113 #include <assert.h>
 114
 115 ST_DATA const char *target_machine_defs =
 116     "__x86_64__\0"
 117     "__amd64__\0"
 118     ;
 119
 120 ST_DATA const int reg_classes[NB_REGS] = {
 121     /* eax */ RC_INT | RC_RAX,
 122     /* ecx */ RC_INT | RC_RCX,
 123     /* edx */ RC_INT | RC_RDX,
 124     0,
 125     0,
 126     0,
 127     0,
 128     0,
 129     RC_R8,
 130     RC_R9,
 131     RC_R10,
 132     RC_R11,
 133     0,
 134     0,
 135     0,
 136     0,
 137     /* xmm0 */ RC_FLOAT | RC_XMM0,
 138     /* xmm1 */ RC_FLOAT | RC_XMM1,
 139     /* xmm2 */ RC_FLOAT | RC_XMM2,
 140     /* xmm3 */ RC_FLOAT | RC_XMM3,
 141     /* xmm4 */ RC_FLOAT | RC_XMM4,
 142     /* xmm5 */ RC_FLOAT | RC_XMM5,
 143     /* xmm6 an xmm7 are included so gv() can be used on them,
 144        but they are not tagged with RC_FLOAT because they are
 145        callee saved on Windows */
 146     RC_XMM6,
 147     RC_XMM7,
 148     /* st0 */ RC_ST0
 149 };
 150
 151 static unsigned long func_sub_sp_offset;
 152 static int func_ret_sub;
 153
 154 #if defined(CONFIG_TCC_BCHECK)
 155 static addr_t func_bound_offset;
 156 static unsigned long func_bound_ind;
 157 ST_DATA int func_bound_add_epilog;
 158 #endif
 159
 160 #ifdef TCC_TARGET_PE
 161 static int func_scratch, func_alloca;
 162 #endif
 163
 164 /* XXX: make it faster ? */
 165 ST_FUNC void g(int c)
 166 {
 167     int ind1;
 168     if (nocode_wanted)
 169         return;
 170     ind1 = ind + 1;
 171     if (ind1 > cur_text_section->data_allocated)
 172         section_realloc(cur_text_section, ind1);
 173     cur_text_section->data[ind] = c;
 174     ind = ind1;
 175 }
 176
 177 ST_FUNC void o(unsigned int c)
 178 {
 179     while (c) {
 180         g(c);
 181         c = c >> 8;
 182     }
 183 }
 184
 185 ST_FUNC void gen_le16(int v)
 186 {
 187     g(v);
 188     g(v >> 8);
 189 }
 190
 191 ST_FUNC void gen_le32(int c)
 192 {
 193     g(c);
 194     g(c >> 8);
 195     g(c >> 16);
 196     g(c >> 24);
 197 }
 198
 199 ST_FUNC void gen_le64(int64_t c)
 200 {
 201     g(c);
 202     g(c >> 8);
 203     g(c >> 16);
 204     g(c >> 24);
 205     g(c >> 32);
 206     g(c >> 40);
 207     g(c >> 48);
 208     g(c >> 56);
 209 }
 210
 211 static void orex(int ll, int r, int r2, int b)
 212 {
 213     if ((r & VT_VALMASK) >= VT_CONST)
 214         r = 0;
 215     if ((r2 & VT_VALMASK) >= VT_CONST)
 216         r2 = 0;
 217     if (ll || REX_BASE(r) || REX_BASE(r2))
 218         o(0x40 | REX_BASE(r) | (REX_BASE(r2) << 2) | (ll << 3));
 219     o(b);
 220 }
 221
 222 /* output a symbol and patch all calls to it */
 223 ST_FUNC void gsym_addr(int t, int a)
 224 {
 225     while (t) {
 226         unsigned char *ptr = cur_text_section->data + t;
 227         uint32_t n = read32le(ptr); /* next value */
 228         write32le(ptr, a < 0 ? -a : a - t - 4);
 229         t = n;
 230     }
 231 }
 232
 233 static int is64_type(int t)
 234 {
 235     return ((t & VT_BTYPE) == VT_PTR ||
 236             (t & VT_BTYPE) == VT_FUNC ||
 237             (t & VT_BTYPE) == VT_LLONG);
 238 }
 239
 240 /* instruction + 4 bytes data. Return the address of the data */
 241 static int oad(int c, int s)
 242 {
 243     int t;
 244     if (nocode_wanted)
 245         return s;
 246     o(c);
 247     t = ind;
 248     gen_le32(s);
 249     return t;
 250 }
 251
 252 /* generate jmp to a label */
 253 #define gjmp2(instr,lbl) oad(instr,lbl)
 254
 255 ST_FUNC void gen_addr32(int r, Sym *sym, int c)
 256 {
 257     if (r & VT_SYM)
 258         greloca(cur_text_section, sym, ind, R_X86_64_32S, c), c=0;
 259     gen_le32(c);
 260 }
 261
 262 /* output constant with relocation if 'r & VT_SYM' is true */
 263 ST_FUNC void gen_addr64(int r, Sym *sym, int64_t c)
 264 {
 265     if (r & VT_SYM)
 266         greloca(cur_text_section, sym, ind, R_X86_64_64, c), c=0;
 267     gen_le64(c);
 268 }
 269
 270 /* output constant with relocation if 'r & VT_SYM' is true */
 271 ST_FUNC void gen_addrpc32(int r, Sym *sym, int c)
 272 {
 273     if (r & VT_SYM)
 274         greloca(cur_text_section, sym, ind, R_X86_64_PC32, c-4), c=4;
 275     gen_le32(c-4);
 276 }
 277
 278 /* output got address with relocation */
 279 static void gen_gotpcrel(int r, Sym *sym, int c)
 280 {
 281 #ifdef TCC_TARGET_PE
 282     tcc_error("internal error: no GOT on PE: %s %x %x | %02x %02x %02x\n",
 283         get_tok_str(sym->v, NULL), c, r,
 284         cur_text_section->data[ind-3],
 285         cur_text_section->data[ind-2],
 286         cur_text_section->data[ind-1]
 287         );
 288 #endif
 289     greloca(cur_text_section, sym, ind, R_X86_64_GOTPCREL, -4);
 290     gen_le32(0);
 291     if (c) {
 292         /* we use add c, %xxx for displacement */
 293         orex(1, r, 0, 0x81);
 294         o(0xc0 + REG_VALUE(r));
 295         gen_le32(c);
 296     }
 297 }
 298
 299 static void gen_modrm_impl(int op_reg, int r, Sym *sym, int c, int is_got)
 300 {
 301     op_reg = REG_VALUE(op_reg) << 3;
 302     if ((r & VT_VALMASK) == VT_CONST) {
 303         /* constant memory reference */
 304         if (!(r & VT_SYM)) {
 305             /* Absolute memory reference */
 306             o(0x04 | op_reg); /* [sib] | destreg */
 307             oad(0x25, c);     /* disp32 */
 308         } else {
 309             o(0x05 | op_reg); /* (%rip)+disp32 | destreg */
 310             if (is_got) {
 311                 gen_gotpcrel(r, sym, c);
 312             } else {
 313                 gen_addrpc32(r, sym, c);
 314             }
 315         }
 316     } else if ((r & VT_VALMASK) == VT_LOCAL) {
 317         /* currently, we use only ebp as base */
 318         if (c == (char)c) {
 319             /* short reference */
 320             o(0x45 | op_reg);
 321             g(c);
 322         } else {
 323             oad(0x85 | op_reg, c);
 324         }
 325     } else if ((r & VT_VALMASK) >= TREG_MEM) {
 326         if (c) {
 327             g(0x80 | op_reg | REG_VALUE(r));
 328             gen_le32(c);
 329         } else {
 330             g(0x00 | op_reg | REG_VALUE(r));
 331         }
 332     } else {
 333         g(0x00 | op_reg | REG_VALUE(r));
 334     }
 335 }
 336
 337 /* generate a modrm reference. 'op_reg' contains the additional 3
 338    opcode bits */
 339 static void gen_modrm(int op_reg, int r, Sym *sym, int c)
 340 {
 341     gen_modrm_impl(op_reg, r, sym, c, 0);
 342 }
 343
 344 /* generate a modrm reference. 'op_reg' contains the additional 3
 345    opcode bits */
 346 static void gen_modrm64(int opcode, int op_reg, int r, Sym *sym, int c)
 347 {
 348     int is_got;
 349     is_got = (op_reg & TREG_MEM) && !(sym->type.t & VT_STATIC);
 350     orex(1, r, op_reg, opcode);
 351     gen_modrm_impl(op_reg, r, sym, c, is_got);
 352 }
 353
 354
 355 /* load 'r' from value 'sv' */
 356 void load(int r, SValue *sv)
 357 {
 358     int v, t, ft, fc, fr;
 359     SValue v1;
 360
 361 #ifdef TCC_TARGET_PE
 362     SValue v2;
 363     sv = pe_getimport(sv, &v2);
 364 #endif
 365
 366     fr = sv->r;
 367     ft = sv->type.t & ~VT_DEFSIGN;
 368     fc = sv->c.i;
 369     if (fc != sv->c.i && (fr & VT_SYM))
 370       tcc_error("64 bit addend in load");
 371
 372     ft &= ~(VT_VOLATILE | VT_CONSTANT);
 373
 374 #ifndef TCC_TARGET_PE
 375     /* we use indirect access via got */
 376     if ((fr & VT_VALMASK) == VT_CONST && (fr & VT_SYM) &&
 377         (fr & VT_LVAL) && !(sv->sym->type.t & VT_STATIC)) {
 378         /* use the result register as a temporal register */
 379         int tr = r | TREG_MEM;
 380         if (is_float(ft)) {
 381             /* we cannot use float registers as a temporal register */
 382             tr = get_reg(RC_INT) | TREG_MEM;
 383         }
 384         gen_modrm64(0x8b, tr, fr, sv->sym, 0);
 385
 386         /* load from the temporal register */
 387         fr = tr | VT_LVAL;
 388     }
 389 #endif
 390
 391     v = fr & VT_VALMASK;
 392     if (fr & VT_LVAL) {
 393         int b, ll;
 394         if (v == VT_LLOCAL) {
 395             v1.type.t = VT_PTR;
 396             v1.r = VT_LOCAL | VT_LVAL;
 397             v1.c.i = fc;
 398             fr = r;
 399             if (!(reg_classes[fr] & (RC_INT|RC_R11)))
 400                 fr = get_reg(RC_INT);
 401             load(fr, &v1);
 402         }
 403         if (fc != sv->c.i) {
 404             /* If the addends doesn't fit into a 32bit signed
 405                we must use a 64bit move.  We've checked above
 406                that this doesn't have a sym associated.  */
 407             v1.type.t = VT_LLONG;
 408             v1.r = VT_CONST;
 409             v1.c.i = sv->c.i;
 410             fr = r;
 411             if (!(reg_classes[fr] & (RC_INT|RC_R11)))
 412                 fr = get_reg(RC_INT);
 413             load(fr, &v1);
 414             fc = 0;
 415         }
 416         ll = 0;
 417         /* Like GCC we can load from small enough properly sized
 418            structs and unions as well.
 419            XXX maybe move to generic operand handling, but should
 420            occur only with asm, so tccasm.c might also be a better place */
 421         if ((ft & VT_BTYPE) == VT_STRUCT) {
 422             int align;
 423             switch (type_size(&sv->type, &align)) {
 424                 case 1: ft = VT_BYTE; break;
 425                 case 2: ft = VT_SHORT; break;
 426                 case 4: ft = VT_INT; break;
 427                 case 8: ft = VT_LLONG; break;
 428                 default:
 429                     tcc_error("invalid aggregate type for register load");
 430                     break;
 431             }
 432         }
 433         if ((ft & VT_BTYPE) == VT_FLOAT) {
 434             b = 0x6e0f66;
 435             r = REG_VALUE(r); /* movd */
 436         } else if ((ft & VT_BTYPE) == VT_DOUBLE) {
 437             b = 0x7e0ff3; /* movq */
 438             r = REG_VALUE(r);
 439         } else if ((ft & VT_BTYPE) == VT_LDOUBLE) {
 440             b = 0xdb, r = 5; /* fldt */
 441         } else if ((ft & VT_TYPE) == VT_BYTE || (ft & VT_TYPE) == VT_BOOL) {
 442             b = 0xbe0f;   /* movsbl */
 443         } else if ((ft & VT_TYPE) == (VT_BYTE | VT_UNSIGNED)) {
 444             b = 0xb60f;   /* movzbl */
 445         } else if ((ft & VT_TYPE) == VT_SHORT) {
 446             b = 0xbf0f;   /* movswl */
 447         } else if ((ft & VT_TYPE) == (VT_SHORT | VT_UNSIGNED)) {
 448             b = 0xb70f;   /* movzwl */
 449         } else if ((ft & VT_TYPE) == (VT_VOID)) {
 450             /* Can happen with zero size structs */
 451             return;
 452         } else {
 453             assert(((ft & VT_BTYPE) == VT_INT)
 454                    || ((ft & VT_BTYPE) == VT_LLONG)
 455                    || ((ft & VT_BTYPE) == VT_PTR)
 456                    || ((ft & VT_BTYPE) == VT_FUNC)
 457                 );
 458             ll = is64_type(ft);
 459             b = 0x8b;
 460         }
 461         if (ll) {
 462             gen_modrm64(b, r, fr, sv->sym, fc);
 463         } else {
 464             orex(ll, fr, r, b);
 465             gen_modrm(r, fr, sv->sym, fc);
 466         }
 467     } else {
 468         if (v == VT_CONST) {
 469             if (fr & VT_SYM) {
 470 #ifdef TCC_TARGET_PE
 471                 orex(1,0,r,0x8d);
 472                 o(0x05 + REG_VALUE(r) * 8); /* lea xx(%rip), r */
 473                 gen_addrpc32(fr, sv->sym, fc);
 474 #else
 475                 if (sv->sym->type.t & VT_STATIC) {
 476                     orex(1,0,r,0x8d);
 477                     o(0x05 + REG_VALUE(r) * 8); /* lea xx(%rip), r */
 478                     gen_addrpc32(fr, sv->sym, fc);
 479                 } else {
 480                     orex(1,0,r,0x8b);
 481                     o(0x05 + REG_VALUE(r) * 8); /* mov xx(%rip), r */
 482                     gen_gotpcrel(r, sv->sym, fc);
 483                 }
 484 #endif
 485             } else if (is64_type(ft)) {
 486                 orex(1,r,0, 0xb8 + REG_VALUE(r)); /* mov $xx, r */
 487                 gen_le64(sv->c.i);
 488             } else {
 489                 orex(0,r,0, 0xb8 + REG_VALUE(r)); /* mov $xx, r */
 490                 gen_le32(fc);
 491             }
 492         } else if (v == VT_LOCAL) {
 493             orex(1,0,r,0x8d); /* lea xxx(%ebp), r */
 494             gen_modrm(r, VT_LOCAL, sv->sym, fc);
 495         } else if (v == VT_CMP) {
 496             if (fc & 0x100)
 497               {
 498                 v = vtop->cmp_r;
 499                 fc &= ~0x100;
 500                 /* This was a float compare.  If the parity bit is
 501                    set the result was unordered, meaning false for everything
 502                    except TOK_NE, and true for TOK_NE.  */
 503                 orex(0, r, 0, 0xb0 + REG_VALUE(r)); /* mov $0/1,%al */
 504                 g(v ^ fc ^ (v == TOK_NE));
 505                 o(0x037a + (REX_BASE(r) << 8));
 506               }
 507             orex(0,r,0, 0x0f); /* setxx %br */
 508             o(fc);
 509             o(0xc0 + REG_VALUE(r));
 510             orex(0,r,0, 0x0f);
 511             o(0xc0b6 + REG_VALUE(r) * 0x900); /* movzbl %al, %eax */
 512         } else if (v == VT_JMP || v == VT_JMPI) {
 513             t = v & 1;
 514             orex(0,r,0,0);
 515             oad(0xb8 + REG_VALUE(r), t); /* mov $1, r */
 516             o(0x05eb + (REX_BASE(r) << 8)); /* jmp after */
 517             gsym(fc);
 518             orex(0,r,0,0);
 519             oad(0xb8 + REG_VALUE(r), t ^ 1); /* mov $0, r */
 520         } else if (v != r) {
 521             if ((r >= TREG_XMM0) && (r <= TREG_XMM7)) {
 522                 if (v == TREG_ST0) {
 523                     /* gen_cvt_ftof(VT_DOUBLE); */
 524                     o(0xf0245cdd); /* fstpl -0x10(%rsp) */
 525                     /* movsd -0x10(%rsp),%xmmN */
 526                     o(0x100ff2);
 527                     o(0x44 + REG_VALUE(r)*8); /* %xmmN */
 528                     o(0xf024);
 529                 } else {
 530                     assert((v >= TREG_XMM0) && (v <= TREG_XMM7));
 531                     if ((ft & VT_BTYPE) == VT_FLOAT) {
 532                         o(0x100ff3);
 533                     } else {
 534                         assert((ft & VT_BTYPE) == VT_DOUBLE);
 535                         o(0x100ff2);
 536                     }
 537                     o(0xc0 + REG_VALUE(v) + REG_VALUE(r)*8);
 538                 }
 539             } else if (r == TREG_ST0) {
 540                 assert((v >= TREG_XMM0) && (v <= TREG_XMM7));
 541                 /* gen_cvt_ftof(VT_LDOUBLE); */
 542                 /* movsd %xmmN,-0x10(%rsp) */
 543                 o(0x110ff2);
 544                 o(0x44 + REG_VALUE(r)*8); /* %xmmN */
 545                 o(0xf024);
 546                 o(0xf02444dd); /* fldl -0x10(%rsp) */
 547             } else {
 548                 orex(is64_type(ft), r, v, 0x89);
 549                 o(0xc0 + REG_VALUE(r) + REG_VALUE(v) * 8); /* mov v, r */
 550             }
 551         }
 552     }
 553 }
 554
 555 /* store register 'r' in lvalue 'v' */
 556 void store(int r, SValue *v)
 557 {
 558     int fr, bt, ft, fc;
 559     int op64 = 0;
 560     /* store the REX prefix in this variable when PIC is enabled */
 561     int pic = 0;
 562
 563 #ifdef TCC_TARGET_PE
 564     SValue v2;
 565     v = pe_getimport(v, &v2);
 566 #endif
 567
 568     fr = v->r & VT_VALMASK;
 569     ft = v->type.t;
 570     fc = v->c.i;
 571     if (fc != v->c.i && (fr & VT_SYM))
 572       tcc_error("64 bit addend in store");
 573     ft &= ~(VT_VOLATILE | VT_CONSTANT);
 574     bt = ft & VT_BTYPE;
 575
 576 #ifndef TCC_TARGET_PE
 577     /* we need to access the variable via got */
 578     if (fr == VT_CONST && (v->r & VT_SYM)) {
 579         /* mov xx(%rip), %r11 */
 580         o(0x1d8b4c);
 581         gen_gotpcrel(TREG_R11, v->sym, v->c.i);
 582         pic = is64_type(bt) ? 0x49 : 0x41;
 583     }
 584 #endif
 585
 586     /* XXX: incorrect if float reg to reg */
 587     if (bt == VT_FLOAT) {
 588         o(0x66);
 589         o(pic);
 590         o(0x7e0f); /* movd */
 591         r = REG_VALUE(r);
 592     } else if (bt == VT_DOUBLE) {
 593         o(0x66);
 594         o(pic);
 595         o(0xd60f); /* movq */
 596         r = REG_VALUE(r);
 597     } else if (bt == VT_LDOUBLE) {
 598         o(0xc0d9); /* fld %st(0) */
 599         o(pic);
 600         o(0xdb); /* fstpt */
 601         r = 7;
 602     } else {
 603         if (bt == VT_SHORT)
 604             o(0x66);
 605         o(pic);
 606         if (bt == VT_BYTE || bt == VT_BOOL)
 607             orex(0, 0, r, 0x88);
 608         else if (is64_type(bt))
 609             op64 = 0x89;
 610         else
 611             orex(0, 0, r, 0x89);
 612     }
 613     if (pic) {
 614         /* xxx r, (%r11) where xxx is mov, movq, fld, or etc */
 615         if (op64)
 616             o(op64);
 617         o(3 + (r << 3));
 618     } else if (op64) {
 619         if (fr == VT_CONST || fr == VT_LOCAL || (v->r & VT_LVAL)) {
 620             gen_modrm64(op64, r, v->r, v->sym, fc);
 621         } else if (fr != r) {
 622             orex(1, fr, r, op64);
 623             o(0xc0 + fr + r * 8); /* mov r, fr */
 624         }
 625     } else {
 626         if (fr == VT_CONST || fr == VT_LOCAL || (v->r & VT_LVAL)) {
 627             gen_modrm(r, v->r, v->sym, fc);
 628         } else if (fr != r) {
 629             o(0xc0 + fr + r * 8); /* mov r, fr */
 630         }
 631     }
 632 }
 633
 634 /* 'is_jmp' is '1' if it is a jump */
 635 static void gcall_or_jmp(int is_jmp)
 636 {
 637     int r;
 638     if ((vtop->r & (VT_VALMASK | VT_LVAL)) == VT_CONST &&
 639         ((vtop->r & VT_SYM) && (vtop->c.i-4) == (int)(vtop->c.i-4))) {
 640         /* constant symbolic case -> simple relocation */
 641 #ifdef TCC_TARGET_PE
 642         greloca(cur_text_section, vtop->sym, ind + 1, R_X86_64_PC32, (int)(vtop->c.i-4));
 643 #else
 644         greloca(cur_text_section, vtop->sym, ind + 1, R_X86_64_PLT32, (int)(vtop->c.i-4));
 645 #endif
 646         oad(0xe8 + is_jmp, 0); /* call/jmp im */
 647     } else {
 648         /* otherwise, indirect call */
 649         r = TREG_R11;
 650         load(r, vtop);
 651         o(0x41); /* REX */
 652         o(0xff); /* call/jmp *r */
 653         o(0xd0 + REG_VALUE(r) + (is_jmp << 4));
 654     }
 655 }
 656
 657 #if defined(CONFIG_TCC_BCHECK)
 658
 659 static void gen_bounds_call(int v)
 660 {
 661     Sym *sym = external_helper_sym(v);
 662     oad(0xe8, 0);
 663 #ifdef TCC_TARGET_PE
 664     greloca(cur_text_section, sym, ind-4, R_X86_64_PC32, -4);
 665 #else
 666     greloca(cur_text_section, sym, ind-4, R_X86_64_PLT32, -4);
 667 #endif
 668 }
 669
 670 #ifdef TCC_TARGET_PE
 671 # define TREG_FASTCALL_1 TREG_RCX
 672 #else
 673 # define TREG_FASTCALL_1 TREG_RDI
 674 #endif
 675
 676 static void gen_bounds_prolog(void)
 677 {
 678     /* leave some room for bound checking code */
 679     func_bound_offset = lbounds_section->data_offset;
 680     func_bound_ind = ind;
 681     func_bound_add_epilog = 0;
 682     o(0x0d8d48 + ((TREG_FASTCALL_1 == TREG_RDI) * 0x300000)); /*lbound section pointer */
 683     gen_le32 (0);
 684     oad(0xb8, 0); /* call to function */
 685 }
 686
 687 static void gen_bounds_epilog(void)
 688 {
 689     addr_t saved_ind;
 690     addr_t *bounds_ptr;
 691     Sym *sym_data;
 692     int offset_modified = func_bound_offset != lbounds_section->data_offset;
 693
 694     if (!offset_modified && !func_bound_add_epilog)
 695         return;
 696
 697     /* add end of table info */
 698     bounds_ptr = section_ptr_add(lbounds_section, sizeof(addr_t));
 699     *bounds_ptr = 0;
 700
 701     sym_data = get_sym_ref(&char_pointer_type, lbounds_section,
 702                            func_bound_offset, lbounds_section->data_offset);
 703
 704     /* generate bound local allocation */
 705     if (offset_modified) {
 706         saved_ind = ind;
 707         ind = func_bound_ind;
 708         greloca(cur_text_section, sym_data, ind + 3, R_X86_64_PC32, -4);
 709         ind = ind + 7;
 710         gen_bounds_call(TOK___bound_local_new);
 711         ind = saved_ind;
 712     }
 713
 714     /* generate bound check local freeing */
 715     o(0x5250); /* save returned value, if any */
 716     greloca(cur_text_section, sym_data, ind + 3, R_X86_64_PC32, -4);
 717     o(0x0d8d48 + ((TREG_FASTCALL_1 == TREG_RDI) * 0x300000)); /* lea xxx(%rip), %rcx/rdi */
 718     gen_le32 (0);
 719     gen_bounds_call(TOK___bound_local_delete);
 720     o(0x585a); /* restore returned value, if any */
 721 }
 722 #endif
 723
 724 #ifdef TCC_TARGET_PE
 725
 726 #define REGN 4
 727 static const uint8_t arg_regs[REGN] = {
 728     TREG_RCX, TREG_RDX, TREG_R8, TREG_R9
 729 };
 730
 731 /* Prepare arguments in R10 and R11 rather than RCX and RDX
 732    because gv() will not ever use these */
 733 static int arg_prepare_reg(int idx) {
 734   if (idx == 0 || idx == 1)
 735       /* idx=0: r10, idx=1: r11 */
 736       return idx + 10;
 737   else
 738       return arg_regs[idx];
 739 }
 740
 741 /* Generate function call. The function address is pushed first, then
 742    all the parameters in call order. This functions pops all the
 743    parameters and the function address. */
 744
 745 static void gen_offs_sp(int b, int r, int d)
 746 {
 747     orex(1,0,r & 0x100 ? 0 : r, b);
 748     if (d == (char)d) {
 749         o(0x2444 | (REG_VALUE(r) << 3));
 750         g(d);
 751     } else {
 752         o(0x2484 | (REG_VALUE(r) << 3));
 753         gen_le32(d);
 754     }
 755 }
 756
 757 static int using_regs(int size)
 758 {
 759     return !(size > 8 || (size & (size - 1)));
 760 }
 761
 762 /* Return the number of registers needed to return the struct, or 0 if
 763    returning via struct pointer. */
 764 ST_FUNC int gfunc_sret(CType *vt, int variadic, CType *ret, int *ret_align, int *regsize)
 765 {
 766     int size, align;
 767     *ret_align = 1; // Never have to re-align return values for x86-64
 768     *regsize = 8;
 769     size = type_size(vt, &align);
 770     if (!using_regs(size))
 771         return 0;
 772     if (size == 8)
 773         ret->t = VT_LLONG;
 774     else if (size == 4)
 775         ret->t = VT_INT;
 776     else if (size == 2)
 777         ret->t = VT_SHORT;
 778     else
 779         ret->t = VT_BYTE;
 780     ret->ref = NULL;
 781     return 1;
 782 }
 783
 784 static int is_sse_float(int t) {
 785     int bt;
 786     bt = t & VT_BTYPE;
 787     return bt == VT_DOUBLE || bt == VT_FLOAT;
 788 }
 789
 790 static int gfunc_arg_size(CType *type) {
 791     int align;
 792     if (type->t & (VT_ARRAY|VT_BITFIELD))
 793         return 8;
 794     return type_size(type, &align);
 795 }
 796
 797 void gfunc_call(int nb_args)
 798 {
 799     int size, r, args_size, i, d, bt, struct_size;
 800     int arg;
 801
 802 #ifdef CONFIG_TCC_BCHECK
 803     if (tcc_state->do_bounds_check)
 804         gbound_args(nb_args);
 805 #endif
 806
 807     args_size = (nb_args < REGN ? REGN : nb_args) * PTR_SIZE;
 808     arg = nb_args;
 809
 810     /* for struct arguments, we need to call memcpy and the function
 811        call breaks register passing arguments we are preparing.
 812        So, we process arguments which will be passed by stack first. */
 813     struct_size = args_size;
 814     for(i = 0; i < nb_args; i++) {
 815         SValue *sv;
 816
 817         --arg;
 818         sv = &vtop[-i];
 819         bt = (sv->type.t & VT_BTYPE);
 820         size = gfunc_arg_size(&sv->type);
 821
 822         if (using_regs(size))
 823             continue; /* arguments smaller than 8 bytes passed in registers or on stack */
 824
 825         if (bt == VT_STRUCT) {
 826             /* align to stack align size */
 827             size = (size + 15) & ~15;
 828             /* generate structure store */
 829             r = get_reg(RC_INT);
 830             gen_offs_sp(0x8d, r, struct_size);
 831             struct_size += size;
 832
 833             /* generate memcpy call */
 834             vset(&sv->type, r | VT_LVAL, 0);
 835             vpushv(sv);
 836             vstore();
 837             --vtop;
 838         } else if (bt == VT_LDOUBLE) {
 839             gv(RC_ST0);
 840             gen_offs_sp(0xdb, 0x107, struct_size);
 841             struct_size += 16;
 842         }
 843     }
 844
 845     if (func_scratch < struct_size)
 846         func_scratch = struct_size;
 847
 848     arg = nb_args;
 849     struct_size = args_size;
 850
 851     for(i = 0; i < nb_args; i++) {
 852         --arg;
 853         bt = (vtop->type.t & VT_BTYPE);
 854
 855         size = gfunc_arg_size(&vtop->type);
 856         if (!using_regs(size)) {
 857             /* align to stack align size */
 858             size = (size + 15) & ~15;
 859             if (arg >= REGN) {
 860                 d = get_reg(RC_INT);
 861                 gen_offs_sp(0x8d, d, struct_size);
 862                 gen_offs_sp(0x89, d, arg*8);
 863             } else {
 864                 d = arg_prepare_reg(arg);
 865                 gen_offs_sp(0x8d, d, struct_size);
 866             }
 867             struct_size += size;
 868         } else {
 869             if (is_sse_float(vtop->type.t)) {
 870                 if (tcc_state->nosse)
 871                   tcc_error("SSE disabled");
 872                 if (arg >= REGN) {
 873                     gv(RC_XMM0);
 874                     /* movq %xmm0, j*8(%rsp) */
 875                     gen_offs_sp(0xd60f66, 0x100, arg*8);
 876                 } else {
 877                     /* Load directly to xmmN register */
 878                     gv(RC_XMM0 << arg);
 879                     d = arg_prepare_reg(arg);
 880                     /* mov %xmmN, %rxx */
 881                     o(0x66);
 882                     orex(1,d,0, 0x7e0f);
 883                     o(0xc0 + arg*8 + REG_VALUE(d));
 884                 }
 885             } else {
 886                 if (bt == VT_STRUCT) {
 887                     vtop->type.ref = NULL;
 888                     vtop->type.t = size > 4 ? VT_LLONG : size > 2 ? VT_INT
 889                         : size > 1 ? VT_SHORT : VT_BYTE;
 890                 }
 891
 892                 r = gv(RC_INT);
 893                 if (arg >= REGN) {
 894                     gen_offs_sp(0x89, r, arg*8);
 895                 } else {
 896                     d = arg_prepare_reg(arg);
 897                     orex(1,d,r,0x89); /* mov */
 898                     o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d));
 899                 }
 900             }
 901         }
 902         vtop--;
 903     }
 904     save_regs(0);
 905     /* Copy R10 and R11 into RCX and RDX, respectively */
 906     if (nb_args > 0) {
 907         o(0xd1894c); /* mov %r10, %rcx */
 908         if (nb_args > 1) {
 909             o(0xda894c); /* mov %r11, %rdx */
 910         }
 911     }
 912
 913     gcall_or_jmp(0);
 914
 915     if ((vtop->r & VT_SYM) && vtop->sym->v == TOK_alloca) {
 916         /* need to add the "func_scratch" area after alloca */
 917         o(0x48); func_alloca = oad(0x05, func_alloca); /* add $NN, %rax */
 918 #ifdef CONFIG_TCC_BCHECK
 919         if (tcc_state->do_bounds_check)
 920             gen_bounds_call(TOK___bound_alloca_nr); /* new region */
 921 #endif
 922     }
 923     vtop--;
 924 }
 925
 926
 927 #define FUNC_PROLOG_SIZE 11
 928
 929 /* generate function prolog of type 't' */
 930 void gfunc_prolog(Sym *func_sym)
 931 {
 932     CType *func_type = &func_sym->type;
 933     int addr, reg_param_index, bt, size;
 934     Sym *sym;
 935     CType *type;
 936
 937     func_ret_sub = 0;
 938     func_scratch = 32;
 939     func_alloca = 0;
 940     loc = 0;
 941
 942     addr = PTR_SIZE * 2;
 943     ind += FUNC_PROLOG_SIZE;
 944     func_sub_sp_offset = ind;
 945     reg_param_index = 0;
 946
 947     sym = func_type->ref;
 948
 949     /* if the function returns a structure, then add an
 950        implicit pointer parameter */
 951     size = gfunc_arg_size(&func_vt);
 952     if (!using_regs(size)) {
 953         gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 954         func_vc = addr;
 955         reg_param_index++;
 956         addr += 8;
 957     }
 958
 959     /* define parameters */
 960     while ((sym = sym->next) != NULL) {
 961         type = &sym->type;
 962         bt = type->t & VT_BTYPE;
 963         size = gfunc_arg_size(type);
 964         if (!using_regs(size)) {
 965             if (reg_param_index < REGN) {
 966                 gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 967             }
 968             sym_push(sym->v & ~SYM_FIELD, type,
 969                      VT_LLOCAL | VT_LVAL, addr);
 970         } else {
 971             if (reg_param_index < REGN) {
 972                 /* save arguments passed by register */
 973                 if ((bt == VT_FLOAT) || (bt == VT_DOUBLE)) {
 974                     if (tcc_state->nosse)
 975                       tcc_error("SSE disabled");
 976                     o(0xd60f66); /* movq */
 977                     gen_modrm(reg_param_index, VT_LOCAL, NULL, addr);
 978                 } else {
 979                     gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 980                 }
 981             }
 982             sym_push(sym->v & ~SYM_FIELD, type,
 983                      VT_LOCAL | VT_LVAL, addr);
 984         }
 985         addr += 8;
 986         reg_param_index++;
 987     }
 988
 989     while (reg_param_index < REGN) {
 990         if (func_var) {
 991             gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, addr);
 992             addr += 8;
 993         }
 994         reg_param_index++;
 995     }
 996 #ifdef CONFIG_TCC_BCHECK
 997     if (tcc_state->do_bounds_check)
 998         gen_bounds_prolog();
 999 #endif
1000 }
1001
1002 /* generate function epilog */
1003 void gfunc_epilog(void)
1004 {
1005     int v, saved_ind;
1006
1007     /* align local size to word & save local variables */
1008     func_scratch = (func_scratch + 15) & -16;
1009     loc = (loc & -16) - func_scratch;
1010
1011 #ifdef CONFIG_TCC_BCHECK
1012     if (tcc_state->do_bounds_check)
1013         gen_bounds_epilog();
1014 #endif
1015
1016     o(0xc9); /* leave */
1017     if (func_ret_sub == 0) {
1018         o(0xc3); /* ret */
1019     } else {
1020         o(0xc2); /* ret n */
1021         g(func_ret_sub);
1022         g(func_ret_sub >> 8);
1023     }
1024
1025     saved_ind = ind;
1026     ind = func_sub_sp_offset - FUNC_PROLOG_SIZE;
1027     v = -loc;
1028
1029     if (v >= 4096) {
1030         Sym *sym = external_helper_sym(TOK___chkstk);
1031         oad(0xb8, v); /* mov stacksize, %eax */
1032         oad(0xe8, 0); /* call __chkstk, (does the stackframe too) */
1033         greloca(cur_text_section, sym, ind-4, R_X86_64_PC32, -4);
1034         o(0x90); /* fill for FUNC_PROLOG_SIZE = 11 bytes */
1035     } else {
1036         o(0xe5894855);  /* push %rbp, mov %rsp, %rbp */
1037         o(0xec8148);  /* sub rsp, stacksize */
1038         gen_le32(v);
1039     }
1040
1041     /* add the "func_scratch" area after each alloca seen */
1042     gsym_addr(func_alloca, -func_scratch);
1043
1044     cur_text_section->data_offset = saved_ind;
1045     pe_add_unwind_data(ind, saved_ind, v);
1046     ind = cur_text_section->data_offset;
1047 }
1048
1049 #else
1050
1051 static void gadd_sp(int val)
1052 {
1053     if (val == (char)val) {
1054         o(0xc48348);
1055         g(val);
1056     } else {
1057         oad(0xc48148, val); /* add $xxx, %rsp */
1058     }
1059 }
1060
1061 typedef enum X86_64_Mode {
1062   x86_64_mode_none,
1063   x86_64_mode_memory,
1064   x86_64_mode_integer,
1065   x86_64_mode_sse,
1066   x86_64_mode_x87
1067 } X86_64_Mode;
1068
1069 static X86_64_Mode classify_x86_64_merge(X86_64_Mode a, X86_64_Mode b)
1070 {
1071     if (a == b)
1072         return a;
1073     else if (a == x86_64_mode_none)
1074         return b;
1075     else if (b == x86_64_mode_none)
1076         return a;
1077     else if ((a == x86_64_mode_memory) || (b == x86_64_mode_memory))
1078         return x86_64_mode_memory;
1079     else if ((a == x86_64_mode_integer) || (b == x86_64_mode_integer))
1080         return x86_64_mode_integer;
1081     else if ((a == x86_64_mode_x87) || (b == x86_64_mode_x87))
1082         return x86_64_mode_memory;
1083     else
1084         return x86_64_mode_sse;
1085 }
1086
1087 static X86_64_Mode classify_x86_64_inner(CType *ty)
1088 {
1089     X86_64_Mode mode;
1090     Sym *f;
1091
1092     switch (ty->t & VT_BTYPE) {
1093     case VT_VOID: return x86_64_mode_none;
1094
1095     case VT_INT:
1096     case VT_BYTE:
1097     case VT_SHORT:
1098     case VT_LLONG:
1099     case VT_BOOL:
1100     case VT_PTR:
1101     case VT_FUNC:
1102         return x86_64_mode_integer;
1103
1104     case VT_FLOAT:
1105     case VT_DOUBLE: return x86_64_mode_sse;
1106
1107     case VT_LDOUBLE: return x86_64_mode_x87;
1108
1109     case VT_STRUCT:
1110         f = ty->ref;
1111
1112         mode = x86_64_mode_none;
1113         for (f = f->next; f; f = f->next)
1114             mode = classify_x86_64_merge(mode, classify_x86_64_inner(&f->type));
1115
1116         return mode;
1117     }
1118     assert(0);
1119     return 0;
1120 }
1121
1122 static X86_64_Mode classify_x86_64_arg(CType *ty, CType *ret, int *psize, int *palign, int *reg_count)
1123 {
1124     X86_64_Mode mode;
1125     int size, align, ret_t = 0;
1126
1127     if (ty->t & (VT_BITFIELD|VT_ARRAY)) {
1128         *psize = 8;
1129         *palign = 8;
1130         *reg_count = 1;
1131         ret_t = ty->t;
1132         mode = x86_64_mode_integer;
1133     } else {
1134         size = type_size(ty, &align);
1135         *psize = (size + 7) & ~7;
1136         *palign = (align + 7) & ~7;
1137
1138         if (size > 16) {
1139             mode = x86_64_mode_memory;
1140         } else {
1141             mode = classify_x86_64_inner(ty);
1142             switch (mode) {
1143             case x86_64_mode_integer:
1144                 if (size > 8) {
1145                     *reg_count = 2;
1146                     ret_t = VT_QLONG;
1147                 } else {
1148                     *reg_count = 1;
1149                     if (size > 4)
1150                         ret_t = VT_LLONG;
1151                     else if (size > 2)
1152                         ret_t = VT_INT;
1153                     else if (size > 1)
1154                         ret_t = VT_SHORT;
1155                     else
1156                         ret_t = VT_BYTE;
1157                     if ((ty->t & VT_BTYPE) == VT_STRUCT || (ty->t & VT_UNSIGNED))
1158                         ret_t |= VT_UNSIGNED;
1159                 }
1160                 break;
1161
1162             case x86_64_mode_x87:
1163                 *reg_count = 1;
1164                 ret_t = VT_LDOUBLE;
1165                 break;
1166
1167             case x86_64_mode_sse:
1168                 if (size > 8) {
1169                     *reg_count = 2;
1170                     ret_t = VT_QFLOAT;
1171                 } else {
1172                     *reg_count = 1;
1173                     ret_t = (size > 4) ? VT_DOUBLE : VT_FLOAT;
1174                 }
1175                 break;
1176             default: break; /* nothing to be done for x86_64_mode_memory and x86_64_mode_none*/
1177             }
1178         }
1179     }
1180
1181     if (ret) {
1182         ret->ref = NULL;
1183         ret->t = ret_t;
1184     }
1185
1186     return mode;
1187 }
1188
1189 ST_FUNC int classify_x86_64_va_arg(CType *ty)
1190 {
1191     /* This definition must be synced with stdarg.h */
1192     enum __va_arg_type {
1193         __va_gen_reg, __va_float_reg, __va_stack
1194     };
1195     int size, align, reg_count;
1196     X86_64_Mode mode = classify_x86_64_arg(ty, NULL, &size, &align, &reg_count);
1197     switch (mode) {
1198     default: return __va_stack;
1199     case x86_64_mode_integer: return __va_gen_reg;
1200     case x86_64_mode_sse: return __va_float_reg;
1201     }
1202 }
1203
1204 /* Return the number of registers needed to return the struct, or 0 if
1205    returning via struct pointer. */
1206 ST_FUNC int gfunc_sret(CType *vt, int variadic, CType *ret, int *ret_align, int *regsize)
1207 {
1208     int size, align, reg_count;
1209     *ret_align = 1; // Never have to re-align return values for x86-64
1210     *regsize = 8;
1211     return (classify_x86_64_arg(vt, ret, &size, &align, &reg_count) != x86_64_mode_memory);
1212 }
1213
1214 #define REGN 6
1215 static const uint8_t arg_regs[REGN] = {
1216     TREG_RDI, TREG_RSI, TREG_RDX, TREG_RCX, TREG_R8, TREG_R9
1217 };
1218
1219 static int arg_prepare_reg(int idx) {
1220   if (idx == 2 || idx == 3)
1221       /* idx=2: r10, idx=3: r11 */
1222       return idx + 8;
1223   else
1224       return arg_regs[idx];
1225 }
1226
1227 /* Generate function call. The function address is pushed first, then
1228    all the parameters in call order. This functions pops all the
1229    parameters and the function address. */
1230 void gfunc_call(int nb_args)
1231 {
1232     X86_64_Mode mode;
1233     CType type;
1234     int size, align, r, args_size, stack_adjust, i, reg_count, k;
1235     int nb_reg_args = 0;
1236     int nb_sse_args = 0;
1237     int sse_reg, gen_reg;
1238     char *onstack = tcc_malloc((nb_args + 1) * sizeof (char));
1239
1240 #ifdef CONFIG_TCC_BCHECK
1241     if (tcc_state->do_bounds_check)
1242         gbound_args(nb_args);
1243 #endif
1244
1245     /* calculate the number of integer/float register arguments, remember
1246        arguments to be passed via stack (in onstack[]), and also remember
1247        if we have to align the stack pointer to 16 (onstack[i] == 2).  Needs
1248        to be done in a left-to-right pass over arguments.  */
1249     stack_adjust = 0;
1250     for(i = nb_args - 1; i >= 0; i--) {
1251         mode = classify_x86_64_arg(&vtop[-i].type, NULL, &size, &align, &reg_count);
1252         if (size == 0) continue;
1253         if (mode == x86_64_mode_sse && nb_sse_args + reg_count <= 8) {
1254             nb_sse_args += reg_count;
1255             onstack[i] = 0;
1256         } else if (mode == x86_64_mode_integer && nb_reg_args + reg_count <= REGN) {
1257             nb_reg_args += reg_count;
1258             onstack[i] = 0;
1259         } else if (mode == x86_64_mode_none) {
1260             onstack[i] = 0;
1261         } else {
1262             if (align == 16 && (stack_adjust &= 15)) {
1263                 onstack[i] = 2;
1264                 stack_adjust = 0;
1265             } else
1266               onstack[i] = 1;
1267             stack_adjust += size;
1268         }
1269     }
1270
1271     if (nb_sse_args && tcc_state->nosse)
1272       tcc_error("SSE disabled but floating point arguments passed");
1273
1274     /* fetch cpu flag before generating any code */
1275     if ((vtop->r & VT_VALMASK) == VT_CMP)
1276       gv(RC_INT);
1277
1278     /* for struct arguments, we need to call memcpy and the function
1279        call breaks register passing arguments we are preparing.
1280        So, we process arguments which will be passed by stack first. */
1281     gen_reg = nb_reg_args;
1282     sse_reg = nb_sse_args;
1283     args_size = 0;
1284     stack_adjust &= 15;
1285     for (i = k = 0; i < nb_args;) {
1286         mode = classify_x86_64_arg(&vtop[-i].type, NULL, &size, &align, &reg_count);
1287         if (size) {
1288             if (!onstack[i + k]) {
1289                 ++i;
1290                 continue;
1291             }
1292             /* Possibly adjust stack to align SSE boundary.  We're processing
1293                args from right to left while allocating happens left to right
1294                (stack grows down), so the adjustment needs to happen _after_
1295                an argument that requires it.  */
1296             if (stack_adjust) {
1297                 o(0x50); /* push %rax; aka sub $8,%rsp */
1298                 args_size += 8;
1299                 stack_adjust = 0;
1300             }
1301             if (onstack[i + k] == 2)
1302                 stack_adjust = 1;
1303         }
1304
1305         vrotb(i+1);
1306
1307         switch (vtop->type.t & VT_BTYPE) {
1308             case VT_STRUCT:
1309                 /* allocate the necessary size on stack */
1310                 o(0x48);
1311                 oad(0xec81, size); /* sub $xxx, %rsp */
1312                 /* generate structure store */
1313                 r = get_reg(RC_INT);
1314                 orex(1, r, 0, 0x89); /* mov %rsp, r */
1315                 o(0xe0 + REG_VALUE(r));
1316                 vset(&vtop->type, r | VT_LVAL, 0);
1317                 vswap();
1318                 vstore();
1319                 break;
1320
1321             case VT_LDOUBLE:
1322                 gv(RC_ST0);
1323                 oad(0xec8148, size); /* sub $xxx, %rsp */
1324                 o(0x7cdb); /* fstpt 0(%rsp) */
1325                 g(0x24);
1326                 g(0x00);
1327                 break;
1328
1329             case VT_FLOAT:
1330             case VT_DOUBLE:
1331                 assert(mode == x86_64_mode_sse);
1332                 r = gv(RC_FLOAT);
1333                 o(0x50); /* push $rax */
1334                 /* movq %xmmN, (%rsp) */
1335                 o(0xd60f66);
1336                 o(0x04 + REG_VALUE(r)*8);
1337                 o(0x24);
1338                 break;
1339
1340             default:
1341                 assert(mode == x86_64_mode_integer);
1342                 /* simple type */
1343                 /* XXX: implicit cast ? */
1344                 r = gv(RC_INT);
1345                 orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */
1346                 break;
1347         }
1348         args_size += size;
1349
1350         vpop();
1351         --nb_args;
1352         k++;
1353     }
1354
1355     tcc_free(onstack);
1356
1357     /* XXX This should be superfluous.  */
1358     save_regs(0); /* save used temporary registers */
1359
1360     /* then, we prepare register passing arguments.
1361        Note that we cannot set RDX and RCX in this loop because gv()
1362        may break these temporary registers. Let's use R10 and R11
1363        instead of them */
1364     assert(gen_reg <= REGN);
1365     assert(sse_reg <= 8);
1366     for(i = 0; i < nb_args; i++) {
1367         mode = classify_x86_64_arg(&vtop->type, &type, &size, &align, &reg_count);
1368         if (size == 0) continue;
1369         /* Alter stack entry type so that gv() knows how to treat it */
1370         vtop->type = type;
1371         if (mode == x86_64_mode_sse) {
1372             if (reg_count == 2) {
1373                 sse_reg -= 2;
1374                 gv(RC_FRET); /* Use pair load into xmm0 & xmm1 */
1375                 if (sse_reg) { /* avoid redundant movaps %xmm0, %xmm0 */
1376                     /* movaps %xmm1, %xmmN */
1377                     o(0x280f);
1378                     o(0xc1 + ((sse_reg+1) << 3));
1379                     /* movaps %xmm0, %xmmN */
1380                     o(0x280f);
1381                     o(0xc0 + (sse_reg << 3));
1382                 }
1383             } else {
1384                 assert(reg_count == 1);
1385                 --sse_reg;
1386                 /* Load directly to register */
1387                 gv(RC_XMM0 << sse_reg);
1388             }
1389         } else if (mode == x86_64_mode_integer) {
1390             /* simple type */
1391             /* XXX: implicit cast ? */
1392             int d;
1393             gen_reg -= reg_count;
1394             r = gv(RC_INT);
1395             d = arg_prepare_reg(gen_reg);
1396             orex(1,d,r,0x89); /* mov */
1397             o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d));
1398             if (reg_count == 2) {
1399                 d = arg_prepare_reg(gen_reg+1);
1400                 orex(1,d,vtop->r2,0x89); /* mov */
1401                 o(0xc0 + REG_VALUE(vtop->r2) * 8 + REG_VALUE(d));
1402             }
1403         }
1404         vtop--;
1405     }
1406     assert(gen_reg == 0);
1407     assert(sse_reg == 0);
1408
1409     /* We shouldn't have many operands on the stack anymore, but the
1410        call address itself is still there, and it might be in %eax
1411        (or edx/ecx) currently, which the below writes would clobber.
1412        So evict all remaining operands here.  */
1413     save_regs(0);
1414
1415     /* Copy R10 and R11 into RDX and RCX, respectively */
1416     if (nb_reg_args > 2) {
1417         o(0xd2894c); /* mov %r10, %rdx */
1418         if (nb_reg_args > 3) {
1419             o(0xd9894c); /* mov %r11, %rcx */
1420         }
1421     }
1422
1423     if (vtop->type.ref->f.func_type != FUNC_NEW) /* implies FUNC_OLD or FUNC_ELLIPSIS */
1424         oad(0xb8, nb_sse_args < 8 ? nb_sse_args : 8); /* mov nb_sse_args, %eax */
1425     gcall_or_jmp(0);
1426     if (args_size)
1427         gadd_sp(args_size);
1428     vtop--;
1429 }
1430
1431 #define FUNC_PROLOG_SIZE 11
1432
1433 static void push_arg_reg(int i) {
1434     loc -= 8;
1435     gen_modrm64(0x89, arg_regs[i], VT_LOCAL, NULL, loc);
1436 }
1437
1438 /* generate function prolog of type 't' */
1439 void gfunc_prolog(Sym *func_sym)
1440 {
1441     CType *func_type = &func_sym->type;
1442     X86_64_Mode mode;
1443     int i, addr, align, size, reg_count;
1444     int param_addr = 0, reg_param_index, sse_param_index;
1445     Sym *sym;
1446     CType *type;
1447
1448     sym = func_type->ref;
1449     addr = PTR_SIZE * 2;
1450     loc = 0;
1451     ind += FUNC_PROLOG_SIZE;
1452     func_sub_sp_offset = ind;
1453     func_ret_sub = 0;
1454
1455     if (func_var) {
1456         int seen_reg_num, seen_sse_num, seen_stack_size;
1457         seen_reg_num = seen_sse_num = 0;
1458         /* frame pointer and return address */
1459         seen_stack_size = PTR_SIZE * 2;
1460         /* count the number of seen parameters */
1461         sym = func_type->ref;
1462         while ((sym = sym->next) != NULL) {
1463             type = &sym->type;
1464             mode = classify_x86_64_arg(type, NULL, &size, &align, &reg_count);
1465             switch (mode) {
1466             default:
1467             stack_arg:
1468                 seen_stack_size = ((seen_stack_size + align - 1) & -align) + size;
1469                 break;
1470
1471             case x86_64_mode_integer:
1472                 if (seen_reg_num + reg_count > REGN)
1473                     goto stack_arg;
1474                 seen_reg_num += reg_count;
1475                 break;
1476
1477             case x86_64_mode_sse:
1478                 if (seen_sse_num + reg_count > 8)
1479                     goto stack_arg;
1480                 seen_sse_num += reg_count;
1481                 break;
1482             }
1483         }
1484
1485         loc -= 24;
1486         /* movl $0x????????, -0x18(%rbp) */
1487         o(0xe845c7);
1488         gen_le32(seen_reg_num * 8);
1489         /* movl $0x????????, -0x14(%rbp) */
1490         o(0xec45c7);
1491         gen_le32(seen_sse_num * 16 + 48);
1492         /* leaq $0x????????, %r11 */
1493         o(0x9d8d4c);
1494         gen_le32(seen_stack_size);
1495         /* movq %r11, -0x10(%rbp) */
1496         o(0xf05d894c);
1497         /* leaq $-192(%rbp), %r11 */
1498         o(0x9d8d4c);
1499         gen_le32(-176 - 24);
1500         /* movq %r11, -0x8(%rbp) */
1501         o(0xf85d894c);
1502
1503         /* save all register passing arguments */
1504         for (i = 0; i < 8; i++) {
1505             loc -= 16;
1506             if (!tcc_state->nosse) {
1507                 o(0xd60f66); /* movq */
1508                 gen_modrm(7 - i, VT_LOCAL, NULL, loc);
1509             }
1510             /* movq $0, loc+8(%rbp) */
1511             o(0x85c748);
1512             gen_le32(loc + 8);
1513             gen_le32(0);
1514         }
1515         for (i = 0; i < REGN; i++) {
1516             push_arg_reg(REGN-1-i);
1517         }
1518     }
1519
1520     sym = func_type->ref;
1521     reg_param_index = 0;
1522     sse_param_index = 0;
1523
1524     /* if the function returns a structure, then add an
1525        implicit pointer parameter */
1526     mode = classify_x86_64_arg(&func_vt, NULL, &size, &align, &reg_count);
1527     if (mode == x86_64_mode_memory) {
1528         push_arg_reg(reg_param_index);
1529         func_vc = loc;
1530         reg_param_index++;
1531     }
1532     /* define parameters */
1533     while ((sym = sym->next) != NULL) {
1534         type = &sym->type;
1535         mode = classify_x86_64_arg(type, NULL, &size, &align, &reg_count);
1536         switch (mode) {
1537         case x86_64_mode_sse:
1538             if (tcc_state->nosse)
1539                 tcc_error("SSE disabled but floating point arguments used");
1540             if (sse_param_index + reg_count <= 8) {
1541                 /* save arguments passed by register */
1542                 loc -= reg_count * 8;
1543                 param_addr = loc;
1544                 for (i = 0; i < reg_count; ++i) {
1545                     o(0xd60f66); /* movq */
1546                     gen_modrm(sse_param_index, VT_LOCAL, NULL, param_addr + i*8);
1547                     ++sse_param_index;
1548                 }
1549             } else {
1550                 addr = (addr + align - 1) & -align;
1551                 param_addr = addr;
1552                 addr += size;
1553             }
1554             break;
1555
1556         case x86_64_mode_memory:
1557         case x86_64_mode_x87:
1558             addr = (addr + align - 1) & -align;
1559             param_addr = addr;
1560             addr += size;
1561             break;
1562
1563         case x86_64_mode_integer: {
1564             if (reg_param_index + reg_count <= REGN) {
1565                 /* save arguments passed by register */
1566                 loc -= reg_count * 8;
1567                 param_addr = loc;
1568                 for (i = 0; i < reg_count; ++i) {
1569                     gen_modrm64(0x89, arg_regs[reg_param_index], VT_LOCAL, NULL, param_addr + i*8);
1570                     ++reg_param_index;
1571                 }
1572             } else {
1573                 addr = (addr + align - 1) & -align;
1574                 param_addr = addr;
1575                 addr += size;
1576             }
1577             break;
1578         }
1579         default: break; /* nothing to be done for x86_64_mode_none */
1580         }
1581         sym_push(sym->v & ~SYM_FIELD, type,
1582                  VT_LOCAL | VT_LVAL, param_addr);
1583     }
1584
1585 #ifdef CONFIG_TCC_BCHECK
1586     if (tcc_state->do_bounds_check)
1587         gen_bounds_prolog();
1588 #endif
1589 }
1590
1591 /* generate function epilog */
1592 void gfunc_epilog(void)
1593 {
1594     int v, saved_ind;
1595
1596 #ifdef CONFIG_TCC_BCHECK
1597     if (tcc_state->do_bounds_check)
1598         gen_bounds_epilog();
1599 #endif
1600     o(0xc9); /* leave */
1601     if (func_ret_sub == 0) {
1602         o(0xc3); /* ret */
1603     } else {
1604         o(0xc2); /* ret n */
1605         g(func_ret_sub);
1606         g(func_ret_sub >> 8);
1607     }
1608     /* align local size to word & save local variables */
1609     v = (-loc + 15) & -16;
1610     saved_ind = ind;
1611     ind = func_sub_sp_offset - FUNC_PROLOG_SIZE;
1612     o(0xe5894855);  /* push %rbp, mov %rsp, %rbp */
1613     o(0xec8148);  /* sub rsp, stacksize */
1614     gen_le32(v);
1615     ind = saved_ind;
1616 }
1617
1618 #endif /* not PE */
1619
1620 ST_FUNC void gen_fill_nops(int bytes)
1621 {
1622     while (bytes--)
1623       g(0x90);
1624 }
1625
1626 /* generate a jump to a label */
1627 int gjmp(int t)
1628 {
1629     return gjmp2(0xe9, t);
1630 }
1631
1632 /* generate a jump to a fixed address */
1633 void gjmp_addr(int a)
1634 {
1635     int r;
1636     r = a - ind - 2;
1637     if (r == (char)r) {
1638         g(0xeb);
1639         g(r);
1640     } else {
1641         oad(0xe9, a - ind - 5);
1642     }
1643 }
1644
1645 ST_FUNC int gjmp_append(int n, int t)
1646 {
1647     void *p;
1648     /* insert vtop->c jump list in t */
1649     if (n) {
1650         uint32_t n1 = n, n2;
1651         while ((n2 = read32le(p = cur_text_section->data + n1)))
1652             n1 = n2;
1653         write32le(p, t);
1654         t = n;
1655     }
1656     return t;
1657 }
1658
1659 ST_FUNC int gjmp_cond(int op, int t)
1660 {
1661         if (op & 0x100)
1662           {
1663             /* This was a float compare.  If the parity flag is set
1664                the result was unordered.  For anything except != this
1665                means false and we don't jump (anding both conditions).
1666                For != this means true (oring both).
1667                Take care about inverting the test.  We need to jump
1668                to our target if the result was unordered and test wasn't NE,
1669                otherwise if unordered we don't want to jump.  */
1670             int v = vtop->cmp_r;
1671             op &= ~0x100;
1672             if (op ^ v ^ (v != TOK_NE))
1673               o(0x067a);  /* jp +6 */
1674             else
1675               {
1676                 g(0x0f);
1677                 t = gjmp2(0x8a, t); /* jp t */
1678               }
1679           }
1680         g(0x0f);
1681         t = gjmp2(op - 16, t);
1682         return t;
1683 }
1684
1685 /* generate an integer binary operation */
1686 void gen_opi(int op)
1687 {
1688     int r, fr, opc, c;
1689     int ll, uu, cc;
1690
1691     ll = is64_type(vtop[-1].type.t);
1692     uu = (vtop[-1].type.t & VT_UNSIGNED) != 0;
1693     cc = (vtop->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST;
1694
1695     switch(op) {
1696     case '+':
1697     case TOK_ADDC1: /* add with carry generation */
1698         opc = 0;
1699     gen_op8:
1700         if (cc && (!ll || (int)vtop->c.i == vtop->c.i)) {
1701             /* constant case */
1702             vswap();
1703             r = gv(RC_INT);
1704             vswap();
1705             c = vtop->c.i;
1706             if (c == (char)c) {
1707                 /* XXX: generate inc and dec for smaller code ? */
1708                 orex(ll, r, 0, 0x83);
1709                 o(0xc0 | (opc << 3) | REG_VALUE(r));
1710                 g(c);
1711             } else {
1712                 orex(ll, r, 0, 0x81);
1713                 oad(0xc0 | (opc << 3) | REG_VALUE(r), c);
1714             }
1715         } else {
1716             gv2(RC_INT, RC_INT);
1717             r = vtop[-1].r;
1718             fr = vtop[0].r;
1719             orex(ll, r, fr, (opc << 3) | 0x01);
1720             o(0xc0 + REG_VALUE(r) + REG_VALUE(fr) * 8);
1721         }
1722         vtop--;
1723         if (op >= TOK_ULT && op <= TOK_GT)
1724             vset_VT_CMP(op);
1725         break;
1726     case '-':
1727     case TOK_SUBC1: /* sub with carry generation */
1728         opc = 5;
1729         goto gen_op8;
1730     case TOK_ADDC2: /* add with carry use */
1731         opc = 2;
1732         goto gen_op8;
1733     case TOK_SUBC2: /* sub with carry use */
1734         opc = 3;
1735         goto gen_op8;
1736     case '&':
1737         opc = 4;
1738         goto gen_op8;
1739     case '^':
1740         opc = 6;
1741         goto gen_op8;
1742     case '|':
1743         opc = 1;
1744         goto gen_op8;
1745     case '*':
1746         gv2(RC_INT, RC_INT);
1747         r = vtop[-1].r;
1748         fr = vtop[0].r;
1749         orex(ll, fr, r, 0xaf0f); /* imul fr, r */
1750         o(0xc0 + REG_VALUE(fr) + REG_VALUE(r) * 8);
1751         vtop--;
1752         break;
1753     case TOK_SHL:
1754         opc = 4;
1755         goto gen_shift;
1756     case TOK_SHR:
1757         opc = 5;
1758         goto gen_shift;
1759     case TOK_SAR:
1760         opc = 7;
1761     gen_shift:
1762         opc = 0xc0 | (opc << 3);
1763         if (cc) {
1764             /* constant case */
1765             vswap();
1766             r = gv(RC_INT);
1767             vswap();
1768             orex(ll, r, 0, 0xc1); /* shl/shr/sar $xxx, r */
1769             o(opc | REG_VALUE(r));
1770             g(vtop->c.i & (ll ? 63 : 31));
1771         } else {
1772             /* we generate the shift in ecx */
1773             gv2(RC_INT, RC_RCX);
1774             r = vtop[-1].r;
1775             orex(ll, r, 0, 0xd3); /* shl/shr/sar %cl, r */
1776             o(opc | REG_VALUE(r));
1777         }
1778         vtop--;
1779         break;
1780     case TOK_UDIV:
1781     case TOK_UMOD:
1782         uu = 1;
1783         goto divmod;
1784     case '/':
1785     case '%':
1786     case TOK_PDIV:
1787         uu = 0;
1788     divmod:
1789         /* first operand must be in eax */
1790         /* XXX: need better constraint for second operand */
1791         gv2(RC_RAX, RC_RCX);
1792         r = vtop[-1].r;
1793         fr = vtop[0].r;
1794         vtop--;
1795         save_reg(TREG_RDX);
1796         orex(ll, 0, 0, uu ? 0xd231 : 0x99); /* xor %edx,%edx : cqto */
1797         orex(ll, fr, 0, 0xf7); /* div fr, %eax */
1798         o((uu ? 0xf0 : 0xf8) + REG_VALUE(fr));
1799         if (op == '%' || op == TOK_UMOD)
1800             r = TREG_RDX;
1801         else
1802             r = TREG_RAX;
1803         vtop->r = r;
1804         break;
1805     default:
1806         opc = 7;
1807         goto gen_op8;
1808     }
1809 }
1810
1811 void gen_opl(int op)
1812 {
1813     gen_opi(op);
1814 }
1815
1816 /* generate a floating point operation 'v = t1 op t2' instruction. The
1817    two operands are guaranteed to have the same floating point type */
1818 /* XXX: need to use ST1 too */
1819 void gen_opf(int op)
1820 {
1821     int a, ft, fc, swapped, r;
1822     int float_type =
1823         (vtop->type.t & VT_BTYPE) == VT_LDOUBLE ? RC_ST0 : RC_FLOAT;
1824
1825     /* convert constants to memory references */
1826     if ((vtop[-1].r & (VT_VALMASK | VT_LVAL)) == VT_CONST) {
1827         vswap();
1828         gv(float_type);
1829         vswap();
1830     }
1831     if ((vtop[0].r & (VT_VALMASK | VT_LVAL)) == VT_CONST)
1832         gv(float_type);
1833
1834     /* must put at least one value in the floating point register */
1835     if ((vtop[-1].r & VT_LVAL) &&
1836         (vtop[0].r & VT_LVAL)) {
1837         vswap();
1838         gv(float_type);
1839         vswap();
1840     }
1841     swapped = 0;
1842     /* swap the stack if needed so that t1 is the register and t2 is
1843        the memory reference */
1844     if (vtop[-1].r & VT_LVAL) {
1845         vswap();
1846         swapped = 1;
1847     }
1848     if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
1849         if (op >= TOK_ULT && op <= TOK_GT) {
1850             /* load on stack second operand */
1851             load(TREG_ST0, vtop);
1852             save_reg(TREG_RAX); /* eax is used by FP comparison code */
1853             if (op == TOK_GE || op == TOK_GT)
1854                 swapped = !swapped;
1855             else if (op == TOK_EQ || op == TOK_NE)
1856                 swapped = 0;
1857             if (swapped)
1858                 o(0xc9d9); /* fxch %st(1) */
1859             if (op == TOK_EQ || op == TOK_NE)
1860                 o(0xe9da); /* fucompp */
1861             else
1862                 o(0xd9de); /* fcompp */
1863             o(0xe0df); /* fnstsw %ax */
1864             if (op == TOK_EQ) {
1865                 o(0x45e480); /* and $0x45, %ah */
1866                 o(0x40fC80); /* cmp $0x40, %ah */
1867             } else if (op == TOK_NE) {
1868                 o(0x45e480); /* and $0x45, %ah */
1869                 o(0x40f480); /* xor $0x40, %ah */
1870                 op = TOK_NE;
1871             } else if (op == TOK_GE || op == TOK_LE) {
1872                 o(0x05c4f6); /* test $0x05, %ah */
1873                 op = TOK_EQ;
1874             } else {
1875                 o(0x45c4f6); /* test $0x45, %ah */
1876                 op = TOK_EQ;
1877             }
1878             vtop--;
1879             vset_VT_CMP(op);
1880         } else {
1881             /* no memory reference possible for long double operations */
1882             load(TREG_ST0, vtop);
1883             swapped = !swapped;
1884
1885             switch(op) {
1886             default:
1887             case '+':
1888                 a = 0;
1889                 break;
1890             case '-':
1891                 a = 4;
1892                 if (swapped)
1893                     a++;
1894                 break;
1895             case '*':
1896                 a = 1;
1897                 break;
1898             case '/':
1899                 a = 6;
1900                 if (swapped)
1901                     a++;
1902                 break;
1903             }
1904             ft = vtop->type.t;
1905             fc = vtop->c.i;
1906             o(0xde); /* fxxxp %st, %st(1) */
1907             o(0xc1 + (a << 3));
1908             vtop--;
1909         }
1910     } else {
1911         if (op >= TOK_ULT && op <= TOK_GT) {
1912             /* if saved lvalue, then we must reload it */
1913             r = vtop->r;
1914             fc = vtop->c.i;
1915             if ((r & VT_VALMASK) == VT_LLOCAL) {
1916                 SValue v1;
1917                 r = get_reg(RC_INT);
1918                 v1.type.t = VT_PTR;
1919                 v1.r = VT_LOCAL | VT_LVAL;
1920                 v1.c.i = fc;
1921                 load(r, &v1);
1922                 fc = 0;
1923                 vtop->r = r = r | VT_LVAL;
1924             }
1925
1926             if (op == TOK_EQ || op == TOK_NE) {
1927                 swapped = 0;
1928             } else {
1929                 if (op == TOK_LE || op == TOK_LT)
1930                     swapped = !swapped;
1931                 if (op == TOK_LE || op == TOK_GE) {
1932                     op = 0x93; /* setae */
1933                 } else {
1934                     op = 0x97; /* seta */
1935                 }
1936             }
1937
1938             if (swapped) {
1939                 gv(RC_FLOAT);
1940                 vswap();
1941             }
1942             assert(!(vtop[-1].r & VT_LVAL));
1943
1944             if ((vtop->type.t & VT_BTYPE) == VT_DOUBLE)
1945                 o(0x66);
1946             if (op == TOK_EQ || op == TOK_NE)
1947                 o(0x2e0f); /* ucomisd */
1948             else
1949                 o(0x2f0f); /* comisd */
1950
1951             if (vtop->r & VT_LVAL) {
1952                 gen_modrm(vtop[-1].r, r, vtop->sym, fc);
1953             } else {
1954                 o(0xc0 + REG_VALUE(vtop[0].r) + REG_VALUE(vtop[-1].r)*8);
1955             }
1956
1957             vtop--;
1958             vset_VT_CMP(op | 0x100);
1959             vtop->cmp_r = op;
1960         } else {
1961             assert((vtop->type.t & VT_BTYPE) != VT_LDOUBLE);
1962             switch(op) {
1963             default:
1964             case '+':
1965                 a = 0;
1966                 break;
1967             case '-':
1968                 a = 4;
1969                 break;
1970             case '*':
1971                 a = 1;
1972                 break;
1973             case '/':
1974                 a = 6;
1975                 break;
1976             }
1977             ft = vtop->type.t;
1978             fc = vtop->c.i;
1979             assert((ft & VT_BTYPE) != VT_LDOUBLE);
1980
1981             r = vtop->r;
1982             /* if saved lvalue, then we must reload it */
1983             if ((vtop->r & VT_VALMASK) == VT_LLOCAL) {
1984                 SValue v1;
1985                 r = get_reg(RC_INT);
1986                 v1.type.t = VT_PTR;
1987                 v1.r = VT_LOCAL | VT_LVAL;
1988                 v1.c.i = fc;
1989                 load(r, &v1);
1990                 fc = 0;
1991                 vtop->r = r = r | VT_LVAL;
1992             }
1993
1994             assert(!(vtop[-1].r & VT_LVAL));
1995             if (swapped) {
1996                 assert(vtop->r & VT_LVAL);
1997                 gv(RC_FLOAT);
1998                 vswap();
1999             }
2000
2001             if ((ft & VT_BTYPE) == VT_DOUBLE) {
2002                 o(0xf2);
2003             } else {
2004                 o(0xf3);
2005             }
2006             o(0x0f);
2007             o(0x58 + a);
2008
2009             if (vtop->r & VT_LVAL) {
2010                 gen_modrm(vtop[-1].r, r, vtop->sym, fc);
2011             } else {
2012                 o(0xc0 + REG_VALUE(vtop[0].r) + REG_VALUE(vtop[-1].r)*8);
2013             }
2014
2015             vtop--;
2016         }
2017     }
2018 }
2019
2020 /* convert integers to fp 't' type. Must handle 'int', 'unsigned int'
2021    and 'long long' cases. */
2022 void gen_cvt_itof(int t)
2023 {
2024     if ((t & VT_BTYPE) == VT_LDOUBLE) {
2025         save_reg(TREG_ST0);
2026         gv(RC_INT);
2027         if ((vtop->type.t & VT_BTYPE) == VT_LLONG) {
2028             /* signed long long to float/double/long double (unsigned case
2029                is handled generically) */
2030             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
2031             o(0x242cdf); /* fildll (%rsp) */
2032             o(0x08c48348); /* add $8, %rsp */
2033         } else if ((vtop->type.t & (VT_BTYPE | VT_UNSIGNED)) ==
2034                    (VT_INT | VT_UNSIGNED)) {
2035             /* unsigned int to float/double/long double */
2036             o(0x6a); /* push $0 */
2037             g(0x00);
2038             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
2039             o(0x242cdf); /* fildll (%rsp) */
2040             o(0x10c48348); /* add $16, %rsp */
2041         } else {
2042             /* int to float/double/long double */
2043             o(0x50 + (vtop->r & VT_VALMASK)); /* push r */
2044             o(0x2404db); /* fildl (%rsp) */
2045             o(0x08c48348); /* add $8, %rsp */
2046         }
2047         vtop->r = TREG_ST0;
2048     } else {
2049         int r = get_reg(RC_FLOAT);
2050         gv(RC_INT);
2051         o(0xf2 + ((t & VT_BTYPE) == VT_FLOAT?1:0));
2052         if ((vtop->type.t & (VT_BTYPE | VT_UNSIGNED)) ==
2053             (VT_INT | VT_UNSIGNED) ||
2054             (vtop->type.t & VT_BTYPE) == VT_LLONG) {
2055             o(0x48); /* REX */
2056         }
2057         o(0x2a0f);
2058         o(0xc0 + (vtop->r & VT_VALMASK) + REG_VALUE(r)*8); /* cvtsi2sd */
2059         vtop->r = r;
2060     }
2061 }
2062
2063 /* convert from one floating point type to another */
2064 void gen_cvt_ftof(int t)
2065 {
2066     int ft, bt, tbt;
2067
2068     ft = vtop->type.t;
2069     bt = ft & VT_BTYPE;
2070     tbt = t & VT_BTYPE;
2071
2072     if (bt == VT_FLOAT) {
2073         gv(RC_FLOAT);
2074         if (tbt == VT_DOUBLE) {
2075             o(0x140f); /* unpcklps */
2076             o(0xc0 + REG_VALUE(vtop->r)*9);
2077             o(0x5a0f); /* cvtps2pd */
2078             o(0xc0 + REG_VALUE(vtop->r)*9);
2079         } else if (tbt == VT_LDOUBLE) {
2080             save_reg(RC_ST0);
2081             /* movss %xmm0,-0x10(%rsp) */
2082             o(0x110ff3);
2083             o(0x44 + REG_VALUE(vtop->r)*8);
2084             o(0xf024);
2085             o(0xf02444d9); /* flds -0x10(%rsp) */
2086             vtop->r = TREG_ST0;
2087         }
2088     } else if (bt == VT_DOUBLE) {
2089         gv(RC_FLOAT);
2090         if (tbt == VT_FLOAT) {
2091             o(0x140f66); /* unpcklpd */
2092             o(0xc0 + REG_VALUE(vtop->r)*9);
2093             o(0x5a0f66); /* cvtpd2ps */
2094             o(0xc0 + REG_VALUE(vtop->r)*9);
2095         } else if (tbt == VT_LDOUBLE) {
2096             save_reg(RC_ST0);
2097             /* movsd %xmm0,-0x10(%rsp) */
2098             o(0x110ff2);
2099             o(0x44 + REG_VALUE(vtop->r)*8);
2100             o(0xf024);
2101             o(0xf02444dd); /* fldl -0x10(%rsp) */
2102             vtop->r = TREG_ST0;
2103         }
2104     } else {
2105         int r;
2106         gv(RC_ST0);
2107         r = get_reg(RC_FLOAT);
2108         if (tbt == VT_DOUBLE) {
2109             o(0xf0245cdd); /* fstpl -0x10(%rsp) */
2110             /* movsd -0x10(%rsp),%xmm0 */
2111             o(0x100ff2);
2112             o(0x44 + REG_VALUE(r)*8);
2113             o(0xf024);
2114             vtop->r = r;
2115         } else if (tbt == VT_FLOAT) {
2116             o(0xf0245cd9); /* fstps -0x10(%rsp) */
2117             /* movss -0x10(%rsp),%xmm0 */
2118             o(0x100ff3);
2119             o(0x44 + REG_VALUE(r)*8);
2120             o(0xf024);
2121             vtop->r = r;
2122         }
2123     }
2124 }
2125
2126 /* convert fp to int 't' type */
2127 void gen_cvt_ftoi(int t)
2128 {
2129     int ft, bt, size, r;
2130     ft = vtop->type.t;
2131     bt = ft & VT_BTYPE;
2132     if (bt == VT_LDOUBLE) {
2133         gen_cvt_ftof(VT_DOUBLE);
2134         bt = VT_DOUBLE;
2135     }
2136
2137     gv(RC_FLOAT);
2138     if (t != VT_INT)
2139         size = 8;
2140     else
2141         size = 4;
2142
2143     r = get_reg(RC_INT);
2144     if (bt == VT_FLOAT) {
2145         o(0xf3);
2146     } else if (bt == VT_DOUBLE) {
2147         o(0xf2);
2148     } else {
2149         assert(0);
2150     }
2151     orex(size == 8, r, 0, 0x2c0f); /* cvttss2si or cvttsd2si */
2152     o(0xc0 + REG_VALUE(vtop->r) + REG_VALUE(r)*8);
2153     vtop->r = r;
2154 }
2155
2156 // Generate sign extension from 32 to 64 bits:
2157 ST_FUNC void gen_cvt_sxtw(void)
2158 {
2159     int r = gv(RC_INT);
2160     /* x86_64 specific: movslq */
2161     o(0x6348);
2162     o(0xc0 + (REG_VALUE(r) << 3) + REG_VALUE(r));
2163 }
2164
2165 /* char/short to int conversion */
2166 ST_FUNC void gen_cvt_csti(int t)
2167 {
2168     int r, sz, xl, ll;
2169     r = gv(RC_INT);
2170     sz = !(t & VT_UNSIGNED);
2171     xl = (t & VT_BTYPE) == VT_SHORT;
2172     ll = (vtop->type.t & VT_BTYPE) == VT_LLONG;
2173     orex(ll, r, 0, 0xc0b60f /* mov[sz] %a[xl], %eax */
2174         | (sz << 3 | xl) << 8
2175         | (REG_VALUE(r) << 3 | REG_VALUE(r)) << 16
2176         );
2177 }
2178
2179 /* computed goto support */
2180 void ggoto(void)
2181 {
2182     gcall_or_jmp(1);
2183     vtop--;
2184 }
2185
2186 /* Save the stack pointer onto the stack and return the location of its address */
2187 ST_FUNC void gen_vla_sp_save(int addr) {
2188     /* mov %rsp,addr(%rbp)*/
2189     gen_modrm64(0x89, TREG_RSP, VT_LOCAL, NULL, addr);
2190 }
2191
2192 /* Restore the SP from a location on the stack */
2193 ST_FUNC void gen_vla_sp_restore(int addr) {
2194     gen_modrm64(0x8b, TREG_RSP, VT_LOCAL, NULL, addr);
2195 }
2196
2197 #ifdef TCC_TARGET_PE
2198 /* Save result of gen_vla_alloc onto the stack */
2199 ST_FUNC void gen_vla_result(int addr) {
2200     /* mov %rax,addr(%rbp)*/
2201     gen_modrm64(0x89, TREG_RAX, VT_LOCAL, NULL, addr);
2202 }
2203 #endif
2204
2205 /* Subtract from the stack pointer, and push the resulting value onto the stack */
2206 ST_FUNC void gen_vla_alloc(CType *type, int align) {
2207     int use_call = 0;
2208
2209 #if defined(CONFIG_TCC_BCHECK)
2210     use_call = tcc_state->do_bounds_check;
2211 #endif
2212 #ifdef TCC_TARGET_PE    /* alloca does more than just adjust %rsp on Windows */
2213     use_call = 1;
2214 #endif
2215     if (use_call)
2216     {
2217         vpush_helper_func(TOK_alloca);
2218         vswap(); /* Move alloca ref past allocation size */
2219         gfunc_call(1);
2220     }
2221     else {
2222         int r;
2223         r = gv(RC_INT); /* allocation size */
2224         /* sub r,%rsp */
2225         o(0x2b48);
2226         o(0xe0 | REG_VALUE(r));
2227         /* We align to 16 bytes rather than align */
2228         /* and ~15, %rsp */
2229         o(0xf0e48348);
2230         vpop();
2231     }
2232 }
2233
2234
2235 /* end of x86-64 code generator */
2236 /*************************************************************/
2237 #endif /* ! TARGET_DEFS_ONLY */
2238 /******************************************************/