src/lj_opt_split.c

   1 /*
   2 ** SPLIT: Split 64 bit IR instructions into 32 bit IR instructions.
   3 ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_opt_split_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10
  11 #if LJ_HASJIT && (LJ_SOFTFP || (LJ_32 && LJ_HASFFI))
  12
  13 #include "lj_err.h"
  14 #include "lj_str.h"
  15 #include "lj_ir.h"
  16 #include "lj_jit.h"
  17 #include "lj_ircall.h"
  18 #include "lj_iropt.h"
  19 #include "lj_vm.h"
  20
  21 /* SPLIT pass:
  22 **
  23 ** This pass splits up 64 bit IR instructions into multiple 32 bit IR
  24 ** instructions. It's only active for soft-float targets or for 32 bit CPUs
  25 ** which lack native 64 bit integer operations (the FFI is currently the
  26 ** only emitter for 64 bit integer instructions).
  27 **
  28 ** Splitting the IR in a separate pass keeps each 32 bit IR assembler
  29 ** backend simple. Only a small amount of extra functionality needs to be
  30 ** implemented. This is much easier than adding support for allocating
  31 ** register pairs to each backend (believe me, I tried). A few simple, but
  32 ** important optimizations can be performed by the SPLIT pass, which would
  33 ** be tedious to do in the backend.
  34 **
  35 ** The basic idea is to replace each 64 bit IR instruction with its 32 bit
  36 ** equivalent plus an extra HIOP instruction. The splitted IR is not passed
  37 ** through FOLD or any other optimizations, so each HIOP is guaranteed to
  38 ** immediately follow it's counterpart. The actual functionality of HIOP is
  39 ** inferred from the previous instruction.
  40 **
  41 ** The operands of HIOP hold the hiword input references. The output of HIOP
  42 ** is the hiword output reference, which is also used to hold the hiword
  43 ** register or spill slot information. The register allocator treats this
  44 ** instruction independently of any other instruction, which improves code
  45 ** quality compared to using fixed register pairs.
  46 **
  47 ** It's easier to split up some instructions into two regular 32 bit
  48 ** instructions. E.g. XLOAD is split up into two XLOADs with two different
  49 ** addresses. Obviously 64 bit constants need to be split up into two 32 bit
  50 ** constants, too. Some hiword instructions can be entirely omitted, e.g.
  51 ** when zero-extending a 32 bit value to 64 bits. 64 bit arguments for calls
  52 ** are split up into two 32 bit arguments each.
  53 **
  54 ** On soft-float targets, floating-point instructions are directly converted
  55 ** to soft-float calls by the SPLIT pass (except for comparisons and MIN/MAX).
  56 ** HIOP for number results has the type IRT_SOFTFP ("sfp" in -jdump).
  57 **
  58 ** Here's the IR and x64 machine code for 'x.b = x.a + 1' for a struct with
  59 ** two int64_t fields:
  60 **
  61 ** 0100    p32 ADD    base  +8
  62 ** 0101    i64 XLOAD  0100
  63 ** 0102    i64 ADD    0101  +1
  64 ** 0103    p32 ADD    base  +16
  65 ** 0104    i64 XSTORE 0103  0102
  66 **
  67 **         mov rax, [esi+0x8]
  68 **         add rax, +0x01
  69 **         mov [esi+0x10], rax
  70 **
  71 ** Here's the transformed IR and the x86 machine code after the SPLIT pass:
  72 **
  73 ** 0100    p32 ADD    base  +8
  74 ** 0101    int XLOAD  0100
  75 ** 0102    p32 ADD    base  +12
  76 ** 0103    int XLOAD  0102
  77 ** 0104    int ADD    0101  +1
  78 ** 0105    int HIOP   0103  +0
  79 ** 0106    p32 ADD    base  +16
  80 ** 0107    int XSTORE 0106  0104
  81 ** 0108    p32 ADD    base  +20
  82 ** 0109    int XSTORE 0108  0105
  83 **
  84 **         mov eax, [esi+0x8]
  85 **         mov ecx, [esi+0xc]
  86 **         add eax, +0x01
  87 **         adc ecx, +0x00
  88 **         mov [esi+0x10], eax
  89 **         mov [esi+0x14], ecx
  90 **
  91 ** You may notice the reassociated hiword address computation, which is
  92 ** later fused into the mov operands by the assembler.
  93 */
  94
  95 /* Some local macros to save typing. Undef'd at the end. */
  96 #define IR(ref)         (&J->cur.ir[(ref)])
  97
  98 /* Directly emit the transformed IR without updating chains etc. */
  99 static IRRef split_emit(jit_State *J, uint16_t ot, IRRef1 op1, IRRef1 op2)
 100 {
 101   IRRef nref = lj_ir_nextins(J);
 102   IRIns *ir = IR(nref);
 103   ir->ot = ot;
 104   ir->op1 = op1;
 105   ir->op2 = op2;
 106   return nref;
 107 }
 108
 109 #if LJ_SOFTFP
 110 /* Emit a (checked) number to integer conversion. */
 111 static IRRef split_num2int(jit_State *J, IRRef lo, IRRef hi, int check)
 112 {
 113   IRRef tmp, res;
 114 #if LJ_LE
 115   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), lo, hi);
 116 #else
 117   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hi, lo);
 118 #endif
 119   res = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_softfp_d2i);
 120   if (check) {
 121     tmp = split_emit(J, IRTI(IR_CALLN), res, IRCALL_softfp_i2d);
 122     split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 123     split_emit(J, IRTGI(IR_EQ), tmp, lo);
 124     split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP), tmp+1, hi);
 125   }
 126   return res;
 127 }
 128
 129 /* Emit a CALLN with one split 64 bit argument. */
 130 static IRRef split_call_l(jit_State *J, IRRef1 *hisubst, IRIns *oir,
 131                           IRIns *ir, IRCallID id)
 132 {
 133   IRRef tmp, op1 = ir->op1;
 134   J->cur.nins--;
 135 #if LJ_LE
 136   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 137 #else
 138   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 139 #endif
 140   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
 141   return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 142 }
 143
 144 /* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
 145 static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
 146                            IRIns *ir, IRCallID id)
 147 {
 148   IRRef tmp, op1 = ir->op1, op2 = ir->op2;
 149   J->cur.nins--;
 150 #if LJ_LE
 151   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 152 #else
 153   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 154 #endif
 155   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
 156   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
 157   return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 158 }
 159 #endif
 160
 161 /* Emit a CALLN with two split 64 bit arguments. */
 162 static IRRef split_call_ll(jit_State *J, IRRef1 *hisubst, IRIns *oir,
 163                            IRIns *ir, IRCallID id)
 164 {
 165   IRRef tmp, op1 = ir->op1, op2 = ir->op2;
 166   J->cur.nins--;
 167 #if LJ_LE
 168   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 169   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
 170   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
 171 #else
 172   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 173   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
 174   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
 175 #endif
 176   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
 177   return split_emit(J,
 178     IRT(IR_HIOP, (LJ_SOFTFP && irt_isnum(ir->t)) ? IRT_SOFTFP : IRT_INT),
 179     tmp, tmp);
 180 }
 181
 182 /* Get a pointer to the other 32 bit word (LE: hiword, BE: loword). */
 183 static IRRef split_ptr(jit_State *J, IRIns *oir, IRRef ref)
 184 {
 185   IRRef nref = oir[ref].prev;
 186   IRIns *ir = IR(nref);
 187   int32_t ofs = 4;
 188   if (ir->o == IR_ADD && irref_isk(ir->op2) && !irt_isphi(oir[ref].t)) {
 189     /* Reassociate address. */
 190     ofs += IR(ir->op2)->i;
 191     nref = ir->op1;
 192     if (ofs == 0) return nref;
 193   }
 194   return split_emit(J, IRTI(IR_ADD), nref, lj_ir_kint(J, ofs));
 195 }
 196
 197 /* Transform the old IR to the new IR. */
 198 static void split_ir(jit_State *J)
 199 {
 200   IRRef nins = J->cur.nins, nk = J->cur.nk;
 201   MSize irlen = nins - nk;
 202   MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
 203   IRIns *oir = (IRIns *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, need);
 204   IRRef1 *hisubst;
 205   IRRef ref;
 206
 207   /* Copy old IR to buffer. */
 208   memcpy(oir, IR(nk), irlen*sizeof(IRIns));
 209   /* Bias hiword substitution table and old IR. Loword kept in field prev. */
 210   hisubst = (IRRef1 *)&oir[irlen] - nk;
 211   oir -= nk;
 212
 213   /* Remove all IR instructions, but retain IR constants. */
 214   J->cur.nins = REF_FIRST;
 215   J->loopref = 0;
 216
 217   /* Process constants and fixed references. */
 218   for (ref = nk; ref <= REF_BASE; ref++) {
 219     IRIns *ir = &oir[ref];
 220     if ((LJ_SOFTFP && ir->o == IR_KNUM) || ir->o == IR_KINT64) {
 221       /* Split up 64 bit constant. */
 222       TValue tv = *ir_k64(ir);
 223       ir->prev = lj_ir_kint(J, (int32_t)tv.u32.lo);
 224       hisubst[ref] = lj_ir_kint(J, (int32_t)tv.u32.hi);
 225     } else {
 226       ir->prev = ref;  /* Identity substitution for loword. */
 227       hisubst[ref] = 0;
 228     }
 229   }
 230
 231   /* Process old IR instructions. */
 232   for (ref = REF_FIRST; ref < nins; ref++) {
 233     IRIns *ir = &oir[ref];
 234     IRRef nref = lj_ir_nextins(J);
 235     IRIns *nir = IR(nref);
 236     IRRef hi = 0;
 237
 238     /* Copy-substitute old instruction to new instruction. */
 239     nir->op1 = ir->op1 < nk ? ir->op1 : oir[ir->op1].prev;
 240     nir->op2 = ir->op2 < nk ? ir->op2 : oir[ir->op2].prev;
 241     ir->prev = nref;  /* Loword substitution. */
 242     nir->o = ir->o;
 243     nir->t.irt = ir->t.irt & ~(IRT_MARK|IRT_ISPHI);
 244     hisubst[ref] = 0;
 245
 246     /* Split 64 bit instructions. */
 247 #if LJ_SOFTFP
 248     if (irt_isnum(ir->t)) {
 249       nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD);  /* Turn into INT op. */
 250       /* Note: hi ref = lo ref + 1! Required for SNAP_SOFTFPNUM logic. */
 251       switch (ir->o) {
 252       case IR_ADD:
 253         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_add);
 254         break;
 255       case IR_SUB:
 256         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_sub);
 257         break;
 258       case IR_MUL:
 259         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_mul);
 260         break;
 261       case IR_DIV:
 262         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_div);
 263         break;
 264       case IR_POW:
 265         hi = split_call_li(J, hisubst, oir, ir, IRCALL_lj_vm_powi);
 266         break;
 267       case IR_FPMATH:
 268         /* Try to rejoin pow from EXP2, MUL and LOG2. */
 269         if (nir->op2 == IRFPM_EXP2 && nir->op1 > J->loopref) {
 270           IRIns *irp = IR(nir->op1);
 271           if (irp->o == IR_CALLN && irp->op2 == IRCALL_softfp_mul) {
 272             IRIns *irm4 = IR(irp->op1);
 273             IRIns *irm3 = IR(irm4->op1);
 274             IRIns *irm12 = IR(irm3->op1);
 275             IRIns *irl1 = IR(irm12->op1);
 276             if (irm12->op1 > J->loopref && irl1->o == IR_CALLN &&
 277                 irl1->op2 == IRCALL_lj_vm_log2) {
 278               IRRef tmp = irl1->op1;  /* Recycle first two args from LOG2. */
 279               IRRef arg3 = irm3->op2, arg4 = irm4->op2;
 280               J->cur.nins--;
 281               tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg3);
 282               tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg4);
 283               ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_pow);
 284               hi = split_emit(J, IRT(IR_HIOP, LJ_SOFTFP), tmp, tmp);
 285               break;
 286             }
 287           }
 288         }
 289         hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_floor + ir->op2);
 290         break;
 291       case IR_ATAN2:
 292         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_atan2);
 293         break;
 294       case IR_LDEXP:
 295         hi = split_call_li(J, hisubst, oir, ir, IRCALL_ldexp);
 296         break;
 297       case IR_NEG: case IR_ABS:
 298         nir->o = IR_CONV;  /* Pass through loword. */
 299         nir->op2 = (IRT_INT << 5) | IRT_INT;
 300         hi = split_emit(J, IRT(ir->o == IR_NEG ? IR_BXOR : IR_BAND, IRT_SOFTFP),
 301                         hisubst[ir->op1], hisubst[ir->op2]);
 302         break;
 303       case IR_SLOAD:
 304         if ((nir->op2 & IRSLOAD_CONVERT)) {  /* Convert from int to number. */
 305           nir->op2 &= ~IRSLOAD_CONVERT;
 306           ir->prev = nref = split_emit(J, IRTI(IR_CALLN), nref,
 307                                        IRCALL_softfp_i2d);
 308           hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 309           break;
 310         }
 311         /* fallthrough */
 312       case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
 313       case IR_STRTO:
 314         hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 315         break;
 316       case IR_XLOAD:
 317         hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP),
 318                         split_ptr(J, oir, ir->op1), ir->op2);
 319 #if LJ_BE
 320         ir->prev = hi; hi = nref;
 321 #endif
 322         break;
 323       case IR_ASTORE: case IR_HSTORE: case IR_USTORE:
 324         split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nir->op1, hisubst[ir->op2]);
 325         break;
 326       case IR_XSTORE: {
 327 #if LJ_LE
 328         IRRef hiref = hisubst[ir->op2];
 329 #else
 330         IRRef hiref = nir->op2; nir->op2 = hisubst[ir->op2];
 331 #endif
 332         split_emit(J, IRT(IR_XSTORE, IRT_SOFTFP),
 333                    split_ptr(J, oir, ir->op1), hiref);
 334         break;
 335         }
 336       case IR_CONV: {  /* Conversion to number. Others handled below. */
 337         IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 338         UNUSED(st);
 339 #if LJ_32 && LJ_HASFFI
 340         if (st == IRT_I64 || st == IRT_U64) {
 341           hi = split_call_l(J, hisubst, oir, ir,
 342                  st == IRT_I64 ? IRCALL_fp64_l2d : IRCALL_fp64_ul2d);
 343           break;
 344         }
 345 #endif
 346         lua_assert(st == IRT_INT ||
 347                    (LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT)));
 348         nir->o = IR_CALLN;
 349 #if LJ_32 && LJ_HASFFI
 350         nir->op2 = st == IRT_INT ? IRCALL_softfp_i2d :
 351                    st == IRT_FLOAT ? IRCALL_softfp_f2d :
 352                    IRCALL_softfp_ui2d;
 353 #else
 354         nir->op2 = IRCALL_softfp_i2d;
 355 #endif
 356         hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 357         break;
 358         }
 359       case IR_CALLS:
 360       case IR_CALLXS:
 361         goto split_call;
 362       case IR_PHI:
 363         if (nir->op1 == nir->op2)
 364           J->cur.nins--;  /* Drop useless PHIs. */
 365         if (hisubst[ir->op1] != hisubst[ir->op2])
 366           split_emit(J, IRT(IR_PHI, IRT_SOFTFP),
 367                      hisubst[ir->op1], hisubst[ir->op2]);
 368         break;
 369       default:
 370         lua_assert(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX);
 371         hi = split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP),
 372                         hisubst[ir->op1], hisubst[ir->op2]);
 373         break;
 374       }
 375     } else
 376 #endif
 377 #if LJ_32 && LJ_HASFFI
 378     if (irt_isint64(ir->t)) {
 379       IRRef hiref = hisubst[ir->op1];
 380       nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD);  /* Turn into INT op. */
 381       switch (ir->o) {
 382       case IR_ADD:
 383       case IR_SUB:
 384         /* Use plain op for hiword if loword cannot produce a carry/borrow. */
 385         if (irref_isk(nir->op2) && IR(nir->op2)->i == 0) {
 386           ir->prev = nir->op1;  /* Pass through loword. */
 387           nir->op1 = hiref; nir->op2 = hisubst[ir->op2];
 388           hi = nref;
 389           break;
 390         }
 391         /* fallthrough */
 392       case IR_NEG:
 393         hi = split_emit(J, IRTI(IR_HIOP), hiref, hisubst[ir->op2]);
 394         break;
 395       case IR_MUL:
 396         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_lj_carith_mul64);
 397         break;
 398       case IR_DIV:
 399         hi = split_call_ll(J, hisubst, oir, ir,
 400                            irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
 401                                               IRCALL_lj_carith_divu64);
 402         break;
 403       case IR_MOD:
 404         hi = split_call_ll(J, hisubst, oir, ir,
 405                            irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
 406                                               IRCALL_lj_carith_modu64);
 407         break;
 408       case IR_POW:
 409         hi = split_call_ll(J, hisubst, oir, ir,
 410                            irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
 411                                               IRCALL_lj_carith_powu64);
 412         break;
 413       case IR_FLOAD:
 414         lua_assert(ir->op2 == IRFL_CDATA_INT64);
 415         hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4);
 416 #if LJ_BE
 417         ir->prev = hi; hi = nref;
 418 #endif
 419         break;
 420       case IR_XLOAD:
 421         hi = split_emit(J, IRTI(IR_XLOAD), split_ptr(J, oir, ir->op1), ir->op2);
 422 #if LJ_BE
 423         ir->prev = hi; hi = nref;
 424 #endif
 425         break;
 426       case IR_XSTORE:
 427 #if LJ_LE
 428         hiref = hisubst[ir->op2];
 429 #else
 430         hiref = nir->op2; nir->op2 = hisubst[ir->op2];
 431 #endif
 432         split_emit(J, IRTI(IR_XSTORE), split_ptr(J, oir, ir->op1), hiref);
 433         break;
 434       case IR_CONV: {  /* Conversion to 64 bit integer. Others handled below. */
 435         IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 436 #if LJ_SOFTFP
 437         if (st == IRT_NUM) {  /* NUM to 64 bit int conv. */
 438           hi = split_call_l(J, hisubst, oir, ir,
 439                  irt_isi64(ir->t) ? IRCALL_fp64_d2l : IRCALL_fp64_d2ul);
 440         } else if (st == IRT_FLOAT) {  /* FLOAT to 64 bit int conv. */
 441           nir->o = IR_CALLN;
 442           nir->op2 = irt_isi64(ir->t) ? IRCALL_fp64_f2l : IRCALL_fp64_f2ul;
 443           hi = split_emit(J, IRTI(IR_HIOP), nref, nref);
 444         }
 445 #else
 446         if (st == IRT_NUM || st == IRT_FLOAT) {  /* FP to 64 bit int conv. */
 447           hi = split_emit(J, IRTI(IR_HIOP), nir->op1, nref);
 448         }
 449 #endif
 450         else if (st == IRT_I64 || st == IRT_U64) {  /* 64/64 bit cast. */
 451           /* Drop cast, since assembler doesn't care. */
 452           goto fwdlo;
 453         } else if ((ir->op2 & IRCONV_SEXT)) {  /* Sign-extend to 64 bit. */
 454           IRRef k31 = lj_ir_kint(J, 31);
 455           nir = IR(nref);  /* May have been reallocated. */
 456           ir->prev = nir->op1;  /* Pass through loword. */
 457           nir->o = IR_BSAR;  /* hi = bsar(lo, 31). */
 458           nir->op2 = k31;
 459           hi = nref;
 460         } else {  /* Zero-extend to 64 bit. */
 461           hi = lj_ir_kint(J, 0);
 462           goto fwdlo;
 463         }
 464         break;
 465         }
 466       case IR_CALLXS:
 467         goto split_call;
 468       case IR_PHI: {
 469         IRRef hiref2;
 470         if ((irref_isk(nir->op1) && irref_isk(nir->op2)) ||
 471             nir->op1 == nir->op2)
 472           J->cur.nins--;  /* Drop useless PHIs. */
 473         hiref2 = hisubst[ir->op2];
 474         if (!((irref_isk(hiref) && irref_isk(hiref2)) || hiref == hiref2))
 475           split_emit(J, IRTI(IR_PHI), hiref, hiref2);
 476         break;
 477         }
 478       default:
 479         lua_assert(ir->o <= IR_NE);  /* Comparisons. */
 480         split_emit(J, IRTGI(IR_HIOP), hiref, hisubst[ir->op2]);
 481         break;
 482       }
 483     } else
 484 #endif
 485 #if LJ_SOFTFP
 486     if (ir->o == IR_SLOAD) {
 487       if ((nir->op2 & IRSLOAD_CONVERT)) {  /* Convert from number to int. */
 488         nir->op2 &= ~IRSLOAD_CONVERT;
 489         if (!(nir->op2 & IRSLOAD_TYPECHECK))
 490           nir->t.irt = IRT_INT;  /* Drop guard. */
 491         split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 492         ir->prev = split_num2int(J, nref, nref+1, irt_isguard(ir->t));
 493       }
 494     } else if (ir->o == IR_TOBIT) {
 495       IRRef tmp, op1 = ir->op1;
 496       J->cur.nins--;
 497 #if LJ_LE
 498       tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 499 #else
 500       tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 501 #endif
 502       ir->prev = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_lj_vm_tobit);
 503     } else if (ir->o == IR_TOSTR) {
 504       if (hisubst[ir->op1]) {
 505         if (irref_isk(ir->op1))
 506           nir->op1 = ir->op1;
 507         else
 508           split_emit(J, IRT(IR_HIOP, IRT_NIL), hisubst[ir->op1], nref);
 509       }
 510     } else if (ir->o == IR_HREF || ir->o == IR_NEWREF) {
 511       if (irref_isk(ir->op2) && hisubst[ir->op2])
 512         nir->op2 = ir->op2;
 513     } else
 514 #endif
 515     if (ir->o == IR_CONV) {  /* See above, too. */
 516       IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 517 #if LJ_32 && LJ_HASFFI
 518       if (st == IRT_I64 || st == IRT_U64) {  /* Conversion from 64 bit int. */
 519 #if LJ_SOFTFP
 520         if (irt_isfloat(ir->t)) {
 521           split_call_l(J, hisubst, oir, ir,
 522                        st == IRT_I64 ? IRCALL_fp64_l2f : IRCALL_fp64_ul2f);
 523           J->cur.nins--;  /* Drop unused HIOP. */
 524         }
 525 #else
 526         if (irt_isfp(ir->t)) {  /* 64 bit integer to FP conversion. */
 527           ir->prev = split_emit(J, IRT(IR_HIOP, irt_type(ir->t)),
 528                                 hisubst[ir->op1], nref);
 529         }
 530 #endif
 531         else {  /* Truncate to lower 32 bits. */
 532         fwdlo:
 533           ir->prev = nir->op1;  /* Forward loword. */
 534           /* Replace with NOP to avoid messing up the snapshot logic. */
 535           nir->ot = IRT(IR_NOP, IRT_NIL);
 536           nir->op1 = nir->op2 = 0;
 537         }
 538       }
 539 #endif
 540 #if LJ_SOFTFP && LJ_32 && LJ_HASFFI
 541       else if (irt_isfloat(ir->t)) {
 542         if (st == IRT_NUM) {
 543           split_call_l(J, hisubst, oir, ir, IRCALL_softfp_d2f);
 544           J->cur.nins--;  /* Drop unused HIOP. */
 545         } else {
 546           nir->o = IR_CALLN;
 547           nir->op2 = st == IRT_INT ? IRCALL_softfp_i2f : IRCALL_softfp_ui2f;
 548         }
 549       } else if (st == IRT_FLOAT) {
 550         nir->o = IR_CALLN;
 551         nir->op2 = irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui;
 552       } else
 553 #endif
 554 #if LJ_SOFTFP
 555       if (st == IRT_NUM || (LJ_32 && LJ_HASFFI && st == IRT_FLOAT)) {
 556         if (irt_isguard(ir->t)) {
 557           lua_assert(st == IRT_NUM && irt_isint(ir->t));
 558           J->cur.nins--;
 559           ir->prev = split_num2int(J, nir->op1, hisubst[ir->op1], 1);
 560         } else {
 561           split_call_l(J, hisubst, oir, ir,
 562 #if LJ_32 && LJ_HASFFI
 563             st == IRT_NUM ?
 564               (irt_isint(ir->t) ? IRCALL_softfp_d2i : IRCALL_softfp_d2ui) :
 565               (irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui)
 566 #else
 567             IRCALL_softfp_d2i
 568 #endif
 569           );
 570           J->cur.nins--;  /* Drop unused HIOP. */
 571         }
 572       }
 573 #endif
 574     } else if (ir->o == IR_CALLXS) {
 575       IRRef hiref;
 576     split_call:
 577       hiref = hisubst[ir->op1];
 578       if (hiref) {
 579         IROpT ot = nir->ot;
 580         IRRef op2 = nir->op2;
 581         nir->ot = IRT(IR_CARG, IRT_NIL);
 582 #if LJ_LE
 583         nir->op2 = hiref;
 584 #else
 585         nir->op2 = nir->op1; nir->op1 = hiref;
 586 #endif
 587         ir->prev = nref = split_emit(J, ot, nref, op2);
 588       }
 589       if (LJ_SOFTFP ? irt_is64(ir->t) : irt_isint64(ir->t))
 590         hi = split_emit(J,
 591           IRT(IR_HIOP, (LJ_SOFTFP && irt_isnum(ir->t)) ? IRT_SOFTFP : IRT_INT),
 592           nref, nref);
 593     } else if (ir->o == IR_CARG) {
 594       IRRef hiref = hisubst[ir->op1];
 595       if (hiref) {
 596         IRRef op2 = nir->op2;
 597 #if LJ_LE
 598         nir->op2 = hiref;
 599 #else
 600         nir->op2 = nir->op1; nir->op1 = hiref;
 601 #endif
 602         ir->prev = nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
 603         nir = IR(nref);
 604       }
 605       hiref = hisubst[ir->op2];
 606       if (hiref) {
 607 #if !LJ_TARGET_X86
 608         int carg = 0;
 609         IRIns *cir;
 610         for (cir = IR(nir->op1); cir->o == IR_CARG; cir = IR(cir->op1))
 611           carg++;
 612         if ((carg & 1) == 0) {  /* Align 64 bit arguments. */
 613           IRRef op2 = nir->op2;
 614           nir->op2 = REF_NIL;
 615           nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
 616           nir = IR(nref);
 617         }
 618 #endif
 619 #if LJ_BE
 620         { IRRef tmp = nir->op2; nir->op2 = hiref; hiref = tmp; }
 621 #endif
 622         ir->prev = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, hiref);
 623       }
 624     } else if (ir->o == IR_CNEWI) {
 625       if (hisubst[ir->op2])
 626         split_emit(J, IRT(IR_HIOP, IRT_NIL), nref, hisubst[ir->op2]);
 627     } else if (ir->o == IR_LOOP) {
 628       J->loopref = nref;  /* Needed by assembler. */
 629     }
 630     hisubst[ref] = hi;  /* Store hiword substitution. */
 631   }
 632
 633   /* Add PHI marks. */
 634   for (ref = J->cur.nins-1; ref >= REF_FIRST; ref--) {
 635     IRIns *ir = IR(ref);
 636     if (ir->o != IR_PHI) break;
 637     if (!irref_isk(ir->op1)) irt_setphi(IR(ir->op1)->t);
 638     if (ir->op2 > J->loopref) irt_setphi(IR(ir->op2)->t);
 639   }
 640
 641   /* Substitute snapshot maps. */
 642   oir[nins].prev = J->cur.nins;  /* Substitution for last snapshot. */
 643   {
 644     SnapNo i, nsnap = J->cur.nsnap;
 645     for (i = 0; i < nsnap; i++) {
 646       SnapShot *snap = &J->cur.snap[i];
 647       SnapEntry *map = &J->cur.snapmap[snap->mapofs];
 648       MSize n, nent = snap->nent;
 649       snap->ref = snap->ref == REF_FIRST ? REF_FIRST : oir[snap->ref].prev;
 650       for (n = 0; n < nent; n++) {
 651         SnapEntry sn = map[n];
 652         IRIns *ir = &oir[snap_ref(sn)];
 653         if (!(LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && irref_isk(snap_ref(sn))))
 654           map[n] = ((sn & 0xffff0000) | ir->prev);
 655       }
 656     }
 657   }
 658 }
 659
 660 /* Protected callback for split pass. */
 661 static TValue *cpsplit(lua_State *L, lua_CFunction dummy, void *ud)
 662 {
 663   jit_State *J = (jit_State *)ud;
 664   split_ir(J);
 665   UNUSED(L); UNUSED(dummy);
 666   return NULL;
 667 }
 668
 669 #if defined(LUA_USE_ASSERT) || LJ_SOFTFP
 670 /* Slow, but sure way to check whether a SPLIT pass is needed. */
 671 static int split_needsplit(jit_State *J)
 672 {
 673   IRIns *ir, *irend;
 674   IRRef ref;
 675   for (ir = IR(REF_FIRST), irend = IR(J->cur.nins); ir < irend; ir++)
 676     if (LJ_SOFTFP ? irt_is64orfp(ir->t) : irt_isint64(ir->t))
 677       return 1;
 678   if (LJ_SOFTFP) {
 679     for (ref = J->chain[IR_SLOAD]; ref; ref = IR(ref)->prev)
 680       if ((IR(ref)->op2 & IRSLOAD_CONVERT))
 681         return 1;
 682   }
 683   for (ref = J->chain[IR_CONV]; ref; ref = IR(ref)->prev) {
 684     IRType st = (IR(ref)->op2 & IRCONV_SRCMASK);
 685     if ((LJ_SOFTFP && (st == IRT_NUM || st == IRT_FLOAT)) ||
 686         st == IRT_I64 || st == IRT_U64)
 687       return 1;
 688   }
 689   return 0;  /* Nope. */
 690 }
 691 #endif
 692
 693 /* SPLIT pass. */
 694 void lj_opt_split(jit_State *J)
 695 {
 696 #if LJ_SOFTFP
 697   if (!J->needsplit)
 698     J->needsplit = split_needsplit(J);
 699 #else
 700   lua_assert(J->needsplit >= split_needsplit(J));  /* Verify flag. */
 701 #endif
 702   if (J->needsplit) {
 703     int errcode = lj_vm_cpcall(J->L, NULL, J, cpsplit);
 704     if (errcode) {
 705       /* Completely reset the trace to avoid inconsistent dump on abort. */
 706       J->cur.nins = J->cur.nk = REF_BASE;
 707       J->cur.nsnap = 0;
 708       lj_err_throw(J->L, errcode);  /* Propagate errors. */
 709     }
 710   }
 711 }
 712
 713 #undef IR
 714
 715 #endif