src/lj_opt_split.c

   1 /*
   2 ** SPLIT: Split 64 bit IR instructions into 32 bit IR instructions.
   3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #define lj_opt_split_c
   7 #define LUA_CORE
   8
   9 #include "lj_obj.h"
  10
  11 #if LJ_HASJIT && (LJ_SOFTFP || (LJ_32 && LJ_HASFFI))
  12
  13 #include "lj_err.h"
  14 #include "lj_str.h"
  15 #include "lj_ir.h"
  16 #include "lj_jit.h"
  17 #include "lj_ircall.h"
  18 #include "lj_iropt.h"
  19 #include "lj_vm.h"
  20
  21 /* SPLIT pass:
  22 **
  23 ** This pass splits up 64 bit IR instructions into multiple 32 bit IR
  24 ** instructions. It's only active for soft-float targets or for 32 bit CPUs
  25 ** which lack native 64 bit integer operations (the FFI is currently the
  26 ** only emitter for 64 bit integer instructions).
  27 **
  28 ** Splitting the IR in a separate pass keeps each 32 bit IR assembler
  29 ** backend simple. Only a small amount of extra functionality needs to be
  30 ** implemented. This is much easier than adding support for allocating
  31 ** register pairs to each backend (believe me, I tried). A few simple, but
  32 ** important optimizations can be performed by the SPLIT pass, which would
  33 ** be tedious to do in the backend.
  34 **
  35 ** The basic idea is to replace each 64 bit IR instruction with its 32 bit
  36 ** equivalent plus an extra HIOP instruction. The splitted IR is not passed
  37 ** through FOLD or any other optimizations, so each HIOP is guaranteed to
  38 ** immediately follow it's counterpart. The actual functionality of HIOP is
  39 ** inferred from the previous instruction.
  40 **
  41 ** The operands of HIOP hold the hiword input references. The output of HIOP
  42 ** is the hiword output reference, which is also used to hold the hiword
  43 ** register or spill slot information. The register allocator treats this
  44 ** instruction independently of any other instruction, which improves code
  45 ** quality compared to using fixed register pairs.
  46 **
  47 ** It's easier to split up some instructions into two regular 32 bit
  48 ** instructions. E.g. XLOAD is split up into two XLOADs with two different
  49 ** addresses. Obviously 64 bit constants need to be split up into two 32 bit
  50 ** constants, too. Some hiword instructions can be entirely omitted, e.g.
  51 ** when zero-extending a 32 bit value to 64 bits. 64 bit arguments for calls
  52 ** are split up into two 32 bit arguments each.
  53 **
  54 ** On soft-float targets, floating-point instructions are directly converted
  55 ** to soft-float calls by the SPLIT pass (except for comparisons and MIN/MAX).
  56 ** HIOP for number results has the type IRT_SOFTFP ("sfp" in -jdump).
  57 **
  58 ** Here's the IR and x64 machine code for 'x.b = x.a + 1' for a struct with
  59 ** two int64_t fields:
  60 **
  61 ** 0100    p32 ADD    base  +8
  62 ** 0101    i64 XLOAD  0100
  63 ** 0102    i64 ADD    0101  +1
  64 ** 0103    p32 ADD    base  +16
  65 ** 0104    i64 XSTORE 0103  0102
  66 **
  67 **         mov rax, [esi+0x8]
  68 **         add rax, +0x01
  69 **         mov [esi+0x10], rax
  70 **
  71 ** Here's the transformed IR and the x86 machine code after the SPLIT pass:
  72 **
  73 ** 0100    p32 ADD    base  +8
  74 ** 0101    int XLOAD  0100
  75 ** 0102    p32 ADD    base  +12
  76 ** 0103    int XLOAD  0102
  77 ** 0104    int ADD    0101  +1
  78 ** 0105    int HIOP   0103  +0
  79 ** 0106    p32 ADD    base  +16
  80 ** 0107    int XSTORE 0106  0104
  81 ** 0108    p32 ADD    base  +20
  82 ** 0109    int XSTORE 0108  0105
  83 **
  84 **         mov eax, [esi+0x8]
  85 **         mov ecx, [esi+0xc]
  86 **         add eax, +0x01
  87 **         adc ecx, +0x00
  88 **         mov [esi+0x10], eax
  89 **         mov [esi+0x14], ecx
  90 **
  91 ** You may notice the reassociated hiword address computation, which is
  92 ** later fused into the mov operands by the assembler.
  93 */
  94
  95 /* Some local macros to save typing. Undef'd at the end. */
  96 #define IR(ref)         (&J->cur.ir[(ref)])
  97
  98 /* Directly emit the transformed IR without updating chains etc. */
  99 static IRRef split_emit(jit_State *J, uint16_t ot, IRRef1 op1, IRRef1 op2)
 100 {
 101   IRRef nref = lj_ir_nextins(J);
 102   IRIns *ir = IR(nref);
 103   ir->ot = ot;
 104   ir->op1 = op1;
 105   ir->op2 = op2;
 106   return nref;
 107 }
 108
 109 #if LJ_SOFTFP
 110 /* Emit a (checked) number to integer conversion. */
 111 static IRRef split_num2int(jit_State *J, IRRef lo, IRRef hi, int check)
 112 {
 113   IRRef tmp, res;
 114 #if LJ_LE
 115   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), lo, hi);
 116 #else
 117   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hi, lo);
 118 #endif
 119   res = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_softfp_d2i);
 120   if (check) {
 121     tmp = split_emit(J, IRTI(IR_CALLN), res, IRCALL_softfp_i2d);
 122     split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 123     split_emit(J, IRTGI(IR_EQ), tmp, lo);
 124     split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP), tmp+1, hi);
 125   }
 126   return res;
 127 }
 128
 129 /* Emit a CALLN with one split 64 bit argument. */
 130 static IRRef split_call_l(jit_State *J, IRRef1 *hisubst, IRIns *oir,
 131                           IRIns *ir, IRCallID id)
 132 {
 133   IRRef tmp, op1 = ir->op1;
 134   J->cur.nins--;
 135 #if LJ_LE
 136   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 137 #else
 138   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 139 #endif
 140   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
 141   return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 142 }
 143
 144 /* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
 145 static IRRef split_call_li(jit_State *J, IRRef1 *hisubst, IRIns *oir,
 146                            IRIns *ir, IRCallID id)
 147 {
 148   IRRef tmp, op1 = ir->op1, op2 = ir->op2;
 149   J->cur.nins--;
 150 #if LJ_LE
 151   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 152 #else
 153   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 154 #endif
 155   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
 156   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
 157   return split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 158 }
 159 #endif
 160
 161 /* Emit a CALLN with two split 64 bit arguments. */
 162 static IRRef split_call_ll(jit_State *J, IRRef1 *hisubst, IRIns *oir,
 163                            IRIns *ir, IRCallID id)
 164 {
 165   IRRef tmp, op1 = ir->op1, op2 = ir->op2;
 166   J->cur.nins--;
 167 #if LJ_LE
 168   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 169   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
 170   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
 171 #else
 172   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 173   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, hisubst[op2]);
 174   tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, oir[op2].prev);
 175 #endif
 176   ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, id);
 177   return split_emit(J,
 178     IRT(IR_HIOP, (LJ_SOFTFP && irt_isnum(ir->t)) ? IRT_SOFTFP : IRT_INT),
 179     tmp, tmp);
 180 }
 181
 182 /* Get a pointer to the other 32 bit word (LE: hiword, BE: loword). */
 183 static IRRef split_ptr(jit_State *J, IRIns *oir, IRRef ref)
 184 {
 185   IRRef nref = oir[ref].prev;
 186   IRIns *ir = IR(nref);
 187   int32_t ofs = 4;
 188   if (ir->o == IR_ADD && irref_isk(ir->op2) && !irt_isphi(oir[ref].t)) {
 189     /* Reassociate address. */
 190     ofs += IR(ir->op2)->i;
 191     nref = ir->op1;
 192     if (ofs == 0) return nref;
 193   }
 194   return split_emit(J, IRTI(IR_ADD), nref, lj_ir_kint(J, ofs));
 195 }
 196
 197 /* Transform the old IR to the new IR. */
 198 static void split_ir(jit_State *J)
 199 {
 200   IRRef nins = J->cur.nins, nk = J->cur.nk;
 201   MSize irlen = nins - nk;
 202   MSize need = (irlen+1)*(sizeof(IRIns) + sizeof(IRRef1));
 203   IRIns *oir = (IRIns *)lj_str_needbuf(J->L, &G(J->L)->tmpbuf, need);
 204   IRRef1 *hisubst;
 205   IRRef ref;
 206
 207   /* Copy old IR to buffer. */
 208   memcpy(oir, IR(nk), irlen*sizeof(IRIns));
 209   /* Bias hiword substitution table and old IR. Loword kept in field prev. */
 210   hisubst = (IRRef1 *)&oir[irlen] - nk;
 211   oir -= nk;
 212
 213   /* Remove all IR instructions, but retain IR constants. */
 214   J->cur.nins = REF_FIRST;
 215   J->loopref = 0;
 216
 217   /* Process constants and fixed references. */
 218   for (ref = nk; ref <= REF_BASE; ref++) {
 219     IRIns *ir = &oir[ref];
 220     if ((LJ_SOFTFP && ir->o == IR_KNUM) || ir->o == IR_KINT64) {
 221       /* Split up 64 bit constant. */
 222       TValue tv = *ir_k64(ir);
 223       ir->prev = lj_ir_kint(J, (int32_t)tv.u32.lo);
 224       hisubst[ref] = lj_ir_kint(J, (int32_t)tv.u32.hi);
 225     } else {
 226       ir->prev = ref;  /* Identity substitution for loword. */
 227       hisubst[ref] = 0;
 228     }
 229   }
 230
 231   /* Process old IR instructions. */
 232   for (ref = REF_FIRST; ref < nins; ref++) {
 233     IRIns *ir = &oir[ref];
 234     IRRef nref = lj_ir_nextins(J);
 235     IRIns *nir = IR(nref);
 236     IRRef hi = 0;
 237
 238     /* Copy-substitute old instruction to new instruction. */
 239     nir->op1 = ir->op1 < nk ? ir->op1 : oir[ir->op1].prev;
 240     nir->op2 = ir->op2 < nk ? ir->op2 : oir[ir->op2].prev;
 241     ir->prev = nref;  /* Loword substitution. */
 242     nir->o = ir->o;
 243     nir->t.irt = ir->t.irt & ~(IRT_MARK|IRT_ISPHI);
 244     hisubst[ref] = 0;
 245
 246     /* Split 64 bit instructions. */
 247 #if LJ_SOFTFP
 248     if (irt_isnum(ir->t)) {
 249       nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD);  /* Turn into INT op. */
 250       /* Note: hi ref = lo ref + 1! Required for SNAP_SOFTFPNUM logic. */
 251       switch (ir->o) {
 252       case IR_ADD:
 253         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_add);
 254         break;
 255       case IR_SUB:
 256         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_sub);
 257         break;
 258       case IR_MUL:
 259         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_mul);
 260         break;
 261       case IR_DIV:
 262         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_softfp_div);
 263         break;
 264       case IR_POW:
 265         hi = split_call_li(J, hisubst, oir, ir, IRCALL_lj_vm_powi);
 266         break;
 267       case IR_FPMATH:
 268         /* Try to rejoin pow from EXP2, MUL and LOG2. */
 269         if (nir->op2 == IRFPM_EXP2 && nir->op1 > J->loopref) {
 270           IRIns *irp = IR(nir->op1);
 271           if (irp->o == IR_CALLN && irp->op2 == IRCALL_softfp_mul) {
 272             IRIns *irm4 = IR(irp->op1);
 273             IRIns *irm3 = IR(irm4->op1);
 274             IRIns *irm12 = IR(irm3->op1);
 275             IRIns *irl1 = IR(irm12->op1);
 276             if (irm12->op1 > J->loopref && irl1->o == IR_CALLN &&
 277                 irl1->op2 == IRCALL_lj_vm_log2) {
 278               IRRef tmp = irl1->op1;  /* Recycle first two args from LOG2. */
 279               IRRef arg3 = irm3->op2, arg4 = irm4->op2;
 280               J->cur.nins--;
 281               tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg3);
 282               tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), tmp, arg4);
 283               ir->prev = tmp = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_pow);
 284               hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), tmp, tmp);
 285               break;
 286             }
 287           }
 288         }
 289         hi = split_call_l(J, hisubst, oir, ir, IRCALL_lj_vm_floor + ir->op2);
 290         break;
 291       case IR_ATAN2:
 292         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_atan2);
 293         break;
 294       case IR_LDEXP:
 295         hi = split_call_li(J, hisubst, oir, ir, IRCALL_ldexp);
 296         break;
 297       case IR_NEG: case IR_ABS:
 298         nir->o = IR_CONV;  /* Pass through loword. */
 299         nir->op2 = (IRT_INT << 5) | IRT_INT;
 300         hi = split_emit(J, IRT(ir->o == IR_NEG ? IR_BXOR : IR_BAND, IRT_SOFTFP),
 301                         hisubst[ir->op1], hisubst[ir->op2]);
 302         break;
 303       case IR_SLOAD:
 304         if ((nir->op2 & IRSLOAD_CONVERT)) {  /* Convert from int to number. */
 305           nir->op2 &= ~IRSLOAD_CONVERT;
 306           ir->prev = nref = split_emit(J, IRTI(IR_CALLN), nref,
 307                                        IRCALL_softfp_i2d);
 308           hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 309           break;
 310         }
 311         /* fallthrough */
 312       case IR_ALOAD: case IR_HLOAD: case IR_ULOAD: case IR_VLOAD:
 313       case IR_STRTO:
 314         hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 315         break;
 316       case IR_XLOAD: {
 317         IRIns inslo = *nir;  /* Save/undo the emit of the lo XLOAD. */
 318         J->cur.nins--;
 319         hi = split_ptr(J, oir, ir->op1);  /* Insert the hiref ADD. */
 320         nref = lj_ir_nextins(J);
 321         nir = IR(nref);
 322         *nir = inslo;  /* Re-emit lo XLOAD immediately before hi XLOAD. */
 323         hi = split_emit(J, IRT(IR_XLOAD, IRT_SOFTFP), hi, ir->op2);
 324 #if LJ_LE
 325         ir->prev = nref;
 326 #else
 327         ir->prev = hi; hi = nref;
 328 #endif
 329         break;
 330         }
 331       case IR_ASTORE: case IR_HSTORE: case IR_USTORE:
 332         split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nir->op1, hisubst[ir->op2]);
 333         break;
 334       case IR_XSTORE: {
 335 #if LJ_LE
 336         IRRef hiref = hisubst[ir->op2];
 337 #else
 338         IRRef hiref = nir->op2; nir->op2 = hisubst[ir->op2];
 339 #endif
 340         split_emit(J, IRT(IR_XSTORE, IRT_SOFTFP),
 341                    split_ptr(J, oir, ir->op1), hiref);
 342         break;
 343         }
 344       case IR_CONV: {  /* Conversion to number. Others handled below. */
 345         IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 346         UNUSED(st);
 347 #if LJ_32 && LJ_HASFFI
 348         if (st == IRT_I64 || st == IRT_U64) {
 349           hi = split_call_l(J, hisubst, oir, ir,
 350                  st == IRT_I64 ? IRCALL_fp64_l2d : IRCALL_fp64_ul2d);
 351           break;
 352         }
 353 #endif
 354         lua_assert(st == IRT_INT ||
 355                    (LJ_32 && LJ_HASFFI && (st == IRT_U32 || st == IRT_FLOAT)));
 356         nir->o = IR_CALLN;
 357 #if LJ_32 && LJ_HASFFI
 358         nir->op2 = st == IRT_INT ? IRCALL_softfp_i2d :
 359                    st == IRT_FLOAT ? IRCALL_softfp_f2d :
 360                    IRCALL_softfp_ui2d;
 361 #else
 362         nir->op2 = IRCALL_softfp_i2d;
 363 #endif
 364         hi = split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 365         break;
 366         }
 367       case IR_CALLN:
 368       case IR_CALLL:
 369       case IR_CALLS:
 370       case IR_CALLXS:
 371         goto split_call;
 372       case IR_PHI:
 373         if (nir->op1 == nir->op2)
 374           J->cur.nins--;  /* Drop useless PHIs. */
 375         if (hisubst[ir->op1] != hisubst[ir->op2])
 376           split_emit(J, IRT(IR_PHI, IRT_SOFTFP),
 377                      hisubst[ir->op1], hisubst[ir->op2]);
 378         break;
 379       default:
 380         lua_assert(ir->o <= IR_NE || ir->o == IR_MIN || ir->o == IR_MAX);
 381         hi = split_emit(J, IRTG(IR_HIOP, IRT_SOFTFP),
 382                         hisubst[ir->op1], hisubst[ir->op2]);
 383         break;
 384       }
 385     } else
 386 #endif
 387 #if LJ_32 && LJ_HASFFI
 388     if (irt_isint64(ir->t)) {
 389       IRRef hiref = hisubst[ir->op1];
 390       nir->t.irt = IRT_INT | (nir->t.irt & IRT_GUARD);  /* Turn into INT op. */
 391       switch (ir->o) {
 392       case IR_ADD:
 393       case IR_SUB:
 394         /* Use plain op for hiword if loword cannot produce a carry/borrow. */
 395         if (irref_isk(nir->op2) && IR(nir->op2)->i == 0) {
 396           ir->prev = nir->op1;  /* Pass through loword. */
 397           nir->op1 = hiref; nir->op2 = hisubst[ir->op2];
 398           hi = nref;
 399           break;
 400         }
 401         /* fallthrough */
 402       case IR_NEG:
 403         hi = split_emit(J, IRTI(IR_HIOP), hiref, hisubst[ir->op2]);
 404         break;
 405       case IR_MUL:
 406         hi = split_call_ll(J, hisubst, oir, ir, IRCALL_lj_carith_mul64);
 407         break;
 408       case IR_DIV:
 409         hi = split_call_ll(J, hisubst, oir, ir,
 410                            irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
 411                                               IRCALL_lj_carith_divu64);
 412         break;
 413       case IR_MOD:
 414         hi = split_call_ll(J, hisubst, oir, ir,
 415                            irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
 416                                               IRCALL_lj_carith_modu64);
 417         break;
 418       case IR_POW:
 419         hi = split_call_ll(J, hisubst, oir, ir,
 420                            irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
 421                                               IRCALL_lj_carith_powu64);
 422         break;
 423       case IR_FLOAD:
 424         lua_assert(ir->op2 == IRFL_CDATA_INT64);
 425         hi = split_emit(J, IRTI(IR_FLOAD), nir->op1, IRFL_CDATA_INT64_4);
 426 #if LJ_BE
 427         ir->prev = hi; hi = nref;
 428 #endif
 429         break;
 430       case IR_XLOAD:
 431         hi = split_emit(J, IRTI(IR_XLOAD), split_ptr(J, oir, ir->op1), ir->op2);
 432 #if LJ_BE
 433         ir->prev = hi; hi = nref;
 434 #endif
 435         break;
 436       case IR_XSTORE:
 437 #if LJ_LE
 438         hiref = hisubst[ir->op2];
 439 #else
 440         hiref = nir->op2; nir->op2 = hisubst[ir->op2];
 441 #endif
 442         split_emit(J, IRTI(IR_XSTORE), split_ptr(J, oir, ir->op1), hiref);
 443         break;
 444       case IR_CONV: {  /* Conversion to 64 bit integer. Others handled below. */
 445         IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 446 #if LJ_SOFTFP
 447         if (st == IRT_NUM) {  /* NUM to 64 bit int conv. */
 448           hi = split_call_l(J, hisubst, oir, ir,
 449                  irt_isi64(ir->t) ? IRCALL_fp64_d2l : IRCALL_fp64_d2ul);
 450         } else if (st == IRT_FLOAT) {  /* FLOAT to 64 bit int conv. */
 451           nir->o = IR_CALLN;
 452           nir->op2 = irt_isi64(ir->t) ? IRCALL_fp64_f2l : IRCALL_fp64_f2ul;
 453           hi = split_emit(J, IRTI(IR_HIOP), nref, nref);
 454         }
 455 #else
 456         if (st == IRT_NUM || st == IRT_FLOAT) {  /* FP to 64 bit int conv. */
 457           hi = split_emit(J, IRTI(IR_HIOP), nir->op1, nref);
 458         }
 459 #endif
 460         else if (st == IRT_I64 || st == IRT_U64) {  /* 64/64 bit cast. */
 461           /* Drop cast, since assembler doesn't care. */
 462           goto fwdlo;
 463         } else if ((ir->op2 & IRCONV_SEXT)) {  /* Sign-extend to 64 bit. */
 464           IRRef k31 = lj_ir_kint(J, 31);
 465           nir = IR(nref);  /* May have been reallocated. */
 466           ir->prev = nir->op1;  /* Pass through loword. */
 467           nir->o = IR_BSAR;  /* hi = bsar(lo, 31). */
 468           nir->op2 = k31;
 469           hi = nref;
 470         } else {  /* Zero-extend to 64 bit. */
 471           hi = lj_ir_kint(J, 0);
 472           goto fwdlo;
 473         }
 474         break;
 475         }
 476       case IR_CALLXS:
 477         goto split_call;
 478       case IR_PHI: {
 479         IRRef hiref2;
 480         if ((irref_isk(nir->op1) && irref_isk(nir->op2)) ||
 481             nir->op1 == nir->op2)
 482           J->cur.nins--;  /* Drop useless PHIs. */
 483         hiref2 = hisubst[ir->op2];
 484         if (!((irref_isk(hiref) && irref_isk(hiref2)) || hiref == hiref2))
 485           split_emit(J, IRTI(IR_PHI), hiref, hiref2);
 486         break;
 487         }
 488       default:
 489         lua_assert(ir->o <= IR_NE);  /* Comparisons. */
 490         split_emit(J, IRTGI(IR_HIOP), hiref, hisubst[ir->op2]);
 491         break;
 492       }
 493     } else
 494 #endif
 495 #if LJ_SOFTFP
 496     if (ir->o == IR_SLOAD) {
 497       if ((nir->op2 & IRSLOAD_CONVERT)) {  /* Convert from number to int. */
 498         nir->op2 &= ~IRSLOAD_CONVERT;
 499         if (!(nir->op2 & IRSLOAD_TYPECHECK))
 500           nir->t.irt = IRT_INT;  /* Drop guard. */
 501         split_emit(J, IRT(IR_HIOP, IRT_SOFTFP), nref, nref);
 502         ir->prev = split_num2int(J, nref, nref+1, irt_isguard(ir->t));
 503       }
 504     } else if (ir->o == IR_TOBIT) {
 505       IRRef tmp, op1 = ir->op1;
 506       J->cur.nins--;
 507 #if LJ_LE
 508       tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), oir[op1].prev, hisubst[op1]);
 509 #else
 510       tmp = split_emit(J, IRT(IR_CARG, IRT_NIL), hisubst[op1], oir[op1].prev);
 511 #endif
 512       ir->prev = split_emit(J, IRTI(IR_CALLN), tmp, IRCALL_lj_vm_tobit);
 513     } else if (ir->o == IR_TOSTR) {
 514       if (hisubst[ir->op1]) {
 515         if (irref_isk(ir->op1))
 516           nir->op1 = ir->op1;
 517         else
 518           split_emit(J, IRT(IR_HIOP, IRT_NIL), hisubst[ir->op1], nref);
 519       }
 520     } else if (ir->o == IR_HREF || ir->o == IR_NEWREF) {
 521       if (irref_isk(ir->op2) && hisubst[ir->op2])
 522         nir->op2 = ir->op2;
 523     } else
 524 #endif
 525     if (ir->o == IR_CONV) {  /* See above, too. */
 526       IRType st = (IRType)(ir->op2 & IRCONV_SRCMASK);
 527 #if LJ_32 && LJ_HASFFI
 528       if (st == IRT_I64 || st == IRT_U64) {  /* Conversion from 64 bit int. */
 529 #if LJ_SOFTFP
 530         if (irt_isfloat(ir->t)) {
 531           split_call_l(J, hisubst, oir, ir,
 532                        st == IRT_I64 ? IRCALL_fp64_l2f : IRCALL_fp64_ul2f);
 533           J->cur.nins--;  /* Drop unused HIOP. */
 534         }
 535 #else
 536         if (irt_isfp(ir->t)) {  /* 64 bit integer to FP conversion. */
 537           ir->prev = split_emit(J, IRT(IR_HIOP, irt_type(ir->t)),
 538                                 hisubst[ir->op1], nref);
 539         }
 540 #endif
 541         else {  /* Truncate to lower 32 bits. */
 542         fwdlo:
 543           ir->prev = nir->op1;  /* Forward loword. */
 544           /* Replace with NOP to avoid messing up the snapshot logic. */
 545           nir->ot = IRT(IR_NOP, IRT_NIL);
 546           nir->op1 = nir->op2 = 0;
 547         }
 548       }
 549 #endif
 550 #if LJ_SOFTFP && LJ_32 && LJ_HASFFI
 551       else if (irt_isfloat(ir->t)) {
 552         if (st == IRT_NUM) {
 553           split_call_l(J, hisubst, oir, ir, IRCALL_softfp_d2f);
 554           J->cur.nins--;  /* Drop unused HIOP. */
 555         } else {
 556           nir->o = IR_CALLN;
 557           nir->op2 = st == IRT_INT ? IRCALL_softfp_i2f : IRCALL_softfp_ui2f;
 558         }
 559       } else if (st == IRT_FLOAT) {
 560         nir->o = IR_CALLN;
 561         nir->op2 = irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui;
 562       } else
 563 #endif
 564 #if LJ_SOFTFP
 565       if (st == IRT_NUM || (LJ_32 && LJ_HASFFI && st == IRT_FLOAT)) {
 566         if (irt_isguard(ir->t)) {
 567           lua_assert(st == IRT_NUM && irt_isint(ir->t));
 568           J->cur.nins--;
 569           ir->prev = split_num2int(J, nir->op1, hisubst[ir->op1], 1);
 570         } else {
 571           split_call_l(J, hisubst, oir, ir,
 572 #if LJ_32 && LJ_HASFFI
 573             st == IRT_NUM ?
 574               (irt_isint(ir->t) ? IRCALL_softfp_d2i : IRCALL_softfp_d2ui) :
 575               (irt_isint(ir->t) ? IRCALL_softfp_f2i : IRCALL_softfp_f2ui)
 576 #else
 577             IRCALL_softfp_d2i
 578 #endif
 579           );
 580           J->cur.nins--;  /* Drop unused HIOP. */
 581         }
 582       }
 583 #endif
 584     } else if (ir->o == IR_CALLXS) {
 585       IRRef hiref;
 586     split_call:
 587       hiref = hisubst[ir->op1];
 588       if (hiref) {
 589         IROpT ot = nir->ot;
 590         IRRef op2 = nir->op2;
 591         nir->ot = IRT(IR_CARG, IRT_NIL);
 592 #if LJ_LE
 593         nir->op2 = hiref;
 594 #else
 595         nir->op2 = nir->op1; nir->op1 = hiref;
 596 #endif
 597         ir->prev = nref = split_emit(J, ot, nref, op2);
 598       }
 599       if (LJ_SOFTFP ? irt_is64(ir->t) : irt_isint64(ir->t))
 600         hi = split_emit(J,
 601           IRT(IR_HIOP, (LJ_SOFTFP && irt_isnum(ir->t)) ? IRT_SOFTFP : IRT_INT),
 602           nref, nref);
 603     } else if (ir->o == IR_CARG) {
 604       IRRef hiref = hisubst[ir->op1];
 605       if (hiref) {
 606         IRRef op2 = nir->op2;
 607 #if LJ_LE
 608         nir->op2 = hiref;
 609 #else
 610         nir->op2 = nir->op1; nir->op1 = hiref;
 611 #endif
 612         ir->prev = nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
 613         nir = IR(nref);
 614       }
 615       hiref = hisubst[ir->op2];
 616       if (hiref) {
 617 #if !LJ_TARGET_X86
 618         int carg = 0;
 619         IRIns *cir;
 620         for (cir = IR(nir->op1); cir->o == IR_CARG; cir = IR(cir->op1))
 621           carg++;
 622         if ((carg & 1) == 0) {  /* Align 64 bit arguments. */
 623           IRRef op2 = nir->op2;
 624           nir->op2 = REF_NIL;
 625           nref = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, op2);
 626           nir = IR(nref);
 627         }
 628 #endif
 629 #if LJ_BE
 630         { IRRef tmp = nir->op2; nir->op2 = hiref; hiref = tmp; }
 631 #endif
 632         ir->prev = split_emit(J, IRT(IR_CARG, IRT_NIL), nref, hiref);
 633       }
 634     } else if (ir->o == IR_CNEWI) {
 635       if (hisubst[ir->op2])
 636         split_emit(J, IRT(IR_HIOP, IRT_NIL), nref, hisubst[ir->op2]);
 637     } else if (ir->o == IR_LOOP) {
 638       J->loopref = nref;  /* Needed by assembler. */
 639     }
 640     hisubst[ref] = hi;  /* Store hiword substitution. */
 641   }
 642
 643   /* Add PHI marks. */
 644   for (ref = J->cur.nins-1; ref >= REF_FIRST; ref--) {
 645     IRIns *ir = IR(ref);
 646     if (ir->o != IR_PHI) break;
 647     if (!irref_isk(ir->op1)) irt_setphi(IR(ir->op1)->t);
 648     if (ir->op2 > J->loopref) irt_setphi(IR(ir->op2)->t);
 649   }
 650
 651   /* Substitute snapshot maps. */
 652   oir[nins].prev = J->cur.nins;  /* Substitution for last snapshot. */
 653   {
 654     SnapNo i, nsnap = J->cur.nsnap;
 655     for (i = 0; i < nsnap; i++) {
 656       SnapShot *snap = &J->cur.snap[i];
 657       SnapEntry *map = &J->cur.snapmap[snap->mapofs];
 658       MSize n, nent = snap->nent;
 659       snap->ref = snap->ref == REF_FIRST ? REF_FIRST : oir[snap->ref].prev;
 660       for (n = 0; n < nent; n++) {
 661         SnapEntry sn = map[n];
 662         IRIns *ir = &oir[snap_ref(sn)];
 663         if (!(LJ_SOFTFP && (sn & SNAP_SOFTFPNUM) && irref_isk(snap_ref(sn))))
 664           map[n] = ((sn & 0xffff0000) | ir->prev);
 665       }
 666     }
 667   }
 668 }
 669
 670 /* Protected callback for split pass. */
 671 static TValue *cpsplit(lua_State *L, lua_CFunction dummy, void *ud)
 672 {
 673   jit_State *J = (jit_State *)ud;
 674   split_ir(J);
 675   UNUSED(L); UNUSED(dummy);
 676   return NULL;
 677 }
 678
 679 #if defined(LUA_USE_ASSERT) || LJ_SOFTFP
 680 /* Slow, but sure way to check whether a SPLIT pass is needed. */
 681 static int split_needsplit(jit_State *J)
 682 {
 683   IRIns *ir, *irend;
 684   IRRef ref;
 685   for (ir = IR(REF_FIRST), irend = IR(J->cur.nins); ir < irend; ir++)
 686     if (LJ_SOFTFP ? irt_is64orfp(ir->t) : irt_isint64(ir->t))
 687       return 1;
 688   if (LJ_SOFTFP) {
 689     for (ref = J->chain[IR_SLOAD]; ref; ref = IR(ref)->prev)
 690       if ((IR(ref)->op2 & IRSLOAD_CONVERT))
 691         return 1;
 692   }
 693   for (ref = J->chain[IR_CONV]; ref; ref = IR(ref)->prev) {
 694     IRType st = (IR(ref)->op2 & IRCONV_SRCMASK);
 695     if ((LJ_SOFTFP && (st == IRT_NUM || st == IRT_FLOAT)) ||
 696         st == IRT_I64 || st == IRT_U64)
 697       return 1;
 698   }
 699   return 0;  /* Nope. */
 700 }
 701 #endif
 702
 703 /* SPLIT pass. */
 704 void lj_opt_split(jit_State *J)
 705 {
 706 #if LJ_SOFTFP
 707   if (!J->needsplit)
 708     J->needsplit = split_needsplit(J);
 709 #else
 710   lua_assert(J->needsplit >= split_needsplit(J));  /* Verify flag. */
 711 #endif
 712   if (J->needsplit) {
 713     int errcode = lj_vm_cpcall(J->L, NULL, J, cpsplit);
 714     if (errcode) {
 715       /* Completely reset the trace to avoid inconsistent dump on abort. */
 716       J->cur.nins = J->cur.nk = REF_BASE;
 717       J->cur.nsnap = 0;
 718       lj_err_throw(J->L, errcode);  /* Propagate errors. */
 719     }
 720   }
 721 }
 722
 723 #undef IR
 724
 725 #endif