2 ** SPLIT: Split 64 bit IR instructions into 32 bit IR instructions.
3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
11 #if LJ_HASJIT && (LJ_SOFTFP || (LJ_32 && LJ_HASFFI))
17 #include "lj_ircall.h"
23 ** This pass splits up 64 bit IR instructions into multiple 32 bit IR
24 ** instructions. It's only active for soft-float targets or for 32 bit CPUs
25 ** which lack native 64 bit integer operations (the FFI is currently the
26 ** only emitter for 64 bit integer instructions).
28 ** Splitting the IR in a separate pass keeps each 32 bit IR assembler
29 ** backend simple. Only a small amount of extra functionality needs to be
30 ** implemented. This is much easier than adding support for allocating
31 ** register pairs to each backend (believe me, I tried). A few simple, but
32 ** important optimizations can be performed by the SPLIT pass, which would
33 ** be tedious to do in the backend.
35 ** The basic idea is to replace each 64 bit IR instruction with its 32 bit
36 ** equivalent plus an extra HIOP instruction. The splitted IR is not passed
37 ** through FOLD or any other optimizations, so each HIOP is guaranteed to
38 ** immediately follow it's counterpart. The actual functionality of HIOP is
39 ** inferred from the previous instruction.
41 ** The operands of HIOP hold the hiword input references. The output of HIOP
42 ** is the hiword output reference, which is also used to hold the hiword
43 ** register or spill slot information. The register allocator treats this
44 ** instruction independently of any other instruction, which improves code
45 ** quality compared to using fixed register pairs.
47 ** It's easier to split up some instructions into two regular 32 bit
48 ** instructions. E.g. XLOAD is split up into two XLOADs with two different
49 ** addresses. Obviously 64 bit constants need to be split up into two 32 bit
50 ** constants, too. Some hiword instructions can be entirely omitted, e.g.
51 ** when zero-extending a 32 bit value to 64 bits. 64 bit arguments for calls
52 ** are split up into two 32 bit arguments each.
54 ** On soft-float targets, floating-point instructions are directly converted
55 ** to soft-float calls by the SPLIT pass (except for comparisons and MIN/MAX).
56 ** HIOP for number results has the type IRT_SOFTFP ("sfp" in -jdump).
58 ** Here's the IR and x64 machine code for 'x.b = x.a + 1' for a struct with
59 ** two int64_t fields:
61 ** 0100 p32 ADD base +8
62 ** 0101 i64 XLOAD 0100
63 ** 0102 i64 ADD 0101 +1
64 ** 0103 p32 ADD base +16
65 ** 0104 i64 XSTORE 0103 0102
69 ** mov [esi+0x10], rax
71 ** Here's the transformed IR and the x86 machine code after the SPLIT pass:
73 ** 0100 p32 ADD base +8
74 ** 0101 int XLOAD 0100
75 ** 0102 p32 ADD base +12
76 ** 0103 int XLOAD 0102
77 ** 0104 int ADD 0101 +1
78 ** 0105 int HIOP 0103 +0
79 ** 0106 p32 ADD base +16
80 ** 0107 int XSTORE 0106 0104
81 ** 0108 int HIOP 0106 0105
87 ** mov [esi+0x10], eax
88 ** mov [esi+0x14], ecx
90 ** You may notice the reassociated hiword address computation, which is
91 ** later fused into the mov operands by the assembler.
94 /* Some local macros to save typing. Undef'd at the end. */
95 #define IR(ref) (&J->cur.ir[(ref)])
97 /* Directly emit the transformed IR without updating chains etc. */
98 static IRRef
split_emit(jit_State
*J
, uint16_t ot
, IRRef1 op1
, IRRef1 op2
)
100 IRRef nref
= lj_ir_nextins(J
);
101 IRIns
*ir
= IR(nref
);
109 /* Emit a (checked) number to integer conversion. */
110 static IRRef
split_num2int(jit_State
*J
, IRRef lo
, IRRef hi
, int check
)
114 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), lo
, hi
);
116 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hi
, lo
);
118 res
= split_emit(J
, IRTI(IR_CALLN
), tmp
, IRCALL_softfp_d2i
);
120 tmp
= split_emit(J
, IRTI(IR_CALLN
), res
, IRCALL_softfp_i2d
);
121 split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
122 split_emit(J
, IRTGI(IR_EQ
), tmp
, lo
);
123 split_emit(J
, IRTG(IR_HIOP
, IRT_SOFTFP
), tmp
+1, hi
);
128 /* Emit a CALLN with one split 64 bit argument. */
129 static IRRef
split_call_l(jit_State
*J
, IRRef1
*hisubst
, IRIns
*oir
,
130 IRIns
*ir
, IRCallID id
)
132 IRRef tmp
, op1
= ir
->op1
;
135 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
137 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
139 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, id
);
140 return split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
143 /* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
144 static IRRef
split_call_li(jit_State
*J
, IRRef1
*hisubst
, IRIns
*oir
,
145 IRIns
*ir
, IRCallID id
)
147 IRRef tmp
, op1
= ir
->op1
, op2
= ir
->op2
;
150 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
152 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
154 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, oir
[op2
].prev
);
155 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, id
);
156 return split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
160 /* Emit a CALLN with two split 64 bit arguments. */
161 static IRRef
split_call_ll(jit_State
*J
, IRRef1
*hisubst
, IRIns
*oir
,
162 IRIns
*ir
, IRCallID id
)
164 IRRef tmp
, op1
= ir
->op1
, op2
= ir
->op2
;
167 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
168 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, oir
[op2
].prev
);
169 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, hisubst
[op2
]);
171 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
172 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, hisubst
[op2
]);
173 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, oir
[op2
].prev
);
175 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, id
);
177 IRT(IR_HIOP
, (LJ_SOFTFP
&& irt_isnum(ir
->t
)) ? IRT_SOFTFP
: IRT_INT
),
181 /* Get a pointer to the other 32 bit word (LE: hiword, BE: loword). */
182 static IRRef
split_ptr(jit_State
*J
, IRIns
*oir
, IRRef ref
)
184 IRRef nref
= oir
[ref
].prev
;
185 IRIns
*ir
= IR(nref
);
187 if (ir
->o
== IR_ADD
&& irref_isk(ir
->op2
) && !irt_isphi(oir
[ref
].t
)) {
188 /* Reassociate address. */
189 ofs
+= IR(ir
->op2
)->i
;
191 if (ofs
== 0) return nref
;
193 return split_emit(J
, IRTI(IR_ADD
), nref
, lj_ir_kint(J
, ofs
));
196 /* Transform the old IR to the new IR. */
197 static void split_ir(jit_State
*J
)
199 IRRef nins
= J
->cur
.nins
, nk
= J
->cur
.nk
;
200 MSize irlen
= nins
- nk
;
201 MSize need
= (irlen
+1)*(sizeof(IRIns
) + sizeof(IRRef1
));
202 IRIns
*oir
= (IRIns
*)lj_str_needbuf(J
->L
, &G(J
->L
)->tmpbuf
, need
);
206 /* Copy old IR to buffer. */
207 memcpy(oir
, IR(nk
), irlen
*sizeof(IRIns
));
208 /* Bias hiword substitution table and old IR. Loword kept in field prev. */
209 hisubst
= (IRRef1
*)&oir
[irlen
] - nk
;
212 /* Remove all IR instructions, but retain IR constants. */
213 J
->cur
.nins
= REF_FIRST
;
216 /* Process constants and fixed references. */
217 for (ref
= nk
; ref
<= REF_BASE
; ref
++) {
218 IRIns
*ir
= &oir
[ref
];
219 if ((LJ_SOFTFP
&& ir
->o
== IR_KNUM
) || ir
->o
== IR_KINT64
) {
220 /* Split up 64 bit constant. */
221 TValue tv
= *ir_k64(ir
);
222 ir
->prev
= lj_ir_kint(J
, (int32_t)tv
.u32
.lo
);
223 hisubst
[ref
] = lj_ir_kint(J
, (int32_t)tv
.u32
.hi
);
225 ir
->prev
= ref
; /* Identity substitution for loword. */
230 /* Process old IR instructions. */
231 for (ref
= REF_FIRST
; ref
< nins
; ref
++) {
232 IRIns
*ir
= &oir
[ref
];
233 IRRef nref
= lj_ir_nextins(J
);
234 IRIns
*nir
= IR(nref
);
237 /* Copy-substitute old instruction to new instruction. */
238 nir
->op1
= ir
->op1
< nk
? ir
->op1
: oir
[ir
->op1
].prev
;
239 nir
->op2
= ir
->op2
< nk
? ir
->op2
: oir
[ir
->op2
].prev
;
240 ir
->prev
= nref
; /* Loword substitution. */
242 nir
->t
.irt
= ir
->t
.irt
& ~(IRT_MARK
|IRT_ISPHI
);
245 /* Split 64 bit instructions. */
247 if (irt_isnum(ir
->t
)) {
248 nir
->t
.irt
= IRT_INT
| (nir
->t
.irt
& IRT_GUARD
); /* Turn into INT op. */
249 /* Note: hi ref = lo ref + 1! Required for SNAP_SOFTFPNUM logic. */
252 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_add
);
255 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_sub
);
258 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_mul
);
261 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_div
);
264 hi
= split_call_li(J
, hisubst
, oir
, ir
, IRCALL_lj_vm_powi
);
267 /* Try to rejoin pow from EXP2, MUL and LOG2. */
268 if (nir
->op2
== IRFPM_EXP2
&& nir
->op1
> J
->loopref
) {
269 IRIns
*irp
= IR(nir
->op1
);
270 if (irp
->o
== IR_CALLN
&& irp
->op2
== IRCALL_softfp_mul
) {
271 IRIns
*irm4
= IR(irp
->op1
);
272 IRIns
*irm3
= IR(irm4
->op1
);
273 IRIns
*irm12
= IR(irm3
->op1
);
274 IRIns
*irl1
= IR(irm12
->op1
);
275 if (irm12
->op1
> J
->loopref
&& irl1
->o
== IR_CALLN
&&
276 irl1
->op2
== IRCALL_lj_vm_log2
) {
277 IRRef tmp
= irl1
->op1
; /* Recycle first two args from LOG2. */
278 IRRef arg3
= irm3
->op2
, arg4
= irm4
->op2
;
280 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, arg3
);
281 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, arg4
);
282 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, IRCALL_pow
);
283 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
288 hi
= split_call_l(J
, hisubst
, oir
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
291 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_atan2
);
294 hi
= split_call_li(J
, hisubst
, oir
, ir
, IRCALL_ldexp
);
296 case IR_NEG
: case IR_ABS
:
297 nir
->o
= IR_CONV
; /* Pass through loword. */
298 nir
->op2
= (IRT_INT
<< 5) | IRT_INT
;
299 hi
= split_emit(J
, IRT(ir
->o
== IR_NEG
? IR_BXOR
: IR_BAND
, IRT_SOFTFP
),
300 hisubst
[ir
->op1
], hisubst
[ir
->op2
]);
303 if ((nir
->op2
& IRSLOAD_CONVERT
)) { /* Convert from int to number. */
304 nir
->op2
&= ~IRSLOAD_CONVERT
;
305 ir
->prev
= nref
= split_emit(J
, IRTI(IR_CALLN
), nref
,
307 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
311 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
313 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
316 IRIns inslo
= *nir
; /* Save/undo the emit of the lo XLOAD. */
318 hi
= split_ptr(J
, oir
, ir
->op1
); /* Insert the hiref ADD. */
319 nref
= lj_ir_nextins(J
);
321 *nir
= inslo
; /* Re-emit lo XLOAD immediately before hi XLOAD. */
322 hi
= split_emit(J
, IRT(IR_XLOAD
, IRT_SOFTFP
), hi
, ir
->op2
);
326 ir
->prev
= hi
; hi
= nref
;
330 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: case IR_XSTORE
:
331 split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nir
->op1
, hisubst
[ir
->op2
]);
333 case IR_CONV
: { /* Conversion to number. Others handled below. */
334 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
336 #if LJ_32 && LJ_HASFFI
337 if (st
== IRT_I64
|| st
== IRT_U64
) {
338 hi
= split_call_l(J
, hisubst
, oir
, ir
,
339 st
== IRT_I64
? IRCALL_fp64_l2d
: IRCALL_fp64_ul2d
);
343 lua_assert(st
== IRT_INT
||
344 (LJ_32
&& LJ_HASFFI
&& (st
== IRT_U32
|| st
== IRT_FLOAT
)));
346 #if LJ_32 && LJ_HASFFI
347 nir
->op2
= st
== IRT_INT
? IRCALL_softfp_i2d
:
348 st
== IRT_FLOAT
? IRCALL_softfp_f2d
:
351 nir
->op2
= IRCALL_softfp_i2d
;
353 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
362 if (nir
->op1
== nir
->op2
)
363 J
->cur
.nins
--; /* Drop useless PHIs. */
364 if (hisubst
[ir
->op1
] != hisubst
[ir
->op2
])
365 split_emit(J
, IRT(IR_PHI
, IRT_SOFTFP
),
366 hisubst
[ir
->op1
], hisubst
[ir
->op2
]);
369 J
->cur
.nins
--; /* Drop joining HIOP. */
374 lua_assert(ir
->o
<= IR_NE
|| ir
->o
== IR_MIN
|| ir
->o
== IR_MAX
);
375 hi
= split_emit(J
, IRTG(IR_HIOP
, IRT_SOFTFP
),
376 hisubst
[ir
->op1
], hisubst
[ir
->op2
]);
381 #if LJ_32 && LJ_HASFFI
382 if (irt_isint64(ir
->t
)) {
383 IRRef hiref
= hisubst
[ir
->op1
];
384 nir
->t
.irt
= IRT_INT
| (nir
->t
.irt
& IRT_GUARD
); /* Turn into INT op. */
388 /* Use plain op for hiword if loword cannot produce a carry/borrow. */
389 if (irref_isk(nir
->op2
) && IR(nir
->op2
)->i
== 0) {
390 ir
->prev
= nir
->op1
; /* Pass through loword. */
391 nir
->op1
= hiref
; nir
->op2
= hisubst
[ir
->op2
];
397 hi
= split_emit(J
, IRTI(IR_HIOP
), hiref
, hisubst
[ir
->op2
]);
400 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_lj_carith_mul64
);
403 hi
= split_call_ll(J
, hisubst
, oir
, ir
,
404 irt_isi64(ir
->t
) ? IRCALL_lj_carith_divi64
:
405 IRCALL_lj_carith_divu64
);
408 hi
= split_call_ll(J
, hisubst
, oir
, ir
,
409 irt_isi64(ir
->t
) ? IRCALL_lj_carith_modi64
:
410 IRCALL_lj_carith_modu64
);
413 hi
= split_call_ll(J
, hisubst
, oir
, ir
,
414 irt_isi64(ir
->t
) ? IRCALL_lj_carith_powi64
:
415 IRCALL_lj_carith_powu64
);
418 lua_assert(ir
->op2
== IRFL_CDATA_INT64
);
419 hi
= split_emit(J
, IRTI(IR_FLOAD
), nir
->op1
, IRFL_CDATA_INT64_4
);
421 ir
->prev
= hi
; hi
= nref
;
425 hi
= split_emit(J
, IRTI(IR_XLOAD
), split_ptr(J
, oir
, ir
->op1
), ir
->op2
);
427 ir
->prev
= hi
; hi
= nref
;
431 split_emit(J
, IRTI(IR_HIOP
), nir
->op1
, hisubst
[ir
->op2
]);
433 case IR_CONV
: { /* Conversion to 64 bit integer. Others handled below. */
434 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
436 if (st
== IRT_NUM
) { /* NUM to 64 bit int conv. */
437 hi
= split_call_l(J
, hisubst
, oir
, ir
,
438 irt_isi64(ir
->t
) ? IRCALL_fp64_d2l
: IRCALL_fp64_d2ul
);
439 } else if (st
== IRT_FLOAT
) { /* FLOAT to 64 bit int conv. */
441 nir
->op2
= irt_isi64(ir
->t
) ? IRCALL_fp64_f2l
: IRCALL_fp64_f2ul
;
442 hi
= split_emit(J
, IRTI(IR_HIOP
), nref
, nref
);
445 if (st
== IRT_NUM
|| st
== IRT_FLOAT
) { /* FP to 64 bit int conv. */
446 hi
= split_emit(J
, IRTI(IR_HIOP
), nir
->op1
, nref
);
449 else if (st
== IRT_I64
|| st
== IRT_U64
) { /* 64/64 bit cast. */
450 /* Drop cast, since assembler doesn't care. */
452 } else if ((ir
->op2
& IRCONV_SEXT
)) { /* Sign-extend to 64 bit. */
453 IRRef k31
= lj_ir_kint(J
, 31);
454 nir
= IR(nref
); /* May have been reallocated. */
455 ir
->prev
= nir
->op1
; /* Pass through loword. */
456 nir
->o
= IR_BSAR
; /* hi = bsar(lo, 31). */
459 } else { /* Zero-extend to 64 bit. */
460 hi
= lj_ir_kint(J
, 0);
469 if ((irref_isk(nir
->op1
) && irref_isk(nir
->op2
)) ||
470 nir
->op1
== nir
->op2
)
471 J
->cur
.nins
--; /* Drop useless PHIs. */
472 hiref2
= hisubst
[ir
->op2
];
473 if (!((irref_isk(hiref
) && irref_isk(hiref2
)) || hiref
== hiref2
))
474 split_emit(J
, IRTI(IR_PHI
), hiref
, hiref2
);
478 J
->cur
.nins
--; /* Drop joining HIOP. */
483 lua_assert(ir
->o
<= IR_NE
); /* Comparisons. */
484 split_emit(J
, IRTGI(IR_HIOP
), hiref
, hisubst
[ir
->op2
]);
490 if (ir
->o
== IR_SLOAD
) {
491 if ((nir
->op2
& IRSLOAD_CONVERT
)) { /* Convert from number to int. */
492 nir
->op2
&= ~IRSLOAD_CONVERT
;
493 if (!(nir
->op2
& IRSLOAD_TYPECHECK
))
494 nir
->t
.irt
= IRT_INT
; /* Drop guard. */
495 split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
496 ir
->prev
= split_num2int(J
, nref
, nref
+1, irt_isguard(ir
->t
));
498 } else if (ir
->o
== IR_TOBIT
) {
499 IRRef tmp
, op1
= ir
->op1
;
502 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
504 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
506 ir
->prev
= split_emit(J
, IRTI(IR_CALLN
), tmp
, IRCALL_lj_vm_tobit
);
507 } else if (ir
->o
== IR_TOSTR
) {
508 if (hisubst
[ir
->op1
]) {
509 if (irref_isk(ir
->op1
))
512 split_emit(J
, IRT(IR_HIOP
, IRT_NIL
), hisubst
[ir
->op1
], nref
);
514 } else if (ir
->o
== IR_HREF
|| ir
->o
== IR_NEWREF
) {
515 if (irref_isk(ir
->op2
) && hisubst
[ir
->op2
])
519 if (ir
->o
== IR_CONV
) { /* See above, too. */
520 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
521 #if LJ_32 && LJ_HASFFI
522 if (st
== IRT_I64
|| st
== IRT_U64
) { /* Conversion from 64 bit int. */
524 if (irt_isfloat(ir
->t
)) {
525 split_call_l(J
, hisubst
, oir
, ir
,
526 st
== IRT_I64
? IRCALL_fp64_l2f
: IRCALL_fp64_ul2f
);
527 J
->cur
.nins
--; /* Drop unused HIOP. */
530 if (irt_isfp(ir
->t
)) { /* 64 bit integer to FP conversion. */
531 ir
->prev
= split_emit(J
, IRT(IR_HIOP
, irt_type(ir
->t
)),
532 hisubst
[ir
->op1
], nref
);
535 else { /* Truncate to lower 32 bits. */
537 ir
->prev
= nir
->op1
; /* Forward loword. */
538 /* Replace with NOP to avoid messing up the snapshot logic. */
539 nir
->ot
= IRT(IR_NOP
, IRT_NIL
);
540 nir
->op1
= nir
->op2
= 0;
544 #if LJ_SOFTFP && LJ_32 && LJ_HASFFI
545 else if (irt_isfloat(ir
->t
)) {
547 split_call_l(J
, hisubst
, oir
, ir
, IRCALL_softfp_d2f
);
548 J
->cur
.nins
--; /* Drop unused HIOP. */
551 nir
->op2
= st
== IRT_INT
? IRCALL_softfp_i2f
: IRCALL_softfp_ui2f
;
553 } else if (st
== IRT_FLOAT
) {
555 nir
->op2
= irt_isint(ir
->t
) ? IRCALL_softfp_f2i
: IRCALL_softfp_f2ui
;
559 if (st
== IRT_NUM
|| (LJ_32
&& LJ_HASFFI
&& st
== IRT_FLOAT
)) {
560 if (irt_isguard(ir
->t
)) {
561 lua_assert(st
== IRT_NUM
&& irt_isint(ir
->t
));
563 ir
->prev
= split_num2int(J
, nir
->op1
, hisubst
[ir
->op1
], 1);
565 split_call_l(J
, hisubst
, oir
, ir
,
566 #if LJ_32 && LJ_HASFFI
568 (irt_isint(ir
->t
) ? IRCALL_softfp_d2i
: IRCALL_softfp_d2ui
) :
569 (irt_isint(ir
->t
) ? IRCALL_softfp_f2i
: IRCALL_softfp_f2ui
)
574 J
->cur
.nins
--; /* Drop unused HIOP. */
578 } else if (ir
->o
== IR_CALLXS
) {
581 hiref
= hisubst
[ir
->op1
];
584 IRRef op2
= nir
->op2
;
585 nir
->ot
= IRT(IR_CARG
, IRT_NIL
);
589 nir
->op2
= nir
->op1
; nir
->op1
= hiref
;
591 ir
->prev
= nref
= split_emit(J
, ot
, nref
, op2
);
593 if (LJ_SOFTFP
? irt_is64(ir
->t
) : irt_isint64(ir
->t
))
595 IRT(IR_HIOP
, (LJ_SOFTFP
&& irt_isnum(ir
->t
)) ? IRT_SOFTFP
: IRT_INT
),
597 } else if (ir
->o
== IR_CARG
) {
598 IRRef hiref
= hisubst
[ir
->op1
];
600 IRRef op2
= nir
->op2
;
604 nir
->op2
= nir
->op1
; nir
->op1
= hiref
;
606 ir
->prev
= nref
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), nref
, op2
);
609 hiref
= hisubst
[ir
->op2
];
614 for (cir
= IR(nir
->op1
); cir
->o
== IR_CARG
; cir
= IR(cir
->op1
))
616 if ((carg
& 1) == 0) { /* Align 64 bit arguments. */
617 IRRef op2
= nir
->op2
;
619 nref
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), nref
, op2
);
624 { IRRef tmp
= nir
->op2
; nir
->op2
= hiref
; hiref
= tmp
; }
626 ir
->prev
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), nref
, hiref
);
628 } else if (ir
->o
== IR_CNEWI
) {
629 if (hisubst
[ir
->op2
])
630 split_emit(J
, IRT(IR_HIOP
, IRT_NIL
), nref
, hisubst
[ir
->op2
]);
631 } else if (ir
->o
== IR_LOOP
) {
632 J
->loopref
= nref
; /* Needed by assembler. */
634 hisubst
[ref
] = hi
; /* Store hiword substitution. */
638 for (ref
= J
->cur
.nins
-1; ref
>= REF_FIRST
; ref
--) {
640 if (ir
->o
!= IR_PHI
) break;
641 if (!irref_isk(ir
->op1
)) irt_setphi(IR(ir
->op1
)->t
);
642 if (ir
->op2
> J
->loopref
) irt_setphi(IR(ir
->op2
)->t
);
645 /* Substitute snapshot maps. */
646 oir
[nins
].prev
= J
->cur
.nins
; /* Substitution for last snapshot. */
648 SnapNo i
, nsnap
= J
->cur
.nsnap
;
649 for (i
= 0; i
< nsnap
; i
++) {
650 SnapShot
*snap
= &J
->cur
.snap
[i
];
651 SnapEntry
*map
= &J
->cur
.snapmap
[snap
->mapofs
];
652 MSize n
, nent
= snap
->nent
;
653 snap
->ref
= snap
->ref
== REF_FIRST
? REF_FIRST
: oir
[snap
->ref
].prev
;
654 for (n
= 0; n
< nent
; n
++) {
655 SnapEntry sn
= map
[n
];
656 IRIns
*ir
= &oir
[snap_ref(sn
)];
657 if (!(LJ_SOFTFP
&& (sn
& SNAP_SOFTFPNUM
) && irref_isk(snap_ref(sn
))))
658 map
[n
] = ((sn
& 0xffff0000) | ir
->prev
);
664 /* Protected callback for split pass. */
665 static TValue
*cpsplit(lua_State
*L
, lua_CFunction dummy
, void *ud
)
667 jit_State
*J
= (jit_State
*)ud
;
669 UNUSED(L
); UNUSED(dummy
);
673 #if defined(LUA_USE_ASSERT) || LJ_SOFTFP
674 /* Slow, but sure way to check whether a SPLIT pass is needed. */
675 static int split_needsplit(jit_State
*J
)
679 for (ir
= IR(REF_FIRST
), irend
= IR(J
->cur
.nins
); ir
< irend
; ir
++)
680 if (LJ_SOFTFP
? irt_is64orfp(ir
->t
) : irt_isint64(ir
->t
))
683 for (ref
= J
->chain
[IR_SLOAD
]; ref
; ref
= IR(ref
)->prev
)
684 if ((IR(ref
)->op2
& IRSLOAD_CONVERT
))
686 if (J
->chain
[IR_TOBIT
])
689 for (ref
= J
->chain
[IR_CONV
]; ref
; ref
= IR(ref
)->prev
) {
690 IRType st
= (IR(ref
)->op2
& IRCONV_SRCMASK
);
691 if ((LJ_SOFTFP
&& (st
== IRT_NUM
|| st
== IRT_FLOAT
)) ||
692 st
== IRT_I64
|| st
== IRT_U64
)
695 return 0; /* Nope. */
700 void lj_opt_split(jit_State
*J
)
704 J
->needsplit
= split_needsplit(J
);
706 lua_assert(J
->needsplit
>= split_needsplit(J
)); /* Verify flag. */
709 int errcode
= lj_vm_cpcall(J
->L
, NULL
, J
, cpsplit
);
711 /* Completely reset the trace to avoid inconsistent dump on abort. */
712 J
->cur
.nins
= J
->cur
.nk
= REF_BASE
;
714 lj_err_throw(J
->L
, errcode
); /* Propagate errors. */