2 ** SPLIT: Split 64 bit IR instructions into 32 bit IR instructions.
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
11 #if LJ_HASJIT && (LJ_SOFTFP || (LJ_32 && LJ_HASFFI))
17 #include "lj_ircall.h"
23 ** This pass splits up 64 bit IR instructions into multiple 32 bit IR
24 ** instructions. It's only active for soft-float targets or for 32 bit CPUs
25 ** which lack native 64 bit integer operations (the FFI is currently the
26 ** only emitter for 64 bit integer instructions).
28 ** Splitting the IR in a separate pass keeps each 32 bit IR assembler
29 ** backend simple. Only a small amount of extra functionality needs to be
30 ** implemented. This is much easier than adding support for allocating
31 ** register pairs to each backend (believe me, I tried). A few simple, but
32 ** important optimizations can be performed by the SPLIT pass, which would
33 ** be tedious to do in the backend.
35 ** The basic idea is to replace each 64 bit IR instruction with its 32 bit
36 ** equivalent plus an extra HIOP instruction. The splitted IR is not passed
37 ** through FOLD or any other optimizations, so each HIOP is guaranteed to
38 ** immediately follow it's counterpart. The actual functionality of HIOP is
39 ** inferred from the previous instruction.
41 ** The operands of HIOP hold the hiword input references. The output of HIOP
42 ** is the hiword output reference, which is also used to hold the hiword
43 ** register or spill slot information. The register allocator treats this
44 ** instruction independently of any other instruction, which improves code
45 ** quality compared to using fixed register pairs.
47 ** It's easier to split up some instructions into two regular 32 bit
48 ** instructions. E.g. XLOAD is split up into two XLOADs with two different
49 ** addresses. Obviously 64 bit constants need to be split up into two 32 bit
50 ** constants, too. Some hiword instructions can be entirely omitted, e.g.
51 ** when zero-extending a 32 bit value to 64 bits. 64 bit arguments for calls
52 ** are split up into two 32 bit arguments each.
54 ** On soft-float targets, floating-point instructions are directly converted
55 ** to soft-float calls by the SPLIT pass (except for comparisons and MIN/MAX).
56 ** HIOP for number results has the type IRT_SOFTFP ("sfp" in -jdump).
58 ** Here's the IR and x64 machine code for 'x.b = x.a + 1' for a struct with
59 ** two int64_t fields:
61 ** 0100 p32 ADD base +8
62 ** 0101 i64 XLOAD 0100
63 ** 0102 i64 ADD 0101 +1
64 ** 0103 p32 ADD base +16
65 ** 0104 i64 XSTORE 0103 0102
69 ** mov [esi+0x10], rax
71 ** Here's the transformed IR and the x86 machine code after the SPLIT pass:
73 ** 0100 p32 ADD base +8
74 ** 0101 int XLOAD 0100
75 ** 0102 p32 ADD base +12
76 ** 0103 int XLOAD 0102
77 ** 0104 int ADD 0101 +1
78 ** 0105 int HIOP 0103 +0
79 ** 0106 p32 ADD base +16
80 ** 0107 int XSTORE 0106 0104
81 ** 0108 int HIOP 0106 0105
87 ** mov [esi+0x10], eax
88 ** mov [esi+0x14], ecx
90 ** You may notice the reassociated hiword address computation, which is
91 ** later fused into the mov operands by the assembler.
94 /* Some local macros to save typing. Undef'd at the end. */
95 #define IR(ref) (&J->cur.ir[(ref)])
97 /* Directly emit the transformed IR without updating chains etc. */
98 static IRRef
split_emit(jit_State
*J
, uint16_t ot
, IRRef1 op1
, IRRef1 op2
)
100 IRRef nref
= lj_ir_nextins(J
);
101 IRIns
*ir
= IR(nref
);
109 /* Emit a (checked) number to integer conversion. */
110 static IRRef
split_num2int(jit_State
*J
, IRRef lo
, IRRef hi
, int check
)
114 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), lo
, hi
);
116 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hi
, lo
);
118 res
= split_emit(J
, IRTI(IR_CALLN
), tmp
, IRCALL_softfp_d2i
);
120 tmp
= split_emit(J
, IRTI(IR_CALLN
), res
, IRCALL_softfp_i2d
);
121 split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
122 split_emit(J
, IRTGI(IR_EQ
), tmp
, lo
);
123 split_emit(J
, IRTG(IR_HIOP
, IRT_SOFTFP
), tmp
+1, hi
);
128 /* Emit a CALLN with one split 64 bit argument. */
129 static IRRef
split_call_l(jit_State
*J
, IRRef1
*hisubst
, IRIns
*oir
,
130 IRIns
*ir
, IRCallID id
)
132 IRRef tmp
, op1
= ir
->op1
;
135 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
137 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
139 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, id
);
140 return split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
143 /* Emit a CALLN with one split 64 bit argument and a 32 bit argument. */
144 static IRRef
split_call_li(jit_State
*J
, IRRef1
*hisubst
, IRIns
*oir
,
145 IRIns
*ir
, IRCallID id
)
147 IRRef tmp
, op1
= ir
->op1
, op2
= ir
->op2
;
150 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
152 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
154 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, oir
[op2
].prev
);
155 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, id
);
156 return split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
160 /* Emit a CALLN with two split 64 bit arguments. */
161 static IRRef
split_call_ll(jit_State
*J
, IRRef1
*hisubst
, IRIns
*oir
,
162 IRIns
*ir
, IRCallID id
)
164 IRRef tmp
, op1
= ir
->op1
, op2
= ir
->op2
;
167 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
168 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, oir
[op2
].prev
);
169 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, hisubst
[op2
]);
171 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
172 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, hisubst
[op2
]);
173 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, oir
[op2
].prev
);
175 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, id
);
177 IRT(IR_HIOP
, (LJ_SOFTFP
&& irt_isnum(ir
->t
)) ? IRT_SOFTFP
: IRT_INT
),
181 /* Get a pointer to the other 32 bit word (LE: hiword, BE: loword). */
182 static IRRef
split_ptr(jit_State
*J
, IRIns
*oir
, IRRef ref
)
184 IRRef nref
= oir
[ref
].prev
;
185 IRIns
*ir
= IR(nref
);
187 if (ir
->o
== IR_KPTR
)
188 return lj_ir_kptr(J
, (char *)ir_kptr(ir
) + ofs
);
189 if (ir
->o
== IR_ADD
&& irref_isk(ir
->op2
) && !irt_isphi(oir
[ref
].t
)) {
190 /* Reassociate address. */
191 ofs
+= IR(ir
->op2
)->i
;
193 if (ofs
== 0) return nref
;
195 return split_emit(J
, IRTI(IR_ADD
), nref
, lj_ir_kint(J
, ofs
));
198 /* Substitute references of a snapshot. */
199 static void split_subst_snap(jit_State
*J
, SnapShot
*snap
, IRIns
*oir
)
201 SnapEntry
*map
= &J
->cur
.snapmap
[snap
->mapofs
];
202 MSize n
, nent
= snap
->nent
;
203 for (n
= 0; n
< nent
; n
++) {
204 SnapEntry sn
= map
[n
];
205 IRIns
*ir
= &oir
[snap_ref(sn
)];
206 if (!(LJ_SOFTFP
&& (sn
& SNAP_SOFTFPNUM
) && irref_isk(snap_ref(sn
))))
207 map
[n
] = ((sn
& 0xffff0000) | ir
->prev
);
211 /* Transform the old IR to the new IR. */
212 static void split_ir(jit_State
*J
)
214 IRRef nins
= J
->cur
.nins
, nk
= J
->cur
.nk
;
215 MSize irlen
= nins
- nk
;
216 MSize need
= (irlen
+1)*(sizeof(IRIns
) + sizeof(IRRef1
));
217 IRIns
*oir
= (IRIns
*)lj_str_needbuf(J
->L
, &G(J
->L
)->tmpbuf
, need
);
222 /* Copy old IR to buffer. */
223 memcpy(oir
, IR(nk
), irlen
*sizeof(IRIns
));
224 /* Bias hiword substitution table and old IR. Loword kept in field prev. */
225 hisubst
= (IRRef1
*)&oir
[irlen
] - nk
;
228 /* Remove all IR instructions, but retain IR constants. */
229 J
->cur
.nins
= REF_FIRST
;
232 /* Process constants and fixed references. */
233 for (ref
= nk
; ref
<= REF_BASE
; ref
++) {
234 IRIns
*ir
= &oir
[ref
];
235 if ((LJ_SOFTFP
&& ir
->o
== IR_KNUM
) || ir
->o
== IR_KINT64
) {
236 /* Split up 64 bit constant. */
237 TValue tv
= *ir_k64(ir
);
238 ir
->prev
= lj_ir_kint(J
, (int32_t)tv
.u32
.lo
);
239 hisubst
[ref
] = lj_ir_kint(J
, (int32_t)tv
.u32
.hi
);
241 ir
->prev
= ref
; /* Identity substitution for loword. */
246 /* Process old IR instructions. */
249 for (ref
= REF_FIRST
; ref
< nins
; ref
++) {
250 IRIns
*ir
= &oir
[ref
];
251 IRRef nref
= lj_ir_nextins(J
);
252 IRIns
*nir
= IR(nref
);
257 split_subst_snap(J
, snap
++, oir
);
258 snref
= snap
< &J
->cur
.snap
[J
->cur
.nsnap
] ? snap
->ref
: ~(IRRef
)0;
261 /* Copy-substitute old instruction to new instruction. */
262 nir
->op1
= ir
->op1
< nk
? ir
->op1
: oir
[ir
->op1
].prev
;
263 nir
->op2
= ir
->op2
< nk
? ir
->op2
: oir
[ir
->op2
].prev
;
264 ir
->prev
= nref
; /* Loword substitution. */
266 nir
->t
.irt
= ir
->t
.irt
& ~(IRT_MARK
|IRT_ISPHI
);
269 /* Split 64 bit instructions. */
271 if (irt_isnum(ir
->t
)) {
272 nir
->t
.irt
= IRT_INT
| (nir
->t
.irt
& IRT_GUARD
); /* Turn into INT op. */
273 /* Note: hi ref = lo ref + 1! Required for SNAP_SOFTFPNUM logic. */
276 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_add
);
279 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_sub
);
282 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_mul
);
285 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_softfp_div
);
288 hi
= split_call_li(J
, hisubst
, oir
, ir
, IRCALL_lj_vm_powi
);
291 /* Try to rejoin pow from EXP2, MUL and LOG2. */
292 if (nir
->op2
== IRFPM_EXP2
&& nir
->op1
> J
->loopref
) {
293 IRIns
*irp
= IR(nir
->op1
);
294 if (irp
->o
== IR_CALLN
&& irp
->op2
== IRCALL_softfp_mul
) {
295 IRIns
*irm4
= IR(irp
->op1
);
296 IRIns
*irm3
= IR(irm4
->op1
);
297 IRIns
*irm12
= IR(irm3
->op1
);
298 IRIns
*irl1
= IR(irm12
->op1
);
299 if (irm12
->op1
> J
->loopref
&& irl1
->o
== IR_CALLN
&&
300 irl1
->op2
== IRCALL_lj_vm_log2
) {
301 IRRef tmp
= irl1
->op1
; /* Recycle first two args from LOG2. */
302 IRRef arg3
= irm3
->op2
, arg4
= irm4
->op2
;
304 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, arg3
);
305 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), tmp
, arg4
);
306 ir
->prev
= tmp
= split_emit(J
, IRTI(IR_CALLN
), tmp
, IRCALL_pow
);
307 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), tmp
, tmp
);
312 hi
= split_call_l(J
, hisubst
, oir
, ir
, IRCALL_lj_vm_floor
+ ir
->op2
);
315 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_atan2
);
318 hi
= split_call_li(J
, hisubst
, oir
, ir
, IRCALL_ldexp
);
320 case IR_NEG
: case IR_ABS
:
321 nir
->o
= IR_CONV
; /* Pass through loword. */
322 nir
->op2
= (IRT_INT
<< 5) | IRT_INT
;
323 hi
= split_emit(J
, IRT(ir
->o
== IR_NEG
? IR_BXOR
: IR_BAND
, IRT_SOFTFP
),
324 hisubst
[ir
->op1
], hisubst
[ir
->op2
]);
327 if ((nir
->op2
& IRSLOAD_CONVERT
)) { /* Convert from int to number. */
328 nir
->op2
&= ~IRSLOAD_CONVERT
;
329 ir
->prev
= nref
= split_emit(J
, IRTI(IR_CALLN
), nref
,
331 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
335 case IR_ALOAD
: case IR_HLOAD
: case IR_ULOAD
: case IR_VLOAD
:
337 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
340 IRIns inslo
= *nir
; /* Save/undo the emit of the lo XLOAD. */
342 hi
= split_ptr(J
, oir
, ir
->op1
); /* Insert the hiref ADD. */
343 nref
= lj_ir_nextins(J
);
345 *nir
= inslo
; /* Re-emit lo XLOAD immediately before hi XLOAD. */
346 hi
= split_emit(J
, IRT(IR_XLOAD
, IRT_SOFTFP
), hi
, ir
->op2
);
350 ir
->prev
= hi
; hi
= nref
;
354 case IR_ASTORE
: case IR_HSTORE
: case IR_USTORE
: case IR_XSTORE
:
355 split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nir
->op1
, hisubst
[ir
->op2
]);
357 case IR_CONV
: { /* Conversion to number. Others handled below. */
358 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
360 #if LJ_32 && LJ_HASFFI
361 if (st
== IRT_I64
|| st
== IRT_U64
) {
362 hi
= split_call_l(J
, hisubst
, oir
, ir
,
363 st
== IRT_I64
? IRCALL_fp64_l2d
: IRCALL_fp64_ul2d
);
367 lua_assert(st
== IRT_INT
||
368 (LJ_32
&& LJ_HASFFI
&& (st
== IRT_U32
|| st
== IRT_FLOAT
)));
370 #if LJ_32 && LJ_HASFFI
371 nir
->op2
= st
== IRT_INT
? IRCALL_softfp_i2d
:
372 st
== IRT_FLOAT
? IRCALL_softfp_f2d
:
375 nir
->op2
= IRCALL_softfp_i2d
;
377 hi
= split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
386 if (nir
->op1
== nir
->op2
)
387 J
->cur
.nins
--; /* Drop useless PHIs. */
388 if (hisubst
[ir
->op1
] != hisubst
[ir
->op2
])
389 split_emit(J
, IRT(IR_PHI
, IRT_SOFTFP
),
390 hisubst
[ir
->op1
], hisubst
[ir
->op2
]);
393 J
->cur
.nins
--; /* Drop joining HIOP. */
398 lua_assert(ir
->o
<= IR_NE
|| ir
->o
== IR_MIN
|| ir
->o
== IR_MAX
);
399 hi
= split_emit(J
, IRTG(IR_HIOP
, IRT_SOFTFP
),
400 hisubst
[ir
->op1
], hisubst
[ir
->op2
]);
405 #if LJ_32 && LJ_HASFFI
406 if (irt_isint64(ir
->t
)) {
407 IRRef hiref
= hisubst
[ir
->op1
];
408 nir
->t
.irt
= IRT_INT
| (nir
->t
.irt
& IRT_GUARD
); /* Turn into INT op. */
412 /* Use plain op for hiword if loword cannot produce a carry/borrow. */
413 if (irref_isk(nir
->op2
) && IR(nir
->op2
)->i
== 0) {
414 ir
->prev
= nir
->op1
; /* Pass through loword. */
415 nir
->op1
= hiref
; nir
->op2
= hisubst
[ir
->op2
];
421 hi
= split_emit(J
, IRTI(IR_HIOP
), hiref
, hisubst
[ir
->op2
]);
424 hi
= split_call_ll(J
, hisubst
, oir
, ir
, IRCALL_lj_carith_mul64
);
427 hi
= split_call_ll(J
, hisubst
, oir
, ir
,
428 irt_isi64(ir
->t
) ? IRCALL_lj_carith_divi64
:
429 IRCALL_lj_carith_divu64
);
432 hi
= split_call_ll(J
, hisubst
, oir
, ir
,
433 irt_isi64(ir
->t
) ? IRCALL_lj_carith_modi64
:
434 IRCALL_lj_carith_modu64
);
437 hi
= split_call_ll(J
, hisubst
, oir
, ir
,
438 irt_isi64(ir
->t
) ? IRCALL_lj_carith_powi64
:
439 IRCALL_lj_carith_powu64
);
442 lua_assert(ir
->op2
== IRFL_CDATA_INT64
);
443 hi
= split_emit(J
, IRTI(IR_FLOAD
), nir
->op1
, IRFL_CDATA_INT64_4
);
445 ir
->prev
= hi
; hi
= nref
;
449 hi
= split_emit(J
, IRTI(IR_XLOAD
), split_ptr(J
, oir
, ir
->op1
), ir
->op2
);
451 ir
->prev
= hi
; hi
= nref
;
455 split_emit(J
, IRTI(IR_HIOP
), nir
->op1
, hisubst
[ir
->op2
]);
457 case IR_CONV
: { /* Conversion to 64 bit integer. Others handled below. */
458 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
460 if (st
== IRT_NUM
) { /* NUM to 64 bit int conv. */
461 hi
= split_call_l(J
, hisubst
, oir
, ir
,
462 irt_isi64(ir
->t
) ? IRCALL_fp64_d2l
: IRCALL_fp64_d2ul
);
463 } else if (st
== IRT_FLOAT
) { /* FLOAT to 64 bit int conv. */
465 nir
->op2
= irt_isi64(ir
->t
) ? IRCALL_fp64_f2l
: IRCALL_fp64_f2ul
;
466 hi
= split_emit(J
, IRTI(IR_HIOP
), nref
, nref
);
469 if (st
== IRT_NUM
|| st
== IRT_FLOAT
) { /* FP to 64 bit int conv. */
470 hi
= split_emit(J
, IRTI(IR_HIOP
), nir
->op1
, nref
);
473 else if (st
== IRT_I64
|| st
== IRT_U64
) { /* 64/64 bit cast. */
474 /* Drop cast, since assembler doesn't care. But fwd both parts. */
477 } else if ((ir
->op2
& IRCONV_SEXT
)) { /* Sign-extend to 64 bit. */
478 IRRef k31
= lj_ir_kint(J
, 31);
479 nir
= IR(nref
); /* May have been reallocated. */
480 ir
->prev
= nir
->op1
; /* Pass through loword. */
481 nir
->o
= IR_BSAR
; /* hi = bsar(lo, 31). */
484 } else { /* Zero-extend to 64 bit. */
485 hi
= lj_ir_kint(J
, 0);
494 if ((irref_isk(nir
->op1
) && irref_isk(nir
->op2
)) ||
495 nir
->op1
== nir
->op2
)
496 J
->cur
.nins
--; /* Drop useless PHIs. */
497 hiref2
= hisubst
[ir
->op2
];
498 if (!((irref_isk(hiref
) && irref_isk(hiref2
)) || hiref
== hiref2
))
499 split_emit(J
, IRTI(IR_PHI
), hiref
, hiref2
);
503 J
->cur
.nins
--; /* Drop joining HIOP. */
508 lua_assert(ir
->o
<= IR_NE
); /* Comparisons. */
509 split_emit(J
, IRTGI(IR_HIOP
), hiref
, hisubst
[ir
->op2
]);
515 if (ir
->o
== IR_SLOAD
) {
516 if ((nir
->op2
& IRSLOAD_CONVERT
)) { /* Convert from number to int. */
517 nir
->op2
&= ~IRSLOAD_CONVERT
;
518 if (!(nir
->op2
& IRSLOAD_TYPECHECK
))
519 nir
->t
.irt
= IRT_INT
; /* Drop guard. */
520 split_emit(J
, IRT(IR_HIOP
, IRT_SOFTFP
), nref
, nref
);
521 ir
->prev
= split_num2int(J
, nref
, nref
+1, irt_isguard(ir
->t
));
523 } else if (ir
->o
== IR_TOBIT
) {
524 IRRef tmp
, op1
= ir
->op1
;
527 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), oir
[op1
].prev
, hisubst
[op1
]);
529 tmp
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), hisubst
[op1
], oir
[op1
].prev
);
531 ir
->prev
= split_emit(J
, IRTI(IR_CALLN
), tmp
, IRCALL_lj_vm_tobit
);
532 } else if (ir
->o
== IR_TOSTR
) {
533 if (hisubst
[ir
->op1
]) {
534 if (irref_isk(ir
->op1
))
537 split_emit(J
, IRT(IR_HIOP
, IRT_NIL
), hisubst
[ir
->op1
], nref
);
539 } else if (ir
->o
== IR_HREF
|| ir
->o
== IR_NEWREF
) {
540 if (irref_isk(ir
->op2
) && hisubst
[ir
->op2
])
544 if (ir
->o
== IR_CONV
) { /* See above, too. */
545 IRType st
= (IRType
)(ir
->op2
& IRCONV_SRCMASK
);
546 #if LJ_32 && LJ_HASFFI
547 if (st
== IRT_I64
|| st
== IRT_U64
) { /* Conversion from 64 bit int. */
549 if (irt_isfloat(ir
->t
)) {
550 split_call_l(J
, hisubst
, oir
, ir
,
551 st
== IRT_I64
? IRCALL_fp64_l2f
: IRCALL_fp64_ul2f
);
552 J
->cur
.nins
--; /* Drop unused HIOP. */
555 if (irt_isfp(ir
->t
)) { /* 64 bit integer to FP conversion. */
556 ir
->prev
= split_emit(J
, IRT(IR_HIOP
, irt_type(ir
->t
)),
557 hisubst
[ir
->op1
], nref
);
560 else { /* Truncate to lower 32 bits. */
562 ir
->prev
= nir
->op1
; /* Forward loword. */
563 /* Replace with NOP to avoid messing up the snapshot logic. */
564 nir
->ot
= IRT(IR_NOP
, IRT_NIL
);
565 nir
->op1
= nir
->op2
= 0;
569 #if LJ_SOFTFP && LJ_32 && LJ_HASFFI
570 else if (irt_isfloat(ir
->t
)) {
572 split_call_l(J
, hisubst
, oir
, ir
, IRCALL_softfp_d2f
);
573 J
->cur
.nins
--; /* Drop unused HIOP. */
576 nir
->op2
= st
== IRT_INT
? IRCALL_softfp_i2f
: IRCALL_softfp_ui2f
;
578 } else if (st
== IRT_FLOAT
) {
580 nir
->op2
= irt_isint(ir
->t
) ? IRCALL_softfp_f2i
: IRCALL_softfp_f2ui
;
584 if (st
== IRT_NUM
|| (LJ_32
&& LJ_HASFFI
&& st
== IRT_FLOAT
)) {
585 if (irt_isguard(ir
->t
)) {
586 lua_assert(st
== IRT_NUM
&& irt_isint(ir
->t
));
588 ir
->prev
= split_num2int(J
, nir
->op1
, hisubst
[ir
->op1
], 1);
590 split_call_l(J
, hisubst
, oir
, ir
,
591 #if LJ_32 && LJ_HASFFI
593 (irt_isint(ir
->t
) ? IRCALL_softfp_d2i
: IRCALL_softfp_d2ui
) :
594 (irt_isint(ir
->t
) ? IRCALL_softfp_f2i
: IRCALL_softfp_f2ui
)
599 J
->cur
.nins
--; /* Drop unused HIOP. */
603 } else if (ir
->o
== IR_CALLXS
) {
606 hiref
= hisubst
[ir
->op1
];
609 IRRef op2
= nir
->op2
;
610 nir
->ot
= IRT(IR_CARG
, IRT_NIL
);
614 nir
->op2
= nir
->op1
; nir
->op1
= hiref
;
616 ir
->prev
= nref
= split_emit(J
, ot
, nref
, op2
);
618 if (LJ_SOFTFP
? irt_is64(ir
->t
) : irt_isint64(ir
->t
))
620 IRT(IR_HIOP
, (LJ_SOFTFP
&& irt_isnum(ir
->t
)) ? IRT_SOFTFP
: IRT_INT
),
622 } else if (ir
->o
== IR_CARG
) {
623 IRRef hiref
= hisubst
[ir
->op1
];
625 IRRef op2
= nir
->op2
;
629 nir
->op2
= nir
->op1
; nir
->op1
= hiref
;
631 ir
->prev
= nref
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), nref
, op2
);
634 hiref
= hisubst
[ir
->op2
];
639 for (cir
= IR(nir
->op1
); cir
->o
== IR_CARG
; cir
= IR(cir
->op1
))
641 if ((carg
& 1) == 0) { /* Align 64 bit arguments. */
642 IRRef op2
= nir
->op2
;
644 nref
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), nref
, op2
);
649 { IRRef tmp
= nir
->op2
; nir
->op2
= hiref
; hiref
= tmp
; }
651 ir
->prev
= split_emit(J
, IRT(IR_CARG
, IRT_NIL
), nref
, hiref
);
653 } else if (ir
->o
== IR_CNEWI
) {
654 if (hisubst
[ir
->op2
])
655 split_emit(J
, IRT(IR_HIOP
, IRT_NIL
), nref
, hisubst
[ir
->op2
]);
656 } else if (ir
->o
== IR_LOOP
) {
657 J
->loopref
= nref
; /* Needed by assembler. */
659 hisubst
[ref
] = hi
; /* Store hiword substitution. */
661 if (snref
== nins
) { /* Substitution for last snapshot. */
662 snap
->ref
= J
->cur
.nins
;
663 split_subst_snap(J
, snap
, oir
);
667 for (ref
= J
->cur
.nins
-1; ref
>= REF_FIRST
; ref
--) {
669 if (ir
->o
!= IR_PHI
) break;
670 if (!irref_isk(ir
->op1
)) irt_setphi(IR(ir
->op1
)->t
);
671 if (ir
->op2
> J
->loopref
) irt_setphi(IR(ir
->op2
)->t
);
675 /* Protected callback for split pass. */
676 static TValue
*cpsplit(lua_State
*L
, lua_CFunction dummy
, void *ud
)
678 jit_State
*J
= (jit_State
*)ud
;
680 UNUSED(L
); UNUSED(dummy
);
684 #if defined(LUA_USE_ASSERT) || LJ_SOFTFP
685 /* Slow, but sure way to check whether a SPLIT pass is needed. */
686 static int split_needsplit(jit_State
*J
)
690 for (ir
= IR(REF_FIRST
), irend
= IR(J
->cur
.nins
); ir
< irend
; ir
++)
691 if (LJ_SOFTFP
? irt_is64orfp(ir
->t
) : irt_isint64(ir
->t
))
694 for (ref
= J
->chain
[IR_SLOAD
]; ref
; ref
= IR(ref
)->prev
)
695 if ((IR(ref
)->op2
& IRSLOAD_CONVERT
))
697 if (J
->chain
[IR_TOBIT
])
700 for (ref
= J
->chain
[IR_CONV
]; ref
; ref
= IR(ref
)->prev
) {
701 IRType st
= (IR(ref
)->op2
& IRCONV_SRCMASK
);
702 if ((LJ_SOFTFP
&& (st
== IRT_NUM
|| st
== IRT_FLOAT
)) ||
703 st
== IRT_I64
|| st
== IRT_U64
)
706 return 0; /* Nope. */
711 void lj_opt_split(jit_State
*J
)
715 J
->needsplit
= split_needsplit(J
);
717 lua_assert(J
->needsplit
>= split_needsplit(J
)); /* Verify flag. */
720 int errcode
= lj_vm_cpcall(J
->L
, NULL
, J
, cpsplit
);
722 /* Completely reset the trace to avoid inconsistent dump on abort. */
723 J
->cur
.nins
= J
->cur
.nk
= REF_BASE
;
725 lj_err_throw(J
->L
, errcode
); /* Propagate errors. */