2 ** Trace recorder for C data operations.
3 ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
11 #if LJ_HASJIT && LJ_HASFFI
18 #include "lj_cparse.h"
20 #include "lj_carith.h"
26 #include "lj_ircall.h"
29 #include "lj_record.h"
30 #include "lj_ffrecord.h"
32 #include "lj_crecord.h"
33 #include "lj_dispatch.h"
34 #include "lj_strfmt.h"
36 /* Some local macros to save typing. Undef'd at the end. */
37 #define IR(ref) (&J->cur.ir[(ref)])
39 /* Pass IR on to next optimization in chain (FOLD). */
40 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
42 #define emitconv(a, dt, st, flags) \
43 emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
45 /* -- C type checks ------------------------------------------------------- */
47 static GCcdata
*argv2cdata(jit_State
*J
, TRef tr
, cTValue
*o
)
51 if (!tref_iscdata(tr
))
52 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
54 /* Specialize to the CTypeID. */
55 trtypeid
= emitir(IRT(IR_FLOAD
, IRT_U16
), tr
, IRFL_CDATA_CTYPEID
);
56 emitir(IRTG(IR_EQ
, IRT_INT
), trtypeid
, lj_ir_kint(J
, (int32_t)cd
->ctypeid
));
60 /* Specialize to the CTypeID held by a cdata constructor. */
61 static CTypeID
crec_constructor(jit_State
*J
, GCcdata
*cd
, TRef tr
)
64 lua_assert(tref_iscdata(tr
) && cd
->ctypeid
== CTID_CTYPEID
);
65 id
= *(CTypeID
*)cdataptr(cd
);
66 tr
= emitir(IRT(IR_FLOAD
, IRT_INT
), tr
, IRFL_CDATA_INT
);
67 emitir(IRTG(IR_EQ
, IRT_INT
), tr
, lj_ir_kint(J
, (int32_t)id
));
71 static CTypeID
argv2ctype(jit_State
*J
, TRef tr
, cTValue
*o
)
77 /* Specialize to the string containing the C type declaration. */
78 emitir(IRTG(IR_EQ
, IRT_STR
), tr
, lj_ir_kstr(J
, s
));
80 cp
.cts
= ctype_ctsG(J2G(J
));
82 cp
.srcname
= strdata(s
);
85 cp
.mode
= CPARSE_MODE_ABSTRACT
|CPARSE_MODE_NOIMPLICIT
;
86 if (lj_cparse(&cp
) || cp
.cts
->top
> oldtop
) /* Avoid new struct defs. */
87 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
90 GCcdata
*cd
= argv2cdata(J
, tr
, o
);
91 return cd
->ctypeid
== CTID_CTYPEID
? crec_constructor(J
, cd
, tr
) :
96 /* Convert CType to IRType (if possible). */
97 static IRType
crec_ct2irt(CTState
*cts
, CType
*ct
)
99 if (ctype_isenum(ct
->info
)) ct
= ctype_child(cts
, ct
);
100 if (LJ_LIKELY(ctype_isnum(ct
->info
))) {
101 if ((ct
->info
& CTF_FP
)) {
102 if (ct
->size
== sizeof(double))
104 else if (ct
->size
== sizeof(float))
107 uint32_t b
= lj_fls(ct
->size
);
109 return IRT_I8
+ 2*b
+ ((ct
->info
& CTF_UNSIGNED
) ? 1 : 0);
111 } else if (ctype_isptr(ct
->info
)) {
112 return (LJ_64
&& ct
->size
== 8) ? IRT_P64
: IRT_P32
;
113 } else if (ctype_iscomplex(ct
->info
)) {
114 if (ct
->size
== 2*sizeof(double))
116 else if (ct
->size
== 2*sizeof(float))
122 /* -- Optimized memory fill and copy -------------------------------------- */
124 /* Maximum length and unroll of inlined copy/fill. */
125 #define CREC_COPY_MAXUNROLL 16
126 #define CREC_COPY_MAXLEN 128
128 #define CREC_FILL_MAXUNROLL 16
130 /* Number of windowed registers used for optimized memory copy. */
132 #define CREC_COPY_REGWIN 2
133 #elif LJ_TARGET_PPC || LJ_TARGET_MIPS
134 #define CREC_COPY_REGWIN 8
136 #define CREC_COPY_REGWIN 4
139 /* List of memory offsets for copy/fill. */
140 typedef struct CRecMemList
{
141 CTSize ofs
; /* Offset in bytes. */
142 IRType tp
; /* Type of load/store. */
143 TRef trofs
; /* TRef of interned offset. */
144 TRef trval
; /* TRef of load value. */
147 /* Generate copy list for element-wise struct copy. */
148 static MSize
crec_copy_struct(CRecMemList
*ml
, CTState
*cts
, CType
*ct
)
150 CTypeID fid
= ct
->sib
;
153 CType
*df
= ctype_get(cts
, fid
);
155 if (ctype_isfield(df
->info
)) {
158 if (!gcref(df
->name
)) continue; /* Ignore unnamed fields. */
159 cct
= ctype_rawchild(cts
, df
); /* Field type. */
160 tp
= crec_ct2irt(cts
, cct
);
161 if (tp
== IRT_CDATA
) return 0; /* NYI: aggregates. */
162 if (mlp
>= CREC_COPY_MAXUNROLL
) return 0;
163 ml
[mlp
].ofs
= df
->size
;
166 if (ctype_iscomplex(cct
->info
)) {
167 if (mlp
>= CREC_COPY_MAXUNROLL
) return 0;
168 ml
[mlp
].ofs
= df
->size
+ (cct
->size
>> 1);
172 } else if (!ctype_isconstval(df
->info
)) {
173 /* NYI: bitfields and sub-structures. */
180 /* Generate unrolled copy list, from highest to lowest step size/alignment. */
181 static MSize
crec_copy_unroll(CRecMemList
*ml
, CTSize len
, CTSize step
,
186 if (tp
== IRT_CDATA
) tp
= IRT_U8
+ 2*lj_fls(step
);
188 while (ofs
+ step
<= len
) {
189 if (mlp
>= CREC_COPY_MAXUNROLL
) return 0;
202 ** Emit copy list with windowed loads/stores.
203 ** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
205 static void crec_copy_emit(jit_State
*J
, CRecMemList
*ml
, MSize mlp
,
206 TRef trdst
, TRef trsrc
)
208 MSize i
, j
, rwin
= 0;
209 for (i
= 0, j
= 0; i
< mlp
; ) {
210 TRef trofs
= lj_ir_kintp(J
, ml
[i
].ofs
);
211 TRef trsptr
= emitir(IRT(IR_ADD
, IRT_PTR
), trsrc
, trofs
);
212 ml
[i
].trval
= emitir(IRT(IR_XLOAD
, ml
[i
].tp
), trsptr
, 0);
215 rwin
+= (LJ_SOFTFP
&& ml
[i
].tp
== IRT_NUM
) ? 2 : 1;
216 if (rwin
>= CREC_COPY_REGWIN
|| i
>= mlp
) { /* Flush buffered stores. */
218 for ( ; j
< i
; j
++) {
219 TRef trdptr
= emitir(IRT(IR_ADD
, IRT_PTR
), trdst
, ml
[j
].trofs
);
220 emitir(IRT(IR_XSTORE
, ml
[j
].tp
), trdptr
, ml
[j
].trval
);
226 /* Optimized memory copy. */
227 static void crec_copy(jit_State
*J
, TRef trdst
, TRef trsrc
, TRef trlen
,
230 if (tref_isk(trlen
)) { /* Length must be constant. */
231 CRecMemList ml
[CREC_COPY_MAXUNROLL
];
233 CTSize step
= 1, len
= (CTSize
)IR(tref_ref(trlen
))->i
;
234 IRType tp
= IRT_CDATA
;
236 if (len
== 0) return; /* Shortcut. */
237 if (len
> CREC_COPY_MAXLEN
) goto fallback
;
239 CTState
*cts
= ctype_ctsG(J2G(J
));
240 lua_assert(ctype_isarray(ct
->info
) || ctype_isstruct(ct
->info
));
241 if (ctype_isarray(ct
->info
)) {
242 CType
*cct
= ctype_rawchild(cts
, ct
);
243 tp
= crec_ct2irt(cts
, cct
);
244 if (tp
== IRT_CDATA
) goto rawcopy
;
245 step
= lj_ir_type_size
[tp
];
246 lua_assert((len
& (step
-1)) == 0);
247 } else if ((ct
->info
& CTF_UNION
)) {
248 step
= (1u << ctype_align(ct
->info
));
251 mlp
= crec_copy_struct(ml
, cts
, ct
);
257 if (LJ_TARGET_UNALIGNED
|| step
>= CTSIZE_PTR
)
260 mlp
= crec_copy_unroll(ml
, len
, step
, tp
);
263 crec_copy_emit(J
, ml
, mlp
, trdst
, trsrc
);
265 emitir(IRT(IR_XBAR
, IRT_NIL
), 0, 0);
270 /* Call memcpy. Always needs a barrier to disable alias analysis. */
271 lj_ir_call(J
, IRCALL_memcpy
, trdst
, trsrc
, trlen
);
272 emitir(IRT(IR_XBAR
, IRT_NIL
), 0, 0);
275 /* Generate unrolled fill list, from highest to lowest step size/alignment. */
276 static MSize
crec_fill_unroll(CRecMemList
*ml
, CTSize len
, CTSize step
)
280 IRType tp
= IRT_U8
+ 2*lj_fls(step
);
282 while (ofs
+ step
<= len
) {
283 if (mlp
>= CREC_COPY_MAXUNROLL
) return 0;
296 ** Emit stores for fill list.
297 ** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
299 static void crec_fill_emit(jit_State
*J
, CRecMemList
*ml
, MSize mlp
,
300 TRef trdst
, TRef trfill
)
303 for (i
= 0; i
< mlp
; i
++) {
304 TRef trofs
= lj_ir_kintp(J
, ml
[i
].ofs
);
305 TRef trdptr
= emitir(IRT(IR_ADD
, IRT_PTR
), trdst
, trofs
);
306 emitir(IRT(IR_XSTORE
, ml
[i
].tp
), trdptr
, trfill
);
310 /* Optimized memory fill. */
311 static void crec_fill(jit_State
*J
, TRef trdst
, TRef trlen
, TRef trfill
,
314 if (tref_isk(trlen
)) { /* Length must be constant. */
315 CRecMemList ml
[CREC_FILL_MAXUNROLL
];
317 CTSize len
= (CTSize
)IR(tref_ref(trlen
))->i
;
318 if (len
== 0) return; /* Shortcut. */
319 if (LJ_TARGET_UNALIGNED
|| step
>= CTSIZE_PTR
)
321 if (step
* CREC_FILL_MAXUNROLL
< len
) goto fallback
;
322 mlp
= crec_fill_unroll(ml
, len
, step
);
323 if (!mlp
) goto fallback
;
324 if (tref_isk(trfill
) || ml
[0].tp
!= IRT_U8
)
325 trfill
= emitconv(trfill
, IRT_INT
, IRT_U8
, 0);
326 if (ml
[0].tp
!= IRT_U8
) { /* Scatter U8 to U16/U32/U64. */
327 if (CTSIZE_PTR
== 8 && ml
[0].tp
== IRT_U64
) {
328 if (tref_isk(trfill
)) /* Pointless on x64 with zero-extended regs. */
329 trfill
= emitconv(trfill
, IRT_U64
, IRT_U32
, 0);
330 trfill
= emitir(IRT(IR_MUL
, IRT_U64
), trfill
,
331 lj_ir_kint64(J
, U64x(01010101,01010101)));
333 trfill
= emitir(IRTI(IR_MUL
), trfill
,
334 lj_ir_kint(J
, ml
[0].tp
== IRT_U16
? 0x0101 : 0x01010101));
337 crec_fill_emit(J
, ml
, mlp
, trdst
, trfill
);
340 /* Call memset. Always needs a barrier to disable alias analysis. */
341 lj_ir_call(J
, IRCALL_memset
, trdst
, trfill
, trlen
); /* Note: arg order! */
343 emitir(IRT(IR_XBAR
, IRT_NIL
), 0, 0);
346 /* -- Convert C type to C type -------------------------------------------- */
349 ** This code mirrors the code in lj_cconv.c. It performs the same steps
350 ** for the trace recorder that lj_cconv.c does for the interpreter.
352 ** One major difference is that we can get away with much fewer checks
353 ** here. E.g. checks for casts, constness or correct types can often be
354 ** omitted, even if they might fail. The interpreter subsequently throws
355 ** an error, which aborts the trace.
357 ** All operations are specialized to their C types, so the on-trace
358 ** outcome must be the same as the outcome in the interpreter. If the
359 ** interpreter doesn't throw an error, then the trace is correct, too.
360 ** Care must be taken not to generate invalid (temporary) IR or to
364 /* Determine whether a passed number or cdata number is non-zero. */
365 static int crec_isnonzero(CType
*s
, void *p
)
371 if ((s
->info
& CTF_FP
)) {
372 if (s
->size
== sizeof(float))
373 return (*(float *)p
!= 0);
375 return (*(double *)p
!= 0);
378 return (*(uint8_t *)p
!= 0);
379 else if (s
->size
== 2)
380 return (*(uint16_t *)p
!= 0);
381 else if (s
->size
== 4)
382 return (*(uint32_t *)p
!= 0);
384 return (*(uint64_t *)p
!= 0);
388 static TRef
crec_ct_ct(jit_State
*J
, CType
*d
, CType
*s
, TRef dp
, TRef sp
,
391 IRType dt
= crec_ct2irt(ctype_ctsG(J2G(J
)), d
);
392 IRType st
= crec_ct2irt(ctype_ctsG(J2G(J
)), s
);
393 CTSize dsize
= d
->size
, ssize
= s
->size
;
394 CTInfo dinfo
= d
->info
, sinfo
= s
->info
;
396 if (ctype_type(dinfo
) > CT_MAYCONVERT
|| ctype_type(sinfo
) > CT_MAYCONVERT
)
400 ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
401 ** numbers up to 8 bytes. Otherwise sp holds a pointer.
404 switch (cconv_idx2(dinfo
, sinfo
)) {
405 /* Destination is a bool. */
407 goto xstore
; /* Source operand is already normalized. */
410 if (st
!= IRT_CDATA
) {
411 /* Specialize to the result of a comparison against 0. */
412 TRef zero
= (st
== IRT_NUM
|| st
== IRT_FLOAT
) ? lj_ir_knum(J
, 0) :
413 (st
== IRT_I64
|| st
== IRT_U64
) ? lj_ir_kint64(J
, 0) :
415 int isnz
= crec_isnonzero(s
, svisnz
);
416 emitir(IRTG(isnz
? IR_NE
: IR_EQ
, st
), sp
, zero
);
417 sp
= lj_ir_kint(J
, isnz
);
422 /* Destination is an integer. */
426 if (dt
== IRT_CDATA
|| st
== IRT_CDATA
) goto err_nyi
;
427 /* Extend 32 to 64 bit integer. */
428 if (dsize
== 8 && ssize
< 8 && !(LJ_64
&& (sinfo
& CTF_UNSIGNED
)))
429 sp
= emitconv(sp
, dt
, ssize
< 4 ? IRT_INT
: st
,
430 (sinfo
& CTF_UNSIGNED
) ? 0 : IRCONV_SEXT
);
431 else if (dsize
< 8 && ssize
== 8) /* Truncate from 64 bit integer. */
432 sp
= emitconv(sp
, dsize
< 4 ? IRT_INT
: dt
, st
, 0);
433 else if (st
== IRT_INT
)
434 sp
= lj_opt_narrow_toint(J
, sp
);
436 if (dt
== IRT_I64
|| dt
== IRT_U64
) lj_needsplit(J
);
437 if (dp
== 0) return sp
;
438 emitir(IRT(IR_XSTORE
, dt
), dp
, sp
);
441 sp
= emitir(IRT(IR_XLOAD
, st
), sp
, 0); /* Load re. */
444 if (dt
== IRT_CDATA
|| st
== IRT_CDATA
) goto err_nyi
;
445 sp
= emitconv(sp
, dsize
< 4 ? IRT_INT
: dt
, st
, IRCONV_ANY
);
449 sinfo
= CTINFO(CT_NUM
, CTF_UNSIGNED
);
452 if (((dsize
^ ssize
) & 8) == 0) { /* Must insert no-op type conversion. */
453 sp
= emitconv(sp
, dsize
< 4 ? IRT_INT
: dt
, IRT_PTR
, 0);
458 /* Destination is a floating-point number. */
462 if (dt
== IRT_CDATA
|| st
== IRT_CDATA
) goto err_nyi
;
463 sp
= emitconv(sp
, dt
, ssize
< 4 ? IRT_INT
: st
, 0);
466 sp
= emitir(IRT(IR_XLOAD
, st
), sp
, 0); /* Load re. */
470 if (dt
== IRT_CDATA
|| st
== IRT_CDATA
) goto err_nyi
;
471 if (dt
!= st
) sp
= emitconv(sp
, dt
, st
, 0);
474 /* Destination is a complex number. */
478 TRef ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), dp
, lj_ir_kintp(J
, (dsize
>> 1)));
479 emitir(IRT(IR_XSTORE
, dt
), ptr
, lj_ir_knum(J
, 0));
482 if ((sinfo
& CTF_FP
)) goto conv_F_F
; else goto conv_F_I
;
485 if (dt
== IRT_CDATA
|| st
== IRT_CDATA
) goto err_nyi
;
488 re
= emitir(IRT(IR_XLOAD
, st
), sp
, 0);
489 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), sp
, lj_ir_kintp(J
, (ssize
>> 1)));
490 im
= emitir(IRT(IR_XLOAD
, st
), ptr
, 0);
492 re
= emitconv(re
, dt
, st
, 0);
493 im
= emitconv(im
, dt
, st
, 0);
495 emitir(IRT(IR_XSTORE
, dt
), dp
, re
);
496 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), dp
, lj_ir_kintp(J
, (dsize
>> 1)));
497 emitir(IRT(IR_XSTORE
, dt
), ptr
, im
);
501 /* Destination is a vector. */
508 /* Destination is a pointer. */
512 /* There are only 32 bit pointers/addresses on 32 bit machines.
513 ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
517 if (st
== IRT_CDATA
) goto err_nyi
;
518 if (!LJ_64
&& ssize
== 8) /* Truncate from 64 bit integer. */
519 sp
= emitconv(sp
, IRT_U32
, st
, 0);
522 if (st
== IRT_CDATA
) goto err_nyi
;
523 /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
524 sp
= emitconv(sp
, (LJ_64
&& dsize
== 8) ? IRT_I64
: IRT_U32
,
528 /* Destination is an array. */
530 /* Destination is a struct/union. */
532 if (dp
== 0) goto err_conv
;
533 crec_copy(J
, dp
, sp
, lj_ir_kint(J
, dsize
), d
);
539 lj_trace_err(J
, LJ_TRERR_NYICONV
);
545 /* -- Convert C type to TValue (load) ------------------------------------- */
547 static TRef
crec_tv_ct(jit_State
*J
, CType
*s
, CTypeID sid
, TRef sp
)
549 CTState
*cts
= ctype_ctsG(J2G(J
));
550 IRType t
= crec_ct2irt(cts
, s
);
551 CTInfo sinfo
= s
->info
;
552 if (ctype_isnum(sinfo
)) {
555 goto err_nyi
; /* NYI: copyval of >64 bit integers. */
556 tr
= emitir(IRT(IR_XLOAD
, t
), sp
, 0);
557 if (t
== IRT_FLOAT
|| t
== IRT_U32
) { /* Keep uint32_t/float as numbers. */
558 return emitconv(tr
, IRT_NUM
, t
, 0);
559 } else if (t
== IRT_I64
|| t
== IRT_U64
) { /* Box 64 bit integer. */
562 } else if ((sinfo
& CTF_BOOL
)) {
563 /* Assume not equal to zero. Fixup and emit pending guard later. */
564 lj_ir_set(J
, IRTGI(IR_NE
), tr
, lj_ir_kint(J
, 0));
565 J
->postproc
= LJ_POST_FIXGUARD
;
570 } else if (ctype_isptr(sinfo
) || ctype_isenum(sinfo
)) {
571 sp
= emitir(IRT(IR_XLOAD
, t
), sp
, 0); /* Box pointers and enums. */
572 } else if (ctype_isrefarray(sinfo
) || ctype_isstruct(sinfo
)) {
574 sid
= lj_ctype_intern(cts
, CTINFO_REF(sid
), CTSIZE_PTR
); /* Create ref. */
575 } else if (ctype_iscomplex(sinfo
)) { /* Unbox/box complex. */
576 ptrdiff_t esz
= (ptrdiff_t)(s
->size
>> 1);
577 TRef ptr
, tr1
, tr2
, dp
;
578 dp
= emitir(IRTG(IR_CNEW
, IRT_CDATA
), lj_ir_kint(J
, sid
), TREF_NIL
);
579 tr1
= emitir(IRT(IR_XLOAD
, t
), sp
, 0);
580 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), sp
, lj_ir_kintp(J
, esz
));
581 tr2
= emitir(IRT(IR_XLOAD
, t
), ptr
, 0);
582 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), dp
, lj_ir_kintp(J
, sizeof(GCcdata
)));
583 emitir(IRT(IR_XSTORE
, t
), ptr
, tr1
);
584 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), dp
, lj_ir_kintp(J
, sizeof(GCcdata
)+esz
));
585 emitir(IRT(IR_XSTORE
, t
), ptr
, tr2
);
588 /* NYI: copyval of vectors. */
590 lj_trace_err(J
, LJ_TRERR_NYICONV
);
592 /* Box pointer, ref, enum or 64 bit integer. */
593 return emitir(IRTG(IR_CNEWI
, IRT_CDATA
), lj_ir_kint(J
, sid
), sp
);
596 /* -- Convert TValue to C type (store) ------------------------------------ */
598 static TRef
crec_ct_tv(jit_State
*J
, CType
*d
, TRef dp
, TRef sp
, cTValue
*sval
)
600 CTState
*cts
= ctype_ctsG(J2G(J
));
601 CTypeID sid
= CTID_P_VOID
;
604 if (LJ_LIKELY(tref_isinteger(sp
))) {
606 svisnz
= (void *)(intptr_t)(tvisint(sval
)?(intV(sval
)!=0):!tviszero(sval
));
607 } else if (tref_isnum(sp
)) {
609 svisnz
= (void *)(intptr_t)(tvisint(sval
)?(intV(sval
)!=0):!tviszero(sval
));
610 } else if (tref_isbool(sp
)) {
611 sp
= lj_ir_kint(J
, tref_istrue(sp
) ? 1 : 0);
613 } else if (tref_isnil(sp
)) {
614 sp
= lj_ir_kptr(J
, NULL
);
615 } else if (tref_isudata(sp
)) {
616 GCudata
*ud
= udataV(sval
);
617 if (ud
->udtype
== UDTYPE_IO_FILE
) {
618 TRef tr
= emitir(IRT(IR_FLOAD
, IRT_U8
), sp
, IRFL_UDATA_UDTYPE
);
619 emitir(IRTGI(IR_EQ
), tr
, lj_ir_kint(J
, UDTYPE_IO_FILE
));
620 sp
= emitir(IRT(IR_FLOAD
, IRT_PTR
), sp
, IRFL_UDATA_FILE
);
622 sp
= emitir(IRT(IR_ADD
, IRT_PTR
), sp
, lj_ir_kintp(J
, sizeof(GCudata
)));
624 } else if (tref_isstr(sp
)) {
625 if (ctype_isenum(d
->info
)) { /* Match string against enum constant. */
626 GCstr
*str
= strV(sval
);
628 CType
*cct
= lj_ctype_getfield(cts
, d
, str
, &ofs
);
629 /* Specialize to the name of the enum constant. */
630 emitir(IRTG(IR_EQ
, IRT_STR
), sp
, lj_ir_kstr(J
, str
));
631 if (cct
&& ctype_isconstval(cct
->info
)) {
632 lua_assert(ctype_child(cts
, cct
)->size
== 4);
633 svisnz
= (void *)(intptr_t)(ofs
!= 0);
634 sp
= lj_ir_kint(J
, (int32_t)ofs
);
635 sid
= ctype_cid(cct
->info
);
636 } /* else: interpreter will throw. */
637 } else if (ctype_isrefarray(d
->info
)) { /* Copy string to array. */
638 lj_trace_err(J
, LJ_TRERR_BADTYPE
); /* NYI */
639 } else { /* Otherwise pass the string data as a const char[]. */
640 /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
641 sp
= emitir(IRT(IR_ADD
, IRT_PTR
), sp
, lj_ir_kintp(J
, sizeof(GCstr
)));
644 } else if (tref_islightud(sp
)) {
646 sp
= emitir(IRT(IR_BAND
, IRT_P64
), sp
,
647 lj_ir_kint64(J
, U64x(00007fff
,ffffffff
)));
649 } else { /* NYI: tref_istab(sp). */
651 sid
= argv2cdata(J
, sp
, sval
)->ctypeid
;
652 s
= ctype_raw(cts
, sid
);
653 svisnz
= cdataptr(cdataV(sval
));
654 if (ctype_isfunc(s
->info
)) {
655 sid
= lj_ctype_intern(cts
, CTINFO(CT_PTR
, CTALIGN_PTR
|sid
), CTSIZE_PTR
);
656 s
= ctype_get(cts
, sid
);
659 t
= crec_ct2irt(cts
, s
);
661 if (ctype_isptr(s
->info
)) {
662 sp
= emitir(IRT(IR_FLOAD
, t
), sp
, IRFL_CDATA_PTR
);
663 if (ctype_isref(s
->info
)) {
664 svisnz
= *(void **)svisnz
;
665 s
= ctype_rawchild(cts
, s
);
666 if (ctype_isenum(s
->info
)) s
= ctype_child(cts
, s
);
667 t
= crec_ct2irt(cts
, s
);
671 } else if (t
== IRT_I64
|| t
== IRT_U64
) {
672 sp
= emitir(IRT(IR_FLOAD
, t
), sp
, IRFL_CDATA_INT64
);
675 } else if (t
== IRT_INT
|| t
== IRT_U32
) {
676 if (ctype_isenum(s
->info
)) s
= ctype_child(cts
, s
);
677 sp
= emitir(IRT(IR_FLOAD
, t
), sp
, IRFL_CDATA_INT
);
680 sp
= emitir(IRT(IR_ADD
, IRT_PTR
), sp
, lj_ir_kintp(J
, sizeof(GCcdata
)));
682 if (ctype_isnum(s
->info
) && t
!= IRT_CDATA
)
683 sp
= emitir(IRT(IR_XLOAD
, t
), sp
, 0); /* Load number value. */
686 s
= ctype_get(cts
, sid
);
688 if (ctype_isenum(d
->info
)) d
= ctype_child(cts
, d
);
689 return crec_ct_ct(J
, d
, s
, dp
, sp
, svisnz
);
692 /* -- C data metamethods -------------------------------------------------- */
694 /* This would be rather difficult in FOLD, so do it here:
695 ** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
696 ** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
698 static TRef
crec_reassoc_ofs(jit_State
*J
, TRef tr
, ptrdiff_t *ofsp
, MSize sz
)
700 IRIns
*ir
= IR(tref_ref(tr
));
701 if (LJ_LIKELY(J
->flags
& JIT_F_OPT_FOLD
) && irref_isk(ir
->op2
) &&
702 (ir
->o
== IR_ADD
|| ir
->o
== IR_ADDOV
|| ir
->o
== IR_SUBOV
)) {
703 IRIns
*irk
= IR(ir
->op2
);
705 if (LJ_64
&& irk
->o
== IR_KINT64
)
706 k
= (ptrdiff_t)ir_kint64(irk
)->u64
* sz
;
708 k
= (ptrdiff_t)irk
->i
* sz
;
709 if (ir
->o
== IR_SUBOV
) *ofsp
-= k
; else *ofsp
+= k
;
710 tr
= ir
->op1
; /* Not a TRef, but the caller doesn't care. */
715 /* Record ctype __index/__newindex metamethods. */
716 static void crec_index_meta(jit_State
*J
, CTState
*cts
, CType
*ct
,
719 CTypeID id
= ctype_typeid(cts
, ct
);
720 cTValue
*tv
= lj_ctype_meta(cts
, id
, rd
->data
? MM_newindex
: MM_index
);
722 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
724 J
->base
[-1] = lj_ir_kfunc(J
, funcV(tv
)) | TREF_FRAME
;
725 rd
->nres
= -1; /* Pending tailcall. */
726 } else if (rd
->data
== 0 && tvistab(tv
) && tref_isstr(J
->base
[1])) {
727 /* Specialize to result of __index lookup. */
728 cTValue
*o
= lj_tab_get(J
->L
, tabV(tv
), &rd
->argv
[1]);
729 J
->base
[0] = lj_record_constify(J
, o
);
731 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
732 /* Always specialize to the key. */
733 emitir(IRTG(IR_EQ
, IRT_STR
), J
->base
[1], lj_ir_kstr(J
, strV(&rd
->argv
[1])));
735 /* NYI: resolving of non-function metamethods. */
736 /* NYI: non-string keys for __index table. */
737 /* NYI: stores to __newindex table. */
738 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
742 void LJ_FASTCALL
recff_cdata_index(jit_State
*J
, RecordFFData
*rd
)
744 TRef idx
, ptr
= J
->base
[0];
745 ptrdiff_t ofs
= sizeof(GCcdata
);
746 GCcdata
*cd
= argv2cdata(J
, ptr
, &rd
->argv
[0]);
747 CTState
*cts
= ctype_ctsG(J2G(J
));
748 CType
*ct
= ctype_raw(cts
, cd
->ctypeid
);
751 /* Resolve pointer or reference for cdata object. */
752 if (ctype_isptr(ct
->info
)) {
753 IRType t
= (LJ_64
&& ct
->size
== 8) ? IRT_P64
: IRT_P32
;
754 if (ctype_isref(ct
->info
)) ct
= ctype_rawchild(cts
, ct
);
755 ptr
= emitir(IRT(IR_FLOAD
, t
), ptr
, IRFL_CDATA_PTR
);
757 ptr
= crec_reassoc_ofs(J
, ptr
, &ofs
, 1);
762 if (tref_isnumber(idx
)) {
763 idx
= lj_opt_narrow_cindex(J
, idx
);
764 if (ctype_ispointer(ct
->info
)) {
767 if ((ct
->info
& CTF_COMPLEX
))
768 idx
= emitir(IRT(IR_BAND
, IRT_INTP
), idx
, lj_ir_kintp(J
, 1));
769 sz
= lj_ctype_size(cts
, (sid
= ctype_cid(ct
->info
)));
770 idx
= crec_reassoc_ofs(J
, idx
, &ofs
, sz
);
771 #if LJ_TARGET_ARM || LJ_TARGET_PPC
772 /* Hoist base add to allow fusion of index/shift into operands. */
773 if (LJ_LIKELY(J
->flags
& JIT_F_OPT_LOOP
) && ofs
775 && (sz
== 1 || sz
== 4)
778 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), ptr
, lj_ir_kintp(J
, ofs
));
782 idx
= emitir(IRT(IR_MUL
, IRT_INTP
), idx
, lj_ir_kintp(J
, sz
));
783 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), idx
, ptr
);
785 } else if (tref_iscdata(idx
)) {
786 GCcdata
*cdk
= cdataV(&rd
->argv
[1]);
787 CType
*ctk
= ctype_raw(cts
, cdk
->ctypeid
);
788 IRType t
= crec_ct2irt(cts
, ctk
);
789 if (ctype_ispointer(ct
->info
) && t
>= IRT_I8
&& t
<= IRT_U64
) {
790 if (ctk
->size
== 8) {
791 idx
= emitir(IRT(IR_FLOAD
, t
), idx
, IRFL_CDATA_INT64
);
792 } else if (ctk
->size
== 4) {
793 idx
= emitir(IRT(IR_FLOAD
, t
), idx
, IRFL_CDATA_INT
);
795 idx
= emitir(IRT(IR_ADD
, IRT_PTR
), idx
,
796 lj_ir_kintp(J
, sizeof(GCcdata
)));
797 idx
= emitir(IRT(IR_XLOAD
, t
), idx
, 0);
799 if (LJ_64
&& ctk
->size
< sizeof(intptr_t) && !(ctk
->info
& CTF_UNSIGNED
))
800 idx
= emitconv(idx
, IRT_INTP
, IRT_INT
, IRCONV_SEXT
);
801 if (!LJ_64
&& ctk
->size
> sizeof(intptr_t)) {
802 idx
= emitconv(idx
, IRT_INTP
, t
, 0);
807 } else if (tref_isstr(idx
)) {
808 GCstr
*name
= strV(&rd
->argv
[1]);
809 if (cd
&& cd
->ctypeid
== CTID_CTYPEID
)
810 ct
= ctype_raw(cts
, crec_constructor(J
, cd
, ptr
));
811 if (ctype_isstruct(ct
->info
)) {
814 fct
= lj_ctype_getfield(cts
, ct
, name
, &fofs
);
816 /* Always specialize to the field name. */
817 emitir(IRTG(IR_EQ
, IRT_STR
), idx
, lj_ir_kstr(J
, name
));
818 if (ctype_isconstval(fct
->info
)) {
819 if (fct
->size
>= 0x80000000u
&&
820 (ctype_child(cts
, fct
)->info
& CTF_UNSIGNED
)) {
821 J
->base
[0] = lj_ir_knum(J
, (lua_Number
)(uint32_t)fct
->size
);
824 J
->base
[0] = lj_ir_kint(J
, (int32_t)fct
->size
);
825 return; /* Interpreter will throw for newindex. */
826 } else if (ctype_isbitfield(fct
->info
)) {
827 lj_trace_err(J
, LJ_TRERR_NYICONV
);
829 lua_assert(ctype_isfield(fct
->info
));
830 sid
= ctype_cid(fct
->info
);
832 ofs
+= (ptrdiff_t)fofs
;
834 } else if (ctype_iscomplex(ct
->info
)) {
835 if (name
->len
== 2 &&
836 ((strdata(name
)[0] == 'r' && strdata(name
)[1] == 'e') ||
837 (strdata(name
)[0] == 'i' && strdata(name
)[1] == 'm'))) {
838 /* Always specialize to the field name. */
839 emitir(IRTG(IR_EQ
, IRT_STR
), idx
, lj_ir_kstr(J
, name
));
840 if (strdata(name
)[0] == 'i') ofs
+= (ct
->size
>> 1);
841 sid
= ctype_cid(ct
->info
);
846 if (ctype_isptr(ct
->info
)) { /* Automatically perform '->'. */
847 CType
*cct
= ctype_rawchild(cts
, ct
);
848 if (ctype_isstruct(cct
->info
)) {
851 if (tref_isstr(idx
)) goto again
;
854 crec_index_meta(J
, cts
, ct
, rd
);
859 ptr
= emitir(IRT(IR_ADD
, IRT_PTR
), ptr
, lj_ir_kintp(J
, ofs
));
861 /* Resolve reference for field. */
862 ct
= ctype_get(cts
, sid
);
863 if (ctype_isref(ct
->info
)) {
864 ptr
= emitir(IRT(IR_XLOAD
, IRT_PTR
), ptr
, 0);
865 sid
= ctype_cid(ct
->info
);
866 ct
= ctype_get(cts
, sid
);
869 while (ctype_isattrib(ct
->info
))
870 ct
= ctype_child(cts
, ct
); /* Skip attributes. */
872 if (rd
->data
== 0) { /* __index metamethod. */
873 J
->base
[0] = crec_tv_ct(J
, ct
, sid
, ptr
);
874 } else { /* __newindex metamethod. */
877 crec_ct_tv(J
, ct
, ptr
, J
->base
[2], &rd
->argv
[2]);
881 /* Record setting a finalizer. */
882 static void crec_finalizer(jit_State
*J
, TRef trcd
, TRef trfin
, cTValue
*fin
)
885 if (!trfin
) trfin
= lj_ir_kptr(J
, gcval(fin
));
886 } else if (tvisnil(fin
)) {
887 trfin
= lj_ir_kptr(J
, NULL
);
889 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
891 lj_ir_call(J
, IRCALL_lj_cdata_setfin
, trcd
,
892 trfin
, lj_ir_kint(J
, (int32_t)itype(fin
)));
896 /* Record cdata allocation. */
897 static void crec_alloc(jit_State
*J
, RecordFFData
*rd
, CTypeID id
)
899 CTState
*cts
= ctype_ctsG(J2G(J
));
901 CTInfo info
= lj_ctype_info(cts
, id
, &sz
);
902 CType
*d
= ctype_raw(cts
, id
);
903 TRef trcd
, trid
= lj_ir_kint(J
, id
);
905 /* Use special instruction to box pointer or 32/64 bit integer. */
906 if (ctype_isptr(info
) || (ctype_isinteger(info
) && (sz
== 4 || sz
== 8))) {
907 TRef sp
= J
->base
[1] ? crec_ct_tv(J
, d
, 0, J
->base
[1], &rd
->argv
[1]) :
908 ctype_isptr(info
) ? lj_ir_kptr(J
, NULL
) :
909 sz
== 4 ? lj_ir_kint(J
, 0) :
910 (lj_needsplit(J
), lj_ir_kint64(J
, 0));
911 J
->base
[0] = emitir(IRTG(IR_CNEWI
, IRT_CDATA
), trid
, sp
);
914 TRef trsz
= TREF_NIL
;
915 if ((info
& CTF_VLA
)) { /* Calculate VLA/VLS size at runtime. */
917 if (!J
->base
[1] || J
->base
[2])
918 lj_trace_err(J
, LJ_TRERR_NYICONV
); /* NYI: init VLA/VLS. */
919 trsz
= crec_ct_tv(J
, ctype_get(cts
, CTID_INT32
), 0,
920 J
->base
[1], &rd
->argv
[1]);
921 sz0
= lj_ctype_vlsize(cts
, d
, 0);
922 sz1
= lj_ctype_vlsize(cts
, d
, 1);
923 trsz
= emitir(IRTGI(IR_MULOV
), trsz
, lj_ir_kint(J
, (int32_t)(sz1
-sz0
)));
924 trsz
= emitir(IRTGI(IR_ADDOV
), trsz
, lj_ir_kint(J
, (int32_t)sz0
));
925 J
->base
[1] = 0; /* Simplify logic below. */
926 } else if (ctype_align(info
) > CT_MEMALIGN
) {
927 trsz
= lj_ir_kint(J
, sz
);
929 trcd
= emitir(IRTG(IR_CNEW
, IRT_CDATA
), trid
, trsz
);
930 if (sz
> 128 || (info
& CTF_VLA
)) {
933 special
: /* Only handle bulk zero-fill for large/VLA/VLS types. */
935 lj_trace_err(J
, LJ_TRERR_NYICONV
); /* NYI: init large/VLA/VLS types. */
936 dp
= emitir(IRT(IR_ADD
, IRT_PTR
), trcd
, lj_ir_kintp(J
, sizeof(GCcdata
)));
937 if (trsz
== TREF_NIL
) trsz
= lj_ir_kint(J
, sz
);
938 align
= ctype_align(info
);
939 if (align
< CT_MEMALIGN
) align
= CT_MEMALIGN
;
940 crec_fill(J
, dp
, trsz
, lj_ir_kint(J
, 0), (1u << align
));
941 } else if (J
->base
[1] && !J
->base
[2] &&
942 !lj_cconv_multi_init(cts
, d
, &rd
->argv
[1])) {
944 } else if (ctype_isarray(d
->info
)) {
945 CType
*dc
= ctype_rawchild(cts
, d
); /* Array element type. */
946 CTSize ofs
, esize
= dc
->size
;
952 if (!(ctype_isnum(dc
->info
) || ctype_isptr(dc
->info
)) ||
953 esize
* CREC_FILL_MAXUNROLL
< sz
)
955 for (i
= 1, ofs
= 0; ofs
< sz
; ofs
+= esize
) {
956 TRef dp
= emitir(IRT(IR_ADD
, IRT_PTR
), trcd
,
957 lj_ir_kintp(J
, ofs
+ sizeof(GCcdata
)));
963 sp
= ctype_isnum(dc
->info
) ? lj_ir_kint(J
, 0) : TREF_NIL
;
965 crec_ct_tv(J
, dc
, dp
, sp
, sval
);
967 } else if (ctype_isstruct(d
->info
)) {
968 CTypeID fid
= d
->sib
;
971 CType
*df
= ctype_get(cts
, fid
);
973 if (ctype_isfield(df
->info
)) {
979 if (!gcref(df
->name
)) continue; /* Ignore unnamed fields. */
980 dc
= ctype_rawchild(cts
, df
); /* Field type. */
981 if (!(ctype_isnum(dc
->info
) || ctype_isptr(dc
->info
) ||
982 ctype_isenum(dc
->info
)))
983 lj_trace_err(J
, LJ_TRERR_NYICONV
); /* NYI: init aggregates. */
989 sp
= ctype_isptr(dc
->info
) ? TREF_NIL
: lj_ir_kint(J
, 0);
991 dp
= emitir(IRT(IR_ADD
, IRT_PTR
), trcd
,
992 lj_ir_kintp(J
, df
->size
+ sizeof(GCcdata
)));
993 crec_ct_tv(J
, dc
, dp
, sp
, sval
);
994 } else if (!ctype_isconstval(df
->info
)) {
995 /* NYI: init bitfields and sub-structures. */
996 lj_trace_err(J
, LJ_TRERR_NYICONV
);
1002 dp
= emitir(IRT(IR_ADD
, IRT_PTR
), trcd
, lj_ir_kintp(J
, sizeof(GCcdata
)));
1004 crec_ct_tv(J
, d
, dp
, J
->base
[1], &rd
->argv
[1]);
1008 crec_ct_tv(J
, d
, dp
, lj_ir_kint(J
, 0), &tv
);
1013 /* Handle __gc metamethod. */
1014 fin
= lj_ctype_meta(cts
, id
, MM_gc
);
1016 crec_finalizer(J
, trcd
, 0, fin
);
1019 /* Record argument conversions. */
1020 static TRef
crec_call_args(jit_State
*J
, RecordFFData
*rd
,
1021 CTState
*cts
, CType
*ct
)
1023 TRef args
[CCI_NARGS_MAX
];
1030 TRef
*arg0
= NULL
, *arg1
= NULL
;
1033 if (ctype_cconv(ct
->info
) == CTCC_THISCALL
)
1035 else if (ctype_cconv(ct
->info
) == CTCC_FASTCALL
)
1039 /* Skip initial attributes. */
1042 CType
*ctf
= ctype_get(cts
, fid
);
1043 if (!ctype_isattrib(ctf
->info
)) break;
1047 for (n
= 0, base
= J
->base
+1, o
= rd
->argv
+1; *base
; n
++, base
++, o
++) {
1051 if (n
>= CCI_NARGS_MAX
)
1052 lj_trace_err(J
, LJ_TRERR_NYICALL
);
1054 if (fid
) { /* Get argument type from field. */
1055 CType
*ctf
= ctype_get(cts
, fid
);
1057 lua_assert(ctype_isfield(ctf
->info
));
1058 did
= ctype_cid(ctf
->info
);
1060 if (!(ct
->info
& CTF_VARARG
))
1061 lj_trace_err(J
, LJ_TRERR_NYICALL
); /* Too many arguments. */
1062 did
= lj_ccall_ctid_vararg(cts
, o
); /* Infer vararg type. */
1064 d
= ctype_raw(cts
, did
);
1065 if (!(ctype_isnum(d
->info
) || ctype_isptr(d
->info
) ||
1066 ctype_isenum(d
->info
)))
1067 lj_trace_err(J
, LJ_TRERR_NYICALL
);
1068 tr
= crec_ct_tv(J
, d
, 0, *base
, o
);
1069 if (ctype_isinteger_or_bool(d
->info
)) {
1071 if ((d
->info
& CTF_UNSIGNED
))
1072 tr
= emitconv(tr
, IRT_INT
, d
->size
==1 ? IRT_U8
: IRT_U16
, 0);
1074 tr
= emitconv(tr
, IRT_INT
, d
->size
==1 ? IRT_I8
: IRT_I16
,IRCONV_SEXT
);
1076 } else if (LJ_SOFTFP
&& ctype_isfp(d
->info
) && d
->size
> 4) {
1080 /* 64 bit args must not end up in registers for fastcall/thiscall. */
1082 if (!ctype_isfp(d
->info
)) {
1083 /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1084 if (tref_typerange(tr
, IRT_I64
, IRT_U64
)) {
1086 arg0
= &args
[n
]; args
[n
++] = TREF_NIL
; ngpr
--;
1088 arg1
= &args
[n
]; args
[n
++] = TREF_NIL
; ngpr
--;
1092 if (arg0
) { *arg0
= tr
; arg0
= NULL
; n
--; continue; }
1093 if (arg1
) { *arg1
= tr
; arg1
= NULL
; n
--; continue; }
1098 if (!ctype_isfp(d
->info
) && ngpr
) {
1099 if (tref_typerange(tr
, IRT_I64
, IRT_U64
)) {
1100 /* No reordering for other x86 ABIs. Simply add alignment args. */
1101 do { args
[n
++] = TREF_NIL
; } while (--ngpr
);
1111 for (i
= 1; i
< n
; i
++)
1112 tr
= emitir(IRT(IR_CARG
, IRT_NIL
), tr
, args
[i
]);
1116 /* Create a snapshot for the caller, simulating a 'false' return value. */
1117 static void crec_snap_caller(jit_State
*J
)
1119 lua_State
*L
= J
->L
;
1120 TValue
*base
= L
->base
, *top
= L
->top
;
1121 const BCIns
*pc
= J
->pc
;
1122 TRef ftr
= J
->base
[-1];
1124 if (!frame_islua(base
-1) || J
->framedepth
<= 0)
1125 lj_trace_err(J
, LJ_TRERR_NYICALL
);
1126 J
->pc
= frame_pc(base
-1); delta
= 1+LJ_FR2
+bc_a(J
->pc
[-1]);
1127 L
->top
= base
; L
->base
= base
- delta
;
1128 J
->base
[-1] = TREF_FALSE
;
1129 J
->base
-= delta
; J
->baseslot
-= (BCReg
)delta
;
1130 J
->maxslot
= (BCReg
)delta
; J
->framedepth
--;
1132 L
->base
= base
; L
->top
= top
;
1133 J
->framedepth
++; J
->maxslot
= 1;
1134 J
->base
+= delta
; J
->baseslot
+= (BCReg
)delta
;
1135 J
->base
[-1] = ftr
; J
->pc
= pc
;
1138 /* Record function call. */
1139 static int crec_call(jit_State
*J
, RecordFFData
*rd
, GCcdata
*cd
)
1141 CTState
*cts
= ctype_ctsG(J2G(J
));
1142 CType
*ct
= ctype_raw(cts
, cd
->ctypeid
);
1143 IRType tp
= IRT_PTR
;
1144 if (ctype_isptr(ct
->info
)) {
1145 tp
= (LJ_64
&& ct
->size
== 8) ? IRT_P64
: IRT_P32
;
1146 ct
= ctype_rawchild(cts
, ct
);
1148 if (ctype_isfunc(ct
->info
)) {
1149 TRef func
= emitir(IRT(IR_FLOAD
, tp
), J
->base
[0], IRFL_CDATA_PTR
);
1150 CType
*ctr
= ctype_rawchild(cts
, ct
);
1151 IRType t
= crec_ct2irt(cts
, ctr
);
1154 /* Check for blacklisted C functions that might call a callback. */
1156 cdata_getptr(cdataptr(cd
), (LJ_64
&& tp
== IRT_P64
) ? 8 : 4));
1157 if (tvistrue(lj_tab_get(J
->L
, cts
->miscmap
, &tv
)))
1158 lj_trace_err(J
, LJ_TRERR_BLACKL
);
1159 if (ctype_isvoid(ctr
->info
)) {
1162 } else if (!(ctype_isnum(ctr
->info
) || ctype_isptr(ctr
->info
) ||
1163 ctype_isenum(ctr
->info
)) || t
== IRT_CDATA
) {
1164 lj_trace_err(J
, LJ_TRERR_NYICALL
);
1166 if ((ct
->info
& CTF_VARARG
)
1168 || ctype_cconv(ct
->info
) != CTCC_CDECL
1171 func
= emitir(IRT(IR_CARG
, IRT_NIL
), func
,
1172 lj_ir_kint(J
, ctype_typeid(cts
, ct
)));
1173 tr
= emitir(IRT(IR_CALLXS
, t
), crec_call_args(J
, rd
, cts
, ct
), func
);
1174 if (ctype_isbool(ctr
->info
)) {
1175 if (frame_islua(J
->L
->base
-1) && bc_b(frame_pc(J
->L
->base
-1)[-1]) == 1) {
1176 /* Don't check result if ignored. */
1179 crec_snap_caller(J
);
1180 #if LJ_TARGET_X86ORX64
1181 /* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1182 lj_ir_set(J
, IRTG(IR_NE
, IRT_U8
), tr
, lj_ir_kint(J
, 0));
1184 lj_ir_set(J
, IRTGI(IR_NE
), tr
, lj_ir_kint(J
, 0));
1186 J
->postproc
= LJ_POST_FIXGUARDSNAP
;
1189 } else if (t
== IRT_PTR
|| (LJ_64
&& t
== IRT_P32
) ||
1190 t
== IRT_I64
|| t
== IRT_U64
|| ctype_isenum(ctr
->info
)) {
1191 TRef trid
= lj_ir_kint(J
, ctype_cid(ct
->info
));
1192 tr
= emitir(IRTG(IR_CNEWI
, IRT_CDATA
), trid
, tr
);
1193 if (t
== IRT_I64
|| t
== IRT_U64
) lj_needsplit(J
);
1194 } else if (t
== IRT_FLOAT
|| t
== IRT_U32
) {
1195 tr
= emitconv(tr
, IRT_NUM
, t
, 0);
1196 } else if (t
== IRT_I8
|| t
== IRT_I16
) {
1197 tr
= emitconv(tr
, IRT_INT
, t
, IRCONV_SEXT
);
1198 } else if (t
== IRT_U8
|| t
== IRT_U16
) {
1199 tr
= emitconv(tr
, IRT_INT
, t
, 0);
1208 void LJ_FASTCALL
recff_cdata_call(jit_State
*J
, RecordFFData
*rd
)
1210 CTState
*cts
= ctype_ctsG(J2G(J
));
1211 GCcdata
*cd
= argv2cdata(J
, J
->base
[0], &rd
->argv
[0]);
1212 CTypeID id
= cd
->ctypeid
;
1216 if (id
== CTID_CTYPEID
) {
1217 id
= crec_constructor(J
, cd
, J
->base
[0]);
1219 } else if (crec_call(J
, rd
, cd
)) {
1222 /* Record ctype __call/__new metamethod. */
1223 ct
= ctype_raw(cts
, id
);
1224 tv
= lj_ctype_meta(cts
, ctype_isptr(ct
->info
) ? ctype_cid(ct
->info
) : id
, mm
);
1227 J
->base
[-1] = lj_ir_kfunc(J
, funcV(tv
)) | TREF_FRAME
;
1228 rd
->nres
= -1; /* Pending tailcall. */
1231 } else if (mm
== MM_new
) {
1232 crec_alloc(J
, rd
, id
);
1235 /* No metamethod or NYI: non-function metamethods. */
1236 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1239 static TRef
crec_arith_int64(jit_State
*J
, TRef
*sp
, CType
**s
, MMS mm
)
1241 if (ctype_isnum(s
[0]->info
) && ctype_isnum(s
[1]->info
)) {
1248 if (((s
[0]->info
& CTF_UNSIGNED
) && s
[0]->size
== 8) ||
1249 ((s
[1]->info
& CTF_UNSIGNED
) && s
[1]->size
== 8)) {
1250 dt
= IRT_U64
; id
= CTID_UINT64
;
1252 dt
= IRT_I64
; id
= CTID_INT64
;
1254 !((s
[0]->info
| s
[1]->info
) & CTF_FP
) &&
1255 s
[0]->size
== 4 && s
[1]->size
== 4) { /* Try to narrow comparison. */
1256 if (!((s
[0]->info
^ s
[1]->info
) & CTF_UNSIGNED
) ||
1257 (tref_isk(sp
[1]) && IR(tref_ref(sp
[1]))->i
>= 0)) {
1258 dt
= (s
[0]->info
& CTF_UNSIGNED
) ? IRT_U32
: IRT_INT
;
1260 } else if (tref_isk(sp
[0]) && IR(tref_ref(sp
[0]))->i
>= 0) {
1261 dt
= (s
[1]->info
& CTF_UNSIGNED
) ? IRT_U32
: IRT_INT
;
1266 for (i
= 0; i
< 2; i
++) {
1267 IRType st
= tref_type(sp
[i
]);
1268 if (st
== IRT_NUM
|| st
== IRT_FLOAT
)
1269 sp
[i
] = emitconv(sp
[i
], dt
, st
, IRCONV_ANY
);
1270 else if (!(st
== IRT_I64
|| st
== IRT_U64
))
1271 sp
[i
] = emitconv(sp
[i
], dt
, IRT_INT
,
1272 (s
[i
]->info
& CTF_UNSIGNED
) ? 0 : IRCONV_SEXT
);
1276 /* Assume true comparison. Fixup and emit pending guard later. */
1280 op
= mm
== MM_lt
? IR_LT
: IR_LE
;
1281 if (dt
== IRT_U32
|| dt
== IRT_U64
)
1282 op
+= (IR_ULT
-IR_LT
);
1284 lj_ir_set(J
, IRTG(op
, dt
), sp
[0], sp
[1]);
1285 J
->postproc
= LJ_POST_FIXGUARD
;
1288 tr
= emitir(IRT(mm
+(int)IR_ADD
-(int)MM_add
, dt
), sp
[0], sp
[1]);
1290 return emitir(IRTG(IR_CNEWI
, IRT_CDATA
), lj_ir_kint(J
, id
), tr
);
1295 static TRef
crec_arith_ptr(jit_State
*J
, TRef
*sp
, CType
**s
, MMS mm
)
1297 CTState
*cts
= ctype_ctsG(J2G(J
));
1299 if (ctype_isptr(ctp
->info
) || ctype_isrefarray(ctp
->info
)) {
1300 if ((mm
== MM_sub
|| mm
== MM_eq
|| mm
== MM_lt
|| mm
== MM_le
) &&
1301 (ctype_isptr(s
[1]->info
) || ctype_isrefarray(s
[1]->info
))) {
1302 if (mm
== MM_sub
) { /* Pointer difference. */
1304 CTSize sz
= lj_ctype_size(cts
, ctype_cid(ctp
->info
));
1305 if (sz
== 0 || (sz
& (sz
-1)) != 0)
1306 return 0; /* NYI: integer division. */
1307 tr
= emitir(IRT(IR_SUB
, IRT_INTP
), sp
[0], sp
[1]);
1308 tr
= emitir(IRT(IR_BSAR
, IRT_INTP
), tr
, lj_ir_kint(J
, lj_fls(sz
)));
1310 tr
= emitconv(tr
, IRT_NUM
, IRT_INTP
, 0);
1313 } else { /* Pointer comparison (unsigned). */
1314 /* Assume true comparison. Fixup and emit pending guard later. */
1315 IROp op
= mm
== MM_eq
? IR_EQ
: mm
== MM_lt
? IR_ULT
: IR_ULE
;
1316 lj_ir_set(J
, IRTG(op
, IRT_PTR
), sp
[0], sp
[1]);
1317 J
->postproc
= LJ_POST_FIXGUARD
;
1321 if (!((mm
== MM_add
|| mm
== MM_sub
) && ctype_isnum(s
[1]->info
)))
1323 } else if (mm
== MM_add
&& ctype_isnum(ctp
->info
) &&
1324 (ctype_isptr(s
[1]->info
) || ctype_isrefarray(s
[1]->info
))) {
1325 TRef tr
= sp
[0]; sp
[0] = sp
[1]; sp
[1] = tr
; /* Swap pointer and index. */
1332 IRType t
= tref_type(tr
);
1333 CTSize sz
= lj_ctype_size(cts
, ctype_cid(ctp
->info
));
1336 if (t
== IRT_NUM
|| t
== IRT_FLOAT
)
1337 tr
= emitconv(tr
, IRT_INTP
, t
, IRCONV_ANY
);
1338 else if (!(t
== IRT_I64
|| t
== IRT_U64
))
1339 tr
= emitconv(tr
, IRT_INTP
, IRT_INT
,
1340 ((t
- IRT_I8
) & 1) ? 0 : IRCONV_SEXT
);
1342 if (!tref_typerange(sp
[1], IRT_I8
, IRT_U32
)) {
1343 tr
= emitconv(tr
, IRT_INTP
, t
,
1344 (t
== IRT_NUM
|| t
== IRT_FLOAT
) ? IRCONV_ANY
: 0);
1347 tr
= emitir(IRT(IR_MUL
, IRT_INTP
), tr
, lj_ir_kintp(J
, sz
));
1348 tr
= emitir(IRT(mm
+(int)IR_ADD
-(int)MM_add
, IRT_PTR
), sp
[0], tr
);
1349 id
= lj_ctype_intern(cts
, CTINFO(CT_PTR
, CTALIGN_PTR
|ctype_cid(ctp
->info
)),
1351 return emitir(IRTG(IR_CNEWI
, IRT_CDATA
), lj_ir_kint(J
, id
), tr
);
1355 /* Record ctype arithmetic metamethods. */
1356 static TRef
crec_arith_meta(jit_State
*J
, TRef
*sp
, CType
**s
, CTState
*cts
,
1361 if (tviscdata(&rd
->argv
[0])) {
1362 CTypeID id
= argv2cdata(J
, J
->base
[0], &rd
->argv
[0])->ctypeid
;
1363 CType
*ct
= ctype_raw(cts
, id
);
1364 if (ctype_isptr(ct
->info
)) id
= ctype_cid(ct
->info
);
1365 tv
= lj_ctype_meta(cts
, id
, (MMS
)rd
->data
);
1367 if (!tv
&& J
->base
[1] && tviscdata(&rd
->argv
[1])) {
1368 CTypeID id
= argv2cdata(J
, J
->base
[1], &rd
->argv
[1])->ctypeid
;
1369 CType
*ct
= ctype_raw(cts
, id
);
1370 if (ctype_isptr(ct
->info
)) id
= ctype_cid(ct
->info
);
1371 tv
= lj_ctype_meta(cts
, id
, (MMS
)rd
->data
);
1376 J
->base
[-1] = lj_ir_kfunc(J
, funcV(tv
)) | TREF_FRAME
;
1377 rd
->nres
= -1; /* Pending tailcall. */
1379 } /* NYI: non-function metamethods. */
1380 } else if ((MMS
)rd
->data
== MM_eq
) { /* Fallback cdata pointer comparison. */
1381 if (sp
[0] && sp
[1] && ctype_isnum(s
[0]->info
) == ctype_isnum(s
[1]->info
)) {
1382 /* Assume true comparison. Fixup and emit pending guard later. */
1383 lj_ir_set(J
, IRTG(IR_EQ
, IRT_PTR
), sp
[0], sp
[1]);
1384 J
->postproc
= LJ_POST_FIXGUARD
;
1390 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1394 void LJ_FASTCALL
recff_cdata_arith(jit_State
*J
, RecordFFData
*rd
)
1396 CTState
*cts
= ctype_ctsG(J2G(J
));
1400 for (i
= 0; i
< 2; i
++) {
1401 TRef tr
= J
->base
[i
];
1402 CType
*ct
= ctype_get(cts
, CTID_DOUBLE
);
1404 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1405 } else if (tref_iscdata(tr
)) {
1406 CTypeID id
= argv2cdata(J
, tr
, &rd
->argv
[i
])->ctypeid
;
1408 ct
= ctype_raw(cts
, id
);
1409 t
= crec_ct2irt(cts
, ct
);
1410 if (ctype_isptr(ct
->info
)) { /* Resolve pointer or reference. */
1411 tr
= emitir(IRT(IR_FLOAD
, t
), tr
, IRFL_CDATA_PTR
);
1412 if (ctype_isref(ct
->info
)) {
1413 ct
= ctype_rawchild(cts
, ct
);
1414 t
= crec_ct2irt(cts
, ct
);
1416 } else if (t
== IRT_I64
|| t
== IRT_U64
) {
1417 tr
= emitir(IRT(IR_FLOAD
, t
), tr
, IRFL_CDATA_INT64
);
1420 } else if (t
== IRT_INT
|| t
== IRT_U32
) {
1421 tr
= emitir(IRT(IR_FLOAD
, t
), tr
, IRFL_CDATA_INT
);
1422 if (ctype_isenum(ct
->info
)) ct
= ctype_child(cts
, ct
);
1424 } else if (ctype_isfunc(ct
->info
)) {
1425 tr
= emitir(IRT(IR_FLOAD
, IRT_PTR
), tr
, IRFL_CDATA_PTR
);
1427 lj_ctype_intern(cts
, CTINFO(CT_PTR
, CTALIGN_PTR
|id
), CTSIZE_PTR
));
1430 tr
= emitir(IRT(IR_ADD
, IRT_PTR
), tr
, lj_ir_kintp(J
, sizeof(GCcdata
)));
1432 if (ctype_isenum(ct
->info
)) ct
= ctype_child(cts
, ct
);
1433 if (ctype_isnum(ct
->info
)) {
1434 if (t
== IRT_CDATA
) {
1437 if (t
== IRT_I64
|| t
== IRT_U64
) lj_needsplit(J
);
1438 tr
= emitir(IRT(IR_XLOAD
, t
), tr
, 0);
1441 } else if (tref_isnil(tr
)) {
1442 tr
= lj_ir_kptr(J
, NULL
);
1443 ct
= ctype_get(cts
, CTID_P_VOID
);
1444 } else if (tref_isinteger(tr
)) {
1445 ct
= ctype_get(cts
, CTID_INT32
);
1446 } else if (tref_isstr(tr
)) {
1447 TRef tr2
= J
->base
[1-i
];
1448 CTypeID id
= argv2cdata(J
, tr2
, &rd
->argv
[1-i
])->ctypeid
;
1449 ct
= ctype_raw(cts
, id
);
1450 if (ctype_isenum(ct
->info
)) { /* Match string against enum constant. */
1451 GCstr
*str
= strV(&rd
->argv
[i
]);
1453 CType
*cct
= lj_ctype_getfield(cts
, ct
, str
, &ofs
);
1454 if (cct
&& ctype_isconstval(cct
->info
)) {
1455 /* Specialize to the name of the enum constant. */
1456 emitir(IRTG(IR_EQ
, IRT_STR
), tr
, lj_ir_kstr(J
, str
));
1457 ct
= ctype_child(cts
, cct
);
1458 tr
= lj_ir_kint(J
, (int32_t)ofs
);
1459 } else { /* Interpreter will throw or return false. */
1460 ct
= ctype_get(cts
, CTID_P_VOID
);
1462 } else if (ctype_isptr(ct
->info
)) {
1463 tr
= emitir(IRT(IR_ADD
, IRT_PTR
), tr
, lj_ir_kintp(J
, sizeof(GCstr
)));
1465 ct
= ctype_get(cts
, CTID_P_VOID
);
1467 } else if (!tref_isnum(tr
)) {
1469 ct
= ctype_get(cts
, CTID_P_VOID
);
1477 if (!(tr
= crec_arith_int64(J
, sp
, s
, (MMS
)rd
->data
)) &&
1478 !(tr
= crec_arith_ptr(J
, sp
, s
, (MMS
)rd
->data
)) &&
1479 !(tr
= crec_arith_meta(J
, sp
, s
, cts
, rd
)))
1482 /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1483 if (J
->postproc
== LJ_POST_FIXGUARD
&& frame_iscont(J
->L
->base
-1) &&
1484 !irt_isguard(J
->guardemit
)) {
1485 const BCIns
*pc
= frame_contpc(J
->L
->base
-1) - 1;
1486 if (bc_op(*pc
) <= BC_ISNEP
) {
1487 J2G(J
)->tmptv
.u64
= (uint64_t)(uintptr_t)pc
;
1488 J
->postproc
= LJ_POST_FIXCOMP
;
1494 /* -- C library namespace metamethods ------------------------------------- */
1496 void LJ_FASTCALL
recff_clib_index(jit_State
*J
, RecordFFData
*rd
)
1498 CTState
*cts
= ctype_ctsG(J2G(J
));
1499 if (tref_isudata(J
->base
[0]) && tref_isstr(J
->base
[1]) &&
1500 udataV(&rd
->argv
[0])->udtype
== UDTYPE_FFI_CLIB
) {
1501 CLibrary
*cl
= (CLibrary
*)uddata(udataV(&rd
->argv
[0]));
1502 GCstr
*name
= strV(&rd
->argv
[1]);
1504 CTypeID id
= lj_ctype_getname(cts
, &ct
, name
, CLNS_INDEX
);
1505 cTValue
*tv
= lj_tab_getstr(cl
->cache
, name
);
1506 rd
->nres
= rd
->data
;
1507 if (id
&& tv
&& !tvisnil(tv
)) {
1508 /* Specialize to the symbol name and make the result a constant. */
1509 emitir(IRTG(IR_EQ
, IRT_STR
), J
->base
[1], lj_ir_kstr(J
, name
));
1510 if (ctype_isconstval(ct
->info
)) {
1511 if (ct
->size
>= 0x80000000u
&&
1512 (ctype_child(cts
, ct
)->info
& CTF_UNSIGNED
))
1513 J
->base
[0] = lj_ir_knum(J
, (lua_Number
)(uint32_t)ct
->size
);
1515 J
->base
[0] = lj_ir_kint(J
, (int32_t)ct
->size
);
1516 } else if (ctype_isextern(ct
->info
)) {
1517 CTypeID sid
= ctype_cid(ct
->info
);
1518 void *sp
= *(void **)cdataptr(cdataV(tv
));
1520 ct
= ctype_raw(cts
, sid
);
1521 if (LJ_64
&& !checkptr32(sp
))
1522 ptr
= lj_ir_kintp(J
, (uintptr_t)sp
);
1524 ptr
= lj_ir_kptr(J
, sp
);
1526 J
->base
[0] = crec_tv_ct(J
, ct
, sid
, ptr
);
1529 crec_ct_tv(J
, ct
, ptr
, J
->base
[2], &rd
->argv
[2]);
1532 J
->base
[0] = lj_ir_kgc(J
, obj2gco(cdataV(tv
)), IRT_CDATA
);
1535 lj_trace_err(J
, LJ_TRERR_NOCACHE
);
1537 } /* else: interpreter will throw. */
1540 /* -- FFI library functions ----------------------------------------------- */
1542 static TRef
crec_toint(jit_State
*J
, CTState
*cts
, TRef sp
, TValue
*sval
)
1544 return crec_ct_tv(J
, ctype_get(cts
, CTID_INT32
), 0, sp
, sval
);
1547 void LJ_FASTCALL
recff_ffi_new(jit_State
*J
, RecordFFData
*rd
)
1549 crec_alloc(J
, rd
, argv2ctype(J
, J
->base
[0], &rd
->argv
[0]));
1552 void LJ_FASTCALL
recff_ffi_errno(jit_State
*J
, RecordFFData
*rd
)
1556 lj_trace_err(J
, LJ_TRERR_NYICALL
);
1557 J
->base
[0] = lj_ir_call(J
, IRCALL_lj_vm_errno
);
1560 void LJ_FASTCALL
recff_ffi_string(jit_State
*J
, RecordFFData
*rd
)
1562 CTState
*cts
= ctype_ctsG(J2G(J
));
1563 TRef tr
= J
->base
[0];
1565 TRef trlen
= J
->base
[1];
1566 if (!tref_isnil(trlen
)) {
1567 trlen
= crec_toint(J
, cts
, trlen
, &rd
->argv
[1]);
1568 tr
= crec_ct_tv(J
, ctype_get(cts
, CTID_P_CVOID
), 0, tr
, &rd
->argv
[0]);
1570 tr
= crec_ct_tv(J
, ctype_get(cts
, CTID_P_CCHAR
), 0, tr
, &rd
->argv
[0]);
1571 trlen
= lj_ir_call(J
, IRCALL_strlen
, tr
);
1573 J
->base
[0] = emitir(IRT(IR_XSNEW
, IRT_STR
), tr
, trlen
);
1574 } /* else: interpreter will throw. */
1577 void LJ_FASTCALL
recff_ffi_copy(jit_State
*J
, RecordFFData
*rd
)
1579 CTState
*cts
= ctype_ctsG(J2G(J
));
1580 TRef trdst
= J
->base
[0], trsrc
= J
->base
[1], trlen
= J
->base
[2];
1581 if (trdst
&& trsrc
&& (trlen
|| tref_isstr(trsrc
))) {
1582 trdst
= crec_ct_tv(J
, ctype_get(cts
, CTID_P_VOID
), 0, trdst
, &rd
->argv
[0]);
1583 trsrc
= crec_ct_tv(J
, ctype_get(cts
, CTID_P_CVOID
), 0, trsrc
, &rd
->argv
[1]);
1585 trlen
= crec_toint(J
, cts
, trlen
, &rd
->argv
[2]);
1587 trlen
= emitir(IRTI(IR_FLOAD
), J
->base
[1], IRFL_STR_LEN
);
1588 trlen
= emitir(IRTI(IR_ADD
), trlen
, lj_ir_kint(J
, 1));
1591 crec_copy(J
, trdst
, trsrc
, trlen
, NULL
);
1592 } /* else: interpreter will throw. */
1595 void LJ_FASTCALL
recff_ffi_fill(jit_State
*J
, RecordFFData
*rd
)
1597 CTState
*cts
= ctype_ctsG(J2G(J
));
1598 TRef trdst
= J
->base
[0], trlen
= J
->base
[1], trfill
= J
->base
[2];
1599 if (trdst
&& trlen
) {
1601 if (tviscdata(&rd
->argv
[0])) { /* Get alignment of original destination. */
1603 CType
*ct
= ctype_raw(cts
, cdataV(&rd
->argv
[0])->ctypeid
);
1604 if (ctype_isptr(ct
->info
))
1605 ct
= ctype_rawchild(cts
, ct
);
1606 step
= (1u<<ctype_align(lj_ctype_info(cts
, ctype_typeid(cts
, ct
), &sz
)));
1608 trdst
= crec_ct_tv(J
, ctype_get(cts
, CTID_P_VOID
), 0, trdst
, &rd
->argv
[0]);
1609 trlen
= crec_toint(J
, cts
, trlen
, &rd
->argv
[1]);
1611 trfill
= crec_toint(J
, cts
, trfill
, &rd
->argv
[2]);
1613 trfill
= lj_ir_kint(J
, 0);
1615 crec_fill(J
, trdst
, trlen
, trfill
, step
);
1616 } /* else: interpreter will throw. */
1619 void LJ_FASTCALL
recff_ffi_typeof(jit_State
*J
, RecordFFData
*rd
)
1621 if (tref_iscdata(J
->base
[0])) {
1622 TRef trid
= lj_ir_kint(J
, argv2ctype(J
, J
->base
[0], &rd
->argv
[0]));
1623 J
->base
[0] = emitir(IRTG(IR_CNEWI
, IRT_CDATA
),
1624 lj_ir_kint(J
, CTID_CTYPEID
), trid
);
1626 setfuncV(J
->L
, &J
->errinfo
, J
->fn
);
1627 lj_trace_err_info(J
, LJ_TRERR_NYIFFU
);
1631 void LJ_FASTCALL
recff_ffi_istype(jit_State
*J
, RecordFFData
*rd
)
1633 argv2ctype(J
, J
->base
[0], &rd
->argv
[0]);
1634 if (tref_iscdata(J
->base
[1])) {
1635 argv2ctype(J
, J
->base
[1], &rd
->argv
[1]);
1636 J
->postproc
= LJ_POST_FIXBOOL
;
1637 J
->base
[0] = TREF_TRUE
;
1639 J
->base
[0] = TREF_FALSE
;
1643 void LJ_FASTCALL
recff_ffi_abi(jit_State
*J
, RecordFFData
*rd
)
1645 if (tref_isstr(J
->base
[0])) {
1646 /* Specialize to the ABI string to make the boolean result a constant. */
1647 emitir(IRTG(IR_EQ
, IRT_STR
), J
->base
[0], lj_ir_kstr(J
, strV(&rd
->argv
[0])));
1648 J
->postproc
= LJ_POST_FIXBOOL
;
1649 J
->base
[0] = TREF_TRUE
;
1651 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1655 /* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
1656 void LJ_FASTCALL
recff_ffi_xof(jit_State
*J
, RecordFFData
*rd
)
1658 CTypeID id
= argv2ctype(J
, J
->base
[0], &rd
->argv
[0]);
1659 if (rd
->data
== FF_ffi_sizeof
) {
1660 CType
*ct
= lj_ctype_rawref(ctype_ctsG(J2G(J
)), id
);
1661 if (ctype_isvltype(ct
->info
))
1662 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1663 } else if (rd
->data
== FF_ffi_offsetof
) { /* Specialize to the field name. */
1664 if (!tref_isstr(J
->base
[1]))
1665 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1666 emitir(IRTG(IR_EQ
, IRT_STR
), J
->base
[1], lj_ir_kstr(J
, strV(&rd
->argv
[1])));
1667 rd
->nres
= 3; /* Just in case. */
1669 J
->postproc
= LJ_POST_FIXCONST
;
1670 J
->base
[0] = J
->base
[1] = J
->base
[2] = TREF_NIL
;
1673 void LJ_FASTCALL
recff_ffi_gc(jit_State
*J
, RecordFFData
*rd
)
1675 argv2cdata(J
, J
->base
[0], &rd
->argv
[0]);
1677 lj_trace_err(J
, LJ_TRERR_BADTYPE
);
1678 crec_finalizer(J
, J
->base
[0], J
->base
[1], &rd
->argv
[1]);
1681 /* -- 64 bit bit.* library functions -------------------------------------- */
1683 /* Determine bit operation type from argument type. */
1684 static CTypeID
crec_bit64_type(CTState
*cts
, cTValue
*tv
)
1686 if (tviscdata(tv
)) {
1687 CType
*ct
= lj_ctype_rawref(cts
, cdataV(tv
)->ctypeid
);
1688 if (ctype_isenum(ct
->info
)) ct
= ctype_child(cts
, ct
);
1689 if ((ct
->info
& (CTMASK_NUM
|CTF_BOOL
|CTF_FP
|CTF_UNSIGNED
)) ==
1690 CTINFO(CT_NUM
, CTF_UNSIGNED
) && ct
->size
== 8)
1691 return CTID_UINT64
; /* Use uint64_t, since it has the highest rank. */
1692 return CTID_INT64
; /* Otherwise use int64_t. */
1694 return 0; /* Use regular 32 bit ops. */
1697 void LJ_FASTCALL
recff_bit64_tobit(jit_State
*J
, RecordFFData
*rd
)
1699 CTState
*cts
= ctype_ctsG(J2G(J
));
1700 TRef tr
= crec_ct_tv(J
, ctype_get(cts
, CTID_INT64
), 0,
1701 J
->base
[0], &rd
->argv
[0]);
1702 if (!tref_isinteger(tr
))
1703 tr
= emitconv(tr
, IRT_INT
, tref_type(tr
), 0);
1707 int LJ_FASTCALL
recff_bit64_unary(jit_State
*J
, RecordFFData
*rd
)
1709 CTState
*cts
= ctype_ctsG(J2G(J
));
1710 CTypeID id
= crec_bit64_type(cts
, &rd
->argv
[0]);
1712 TRef tr
= crec_ct_tv(J
, ctype_get(cts
, id
), 0, J
->base
[0], &rd
->argv
[0]);
1713 tr
= emitir(IRT(rd
->data
, id
-CTID_INT64
+IRT_I64
), tr
, 0);
1714 J
->base
[0] = emitir(IRTG(IR_CNEWI
, IRT_CDATA
), lj_ir_kint(J
, id
), tr
);
1720 int LJ_FASTCALL
recff_bit64_nary(jit_State
*J
, RecordFFData
*rd
)
1722 CTState
*cts
= ctype_ctsG(J2G(J
));
1725 for (i
= 0; J
->base
[i
] != 0; i
++) {
1726 CTypeID aid
= crec_bit64_type(cts
, &rd
->argv
[i
]);
1727 if (id
< aid
) id
= aid
; /* Determine highest type rank of all arguments. */
1730 CType
*ct
= ctype_get(cts
, id
);
1731 uint32_t ot
= IRT(rd
->data
, id
-CTID_INT64
+IRT_I64
);
1732 TRef tr
= crec_ct_tv(J
, ct
, 0, J
->base
[0], &rd
->argv
[0]);
1733 for (i
= 1; J
->base
[i
] != 0; i
++) {
1734 TRef tr2
= crec_ct_tv(J
, ct
, 0, J
->base
[i
], &rd
->argv
[i
]);
1735 tr
= emitir(ot
, tr
, tr2
);
1737 J
->base
[0] = emitir(IRTG(IR_CNEWI
, IRT_CDATA
), lj_ir_kint(J
, id
), tr
);
1743 int LJ_FASTCALL
recff_bit64_shift(jit_State
*J
, RecordFFData
*rd
)
1745 CTState
*cts
= ctype_ctsG(J2G(J
));
1748 if (J
->base
[0] && tref_iscdata(J
->base
[1])) {
1749 tsh
= crec_ct_tv(J
, ctype_get(cts
, CTID_INT64
), 0,
1750 J
->base
[1], &rd
->argv
[1]);
1751 if (!tref_isinteger(tsh
))
1752 tsh
= emitconv(tsh
, IRT_INT
, tref_type(tsh
), 0);
1755 id
= crec_bit64_type(cts
, &rd
->argv
[0]);
1757 TRef tr
= crec_ct_tv(J
, ctype_get(cts
, id
), 0, J
->base
[0], &rd
->argv
[0]);
1758 uint32_t op
= rd
->data
;
1759 if (!tsh
) tsh
= lj_opt_narrow_tobit(J
, J
->base
[1]);
1760 if (!(op
< IR_BROL
? LJ_TARGET_MASKSHIFT
: LJ_TARGET_MASKROT
) &&
1762 tsh
= emitir(IRTI(IR_BAND
), tsh
, lj_ir_kint(J
, 63));
1763 #ifdef LJ_TARGET_UNIFYROT
1764 if (op
== (LJ_TARGET_UNIFYROT
== 1 ? IR_BROR
: IR_BROL
)) {
1765 op
= LJ_TARGET_UNIFYROT
== 1 ? IR_BROL
: IR_BROR
;
1766 tsh
= emitir(IRTI(IR_NEG
), tsh
, tsh
);
1769 tr
= emitir(IRT(op
, id
-CTID_INT64
+IRT_I64
), tr
, tsh
);
1770 J
->base
[0] = emitir(IRTG(IR_CNEWI
, IRT_CDATA
), lj_ir_kint(J
, id
), tr
);
1776 TRef
recff_bit64_tohex(jit_State
*J
, RecordFFData
*rd
, TRef hdr
)
1778 CTState
*cts
= ctype_ctsG(J2G(J
));
1779 CTypeID id
= crec_bit64_type(cts
, &rd
->argv
[0]);
1780 TRef tr
, trsf
= J
->base
[1];
1781 SFormat sf
= (STRFMT_UINT
|STRFMT_T_HEX
);
1785 n
= (int32_t)lj_carith_check64(J
->L
, 2, &id2
);
1787 trsf
= crec_ct_tv(J
, ctype_get(cts
, CTID_INT32
), 0, trsf
, &rd
->argv
[1]);
1789 trsf
= lj_opt_narrow_tobit(J
, trsf
);
1790 emitir(IRTGI(IR_EQ
), trsf
, lj_ir_kint(J
, n
)); /* Specialize to n. */
1794 if (n
< 0) { n
= -n
; sf
|= STRFMT_F_UPPER
; }
1795 sf
|= ((SFormat
)((n
+1)&255) << STRFMT_SH_PREC
);
1797 tr
= crec_ct_tv(J
, ctype_get(cts
, id
), 0, J
->base
[0], &rd
->argv
[0]);
1799 tr
= emitir(IRT(IR_BAND
, IRT_U64
), tr
,
1800 lj_ir_kint64(J
, ((uint64_t)1 << 4*n
)-1));
1802 tr
= lj_opt_narrow_tobit(J
, J
->base
[0]);
1804 tr
= emitir(IRTI(IR_BAND
), tr
, lj_ir_kint(J
, (int32_t)((1u << 4*n
)-1)));
1805 tr
= emitconv(tr
, IRT_U64
, IRT_INT
, 0); /* No sign-extension. */
1808 return lj_ir_call(J
, IRCALL_lj_strfmt_putfxint
, hdr
, lj_ir_kint(J
, sf
), tr
);
1811 /* -- Miscellaneous library functions ------------------------------------- */
1813 void LJ_FASTCALL
lj_crecord_tonumber(jit_State
*J
, RecordFFData
*rd
)
1815 CTState
*cts
= ctype_ctsG(J2G(J
));
1816 CType
*d
, *ct
= lj_ctype_rawref(cts
, cdataV(&rd
->argv
[0])->ctypeid
);
1817 if (ctype_isenum(ct
->info
)) ct
= ctype_child(cts
, ct
);
1818 if (ctype_isnum(ct
->info
) || ctype_iscomplex(ct
->info
)) {
1819 if (ctype_isinteger_or_bool(ct
->info
) && ct
->size
<= 4 &&
1820 !(ct
->size
== 4 && (ct
->info
& CTF_UNSIGNED
)))
1821 d
= ctype_get(cts
, CTID_INT32
);
1823 d
= ctype_get(cts
, CTID_DOUBLE
);
1824 J
->base
[0] = crec_ct_tv(J
, d
, 0, J
->base
[0], &rd
->argv
[0]);
1826 J
->base
[0] = TREF_NIL
;