2 ** FFI C callback handling.
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
18 #include "lj_ccallback.h"
19 #include "lj_target.h"
24 /* -- Target-specific handling of callback slots -------------------------- */
26 #define CALLBACK_MCODE_SIZE (LJ_PAGESIZE * LJ_NUM_CBPAGE)
30 /* Callbacks disabled. */
31 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
32 #define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
33 #define CALLBACK_MAX_SLOT 0
35 #elif LJ_TARGET_X86ORX64
37 #define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0)
38 #define CALLBACK_MCODE_GROUP (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
40 #define CALLBACK_SLOT2OFS(slot) \
41 (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
43 static MSize
CALLBACK_OFS2SLOT(MSize ofs
)
46 ofs
-= CALLBACK_MCODE_HEAD
;
47 group
= ofs
/ (32*4 + CALLBACK_MCODE_GROUP
);
48 return (ofs
% (32*4 + CALLBACK_MCODE_GROUP
))/4 + group
*32;
51 #define CALLBACK_MAX_SLOT \
52 (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
56 #define CALLBACK_MCODE_HEAD 32
60 #define CALLBACK_MCODE_HEAD 32
64 #define CALLBACK_MCODE_HEAD 24
66 #elif LJ_TARGET_MIPS32
68 #define CALLBACK_MCODE_HEAD 20
70 #elif LJ_TARGET_MIPS64
72 #define CALLBACK_MCODE_HEAD 52
76 /* Missing support for this architecture. */
77 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
78 #define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
79 #define CALLBACK_MAX_SLOT 0
83 #ifndef CALLBACK_SLOT2OFS
84 #define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot))
85 #define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8)
86 #define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
89 /* Convert callback slot number to callback function pointer. */
90 static void *callback_slot2ptr(CTState
*cts
, MSize slot
)
92 return (uint8_t *)cts
->cb
.mcode
+ CALLBACK_SLOT2OFS(slot
);
95 /* Convert callback function pointer to slot number. */
96 MSize
lj_ccallback_ptr2slot(CTState
*cts
, void *p
)
98 uintptr_t ofs
= (uintptr_t)((uint8_t *)p
-(uint8_t *)cts
->cb
.mcode
);
99 if (ofs
< CALLBACK_MCODE_SIZE
) {
100 MSize slot
= CALLBACK_OFS2SLOT((MSize
)ofs
);
101 if (CALLBACK_SLOT2OFS(slot
) == (MSize
)ofs
)
104 return ~0u; /* Not a known callback function pointer. */
107 /* Initialize machine code for callback function pointers. */
109 /* Disabled callback support. */
110 #define callback_mcode_init(g, p) (p)
111 #elif LJ_TARGET_X86ORX64
112 static void *callback_mcode_init(global_State
*g
, uint8_t *page
)
115 uint8_t *target
= (uint8_t *)(void *)lj_vm_ffi_callback
;
118 *(void **)p
= target
; p
+= 8;
120 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
121 /* mov al, slot; jmp group */
122 *p
++ = XI_MOVrib
| RID_EAX
; *p
++ = (uint8_t)slot
;
123 if ((slot
& 31) == 31 || slot
== CALLBACK_MAX_SLOT
-1) {
124 /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
125 *p
++ = XI_PUSH
+ RID_EBP
;
126 *p
++ = XI_MOVrib
| (RID_EAX
+4); *p
++ = (uint8_t)(slot
>> 8);
128 *p
++ = 0x48; *p
++ = XI_MOVri
| RID_EBP
;
129 *(uint64_t *)p
= (uint64_t)(g
); p
+= 8;
131 *p
++ = XI_MOVri
| RID_EBP
;
132 *(int32_t *)p
= i32ptr(g
); p
+= 4;
135 /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
136 *p
++ = XI_GROUP5
; *p
++ = XM_OFS0
+ (XOg_JMP
<<3) + RID_EBP
;
137 *(int32_t *)p
= (int32_t)(page
-(p
+4)); p
+= 4;
139 /* jmp lj_vm_ffi_callback. */
140 *p
++ = XI_JMP
; *(int32_t *)p
= target
-(p
+4); p
+= 4;
143 *p
++ = XI_JMPs
; *p
++ = (uint8_t)((2+2)*(31-(slot
&31)) - 2);
149 static void *callback_mcode_init(global_State
*g
, uint32_t *page
)
152 void *target
= (void *)lj_vm_ffi_callback
;
154 /* This must match with the saveregs macro in buildvm_arm.dasc. */
155 *p
++ = ARMI_SUB
|ARMF_D(RID_R12
)|ARMF_N(RID_R12
)|ARMF_M(RID_PC
);
156 *p
++ = ARMI_PUSH
|ARMF_N(RID_SP
)|RSET_RANGE(RID_R4
,RID_R11
+1)|RID2RSET(RID_LR
);
157 *p
++ = ARMI_SUB
|ARMI_K12
|ARMF_D(RID_R12
)|ARMF_N(RID_R12
)|CALLBACK_MCODE_HEAD
;
158 *p
++ = ARMI_STR
|ARMI_LS_P
|ARMI_LS_W
|ARMF_D(RID_R12
)|ARMF_N(RID_SP
)|(CFRAME_SIZE
-4*9);
159 *p
++ = ARMI_LDR
|ARMI_LS_P
|ARMI_LS_U
|ARMF_D(RID_R12
)|ARMF_N(RID_PC
);
160 *p
++ = ARMI_LDR
|ARMI_LS_P
|ARMI_LS_U
|ARMF_D(RID_PC
)|ARMF_N(RID_PC
);
162 *p
++ = u32ptr(target
);
163 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
164 *p
++ = ARMI_MOV
|ARMF_D(RID_R12
)|ARMF_M(RID_PC
);
165 *p
= ARMI_B
| ((page
-p
-2) & 0x00ffffffu
);
170 #elif LJ_TARGET_ARM64
171 static void *callback_mcode_init(global_State
*g
, uint32_t *page
)
174 ASMFunction target
= lj_vm_ffi_callback
;
176 *p
++ = A64I_LE(A64I_LDRLx
| A64F_D(RID_X11
) | A64F_S19(4));
177 *p
++ = A64I_LE(A64I_LDRLx
| A64F_D(RID_X10
) | A64F_S19(5));
178 *p
++ = A64I_LE(A64I_BR_AUTH
| A64F_N(RID_X11
));
179 *p
++ = A64I_LE(A64I_NOP
);
180 ((ASMFunction
*)p
)[0] = target
;
183 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
184 *p
++ = A64I_LE(A64I_MOVZw
| A64F_D(RID_X9
) | A64F_U16(slot
));
185 *p
= A64I_LE(A64I_B
| A64F_S26((page
-p
) & 0x03ffffffu
));
191 static void *callback_mcode_init(global_State
*g
, uint32_t *page
)
194 void *target
= (void *)lj_vm_ffi_callback
;
196 *p
++ = PPCI_LIS
| PPCF_T(RID_TMP
) | (u32ptr(target
) >> 16);
197 *p
++ = PPCI_LIS
| PPCF_T(RID_R12
) | (u32ptr(g
) >> 16);
198 *p
++ = PPCI_ORI
| PPCF_A(RID_TMP
)|PPCF_T(RID_TMP
) | (u32ptr(target
) & 0xffff);
199 *p
++ = PPCI_ORI
| PPCF_A(RID_R12
)|PPCF_T(RID_R12
) | (u32ptr(g
) & 0xffff);
200 *p
++ = PPCI_MTCTR
| PPCF_T(RID_TMP
);
202 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
203 *p
++ = PPCI_LI
| PPCF_T(RID_R11
) | slot
;
204 *p
= PPCI_B
| (((page
-p
) & 0x00ffffffu
) << 2);
210 static void *callback_mcode_init(global_State
*g
, uint32_t *page
)
213 uintptr_t target
= (uintptr_t)(void *)lj_vm_ffi_callback
;
214 uintptr_t ug
= (uintptr_t)(void *)g
;
217 *p
++ = MIPSI_LUI
| MIPSF_T(RID_R3
) | (target
>> 16);
218 *p
++ = MIPSI_LUI
| MIPSF_T(RID_R2
) | (ug
>> 16);
220 *p
++ = MIPSI_LUI
| MIPSF_T(RID_R3
) | (target
>> 48);
221 *p
++ = MIPSI_LUI
| MIPSF_T(RID_R2
) | (ug
>> 48);
222 *p
++ = MIPSI_ORI
| MIPSF_T(RID_R3
)|MIPSF_S(RID_R3
) | ((target
>> 32) & 0xffff);
223 *p
++ = MIPSI_ORI
| MIPSF_T(RID_R2
)|MIPSF_S(RID_R2
) | ((ug
>> 32) & 0xffff);
224 *p
++ = MIPSI_DSLL
| MIPSF_D(RID_R3
)|MIPSF_T(RID_R3
) | MIPSF_A(16);
225 *p
++ = MIPSI_DSLL
| MIPSF_D(RID_R2
)|MIPSF_T(RID_R2
) | MIPSF_A(16);
226 *p
++ = MIPSI_ORI
| MIPSF_T(RID_R3
)|MIPSF_S(RID_R3
) | ((target
>> 16) & 0xffff);
227 *p
++ = MIPSI_ORI
| MIPSF_T(RID_R2
)|MIPSF_S(RID_R2
) | ((ug
>> 16) & 0xffff);
228 *p
++ = MIPSI_DSLL
| MIPSF_D(RID_R3
)|MIPSF_T(RID_R3
) | MIPSF_A(16);
229 *p
++ = MIPSI_DSLL
| MIPSF_D(RID_R2
)|MIPSF_T(RID_R2
) | MIPSF_A(16);
231 *p
++ = MIPSI_ORI
| MIPSF_T(RID_R3
)|MIPSF_S(RID_R3
) | (target
& 0xffff);
232 *p
++ = MIPSI_JR
| MIPSF_S(RID_R3
);
233 *p
++ = MIPSI_ORI
| MIPSF_T(RID_R2
)|MIPSF_S(RID_R2
) | (ug
& 0xffff);
234 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
235 *p
= MIPSI_B
| ((page
-p
-1) & 0x0000ffffu
);
237 *p
++ = MIPSI_LI
| MIPSF_T(RID_R1
) | slot
;
242 /* Missing support for this architecture. */
243 #define callback_mcode_init(g, p) (p)
246 /* -- Machine code management --------------------------------------------- */
248 #if LJ_TARGET_WINDOWS
250 #define WIN32_LEAN_AND_MEAN
253 #elif LJ_TARGET_POSIX
255 #include <sys/mman.h>
256 #ifndef MAP_ANONYMOUS
257 #define MAP_ANONYMOUS MAP_ANON
260 #define CCPROT_CREATE (PROT_MPROTECT(PROT_EXEC))
262 #define CCPROT_CREATE 0
267 /* Allocate and initialize area for callback function pointers. */
268 static void callback_mcode_new(CTState
*cts
)
270 size_t sz
= (size_t)CALLBACK_MCODE_SIZE
;
272 if (CALLBACK_MAX_SLOT
== 0)
273 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
274 #if LJ_TARGET_WINDOWS
275 p
= LJ_WIN_VALLOC(NULL
, sz
, MEM_RESERVE
|MEM_COMMIT
, PAGE_READWRITE
);
277 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
278 #elif LJ_TARGET_POSIX
279 p
= mmap(NULL
, sz
, (PROT_READ
|PROT_WRITE
|CCPROT_CREATE
), MAP_PRIVATE
|MAP_ANONYMOUS
,
282 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
284 /* Fallback allocator. Fails if memory is not executable by default. */
285 p
= lj_mem_new(cts
->L
, sz
);
288 pe
= callback_mcode_init(cts
->g
, p
);
290 lj_assertCTS((size_t)((char *)pe
- (char *)p
) <= sz
,
291 "miscalculated CALLBACK_MAX_SLOT");
292 lj_mcode_sync(p
, (char *)p
+ sz
);
293 #if LJ_TARGET_WINDOWS
296 LJ_WIN_VPROTECT(p
, sz
, PAGE_EXECUTE_READ
, &oprot
);
298 #elif LJ_TARGET_POSIX
299 mprotect(p
, sz
, (PROT_READ
|PROT_EXEC
));
303 /* Free area for callback function pointers. */
304 void lj_ccallback_mcode_free(CTState
*cts
)
306 size_t sz
= (size_t)CALLBACK_MCODE_SIZE
;
307 void *p
= cts
->cb
.mcode
;
308 if (p
== NULL
) return;
309 #if LJ_TARGET_WINDOWS
310 VirtualFree(p
, 0, MEM_RELEASE
);
312 #elif LJ_TARGET_POSIX
315 lj_mem_free(cts
->g
, p
, sz
);
319 /* -- C callback entry ---------------------------------------------------- */
321 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */
324 #define CALLBACK_HANDLE_REGARG \
325 if (!isfp) { /* Only non-FP values may be passed in registers. */ \
326 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \
327 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \
328 } else if (ngpr + 1 <= maxgpr) { \
329 sp = &cts->cb.gpr[ngpr]; \
335 #elif LJ_TARGET_X64 && LJ_ABI_WIN
337 /* Windows/x64 argument registers are strictly positional (use ngpr). */
338 #define CALLBACK_HANDLE_REGARG \
340 if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
342 if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
347 #define CALLBACK_HANDLE_REGARG \
349 if (nfpr + n <= CCALL_NARG_FPR) { \
350 sp = &cts->cb.fpr[nfpr]; \
355 if (ngpr + n <= maxgpr) { \
356 sp = &cts->cb.gpr[ngpr]; \
366 #define CALLBACK_HANDLE_REGARG_FP1 UNUSED(isfp);
367 #define CALLBACK_HANDLE_REGARG_FP2
371 #define CALLBACK_HANDLE_REGARG_FP1 \
375 sp = &cts->cb.fpr[fprodd-1]; \
378 } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
379 sp = &cts->cb.fpr[nfpr++]; \
384 if (nfpr + 1 <= CCALL_NARG_FPR) { \
385 sp = &cts->cb.fpr[nfpr++]; \
389 fprodd = 0; /* No reordering after the first FP value is on stack. */ \
392 #define CALLBACK_HANDLE_REGARG_FP2 }
396 #define CALLBACK_HANDLE_REGARG \
397 CALLBACK_HANDLE_REGARG_FP1 \
398 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
399 if (ngpr + n <= maxgpr) { \
400 sp = &cts->cb.gpr[ngpr]; \
403 } CALLBACK_HANDLE_REGARG_FP2
405 #elif LJ_TARGET_ARM64
407 #define CALLBACK_HANDLE_REGARG \
409 if (nfpr + n <= CCALL_NARG_FPR) { \
410 sp = &cts->cb.fpr[nfpr]; \
414 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \
417 if (!LJ_TARGET_OSX && n > 1) \
418 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
419 if (ngpr + n <= maxgpr) { \
420 sp = &cts->cb.gpr[ngpr]; \
424 ngpr = CCALL_NARG_GPR; /* Prevent reordering. */ \
430 #define CALLBACK_HANDLE_GPR \
432 lj_assertCTS(((LJ_ABI_SOFTFP && ctype_isnum(cta->info)) || /* double. */ \
433 ctype_isinteger(cta->info)) && n == 2, /* int64_t. */ \
435 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
437 if (ngpr + n <= maxgpr) { \
438 sp = &cts->cb.gpr[ngpr]; \
444 #define CALLBACK_HANDLE_REGARG \
445 CALLBACK_HANDLE_GPR \
448 #define CALLBACK_HANDLE_REGARG \
450 if (nfpr + 1 <= CCALL_NARG_FPR) { \
451 sp = &cts->cb.fpr[nfpr++]; \
452 cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
455 } else { /* Try to pass argument in GPRs. */ \
456 CALLBACK_HANDLE_GPR \
461 #define CALLBACK_HANDLE_RET \
462 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
463 *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */
466 #elif LJ_TARGET_MIPS32
468 #define CALLBACK_HANDLE_GPR \
469 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
470 if (ngpr + n <= maxgpr) { \
471 sp = &cts->cb.gpr[ngpr]; \
476 #if !LJ_ABI_SOFTFP /* MIPS32 hard-float */
477 #define CALLBACK_HANDLE_REGARG \
478 if (isfp && nfpr < CCALL_NARG_FPR) { /* Try to pass argument in FPRs. */ \
479 sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
482 } else { /* Try to pass argument in GPRs. */ \
483 nfpr = CCALL_NARG_FPR; \
484 CALLBACK_HANDLE_GPR \
486 #else /* MIPS32 soft-float */
487 #define CALLBACK_HANDLE_REGARG \
488 CALLBACK_HANDLE_GPR \
492 #define CALLBACK_HANDLE_RET \
493 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
494 ((float *)dp)[1] = *(float *)dp;
496 #elif LJ_TARGET_MIPS64
498 #if !LJ_ABI_SOFTFP /* MIPS64 hard-float */
499 #define CALLBACK_HANDLE_REGARG \
500 if (ngpr + n <= maxgpr) { \
501 sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \
505 #else /* MIPS64 soft-float */
506 #define CALLBACK_HANDLE_REGARG \
507 if (ngpr + n <= maxgpr) { \
509 sp = (void*) &cts->cb.gpr[ngpr]; \
515 #define CALLBACK_HANDLE_RET \
516 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
517 ((float *)dp)[1] = *(float *)dp;
520 #error "Missing calling convention definitions for this architecture"
523 /* Convert and push callback arguments to Lua stack. */
524 static void callback_conv_args(CTState
*cts
, lua_State
*L
)
527 intptr_t *stack
= cts
->cb
.stack
;
528 MSize slot
= cts
->cb
.slot
;
529 CTypeID id
= 0, rid
, fid
;
534 MSize ngpr
= 0, nsp
= 0, maxgpr
= CCALL_NARG_GPR
;
542 if (slot
< cts
->cb
.sizeid
&& (id
= cts
->cb
.cbid
[slot
]) != 0) {
543 ct
= ctype_get(cts
, id
);
544 rid
= ctype_cid(ct
->info
); /* Return type. x86: +(spadj<<16). */
545 fn
= funcV(lj_tab_getint(cts
->miscmap
, (int32_t)slot
));
547 } else { /* Must set up frame first, before throwing the error. */
553 /* Continuation returns from callback. */
555 (o
++)->u64
= LJ_CONT_FFI_CALLBACK
;
558 o
->u32
.lo
= LJ_CONT_FFI_CALLBACK
;
562 setframe_gc(o
, obj2gco(fn
), fntp
);
564 setframe_ftsz(o
, ((char *)(o
+1) - (char *)L
->base
) + FRAME_CONT
);
565 L
->top
= L
->base
= ++o
;
567 lj_err_caller(cts
->L
, LJ_ERR_FFI_BADCBACK
);
569 setcframe_pc(L
->cframe
, proto_bc(funcproto(fn
))+1);
570 lj_state_checkstack(L
, LUA_MINSTACK
); /* May throw. */
571 o
= L
->base
; /* Might have been reallocated. */
574 /* x86 has several different calling conventions. */
575 switch (ctype_cconv(ct
->info
)) {
576 case CTCC_FASTCALL
: maxgpr
= 2; break;
577 case CTCC_THISCALL
: maxgpr
= 1; break;
578 default: maxgpr
= 0; break;
584 CType
*ctf
= ctype_get(cts
, fid
);
585 if (!ctype_isattrib(ctf
->info
)) {
591 lj_assertCTS(ctype_isfield(ctf
->info
), "field expected");
592 cta
= ctype_rawchild(cts
, ctf
);
593 isfp
= ctype_isfp(cta
->info
);
594 sz
= (cta
->size
+ CTSIZE_PTR
-1) & ~(CTSIZE_PTR
-1);
595 n
= sz
/ CTSIZE_PTR
; /* Number of GPRs or stack slots needed. */
597 CALLBACK_HANDLE_REGARG
/* Handle register arguments. */
599 /* Otherwise pass argument on stack. */
600 if (CCALL_ALIGN_STACKARG
&& LJ_32
&& sz
== 8)
601 nsp
= (nsp
+ 1) & ~1u; /* Align 64 bit argument on stack. */
606 if (LJ_BE
&& cta
->size
< CTSIZE_PTR
611 sp
= (void *)((uint8_t *)sp
+ CTSIZE_PTR
-cta
->size
);
612 gcsteps
+= lj_cconv_tv_ct(cts
, cta
, 0, o
++, sp
);
618 /* Store stack adjustment for returns from non-cdecl callbacks. */
619 if (ctype_cconv(ct
->info
) != CTCC_CDECL
) {
621 (L
->base
-3)->u64
|= (nsp
<< (16+2));
623 (L
->base
-2)->u32
.hi
|= (nsp
<< (16+2));
627 while (gcsteps
-- > 0)
631 /* Convert Lua object to callback result. */
632 static void callback_conv_result(CTState
*cts
, lua_State
*L
, TValue
*o
)
635 CType
*ctr
= ctype_raw(cts
, (uint16_t)(L
->base
-3)->u64
);
637 CType
*ctr
= ctype_raw(cts
, (uint16_t)(L
->base
-2)->u32
.hi
);
642 if (!ctype_isvoid(ctr
->info
)) {
643 uint8_t *dp
= (uint8_t *)&cts
->cb
.gpr
[0];
645 if (ctype_isfp(ctr
->info
))
646 dp
= (uint8_t *)&cts
->cb
.fpr
[0];
648 #if LJ_TARGET_ARM64 && LJ_BE
649 if (ctype_isfp(ctr
->info
) && ctr
->size
== sizeof(float))
650 dp
= (uint8_t *)&cts
->cb
.fpr
[0].f
[1];
652 lj_cconv_ct_tv(cts
, ctr
, dp
, o
, 0);
653 #ifdef CALLBACK_HANDLE_RET
656 /* Extend returned integers to (at least) 32 bits. */
657 if (ctype_isinteger_or_bool(ctr
->info
) && ctr
->size
< 4) {
658 if (ctr
->info
& CTF_UNSIGNED
)
659 *(uint32_t *)dp
= ctr
->size
== 1 ? (uint32_t)*(uint8_t *)dp
:
660 (uint32_t)*(uint16_t *)dp
;
662 *(int32_t *)dp
= ctr
->size
== 1 ? (int32_t)*(int8_t *)dp
:
663 (int32_t)*(int16_t *)dp
;
665 #if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
666 /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
667 if (ctr
->size
<= 4 &&
668 (LJ_ABI_SOFTFP
|| ctype_isinteger_or_bool(ctr
->info
)))
669 *(int64_t *)dp
= (int64_t)*(int32_t *)dp
;
672 if (ctype_isfp(ctr
->info
))
673 cts
->cb
.gpr
[2] = ctr
->size
== sizeof(float) ? 1 : 2;
678 /* Enter callback. */
679 lua_State
* LJ_FASTCALL
lj_ccallback_enter(CTState
*cts
, void *cf
)
681 lua_State
*L
= cts
->L
;
682 global_State
*g
= cts
->g
;
683 lj_assertG(L
!= NULL
, "uninitialized cts->L in callback");
684 if (tvref(g
->jit_base
)) {
685 setstrV(L
, L
->top
++, lj_err_str(L
, LJ_ERR_FFI_BADCBACK
));
686 if (g
->panic
) g
->panic(L
);
689 lj_trace_abort(g
); /* Never record across callback. */
691 cframe_prev(cf
) = L
->cframe
;
693 cframe_errfunc(cf
) = -1;
696 callback_conv_args(cts
, L
);
697 return L
; /* Now call the function on this stack. */
700 /* Leave callback. */
701 void LJ_FASTCALL
lj_ccallback_leave(CTState
*cts
, TValue
*o
)
703 lua_State
*L
= cts
->L
;
705 TValue
*obase
= L
->base
;
706 L
->base
= L
->top
; /* Keep continuation frame for throwing errors. */
708 /* PC of RET* is lost. Point to last line for result conv. errors. */
711 GCproto
*pt
= funcproto(fn
);
712 setcframe_pc(L
->cframe
, proto_bc(pt
)+pt
->sizebc
+1);
715 callback_conv_result(cts
, L
, o
);
716 /* Finally drop C frame and continuation frame. */
717 L
->top
-= 2+2*LJ_FR2
;
719 L
->cframe
= cframe_prev(L
->cframe
);
720 cts
->cb
.slot
= 0; /* Blacklist C function that called the callback. */
723 /* -- C callback management ----------------------------------------------- */
725 /* Get an unused slot in the callback slot table. */
726 static MSize
callback_slot_new(CTState
*cts
, CType
*ct
)
728 CTypeID id
= ctype_typeid(cts
, ct
);
729 CTypeID1
*cbid
= cts
->cb
.cbid
;
731 for (top
= cts
->cb
.topid
; top
< cts
->cb
.sizeid
; top
++)
732 if (LJ_LIKELY(cbid
[top
] == 0))
734 #if CALLBACK_MAX_SLOT
735 if (top
>= CALLBACK_MAX_SLOT
)
737 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
739 callback_mcode_new(cts
);
740 lj_mem_growvec(cts
->L
, cbid
, cts
->cb
.sizeid
, CALLBACK_MAX_SLOT
, CTypeID1
);
742 memset(cbid
+top
, 0, (cts
->cb
.sizeid
-top
)*sizeof(CTypeID1
));
745 cts
->cb
.topid
= top
+1;
749 /* Check for function pointer and supported argument/result types. */
750 static CType
*callback_checkfunc(CTState
*cts
, CType
*ct
)
753 if (!ctype_isptr(ct
->info
) || (LJ_64
&& ct
->size
!= CTSIZE_PTR
))
755 ct
= ctype_rawchild(cts
, ct
);
756 if (ctype_isfunc(ct
->info
)) {
757 CType
*ctr
= ctype_rawchild(cts
, ct
);
758 CTypeID fid
= ct
->sib
;
759 if (!(ctype_isvoid(ctr
->info
) || ctype_isenum(ctr
->info
) ||
760 ctype_isptr(ctr
->info
) || (ctype_isnum(ctr
->info
) && ctr
->size
<= 8)))
762 if ((ct
->info
& CTF_VARARG
))
765 CType
*ctf
= ctype_get(cts
, fid
);
766 if (!ctype_isattrib(ctf
->info
)) {
768 lj_assertCTS(ctype_isfield(ctf
->info
), "field expected");
769 cta
= ctype_rawchild(cts
, ctf
);
770 if (!(ctype_isenum(cta
->info
) || ctype_isptr(cta
->info
) ||
771 (ctype_isnum(cta
->info
) && cta
->size
<= 8)) ||
772 ++narg
>= LUA_MINSTACK
-3)
782 /* Create a new callback and return the callback function pointer. */
783 void *lj_ccallback_new(CTState
*cts
, CType
*ct
, GCfunc
*fn
)
785 ct
= callback_checkfunc(cts
, ct
);
787 MSize slot
= callback_slot_new(cts
, ct
);
788 GCtab
*t
= cts
->miscmap
;
789 setfuncV(cts
->L
, lj_tab_setint(cts
->L
, t
, (int32_t)slot
), fn
);
790 lj_gc_anybarriert(cts
->L
, t
);
791 return callback_slot2ptr(cts
, slot
);
793 return NULL
; /* Bad conversion. */