2 ** FFI C callback handling.
3 ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
18 #include "lj_ccallback.h"
19 #include "lj_target.h"
23 /* -- Target-specific handling of callback slots -------------------------- */
25 #define CALLBACK_MCODE_SIZE (LJ_PAGESIZE * LJ_NUM_CBPAGE)
27 #if LJ_TARGET_X86ORX64
29 #define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0)
30 #define CALLBACK_MCODE_GROUP (-2+1+2+5+(LJ_64 ? 6 : 5))
32 #define CALLBACK_SLOT2OFS(slot) \
33 (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
35 static MSize
CALLBACK_OFS2SLOT(MSize ofs
)
38 ofs
-= CALLBACK_MCODE_HEAD
;
39 group
= ofs
/ (32*4 + CALLBACK_MCODE_GROUP
);
40 return (ofs
% (32*4 + CALLBACK_MCODE_GROUP
))/4 + group
*32;
43 #define CALLBACK_MAX_SLOT \
44 (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
48 #define CALLBACK_MCODE_HEAD 32
49 #define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot))
50 #define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8)
51 #define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
55 #define CALLBACK_MCODE_HEAD 24
56 #define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot))
57 #define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8)
58 #define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
62 /* Missing support for this architecture. */
63 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
64 #define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
65 #define CALLBACK_MAX_SLOT 0
69 /* Convert callback slot number to callback function pointer. */
70 static void *callback_slot2ptr(CTState
*cts
, MSize slot
)
72 return (uint8_t *)cts
->cb
.mcode
+ CALLBACK_SLOT2OFS(slot
);
75 /* Convert callback function pointer to slot number. */
76 MSize
lj_ccallback_ptr2slot(CTState
*cts
, void *p
)
78 uintptr_t ofs
= (uintptr_t)((uint8_t *)p
-(uint8_t *)cts
->cb
.mcode
);
79 if (ofs
< CALLBACK_MCODE_SIZE
) {
80 MSize slot
= CALLBACK_OFS2SLOT((MSize
)ofs
);
81 if (CALLBACK_SLOT2OFS(slot
) == (MSize
)ofs
)
84 return ~0u; /* Not a known callback function pointer. */
87 /* Initialize machine code for callback function pointers. */
88 #if LJ_TARGET_X86ORX64
89 static void callback_mcode_init(global_State
*g
, uint8_t *page
)
92 uint8_t *target
= (uint8_t *)(void *)lj_vm_ffi_callback
;
95 *(void **)p
= target
; p
+= 8;
97 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
98 /* mov al, slot; jmp group */
99 *p
++ = XI_MOVrib
| RID_EAX
; *p
++ = (uint8_t)slot
;
100 if ((slot
& 31) == 31 || slot
== CALLBACK_MAX_SLOT
-1) {
101 /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
102 *p
++ = XI_PUSH
+ RID_EBP
;
103 *p
++ = XI_MOVrib
| (RID_EAX
+4); *p
++ = (uint8_t)(slot
>> 8);
104 *p
++ = XI_MOVri
| RID_EBP
;
105 *(int32_t *)p
= i32ptr(g
); p
+= 4;
107 /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
108 *p
++ = XI_GROUP5
; *p
++ = XM_OFS0
+ (XOg_JMP
<<3) + RID_EBP
;
109 *(int32_t *)p
= (int32_t)(page
-(p
+4)); p
+= 4;
111 /* jmp lj_vm_ffi_callback. */
112 *p
++ = XI_JMP
; *(int32_t *)p
= target
-(p
+4); p
+= 4;
115 *p
++ = XI_JMPs
; *p
++ = (uint8_t)((2+2)*(31-(slot
&31)) - 2);
118 lua_assert(p
- page
<= CALLBACK_MCODE_SIZE
);
121 static void callback_mcode_init(global_State
*g
, uint32_t *page
)
124 void *target
= (void *)lj_vm_ffi_callback
;
126 /* This must match with the saveregs macro in buildvm_arm.dasc. */
127 *p
++ = ARMI_SUB
|ARMF_D(RID_R12
)|ARMF_N(RID_R12
)|ARMF_M(RID_PC
);
128 *p
++ = ARMI_PUSH
|ARMF_N(RID_SP
)|RSET_RANGE(RID_R4
,RID_R11
+1)|RID2RSET(RID_LR
);
129 *p
++ = ARMI_SUB
|ARMI_K12
|ARMF_D(RID_R12
)|ARMF_N(RID_R12
)|CALLBACK_MCODE_HEAD
;
130 *p
++ = ARMI_STR
|ARMI_LS_P
|ARMI_LS_W
|ARMF_D(RID_R12
)|ARMF_N(RID_SP
)|(CFRAME_SIZE
-4*9);
131 *p
++ = ARMI_LDR
|ARMI_LS_P
|ARMI_LS_U
|ARMF_D(RID_R12
)|ARMF_N(RID_PC
);
132 *p
++ = ARMI_LDR
|ARMI_LS_P
|ARMI_LS_U
|ARMF_D(RID_PC
)|ARMF_N(RID_PC
);
134 *p
++ = u32ptr(target
);
135 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
136 *p
++ = ARMI_MOV
|ARMF_D(RID_R12
)|ARMF_M(RID_PC
);
137 *p
= ARMI_B
| ((page
-p
-2) & 0x00ffffffu
);
140 lua_assert(p
- page
<= CALLBACK_MCODE_SIZE
);
143 static void callback_mcode_init(global_State
*g
, uint32_t *page
)
146 void *target
= (void *)lj_vm_ffi_callback
;
148 *p
++ = PPCI_LIS
| PPCF_T(RID_TMP
) | (u32ptr(target
) >> 16);
149 *p
++ = PPCI_LIS
| PPCF_T(RID_R12
) | (u32ptr(g
) >> 16);
150 *p
++ = PPCI_ORI
| PPCF_A(RID_TMP
)|PPCF_T(RID_TMP
) | (u32ptr(target
) & 0xffff);
151 *p
++ = PPCI_ORI
| PPCF_A(RID_R12
)|PPCF_T(RID_R12
) | (u32ptr(g
) & 0xffff);
152 *p
++ = PPCI_MTCTR
| PPCF_T(RID_TMP
);
154 for (slot
= 0; slot
< CALLBACK_MAX_SLOT
; slot
++) {
155 *p
++ = PPCI_LI
| PPCF_T(RID_R11
) | slot
;
156 *p
= PPCI_B
| (((page
-p
) & 0x00ffffffu
) << 2);
159 lua_assert(p
- page
<= CALLBACK_MCODE_SIZE
);
162 /* Missing support for this architecture. */
163 #define callback_mcode_init(g, p) UNUSED(p)
166 /* -- Machine code management --------------------------------------------- */
168 #if LJ_TARGET_WINDOWS
170 #define WIN32_LEAN_AND_MEAN
173 #elif LJ_TARGET_POSIX
175 #include <sys/mman.h>
176 #ifndef MAP_ANONYMOUS
177 #define MAP_ANONYMOUS MAP_ANON
182 /* Allocate and initialize area for callback function pointers. */
183 static void callback_mcode_new(CTState
*cts
)
185 size_t sz
= (size_t)CALLBACK_MCODE_SIZE
;
187 if (CALLBACK_MAX_SLOT
== 0)
188 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
189 #if LJ_TARGET_WINDOWS
190 p
= VirtualAlloc(NULL
, sz
, MEM_RESERVE
|MEM_COMMIT
, PAGE_READWRITE
);
192 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
193 #elif LJ_TARGET_POSIX
194 p
= mmap(NULL
, sz
, (PROT_READ
|PROT_WRITE
), MAP_PRIVATE
|MAP_ANONYMOUS
,
197 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
199 /* Fallback allocator. Fails if memory is not executable by default. */
200 p
= lj_mem_new(cts
->L
, sz
);
203 callback_mcode_init(cts
->g
, p
);
204 lj_mcode_sync(p
, (char *)p
+ sz
);
205 #if LJ_TARGET_WINDOWS
208 VirtualProtect(p
, sz
, PAGE_EXECUTE_READ
, &oprot
);
210 #elif LJ_TARGET_POSIX
211 mprotect(p
, sz
, (PROT_READ
|PROT_EXEC
));
215 /* Free area for callback function pointers. */
216 void lj_ccallback_mcode_free(CTState
*cts
)
218 size_t sz
= (size_t)CALLBACK_MCODE_SIZE
;
219 void *p
= cts
->cb
.mcode
;
220 if (p
== NULL
) return;
221 #if LJ_TARGET_WINDOWS
222 VirtualFree(p
, 0, MEM_RELEASE
);
224 #elif LJ_TARGET_POSIX
227 lj_mem_free(cts
->g
, p
, sz
);
231 /* -- C callback entry ---------------------------------------------------- */
233 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */
236 #define CALLBACK_HANDLE_REGARG \
237 if (!isfp) { /* Only non-FP values may be passed in registers. */ \
238 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \
239 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \
240 } else if (ngpr + 1 <= maxgpr) { \
241 sp = &cts->cb.gpr[ngpr]; \
247 #elif LJ_TARGET_X64 && LJ_ABI_WIN
249 /* Windows/x64 argument registers are strictly positional (use ngpr). */
250 #define CALLBACK_HANDLE_REGARG \
252 if (ngpr < 4) { sp = &cts->cb.fpr[ngpr++]; nfpr = ngpr; goto done; } \
254 if (ngpr < 4) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
259 #define CALLBACK_HANDLE_REGARG \
261 if (nfpr + n <= CCALL_NARG_FPR) { \
262 sp = &cts->cb.fpr[nfpr]; \
267 if (ngpr + n <= maxgpr) { \
268 sp = &cts->cb.gpr[ngpr]; \
276 #define CALLBACK_HANDLE_REGARG \
277 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
278 if (ngpr + n <= maxgpr) { \
279 sp = &cts->cb.gpr[ngpr]; \
286 #define CALLBACK_HANDLE_REGARG \
288 if (nfpr + 1 <= CCALL_NARG_FPR) { \
289 sp = &cts->cb.fpr[nfpr]; \
291 cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
294 } else { /* Try to pass argument in GPRs. */ \
296 lua_assert(ctype_isinteger(cta->info) && n == 2); /* int64_t. */ \
297 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
299 if (ngpr + n <= maxgpr) { \
300 sp = &cts->cb.gpr[ngpr]; \
306 #define CALLBACK_HANDLE_RET \
307 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
308 *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */
311 #error "Missing calling convention definitions for this architecture"
314 /* Convert and push callback arguments to Lua stack. */
315 static void callback_conv_args(CTState
*cts
, lua_State
*L
)
318 intptr_t *stack
= cts
->cb
.stack
;
319 MSize slot
= cts
->cb
.slot
;
320 CTypeID id
= 0, rid
, fid
;
323 MSize ngpr
= 0, nsp
= 0, maxgpr
= CCALL_NARG_GPR
;
328 if (slot
< cts
->cb
.sizeid
&& (id
= cts
->cb
.cbid
[slot
]) != 0) {
329 ct
= ctype_get(cts
, id
);
330 rid
= ctype_cid(ct
->info
);
331 fn
= funcV(lj_tab_getint(cts
->miscmap
, (int32_t)slot
));
332 } else { /* Must set up frame first, before throwing the error. */
337 o
->u32
.lo
= LJ_CONT_FFI_CALLBACK
; /* Continuation returns from callback. */
338 o
->u32
.hi
= rid
; /* Return type. x86: +(spadj<<16). */
340 setframe_gc(o
, obj2gco(fn
));
341 setframe_ftsz(o
, (int)((char *)(o
+1) - (char *)L
->base
) + FRAME_CONT
);
342 L
->top
= L
->base
= ++o
;
344 lj_err_caller(cts
->L
, LJ_ERR_FFI_BADCBACK
);
346 setcframe_pc(L
->cframe
, proto_bc(funcproto(fn
))+1);
347 lj_state_checkstack(L
, LUA_MINSTACK
); /* May throw. */
348 o
= L
->base
; /* Might have been reallocated. */
351 /* x86 has several different calling conventions. */
352 switch (ctype_cconv(ct
->info
)) {
353 case CTCC_FASTCALL
: maxgpr
= 2; break;
354 case CTCC_THISCALL
: maxgpr
= 1; break;
355 default: maxgpr
= 0; break;
361 CType
*ctf
= ctype_get(cts
, fid
);
362 if (!ctype_isattrib(ctf
->info
)) {
368 lua_assert(ctype_isfield(ctf
->info
));
369 cta
= ctype_rawchild(cts
, ctf
);
370 if (ctype_isenum(cta
->info
)) cta
= ctype_child(cts
, cta
);
371 isfp
= ctype_isfp(cta
->info
);
372 sz
= (cta
->size
+ CTSIZE_PTR
-1) & ~(CTSIZE_PTR
-1);
373 n
= sz
/ CTSIZE_PTR
; /* Number of GPRs or stack slots needed. */
375 CALLBACK_HANDLE_REGARG
/* Handle register arguments. */
377 /* Otherwise pass argument on stack. */
378 if (CCALL_ALIGN_STACKARG
&& LJ_32
&& sz
== 8)
379 nsp
= (nsp
+ 1) & ~1u; /* Align 64 bit argument on stack. */
384 if (LJ_BE
&& cta
->size
< CTSIZE_PTR
)
385 sp
= (void *)((uint8_t *)sp
+ CTSIZE_PTR
-cta
->size
);
386 lj_cconv_tv_ct(cts
, cta
, 0, o
++, sp
);
392 /* Store stack adjustment for returns from fastcall/stdcall callbacks. */
393 switch (ctype_cconv(ct
->info
)) {
394 case CTCC_FASTCALL
: case CTCC_STDCALL
:
395 (L
->base
-2)->u32
.hi
|= (nsp
<< (16+2));
401 /* Convert Lua object to callback result. */
402 static void callback_conv_result(CTState
*cts
, lua_State
*L
, TValue
*o
)
404 CType
*ctr
= ctype_raw(cts
, (uint16_t)(L
->base
-2)->u32
.hi
);
408 if (!ctype_isvoid(ctr
->info
)) {
409 uint8_t *dp
= (uint8_t *)&cts
->cb
.gpr
[0];
411 if (ctype_isfp(ctr
->info
))
412 dp
= (uint8_t *)&cts
->cb
.fpr
[0];
414 lj_cconv_ct_tv(cts
, ctr
, dp
, o
, 0);
415 #ifdef CALLBACK_HANDLE_RET
418 /* Extend returned integers to (at least) 32 bits. */
419 if (ctype_isinteger_or_bool(ctr
->info
) && ctr
->size
< 4) {
420 if (ctr
->info
& CTF_UNSIGNED
)
421 *(uint32_t *)dp
= ctr
->size
== 1 ? (uint32_t)*(uint8_t *)dp
:
422 (uint32_t)*(uint16_t *)dp
;
424 *(int32_t *)dp
= ctr
->size
== 1 ? (int32_t)*(int8_t *)dp
:
425 (int32_t)*(int16_t *)dp
;
428 if (ctype_isfp(ctr
->info
))
429 cts
->cb
.gpr
[2] = ctr
->size
== sizeof(float) ? 1 : 2;
434 /* Enter callback. */
435 lua_State
* LJ_FASTCALL
lj_ccallback_enter(CTState
*cts
, void *cf
)
437 lua_State
*L
= cts
->L
;
438 lua_assert(L
!= NULL
);
439 if (gcref(cts
->g
->jit_L
))
440 lj_err_caller(gco2th(gcref(cts
->g
->jit_L
)), LJ_ERR_FFI_BADCBACK
);
442 cframe_prev(cf
) = L
->cframe
;
444 cframe_errfunc(cf
) = -1;
447 callback_conv_args(cts
, L
);
448 return L
; /* Now call the function on this stack. */
451 /* Leave callback. */
452 void LJ_FASTCALL
lj_ccallback_leave(CTState
*cts
, TValue
*o
)
454 lua_State
*L
= cts
->L
;
456 TValue
*obase
= L
->base
;
457 L
->base
= L
->top
; /* Keep continuation frame for throwing errors. */
458 /* PC of RET* is lost. Point to last line for result conv. errors. */
461 GCproto
*pt
= funcproto(fn
);
462 setcframe_pc(L
->cframe
, proto_bc(pt
)+pt
->sizebc
+1);
464 callback_conv_result(cts
, L
, o
);
465 /* Finally drop C frame and continuation frame. */
466 L
->cframe
= cframe_prev(L
->cframe
);
471 /* -- C callback management ----------------------------------------------- */
473 /* Get an unused slot in the callback slot table. */
474 static MSize
callback_slot_new(CTState
*cts
, CType
*ct
)
476 CTypeID id
= ctype_typeid(cts
, ct
);
477 CTypeID1
*cbid
= cts
->cb
.cbid
;
479 for (top
= cts
->cb
.topid
; top
< cts
->cb
.sizeid
; top
++)
480 if (LJ_LIKELY(cbid
[top
] == 0))
482 #if CALLBACK_MAX_SLOT
483 if (top
>= CALLBACK_MAX_SLOT
)
485 lj_err_caller(cts
->L
, LJ_ERR_FFI_CBACKOV
);
487 callback_mcode_new(cts
);
488 lj_mem_growvec(cts
->L
, cbid
, cts
->cb
.sizeid
, CALLBACK_MAX_SLOT
, CTypeID1
);
490 memset(cbid
+top
, 0, (cts
->cb
.sizeid
-top
)*sizeof(CTypeID1
));
493 cts
->cb
.topid
= top
+1;
497 /* Check for function pointer and supported argument/result types. */
498 static CType
*callback_checkfunc(CTState
*cts
, CType
*ct
)
501 if (!ctype_isptr(ct
->info
) || (LJ_64
&& ct
->size
!= CTSIZE_PTR
))
503 ct
= ctype_rawchild(cts
, ct
);
504 if (ctype_isfunc(ct
->info
)) {
505 CType
*ctr
= ctype_rawchild(cts
, ct
);
506 CTypeID fid
= ct
->sib
;
507 if (!(ctype_isvoid(ctr
->info
) || ctype_isenum(ctr
->info
) ||
508 ctype_isptr(ctr
->info
) || (ctype_isnum(ctr
->info
) && ctr
->size
<= 8)))
510 if ((ct
->info
& CTF_VARARG
))
513 CType
*ctf
= ctype_get(cts
, fid
);
514 if (!ctype_isattrib(ctf
->info
)) {
516 lua_assert(ctype_isfield(ctf
->info
));
517 cta
= ctype_rawchild(cts
, ctf
);
518 if (!(ctype_isenum(cta
->info
) || ctype_isptr(cta
->info
) ||
519 (ctype_isnum(cta
->info
) && cta
->size
<= 8)) ||
520 ++narg
>= LUA_MINSTACK
-3)
530 /* Create a new callback and return the callback function pointer. */
531 void *lj_ccallback_new(CTState
*cts
, CType
*ct
, GCfunc
*fn
)
533 ct
= callback_checkfunc(cts
, ct
);
535 MSize slot
= callback_slot_new(cts
, ct
);
536 GCtab
*t
= cts
->miscmap
;
537 setfuncV(cts
->L
, lj_tab_setint(cts
->L
, t
, (int32_t)slot
), fn
);
538 lj_gc_anybarriert(cts
->L
, t
);
539 return callback_slot2ptr(cts
, slot
);
541 return NULL
; /* Bad conversion. */