src/lj_ccallback.c

   1 /*
   2 ** FFI C callback handling.
   3 ** Copyright (C) 2005-2014 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #include "lj_obj.h"
   7
   8 #if LJ_HASFFI
   9
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_tab.h"
  13 #include "lj_state.h"
  14 #include "lj_frame.h"
  15 #include "lj_ctype.h"
  16 #include "lj_cconv.h"
  17 #include "lj_ccall.h"
  18 #include "lj_ccallback.h"
  19 #include "lj_target.h"
  20 #include "lj_mcode.h"
  21 #include "lj_trace.h"
  22 #include "lj_vm.h"
  23
  24 /* -- Target-specific handling of callback slots -------------------------- */
  25
  26 #define CALLBACK_MCODE_SIZE     (LJ_PAGESIZE * LJ_NUM_CBPAGE)
  27
  28 #if LJ_OS_NOJIT
  29
  30 /* Disabled callback support. */
  31 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
  32 #define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
  33 #define CALLBACK_MAX_SLOT       0
  34
  35 #elif LJ_TARGET_X86ORX64
  36
  37 #define CALLBACK_MCODE_HEAD     (LJ_64 ? 8 : 0)
  38 #define CALLBACK_MCODE_GROUP    (-2+1+2+5+(LJ_64 ? 6 : 5))
  39
  40 #define CALLBACK_SLOT2OFS(slot) \
  41   (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
  42
  43 static MSize CALLBACK_OFS2SLOT(MSize ofs)
  44 {
  45   MSize group;
  46   ofs -= CALLBACK_MCODE_HEAD;
  47   group = ofs / (32*4 + CALLBACK_MCODE_GROUP);
  48   return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32;
  49 }
  50
  51 #define CALLBACK_MAX_SLOT \
  52   (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
  53
  54 #elif LJ_TARGET_ARM
  55
  56 #define CALLBACK_MCODE_HEAD             32
  57 #define CALLBACK_SLOT2OFS(slot)         (CALLBACK_MCODE_HEAD + 8*(slot))
  58 #define CALLBACK_OFS2SLOT(ofs)          (((ofs)-CALLBACK_MCODE_HEAD)/8)
  59 #define CALLBACK_MAX_SLOT               (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
  60
  61 #elif LJ_TARGET_PPC
  62
  63 #define CALLBACK_MCODE_HEAD             24
  64 #define CALLBACK_SLOT2OFS(slot)         (CALLBACK_MCODE_HEAD + 8*(slot))
  65 #define CALLBACK_OFS2SLOT(ofs)          (((ofs)-CALLBACK_MCODE_HEAD)/8)
  66 #define CALLBACK_MAX_SLOT               (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
  67
  68 #elif LJ_TARGET_MIPS
  69
  70 #define CALLBACK_MCODE_HEAD             24
  71 #define CALLBACK_SLOT2OFS(slot)         (CALLBACK_MCODE_HEAD + 8*(slot))
  72 #define CALLBACK_OFS2SLOT(ofs)          (((ofs)-CALLBACK_MCODE_HEAD)/8)
  73 #define CALLBACK_MAX_SLOT               (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
  74
  75 #else
  76
  77 /* Missing support for this architecture. */
  78 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
  79 #define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
  80 #define CALLBACK_MAX_SLOT       0
  81
  82 #endif
  83
  84 /* Convert callback slot number to callback function pointer. */
  85 static void *callback_slot2ptr(CTState *cts, MSize slot)
  86 {
  87   return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot);
  88 }
  89
  90 /* Convert callback function pointer to slot number. */
  91 MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
  92 {
  93   uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode);
  94   if (ofs < CALLBACK_MCODE_SIZE) {
  95     MSize slot = CALLBACK_OFS2SLOT((MSize)ofs);
  96     if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs)
  97       return slot;
  98   }
  99   return ~0u;  /* Not a known callback function pointer. */
 100 }
 101
 102 /* Initialize machine code for callback function pointers. */
 103 #if LJ_OS_NOJIT
 104 /* Disabled callback support. */
 105 #define callback_mcode_init(g, p)       UNUSED(p)
 106 #elif LJ_TARGET_X86ORX64
 107 static void callback_mcode_init(global_State *g, uint8_t *page)
 108 {
 109   uint8_t *p = page;
 110   uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
 111   MSize slot;
 112 #if LJ_64
 113   *(void **)p = target; p += 8;
 114 #endif
 115   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 116     /* mov al, slot; jmp group */
 117     *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot;
 118     if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) {
 119       /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
 120       *p++ = XI_PUSH + RID_EBP;
 121       *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
 122       *p++ = XI_MOVri | RID_EBP;
 123       *(int32_t *)p = i32ptr(g); p += 4;
 124 #if LJ_64
 125       /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
 126       *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
 127       *(int32_t *)p = (int32_t)(page-(p+4)); p += 4;
 128 #else
 129       /* jmp lj_vm_ffi_callback. */
 130       *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4;
 131 #endif
 132     } else {
 133       *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
 134     }
 135   }
 136   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 137 }
 138 #elif LJ_TARGET_ARM
 139 static void callback_mcode_init(global_State *g, uint32_t *page)
 140 {
 141   uint32_t *p = page;
 142   void *target = (void *)lj_vm_ffi_callback;
 143   MSize slot;
 144   /* This must match with the saveregs macro in buildvm_arm.dasc. */
 145   *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC);
 146   *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR);
 147   *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD;
 148   *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9);
 149   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC);
 150   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC);
 151   *p++ = u32ptr(g);
 152   *p++ = u32ptr(target);
 153   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 154     *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC);
 155     *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
 156     p++;
 157   }
 158   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 159 }
 160 #elif LJ_TARGET_PPC
 161 static void callback_mcode_init(global_State *g, uint32_t *page)
 162 {
 163   uint32_t *p = page;
 164   void *target = (void *)lj_vm_ffi_callback;
 165   MSize slot;
 166   *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16);
 167   *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16);
 168   *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff);
 169   *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff);
 170   *p++ = PPCI_MTCTR | PPCF_T(RID_TMP);
 171   *p++ = PPCI_BCTR;
 172   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 173     *p++ = PPCI_LI | PPCF_T(RID_R11) | slot;
 174     *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
 175     p++;
 176   }
 177   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 178 }
 179 #elif LJ_TARGET_MIPS
 180 static void callback_mcode_init(global_State *g, uint32_t *page)
 181 {
 182   uint32_t *p = page;
 183   void *target = (void *)lj_vm_ffi_callback;
 184   MSize slot;
 185   *p++ = MIPSI_SW | MIPSF_T(RID_R1)|MIPSF_S(RID_SP) | 0;
 186   *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (u32ptr(target) >> 16);
 187   *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (u32ptr(g) >> 16);
 188   *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) |(u32ptr(target)&0xffff);
 189   *p++ = MIPSI_JR | MIPSF_S(RID_R3);
 190   *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (u32ptr(g)&0xffff);
 191   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 192     *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
 193     p++;
 194     *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
 195   }
 196   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 197 }
 198 #else
 199 /* Missing support for this architecture. */
 200 #define callback_mcode_init(g, p)       UNUSED(p)
 201 #endif
 202
 203 /* -- Machine code management --------------------------------------------- */
 204
 205 #if LJ_TARGET_WINDOWS
 206
 207 #define WIN32_LEAN_AND_MEAN
 208 #include <windows.h>
 209
 210 #elif LJ_TARGET_POSIX
 211
 212 #include <sys/mman.h>
 213 #ifndef MAP_ANONYMOUS
 214 #define MAP_ANONYMOUS   MAP_ANON
 215 #endif
 216
 217 #endif
 218
 219 /* Allocate and initialize area for callback function pointers. */
 220 static void callback_mcode_new(CTState *cts)
 221 {
 222   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
 223   void *p;
 224   if (CALLBACK_MAX_SLOT == 0)
 225     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 226 #if LJ_TARGET_WINDOWS
 227   p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
 228   if (!p)
 229     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 230 #elif LJ_TARGET_POSIX
 231   p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS,
 232            -1, 0);
 233   if (p == MAP_FAILED)
 234     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 235 #else
 236   /* Fallback allocator. Fails if memory is not executable by default. */
 237   p = lj_mem_new(cts->L, sz);
 238 #endif
 239   cts->cb.mcode = p;
 240   callback_mcode_init(cts->g, p);
 241   lj_mcode_sync(p, (char *)p + sz);
 242 #if LJ_TARGET_WINDOWS
 243   {
 244     DWORD oprot;
 245     VirtualProtect(p, sz, PAGE_EXECUTE_READ, &oprot);
 246   }
 247 #elif LJ_TARGET_POSIX
 248   mprotect(p, sz, (PROT_READ|PROT_EXEC));
 249 #endif
 250 }
 251
 252 /* Free area for callback function pointers. */
 253 void lj_ccallback_mcode_free(CTState *cts)
 254 {
 255   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
 256   void *p = cts->cb.mcode;
 257   if (p == NULL) return;
 258 #if LJ_TARGET_WINDOWS
 259   VirtualFree(p, 0, MEM_RELEASE);
 260   UNUSED(sz);
 261 #elif LJ_TARGET_POSIX
 262   munmap(p, sz);
 263 #else
 264   lj_mem_free(cts->g, p, sz);
 265 #endif
 266 }
 267
 268 /* -- C callback entry ---------------------------------------------------- */
 269
 270 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */
 271 #if LJ_TARGET_X86
 272
 273 #define CALLBACK_HANDLE_REGARG \
 274   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
 275     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
 276       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
 277     } else if (ngpr + 1 <= maxgpr) { \
 278       sp = &cts->cb.gpr[ngpr]; \
 279       ngpr += n; \
 280       goto done; \
 281     } \
 282   }
 283
 284 #elif LJ_TARGET_X64 && LJ_ABI_WIN
 285
 286 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 287 #define CALLBACK_HANDLE_REGARG \
 288   if (isfp) { \
 289     if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
 290   } else { \
 291     if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
 292   }
 293
 294 #elif LJ_TARGET_X64
 295
 296 #define CALLBACK_HANDLE_REGARG \
 297   if (isfp) { \
 298     if (nfpr + n <= CCALL_NARG_FPR) { \
 299       sp = &cts->cb.fpr[nfpr]; \
 300       nfpr += n; \
 301       goto done; \
 302     } \
 303   } else { \
 304     if (ngpr + n <= maxgpr) { \
 305       sp = &cts->cb.gpr[ngpr]; \
 306       ngpr += n; \
 307       goto done; \
 308     } \
 309   }
 310
 311 #elif LJ_TARGET_ARM
 312
 313 #if LJ_ABI_SOFTFP
 314
 315 #define CALLBACK_HANDLE_REGARG_FP1      UNUSED(isfp);
 316 #define CALLBACK_HANDLE_REGARG_FP2
 317
 318 #else
 319
 320 #define CALLBACK_HANDLE_REGARG_FP1 \
 321   if (isfp) { \
 322     if (n == 1) { \
 323       if (fprodd) { \
 324         sp = &cts->cb.fpr[fprodd-1]; \
 325         fprodd = 0; \
 326         goto done; \
 327       } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
 328         sp = &cts->cb.fpr[nfpr++]; \
 329         fprodd = nfpr; \
 330         goto done; \
 331       } \
 332     } else { \
 333       if (nfpr + 1 <= CCALL_NARG_FPR) { \
 334         sp = &cts->cb.fpr[nfpr++]; \
 335         goto done; \
 336       } \
 337     } \
 338     fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
 339   } else {
 340
 341 #define CALLBACK_HANDLE_REGARG_FP2      }
 342
 343 #endif
 344
 345 #define CALLBACK_HANDLE_REGARG \
 346   CALLBACK_HANDLE_REGARG_FP1 \
 347   if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 348   if (ngpr + n <= maxgpr) { \
 349     sp = &cts->cb.gpr[ngpr]; \
 350     ngpr += n; \
 351     goto done; \
 352   } CALLBACK_HANDLE_REGARG_FP2
 353
 354 #elif LJ_TARGET_PPC
 355
 356 #define CALLBACK_HANDLE_REGARG \
 357   if (isfp) { \
 358     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 359       sp = &cts->cb.fpr[nfpr++]; \
 360       cta = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 361       goto done; \
 362     } \
 363   } else {  /* Try to pass argument in GPRs. */ \
 364     if (n > 1) { \
 365       lua_assert(ctype_isinteger(cta->info) && n == 2);  /* int64_t. */ \
 366       ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 367     } \
 368     if (ngpr + n <= maxgpr) { \
 369       sp = &cts->cb.gpr[ngpr]; \
 370       ngpr += n; \
 371       goto done; \
 372     } \
 373   }
 374
 375 #define CALLBACK_HANDLE_RET \
 376   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 377     *(double *)dp = *(float *)dp;  /* FPRs always hold doubles. */
 378
 379 #elif LJ_TARGET_MIPS
 380
 381 #define CALLBACK_HANDLE_REGARG \
 382   if (isfp && nfpr < CCALL_NARG_FPR) {  /* Try to pass argument in FPRs. */ \
 383     sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
 384     nfpr++; ngpr += n; \
 385     goto done; \
 386   } else {  /* Try to pass argument in GPRs. */ \
 387     nfpr = CCALL_NARG_FPR; \
 388     if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 389     if (ngpr + n <= maxgpr) { \
 390       sp = &cts->cb.gpr[ngpr]; \
 391       ngpr += n; \
 392       goto done; \
 393     } \
 394   }
 395
 396 #define CALLBACK_HANDLE_RET \
 397   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 398     ((float *)dp)[1] = *(float *)dp;
 399
 400 #else
 401 #error "Missing calling convention definitions for this architecture"
 402 #endif
 403
 404 /* Convert and push callback arguments to Lua stack. */
 405 static void callback_conv_args(CTState *cts, lua_State *L)
 406 {
 407   TValue *o = L->top;
 408   intptr_t *stack = cts->cb.stack;
 409   MSize slot = cts->cb.slot;
 410   CTypeID id = 0, rid, fid;
 411   int gcsteps = 0;
 412   CType *ct;
 413   GCfunc *fn;
 414   MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
 415 #if CCALL_NARG_FPR
 416   MSize nfpr = 0;
 417 #if LJ_TARGET_ARM
 418   MSize fprodd = 0;
 419 #endif
 420 #endif
 421
 422   if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
 423     ct = ctype_get(cts, id);
 424     rid = ctype_cid(ct->info);
 425     fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
 426   } else {  /* Must set up frame first, before throwing the error. */
 427     ct = NULL;
 428     rid = 0;
 429     fn = (GCfunc *)L;
 430   }
 431   o->u32.lo = LJ_CONT_FFI_CALLBACK;  /* Continuation returns from callback. */
 432   o->u32.hi = rid;  /* Return type. x86: +(spadj<<16). */
 433   o++;
 434   setframe_gc(o, obj2gco(fn));
 435   setframe_ftsz(o, (int)((char *)(o+1) - (char *)L->base) + FRAME_CONT);
 436   L->top = L->base = ++o;
 437   if (!ct)
 438     lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
 439   if (isluafunc(fn))
 440     setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1);
 441   lj_state_checkstack(L, LUA_MINSTACK);  /* May throw. */
 442   o = L->base;  /* Might have been reallocated. */
 443
 444 #if LJ_TARGET_X86
 445   /* x86 has several different calling conventions. */
 446   switch (ctype_cconv(ct->info)) {
 447   case CTCC_FASTCALL: maxgpr = 2; break;
 448   case CTCC_THISCALL: maxgpr = 1; break;
 449   default: maxgpr = 0; break;
 450   }
 451 #endif
 452
 453   fid = ct->sib;
 454   while (fid) {
 455     CType *ctf = ctype_get(cts, fid);
 456     if (!ctype_isattrib(ctf->info)) {
 457       CType *cta;
 458       void *sp;
 459       CTSize sz;
 460       int isfp;
 461       MSize n;
 462       lua_assert(ctype_isfield(ctf->info));
 463       cta = ctype_rawchild(cts, ctf);
 464       isfp = ctype_isfp(cta->info);
 465       sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
 466       n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
 467
 468       CALLBACK_HANDLE_REGARG  /* Handle register arguments. */
 469
 470       /* Otherwise pass argument on stack. */
 471       if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8)
 472         nsp = (nsp + 1) & ~1u;  /* Align 64 bit argument on stack. */
 473       sp = &stack[nsp];
 474       nsp += n;
 475
 476     done:
 477       if (LJ_BE && cta->size < CTSIZE_PTR)
 478         sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
 479       gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
 480     }
 481     fid = ctf->sib;
 482   }
 483   L->top = o;
 484 #if LJ_TARGET_X86
 485   /* Store stack adjustment for returns from non-cdecl callbacks. */
 486   if (ctype_cconv(ct->info) != CTCC_CDECL)
 487     (L->base-2)->u32.hi |= (nsp << (16+2));
 488 #endif
 489   while (gcsteps-- > 0)
 490     lj_gc_check(L);
 491 }
 492
 493 /* Convert Lua object to callback result. */
 494 static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
 495 {
 496   CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
 497 #if LJ_TARGET_X86
 498   cts->cb.gpr[2] = 0;
 499 #endif
 500   if (!ctype_isvoid(ctr->info)) {
 501     uint8_t *dp = (uint8_t *)&cts->cb.gpr[0];
 502 #if CCALL_NUM_FPR
 503     if (ctype_isfp(ctr->info))
 504       dp = (uint8_t *)&cts->cb.fpr[0];
 505 #endif
 506     lj_cconv_ct_tv(cts, ctr, dp, o, 0);
 507 #ifdef CALLBACK_HANDLE_RET
 508     CALLBACK_HANDLE_RET
 509 #endif
 510     /* Extend returned integers to (at least) 32 bits. */
 511     if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) {
 512       if (ctr->info & CTF_UNSIGNED)
 513         *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp :
 514                                            (uint32_t)*(uint16_t *)dp;
 515       else
 516         *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
 517                                           (int32_t)*(int16_t *)dp;
 518     }
 519 #if LJ_TARGET_X86
 520     if (ctype_isfp(ctr->info))
 521       cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
 522 #endif
 523   }
 524 }
 525
 526 /* Enter callback. */
 527 lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
 528 {
 529   lua_State *L = cts->L;
 530   global_State *g = cts->g;
 531   lua_assert(L != NULL);
 532   if (tvref(g->jit_base)) {
 533     setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
 534     if (g->panic) g->panic(L);
 535     exit(EXIT_FAILURE);
 536   }
 537   lj_trace_abort(g);  /* Never record across callback. */
 538   /* Setup C frame. */
 539   cframe_prev(cf) = L->cframe;
 540   setcframe_L(cf, L);
 541   cframe_errfunc(cf) = -1;
 542   cframe_nres(cf) = 0;
 543   L->cframe = cf;
 544   callback_conv_args(cts, L);
 545   return L;  /* Now call the function on this stack. */
 546 }
 547
 548 /* Leave callback. */
 549 void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
 550 {
 551   lua_State *L = cts->L;
 552   GCfunc *fn;
 553   TValue *obase = L->base;
 554   L->base = L->top;  /* Keep continuation frame for throwing errors. */
 555   if (o >= L->base) {
 556     /* PC of RET* is lost. Point to last line for result conv. errors. */
 557     fn = curr_func(L);
 558     if (isluafunc(fn)) {
 559       GCproto *pt = funcproto(fn);
 560       setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1);
 561     }
 562   }
 563   callback_conv_result(cts, L, o);
 564   /* Finally drop C frame and continuation frame. */
 565   L->top -= 2;
 566   L->base = obase;
 567   L->cframe = cframe_prev(L->cframe);
 568   cts->cb.slot = 0;  /* Blacklist C function that called the callback. */
 569 }
 570
 571 /* -- C callback management ----------------------------------------------- */
 572
 573 /* Get an unused slot in the callback slot table. */
 574 static MSize callback_slot_new(CTState *cts, CType *ct)
 575 {
 576   CTypeID id = ctype_typeid(cts, ct);
 577   CTypeID1 *cbid = cts->cb.cbid;
 578   MSize top;
 579   for (top = cts->cb.topid; top < cts->cb.sizeid; top++)
 580     if (LJ_LIKELY(cbid[top] == 0))
 581       goto found;
 582 #if CALLBACK_MAX_SLOT
 583   if (top >= CALLBACK_MAX_SLOT)
 584 #endif
 585     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 586   if (!cts->cb.mcode)
 587     callback_mcode_new(cts);
 588   lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1);
 589   cts->cb.cbid = cbid;
 590   memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1));
 591 found:
 592   cbid[top] = id;
 593   cts->cb.topid = top+1;
 594   return top;
 595 }
 596
 597 /* Check for function pointer and supported argument/result types. */
 598 static CType *callback_checkfunc(CTState *cts, CType *ct)
 599 {
 600   int narg = 0;
 601   if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR))
 602     return NULL;
 603   ct = ctype_rawchild(cts, ct);
 604   if (ctype_isfunc(ct->info)) {
 605     CType *ctr = ctype_rawchild(cts, ct);
 606     CTypeID fid = ct->sib;
 607     if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) ||
 608           ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8)))
 609       return NULL;
 610     if ((ct->info & CTF_VARARG))
 611       return NULL;
 612     while (fid) {
 613       CType *ctf = ctype_get(cts, fid);
 614       if (!ctype_isattrib(ctf->info)) {
 615         CType *cta;
 616         lua_assert(ctype_isfield(ctf->info));
 617         cta = ctype_rawchild(cts, ctf);
 618         if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
 619               (ctype_isnum(cta->info) && cta->size <= 8)) ||
 620             ++narg >= LUA_MINSTACK-3)
 621           return NULL;
 622       }
 623       fid = ctf->sib;
 624     }
 625     return ct;
 626   }
 627   return NULL;
 628 }
 629
 630 /* Create a new callback and return the callback function pointer. */
 631 void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn)
 632 {
 633   ct = callback_checkfunc(cts, ct);
 634   if (ct) {
 635     MSize slot = callback_slot_new(cts, ct);
 636     GCtab *t = cts->miscmap;
 637     setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn);
 638     lj_gc_anybarriert(cts->L, t);
 639     return callback_slot2ptr(cts, slot);
 640   }
 641   return NULL;  /* Bad conversion. */
 642 }
 643
 644 #endif