source/libs/luajit/LuaJIT-src/src/lj_ccallback.c

   1 /*
   2 ** FFI C callback handling.
   3 ** Copyright (C) 2005-2015 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #include "lj_obj.h"
   7
   8 #if LJ_HASFFI
   9
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_tab.h"
  13 #include "lj_state.h"
  14 #include "lj_frame.h"
  15 #include "lj_ctype.h"
  16 #include "lj_cconv.h"
  17 #include "lj_ccall.h"
  18 #include "lj_ccallback.h"
  19 #include "lj_target.h"
  20 #include "lj_mcode.h"
  21 #include "lj_trace.h"
  22 #include "lj_vm.h"
  23
  24 /* -- Target-specific handling of callback slots -------------------------- */
  25
  26 #define CALLBACK_MCODE_SIZE     (LJ_PAGESIZE * LJ_NUM_CBPAGE)
  27
  28 #if LJ_OS_NOJIT
  29
  30 /* Callbacks disabled. */
  31 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
  32 #define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
  33 #define CALLBACK_MAX_SLOT       0
  34
  35 #elif LJ_TARGET_X86ORX64
  36
  37 #define CALLBACK_MCODE_HEAD     (LJ_64 ? 8 : 0)
  38 #define CALLBACK_MCODE_GROUP    (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
  39
  40 #define CALLBACK_SLOT2OFS(slot) \
  41   (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
  42
  43 static MSize CALLBACK_OFS2SLOT(MSize ofs)
  44 {
  45   MSize group;
  46   ofs -= CALLBACK_MCODE_HEAD;
  47   group = ofs / (32*4 + CALLBACK_MCODE_GROUP);
  48   return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32;
  49 }
  50
  51 #define CALLBACK_MAX_SLOT \
  52   (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
  53
  54 #elif LJ_TARGET_ARM
  55
  56 #define CALLBACK_MCODE_HEAD             32
  57
  58 #elif LJ_TARGET_ARM64
  59
  60 #define CALLBACK_MCODE_HEAD             32
  61
  62 #elif LJ_TARGET_PPC
  63
  64 #define CALLBACK_MCODE_HEAD             24
  65
  66 #elif LJ_TARGET_MIPS
  67
  68 #define CALLBACK_MCODE_HEAD             24
  69
  70 #else
  71
  72 /* Missing support for this architecture. */
  73 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
  74 #define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
  75 #define CALLBACK_MAX_SLOT       0
  76
  77 #endif
  78
  79 #ifndef CALLBACK_SLOT2OFS
  80 #define CALLBACK_SLOT2OFS(slot)         (CALLBACK_MCODE_HEAD + 8*(slot))
  81 #define CALLBACK_OFS2SLOT(ofs)          (((ofs)-CALLBACK_MCODE_HEAD)/8)
  82 #define CALLBACK_MAX_SLOT               (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
  83 #endif
  84
  85 /* Convert callback slot number to callback function pointer. */
  86 static void *callback_slot2ptr(CTState *cts, MSize slot)
  87 {
  88   return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot);
  89 }
  90
  91 /* Convert callback function pointer to slot number. */
  92 MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
  93 {
  94   uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode);
  95   if (ofs < CALLBACK_MCODE_SIZE) {
  96     MSize slot = CALLBACK_OFS2SLOT((MSize)ofs);
  97     if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs)
  98       return slot;
  99   }
 100   return ~0u;  /* Not a known callback function pointer. */
 101 }
 102
 103 /* Initialize machine code for callback function pointers. */
 104 #if LJ_OS_NOJIT
 105 /* Disabled callback support. */
 106 #define callback_mcode_init(g, p)       UNUSED(p)
 107 #elif LJ_TARGET_X86ORX64
 108 static void callback_mcode_init(global_State *g, uint8_t *page)
 109 {
 110   uint8_t *p = page;
 111   uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
 112   MSize slot;
 113 #if LJ_64
 114   *(void **)p = target; p += 8;
 115 #endif
 116   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 117     /* mov al, slot; jmp group */
 118     *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot;
 119     if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) {
 120       /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
 121       *p++ = XI_PUSH + RID_EBP;
 122       *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
 123 #if LJ_GC64
 124       *p++ = 0x48; *p++ = XI_MOVri | RID_EBP;
 125       *(uint64_t *)p = (uint64_t)(g); p += 8;
 126 #else
 127       *p++ = XI_MOVri | RID_EBP;
 128       *(int32_t *)p = i32ptr(g); p += 4;
 129 #endif
 130 #if LJ_64
 131       /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
 132       *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
 133       *(int32_t *)p = (int32_t)(page-(p+4)); p += 4;
 134 #else
 135       /* jmp lj_vm_ffi_callback. */
 136       *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4;
 137 #endif
 138     } else {
 139       *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
 140     }
 141   }
 142   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 143 }
 144 #elif LJ_TARGET_ARM
 145 static void callback_mcode_init(global_State *g, uint32_t *page)
 146 {
 147   uint32_t *p = page;
 148   void *target = (void *)lj_vm_ffi_callback;
 149   MSize slot;
 150   /* This must match with the saveregs macro in buildvm_arm.dasc. */
 151   *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC);
 152   *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR);
 153   *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD;
 154   *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9);
 155   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC);
 156   *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC);
 157   *p++ = u32ptr(g);
 158   *p++ = u32ptr(target);
 159   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 160     *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC);
 161     *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
 162     p++;
 163   }
 164   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 165 }
 166 #elif LJ_TARGET_ARM64
 167 static void callback_mcode_init(global_State *g, uint32_t *page)
 168 {
 169   uint32_t *p = page;
 170   void *target = (void *)lj_vm_ffi_callback;
 171   MSize slot;
 172   *p++ = A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4);
 173   *p++ = A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5);
 174   *p++ = A64I_BR | A64F_N(RID_X11);
 175   *p++ = A64I_NOP;
 176   ((void **)p)[0] = target;
 177   ((void **)p)[1] = g;
 178   p += 4;
 179   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 180     *p++ = A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot);
 181     *p = A64I_B | A64F_S26((page-p) & 0x03ffffffu);
 182     p++;
 183   }
 184   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 185 }
 186 #elif LJ_TARGET_PPC
 187 static void callback_mcode_init(global_State *g, uint32_t *page)
 188 {
 189   uint32_t *p = page;
 190   void *target = (void *)lj_vm_ffi_callback;
 191   MSize slot;
 192   *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16);
 193   *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16);
 194   *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff);
 195   *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff);
 196   *p++ = PPCI_MTCTR | PPCF_T(RID_TMP);
 197   *p++ = PPCI_BCTR;
 198   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 199     *p++ = PPCI_LI | PPCF_T(RID_R11) | slot;
 200     *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
 201     p++;
 202   }
 203   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 204 }
 205 #elif LJ_TARGET_MIPS
 206 static void callback_mcode_init(global_State *g, uint32_t *page)
 207 {
 208   uint32_t *p = page;
 209   void *target = (void *)lj_vm_ffi_callback;
 210   MSize slot;
 211   *p++ = MIPSI_SW | MIPSF_T(RID_R1)|MIPSF_S(RID_SP) | 0;
 212   *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (u32ptr(target) >> 16);
 213   *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (u32ptr(g) >> 16);
 214   *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) |(u32ptr(target)&0xffff);
 215   *p++ = MIPSI_JR | MIPSF_S(RID_R3);
 216   *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (u32ptr(g)&0xffff);
 217   for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
 218     *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
 219     p++;
 220     *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
 221   }
 222   lua_assert(p - page <= CALLBACK_MCODE_SIZE);
 223 }
 224 #else
 225 /* Missing support for this architecture. */
 226 #define callback_mcode_init(g, p)       UNUSED(p)
 227 #endif
 228
 229 /* -- Machine code management --------------------------------------------- */
 230
 231 #if LJ_TARGET_WINDOWS
 232
 233 #define WIN32_LEAN_AND_MEAN
 234 #include <windows.h>
 235
 236 #elif LJ_TARGET_POSIX
 237
 238 #include <sys/mman.h>
 239 #ifndef MAP_ANONYMOUS
 240 #define MAP_ANONYMOUS   MAP_ANON
 241 #endif
 242
 243 #endif
 244
 245 /* Allocate and initialize area for callback function pointers. */
 246 static void callback_mcode_new(CTState *cts)
 247 {
 248   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
 249   void *p;
 250   if (CALLBACK_MAX_SLOT == 0)
 251     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 252 #if LJ_TARGET_WINDOWS
 253   p = VirtualAlloc(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
 254   if (!p)
 255     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 256 #elif LJ_TARGET_POSIX
 257   p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS,
 258            -1, 0);
 259   if (p == MAP_FAILED)
 260     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 261 #else
 262   /* Fallback allocator. Fails if memory is not executable by default. */
 263   p = lj_mem_new(cts->L, sz);
 264 #endif
 265   cts->cb.mcode = p;
 266   callback_mcode_init(cts->g, p);
 267   lj_mcode_sync(p, (char *)p + sz);
 268 #if LJ_TARGET_WINDOWS
 269   {
 270     DWORD oprot;
 271     VirtualProtect(p, sz, PAGE_EXECUTE_READ, &oprot);
 272   }
 273 #elif LJ_TARGET_POSIX
 274   mprotect(p, sz, (PROT_READ|PROT_EXEC));
 275 #endif
 276 }
 277
 278 /* Free area for callback function pointers. */
 279 void lj_ccallback_mcode_free(CTState *cts)
 280 {
 281   size_t sz = (size_t)CALLBACK_MCODE_SIZE;
 282   void *p = cts->cb.mcode;
 283   if (p == NULL) return;
 284 #if LJ_TARGET_WINDOWS
 285   VirtualFree(p, 0, MEM_RELEASE);
 286   UNUSED(sz);
 287 #elif LJ_TARGET_POSIX
 288   munmap(p, sz);
 289 #else
 290   lj_mem_free(cts->g, p, sz);
 291 #endif
 292 }
 293
 294 /* -- C callback entry ---------------------------------------------------- */
 295
 296 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */
 297 #if LJ_TARGET_X86
 298
 299 #define CALLBACK_HANDLE_REGARG \
 300   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
 301     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
 302       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
 303     } else if (ngpr + 1 <= maxgpr) { \
 304       sp = &cts->cb.gpr[ngpr]; \
 305       ngpr += n; \
 306       goto done; \
 307     } \
 308   }
 309
 310 #elif LJ_TARGET_X64 && LJ_ABI_WIN
 311
 312 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 313 #define CALLBACK_HANDLE_REGARG \
 314   if (isfp) { \
 315     if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
 316   } else { \
 317     if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
 318   }
 319
 320 #elif LJ_TARGET_X64
 321
 322 #define CALLBACK_HANDLE_REGARG \
 323   if (isfp) { \
 324     if (nfpr + n <= CCALL_NARG_FPR) { \
 325       sp = &cts->cb.fpr[nfpr]; \
 326       nfpr += n; \
 327       goto done; \
 328     } \
 329   } else { \
 330     if (ngpr + n <= maxgpr) { \
 331       sp = &cts->cb.gpr[ngpr]; \
 332       ngpr += n; \
 333       goto done; \
 334     } \
 335   }
 336
 337 #elif LJ_TARGET_ARM
 338
 339 #if LJ_ABI_SOFTFP
 340
 341 #define CALLBACK_HANDLE_REGARG_FP1      UNUSED(isfp);
 342 #define CALLBACK_HANDLE_REGARG_FP2
 343
 344 #else
 345
 346 #define CALLBACK_HANDLE_REGARG_FP1 \
 347   if (isfp) { \
 348     if (n == 1) { \
 349       if (fprodd) { \
 350         sp = &cts->cb.fpr[fprodd-1]; \
 351         fprodd = 0; \
 352         goto done; \
 353       } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
 354         sp = &cts->cb.fpr[nfpr++]; \
 355         fprodd = nfpr; \
 356         goto done; \
 357       } \
 358     } else { \
 359       if (nfpr + 1 <= CCALL_NARG_FPR) { \
 360         sp = &cts->cb.fpr[nfpr++]; \
 361         goto done; \
 362       } \
 363     } \
 364     fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
 365   } else {
 366
 367 #define CALLBACK_HANDLE_REGARG_FP2      }
 368
 369 #endif
 370
 371 #define CALLBACK_HANDLE_REGARG \
 372   CALLBACK_HANDLE_REGARG_FP1 \
 373   if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 374   if (ngpr + n <= maxgpr) { \
 375     sp = &cts->cb.gpr[ngpr]; \
 376     ngpr += n; \
 377     goto done; \
 378   } CALLBACK_HANDLE_REGARG_FP2
 379
 380 #elif LJ_TARGET_ARM64
 381
 382 #define CALLBACK_HANDLE_REGARG \
 383   if (isfp) { \
 384     if (nfpr + n <= CCALL_NARG_FPR) { \
 385       sp = &cts->cb.fpr[nfpr]; \
 386       nfpr += n; \
 387       goto done; \
 388     } else { \
 389       nfpr = CCALL_NARG_FPR;  /* Prevent reordering. */ \
 390     } \
 391   } else { \
 392     if (!LJ_TARGET_IOS && n > 1) \
 393       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 394     if (ngpr + n <= maxgpr) { \
 395       sp = &cts->cb.gpr[ngpr]; \
 396       ngpr += n; \
 397       goto done; \
 398     } else { \
 399       ngpr = CCALL_NARG_GPR;  /* Prevent reordering. */ \
 400     } \
 401   }
 402
 403 #elif LJ_TARGET_PPC
 404
 405 #define CALLBACK_HANDLE_REGARG \
 406   if (isfp) { \
 407     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 408       sp = &cts->cb.fpr[nfpr++]; \
 409       cta = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 410       goto done; \
 411     } \
 412   } else {  /* Try to pass argument in GPRs. */ \
 413     if (n > 1) { \
 414       lua_assert(ctype_isinteger(cta->info) && n == 2);  /* int64_t. */ \
 415       ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 416     } \
 417     if (ngpr + n <= maxgpr) { \
 418       sp = &cts->cb.gpr[ngpr]; \
 419       ngpr += n; \
 420       goto done; \
 421     } \
 422   }
 423
 424 #define CALLBACK_HANDLE_RET \
 425   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 426     *(double *)dp = *(float *)dp;  /* FPRs always hold doubles. */
 427
 428 #elif LJ_TARGET_MIPS
 429
 430 #define CALLBACK_HANDLE_REGARG \
 431   if (isfp && nfpr < CCALL_NARG_FPR) {  /* Try to pass argument in FPRs. */ \
 432     sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
 433     nfpr++; ngpr += n; \
 434     goto done; \
 435   } else {  /* Try to pass argument in GPRs. */ \
 436     nfpr = CCALL_NARG_FPR; \
 437     if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 438     if (ngpr + n <= maxgpr) { \
 439       sp = &cts->cb.gpr[ngpr]; \
 440       ngpr += n; \
 441       goto done; \
 442     } \
 443   }
 444
 445 #define CALLBACK_HANDLE_RET \
 446   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 447     ((float *)dp)[1] = *(float *)dp;
 448
 449 #else
 450 #error "Missing calling convention definitions for this architecture"
 451 #endif
 452
 453 /* Convert and push callback arguments to Lua stack. */
 454 static void callback_conv_args(CTState *cts, lua_State *L)
 455 {
 456   TValue *o = L->top;
 457   intptr_t *stack = cts->cb.stack;
 458   MSize slot = cts->cb.slot;
 459   CTypeID id = 0, rid, fid;
 460   int gcsteps = 0;
 461   CType *ct;
 462   GCfunc *fn;
 463   int fntp;
 464   MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
 465 #if CCALL_NARG_FPR
 466   MSize nfpr = 0;
 467 #if LJ_TARGET_ARM
 468   MSize fprodd = 0;
 469 #endif
 470 #endif
 471
 472   if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
 473     ct = ctype_get(cts, id);
 474     rid = ctype_cid(ct->info);  /* Return type. x86: +(spadj<<16). */
 475     fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
 476     fntp = LJ_TFUNC;
 477   } else {  /* Must set up frame first, before throwing the error. */
 478     ct = NULL;
 479     rid = 0;
 480     fn = (GCfunc *)L;
 481     fntp = LJ_TTHREAD;
 482   }
 483   /* Continuation returns from callback. */
 484   if (LJ_FR2) {
 485     (o++)->u64 = LJ_CONT_FFI_CALLBACK;
 486     (o++)->u64 = rid;
 487     o++;
 488   } else {
 489     o->u32.lo = LJ_CONT_FFI_CALLBACK;
 490     o->u32.hi = rid;
 491     o++;
 492   }
 493   setframe_gc(o, obj2gco(fn), fntp);
 494   setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT);
 495   L->top = L->base = ++o;
 496   if (!ct)
 497     lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
 498   if (isluafunc(fn))
 499     setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1);
 500   lj_state_checkstack(L, LUA_MINSTACK);  /* May throw. */
 501   o = L->base;  /* Might have been reallocated. */
 502
 503 #if LJ_TARGET_X86
 504   /* x86 has several different calling conventions. */
 505   switch (ctype_cconv(ct->info)) {
 506   case CTCC_FASTCALL: maxgpr = 2; break;
 507   case CTCC_THISCALL: maxgpr = 1; break;
 508   default: maxgpr = 0; break;
 509   }
 510 #endif
 511
 512   fid = ct->sib;
 513   while (fid) {
 514     CType *ctf = ctype_get(cts, fid);
 515     if (!ctype_isattrib(ctf->info)) {
 516       CType *cta;
 517       void *sp;
 518       CTSize sz;
 519       int isfp;
 520       MSize n;
 521       lua_assert(ctype_isfield(ctf->info));
 522       cta = ctype_rawchild(cts, ctf);
 523       isfp = ctype_isfp(cta->info);
 524       sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
 525       n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
 526
 527       CALLBACK_HANDLE_REGARG  /* Handle register arguments. */
 528
 529       /* Otherwise pass argument on stack. */
 530       if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8)
 531         nsp = (nsp + 1) & ~1u;  /* Align 64 bit argument on stack. */
 532       sp = &stack[nsp];
 533       nsp += n;
 534
 535     done:
 536       if (LJ_BE && cta->size < CTSIZE_PTR)
 537         sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
 538       gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
 539     }
 540     fid = ctf->sib;
 541   }
 542   L->top = o;
 543 #if LJ_TARGET_X86
 544   /* Store stack adjustment for returns from non-cdecl callbacks. */
 545   if (ctype_cconv(ct->info) != CTCC_CDECL) {
 546 #if LJ_FR2
 547     (L->base-3)->u64 |= (nsp << (16+2));
 548 #else
 549     (L->base-2)->u32.hi |= (nsp << (16+2));
 550 #endif
 551   }
 552 #endif
 553   while (gcsteps-- > 0)
 554     lj_gc_check(L);
 555 }
 556
 557 /* Convert Lua object to callback result. */
 558 static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
 559 {
 560 #if LJ_FR2
 561   CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64);
 562 #else
 563   CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
 564 #endif
 565 #if LJ_TARGET_X86
 566   cts->cb.gpr[2] = 0;
 567 #endif
 568   if (!ctype_isvoid(ctr->info)) {
 569     uint8_t *dp = (uint8_t *)&cts->cb.gpr[0];
 570 #if CCALL_NUM_FPR
 571     if (ctype_isfp(ctr->info))
 572       dp = (uint8_t *)&cts->cb.fpr[0];
 573 #endif
 574     lj_cconv_ct_tv(cts, ctr, dp, o, 0);
 575 #ifdef CALLBACK_HANDLE_RET
 576     CALLBACK_HANDLE_RET
 577 #endif
 578     /* Extend returned integers to (at least) 32 bits. */
 579     if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) {
 580       if (ctr->info & CTF_UNSIGNED)
 581         *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp :
 582                                            (uint32_t)*(uint16_t *)dp;
 583       else
 584         *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
 585                                           (int32_t)*(int16_t *)dp;
 586     }
 587 #if LJ_TARGET_X86
 588     if (ctype_isfp(ctr->info))
 589       cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
 590 #endif
 591   }
 592 }
 593
 594 /* Enter callback. */
 595 lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
 596 {
 597   lua_State *L = cts->L;
 598   global_State *g = cts->g;
 599   lua_assert(L != NULL);
 600   if (tvref(g->jit_base)) {
 601     setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
 602     if (g->panic) g->panic(L);
 603     exit(EXIT_FAILURE);
 604   }
 605   lj_trace_abort(g);  /* Never record across callback. */
 606   /* Setup C frame. */
 607   cframe_prev(cf) = L->cframe;
 608   setcframe_L(cf, L);
 609   cframe_errfunc(cf) = -1;
 610   cframe_nres(cf) = 0;
 611   L->cframe = cf;
 612   callback_conv_args(cts, L);
 613   return L;  /* Now call the function on this stack. */
 614 }
 615
 616 /* Leave callback. */
 617 void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
 618 {
 619   lua_State *L = cts->L;
 620   GCfunc *fn;
 621   TValue *obase = L->base;
 622   L->base = L->top;  /* Keep continuation frame for throwing errors. */
 623   if (o >= L->base) {
 624     /* PC of RET* is lost. Point to last line for result conv. errors. */
 625     fn = curr_func(L);
 626     if (isluafunc(fn)) {
 627       GCproto *pt = funcproto(fn);
 628       setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1);
 629     }
 630   }
 631   callback_conv_result(cts, L, o);
 632   /* Finally drop C frame and continuation frame. */
 633   L->top -= 2+2*LJ_FR2;
 634   L->base = obase;
 635   L->cframe = cframe_prev(L->cframe);
 636   cts->cb.slot = 0;  /* Blacklist C function that called the callback. */
 637 }
 638
 639 /* -- C callback management ----------------------------------------------- */
 640
 641 /* Get an unused slot in the callback slot table. */
 642 static MSize callback_slot_new(CTState *cts, CType *ct)
 643 {
 644   CTypeID id = ctype_typeid(cts, ct);
 645   CTypeID1 *cbid = cts->cb.cbid;
 646   MSize top;
 647   for (top = cts->cb.topid; top < cts->cb.sizeid; top++)
 648     if (LJ_LIKELY(cbid[top] == 0))
 649       goto found;
 650 #if CALLBACK_MAX_SLOT
 651   if (top >= CALLBACK_MAX_SLOT)
 652 #endif
 653     lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
 654   if (!cts->cb.mcode)
 655     callback_mcode_new(cts);
 656   lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1);
 657   cts->cb.cbid = cbid;
 658   memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1));
 659 found:
 660   cbid[top] = id;
 661   cts->cb.topid = top+1;
 662   return top;
 663 }
 664
 665 /* Check for function pointer and supported argument/result types. */
 666 static CType *callback_checkfunc(CTState *cts, CType *ct)
 667 {
 668   int narg = 0;
 669   if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR))
 670     return NULL;
 671   ct = ctype_rawchild(cts, ct);
 672   if (ctype_isfunc(ct->info)) {
 673     CType *ctr = ctype_rawchild(cts, ct);
 674     CTypeID fid = ct->sib;
 675     if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) ||
 676           ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8)))
 677       return NULL;
 678     if ((ct->info & CTF_VARARG))
 679       return NULL;
 680     while (fid) {
 681       CType *ctf = ctype_get(cts, fid);
 682       if (!ctype_isattrib(ctf->info)) {
 683         CType *cta;
 684         lua_assert(ctype_isfield(ctf->info));
 685         cta = ctype_rawchild(cts, ctf);
 686         if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
 687               (ctype_isnum(cta->info) && cta->size <= 8)) ||
 688             ++narg >= LUA_MINSTACK-3)
 689           return NULL;
 690       }
 691       fid = ctf->sib;
 692     }
 693     return ct;
 694   }
 695   return NULL;
 696 }
 697
 698 /* Create a new callback and return the callback function pointer. */
 699 void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn)
 700 {
 701   ct = callback_checkfunc(cts, ct);
 702   if (ct) {
 703     MSize slot = callback_slot_new(cts, ct);
 704     GCtab *t = cts->miscmap;
 705     setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn);
 706     lj_gc_anybarriert(cts->L, t);
 707     return callback_slot2ptr(cts, slot);
 708   }
 709   return NULL;  /* Bad conversion. */
 710 }
 711
 712 #endif