src/lj_ccall.c

   1 /*
   2 ** FFI C call handling.
   3 ** Copyright (C) 2005-2011 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #include "lj_obj.h"
   7
   8 #if LJ_HASFFI
   9
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_str.h"
  13 #include "lj_ctype.h"
  14 #include "lj_cconv.h"
  15 #include "lj_cdata.h"
  16 #include "lj_ccall.h"
  17 #include "lj_trace.h"
  18
  19 /* Target-specific handling of register arguments. */
  20 #if LJ_TARGET_X86
  21 /* -- x86 calling conventions --------------------------------------------- */
  22
  23 #if LJ_ABI_WIN
  24
  25 #define CCALL_HANDLE_STRUCTRET \
  26   /* Return structs bigger than 8 by reference (on stack only). */ \
  27   cc->retref = (sz > 8); \
  28   if (cc->retref) cc->stack[nsp++] = (GPRArg)dp;
  29
  30 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  31
  32 #else
  33
  34 #define CCALL_HANDLE_STRUCTRET \
  35   cc->retref = 1;  /* Return all structs by reference (in reg or on stack). */ \
  36   if (ngpr < maxgpr) \
  37     cc->gpr[ngpr++] = (GPRArg)dp; \
  38   else \
  39     cc->stack[nsp++] = (GPRArg)dp;
  40
  41 #define CCALL_HANDLE_COMPLEXRET \
  42   /* Return complex float in GPRs and complex double by reference. */ \
  43   cc->retref = (sz > 8); \
  44   if (cc->retref) { \
  45     if (ngpr < maxgpr) \
  46       cc->gpr[ngpr++] = (GPRArg)dp; \
  47     else \
  48       cc->stack[nsp++] = (GPRArg)dp; \
  49   }
  50
  51 #endif
  52
  53 #define CCALL_HANDLE_COMPLEXRET2 \
  54   if (!cc->retref) \
  55     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  56
  57 #define CCALL_HANDLE_STRUCTARG \
  58   ngpr = maxgpr;  /* Pass all structs by value on the stack. */
  59
  60 #define CCALL_HANDLE_COMPLEXARG \
  61   isfp = 1;  /* Pass complex by value on stack. */
  62
  63 #define CCALL_HANDLE_REGARG \
  64   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
  65     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
  66       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
  67     } else if (ngpr + 1 <= maxgpr) { \
  68       dp = &cc->gpr[ngpr]; \
  69       ngpr += n; \
  70       goto done; \
  71     } \
  72   }
  73
  74 #elif LJ_TARGET_X64 && LJ_ABI_WIN
  75 /* -- Windows/x64 calling conventions ------------------------------------- */
  76
  77 #define CCALL_HANDLE_STRUCTRET \
  78   /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
  79   cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
  80   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
  81
  82 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  83
  84 #define CCALL_HANDLE_COMPLEXRET2 \
  85   if (!cc->retref) \
  86     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  87
  88 #define CCALL_HANDLE_STRUCTARG \
  89   /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
  90   if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
  91     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
  92     sz = CTSIZE_PTR;  /* Pass all other structs by reference. */ \
  93   }
  94
  95 #define CCALL_HANDLE_COMPLEXARG \
  96   /* Pass complex float in a GPR and complex double by reference. */ \
  97   if (sz != 2*sizeof(float)) { \
  98     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
  99     sz = CTSIZE_PTR; \
 100   }
 101
 102 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 103 #define CCALL_HANDLE_REGARG \
 104   if (isfp) { \
 105     if (ngpr < 4) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
 106   } else { \
 107     if (ngpr < 4) { dp = &cc->gpr[ngpr++]; goto done; } \
 108   }
 109
 110 #elif LJ_TARGET_X64
 111 /* -- POSIX/x64 calling conventions --------------------------------------- */
 112
 113 #define CCALL_HANDLE_STRUCTRET \
 114   int rcl[2]; rcl[0] = rcl[1] = 0; \
 115   if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
 116     cc->retref = 1;  /* Return struct by reference. */ \
 117     cc->gpr[ngpr++] = (GPRArg)dp; \
 118   } else { \
 119     cc->retref = 0;  /* Return small structs in registers. */ \
 120   }
 121
 122 #define CCALL_HANDLE_STRUCTRET2 \
 123   int rcl[2]; rcl[0] = rcl[1] = 0; \
 124   ccall_classify_struct(cts, ctr, rcl, 0); \
 125   ccall_struct_ret(cc, rcl, dp, ctr->size);
 126
 127 #define CCALL_HANDLE_COMPLEXRET \
 128   /* Complex values are returned in one or two FPRs. */ \
 129   cc->retref = 0;
 130
 131 #define CCALL_HANDLE_COMPLEXRET2 \
 132   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPR. */ \
 133     *(int64_t *)dp = cc->fpr[0].l[0]; \
 134   } else {  /* Copy non-contiguous complex double from FPRs. */ \
 135     ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
 136     ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
 137   }
 138
 139 #define CCALL_HANDLE_STRUCTARG \
 140   int rcl[2]; rcl[0] = rcl[1] = 0; \
 141   if (!ccall_classify_struct(cts, d, rcl, 0)) { \
 142     cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
 143     if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
 144     nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
 145     continue; \
 146   }  /* Pass all other structs by value on stack. */
 147
 148 #define CCALL_HANDLE_COMPLEXARG \
 149   isfp = 2;  /* Pass complex in FPRs or on stack. Needs postprocessing. */
 150
 151 #define CCALL_HANDLE_REGARG \
 152   if (isfp) {  /* Try to pass argument in FPRs. */ \
 153     if (nfpr + n <= CCALL_NARG_FPR) { \
 154       dp = &cc->fpr[nfpr]; \
 155       nfpr += n; \
 156       goto done; \
 157     } \
 158   } else {  /* Try to pass argument in GPRs. */ \
 159     /* Note that reordering is explicitly allowed in the x64 ABI. */ \
 160     if (n <= 2 && ngpr + n <= maxgpr) { \
 161       dp = &cc->gpr[ngpr]; \
 162       ngpr += n; \
 163       goto done; \
 164     } \
 165   }
 166
 167 #elif LJ_TARGET_ARM
 168 /* -- ARM calling conventions --------------------------------------------- */
 169
 170 #define CCALL_HANDLE_STRUCTRET \
 171   /* Return structs of size <= 4 in a GPR. */ \
 172   cc->retref = !(sz <= 4); \
 173   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 174
 175 #define CCALL_HANDLE_COMPLEXRET \
 176   cc->retref = 1;  /* Return all complex values by reference. */ \
 177   cc->gpr[ngpr++] = (GPRArg)dp;
 178
 179 #define CCALL_HANDLE_COMPLEXRET2 \
 180   UNUSED(dp); /* Nothing to do. */
 181
 182 #define CCALL_HANDLE_STRUCTARG \
 183   /* Pass all structs by value in registers and/or on the stack. */
 184
 185 #define CCALL_HANDLE_COMPLEXARG \
 186   /* Pass complex by value in 2 or 4 GPRs. */
 187
 188 /* ARM has a softfp ABI. */
 189 #define CCALL_HANDLE_REGARG \
 190   if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 191     if (ngpr < maxgpr) \
 192       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 193     else \
 194       nsp = (nsp + 1u) & ~1u;  /* Align argument on stack. */ \
 195   } \
 196   if (ngpr < maxgpr) { \
 197     dp = &cc->gpr[ngpr]; \
 198     if (ngpr + n > maxgpr) { \
 199       nsp += ngpr + n - maxgpr;  /* Assumes contiguous gpr/stack fields. */ \
 200       if (nsp > CCALL_MAXSTACK) goto err_nyi;  /* Too many arguments. */ \
 201       ngpr = maxgpr; \
 202     } else { \
 203       ngpr += n; \
 204     } \
 205     goto done; \
 206   }
 207
 208 #elif LJ_TARGET_PPC
 209 /* -- PPC calling conventions --------------------------------------------- */
 210
 211 #define CCALL_HANDLE_STRUCTRET \
 212   cc->retref = 1;  /* Return all structs by reference. */ \
 213   cc->gpr[ngpr++] = (GPRArg)dp;
 214
 215 #define CCALL_HANDLE_COMPLEXRET \
 216   /* Complex values are returned in 2 or 4 GPRs. */ \
 217   cc->retref = 0;
 218
 219 #define CCALL_HANDLE_COMPLEXRET2 \
 220   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 221
 222 #define CCALL_HANDLE_STRUCTARG \
 223   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 224   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 225
 226 #define CCALL_HANDLE_COMPLEXARG \
 227   /* Pass complex by value in 2 or 4 GPRs. */
 228
 229 #define CCALL_HANDLE_REGARG \
 230   if (isfp) {  /* Try to pass argument in FPRs. */ \
 231     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 232       dp = &cc->fpr[nfpr]; \
 233       nfpr += 1; \
 234       d = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 235       goto done; \
 236     } \
 237   } else {  /* Try to pass argument in GPRs. */ \
 238     if (n > 1) { \
 239       lua_assert(n == 2 || n == 4);  /* int64_t or complex (float). */ \
 240       if (ctype_isinteger(d->info)) \
 241         ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 242       else if (ngpr + n > maxgpr) \
 243         ngpr = maxgpr;  /* Prevent reordering. */ \
 244     } \
 245     if (ngpr + n <= maxgpr) { \
 246       dp = &cc->gpr[ngpr]; \
 247       ngpr += n; \
 248       goto done; \
 249     } \
 250   }
 251
 252 #define CCALL_HANDLE_RET \
 253   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 254     ctr = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */
 255
 256 #elif LJ_TARGET_PPCSPE
 257 /* -- PPC/SPE calling conventions ----------------------------------------- */
 258
 259 #define CCALL_HANDLE_STRUCTRET \
 260   cc->retref = 1;  /* Return all structs by reference. */ \
 261   cc->gpr[ngpr++] = (GPRArg)dp;
 262
 263 #define CCALL_HANDLE_COMPLEXRET \
 264   /* Complex values are returned in 2 or 4 GPRs. */ \
 265   cc->retref = 0;
 266
 267 #define CCALL_HANDLE_COMPLEXRET2 \
 268   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 269
 270 #define CCALL_HANDLE_STRUCTARG \
 271   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 272   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 273
 274 #define CCALL_HANDLE_COMPLEXARG \
 275   /* Pass complex by value in 2 or 4 GPRs. */
 276
 277 /* PPC/SPE has a softfp ABI. */
 278 #define CCALL_HANDLE_REGARG \
 279   if (n > 1) {  /* Doesn't fit in a single GPR? */ \
 280     lua_assert(n == 2 || n == 4);  /* int64_t, double or complex (float). */ \
 281     if (n == 2) \
 282       ngpr = (ngpr + 1u) & ~1u;  /* Only align 64 bit value to regpair. */ \
 283     else if (ngpr + n > maxgpr) \
 284       ngpr = maxgpr;  /* Prevent reordering. */ \
 285   } \
 286   if (ngpr + n <= maxgpr) { \
 287     dp = &cc->gpr[ngpr]; \
 288     ngpr += n; \
 289     goto done; \
 290   }
 291
 292 #else
 293 #error "missing calling convention definitions for this architecture"
 294 #endif
 295
 296 #ifndef CCALL_HANDLE_STRUCTRET2
 297 #define CCALL_HANDLE_STRUCTRET2 \
 298   memcpy(dp, sp, ctr->size);  /* Copy struct return value from GPRs. */
 299 #endif
 300
 301 /* -- x64 struct classification ------------------------------------------- */
 302
 303 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 304
 305 /* Register classes for x64 struct classification. */
 306 #define CCALL_RCL_INT   1
 307 #define CCALL_RCL_SSE   2
 308 #define CCALL_RCL_MEM   4
 309 /* NYI: classify vectors. */
 310
 311 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
 312
 313 /* Classify a C type. */
 314 static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 315 {
 316   if (ctype_isarray(ct->info)) {
 317     CType *cct = ctype_rawchild(cts, ct);
 318     CTSize eofs, esz = cct->size, asz = ct->size;
 319     for (eofs = 0; eofs < asz; eofs += esz)
 320       ccall_classify_ct(cts, cct, rcl, ofs+eofs);
 321   } else if (ctype_isstruct(ct->info)) {
 322     ccall_classify_struct(cts, ct, rcl, ofs);
 323   } else {
 324     int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
 325     lua_assert(ctype_hassize(ct->info));
 326     if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM;  /* Unaligned. */
 327     rcl[(ofs >= 8)] |= cl;
 328   }
 329 }
 330
 331 /* Recursively classify a struct based on its fields. */
 332 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 333 {
 334   if (ct->size > 16) return CCALL_RCL_MEM;  /* Too big, gets memory class. */
 335   while (ct->sib) {
 336     CTSize fofs;
 337     ct = ctype_get(cts, ct->sib);
 338     fofs = ofs+ct->size;
 339     if (ctype_isfield(ct->info))
 340       ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 341     else if (ctype_isbitfield(ct->info))
 342       rcl[(fofs >= 8)] |= CCALL_RCL_INT;  /* NYI: unaligned bitfields? */
 343     else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
 344       ccall_classify_struct(cts, ctype_child(cts, ct), rcl, fofs);
 345   }
 346   return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM);  /* Memory class? */
 347 }
 348
 349 /* Try to split up a small struct into registers. */
 350 static int ccall_struct_reg(CCallState *cc, GPRArg *dp, int *rcl)
 351 {
 352   MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
 353   uint32_t i;
 354   for (i = 0; i < 2; i++) {
 355     lua_assert(!(rcl[i] & CCALL_RCL_MEM));
 356     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 357       if (ngpr >= CCALL_NARG_GPR) return 1;  /* Register overflow. */
 358       cc->gpr[ngpr++] = dp[i];
 359     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 360       if (nfpr >= CCALL_NARG_FPR) return 1;  /* Register overflow. */
 361       cc->fpr[nfpr++].l[0] = dp[i];
 362     }
 363   }
 364   cc->ngpr = ngpr; cc->nfpr = nfpr;
 365   return 0;  /* Ok. */
 366 }
 367
 368 /* Pass a small struct argument. */
 369 static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
 370                             TValue *o, int narg)
 371 {
 372   GPRArg dp[2];
 373   dp[0] = dp[1] = 0;
 374   /* Convert to temp. struct. */
 375   lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 376   if (ccall_struct_reg(cc, dp, rcl)) {  /* Register overflow? Pass on stack. */
 377     MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
 378     if (nsp + n > CCALL_MAXSTACK) return 1;  /* Too many arguments. */
 379     cc->nsp = nsp + n;
 380     memcpy(&cc->stack[nsp], dp, n*CTSIZE_PTR);
 381   }
 382   return 0;  /* Ok. */
 383 }
 384
 385 /* Combine returned small struct. */
 386 static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
 387 {
 388   GPRArg sp[2];
 389   MSize ngpr = 0, nfpr = 0;
 390   uint32_t i;
 391   for (i = 0; i < 2; i++) {
 392     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 393       sp[i] = cc->gpr[ngpr++];
 394     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 395       sp[i] = cc->fpr[nfpr++].l[0];
 396     }
 397   }
 398   memcpy(dp, sp, sz);
 399 }
 400 #endif
 401
 402 /* -- Common C call handling ---------------------------------------------- */
 403
 404 /* Infer the destination CTypeID for a vararg argument. */
 405 static CTypeID ccall_ctid_vararg(CTState *cts, cTValue *o)
 406 {
 407   if (tvisnumber(o)) {
 408     return CTID_DOUBLE;
 409   } else if (tviscdata(o)) {
 410     CTypeID id = cdataV(o)->typeid;
 411     CType *s = ctype_get(cts, id);
 412     if (ctype_isrefarray(s->info)) {
 413       return lj_ctype_intern(cts,
 414                CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
 415     } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
 416       /* NYI: how to pass a struct by value in a vararg argument? */
 417       return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
 418     } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
 419       return CTID_DOUBLE;
 420     } else {
 421       return id;
 422     }
 423   } else if (tvisstr(o)) {
 424     return CTID_P_CCHAR;
 425   } else if (tvisbool(o)) {
 426     return CTID_BOOL;
 427   } else {
 428     return CTID_P_VOID;
 429   }
 430 }
 431
 432 /* Setup arguments for C call. */
 433 static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
 434                           CCallState *cc)
 435 {
 436   int gcsteps = 0;
 437   TValue *o, *top = L->top;
 438   CTypeID fid;
 439   CType *ctr;
 440   MSize maxgpr, ngpr = 0, nsp = 0, narg;
 441 #if CCALL_NARG_FPR
 442   MSize nfpr = 0;
 443 #endif
 444
 445   /* Clear unused regs to get some determinism in case of misdeclaration. */
 446   memset(cc->gpr, 0, sizeof(cc->gpr));
 447 #if CCALL_NUM_FPR
 448   memset(cc->fpr, 0, sizeof(cc->fpr));
 449 #endif
 450
 451 #if LJ_TARGET_X86
 452   /* x86 has several different calling conventions. */
 453   cc->resx87 = 0;
 454   switch (ctype_cconv(ct->info)) {
 455   case CTCC_FASTCALL: maxgpr = 2; break;
 456   case CTCC_THISCALL: maxgpr = 1; break;
 457   default: maxgpr = 0; break;
 458   }
 459 #else
 460   maxgpr = CCALL_NARG_GPR;
 461 #endif
 462
 463   /* Perform required setup for some result types. */
 464   ctr = ctype_rawchild(cts, ct);
 465   if (ctype_isvector(ctr->info)) {
 466     if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
 467       goto err_nyi;
 468   } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
 469     /* Preallocate cdata object and anchor it after arguments. */
 470     CTSize sz = ctr->size;
 471     GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
 472     void *dp = cdataptr(cd);
 473     setcdataV(L, L->top++, cd);
 474     if (ctype_isstruct(ctr->info)) {
 475       CCALL_HANDLE_STRUCTRET
 476     } else {
 477       CCALL_HANDLE_COMPLEXRET
 478     }
 479 #if LJ_TARGET_X86
 480   } else if (ctype_isfp(ctr->info)) {
 481     cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
 482 #endif
 483   }
 484
 485   /* Skip initial attributes. */
 486   fid = ct->sib;
 487   while (fid) {
 488     CType *ctf = ctype_get(cts, fid);
 489     if (!ctype_isattrib(ctf->info)) break;
 490     fid = ctf->sib;
 491   }
 492
 493   /* Walk through all passed arguments. */
 494   for (o = L->base+1, narg = 1; o < top; o++, narg++) {
 495     CTypeID did;
 496     CType *d;
 497     CTSize sz;
 498     MSize n, isfp = 0, isva = 0;
 499     void *dp, *rp = NULL;
 500
 501     if (fid) {  /* Get argument type from field. */
 502       CType *ctf = ctype_get(cts, fid);
 503       fid = ctf->sib;
 504       lua_assert(ctype_isfield(ctf->info));
 505       did = ctype_cid(ctf->info);
 506     } else {
 507       if (!(ct->info & CTF_VARARG))
 508         lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too many arguments. */
 509       did = ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
 510       isva = 1;
 511     }
 512     d = ctype_raw(cts, did);
 513     sz = d->size;
 514
 515     /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
 516     if (ctype_isnum(d->info)) {
 517       if (sz > 8) goto err_nyi;
 518       if ((d->info & CTF_FP))
 519         isfp = 1;
 520     } else if (ctype_isvector(d->info)) {
 521       if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
 522         isfp = 1;
 523       else
 524         goto err_nyi;
 525     } else if (ctype_isstruct(d->info)) {
 526       CCALL_HANDLE_STRUCTARG
 527     } else if (ctype_iscomplex(d->info)) {
 528       CCALL_HANDLE_COMPLEXARG
 529     } else {
 530       sz = CTSIZE_PTR;
 531     }
 532     sz = (sz + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
 533     n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
 534
 535     CCALL_HANDLE_REGARG  /* Handle register arguments. */
 536
 537     /* Otherwise pass argument on stack. */
 538     if (CCALL_ALIGN_STACKARG && !rp && (d->info & CTF_ALIGN) > CTALIGN_PTR) {
 539       MSize align = (1u << ctype_align(d->info-CTALIGN_PTR)) -1;
 540       nsp = (nsp + align) & ~align;  /* Align argument on stack. */
 541     }
 542     if (nsp + n > CCALL_MAXSTACK) {  /* Too many arguments. */
 543     err_nyi:
 544       lj_err_caller(L, LJ_ERR_FFI_NYICALL);
 545     }
 546     dp = &cc->stack[nsp];
 547     nsp += n;
 548     isva = 0;
 549
 550   done:
 551     if (rp) {  /* Pass by reference. */
 552       gcsteps++;
 553       *(void **)dp = rp;
 554       dp = rp;
 555     }
 556     lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 557     /* Extend passed integers to 32 bits at least. */
 558     if (ctype_isinteger_or_bool(d->info) && d->size < 4) {
 559       if (d->info & CTF_UNSIGNED)
 560         *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
 561                                          (uint32_t)*(uint16_t *)dp;
 562       else
 563         *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
 564                                         (int32_t)*(int16_t *)dp;
 565     }
 566 #if LJ_TARGET_X64 && LJ_ABI_WIN
 567     if (isva) {  /* Windows/x64 mirrors varargs in both register sets. */
 568       if (nfpr == ngpr)
 569         cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
 570       else
 571         cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
 572     }
 573 #else
 574     UNUSED(isva);
 575 #endif
 576 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 577     if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
 578       cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1];  /* Split complex double. */
 579       cc->fpr[nfpr-2].d[1] = 0;
 580     }
 581 #else
 582     UNUSED(isfp);
 583 #endif
 584   }
 585   if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too few arguments. */
 586
 587 #if LJ_TARGET_X64 || LJ_TARGET_PPC
 588   cc->nfpr = nfpr;  /* Required for vararg functions. */
 589 #endif
 590   cc->nsp = nsp;
 591   cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA)*CTSIZE_PTR;
 592   if (nsp > CCALL_SPS_FREE)
 593     cc->spadj += (((nsp-CCALL_SPS_FREE)*CTSIZE_PTR + 15u) & ~15u);
 594   return gcsteps;
 595 }
 596
 597 /* Get results from C call. */
 598 static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
 599                              CCallState *cc, int *ret)
 600 {
 601   CType *ctr = ctype_rawchild(cts, ct);
 602   uint8_t *sp = (uint8_t *)&cc->gpr[0];
 603   if (ctype_isvoid(ctr->info)) {
 604     *ret = 0;  /* Zero results. */
 605     return 0;  /* No additional GC step. */
 606   }
 607   *ret = 1;  /* One result. */
 608   if (ctype_isstruct(ctr->info)) {
 609     /* Return cdata object which is already on top of stack. */
 610     if (!cc->retref) {
 611       void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 612       CCALL_HANDLE_STRUCTRET2
 613     }
 614     return 1;  /* One GC step. */
 615   }
 616   if (ctype_iscomplex(ctr->info)) {
 617     /* Return cdata object which is already on top of stack. */
 618     void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 619     CCALL_HANDLE_COMPLEXRET2
 620     return 1;  /* One GC step. */
 621   }
 622   if (LJ_BE && ctype_isinteger_or_bool(ctr->info) && ctr->size < CTSIZE_PTR)
 623     sp += (CTSIZE_PTR - ctr->size);
 624 #ifdef CCALL_HANDLE_RET
 625   CCALL_HANDLE_RET
 626 #endif
 627 #if CCALL_NUM_FPR
 628   if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
 629     sp = (uint8_t *)&cc->fpr[0];
 630 #endif
 631   /* No reference types end up here, so there's no need for the CTypeID. */
 632   lua_assert(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)));
 633   if (ctype_isenum(ctr->info)) ctr = ctype_child(cts, ctr);
 634   return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
 635 }
 636
 637 /* Call C function. */
 638 int lj_ccall_func(lua_State *L, GCcdata *cd)
 639 {
 640   CTState *cts = ctype_cts(L);
 641   CType *ct = ctype_raw(cts, cd->typeid);
 642   CTSize sz = CTSIZE_PTR;
 643   if (ctype_isptr(ct->info)) {
 644     sz = ct->size;
 645     ct = ctype_rawchild(cts, ct);
 646   }
 647   if (ctype_isfunc(ct->info)) {
 648     CCallState cc;
 649     int gcsteps, ret;
 650     cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
 651     gcsteps = ccall_set_args(L, cts, ct, &cc);
 652     lj_vm_ffi_call(&cc);
 653     gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
 654 #if LJ_TARGET_X86 && LJ_ABI_WIN
 655     /* Automatically detect __stdcall and fix up C function declaration. */
 656     if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
 657       CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
 658       lj_trace_abort(G(L));
 659     }
 660 #endif
 661     while (gcsteps-- > 0)
 662       lj_gc_check(L);
 663     return ret;
 664   }
 665   return -1;  /* Not a function. */
 666 }
 667
 668 #endif