src/lj_ccall.c

   1 /*
   2 ** FFI C call handling.
   3 ** Copyright (C) 2005-2012 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #include "lj_obj.h"
   7
   8 #if LJ_HASFFI
   9
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_str.h"
  13 #include "lj_tab.h"
  14 #include "lj_ctype.h"
  15 #include "lj_cconv.h"
  16 #include "lj_cdata.h"
  17 #include "lj_ccall.h"
  18 #include "lj_trace.h"
  19
  20 /* Target-specific handling of register arguments. */
  21 #if LJ_TARGET_X86
  22 /* -- x86 calling conventions --------------------------------------------- */
  23
  24 #if LJ_ABI_WIN
  25
  26 #define CCALL_HANDLE_STRUCTRET \
  27   /* Return structs bigger than 8 by reference (on stack only). */ \
  28   cc->retref = (sz > 8); \
  29   if (cc->retref) cc->stack[nsp++] = (GPRArg)dp;
  30
  31 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  32
  33 #else
  34
  35 #define CCALL_HANDLE_STRUCTRET \
  36   cc->retref = 1;  /* Return all structs by reference (in reg or on stack). */ \
  37   if (ngpr < maxgpr) \
  38     cc->gpr[ngpr++] = (GPRArg)dp; \
  39   else \
  40     cc->stack[nsp++] = (GPRArg)dp;
  41
  42 #define CCALL_HANDLE_COMPLEXRET \
  43   /* Return complex float in GPRs and complex double by reference. */ \
  44   cc->retref = (sz > 8); \
  45   if (cc->retref) { \
  46     if (ngpr < maxgpr) \
  47       cc->gpr[ngpr++] = (GPRArg)dp; \
  48     else \
  49       cc->stack[nsp++] = (GPRArg)dp; \
  50   }
  51
  52 #endif
  53
  54 #define CCALL_HANDLE_COMPLEXRET2 \
  55   if (!cc->retref) \
  56     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  57
  58 #define CCALL_HANDLE_STRUCTARG \
  59   ngpr = maxgpr;  /* Pass all structs by value on the stack. */
  60
  61 #define CCALL_HANDLE_COMPLEXARG \
  62   isfp = 1;  /* Pass complex by value on stack. */
  63
  64 #define CCALL_HANDLE_REGARG \
  65   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
  66     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
  67       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
  68     } else if (ngpr + 1 <= maxgpr) { \
  69       dp = &cc->gpr[ngpr]; \
  70       ngpr += n; \
  71       goto done; \
  72     } \
  73   }
  74
  75 #elif LJ_TARGET_X64 && LJ_ABI_WIN
  76 /* -- Windows/x64 calling conventions ------------------------------------- */
  77
  78 #define CCALL_HANDLE_STRUCTRET \
  79   /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
  80   cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
  81   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
  82
  83 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  84
  85 #define CCALL_HANDLE_COMPLEXRET2 \
  86   if (!cc->retref) \
  87     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  88
  89 #define CCALL_HANDLE_STRUCTARG \
  90   /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
  91   if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
  92     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
  93     sz = CTSIZE_PTR;  /* Pass all other structs by reference. */ \
  94   }
  95
  96 #define CCALL_HANDLE_COMPLEXARG \
  97   /* Pass complex float in a GPR and complex double by reference. */ \
  98   if (sz != 2*sizeof(float)) { \
  99     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 100     sz = CTSIZE_PTR; \
 101   }
 102
 103 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 104 #define CCALL_HANDLE_REGARG \
 105   if (isfp) { \
 106     if (ngpr < 4) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
 107   } else { \
 108     if (ngpr < 4) { dp = &cc->gpr[ngpr++]; goto done; } \
 109   }
 110
 111 #elif LJ_TARGET_X64
 112 /* -- POSIX/x64 calling conventions --------------------------------------- */
 113
 114 #define CCALL_HANDLE_STRUCTRET \
 115   int rcl[2]; rcl[0] = rcl[1] = 0; \
 116   if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
 117     cc->retref = 1;  /* Return struct by reference. */ \
 118     cc->gpr[ngpr++] = (GPRArg)dp; \
 119   } else { \
 120     cc->retref = 0;  /* Return small structs in registers. */ \
 121   }
 122
 123 #define CCALL_HANDLE_STRUCTRET2 \
 124   int rcl[2]; rcl[0] = rcl[1] = 0; \
 125   ccall_classify_struct(cts, ctr, rcl, 0); \
 126   ccall_struct_ret(cc, rcl, dp, ctr->size);
 127
 128 #define CCALL_HANDLE_COMPLEXRET \
 129   /* Complex values are returned in one or two FPRs. */ \
 130   cc->retref = 0;
 131
 132 #define CCALL_HANDLE_COMPLEXRET2 \
 133   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPR. */ \
 134     *(int64_t *)dp = cc->fpr[0].l[0]; \
 135   } else {  /* Copy non-contiguous complex double from FPRs. */ \
 136     ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
 137     ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
 138   }
 139
 140 #define CCALL_HANDLE_STRUCTARG \
 141   int rcl[2]; rcl[0] = rcl[1] = 0; \
 142   if (!ccall_classify_struct(cts, d, rcl, 0)) { \
 143     cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
 144     if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
 145     nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
 146     continue; \
 147   }  /* Pass all other structs by value on stack. */
 148
 149 #define CCALL_HANDLE_COMPLEXARG \
 150   isfp = 2;  /* Pass complex in FPRs or on stack. Needs postprocessing. */
 151
 152 #define CCALL_HANDLE_REGARG \
 153   if (isfp) {  /* Try to pass argument in FPRs. */ \
 154     if (nfpr + n <= CCALL_NARG_FPR) { \
 155       dp = &cc->fpr[nfpr]; \
 156       nfpr += n; \
 157       goto done; \
 158     } \
 159   } else {  /* Try to pass argument in GPRs. */ \
 160     /* Note that reordering is explicitly allowed in the x64 ABI. */ \
 161     if (n <= 2 && ngpr + n <= maxgpr) { \
 162       dp = &cc->gpr[ngpr]; \
 163       ngpr += n; \
 164       goto done; \
 165     } \
 166   }
 167
 168 #elif LJ_TARGET_ARM
 169 /* -- ARM calling conventions --------------------------------------------- */
 170
 171 #define CCALL_HANDLE_STRUCTRET \
 172   /* Return structs of size <= 4 in a GPR. */ \
 173   cc->retref = !(sz <= 4); \
 174   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 175
 176 #define CCALL_HANDLE_COMPLEXRET \
 177   cc->retref = 1;  /* Return all complex values by reference. */ \
 178   cc->gpr[ngpr++] = (GPRArg)dp;
 179
 180 #define CCALL_HANDLE_COMPLEXRET2 \
 181   UNUSED(dp); /* Nothing to do. */
 182
 183 #define CCALL_HANDLE_STRUCTARG \
 184   /* Pass all structs by value in registers and/or on the stack. */
 185
 186 #define CCALL_HANDLE_COMPLEXARG \
 187   /* Pass complex by value in 2 or 4 GPRs. */
 188
 189 /* ARM has a softfp ABI. */
 190 #define CCALL_HANDLE_REGARG \
 191   if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 192     if (ngpr < maxgpr) \
 193       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 194     else \
 195       nsp = (nsp + 1u) & ~1u;  /* Align argument on stack. */ \
 196   } \
 197   if (ngpr < maxgpr) { \
 198     dp = &cc->gpr[ngpr]; \
 199     if (ngpr + n > maxgpr) { \
 200       nsp += ngpr + n - maxgpr;  /* Assumes contiguous gpr/stack fields. */ \
 201       if (nsp > CCALL_MAXSTACK) goto err_nyi;  /* Too many arguments. */ \
 202       ngpr = maxgpr; \
 203     } else { \
 204       ngpr += n; \
 205     } \
 206     goto done; \
 207   }
 208
 209 #elif LJ_TARGET_PPC
 210 /* -- PPC calling conventions --------------------------------------------- */
 211
 212 #define CCALL_HANDLE_STRUCTRET \
 213   cc->retref = 1;  /* Return all structs by reference. */ \
 214   cc->gpr[ngpr++] = (GPRArg)dp;
 215
 216 #define CCALL_HANDLE_COMPLEXRET \
 217   /* Complex values are returned in 2 or 4 GPRs. */ \
 218   cc->retref = 0;
 219
 220 #define CCALL_HANDLE_COMPLEXRET2 \
 221   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 222
 223 #define CCALL_HANDLE_STRUCTARG \
 224   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 225   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 226
 227 #define CCALL_HANDLE_COMPLEXARG \
 228   /* Pass complex by value in 2 or 4 GPRs. */
 229
 230 #define CCALL_HANDLE_REGARG \
 231   if (isfp) {  /* Try to pass argument in FPRs. */ \
 232     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 233       dp = &cc->fpr[nfpr]; \
 234       nfpr += 1; \
 235       d = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 236       goto done; \
 237     } \
 238   } else {  /* Try to pass argument in GPRs. */ \
 239     if (n > 1) { \
 240       lua_assert(n == 2 || n == 4);  /* int64_t or complex (float). */ \
 241       if (ctype_isinteger(d->info)) \
 242         ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 243       else if (ngpr + n > maxgpr) \
 244         ngpr = maxgpr;  /* Prevent reordering. */ \
 245     } \
 246     if (ngpr + n <= maxgpr) { \
 247       dp = &cc->gpr[ngpr]; \
 248       ngpr += n; \
 249       goto done; \
 250     } \
 251   }
 252
 253 #define CCALL_HANDLE_RET \
 254   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 255     ctr = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */
 256
 257 #elif LJ_TARGET_PPCSPE
 258 /* -- PPC/SPE calling conventions ----------------------------------------- */
 259
 260 #define CCALL_HANDLE_STRUCTRET \
 261   cc->retref = 1;  /* Return all structs by reference. */ \
 262   cc->gpr[ngpr++] = (GPRArg)dp;
 263
 264 #define CCALL_HANDLE_COMPLEXRET \
 265   /* Complex values are returned in 2 or 4 GPRs. */ \
 266   cc->retref = 0;
 267
 268 #define CCALL_HANDLE_COMPLEXRET2 \
 269   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 270
 271 #define CCALL_HANDLE_STRUCTARG \
 272   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 273   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 274
 275 #define CCALL_HANDLE_COMPLEXARG \
 276   /* Pass complex by value in 2 or 4 GPRs. */
 277
 278 /* PPC/SPE has a softfp ABI. */
 279 #define CCALL_HANDLE_REGARG \
 280   if (n > 1) {  /* Doesn't fit in a single GPR? */ \
 281     lua_assert(n == 2 || n == 4);  /* int64_t, double or complex (float). */ \
 282     if (n == 2) \
 283       ngpr = (ngpr + 1u) & ~1u;  /* Only align 64 bit value to regpair. */ \
 284     else if (ngpr + n > maxgpr) \
 285       ngpr = maxgpr;  /* Prevent reordering. */ \
 286   } \
 287   if (ngpr + n <= maxgpr) { \
 288     dp = &cc->gpr[ngpr]; \
 289     ngpr += n; \
 290     goto done; \
 291   }
 292
 293 #elif LJ_TARGET_MIPS
 294 /* -- MIPS calling conventions -------------------------------------------- */
 295
 296 #define CCALL_HANDLE_STRUCTRET \
 297   cc->retref = 1;  /* Return all structs by reference. */ \
 298   cc->gpr[ngpr++] = (GPRArg)dp;
 299
 300 #define CCALL_HANDLE_COMPLEXRET \
 301   /* Complex values are returned in 1 or 2 FPRs. */ \
 302   cc->retref = 0;
 303
 304 #define CCALL_HANDLE_COMPLEXRET2 \
 305   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 306     ((float *)dp)[0] = cc->fpr[0].f; \
 307     ((float *)dp)[1] = cc->fpr[1].f; \
 308   } else {  /* Copy complex double from FPRs. */ \
 309     ((double *)dp)[0] = cc->fpr[0].d; \
 310     ((double *)dp)[1] = cc->fpr[1].d; \
 311   }
 312
 313 #define CCALL_HANDLE_STRUCTARG \
 314   /* Pass all structs by value in registers and/or on the stack. */
 315
 316 #define CCALL_HANDLE_COMPLEXARG \
 317   /* Pass complex by value in 2 or 4 GPRs. */
 318
 319 #define CCALL_HANDLE_REGARG \
 320   if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
 321     /* Try to pass argument in FPRs. */ \
 322     dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
 323     nfpr++; ngpr += n; \
 324     goto done; \
 325   } else {  /* Try to pass argument in GPRs. */ \
 326     nfpr = CCALL_NARG_FPR; \
 327     if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
 328       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 329     if (ngpr < maxgpr) { \
 330       dp = &cc->gpr[ngpr]; \
 331       if (ngpr + n > maxgpr) { \
 332         nsp += ngpr + n - maxgpr;  /* Assumes contiguous gpr/stack fields. */ \
 333         if (nsp > CCALL_MAXSTACK) goto err_nyi;  /* Too many arguments. */ \
 334         ngpr = maxgpr; \
 335       } else { \
 336         ngpr += n; \
 337       } \
 338       goto done; \
 339     } \
 340   }
 341
 342 #define CCALL_HANDLE_RET \
 343   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 344     sp = (uint8_t *)&cc->fpr[0].f;
 345
 346 #else
 347 #error "Missing calling convention definitions for this architecture"
 348 #endif
 349
 350 #ifndef CCALL_HANDLE_STRUCTRET2
 351 #define CCALL_HANDLE_STRUCTRET2 \
 352   memcpy(dp, sp, ctr->size);  /* Copy struct return value from GPRs. */
 353 #endif
 354
 355 /* -- x64 struct classification ------------------------------------------- */
 356
 357 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 358
 359 /* Register classes for x64 struct classification. */
 360 #define CCALL_RCL_INT   1
 361 #define CCALL_RCL_SSE   2
 362 #define CCALL_RCL_MEM   4
 363 /* NYI: classify vectors. */
 364
 365 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
 366
 367 /* Classify a C type. */
 368 static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 369 {
 370   if (ctype_isarray(ct->info)) {
 371     CType *cct = ctype_rawchild(cts, ct);
 372     CTSize eofs, esz = cct->size, asz = ct->size;
 373     for (eofs = 0; eofs < asz; eofs += esz)
 374       ccall_classify_ct(cts, cct, rcl, ofs+eofs);
 375   } else if (ctype_isstruct(ct->info)) {
 376     ccall_classify_struct(cts, ct, rcl, ofs);
 377   } else {
 378     int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
 379     lua_assert(ctype_hassize(ct->info));
 380     if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM;  /* Unaligned. */
 381     rcl[(ofs >= 8)] |= cl;
 382   }
 383 }
 384
 385 /* Recursively classify a struct based on its fields. */
 386 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 387 {
 388   if (ct->size > 16) return CCALL_RCL_MEM;  /* Too big, gets memory class. */
 389   while (ct->sib) {
 390     CTSize fofs;
 391     ct = ctype_get(cts, ct->sib);
 392     fofs = ofs+ct->size;
 393     if (ctype_isfield(ct->info))
 394       ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 395     else if (ctype_isbitfield(ct->info))
 396       rcl[(fofs >= 8)] |= CCALL_RCL_INT;  /* NYI: unaligned bitfields? */
 397     else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
 398       ccall_classify_struct(cts, ctype_child(cts, ct), rcl, fofs);
 399   }
 400   return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM);  /* Memory class? */
 401 }
 402
 403 /* Try to split up a small struct into registers. */
 404 static int ccall_struct_reg(CCallState *cc, GPRArg *dp, int *rcl)
 405 {
 406   MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
 407   uint32_t i;
 408   for (i = 0; i < 2; i++) {
 409     lua_assert(!(rcl[i] & CCALL_RCL_MEM));
 410     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 411       if (ngpr >= CCALL_NARG_GPR) return 1;  /* Register overflow. */
 412       cc->gpr[ngpr++] = dp[i];
 413     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 414       if (nfpr >= CCALL_NARG_FPR) return 1;  /* Register overflow. */
 415       cc->fpr[nfpr++].l[0] = dp[i];
 416     }
 417   }
 418   cc->ngpr = ngpr; cc->nfpr = nfpr;
 419   return 0;  /* Ok. */
 420 }
 421
 422 /* Pass a small struct argument. */
 423 static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
 424                             TValue *o, int narg)
 425 {
 426   GPRArg dp[2];
 427   dp[0] = dp[1] = 0;
 428   /* Convert to temp. struct. */
 429   lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 430   if (ccall_struct_reg(cc, dp, rcl)) {  /* Register overflow? Pass on stack. */
 431     MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
 432     if (nsp + n > CCALL_MAXSTACK) return 1;  /* Too many arguments. */
 433     cc->nsp = nsp + n;
 434     memcpy(&cc->stack[nsp], dp, n*CTSIZE_PTR);
 435   }
 436   return 0;  /* Ok. */
 437 }
 438
 439 /* Combine returned small struct. */
 440 static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
 441 {
 442   GPRArg sp[2];
 443   MSize ngpr = 0, nfpr = 0;
 444   uint32_t i;
 445   for (i = 0; i < 2; i++) {
 446     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 447       sp[i] = cc->gpr[ngpr++];
 448     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 449       sp[i] = cc->fpr[nfpr++].l[0];
 450     }
 451   }
 452   memcpy(dp, sp, sz);
 453 }
 454 #endif
 455
 456 /* -- Common C call handling ---------------------------------------------- */
 457
 458 /* Infer the destination CTypeID for a vararg argument. */
 459 CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
 460 {
 461   if (tvisnumber(o)) {
 462     return CTID_DOUBLE;
 463   } else if (tviscdata(o)) {
 464     CTypeID id = cdataV(o)->typeid;
 465     CType *s = ctype_get(cts, id);
 466     if (ctype_isrefarray(s->info)) {
 467       return lj_ctype_intern(cts,
 468                CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
 469     } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
 470       /* NYI: how to pass a struct by value in a vararg argument? */
 471       return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
 472     } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
 473       return CTID_DOUBLE;
 474     } else {
 475       return id;
 476     }
 477   } else if (tvisstr(o)) {
 478     return CTID_P_CCHAR;
 479   } else if (tvisbool(o)) {
 480     return CTID_BOOL;
 481   } else {
 482     return CTID_P_VOID;
 483   }
 484 }
 485
 486 /* Setup arguments for C call. */
 487 static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
 488                           CCallState *cc)
 489 {
 490   int gcsteps = 0;
 491   TValue *o, *top = L->top;
 492   CTypeID fid;
 493   CType *ctr;
 494   MSize maxgpr, ngpr = 0, nsp = 0, narg;
 495 #if CCALL_NARG_FPR
 496   MSize nfpr = 0;
 497 #endif
 498
 499   /* Clear unused regs to get some determinism in case of misdeclaration. */
 500   memset(cc->gpr, 0, sizeof(cc->gpr));
 501 #if CCALL_NUM_FPR
 502   memset(cc->fpr, 0, sizeof(cc->fpr));
 503 #endif
 504
 505 #if LJ_TARGET_X86
 506   /* x86 has several different calling conventions. */
 507   cc->resx87 = 0;
 508   switch (ctype_cconv(ct->info)) {
 509   case CTCC_FASTCALL: maxgpr = 2; break;
 510   case CTCC_THISCALL: maxgpr = 1; break;
 511   default: maxgpr = 0; break;
 512   }
 513 #else
 514   maxgpr = CCALL_NARG_GPR;
 515 #endif
 516
 517   /* Perform required setup for some result types. */
 518   ctr = ctype_rawchild(cts, ct);
 519   if (ctype_isvector(ctr->info)) {
 520     if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
 521       goto err_nyi;
 522   } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
 523     /* Preallocate cdata object and anchor it after arguments. */
 524     CTSize sz = ctr->size;
 525     GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
 526     void *dp = cdataptr(cd);
 527     setcdataV(L, L->top++, cd);
 528     if (ctype_isstruct(ctr->info)) {
 529       CCALL_HANDLE_STRUCTRET
 530     } else {
 531       CCALL_HANDLE_COMPLEXRET
 532     }
 533 #if LJ_TARGET_X86
 534   } else if (ctype_isfp(ctr->info)) {
 535     cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
 536 #endif
 537   }
 538
 539   /* Skip initial attributes. */
 540   fid = ct->sib;
 541   while (fid) {
 542     CType *ctf = ctype_get(cts, fid);
 543     if (!ctype_isattrib(ctf->info)) break;
 544     fid = ctf->sib;
 545   }
 546
 547   /* Walk through all passed arguments. */
 548   for (o = L->base+1, narg = 1; o < top; o++, narg++) {
 549     CTypeID did;
 550     CType *d;
 551     CTSize sz;
 552     MSize n, isfp = 0, isva = 0;
 553     void *dp, *rp = NULL;
 554
 555     if (fid) {  /* Get argument type from field. */
 556       CType *ctf = ctype_get(cts, fid);
 557       fid = ctf->sib;
 558       lua_assert(ctype_isfield(ctf->info));
 559       did = ctype_cid(ctf->info);
 560     } else {
 561       if (!(ct->info & CTF_VARARG))
 562         lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too many arguments. */
 563       did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
 564       isva = 1;
 565     }
 566     d = ctype_raw(cts, did);
 567     sz = d->size;
 568
 569     /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
 570     if (ctype_isnum(d->info)) {
 571       if (sz > 8) goto err_nyi;
 572       if ((d->info & CTF_FP))
 573         isfp = 1;
 574     } else if (ctype_isvector(d->info)) {
 575       if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
 576         isfp = 1;
 577       else
 578         goto err_nyi;
 579     } else if (ctype_isstruct(d->info)) {
 580       CCALL_HANDLE_STRUCTARG
 581     } else if (ctype_iscomplex(d->info)) {
 582       CCALL_HANDLE_COMPLEXARG
 583     } else {
 584       sz = CTSIZE_PTR;
 585     }
 586     sz = (sz + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
 587     n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
 588
 589     CCALL_HANDLE_REGARG  /* Handle register arguments. */
 590
 591     /* Otherwise pass argument on stack. */
 592     if (CCALL_ALIGN_STACKARG && !rp && (d->info & CTF_ALIGN) > CTALIGN_PTR) {
 593       MSize align = (1u << ctype_align(d->info-CTALIGN_PTR)) -1;
 594       nsp = (nsp + align) & ~align;  /* Align argument on stack. */
 595     }
 596     if (nsp + n > CCALL_MAXSTACK) {  /* Too many arguments. */
 597     err_nyi:
 598       lj_err_caller(L, LJ_ERR_FFI_NYICALL);
 599     }
 600     dp = &cc->stack[nsp];
 601     nsp += n;
 602     isva = 0;
 603
 604   done:
 605     if (rp) {  /* Pass by reference. */
 606       gcsteps++;
 607       *(void **)dp = rp;
 608       dp = rp;
 609     }
 610     lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 611     /* Extend passed integers to 32 bits at least. */
 612     if (ctype_isinteger_or_bool(d->info) && d->size < 4) {
 613       if (d->info & CTF_UNSIGNED)
 614         *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
 615                                          (uint32_t)*(uint16_t *)dp;
 616       else
 617         *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
 618                                         (int32_t)*(int16_t *)dp;
 619     }
 620 #if LJ_TARGET_X64 && LJ_ABI_WIN
 621     if (isva) {  /* Windows/x64 mirrors varargs in both register sets. */
 622       if (nfpr == ngpr)
 623         cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
 624       else
 625         cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
 626     }
 627 #else
 628     UNUSED(isva);
 629 #endif
 630 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 631     if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
 632       cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1];  /* Split complex double. */
 633       cc->fpr[nfpr-2].d[1] = 0;
 634     }
 635 #else
 636     UNUSED(isfp);
 637 #endif
 638   }
 639   if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too few arguments. */
 640
 641 #if LJ_TARGET_X64 || LJ_TARGET_PPC
 642   cc->nfpr = nfpr;  /* Required for vararg functions. */
 643 #endif
 644   cc->nsp = nsp;
 645   cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA)*CTSIZE_PTR;
 646   if (nsp > CCALL_SPS_FREE)
 647     cc->spadj += (((nsp-CCALL_SPS_FREE)*CTSIZE_PTR + 15u) & ~15u);
 648   return gcsteps;
 649 }
 650
 651 /* Get results from C call. */
 652 static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
 653                              CCallState *cc, int *ret)
 654 {
 655   CType *ctr = ctype_rawchild(cts, ct);
 656   uint8_t *sp = (uint8_t *)&cc->gpr[0];
 657   if (ctype_isvoid(ctr->info)) {
 658     *ret = 0;  /* Zero results. */
 659     return 0;  /* No additional GC step. */
 660   }
 661   *ret = 1;  /* One result. */
 662   if (ctype_isstruct(ctr->info)) {
 663     /* Return cdata object which is already on top of stack. */
 664     if (!cc->retref) {
 665       void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 666       CCALL_HANDLE_STRUCTRET2
 667     }
 668     return 1;  /* One GC step. */
 669   }
 670   if (ctype_iscomplex(ctr->info)) {
 671     /* Return cdata object which is already on top of stack. */
 672     void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 673     CCALL_HANDLE_COMPLEXRET2
 674     return 1;  /* One GC step. */
 675   }
 676   if (LJ_BE && ctype_isinteger_or_bool(ctr->info) && ctr->size < CTSIZE_PTR)
 677     sp += (CTSIZE_PTR - ctr->size);
 678 #if CCALL_NUM_FPR
 679   if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
 680     sp = (uint8_t *)&cc->fpr[0];
 681 #endif
 682 #ifdef CCALL_HANDLE_RET
 683   CCALL_HANDLE_RET
 684 #endif
 685   /* No reference types end up here, so there's no need for the CTypeID. */
 686   lua_assert(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)));
 687   if (ctype_isenum(ctr->info)) ctr = ctype_child(cts, ctr);
 688   return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
 689 }
 690
 691 /* Call C function. */
 692 int lj_ccall_func(lua_State *L, GCcdata *cd)
 693 {
 694   CTState *cts = ctype_cts(L);
 695   CType *ct = ctype_raw(cts, cd->typeid);
 696   CTSize sz = CTSIZE_PTR;
 697   if (ctype_isptr(ct->info)) {
 698     sz = ct->size;
 699     ct = ctype_rawchild(cts, ct);
 700   }
 701   if (ctype_isfunc(ct->info)) {
 702     CCallState cc;
 703     int gcsteps, ret;
 704     cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
 705     gcsteps = ccall_set_args(L, cts, ct, &cc);
 706     cts->cb.slot = ~0u;
 707     lj_vm_ffi_call(&cc);
 708     if (cts->cb.slot != ~0u) {  /* Blacklist function that called a callback. */
 709       TValue tv;
 710       setlightudV(&tv, (void *)cc.func);
 711       setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
 712     }
 713     gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
 714 #if LJ_TARGET_X86 && LJ_ABI_WIN
 715     /* Automatically detect __stdcall and fix up C function declaration. */
 716     if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
 717       CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
 718       lj_trace_abort(G(L));
 719     }
 720 #endif
 721     while (gcsteps-- > 0)
 722       lj_gc_check(L);
 723     return ret;
 724   }
 725   return -1;  /* Not a function. */
 726 }
 727
 728 #endif