src/lj_ccall.c

   1 /*
   2 ** FFI C call handling.
   3 ** Copyright (C) 2005-2013 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #include "lj_obj.h"
   7
   8 #if LJ_HASFFI
   9
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_str.h"
  13 #include "lj_tab.h"
  14 #include "lj_ctype.h"
  15 #include "lj_cconv.h"
  16 #include "lj_cdata.h"
  17 #include "lj_ccall.h"
  18 #include "lj_trace.h"
  19
  20 /* Target-specific handling of register arguments. */
  21 #if LJ_TARGET_X86
  22 /* -- x86 calling conventions --------------------------------------------- */
  23
  24 #if LJ_ABI_WIN
  25
  26 #define CCALL_HANDLE_STRUCTRET \
  27   /* Return structs bigger than 8 by reference (on stack only). */ \
  28   cc->retref = (sz > 8); \
  29   if (cc->retref) cc->stack[nsp++] = (GPRArg)dp;
  30
  31 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  32
  33 #else
  34
  35 #define CCALL_HANDLE_STRUCTRET \
  36   cc->retref = 1;  /* Return all structs by reference (in reg or on stack). */ \
  37   if (ngpr < maxgpr) \
  38     cc->gpr[ngpr++] = (GPRArg)dp; \
  39   else \
  40     cc->stack[nsp++] = (GPRArg)dp;
  41
  42 #define CCALL_HANDLE_COMPLEXRET \
  43   /* Return complex float in GPRs and complex double by reference. */ \
  44   cc->retref = (sz > 8); \
  45   if (cc->retref) { \
  46     if (ngpr < maxgpr) \
  47       cc->gpr[ngpr++] = (GPRArg)dp; \
  48     else \
  49       cc->stack[nsp++] = (GPRArg)dp; \
  50   }
  51
  52 #endif
  53
  54 #define CCALL_HANDLE_COMPLEXRET2 \
  55   if (!cc->retref) \
  56     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  57
  58 #define CCALL_HANDLE_STRUCTARG \
  59   ngpr = maxgpr;  /* Pass all structs by value on the stack. */
  60
  61 #define CCALL_HANDLE_COMPLEXARG \
  62   isfp = 1;  /* Pass complex by value on stack. */
  63
  64 #define CCALL_HANDLE_REGARG \
  65   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
  66     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
  67       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
  68     } else if (ngpr + 1 <= maxgpr) { \
  69       dp = &cc->gpr[ngpr]; \
  70       ngpr += n; \
  71       goto done; \
  72     } \
  73   }
  74
  75 #elif LJ_TARGET_X64 && LJ_ABI_WIN
  76 /* -- Windows/x64 calling conventions ------------------------------------- */
  77
  78 #define CCALL_HANDLE_STRUCTRET \
  79   /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
  80   cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
  81   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
  82
  83 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  84
  85 #define CCALL_HANDLE_COMPLEXRET2 \
  86   if (!cc->retref) \
  87     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  88
  89 #define CCALL_HANDLE_STRUCTARG \
  90   /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
  91   if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
  92     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
  93     sz = CTSIZE_PTR;  /* Pass all other structs by reference. */ \
  94   }
  95
  96 #define CCALL_HANDLE_COMPLEXARG \
  97   /* Pass complex float in a GPR and complex double by reference. */ \
  98   if (sz != 2*sizeof(float)) { \
  99     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 100     sz = CTSIZE_PTR; \
 101   }
 102
 103 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 104 #define CCALL_HANDLE_REGARG \
 105   if (isfp) { \
 106     if (ngpr < maxgpr) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
 107   } else { \
 108     if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
 109   }
 110
 111 #elif LJ_TARGET_X64
 112 /* -- POSIX/x64 calling conventions --------------------------------------- */
 113
 114 #define CCALL_HANDLE_STRUCTRET \
 115   int rcl[2]; rcl[0] = rcl[1] = 0; \
 116   if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
 117     cc->retref = 1;  /* Return struct by reference. */ \
 118     cc->gpr[ngpr++] = (GPRArg)dp; \
 119   } else { \
 120     cc->retref = 0;  /* Return small structs in registers. */ \
 121   }
 122
 123 #define CCALL_HANDLE_STRUCTRET2 \
 124   int rcl[2]; rcl[0] = rcl[1] = 0; \
 125   ccall_classify_struct(cts, ctr, rcl, 0); \
 126   ccall_struct_ret(cc, rcl, dp, ctr->size);
 127
 128 #define CCALL_HANDLE_COMPLEXRET \
 129   /* Complex values are returned in one or two FPRs. */ \
 130   cc->retref = 0;
 131
 132 #define CCALL_HANDLE_COMPLEXRET2 \
 133   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPR. */ \
 134     *(int64_t *)dp = cc->fpr[0].l[0]; \
 135   } else {  /* Copy non-contiguous complex double from FPRs. */ \
 136     ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
 137     ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
 138   }
 139
 140 #define CCALL_HANDLE_STRUCTARG \
 141   int rcl[2]; rcl[0] = rcl[1] = 0; \
 142   if (!ccall_classify_struct(cts, d, rcl, 0)) { \
 143     cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
 144     if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
 145     nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
 146     continue; \
 147   }  /* Pass all other structs by value on stack. */
 148
 149 #define CCALL_HANDLE_COMPLEXARG \
 150   isfp = 2;  /* Pass complex in FPRs or on stack. Needs postprocessing. */
 151
 152 #define CCALL_HANDLE_REGARG \
 153   if (isfp) {  /* Try to pass argument in FPRs. */ \
 154     if (nfpr + n <= CCALL_NARG_FPR) { \
 155       dp = &cc->fpr[nfpr]; \
 156       nfpr += n; \
 157       goto done; \
 158     } \
 159   } else {  /* Try to pass argument in GPRs. */ \
 160     /* Note that reordering is explicitly allowed in the x64 ABI. */ \
 161     if (n <= 2 && ngpr + n <= maxgpr) { \
 162       dp = &cc->gpr[ngpr]; \
 163       ngpr += n; \
 164       goto done; \
 165     } \
 166   }
 167
 168 #elif LJ_TARGET_ARM
 169 /* -- ARM calling conventions --------------------------------------------- */
 170
 171 #if LJ_ABI_SOFTFP
 172
 173 #define CCALL_HANDLE_STRUCTRET \
 174   /* Return structs of size <= 4 in a GPR. */ \
 175   cc->retref = !(sz <= 4); \
 176   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 177
 178 #define CCALL_HANDLE_COMPLEXRET \
 179   cc->retref = 1;  /* Return all complex values by reference. */ \
 180   cc->gpr[ngpr++] = (GPRArg)dp;
 181
 182 #define CCALL_HANDLE_COMPLEXRET2 \
 183   UNUSED(dp); /* Nothing to do. */
 184
 185 #define CCALL_HANDLE_STRUCTARG \
 186   /* Pass all structs by value in registers and/or on the stack. */
 187
 188 #define CCALL_HANDLE_COMPLEXARG \
 189   /* Pass complex by value in 2 or 4 GPRs. */
 190
 191 #define CCALL_HANDLE_REGARG_FP1
 192 #define CCALL_HANDLE_REGARG_FP2
 193
 194 #else
 195
 196 #define CCALL_HANDLE_STRUCTRET \
 197   cc->retref = !ccall_classify_struct(cts, ctr, ct); \
 198   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 199
 200 #define CCALL_HANDLE_STRUCTRET2 \
 201   if (ccall_classify_struct(cts, ctr, ct) > 1) sp = (uint8_t *)&cc->fpr[0]; \
 202   memcpy(dp, sp, ctr->size);
 203
 204 #define CCALL_HANDLE_COMPLEXRET \
 205   if (!(ct->info & CTF_VARARG)) cc->retref = 0;  /* Return complex in FPRs. */
 206
 207 #define CCALL_HANDLE_COMPLEXRET2 \
 208   if (!(ct->info & CTF_VARARG)) memcpy(dp, &cc->fpr[0], ctr->size);
 209
 210 #define CCALL_HANDLE_STRUCTARG \
 211   isfp = (ccall_classify_struct(cts, d, ct) > 1);
 212   /* Pass all structs by value in registers and/or on the stack. */
 213
 214 #define CCALL_HANDLE_COMPLEXARG \
 215   isfp = 1;  /* Pass complex by value in FPRs or on stack. */
 216
 217 #define CCALL_HANDLE_REGARG_FP1 \
 218   if (isfp && !(ct->info & CTF_VARARG)) { \
 219     if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 220       if (nfpr + (n >> 1) <= CCALL_NARG_FPR) { \
 221         dp = &cc->fpr[nfpr]; \
 222         nfpr += (n >> 1); \
 223         goto done; \
 224       } \
 225     } else { \
 226       if (sz > 1 && fprodd != nfpr) fprodd = 0; \
 227       if (fprodd) { \
 228         if (2*nfpr+n <= 2*CCALL_NARG_FPR+1) { \
 229           dp = (void *)&cc->fpr[fprodd-1].f[1]; \
 230           nfpr += (n >> 1); \
 231           if ((n & 1)) fprodd = 0; else fprodd = nfpr-1; \
 232           goto done; \
 233         } \
 234       } else { \
 235         if (2*nfpr+n <= 2*CCALL_NARG_FPR) { \
 236           dp = (void *)&cc->fpr[nfpr]; \
 237           nfpr += (n >> 1); \
 238           if ((n & 1)) fprodd = ++nfpr; else fprodd = 0; \
 239           goto done; \
 240         } \
 241       } \
 242     } \
 243     fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
 244   } else {
 245
 246 #define CCALL_HANDLE_REGARG_FP2 }
 247
 248 #endif
 249
 250 #define CCALL_HANDLE_REGARG \
 251   CCALL_HANDLE_REGARG_FP1 \
 252   if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 253     if (ngpr < maxgpr) \
 254       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 255   } \
 256   if (ngpr < maxgpr) { \
 257     dp = &cc->gpr[ngpr]; \
 258     if (ngpr + n > maxgpr) { \
 259       nsp += ngpr + n - maxgpr;  /* Assumes contiguous gpr/stack fields. */ \
 260       if (nsp > CCALL_MAXSTACK) goto err_nyi;  /* Too many arguments. */ \
 261       ngpr = maxgpr; \
 262     } else { \
 263       ngpr += n; \
 264     } \
 265     goto done; \
 266   } CCALL_HANDLE_REGARG_FP2
 267
 268 #define CCALL_HANDLE_RET \
 269   if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
 270
 271 #elif LJ_TARGET_PPC
 272 /* -- PPC calling conventions --------------------------------------------- */
 273
 274 #define CCALL_HANDLE_STRUCTRET \
 275   cc->retref = 1;  /* Return all structs by reference. */ \
 276   cc->gpr[ngpr++] = (GPRArg)dp;
 277
 278 #define CCALL_HANDLE_COMPLEXRET \
 279   /* Complex values are returned in 2 or 4 GPRs. */ \
 280   cc->retref = 0;
 281
 282 #define CCALL_HANDLE_COMPLEXRET2 \
 283   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 284
 285 #define CCALL_HANDLE_STRUCTARG \
 286   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 287   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 288
 289 #define CCALL_HANDLE_COMPLEXARG \
 290   /* Pass complex by value in 2 or 4 GPRs. */
 291
 292 #define CCALL_HANDLE_REGARG \
 293   if (isfp) {  /* Try to pass argument in FPRs. */ \
 294     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 295       dp = &cc->fpr[nfpr]; \
 296       nfpr += 1; \
 297       d = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 298       goto done; \
 299     } \
 300   } else {  /* Try to pass argument in GPRs. */ \
 301     if (n > 1) { \
 302       lua_assert(n == 2 || n == 4);  /* int64_t or complex (float). */ \
 303       if (ctype_isinteger(d->info)) \
 304         ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 305       else if (ngpr + n > maxgpr) \
 306         ngpr = maxgpr;  /* Prevent reordering. */ \
 307     } \
 308     if (ngpr + n <= maxgpr) { \
 309       dp = &cc->gpr[ngpr]; \
 310       ngpr += n; \
 311       goto done; \
 312     } \
 313   }
 314
 315 #define CCALL_HANDLE_RET \
 316   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 317     ctr = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */
 318
 319 #elif LJ_TARGET_PPCSPE
 320 /* -- PPC/SPE calling conventions ----------------------------------------- */
 321
 322 #define CCALL_HANDLE_STRUCTRET \
 323   cc->retref = 1;  /* Return all structs by reference. */ \
 324   cc->gpr[ngpr++] = (GPRArg)dp;
 325
 326 #define CCALL_HANDLE_COMPLEXRET \
 327   /* Complex values are returned in 2 or 4 GPRs. */ \
 328   cc->retref = 0;
 329
 330 #define CCALL_HANDLE_COMPLEXRET2 \
 331   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 332
 333 #define CCALL_HANDLE_STRUCTARG \
 334   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 335   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 336
 337 #define CCALL_HANDLE_COMPLEXARG \
 338   /* Pass complex by value in 2 or 4 GPRs. */
 339
 340 /* PPC/SPE has a softfp ABI. */
 341 #define CCALL_HANDLE_REGARG \
 342   if (n > 1) {  /* Doesn't fit in a single GPR? */ \
 343     lua_assert(n == 2 || n == 4);  /* int64_t, double or complex (float). */ \
 344     if (n == 2) \
 345       ngpr = (ngpr + 1u) & ~1u;  /* Only align 64 bit value to regpair. */ \
 346     else if (ngpr + n > maxgpr) \
 347       ngpr = maxgpr;  /* Prevent reordering. */ \
 348   } \
 349   if (ngpr + n <= maxgpr) { \
 350     dp = &cc->gpr[ngpr]; \
 351     ngpr += n; \
 352     goto done; \
 353   }
 354
 355 #elif LJ_TARGET_MIPS
 356 /* -- MIPS calling conventions -------------------------------------------- */
 357
 358 #define CCALL_HANDLE_STRUCTRET \
 359   cc->retref = 1;  /* Return all structs by reference. */ \
 360   cc->gpr[ngpr++] = (GPRArg)dp;
 361
 362 #define CCALL_HANDLE_COMPLEXRET \
 363   /* Complex values are returned in 1 or 2 FPRs. */ \
 364   cc->retref = 0;
 365
 366 #define CCALL_HANDLE_COMPLEXRET2 \
 367   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 368     ((float *)dp)[0] = cc->fpr[0].f; \
 369     ((float *)dp)[1] = cc->fpr[1].f; \
 370   } else {  /* Copy complex double from FPRs. */ \
 371     ((double *)dp)[0] = cc->fpr[0].d; \
 372     ((double *)dp)[1] = cc->fpr[1].d; \
 373   }
 374
 375 #define CCALL_HANDLE_STRUCTARG \
 376   /* Pass all structs by value in registers and/or on the stack. */
 377
 378 #define CCALL_HANDLE_COMPLEXARG \
 379   /* Pass complex by value in 2 or 4 GPRs. */
 380
 381 #define CCALL_HANDLE_REGARG \
 382   if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
 383     /* Try to pass argument in FPRs. */ \
 384     dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
 385     nfpr++; ngpr += n; \
 386     goto done; \
 387   } else {  /* Try to pass argument in GPRs. */ \
 388     nfpr = CCALL_NARG_FPR; \
 389     if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
 390       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 391     if (ngpr < maxgpr) { \
 392       dp = &cc->gpr[ngpr]; \
 393       if (ngpr + n > maxgpr) { \
 394         nsp += ngpr + n - maxgpr;  /* Assumes contiguous gpr/stack fields. */ \
 395         if (nsp > CCALL_MAXSTACK) goto err_nyi;  /* Too many arguments. */ \
 396         ngpr = maxgpr; \
 397       } else { \
 398         ngpr += n; \
 399       } \
 400       goto done; \
 401     } \
 402   }
 403
 404 #define CCALL_HANDLE_RET \
 405   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 406     sp = (uint8_t *)&cc->fpr[0].f;
 407
 408 #else
 409 #error "Missing calling convention definitions for this architecture"
 410 #endif
 411
 412 #ifndef CCALL_HANDLE_STRUCTRET2
 413 #define CCALL_HANDLE_STRUCTRET2 \
 414   memcpy(dp, sp, ctr->size);  /* Copy struct return value from GPRs. */
 415 #endif
 416
 417 /* -- x64 struct classification ------------------------------------------- */
 418
 419 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 420
 421 /* Register classes for x64 struct classification. */
 422 #define CCALL_RCL_INT   1
 423 #define CCALL_RCL_SSE   2
 424 #define CCALL_RCL_MEM   4
 425 /* NYI: classify vectors. */
 426
 427 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
 428
 429 /* Classify a C type. */
 430 static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 431 {
 432   if (ctype_isarray(ct->info)) {
 433     CType *cct = ctype_rawchild(cts, ct);
 434     CTSize eofs, esz = cct->size, asz = ct->size;
 435     for (eofs = 0; eofs < asz; eofs += esz)
 436       ccall_classify_ct(cts, cct, rcl, ofs+eofs);
 437   } else if (ctype_isstruct(ct->info)) {
 438     ccall_classify_struct(cts, ct, rcl, ofs);
 439   } else {
 440     int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
 441     lua_assert(ctype_hassize(ct->info));
 442     if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM;  /* Unaligned. */
 443     rcl[(ofs >= 8)] |= cl;
 444   }
 445 }
 446
 447 /* Recursively classify a struct based on its fields. */
 448 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 449 {
 450   if (ct->size > 16) return CCALL_RCL_MEM;  /* Too big, gets memory class. */
 451   while (ct->sib) {
 452     CTSize fofs;
 453     ct = ctype_get(cts, ct->sib);
 454     fofs = ofs+ct->size;
 455     if (ctype_isfield(ct->info))
 456       ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 457     else if (ctype_isbitfield(ct->info))
 458       rcl[(fofs >= 8)] |= CCALL_RCL_INT;  /* NYI: unaligned bitfields? */
 459     else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
 460       ccall_classify_struct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 461   }
 462   return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM);  /* Memory class? */
 463 }
 464
 465 /* Try to split up a small struct into registers. */
 466 static int ccall_struct_reg(CCallState *cc, GPRArg *dp, int *rcl)
 467 {
 468   MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
 469   uint32_t i;
 470   for (i = 0; i < 2; i++) {
 471     lua_assert(!(rcl[i] & CCALL_RCL_MEM));
 472     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 473       if (ngpr >= CCALL_NARG_GPR) return 1;  /* Register overflow. */
 474       cc->gpr[ngpr++] = dp[i];
 475     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 476       if (nfpr >= CCALL_NARG_FPR) return 1;  /* Register overflow. */
 477       cc->fpr[nfpr++].l[0] = dp[i];
 478     }
 479   }
 480   cc->ngpr = ngpr; cc->nfpr = nfpr;
 481   return 0;  /* Ok. */
 482 }
 483
 484 /* Pass a small struct argument. */
 485 static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
 486                             TValue *o, int narg)
 487 {
 488   GPRArg dp[2];
 489   dp[0] = dp[1] = 0;
 490   /* Convert to temp. struct. */
 491   lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 492   if (ccall_struct_reg(cc, dp, rcl)) {  /* Register overflow? Pass on stack. */
 493     MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
 494     if (nsp + n > CCALL_MAXSTACK) return 1;  /* Too many arguments. */
 495     cc->nsp = nsp + n;
 496     memcpy(&cc->stack[nsp], dp, n*CTSIZE_PTR);
 497   }
 498   return 0;  /* Ok. */
 499 }
 500
 501 /* Combine returned small struct. */
 502 static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
 503 {
 504   GPRArg sp[2];
 505   MSize ngpr = 0, nfpr = 0;
 506   uint32_t i;
 507   for (i = 0; i < 2; i++) {
 508     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 509       sp[i] = cc->gpr[ngpr++];
 510     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 511       sp[i] = cc->fpr[nfpr++].l[0];
 512     }
 513   }
 514   memcpy(dp, sp, sz);
 515 }
 516 #endif
 517
 518 /* -- ARM hard-float ABI struct classification ---------------------------- */
 519
 520 #if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
 521
 522 /* Classify a struct based on its fields. */
 523 static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
 524 {
 525   CTSize sz = ct->size;
 526   unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
 527   if ((ctf->info & CTF_VARARG)) goto noth;
 528   while (ct->sib) {
 529     ct = ctype_get(cts, ct->sib);
 530     if (ctype_isfield(ct->info)) {
 531       CType *sct = ctype_rawchild(cts, ct);
 532       if (ctype_isfp(sct->info)) {
 533         r |= sct->size;
 534         if (!isu) n++; else if (n == 0) n = 1;
 535       } else if (ctype_iscomplex(sct->info)) {
 536         r |= (sct->size >> 1);
 537         if (!isu) n += 2; else if (n < 2) n = 2;
 538       } else {
 539         goto noth;
 540       }
 541     } else if (ctype_isbitfield(ct->info)) {
 542       goto noth;
 543     } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 544       CType *sct = ctype_rawchild(cts, ct);
 545       if (sct->size > 0) {
 546         unsigned int s = ccall_classify_struct(cts, sct, ctf);
 547         if (s <= 1) goto noth;
 548         r |= (s & 255);
 549         if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
 550       }
 551     }
 552   }
 553   if ((r == 4 || r == 8) && n <= 4)
 554     return r + (n << 8);
 555 noth:  /* Not a homogeneous float/double aggregate. */
 556   return (sz <= 4);  /* Return structs of size <= 4 in a GPR. */
 557 }
 558
 559 #endif
 560
 561 /* -- Common C call handling ---------------------------------------------- */
 562
 563 /* Infer the destination CTypeID for a vararg argument. */
 564 CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
 565 {
 566   if (tvisnumber(o)) {
 567     return CTID_DOUBLE;
 568   } else if (tviscdata(o)) {
 569     CTypeID id = cdataV(o)->ctypeid;
 570     CType *s = ctype_get(cts, id);
 571     if (ctype_isrefarray(s->info)) {
 572       return lj_ctype_intern(cts,
 573                CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
 574     } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
 575       /* NYI: how to pass a struct by value in a vararg argument? */
 576       return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
 577     } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
 578       return CTID_DOUBLE;
 579     } else {
 580       return id;
 581     }
 582   } else if (tvisstr(o)) {
 583     return CTID_P_CCHAR;
 584   } else if (tvisbool(o)) {
 585     return CTID_BOOL;
 586   } else {
 587     return CTID_P_VOID;
 588   }
 589 }
 590
 591 /* Setup arguments for C call. */
 592 static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
 593                           CCallState *cc)
 594 {
 595   int gcsteps = 0;
 596   TValue *o, *top = L->top;
 597   CTypeID fid;
 598   CType *ctr;
 599   MSize maxgpr, ngpr = 0, nsp = 0, narg;
 600 #if CCALL_NARG_FPR
 601   MSize nfpr = 0;
 602 #if LJ_TARGET_ARM
 603   MSize fprodd = 0;
 604 #endif
 605 #endif
 606
 607   /* Clear unused regs to get some determinism in case of misdeclaration. */
 608   memset(cc->gpr, 0, sizeof(cc->gpr));
 609 #if CCALL_NUM_FPR
 610   memset(cc->fpr, 0, sizeof(cc->fpr));
 611 #endif
 612
 613 #if LJ_TARGET_X86
 614   /* x86 has several different calling conventions. */
 615   cc->resx87 = 0;
 616   switch (ctype_cconv(ct->info)) {
 617   case CTCC_FASTCALL: maxgpr = 2; break;
 618   case CTCC_THISCALL: maxgpr = 1; break;
 619   default: maxgpr = 0; break;
 620   }
 621 #else
 622   maxgpr = CCALL_NARG_GPR;
 623 #endif
 624
 625   /* Perform required setup for some result types. */
 626   ctr = ctype_rawchild(cts, ct);
 627   if (ctype_isvector(ctr->info)) {
 628     if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
 629       goto err_nyi;
 630   } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
 631     /* Preallocate cdata object and anchor it after arguments. */
 632     CTSize sz = ctr->size;
 633     GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
 634     void *dp = cdataptr(cd);
 635     setcdataV(L, L->top++, cd);
 636     if (ctype_isstruct(ctr->info)) {
 637       CCALL_HANDLE_STRUCTRET
 638     } else {
 639       CCALL_HANDLE_COMPLEXRET
 640     }
 641 #if LJ_TARGET_X86
 642   } else if (ctype_isfp(ctr->info)) {
 643     cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
 644 #endif
 645   }
 646
 647   /* Skip initial attributes. */
 648   fid = ct->sib;
 649   while (fid) {
 650     CType *ctf = ctype_get(cts, fid);
 651     if (!ctype_isattrib(ctf->info)) break;
 652     fid = ctf->sib;
 653   }
 654
 655   /* Walk through all passed arguments. */
 656   for (o = L->base+1, narg = 1; o < top; o++, narg++) {
 657     CTypeID did;
 658     CType *d;
 659     CTSize sz;
 660     MSize n, isfp = 0, isva = 0;
 661     void *dp, *rp = NULL;
 662
 663     if (fid) {  /* Get argument type from field. */
 664       CType *ctf = ctype_get(cts, fid);
 665       fid = ctf->sib;
 666       lua_assert(ctype_isfield(ctf->info));
 667       did = ctype_cid(ctf->info);
 668     } else {
 669       if (!(ct->info & CTF_VARARG))
 670         lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too many arguments. */
 671       did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
 672       isva = 1;
 673     }
 674     d = ctype_raw(cts, did);
 675     sz = d->size;
 676
 677     /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
 678     if (ctype_isnum(d->info)) {
 679       if (sz > 8) goto err_nyi;
 680       if ((d->info & CTF_FP))
 681         isfp = 1;
 682     } else if (ctype_isvector(d->info)) {
 683       if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
 684         isfp = 1;
 685       else
 686         goto err_nyi;
 687     } else if (ctype_isstruct(d->info)) {
 688       CCALL_HANDLE_STRUCTARG
 689     } else if (ctype_iscomplex(d->info)) {
 690       CCALL_HANDLE_COMPLEXARG
 691     } else {
 692       sz = CTSIZE_PTR;
 693     }
 694     sz = (sz + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
 695     n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
 696
 697     CCALL_HANDLE_REGARG  /* Handle register arguments. */
 698
 699     /* Otherwise pass argument on stack. */
 700     if (CCALL_ALIGN_STACKARG && !rp && (d->info & CTF_ALIGN) > CTALIGN_PTR) {
 701       MSize align = (1u << ctype_align(d->info-CTALIGN_PTR)) -1;
 702       nsp = (nsp + align) & ~align;  /* Align argument on stack. */
 703     }
 704     if (nsp + n > CCALL_MAXSTACK) {  /* Too many arguments. */
 705     err_nyi:
 706       lj_err_caller(L, LJ_ERR_FFI_NYICALL);
 707     }
 708     dp = &cc->stack[nsp];
 709     nsp += n;
 710     isva = 0;
 711
 712   done:
 713     if (rp) {  /* Pass by reference. */
 714       gcsteps++;
 715       *(void **)dp = rp;
 716       dp = rp;
 717     }
 718     lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 719     /* Extend passed integers to 32 bits at least. */
 720     if (ctype_isinteger_or_bool(d->info) && d->size < 4) {
 721       if (d->info & CTF_UNSIGNED)
 722         *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
 723                                          (uint32_t)*(uint16_t *)dp;
 724       else
 725         *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
 726                                         (int32_t)*(int16_t *)dp;
 727     }
 728 #if LJ_TARGET_X64 && LJ_ABI_WIN
 729     if (isva) {  /* Windows/x64 mirrors varargs in both register sets. */
 730       if (nfpr == ngpr)
 731         cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
 732       else
 733         cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
 734     }
 735 #else
 736     UNUSED(isva);
 737 #endif
 738 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 739     if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
 740       cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1];  /* Split complex double. */
 741       cc->fpr[nfpr-2].d[1] = 0;
 742     }
 743 #else
 744     UNUSED(isfp);
 745 #endif
 746   }
 747   if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too few arguments. */
 748
 749 #if LJ_TARGET_X64 || LJ_TARGET_PPC
 750   cc->nfpr = nfpr;  /* Required for vararg functions. */
 751 #endif
 752   cc->nsp = nsp;
 753   cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA)*CTSIZE_PTR;
 754   if (nsp > CCALL_SPS_FREE)
 755     cc->spadj += (((nsp-CCALL_SPS_FREE)*CTSIZE_PTR + 15u) & ~15u);
 756   return gcsteps;
 757 }
 758
 759 /* Get results from C call. */
 760 static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
 761                              CCallState *cc, int *ret)
 762 {
 763   CType *ctr = ctype_rawchild(cts, ct);
 764   uint8_t *sp = (uint8_t *)&cc->gpr[0];
 765   if (ctype_isvoid(ctr->info)) {
 766     *ret = 0;  /* Zero results. */
 767     return 0;  /* No additional GC step. */
 768   }
 769   *ret = 1;  /* One result. */
 770   if (ctype_isstruct(ctr->info)) {
 771     /* Return cdata object which is already on top of stack. */
 772     if (!cc->retref) {
 773       void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 774       CCALL_HANDLE_STRUCTRET2
 775     }
 776     return 1;  /* One GC step. */
 777   }
 778   if (ctype_iscomplex(ctr->info)) {
 779     /* Return cdata object which is already on top of stack. */
 780     void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 781     CCALL_HANDLE_COMPLEXRET2
 782     return 1;  /* One GC step. */
 783   }
 784   if (LJ_BE && ctype_isinteger_or_bool(ctr->info) && ctr->size < CTSIZE_PTR)
 785     sp += (CTSIZE_PTR - ctr->size);
 786 #if CCALL_NUM_FPR
 787   if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
 788     sp = (uint8_t *)&cc->fpr[0];
 789 #endif
 790 #ifdef CCALL_HANDLE_RET
 791   CCALL_HANDLE_RET
 792 #endif
 793   /* No reference types end up here, so there's no need for the CTypeID. */
 794   lua_assert(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)));
 795   return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
 796 }
 797
 798 /* Call C function. */
 799 int lj_ccall_func(lua_State *L, GCcdata *cd)
 800 {
 801   CTState *cts = ctype_cts(L);
 802   CType *ct = ctype_raw(cts, cd->ctypeid);
 803   CTSize sz = CTSIZE_PTR;
 804   if (ctype_isptr(ct->info)) {
 805     sz = ct->size;
 806     ct = ctype_rawchild(cts, ct);
 807   }
 808   if (ctype_isfunc(ct->info)) {
 809     CCallState cc;
 810     int gcsteps, ret;
 811     cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
 812     gcsteps = ccall_set_args(L, cts, ct, &cc);
 813     ct = (CType *)((intptr_t)ct-(intptr_t)cts->tab);
 814     cts->cb.slot = ~0u;
 815     lj_vm_ffi_call(&cc);
 816     if (cts->cb.slot != ~0u) {  /* Blacklist function that called a callback. */
 817       TValue tv;
 818       setlightudV(&tv, (void *)cc.func);
 819       setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
 820     }
 821     ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab);  /* May be reallocated. */
 822     gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
 823 #if LJ_TARGET_X86 && LJ_ABI_WIN
 824     /* Automatically detect __stdcall and fix up C function declaration. */
 825     if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
 826       CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
 827       lj_trace_abort(G(L));
 828     }
 829 #endif
 830     while (gcsteps-- > 0)
 831       lj_gc_check(L);
 832     return ret;
 833   }
 834   return -1;  /* Not a function. */
 835 }
 836
 837 #endif