src/lj_ccall.c

   1 /*
   2 ** FFI C call handling.
   3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #include "lj_obj.h"
   7
   8 #if LJ_HASFFI
   9
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_tab.h"
  13 #include "lj_ctype.h"
  14 #include "lj_cconv.h"
  15 #include "lj_cdata.h"
  16 #include "lj_ccall.h"
  17 #include "lj_trace.h"
  18
  19 /* Target-specific handling of register arguments. */
  20 #if LJ_TARGET_X86
  21 /* -- x86 calling conventions --------------------------------------------- */
  22
  23 #define CCALL_PUSH(arg) \
  24   *(GPRArg *)((uint8_t *)cc->stack + nsp) = (GPRArg)(arg), nsp += CTSIZE_PTR
  25
  26 #if LJ_ABI_WIN
  27
  28 #define CCALL_HANDLE_STRUCTRET \
  29   /* Return structs bigger than 8 by reference (on stack only). */ \
  30   cc->retref = (sz > 8); \
  31   if (cc->retref) CCALL_PUSH(dp);
  32
  33 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  34
  35 #else
  36
  37 #if LJ_TARGET_OSX
  38
  39 #define CCALL_HANDLE_STRUCTRET \
  40   /* Return structs of size 1, 2, 4 or 8 in registers. */ \
  41   cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
  42   if (cc->retref) { \
  43     if (ngpr < maxgpr) \
  44       cc->gpr[ngpr++] = (GPRArg)dp; \
  45     else \
  46       CCALL_PUSH(dp); \
  47   } else {  /* Struct with single FP field ends up in FPR. */ \
  48     cc->resx87 = ccall_classify_struct(cts, ctr); \
  49   }
  50
  51 #define CCALL_HANDLE_STRUCTRET2 \
  52   if (cc->resx87) sp = (uint8_t *)&cc->fpr[0]; \
  53   memcpy(dp, sp, ctr->size);
  54
  55 #else
  56
  57 #define CCALL_HANDLE_STRUCTRET \
  58   cc->retref = 1;  /* Return all structs by reference (in reg or on stack). */ \
  59   if (ngpr < maxgpr) \
  60     cc->gpr[ngpr++] = (GPRArg)dp; \
  61   else \
  62     CCALL_PUSH(dp);
  63
  64 #endif
  65
  66 #define CCALL_HANDLE_COMPLEXRET \
  67   /* Return complex float in GPRs and complex double by reference. */ \
  68   cc->retref = (sz > 8); \
  69   if (cc->retref) { \
  70     if (ngpr < maxgpr) \
  71       cc->gpr[ngpr++] = (GPRArg)dp; \
  72     else \
  73       CCALL_PUSH(dp); \
  74   }
  75
  76 #endif
  77
  78 #define CCALL_HANDLE_COMPLEXRET2 \
  79   if (!cc->retref) \
  80     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  81
  82 #define CCALL_HANDLE_STRUCTARG \
  83   ngpr = maxgpr;  /* Pass all structs by value on the stack. */
  84
  85 #define CCALL_HANDLE_COMPLEXARG \
  86   isfp = 1;  /* Pass complex by value on stack. */
  87
  88 #define CCALL_HANDLE_REGARG \
  89   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
  90     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
  91       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
  92     } else if (ngpr + 1 <= maxgpr) { \
  93       dp = &cc->gpr[ngpr]; \
  94       ngpr += n; \
  95       goto done; \
  96     } \
  97   }
  98
  99 #elif LJ_TARGET_X64 && LJ_ABI_WIN
 100 /* -- Windows/x64 calling conventions ------------------------------------- */
 101
 102 #define CCALL_HANDLE_STRUCTRET \
 103   /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
 104   cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
 105   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 106
 107 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
 108
 109 #define CCALL_HANDLE_COMPLEXRET2 \
 110   if (!cc->retref) \
 111     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
 112
 113 #define CCALL_HANDLE_STRUCTARG \
 114   /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
 115   if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
 116     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 117     sz = CTSIZE_PTR;  /* Pass all other structs by reference. */ \
 118   }
 119
 120 #define CCALL_HANDLE_COMPLEXARG \
 121   /* Pass complex float in a GPR and complex double by reference. */ \
 122   if (sz != 2*sizeof(float)) { \
 123     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 124     sz = CTSIZE_PTR; \
 125   }
 126
 127 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 128 #define CCALL_HANDLE_REGARG \
 129   if (isfp) { \
 130     if (ngpr < maxgpr) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
 131   } else { \
 132     if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
 133   }
 134
 135 #elif LJ_TARGET_X64
 136 /* -- POSIX/x64 calling conventions --------------------------------------- */
 137
 138 #define CCALL_HANDLE_STRUCTRET \
 139   int rcl[2]; rcl[0] = rcl[1] = 0; \
 140   if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
 141     cc->retref = 1;  /* Return struct by reference. */ \
 142     cc->gpr[ngpr++] = (GPRArg)dp; \
 143   } else { \
 144     cc->retref = 0;  /* Return small structs in registers. */ \
 145   }
 146
 147 #define CCALL_HANDLE_STRUCTRET2 \
 148   int rcl[2]; rcl[0] = rcl[1] = 0; \
 149   ccall_classify_struct(cts, ctr, rcl, 0); \
 150   ccall_struct_ret(cc, rcl, dp, ctr->size);
 151
 152 #define CCALL_HANDLE_COMPLEXRET \
 153   /* Complex values are returned in one or two FPRs. */ \
 154   cc->retref = 0;
 155
 156 #define CCALL_HANDLE_COMPLEXRET2 \
 157   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPR. */ \
 158     *(int64_t *)dp = cc->fpr[0].l[0]; \
 159   } else {  /* Copy non-contiguous complex double from FPRs. */ \
 160     ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
 161     ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
 162   }
 163
 164 #define CCALL_HANDLE_STRUCTARG \
 165   int rcl[2]; rcl[0] = rcl[1] = 0; \
 166   if (!ccall_classify_struct(cts, d, rcl, 0)) { \
 167     cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
 168     if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
 169     nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
 170     continue; \
 171   }  /* Pass all other structs by value on stack. */
 172
 173 #define CCALL_HANDLE_COMPLEXARG \
 174   isfp = 2;  /* Pass complex in FPRs or on stack. Needs postprocessing. */
 175
 176 #define CCALL_HANDLE_REGARG \
 177   if (isfp) {  /* Try to pass argument in FPRs. */ \
 178     int n2 = ctype_isvector(d->info) ? 1 : n; \
 179     if (nfpr + n2 <= CCALL_NARG_FPR) { \
 180       dp = &cc->fpr[nfpr]; \
 181       nfpr += n2; \
 182       goto done; \
 183     } \
 184   } else {  /* Try to pass argument in GPRs. */ \
 185     /* Note that reordering is explicitly allowed in the x64 ABI. */ \
 186     if (n <= 2 && ngpr + n <= maxgpr) { \
 187       dp = &cc->gpr[ngpr]; \
 188       ngpr += n; \
 189       goto done; \
 190     } \
 191   }
 192
 193 #elif LJ_TARGET_ARM
 194 /* -- ARM calling conventions --------------------------------------------- */
 195
 196 #if LJ_ABI_SOFTFP
 197
 198 #define CCALL_HANDLE_STRUCTRET \
 199   /* Return structs of size <= 4 in a GPR. */ \
 200   cc->retref = !(sz <= 4); \
 201   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 202
 203 #define CCALL_HANDLE_COMPLEXRET \
 204   cc->retref = 1;  /* Return all complex values by reference. */ \
 205   cc->gpr[ngpr++] = (GPRArg)dp;
 206
 207 #define CCALL_HANDLE_COMPLEXRET2 \
 208   UNUSED(dp); /* Nothing to do. */
 209
 210 #define CCALL_HANDLE_STRUCTARG \
 211   /* Pass all structs by value in registers and/or on the stack. */
 212
 213 #define CCALL_HANDLE_COMPLEXARG \
 214   /* Pass complex by value in 2 or 4 GPRs. */
 215
 216 #define CCALL_HANDLE_REGARG_FP1
 217 #define CCALL_HANDLE_REGARG_FP2
 218
 219 #else
 220
 221 #define CCALL_HANDLE_STRUCTRET \
 222   cc->retref = !ccall_classify_struct(cts, ctr, ct); \
 223   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 224
 225 #define CCALL_HANDLE_STRUCTRET2 \
 226   if (ccall_classify_struct(cts, ctr, ct) > 1) sp = (uint8_t *)&cc->fpr[0]; \
 227   memcpy(dp, sp, ctr->size);
 228
 229 #define CCALL_HANDLE_COMPLEXRET \
 230   if (!(ct->info & CTF_VARARG)) cc->retref = 0;  /* Return complex in FPRs. */
 231
 232 #define CCALL_HANDLE_COMPLEXRET2 \
 233   if (!(ct->info & CTF_VARARG)) memcpy(dp, &cc->fpr[0], ctr->size);
 234
 235 #define CCALL_HANDLE_STRUCTARG \
 236   isfp = (ccall_classify_struct(cts, d, ct) > 1);
 237   /* Pass all structs by value in registers and/or on the stack. */
 238
 239 #define CCALL_HANDLE_COMPLEXARG \
 240   isfp = 1;  /* Pass complex by value in FPRs or on stack. */
 241
 242 #define CCALL_HANDLE_REGARG_FP1 \
 243   if (isfp && !(ct->info & CTF_VARARG)) { \
 244     if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 245       if (nfpr + (n >> 1) <= CCALL_NARG_FPR) { \
 246         dp = &cc->fpr[nfpr]; \
 247         nfpr += (n >> 1); \
 248         goto done; \
 249       } \
 250     } else { \
 251       if (sz > 1 && fprodd != nfpr) fprodd = 0; \
 252       if (fprodd) { \
 253         if (2*nfpr+n <= 2*CCALL_NARG_FPR+1) { \
 254           dp = (void *)&cc->fpr[fprodd-1].f[1]; \
 255           nfpr += (n >> 1); \
 256           if ((n & 1)) fprodd = 0; else fprodd = nfpr-1; \
 257           goto done; \
 258         } \
 259       } else { \
 260         if (2*nfpr+n <= 2*CCALL_NARG_FPR) { \
 261           dp = (void *)&cc->fpr[nfpr]; \
 262           nfpr += (n >> 1); \
 263           if ((n & 1)) fprodd = ++nfpr; else fprodd = 0; \
 264           goto done; \
 265         } \
 266       } \
 267     } \
 268     fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
 269   } else {
 270
 271 #define CCALL_HANDLE_REGARG_FP2 }
 272
 273 #endif
 274
 275 #define CCALL_HANDLE_REGARG \
 276   CCALL_HANDLE_REGARG_FP1 \
 277   if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 278     if (ngpr < maxgpr) \
 279       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 280   } \
 281   if (ngpr < maxgpr) { \
 282     dp = &cc->gpr[ngpr]; \
 283     if (ngpr + n > maxgpr) { \
 284       nsp += (ngpr + n - maxgpr) * CTSIZE_PTR;  /* Assumes contiguous gpr/stack fields. */ \
 285       if (nsp > CCALL_SIZE_STACK) goto err_nyi;  /* Too many arguments. */ \
 286       ngpr = maxgpr; \
 287     } else { \
 288       ngpr += n; \
 289     } \
 290     goto done; \
 291   } CCALL_HANDLE_REGARG_FP2
 292
 293 #define CCALL_HANDLE_RET \
 294   if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
 295
 296 #elif LJ_TARGET_ARM64
 297 /* -- ARM64 calling conventions ------------------------------------------- */
 298
 299 #define CCALL_HANDLE_STRUCTRET \
 300   cc->retref = !ccall_classify_struct(cts, ctr); \
 301   if (cc->retref) cc->retp = dp;
 302
 303 #define CCALL_HANDLE_STRUCTRET2 \
 304   unsigned int cl = ccall_classify_struct(cts, ctr); \
 305   if ((cl & 4)) { /* Combine float HFA from separate registers. */ \
 306     CTSize i = (cl >> 8) - 1; \
 307     do { ((uint32_t *)dp)[i] = cc->fpr[i].lo; } while (i--); \
 308   } else { \
 309     if (cl > 1) sp = (uint8_t *)&cc->fpr[0]; \
 310     memcpy(dp, sp, ctr->size); \
 311   }
 312
 313 #define CCALL_HANDLE_COMPLEXRET \
 314   /* Complex values are returned in one or two FPRs. */ \
 315   cc->retref = 0;
 316
 317 #define CCALL_HANDLE_COMPLEXRET2 \
 318   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 319     ((float *)dp)[0] = cc->fpr[0].f; \
 320     ((float *)dp)[1] = cc->fpr[1].f; \
 321   } else {  /* Copy complex double from FPRs. */ \
 322     ((double *)dp)[0] = cc->fpr[0].d; \
 323     ((double *)dp)[1] = cc->fpr[1].d; \
 324   }
 325
 326 #define CCALL_HANDLE_STRUCTARG \
 327   unsigned int cl = ccall_classify_struct(cts, d); \
 328   if (cl == 0) {  /* Pass struct by reference. */ \
 329     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 330     sz = CTSIZE_PTR; \
 331   } else if (cl > 1) {  /* Pass struct in FPRs or on stack. */ \
 332     isfp = (cl & 4) ? 2 : 1; \
 333   }  /* else: Pass struct in GPRs or on stack. */
 334
 335 #define CCALL_HANDLE_COMPLEXARG \
 336   /* Pass complex by value in separate (!) FPRs or on stack. */ \
 337   isfp = sz == 2*sizeof(float) ? 2 : 1;
 338
 339 #define CCALL_HANDLE_REGARG \
 340   if (LJ_TARGET_OSX && isva) { \
 341     /* IOS: All variadic arguments are on the stack. */ \
 342   } else if (isfp) {  /* Try to pass argument in FPRs. */ \
 343     int n2 = ctype_isvector(d->info) ? 1 : \
 344              isfp == 1 ? n : (d->size >> (4-isfp)); \
 345     if (nfpr + n2 <= CCALL_NARG_FPR) { \
 346       dp = &cc->fpr[nfpr]; \
 347       nfpr += n2; \
 348       goto done; \
 349     } else { \
 350       nfpr = CCALL_NARG_FPR;  /* Prevent reordering. */ \
 351     } \
 352   } else {  /* Try to pass argument in GPRs. */ \
 353     if (!LJ_TARGET_OSX && (d->info & CTF_ALIGN) > CTALIGN_PTR) \
 354       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 355     if (ngpr + n <= maxgpr) { \
 356       dp = &cc->gpr[ngpr]; \
 357       ngpr += n; \
 358       goto done; \
 359     } else { \
 360       ngpr = maxgpr;  /* Prevent reordering. */ \
 361     } \
 362   }
 363
 364 #if LJ_BE
 365 #define CCALL_HANDLE_RET \
 366   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 367     sp = (uint8_t *)&cc->fpr[0].f;
 368 #endif
 369
 370
 371 #elif LJ_TARGET_PPC
 372 /* -- PPC calling conventions --------------------------------------------- */
 373
 374 #define CCALL_HANDLE_STRUCTRET \
 375   cc->retref = 1;  /* Return all structs by reference. */ \
 376   cc->gpr[ngpr++] = (GPRArg)dp;
 377
 378 #define CCALL_HANDLE_COMPLEXRET \
 379   /* Complex values are returned in 2 or 4 GPRs. */ \
 380   cc->retref = 0;
 381
 382 #define CCALL_HANDLE_COMPLEXRET2 \
 383   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 384
 385 #define CCALL_HANDLE_STRUCTARG \
 386   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 387   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 388
 389 #define CCALL_HANDLE_COMPLEXARG \
 390   /* Pass complex by value in 2 or 4 GPRs. */
 391
 392 #define CCALL_HANDLE_GPR \
 393   /* Try to pass argument in GPRs. */ \
 394   if (n > 1) { \
 395     /* int64_t or complex (float). */ \
 396     lj_assertL(n == 2 || n == 4, "bad GPR size %d", n); \
 397     if (ctype_isinteger(d->info) || ctype_isfp(d->info)) \
 398       ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 399     else if (ngpr + n > maxgpr) \
 400       ngpr = maxgpr;  /* Prevent reordering. */ \
 401   } \
 402   if (ngpr + n <= maxgpr) { \
 403     dp = &cc->gpr[ngpr]; \
 404     ngpr += n; \
 405     goto done; \
 406   } \
 407
 408 #if LJ_ABI_SOFTFP
 409 #define CCALL_HANDLE_REGARG  CCALL_HANDLE_GPR
 410 #else
 411 #define CCALL_HANDLE_REGARG \
 412   if (isfp) {  /* Try to pass argument in FPRs. */ \
 413     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 414       dp = &cc->fpr[nfpr]; \
 415       nfpr += 1; \
 416       d = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 417       goto done; \
 418     } \
 419   } else { \
 420     CCALL_HANDLE_GPR \
 421   }
 422 #endif
 423
 424 #if !LJ_ABI_SOFTFP
 425 #define CCALL_HANDLE_RET \
 426   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 427     ctr = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */
 428 #endif
 429
 430 #elif LJ_TARGET_MIPS32
 431 /* -- MIPS o32 calling conventions ---------------------------------------- */
 432
 433 #define CCALL_HANDLE_STRUCTRET \
 434   cc->retref = 1;  /* Return all structs by reference. */ \
 435   cc->gpr[ngpr++] = (GPRArg)dp;
 436
 437 #define CCALL_HANDLE_COMPLEXRET \
 438   /* Complex values are returned in 1 or 2 FPRs. */ \
 439   cc->retref = 0;
 440
 441 #if LJ_ABI_SOFTFP
 442 #define CCALL_HANDLE_COMPLEXRET2 \
 443   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from GPRs. */ \
 444     ((intptr_t *)dp)[0] = cc->gpr[0]; \
 445     ((intptr_t *)dp)[1] = cc->gpr[1]; \
 446   } else {  /* Copy complex double from GPRs. */ \
 447     ((intptr_t *)dp)[0] = cc->gpr[0]; \
 448     ((intptr_t *)dp)[1] = cc->gpr[1]; \
 449     ((intptr_t *)dp)[2] = cc->gpr[2]; \
 450     ((intptr_t *)dp)[3] = cc->gpr[3]; \
 451   }
 452 #else
 453 #define CCALL_HANDLE_COMPLEXRET2 \
 454   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 455     ((float *)dp)[0] = cc->fpr[0].f; \
 456     ((float *)dp)[1] = cc->fpr[1].f; \
 457   } else {  /* Copy complex double from FPRs. */ \
 458     ((double *)dp)[0] = cc->fpr[0].d; \
 459     ((double *)dp)[1] = cc->fpr[1].d; \
 460   }
 461 #endif
 462
 463 #define CCALL_HANDLE_STRUCTARG \
 464   /* Pass all structs by value in registers and/or on the stack. */
 465
 466 #define CCALL_HANDLE_COMPLEXARG \
 467   /* Pass complex by value in 2 or 4 GPRs. */
 468
 469 #define CCALL_HANDLE_GPR \
 470   if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
 471     ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 472   if (ngpr < maxgpr) { \
 473     dp = &cc->gpr[ngpr]; \
 474     if (ngpr + n > maxgpr) { \
 475      nsp += (ngpr + n - maxgpr) * CTSIZE_PTR;  /* Assumes contiguous gpr/stack fields. */ \
 476      if (nsp > CCALL_SIZE_STACK) goto err_nyi;  /* Too many arguments. */ \
 477      ngpr = maxgpr; \
 478     } else { \
 479      ngpr += n; \
 480     } \
 481     goto done; \
 482   }
 483
 484 #if !LJ_ABI_SOFTFP      /* MIPS32 hard-float */
 485 #define CCALL_HANDLE_REGARG \
 486   if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
 487     /* Try to pass argument in FPRs. */ \
 488     dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
 489     nfpr++; ngpr += n; \
 490     goto done; \
 491   } else {  /* Try to pass argument in GPRs. */ \
 492     nfpr = CCALL_NARG_FPR; \
 493     CCALL_HANDLE_GPR \
 494   }
 495 #else                   /* MIPS32 soft-float */
 496 #define CCALL_HANDLE_REGARG CCALL_HANDLE_GPR
 497 #endif
 498
 499 #if !LJ_ABI_SOFTFP
 500 /* On MIPS64 soft-float, position of float return values is endian-dependant. */
 501 #define CCALL_HANDLE_RET \
 502   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 503     sp = (uint8_t *)&cc->fpr[0].f;
 504 #endif
 505
 506 #elif LJ_TARGET_MIPS64
 507 /* -- MIPS n64 calling conventions ---------------------------------------- */
 508
 509 #define CCALL_HANDLE_STRUCTRET \
 510   cc->retref = !(sz <= 16); \
 511   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 512
 513 #define CCALL_HANDLE_STRUCTRET2 \
 514   ccall_copy_struct(cc, ctr, dp, sp, ccall_classify_struct(cts, ctr, ct));
 515
 516 #define CCALL_HANDLE_COMPLEXRET \
 517   /* Complex values are returned in 1 or 2 FPRs. */ \
 518   cc->retref = 0;
 519
 520 #if LJ_ABI_SOFTFP       /* MIPS64 soft-float */
 521
 522 #define CCALL_HANDLE_COMPLEXRET2 \
 523   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from GPRs. */ \
 524     ((intptr_t *)dp)[0] = cc->gpr[0]; \
 525   } else {  /* Copy complex double from GPRs. */ \
 526     ((intptr_t *)dp)[0] = cc->gpr[0]; \
 527     ((intptr_t *)dp)[1] = cc->gpr[1]; \
 528   }
 529
 530 #define CCALL_HANDLE_COMPLEXARG \
 531   /* Pass complex by value in 2 or 4 GPRs. */
 532
 533 /* Position of soft-float 'float' return value depends on endianess.  */
 534 #define CCALL_HANDLE_RET \
 535   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 536     sp = (uint8_t *)cc->gpr + LJ_ENDIAN_SELECT(0, 4);
 537
 538 #else                   /* MIPS64 hard-float */
 539
 540 #define CCALL_HANDLE_COMPLEXRET2 \
 541   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 542     ((float *)dp)[0] = cc->fpr[0].f; \
 543     ((float *)dp)[1] = cc->fpr[1].f; \
 544   } else {  /* Copy complex double from FPRs. */ \
 545     ((double *)dp)[0] = cc->fpr[0].d; \
 546     ((double *)dp)[1] = cc->fpr[1].d; \
 547   }
 548
 549 #define CCALL_HANDLE_COMPLEXARG \
 550   if (sz == 2*sizeof(float)) { \
 551     isfp = 2; \
 552     if (ngpr < maxgpr) \
 553       sz *= 2; \
 554   }
 555
 556 #define CCALL_HANDLE_RET \
 557   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 558     sp = (uint8_t *)&cc->fpr[0].f;
 559
 560 #endif
 561
 562 #define CCALL_HANDLE_STRUCTARG \
 563   /* Pass all structs by value in registers and/or on the stack. */
 564
 565 #define CCALL_HANDLE_REGARG \
 566   if (ngpr < maxgpr) { \
 567     dp = &cc->gpr[ngpr]; \
 568     if (ngpr + n > maxgpr) { \
 569       nsp += (ngpr + n - maxgpr) * CTSIZE_PTR;  /* Assumes contiguous gpr/stack fields. */ \
 570       if (nsp > CCALL_SIZE_STACK) goto err_nyi;  /* Too many arguments. */ \
 571       ngpr = maxgpr; \
 572     } else { \
 573       ngpr += n; \
 574     } \
 575     goto done; \
 576   }
 577
 578 #else
 579 #error "Missing calling convention definitions for this architecture"
 580 #endif
 581
 582 #ifndef CCALL_HANDLE_STRUCTRET2
 583 #define CCALL_HANDLE_STRUCTRET2 \
 584   memcpy(dp, sp, ctr->size);  /* Copy struct return value from GPRs. */
 585 #endif
 586
 587 /* -- x86 OSX ABI struct classification ----------------------------------- */
 588
 589 #if LJ_TARGET_X86 && LJ_TARGET_OSX
 590
 591 /* Check for struct with single FP field. */
 592 static int ccall_classify_struct(CTState *cts, CType *ct)
 593 {
 594   CTSize sz = ct->size;
 595   if (!(sz == sizeof(float) || sz == sizeof(double))) return 0;
 596   if ((ct->info & CTF_UNION)) return 0;
 597   while (ct->sib) {
 598     ct = ctype_get(cts, ct->sib);
 599     if (ctype_isfield(ct->info)) {
 600       CType *sct = ctype_rawchild(cts, ct);
 601       if (ctype_isfp(sct->info)) {
 602         if (sct->size == sz)
 603           return (sz >> 2);  /* Return 1 for float or 2 for double. */
 604       } else if (ctype_isstruct(sct->info)) {
 605         if (sct->size)
 606           return ccall_classify_struct(cts, sct);
 607       } else {
 608         break;
 609       }
 610     } else if (ctype_isbitfield(ct->info)) {
 611       break;
 612     } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 613       CType *sct = ctype_rawchild(cts, ct);
 614       if (sct->size)
 615         return ccall_classify_struct(cts, sct);
 616     }
 617   }
 618   return 0;
 619 }
 620
 621 #endif
 622
 623 /* -- x64 struct classification ------------------------------------------- */
 624
 625 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 626
 627 /* Register classes for x64 struct classification. */
 628 #define CCALL_RCL_INT   1
 629 #define CCALL_RCL_SSE   2
 630 #define CCALL_RCL_MEM   4
 631 /* NYI: classify vectors. */
 632
 633 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
 634
 635 /* Classify a C type. */
 636 static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 637 {
 638   if (ctype_isarray(ct->info)) {
 639     CType *cct = ctype_rawchild(cts, ct);
 640     CTSize eofs, esz = cct->size, asz = ct->size;
 641     for (eofs = 0; eofs < asz; eofs += esz)
 642       ccall_classify_ct(cts, cct, rcl, ofs+eofs);
 643   } else if (ctype_isstruct(ct->info)) {
 644     ccall_classify_struct(cts, ct, rcl, ofs);
 645   } else {
 646     int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
 647     lj_assertCTS(ctype_hassize(ct->info),
 648                  "classify ctype %08x without size", ct->info);
 649     if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM;  /* Unaligned. */
 650     rcl[(ofs >= 8)] |= cl;
 651   }
 652 }
 653
 654 /* Recursively classify a struct based on its fields. */
 655 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 656 {
 657   if (ct->size > 16) return CCALL_RCL_MEM;  /* Too big, gets memory class. */
 658   while (ct->sib) {
 659     CTSize fofs;
 660     ct = ctype_get(cts, ct->sib);
 661     fofs = ofs+ct->size;
 662     if (ctype_isfield(ct->info))
 663       ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 664     else if (ctype_isbitfield(ct->info))
 665       rcl[(fofs >= 8)] |= CCALL_RCL_INT;  /* NYI: unaligned bitfields? */
 666     else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
 667       ccall_classify_struct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 668   }
 669   return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM);  /* Memory class? */
 670 }
 671
 672 /* Try to split up a small struct into registers. */
 673 static int ccall_struct_reg(CCallState *cc, CTState *cts, GPRArg *dp, int *rcl)
 674 {
 675   MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
 676   uint32_t i;
 677   UNUSED(cts);
 678   for (i = 0; i < 2; i++) {
 679     lj_assertCTS(!(rcl[i] & CCALL_RCL_MEM), "pass mem struct in reg");
 680     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 681       if (ngpr >= CCALL_NARG_GPR) return 1;  /* Register overflow. */
 682       cc->gpr[ngpr++] = dp[i];
 683     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 684       if (nfpr >= CCALL_NARG_FPR) return 1;  /* Register overflow. */
 685       cc->fpr[nfpr++].l[0] = dp[i];
 686     }
 687   }
 688   cc->ngpr = ngpr; cc->nfpr = nfpr;
 689   return 0;  /* Ok. */
 690 }
 691
 692 /* Pass a small struct argument. */
 693 static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
 694                             TValue *o, int narg)
 695 {
 696   GPRArg dp[2];
 697   dp[0] = dp[1] = 0;
 698   /* Convert to temp. struct. */
 699   lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 700   if (ccall_struct_reg(cc, cts, dp, rcl)) {
 701     /* Register overflow? Pass on stack. */
 702     MSize nsp = cc->nsp, sz = rcl[1] ? 2*CTSIZE_PTR : CTSIZE_PTR;
 703     if (nsp + sz > CCALL_SIZE_STACK)
 704       return 1;  /* Too many arguments. */
 705     cc->nsp = nsp + sz;
 706     memcpy((uint8_t *)cc->stack + nsp, dp, sz);
 707   }
 708   return 0;  /* Ok. */
 709 }
 710
 711 /* Combine returned small struct. */
 712 static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
 713 {
 714   GPRArg sp[2];
 715   MSize ngpr = 0, nfpr = 0;
 716   uint32_t i;
 717   for (i = 0; i < 2; i++) {
 718     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 719       sp[i] = cc->gpr[ngpr++];
 720     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 721       sp[i] = cc->fpr[nfpr++].l[0];
 722     }
 723   }
 724   memcpy(dp, sp, sz);
 725 }
 726 #endif
 727
 728 /* -- ARM hard-float ABI struct classification ---------------------------- */
 729
 730 #if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
 731
 732 /* Classify a struct based on its fields. */
 733 static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
 734 {
 735   CTSize sz = ct->size;
 736   unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
 737   if ((ctf->info & CTF_VARARG)) goto noth;
 738   while (ct->sib) {
 739     CType *sct;
 740     ct = ctype_get(cts, ct->sib);
 741     if (ctype_isfield(ct->info)) {
 742       sct = ctype_rawchild(cts, ct);
 743       if (ctype_isfp(sct->info)) {
 744         r |= sct->size;
 745         if (!isu) n++; else if (n == 0) n = 1;
 746       } else if (ctype_iscomplex(sct->info)) {
 747         r |= (sct->size >> 1);
 748         if (!isu) n += 2; else if (n < 2) n = 2;
 749       } else if (ctype_isstruct(sct->info)) {
 750         goto substruct;
 751       } else {
 752         goto noth;
 753       }
 754     } else if (ctype_isbitfield(ct->info)) {
 755       goto noth;
 756     } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 757       sct = ctype_rawchild(cts, ct);
 758     substruct:
 759       if (sct->size > 0) {
 760         unsigned int s = ccall_classify_struct(cts, sct, ctf);
 761         if (s <= 1) goto noth;
 762         r |= (s & 255);
 763         if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
 764       }
 765     }
 766   }
 767   if ((r == 4 || r == 8) && n <= 4)
 768     return r + (n << 8);
 769 noth:  /* Not a homogeneous float/double aggregate. */
 770   return (sz <= 4);  /* Return structs of size <= 4 in a GPR. */
 771 }
 772
 773 #endif
 774
 775 /* -- ARM64 ABI struct classification ------------------------------------- */
 776
 777 #if LJ_TARGET_ARM64
 778
 779 /* Classify a struct based on its fields. */
 780 static unsigned int ccall_classify_struct(CTState *cts, CType *ct)
 781 {
 782   CTSize sz = ct->size;
 783   unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
 784   while (ct->sib) {
 785     CType *sct;
 786     ct = ctype_get(cts, ct->sib);
 787     if (ctype_isfield(ct->info)) {
 788       sct = ctype_rawchild(cts, ct);
 789       if (ctype_isfp(sct->info)) {
 790         r |= sct->size;
 791         if (!isu) n++; else if (n == 0) n = 1;
 792       } else if (ctype_iscomplex(sct->info)) {
 793         r |= (sct->size >> 1);
 794         if (!isu) n += 2; else if (n < 2) n = 2;
 795       } else if (ctype_isstruct(sct->info)) {
 796         goto substruct;
 797       } else {
 798         goto noth;
 799       }
 800     } else if (ctype_isbitfield(ct->info)) {
 801       goto noth;
 802     } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 803       sct = ctype_rawchild(cts, ct);
 804     substruct:
 805       if (sct->size > 0) {
 806         unsigned int s = ccall_classify_struct(cts, sct);
 807         if (s <= 1) goto noth;
 808         r |= (s & 255);
 809         if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
 810       }
 811     }
 812   }
 813   if ((r == 4 || r == 8) && n <= 4)
 814     return r + (n << 8);
 815 noth:  /* Not a homogeneous float/double aggregate. */
 816   return (sz <= 16);  /* Return structs of size <= 16 in GPRs. */
 817 }
 818
 819 #endif
 820
 821 /* -- MIPS64 ABI struct classification ---------------------------- */
 822
 823 #if LJ_TARGET_MIPS64
 824
 825 #define FTYPE_FLOAT     1
 826 #define FTYPE_DOUBLE    2
 827
 828 /* Classify FP fields (max. 2) and their types. */
 829 static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
 830 {
 831   int n = 0, ft = 0;
 832   if ((ctf->info & CTF_VARARG) || (ct->info & CTF_UNION))
 833     goto noth;
 834   while (ct->sib) {
 835     CType *sct;
 836     ct = ctype_get(cts, ct->sib);
 837     if (n == 2) {
 838       goto noth;
 839     } else if (ctype_isfield(ct->info)) {
 840       sct = ctype_rawchild(cts, ct);
 841       if (ctype_isfp(sct->info)) {
 842         ft |= (sct->size == 4 ? FTYPE_FLOAT : FTYPE_DOUBLE) << 2*n;
 843         n++;
 844       } else {
 845         goto noth;
 846       }
 847     } else if (ctype_isbitfield(ct->info) ||
 848                ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 849       goto noth;
 850     }
 851   }
 852   if (n <= 2)
 853     return ft;
 854 noth:  /* Not a homogeneous float/double aggregate. */
 855   return 0;  /* Struct is in GPRs. */
 856 }
 857
 858 static void ccall_copy_struct(CCallState *cc, CType *ctr, void *dp, void *sp,
 859                               int ft)
 860 {
 861   if (LJ_ABI_SOFTFP ? ft :
 862       ((ft & 3) == FTYPE_FLOAT || (ft >> 2) == FTYPE_FLOAT)) {
 863     int i, ofs = 0;
 864     for (i = 0; ft != 0; i++, ft >>= 2) {
 865       if ((ft & 3) == FTYPE_FLOAT) {
 866 #if LJ_ABI_SOFTFP
 867         /* The 2nd FP struct result is in CARG1 (gpr[2]) and not CRET2. */
 868         memcpy((uint8_t *)dp + ofs,
 869                (uint8_t *)&cc->gpr[2*i] + LJ_ENDIAN_SELECT(0, 4), 4);
 870 #else
 871         *(float *)((uint8_t *)dp + ofs) = cc->fpr[i].f;
 872 #endif
 873         ofs += 4;
 874       } else {
 875         ofs = (ofs + 7) & ~7;  /* 64 bit alignment. */
 876 #if LJ_ABI_SOFTFP
 877         *(intptr_t *)((uint8_t *)dp + ofs) = cc->gpr[2*i];
 878 #else
 879         *(double *)((uint8_t *)dp + ofs) = cc->fpr[i].d;
 880 #endif
 881         ofs += 8;
 882       }
 883     }
 884   } else {
 885 #if !LJ_ABI_SOFTFP
 886     if (ft) sp = (uint8_t *)&cc->fpr[0];
 887 #endif
 888     memcpy(dp, sp, ctr->size);
 889   }
 890 }
 891
 892 #endif
 893
 894 /* -- Common C call handling ---------------------------------------------- */
 895
 896 /* Infer the destination CTypeID for a vararg argument. */
 897 CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
 898 {
 899   if (tvisnumber(o)) {
 900     return CTID_DOUBLE;
 901   } else if (tviscdata(o)) {
 902     CTypeID id = cdataV(o)->ctypeid;
 903     CType *s = ctype_get(cts, id);
 904     if (ctype_isrefarray(s->info)) {
 905       return lj_ctype_intern(cts,
 906                CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
 907     } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
 908       /* NYI: how to pass a struct by value in a vararg argument? */
 909       return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
 910     } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
 911       return CTID_DOUBLE;
 912     } else {
 913       return id;
 914     }
 915   } else if (tvisstr(o)) {
 916     return CTID_P_CCHAR;
 917   } else if (tvisbool(o)) {
 918     return CTID_BOOL;
 919   } else {
 920     return CTID_P_VOID;
 921   }
 922 }
 923
 924 /* Setup arguments for C call. */
 925 static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
 926                           CCallState *cc)
 927 {
 928   int gcsteps = 0;
 929   TValue *o, *top = L->top;
 930   CTypeID fid;
 931   CType *ctr;
 932   MSize maxgpr, ngpr = 0, nsp = 0, narg;
 933 #if CCALL_NARG_FPR
 934   MSize nfpr = 0;
 935 #if LJ_TARGET_ARM
 936   MSize fprodd = 0;
 937 #endif
 938 #endif
 939
 940   /* Clear unused regs to get some determinism in case of misdeclaration. */
 941   memset(cc->gpr, 0, sizeof(cc->gpr));
 942 #if CCALL_NUM_FPR
 943   memset(cc->fpr, 0, sizeof(cc->fpr));
 944 #endif
 945
 946 #if LJ_TARGET_X86
 947   /* x86 has several different calling conventions. */
 948   cc->resx87 = 0;
 949   switch (ctype_cconv(ct->info)) {
 950   case CTCC_FASTCALL: maxgpr = 2; break;
 951   case CTCC_THISCALL: maxgpr = 1; break;
 952   default: maxgpr = 0; break;
 953   }
 954 #else
 955   maxgpr = CCALL_NARG_GPR;
 956 #endif
 957
 958   /* Perform required setup for some result types. */
 959   ctr = ctype_rawchild(cts, ct);
 960   if (ctype_isvector(ctr->info)) {
 961     if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
 962       goto err_nyi;
 963   } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
 964     /* Preallocate cdata object and anchor it after arguments. */
 965     CTSize sz = ctr->size;
 966     GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
 967     void *dp = cdataptr(cd);
 968     setcdataV(L, L->top++, cd);
 969     if (ctype_isstruct(ctr->info)) {
 970       CCALL_HANDLE_STRUCTRET
 971     } else {
 972       CCALL_HANDLE_COMPLEXRET
 973     }
 974 #if LJ_TARGET_X86
 975   } else if (ctype_isfp(ctr->info)) {
 976     cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
 977 #endif
 978   }
 979
 980   /* Skip initial attributes. */
 981   fid = ct->sib;
 982   while (fid) {
 983     CType *ctf = ctype_get(cts, fid);
 984     if (!ctype_isattrib(ctf->info)) break;
 985     fid = ctf->sib;
 986   }
 987
 988 #if LJ_TARGET_ARM64 && LJ_ABI_WIN
 989   if ((ct->info & CTF_VARARG)) {
 990     nsp -= maxgpr * CTSIZE_PTR;  /* May end up with negative nsp. */
 991     ngpr = maxgpr;
 992     nfpr = CCALL_NARG_FPR;
 993   }
 994 #endif
 995
 996   /* Walk through all passed arguments. */
 997   for (o = L->base+1, narg = 1; o < top; o++, narg++) {
 998     CTypeID did;
 999     CType *d;
1000     CTSize sz;
1001     MSize n, isfp = 0, isva = 0;
1002     void *dp, *rp = NULL;
1003
1004     if (fid) {  /* Get argument type from field. */
1005       CType *ctf = ctype_get(cts, fid);
1006       fid = ctf->sib;
1007       lj_assertL(ctype_isfield(ctf->info), "field expected");
1008       did = ctype_cid(ctf->info);
1009     } else {
1010       if (!(ct->info & CTF_VARARG))
1011         lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too many arguments. */
1012       did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
1013       isva = 1;
1014     }
1015     d = ctype_raw(cts, did);
1016     sz = d->size;
1017
1018     /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
1019     if (ctype_isnum(d->info)) {
1020       if (sz > 8) goto err_nyi;
1021       if ((d->info & CTF_FP))
1022         isfp = 1;
1023     } else if (ctype_isvector(d->info)) {
1024       if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
1025         isfp = 1;
1026       else
1027         goto err_nyi;
1028     } else if (ctype_isstruct(d->info)) {
1029       CCALL_HANDLE_STRUCTARG
1030     } else if (ctype_iscomplex(d->info)) {
1031       CCALL_HANDLE_COMPLEXARG
1032     } else if (!(CCALL_PACK_STACKARG && ctype_isenum(d->info))) {
1033       sz = CTSIZE_PTR;
1034     }
1035     n = (sz + CTSIZE_PTR-1) / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
1036
1037     CCALL_HANDLE_REGARG  /* Handle register arguments. */
1038
1039     /* Otherwise pass argument on stack. */
1040     if (CCALL_ALIGN_STACKARG) {  /* Align argument on stack. */
1041       MSize align = (1u << ctype_align(d->info)) - 1;
1042       if (rp || (CCALL_PACK_STACKARG && isva && align < CTSIZE_PTR-1))
1043         align = CTSIZE_PTR-1;
1044       nsp = (nsp + align) & ~align;
1045     }
1046 #if LJ_TARGET_ARM64 && LJ_ABI_WIN
1047     /* A negative nsp points into cc->gpr. Blame MS for their messy ABI. */
1048     dp = ((uint8_t *)cc->stack) + (int32_t)nsp;
1049 #else
1050     dp = ((uint8_t *)cc->stack) + nsp;
1051 #endif
1052     nsp += CCALL_PACK_STACKARG ? sz : n * CTSIZE_PTR;
1053     if ((int32_t)nsp > CCALL_SIZE_STACK) {  /* Too many arguments. */
1054     err_nyi:
1055       lj_err_caller(L, LJ_ERR_FFI_NYICALL);
1056     }
1057     isva = 0;
1058
1059   done:
1060     if (rp) {  /* Pass by reference. */
1061       gcsteps++;
1062       *(void **)dp = rp;
1063       dp = rp;
1064     }
1065     lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
1066     /* Extend passed integers to 32 bits at least. */
1067     if (ctype_isinteger_or_bool(d->info) && d->size < 4 &&
1068         (!CCALL_PACK_STACKARG || !((uintptr_t)dp & 3))) {  /* Assumes LJ_LE. */
1069       if (d->info & CTF_UNSIGNED)
1070         *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
1071                                          (uint32_t)*(uint16_t *)dp;
1072       else
1073         *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
1074                                         (int32_t)*(int16_t *)dp;
1075     }
1076 #if LJ_TARGET_ARM64 && LJ_BE
1077     if (isfp && d->size == sizeof(float))
1078       ((float *)dp)[1] = ((float *)dp)[0];  /* Floats occupy high slot. */
1079 #endif
1080 #if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
1081     if ((ctype_isinteger_or_bool(d->info) || ctype_isenum(d->info)
1082 #if LJ_TARGET_MIPS64
1083          || (isfp && nsp == 0)
1084 #endif
1085          ) && d->size <= 4) {
1086       *(int64_t *)dp = (int64_t)*(int32_t *)dp;  /* Sign-extend to 64 bit. */
1087     }
1088 #endif
1089 #if LJ_TARGET_X64 && LJ_ABI_WIN
1090     if (isva) {  /* Windows/x64 mirrors varargs in both register sets. */
1091       if (nfpr == ngpr)
1092         cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
1093       else
1094         cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
1095     }
1096 #else
1097     UNUSED(isva);
1098 #endif
1099 #if LJ_TARGET_X64 && !LJ_ABI_WIN
1100     if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
1101       cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1];  /* Split complex double. */
1102       cc->fpr[nfpr-2].d[1] = 0;
1103     }
1104 #elif LJ_TARGET_ARM64 || (LJ_TARGET_MIPS64 && !LJ_ABI_SOFTFP)
1105     if (isfp == 2 && (uint8_t *)dp < (uint8_t *)cc->stack) {
1106       /* Split float HFA or complex float into separate registers. */
1107       CTSize i = (sz >> 2) - 1;
1108       do { ((uint64_t *)dp)[i] = ((uint32_t *)dp)[i]; } while (i--);
1109     }
1110 #else
1111     UNUSED(isfp);
1112 #endif
1113   }
1114   if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too few arguments. */
1115 #if LJ_TARGET_ARM64 && LJ_ABI_WIN
1116   if ((int32_t)nsp < 0) nsp = 0;
1117 #endif
1118
1119 #if LJ_TARGET_X64 || (LJ_TARGET_PPC && !LJ_ABI_SOFTFP)
1120   cc->nfpr = nfpr;  /* Required for vararg functions. */
1121 #endif
1122   cc->nsp = (nsp + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
1123   cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA) * CTSIZE_PTR;
1124   if (cc->nsp > CCALL_SPS_FREE * CTSIZE_PTR)
1125     cc->spadj += (((cc->nsp - CCALL_SPS_FREE * CTSIZE_PTR) + 15u) & ~15u);
1126   return gcsteps;
1127 }
1128
1129 /* Get results from C call. */
1130 static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
1131                              CCallState *cc, int *ret)
1132 {
1133   CType *ctr = ctype_rawchild(cts, ct);
1134   uint8_t *sp = (uint8_t *)&cc->gpr[0];
1135   if (ctype_isvoid(ctr->info)) {
1136     *ret = 0;  /* Zero results. */
1137     return 0;  /* No additional GC step. */
1138   }
1139   *ret = 1;  /* One result. */
1140   if (ctype_isstruct(ctr->info)) {
1141     /* Return cdata object which is already on top of stack. */
1142     if (!cc->retref) {
1143       void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
1144       CCALL_HANDLE_STRUCTRET2
1145     }
1146     return 1;  /* One GC step. */
1147   }
1148   if (ctype_iscomplex(ctr->info)) {
1149     /* Return cdata object which is already on top of stack. */
1150     void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
1151     CCALL_HANDLE_COMPLEXRET2
1152     return 1;  /* One GC step. */
1153   }
1154   if (LJ_BE && ctr->size < CTSIZE_PTR &&
1155       (ctype_isinteger_or_bool(ctr->info) || ctype_isenum(ctr->info)))
1156     sp += (CTSIZE_PTR - ctr->size);
1157 #if CCALL_NUM_FPR
1158   if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
1159     sp = (uint8_t *)&cc->fpr[0];
1160 #endif
1161 #ifdef CCALL_HANDLE_RET
1162   CCALL_HANDLE_RET
1163 #endif
1164   /* No reference types end up here, so there's no need for the CTypeID. */
1165   lj_assertL(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)),
1166              "unexpected reference ctype");
1167   return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
1168 }
1169
1170 /* Call C function. */
1171 int lj_ccall_func(lua_State *L, GCcdata *cd)
1172 {
1173   CTState *cts = ctype_cts(L);
1174   CType *ct = ctype_raw(cts, cd->ctypeid);
1175   CTSize sz = CTSIZE_PTR;
1176   if (ctype_isptr(ct->info)) {
1177     sz = ct->size;
1178     ct = ctype_rawchild(cts, ct);
1179   }
1180   if (ctype_isfunc(ct->info)) {
1181     CCallState cc;
1182     int gcsteps, ret;
1183     cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
1184     gcsteps = ccall_set_args(L, cts, ct, &cc);
1185     ct = (CType *)((intptr_t)ct-(intptr_t)cts->tab);
1186     cts->cb.slot = ~0u;
1187     lj_vm_ffi_call(&cc);
1188     if (cts->cb.slot != ~0u) {  /* Blacklist function that called a callback. */
1189       TValue tv;
1190       tv.u64 = ((uintptr_t)(void *)cc.func >> 2) | U64x(800000000, 00000000);
1191       setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
1192     }
1193     ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab);  /* May be reallocated. */
1194     gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
1195 #if LJ_TARGET_X86 && LJ_ABI_WIN
1196     /* Automatically detect __stdcall and fix up C function declaration. */
1197     if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
1198       CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
1199       lj_trace_abort(G(L));
1200     }
1201 #endif
1202     while (gcsteps-- > 0)
1203       lj_gc_check(L);
1204     return ret;
1205   }
1206   return -1;  /* Not a function. */
1207 }
1208
1209 #endif