src/lj_ccall.c

   1 /*
   2 ** FFI C call handling.
   3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
   4 */
   5
   6 #include "lj_obj.h"
   7
   8 #if LJ_HASFFI
   9
  10 #include "lj_gc.h"
  11 #include "lj_err.h"
  12 #include "lj_str.h"
  13 #include "lj_tab.h"
  14 #include "lj_ctype.h"
  15 #include "lj_cconv.h"
  16 #include "lj_cdata.h"
  17 #include "lj_ccall.h"
  18 #include "lj_trace.h"
  19
  20 /* Target-specific handling of register arguments. */
  21 #if LJ_TARGET_X86
  22 /* -- x86 calling conventions --------------------------------------------- */
  23
  24 #if LJ_ABI_WIN
  25
  26 #define CCALL_HANDLE_STRUCTRET \
  27   /* Return structs bigger than 8 by reference (on stack only). */ \
  28   cc->retref = (sz > 8); \
  29   if (cc->retref) cc->stack[nsp++] = (GPRArg)dp;
  30
  31 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
  32
  33 #else
  34
  35 #if LJ_TARGET_OSX
  36
  37 #define CCALL_HANDLE_STRUCTRET \
  38   /* Return structs of size 1, 2, 4 or 8 in registers. */ \
  39   cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
  40   if (cc->retref) { \
  41     if (ngpr < maxgpr) \
  42       cc->gpr[ngpr++] = (GPRArg)dp; \
  43     else \
  44       cc->stack[nsp++] = (GPRArg)dp; \
  45   } else {  /* Struct with single FP field ends up in FPR. */ \
  46     cc->resx87 = ccall_classify_struct(cts, ctr); \
  47   }
  48
  49 #define CCALL_HANDLE_STRUCTRET2 \
  50   if (cc->resx87) sp = (uint8_t *)&cc->fpr[0]; \
  51   memcpy(dp, sp, ctr->size);
  52
  53 #else
  54
  55 #define CCALL_HANDLE_STRUCTRET \
  56   cc->retref = 1;  /* Return all structs by reference (in reg or on stack). */ \
  57   if (ngpr < maxgpr) \
  58     cc->gpr[ngpr++] = (GPRArg)dp; \
  59   else \
  60     cc->stack[nsp++] = (GPRArg)dp;
  61
  62 #endif
  63
  64 #define CCALL_HANDLE_COMPLEXRET \
  65   /* Return complex float in GPRs and complex double by reference. */ \
  66   cc->retref = (sz > 8); \
  67   if (cc->retref) { \
  68     if (ngpr < maxgpr) \
  69       cc->gpr[ngpr++] = (GPRArg)dp; \
  70     else \
  71       cc->stack[nsp++] = (GPRArg)dp; \
  72   }
  73
  74 #endif
  75
  76 #define CCALL_HANDLE_COMPLEXRET2 \
  77   if (!cc->retref) \
  78     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
  79
  80 #define CCALL_HANDLE_STRUCTARG \
  81   ngpr = maxgpr;  /* Pass all structs by value on the stack. */
  82
  83 #define CCALL_HANDLE_COMPLEXARG \
  84   isfp = 1;  /* Pass complex by value on stack. */
  85
  86 #define CCALL_HANDLE_REGARG \
  87   if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
  88     if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
  89       if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
  90     } else if (ngpr + 1 <= maxgpr) { \
  91       dp = &cc->gpr[ngpr]; \
  92       ngpr += n; \
  93       goto done; \
  94     } \
  95   }
  96
  97 #elif LJ_TARGET_X64 && LJ_ABI_WIN
  98 /* -- Windows/x64 calling conventions ------------------------------------- */
  99
 100 #define CCALL_HANDLE_STRUCTRET \
 101   /* Return structs of size 1, 2, 4 or 8 in a GPR. */ \
 102   cc->retref = !(sz == 1 || sz == 2 || sz == 4 || sz == 8); \
 103   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 104
 105 #define CCALL_HANDLE_COMPLEXRET CCALL_HANDLE_STRUCTRET
 106
 107 #define CCALL_HANDLE_COMPLEXRET2 \
 108   if (!cc->retref) \
 109     *(int64_t *)dp = *(int64_t *)sp;  /* Copy complex float from GPRs. */
 110
 111 #define CCALL_HANDLE_STRUCTARG \
 112   /* Pass structs of size 1, 2, 4 or 8 in a GPR by value. */ \
 113   if (!(sz == 1 || sz == 2 || sz == 4 || sz == 8)) { \
 114     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 115     sz = CTSIZE_PTR;  /* Pass all other structs by reference. */ \
 116   }
 117
 118 #define CCALL_HANDLE_COMPLEXARG \
 119   /* Pass complex float in a GPR and complex double by reference. */ \
 120   if (sz != 2*sizeof(float)) { \
 121     rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 122     sz = CTSIZE_PTR; \
 123   }
 124
 125 /* Windows/x64 argument registers are strictly positional (use ngpr). */
 126 #define CCALL_HANDLE_REGARG \
 127   if (isfp) { \
 128     if (ngpr < maxgpr) { dp = &cc->fpr[ngpr++]; nfpr = ngpr; goto done; } \
 129   } else { \
 130     if (ngpr < maxgpr) { dp = &cc->gpr[ngpr++]; goto done; } \
 131   }
 132
 133 #elif LJ_TARGET_X64
 134 /* -- POSIX/x64 calling conventions --------------------------------------- */
 135
 136 #define CCALL_HANDLE_STRUCTRET \
 137   int rcl[2]; rcl[0] = rcl[1] = 0; \
 138   if (ccall_classify_struct(cts, ctr, rcl, 0)) { \
 139     cc->retref = 1;  /* Return struct by reference. */ \
 140     cc->gpr[ngpr++] = (GPRArg)dp; \
 141   } else { \
 142     cc->retref = 0;  /* Return small structs in registers. */ \
 143   }
 144
 145 #define CCALL_HANDLE_STRUCTRET2 \
 146   int rcl[2]; rcl[0] = rcl[1] = 0; \
 147   ccall_classify_struct(cts, ctr, rcl, 0); \
 148   ccall_struct_ret(cc, rcl, dp, ctr->size);
 149
 150 #define CCALL_HANDLE_COMPLEXRET \
 151   /* Complex values are returned in one or two FPRs. */ \
 152   cc->retref = 0;
 153
 154 #define CCALL_HANDLE_COMPLEXRET2 \
 155   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPR. */ \
 156     *(int64_t *)dp = cc->fpr[0].l[0]; \
 157   } else {  /* Copy non-contiguous complex double from FPRs. */ \
 158     ((int64_t *)dp)[0] = cc->fpr[0].l[0]; \
 159     ((int64_t *)dp)[1] = cc->fpr[1].l[0]; \
 160   }
 161
 162 #define CCALL_HANDLE_STRUCTARG \
 163   int rcl[2]; rcl[0] = rcl[1] = 0; \
 164   if (!ccall_classify_struct(cts, d, rcl, 0)) { \
 165     cc->nsp = nsp; cc->ngpr = ngpr; cc->nfpr = nfpr; \
 166     if (ccall_struct_arg(cc, cts, d, rcl, o, narg)) goto err_nyi; \
 167     nsp = cc->nsp; ngpr = cc->ngpr; nfpr = cc->nfpr; \
 168     continue; \
 169   }  /* Pass all other structs by value on stack. */
 170
 171 #define CCALL_HANDLE_COMPLEXARG \
 172   isfp = 2;  /* Pass complex in FPRs or on stack. Needs postprocessing. */
 173
 174 #define CCALL_HANDLE_REGARG \
 175   if (isfp) {  /* Try to pass argument in FPRs. */ \
 176     int n2 = ctype_isvector(d->info) ? 1 : n; \
 177     if (nfpr + n2 <= CCALL_NARG_FPR) { \
 178       dp = &cc->fpr[nfpr]; \
 179       nfpr += n2; \
 180       goto done; \
 181     } \
 182   } else {  /* Try to pass argument in GPRs. */ \
 183     /* Note that reordering is explicitly allowed in the x64 ABI. */ \
 184     if (n <= 2 && ngpr + n <= maxgpr) { \
 185       dp = &cc->gpr[ngpr]; \
 186       ngpr += n; \
 187       goto done; \
 188     } \
 189   }
 190
 191 #elif LJ_TARGET_ARM
 192 /* -- ARM calling conventions --------------------------------------------- */
 193
 194 #if LJ_ABI_SOFTFP
 195
 196 #define CCALL_HANDLE_STRUCTRET \
 197   /* Return structs of size <= 4 in a GPR. */ \
 198   cc->retref = !(sz <= 4); \
 199   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 200
 201 #define CCALL_HANDLE_COMPLEXRET \
 202   cc->retref = 1;  /* Return all complex values by reference. */ \
 203   cc->gpr[ngpr++] = (GPRArg)dp;
 204
 205 #define CCALL_HANDLE_COMPLEXRET2 \
 206   UNUSED(dp); /* Nothing to do. */
 207
 208 #define CCALL_HANDLE_STRUCTARG \
 209   /* Pass all structs by value in registers and/or on the stack. */
 210
 211 #define CCALL_HANDLE_COMPLEXARG \
 212   /* Pass complex by value in 2 or 4 GPRs. */
 213
 214 #define CCALL_HANDLE_REGARG_FP1
 215 #define CCALL_HANDLE_REGARG_FP2
 216
 217 #else
 218
 219 #define CCALL_HANDLE_STRUCTRET \
 220   cc->retref = !ccall_classify_struct(cts, ctr, ct); \
 221   if (cc->retref) cc->gpr[ngpr++] = (GPRArg)dp;
 222
 223 #define CCALL_HANDLE_STRUCTRET2 \
 224   if (ccall_classify_struct(cts, ctr, ct) > 1) sp = (uint8_t *)&cc->fpr[0]; \
 225   memcpy(dp, sp, ctr->size);
 226
 227 #define CCALL_HANDLE_COMPLEXRET \
 228   if (!(ct->info & CTF_VARARG)) cc->retref = 0;  /* Return complex in FPRs. */
 229
 230 #define CCALL_HANDLE_COMPLEXRET2 \
 231   if (!(ct->info & CTF_VARARG)) memcpy(dp, &cc->fpr[0], ctr->size);
 232
 233 #define CCALL_HANDLE_STRUCTARG \
 234   isfp = (ccall_classify_struct(cts, d, ct) > 1);
 235   /* Pass all structs by value in registers and/or on the stack. */
 236
 237 #define CCALL_HANDLE_COMPLEXARG \
 238   isfp = 1;  /* Pass complex by value in FPRs or on stack. */
 239
 240 #define CCALL_HANDLE_REGARG_FP1 \
 241   if (isfp && !(ct->info & CTF_VARARG)) { \
 242     if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 243       if (nfpr + (n >> 1) <= CCALL_NARG_FPR) { \
 244         dp = &cc->fpr[nfpr]; \
 245         nfpr += (n >> 1); \
 246         goto done; \
 247       } \
 248     } else { \
 249       if (sz > 1 && fprodd != nfpr) fprodd = 0; \
 250       if (fprodd) { \
 251         if (2*nfpr+n <= 2*CCALL_NARG_FPR+1) { \
 252           dp = (void *)&cc->fpr[fprodd-1].f[1]; \
 253           nfpr += (n >> 1); \
 254           if ((n & 1)) fprodd = 0; else fprodd = nfpr-1; \
 255           goto done; \
 256         } \
 257       } else { \
 258         if (2*nfpr+n <= 2*CCALL_NARG_FPR) { \
 259           dp = (void *)&cc->fpr[nfpr]; \
 260           nfpr += (n >> 1); \
 261           if ((n & 1)) fprodd = ++nfpr; else fprodd = 0; \
 262           goto done; \
 263         } \
 264       } \
 265     } \
 266     fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
 267   } else {
 268
 269 #define CCALL_HANDLE_REGARG_FP2 }
 270
 271 #endif
 272
 273 #define CCALL_HANDLE_REGARG \
 274   CCALL_HANDLE_REGARG_FP1 \
 275   if ((d->info & CTF_ALIGN) > CTALIGN_PTR) { \
 276     if (ngpr < maxgpr) \
 277       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 278   } \
 279   if (ngpr < maxgpr) { \
 280     dp = &cc->gpr[ngpr]; \
 281     if (ngpr + n > maxgpr) { \
 282       nsp += ngpr + n - maxgpr;  /* Assumes contiguous gpr/stack fields. */ \
 283       if (nsp > CCALL_MAXSTACK) goto err_nyi;  /* Too many arguments. */ \
 284       ngpr = maxgpr; \
 285     } else { \
 286       ngpr += n; \
 287     } \
 288     goto done; \
 289   } CCALL_HANDLE_REGARG_FP2
 290
 291 #define CCALL_HANDLE_RET \
 292   if ((ct->info & CTF_VARARG)) sp = (uint8_t *)&cc->gpr[0];
 293
 294 #elif LJ_TARGET_PPC
 295 /* -- PPC calling conventions --------------------------------------------- */
 296
 297 #define CCALL_HANDLE_STRUCTRET \
 298   cc->retref = 1;  /* Return all structs by reference. */ \
 299   cc->gpr[ngpr++] = (GPRArg)dp;
 300
 301 #define CCALL_HANDLE_COMPLEXRET \
 302   /* Complex values are returned in 2 or 4 GPRs. */ \
 303   cc->retref = 0;
 304
 305 #define CCALL_HANDLE_COMPLEXRET2 \
 306   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 307
 308 #define CCALL_HANDLE_STRUCTARG \
 309   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 310   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 311
 312 #define CCALL_HANDLE_COMPLEXARG \
 313   /* Pass complex by value in 2 or 4 GPRs. */
 314
 315 #define CCALL_HANDLE_REGARG \
 316   if (isfp) {  /* Try to pass argument in FPRs. */ \
 317     if (nfpr + 1 <= CCALL_NARG_FPR) { \
 318       dp = &cc->fpr[nfpr]; \
 319       nfpr += 1; \
 320       d = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
 321       goto done; \
 322     } \
 323   } else {  /* Try to pass argument in GPRs. */ \
 324     if (n > 1) { \
 325       lua_assert(n == 2 || n == 4);  /* int64_t or complex (float). */ \
 326       if (ctype_isinteger(d->info)) \
 327         ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
 328       else if (ngpr + n > maxgpr) \
 329         ngpr = maxgpr;  /* Prevent reordering. */ \
 330     } \
 331     if (ngpr + n <= maxgpr) { \
 332       dp = &cc->gpr[ngpr]; \
 333       ngpr += n; \
 334       goto done; \
 335     } \
 336   }
 337
 338 #define CCALL_HANDLE_RET \
 339   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 340     ctr = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */
 341
 342 #elif LJ_TARGET_PPCSPE
 343 /* -- PPC/SPE calling conventions ----------------------------------------- */
 344
 345 #define CCALL_HANDLE_STRUCTRET \
 346   cc->retref = 1;  /* Return all structs by reference. */ \
 347   cc->gpr[ngpr++] = (GPRArg)dp;
 348
 349 #define CCALL_HANDLE_COMPLEXRET \
 350   /* Complex values are returned in 2 or 4 GPRs. */ \
 351   cc->retref = 0;
 352
 353 #define CCALL_HANDLE_COMPLEXRET2 \
 354   memcpy(dp, sp, ctr->size);  /* Copy complex from GPRs. */
 355
 356 #define CCALL_HANDLE_STRUCTARG \
 357   rp = cdataptr(lj_cdata_new(cts, did, sz)); \
 358   sz = CTSIZE_PTR;  /* Pass all structs by reference. */
 359
 360 #define CCALL_HANDLE_COMPLEXARG \
 361   /* Pass complex by value in 2 or 4 GPRs. */
 362
 363 /* PPC/SPE has a softfp ABI. */
 364 #define CCALL_HANDLE_REGARG \
 365   if (n > 1) {  /* Doesn't fit in a single GPR? */ \
 366     lua_assert(n == 2 || n == 4);  /* int64_t, double or complex (float). */ \
 367     if (n == 2) \
 368       ngpr = (ngpr + 1u) & ~1u;  /* Only align 64 bit value to regpair. */ \
 369     else if (ngpr + n > maxgpr) \
 370       ngpr = maxgpr;  /* Prevent reordering. */ \
 371   } \
 372   if (ngpr + n <= maxgpr) { \
 373     dp = &cc->gpr[ngpr]; \
 374     ngpr += n; \
 375     goto done; \
 376   }
 377
 378 #elif LJ_TARGET_MIPS
 379 /* -- MIPS calling conventions -------------------------------------------- */
 380
 381 #define CCALL_HANDLE_STRUCTRET \
 382   cc->retref = 1;  /* Return all structs by reference. */ \
 383   cc->gpr[ngpr++] = (GPRArg)dp;
 384
 385 #define CCALL_HANDLE_COMPLEXRET \
 386   /* Complex values are returned in 1 or 2 FPRs. */ \
 387   cc->retref = 0;
 388
 389 #define CCALL_HANDLE_COMPLEXRET2 \
 390   if (ctr->size == 2*sizeof(float)) {  /* Copy complex float from FPRs. */ \
 391     ((float *)dp)[0] = cc->fpr[0].f; \
 392     ((float *)dp)[1] = cc->fpr[1].f; \
 393   } else {  /* Copy complex double from FPRs. */ \
 394     ((double *)dp)[0] = cc->fpr[0].d; \
 395     ((double *)dp)[1] = cc->fpr[1].d; \
 396   }
 397
 398 #define CCALL_HANDLE_STRUCTARG \
 399   /* Pass all structs by value in registers and/or on the stack. */
 400
 401 #define CCALL_HANDLE_COMPLEXARG \
 402   /* Pass complex by value in 2 or 4 GPRs. */
 403
 404 #define CCALL_HANDLE_REGARG \
 405   if (isfp && nfpr < CCALL_NARG_FPR && !(ct->info & CTF_VARARG)) { \
 406     /* Try to pass argument in FPRs. */ \
 407     dp = n == 1 ? (void *)&cc->fpr[nfpr].f : (void *)&cc->fpr[nfpr].d; \
 408     nfpr++; ngpr += n; \
 409     goto done; \
 410   } else {  /* Try to pass argument in GPRs. */ \
 411     nfpr = CCALL_NARG_FPR; \
 412     if ((d->info & CTF_ALIGN) > CTALIGN_PTR) \
 413       ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
 414     if (ngpr < maxgpr) { \
 415       dp = &cc->gpr[ngpr]; \
 416       if (ngpr + n > maxgpr) { \
 417         nsp += ngpr + n - maxgpr;  /* Assumes contiguous gpr/stack fields. */ \
 418         if (nsp > CCALL_MAXSTACK) goto err_nyi;  /* Too many arguments. */ \
 419         ngpr = maxgpr; \
 420       } else { \
 421         ngpr += n; \
 422       } \
 423       goto done; \
 424     } \
 425   }
 426
 427 #define CCALL_HANDLE_RET \
 428   if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
 429     sp = (uint8_t *)&cc->fpr[0].f;
 430
 431 #else
 432 #error "Missing calling convention definitions for this architecture"
 433 #endif
 434
 435 #ifndef CCALL_HANDLE_STRUCTRET2
 436 #define CCALL_HANDLE_STRUCTRET2 \
 437   memcpy(dp, sp, ctr->size);  /* Copy struct return value from GPRs. */
 438 #endif
 439
 440 /* -- x86 OSX ABI struct classification ----------------------------------- */
 441
 442 #if LJ_TARGET_X86 && LJ_TARGET_OSX
 443
 444 /* Check for struct with single FP field. */
 445 static int ccall_classify_struct(CTState *cts, CType *ct)
 446 {
 447   CTSize sz = ct->size;
 448   if (!(sz == sizeof(float) || sz == sizeof(double))) return 0;
 449   if ((ct->info & CTF_UNION)) return 0;
 450   while (ct->sib) {
 451     ct = ctype_get(cts, ct->sib);
 452     if (ctype_isfield(ct->info)) {
 453       CType *sct = ctype_rawchild(cts, ct);
 454       if (ctype_isfp(sct->info)) {
 455         if (sct->size == sz)
 456           return (sz >> 2);  /* Return 1 for float or 2 for double. */
 457       } else if (ctype_isstruct(sct->info)) {
 458         if (sct->size)
 459           return ccall_classify_struct(cts, sct);
 460       } else {
 461         break;
 462       }
 463     } else if (ctype_isbitfield(ct->info)) {
 464       break;
 465     } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 466       CType *sct = ctype_rawchild(cts, ct);
 467       if (sct->size)
 468         return ccall_classify_struct(cts, sct);
 469     }
 470   }
 471   return 0;
 472 }
 473
 474 #endif
 475
 476 /* -- x64 struct classification ------------------------------------------- */
 477
 478 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 479
 480 /* Register classes for x64 struct classification. */
 481 #define CCALL_RCL_INT   1
 482 #define CCALL_RCL_SSE   2
 483 #define CCALL_RCL_MEM   4
 484 /* NYI: classify vectors. */
 485
 486 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs);
 487
 488 /* Classify a C type. */
 489 static void ccall_classify_ct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 490 {
 491   if (ctype_isarray(ct->info)) {
 492     CType *cct = ctype_rawchild(cts, ct);
 493     CTSize eofs, esz = cct->size, asz = ct->size;
 494     for (eofs = 0; eofs < asz; eofs += esz)
 495       ccall_classify_ct(cts, cct, rcl, ofs+eofs);
 496   } else if (ctype_isstruct(ct->info)) {
 497     ccall_classify_struct(cts, ct, rcl, ofs);
 498   } else {
 499     int cl = ctype_isfp(ct->info) ? CCALL_RCL_SSE : CCALL_RCL_INT;
 500     lua_assert(ctype_hassize(ct->info));
 501     if ((ofs & (ct->size-1))) cl = CCALL_RCL_MEM;  /* Unaligned. */
 502     rcl[(ofs >= 8)] |= cl;
 503   }
 504 }
 505
 506 /* Recursively classify a struct based on its fields. */
 507 static int ccall_classify_struct(CTState *cts, CType *ct, int *rcl, CTSize ofs)
 508 {
 509   if (ct->size > 16) return CCALL_RCL_MEM;  /* Too big, gets memory class. */
 510   while (ct->sib) {
 511     CTSize fofs;
 512     ct = ctype_get(cts, ct->sib);
 513     fofs = ofs+ct->size;
 514     if (ctype_isfield(ct->info))
 515       ccall_classify_ct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 516     else if (ctype_isbitfield(ct->info))
 517       rcl[(fofs >= 8)] |= CCALL_RCL_INT;  /* NYI: unaligned bitfields? */
 518     else if (ctype_isxattrib(ct->info, CTA_SUBTYPE))
 519       ccall_classify_struct(cts, ctype_rawchild(cts, ct), rcl, fofs);
 520   }
 521   return ((rcl[0]|rcl[1]) & CCALL_RCL_MEM);  /* Memory class? */
 522 }
 523
 524 /* Try to split up a small struct into registers. */
 525 static int ccall_struct_reg(CCallState *cc, GPRArg *dp, int *rcl)
 526 {
 527   MSize ngpr = cc->ngpr, nfpr = cc->nfpr;
 528   uint32_t i;
 529   for (i = 0; i < 2; i++) {
 530     lua_assert(!(rcl[i] & CCALL_RCL_MEM));
 531     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 532       if (ngpr >= CCALL_NARG_GPR) return 1;  /* Register overflow. */
 533       cc->gpr[ngpr++] = dp[i];
 534     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 535       if (nfpr >= CCALL_NARG_FPR) return 1;  /* Register overflow. */
 536       cc->fpr[nfpr++].l[0] = dp[i];
 537     }
 538   }
 539   cc->ngpr = ngpr; cc->nfpr = nfpr;
 540   return 0;  /* Ok. */
 541 }
 542
 543 /* Pass a small struct argument. */
 544 static int ccall_struct_arg(CCallState *cc, CTState *cts, CType *d, int *rcl,
 545                             TValue *o, int narg)
 546 {
 547   GPRArg dp[2];
 548   dp[0] = dp[1] = 0;
 549   /* Convert to temp. struct. */
 550   lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 551   if (ccall_struct_reg(cc, dp, rcl)) {  /* Register overflow? Pass on stack. */
 552     MSize nsp = cc->nsp, n = rcl[1] ? 2 : 1;
 553     if (nsp + n > CCALL_MAXSTACK) return 1;  /* Too many arguments. */
 554     cc->nsp = nsp + n;
 555     memcpy(&cc->stack[nsp], dp, n*CTSIZE_PTR);
 556   }
 557   return 0;  /* Ok. */
 558 }
 559
 560 /* Combine returned small struct. */
 561 static void ccall_struct_ret(CCallState *cc, int *rcl, uint8_t *dp, CTSize sz)
 562 {
 563   GPRArg sp[2];
 564   MSize ngpr = 0, nfpr = 0;
 565   uint32_t i;
 566   for (i = 0; i < 2; i++) {
 567     if ((rcl[i] & CCALL_RCL_INT)) {  /* Integer class takes precedence. */
 568       sp[i] = cc->gpr[ngpr++];
 569     } else if ((rcl[i] & CCALL_RCL_SSE)) {
 570       sp[i] = cc->fpr[nfpr++].l[0];
 571     }
 572   }
 573   memcpy(dp, sp, sz);
 574 }
 575 #endif
 576
 577 /* -- ARM hard-float ABI struct classification ---------------------------- */
 578
 579 #if LJ_TARGET_ARM && !LJ_ABI_SOFTFP
 580
 581 /* Classify a struct based on its fields. */
 582 static unsigned int ccall_classify_struct(CTState *cts, CType *ct, CType *ctf)
 583 {
 584   CTSize sz = ct->size;
 585   unsigned int r = 0, n = 0, isu = (ct->info & CTF_UNION);
 586   if ((ctf->info & CTF_VARARG)) goto noth;
 587   while (ct->sib) {
 588     CType *sct;
 589     ct = ctype_get(cts, ct->sib);
 590     if (ctype_isfield(ct->info)) {
 591       sct = ctype_rawchild(cts, ct);
 592       if (ctype_isfp(sct->info)) {
 593         r |= sct->size;
 594         if (!isu) n++; else if (n == 0) n = 1;
 595       } else if (ctype_iscomplex(sct->info)) {
 596         r |= (sct->size >> 1);
 597         if (!isu) n += 2; else if (n < 2) n = 2;
 598       } else if (ctype_isstruct(sct->info)) {
 599         goto substruct;
 600       } else {
 601         goto noth;
 602       }
 603     } else if (ctype_isbitfield(ct->info)) {
 604       goto noth;
 605     } else if (ctype_isxattrib(ct->info, CTA_SUBTYPE)) {
 606       sct = ctype_rawchild(cts, ct);
 607     substruct:
 608       if (sct->size > 0) {
 609         unsigned int s = ccall_classify_struct(cts, sct, ctf);
 610         if (s <= 1) goto noth;
 611         r |= (s & 255);
 612         if (!isu) n += (s >> 8); else if (n < (s >>8)) n = (s >> 8);
 613       }
 614     }
 615   }
 616   if ((r == 4 || r == 8) && n <= 4)
 617     return r + (n << 8);
 618 noth:  /* Not a homogeneous float/double aggregate. */
 619   return (sz <= 4);  /* Return structs of size <= 4 in a GPR. */
 620 }
 621
 622 #endif
 623
 624 /* -- Common C call handling ---------------------------------------------- */
 625
 626 /* Infer the destination CTypeID for a vararg argument. */
 627 CTypeID lj_ccall_ctid_vararg(CTState *cts, cTValue *o)
 628 {
 629   if (tvisnumber(o)) {
 630     return CTID_DOUBLE;
 631   } else if (tviscdata(o)) {
 632     CTypeID id = cdataV(o)->ctypeid;
 633     CType *s = ctype_get(cts, id);
 634     if (ctype_isrefarray(s->info)) {
 635       return lj_ctype_intern(cts,
 636                CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(s->info)), CTSIZE_PTR);
 637     } else if (ctype_isstruct(s->info) || ctype_isfunc(s->info)) {
 638       /* NYI: how to pass a struct by value in a vararg argument? */
 639       return lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR);
 640     } else if (ctype_isfp(s->info) && s->size == sizeof(float)) {
 641       return CTID_DOUBLE;
 642     } else {
 643       return id;
 644     }
 645   } else if (tvisstr(o)) {
 646     return CTID_P_CCHAR;
 647   } else if (tvisbool(o)) {
 648     return CTID_BOOL;
 649   } else {
 650     return CTID_P_VOID;
 651   }
 652 }
 653
 654 /* Setup arguments for C call. */
 655 static int ccall_set_args(lua_State *L, CTState *cts, CType *ct,
 656                           CCallState *cc)
 657 {
 658   int gcsteps = 0;
 659   TValue *o, *top = L->top;
 660   CTypeID fid;
 661   CType *ctr;
 662   MSize maxgpr, ngpr = 0, nsp = 0, narg;
 663 #if CCALL_NARG_FPR
 664   MSize nfpr = 0;
 665 #if LJ_TARGET_ARM
 666   MSize fprodd = 0;
 667 #endif
 668 #endif
 669
 670   /* Clear unused regs to get some determinism in case of misdeclaration. */
 671   memset(cc->gpr, 0, sizeof(cc->gpr));
 672 #if CCALL_NUM_FPR
 673   memset(cc->fpr, 0, sizeof(cc->fpr));
 674 #endif
 675
 676 #if LJ_TARGET_X86
 677   /* x86 has several different calling conventions. */
 678   cc->resx87 = 0;
 679   switch (ctype_cconv(ct->info)) {
 680   case CTCC_FASTCALL: maxgpr = 2; break;
 681   case CTCC_THISCALL: maxgpr = 1; break;
 682   default: maxgpr = 0; break;
 683   }
 684 #else
 685   maxgpr = CCALL_NARG_GPR;
 686 #endif
 687
 688   /* Perform required setup for some result types. */
 689   ctr = ctype_rawchild(cts, ct);
 690   if (ctype_isvector(ctr->info)) {
 691     if (!(CCALL_VECTOR_REG && (ctr->size == 8 || ctr->size == 16)))
 692       goto err_nyi;
 693   } else if (ctype_iscomplex(ctr->info) || ctype_isstruct(ctr->info)) {
 694     /* Preallocate cdata object and anchor it after arguments. */
 695     CTSize sz = ctr->size;
 696     GCcdata *cd = lj_cdata_new(cts, ctype_cid(ct->info), sz);
 697     void *dp = cdataptr(cd);
 698     setcdataV(L, L->top++, cd);
 699     if (ctype_isstruct(ctr->info)) {
 700       CCALL_HANDLE_STRUCTRET
 701     } else {
 702       CCALL_HANDLE_COMPLEXRET
 703     }
 704 #if LJ_TARGET_X86
 705   } else if (ctype_isfp(ctr->info)) {
 706     cc->resx87 = ctr->size == sizeof(float) ? 1 : 2;
 707 #endif
 708   }
 709
 710   /* Skip initial attributes. */
 711   fid = ct->sib;
 712   while (fid) {
 713     CType *ctf = ctype_get(cts, fid);
 714     if (!ctype_isattrib(ctf->info)) break;
 715     fid = ctf->sib;
 716   }
 717
 718   /* Walk through all passed arguments. */
 719   for (o = L->base+1, narg = 1; o < top; o++, narg++) {
 720     CTypeID did;
 721     CType *d;
 722     CTSize sz;
 723     MSize n, isfp = 0, isva = 0;
 724     void *dp, *rp = NULL;
 725
 726     if (fid) {  /* Get argument type from field. */
 727       CType *ctf = ctype_get(cts, fid);
 728       fid = ctf->sib;
 729       lua_assert(ctype_isfield(ctf->info));
 730       did = ctype_cid(ctf->info);
 731     } else {
 732       if (!(ct->info & CTF_VARARG))
 733         lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too many arguments. */
 734       did = lj_ccall_ctid_vararg(cts, o);  /* Infer vararg type. */
 735       isva = 1;
 736     }
 737     d = ctype_raw(cts, did);
 738     sz = d->size;
 739
 740     /* Find out how (by value/ref) and where (GPR/FPR) to pass an argument. */
 741     if (ctype_isnum(d->info)) {
 742       if (sz > 8) goto err_nyi;
 743       if ((d->info & CTF_FP))
 744         isfp = 1;
 745     } else if (ctype_isvector(d->info)) {
 746       if (CCALL_VECTOR_REG && (sz == 8 || sz == 16))
 747         isfp = 1;
 748       else
 749         goto err_nyi;
 750     } else if (ctype_isstruct(d->info)) {
 751       CCALL_HANDLE_STRUCTARG
 752     } else if (ctype_iscomplex(d->info)) {
 753       CCALL_HANDLE_COMPLEXARG
 754     } else {
 755       sz = CTSIZE_PTR;
 756     }
 757     sz = (sz + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
 758     n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
 759
 760     CCALL_HANDLE_REGARG  /* Handle register arguments. */
 761
 762     /* Otherwise pass argument on stack. */
 763     if (CCALL_ALIGN_STACKARG && !rp && (d->info & CTF_ALIGN) > CTALIGN_PTR) {
 764       MSize align = (1u << ctype_align(d->info-CTALIGN_PTR)) -1;
 765       nsp = (nsp + align) & ~align;  /* Align argument on stack. */
 766     }
 767     if (nsp + n > CCALL_MAXSTACK) {  /* Too many arguments. */
 768     err_nyi:
 769       lj_err_caller(L, LJ_ERR_FFI_NYICALL);
 770     }
 771     dp = &cc->stack[nsp];
 772     nsp += n;
 773     isva = 0;
 774
 775   done:
 776     if (rp) {  /* Pass by reference. */
 777       gcsteps++;
 778       *(void **)dp = rp;
 779       dp = rp;
 780     }
 781     lj_cconv_ct_tv(cts, d, (uint8_t *)dp, o, CCF_ARG(narg));
 782     /* Extend passed integers to 32 bits at least. */
 783     if (ctype_isinteger_or_bool(d->info) && d->size < 4) {
 784       if (d->info & CTF_UNSIGNED)
 785         *(uint32_t *)dp = d->size == 1 ? (uint32_t)*(uint8_t *)dp :
 786                                          (uint32_t)*(uint16_t *)dp;
 787       else
 788         *(int32_t *)dp = d->size == 1 ? (int32_t)*(int8_t *)dp :
 789                                         (int32_t)*(int16_t *)dp;
 790     }
 791 #if LJ_TARGET_X64 && LJ_ABI_WIN
 792     if (isva) {  /* Windows/x64 mirrors varargs in both register sets. */
 793       if (nfpr == ngpr)
 794         cc->gpr[ngpr-1] = cc->fpr[ngpr-1].l[0];
 795       else
 796         cc->fpr[ngpr-1].l[0] = cc->gpr[ngpr-1];
 797     }
 798 #else
 799     UNUSED(isva);
 800 #endif
 801 #if LJ_TARGET_X64 && !LJ_ABI_WIN
 802     if (isfp == 2 && n == 2 && (uint8_t *)dp == (uint8_t *)&cc->fpr[nfpr-2]) {
 803       cc->fpr[nfpr-1].d[0] = cc->fpr[nfpr-2].d[1];  /* Split complex double. */
 804       cc->fpr[nfpr-2].d[1] = 0;
 805     }
 806 #else
 807     UNUSED(isfp);
 808 #endif
 809   }
 810   if (fid) lj_err_caller(L, LJ_ERR_FFI_NUMARG);  /* Too few arguments. */
 811
 812 #if LJ_TARGET_X64 || LJ_TARGET_PPC
 813   cc->nfpr = nfpr;  /* Required for vararg functions. */
 814 #endif
 815   cc->nsp = nsp;
 816   cc->spadj = (CCALL_SPS_FREE + CCALL_SPS_EXTRA)*CTSIZE_PTR;
 817   if (nsp > CCALL_SPS_FREE)
 818     cc->spadj += (((nsp-CCALL_SPS_FREE)*CTSIZE_PTR + 15u) & ~15u);
 819   return gcsteps;
 820 }
 821
 822 /* Get results from C call. */
 823 static int ccall_get_results(lua_State *L, CTState *cts, CType *ct,
 824                              CCallState *cc, int *ret)
 825 {
 826   CType *ctr = ctype_rawchild(cts, ct);
 827   uint8_t *sp = (uint8_t *)&cc->gpr[0];
 828   if (ctype_isvoid(ctr->info)) {
 829     *ret = 0;  /* Zero results. */
 830     return 0;  /* No additional GC step. */
 831   }
 832   *ret = 1;  /* One result. */
 833   if (ctype_isstruct(ctr->info)) {
 834     /* Return cdata object which is already on top of stack. */
 835     if (!cc->retref) {
 836       void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 837       CCALL_HANDLE_STRUCTRET2
 838     }
 839     return 1;  /* One GC step. */
 840   }
 841   if (ctype_iscomplex(ctr->info)) {
 842     /* Return cdata object which is already on top of stack. */
 843     void *dp = cdataptr(cdataV(L->top-1));  /* Use preallocated object. */
 844     CCALL_HANDLE_COMPLEXRET2
 845     return 1;  /* One GC step. */
 846   }
 847   if (LJ_BE && ctype_isinteger_or_bool(ctr->info) && ctr->size < CTSIZE_PTR)
 848     sp += (CTSIZE_PTR - ctr->size);
 849 #if CCALL_NUM_FPR
 850   if (ctype_isfp(ctr->info) || ctype_isvector(ctr->info))
 851     sp = (uint8_t *)&cc->fpr[0];
 852 #endif
 853 #ifdef CCALL_HANDLE_RET
 854   CCALL_HANDLE_RET
 855 #endif
 856   /* No reference types end up here, so there's no need for the CTypeID. */
 857   lua_assert(!(ctype_isrefarray(ctr->info) || ctype_isstruct(ctr->info)));
 858   return lj_cconv_tv_ct(cts, ctr, 0, L->top-1, sp);
 859 }
 860
 861 /* Call C function. */
 862 int lj_ccall_func(lua_State *L, GCcdata *cd)
 863 {
 864   CTState *cts = ctype_cts(L);
 865   CType *ct = ctype_raw(cts, cd->ctypeid);
 866   CTSize sz = CTSIZE_PTR;
 867   if (ctype_isptr(ct->info)) {
 868     sz = ct->size;
 869     ct = ctype_rawchild(cts, ct);
 870   }
 871   if (ctype_isfunc(ct->info)) {
 872     CCallState cc;
 873     int gcsteps, ret;
 874     cc.func = (void (*)(void))cdata_getptr(cdataptr(cd), sz);
 875     gcsteps = ccall_set_args(L, cts, ct, &cc);
 876     ct = (CType *)((intptr_t)ct-(intptr_t)cts->tab);
 877     cts->cb.slot = ~0u;
 878     lj_vm_ffi_call(&cc);
 879     if (cts->cb.slot != ~0u) {  /* Blacklist function that called a callback. */
 880       TValue tv;
 881       setlightudV(&tv, (void *)cc.func);
 882       setboolV(lj_tab_set(L, cts->miscmap, &tv), 1);
 883     }
 884     ct = (CType *)((intptr_t)ct+(intptr_t)cts->tab);  /* May be reallocated. */
 885     gcsteps += ccall_get_results(L, cts, ct, &cc, &ret);
 886 #if LJ_TARGET_X86 && LJ_ABI_WIN
 887     /* Automatically detect __stdcall and fix up C function declaration. */
 888     if (cc.spadj && ctype_cconv(ct->info) == CTCC_CDECL) {
 889       CTF_INSERT(ct->info, CCONV, CTCC_STDCALL);
 890       lj_trace_abort(G(L));
 891     }
 892 #endif
 893     while (gcsteps-- > 0)
 894       lj_gc_check(L);
 895     return ret;
 896   }
 897   return -1;  /* Not a function. */
 898 }
 899
 900 #endif