libffi/src/arm/ffi.c

   1 /* -----------------------------------------------------------------------
   2    ffi.c - Copyright (c) 2011 Timothy Wall
   3            Copyright (c) 2011 Plausible Labs Cooperative, Inc.
   4            Copyright (c) 2011 Anthony Green
   5            Copyright (c) 2011 Free Software Foundation
   6            Copyright (c) 1998, 2008, 2011  Red Hat, Inc.
   7
   8    ARM Foreign Function Interface
   9
  10    Permission is hereby granted, free of charge, to any person obtaining
  11    a copy of this software and associated documentation files (the
  12    ``Software''), to deal in the Software without restriction, including
  13    without limitation the rights to use, copy, modify, merge, publish,
  14    distribute, sublicense, and/or sell copies of the Software, and to
  15    permit persons to whom the Software is furnished to do so, subject to
  16    the following conditions:
  17
  18    The above copyright notice and this permission notice shall be included
  19    in all copies or substantial portions of the Software.
  20
  21    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  22    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  23    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  24    NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  25    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  26    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  27    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  28    DEALINGS IN THE SOFTWARE.
  29    ----------------------------------------------------------------------- */
  30
  31 #include <ffi.h>
  32 #include <ffi_common.h>
  33 #include <stdlib.h>
  34 #include "internal.h"
  35
  36 /* Forward declares. */
  37 static int vfp_type_p (const ffi_type *);
  38 static void layout_vfp_args (ffi_cif *);
  39
  40 static void *
  41 ffi_align (ffi_type *ty, void *p)
  42 {
  43   /* Align if necessary */
  44   size_t alignment;
  45 #ifdef _WIN32_WCE
  46   alignment = 4;
  47 #else
  48   alignment = ty->alignment;
  49   if (alignment < 4)
  50     alignment = 4;
  51 #endif
  52   return (void *) ALIGN (p, alignment);
  53 }
  54
  55 static size_t
  56 ffi_put_arg (ffi_type *ty, void *src, void *dst)
  57 {
  58   size_t z = ty->size;
  59
  60   switch (ty->type)
  61     {
  62     case FFI_TYPE_SINT8:
  63       *(UINT32 *)dst = *(SINT8 *)src;
  64       break;
  65     case FFI_TYPE_UINT8:
  66       *(UINT32 *)dst = *(UINT8 *)src;
  67       break;
  68     case FFI_TYPE_SINT16:
  69       *(UINT32 *)dst = *(SINT16 *)src;
  70       break;
  71     case FFI_TYPE_UINT16:
  72       *(UINT32 *)dst = *(UINT16 *)src;
  73       break;
  74
  75     case FFI_TYPE_INT:
  76     case FFI_TYPE_SINT32:
  77     case FFI_TYPE_UINT32:
  78     case FFI_TYPE_POINTER:
  79     case FFI_TYPE_FLOAT:
  80       *(UINT32 *)dst = *(UINT32 *)src;
  81       break;
  82
  83     case FFI_TYPE_SINT64:
  84     case FFI_TYPE_UINT64:
  85     case FFI_TYPE_DOUBLE:
  86       *(UINT64 *)dst = *(UINT64 *)src;
  87       break;
  88
  89     case FFI_TYPE_STRUCT:
  90     case FFI_TYPE_COMPLEX:
  91       memcpy (dst, src, z);
  92       break;
  93
  94     default:
  95       abort();
  96     }
  97
  98   return ALIGN (z, 4);
  99 }
 100
 101 /* ffi_prep_args is called once stack space has been allocated
 102    for the function's arguments.
 103
 104    The vfp_space parameter is the load area for VFP regs, the return
 105    value is cif->vfp_used (word bitset of VFP regs used for passing
 106    arguments). These are only used for the VFP hard-float ABI.
 107 */
 108 static void
 109 ffi_prep_args_SYSV (ffi_cif *cif, int flags, void *rvalue,
 110                     void **avalue, char *argp)
 111 {
 112   ffi_type **arg_types = cif->arg_types;
 113   int i, n;
 114
 115   if (flags == ARM_TYPE_STRUCT)
 116     {
 117       *(void **) argp = rvalue;
 118       argp += 4;
 119     }
 120
 121   for (i = 0, n = cif->nargs; i < n; i++)
 122     {
 123       ffi_type *ty = arg_types[i];
 124       argp = ffi_align (ty, argp);
 125       argp += ffi_put_arg (ty, avalue[i], argp);
 126     }
 127 }
 128
 129 static void
 130 ffi_prep_args_VFP (ffi_cif *cif, int flags, void *rvalue,
 131                    void **avalue, char *stack, char *vfp_space)
 132 {
 133   ffi_type **arg_types = cif->arg_types;
 134   int i, n, vi = 0;
 135   char *argp, *regp, *eo_regp;
 136   char stack_used = 0;
 137   char done_with_regs = 0;
 138
 139   /* The first 4 words on the stack are used for values
 140      passed in core registers.  */
 141   regp = stack;
 142   eo_regp = argp = regp + 16;
 143
 144   /* If the function returns an FFI_TYPE_STRUCT in memory,
 145      that address is passed in r0 to the function.  */
 146   if (flags == ARM_TYPE_STRUCT)
 147     {
 148       *(void **) regp = rvalue;
 149       regp += 4;
 150     }
 151
 152   for (i = 0, n = cif->nargs; i < n; i++)
 153     {
 154       ffi_type *ty = arg_types[i];
 155       void *a = avalue[i];
 156       int is_vfp_type = vfp_type_p (ty);
 157
 158       /* Allocated in VFP registers. */
 159       if (vi < cif->vfp_nargs && is_vfp_type)
 160         {
 161           char *vfp_slot = vfp_space + cif->vfp_args[vi++] * 4;
 162           ffi_put_arg (ty, a, vfp_slot);
 163           continue;
 164         }
 165       /* Try allocating in core registers. */
 166       else if (!done_with_regs && !is_vfp_type)
 167         {
 168           char *tregp = ffi_align (ty, regp);
 169           size_t size = ty->size;
 170           size = (size < 4) ? 4 : size; // pad
 171           /* Check if there is space left in the aligned register
 172              area to place the argument.  */
 173           if (tregp + size <= eo_regp)
 174             {
 175               regp = tregp + ffi_put_arg (ty, a, tregp);
 176               done_with_regs = (regp == argp);
 177               // ensure we did not write into the stack area
 178               FFI_ASSERT (regp <= argp);
 179               continue;
 180             }
 181           /* In case there are no arguments in the stack area yet,
 182              the argument is passed in the remaining core registers
 183              and on the stack.  */
 184           else if (!stack_used)
 185             {
 186               stack_used = 1;
 187               done_with_regs = 1;
 188               argp = tregp + ffi_put_arg (ty, a, tregp);
 189               FFI_ASSERT (eo_regp < argp);
 190               continue;
 191             }
 192         }
 193       /* Base case, arguments are passed on the stack */
 194       stack_used = 1;
 195       argp = ffi_align (ty, argp);
 196       argp += ffi_put_arg (ty, a, argp);
 197     }
 198 }
 199
 200 /* Perform machine dependent cif processing */
 201 ffi_status
 202 ffi_prep_cif_machdep (ffi_cif *cif)
 203 {
 204   int flags = 0, cabi = cif->abi;
 205   size_t bytes = cif->bytes;
 206
 207   /* Map out the register placements of VFP register args.  The VFP
 208      hard-float calling conventions are slightly more sophisticated
 209      than the base calling conventions, so we do it here instead of
 210      in ffi_prep_args(). */
 211   if (cabi == FFI_VFP)
 212     layout_vfp_args (cif);
 213
 214   /* Set the return type flag */
 215   switch (cif->rtype->type)
 216     {
 217     case FFI_TYPE_VOID:
 218       flags = ARM_TYPE_VOID;
 219       break;
 220
 221     case FFI_TYPE_INT:
 222     case FFI_TYPE_UINT8:
 223     case FFI_TYPE_SINT8:
 224     case FFI_TYPE_UINT16:
 225     case FFI_TYPE_SINT16:
 226     case FFI_TYPE_UINT32:
 227     case FFI_TYPE_SINT32:
 228     case FFI_TYPE_POINTER:
 229       flags = ARM_TYPE_INT;
 230       break;
 231
 232     case FFI_TYPE_SINT64:
 233     case FFI_TYPE_UINT64:
 234       flags = ARM_TYPE_INT64;
 235       break;
 236
 237     case FFI_TYPE_FLOAT:
 238       flags = (cabi == FFI_VFP ? ARM_TYPE_VFP_S : ARM_TYPE_INT);
 239       break;
 240     case FFI_TYPE_DOUBLE:
 241       flags = (cabi == FFI_VFP ? ARM_TYPE_VFP_D : ARM_TYPE_INT64);
 242       break;
 243
 244     case FFI_TYPE_STRUCT:
 245     case FFI_TYPE_COMPLEX:
 246       if (cabi == FFI_VFP)
 247         {
 248           int h = vfp_type_p (cif->rtype);
 249
 250           flags = ARM_TYPE_VFP_N;
 251           if (h == 0x100 + FFI_TYPE_FLOAT)
 252             flags = ARM_TYPE_VFP_S;
 253           if (h == 0x100 + FFI_TYPE_DOUBLE)
 254             flags = ARM_TYPE_VFP_D;
 255           if (h != 0)
 256               break;
 257         }
 258
 259       /* A Composite Type not larger than 4 bytes is returned in r0.
 260          A Composite Type larger than 4 bytes, or whose size cannot
 261          be determined statically ... is stored in memory at an
 262          address passed [in r0].  */
 263       if (cif->rtype->size <= 4)
 264         flags = ARM_TYPE_INT;
 265       else
 266         {
 267           flags = ARM_TYPE_STRUCT;
 268           bytes += 4;
 269         }
 270       break;
 271
 272     default:
 273       abort();
 274     }
 275
 276   /* Round the stack up to a multiple of 8 bytes.  This isn't needed
 277      everywhere, but it is on some platforms, and it doesn't harm anything
 278      when it isn't needed.  */
 279   bytes = ALIGN (bytes, 8);
 280
 281   /* Minimum stack space is the 4 register arguments that we pop.  */
 282   if (bytes < 4*4)
 283     bytes = 4*4;
 284
 285   cif->bytes = bytes;
 286   cif->flags = flags;
 287
 288   return FFI_OK;
 289 }
 290
 291 /* Perform machine dependent cif processing for variadic calls */
 292 ffi_status
 293 ffi_prep_cif_machdep_var (ffi_cif * cif,
 294                           unsigned int nfixedargs, unsigned int ntotalargs)
 295 {
 296   /* VFP variadic calls actually use the SYSV ABI */
 297   if (cif->abi == FFI_VFP)
 298     cif->abi = FFI_SYSV;
 299
 300   return ffi_prep_cif_machdep (cif);
 301 }
 302
 303 /* Prototypes for assembly functions, in sysv.S.  */
 304
 305 struct call_frame
 306 {
 307   void *fp;
 308   void *lr;
 309   void *rvalue;
 310   int flags;
 311   void *closure;
 312 };
 313
 314 extern void ffi_call_SYSV (void *stack, struct call_frame *,
 315                            void (*fn) (void)) FFI_HIDDEN;
 316 extern void ffi_call_VFP (void *vfp_space, struct call_frame *,
 317                            void (*fn) (void), unsigned vfp_used) FFI_HIDDEN;
 318
 319 static void
 320 ffi_call_int (ffi_cif * cif, void (*fn) (void), void *rvalue,
 321               void **avalue, void *closure)
 322 {
 323   int flags = cif->flags;
 324   ffi_type *rtype = cif->rtype;
 325   size_t bytes, rsize, vfp_size;
 326   char *stack, *vfp_space, *new_rvalue;
 327   struct call_frame *frame;
 328
 329   rsize = 0;
 330   if (rvalue == NULL)
 331     {
 332       /* If the return value is a struct and we don't have a return
 333          value address then we need to make one.  Otherwise the return
 334          value is in registers and we can ignore them.  */
 335       if (flags == ARM_TYPE_STRUCT)
 336         rsize = rtype->size;
 337       else
 338         flags = ARM_TYPE_VOID;
 339     }
 340   else if (flags == ARM_TYPE_VFP_N)
 341     {
 342       /* Largest case is double x 4. */
 343       rsize = 32;
 344     }
 345   else if (flags == ARM_TYPE_INT && rtype->type == FFI_TYPE_STRUCT)
 346     rsize = 4;
 347
 348   /* Largest case.  */
 349   vfp_size = (cif->abi == FFI_VFP && cif->vfp_used ? 8*8: 0);
 350
 351   bytes = cif->bytes;
 352   stack = alloca (vfp_size + bytes + sizeof(struct call_frame) + rsize);
 353
 354   vfp_space = NULL;
 355   if (vfp_size)
 356     {
 357       vfp_space = stack;
 358       stack += vfp_size;
 359     }
 360
 361   frame = (struct call_frame *)(stack + bytes);
 362
 363   new_rvalue = rvalue;
 364   if (rsize)
 365     new_rvalue = (void *)(frame + 1);
 366
 367   frame->rvalue = new_rvalue;
 368   frame->flags = flags;
 369   frame->closure = closure;
 370
 371   if (vfp_space)
 372     {
 373       ffi_prep_args_VFP (cif, flags, new_rvalue, avalue, stack, vfp_space);
 374       ffi_call_VFP (vfp_space, frame, fn, cif->vfp_used);
 375     }
 376   else
 377     {
 378       ffi_prep_args_SYSV (cif, flags, new_rvalue, avalue, stack);
 379       ffi_call_SYSV (stack, frame, fn);
 380     }
 381
 382   if (rvalue && rvalue != new_rvalue)
 383     memcpy (rvalue, new_rvalue, rtype->size);
 384 }
 385
 386 void
 387 ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
 388 {
 389   ffi_call_int (cif, fn, rvalue, avalue, NULL);
 390 }
 391
 392 void
 393 ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
 394              void **avalue, void *closure)
 395 {
 396   ffi_call_int (cif, fn, rvalue, avalue, closure);
 397 }
 398
 399 static void *
 400 ffi_prep_incoming_args_SYSV (ffi_cif *cif, void *rvalue,
 401                              char *argp, void **avalue)
 402 {
 403   ffi_type **arg_types = cif->arg_types;
 404   int i, n;
 405
 406   if (cif->flags == ARM_TYPE_STRUCT)
 407     {
 408       rvalue = *(void **) argp;
 409       argp += 4;
 410     }
 411
 412   for (i = 0, n = cif->nargs; i < n; i++)
 413     {
 414       ffi_type *ty = arg_types[i];
 415       size_t z = ty->size;
 416
 417       argp = ffi_align (ty, argp);
 418       avalue[i] = (void *) argp;
 419       argp += z;
 420     }
 421
 422   return rvalue;
 423 }
 424
 425 static void *
 426 ffi_prep_incoming_args_VFP (ffi_cif *cif, void *rvalue, char *stack,
 427                             char *vfp_space, void **avalue)
 428 {
 429   ffi_type **arg_types = cif->arg_types;
 430   int i, n, vi = 0;
 431   char *argp, *regp, *eo_regp;
 432   char done_with_regs = 0;
 433   char stack_used = 0;
 434
 435   regp = stack;
 436   eo_regp = argp = regp + 16;
 437
 438   if (cif->flags == ARM_TYPE_STRUCT)
 439     {
 440       rvalue = *(void **) regp;
 441       regp += 4;
 442     }
 443
 444   for (i = 0, n = cif->nargs; i < n; i++)
 445     {
 446       ffi_type *ty = arg_types[i];
 447       int is_vfp_type = vfp_type_p (ty);
 448       size_t z = ty->size;
 449
 450       if (vi < cif->vfp_nargs && is_vfp_type)
 451         {
 452           avalue[i] = vfp_space + cif->vfp_args[vi++] * 4;
 453           continue;
 454         }
 455       else if (!done_with_regs && !is_vfp_type)
 456         {
 457           char *tregp = ffi_align (ty, regp);
 458
 459           z = (z < 4) ? 4 : z;  // pad
 460
 461           /* If the arguments either fits into the registers or uses registers
 462              and stack, while we haven't read other things from the stack */
 463           if (tregp + z <= eo_regp || !stack_used)
 464             {
 465               /* Because we're little endian, this is what it turns into.  */
 466               avalue[i] = (void *) tregp;
 467               regp = tregp + z;
 468
 469               /* If we read past the last core register, make sure we
 470                  have not read from the stack before and continue
 471                  reading after regp.  */
 472               if (regp > eo_regp)
 473                 {
 474                   FFI_ASSERT (!stack_used);
 475                   argp = regp;
 476                 }
 477               if (regp >= eo_regp)
 478                 {
 479                   done_with_regs = 1;
 480                   stack_used = 1;
 481                 }
 482               continue;
 483             }
 484         }
 485
 486       stack_used = 1;
 487       argp = ffi_align (ty, argp);
 488       avalue[i] = (void *) argp;
 489       argp += z;
 490     }
 491
 492   return rvalue;
 493 }
 494
 495 struct closure_frame
 496 {
 497   char vfp_space[8*8] __attribute__((aligned(8)));
 498   char result[8*4];
 499   char argp[];
 500 };
 501
 502 int FFI_HIDDEN
 503 ffi_closure_inner_SYSV (ffi_cif *cif,
 504                         void (*fun) (ffi_cif *, void *, void **, void *),
 505                         void *user_data,
 506                         struct closure_frame *frame)
 507 {
 508   void **avalue = (void **) alloca (cif->nargs * sizeof (void *));
 509   void *rvalue = ffi_prep_incoming_args_SYSV (cif, frame->result,
 510                                               frame->argp, avalue);
 511   fun (cif, rvalue, avalue, user_data);
 512   return cif->flags;
 513 }
 514
 515 int FFI_HIDDEN
 516 ffi_closure_inner_VFP (ffi_cif *cif,
 517                        void (*fun) (ffi_cif *, void *, void **, void *),
 518                        void *user_data,
 519                        struct closure_frame *frame)
 520 {
 521   void **avalue = (void **) alloca (cif->nargs * sizeof (void *));
 522   void *rvalue = ffi_prep_incoming_args_VFP (cif, frame->result, frame->argp,
 523                                              frame->vfp_space, avalue);
 524   fun (cif, rvalue, avalue, user_data);
 525   return cif->flags;
 526 }
 527
 528 void ffi_closure_SYSV (void) FFI_HIDDEN;
 529 void ffi_closure_VFP (void) FFI_HIDDEN;
 530 void ffi_go_closure_SYSV (void) FFI_HIDDEN;
 531 void ffi_go_closure_VFP (void) FFI_HIDDEN;
 532
 533 #if FFI_EXEC_TRAMPOLINE_TABLE
 534
 535 #include <mach/mach.h>
 536 #include <pthread.h>
 537 #include <stdio.h>
 538 #include <stdlib.h>
 539
 540 extern void *ffi_closure_trampoline_table_page;
 541
 542 typedef struct ffi_trampoline_table ffi_trampoline_table;
 543 typedef struct ffi_trampoline_table_entry ffi_trampoline_table_entry;
 544
 545 struct ffi_trampoline_table
 546 {
 547   /* contiguous writable and executable pages */
 548   vm_address_t config_page;
 549   vm_address_t trampoline_page;
 550
 551   /* free list tracking */
 552   uint16_t free_count;
 553   ffi_trampoline_table_entry *free_list;
 554   ffi_trampoline_table_entry *free_list_pool;
 555
 556   ffi_trampoline_table *prev;
 557   ffi_trampoline_table *next;
 558 };
 559
 560 struct ffi_trampoline_table_entry
 561 {
 562   void *(*trampoline) ();
 563   ffi_trampoline_table_entry *next;
 564 };
 565
 566 /* Override the standard architecture trampoline size */
 567 // XXX TODO - Fix
 568 #undef FFI_TRAMPOLINE_SIZE
 569 #define FFI_TRAMPOLINE_SIZE 12
 570
 571 /* The trampoline configuration is placed at 4080 bytes prior to the trampoline's entry point */
 572 #define FFI_TRAMPOLINE_CODELOC_CONFIG(codeloc) ((void **) (((uint8_t *) codeloc) - 4080));
 573
 574 /* The first 16 bytes of the config page are unused, as they are unaddressable from the trampoline page. */
 575 #define FFI_TRAMPOLINE_CONFIG_PAGE_OFFSET 16
 576
 577 /* Total number of trampolines that fit in one trampoline table */
 578 #define FFI_TRAMPOLINE_COUNT ((PAGE_SIZE - FFI_TRAMPOLINE_CONFIG_PAGE_OFFSET) / FFI_TRAMPOLINE_SIZE)
 579
 580 static pthread_mutex_t ffi_trampoline_lock = PTHREAD_MUTEX_INITIALIZER;
 581 static ffi_trampoline_table *ffi_trampoline_tables = NULL;
 582
 583 static ffi_trampoline_table *
 584 ffi_trampoline_table_alloc ()
 585 {
 586   ffi_trampoline_table *table = NULL;
 587
 588   /* Loop until we can allocate two contiguous pages */
 589   while (table == NULL)
 590     {
 591       vm_address_t config_page = 0x0;
 592       kern_return_t kt;
 593
 594       /* Try to allocate two pages */
 595       kt =
 596         vm_allocate (mach_task_self (), &config_page, PAGE_SIZE * 2,
 597                      VM_FLAGS_ANYWHERE);
 598       if (kt != KERN_SUCCESS)
 599         {
 600           fprintf (stderr, "vm_allocate() failure: %d at %s:%d\n", kt,
 601                    __FILE__, __LINE__);
 602           break;
 603         }
 604
 605       /* Now drop the second half of the allocation to make room for the trampoline table */
 606       vm_address_t trampoline_page = config_page + PAGE_SIZE;
 607       kt = vm_deallocate (mach_task_self (), trampoline_page, PAGE_SIZE);
 608       if (kt != KERN_SUCCESS)
 609         {
 610           fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt,
 611                    __FILE__, __LINE__);
 612           break;
 613         }
 614
 615       /* Remap the trampoline table to directly follow the config page */
 616       vm_prot_t cur_prot;
 617       vm_prot_t max_prot;
 618
 619       kt =
 620         vm_remap (mach_task_self (), &trampoline_page, PAGE_SIZE, 0x0, FALSE,
 621                   mach_task_self (),
 622                   (vm_address_t) & ffi_closure_trampoline_table_page, FALSE,
 623                   &cur_prot, &max_prot, VM_INHERIT_SHARE);
 624
 625       /* If we lost access to the destination trampoline page, drop our config allocation mapping and retry */
 626       if (kt != KERN_SUCCESS)
 627         {
 628           /* Log unexpected failures */
 629           if (kt != KERN_NO_SPACE)
 630             {
 631               fprintf (stderr, "vm_remap() failure: %d at %s:%d\n", kt,
 632                        __FILE__, __LINE__);
 633             }
 634
 635           vm_deallocate (mach_task_self (), config_page, PAGE_SIZE);
 636           continue;
 637         }
 638
 639       /* We have valid trampoline and config pages */
 640       table = calloc (1, sizeof (ffi_trampoline_table));
 641       table->free_count = FFI_TRAMPOLINE_COUNT;
 642       table->config_page = config_page;
 643       table->trampoline_page = trampoline_page;
 644
 645       /* Create and initialize the free list */
 646       table->free_list_pool =
 647         calloc (FFI_TRAMPOLINE_COUNT, sizeof (ffi_trampoline_table_entry));
 648
 649       uint16_t i;
 650       for (i = 0; i < table->free_count; i++)
 651         {
 652           ffi_trampoline_table_entry *entry = &table->free_list_pool[i];
 653           entry->trampoline =
 654             (void *) (table->trampoline_page + (i * FFI_TRAMPOLINE_SIZE));
 655
 656           if (i < table->free_count - 1)
 657             entry->next = &table->free_list_pool[i + 1];
 658         }
 659
 660       table->free_list = table->free_list_pool;
 661     }
 662
 663   return table;
 664 }
 665
 666 void *
 667 ffi_closure_alloc (size_t size, void **code)
 668 {
 669   /* Create the closure */
 670   ffi_closure *closure = malloc (size);
 671   if (closure == NULL)
 672     return NULL;
 673
 674   pthread_mutex_lock (&ffi_trampoline_lock);
 675
 676   /* Check for an active trampoline table with available entries. */
 677   ffi_trampoline_table *table = ffi_trampoline_tables;
 678   if (table == NULL || table->free_list == NULL)
 679     {
 680       table = ffi_trampoline_table_alloc ();
 681       if (table == NULL)
 682         {
 683           free (closure);
 684           return NULL;
 685         }
 686
 687       /* Insert the new table at the top of the list */
 688       table->next = ffi_trampoline_tables;
 689       if (table->next != NULL)
 690         table->next->prev = table;
 691
 692       ffi_trampoline_tables = table;
 693     }
 694
 695   /* Claim the free entry */
 696   ffi_trampoline_table_entry *entry = ffi_trampoline_tables->free_list;
 697   ffi_trampoline_tables->free_list = entry->next;
 698   ffi_trampoline_tables->free_count--;
 699   entry->next = NULL;
 700
 701   pthread_mutex_unlock (&ffi_trampoline_lock);
 702
 703   /* Initialize the return values */
 704   *code = entry->trampoline;
 705   closure->trampoline_table = table;
 706   closure->trampoline_table_entry = entry;
 707
 708   return closure;
 709 }
 710
 711 void
 712 ffi_closure_free (void *ptr)
 713 {
 714   ffi_closure *closure = ptr;
 715
 716   pthread_mutex_lock (&ffi_trampoline_lock);
 717
 718   /* Fetch the table and entry references */
 719   ffi_trampoline_table *table = closure->trampoline_table;
 720   ffi_trampoline_table_entry *entry = closure->trampoline_table_entry;
 721
 722   /* Return the entry to the free list */
 723   entry->next = table->free_list;
 724   table->free_list = entry;
 725   table->free_count++;
 726
 727   /* If all trampolines within this table are free, and at least one other table exists, deallocate
 728    * the table */
 729   if (table->free_count == FFI_TRAMPOLINE_COUNT
 730       && ffi_trampoline_tables != table)
 731     {
 732       /* Remove from the list */
 733       if (table->prev != NULL)
 734         table->prev->next = table->next;
 735
 736       if (table->next != NULL)
 737         table->next->prev = table->prev;
 738
 739       /* Deallocate pages */
 740       kern_return_t kt;
 741       kt = vm_deallocate (mach_task_self (), table->config_page, PAGE_SIZE);
 742       if (kt != KERN_SUCCESS)
 743         fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt,
 744                  __FILE__, __LINE__);
 745
 746       kt =
 747         vm_deallocate (mach_task_self (), table->trampoline_page, PAGE_SIZE);
 748       if (kt != KERN_SUCCESS)
 749         fprintf (stderr, "vm_deallocate() failure: %d at %s:%d\n", kt,
 750                  __FILE__, __LINE__);
 751
 752       /* Deallocate free list */
 753       free (table->free_list_pool);
 754       free (table);
 755     }
 756   else if (ffi_trampoline_tables != table)
 757     {
 758       /* Otherwise, bump this table to the top of the list */
 759       table->prev = NULL;
 760       table->next = ffi_trampoline_tables;
 761       if (ffi_trampoline_tables != NULL)
 762         ffi_trampoline_tables->prev = table;
 763
 764       ffi_trampoline_tables = table;
 765     }
 766
 767   pthread_mutex_unlock (&ffi_trampoline_lock);
 768
 769   /* Free the closure */
 770   free (closure);
 771 }
 772
 773 #else
 774
 775 extern unsigned int ffi_arm_trampoline[2] FFI_HIDDEN;
 776
 777 #endif
 778
 779 /* the cif must already be prep'ed */
 780
 781 ffi_status
 782 ffi_prep_closure_loc (ffi_closure * closure,
 783                       ffi_cif * cif,
 784                       void (*fun) (ffi_cif *, void *, void **, void *),
 785                       void *user_data, void *codeloc)
 786 {
 787   void (*closure_func) (void) = ffi_closure_SYSV;
 788
 789   if (cif->abi == FFI_VFP)
 790     {
 791       /* We only need take the vfp path if there are vfp arguments.  */
 792       if (cif->vfp_used)
 793         closure_func = ffi_closure_VFP;
 794     }
 795   else if (cif->abi != FFI_SYSV)
 796     return FFI_BAD_ABI;
 797
 798 #if FFI_EXEC_TRAMPOLINE_TABLE
 799   void **config = FFI_TRAMPOLINE_CODELOC_CONFIG (codeloc);
 800   config[0] = closure;
 801   config[1] = closure_func;
 802 #else
 803   memcpy (closure->tramp, ffi_arm_trampoline, 8);
 804   __clear_cache(closure->tramp, closure->tramp + 8);    /* clear data map */
 805   __clear_cache(codeloc, codeloc + 8);                  /* clear insn map */
 806   *(void (**)(void))(closure->tramp + 8) = closure_func;
 807 #endif
 808
 809   closure->cif = cif;
 810   closure->fun = fun;
 811   closure->user_data = user_data;
 812
 813   return FFI_OK;
 814 }
 815
 816 ffi_status
 817 ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif,
 818                      void (*fun) (ffi_cif *, void *, void **, void *))
 819 {
 820   void (*closure_func) (void) = ffi_go_closure_SYSV;
 821
 822   if (cif->abi == FFI_VFP)
 823     {
 824       /* We only need take the vfp path if there are vfp arguments.  */
 825       if (cif->vfp_used)
 826         closure_func = ffi_go_closure_VFP;
 827     }
 828   else if (cif->abi != FFI_SYSV)
 829     return FFI_BAD_ABI;
 830
 831   closure->tramp = closure_func;
 832   closure->cif = cif;
 833   closure->fun = fun;
 834
 835   return FFI_OK;
 836 }
 837
 838 /* Below are routines for VFP hard-float support. */
 839
 840 /* A subroutine of vfp_type_p.  Given a structure type, return the type code
 841    of the first non-structure element.  Recurse for structure elements.
 842    Return -1 if the structure is in fact empty, i.e. no nested elements.  */
 843
 844 static int
 845 is_hfa0 (const ffi_type *ty)
 846 {
 847   ffi_type **elements = ty->elements;
 848   int i, ret = -1;
 849
 850   if (elements != NULL)
 851     for (i = 0; elements[i]; ++i)
 852       {
 853         ret = elements[i]->type;
 854         if (ret == FFI_TYPE_STRUCT || ret == FFI_TYPE_COMPLEX)
 855           {
 856             ret = is_hfa0 (elements[i]);
 857             if (ret < 0)
 858               continue;
 859           }
 860         break;
 861       }
 862
 863   return ret;
 864 }
 865
 866 /* A subroutine of vfp_type_p.  Given a structure type, return true if all
 867    of the non-structure elements are the same as CANDIDATE.  */
 868
 869 static int
 870 is_hfa1 (const ffi_type *ty, int candidate)
 871 {
 872   ffi_type **elements = ty->elements;
 873   int i;
 874
 875   if (elements != NULL)
 876     for (i = 0; elements[i]; ++i)
 877       {
 878         int t = elements[i]->type;
 879         if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX)
 880           {
 881             if (!is_hfa1 (elements[i], candidate))
 882               return 0;
 883           }
 884         else if (t != candidate)
 885           return 0;
 886       }
 887
 888   return 1;
 889 }
 890
 891 /* Determine if TY is an homogenous floating point aggregate (HFA).
 892    That is, a structure consisting of 1 to 4 members of all the same type,
 893    where that type is a floating point scalar.
 894
 895    Returns non-zero iff TY is an HFA.  The result is an encoded value where
 896    bits 0-7 contain the type code, and bits 8-10 contain the element count.  */
 897
 898 static int
 899 vfp_type_p (const ffi_type *ty)
 900 {
 901   ffi_type **elements;
 902   int candidate, i;
 903   size_t size, ele_count;
 904
 905   /* Quickest tests first.  */
 906   candidate = ty->type;
 907   switch (ty->type)
 908     {
 909     default:
 910       return 0;
 911     case FFI_TYPE_FLOAT:
 912     case FFI_TYPE_DOUBLE:
 913       ele_count = 1;
 914       goto done;
 915     case FFI_TYPE_COMPLEX:
 916       candidate = ty->elements[0]->type;
 917       if (candidate != FFI_TYPE_FLOAT && candidate != FFI_TYPE_DOUBLE)
 918         return 0;
 919       ele_count = 2;
 920       goto done;
 921     case FFI_TYPE_STRUCT:
 922       break;
 923     }
 924
 925   /* No HFA types are smaller than 4 bytes, or larger than 32 bytes.  */
 926   size = ty->size;
 927   if (size < 4 || size > 32)
 928     return 0;
 929
 930   /* Find the type of the first non-structure member.  */
 931   elements = ty->elements;
 932   candidate = elements[0]->type;
 933   if (candidate == FFI_TYPE_STRUCT || candidate == FFI_TYPE_COMPLEX)
 934     {
 935       for (i = 0; ; ++i)
 936         {
 937           candidate = is_hfa0 (elements[i]);
 938           if (candidate >= 0)
 939             break;
 940         }
 941     }
 942
 943   /* If the first member is not a floating point type, it's not an HFA.
 944      Also quickly re-check the size of the structure.  */
 945   switch (candidate)
 946     {
 947     case FFI_TYPE_FLOAT:
 948       ele_count = size / sizeof(float);
 949       if (size != ele_count * sizeof(float))
 950         return 0;
 951       break;
 952     case FFI_TYPE_DOUBLE:
 953       ele_count = size / sizeof(double);
 954       if (size != ele_count * sizeof(double))
 955         return 0;
 956       break;
 957     default:
 958       return 0;
 959     }
 960   if (ele_count > 4)
 961     return 0;
 962
 963   /* Finally, make sure that all scalar elements are the same type.  */
 964   for (i = 0; elements[i]; ++i)
 965     {
 966       int t = elements[i]->type;
 967       if (t == FFI_TYPE_STRUCT || t == FFI_TYPE_COMPLEX)
 968         {
 969           if (!is_hfa1 (elements[i], candidate))
 970             return 0;
 971         }
 972       else if (t != candidate)
 973         return 0;
 974     }
 975
 976   /* All tests succeeded.  Encode the result.  */
 977  done:
 978   return (ele_count << 8) | candidate;
 979 }
 980
 981 static int
 982 place_vfp_arg (ffi_cif *cif, int h)
 983 {
 984   unsigned short reg = cif->vfp_reg_free;
 985   int align = 1, nregs = h >> 8;
 986
 987   if ((h & 0xff) == FFI_TYPE_DOUBLE)
 988     align = 2, nregs *= 2;
 989
 990   /* Align register number. */
 991   if ((reg & 1) && align == 2)
 992     reg++;
 993
 994   while (reg + nregs <= 16)
 995     {
 996       int s, new_used = 0;
 997       for (s = reg; s < reg + nregs; s++)
 998         {
 999           new_used |= (1 << s);
1000           if (cif->vfp_used & (1 << s))
1001             {
1002               reg += align;
1003               goto next_reg;
1004             }
1005         }
1006       /* Found regs to allocate. */
1007       cif->vfp_used |= new_used;
1008       cif->vfp_args[cif->vfp_nargs++] = reg;
1009
1010       /* Update vfp_reg_free. */
1011       if (cif->vfp_used & (1 << cif->vfp_reg_free))
1012         {
1013           reg += nregs;
1014           while (cif->vfp_used & (1 << reg))
1015             reg += 1;
1016           cif->vfp_reg_free = reg;
1017         }
1018       return 0;
1019     next_reg:;
1020     }
1021   // done, mark all regs as used
1022   cif->vfp_reg_free = 16;
1023   cif->vfp_used = 0xFFFF;
1024   return 1;
1025 }
1026
1027 static void
1028 layout_vfp_args (ffi_cif * cif)
1029 {
1030   int i;
1031   /* Init VFP fields */
1032   cif->vfp_used = 0;
1033   cif->vfp_nargs = 0;
1034   cif->vfp_reg_free = 0;
1035   memset (cif->vfp_args, -1, 16);       /* Init to -1. */
1036
1037   for (i = 0; i < cif->nargs; i++)
1038     {
1039       int h = vfp_type_p (cif->arg_types[i]);
1040       if (h && place_vfp_arg (cif, h) == 1)
1041         break;
1042     }
1043 }