libffi/src/aarch64/ffi.c

   1 /* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.
   2
   3 Permission is hereby granted, free of charge, to any person obtaining
   4 a copy of this software and associated documentation files (the
   5 ``Software''), to deal in the Software without restriction, including
   6 without limitation the rights to use, copy, modify, merge, publish,
   7 distribute, sublicense, and/or sell copies of the Software, and to
   8 permit persons to whom the Software is furnished to do so, subject to
   9 the following conditions:
  10
  11 The above copyright notice and this permission notice shall be
  12 included in all copies or substantial portions of the Software.
  13
  14 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  15 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  16 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  17 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  18 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  19 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  20 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
  21
  22 #include <stdio.h>
  23
  24 #include <ffi.h>
  25 #include <ffi_common.h>
  26
  27 #include <stdlib.h>
  28
  29 /* Stack alignment requirement in bytes */
  30 #define AARCH64_STACK_ALIGN 16
  31
  32 #define N_X_ARG_REG 8
  33 #define N_V_ARG_REG 8
  34
  35 #define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT)
  36
  37 union _d
  38 {
  39   UINT64 d;
  40   UINT32 s[2];
  41 };
  42
  43 struct call_context
  44 {
  45   UINT64 x [AARCH64_N_XREG];
  46   struct
  47   {
  48     union _d d[2];
  49   } v [AARCH64_N_VREG];
  50 };
  51
  52 static void *
  53 get_x_addr (struct call_context *context, unsigned n)
  54 {
  55   return &context->x[n];
  56 }
  57
  58 static void *
  59 get_s_addr (struct call_context *context, unsigned n)
  60 {
  61 #if defined __AARCH64EB__
  62   return &context->v[n].d[1].s[1];
  63 #else
  64   return &context->v[n].d[0].s[0];
  65 #endif
  66 }
  67
  68 static void *
  69 get_d_addr (struct call_context *context, unsigned n)
  70 {
  71 #if defined __AARCH64EB__
  72   return &context->v[n].d[1];
  73 #else
  74   return &context->v[n].d[0];
  75 #endif
  76 }
  77
  78 static void *
  79 get_v_addr (struct call_context *context, unsigned n)
  80 {
  81   return &context->v[n];
  82 }
  83
  84 /* Return the memory location at which a basic type would reside
  85    were it to have been stored in register n.  */
  86
  87 static void *
  88 get_basic_type_addr (unsigned short type, struct call_context *context,
  89                      unsigned n)
  90 {
  91   switch (type)
  92     {
  93     case FFI_TYPE_FLOAT:
  94       return get_s_addr (context, n);
  95     case FFI_TYPE_DOUBLE:
  96       return get_d_addr (context, n);
  97     case FFI_TYPE_LONGDOUBLE:
  98       return get_v_addr (context, n);
  99     case FFI_TYPE_UINT8:
 100     case FFI_TYPE_SINT8:
 101     case FFI_TYPE_UINT16:
 102     case FFI_TYPE_SINT16:
 103     case FFI_TYPE_UINT32:
 104     case FFI_TYPE_SINT32:
 105     case FFI_TYPE_INT:
 106     case FFI_TYPE_POINTER:
 107     case FFI_TYPE_UINT64:
 108     case FFI_TYPE_SINT64:
 109       return get_x_addr (context, n);
 110     default:
 111       FFI_ASSERT (0);
 112       return NULL;
 113     }
 114 }
 115
 116 /* Return the alignment width for each of the basic types.  */
 117
 118 static size_t
 119 get_basic_type_alignment (unsigned short type)
 120 {
 121   switch (type)
 122     {
 123     case FFI_TYPE_FLOAT:
 124     case FFI_TYPE_DOUBLE:
 125       return sizeof (UINT64);
 126     case FFI_TYPE_LONGDOUBLE:
 127       return sizeof (long double);
 128     case FFI_TYPE_UINT8:
 129     case FFI_TYPE_SINT8:
 130     case FFI_TYPE_UINT16:
 131     case FFI_TYPE_SINT16:
 132     case FFI_TYPE_UINT32:
 133     case FFI_TYPE_INT:
 134     case FFI_TYPE_SINT32:
 135     case FFI_TYPE_POINTER:
 136     case FFI_TYPE_UINT64:
 137     case FFI_TYPE_SINT64:
 138       return sizeof (UINT64);
 139
 140     default:
 141       FFI_ASSERT (0);
 142       return 0;
 143     }
 144 }
 145
 146 /* Return the size in bytes for each of the basic types.  */
 147
 148 static size_t
 149 get_basic_type_size (unsigned short type)
 150 {
 151   switch (type)
 152     {
 153     case FFI_TYPE_FLOAT:
 154       return sizeof (UINT32);
 155     case FFI_TYPE_DOUBLE:
 156       return sizeof (UINT64);
 157     case FFI_TYPE_LONGDOUBLE:
 158       return sizeof (long double);
 159     case FFI_TYPE_UINT8:
 160       return sizeof (UINT8);
 161     case FFI_TYPE_SINT8:
 162       return sizeof (SINT8);
 163     case FFI_TYPE_UINT16:
 164       return sizeof (UINT16);
 165     case FFI_TYPE_SINT16:
 166       return sizeof (SINT16);
 167     case FFI_TYPE_UINT32:
 168       return sizeof (UINT32);
 169     case FFI_TYPE_INT:
 170     case FFI_TYPE_SINT32:
 171       return sizeof (SINT32);
 172     case FFI_TYPE_POINTER:
 173     case FFI_TYPE_UINT64:
 174       return sizeof (UINT64);
 175     case FFI_TYPE_SINT64:
 176       return sizeof (SINT64);
 177
 178     default:
 179       FFI_ASSERT (0);
 180       return 0;
 181     }
 182 }
 183
 184 extern void
 185 ffi_call_SYSV (unsigned (*)(struct call_context *context, unsigned char *,
 186                             extended_cif *),
 187                struct call_context *context,
 188                extended_cif *,
 189                unsigned,
 190                void (*fn)(void));
 191
 192 extern void
 193 ffi_closure_SYSV (ffi_closure *);
 194
 195 /* Test for an FFI floating point representation.  */
 196
 197 static unsigned
 198 is_floating_type (unsigned short type)
 199 {
 200   return (type == FFI_TYPE_FLOAT || type == FFI_TYPE_DOUBLE
 201           || type == FFI_TYPE_LONGDOUBLE);
 202 }
 203
 204 /* Test for a homogeneous structure.  */
 205
 206 static unsigned short
 207 get_homogeneous_type (ffi_type *ty)
 208 {
 209   if (ty->type == FFI_TYPE_STRUCT && ty->elements)
 210     {
 211       unsigned i;
 212       unsigned short candidate_type
 213         = get_homogeneous_type (ty->elements[0]);
 214       for (i =1; ty->elements[i]; i++)
 215         {
 216           unsigned short iteration_type = 0;
 217           /* If we have a nested struct, we must find its homogeneous type.
 218              If that fits with our candidate type, we are still
 219              homogeneous.  */
 220           if (ty->elements[i]->type == FFI_TYPE_STRUCT
 221               && ty->elements[i]->elements)
 222             {
 223               iteration_type = get_homogeneous_type (ty->elements[i]);
 224             }
 225           else
 226             {
 227               iteration_type = ty->elements[i]->type;
 228             }
 229
 230           /* If we are not homogeneous, return FFI_TYPE_STRUCT.  */
 231           if (candidate_type != iteration_type)
 232             return FFI_TYPE_STRUCT;
 233         }
 234       return candidate_type;
 235     }
 236
 237   /* Base case, we have no more levels of nesting, so we
 238      are a basic type, and so, trivially homogeneous in that type.  */
 239   return ty->type;
 240 }
 241
 242 /* Determine the number of elements within a STRUCT.
 243
 244    Note, we must handle nested structs.
 245
 246    If ty is not a STRUCT this function will return 0.  */
 247
 248 static unsigned
 249 element_count (ffi_type *ty)
 250 {
 251   if (ty->type == FFI_TYPE_STRUCT && ty->elements)
 252     {
 253       unsigned n;
 254       unsigned elems = 0;
 255       for (n = 0; ty->elements[n]; n++)
 256         {
 257           if (ty->elements[n]->type == FFI_TYPE_STRUCT
 258               && ty->elements[n]->elements)
 259             elems += element_count (ty->elements[n]);
 260           else
 261             elems++;
 262         }
 263       return elems;
 264     }
 265   return 0;
 266 }
 267
 268 /* Test for a homogeneous floating point aggregate.
 269
 270    A homogeneous floating point aggregate is a homogeneous aggregate of
 271    a half- single- or double- precision floating point type with one
 272    to four elements.  Note that this includes nested structs of the
 273    basic type.  */
 274
 275 static int
 276 is_hfa (ffi_type *ty)
 277 {
 278   if (ty->type == FFI_TYPE_STRUCT
 279       && ty->elements[0]
 280       && is_floating_type (get_homogeneous_type (ty)))
 281     {
 282       unsigned n = element_count (ty);
 283       return n >= 1 && n <= 4;
 284     }
 285   return 0;
 286 }
 287
 288 /* Test if an ffi_type is a candidate for passing in a register.
 289
 290    This test does not check that sufficient registers of the
 291    appropriate class are actually available, merely that IFF
 292    sufficient registers are available then the argument will be passed
 293    in register(s).
 294
 295    Note that an ffi_type that is deemed to be a register candidate
 296    will always be returned in registers.
 297
 298    Returns 1 if a register candidate else 0.  */
 299
 300 static int
 301 is_register_candidate (ffi_type *ty)
 302 {
 303   switch (ty->type)
 304     {
 305     case FFI_TYPE_VOID:
 306     case FFI_TYPE_FLOAT:
 307     case FFI_TYPE_DOUBLE:
 308     case FFI_TYPE_LONGDOUBLE:
 309     case FFI_TYPE_UINT8:
 310     case FFI_TYPE_UINT16:
 311     case FFI_TYPE_UINT32:
 312     case FFI_TYPE_UINT64:
 313     case FFI_TYPE_POINTER:
 314     case FFI_TYPE_SINT8:
 315     case FFI_TYPE_SINT16:
 316     case FFI_TYPE_SINT32:
 317     case FFI_TYPE_INT:
 318     case FFI_TYPE_SINT64:
 319       return 1;
 320
 321     case FFI_TYPE_STRUCT:
 322       if (is_hfa (ty))
 323         {
 324           return 1;
 325         }
 326       else if (ty->size > 16)
 327         {
 328           /* Too large. Will be replaced with a pointer to memory. The
 329              pointer MAY be passed in a register, but the value will
 330              not. This test specifically fails since the argument will
 331              never be passed by value in registers. */
 332           return 0;
 333         }
 334       else
 335         {
 336           /* Might be passed in registers depending on the number of
 337              registers required. */
 338           return (ty->size + 7) / 8 < N_X_ARG_REG;
 339         }
 340       break;
 341
 342     default:
 343       FFI_ASSERT (0);
 344       break;
 345     }
 346
 347   return 0;
 348 }
 349
 350 /* Test if an ffi_type argument or result is a candidate for a vector
 351    register.  */
 352
 353 static int
 354 is_v_register_candidate (ffi_type *ty)
 355 {
 356   return is_floating_type (ty->type)
 357            || (ty->type == FFI_TYPE_STRUCT && is_hfa (ty));
 358 }
 359
 360 /* Representation of the procedure call argument marshalling
 361    state.
 362
 363    The terse state variable names match the names used in the AARCH64
 364    PCS. */
 365
 366 struct arg_state
 367 {
 368   unsigned ngrn;                /* Next general-purpose register number. */
 369   unsigned nsrn;                /* Next vector register number. */
 370   unsigned nsaa;                /* Next stack offset. */
 371 };
 372
 373 /* Initialize a procedure call argument marshalling state.  */
 374 static void
 375 arg_init (struct arg_state *state, unsigned call_frame_size)
 376 {
 377   state->ngrn = 0;
 378   state->nsrn = 0;
 379   state->nsaa = 0;
 380 }
 381
 382 /* Return the number of available consecutive core argument
 383    registers.  */
 384
 385 static unsigned
 386 available_x (struct arg_state *state)
 387 {
 388   return N_X_ARG_REG - state->ngrn;
 389 }
 390
 391 /* Return the number of available consecutive vector argument
 392    registers.  */
 393
 394 static unsigned
 395 available_v (struct arg_state *state)
 396 {
 397   return N_V_ARG_REG - state->nsrn;
 398 }
 399
 400 static void *
 401 allocate_to_x (struct call_context *context, struct arg_state *state)
 402 {
 403   FFI_ASSERT (state->ngrn < N_X_ARG_REG)
 404   return get_x_addr (context, (state->ngrn)++);
 405 }
 406
 407 static void *
 408 allocate_to_s (struct call_context *context, struct arg_state *state)
 409 {
 410   FFI_ASSERT (state->nsrn < N_V_ARG_REG)
 411   return get_s_addr (context, (state->nsrn)++);
 412 }
 413
 414 static void *
 415 allocate_to_d (struct call_context *context, struct arg_state *state)
 416 {
 417   FFI_ASSERT (state->nsrn < N_V_ARG_REG)
 418   return get_d_addr (context, (state->nsrn)++);
 419 }
 420
 421 static void *
 422 allocate_to_v (struct call_context *context, struct arg_state *state)
 423 {
 424   FFI_ASSERT (state->nsrn < N_V_ARG_REG)
 425   return get_v_addr (context, (state->nsrn)++);
 426 }
 427
 428 /* Allocate an aligned slot on the stack and return a pointer to it.  */
 429 static void *
 430 allocate_to_stack (struct arg_state *state, void *stack, unsigned alignment,
 431                    unsigned size)
 432 {
 433   void *allocation;
 434
 435   /* Round up the NSAA to the larger of 8 or the natural
 436      alignment of the argument's type.  */
 437   state->nsaa = ALIGN (state->nsaa, alignment);
 438   state->nsaa = ALIGN (state->nsaa, alignment);
 439   state->nsaa = ALIGN (state->nsaa, 8);
 440
 441   allocation = stack + state->nsaa;
 442
 443   state->nsaa += size;
 444   return allocation;
 445 }
 446
 447 static void
 448 copy_basic_type (void *dest, void *source, unsigned short type)
 449 {
 450   /* This is neccessary to ensure that basic types are copied
 451      sign extended to 64-bits as libffi expects.  */
 452   switch (type)
 453     {
 454     case FFI_TYPE_FLOAT:
 455       *(float *) dest = *(float *) source;
 456       break;
 457     case FFI_TYPE_DOUBLE:
 458       *(double *) dest = *(double *) source;
 459       break;
 460     case FFI_TYPE_LONGDOUBLE:
 461       *(long double *) dest = *(long double *) source;
 462       break;
 463     case FFI_TYPE_UINT8:
 464       *(ffi_arg *) dest = *(UINT8 *) source;
 465       break;
 466     case FFI_TYPE_SINT8:
 467       *(ffi_sarg *) dest = *(SINT8 *) source;
 468       break;
 469     case FFI_TYPE_UINT16:
 470       *(ffi_arg *) dest = *(UINT16 *) source;
 471       break;
 472     case FFI_TYPE_SINT16:
 473       *(ffi_sarg *) dest = *(SINT16 *) source;
 474       break;
 475     case FFI_TYPE_UINT32:
 476       *(ffi_arg *) dest = *(UINT32 *) source;
 477       break;
 478     case FFI_TYPE_INT:
 479     case FFI_TYPE_SINT32:
 480       *(ffi_sarg *) dest = *(SINT32 *) source;
 481       break;
 482     case FFI_TYPE_POINTER:
 483     case FFI_TYPE_UINT64:
 484       *(ffi_arg *) dest = *(UINT64 *) source;
 485       break;
 486     case FFI_TYPE_SINT64:
 487       *(ffi_sarg *) dest = *(SINT64 *) source;
 488       break;
 489
 490     default:
 491       FFI_ASSERT (0);
 492     }
 493 }
 494
 495 static void
 496 copy_hfa_to_reg_or_stack (void *memory,
 497                           ffi_type *ty,
 498                           struct call_context *context,
 499                           unsigned char *stack,
 500                           struct arg_state *state)
 501 {
 502   unsigned elems = element_count (ty);
 503   if (available_v (state) < elems)
 504     {
 505       /* There are insufficient V registers. Further V register allocations
 506          are prevented, the NSAA is adjusted (by allocate_to_stack ())
 507          and the argument is copied to memory at the adjusted NSAA.  */
 508       state->nsrn = N_V_ARG_REG;
 509       memcpy (allocate_to_stack (state, stack, ty->alignment, ty->size),
 510               memory,
 511               ty->size);
 512     }
 513   else
 514     {
 515       int i;
 516       unsigned short type = get_homogeneous_type (ty);
 517       unsigned elems = element_count (ty);
 518       for (i = 0; i < elems; i++)
 519         {
 520           void *reg = allocate_to_v (context, state);
 521           copy_basic_type (reg, memory, type);
 522           memory += get_basic_type_size (type);
 523         }
 524     }
 525 }
 526
 527 /* Either allocate an appropriate register for the argument type, or if
 528    none are available, allocate a stack slot and return a pointer
 529    to the allocated space.  */
 530
 531 static void *
 532 allocate_to_register_or_stack (struct call_context *context,
 533                                unsigned char *stack,
 534                                struct arg_state *state,
 535                                unsigned short type)
 536 {
 537   size_t alignment = get_basic_type_alignment (type);
 538   size_t size = alignment;
 539   switch (type)
 540     {
 541     case FFI_TYPE_FLOAT:
 542       /* This is the only case for which the allocated stack size
 543          should not match the alignment of the type.  */
 544       size = sizeof (UINT32);
 545       /* Fall through.  */
 546     case FFI_TYPE_DOUBLE:
 547       if (state->nsrn < N_V_ARG_REG)
 548         return allocate_to_d (context, state);
 549       state->nsrn = N_V_ARG_REG;
 550       break;
 551     case FFI_TYPE_LONGDOUBLE:
 552       if (state->nsrn < N_V_ARG_REG)
 553         return allocate_to_v (context, state);
 554       state->nsrn = N_V_ARG_REG;
 555       break;
 556     case FFI_TYPE_UINT8:
 557     case FFI_TYPE_SINT8:
 558     case FFI_TYPE_UINT16:
 559     case FFI_TYPE_SINT16:
 560     case FFI_TYPE_UINT32:
 561     case FFI_TYPE_SINT32:
 562     case FFI_TYPE_INT:
 563     case FFI_TYPE_POINTER:
 564     case FFI_TYPE_UINT64:
 565     case FFI_TYPE_SINT64:
 566       if (state->ngrn < N_X_ARG_REG)
 567         return allocate_to_x (context, state);
 568       state->ngrn = N_X_ARG_REG;
 569       break;
 570     default:
 571       FFI_ASSERT (0);
 572     }
 573
 574     return allocate_to_stack (state, stack, alignment, size);
 575 }
 576
 577 /* Copy a value to an appropriate register, or if none are
 578    available, to the stack.  */
 579
 580 static void
 581 copy_to_register_or_stack (struct call_context *context,
 582                            unsigned char *stack,
 583                            struct arg_state *state,
 584                            void *value,
 585                            unsigned short type)
 586 {
 587   copy_basic_type (
 588           allocate_to_register_or_stack (context, stack, state, type),
 589           value,
 590           type);
 591 }
 592
 593 /* Marshall the arguments from FFI representation to procedure call
 594    context and stack.  */
 595
 596 static unsigned
 597 aarch64_prep_args (struct call_context *context, unsigned char *stack,
 598                    extended_cif *ecif)
 599 {
 600   int i;
 601   struct arg_state state;
 602
 603   arg_init (&state, ALIGN(ecif->cif->bytes, 16));
 604
 605   for (i = 0; i < ecif->cif->nargs; i++)
 606     {
 607       ffi_type *ty = ecif->cif->arg_types[i];
 608       switch (ty->type)
 609         {
 610         case FFI_TYPE_VOID:
 611           FFI_ASSERT (0);
 612           break;
 613
 614         /* If the argument is a basic type the argument is allocated to an
 615            appropriate register, or if none are available, to the stack.  */
 616         case FFI_TYPE_FLOAT:
 617         case FFI_TYPE_DOUBLE:
 618         case FFI_TYPE_LONGDOUBLE:
 619         case FFI_TYPE_UINT8:
 620         case FFI_TYPE_SINT8:
 621         case FFI_TYPE_UINT16:
 622         case FFI_TYPE_SINT16:
 623         case FFI_TYPE_UINT32:
 624         case FFI_TYPE_INT:
 625         case FFI_TYPE_SINT32:
 626         case FFI_TYPE_POINTER:
 627         case FFI_TYPE_UINT64:
 628         case FFI_TYPE_SINT64:
 629           copy_to_register_or_stack (context, stack, &state,
 630                                      ecif->avalue[i], ty->type);
 631           break;
 632
 633         case FFI_TYPE_STRUCT:
 634           if (is_hfa (ty))
 635             {
 636               copy_hfa_to_reg_or_stack (ecif->avalue[i], ty, context,
 637                                         stack, &state);
 638             }
 639           else if (ty->size > 16)
 640             {
 641               /* If the argument is a composite type that is larger than 16
 642                  bytes, then the argument has been copied to memory, and
 643                  the argument is replaced by a pointer to the copy.  */
 644
 645               copy_to_register_or_stack (context, stack, &state,
 646                                          &(ecif->avalue[i]), FFI_TYPE_POINTER);
 647             }
 648           else if (available_x (&state) >= (ty->size + 7) / 8)
 649             {
 650               /* If the argument is a composite type and the size in
 651                  double-words is not more than the number of available
 652                  X registers, then the argument is copied into consecutive
 653                  X registers.  */
 654               int j;
 655               for (j = 0; j < (ty->size + 7) / 8; j++)
 656                 {
 657                   memcpy (allocate_to_x (context, &state),
 658                           &(((UINT64 *) ecif->avalue[i])[j]),
 659                           sizeof (UINT64));
 660                 }
 661             }
 662           else
 663             {
 664               /* Otherwise, there are insufficient X registers. Further X
 665                  register allocations are prevented, the NSAA is adjusted
 666                  (by allocate_to_stack ()) and the argument is copied to
 667                  memory at the adjusted NSAA.  */
 668               state.ngrn = N_X_ARG_REG;
 669
 670               memcpy (allocate_to_stack (&state, stack, ty->alignment,
 671                                          ty->size), ecif->avalue + i, ty->size);
 672             }
 673           break;
 674
 675         default:
 676           FFI_ASSERT (0);
 677           break;
 678         }
 679     }
 680
 681   return ecif->cif->aarch64_flags;
 682 }
 683
 684 ffi_status
 685 ffi_prep_cif_machdep (ffi_cif *cif)
 686 {
 687   /* Round the stack up to a multiple of the stack alignment requirement. */
 688   cif->bytes =
 689     (cif->bytes + (AARCH64_STACK_ALIGN - 1)) & ~ (AARCH64_STACK_ALIGN - 1);
 690
 691   /* Initialize our flags. We are interested if this CIF will touch a
 692      vector register, if so we will enable context save and load to
 693      those registers, otherwise not. This is intended to be friendly
 694      to lazy float context switching in the kernel.  */
 695   cif->aarch64_flags = 0;
 696
 697   if (is_v_register_candidate (cif->rtype))
 698     {
 699       cif->aarch64_flags |= AARCH64_FFI_WITH_V;
 700     }
 701   else
 702     {
 703       int i;
 704       for (i = 0; i < cif->nargs; i++)
 705         if (is_v_register_candidate (cif->arg_types[i]))
 706           {
 707             cif->aarch64_flags |= AARCH64_FFI_WITH_V;
 708             break;
 709           }
 710     }
 711
 712   return FFI_OK;
 713 }
 714
 715 /* Call a function with the provided arguments and capture the return
 716    value.  */
 717 void
 718 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
 719 {
 720   extended_cif ecif;
 721
 722   ecif.cif = cif;
 723   ecif.avalue = avalue;
 724   ecif.rvalue = rvalue;
 725
 726   switch (cif->abi)
 727     {
 728     case FFI_SYSV:
 729       {
 730         struct call_context context;
 731         unsigned stack_bytes;
 732
 733         /* Figure out the total amount of stack space we need, the
 734            above call frame space needs to be 16 bytes aligned to
 735            ensure correct alignment of the first object inserted in
 736            that space hence the ALIGN applied to cif->bytes.*/
 737         stack_bytes = ALIGN(cif->bytes, 16);
 738
 739         memset (&context, 0, sizeof (context));
 740         if (is_register_candidate (cif->rtype))
 741           {
 742             ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn);
 743             switch (cif->rtype->type)
 744               {
 745               case FFI_TYPE_VOID:
 746               case FFI_TYPE_FLOAT:
 747               case FFI_TYPE_DOUBLE:
 748               case FFI_TYPE_LONGDOUBLE:
 749               case FFI_TYPE_UINT8:
 750               case FFI_TYPE_SINT8:
 751               case FFI_TYPE_UINT16:
 752               case FFI_TYPE_SINT16:
 753               case FFI_TYPE_UINT32:
 754               case FFI_TYPE_SINT32:
 755               case FFI_TYPE_POINTER:
 756               case FFI_TYPE_UINT64:
 757               case FFI_TYPE_INT:
 758               case FFI_TYPE_SINT64:
 759                 {
 760                   void *addr = get_basic_type_addr (cif->rtype->type,
 761                                                     &context, 0);
 762                   copy_basic_type (rvalue, addr, cif->rtype->type);
 763                   break;
 764                 }
 765
 766               case FFI_TYPE_STRUCT:
 767                 if (is_hfa (cif->rtype))
 768                   {
 769                     int j;
 770                     unsigned short type = get_homogeneous_type (cif->rtype);
 771                     unsigned elems = element_count (cif->rtype);
 772                     for (j = 0; j < elems; j++)
 773                       {
 774                         void *reg = get_basic_type_addr (type, &context, j);
 775                         copy_basic_type (rvalue, reg, type);
 776                         rvalue += get_basic_type_size (type);
 777                       }
 778                   }
 779                 else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
 780                   {
 781                     unsigned size = ALIGN (cif->rtype->size, sizeof (UINT64));
 782                     memcpy (rvalue, get_x_addr (&context, 0), size);
 783                   }
 784                 else
 785                   {
 786                     FFI_ASSERT (0);
 787                   }
 788                 break;
 789
 790               default:
 791                 FFI_ASSERT (0);
 792                 break;
 793               }
 794           }
 795         else
 796           {
 797             memcpy (get_x_addr (&context, 8), &rvalue, sizeof (UINT64));
 798             ffi_call_SYSV (aarch64_prep_args, &context, &ecif,
 799                            stack_bytes, fn);
 800           }
 801         break;
 802       }
 803
 804     default:
 805       FFI_ASSERT (0);
 806       break;
 807     }
 808 }
 809
 810 static unsigned char trampoline [] =
 811 { 0x70, 0x00, 0x00, 0x58,       /* ldr  x16, 1f */
 812   0x91, 0x00, 0x00, 0x10,       /* adr  x17, 2f */
 813   0x00, 0x02, 0x1f, 0xd6        /* br   x16     */
 814 };
 815
 816 /* Build a trampoline.  */
 817
 818 #define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS)                        \
 819   ({unsigned char *__tramp = (unsigned char*)(TRAMP);                   \
 820     UINT64  __fun = (UINT64)(FUN);                                      \
 821     UINT64  __ctx = (UINT64)(CTX);                                      \
 822     UINT64  __flags = (UINT64)(FLAGS);                                  \
 823     memcpy (__tramp, trampoline, sizeof (trampoline));                  \
 824     memcpy (__tramp + 12, &__fun, sizeof (__fun));                      \
 825     memcpy (__tramp + 20, &__ctx, sizeof (__ctx));                      \
 826     memcpy (__tramp + 28, &__flags, sizeof (__flags));                  \
 827     __clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE);              \
 828   })
 829
 830 ffi_status
 831 ffi_prep_closure_loc (ffi_closure* closure,
 832                       ffi_cif* cif,
 833                       void (*fun)(ffi_cif*,void*,void**,void*),
 834                       void *user_data,
 835                       void *codeloc)
 836 {
 837   if (cif->abi != FFI_SYSV)
 838     return FFI_BAD_ABI;
 839
 840   FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_SYSV, codeloc,
 841                        cif->aarch64_flags);
 842
 843   closure->cif  = cif;
 844   closure->user_data = user_data;
 845   closure->fun  = fun;
 846
 847   return FFI_OK;
 848 }
 849
 850 /* Primary handler to setup and invoke a function within a closure.
 851
 852    A closure when invoked enters via the assembler wrapper
 853    ffi_closure_SYSV(). The wrapper allocates a call context on the
 854    stack, saves the interesting registers (from the perspective of
 855    the calling convention) into the context then passes control to
 856    ffi_closure_SYSV_inner() passing the saved context and a pointer to
 857    the stack at the point ffi_closure_SYSV() was invoked.
 858
 859    On the return path the assembler wrapper will reload call context
 860    regsiters.
 861
 862    ffi_closure_SYSV_inner() marshalls the call context into ffi value
 863    desriptors, invokes the wrapped function, then marshalls the return
 864    value back into the call context.  */
 865
 866 void
 867 ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context,
 868                         void *stack)
 869 {
 870   ffi_cif *cif = closure->cif;
 871   void **avalue = (void**) alloca (cif->nargs * sizeof (void*));
 872   void *rvalue = NULL;
 873   int i;
 874   struct arg_state state;
 875
 876   arg_init (&state, ALIGN(cif->bytes, 16));
 877
 878   for (i = 0; i < cif->nargs; i++)
 879     {
 880       ffi_type *ty = cif->arg_types[i];
 881
 882       switch (ty->type)
 883         {
 884         case FFI_TYPE_VOID:
 885           FFI_ASSERT (0);
 886           break;
 887
 888         case FFI_TYPE_UINT8:
 889         case FFI_TYPE_SINT8:
 890         case FFI_TYPE_UINT16:
 891         case FFI_TYPE_SINT16:
 892         case FFI_TYPE_UINT32:
 893         case FFI_TYPE_SINT32:
 894         case FFI_TYPE_INT:
 895         case FFI_TYPE_POINTER:
 896         case FFI_TYPE_UINT64:
 897         case FFI_TYPE_SINT64:
 898         case  FFI_TYPE_FLOAT:
 899         case  FFI_TYPE_DOUBLE:
 900         case  FFI_TYPE_LONGDOUBLE:
 901           avalue[i] = allocate_to_register_or_stack (context, stack,
 902                                                      &state, ty->type);
 903           break;
 904
 905         case FFI_TYPE_STRUCT:
 906           if (is_hfa (ty))
 907             {
 908               unsigned n = element_count (ty);
 909               if (available_v (&state) < n)
 910                 {
 911                   state.nsrn = N_V_ARG_REG;
 912                   avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
 913                                                  ty->size);
 914                 }
 915               else
 916                 {
 917                   switch (get_homogeneous_type (ty))
 918                     {
 919                     case FFI_TYPE_FLOAT:
 920                       {
 921                         /* Eeek! We need a pointer to the structure,
 922                            however the homogeneous float elements are
 923                            being passed in individual S registers,
 924                            therefore the structure is not represented as
 925                            a contiguous sequence of bytes in our saved
 926                            register context. We need to fake up a copy
 927                            of the structure layed out in memory
 928                            correctly. The fake can be tossed once the
 929                            closure function has returned hence alloca()
 930                            is sufficient. */
 931                         int j;
 932                         UINT32 *p = avalue[i] = alloca (ty->size);
 933                         for (j = 0; j < element_count (ty); j++)
 934                           memcpy (&p[j],
 935                                   allocate_to_s (context, &state),
 936                                   sizeof (*p));
 937                         break;
 938                       }
 939
 940                     case FFI_TYPE_DOUBLE:
 941                       {
 942                         /* Eeek! We need a pointer to the structure,
 943                            however the homogeneous float elements are
 944                            being passed in individual S registers,
 945                            therefore the structure is not represented as
 946                            a contiguous sequence of bytes in our saved
 947                            register context. We need to fake up a copy
 948                            of the structure layed out in memory
 949                            correctly. The fake can be tossed once the
 950                            closure function has returned hence alloca()
 951                            is sufficient. */
 952                         int j;
 953                         UINT64 *p = avalue[i] = alloca (ty->size);
 954                         for (j = 0; j < element_count (ty); j++)
 955                           memcpy (&p[j],
 956                                   allocate_to_d (context, &state),
 957                                   sizeof (*p));
 958                         break;
 959                       }
 960
 961                     case FFI_TYPE_LONGDOUBLE:
 962                           memcpy (&avalue[i],
 963                                   allocate_to_v (context, &state),
 964                                   sizeof (*avalue));
 965                       break;
 966
 967                     default:
 968                       FFI_ASSERT (0);
 969                       break;
 970                     }
 971                 }
 972             }
 973           else if (ty->size > 16)
 974             {
 975               /* Replace Composite type of size greater than 16 with a
 976                  pointer.  */
 977               memcpy (&avalue[i],
 978                       allocate_to_register_or_stack (context, stack,
 979                                                      &state, FFI_TYPE_POINTER),
 980                       sizeof (avalue[i]));
 981             }
 982           else if (available_x (&state) >= (ty->size + 7) / 8)
 983             {
 984               avalue[i] = get_x_addr (context, state.ngrn);
 985               state.ngrn += (ty->size + 7) / 8;
 986             }
 987           else
 988             {
 989               state.ngrn = N_X_ARG_REG;
 990
 991               avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
 992                                              ty->size);
 993             }
 994           break;
 995
 996         default:
 997           FFI_ASSERT (0);
 998           break;
 999         }
1000     }
1001
1002   /* Figure out where the return value will be passed, either in
1003      registers or in a memory block allocated by the caller and passed
1004      in x8.  */
1005
1006   if (is_register_candidate (cif->rtype))
1007     {
1008       /* Register candidates are *always* returned in registers. */
1009
1010       /* Allocate a scratchpad for the return value, we will let the
1011          callee scrible the result into the scratch pad then move the
1012          contents into the appropriate return value location for the
1013          call convention.  */
1014       rvalue = alloca (cif->rtype->size);
1015       (closure->fun) (cif, rvalue, avalue, closure->user_data);
1016
1017       /* Copy the return value into the call context so that it is returned
1018          as expected to our caller.  */
1019       switch (cif->rtype->type)
1020         {
1021         case FFI_TYPE_VOID:
1022           break;
1023
1024         case FFI_TYPE_UINT8:
1025         case FFI_TYPE_UINT16:
1026         case FFI_TYPE_UINT32:
1027         case FFI_TYPE_POINTER:
1028         case FFI_TYPE_UINT64:
1029         case FFI_TYPE_SINT8:
1030         case FFI_TYPE_SINT16:
1031         case FFI_TYPE_INT:
1032         case FFI_TYPE_SINT32:
1033         case FFI_TYPE_SINT64:
1034         case FFI_TYPE_FLOAT:
1035         case FFI_TYPE_DOUBLE:
1036         case FFI_TYPE_LONGDOUBLE:
1037           {
1038             void *addr = get_basic_type_addr (cif->rtype->type, context, 0);
1039             copy_basic_type (addr, rvalue, cif->rtype->type);
1040             break;
1041           }
1042         case FFI_TYPE_STRUCT:
1043           if (is_hfa (cif->rtype))
1044             {
1045               int i;
1046               unsigned short type = get_homogeneous_type (cif->rtype);
1047               unsigned elems = element_count (cif->rtype);
1048               for (i = 0; i < elems; i++)
1049                 {
1050                   void *reg = get_basic_type_addr (type, context, i);
1051                   copy_basic_type (reg, rvalue, type);
1052                   rvalue += get_basic_type_size (type);
1053                 }
1054             }
1055           else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
1056             {
1057               unsigned size = ALIGN (cif->rtype->size, sizeof (UINT64)) ;
1058               memcpy (get_x_addr (context, 0), rvalue, size);
1059             }
1060           else
1061             {
1062               FFI_ASSERT (0);
1063             }
1064           break;
1065         default:
1066           FFI_ASSERT (0);
1067           break;
1068         }
1069     }
1070   else
1071     {
1072       memcpy (&rvalue, get_x_addr (context, 8), sizeof (UINT64));
1073       (closure->fun) (cif, rvalue, avalue, closure->user_data);
1074     }
1075 }
1076