libffi/src/powerpc/ffi_linux64.c

   1 /* -----------------------------------------------------------------------
   2    ffi_linux64.c - Copyright (C) 2013 IBM
   3                    Copyright (C) 2011 Anthony Green
   4                    Copyright (C) 2011 Kyle Moffett
   5                    Copyright (C) 2008 Red Hat, Inc
   6                    Copyright (C) 2007, 2008 Free Software Foundation, Inc
   7                    Copyright (c) 1998 Geoffrey Keating
   8
   9    PowerPC Foreign Function Interface
  10
  11    Permission is hereby granted, free of charge, to any person obtaining
  12    a copy of this software and associated documentation files (the
  13    ``Software''), to deal in the Software without restriction, including
  14    without limitation the rights to use, copy, modify, merge, publish,
  15    distribute, sublicense, and/or sell copies of the Software, and to
  16    permit persons to whom the Software is furnished to do so, subject to
  17    the following conditions:
  18
  19    The above copyright notice and this permission notice shall be included
  20    in all copies or substantial portions of the Software.
  21
  22    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
  23    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  25    IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR
  26    OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  27    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  28    OTHER DEALINGS IN THE SOFTWARE.
  29    ----------------------------------------------------------------------- */
  30
  31 #include "ffi.h"
  32
  33 #ifdef POWERPC64
  34 #include "ffi_common.h"
  35 #include "ffi_powerpc.h"
  36
  37
  38 /* About the LINUX64 ABI.  */
  39 enum {
  40   NUM_GPR_ARG_REGISTERS64 = 8,
  41   NUM_FPR_ARG_REGISTERS64 = 13
  42 };
  43 enum { ASM_NEEDS_REGISTERS64 = 4 };
  44
  45
  46 #if HAVE_LONG_DOUBLE_VARIANT && FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
  47 /* Adjust size of ffi_type_longdouble.  */
  48 void FFI_HIDDEN
  49 ffi_prep_types_linux64 (ffi_abi abi)
  50 {
  51   if ((abi & (FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128)) == FFI_LINUX)
  52     {
  53       ffi_type_longdouble.size = 8;
  54       ffi_type_longdouble.alignment = 8;
  55     }
  56   else
  57     {
  58       ffi_type_longdouble.size = 16;
  59       ffi_type_longdouble.alignment = 16;
  60     }
  61 }
  62 #endif
  63
  64
  65 static unsigned int
  66 discover_homogeneous_aggregate (const ffi_type *t, unsigned int *elnum)
  67 {
  68   switch (t->type)
  69     {
  70     case FFI_TYPE_FLOAT:
  71     case FFI_TYPE_DOUBLE:
  72       *elnum = 1;
  73       return (int) t->type;
  74
  75     case FFI_TYPE_STRUCT:;
  76       {
  77         unsigned int base_elt = 0, total_elnum = 0;
  78         ffi_type **el = t->elements;
  79         while (*el)
  80           {
  81             unsigned int el_elt, el_elnum = 0;
  82             el_elt = discover_homogeneous_aggregate (*el, &el_elnum);
  83             if (el_elt == 0
  84                 || (base_elt && base_elt != el_elt))
  85               return 0;
  86             base_elt = el_elt;
  87             total_elnum += el_elnum;
  88 #if _CALL_ELF == 2
  89             if (total_elnum > 8)
  90               return 0;
  91 #else
  92             if (total_elnum > 1)
  93               return 0;
  94 #endif
  95             el++;
  96           }
  97         *elnum = total_elnum;
  98         return base_elt;
  99       }
 100
 101     default:
 102       return 0;
 103     }
 104 }
 105
 106
 107 /* Perform machine dependent cif processing */
 108 static ffi_status
 109 ffi_prep_cif_linux64_core (ffi_cif *cif)
 110 {
 111   ffi_type **ptr;
 112   unsigned bytes;
 113   unsigned i, fparg_count = 0, intarg_count = 0;
 114   unsigned flags = cif->flags;
 115   unsigned int elt, elnum;
 116
 117 #if FFI_TYPE_LONGDOUBLE == FFI_TYPE_DOUBLE
 118   /* If compiled without long double support..  */
 119   if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
 120     return FFI_BAD_ABI;
 121 #endif
 122
 123   /* The machine-independent calculation of cif->bytes doesn't work
 124      for us.  Redo the calculation.  */
 125 #if _CALL_ELF == 2
 126   /* Space for backchain, CR, LR, TOC and the asm's temp regs.  */
 127   bytes = (4 + ASM_NEEDS_REGISTERS64) * sizeof (long);
 128
 129   /* Space for the general registers.  */
 130   bytes += NUM_GPR_ARG_REGISTERS64 * sizeof (long);
 131 #else
 132   /* Space for backchain, CR, LR, cc/ld doubleword, TOC and the asm's temp
 133      regs.  */
 134   bytes = (6 + ASM_NEEDS_REGISTERS64) * sizeof (long);
 135
 136   /* Space for the mandatory parm save area and general registers.  */
 137   bytes += 2 * NUM_GPR_ARG_REGISTERS64 * sizeof (long);
 138 #endif
 139
 140   /* Return value handling.  */
 141   switch (cif->rtype->type)
 142     {
 143 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 144     case FFI_TYPE_LONGDOUBLE:
 145       if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
 146         flags |= FLAG_RETURNS_128BITS;
 147       /* Fall through.  */
 148 #endif
 149     case FFI_TYPE_DOUBLE:
 150       flags |= FLAG_RETURNS_64BITS;
 151       /* Fall through.  */
 152     case FFI_TYPE_FLOAT:
 153       flags |= FLAG_RETURNS_FP;
 154       break;
 155
 156     case FFI_TYPE_UINT128:
 157       flags |= FLAG_RETURNS_128BITS;
 158       /* Fall through.  */
 159     case FFI_TYPE_UINT64:
 160     case FFI_TYPE_SINT64:
 161     case FFI_TYPE_POINTER:
 162       flags |= FLAG_RETURNS_64BITS;
 163       break;
 164
 165     case FFI_TYPE_STRUCT:
 166 #if _CALL_ELF == 2
 167       elt = discover_homogeneous_aggregate (cif->rtype, &elnum);
 168       if (elt)
 169         {
 170           if (elt == FFI_TYPE_DOUBLE)
 171             flags |= FLAG_RETURNS_64BITS;
 172           flags |= FLAG_RETURNS_FP | FLAG_RETURNS_SMST;
 173           break;
 174         }
 175       if (cif->rtype->size <= 16)
 176         {
 177           flags |= FLAG_RETURNS_SMST;
 178           break;
 179         }
 180 #endif
 181       intarg_count++;
 182       flags |= FLAG_RETVAL_REFERENCE;
 183       /* Fall through.  */
 184     case FFI_TYPE_VOID:
 185       flags |= FLAG_RETURNS_NOTHING;
 186       break;
 187
 188     default:
 189       /* Returns 32-bit integer, or similar.  Nothing to do here.  */
 190       break;
 191     }
 192
 193   for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
 194     {
 195       unsigned int align;
 196
 197       switch ((*ptr)->type)
 198         {
 199 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 200         case FFI_TYPE_LONGDOUBLE:
 201           if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
 202             {
 203               fparg_count++;
 204               intarg_count++;
 205             }
 206           /* Fall through.  */
 207 #endif
 208         case FFI_TYPE_DOUBLE:
 209         case FFI_TYPE_FLOAT:
 210           fparg_count++;
 211           intarg_count++;
 212           if (fparg_count > NUM_FPR_ARG_REGISTERS64)
 213             flags |= FLAG_ARG_NEEDS_PSAVE;
 214           break;
 215
 216         case FFI_TYPE_STRUCT:
 217           if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
 218             {
 219               align = (*ptr)->alignment;
 220               if (align > 16)
 221                 align = 16;
 222               align = align / 8;
 223               if (align > 1)
 224                 intarg_count = ALIGN (intarg_count, align);
 225             }
 226           intarg_count += ((*ptr)->size + 7) / 8;
 227           elt = discover_homogeneous_aggregate (*ptr, &elnum);
 228           if (elt)
 229             {
 230               fparg_count += elnum;
 231               if (fparg_count > NUM_FPR_ARG_REGISTERS64)
 232                 flags |= FLAG_ARG_NEEDS_PSAVE;
 233             }
 234           else
 235             {
 236               if (intarg_count > NUM_GPR_ARG_REGISTERS64)
 237                 flags |= FLAG_ARG_NEEDS_PSAVE;
 238             }
 239           break;
 240
 241         case FFI_TYPE_POINTER:
 242         case FFI_TYPE_UINT64:
 243         case FFI_TYPE_SINT64:
 244         case FFI_TYPE_INT:
 245         case FFI_TYPE_UINT32:
 246         case FFI_TYPE_SINT32:
 247         case FFI_TYPE_UINT16:
 248         case FFI_TYPE_SINT16:
 249         case FFI_TYPE_UINT8:
 250         case FFI_TYPE_SINT8:
 251           /* Everything else is passed as a 8-byte word in a GPR, either
 252              the object itself or a pointer to it.  */
 253           intarg_count++;
 254           if (intarg_count > NUM_GPR_ARG_REGISTERS64)
 255             flags |= FLAG_ARG_NEEDS_PSAVE;
 256           break;
 257         default:
 258           FFI_ASSERT (0);
 259         }
 260     }
 261
 262   if (fparg_count != 0)
 263     flags |= FLAG_FP_ARGUMENTS;
 264   if (intarg_count > 4)
 265     flags |= FLAG_4_GPR_ARGUMENTS;
 266
 267   /* Space for the FPR registers, if needed.  */
 268   if (fparg_count != 0)
 269     bytes += NUM_FPR_ARG_REGISTERS64 * sizeof (double);
 270
 271   /* Stack space.  */
 272 #if _CALL_ELF == 2
 273   if ((flags & FLAG_ARG_NEEDS_PSAVE) != 0)
 274     bytes += intarg_count * sizeof (long);
 275 #else
 276   if (intarg_count > NUM_GPR_ARG_REGISTERS64)
 277     bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) * sizeof (long);
 278 #endif
 279
 280   /* The stack space allocated needs to be a multiple of 16 bytes.  */
 281   bytes = (bytes + 15) & ~0xF;
 282
 283   cif->flags = flags;
 284   cif->bytes = bytes;
 285
 286   return FFI_OK;
 287 }
 288
 289 ffi_status FFI_HIDDEN
 290 ffi_prep_cif_linux64 (ffi_cif *cif)
 291 {
 292   if ((cif->abi & FFI_LINUX) != 0)
 293     cif->nfixedargs = cif->nargs;
 294 #if _CALL_ELF != 2
 295   else if (cif->abi == FFI_COMPAT_LINUX64)
 296     {
 297       /* This call is from old code.  Don't touch cif->nfixedargs
 298          since old code will be using a smaller cif.  */
 299       cif->flags |= FLAG_COMPAT;
 300       /* Translate to new abi value.  */
 301       cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
 302     }
 303 #endif
 304   else
 305     return FFI_BAD_ABI;
 306   return ffi_prep_cif_linux64_core (cif);
 307 }
 308
 309 ffi_status FFI_HIDDEN
 310 ffi_prep_cif_linux64_var (ffi_cif *cif,
 311                           unsigned int nfixedargs,
 312                           unsigned int ntotalargs MAYBE_UNUSED)
 313 {
 314   if ((cif->abi & FFI_LINUX) != 0)
 315     cif->nfixedargs = nfixedargs;
 316 #if _CALL_ELF != 2
 317   else if (cif->abi == FFI_COMPAT_LINUX64)
 318     {
 319       /* This call is from old code.  Don't touch cif->nfixedargs
 320          since old code will be using a smaller cif.  */
 321       cif->flags |= FLAG_COMPAT;
 322       /* Translate to new abi value.  */
 323       cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
 324     }
 325 #endif
 326   else
 327     return FFI_BAD_ABI;
 328 #if _CALL_ELF == 2
 329   cif->flags |= FLAG_ARG_NEEDS_PSAVE;
 330 #endif
 331   return ffi_prep_cif_linux64_core (cif);
 332 }
 333
 334
 335 /* ffi_prep_args64 is called by the assembly routine once stack space
 336    has been allocated for the function's arguments.
 337
 338    The stack layout we want looks like this:
 339
 340    |   Ret addr from ffi_call_LINUX64   8bytes  |       higher addresses
 341    |--------------------------------------------|
 342    |   CR save area                     8bytes  |
 343    |--------------------------------------------|
 344    |   Previous backchain pointer       8       |       stack pointer here
 345    |--------------------------------------------|<+ <<< on entry to
 346    |   Saved r28-r31                    4*8     | |     ffi_call_LINUX64
 347    |--------------------------------------------| |
 348    |   GPR registers r3-r10             8*8     | |
 349    |--------------------------------------------| |
 350    |   FPR registers f1-f13 (optional)  13*8    | |
 351    |--------------------------------------------| |
 352    |   Parameter save area                      | |
 353    |--------------------------------------------| |
 354    |   TOC save area                    8       | |
 355    |--------------------------------------------| |     stack   |
 356    |   Linker doubleword                8       | |     grows   |
 357    |--------------------------------------------| |     down    V
 358    |   Compiler doubleword              8       | |
 359    |--------------------------------------------| |     lower addresses
 360    |   Space for callee's LR            8       | |
 361    |--------------------------------------------| |
 362    |   CR save area                     8       | |
 363    |--------------------------------------------| |     stack pointer here
 364    |   Current backchain pointer        8       |-/     during
 365    |--------------------------------------------|   <<< ffi_call_LINUX64
 366
 367 */
 368
 369 void FFI_HIDDEN
 370 ffi_prep_args64 (extended_cif *ecif, unsigned long *const stack)
 371 {
 372   const unsigned long bytes = ecif->cif->bytes;
 373   const unsigned long flags = ecif->cif->flags;
 374
 375   typedef union
 376   {
 377     char *c;
 378     unsigned long *ul;
 379     float *f;
 380     double *d;
 381     size_t p;
 382   } valp;
 383
 384   /* 'stacktop' points at the previous backchain pointer.  */
 385   valp stacktop;
 386
 387   /* 'next_arg' points at the space for gpr3, and grows upwards as
 388      we use GPR registers, then continues at rest.  */
 389   valp gpr_base;
 390   valp gpr_end;
 391   valp rest;
 392   valp next_arg;
 393
 394   /* 'fpr_base' points at the space for fpr3, and grows upwards as
 395      we use FPR registers.  */
 396   valp fpr_base;
 397   unsigned int fparg_count;
 398
 399   unsigned int i, words, nargs, nfixedargs;
 400   ffi_type **ptr;
 401   double double_tmp;
 402   union
 403   {
 404     void **v;
 405     char **c;
 406     signed char **sc;
 407     unsigned char **uc;
 408     signed short **ss;
 409     unsigned short **us;
 410     signed int **si;
 411     unsigned int **ui;
 412     unsigned long **ul;
 413     float **f;
 414     double **d;
 415   } p_argv;
 416   unsigned long gprvalue;
 417   unsigned long align;
 418
 419   stacktop.c = (char *) stack + bytes;
 420   gpr_base.ul = stacktop.ul - ASM_NEEDS_REGISTERS64 - NUM_GPR_ARG_REGISTERS64;
 421   gpr_end.ul = gpr_base.ul + NUM_GPR_ARG_REGISTERS64;
 422 #if _CALL_ELF == 2
 423   rest.ul = stack + 4 + NUM_GPR_ARG_REGISTERS64;
 424 #else
 425   rest.ul = stack + 6 + NUM_GPR_ARG_REGISTERS64;
 426 #endif
 427   fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS64;
 428   fparg_count = 0;
 429   next_arg.ul = gpr_base.ul;
 430
 431   /* Check that everything starts aligned properly.  */
 432   FFI_ASSERT (((unsigned long) (char *) stack & 0xF) == 0);
 433   FFI_ASSERT (((unsigned long) stacktop.c & 0xF) == 0);
 434   FFI_ASSERT ((bytes & 0xF) == 0);
 435
 436   /* Deal with return values that are actually pass-by-reference.  */
 437   if (flags & FLAG_RETVAL_REFERENCE)
 438     *next_arg.ul++ = (unsigned long) (char *) ecif->rvalue;
 439
 440   /* Now for the arguments.  */
 441   p_argv.v = ecif->avalue;
 442   nargs = ecif->cif->nargs;
 443 #if _CALL_ELF != 2
 444   nfixedargs = (unsigned) -1;
 445   if ((flags & FLAG_COMPAT) == 0)
 446 #endif
 447     nfixedargs = ecif->cif->nfixedargs;
 448   for (ptr = ecif->cif->arg_types, i = 0;
 449        i < nargs;
 450        i++, ptr++, p_argv.v++)
 451     {
 452       unsigned int elt, elnum;
 453
 454       switch ((*ptr)->type)
 455         {
 456 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 457         case FFI_TYPE_LONGDOUBLE:
 458           if ((ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
 459             {
 460               double_tmp = (*p_argv.d)[0];
 461               if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
 462                 {
 463                   *fpr_base.d++ = double_tmp;
 464 # if _CALL_ELF != 2
 465                   if ((flags & FLAG_COMPAT) != 0)
 466                     *next_arg.d = double_tmp;
 467 # endif
 468                 }
 469               else
 470                 *next_arg.d = double_tmp;
 471               if (++next_arg.ul == gpr_end.ul)
 472                 next_arg.ul = rest.ul;
 473               fparg_count++;
 474               double_tmp = (*p_argv.d)[1];
 475               if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
 476                 {
 477                   *fpr_base.d++ = double_tmp;
 478 # if _CALL_ELF != 2
 479                   if ((flags & FLAG_COMPAT) != 0)
 480                     *next_arg.d = double_tmp;
 481 # endif
 482                 }
 483               else
 484                 *next_arg.d = double_tmp;
 485               if (++next_arg.ul == gpr_end.ul)
 486                 next_arg.ul = rest.ul;
 487               fparg_count++;
 488               FFI_ASSERT (__LDBL_MANT_DIG__ == 106);
 489               FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
 490               break;
 491             }
 492           /* Fall through.  */
 493 #endif
 494         case FFI_TYPE_DOUBLE:
 495         do_double:
 496           double_tmp = **p_argv.d;
 497           if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
 498             {
 499               *fpr_base.d++ = double_tmp;
 500 #if _CALL_ELF != 2
 501               if ((flags & FLAG_COMPAT) != 0)
 502                 *next_arg.d = double_tmp;
 503 #endif
 504             }
 505           else
 506             *next_arg.d = double_tmp;
 507           if (++next_arg.ul == gpr_end.ul)
 508             next_arg.ul = rest.ul;
 509           fparg_count++;
 510           FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
 511           break;
 512
 513         case FFI_TYPE_FLOAT:
 514         do_float:
 515           double_tmp = **p_argv.f;
 516           if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
 517             {
 518               *fpr_base.d++ = double_tmp;
 519 #if _CALL_ELF != 2
 520               if ((flags & FLAG_COMPAT) != 0)
 521                 {
 522 # ifndef __LITTLE_ENDIAN__
 523                   next_arg.f[1] = (float) double_tmp;
 524 # else
 525                   next_arg.f[0] = (float) double_tmp;
 526 # endif
 527                 }
 528 #endif
 529             }
 530           else
 531             {
 532 # ifndef __LITTLE_ENDIAN__
 533               next_arg.f[1] = (float) double_tmp;
 534 # else
 535               next_arg.f[0] = (float) double_tmp;
 536 # endif
 537             }
 538           if (++next_arg.ul == gpr_end.ul)
 539             next_arg.ul = rest.ul;
 540           fparg_count++;
 541           FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
 542           break;
 543
 544         case FFI_TYPE_STRUCT:
 545           if ((ecif->cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
 546             {
 547               align = (*ptr)->alignment;
 548               if (align > 16)
 549                 align = 16;
 550               if (align > 1)
 551                 next_arg.p = ALIGN (next_arg.p, align);
 552             }
 553           elt = discover_homogeneous_aggregate (*ptr, &elnum);
 554           if (elt)
 555             {
 556 #if _CALL_ELF == 2
 557               union {
 558                 void *v;
 559                 float *f;
 560                 double *d;
 561               } arg;
 562
 563               arg.v = *p_argv.v;
 564               if (elt == FFI_TYPE_FLOAT)
 565                 {
 566                   do
 567                     {
 568                       double_tmp = *arg.f++;
 569                       if (fparg_count < NUM_FPR_ARG_REGISTERS64
 570                           && i < nfixedargs)
 571                         *fpr_base.d++ = double_tmp;
 572                       else
 573                         *next_arg.f = (float) double_tmp;
 574                       if (++next_arg.f == gpr_end.f)
 575                         next_arg.f = rest.f;
 576                       fparg_count++;
 577                     }
 578                   while (--elnum != 0);
 579                   if ((next_arg.p & 3) != 0)
 580                     {
 581                       if (++next_arg.f == gpr_end.f)
 582                         next_arg.f = rest.f;
 583                     }
 584                 }
 585               else
 586                 do
 587                   {
 588                     double_tmp = *arg.d++;
 589                     if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
 590                       *fpr_base.d++ = double_tmp;
 591                     else
 592                       *next_arg.d = double_tmp;
 593                     if (++next_arg.d == gpr_end.d)
 594                       next_arg.d = rest.d;
 595                     fparg_count++;
 596                   }
 597                 while (--elnum != 0);
 598 #else
 599               if (elt == FFI_TYPE_FLOAT)
 600                 goto do_float;
 601               else
 602                 goto do_double;
 603 #endif
 604             }
 605           else
 606             {
 607               words = ((*ptr)->size + 7) / 8;
 608               if (next_arg.ul >= gpr_base.ul && next_arg.ul + words > gpr_end.ul)
 609                 {
 610                   size_t first = gpr_end.c - next_arg.c;
 611                   memcpy (next_arg.c, *p_argv.c, first);
 612                   memcpy (rest.c, *p_argv.c + first, (*ptr)->size - first);
 613                   next_arg.c = rest.c + words * 8 - first;
 614                 }
 615               else
 616                 {
 617                   char *where = next_arg.c;
 618
 619 #ifndef __LITTLE_ENDIAN__
 620                   /* Structures with size less than eight bytes are passed
 621                      left-padded.  */
 622                   if ((*ptr)->size < 8)
 623                     where += 8 - (*ptr)->size;
 624 #endif
 625                   memcpy (where, *p_argv.c, (*ptr)->size);
 626                   next_arg.ul += words;
 627                   if (next_arg.ul == gpr_end.ul)
 628                     next_arg.ul = rest.ul;
 629                 }
 630             }
 631           break;
 632
 633         case FFI_TYPE_UINT8:
 634           gprvalue = **p_argv.uc;
 635           goto putgpr;
 636         case FFI_TYPE_SINT8:
 637           gprvalue = **p_argv.sc;
 638           goto putgpr;
 639         case FFI_TYPE_UINT16:
 640           gprvalue = **p_argv.us;
 641           goto putgpr;
 642         case FFI_TYPE_SINT16:
 643           gprvalue = **p_argv.ss;
 644           goto putgpr;
 645         case FFI_TYPE_UINT32:
 646           gprvalue = **p_argv.ui;
 647           goto putgpr;
 648         case FFI_TYPE_INT:
 649         case FFI_TYPE_SINT32:
 650           gprvalue = **p_argv.si;
 651           goto putgpr;
 652
 653         case FFI_TYPE_UINT64:
 654         case FFI_TYPE_SINT64:
 655         case FFI_TYPE_POINTER:
 656           gprvalue = **p_argv.ul;
 657         putgpr:
 658           *next_arg.ul++ = gprvalue;
 659           if (next_arg.ul == gpr_end.ul)
 660             next_arg.ul = rest.ul;
 661           break;
 662         }
 663     }
 664
 665   FFI_ASSERT (flags & FLAG_4_GPR_ARGUMENTS
 666               || (next_arg.ul >= gpr_base.ul
 667                   && next_arg.ul <= gpr_base.ul + 4));
 668 }
 669
 670
 671 #if _CALL_ELF == 2
 672 #define MIN_CACHE_LINE_SIZE 8
 673
 674 static void
 675 flush_icache (char *wraddr, char *xaddr, int size)
 676 {
 677   int i;
 678   for (i = 0; i < size; i += MIN_CACHE_LINE_SIZE)
 679     __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;"
 680                       : : "r" (xaddr + i), "r" (wraddr + i) : "memory");
 681   __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
 682                     : : "r"(xaddr + size - 1), "r"(wraddr + size - 1)
 683                     : "memory");
 684 }
 685 #endif
 686
 687
 688 ffi_status FFI_HIDDEN
 689 ffi_prep_closure_loc_linux64 (ffi_closure *closure,
 690                               ffi_cif *cif,
 691                               void (*fun) (ffi_cif *, void *, void **, void *),
 692                               void *user_data,
 693                               void *codeloc)
 694 {
 695 #if _CALL_ELF == 2
 696   unsigned int *tramp = (unsigned int *) &closure->tramp[0];
 697
 698   if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
 699     return FFI_BAD_ABI;
 700
 701   tramp[0] = 0xe96c0018;        /* 0:   ld      11,2f-0b(12)    */
 702   tramp[1] = 0xe98c0010;        /*      ld      12,1f-0b(12)    */
 703   tramp[2] = 0x7d8903a6;        /*      mtctr   12              */
 704   tramp[3] = 0x4e800420;        /*      bctr                    */
 705                                 /* 1:   .quad   function_addr   */
 706                                 /* 2:   .quad   context         */
 707   *(void **) &tramp[4] = (void *) ffi_closure_LINUX64;
 708   *(void **) &tramp[6] = codeloc;
 709   flush_icache ((char *) tramp, (char *) codeloc, 4 * 4);
 710 #else
 711   void **tramp = (void **) &closure->tramp[0];
 712
 713   if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
 714     return FFI_BAD_ABI;
 715
 716   /* Copy function address and TOC from ffi_closure_LINUX64 OPD.  */
 717   memcpy (&tramp[0], (void **) ffi_closure_LINUX64, sizeof (void *));
 718   tramp[1] = codeloc;
 719   memcpy (&tramp[2], (void **) ffi_closure_LINUX64 + 1, sizeof (void *));
 720 #endif
 721
 722   closure->cif = cif;
 723   closure->fun = fun;
 724   closure->user_data = user_data;
 725
 726   return FFI_OK;
 727 }
 728
 729
 730 int FFI_HIDDEN
 731 ffi_closure_helper_LINUX64 (ffi_cif *cif,
 732                             void (*fun) (ffi_cif *, void *, void **, void *),
 733                             void *user_data,
 734                             void *rvalue,
 735                             unsigned long *pst,
 736                             ffi_dblfl *pfr)
 737 {
 738   /* rvalue is the pointer to space for return value in closure assembly */
 739   /* pst is the pointer to parameter save area
 740      (r3-r10 are stored into its first 8 slots by ffi_closure_LINUX64) */
 741   /* pfr is the pointer to where f1-f13 are stored in ffi_closure_LINUX64 */
 742
 743   void **avalue;
 744   ffi_type **arg_types;
 745   unsigned long i, avn, nfixedargs;
 746   ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
 747   unsigned long align;
 748
 749   avalue = alloca (cif->nargs * sizeof (void *));
 750
 751   /* Copy the caller's structure return value address so that the
 752      closure returns the data directly to the caller.  */
 753   if (cif->rtype->type == FFI_TYPE_STRUCT
 754       && (cif->flags & FLAG_RETURNS_SMST) == 0)
 755     {
 756       rvalue = (void *) *pst;
 757       pst++;
 758     }
 759
 760   i = 0;
 761   avn = cif->nargs;
 762 #if _CALL_ELF != 2
 763   nfixedargs = (unsigned) -1;
 764   if ((cif->flags & FLAG_COMPAT) == 0)
 765 #endif
 766     nfixedargs = cif->nfixedargs;
 767   arg_types = cif->arg_types;
 768
 769   /* Grab the addresses of the arguments from the stack frame.  */
 770   while (i < avn)
 771     {
 772       unsigned int elt, elnum;
 773
 774       switch (arg_types[i]->type)
 775         {
 776         case FFI_TYPE_SINT8:
 777         case FFI_TYPE_UINT8:
 778 #ifndef __LITTLE_ENDIAN__
 779           avalue[i] = (char *) pst + 7;
 780           pst++;
 781           break;
 782 #endif
 783
 784         case FFI_TYPE_SINT16:
 785         case FFI_TYPE_UINT16:
 786 #ifndef __LITTLE_ENDIAN__
 787           avalue[i] = (char *) pst + 6;
 788           pst++;
 789           break;
 790 #endif
 791
 792         case FFI_TYPE_SINT32:
 793         case FFI_TYPE_UINT32:
 794 #ifndef __LITTLE_ENDIAN__
 795           avalue[i] = (char *) pst + 4;
 796           pst++;
 797           break;
 798 #endif
 799
 800         case FFI_TYPE_SINT64:
 801         case FFI_TYPE_UINT64:
 802         case FFI_TYPE_POINTER:
 803           avalue[i] = pst;
 804           pst++;
 805           break;
 806
 807         case FFI_TYPE_STRUCT:
 808           if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
 809             {
 810               align = arg_types[i]->alignment;
 811               if (align > 16)
 812                 align = 16;
 813               if (align > 1)
 814                 pst = (unsigned long *) ALIGN ((size_t) pst, align);
 815             }
 816           elt = discover_homogeneous_aggregate (arg_types[i], &elnum);
 817           if (elt)
 818             {
 819 #if _CALL_ELF == 2
 820               union {
 821                 void *v;
 822                 unsigned long *ul;
 823                 float *f;
 824                 double *d;
 825                 size_t p;
 826               } to, from;
 827
 828               /* Repackage the aggregate from its parts.  The
 829                  aggregate size is not greater than the space taken by
 830                  the registers so store back to the register/parameter
 831                  save arrays.  */
 832               if (pfr + elnum <= end_pfr)
 833                 to.v = pfr;
 834               else
 835                 to.v = pst;
 836
 837               avalue[i] = to.v;
 838               from.ul = pst;
 839               if (elt == FFI_TYPE_FLOAT)
 840                 {
 841                   do
 842                     {
 843                       if (pfr < end_pfr && i < nfixedargs)
 844                         {
 845                           *to.f = (float) pfr->d;
 846                           pfr++;
 847                         }
 848                       else
 849                         *to.f = *from.f;
 850                       to.f++;
 851                       from.f++;
 852                     }
 853                   while (--elnum != 0);
 854                 }
 855               else
 856                 {
 857                   do
 858                     {
 859                       if (pfr < end_pfr && i < nfixedargs)
 860                         {
 861                           *to.d = pfr->d;
 862                           pfr++;
 863                         }
 864                       else
 865                         *to.d = *from.d;
 866                       to.d++;
 867                       from.d++;
 868                     }
 869                   while (--elnum != 0);
 870                 }
 871 #else
 872               if (elt == FFI_TYPE_FLOAT)
 873                 goto do_float;
 874               else
 875                 goto do_double;
 876 #endif
 877             }
 878           else
 879             {
 880 #ifndef __LITTLE_ENDIAN__
 881               /* Structures with size less than eight bytes are passed
 882                  left-padded.  */
 883               if (arg_types[i]->size < 8)
 884                 avalue[i] = (char *) pst + 8 - arg_types[i]->size;
 885               else
 886 #endif
 887                 avalue[i] = pst;
 888             }
 889           pst += (arg_types[i]->size + 7) / 8;
 890           break;
 891
 892 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
 893         case FFI_TYPE_LONGDOUBLE:
 894           if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
 895             {
 896               if (pfr + 1 < end_pfr && i + 1 < nfixedargs)
 897                 {
 898                   avalue[i] = pfr;
 899                   pfr += 2;
 900                 }
 901               else
 902                 {
 903                   if (pfr < end_pfr && i < nfixedargs)
 904                     {
 905                       /* Passed partly in f13 and partly on the stack.
 906                          Move it all to the stack.  */
 907                       *pst = *(unsigned long *) pfr;
 908                       pfr++;
 909                     }
 910                   avalue[i] = pst;
 911                 }
 912               pst += 2;
 913               break;
 914             }
 915           /* Fall through.  */
 916 #endif
 917         case FFI_TYPE_DOUBLE:
 918         do_double:
 919           /* On the outgoing stack all values are aligned to 8 */
 920           /* there are 13 64bit floating point registers */
 921
 922           if (pfr < end_pfr && i < nfixedargs)
 923             {
 924               avalue[i] = pfr;
 925               pfr++;
 926             }
 927           else
 928             avalue[i] = pst;
 929           pst++;
 930           break;
 931
 932         case FFI_TYPE_FLOAT:
 933         do_float:
 934           if (pfr < end_pfr && i < nfixedargs)
 935             {
 936               /* Float values are stored as doubles in the
 937                  ffi_closure_LINUX64 code.  Fix them here.  */
 938               pfr->f = (float) pfr->d;
 939               avalue[i] = pfr;
 940               pfr++;
 941             }
 942           else
 943             {
 944 #ifndef __LITTLE_ENDIAN__
 945               avalue[i] = (char *) pst + 4;
 946 #else
 947               avalue[i] = pst;
 948 #endif
 949             }
 950           pst++;
 951           break;
 952
 953         default:
 954           FFI_ASSERT (0);
 955         }
 956
 957       i++;
 958     }
 959
 960   (*fun) (cif, rvalue, avalue, user_data);
 961
 962   /* Tell ffi_closure_LINUX64 how to perform return type promotions.  */
 963   if ((cif->flags & FLAG_RETURNS_SMST) != 0)
 964     {
 965       if ((cif->flags & FLAG_RETURNS_FP) == 0)
 966         return FFI_V2_TYPE_SMALL_STRUCT + cif->rtype->size - 1;
 967       else if ((cif->flags & FLAG_RETURNS_64BITS) != 0)
 968         return FFI_V2_TYPE_DOUBLE_HOMOG;
 969       else
 970         return FFI_V2_TYPE_FLOAT_HOMOG;
 971     }
 972   return cif->rtype->type;
 973 }
 974 #endif