libffi/src/x86/ffi64.c

   1 /* -----------------------------------------------------------------------
   2    ffi.c - Copyright (c) 2002, 2007  Bo Thorsen <bo@suse.de>
   3
   4    x86-64 Foreign Function Interface
   5
   6    Permission is hereby granted, free of charge, to any person obtaining
   7    a copy of this software and associated documentation files (the
   8    ``Software''), to deal in the Software without restriction, including
   9    without limitation the rights to use, copy, modify, merge, publish,
  10    distribute, sublicense, and/or sell copies of the Software, and to
  11    permit persons to whom the Software is furnished to do so, subject to
  12    the following conditions:
  13
  14    The above copyright notice and this permission notice shall be included
  15    in all copies or substantial portions of the Software.
  16
  17    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  18    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  20    NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  21    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  22    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24    DEALINGS IN THE SOFTWARE.
  25    ----------------------------------------------------------------------- */
  26
  27 #include <ffi.h>
  28 #include <ffi_common.h>
  29
  30 #include <stdlib.h>
  31 #include <stdarg.h>
  32
  33 #ifdef __x86_64__
  34
  35 #define MAX_GPR_REGS 6
  36 #define MAX_SSE_REGS 8
  37
  38 struct register_args
  39 {
  40   /* Registers for argument passing.  */
  41   UINT64 gpr[MAX_GPR_REGS];
  42   __int128_t sse[MAX_SSE_REGS];
  43 };
  44
  45 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
  46                              void *raddr, void (*fnaddr)(), unsigned ssecount);
  47
  48 /* All reference to register classes here is identical to the code in
  49    gcc/config/i386/i386.c. Do *not* change one without the other.  */
  50
  51 /* Register class used for passing given 64bit part of the argument.
  52    These represent classes as documented by the PS ABI, with the exception
  53    of SSESF, SSEDF classes, that are basically SSE class, just gcc will
  54    use SF or DFmode move instead of DImode to avoid reformating penalties.
  55
  56    Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
  57    whenever possible (upper half does contain padding).  */
  58 enum x86_64_reg_class
  59   {
  60     X86_64_NO_CLASS,
  61     X86_64_INTEGER_CLASS,
  62     X86_64_INTEGERSI_CLASS,
  63     X86_64_SSE_CLASS,
  64     X86_64_SSESF_CLASS,
  65     X86_64_SSEDF_CLASS,
  66     X86_64_SSEUP_CLASS,
  67     X86_64_X87_CLASS,
  68     X86_64_X87UP_CLASS,
  69     X86_64_COMPLEX_X87_CLASS,
  70     X86_64_MEMORY_CLASS
  71   };
  72
  73 #define MAX_CLASSES 4
  74
  75 #define SSE_CLASS_P(X)  ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
  76
  77 /* x86-64 register passing implementation.  See x86-64 ABI for details.  Goal
  78    of this code is to classify each 8bytes of incoming argument by the register
  79    class and assign registers accordingly.  */
  80
  81 /* Return the union class of CLASS1 and CLASS2.
  82    See the x86-64 PS ABI for details.  */
  83
  84 static enum x86_64_reg_class
  85 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
  86 {
  87   /* Rule #1: If both classes are equal, this is the resulting class.  */
  88   if (class1 == class2)
  89     return class1;
  90
  91   /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
  92      the other class.  */
  93   if (class1 == X86_64_NO_CLASS)
  94     return class2;
  95   if (class2 == X86_64_NO_CLASS)
  96     return class1;
  97
  98   /* Rule #3: If one of the classes is MEMORY, the result is MEMORY.  */
  99   if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
 100     return X86_64_MEMORY_CLASS;
 101
 102   /* Rule #4: If one of the classes is INTEGER, the result is INTEGER.  */
 103   if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
 104       || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
 105     return X86_64_INTEGERSI_CLASS;
 106   if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
 107       || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
 108     return X86_64_INTEGER_CLASS;
 109
 110   /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
 111      MEMORY is used.  */
 112   if (class1 == X86_64_X87_CLASS
 113       || class1 == X86_64_X87UP_CLASS
 114       || class1 == X86_64_COMPLEX_X87_CLASS
 115       || class2 == X86_64_X87_CLASS
 116       || class2 == X86_64_X87UP_CLASS
 117       || class2 == X86_64_COMPLEX_X87_CLASS)
 118     return X86_64_MEMORY_CLASS;
 119
 120   /* Rule #6: Otherwise class SSE is used.  */
 121   return X86_64_SSE_CLASS;
 122 }
 123
 124 /* Classify the argument of type TYPE and mode MODE.
 125    CLASSES will be filled by the register class used to pass each word
 126    of the operand.  The number of words is returned.  In case the parameter
 127    should be passed in memory, 0 is returned. As a special case for zero
 128    sized containers, classes[0] will be NO_CLASS and 1 is returned.
 129
 130    See the x86-64 PS ABI for details.
 131 */
 132 static int
 133 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
 134                    size_t byte_offset)
 135 {
 136   switch (type->type)
 137     {
 138     case FFI_TYPE_UINT8:
 139     case FFI_TYPE_SINT8:
 140     case FFI_TYPE_UINT16:
 141     case FFI_TYPE_SINT16:
 142     case FFI_TYPE_UINT32:
 143     case FFI_TYPE_SINT32:
 144     case FFI_TYPE_UINT64:
 145     case FFI_TYPE_SINT64:
 146     case FFI_TYPE_POINTER:
 147       if (byte_offset + type->size <= 4)
 148         classes[0] = X86_64_INTEGERSI_CLASS;
 149       else
 150         classes[0] = X86_64_INTEGER_CLASS;
 151       return 1;
 152     case FFI_TYPE_FLOAT:
 153       if (byte_offset == 0)
 154         classes[0] = X86_64_SSESF_CLASS;
 155       else
 156         classes[0] = X86_64_SSE_CLASS;
 157       return 1;
 158     case FFI_TYPE_DOUBLE:
 159       classes[0] = X86_64_SSEDF_CLASS;
 160       return 1;
 161     case FFI_TYPE_LONGDOUBLE:
 162       classes[0] = X86_64_X87_CLASS;
 163       classes[1] = X86_64_X87UP_CLASS;
 164       return 2;
 165     case FFI_TYPE_STRUCT:
 166       {
 167         const int UNITS_PER_WORD = 8;
 168         int words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
 169         ffi_type **ptr;
 170         int i;
 171         enum x86_64_reg_class subclasses[MAX_CLASSES];
 172
 173         /* If the struct is larger than 16 bytes, pass it on the stack.  */
 174         if (type->size > 16)
 175           return 0;
 176
 177         for (i = 0; i < words; i++)
 178           classes[i] = X86_64_NO_CLASS;
 179
 180         /* Merge the fields of structure.  */
 181         for (ptr = type->elements; *ptr != NULL; ptr++)
 182           {
 183             int num;
 184
 185             byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
 186
 187             num = classify_argument (*ptr, subclasses, byte_offset % 8);
 188             if (num == 0)
 189               return 0;
 190             for (i = 0; i < num; i++)
 191               {
 192                 int pos = byte_offset / 8;
 193                 classes[i + pos] =
 194                   merge_classes (subclasses[i], classes[i + pos]);
 195               }
 196
 197             byte_offset += (*ptr)->size;
 198           }
 199
 200         /* Final merger cleanup.  */
 201         for (i = 0; i < words; i++)
 202           {
 203             /* If one class is MEMORY, everything should be passed in
 204                memory.  */
 205             if (classes[i] == X86_64_MEMORY_CLASS)
 206               return 0;
 207
 208             /* The X86_64_SSEUP_CLASS should be always preceded by
 209                X86_64_SSE_CLASS.  */
 210             if (classes[i] == X86_64_SSEUP_CLASS
 211                 && (i == 0 || classes[i - 1] != X86_64_SSE_CLASS))
 212               classes[i] = X86_64_SSE_CLASS;
 213
 214             /*  X86_64_X87UP_CLASS should be preceded by X86_64_X87_CLASS.  */
 215             if (classes[i] == X86_64_X87UP_CLASS
 216                 && (i == 0 || classes[i - 1] != X86_64_X87_CLASS))
 217               classes[i] = X86_64_SSE_CLASS;
 218           }
 219         return words;
 220       }
 221
 222     default:
 223       FFI_ASSERT(0);
 224     }
 225   return 0; /* Never reached.  */
 226 }
 227
 228 /* Examine the argument and return set number of register required in each
 229    class.  Return zero iff parameter should be passed in memory, otherwise
 230    the number of registers.  */
 231
 232 static int
 233 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
 234                   _Bool in_return, int *pngpr, int *pnsse)
 235 {
 236   int i, n, ngpr, nsse;
 237
 238   n = classify_argument (type, classes, 0);
 239   if (n == 0)
 240     return 0;
 241
 242   ngpr = nsse = 0;
 243   for (i = 0; i < n; ++i)
 244     switch (classes[i])
 245       {
 246       case X86_64_INTEGER_CLASS:
 247       case X86_64_INTEGERSI_CLASS:
 248         ngpr++;
 249         break;
 250       case X86_64_SSE_CLASS:
 251       case X86_64_SSESF_CLASS:
 252       case X86_64_SSEDF_CLASS:
 253         nsse++;
 254         break;
 255       case X86_64_NO_CLASS:
 256       case X86_64_SSEUP_CLASS:
 257         break;
 258       case X86_64_X87_CLASS:
 259       case X86_64_X87UP_CLASS:
 260       case X86_64_COMPLEX_X87_CLASS:
 261         return in_return != 0;
 262       default:
 263         abort ();
 264       }
 265
 266   *pngpr = ngpr;
 267   *pnsse = nsse;
 268
 269   return n;
 270 }
 271
 272 /* Perform machine dependent cif processing.  */
 273
 274 ffi_status
 275 ffi_prep_cif_machdep (ffi_cif *cif)
 276 {
 277   int gprcount, ssecount, i, avn, n, ngpr, nsse, flags;
 278   enum x86_64_reg_class classes[MAX_CLASSES];
 279   size_t bytes;
 280
 281   gprcount = ssecount = 0;
 282
 283   flags = cif->rtype->type;
 284   if (flags != FFI_TYPE_VOID)
 285     {
 286       n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
 287       if (n == 0)
 288         {
 289           /* The return value is passed in memory.  A pointer to that
 290              memory is the first argument.  Allocate a register for it.  */
 291           gprcount++;
 292           /* We don't have to do anything in asm for the return.  */
 293           flags = FFI_TYPE_VOID;
 294         }
 295       else if (flags == FFI_TYPE_STRUCT)
 296         {
 297           /* Mark which registers the result appears in.  */
 298           _Bool sse0 = SSE_CLASS_P (classes[0]);
 299           _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
 300           if (sse0 && !sse1)
 301             flags |= 1 << 8;
 302           else if (!sse0 && sse1)
 303             flags |= 1 << 9;
 304           else if (sse0 && sse1)
 305             flags |= 1 << 10;
 306           /* Mark the true size of the structure.  */
 307           flags |= cif->rtype->size << 12;
 308         }
 309     }
 310
 311   /* Go over all arguments and determine the way they should be passed.
 312      If it's in a register and there is space for it, let that be so. If
 313      not, add it's size to the stack byte count.  */
 314   for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
 315     {
 316       if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
 317           || gprcount + ngpr > MAX_GPR_REGS
 318           || ssecount + nsse > MAX_SSE_REGS)
 319         {
 320           long align = cif->arg_types[i]->alignment;
 321
 322           if (align < 8)
 323             align = 8;
 324
 325           bytes = ALIGN(bytes, align);
 326           bytes += cif->arg_types[i]->size;
 327         }
 328       else
 329         {
 330           gprcount += ngpr;
 331           ssecount += nsse;
 332         }
 333     }
 334   if (ssecount)
 335     flags |= 1 << 11;
 336   cif->flags = flags;
 337   cif->bytes = bytes;
 338
 339   return FFI_OK;
 340 }
 341
 342 void
 343 ffi_call (ffi_cif *cif, void (*fn)(), void *rvalue, void **avalue)
 344 {
 345   enum x86_64_reg_class classes[MAX_CLASSES];
 346   char *stack, *argp;
 347   ffi_type **arg_types;
 348   int gprcount, ssecount, ngpr, nsse, i, avn;
 349   _Bool ret_in_memory;
 350   struct register_args *reg_args;
 351
 352   /* Can't call 32-bit mode from 64-bit mode.  */
 353   FFI_ASSERT (cif->abi == FFI_UNIX64);
 354
 355   /* If the return value is a struct and we don't have a return value
 356      address then we need to make one.  Note the setting of flags to
 357      VOID above in ffi_prep_cif_machdep.  */
 358   ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
 359                    && (cif->flags & 0xff) == FFI_TYPE_VOID);
 360   if (rvalue == NULL && ret_in_memory)
 361     rvalue = alloca (cif->rtype->size);
 362
 363   /* Allocate the space for the arguments, plus 4 words of temp space.  */
 364   stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
 365   reg_args = (struct register_args *) stack;
 366   argp = stack + sizeof (struct register_args);
 367
 368   gprcount = ssecount = 0;
 369
 370   /* If the return value is passed in memory, add the pointer as the
 371      first integer argument.  */
 372   if (ret_in_memory)
 373     reg_args->gpr[gprcount++] = (long) rvalue;
 374
 375   avn = cif->nargs;
 376   arg_types = cif->arg_types;
 377
 378   for (i = 0; i < avn; ++i)
 379     {
 380       size_t size = arg_types[i]->size;
 381       int n;
 382
 383       n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
 384       if (n == 0
 385           || gprcount + ngpr > MAX_GPR_REGS
 386           || ssecount + nsse > MAX_SSE_REGS)
 387         {
 388           long align = arg_types[i]->alignment;
 389
 390           /* Stack arguments are *always* at least 8 byte aligned.  */
 391           if (align < 8)
 392             align = 8;
 393
 394           /* Pass this argument in memory.  */
 395           argp = (void *) ALIGN (argp, align);
 396           memcpy (argp, avalue[i], size);
 397           argp += size;
 398         }
 399       else
 400         {
 401           /* The argument is passed entirely in registers.  */
 402           char *a = (char *) avalue[i];
 403           int j;
 404
 405           for (j = 0; j < n; j++, a += 8, size -= 8)
 406             {
 407               switch (classes[j])
 408                 {
 409                 case X86_64_INTEGER_CLASS:
 410                 case X86_64_INTEGERSI_CLASS:
 411                   reg_args->gpr[gprcount] = 0;
 412                   memcpy (&reg_args->gpr[gprcount], a, size < 8 ? size : 8);
 413                   gprcount++;
 414                   break;
 415                 case X86_64_SSE_CLASS:
 416                 case X86_64_SSEDF_CLASS:
 417                   reg_args->sse[ssecount++] = *(UINT64 *) a;
 418                   break;
 419                 case X86_64_SSESF_CLASS:
 420                   reg_args->sse[ssecount++] = *(UINT32 *) a;
 421                   break;
 422                 default:
 423                   abort();
 424                 }
 425             }
 426         }
 427     }
 428
 429   ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
 430                    cif->flags, rvalue, fn, ssecount);
 431 }
 432
 433
 434 extern void ffi_closure_unix64(void);
 435
 436 ffi_status
 437 ffi_prep_closure_loc (ffi_closure* closure,
 438                       ffi_cif* cif,
 439                       void (*fun)(ffi_cif*, void*, void**, void*),
 440                       void *user_data,
 441                       void *codeloc)
 442 {
 443   volatile unsigned short *tramp;
 444
 445   tramp = (volatile unsigned short *) &closure->tramp[0];
 446
 447   tramp[0] = 0xbb49;            /* mov <code>, %r11     */
 448   *(void * volatile *) &tramp[1] = ffi_closure_unix64;
 449   tramp[5] = 0xba49;            /* mov <data>, %r10     */
 450   *(void * volatile *) &tramp[6] = codeloc;
 451
 452   /* Set the carry bit iff the function uses any sse registers.
 453      This is clc or stc, together with the first byte of the jmp.  */
 454   tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8;
 455
 456   tramp[11] = 0xe3ff;                   /* jmp *%r11    */
 457
 458   closure->cif = cif;
 459   closure->fun = fun;
 460   closure->user_data = user_data;
 461
 462   return FFI_OK;
 463 }
 464
 465 int
 466 ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue,
 467                          struct register_args *reg_args, char *argp)
 468 {
 469   ffi_cif *cif;
 470   void **avalue;
 471   ffi_type **arg_types;
 472   long i, avn;
 473   int gprcount, ssecount, ngpr, nsse;
 474   int ret;
 475
 476   cif = closure->cif;
 477   avalue = alloca(cif->nargs * sizeof(void *));
 478   gprcount = ssecount = 0;
 479
 480   ret = cif->rtype->type;
 481   if (ret != FFI_TYPE_VOID)
 482     {
 483       enum x86_64_reg_class classes[MAX_CLASSES];
 484       int n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
 485       if (n == 0)
 486         {
 487           /* The return value goes in memory.  Arrange for the closure
 488              return value to go directly back to the original caller.  */
 489           rvalue = (void *) reg_args->gpr[gprcount++];
 490           /* We don't have to do anything in asm for the return.  */
 491           ret = FFI_TYPE_VOID;
 492         }
 493       else if (ret == FFI_TYPE_STRUCT && n == 2)
 494         {
 495           /* Mark which register the second word of the structure goes in.  */
 496           _Bool sse0 = SSE_CLASS_P (classes[0]);
 497           _Bool sse1 = SSE_CLASS_P (classes[1]);
 498           if (!sse0 && sse1)
 499             ret |= 1 << 8;
 500           else if (sse0 && !sse1)
 501             ret |= 1 << 9;
 502         }
 503     }
 504
 505   avn = cif->nargs;
 506   arg_types = cif->arg_types;
 507
 508   for (i = 0; i < avn; ++i)
 509     {
 510       enum x86_64_reg_class classes[MAX_CLASSES];
 511       int n;
 512
 513       n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
 514       if (n == 0
 515           || gprcount + ngpr > MAX_GPR_REGS
 516           || ssecount + nsse > MAX_SSE_REGS)
 517         {
 518           long align = arg_types[i]->alignment;
 519
 520           /* Stack arguments are *always* at least 8 byte aligned.  */
 521           if (align < 8)
 522             align = 8;
 523
 524           /* Pass this argument in memory.  */
 525           argp = (void *) ALIGN (argp, align);
 526           avalue[i] = argp;
 527           argp += arg_types[i]->size;
 528         }
 529       /* If the argument is in a single register, or two consecutive
 530          registers, then we can use that address directly.  */
 531       else if (n == 1
 532                || (n == 2
 533                    && SSE_CLASS_P (classes[0]) == SSE_CLASS_P (classes[1])))
 534         {
 535           /* The argument is in a single register.  */
 536           if (SSE_CLASS_P (classes[0]))
 537             {
 538               avalue[i] = &reg_args->sse[ssecount];
 539               ssecount += n;
 540             }
 541           else
 542             {
 543               avalue[i] = &reg_args->gpr[gprcount];
 544               gprcount += n;
 545             }
 546         }
 547       /* Otherwise, allocate space to make them consecutive.  */
 548       else
 549         {
 550           char *a = alloca (16);
 551           int j;
 552
 553           avalue[i] = a;
 554           for (j = 0; j < n; j++, a += 8)
 555             {
 556               if (SSE_CLASS_P (classes[j]))
 557                 memcpy (a, &reg_args->sse[ssecount++], 8);
 558               else
 559                 memcpy (a, &reg_args->gpr[gprcount++], 8);
 560             }
 561         }
 562     }
 563
 564   /* Invoke the closure.  */
 565   closure->fun (cif, rvalue, avalue, closure->user_data);
 566
 567   /* Tell assembly how to perform return type promotions.  */
 568   return ret;
 569 }
 570
 571 #endif /* __x86_64__ */