libffi/src/riscv/ffi.c

   1 /* -----------------------------------------------------------------------
   2    ffi.c - Copyright (c) 2015 Michael Knyszek <mknyszek@berkeley.edu>
   3                          2015 Andrew Waterman <waterman@cs.berkeley.edu>
   4                          2018 Stef O'Rear <sorear2@gmail.com>
   5    Based on MIPS N32/64 port
   6
   7    RISC-V Foreign Function Interface
   8
   9    Permission is hereby granted, free of charge, to any person obtaining
  10    a copy of this software and associated documentation files (the
  11    ``Software''), to deal in the Software without restriction, including
  12    without limitation the rights to use, copy, modify, merge, publish,
  13    distribute, sublicense, and/or sell copies of the Software, and to
  14    permit persons to whom the Software is furnished to do so, subject to
  15    the following conditions:
  16
  17    The above copyright notice and this permission notice shall be included
  18    in all copies or substantial portions of the Software.
  19
  20    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  21    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23    NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  24    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  25    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  26    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  27    DEALINGS IN THE SOFTWARE.
  28    ----------------------------------------------------------------------- */
  29
  30 #include <ffi.h>
  31 #include <ffi_common.h>
  32
  33 #include <stdlib.h>
  34 #include <stdint.h>
  35
  36 #if __riscv_float_abi_double
  37 #define ABI_FLEN 64
  38 #define ABI_FLOAT double
  39 #elif __riscv_float_abi_single
  40 #define ABI_FLEN 32
  41 #define ABI_FLOAT float
  42 #endif
  43
  44 #define NARGREG 8
  45 #define STKALIGN 16
  46 #define MAXCOPYARG (2 * sizeof(double))
  47
  48 typedef struct call_context
  49 {
  50 #if ABI_FLEN
  51     ABI_FLOAT fa[8];
  52 #endif
  53     size_t a[8];
  54     /* used by the assembly code to in-place construct its own stack frame */
  55     char frame[16];
  56 } call_context;
  57
  58 typedef struct call_builder
  59 {
  60     call_context *aregs;
  61     int used_integer;
  62     int used_float;
  63     size_t *used_stack;
  64 } call_builder;
  65
  66 /* integer (not pointer) less than ABI XLEN */
  67 /* FFI_TYPE_INT does not appear to be used */
  68 #if __SIZEOF_POINTER__ == 8
  69 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT64)
  70 #else
  71 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT32)
  72 #endif
  73
  74 #if ABI_FLEN
  75 typedef struct {
  76     char as_elements, type1, offset2, type2;
  77 } float_struct_info;
  78
  79 #if ABI_FLEN >= 64
  80 #define IS_FLOAT(type) ((type) >= FFI_TYPE_FLOAT && (type) <= FFI_TYPE_DOUBLE)
  81 #else
  82 #define IS_FLOAT(type) ((type) == FFI_TYPE_FLOAT)
  83 #endif
  84
  85 static ffi_type **flatten_struct(ffi_type *in, ffi_type **out, ffi_type **out_end) {
  86     int i;
  87     if (out == out_end) return out;
  88     if (in->type != FFI_TYPE_STRUCT) {
  89         *(out++) = in;
  90     } else {
  91         for (i = 0; in->elements[i]; i++)
  92             out = flatten_struct(in->elements[i], out, out_end);
  93     }
  94     return out;
  95 }
  96
  97 /* Structs with at most two fields after flattening, one of which is of
  98    floating point type, are passed in multiple registers if sufficient
  99    registers are available. */
 100 static float_struct_info struct_passed_as_elements(call_builder *cb, ffi_type *top) {
 101     float_struct_info ret = {0, 0, 0, 0};
 102     ffi_type *fields[3];
 103     int num_floats, num_ints;
 104     int num_fields = flatten_struct(top, fields, fields + 3) - fields;
 105
 106     if (num_fields == 1) {
 107         if (IS_FLOAT(fields[0]->type)) {
 108             ret.as_elements = 1;
 109             ret.type1 = fields[0]->type;
 110         }
 111     } else if (num_fields == 2) {
 112         num_floats = IS_FLOAT(fields[0]->type) + IS_FLOAT(fields[1]->type);
 113         num_ints = IS_INT(fields[0]->type) + IS_INT(fields[1]->type);
 114         if (num_floats == 0 || num_floats + num_ints != 2)
 115             return ret;
 116         if (cb->used_float + num_floats > NARGREG || cb->used_integer + (2 - num_floats) > NARGREG)
 117             return ret;
 118         if (!IS_FLOAT(fields[0]->type) && !IS_FLOAT(fields[1]->type))
 119             return ret;
 120
 121         ret.type1 = fields[0]->type;
 122         ret.type2 = fields[1]->type;
 123         ret.offset2 = ALIGN(fields[0]->size, fields[1]->alignment);
 124         ret.as_elements = 1;
 125     }
 126
 127     return ret;
 128 }
 129 #endif
 130
 131 /* allocates a single register, float register, or XLEN-sized stack slot to a datum */
 132 static void marshal_atom(call_builder *cb, int type, void *data) {
 133     size_t value = 0;
 134     switch (type) {
 135         case FFI_TYPE_UINT8: value = *(uint8_t *)data; break;
 136         case FFI_TYPE_SINT8: value = *(int8_t *)data; break;
 137         case FFI_TYPE_UINT16: value = *(uint16_t *)data; break;
 138         case FFI_TYPE_SINT16: value = *(int16_t *)data; break;
 139         /* 32-bit quantities are always sign-extended in the ABI */
 140         case FFI_TYPE_UINT32: value = *(int32_t *)data; break;
 141         case FFI_TYPE_SINT32: value = *(int32_t *)data; break;
 142 #if __SIZEOF_POINTER__ == 8
 143         case FFI_TYPE_UINT64: value = *(uint64_t *)data; break;
 144         case FFI_TYPE_SINT64: value = *(int64_t *)data; break;
 145 #endif
 146         case FFI_TYPE_POINTER: value = *(size_t *)data; break;
 147
 148         /* float values may be recoded in an implementation-defined way
 149            by hardware conforming to 2.1 or earlier, so use asm to
 150            reinterpret floats as doubles */
 151 #if ABI_FLEN >= 32
 152         case FFI_TYPE_FLOAT:
 153             asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(float *)data));
 154             return;
 155 #endif
 156 #if ABI_FLEN >= 64
 157         case FFI_TYPE_DOUBLE:
 158             asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(double *)data));
 159             return;
 160 #endif
 161         default: FFI_ASSERT(0); break;
 162     }
 163
 164     if (cb->used_integer == NARGREG) {
 165         *cb->used_stack++ = value;
 166     } else {
 167         cb->aregs->a[cb->used_integer++] = value;
 168     }
 169 }
 170
 171 static void unmarshal_atom(call_builder *cb, int type, void *data) {
 172     size_t value;
 173     switch (type) {
 174 #if ABI_FLEN >= 32
 175         case FFI_TYPE_FLOAT:
 176             asm("" : "=f"(*(float *)data) : "0"(cb->aregs->fa[cb->used_float++]));
 177             return;
 178 #endif
 179 #if ABI_FLEN >= 64
 180         case FFI_TYPE_DOUBLE:
 181             asm("" : "=f"(*(double *)data) : "0"(cb->aregs->fa[cb->used_float++]));
 182             return;
 183 #endif
 184     }
 185
 186     if (cb->used_integer == NARGREG) {
 187         value = *cb->used_stack++;
 188     } else {
 189         value = cb->aregs->a[cb->used_integer++];
 190     }
 191
 192     switch (type) {
 193         case FFI_TYPE_UINT8: *(uint8_t *)data = value; break;
 194         case FFI_TYPE_SINT8: *(uint8_t *)data = value; break;
 195         case FFI_TYPE_UINT16: *(uint16_t *)data = value; break;
 196         case FFI_TYPE_SINT16: *(uint16_t *)data = value; break;
 197         case FFI_TYPE_UINT32: *(uint32_t *)data = value; break;
 198         case FFI_TYPE_SINT32: *(uint32_t *)data = value; break;
 199 #if __SIZEOF_POINTER__ == 8
 200         case FFI_TYPE_UINT64: *(uint64_t *)data = value; break;
 201         case FFI_TYPE_SINT64: *(uint64_t *)data = value; break;
 202 #endif
 203         case FFI_TYPE_POINTER: *(size_t *)data = value; break;
 204         default: FFI_ASSERT(0); break;
 205     }
 206 }
 207
 208 /* adds an argument to a call, or a not by reference return value */
 209 static void marshal(call_builder *cb, ffi_type *type, int var, void *data) {
 210     size_t realign[2];
 211
 212 #if ABI_FLEN
 213     if (!var && type->type == FFI_TYPE_STRUCT) {
 214         float_struct_info fsi = struct_passed_as_elements(cb, type);
 215         if (fsi.as_elements) {
 216             marshal_atom(cb, fsi.type1, data);
 217             if (fsi.offset2)
 218                 marshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
 219             return;
 220         }
 221     }
 222
 223     if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
 224         marshal_atom(cb, type->type, data);
 225         return;
 226     }
 227 #endif
 228
 229     if (type->size > 2 * __SIZEOF_POINTER__) {
 230         /* pass by reference */
 231         marshal_atom(cb, FFI_TYPE_POINTER, &data);
 232     } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
 233         marshal_atom(cb, type->type, data);
 234     } else {
 235         /* overlong integers, soft-float floats, and structs without special
 236            float handling are treated identically from this point on */
 237
 238         /* variadics are aligned even in registers */
 239         if (type->alignment > __SIZEOF_POINTER__) {
 240             if (var)
 241                 cb->used_integer = ALIGN(cb->used_integer, 2);
 242             cb->used_stack = (size_t *)ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
 243         }
 244
 245         memcpy(realign, data, type->size);
 246         if (type->size > 0)
 247             marshal_atom(cb, FFI_TYPE_POINTER, realign);
 248         if (type->size > __SIZEOF_POINTER__)
 249             marshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
 250     }
 251 }
 252
 253 /* for arguments passed by reference returns the pointer, otherwise the arg is copied (up to MAXCOPYARG bytes) */
 254 static void *unmarshal(call_builder *cb, ffi_type *type, int var, void *data) {
 255     size_t realign[2];
 256     void *pointer;
 257
 258 #if ABI_FLEN
 259     if (!var && type->type == FFI_TYPE_STRUCT) {
 260         float_struct_info fsi = struct_passed_as_elements(cb, type);
 261         if (fsi.as_elements) {
 262             unmarshal_atom(cb, fsi.type1, data);
 263             if (fsi.offset2)
 264                 unmarshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
 265             return data;
 266         }
 267     }
 268
 269     if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
 270         unmarshal_atom(cb, type->type, data);
 271         return data;
 272     }
 273 #endif
 274
 275     if (type->size > 2 * __SIZEOF_POINTER__) {
 276         /* pass by reference */
 277         unmarshal_atom(cb, FFI_TYPE_POINTER, (char*)&pointer);
 278         return pointer;
 279     } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
 280         unmarshal_atom(cb, type->type, data);
 281         return data;
 282     } else {
 283         /* overlong integers, soft-float floats, and structs without special
 284            float handling are treated identically from this point on */
 285
 286         /* variadics are aligned even in registers */
 287         if (type->alignment > __SIZEOF_POINTER__) {
 288             if (var)
 289                 cb->used_integer = ALIGN(cb->used_integer, 2);
 290             cb->used_stack = (size_t *)ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
 291         }
 292
 293         if (type->size > 0)
 294             unmarshal_atom(cb, FFI_TYPE_POINTER, realign);
 295         if (type->size > __SIZEOF_POINTER__)
 296             unmarshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
 297         memcpy(data, realign, type->size);
 298         return data;
 299     }
 300 }
 301
 302 static int passed_by_ref(call_builder *cb, ffi_type *type, int var) {
 303 #if ABI_FLEN
 304     if (!var && type->type == FFI_TYPE_STRUCT) {
 305         float_struct_info fsi = struct_passed_as_elements(cb, type);
 306         if (fsi.as_elements) return 0;
 307     }
 308 #endif
 309
 310     return type->size > 2 * __SIZEOF_POINTER__;
 311 }
 312
 313 /* Perform machine dependent cif processing */
 314 ffi_status ffi_prep_cif_machdep(ffi_cif *cif) {
 315     cif->riscv_nfixedargs = cif->nargs;
 316     return FFI_OK;
 317 }
 318
 319 /* Perform machine dependent cif processing when we have a variadic function */
 320
 321 ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs) {
 322     cif->riscv_nfixedargs = nfixedargs;
 323     return FFI_OK;
 324 }
 325
 326 /* Low level routine for calling functions */
 327 extern void ffi_call_asm (void *stack, struct call_context *regs,
 328                           void (*fn) (void), void *closure) FFI_HIDDEN;
 329
 330 static void
 331 ffi_call_int (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue,
 332               void *closure)
 333 {
 334     /* this is a conservative estimate, assuming a complex return value and
 335        that all remaining arguments are long long / __int128 */
 336     size_t arg_bytes = cif->nargs <= 3 ? 0 :
 337         ALIGN(2 * sizeof(size_t) * (cif->nargs - 3), STKALIGN);
 338     size_t rval_bytes = 0;
 339     if (rvalue == NULL && cif->rtype->size > 2*__SIZEOF_POINTER__)
 340         rval_bytes = ALIGN(cif->rtype->size, STKALIGN);
 341     size_t alloc_size = arg_bytes + rval_bytes + sizeof(call_context);
 342
 343     /* the assembly code will deallocate all stack data at lower addresses
 344        than the argument region, so we need to allocate the frame and the
 345        return value after the arguments in a single allocation */
 346     size_t alloc_base;
 347     /* Argument region must be 16-byte aligned */
 348     if (_Alignof(max_align_t) >= STKALIGN) {
 349         /* since sizeof long double is normally 16, the compiler will
 350            guarantee alloca alignment to at least that much */
 351         alloc_base = (size_t)alloca(alloc_size);
 352     } else {
 353         alloc_base = ALIGN(alloca(alloc_size + STKALIGN - 1), STKALIGN);
 354     }
 355
 356     if (rval_bytes)
 357         rvalue = (void*)(alloc_base + arg_bytes);
 358
 359     call_builder cb;
 360     cb.used_float = cb.used_integer = 0;
 361     cb.aregs = (call_context*)(alloc_base + arg_bytes + rval_bytes);
 362     cb.used_stack = (void*)alloc_base;
 363
 364     int return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
 365     if (return_by_ref)
 366         marshal(&cb, &ffi_type_pointer, 0, &rvalue);
 367
 368     int i;
 369     for (i = 0; i < cif->nargs; i++)
 370         marshal(&cb, cif->arg_types[i], i >= cif->riscv_nfixedargs, avalue[i]);
 371
 372     ffi_call_asm ((void *) alloc_base, cb.aregs, fn, closure);
 373
 374     cb.used_float = cb.used_integer = 0;
 375     if (!return_by_ref && rvalue)
 376         unmarshal(&cb, cif->rtype, 0, rvalue);
 377 }
 378
 379 void
 380 ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
 381 {
 382   ffi_call_int(cif, fn, rvalue, avalue, NULL);
 383 }
 384
 385 void
 386 ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
 387              void **avalue, void *closure)
 388 {
 389   ffi_call_int(cif, fn, rvalue, avalue, closure);
 390 }
 391
 392 extern void ffi_closure_asm(void) FFI_HIDDEN;
 393
 394 ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void (*fun)(ffi_cif*,void*,void**,void*), void *user_data, void *codeloc)
 395 {
 396     uint32_t *tramp = (uint32_t *) &closure->tramp[0];
 397     uint64_t fn = (uint64_t) (uintptr_t) ffi_closure_asm;
 398
 399     if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
 400         return FFI_BAD_ABI;
 401
 402     /* we will call ffi_closure_inner with codeloc, not closure, but as long
 403        as the memory is readable it should work */
 404
 405     tramp[0] = 0x00000317; /* auipc t1, 0 (i.e. t0 <- codeloc) */
 406 #if __SIZEOF_POINTER__ == 8
 407     tramp[1] = 0x01033383; /* ld t2, 16(t1) */
 408 #else
 409     tramp[1] = 0x01032383; /* lw t2, 16(t1) */
 410 #endif
 411     tramp[2] = 0x00038067; /* jr t2 */
 412     tramp[3] = 0x00000013; /* nop */
 413     tramp[4] = fn;
 414     tramp[5] = fn >> 32;
 415
 416     closure->cif = cif;
 417     closure->fun = fun;
 418     closure->user_data = user_data;
 419
 420     __builtin___clear_cache(codeloc, codeloc + FFI_TRAMPOLINE_SIZE);
 421
 422     return FFI_OK;
 423 }
 424
 425 extern void ffi_go_closure_asm (void) FFI_HIDDEN;
 426
 427 ffi_status
 428 ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif,
 429                      void (*fun) (ffi_cif *, void *, void **, void *))
 430 {
 431   if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
 432     return FFI_BAD_ABI;
 433
 434   closure->tramp = (void *) ffi_go_closure_asm;
 435   closure->cif = cif;
 436   closure->fun = fun;
 437
 438   return FFI_OK;
 439 }
 440
 441 /* Called by the assembly code with aregs pointing to saved argument registers
 442    and stack pointing to the stacked arguments.  Return values passed in
 443    registers will be reloaded from aregs. */
 444 void FFI_HIDDEN
 445 ffi_closure_inner (ffi_cif *cif,
 446                    void (*fun) (ffi_cif *, void *, void **, void *),
 447                    void *user_data,
 448                    size_t *stack, call_context *aregs)
 449 {
 450     void **avalue = alloca(cif->nargs * sizeof(void*));
 451     /* storage for arguments which will be copied by unmarshal().  We could
 452        theoretically avoid the copies in many cases and use at most 128 bytes
 453        of memory, but allocating disjoint storage for each argument is
 454        simpler. */
 455     char *astorage = alloca(cif->nargs * MAXCOPYARG);
 456     void *rvalue;
 457     call_builder cb;
 458     int return_by_ref;
 459     int i;
 460
 461     cb.aregs = aregs;
 462     cb.used_integer = cb.used_float = 0;
 463     cb.used_stack = stack;
 464
 465     return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
 466     if (return_by_ref)
 467         unmarshal(&cb, &ffi_type_pointer, 0, &rvalue);
 468     else
 469         rvalue = alloca(cif->rtype->size);
 470
 471     for (i = 0; i < cif->nargs; i++)
 472         avalue[i] = unmarshal(&cb, cif->arg_types[i],
 473             i >= cif->riscv_nfixedargs, astorage + i*MAXCOPYARG);
 474
 475     fun (cif, rvalue, avalue, user_data);
 476
 477     if (!return_by_ref && cif->rtype->type != FFI_TYPE_VOID) {
 478         cb.used_integer = cb.used_float = 0;
 479         marshal(&cb, cif->rtype, 0, rvalue);
 480     }
 481 }