libffi/src/riscv/ffi.c

   1 /* -----------------------------------------------------------------------
   2    ffi.c - Copyright (c) 2015 Michael Knyszek <mknyszek@berkeley.edu>
   3                          2015 Andrew Waterman <waterman@cs.berkeley.edu>
   4                          2018 Stef O'Rear <sorear2@gmail.com>
   5    Based on MIPS N32/64 port
   6
   7    RISC-V Foreign Function Interface
   8
   9    Permission is hereby granted, free of charge, to any person obtaining
  10    a copy of this software and associated documentation files (the
  11    ``Software''), to deal in the Software without restriction, including
  12    without limitation the rights to use, copy, modify, merge, publish,
  13    distribute, sublicense, and/or sell copies of the Software, and to
  14    permit persons to whom the Software is furnished to do so, subject to
  15    the following conditions:
  16
  17    The above copyright notice and this permission notice shall be included
  18    in all copies or substantial portions of the Software.
  19
  20    THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  21    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  22    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  23    NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
  24    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  25    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  26    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  27    DEALINGS IN THE SOFTWARE.
  28    ----------------------------------------------------------------------- */
  29
  30 #include <ffi.h>
  31 #include <ffi_common.h>
  32
  33 #include <stdlib.h>
  34 #include <stdint.h>
  35
  36 #if __riscv_float_abi_double
  37 #define ABI_FLEN 64
  38 #define ABI_FLOAT double
  39 #elif __riscv_float_abi_single
  40 #define ABI_FLEN 32
  41 #define ABI_FLOAT float
  42 #endif
  43
  44 #define NARGREG 8
  45 #define STKALIGN 16
  46 #define MAXCOPYARG (2 * sizeof(double))
  47
  48 typedef struct call_context
  49 {
  50 #if ABI_FLEN
  51     ABI_FLOAT fa[8];
  52 #endif
  53     size_t a[8];
  54     /* used by the assembly code to in-place construct its own stack frame */
  55     char frame[16];
  56 } call_context;
  57
  58 typedef struct call_builder
  59 {
  60     call_context *aregs;
  61     int used_integer;
  62     int used_float;
  63     size_t *used_stack;
  64 } call_builder;
  65
  66 /* integer (not pointer) less than ABI XLEN */
  67 /* FFI_TYPE_INT does not appear to be used */
  68 #if __SIZEOF_POINTER__ == 8
  69 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT64)
  70 #else
  71 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT32)
  72 #endif
  73
  74 #if ABI_FLEN
  75 typedef struct {
  76     char as_elements, type1, offset2, type2;
  77 } float_struct_info;
  78
  79 #if ABI_FLEN >= 64
  80 #define IS_FLOAT(type) ((type) >= FFI_TYPE_FLOAT && (type) <= FFI_TYPE_DOUBLE)
  81 #else
  82 #define IS_FLOAT(type) ((type) == FFI_TYPE_FLOAT)
  83 #endif
  84
  85 static ffi_type **flatten_struct(ffi_type *in, ffi_type **out, ffi_type **out_end) {
  86     int i;
  87     if (out == out_end) return out;
  88     if (in->type != FFI_TYPE_STRUCT) {
  89         *(out++) = in;
  90     } else {
  91         for (i = 0; in->elements[i]; i++)
  92             out = flatten_struct(in->elements[i], out, out_end);
  93     }
  94     return out;
  95 }
  96
  97 /* Structs with at most two fields after flattening, one of which is of
  98    floating point type, are passed in multiple registers if sufficient
  99    registers are available. */
 100 static float_struct_info struct_passed_as_elements(call_builder *cb, ffi_type *top) {
 101     float_struct_info ret = {0, 0, 0, 0};
 102     ffi_type *fields[3];
 103     int num_floats, num_ints;
 104     int num_fields = flatten_struct(top, fields, fields + 3) - fields;
 105
 106     if (num_fields == 1) {
 107         if (IS_FLOAT(fields[0]->type)) {
 108             ret.as_elements = 1;
 109             ret.type1 = fields[0]->type;
 110         }
 111     } else if (num_fields == 2) {
 112         num_floats = IS_FLOAT(fields[0]->type) + IS_FLOAT(fields[1]->type);
 113         num_ints = IS_INT(fields[0]->type) + IS_INT(fields[1]->type);
 114         if (num_floats == 0 || num_floats + num_ints != 2)
 115             return ret;
 116         if (cb->used_float + num_floats > NARGREG || cb->used_integer + (2 - num_floats) > NARGREG)
 117             return ret;
 118         if (!IS_FLOAT(fields[0]->type) && !IS_FLOAT(fields[1]->type))
 119             return ret;
 120
 121         ret.type1 = fields[0]->type;
 122         ret.type2 = fields[1]->type;
 123         ret.offset2 = ALIGN(fields[0]->size, fields[1]->alignment);
 124         ret.as_elements = 1;
 125     }
 126
 127     return ret;
 128 }
 129 #endif
 130
 131 /* allocates a single register, float register, or XLEN-sized stack slot to a datum */
 132 static void marshal_atom(call_builder *cb, int type, void *data) {
 133     size_t value = 0;
 134     switch (type) {
 135         case FFI_TYPE_UINT8: value = *(uint8_t *)data; break;
 136         case FFI_TYPE_SINT8: value = *(int8_t *)data; break;
 137         case FFI_TYPE_UINT16: value = *(uint16_t *)data; break;
 138         case FFI_TYPE_SINT16: value = *(int16_t *)data; break;
 139         /* 32-bit quantities are always sign-extended in the ABI */
 140         case FFI_TYPE_UINT32: value = *(int32_t *)data; break;
 141         case FFI_TYPE_SINT32: value = *(int32_t *)data; break;
 142 #if __SIZEOF_POINTER__ == 8
 143         case FFI_TYPE_UINT64: value = *(uint64_t *)data; break;
 144         case FFI_TYPE_SINT64: value = *(int64_t *)data; break;
 145 #endif
 146         case FFI_TYPE_POINTER: value = *(size_t *)data; break;
 147
 148         /* float values may be recoded in an implementation-defined way
 149            by hardware conforming to 2.1 or earlier, so use asm to
 150            reinterpret floats as doubles */
 151 #if ABI_FLEN >= 32
 152         case FFI_TYPE_FLOAT:
 153             asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(float *)data));
 154             return;
 155 #endif
 156 #if ABI_FLEN >= 64
 157         case FFI_TYPE_DOUBLE:
 158             asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(double *)data));
 159             return;
 160 #endif
 161         default: FFI_ASSERT(0); break;
 162     }
 163
 164     if (cb->used_integer == NARGREG) {
 165         *cb->used_stack++ = value;
 166     } else {
 167         cb->aregs->a[cb->used_integer++] = value;
 168     }
 169 }
 170
 171 static void unmarshal_atom(call_builder *cb, int type, void *data) {
 172     size_t value;
 173     switch (type) {
 174 #if ABI_FLEN >= 32
 175         case FFI_TYPE_FLOAT:
 176             asm("" : "=f"(*(float *)data) : "0"(cb->aregs->fa[cb->used_float++]));
 177             return;
 178 #endif
 179 #if ABI_FLEN >= 64
 180         case FFI_TYPE_DOUBLE:
 181             asm("" : "=f"(*(double *)data) : "0"(cb->aregs->fa[cb->used_float++]));
 182             return;
 183 #endif
 184     }
 185
 186     if (cb->used_integer == NARGREG) {
 187         value = *cb->used_stack++;
 188     } else {
 189         value = cb->aregs->a[cb->used_integer++];
 190     }
 191
 192     switch (type) {
 193         case FFI_TYPE_UINT8: *(uint8_t *)data = value; break;
 194         case FFI_TYPE_SINT8: *(uint8_t *)data = value; break;
 195         case FFI_TYPE_UINT16: *(uint16_t *)data = value; break;
 196         case FFI_TYPE_SINT16: *(uint16_t *)data = value; break;
 197         case FFI_TYPE_UINT32: *(uint32_t *)data = value; break;
 198         case FFI_TYPE_SINT32: *(uint32_t *)data = value; break;
 199 #if __SIZEOF_POINTER__ == 8
 200         case FFI_TYPE_UINT64: *(uint64_t *)data = value; break;
 201         case FFI_TYPE_SINT64: *(uint64_t *)data = value; break;
 202 #endif
 203         case FFI_TYPE_POINTER: *(size_t *)data = value; break;
 204         default: FFI_ASSERT(0); break;
 205     }
 206 }
 207
 208 /* adds an argument to a call, or a not by reference return value */
 209 static void marshal(call_builder *cb, ffi_type *type, int var, void *data) {
 210     size_t realign[2];
 211
 212 #if ABI_FLEN
 213     if (!var && type->type == FFI_TYPE_STRUCT) {
 214         float_struct_info fsi = struct_passed_as_elements(cb, type);
 215         if (fsi.as_elements) {
 216             marshal_atom(cb, fsi.type1, data);
 217             if (fsi.offset2)
 218                 marshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
 219             return;
 220         }
 221     }
 222
 223     if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
 224         marshal_atom(cb, type->type, data);
 225         return;
 226     }
 227 #endif
 228
 229     if (type->size > 2 * __SIZEOF_POINTER__) {
 230         /* pass by reference */
 231         marshal_atom(cb, FFI_TYPE_POINTER, &data);
 232     } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
 233         marshal_atom(cb, type->type, data);
 234     } else {
 235         /* overlong integers, soft-float floats, and structs without special
 236            float handling are treated identically from this point on */
 237
 238         /* variadics are aligned even in registers */
 239         if (type->alignment > __SIZEOF_POINTER__) {
 240             if (var)
 241                 cb->used_integer = ALIGN(cb->used_integer, 2);
 242             cb->used_stack = (size_t *)ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
 243         }
 244
 245         memcpy(realign, data, type->size);
 246         if (type->size > 0)
 247             marshal_atom(cb, FFI_TYPE_POINTER, realign);
 248         if (type->size > __SIZEOF_POINTER__)
 249             marshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
 250     }
 251 }
 252
 253 /* for arguments passed by reference returns the pointer, otherwise the arg is copied (up to MAXCOPYARG bytes) */
 254 static void *unmarshal(call_builder *cb, ffi_type *type, int var, void *data) {
 255     size_t realign[2];
 256     void *pointer;
 257
 258 #if ABI_FLEN
 259     if (!var && type->type == FFI_TYPE_STRUCT) {
 260         float_struct_info fsi = struct_passed_as_elements(cb, type);
 261         if (fsi.as_elements) {
 262             unmarshal_atom(cb, fsi.type1, data);
 263             if (fsi.offset2)
 264                 unmarshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
 265             return data;
 266         }
 267     }
 268
 269     if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
 270         unmarshal_atom(cb, type->type, data);
 271         return data;
 272     }
 273 #endif
 274
 275     if (type->size > 2 * __SIZEOF_POINTER__) {
 276         /* pass by reference */
 277         unmarshal_atom(cb, FFI_TYPE_POINTER, (char*)&pointer);
 278         return pointer;
 279     } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
 280         unmarshal_atom(cb, type->type, data);
 281         return data;
 282     } else {
 283         /* overlong integers, soft-float floats, and structs without special
 284            float handling are treated identically from this point on */
 285
 286         /* variadics are aligned even in registers */
 287         if (type->alignment > __SIZEOF_POINTER__) {
 288             if (var)
 289                 cb->used_integer = ALIGN(cb->used_integer, 2);
 290             cb->used_stack = (size_t *)ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
 291         }
 292
 293         if (type->size > 0)
 294             unmarshal_atom(cb, FFI_TYPE_POINTER, realign);
 295         if (type->size > __SIZEOF_POINTER__)
 296             unmarshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
 297         memcpy(data, realign, type->size);
 298         return data;
 299     }
 300 }
 301
 302 static int passed_by_ref(call_builder *cb, ffi_type *type, int var) {
 303 #if ABI_FLEN
 304     if (!var && type->type == FFI_TYPE_STRUCT) {
 305         float_struct_info fsi = struct_passed_as_elements(cb, type);
 306         if (fsi.as_elements) return 0;
 307     }
 308 #endif
 309
 310     return type->size > 2 * __SIZEOF_POINTER__;
 311 }
 312
 313 /* Perform machine dependent cif processing */
 314 ffi_status ffi_prep_cif_machdep(ffi_cif *cif) {
 315     cif->riscv_nfixedargs = cif->nargs;
 316     return FFI_OK;
 317 }
 318
 319 /* Perform machine dependent cif processing when we have a variadic function */
 320
 321 ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs) {
 322     cif->riscv_nfixedargs = nfixedargs;
 323     return FFI_OK;
 324 }
 325
 326 /* Low level routine for calling functions */
 327 extern void ffi_call_asm(void *stack, struct call_context *regs, void (*fn)(void)) FFI_HIDDEN;
 328
 329 void ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
 330 {
 331     /* this is a conservative estimate, assuming a complex return value and
 332        that all remaining arguments are long long / __int128 */
 333     size_t arg_bytes = cif->nargs <= 3 ? 0 :
 334         ALIGN(2 * sizeof(size_t) * (cif->nargs - 3), STKALIGN);
 335     size_t rval_bytes = 0;
 336     if (rvalue == NULL && cif->rtype->size > 2*__SIZEOF_POINTER__)
 337         rval_bytes = ALIGN(cif->rtype->size, STKALIGN);
 338     size_t alloc_size = arg_bytes + rval_bytes + sizeof(call_context);
 339
 340     /* the assembly code will deallocate all stack data at lower addresses
 341        than the argument region, so we need to allocate the frame and the
 342        return value after the arguments in a single allocation */
 343     size_t alloc_base;
 344     /* Argument region must be 16-byte aligned */
 345     if (_Alignof(max_align_t) >= STKALIGN) {
 346         /* since sizeof long double is normally 16, the compiler will
 347            guarantee alloca alignment to at least that much */
 348         alloc_base = (size_t)alloca(alloc_size);
 349     } else {
 350         alloc_base = ALIGN(alloca(alloc_size + STKALIGN - 1), STKALIGN);
 351     }
 352
 353     if (rval_bytes)
 354         rvalue = (void*)(alloc_base + arg_bytes);
 355
 356     call_builder cb;
 357     cb.used_float = cb.used_integer = 0;
 358     cb.aregs = (call_context*)(alloc_base + arg_bytes + rval_bytes);
 359     cb.used_stack = (void*)alloc_base;
 360
 361     int return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
 362     if (return_by_ref)
 363         marshal(&cb, &ffi_type_pointer, 0, &rvalue);
 364
 365     int i;
 366     for (i = 0; i < cif->nargs; i++)
 367         marshal(&cb, cif->arg_types[i], i >= cif->riscv_nfixedargs, avalue[i]);
 368
 369     ffi_call_asm((void*)alloc_base, cb.aregs, fn);
 370
 371     cb.used_float = cb.used_integer = 0;
 372     if (!return_by_ref && rvalue)
 373         unmarshal(&cb, cif->rtype, 0, rvalue);
 374 }
 375
 376 extern void ffi_closure_asm(void) FFI_HIDDEN;
 377
 378 ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void (*fun)(ffi_cif*,void*,void**,void*), void *user_data, void *codeloc)
 379 {
 380     uint32_t *tramp = (uint32_t *) &closure->tramp[0];
 381     uint64_t fn = (uint64_t) (uintptr_t) ffi_closure_asm;
 382
 383     if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
 384         return FFI_BAD_ABI;
 385
 386     /* we will call ffi_closure_inner with codeloc, not closure, but as long
 387        as the memory is readable it should work */
 388
 389     tramp[0] = 0x00000317; /* auipc t1, 0 (i.e. t0 <- codeloc) */
 390 #if __SIZEOF_POINTER__ == 8
 391     tramp[1] = 0x01033383; /* ld t2, 16(t1) */
 392 #else
 393     tramp[1] = 0x01032383; /* lw t2, 16(t1) */
 394 #endif
 395     tramp[2] = 0x00038067; /* jr t2 */
 396     tramp[3] = 0x00000013; /* nop */
 397     tramp[4] = fn;
 398     tramp[5] = fn >> 32;
 399
 400     closure->cif = cif;
 401     closure->fun = fun;
 402     closure->user_data = user_data;
 403
 404     __builtin___clear_cache(codeloc, codeloc + FFI_TRAMPOLINE_SIZE);
 405
 406     return FFI_OK;
 407 }
 408
 409 /* Called by the assembly code with aregs pointing to saved argument registers
 410    and stack pointing to the stacked arguments.  Return values passed in
 411    registers will be reloaded from aregs. */
 412 void FFI_HIDDEN ffi_closure_inner(size_t *stack, call_context *aregs, ffi_closure *closure) {
 413     ffi_cif *cif = closure->cif;
 414     void **avalue = alloca(cif->nargs * sizeof(void*));
 415     /* storage for arguments which will be copied by unmarshal().  We could
 416        theoretically avoid the copies in many cases and use at most 128 bytes
 417        of memory, but allocating disjoint storage for each argument is
 418        simpler. */
 419     char *astorage = alloca(cif->nargs * MAXCOPYARG);
 420     void *rvalue;
 421     call_builder cb;
 422     int return_by_ref;
 423     int i;
 424
 425     cb.aregs = aregs;
 426     cb.used_integer = cb.used_float = 0;
 427     cb.used_stack = stack;
 428
 429     return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
 430     if (return_by_ref)
 431         unmarshal(&cb, &ffi_type_pointer, 0, &rvalue);
 432     else
 433         rvalue = alloca(cif->rtype->size);
 434
 435     for (i = 0; i < cif->nargs; i++)
 436         avalue[i] = unmarshal(&cb, cif->arg_types[i],
 437             i >= cif->riscv_nfixedargs, astorage + i*MAXCOPYARG);
 438
 439     (closure->fun)(cif, rvalue, avalue, closure->user_data);
 440
 441     if (!return_by_ref && cif->rtype->type != FFI_TYPE_VOID) {
 442         cb.used_integer = cb.used_float = 0;
 443         marshal(&cb, cif->rtype, 0, rvalue);
 444     }
 445 }