1 /* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.
3 Permission is hereby granted, free of charge, to any person obtaining
4 a copy of this software and associated documentation files (the
5 ``Software''), to deal in the Software without restriction, including
6 without limitation the rights to use, copy, modify, merge, publish,
7 distribute, sublicense, and/or sell copies of the Software, and to
8 permit persons to whom the Software is furnished to do so, subject to
9 the following conditions:
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
14 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
15 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
17 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
18 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
19 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
26 #include <ffi_common.h>
29 /* Force FFI_TYPE_LONGDOUBLE to be different than FFI_TYPE_DOUBLE;
30 all further uses in this file will refer to the 128-bit type. */
31 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
32 # if FFI_TYPE_LONGDOUBLE != 4
33 # error FFI_TYPE_LONGDOUBLE out of date
36 # undef FFI_TYPE_LONGDOUBLE
37 # define FFI_TYPE_LONGDOUBLE 4
48 union _d d
[2] __attribute__((aligned(16)));
53 struct _v v
[N_V_ARG_REG
];
54 UINT64 x
[N_X_ARG_REG
];
57 #if defined (__clang__) && defined (__APPLE__)
58 extern void sys_icache_invalidate (void *start
, size_t len
);
62 ffi_clear_cache (void *start
, void *end
)
64 #if defined (__clang__) && defined (__APPLE__)
65 sys_icache_invalidate (start
, (char *)end
- (char *)start
);
66 #elif defined (__GNUC__)
67 __builtin___clear_cache (start
, end
);
69 #error "Missing builtin to flush instruction cache"
73 /* A subroutine of is_vfp_type. Given a structure type, return the type code
74 of the first non-structure element. Recurse for structure elements.
75 Return -1 if the structure is in fact empty, i.e. no nested elements. */
78 is_hfa0 (const ffi_type
*ty
)
80 ffi_type
**elements
= ty
->elements
;
84 for (i
= 0; elements
[i
]; ++i
)
86 ret
= elements
[i
]->type
;
87 if (ret
== FFI_TYPE_STRUCT
|| ret
== FFI_TYPE_COMPLEX
)
89 ret
= is_hfa0 (elements
[i
]);
99 /* A subroutine of is_vfp_type. Given a structure type, return true if all
100 of the non-structure elements are the same as CANDIDATE. */
103 is_hfa1 (const ffi_type
*ty
, int candidate
)
105 ffi_type
**elements
= ty
->elements
;
108 if (elements
!= NULL
)
109 for (i
= 0; elements
[i
]; ++i
)
111 int t
= elements
[i
]->type
;
112 if (t
== FFI_TYPE_STRUCT
|| t
== FFI_TYPE_COMPLEX
)
114 if (!is_hfa1 (elements
[i
], candidate
))
117 else if (t
!= candidate
)
124 /* Determine if TY may be allocated to the FP registers. This is both an
125 fp scalar type as well as an homogenous floating point aggregate (HFA).
126 That is, a structure consisting of 1 to 4 members of all the same type,
127 where that type is an fp scalar.
129 Returns non-zero iff TY is an HFA. The result is the AARCH64_RET_*
130 constant for the type. */
133 is_vfp_type (const ffi_type
*ty
)
137 size_t size
, ele_count
;
139 /* Quickest tests first. */
140 candidate
= ty
->type
;
146 case FFI_TYPE_DOUBLE
:
147 case FFI_TYPE_LONGDOUBLE
:
150 case FFI_TYPE_COMPLEX
:
151 candidate
= ty
->elements
[0]->type
;
155 case FFI_TYPE_DOUBLE
:
156 case FFI_TYPE_LONGDOUBLE
:
161 case FFI_TYPE_STRUCT
:
165 /* No HFA types are smaller than 4 bytes, or larger than 64 bytes. */
167 if (size
< 4 || size
> 64)
170 /* Find the type of the first non-structure member. */
171 elements
= ty
->elements
;
172 candidate
= elements
[0]->type
;
173 if (candidate
== FFI_TYPE_STRUCT
|| candidate
== FFI_TYPE_COMPLEX
)
177 candidate
= is_hfa0 (elements
[i
]);
183 /* If the first member is not a floating point type, it's not an HFA.
184 Also quickly re-check the size of the structure. */
188 ele_count
= size
/ sizeof(float);
189 if (size
!= ele_count
* sizeof(float))
192 case FFI_TYPE_DOUBLE
:
193 ele_count
= size
/ sizeof(double);
194 if (size
!= ele_count
* sizeof(double))
197 case FFI_TYPE_LONGDOUBLE
:
198 ele_count
= size
/ sizeof(long double);
199 if (size
!= ele_count
* sizeof(long double))
208 /* Finally, make sure that all scalar elements are the same type. */
209 for (i
= 0; elements
[i
]; ++i
)
211 int t
= elements
[i
]->type
;
212 if (t
== FFI_TYPE_STRUCT
|| t
== FFI_TYPE_COMPLEX
)
214 if (!is_hfa1 (elements
[i
], candidate
))
217 else if (t
!= candidate
)
221 /* All tests succeeded. Encode the result. */
223 return candidate
* 4 + (4 - ele_count
);
226 /* Representation of the procedure call argument marshalling
229 The terse state variable names match the names used in the AARCH64
234 unsigned ngrn
; /* Next general-purpose register number. */
235 unsigned nsrn
; /* Next vector register number. */
236 size_t nsaa
; /* Next stack offset. */
238 #if defined (__APPLE__)
239 unsigned allocating_variadic
;
243 /* Initialize a procedure call argument marshalling state. */
245 arg_init (struct arg_state
*state
)
250 #if defined (__APPLE__)
251 state
->allocating_variadic
= 0;
255 /* Allocate an aligned slot on the stack and return a pointer to it. */
257 allocate_to_stack (struct arg_state
*state
, void *stack
,
258 size_t alignment
, size_t size
)
260 size_t nsaa
= state
->nsaa
;
262 /* Round up the NSAA to the larger of 8 or the natural
263 alignment of the argument's type. */
264 #if defined (__APPLE__)
265 if (state
->allocating_variadic
&& alignment
< 8)
272 nsaa
= ALIGN (nsaa
, alignment
);
273 state
->nsaa
= nsaa
+ size
;
275 return (char *)stack
+ nsaa
;
279 extend_integer_type (void *source
, int type
)
284 return *(UINT8
*) source
;
286 return *(SINT8
*) source
;
287 case FFI_TYPE_UINT16
:
288 return *(UINT16
*) source
;
289 case FFI_TYPE_SINT16
:
290 return *(SINT16
*) source
;
291 case FFI_TYPE_UINT32
:
292 return *(UINT32
*) source
;
294 case FFI_TYPE_SINT32
:
295 return *(SINT32
*) source
;
296 case FFI_TYPE_UINT64
:
297 case FFI_TYPE_SINT64
:
298 return *(UINT64
*) source
;
300 case FFI_TYPE_POINTER
:
301 return *(uintptr_t *) source
;
308 extend_hfa_type (void *dest
, void *src
, int h
)
310 int f
= h
- AARCH64_RET_S4
;
317 "0: ldp s16, s17, [%3]\n" /* S4 */
318 " ldp s18, s19, [%3, #8]\n"
320 " ldp s16, s17, [%3]\n" /* S3 */
321 " ldr s18, [%3, #8]\n"
323 " ldp s16, s17, [%3]\n" /* S2 */
326 " ldr s16, [%3]\n" /* S1 */
329 " ldp d16, d17, [%3]\n" /* D4 */
330 " ldp d18, d19, [%3, #16]\n"
332 " ldp d16, d17, [%3]\n" /* D3 */
333 " ldr d18, [%3, #16]\n"
335 " ldp d16, d17, [%3]\n" /* D2 */
338 " ldr d16, [%3]\n" /* D1 */
341 " ldp q16, q17, [%3]\n" /* Q4 */
342 " ldp q18, q19, [%3, #16]\n"
344 " ldp q16, q17, [%3]\n" /* Q3 */
345 " ldr q18, [%3, #16]\n"
347 " ldp q16, q17, [%3]\n" /* Q2 */
350 " ldr q16, [%3]\n" /* Q1 */
352 "4: str q19, [%2, #48]\n"
353 "3: str q18, [%2, #32]\n"
354 "2: str q17, [%2, #16]\n"
357 : "r"(f
* 12), "r"(dest
), "r"(src
)
358 : "memory", "v16", "v17", "v18", "v19");
362 compress_hfa_type (void *dest
, void *reg
, int h
)
374 *(float *)dest
= *(float *)reg
;
377 asm ("ldp q16, q17, [%1]\n\t"
378 "st2 { v16.s, v17.s }[0], [%0]"
379 : : "r"(dest
), "r"(reg
) : "memory", "v16", "v17");
382 asm ("ldp q16, q17, [%1]\n\t"
383 "ldr q18, [%1, #32]\n\t"
384 "st3 { v16.s, v17.s, v18.s }[0], [%0]"
385 : : "r"(dest
), "r"(reg
) : "memory", "v16", "v17", "v18");
388 asm ("ldp q16, q17, [%1]\n\t"
389 "ldp q18, q19, [%1, #32]\n\t"
390 "st4 { v16.s, v17.s, v18.s, v19.s }[0], [%0]"
391 : : "r"(dest
), "r"(reg
) : "memory", "v16", "v17", "v18", "v19");
402 *(double *)dest
= *(double *)reg
;
405 asm ("ldp q16, q17, [%1]\n\t"
406 "st2 { v16.d, v17.d }[0], [%0]"
407 : : "r"(dest
), "r"(reg
) : "memory", "v16", "v17");
410 asm ("ldp q16, q17, [%1]\n\t"
411 "ldr q18, [%1, #32]\n\t"
412 "st3 { v16.d, v17.d, v18.d }[0], [%0]"
413 : : "r"(dest
), "r"(reg
) : "memory", "v16", "v17", "v18");
416 asm ("ldp q16, q17, [%1]\n\t"
417 "ldp q18, q19, [%1, #32]\n\t"
418 "st4 { v16.d, v17.d, v18.d, v19.d }[0], [%0]"
419 : : "r"(dest
), "r"(reg
) : "memory", "v16", "v17", "v18", "v19");
424 return memcpy (dest
, reg
, 16 * (4 - (h
& 3)));
430 /* Either allocate an appropriate register for the argument type, or if
431 none are available, allocate a stack slot and return a pointer
432 to the allocated space. */
435 allocate_int_to_reg_or_stack (struct call_context
*context
,
436 struct arg_state
*state
,
437 void *stack
, size_t size
)
439 if (state
->ngrn
< N_X_ARG_REG
)
440 return &context
->x
[state
->ngrn
++];
442 state
->ngrn
= N_X_ARG_REG
;
443 return allocate_to_stack (state
, stack
, size
, size
);
447 ffi_prep_cif_machdep (ffi_cif
*cif
)
449 ffi_type
*rtype
= cif
->rtype
;
450 size_t bytes
= cif
->bytes
;
456 flags
= AARCH64_RET_VOID
;
459 flags
= AARCH64_RET_UINT8
;
461 case FFI_TYPE_UINT16
:
462 flags
= AARCH64_RET_UINT16
;
464 case FFI_TYPE_UINT32
:
465 flags
= AARCH64_RET_UINT32
;
468 flags
= AARCH64_RET_SINT8
;
470 case FFI_TYPE_SINT16
:
471 flags
= AARCH64_RET_SINT16
;
474 case FFI_TYPE_SINT32
:
475 flags
= AARCH64_RET_SINT32
;
477 case FFI_TYPE_SINT64
:
478 case FFI_TYPE_UINT64
:
479 flags
= AARCH64_RET_INT64
;
481 case FFI_TYPE_POINTER
:
482 flags
= (sizeof(void *) == 4 ? AARCH64_RET_UINT32
: AARCH64_RET_INT64
);
486 case FFI_TYPE_DOUBLE
:
487 case FFI_TYPE_LONGDOUBLE
:
488 case FFI_TYPE_STRUCT
:
489 case FFI_TYPE_COMPLEX
:
490 flags
= is_vfp_type (rtype
);
493 size_t s
= rtype
->size
;
496 flags
= AARCH64_RET_VOID
| AARCH64_RET_IN_MEM
;
500 flags
= AARCH64_RET_INT128
;
502 flags
= AARCH64_RET_INT64
;
504 flags
= AARCH64_RET_INT128
| AARCH64_RET_NEED_COPY
;
512 for (i
= 0, n
= cif
->nargs
; i
< n
; i
++)
513 if (is_vfp_type (cif
->arg_types
[i
]))
515 flags
|= AARCH64_FLAG_ARG_V
;
519 /* Round the stack up to a multiple of the stack alignment requirement. */
520 cif
->bytes
= ALIGN(bytes
, 16);
522 #if defined (__APPLE__)
523 cif
->aarch64_nfixedargs
= 0;
529 #if defined (__APPLE__)
530 /* Perform Apple-specific cif processing for variadic calls */
531 ffi_status
ffi_prep_cif_machdep_var(ffi_cif
*cif
,
532 unsigned int nfixedargs
,
533 unsigned int ntotalargs
)
535 ffi_status status
= ffi_prep_cif_machdep (cif
);
536 cif
->aarch64_nfixedargs
= nfixedargs
;
539 #endif /* __APPLE__ */
541 extern void ffi_call_SYSV (struct call_context
*context
, void *frame
,
542 void (*fn
)(void), void *rvalue
, int flags
,
543 void *closure
) FFI_HIDDEN
;
545 /* Call a function with the provided arguments and capture the return
548 ffi_call_int (ffi_cif
*cif
, void (*fn
)(void), void *orig_rvalue
,
549 void **avalue
, void *closure
)
551 struct call_context
*context
;
552 void *stack
, *frame
, *rvalue
;
553 struct arg_state state
;
554 size_t stack_bytes
, rtype_size
, rsize
;
560 rtype_size
= rtype
->size
;
561 stack_bytes
= cif
->bytes
;
563 /* If the target function returns a structure via hidden pointer,
564 then we cannot allow a null rvalue. Otherwise, mash a null
565 rvalue to void return type. */
567 if (flags
& AARCH64_RET_IN_MEM
)
569 if (orig_rvalue
== NULL
)
572 else if (orig_rvalue
== NULL
)
573 flags
&= AARCH64_FLAG_ARG_V
;
574 else if (flags
& AARCH64_RET_NEED_COPY
)
577 /* Allocate consectutive stack for everything we'll need. */
578 context
= alloca (sizeof(struct call_context
) + stack_bytes
+ 32 + rsize
);
580 frame
= stack
+ stack_bytes
;
581 rvalue
= (rsize
? frame
+ 32 : orig_rvalue
);
584 for (i
= 0, nargs
= cif
->nargs
; i
< nargs
; i
++)
586 ffi_type
*ty
= cif
->arg_types
[i
];
598 /* If the argument is a basic type the argument is allocated to an
599 appropriate register, or if none are available, to the stack. */
603 case FFI_TYPE_UINT16
:
604 case FFI_TYPE_SINT16
:
605 case FFI_TYPE_UINT32
:
606 case FFI_TYPE_SINT32
:
607 case FFI_TYPE_UINT64
:
608 case FFI_TYPE_SINT64
:
609 case FFI_TYPE_POINTER
:
612 ffi_arg ext
= extend_integer_type (a
, t
);
613 if (state
.ngrn
< N_X_ARG_REG
)
614 context
->x
[state
.ngrn
++] = ext
;
617 void *d
= allocate_to_stack (&state
, stack
, ty
->alignment
, s
);
618 state
.ngrn
= N_X_ARG_REG
;
619 /* Note that the default abi extends each argument
620 to a full 64-bit slot, while the iOS abi allocates
621 only enough space. */
632 case FFI_TYPE_DOUBLE
:
633 case FFI_TYPE_LONGDOUBLE
:
634 case FFI_TYPE_STRUCT
:
635 case FFI_TYPE_COMPLEX
:
639 h
= is_vfp_type (ty
);
642 int elems
= 4 - (h
& 3);
643 if (state
.nsrn
+ elems
<= N_V_ARG_REG
)
645 dest
= &context
->v
[state
.nsrn
];
647 extend_hfa_type (dest
, a
, h
);
650 state
.nsrn
= N_V_ARG_REG
;
651 dest
= allocate_to_stack (&state
, stack
, ty
->alignment
, s
);
655 /* If the argument is a composite type that is larger than 16
656 bytes, then the argument has been copied to memory, and
657 the argument is replaced by a pointer to the copy. */
659 t
= FFI_TYPE_POINTER
;
664 size_t n
= (s
+ 7) / 8;
665 if (state
.ngrn
+ n
<= N_X_ARG_REG
)
667 /* If the argument is a composite type and the size in
668 double-words is not more than the number of available
669 X registers, then the argument is copied into
670 consecutive X registers. */
671 dest
= &context
->x
[state
.ngrn
];
676 /* Otherwise, there are insufficient X registers. Further
677 X register allocations are prevented, the NSAA is
678 adjusted and the argument is copied to memory at the
680 state
.ngrn
= N_X_ARG_REG
;
681 dest
= allocate_to_stack (&state
, stack
, ty
->alignment
, s
);
692 #if defined (__APPLE__)
693 if (i
+ 1 == cif
->aarch64_nfixedargs
)
695 state
.ngrn
= N_X_ARG_REG
;
696 state
.nsrn
= N_V_ARG_REG
;
697 state
.allocating_variadic
= 1;
702 ffi_call_SYSV (context
, frame
, fn
, rvalue
, flags
, closure
);
704 if (flags
& AARCH64_RET_NEED_COPY
)
705 memcpy (orig_rvalue
, rvalue
, rtype_size
);
709 ffi_call (ffi_cif
*cif
, void (*fn
) (void), void *rvalue
, void **avalue
)
711 ffi_call_int (cif
, fn
, rvalue
, avalue
, NULL
);
714 #ifdef FFI_GO_CLOSURES
716 ffi_call_go (ffi_cif
*cif
, void (*fn
) (void), void *rvalue
,
717 void **avalue
, void *closure
)
719 ffi_call_int (cif
, fn
, rvalue
, avalue
, closure
);
721 #endif /* FFI_GO_CLOSURES */
723 /* Build a trampoline. */
725 extern void ffi_closure_SYSV (void) FFI_HIDDEN
;
726 extern void ffi_closure_SYSV_V (void) FFI_HIDDEN
;
728 #if FFI_EXEC_TRAMPOLINE_TABLE
730 #include <mach/mach.h>
735 extern void *ffi_closure_trampoline_table_page
;
737 typedef struct ffi_trampoline_table ffi_trampoline_table
;
738 typedef struct ffi_trampoline_table_entry ffi_trampoline_table_entry
;
740 struct ffi_trampoline_table
742 /* contiguous writable and executable pages */
743 vm_address_t config_page
;
744 vm_address_t trampoline_page
;
746 /* free list tracking */
748 ffi_trampoline_table_entry
*free_list
;
749 ffi_trampoline_table_entry
*free_list_pool
;
751 ffi_trampoline_table
*prev
;
752 ffi_trampoline_table
*next
;
755 struct ffi_trampoline_table_entry
757 void *(*trampoline
) ();
758 ffi_trampoline_table_entry
*next
;
761 /* The trampoline configuration is placed a page prior to the trampoline's entry point */
762 #define FFI_TRAMPOLINE_CODELOC_CONFIG(codeloc) ((void **) (((uint8_t *) codeloc) - PAGE_SIZE));
764 /* Total number of trampolines that fit in one trampoline table */
765 #define FFI_TRAMPOLINE_COUNT (PAGE_SIZE / FFI_TRAMPOLINE_SIZE)
767 static pthread_mutex_t ffi_trampoline_lock
= PTHREAD_MUTEX_INITIALIZER
;
768 static ffi_trampoline_table
*ffi_trampoline_tables
= NULL
;
770 static ffi_trampoline_table
*
771 ffi_trampoline_table_alloc ()
773 ffi_trampoline_table
*table
= NULL
;
775 /* Loop until we can allocate two contiguous pages */
776 while (table
== NULL
)
778 vm_address_t config_page
= 0x0;
781 /* Try to allocate two pages */
783 vm_allocate (mach_task_self (), &config_page
, PAGE_SIZE
* 2,
785 if (kt
!= KERN_SUCCESS
)
787 fprintf (stderr
, "vm_allocate() failure: %d at %s:%d\n", kt
,
792 /* Now drop the second half of the allocation to make room for the trampoline table */
793 vm_address_t trampoline_page
= config_page
+ PAGE_SIZE
;
794 kt
= vm_deallocate (mach_task_self (), trampoline_page
, PAGE_SIZE
);
795 if (kt
!= KERN_SUCCESS
)
797 fprintf (stderr
, "vm_deallocate() failure: %d at %s:%d\n", kt
,
802 /* Remap the trampoline table to directly follow the config page */
807 vm_remap (mach_task_self (), &trampoline_page
, PAGE_SIZE
, 0x0, FALSE
,
809 (vm_address_t
) & ffi_closure_trampoline_table_page
, FALSE
,
810 &cur_prot
, &max_prot
, VM_INHERIT_SHARE
);
812 /* If we lost access to the destination trampoline page, drop our config allocation mapping and retry */
813 if (kt
!= KERN_SUCCESS
)
815 /* Log unexpected failures */
816 if (kt
!= KERN_NO_SPACE
)
818 fprintf (stderr
, "vm_remap() failure: %d at %s:%d\n", kt
,
822 vm_deallocate (mach_task_self (), config_page
, PAGE_SIZE
);
826 /* We have valid trampoline and config pages */
827 table
= calloc (1, sizeof (ffi_trampoline_table
));
828 table
->free_count
= FFI_TRAMPOLINE_COUNT
;
829 table
->config_page
= config_page
;
830 table
->trampoline_page
= trampoline_page
;
832 /* Create and initialize the free list */
833 table
->free_list_pool
=
834 calloc (FFI_TRAMPOLINE_COUNT
, sizeof (ffi_trampoline_table_entry
));
837 for (i
= 0; i
< table
->free_count
; i
++)
839 ffi_trampoline_table_entry
*entry
= &table
->free_list_pool
[i
];
841 (void *) (table
->trampoline_page
+ (i
* FFI_TRAMPOLINE_SIZE
));
843 if (i
< table
->free_count
- 1)
844 entry
->next
= &table
->free_list_pool
[i
+ 1];
847 table
->free_list
= table
->free_list_pool
;
854 ffi_closure_alloc (size_t size
, void **code
)
856 /* Create the closure */
857 ffi_closure
*closure
= malloc (size
);
861 pthread_mutex_lock (&ffi_trampoline_lock
);
863 /* Check for an active trampoline table with available entries. */
864 ffi_trampoline_table
*table
= ffi_trampoline_tables
;
865 if (table
== NULL
|| table
->free_list
== NULL
)
867 table
= ffi_trampoline_table_alloc ();
874 /* Insert the new table at the top of the list */
875 table
->next
= ffi_trampoline_tables
;
876 if (table
->next
!= NULL
)
877 table
->next
->prev
= table
;
879 ffi_trampoline_tables
= table
;
882 /* Claim the free entry */
883 ffi_trampoline_table_entry
*entry
= ffi_trampoline_tables
->free_list
;
884 ffi_trampoline_tables
->free_list
= entry
->next
;
885 ffi_trampoline_tables
->free_count
--;
888 pthread_mutex_unlock (&ffi_trampoline_lock
);
890 /* Initialize the return values */
891 *code
= entry
->trampoline
;
892 closure
->trampoline_table
= table
;
893 closure
->trampoline_table_entry
= entry
;
899 ffi_closure_free (void *ptr
)
901 ffi_closure
*closure
= ptr
;
903 pthread_mutex_lock (&ffi_trampoline_lock
);
905 /* Fetch the table and entry references */
906 ffi_trampoline_table
*table
= closure
->trampoline_table
;
907 ffi_trampoline_table_entry
*entry
= closure
->trampoline_table_entry
;
909 /* Return the entry to the free list */
910 entry
->next
= table
->free_list
;
911 table
->free_list
= entry
;
914 /* If all trampolines within this table are free, and at least one other table exists, deallocate
916 if (table
->free_count
== FFI_TRAMPOLINE_COUNT
917 && ffi_trampoline_tables
!= table
)
919 /* Remove from the list */
920 if (table
->prev
!= NULL
)
921 table
->prev
->next
= table
->next
;
923 if (table
->next
!= NULL
)
924 table
->next
->prev
= table
->prev
;
926 /* Deallocate pages */
928 kt
= vm_deallocate (mach_task_self (), table
->config_page
, PAGE_SIZE
);
929 if (kt
!= KERN_SUCCESS
)
930 fprintf (stderr
, "vm_deallocate() failure: %d at %s:%d\n", kt
,
934 vm_deallocate (mach_task_self (), table
->trampoline_page
, PAGE_SIZE
);
935 if (kt
!= KERN_SUCCESS
)
936 fprintf (stderr
, "vm_deallocate() failure: %d at %s:%d\n", kt
,
939 /* Deallocate free list */
940 free (table
->free_list_pool
);
943 else if (ffi_trampoline_tables
!= table
)
945 /* Otherwise, bump this table to the top of the list */
947 table
->next
= ffi_trampoline_tables
;
948 if (ffi_trampoline_tables
!= NULL
)
949 ffi_trampoline_tables
->prev
= table
;
951 ffi_trampoline_tables
= table
;
954 pthread_mutex_unlock (&ffi_trampoline_lock
);
956 /* Free the closure */
963 ffi_prep_closure_loc (ffi_closure
*closure
,
965 void (*fun
)(ffi_cif
*,void*,void**,void*),
969 if (cif
->abi
!= FFI_SYSV
)
974 if (cif
->flags
& AARCH64_FLAG_ARG_V
)
975 start
= ffi_closure_SYSV_V
;
977 start
= ffi_closure_SYSV
;
979 #if FFI_EXEC_TRAMPOLINE_TABLE
980 void **config
= FFI_TRAMPOLINE_CODELOC_CONFIG (codeloc
);
984 static const unsigned char trampoline
[16] = {
985 0x90, 0x00, 0x00, 0x58, /* ldr x16, tramp+16 */
986 0xf1, 0xff, 0xff, 0x10, /* adr x17, tramp+0 */
987 0x00, 0x02, 0x1f, 0xd6 /* br x16 */
989 char *tramp
= closure
->tramp
;
991 memcpy (tramp
, trampoline
, sizeof(trampoline
));
993 *(UINT64
*)(tramp
+ 16) = (uintptr_t)start
;
995 ffi_clear_cache(tramp
, tramp
+ FFI_TRAMPOLINE_SIZE
);
1000 closure
->user_data
= user_data
;
1005 #ifdef FFI_GO_CLOSURES
1006 extern void ffi_go_closure_SYSV (void) FFI_HIDDEN
;
1007 extern void ffi_go_closure_SYSV_V (void) FFI_HIDDEN
;
1010 ffi_prep_go_closure (ffi_go_closure
*closure
, ffi_cif
* cif
,
1011 void (*fun
)(ffi_cif
*,void*,void**,void*))
1013 void (*start
)(void);
1015 if (cif
->abi
!= FFI_SYSV
)
1018 if (cif
->flags
& AARCH64_FLAG_ARG_V
)
1019 start
= ffi_go_closure_SYSV_V
;
1021 start
= ffi_go_closure_SYSV
;
1023 closure
->tramp
= start
;
1029 #endif /* FFI_GO_CLOSURES */
1031 /* Primary handler to setup and invoke a function within a closure.
1033 A closure when invoked enters via the assembler wrapper
1034 ffi_closure_SYSV(). The wrapper allocates a call context on the
1035 stack, saves the interesting registers (from the perspective of
1036 the calling convention) into the context then passes control to
1037 ffi_closure_SYSV_inner() passing the saved context and a pointer to
1038 the stack at the point ffi_closure_SYSV() was invoked.
1040 On the return path the assembler wrapper will reload call context
1043 ffi_closure_SYSV_inner() marshalls the call context into ffi value
1044 descriptors, invokes the wrapped function, then marshalls the return
1045 value back into the call context. */
1048 ffi_closure_SYSV_inner (ffi_cif
*cif
,
1049 void (*fun
)(ffi_cif
*,void*,void**,void*),
1051 struct call_context
*context
,
1052 void *stack
, void *rvalue
, void *struct_rvalue
)
1054 void **avalue
= (void**) alloca (cif
->nargs
* sizeof (void*));
1055 int i
, h
, nargs
, flags
;
1056 struct arg_state state
;
1060 for (i
= 0, nargs
= cif
->nargs
; i
< nargs
; i
++)
1062 ffi_type
*ty
= cif
->arg_types
[i
];
1064 size_t n
, s
= ty
->size
;
1073 case FFI_TYPE_UINT8
:
1074 case FFI_TYPE_SINT8
:
1075 case FFI_TYPE_UINT16
:
1076 case FFI_TYPE_SINT16
:
1077 case FFI_TYPE_UINT32
:
1078 case FFI_TYPE_SINT32
:
1079 case FFI_TYPE_UINT64
:
1080 case FFI_TYPE_SINT64
:
1081 case FFI_TYPE_POINTER
:
1082 avalue
[i
] = allocate_int_to_reg_or_stack (context
, &state
, stack
, s
);
1085 case FFI_TYPE_FLOAT
:
1086 case FFI_TYPE_DOUBLE
:
1087 case FFI_TYPE_LONGDOUBLE
:
1088 case FFI_TYPE_STRUCT
:
1089 case FFI_TYPE_COMPLEX
:
1090 h
= is_vfp_type (ty
);
1094 if (state
.nsrn
+ n
<= N_V_ARG_REG
)
1096 void *reg
= &context
->v
[state
.nsrn
];
1099 /* Eeek! We need a pointer to the structure, however the
1100 homogeneous float elements are being passed in individual
1101 registers, therefore for float and double the structure
1102 is not represented as a contiguous sequence of bytes in
1103 our saved register context. We don't need the original
1104 contents of the register storage, so we reformat the
1105 structure into the same memory. */
1106 avalue
[i
] = compress_hfa_type (reg
, reg
, h
);
1110 state
.nsrn
= N_V_ARG_REG
;
1111 avalue
[i
] = allocate_to_stack (&state
, stack
,
1117 /* Replace Composite type of size greater than 16 with a
1119 avalue
[i
] = *(void **)
1120 allocate_int_to_reg_or_stack (context
, &state
, stack
,
1126 if (state
.ngrn
+ n
<= N_X_ARG_REG
)
1128 avalue
[i
] = &context
->x
[state
.ngrn
];
1133 state
.ngrn
= N_X_ARG_REG
;
1134 avalue
[i
] = allocate_to_stack (&state
, stack
,
1146 if (flags
& AARCH64_RET_IN_MEM
)
1147 rvalue
= struct_rvalue
;
1149 fun (cif
, rvalue
, avalue
, user_data
);