libgcc/config/aarch64/heap-trampoline.c

   1 /* Copyright The GNU Toolchain Authors. */
   2
   3 /* libc is required to allocate trampolines.  */
   4 #ifndef inhibit_libc
   5
   6 #include <unistd.h>
   7 #include <sys/mman.h>
   8 #include <stdint.h>
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <string.h>
  12
  13 #if __APPLE__
  14 /* For pthread_jit_write_protect_np */
  15 #include <pthread.h>
  16 #endif
  17
  18 /* HEAP_T_ATTR is provided to allow targets to build the exported functions
  19    as weak definitions.  */
  20 #ifndef HEAP_T_ATTR
  21 #  define HEAP_T_ATTR
  22 #endif
  23
  24 void *allocate_trampoline_page (void);
  25 int get_trampolines_per_page (void);
  26 struct tramp_ctrl_data *allocate_tramp_ctrl (struct tramp_ctrl_data *parent);
  27 void *allocate_trampoline_page (void);
  28
  29 void __gcc_nested_func_ptr_created (void *chain, void *func, void *dst);
  30 void __gcc_nested_func_ptr_deleted (void);
  31
  32 #if defined(__gnu_linux__)
  33 static const uint32_t aarch64_trampoline_insns[] = {
  34   0xd503245f, /* hint    34 */
  35   0x580000b1, /* ldr     x17, .+20 */
  36   0x580000d2, /* ldr     x18, .+24 */
  37   0xd61f0220, /* br      x17 */
  38   0xd5033f9f, /* dsb     sy */
  39   0xd5033fdf /* isb */
  40 };
  41
  42 #elif __APPLE__
  43 static const uint32_t aarch64_trampoline_insns[] = {
  44   0xd503245f, /* hint    34 */
  45   0x580000b1, /* ldr     x17, .+20 */
  46   0x580000d0, /* ldr     x16, .+24 */
  47   0xd61f0220, /* br      x17 */
  48   0xd5033f9f, /* dsb     sy */
  49   0xd5033fdf /* isb */
  50 };
  51
  52 #else
  53 #error "Unsupported AArch64 platform for heap trampolines"
  54 #endif
  55
  56 struct aarch64_trampoline {
  57   uint32_t insns[6];
  58   void *func_ptr;
  59   void *chain_ptr;
  60 };
  61
  62 struct tramp_ctrl_data
  63 {
  64   struct tramp_ctrl_data *prev;
  65
  66   int free_trampolines;
  67
  68   /* This will be pointing to an executable mmap'ed page.  */
  69   struct aarch64_trampoline *trampolines;
  70 };
  71
  72 int
  73 get_trampolines_per_page (void)
  74 {
  75   return getpagesize() / sizeof(struct aarch64_trampoline);
  76 }
  77
  78 static _Thread_local struct tramp_ctrl_data *tramp_ctrl_curr = NULL;
  79
  80 void *
  81 allocate_trampoline_page (void)
  82 {
  83   void *page;
  84
  85 #if defined(__gnu_linux__)
  86   page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
  87                MAP_ANON | MAP_PRIVATE, 0, 0);
  88 #elif __APPLE__
  89   page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
  90                MAP_ANON | MAP_PRIVATE | MAP_JIT, 0, 0);
  91 #else
  92   page = MAP_FAILED;
  93 #endif
  94
  95   return page;
  96 }
  97
  98 struct tramp_ctrl_data *
  99 allocate_tramp_ctrl (struct tramp_ctrl_data *parent)
 100 {
 101   struct tramp_ctrl_data *p = malloc (sizeof (struct tramp_ctrl_data));
 102   if (p == NULL)
 103     return NULL;
 104
 105   p->trampolines = allocate_trampoline_page ();
 106
 107   if (p->trampolines == MAP_FAILED)
 108     return NULL;
 109
 110   p->prev = parent;
 111   p->free_trampolines = get_trampolines_per_page();
 112
 113   return p;
 114 }
 115
 116 HEAP_T_ATTR
 117 void
 118 __gcc_nested_func_ptr_created (void *chain, void *func, void *dst)
 119 {
 120   if (tramp_ctrl_curr == NULL)
 121     {
 122       tramp_ctrl_curr = allocate_tramp_ctrl (NULL);
 123       if (tramp_ctrl_curr == NULL)
 124         abort ();
 125     }
 126
 127   if (tramp_ctrl_curr->free_trampolines == 0)
 128     {
 129       void *tramp_ctrl = allocate_tramp_ctrl (tramp_ctrl_curr);
 130       if (!tramp_ctrl)
 131         abort ();
 132
 133       tramp_ctrl_curr = tramp_ctrl;
 134     }
 135
 136   struct aarch64_trampoline *trampoline
 137     = &tramp_ctrl_curr->trampolines[get_trampolines_per_page ()
 138                                     - tramp_ctrl_curr->free_trampolines];
 139
 140 #if __APPLE__
 141   /* Disable write protection for the MAP_JIT regions in this thread (see
 142      https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon) */
 143   pthread_jit_write_protect_np (0);
 144 #endif
 145
 146   memcpy (trampoline->insns, aarch64_trampoline_insns,
 147           sizeof(aarch64_trampoline_insns));
 148   trampoline->func_ptr = func;
 149   trampoline->chain_ptr = chain;
 150
 151 #if __APPLE__
 152   /* Re-enable write protection.  */
 153   pthread_jit_write_protect_np (1);
 154 #endif
 155
 156   tramp_ctrl_curr->free_trampolines -= 1;
 157
 158   __builtin___clear_cache ((void *)trampoline->insns,
 159                            ((void *)trampoline->insns + sizeof(trampoline->insns)));
 160
 161   *(void **) dst = &trampoline->insns;
 162 }
 163
 164 HEAP_T_ATTR
 165 void
 166 __gcc_nested_func_ptr_deleted (void)
 167 {
 168   if (tramp_ctrl_curr == NULL)
 169     abort ();
 170
 171   tramp_ctrl_curr->free_trampolines += 1;
 172
 173   if (tramp_ctrl_curr->free_trampolines == get_trampolines_per_page ())
 174     {
 175       if (tramp_ctrl_curr->prev == NULL)
 176         return;
 177
 178       munmap (tramp_ctrl_curr->trampolines, getpagesize());
 179       struct tramp_ctrl_data *prev = tramp_ctrl_curr->prev;
 180       free (tramp_ctrl_curr);
 181       tramp_ctrl_curr = prev;
 182     }
 183 }
 184
 185 #endif /* !inhibit_libc */