libgcc/config/aarch64/heap-trampoline.c

   1 /* Copyright The GNU Toolchain Authors. */
   2
   3 /* libc is required to allocate trampolines.  */
   4 #ifndef inhibit_libc
   5
   6 #include <unistd.h>
   7 #include <sys/mman.h>
   8 #include <stdint.h>
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <string.h>
  12
  13 #if __APPLE__
  14 /* For pthread_jit_write_protect_np */
  15 #include <pthread.h>
  16 #endif
  17
  18 void *allocate_trampoline_page (void);
  19 int get_trampolines_per_page (void);
  20 struct tramp_ctrl_data *allocate_tramp_ctrl (struct tramp_ctrl_data *parent);
  21 void *allocate_trampoline_page (void);
  22
  23 void __builtin_nested_func_ptr_created (void *chain, void *func, void **dst);
  24 void __builtin_nested_func_ptr_deleted (void);
  25
  26 #if defined(__gnu_linux__)
  27 static const uint32_t aarch64_trampoline_insns[] = {
  28   0xd503245f, /* hint    34 */
  29   0x580000b1, /* ldr     x17, .+20 */
  30   0x580000d2, /* ldr     x18, .+24 */
  31   0xd61f0220, /* br      x17 */
  32   0xd5033f9f, /* dsb     sy */
  33   0xd5033fdf /* isb */
  34 };
  35
  36 #elif __APPLE__
  37 static const uint32_t aarch64_trampoline_insns[] = {
  38   0xd503245f, /* hint    34 */
  39   0x580000b1, /* ldr     x17, .+20 */
  40   0x580000d0, /* ldr     x16, .+24 */
  41   0xd61f0220, /* br      x17 */
  42   0xd5033f9f, /* dsb     sy */
  43   0xd5033fdf /* isb */
  44 };
  45
  46 #else
  47 #error "Unsupported AArch64 platform for heap trampolines"
  48 #endif
  49
  50 struct aarch64_trampoline {
  51   uint32_t insns[6];
  52   void *func_ptr;
  53   void *chain_ptr;
  54 };
  55
  56 struct tramp_ctrl_data
  57 {
  58   struct tramp_ctrl_data *prev;
  59
  60   int free_trampolines;
  61
  62   /* This will be pointing to an executable mmap'ed page.  */
  63   struct aarch64_trampoline *trampolines;
  64 };
  65
  66 int
  67 get_trampolines_per_page (void)
  68 {
  69   return getpagesize() / sizeof(struct aarch64_trampoline);
  70 }
  71
  72 static _Thread_local struct tramp_ctrl_data *tramp_ctrl_curr = NULL;
  73
  74 void *
  75 allocate_trampoline_page (void)
  76 {
  77   void *page;
  78
  79 #if defined(__gnu_linux__)
  80   page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
  81                MAP_ANON | MAP_PRIVATE, 0, 0);
  82 #elif __APPLE__
  83   page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
  84                MAP_ANON | MAP_PRIVATE | MAP_JIT, 0, 0);
  85 #else
  86   page = MAP_FAILED;
  87 #endif
  88
  89   return page;
  90 }
  91
  92 struct tramp_ctrl_data *
  93 allocate_tramp_ctrl (struct tramp_ctrl_data *parent)
  94 {
  95   struct tramp_ctrl_data *p = malloc (sizeof (struct tramp_ctrl_data));
  96   if (p == NULL)
  97     return NULL;
  98
  99   p->trampolines = allocate_trampoline_page ();
 100
 101   if (p->trampolines == MAP_FAILED)
 102     return NULL;
 103
 104   p->prev = parent;
 105   p->free_trampolines = get_trampolines_per_page();
 106
 107   return p;
 108 }
 109
 110 void
 111 __builtin_nested_func_ptr_created (void *chain, void *func, void **dst)
 112 {
 113   if (tramp_ctrl_curr == NULL)
 114     {
 115       tramp_ctrl_curr = allocate_tramp_ctrl (NULL);
 116       if (tramp_ctrl_curr == NULL)
 117         abort ();
 118     }
 119
 120   if (tramp_ctrl_curr->free_trampolines == 0)
 121     {
 122       void *tramp_ctrl = allocate_tramp_ctrl (tramp_ctrl_curr);
 123       if (!tramp_ctrl)
 124         abort ();
 125
 126       tramp_ctrl_curr = tramp_ctrl;
 127     }
 128
 129   struct aarch64_trampoline *trampoline
 130     = &tramp_ctrl_curr->trampolines[get_trampolines_per_page ()
 131                                     - tramp_ctrl_curr->free_trampolines];
 132
 133 #if __APPLE__
 134   /* Disable write protection for the MAP_JIT regions in this thread (see
 135      https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon) */
 136   pthread_jit_write_protect_np (0);
 137 #endif
 138
 139   memcpy (trampoline->insns, aarch64_trampoline_insns,
 140           sizeof(aarch64_trampoline_insns));
 141   trampoline->func_ptr = func;
 142   trampoline->chain_ptr = chain;
 143
 144 #if __APPLE__
 145   /* Re-enable write protection.  */
 146   pthread_jit_write_protect_np (1);
 147 #endif
 148
 149   tramp_ctrl_curr->free_trampolines -= 1;
 150
 151   __builtin___clear_cache ((void *)trampoline->insns,
 152                            ((void *)trampoline->insns + sizeof(trampoline->insns)));
 153
 154   *dst = &trampoline->insns;
 155 }
 156
 157 void
 158 __builtin_nested_func_ptr_deleted (void)
 159 {
 160   if (tramp_ctrl_curr == NULL)
 161     abort ();
 162
 163   tramp_ctrl_curr->free_trampolines += 1;
 164
 165   if (tramp_ctrl_curr->free_trampolines == get_trampolines_per_page ())
 166     {
 167       if (tramp_ctrl_curr->prev == NULL)
 168         return;
 169
 170       munmap (tramp_ctrl_curr->trampolines, getpagesize());
 171       struct tramp_ctrl_data *prev = tramp_ctrl_curr->prev;
 172       free (tramp_ctrl_curr);
 173       tramp_ctrl_curr = prev;
 174     }
 175 }
 176
 177 #endif /* !inhibit_libc */