libgcc/config/i386/heap-trampoline.c

   1 /* Copyright The GNU Toolchain Authors. */
   2
   3 /* libc is required to allocate trampolines.  */
   4 #ifndef inhibit_libc
   5
   6 #include <unistd.h>
   7 #include <sys/mman.h>
   8 #include <stdint.h>
   9 #include <stdlib.h>
  10 #include <stdio.h>
  11 #include <string.h>
  12
  13 #if __APPLE__ && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
  14 /* For pthread_jit_write_protect_np */
  15 #include <pthread.h>
  16 #endif
  17
  18 /* HEAP_T_ATTR is provided to allow targets to build the exported functions
  19    as weak definitions.  */
  20 #ifndef HEAP_T_ATTR
  21 #  define HEAP_T_ATTR
  22 #endif
  23
  24 void *allocate_trampoline_page (void);
  25 int get_trampolines_per_page (void);
  26 struct tramp_ctrl_data *allocate_tramp_ctrl (struct tramp_ctrl_data *parent);
  27 void *allocate_trampoline_page (void);
  28
  29 void __gcc_nested_func_ptr_created (void *chain, void *func, void *dst);
  30 void __gcc_nested_func_ptr_deleted (void);
  31
  32 #if __x86_64__
  33
  34 #ifdef __LP64__
  35 static const uint8_t trampoline_insns[] = {
  36 #if defined __CET__ && (__CET__ & 1) != 0
  37   /* endbr64.  */
  38   0xf3, 0x0f, 0x1e, 0xfa,
  39 #endif
  40
  41   /* movabsq $<func>,%r11  */
  42   0x49, 0xbb,
  43   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  44
  45   /* movabsq $<chain>,%r10  */
  46   0x49, 0xba,
  47   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  48
  49   /* rex.WB jmpq *%r11  */
  50   0x41, 0xff, 0xe3,
  51
  52   /* Pad to the multiple of 4 bytes.  */
  53   0x90
  54 };
  55 #else
  56 static const uint8_t trampoline_insns[] = {
  57 #if defined __CET__ && (__CET__ & 1) != 0
  58   /* endbr64.  */
  59   0xf3, 0x0f, 0x1e, 0xfa,
  60 #endif
  61
  62   /* movl $<func>,%r11d  */
  63   0x41, 0xbb,
  64   0x00, 0x00, 0x00, 0x00,
  65
  66   /* movl $<chain>,%r10d  */
  67   0x41, 0xba,
  68   0x00, 0x00, 0x00, 0x00,
  69
  70   /* rex.WB jmpq *%r11  */
  71   0x41, 0xff, 0xe3,
  72
  73   /* Pad to the multiple of 4 bytes.  */
  74   0x90
  75 };
  76 #endif
  77
  78 union ix86_trampoline {
  79   uint8_t insns[sizeof(trampoline_insns)];
  80
  81   struct __attribute__((packed)) fields {
  82 #if defined __CET__ && (__CET__ & 1) != 0
  83     uint8_t endbr64[4];
  84 #endif
  85     uint8_t insn_0[2];
  86     void *func_ptr;
  87     uint8_t insn_1[2];
  88     void *chain_ptr;
  89     uint8_t insn_2[3];
  90     uint8_t pad;
  91   } fields;
  92 };
  93
  94 #elif __i386__
  95
  96 static const uint8_t trampoline_insns[] = {
  97   /* movl $<chain>,%ecx  */
  98   0xb9,
  99   0x00, 0x00, 0x00, 0x00,
 100
 101   /* jmpl <func>-. */
 102   0xe9,
 103   0x00, 0x00, 0x00, 0x00,
 104 };
 105
 106 union ix86_trampoline {
 107   uint8_t insns[sizeof(trampoline_insns)];
 108
 109   struct __attribute__((packed)) fields {
 110     uint8_t insn_0[1];
 111     void *chain_ptr;
 112     uint8_t insn_1[1];
 113     uintptr_t func_offset;
 114   } fields;
 115 };
 116
 117 #else
 118 #error unsupported architecture/ABI
 119 #endif
 120
 121 struct tramp_ctrl_data
 122 {
 123   struct tramp_ctrl_data *prev;
 124
 125   int free_trampolines;
 126
 127   /* This will be pointing to an executable mmap'ed page.  */
 128   union ix86_trampoline *trampolines;
 129 };
 130
 131 int
 132 get_trampolines_per_page (void)
 133 {
 134   return getpagesize() / sizeof(union ix86_trampoline);
 135 }
 136
 137 static _Thread_local struct tramp_ctrl_data *tramp_ctrl_curr = NULL;
 138
 139 void *
 140 allocate_trampoline_page (void)
 141 {
 142   void *page;
 143
 144 #if defined(__gnu_linux__)
 145   page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
 146                MAP_ANON | MAP_PRIVATE, 0, 0);
 147 #elif __APPLE__
 148 # if  __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
 149   page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
 150                MAP_ANON | MAP_PRIVATE | MAP_JIT, 0, 0);
 151 # else
 152   page = mmap (0, getpagesize (), PROT_WRITE | PROT_EXEC,
 153                MAP_ANON | MAP_PRIVATE, 0, 0);
 154 # endif
 155 #else
 156   page = MAP_FAILED;
 157 #endif
 158
 159   return page;
 160 }
 161
 162 struct tramp_ctrl_data *
 163 allocate_tramp_ctrl (struct tramp_ctrl_data *parent)
 164 {
 165   struct tramp_ctrl_data *p = malloc (sizeof (struct tramp_ctrl_data));
 166   if (p == NULL)
 167     return NULL;
 168
 169   p->trampolines = allocate_trampoline_page ();
 170
 171   if (p->trampolines == MAP_FAILED)
 172     return NULL;
 173
 174   p->prev = parent;
 175   p->free_trampolines = get_trampolines_per_page();
 176
 177   return p;
 178 }
 179
 180 HEAP_T_ATTR
 181 void
 182 __gcc_nested_func_ptr_created (void *chain, void *func, void *dst)
 183 {
 184   if (tramp_ctrl_curr == NULL)
 185     {
 186       tramp_ctrl_curr = allocate_tramp_ctrl (NULL);
 187       if (tramp_ctrl_curr == NULL)
 188         abort ();
 189     }
 190
 191   if (tramp_ctrl_curr->free_trampolines == 0)
 192     {
 193       void *tramp_ctrl = allocate_tramp_ctrl (tramp_ctrl_curr);
 194       if (!tramp_ctrl)
 195         abort ();
 196
 197       tramp_ctrl_curr = tramp_ctrl;
 198     }
 199
 200   union ix86_trampoline *trampoline
 201     = &tramp_ctrl_curr->trampolines[get_trampolines_per_page ()
 202                                     - tramp_ctrl_curr->free_trampolines];
 203
 204 #if __APPLE__ && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
 205   /* Disable write protection for the MAP_JIT regions in this thread (see
 206      https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon) */
 207   pthread_jit_write_protect_np (0);
 208 #endif
 209
 210   memcpy (trampoline->insns, trampoline_insns,
 211           sizeof(trampoline_insns));
 212   trampoline->fields.chain_ptr = chain;
 213 #if __x86_64__
 214   trampoline->fields.func_ptr = func;
 215 #elif __i386__
 216   uintptr_t off_add = (uintptr_t) &trampoline->fields.func_offset;
 217   off_add += 4;
 218   trampoline->fields.func_offset = (uintptr_t)func - off_add;
 219 #endif
 220
 221 #if __APPLE__ && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101400
 222   /* Re-enable write protection.  */
 223   pthread_jit_write_protect_np (1);
 224 #endif
 225
 226   tramp_ctrl_curr->free_trampolines -= 1;
 227
 228   __builtin___clear_cache ((void *)trampoline->insns,
 229                            ((void *)trampoline->insns + sizeof(trampoline->insns)));
 230
 231   *(void **) dst = &trampoline->insns;
 232 }
 233
 234 HEAP_T_ATTR
 235 void
 236 __gcc_nested_func_ptr_deleted (void)
 237 {
 238   if (tramp_ctrl_curr == NULL)
 239     abort ();
 240
 241   tramp_ctrl_curr->free_trampolines += 1;
 242
 243   if (tramp_ctrl_curr->free_trampolines == get_trampolines_per_page ())
 244     {
 245       if (tramp_ctrl_curr->prev == NULL)
 246         return;
 247
 248       munmap (tramp_ctrl_curr->trampolines, getpagesize());
 249       struct tramp_ctrl_data *prev = tramp_ctrl_curr->prev;
 250       free (tramp_ctrl_curr);
 251       tramp_ctrl_curr = prev;
 252     }
 253 }
 254
 255 #endif /* !inhibit_libc */