memory/build/replace_malloc.c

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   3  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 #ifndef MOZ_MEMORY
   6 #  error Should not compile this file when MOZ_MEMORY is not set
   7 #endif
   8
   9 #ifndef MOZ_REPLACE_MALLOC
  10 #  error Should not compile this file when replace-malloc is disabled
  11 #endif
  12
  13 #ifdef MOZ_NATIVE_JEMALLOC
  14 #  error Should not compile this file when we want to use native jemalloc
  15 #endif
  16
  17 #include "mozmemory_wrap.h"
  18
  19 /* Declare all je_* functions */
  20 #define MALLOC_DECL(name, return_type, ...) \
  21   return_type je_ ## name(__VA_ARGS__);
  22 #include "malloc_decls.h"
  23
  24 #include "mozilla/Likely.h"
  25 /*
  26  * Windows doesn't come with weak imports as they are possible with
  27  * LD_PRELOAD or DYLD_INSERT_LIBRARIES on Linux/OSX. On this platform,
  28  * the replacement functions are defined as variable pointers to the
  29  * function resolved with GetProcAddress() instead of weak definitions
  30  * of functions. On Android, the same needs to happen as well, because
  31  * the Android linker doesn't handle weak linking with non LD_PRELOADed
  32  * libraries, but LD_PRELOADing is not very convenient on Android, with
  33  * the zygote.
  34  */
  35 #ifdef XP_DARWIN
  36 #  define MOZ_REPLACE_WEAK __attribute__((weak_import))
  37 #elif defined(XP_WIN) || defined(MOZ_WIDGET_ANDROID)
  38 #  define MOZ_NO_REPLACE_FUNC_DECL
  39 #elif defined(__GNUC__)
  40 #  define MOZ_REPLACE_WEAK __attribute__((weak))
  41 #endif
  42
  43 #include "replace_malloc.h"
  44
  45 #define MALLOC_DECL(name, return_type, ...) \
  46     je_ ## name,
  47
  48 static const malloc_table_t malloc_table = {
  49 #include "malloc_decls.h"
  50 };
  51
  52 #ifdef MOZ_NO_REPLACE_FUNC_DECL
  53 #  define MALLOC_DECL(name, return_type, ...) \
  54     typedef return_type (replace_ ## name ## _impl_t)(__VA_ARGS__); \
  55     replace_ ## name ## _impl_t *replace_ ## name = NULL;
  56 #  define MALLOC_FUNCS MALLOC_FUNCS_ALL
  57 #  include "malloc_decls.h"
  58
  59 #  ifdef XP_WIN
  60 #    include <windows.h>
  61 static void
  62 replace_malloc_init_funcs()
  63 {
  64   char replace_malloc_lib[1024];
  65   if (GetEnvironmentVariableA("MOZ_REPLACE_MALLOC_LIB", (LPSTR)&replace_malloc_lib,
  66                               sizeof(replace_malloc_lib)) > 0) {
  67     HMODULE handle = LoadLibraryA(replace_malloc_lib);
  68     if (handle) {
  69 #define MALLOC_DECL(name, ...) \
  70   replace_ ## name = (replace_ ## name ## _impl_t *) GetProcAddress(handle, "replace_" # name);
  71
  72 #  define MALLOC_FUNCS MALLOC_FUNCS_ALL
  73 #include "malloc_decls.h"
  74     }
  75   }
  76 }
  77 #  elif defined(MOZ_WIDGET_ANDROID)
  78 #    include <dlfcn.h>
  79 #    include <stdlib.h>
  80 static void
  81 replace_malloc_init_funcs()
  82 {
  83   const char *replace_malloc_lib = getenv("MOZ_REPLACE_MALLOC_LIB");
  84   if (replace_malloc_lib && *replace_malloc_lib) {
  85     void *handle = dlopen(replace_malloc_lib, RTLD_LAZY);
  86     if (handle) {
  87 #define MALLOC_DECL(name, ...) \
  88   replace_ ## name = (replace_ ## name ## _impl_t *) dlsym(handle, "replace_" # name);
  89
  90 #  define MALLOC_FUNCS MALLOC_FUNCS_ALL
  91 #include "malloc_decls.h"
  92     }
  93   }
  94 }
  95 #  else
  96 #    error No implementation for replace_malloc_init_funcs()
  97 #  endif
  98
  99 #endif /* MOZ_NO_REPLACE_FUNC_DECL */
 100
 101 /*
 102  * Below is the malloc implementation overriding jemalloc and calling the
 103  * replacement functions if they exist.
 104  */
 105
 106 /*
 107  * On OSX, MOZ_MEMORY_API is defined to nothing, because malloc functions
 108  * are meant to have hidden visibility. But since the functions are only
 109  * used locally in the zone allocator further below, we can allow the
 110  * compiler to optimize more by switching to static.
 111  */
 112 #ifdef XP_DARWIN
 113 #undef MOZ_MEMORY_API
 114 #define MOZ_MEMORY_API static
 115 #endif
 116
 117 /*
 118  * Malloc implementation functions are MOZ_MEMORY_API, and jemalloc
 119  * specific functions MOZ_JEMALLOC_API; see mozmemory_wrap.h
 120  */
 121 #define MALLOC_DECL(name, return_type, ...) \
 122   MOZ_MEMORY_API return_type name ## _impl(__VA_ARGS__);
 123 #define MALLOC_FUNCS MALLOC_FUNCS_MALLOC
 124 #include "malloc_decls.h"
 125
 126 #define MALLOC_DECL(name, return_type, ...) \
 127   MOZ_JEMALLOC_API return_type name ## _impl(__VA_ARGS__);
 128 #define MALLOC_FUNCS MALLOC_FUNCS_JEMALLOC
 129 #include "malloc_decls.h"
 130
 131 static int replace_malloc_initialized = 0;
 132 static void
 133 init()
 134 {
 135 #ifdef MOZ_NO_REPLACE_FUNC_DECL
 136   replace_malloc_init_funcs();
 137 #endif
 138   // Set this *before* calling replace_init, otherwise if replace_init calls
 139   // malloc() we'll get an infinite loop.
 140   replace_malloc_initialized = 1;
 141   if (replace_init)
 142     replace_init(&malloc_table);
 143 }
 144
 145 void*
 146 malloc_impl(size_t size)
 147 {
 148   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 149     init();
 150   if (MOZ_LIKELY(!replace_malloc))
 151     return je_malloc(size);
 152   return replace_malloc(size);
 153 }
 154
 155 int
 156 posix_memalign_impl(void **memptr, size_t alignment, size_t size)
 157 {
 158   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 159     init();
 160   if (MOZ_LIKELY(!replace_posix_memalign))
 161     return je_posix_memalign(memptr, alignment, size);
 162   return replace_posix_memalign(memptr, alignment, size);
 163 }
 164
 165 void*
 166 aligned_alloc_impl(size_t alignment, size_t size)
 167 {
 168   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 169     init();
 170   if (MOZ_LIKELY(!replace_aligned_alloc))
 171     return je_aligned_alloc(alignment, size);
 172   return replace_aligned_alloc(alignment, size);
 173 }
 174
 175 void*
 176 calloc_impl(size_t num, size_t size)
 177 {
 178   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 179     init();
 180   if (MOZ_LIKELY(!replace_calloc))
 181     return je_calloc(num, size);
 182   return replace_calloc(num, size);
 183 }
 184
 185 void*
 186 realloc_impl(void *ptr, size_t size)
 187 {
 188   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 189     init();
 190   if (MOZ_LIKELY(!replace_realloc))
 191     return je_realloc(ptr, size);
 192   return replace_realloc(ptr, size);
 193 }
 194
 195 void
 196 free_impl(void *ptr)
 197 {
 198   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 199     init();
 200   if (MOZ_LIKELY(!replace_free))
 201     je_free(ptr);
 202   else
 203     replace_free(ptr);
 204 }
 205
 206 void*
 207 memalign_impl(size_t alignment, size_t size)
 208 {
 209   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 210     init();
 211   if (MOZ_LIKELY(!replace_memalign))
 212     return je_memalign(alignment, size);
 213   return replace_memalign(alignment, size);
 214 }
 215
 216 void*
 217 valloc_impl(size_t size)
 218 {
 219   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 220     init();
 221   if (MOZ_LIKELY(!replace_valloc))
 222     return je_valloc(size);
 223   return replace_valloc(size);
 224 }
 225
 226 size_t
 227 malloc_usable_size_impl(usable_ptr_t ptr)
 228 {
 229   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 230     init();
 231   if (MOZ_LIKELY(!replace_malloc_usable_size))
 232     return je_malloc_usable_size(ptr);
 233   return replace_malloc_usable_size(ptr);
 234 }
 235
 236 size_t
 237 malloc_good_size_impl(size_t size)
 238 {
 239   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 240     init();
 241   if (MOZ_LIKELY(!replace_malloc_good_size))
 242     return je_malloc_good_size(size);
 243   return replace_malloc_good_size(size);
 244 }
 245
 246 void
 247 jemalloc_stats_impl(jemalloc_stats_t *stats)
 248 {
 249   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 250     init();
 251   if (MOZ_LIKELY(!replace_jemalloc_stats))
 252     je_jemalloc_stats(stats);
 253   else
 254     replace_jemalloc_stats(stats);
 255 }
 256
 257 void
 258 jemalloc_purge_freed_pages_impl()
 259 {
 260   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 261     init();
 262   if (MOZ_LIKELY(!replace_jemalloc_purge_freed_pages))
 263     je_jemalloc_purge_freed_pages();
 264   else
 265     replace_jemalloc_purge_freed_pages();
 266 }
 267
 268 void
 269 jemalloc_free_dirty_pages_impl()
 270 {
 271   if (MOZ_UNLIKELY(!replace_malloc_initialized))
 272     init();
 273   if (MOZ_LIKELY(!replace_jemalloc_free_dirty_pages))
 274     je_jemalloc_free_dirty_pages();
 275   else
 276     replace_jemalloc_free_dirty_pages();
 277 }
 278
 279 /* The following comment and definitions are from jemalloc.c: */
 280 #if defined(__GLIBC__) && !defined(__UCLIBC__)
 281
 282 /*
 283  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
 284  * to inconsistently reference libc's malloc(3)-compatible functions
 285  * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
 286  *
 287  * These definitions interpose hooks in glibc.  The functions are actually
 288  * passed an extra argument for the caller return address, which will be
 289  * ignored.
 290  */
 291
 292 typedef void (* __free_hook_type)(void *ptr);
 293 typedef void *(* __malloc_hook_type)(size_t size);
 294 typedef void *(* __realloc_hook_type)(void *ptr, size_t size);
 295 typedef void *(* __memalign_hook_type)(size_t alignment, size_t size);
 296
 297 MOZ_MEMORY_API __free_hook_type __free_hook = free_impl;
 298 MOZ_MEMORY_API __malloc_hook_type __malloc_hook = malloc_impl;
 299 MOZ_MEMORY_API __realloc_hook_type __realloc_hook = realloc_impl;
 300 MOZ_MEMORY_API __memalign_hook_type __memalign_hook = memalign_impl;
 301
 302 #endif
 303
 304 /*
 305  * The following is a OSX zone allocator implementation.
 306  * /!\ WARNING. It assumes the underlying malloc implementation's
 307  * malloc_usable_size returns 0 when the given pointer is not owned by
 308  * the allocator. Sadly, OSX does call zone_size with pointers not
 309  * owned by the allocator.
 310  */
 311
 312 #ifdef XP_DARWIN
 313 #include <stdlib.h>
 314 #include <malloc/malloc.h>
 315 #include "mozilla/Assertions.h"
 316
 317 static size_t
 318 zone_size(malloc_zone_t *zone, void *ptr)
 319 {
 320   return malloc_usable_size_impl(ptr);
 321 }
 322
 323 static void *
 324 zone_malloc(malloc_zone_t *zone, size_t size)
 325 {
 326   return malloc_impl(size);
 327 }
 328
 329 static void *
 330 zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
 331 {
 332   return calloc_impl(num, size);
 333 }
 334
 335 static void *
 336 zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
 337 {
 338   if (malloc_usable_size_impl(ptr))
 339     return realloc_impl(ptr, size);
 340   return realloc(ptr, size);
 341 }
 342
 343 static void
 344 zone_free(malloc_zone_t *zone, void *ptr)
 345 {
 346   if (malloc_usable_size_impl(ptr)) {
 347     free_impl(ptr);
 348     return;
 349   }
 350   free(ptr);
 351 }
 352
 353 static void
 354 zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
 355 {
 356   size_t current_size = malloc_usable_size_impl(ptr);
 357   if (current_size) {
 358     MOZ_ASSERT(current_size == size);
 359     free_impl(ptr);
 360     return;
 361   }
 362   free(ptr);
 363 }
 364
 365 static void *
 366 zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
 367 {
 368   void *ptr;
 369   if (posix_memalign_impl(&ptr, alignment, size) == 0)
 370     return ptr;
 371   return NULL;
 372 }
 373
 374 static void *
 375 zone_valloc(malloc_zone_t *zone, size_t size)
 376 {
 377   return valloc_impl(size);
 378 }
 379
 380 static void *
 381 zone_destroy(malloc_zone_t *zone)
 382 {
 383   /* This function should never be called. */
 384   MOZ_CRASH();
 385 }
 386
 387 static size_t
 388 zone_good_size(malloc_zone_t *zone, size_t size)
 389 {
 390   return malloc_good_size_impl(size);
 391 }
 392
 393 #ifdef MOZ_JEMALLOC
 394
 395 #include "jemalloc/internal/jemalloc_internal.h"
 396
 397 static void
 398 zone_force_lock(malloc_zone_t *zone)
 399 {
 400   /* /!\ This calls into jemalloc. It works because we're linked in the
 401    * same library. Stolen from jemalloc's zone.c. */
 402   if (isthreaded)
 403     jemalloc_prefork();
 404 }
 405
 406 static void
 407 zone_force_unlock(malloc_zone_t *zone)
 408 {
 409   /* /!\ This calls into jemalloc. It works because we're linked in the
 410    * same library. Stolen from jemalloc's zone.c. */
 411   if (isthreaded)
 412     jemalloc_postfork_parent();
 413 }
 414
 415 #else
 416
 417 #define JEMALLOC_ZONE_VERSION 6
 418
 419 /* Empty implementations are needed, because fork() calls zone->force_(un)lock
 420  * unconditionally. */
 421 static void
 422 zone_force_lock(malloc_zone_t *zone)
 423 {
 424 }
 425
 426 static void
 427 zone_force_unlock(malloc_zone_t *zone)
 428 {
 429 }
 430
 431 #endif
 432
 433 static malloc_zone_t zone;
 434 static struct malloc_introspection_t zone_introspect;
 435
 436 __attribute__((constructor)) void
 437 register_zone(void)
 438 {
 439   zone.size = (void *)zone_size;
 440   zone.malloc = (void *)zone_malloc;
 441   zone.calloc = (void *)zone_calloc;
 442   zone.valloc = (void *)zone_valloc;
 443   zone.free = (void *)zone_free;
 444   zone.realloc = (void *)zone_realloc;
 445   zone.destroy = (void *)zone_destroy;
 446   zone.zone_name = "replace_malloc_zone";
 447   zone.batch_malloc = NULL;
 448   zone.batch_free = NULL;
 449   zone.introspect = &zone_introspect;
 450   zone.version = JEMALLOC_ZONE_VERSION;
 451   zone.memalign = zone_memalign;
 452   zone.free_definite_size = zone_free_definite_size;
 453 #if (JEMALLOC_ZONE_VERSION >= 8)
 454   zone.pressure_relief = NULL;
 455 #endif
 456   zone_introspect.enumerator = NULL;
 457   zone_introspect.good_size = (void *)zone_good_size;
 458   zone_introspect.check = NULL;
 459   zone_introspect.print = NULL;
 460   zone_introspect.log = NULL;
 461   zone_introspect.force_lock = (void *)zone_force_lock;
 462   zone_introspect.force_unlock = (void *)zone_force_unlock;
 463   zone_introspect.statistics = NULL;
 464   zone_introspect.zone_locked = NULL;
 465 #if (JEMALLOC_ZONE_VERSION >= 7)
 466   zone_introspect.enable_discharge_checking = NULL;
 467   zone_introspect.disable_discharge_checking = NULL;
 468   zone_introspect.discharge = NULL;
 469 #ifdef __BLOCKS__
 470   zone_introspect.enumerate_discharged_pointers = NULL;
 471 #else
 472   zone_introspect.enumerate_unavailable_without_blocks = NULL;
 473 #endif
 474 #endif
 475
 476   /*
 477    * The default purgeable zone is created lazily by OSX's libc.  It uses
 478    * the default zone when it is created for "small" allocations
 479    * (< 15 KiB), but assumes the default zone is a scalable_zone.  This
 480    * obviously fails when the default zone is the jemalloc zone, so
 481    * malloc_default_purgeable_zone is called beforehand so that the
 482    * default purgeable zone is created when the default zone is still
 483    * a scalable_zone.
 484    */
 485   malloc_zone_t *purgeable_zone = malloc_default_purgeable_zone();
 486
 487   /* Register the custom zone.  At this point it won't be the default. */
 488   malloc_zone_register(&zone);
 489
 490   do {
 491     malloc_zone_t *default_zone = malloc_default_zone();
 492     /*
 493      * Unregister and reregister the default zone.  On OSX >= 10.6,
 494      * unregistering takes the last registered zone and places it at the
 495      * location of the specified zone.  Unregistering the default zone thus
 496      * makes the last registered one the default.  On OSX < 10.6,
 497      * unregistering shifts all registered zones.  The first registered zone
 498      * then becomes the default.
 499      */
 500     malloc_zone_unregister(default_zone);
 501     malloc_zone_register(default_zone);
 502     /*
 503      * On OSX 10.6, having the default purgeable zone appear before the default
 504      * zone makes some things crash because it thinks it owns the default
 505      * zone allocated pointers. We thus unregister/re-register it in order to
 506      * ensure it's always after the default zone. On OSX < 10.6, as
 507      * unregistering shifts registered zones, this simply removes the purgeable
 508      * zone from the list and adds it back at the end, after the default zone.
 509      * On OSX >= 10.6, unregistering replaces the purgeable zone with the last
 510      * registered zone above, i.e the default zone. Registering it again then
 511      * puts it at the end, obviously after the default zone.
 512      */
 513     malloc_zone_unregister(purgeable_zone);
 514     malloc_zone_register(purgeable_zone);
 515   } while (malloc_default_zone() != &zone);
 516 }
 517 #endif