libgomp/allocator.c

   1 /* Copyright (C) 2020-2023 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains wrappers for the system allocation routines.  Most
  27    places in the OpenMP API do not make any provision for failure, so in
  28    general we cannot allow memory allocation to fail.  */
  29
  30 #define _GNU_SOURCE
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
  35 #include <dlfcn.h>
  36 #endif
  37
  38 #define omp_max_predefined_alloc omp_thread_mem_alloc
  39
  40 /* These macros may be overridden in config/<target>/allocator.c.
  41    The following definitions (ab)use comma operators to avoid unused
  42    variable errors.  */
  43 #ifndef MEMSPACE_ALLOC
  44 #define MEMSPACE_ALLOC(MEMSPACE, SIZE) \
  45   malloc (((void)(MEMSPACE), (SIZE)))
  46 #endif
  47 #ifndef MEMSPACE_CALLOC
  48 #define MEMSPACE_CALLOC(MEMSPACE, SIZE) \
  49   calloc (1, (((void)(MEMSPACE), (SIZE))))
  50 #endif
  51 #ifndef MEMSPACE_REALLOC
  52 #define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE) \
  53   realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE))))
  54 #endif
  55 #ifndef MEMSPACE_FREE
  56 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE) \
  57   free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
  58 #endif
  59 #ifndef MEMSPACE_VALIDATE
  60 #define MEMSPACE_VALIDATE(MEMSPACE, ACCESS) \
  61   (((void)(MEMSPACE), (void)(ACCESS), 1))
  62 #endif
  63
  64 /* Map the predefined allocators to the correct memory space.
  65    The index to this table is the omp_allocator_handle_t enum value.
  66    When the user calls omp_alloc with a predefined allocator this
  67    table determines what memory they get.  */
  68 static const omp_memspace_handle_t predefined_alloc_mapping[] = {
  69   omp_default_mem_space,   /* omp_null_allocator doesn't actually use this. */
  70   omp_default_mem_space,   /* omp_default_mem_alloc. */
  71   omp_large_cap_mem_space, /* omp_large_cap_mem_alloc. */
  72   omp_const_mem_space,     /* omp_const_mem_alloc. */
  73   omp_high_bw_mem_space,   /* omp_high_bw_mem_alloc. */
  74   omp_low_lat_mem_space,   /* omp_low_lat_mem_alloc. */
  75   omp_low_lat_mem_space,   /* omp_cgroup_mem_alloc (implementation defined). */
  76   omp_low_lat_mem_space,   /* omp_pteam_mem_alloc (implementation defined). */
  77   omp_low_lat_mem_space,   /* omp_thread_mem_alloc (implementation defined). */
  78 };
  79
  80 #define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
  81 _Static_assert (ARRAY_SIZE (predefined_alloc_mapping)
  82                 == omp_max_predefined_alloc + 1,
  83                 "predefined_alloc_mapping must match omp_memspace_handle_t");
  84
  85 enum gomp_numa_memkind_kind
  86 {
  87   GOMP_MEMKIND_NONE = 0,
  88 #define GOMP_MEMKIND_KINDS \
  89   GOMP_MEMKIND_KIND (HBW_INTERLEAVE),           \
  90   GOMP_MEMKIND_KIND (HBW_PREFERRED),            \
  91   GOMP_MEMKIND_KIND (DAX_KMEM_ALL),             \
  92   GOMP_MEMKIND_KIND (DAX_KMEM),                 \
  93   GOMP_MEMKIND_KIND (INTERLEAVE),               \
  94   GOMP_MEMKIND_KIND (DEFAULT)
  95 #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
  96   GOMP_MEMKIND_KINDS,
  97 #undef GOMP_MEMKIND_KIND
  98   GOMP_MEMKIND_COUNT,
  99   GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT
 100 };
 101
 102 struct omp_allocator_data
 103 {
 104   omp_memspace_handle_t memspace;
 105   omp_uintptr_t alignment;
 106   omp_uintptr_t pool_size;
 107   omp_uintptr_t used_pool_size;
 108   omp_allocator_handle_t fb_data;
 109   unsigned int sync_hint : 8;
 110   unsigned int access : 8;
 111   unsigned int fallback : 8;
 112   unsigned int pinned : 1;
 113   unsigned int partition : 7;
 114 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 115   unsigned int memkind : 8;
 116 #endif
 117 #ifndef HAVE_SYNC_BUILTINS
 118   gomp_mutex_t lock;
 119 #endif
 120 };
 121
 122 struct omp_mem_header
 123 {
 124   void *ptr;
 125   size_t size;
 126   omp_allocator_handle_t allocator;
 127   void *pad;
 128 };
 129
 130 struct gomp_libnuma_data
 131 {
 132   void *numa_handle;
 133   void *(*numa_alloc_local) (size_t);
 134   void *(*numa_realloc) (void *, size_t, size_t);
 135   void (*numa_free) (void *, size_t);
 136 };
 137
 138 struct gomp_memkind_data
 139 {
 140   void *memkind_handle;
 141   void *(*memkind_malloc) (void *, size_t);
 142   void *(*memkind_calloc) (void *, size_t, size_t);
 143   void *(*memkind_realloc) (void *, void *, size_t);
 144   void (*memkind_free) (void *, void *);
 145   int (*memkind_check_available) (void *);
 146   void **kinds[GOMP_MEMKIND_COUNT];
 147 };
 148
 149 #ifdef LIBGOMP_USE_LIBNUMA
 150 static struct gomp_libnuma_data *libnuma_data;
 151 static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT;
 152
 153 static void
 154 gomp_init_libnuma (void)
 155 {
 156   void *handle = dlopen ("libnuma.so.1", RTLD_LAZY);
 157   struct gomp_libnuma_data *data;
 158
 159   data = calloc (1, sizeof (struct gomp_libnuma_data));
 160   if (data == NULL)
 161     {
 162       if (handle)
 163         dlclose (handle);
 164       return;
 165     }
 166   if (handle)
 167     {
 168       int (*numa_available) (void);
 169       numa_available
 170         = (__typeof (numa_available)) dlsym (handle, "numa_available");
 171       if (!numa_available || numa_available () != 0)
 172         {
 173           dlclose (handle);
 174           handle = NULL;
 175         }
 176     }
 177   if (!handle)
 178     {
 179       __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 180       return;
 181     }
 182   data->numa_handle = handle;
 183   data->numa_alloc_local
 184     = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local");
 185   data->numa_realloc
 186     = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc");
 187   data->numa_free
 188     = (__typeof (data->numa_free)) dlsym (handle, "numa_free");
 189   __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 190 }
 191
 192 static struct gomp_libnuma_data *
 193 gomp_get_libnuma (void)
 194 {
 195   struct gomp_libnuma_data *data
 196     = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 197   if (data)
 198     return data;
 199   pthread_once (&libnuma_data_once, gomp_init_libnuma);
 200   return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 201 }
 202 #endif
 203
 204 #ifdef LIBGOMP_USE_MEMKIND
 205 static struct gomp_memkind_data *memkind_data;
 206 static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
 207
 208 static void
 209 gomp_init_memkind (void)
 210 {
 211   void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY);
 212   struct gomp_memkind_data *data;
 213   int i;
 214   static const char *kinds[] = {
 215     NULL,
 216 #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
 217     GOMP_MEMKIND_KINDS
 218 #undef GOMP_MEMKIND_KIND
 219   };
 220
 221   data = calloc (1, sizeof (struct gomp_memkind_data));
 222   if (data == NULL)
 223     {
 224       if (handle)
 225         dlclose (handle);
 226       return;
 227     }
 228   if (!handle)
 229     {
 230       __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 231       return;
 232     }
 233   data->memkind_handle = handle;
 234   data->memkind_malloc
 235     = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
 236   data->memkind_calloc
 237     = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
 238   data->memkind_realloc
 239     = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
 240   data->memkind_free
 241     = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
 242   data->memkind_check_available
 243     = (__typeof (data->memkind_check_available))
 244       dlsym (handle, "memkind_check_available");
 245   if (data->memkind_malloc
 246       && data->memkind_calloc
 247       && data->memkind_realloc
 248       && data->memkind_free
 249       && data->memkind_check_available)
 250     for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
 251       {
 252         data->kinds[i] = (void **) dlsym (handle, kinds[i]);
 253         if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
 254           data->kinds[i] = NULL;
 255       }
 256   __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 257 }
 258
 259 static struct gomp_memkind_data *
 260 gomp_get_memkind (void)
 261 {
 262   struct gomp_memkind_data *data
 263     = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 264   if (data)
 265     return data;
 266   pthread_once (&memkind_data_once, gomp_init_memkind);
 267   return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 268 }
 269 #endif
 270
 271 omp_allocator_handle_t
 272 omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
 273                     const omp_alloctrait_t traits[])
 274 {
 275   struct omp_allocator_data data
 276     = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
 277         omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
 278 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 279         GOMP_MEMKIND_NONE
 280 #endif
 281       };
 282   struct omp_allocator_data *ret;
 283   int i;
 284
 285   if (memspace > omp_low_lat_mem_space)
 286     return omp_null_allocator;
 287   for (i = 0; i < ntraits; i++)
 288     switch (traits[i].key)
 289       {
 290       case omp_atk_sync_hint:
 291         switch (traits[i].value)
 292           {
 293           case omp_atv_default:
 294             data.sync_hint = omp_atv_contended;
 295             break;
 296           case omp_atv_contended:
 297           case omp_atv_uncontended:
 298           case omp_atv_serialized:
 299           case omp_atv_private:
 300             data.sync_hint = traits[i].value;
 301             break;
 302           default:
 303             return omp_null_allocator;
 304           }
 305         break;
 306       case omp_atk_alignment:
 307         if (traits[i].value == omp_atv_default)
 308           {
 309             data.alignment = 1;
 310             break;
 311           }
 312         if ((traits[i].value & (traits[i].value - 1)) != 0
 313             || !traits[i].value)
 314           return omp_null_allocator;
 315         data.alignment = traits[i].value;
 316         break;
 317       case omp_atk_access:
 318         switch (traits[i].value)
 319           {
 320           case omp_atv_default:
 321             data.access = omp_atv_all;
 322             break;
 323           case omp_atv_all:
 324           case omp_atv_cgroup:
 325           case omp_atv_pteam:
 326           case omp_atv_thread:
 327             data.access = traits[i].value;
 328             break;
 329           default:
 330             return omp_null_allocator;
 331           }
 332         break;
 333       case omp_atk_pool_size:
 334         if (traits[i].value == omp_atv_default)
 335           data.pool_size = ~(uintptr_t) 0;
 336         else
 337           data.pool_size = traits[i].value;
 338         break;
 339       case omp_atk_fallback:
 340         switch (traits[i].value)
 341           {
 342           case omp_atv_default:
 343             data.fallback = omp_atv_default_mem_fb;
 344             break;
 345           case omp_atv_default_mem_fb:
 346           case omp_atv_null_fb:
 347           case omp_atv_abort_fb:
 348           case omp_atv_allocator_fb:
 349             data.fallback = traits[i].value;
 350             break;
 351           default:
 352             return omp_null_allocator;
 353           }
 354         break;
 355       case omp_atk_fb_data:
 356         data.fb_data = traits[i].value;
 357         break;
 358       case omp_atk_pinned:
 359         switch (traits[i].value)
 360           {
 361           case omp_atv_default:
 362           case omp_atv_false:
 363             data.pinned = omp_atv_false;
 364             break;
 365           case omp_atv_true:
 366             data.pinned = omp_atv_true;
 367             break;
 368           default:
 369             return omp_null_allocator;
 370           }
 371         break;
 372       case omp_atk_partition:
 373         switch (traits[i].value)
 374           {
 375           case omp_atv_default:
 376             data.partition = omp_atv_environment;
 377             break;
 378           case omp_atv_environment:
 379           case omp_atv_nearest:
 380           case omp_atv_blocked:
 381           case omp_atv_interleaved:
 382             data.partition = traits[i].value;
 383             break;
 384           default:
 385             return omp_null_allocator;
 386           }
 387         break;
 388       default:
 389         return omp_null_allocator;
 390       }
 391
 392   if (data.alignment < sizeof (void *))
 393     data.alignment = sizeof (void *);
 394
 395   switch (memspace)
 396     {
 397 #ifdef LIBGOMP_USE_MEMKIND
 398     case omp_high_bw_mem_space:
 399       struct gomp_memkind_data *memkind_data;
 400       memkind_data = gomp_get_memkind ();
 401       if (data.partition == omp_atv_interleaved
 402           && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
 403         {
 404           data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
 405           break;
 406         }
 407       else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
 408         {
 409           data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
 410           break;
 411         }
 412       break;
 413     case omp_large_cap_mem_space:
 414       memkind_data = gomp_get_memkind ();
 415       if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
 416         data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 417       else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
 418         data.memkind = GOMP_MEMKIND_DAX_KMEM;
 419       break;
 420 #endif
 421     default:
 422 #ifdef LIBGOMP_USE_MEMKIND
 423       if (data.partition == omp_atv_interleaved)
 424         {
 425           memkind_data = gomp_get_memkind ();
 426           if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
 427             data.memkind = GOMP_MEMKIND_INTERLEAVE;
 428         }
 429 #endif
 430       break;
 431     }
 432
 433 #ifdef LIBGOMP_USE_LIBNUMA
 434   if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest)
 435     {
 436       libnuma_data = gomp_get_libnuma ();
 437       if (libnuma_data->numa_alloc_local != NULL)
 438         data.memkind = GOMP_MEMKIND_LIBNUMA;
 439     }
 440 #endif
 441
 442   /* No support for this so far.  */
 443   if (data.pinned)
 444     return omp_null_allocator;
 445
 446   /* Reject unsupported memory spaces.  */
 447   if (!MEMSPACE_VALIDATE (data.memspace, data.access))
 448     return omp_null_allocator;
 449
 450   ret = gomp_malloc (sizeof (struct omp_allocator_data));
 451   *ret = data;
 452 #ifndef HAVE_SYNC_BUILTINS
 453   gomp_mutex_init (&ret->lock);
 454 #endif
 455   return (omp_allocator_handle_t) ret;
 456 }
 457
 458 void
 459 omp_destroy_allocator (omp_allocator_handle_t allocator)
 460 {
 461   if (allocator != omp_null_allocator)
 462     {
 463 #ifndef HAVE_SYNC_BUILTINS
 464       gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
 465 #endif
 466       free ((void *) allocator);
 467     }
 468 }
 469
 470 ialias (omp_init_allocator)
 471 ialias (omp_destroy_allocator)
 472
 473 void *
 474 omp_aligned_alloc (size_t alignment, size_t size,
 475                    omp_allocator_handle_t allocator)
 476 {
 477   struct omp_allocator_data *allocator_data;
 478   size_t new_size, new_alignment;
 479   void *ptr, *ret;
 480 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 481   enum gomp_numa_memkind_kind memkind;
 482 #endif
 483
 484   if (__builtin_expect (size == 0, 0))
 485     return NULL;
 486
 487 retry:
 488   new_alignment = alignment;
 489   if (allocator == omp_null_allocator)
 490     {
 491       struct gomp_thread *thr = gomp_thread ();
 492       if (thr->ts.def_allocator == omp_null_allocator)
 493         thr->ts.def_allocator = gomp_def_allocator;
 494       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 495     }
 496
 497   if (allocator > omp_max_predefined_alloc)
 498     {
 499       allocator_data = (struct omp_allocator_data *) allocator;
 500       if (new_alignment < allocator_data->alignment)
 501         new_alignment = allocator_data->alignment;
 502 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 503       memkind = allocator_data->memkind;
 504 #endif
 505     }
 506   else
 507     {
 508       allocator_data = NULL;
 509       if (new_alignment < sizeof (void *))
 510         new_alignment = sizeof (void *);
 511 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 512       memkind = GOMP_MEMKIND_NONE;
 513 #endif
 514 #ifdef LIBGOMP_USE_MEMKIND
 515       if (allocator == omp_high_bw_mem_alloc)
 516         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 517       else if (allocator == omp_large_cap_mem_alloc)
 518         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 519       if (memkind)
 520         {
 521           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 522           if (!memkind_data->kinds[memkind])
 523             memkind = GOMP_MEMKIND_NONE;
 524         }
 525 #endif
 526     }
 527
 528   new_size = sizeof (struct omp_mem_header);
 529   if (new_alignment > sizeof (void *))
 530     new_size += new_alignment - sizeof (void *);
 531   if (__builtin_add_overflow (size, new_size, &new_size))
 532     goto fail;
 533 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
 534   if (allocator == omp_low_lat_mem_alloc)
 535     goto fail;
 536 #endif
 537
 538   if (__builtin_expect (allocator_data
 539                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 540     {
 541       uintptr_t used_pool_size;
 542       if (new_size > allocator_data->pool_size)
 543         goto fail;
 544 #ifdef HAVE_SYNC_BUILTINS
 545       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 546                                         MEMMODEL_RELAXED);
 547       do
 548         {
 549           uintptr_t new_pool_size;
 550           if (__builtin_add_overflow (used_pool_size, new_size,
 551                                       &new_pool_size)
 552               || new_pool_size > allocator_data->pool_size)
 553             goto fail;
 554           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 555                                            &used_pool_size, new_pool_size,
 556                                            true, MEMMODEL_RELAXED,
 557                                            MEMMODEL_RELAXED))
 558             break;
 559         }
 560       while (1);
 561 #else
 562       gomp_mutex_lock (&allocator_data->lock);
 563       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 564                                   &used_pool_size)
 565           || used_pool_size > allocator_data->pool_size)
 566         {
 567           gomp_mutex_unlock (&allocator_data->lock);
 568           goto fail;
 569         }
 570       allocator_data->used_pool_size = used_pool_size;
 571       gomp_mutex_unlock (&allocator_data->lock);
 572 #endif
 573 #ifdef LIBGOMP_USE_LIBNUMA
 574       if (memkind == GOMP_MEMKIND_LIBNUMA)
 575         ptr = libnuma_data->numa_alloc_local (new_size);
 576 # ifdef LIBGOMP_USE_MEMKIND
 577       else
 578 # endif
 579 #endif
 580 #ifdef LIBGOMP_USE_MEMKIND
 581       if (memkind)
 582         {
 583           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 584           void *kind = *memkind_data->kinds[memkind];
 585           ptr = memkind_data->memkind_malloc (kind, new_size);
 586         }
 587       else
 588 #endif
 589         ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size);
 590       if (ptr == NULL)
 591         {
 592 #ifdef HAVE_SYNC_BUILTINS
 593           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 594                               MEMMODEL_RELAXED);
 595 #else
 596           gomp_mutex_lock (&allocator_data->lock);
 597           allocator_data->used_pool_size -= new_size;
 598           gomp_mutex_unlock (&allocator_data->lock);
 599 #endif
 600           goto fail;
 601         }
 602     }
 603   else
 604     {
 605 #ifdef LIBGOMP_USE_LIBNUMA
 606       if (memkind == GOMP_MEMKIND_LIBNUMA)
 607         ptr = libnuma_data->numa_alloc_local (new_size);
 608 # ifdef LIBGOMP_USE_MEMKIND
 609       else
 610 # endif
 611 #endif
 612 #ifdef LIBGOMP_USE_MEMKIND
 613       if (memkind)
 614         {
 615           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 616           void *kind = *memkind_data->kinds[memkind];
 617           ptr = memkind_data->memkind_malloc (kind, new_size);
 618         }
 619       else
 620 #endif
 621         {
 622           omp_memspace_handle_t memspace;
 623           memspace = (allocator_data
 624                       ? allocator_data->memspace
 625                       : predefined_alloc_mapping[allocator]);
 626           ptr = MEMSPACE_ALLOC (memspace, new_size);
 627         }
 628       if (ptr == NULL)
 629         goto fail;
 630     }
 631
 632   if (new_alignment > sizeof (void *))
 633     ret = (void *) (((uintptr_t) ptr
 634                      + sizeof (struct omp_mem_header)
 635                      + new_alignment - sizeof (void *))
 636                     & ~(new_alignment - 1));
 637   else
 638     ret = (char *) ptr + sizeof (struct omp_mem_header);
 639   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 640   ((struct omp_mem_header *) ret)[-1].size = new_size;
 641   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 642   return ret;
 643
 644 fail:;
 645   int fallback = (allocator_data
 646                   ? allocator_data->fallback
 647                   : allocator == omp_default_mem_alloc
 648                   ? omp_atv_null_fb
 649                   : omp_atv_default_mem_fb);
 650   switch (fallback)
 651     {
 652     case omp_atv_default_mem_fb:
 653       allocator = omp_default_mem_alloc;
 654       goto retry;
 655     case omp_atv_null_fb:
 656       break;
 657     default:
 658     case omp_atv_abort_fb:
 659       gomp_fatal ("Out of memory allocating %lu bytes",
 660                   (unsigned long) size);
 661     case omp_atv_allocator_fb:
 662       allocator = allocator_data->fb_data;
 663       goto retry;
 664     }
 665   return NULL;
 666 }
 667
 668 ialias (omp_aligned_alloc)
 669
 670 void *
 671 omp_alloc (size_t size, omp_allocator_handle_t allocator)
 672 {
 673   return ialias_call (omp_aligned_alloc) (1, size, allocator);
 674 }
 675
 676 /* Like omp_aligned_alloc, but apply on top of that:
 677    "For allocations that arise from this ... the null_fb value of the
 678    fallback allocator trait behaves as if the abort_fb had been specified."  */
 679
 680 void *
 681 GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
 682 {
 683   void *ret
 684     = ialias_call (omp_aligned_alloc) (alignment, size,
 685                                        (omp_allocator_handle_t) allocator);
 686   if (__builtin_expect (ret == NULL, 0) && size)
 687     gomp_fatal ("Out of memory allocating %lu bytes",
 688                 (unsigned long) size);
 689   return ret;
 690 }
 691
 692 void
 693 omp_free (void *ptr, omp_allocator_handle_t allocator)
 694 {
 695   struct omp_mem_header *data;
 696   omp_memspace_handle_t memspace = omp_default_mem_space;
 697
 698   if (ptr == NULL)
 699     return;
 700   (void) allocator;
 701   data = &((struct omp_mem_header *) ptr)[-1];
 702   if (data->allocator > omp_max_predefined_alloc)
 703     {
 704       struct omp_allocator_data *allocator_data
 705         = (struct omp_allocator_data *) (data->allocator);
 706       if (allocator_data->pool_size < ~(uintptr_t) 0)
 707         {
 708 #ifdef HAVE_SYNC_BUILTINS
 709           __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
 710                               MEMMODEL_RELAXED);
 711 #else
 712           gomp_mutex_lock (&allocator_data->lock);
 713           allocator_data->used_pool_size -= data->size;
 714           gomp_mutex_unlock (&allocator_data->lock);
 715 #endif
 716         }
 717 #ifdef LIBGOMP_USE_LIBNUMA
 718       if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA)
 719         {
 720           libnuma_data->numa_free (data->ptr, data->size);
 721           return;
 722         }
 723 # ifdef LIBGOMP_USE_MEMKIND
 724       else
 725 # endif
 726 #endif
 727 #ifdef LIBGOMP_USE_MEMKIND
 728       if (allocator_data->memkind)
 729         {
 730           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 731           void *kind = *memkind_data->kinds[allocator_data->memkind];
 732           memkind_data->memkind_free (kind, data->ptr);
 733           return;
 734         }
 735 #endif
 736
 737       memspace = allocator_data->memspace;
 738     }
 739   else
 740     {
 741 #ifdef LIBGOMP_USE_MEMKIND
 742       enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE;
 743       if (data->allocator == omp_high_bw_mem_alloc)
 744         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 745       else if (data->allocator == omp_large_cap_mem_alloc)
 746         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 747       if (memkind)
 748         {
 749           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 750           if (memkind_data->kinds[memkind])
 751             {
 752               void *kind = *memkind_data->kinds[memkind];
 753               memkind_data->memkind_free (kind, data->ptr);
 754               return;
 755             }
 756         }
 757 #endif
 758
 759       memspace = predefined_alloc_mapping[data->allocator];
 760     }
 761
 762   MEMSPACE_FREE (memspace, data->ptr, data->size);
 763 }
 764
 765 ialias (omp_free)
 766
 767 void
 768 GOMP_free (void *ptr, uintptr_t allocator)
 769 {
 770   return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
 771 }
 772
 773 void *
 774 omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
 775                     omp_allocator_handle_t allocator)
 776 {
 777   struct omp_allocator_data *allocator_data;
 778   size_t new_size, size_temp, new_alignment;
 779   void *ptr, *ret;
 780 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 781   enum gomp_numa_memkind_kind memkind;
 782 #endif
 783
 784   if (__builtin_expect (size == 0 || nmemb == 0, 0))
 785     return NULL;
 786
 787 retry:
 788   new_alignment = alignment;
 789   if (allocator == omp_null_allocator)
 790     {
 791       struct gomp_thread *thr = gomp_thread ();
 792       if (thr->ts.def_allocator == omp_null_allocator)
 793         thr->ts.def_allocator = gomp_def_allocator;
 794       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 795     }
 796
 797   if (allocator > omp_max_predefined_alloc)
 798     {
 799       allocator_data = (struct omp_allocator_data *) allocator;
 800       if (new_alignment < allocator_data->alignment)
 801         new_alignment = allocator_data->alignment;
 802 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 803       memkind = allocator_data->memkind;
 804 #endif
 805     }
 806   else
 807     {
 808       allocator_data = NULL;
 809       if (new_alignment < sizeof (void *))
 810         new_alignment = sizeof (void *);
 811 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 812       memkind = GOMP_MEMKIND_NONE;
 813 #endif
 814 #ifdef LIBGOMP_USE_MEMKIND
 815       if (allocator == omp_high_bw_mem_alloc)
 816         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 817       else if (allocator == omp_large_cap_mem_alloc)
 818         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 819       if (memkind)
 820         {
 821           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 822           if (!memkind_data->kinds[memkind])
 823             memkind = GOMP_MEMKIND_NONE;
 824         }
 825 #endif
 826     }
 827
 828   new_size = sizeof (struct omp_mem_header);
 829   if (new_alignment > sizeof (void *))
 830     new_size += new_alignment - sizeof (void *);
 831   if (__builtin_mul_overflow (size, nmemb, &size_temp))
 832     goto fail;
 833   if (__builtin_add_overflow (size_temp, new_size, &new_size))
 834     goto fail;
 835 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
 836   if (allocator == omp_low_lat_mem_alloc)
 837     goto fail;
 838 #endif
 839
 840   if (__builtin_expect (allocator_data
 841                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 842     {
 843       uintptr_t used_pool_size;
 844       if (new_size > allocator_data->pool_size)
 845         goto fail;
 846 #ifdef HAVE_SYNC_BUILTINS
 847       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 848                                         MEMMODEL_RELAXED);
 849       do
 850         {
 851           uintptr_t new_pool_size;
 852           if (__builtin_add_overflow (used_pool_size, new_size,
 853                                       &new_pool_size)
 854               || new_pool_size > allocator_data->pool_size)
 855             goto fail;
 856           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 857                                            &used_pool_size, new_pool_size,
 858                                            true, MEMMODEL_RELAXED,
 859                                            MEMMODEL_RELAXED))
 860             break;
 861         }
 862       while (1);
 863 #else
 864       gomp_mutex_lock (&allocator_data->lock);
 865       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 866                                   &used_pool_size)
 867           || used_pool_size > allocator_data->pool_size)
 868         {
 869           gomp_mutex_unlock (&allocator_data->lock);
 870           goto fail;
 871         }
 872       allocator_data->used_pool_size = used_pool_size;
 873       gomp_mutex_unlock (&allocator_data->lock);
 874 #endif
 875 #ifdef LIBGOMP_USE_LIBNUMA
 876       if (memkind == GOMP_MEMKIND_LIBNUMA)
 877         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 878            memory that is initialized to zero.  */
 879         ptr = libnuma_data->numa_alloc_local (new_size);
 880 # ifdef LIBGOMP_USE_MEMKIND
 881       else
 882 # endif
 883 #endif
 884 #ifdef LIBGOMP_USE_MEMKIND
 885       if (memkind)
 886         {
 887           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 888           void *kind = *memkind_data->kinds[memkind];
 889           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 890         }
 891       else
 892 #endif
 893         ptr = MEMSPACE_CALLOC (allocator_data->memspace, new_size);
 894       if (ptr == NULL)
 895         {
 896 #ifdef HAVE_SYNC_BUILTINS
 897           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 898                               MEMMODEL_RELAXED);
 899 #else
 900           gomp_mutex_lock (&allocator_data->lock);
 901           allocator_data->used_pool_size -= new_size;
 902           gomp_mutex_unlock (&allocator_data->lock);
 903 #endif
 904           goto fail;
 905         }
 906     }
 907   else
 908     {
 909 #ifdef LIBGOMP_USE_LIBNUMA
 910       if (memkind == GOMP_MEMKIND_LIBNUMA)
 911         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 912            memory that is initialized to zero.  */
 913         ptr = libnuma_data->numa_alloc_local (new_size);
 914 # ifdef LIBGOMP_USE_MEMKIND
 915       else
 916 # endif
 917 #endif
 918 #ifdef LIBGOMP_USE_MEMKIND
 919       if (memkind)
 920         {
 921           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 922           void *kind = *memkind_data->kinds[memkind];
 923           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 924         }
 925       else
 926 #endif
 927         {
 928           omp_memspace_handle_t memspace;
 929           memspace = (allocator_data
 930                       ? allocator_data->memspace
 931                       : predefined_alloc_mapping[allocator]);
 932           ptr = MEMSPACE_CALLOC (memspace, new_size);
 933         }
 934       if (ptr == NULL)
 935         goto fail;
 936     }
 937
 938   if (new_alignment > sizeof (void *))
 939     ret = (void *) (((uintptr_t) ptr
 940                      + sizeof (struct omp_mem_header)
 941                      + new_alignment - sizeof (void *))
 942                     & ~(new_alignment - 1));
 943   else
 944     ret = (char *) ptr + sizeof (struct omp_mem_header);
 945   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 946   ((struct omp_mem_header *) ret)[-1].size = new_size;
 947   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 948   return ret;
 949
 950 fail:;
 951   int fallback = (allocator_data
 952                   ? allocator_data->fallback
 953                   : allocator == omp_default_mem_alloc
 954                   ? omp_atv_null_fb
 955                   : omp_atv_default_mem_fb);
 956   switch (fallback)
 957     {
 958     case omp_atv_default_mem_fb:
 959       allocator = omp_default_mem_alloc;
 960       goto retry;
 961     case omp_atv_null_fb:
 962       break;
 963     default:
 964     case omp_atv_abort_fb:
 965       gomp_fatal ("Out of memory allocating %lu bytes",
 966                   (unsigned long) (size * nmemb));
 967     case omp_atv_allocator_fb:
 968       allocator = allocator_data->fb_data;
 969       goto retry;
 970     }
 971   return NULL;
 972 }
 973
 974 ialias (omp_aligned_calloc)
 975
 976 void *
 977 omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
 978 {
 979   return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
 980 }
 981
 982 void *
 983 omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
 984              omp_allocator_handle_t free_allocator)
 985 {
 986   struct omp_allocator_data *allocator_data, *free_allocator_data;
 987   size_t new_size, old_size, new_alignment, old_alignment;
 988   void *new_ptr, *ret;
 989   struct omp_mem_header *data;
 990 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 991   enum gomp_numa_memkind_kind memkind, free_memkind;
 992 #endif
 993
 994   if (__builtin_expect (ptr == NULL, 0))
 995     return ialias_call (omp_aligned_alloc) (1, size, allocator);
 996
 997   if (__builtin_expect (size == 0, 0))
 998     {
 999       ialias_call (omp_free) (ptr, free_allocator);
1000       return NULL;
1001     }
1002
1003   data = &((struct omp_mem_header *) ptr)[-1];
1004   free_allocator = data->allocator;
1005
1006 retry:
1007   new_alignment = sizeof (void *);
1008   if (allocator == omp_null_allocator)
1009     allocator = free_allocator;
1010
1011   if (allocator > omp_max_predefined_alloc)
1012     {
1013       allocator_data = (struct omp_allocator_data *) allocator;
1014       if (new_alignment < allocator_data->alignment)
1015         new_alignment = allocator_data->alignment;
1016 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1017       memkind = allocator_data->memkind;
1018 #endif
1019     }
1020   else
1021     {
1022       allocator_data = NULL;
1023 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1024       memkind = GOMP_MEMKIND_NONE;
1025 #endif
1026 #ifdef LIBGOMP_USE_MEMKIND
1027       if (allocator == omp_high_bw_mem_alloc)
1028         memkind = GOMP_MEMKIND_HBW_PREFERRED;
1029       else if (allocator == omp_large_cap_mem_alloc)
1030         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1031       if (memkind)
1032         {
1033           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1034           if (!memkind_data->kinds[memkind])
1035             memkind = GOMP_MEMKIND_NONE;
1036         }
1037 #endif
1038     }
1039   if (free_allocator > omp_max_predefined_alloc)
1040     {
1041       free_allocator_data = (struct omp_allocator_data *) free_allocator;
1042 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1043       free_memkind = free_allocator_data->memkind;
1044 #endif
1045     }
1046   else
1047     {
1048       free_allocator_data = NULL;
1049 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1050       free_memkind = GOMP_MEMKIND_NONE;
1051 #endif
1052 #ifdef LIBGOMP_USE_MEMKIND
1053       if (free_allocator == omp_high_bw_mem_alloc)
1054         free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
1055       else if (free_allocator == omp_large_cap_mem_alloc)
1056         free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
1057       if (free_memkind)
1058         {
1059           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1060           if (!memkind_data->kinds[free_memkind])
1061             free_memkind = GOMP_MEMKIND_NONE;
1062         }
1063 #endif
1064     }
1065   old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
1066
1067   new_size = sizeof (struct omp_mem_header);
1068   if (new_alignment > sizeof (void *))
1069     new_size += new_alignment - sizeof (void *);
1070   if (__builtin_add_overflow (size, new_size, &new_size))
1071     goto fail;
1072   old_size = data->size;
1073 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
1074   if (allocator == omp_low_lat_mem_alloc)
1075     goto fail;
1076 #endif
1077
1078   if (__builtin_expect (allocator_data
1079                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
1080     {
1081       uintptr_t used_pool_size;
1082       size_t prev_size = 0;
1083       /* Check if we can use realloc.  Don't use it if extra alignment
1084          was used previously or newly, because realloc might return a pointer
1085          with different alignment and then we'd need to memmove the data
1086          again.  */
1087       if (free_allocator_data
1088           && free_allocator_data == allocator_data
1089           && new_alignment == sizeof (void *)
1090           && old_alignment == sizeof (struct omp_mem_header))
1091         prev_size = old_size;
1092       if (new_size > prev_size
1093           && new_size - prev_size > allocator_data->pool_size)
1094         goto fail;
1095 #ifdef HAVE_SYNC_BUILTINS
1096       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
1097                                         MEMMODEL_RELAXED);
1098       do
1099         {
1100           uintptr_t new_pool_size;
1101           if (new_size > prev_size)
1102             {
1103               if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
1104                                           &new_pool_size)
1105                   || new_pool_size > allocator_data->pool_size)
1106                 goto fail;
1107             }
1108           else
1109             new_pool_size = used_pool_size + new_size - prev_size;
1110           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
1111                                            &used_pool_size, new_pool_size,
1112                                            true, MEMMODEL_RELAXED,
1113                                            MEMMODEL_RELAXED))
1114             break;
1115         }
1116       while (1);
1117 #else
1118       gomp_mutex_lock (&allocator_data->lock);
1119       if (new_size > prev_size)
1120         {
1121           if (__builtin_add_overflow (allocator_data->used_pool_size,
1122                                       new_size - prev_size,
1123                                       &used_pool_size)
1124               || used_pool_size > allocator_data->pool_size)
1125             {
1126               gomp_mutex_unlock (&allocator_data->lock);
1127               goto fail;
1128             }
1129         }
1130       else
1131         used_pool_size = (allocator_data->used_pool_size
1132                           + new_size - prev_size);
1133       allocator_data->used_pool_size = used_pool_size;
1134       gomp_mutex_unlock (&allocator_data->lock);
1135 #endif
1136 #ifdef LIBGOMP_USE_LIBNUMA
1137       if (memkind == GOMP_MEMKIND_LIBNUMA)
1138         {
1139           if (prev_size)
1140             new_ptr = libnuma_data->numa_realloc (data->ptr, data->size,
1141                                                   new_size);
1142           else
1143             new_ptr = libnuma_data->numa_alloc_local (new_size);
1144         }
1145 # ifdef LIBGOMP_USE_MEMKIND
1146       else
1147 # endif
1148 #endif
1149 #ifdef LIBGOMP_USE_MEMKIND
1150       if (memkind)
1151         {
1152           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1153           void *kind = *memkind_data->kinds[memkind];
1154           if (prev_size)
1155             new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1156                                                      new_size);
1157           else
1158             new_ptr = memkind_data->memkind_malloc (kind, new_size);
1159         }
1160       else
1161 #endif
1162       if (prev_size)
1163         new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
1164                                     data->size, new_size);
1165       else
1166         new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size);
1167       if (new_ptr == NULL)
1168         {
1169 #ifdef HAVE_SYNC_BUILTINS
1170           __atomic_add_fetch (&allocator_data->used_pool_size,
1171                               prev_size - new_size,
1172                               MEMMODEL_RELAXED);
1173 #else
1174           gomp_mutex_lock (&allocator_data->lock);
1175           allocator_data->used_pool_size -= new_size - prev_size;
1176           gomp_mutex_unlock (&allocator_data->lock);
1177 #endif
1178           goto fail;
1179         }
1180       else if (prev_size)
1181         {
1182           ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1183           ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1184           ((struct omp_mem_header *) ret)[-1].size = new_size;
1185           ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1186           return ret;
1187         }
1188     }
1189   else if (new_alignment == sizeof (void *)
1190            && old_alignment == sizeof (struct omp_mem_header)
1191 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1192            && memkind == free_memkind
1193 #endif
1194            && (free_allocator_data == NULL
1195                || free_allocator_data->pool_size == ~(uintptr_t) 0))
1196     {
1197 #ifdef LIBGOMP_USE_LIBNUMA
1198       if (memkind == GOMP_MEMKIND_LIBNUMA)
1199         new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size);
1200 # ifdef LIBGOMP_USE_MEMKIND
1201       else
1202 # endif
1203 #endif
1204 #ifdef LIBGOMP_USE_MEMKIND
1205       if (memkind)
1206         {
1207           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1208           void *kind = *memkind_data->kinds[memkind];
1209           new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1210                                                    new_size);
1211         }
1212       else
1213 #endif
1214         {
1215           omp_memspace_handle_t memspace;
1216           memspace = (allocator_data
1217                       ? allocator_data->memspace
1218                       : predefined_alloc_mapping[allocator]);
1219           new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size);
1220         }
1221       if (new_ptr == NULL)
1222         goto fail;
1223       ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1224       ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1225       ((struct omp_mem_header *) ret)[-1].size = new_size;
1226       ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1227       return ret;
1228     }
1229   else
1230     {
1231 #ifdef LIBGOMP_USE_LIBNUMA
1232       if (memkind == GOMP_MEMKIND_LIBNUMA)
1233         new_ptr = libnuma_data->numa_alloc_local (new_size);
1234 # ifdef LIBGOMP_USE_MEMKIND
1235       else
1236 # endif
1237 #endif
1238 #ifdef LIBGOMP_USE_MEMKIND
1239       if (memkind)
1240         {
1241           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1242           void *kind = *memkind_data->kinds[memkind];
1243           new_ptr = memkind_data->memkind_malloc (kind, new_size);
1244         }
1245       else
1246 #endif
1247         {
1248           omp_memspace_handle_t memspace;
1249           memspace = (allocator_data
1250                       ? allocator_data->memspace
1251                       : predefined_alloc_mapping[allocator]);
1252           new_ptr = MEMSPACE_ALLOC (memspace, new_size);
1253         }
1254       if (new_ptr == NULL)
1255         goto fail;
1256     }
1257
1258   if (new_alignment > sizeof (void *))
1259     ret = (void *) (((uintptr_t) new_ptr
1260                      + sizeof (struct omp_mem_header)
1261                      + new_alignment - sizeof (void *))
1262                     & ~(new_alignment - 1));
1263   else
1264     ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1265   ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1266   ((struct omp_mem_header *) ret)[-1].size = new_size;
1267   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1268   if (old_size - old_alignment < size)
1269     size = old_size - old_alignment;
1270   memcpy (ret, ptr, size);
1271   if (__builtin_expect (free_allocator_data
1272                         && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1273     {
1274 #ifdef HAVE_SYNC_BUILTINS
1275       __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1276                           MEMMODEL_RELAXED);
1277 #else
1278       gomp_mutex_lock (&free_allocator_data->lock);
1279       free_allocator_data->used_pool_size -= data->size;
1280       gomp_mutex_unlock (&free_allocator_data->lock);
1281 #endif
1282     }
1283 #ifdef LIBGOMP_USE_LIBNUMA
1284   if (free_memkind == GOMP_MEMKIND_LIBNUMA)
1285     {
1286       libnuma_data->numa_free (data->ptr, data->size);
1287       return ret;
1288     }
1289 # ifdef LIBGOMP_USE_MEMKIND
1290   else
1291 # endif
1292 #endif
1293 #ifdef LIBGOMP_USE_MEMKIND
1294   if (free_memkind)
1295     {
1296       struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1297       void *kind = *memkind_data->kinds[free_memkind];
1298       memkind_data->memkind_free (kind, data->ptr);
1299       return ret;
1300     }
1301 #endif
1302   {
1303     omp_memspace_handle_t was_memspace;
1304     was_memspace = (free_allocator_data
1305                     ? free_allocator_data->memspace
1306                     : predefined_alloc_mapping[free_allocator]);
1307     MEMSPACE_FREE (was_memspace, data->ptr, data->size);
1308   }
1309   return ret;
1310
1311 fail:;
1312   int fallback = (allocator_data
1313                   ? allocator_data->fallback
1314                   : allocator == omp_default_mem_alloc
1315                   ? omp_atv_null_fb
1316                   : omp_atv_default_mem_fb);
1317   switch (fallback)
1318     {
1319     case omp_atv_default_mem_fb:
1320       allocator = omp_default_mem_alloc;
1321       goto retry;
1322     case omp_atv_null_fb:
1323       break;
1324     default:
1325     case omp_atv_abort_fb:
1326       gomp_fatal ("Out of memory allocating %lu bytes",
1327                   (unsigned long) size);
1328     case omp_atv_allocator_fb:
1329       allocator = allocator_data->fb_data;
1330       goto retry;
1331     }
1332   return NULL;
1333 }