libgomp/allocator.c

   1 /* Copyright (C) 2020-2023 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains wrappers for the system allocation routines.  Most
  27    places in the OpenMP API do not make any provision for failure, so in
  28    general we cannot allow memory allocation to fail.  */
  29
  30 #define _GNU_SOURCE
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
  35 #include <dlfcn.h>
  36 #endif
  37
  38 #define omp_max_predefined_alloc omp_thread_mem_alloc
  39
  40 enum gomp_numa_memkind_kind
  41 {
  42   GOMP_MEMKIND_NONE = 0,
  43 #define GOMP_MEMKIND_KINDS \
  44   GOMP_MEMKIND_KIND (HBW_INTERLEAVE),           \
  45   GOMP_MEMKIND_KIND (HBW_PREFERRED),            \
  46   GOMP_MEMKIND_KIND (DAX_KMEM_ALL),             \
  47   GOMP_MEMKIND_KIND (DAX_KMEM),                 \
  48   GOMP_MEMKIND_KIND (INTERLEAVE),               \
  49   GOMP_MEMKIND_KIND (DEFAULT)
  50 #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
  51   GOMP_MEMKIND_KINDS,
  52 #undef GOMP_MEMKIND_KIND
  53   GOMP_MEMKIND_COUNT,
  54   GOMP_MEMKIND_LIBNUMA = GOMP_MEMKIND_COUNT
  55 };
  56
  57 struct omp_allocator_data
  58 {
  59   omp_memspace_handle_t memspace;
  60   omp_uintptr_t alignment;
  61   omp_uintptr_t pool_size;
  62   omp_uintptr_t used_pool_size;
  63   omp_allocator_handle_t fb_data;
  64   unsigned int sync_hint : 8;
  65   unsigned int access : 8;
  66   unsigned int fallback : 8;
  67   unsigned int pinned : 1;
  68   unsigned int partition : 7;
  69 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
  70   unsigned int memkind : 8;
  71 #endif
  72 #ifndef HAVE_SYNC_BUILTINS
  73   gomp_mutex_t lock;
  74 #endif
  75 };
  76
  77 struct omp_mem_header
  78 {
  79   void *ptr;
  80   size_t size;
  81   omp_allocator_handle_t allocator;
  82   void *pad;
  83 };
  84
  85 struct gomp_libnuma_data
  86 {
  87   void *numa_handle;
  88   void *(*numa_alloc_local) (size_t);
  89   void *(*numa_realloc) (void *, size_t, size_t);
  90   void (*numa_free) (void *, size_t);
  91 };
  92
  93 struct gomp_memkind_data
  94 {
  95   void *memkind_handle;
  96   void *(*memkind_malloc) (void *, size_t);
  97   void *(*memkind_calloc) (void *, size_t, size_t);
  98   void *(*memkind_realloc) (void *, void *, size_t);
  99   void (*memkind_free) (void *, void *);
 100   int (*memkind_check_available) (void *);
 101   void **kinds[GOMP_MEMKIND_COUNT];
 102 };
 103
 104 #ifdef LIBGOMP_USE_LIBNUMA
 105 static struct gomp_libnuma_data *libnuma_data;
 106 static pthread_once_t libnuma_data_once = PTHREAD_ONCE_INIT;
 107
 108 static void
 109 gomp_init_libnuma (void)
 110 {
 111   void *handle = dlopen ("libnuma.so.1", RTLD_LAZY);
 112   struct gomp_libnuma_data *data;
 113
 114   data = calloc (1, sizeof (struct gomp_libnuma_data));
 115   if (data == NULL)
 116     {
 117       if (handle)
 118         dlclose (handle);
 119       return;
 120     }
 121   if (!handle)
 122     {
 123       __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 124       return;
 125     }
 126   data->numa_handle = handle;
 127   data->numa_alloc_local
 128     = (__typeof (data->numa_alloc_local)) dlsym (handle, "numa_alloc_local");
 129   data->numa_realloc
 130     = (__typeof (data->numa_realloc)) dlsym (handle, "numa_realloc");
 131   data->numa_free
 132     = (__typeof (data->numa_free)) dlsym (handle, "numa_free");
 133   __atomic_store_n (&libnuma_data, data, MEMMODEL_RELEASE);
 134 }
 135
 136 static struct gomp_libnuma_data *
 137 gomp_get_libnuma (void)
 138 {
 139   struct gomp_libnuma_data *data
 140     = __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 141   if (data)
 142     return data;
 143   pthread_once (&libnuma_data_once, gomp_init_libnuma);
 144   return __atomic_load_n (&libnuma_data, MEMMODEL_ACQUIRE);
 145 }
 146 #endif
 147
 148 #ifdef LIBGOMP_USE_MEMKIND
 149 static struct gomp_memkind_data *memkind_data;
 150 static pthread_once_t memkind_data_once = PTHREAD_ONCE_INIT;
 151
 152 static void
 153 gomp_init_memkind (void)
 154 {
 155   void *handle = dlopen ("libmemkind.so.0", RTLD_LAZY);
 156   struct gomp_memkind_data *data;
 157   int i;
 158   static const char *kinds[] = {
 159     NULL,
 160 #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
 161     GOMP_MEMKIND_KINDS
 162 #undef GOMP_MEMKIND_KIND
 163   };
 164
 165   data = calloc (1, sizeof (struct gomp_memkind_data));
 166   if (data == NULL)
 167     {
 168       if (handle)
 169         dlclose (handle);
 170       return;
 171     }
 172   if (!handle)
 173     {
 174       __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 175       return;
 176     }
 177   data->memkind_handle = handle;
 178   data->memkind_malloc
 179     = (__typeof (data->memkind_malloc)) dlsym (handle, "memkind_malloc");
 180   data->memkind_calloc
 181     = (__typeof (data->memkind_calloc)) dlsym (handle, "memkind_calloc");
 182   data->memkind_realloc
 183     = (__typeof (data->memkind_realloc)) dlsym (handle, "memkind_realloc");
 184   data->memkind_free
 185     = (__typeof (data->memkind_free)) dlsym (handle, "memkind_free");
 186   data->memkind_check_available
 187     = (__typeof (data->memkind_check_available))
 188       dlsym (handle, "memkind_check_available");
 189   if (data->memkind_malloc
 190       && data->memkind_calloc
 191       && data->memkind_realloc
 192       && data->memkind_free
 193       && data->memkind_check_available)
 194     for (i = 1; i < GOMP_MEMKIND_COUNT; ++i)
 195       {
 196         data->kinds[i] = (void **) dlsym (handle, kinds[i]);
 197         if (data->kinds[i] && data->memkind_check_available (*data->kinds[i]))
 198           data->kinds[i] = NULL;
 199       }
 200   __atomic_store_n (&memkind_data, data, MEMMODEL_RELEASE);
 201 }
 202
 203 static struct gomp_memkind_data *
 204 gomp_get_memkind (void)
 205 {
 206   struct gomp_memkind_data *data
 207     = __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 208   if (data)
 209     return data;
 210   pthread_once (&memkind_data_once, gomp_init_memkind);
 211   return __atomic_load_n (&memkind_data, MEMMODEL_ACQUIRE);
 212 }
 213 #endif
 214
 215 omp_allocator_handle_t
 216 omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
 217                     const omp_alloctrait_t traits[])
 218 {
 219   struct omp_allocator_data data
 220     = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
 221         omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment,
 222 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 223         GOMP_MEMKIND_NONE
 224 #endif
 225       };
 226   struct omp_allocator_data *ret;
 227   int i;
 228
 229   if (memspace > omp_low_lat_mem_space)
 230     return omp_null_allocator;
 231   for (i = 0; i < ntraits; i++)
 232     switch (traits[i].key)
 233       {
 234       case omp_atk_sync_hint:
 235         switch (traits[i].value)
 236           {
 237           case omp_atv_default:
 238             data.sync_hint = omp_atv_contended;
 239             break;
 240           case omp_atv_contended:
 241           case omp_atv_uncontended:
 242           case omp_atv_serialized:
 243           case omp_atv_private:
 244             data.sync_hint = traits[i].value;
 245             break;
 246           default:
 247             return omp_null_allocator;
 248           }
 249         break;
 250       case omp_atk_alignment:
 251         if (traits[i].value == omp_atv_default)
 252           {
 253             data.alignment = 1;
 254             break;
 255           }
 256         if ((traits[i].value & (traits[i].value - 1)) != 0
 257             || !traits[i].value)
 258           return omp_null_allocator;
 259         data.alignment = traits[i].value;
 260         break;
 261       case omp_atk_access:
 262         switch (traits[i].value)
 263           {
 264           case omp_atv_default:
 265             data.access = omp_atv_all;
 266             break;
 267           case omp_atv_all:
 268           case omp_atv_cgroup:
 269           case omp_atv_pteam:
 270           case omp_atv_thread:
 271             data.access = traits[i].value;
 272             break;
 273           default:
 274             return omp_null_allocator;
 275           }
 276         break;
 277       case omp_atk_pool_size:
 278         if (traits[i].value == omp_atv_default)
 279           data.pool_size = ~(uintptr_t) 0;
 280         else
 281           data.pool_size = traits[i].value;
 282         break;
 283       case omp_atk_fallback:
 284         switch (traits[i].value)
 285           {
 286           case omp_atv_default:
 287             data.fallback = omp_atv_default_mem_fb;
 288             break;
 289           case omp_atv_default_mem_fb:
 290           case omp_atv_null_fb:
 291           case omp_atv_abort_fb:
 292           case omp_atv_allocator_fb:
 293             data.fallback = traits[i].value;
 294             break;
 295           default:
 296             return omp_null_allocator;
 297           }
 298         break;
 299       case omp_atk_fb_data:
 300         data.fb_data = traits[i].value;
 301         break;
 302       case omp_atk_pinned:
 303         switch (traits[i].value)
 304           {
 305           case omp_atv_default:
 306           case omp_atv_false:
 307             data.pinned = omp_atv_false;
 308             break;
 309           case omp_atv_true:
 310             data.pinned = omp_atv_true;
 311             break;
 312           default:
 313             return omp_null_allocator;
 314           }
 315         break;
 316       case omp_atk_partition:
 317         switch (traits[i].value)
 318           {
 319           case omp_atv_default:
 320             data.partition = omp_atv_environment;
 321             break;
 322           case omp_atv_environment:
 323           case omp_atv_nearest:
 324           case omp_atv_blocked:
 325           case omp_atv_interleaved:
 326             data.partition = traits[i].value;
 327             break;
 328           default:
 329             return omp_null_allocator;
 330           }
 331         break;
 332       default:
 333         return omp_null_allocator;
 334       }
 335
 336   if (data.alignment < sizeof (void *))
 337     data.alignment = sizeof (void *);
 338
 339   switch (memspace)
 340     {
 341 #ifdef LIBGOMP_USE_MEMKIND
 342     case omp_high_bw_mem_space:
 343       struct gomp_memkind_data *memkind_data;
 344       memkind_data = gomp_get_memkind ();
 345       if (data.partition == omp_atv_interleaved
 346           && memkind_data->kinds[GOMP_MEMKIND_HBW_INTERLEAVE])
 347         {
 348           data.memkind = GOMP_MEMKIND_HBW_INTERLEAVE;
 349           break;
 350         }
 351       else if (memkind_data->kinds[GOMP_MEMKIND_HBW_PREFERRED])
 352         {
 353           data.memkind = GOMP_MEMKIND_HBW_PREFERRED;
 354           break;
 355         }
 356       break;
 357     case omp_large_cap_mem_space:
 358       memkind_data = gomp_get_memkind ();
 359       if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM_ALL])
 360         data.memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 361       else if (memkind_data->kinds[GOMP_MEMKIND_DAX_KMEM])
 362         data.memkind = GOMP_MEMKIND_DAX_KMEM;
 363       break;
 364 #endif
 365     default:
 366 #ifdef LIBGOMP_USE_MEMKIND
 367       if (data.partition == omp_atv_interleaved)
 368         {
 369           memkind_data = gomp_get_memkind ();
 370           if (memkind_data->kinds[GOMP_MEMKIND_INTERLEAVE])
 371             data.memkind = GOMP_MEMKIND_INTERLEAVE;
 372         }
 373 #endif
 374       break;
 375     }
 376
 377 #ifdef LIBGOMP_USE_LIBNUMA
 378   if (data.memkind == GOMP_MEMKIND_NONE && data.partition == omp_atv_nearest)
 379     {
 380       libnuma_data = gomp_get_libnuma ();
 381       if (libnuma_data->numa_alloc_local != NULL)
 382         data.memkind = GOMP_MEMKIND_LIBNUMA;
 383     }
 384 #endif
 385
 386   /* No support for this so far.  */
 387   if (data.pinned)
 388     return omp_null_allocator;
 389
 390   ret = gomp_malloc (sizeof (struct omp_allocator_data));
 391   *ret = data;
 392 #ifndef HAVE_SYNC_BUILTINS
 393   gomp_mutex_init (&ret->lock);
 394 #endif
 395   return (omp_allocator_handle_t) ret;
 396 }
 397
 398 void
 399 omp_destroy_allocator (omp_allocator_handle_t allocator)
 400 {
 401   if (allocator != omp_null_allocator)
 402     {
 403 #ifndef HAVE_SYNC_BUILTINS
 404       gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
 405 #endif
 406       free ((void *) allocator);
 407     }
 408 }
 409
 410 ialias (omp_init_allocator)
 411 ialias (omp_destroy_allocator)
 412
 413 void *
 414 omp_aligned_alloc (size_t alignment, size_t size,
 415                    omp_allocator_handle_t allocator)
 416 {
 417   struct omp_allocator_data *allocator_data;
 418   size_t new_size, new_alignment;
 419   void *ptr, *ret;
 420 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 421   enum gomp_numa_memkind_kind memkind;
 422 #endif
 423
 424   if (__builtin_expect (size == 0, 0))
 425     return NULL;
 426
 427 retry:
 428   new_alignment = alignment;
 429   if (allocator == omp_null_allocator)
 430     {
 431       struct gomp_thread *thr = gomp_thread ();
 432       if (thr->ts.def_allocator == omp_null_allocator)
 433         thr->ts.def_allocator = gomp_def_allocator;
 434       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 435     }
 436
 437   if (allocator > omp_max_predefined_alloc)
 438     {
 439       allocator_data = (struct omp_allocator_data *) allocator;
 440       if (new_alignment < allocator_data->alignment)
 441         new_alignment = allocator_data->alignment;
 442 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 443       memkind = allocator_data->memkind;
 444 #endif
 445     }
 446   else
 447     {
 448       allocator_data = NULL;
 449       if (new_alignment < sizeof (void *))
 450         new_alignment = sizeof (void *);
 451 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 452       memkind = GOMP_MEMKIND_NONE;
 453 #endif
 454 #ifdef LIBGOMP_USE_MEMKIND
 455       if (allocator == omp_high_bw_mem_alloc)
 456         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 457       else if (allocator == omp_large_cap_mem_alloc)
 458         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 459       if (memkind)
 460         {
 461           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 462           if (!memkind_data->kinds[memkind])
 463             memkind = GOMP_MEMKIND_NONE;
 464         }
 465 #endif
 466     }
 467
 468   new_size = sizeof (struct omp_mem_header);
 469   if (new_alignment > sizeof (void *))
 470     new_size += new_alignment - sizeof (void *);
 471   if (__builtin_add_overflow (size, new_size, &new_size))
 472     goto fail;
 473
 474   if (__builtin_expect (allocator_data
 475                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 476     {
 477       uintptr_t used_pool_size;
 478       if (new_size > allocator_data->pool_size)
 479         goto fail;
 480 #ifdef HAVE_SYNC_BUILTINS
 481       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 482                                         MEMMODEL_RELAXED);
 483       do
 484         {
 485           uintptr_t new_pool_size;
 486           if (__builtin_add_overflow (used_pool_size, new_size,
 487                                       &new_pool_size)
 488               || new_pool_size > allocator_data->pool_size)
 489             goto fail;
 490           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 491                                            &used_pool_size, new_pool_size,
 492                                            true, MEMMODEL_RELAXED,
 493                                            MEMMODEL_RELAXED))
 494             break;
 495         }
 496       while (1);
 497 #else
 498       gomp_mutex_lock (&allocator_data->lock);
 499       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 500                                   &used_pool_size)
 501           || used_pool_size > allocator_data->pool_size)
 502         {
 503           gomp_mutex_unlock (&allocator_data->lock);
 504           goto fail;
 505         }
 506       allocator_data->used_pool_size = used_pool_size;
 507       gomp_mutex_unlock (&allocator_data->lock);
 508 #endif
 509 #ifdef LIBGOMP_USE_LIBNUMA
 510       if (memkind == GOMP_MEMKIND_LIBNUMA)
 511         ptr = libnuma_data->numa_alloc_local (new_size);
 512 # ifdef LIBGOMP_USE_MEMKIND
 513       else
 514 # endif
 515 #endif
 516 #ifdef LIBGOMP_USE_MEMKIND
 517       if (memkind)
 518         {
 519           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 520           void *kind = *memkind_data->kinds[memkind];
 521           ptr = memkind_data->memkind_malloc (kind, new_size);
 522         }
 523       else
 524 #endif
 525         ptr = malloc (new_size);
 526       if (ptr == NULL)
 527         {
 528 #ifdef HAVE_SYNC_BUILTINS
 529           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 530                               MEMMODEL_RELAXED);
 531 #else
 532           gomp_mutex_lock (&allocator_data->lock);
 533           allocator_data->used_pool_size -= new_size;
 534           gomp_mutex_unlock (&allocator_data->lock);
 535 #endif
 536           goto fail;
 537         }
 538     }
 539   else
 540     {
 541 #ifdef LIBGOMP_USE_LIBNUMA
 542       if (memkind == GOMP_MEMKIND_LIBNUMA)
 543         ptr = libnuma_data->numa_alloc_local (new_size);
 544 # ifdef LIBGOMP_USE_MEMKIND
 545       else
 546 # endif
 547 #endif
 548 #ifdef LIBGOMP_USE_MEMKIND
 549       if (memkind)
 550         {
 551           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 552           void *kind = *memkind_data->kinds[memkind];
 553           ptr = memkind_data->memkind_malloc (kind, new_size);
 554         }
 555       else
 556 #endif
 557         ptr = malloc (new_size);
 558       if (ptr == NULL)
 559         goto fail;
 560     }
 561
 562   if (new_alignment > sizeof (void *))
 563     ret = (void *) (((uintptr_t) ptr
 564                      + sizeof (struct omp_mem_header)
 565                      + new_alignment - sizeof (void *))
 566                     & ~(new_alignment - 1));
 567   else
 568     ret = (char *) ptr + sizeof (struct omp_mem_header);
 569   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 570   ((struct omp_mem_header *) ret)[-1].size = new_size;
 571   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 572   return ret;
 573
 574 fail:
 575   if (allocator_data)
 576     {
 577       switch (allocator_data->fallback)
 578         {
 579         case omp_atv_default_mem_fb:
 580           if ((new_alignment > sizeof (void *) && new_alignment > alignment)
 581 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 582               || memkind
 583 #endif
 584               || (allocator_data
 585                   && allocator_data->pool_size < ~(uintptr_t) 0))
 586             {
 587               allocator = omp_default_mem_alloc;
 588               goto retry;
 589             }
 590           /* Otherwise, we've already performed default mem allocation
 591              and if that failed, it won't succeed again (unless it was
 592              intermittent.  Return NULL then, as that is the fallback.  */
 593           break;
 594         case omp_atv_null_fb:
 595           break;
 596         default:
 597         case omp_atv_abort_fb:
 598           gomp_fatal ("Out of memory allocating %lu bytes",
 599                       (unsigned long) size);
 600         case omp_atv_allocator_fb:
 601           allocator = allocator_data->fb_data;
 602           goto retry;
 603         }
 604     }
 605   return NULL;
 606 }
 607
 608 ialias (omp_aligned_alloc)
 609
 610 void *
 611 omp_alloc (size_t size, omp_allocator_handle_t allocator)
 612 {
 613   return ialias_call (omp_aligned_alloc) (1, size, allocator);
 614 }
 615
 616 /* Like omp_aligned_alloc, but apply on top of that:
 617    "For allocations that arise from this ... the null_fb value of the
 618    fallback allocator trait behaves as if the abort_fb had been specified."  */
 619
 620 void *
 621 GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
 622 {
 623   void *ret
 624     = ialias_call (omp_aligned_alloc) (alignment, size,
 625                                        (omp_allocator_handle_t) allocator);
 626   if (__builtin_expect (ret == NULL, 0) && size)
 627     gomp_fatal ("Out of memory allocating %lu bytes",
 628                 (unsigned long) size);
 629   return ret;
 630 }
 631
 632 void
 633 omp_free (void *ptr, omp_allocator_handle_t allocator)
 634 {
 635   struct omp_mem_header *data;
 636
 637   if (ptr == NULL)
 638     return;
 639   (void) allocator;
 640   data = &((struct omp_mem_header *) ptr)[-1];
 641   if (data->allocator > omp_max_predefined_alloc)
 642     {
 643       struct omp_allocator_data *allocator_data
 644         = (struct omp_allocator_data *) (data->allocator);
 645       if (allocator_data->pool_size < ~(uintptr_t) 0)
 646         {
 647 #ifdef HAVE_SYNC_BUILTINS
 648           __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
 649                               MEMMODEL_RELAXED);
 650 #else
 651           gomp_mutex_lock (&allocator_data->lock);
 652           allocator_data->used_pool_size -= data->size;
 653           gomp_mutex_unlock (&allocator_data->lock);
 654 #endif
 655         }
 656 #ifdef LIBGOMP_USE_LIBNUMA
 657       if (allocator_data->memkind == GOMP_MEMKIND_LIBNUMA)
 658         {
 659           libnuma_data->numa_free (data->ptr, data->size);
 660           return;
 661         }
 662 # ifdef LIBGOMP_USE_MEMKIND
 663       else
 664 # endif
 665 #endif
 666 #ifdef LIBGOMP_USE_MEMKIND
 667       if (allocator_data->memkind)
 668         {
 669           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 670           void *kind = *memkind_data->kinds[allocator_data->memkind];
 671           memkind_data->memkind_free (kind, data->ptr);
 672           return;
 673         }
 674 #endif
 675     }
 676 #ifdef LIBGOMP_USE_MEMKIND
 677   else
 678     {
 679       enum gomp_numa_memkind_kind memkind = GOMP_MEMKIND_NONE;
 680       if (data->allocator == omp_high_bw_mem_alloc)
 681         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 682       else if (data->allocator == omp_large_cap_mem_alloc)
 683         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 684       if (memkind)
 685         {
 686           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 687           if (memkind_data->kinds[memkind])
 688             {
 689               void *kind = *memkind_data->kinds[memkind];
 690               memkind_data->memkind_free (kind, data->ptr);
 691               return;
 692             }
 693         }
 694     }
 695 #endif
 696   free (data->ptr);
 697 }
 698
 699 ialias (omp_free)
 700
 701 void
 702 GOMP_free (void *ptr, uintptr_t allocator)
 703 {
 704   return ialias_call (omp_free) (ptr, (omp_allocator_handle_t) allocator);
 705 }
 706
 707 void *
 708 omp_aligned_calloc (size_t alignment, size_t nmemb, size_t size,
 709                     omp_allocator_handle_t allocator)
 710 {
 711   struct omp_allocator_data *allocator_data;
 712   size_t new_size, size_temp, new_alignment;
 713   void *ptr, *ret;
 714 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 715   enum gomp_numa_memkind_kind memkind;
 716 #endif
 717
 718   if (__builtin_expect (size == 0 || nmemb == 0, 0))
 719     return NULL;
 720
 721 retry:
 722   new_alignment = alignment;
 723   if (allocator == omp_null_allocator)
 724     {
 725       struct gomp_thread *thr = gomp_thread ();
 726       if (thr->ts.def_allocator == omp_null_allocator)
 727         thr->ts.def_allocator = gomp_def_allocator;
 728       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 729     }
 730
 731   if (allocator > omp_max_predefined_alloc)
 732     {
 733       allocator_data = (struct omp_allocator_data *) allocator;
 734       if (new_alignment < allocator_data->alignment)
 735         new_alignment = allocator_data->alignment;
 736 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 737       memkind = allocator_data->memkind;
 738 #endif
 739     }
 740   else
 741     {
 742       allocator_data = NULL;
 743       if (new_alignment < sizeof (void *))
 744         new_alignment = sizeof (void *);
 745 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 746       memkind = GOMP_MEMKIND_NONE;
 747 #endif
 748 #ifdef LIBGOMP_USE_MEMKIND
 749       if (allocator == omp_high_bw_mem_alloc)
 750         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 751       else if (allocator == omp_large_cap_mem_alloc)
 752         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 753       if (memkind)
 754         {
 755           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 756           if (!memkind_data->kinds[memkind])
 757             memkind = GOMP_MEMKIND_NONE;
 758         }
 759 #endif
 760     }
 761
 762   new_size = sizeof (struct omp_mem_header);
 763   if (new_alignment > sizeof (void *))
 764     new_size += new_alignment - sizeof (void *);
 765   if (__builtin_mul_overflow (size, nmemb, &size_temp))
 766     goto fail;
 767   if (__builtin_add_overflow (size_temp, new_size, &new_size))
 768     goto fail;
 769
 770   if (__builtin_expect (allocator_data
 771                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 772     {
 773       uintptr_t used_pool_size;
 774       if (new_size > allocator_data->pool_size)
 775         goto fail;
 776 #ifdef HAVE_SYNC_BUILTINS
 777       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 778                                         MEMMODEL_RELAXED);
 779       do
 780         {
 781           uintptr_t new_pool_size;
 782           if (__builtin_add_overflow (used_pool_size, new_size,
 783                                       &new_pool_size)
 784               || new_pool_size > allocator_data->pool_size)
 785             goto fail;
 786           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 787                                            &used_pool_size, new_pool_size,
 788                                            true, MEMMODEL_RELAXED,
 789                                            MEMMODEL_RELAXED))
 790             break;
 791         }
 792       while (1);
 793 #else
 794       gomp_mutex_lock (&allocator_data->lock);
 795       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 796                                   &used_pool_size)
 797           || used_pool_size > allocator_data->pool_size)
 798         {
 799           gomp_mutex_unlock (&allocator_data->lock);
 800           goto fail;
 801         }
 802       allocator_data->used_pool_size = used_pool_size;
 803       gomp_mutex_unlock (&allocator_data->lock);
 804 #endif
 805 #ifdef LIBGOMP_USE_LIBNUMA
 806       if (memkind == GOMP_MEMKIND_LIBNUMA)
 807         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 808            memory that is initialized to zero.  */
 809         ptr = libnuma_data->numa_alloc_local (new_size);
 810 # ifdef LIBGOMP_USE_MEMKIND
 811       else
 812 # endif
 813 #endif
 814 #ifdef LIBGOMP_USE_MEMKIND
 815       if (memkind)
 816         {
 817           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 818           void *kind = *memkind_data->kinds[memkind];
 819           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 820         }
 821       else
 822 #endif
 823         ptr = calloc (1, new_size);
 824       if (ptr == NULL)
 825         {
 826 #ifdef HAVE_SYNC_BUILTINS
 827           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 828                               MEMMODEL_RELAXED);
 829 #else
 830           gomp_mutex_lock (&allocator_data->lock);
 831           allocator_data->used_pool_size -= new_size;
 832           gomp_mutex_unlock (&allocator_data->lock);
 833 #endif
 834           goto fail;
 835         }
 836     }
 837   else
 838     {
 839 #ifdef LIBGOMP_USE_LIBNUMA
 840       if (memkind == GOMP_MEMKIND_LIBNUMA)
 841         /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
 842            memory that is initialized to zero.  */
 843         ptr = libnuma_data->numa_alloc_local (new_size);
 844 # ifdef LIBGOMP_USE_MEMKIND
 845       else
 846 # endif
 847 #endif
 848 #ifdef LIBGOMP_USE_MEMKIND
 849       if (memkind)
 850         {
 851           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 852           void *kind = *memkind_data->kinds[memkind];
 853           ptr = memkind_data->memkind_calloc (kind, 1, new_size);
 854         }
 855       else
 856 #endif
 857         ptr = calloc (1, new_size);
 858       if (ptr == NULL)
 859         goto fail;
 860     }
 861
 862   if (new_alignment > sizeof (void *))
 863     ret = (void *) (((uintptr_t) ptr
 864                      + sizeof (struct omp_mem_header)
 865                      + new_alignment - sizeof (void *))
 866                     & ~(new_alignment - 1));
 867   else
 868     ret = (char *) ptr + sizeof (struct omp_mem_header);
 869   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 870   ((struct omp_mem_header *) ret)[-1].size = new_size;
 871   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 872   return ret;
 873
 874 fail:
 875   if (allocator_data)
 876     {
 877       switch (allocator_data->fallback)
 878         {
 879         case omp_atv_default_mem_fb:
 880           if ((new_alignment > sizeof (void *) && new_alignment > alignment)
 881 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 882               || memkind
 883 #endif
 884               || (allocator_data
 885                   && allocator_data->pool_size < ~(uintptr_t) 0))
 886             {
 887               allocator = omp_default_mem_alloc;
 888               goto retry;
 889             }
 890           /* Otherwise, we've already performed default mem allocation
 891              and if that failed, it won't succeed again (unless it was
 892              intermittent.  Return NULL then, as that is the fallback.  */
 893           break;
 894         case omp_atv_null_fb:
 895           break;
 896         default:
 897         case omp_atv_abort_fb:
 898           gomp_fatal ("Out of memory allocating %lu bytes",
 899                       (unsigned long) (size * nmemb));
 900         case omp_atv_allocator_fb:
 901           allocator = allocator_data->fb_data;
 902           goto retry;
 903         }
 904     }
 905   return NULL;
 906 }
 907
 908 ialias (omp_aligned_calloc)
 909
 910 void *
 911 omp_calloc (size_t nmemb, size_t size, omp_allocator_handle_t allocator)
 912 {
 913   return ialias_call (omp_aligned_calloc) (1, nmemb, size, allocator);
 914 }
 915
 916 void *
 917 omp_realloc (void *ptr, size_t size, omp_allocator_handle_t allocator,
 918              omp_allocator_handle_t free_allocator)
 919 {
 920   struct omp_allocator_data *allocator_data, *free_allocator_data;
 921   size_t new_size, old_size, new_alignment, old_alignment;
 922   void *new_ptr, *ret;
 923   struct omp_mem_header *data;
 924 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 925   enum gomp_numa_memkind_kind memkind, free_memkind;
 926 #endif
 927
 928   if (__builtin_expect (ptr == NULL, 0))
 929     return ialias_call (omp_aligned_alloc) (1, size, allocator);
 930
 931   if (__builtin_expect (size == 0, 0))
 932     {
 933       ialias_call (omp_free) (ptr, free_allocator);
 934       return NULL;
 935     }
 936
 937   data = &((struct omp_mem_header *) ptr)[-1];
 938   free_allocator = data->allocator;
 939
 940 retry:
 941   new_alignment = sizeof (void *);
 942   if (allocator == omp_null_allocator)
 943     allocator = free_allocator;
 944
 945   if (allocator > omp_max_predefined_alloc)
 946     {
 947       allocator_data = (struct omp_allocator_data *) allocator;
 948       if (new_alignment < allocator_data->alignment)
 949         new_alignment = allocator_data->alignment;
 950 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 951       memkind = allocator_data->memkind;
 952 #endif
 953     }
 954   else
 955     {
 956       allocator_data = NULL;
 957 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 958       memkind = GOMP_MEMKIND_NONE;
 959 #endif
 960 #ifdef LIBGOMP_USE_MEMKIND
 961       if (allocator == omp_high_bw_mem_alloc)
 962         memkind = GOMP_MEMKIND_HBW_PREFERRED;
 963       else if (allocator == omp_large_cap_mem_alloc)
 964         memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 965       if (memkind)
 966         {
 967           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 968           if (!memkind_data->kinds[memkind])
 969             memkind = GOMP_MEMKIND_NONE;
 970         }
 971 #endif
 972     }
 973   if (free_allocator > omp_max_predefined_alloc)
 974     {
 975       free_allocator_data = (struct omp_allocator_data *) free_allocator;
 976 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 977       free_memkind = free_allocator_data->memkind;
 978 #endif
 979     }
 980   else
 981     {
 982       free_allocator_data = NULL;
 983 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
 984       free_memkind = GOMP_MEMKIND_NONE;
 985 #endif
 986 #ifdef LIBGOMP_USE_MEMKIND
 987       if (free_allocator == omp_high_bw_mem_alloc)
 988         free_memkind = GOMP_MEMKIND_HBW_PREFERRED;
 989       else if (free_allocator == omp_large_cap_mem_alloc)
 990         free_memkind = GOMP_MEMKIND_DAX_KMEM_ALL;
 991       if (free_memkind)
 992         {
 993           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
 994           if (!memkind_data->kinds[free_memkind])
 995             free_memkind = GOMP_MEMKIND_NONE;
 996         }
 997 #endif
 998     }
 999   old_alignment = (uintptr_t) ptr - (uintptr_t) (data->ptr);
1000
1001   new_size = sizeof (struct omp_mem_header);
1002   if (new_alignment > sizeof (void *))
1003     new_size += new_alignment - sizeof (void *);
1004   if (__builtin_add_overflow (size, new_size, &new_size))
1005     goto fail;
1006   old_size = data->size;
1007
1008   if (__builtin_expect (allocator_data
1009                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
1010     {
1011       uintptr_t used_pool_size;
1012       size_t prev_size = 0;
1013       /* Check if we can use realloc.  Don't use it if extra alignment
1014          was used previously or newly, because realloc might return a pointer
1015          with different alignment and then we'd need to memmove the data
1016          again.  */
1017       if (free_allocator_data
1018           && free_allocator_data == allocator_data
1019           && new_alignment == sizeof (void *)
1020           && old_alignment == sizeof (struct omp_mem_header))
1021         prev_size = old_size;
1022       if (new_size > prev_size
1023           && new_size - prev_size > allocator_data->pool_size)
1024         goto fail;
1025 #ifdef HAVE_SYNC_BUILTINS
1026       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
1027                                         MEMMODEL_RELAXED);
1028       do
1029         {
1030           uintptr_t new_pool_size;
1031           if (new_size > prev_size)
1032             {
1033               if (__builtin_add_overflow (used_pool_size, new_size - prev_size,
1034                                           &new_pool_size)
1035                   || new_pool_size > allocator_data->pool_size)
1036                 goto fail;
1037             }
1038           else
1039             new_pool_size = used_pool_size + new_size - prev_size;
1040           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
1041                                            &used_pool_size, new_pool_size,
1042                                            true, MEMMODEL_RELAXED,
1043                                            MEMMODEL_RELAXED))
1044             break;
1045         }
1046       while (1);
1047 #else
1048       gomp_mutex_lock (&allocator_data->lock);
1049       if (new_size > prev_size)
1050         {
1051           if (__builtin_add_overflow (allocator_data->used_pool_size,
1052                                       new_size - prev_size,
1053                                       &used_pool_size)
1054               || used_pool_size > allocator_data->pool_size)
1055             {
1056               gomp_mutex_unlock (&allocator_data->lock);
1057               goto fail;
1058             }
1059         }
1060       else
1061         used_pool_size = (allocator_data->used_pool_size
1062                           + new_size - prev_size);
1063       allocator_data->used_pool_size = used_pool_size;
1064       gomp_mutex_unlock (&allocator_data->lock);
1065 #endif
1066 #ifdef LIBGOMP_USE_LIBNUMA
1067       if (memkind == GOMP_MEMKIND_LIBNUMA)
1068         {
1069           if (prev_size)
1070             new_ptr = libnuma_data->numa_realloc (data->ptr, data->size,
1071                                                   new_size);
1072           else
1073             new_ptr = libnuma_data->numa_alloc_local (new_size);
1074         }
1075 # ifdef LIBGOMP_USE_MEMKIND
1076       else
1077 # endif
1078 #endif
1079 #ifdef LIBGOMP_USE_MEMKIND
1080       if (memkind)
1081         {
1082           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1083           void *kind = *memkind_data->kinds[memkind];
1084           if (prev_size)
1085             new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1086                                                      new_size);
1087           else
1088             new_ptr = memkind_data->memkind_malloc (kind, new_size);
1089         }
1090       else
1091 #endif
1092       if (prev_size)
1093         new_ptr = realloc (data->ptr, new_size);
1094       else
1095         new_ptr = malloc (new_size);
1096       if (new_ptr == NULL)
1097         {
1098 #ifdef HAVE_SYNC_BUILTINS
1099           __atomic_add_fetch (&allocator_data->used_pool_size,
1100                               prev_size - new_size,
1101                               MEMMODEL_RELAXED);
1102 #else
1103           gomp_mutex_lock (&allocator_data->lock);
1104           allocator_data->used_pool_size -= new_size - prev_size;
1105           gomp_mutex_unlock (&allocator_data->lock);
1106 #endif
1107           goto fail;
1108         }
1109       else if (prev_size)
1110         {
1111           ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1112           ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1113           ((struct omp_mem_header *) ret)[-1].size = new_size;
1114           ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1115           return ret;
1116         }
1117     }
1118   else if (new_alignment == sizeof (void *)
1119            && old_alignment == sizeof (struct omp_mem_header)
1120 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1121            && memkind == free_memkind
1122 #endif
1123            && (free_allocator_data == NULL
1124                || free_allocator_data->pool_size == ~(uintptr_t) 0))
1125     {
1126 #ifdef LIBGOMP_USE_LIBNUMA
1127       if (memkind == GOMP_MEMKIND_LIBNUMA)
1128         new_ptr = libnuma_data->numa_realloc (data->ptr, data->size, new_size);
1129 # ifdef LIBGOMP_USE_MEMKIND
1130       else
1131 # endif
1132 #endif
1133 #ifdef LIBGOMP_USE_MEMKIND
1134       if (memkind)
1135         {
1136           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1137           void *kind = *memkind_data->kinds[memkind];
1138           new_ptr = memkind_data->memkind_realloc (kind, data->ptr,
1139                                                    new_size);
1140         }
1141       else
1142 #endif
1143         new_ptr = realloc (data->ptr, new_size);
1144       if (new_ptr == NULL)
1145         goto fail;
1146       ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1147       ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1148       ((struct omp_mem_header *) ret)[-1].size = new_size;
1149       ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1150       return ret;
1151     }
1152   else
1153     {
1154 #ifdef LIBGOMP_USE_LIBNUMA
1155       if (memkind == GOMP_MEMKIND_LIBNUMA)
1156         new_ptr = libnuma_data->numa_alloc_local (new_size);
1157 # ifdef LIBGOMP_USE_MEMKIND
1158       else
1159 # endif
1160 #endif
1161 #ifdef LIBGOMP_USE_MEMKIND
1162       if (memkind)
1163         {
1164           struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1165           void *kind = *memkind_data->kinds[memkind];
1166           new_ptr = memkind_data->memkind_malloc (kind, new_size);
1167         }
1168       else
1169 #endif
1170         new_ptr = malloc (new_size);
1171       if (new_ptr == NULL)
1172         goto fail;
1173     }
1174
1175   if (new_alignment > sizeof (void *))
1176     ret = (void *) (((uintptr_t) new_ptr
1177                      + sizeof (struct omp_mem_header)
1178                      + new_alignment - sizeof (void *))
1179                     & ~(new_alignment - 1));
1180   else
1181     ret = (char *) new_ptr + sizeof (struct omp_mem_header);
1182   ((struct omp_mem_header *) ret)[-1].ptr = new_ptr;
1183   ((struct omp_mem_header *) ret)[-1].size = new_size;
1184   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
1185   if (old_size - old_alignment < size)
1186     size = old_size - old_alignment;
1187   memcpy (ret, ptr, size);
1188   if (__builtin_expect (free_allocator_data
1189                         && free_allocator_data->pool_size < ~(uintptr_t) 0, 0))
1190     {
1191 #ifdef HAVE_SYNC_BUILTINS
1192       __atomic_add_fetch (&free_allocator_data->used_pool_size, -data->size,
1193                           MEMMODEL_RELAXED);
1194 #else
1195       gomp_mutex_lock (&free_allocator_data->lock);
1196       free_allocator_data->used_pool_size -= data->size;
1197       gomp_mutex_unlock (&free_allocator_data->lock);
1198 #endif
1199     }
1200 #ifdef LIBGOMP_USE_LIBNUMA
1201   if (free_memkind == GOMP_MEMKIND_LIBNUMA)
1202     {
1203       libnuma_data->numa_free (data->ptr, data->size);
1204       return ret;
1205     }
1206 # ifdef LIBGOMP_USE_MEMKIND
1207   else
1208 # endif
1209 #endif
1210 #ifdef LIBGOMP_USE_MEMKIND
1211   if (free_memkind)
1212     {
1213       struct gomp_memkind_data *memkind_data = gomp_get_memkind ();
1214       void *kind = *memkind_data->kinds[free_memkind];
1215       memkind_data->memkind_free (kind, data->ptr);
1216       return ret;
1217     }
1218 #endif
1219   free (data->ptr);
1220   return ret;
1221
1222 fail:
1223   if (allocator_data)
1224     {
1225       switch (allocator_data->fallback)
1226         {
1227         case omp_atv_default_mem_fb:
1228           if (new_alignment > sizeof (void *)
1229 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1230               || memkind
1231 #endif
1232               || (allocator_data
1233                   && allocator_data->pool_size < ~(uintptr_t) 0))
1234             {
1235               allocator = omp_default_mem_alloc;
1236               goto retry;
1237             }
1238           /* Otherwise, we've already performed default mem allocation
1239              and if that failed, it won't succeed again (unless it was
1240              intermittent.  Return NULL then, as that is the fallback.  */
1241           break;
1242         case omp_atv_null_fb:
1243           break;
1244         default:
1245         case omp_atv_abort_fb:
1246           gomp_fatal ("Out of memory allocating %lu bytes",
1247                       (unsigned long) size);
1248         case omp_atv_allocator_fb:
1249           allocator = allocator_data->fb_data;
1250           goto retry;
1251         }
1252     }
1253   return NULL;
1254 }