libgomp/allocator.c

   1 /* Copyright (C) 2020-2021 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains wrappers for the system allocation routines.  Most
  27    places in the OpenMP API do not make any provision for failure, so in
  28    general we cannot allow memory allocation to fail.  */
  29
  30 #define _GNU_SOURCE
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33
  34 #define omp_max_predefined_alloc omp_thread_mem_alloc
  35
  36 struct omp_allocator_data
  37 {
  38   omp_memspace_handle_t memspace;
  39   omp_uintptr_t alignment;
  40   omp_uintptr_t pool_size;
  41   omp_uintptr_t used_pool_size;
  42   omp_allocator_handle_t fb_data;
  43   unsigned int sync_hint : 8;
  44   unsigned int access : 8;
  45   unsigned int fallback : 8;
  46   unsigned int pinned : 1;
  47   unsigned int partition : 7;
  48 #ifndef HAVE_SYNC_BUILTINS
  49   gomp_mutex_t lock;
  50 #endif
  51 };
  52
  53 struct omp_mem_header
  54 {
  55   void *ptr;
  56   size_t size;
  57   omp_allocator_handle_t allocator;
  58   void *pad;
  59 };
  60
  61 omp_allocator_handle_t
  62 omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
  63                     const omp_alloctrait_t traits[])
  64 {
  65   struct omp_allocator_data data
  66     = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
  67         omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment };
  68   struct omp_allocator_data *ret;
  69   int i;
  70
  71   if (memspace > omp_low_lat_mem_space)
  72     return omp_null_allocator;
  73   for (i = 0; i < ntraits; i++)
  74     switch (traits[i].key)
  75       {
  76       case omp_atk_sync_hint:
  77         switch (traits[i].value)
  78           {
  79           case omp_atv_default:
  80             data.sync_hint = omp_atv_contended;
  81             break;
  82           case omp_atv_contended:
  83           case omp_atv_uncontended:
  84           case omp_atv_sequential:
  85           case omp_atv_private:
  86             data.sync_hint = traits[i].value;
  87             break;
  88           default:
  89             return omp_null_allocator;
  90           }
  91         break;
  92       case omp_atk_alignment:
  93         if (traits[i].value == omp_atv_default)
  94           {
  95             data.alignment = 1;
  96             break;
  97           }
  98         if ((traits[i].value & (traits[i].value - 1)) != 0
  99             || !traits[i].value)
 100           return omp_null_allocator;
 101         data.alignment = traits[i].value;
 102         break;
 103       case omp_atk_access:
 104         switch (traits[i].value)
 105           {
 106           case omp_atv_default:
 107             data.access = omp_atv_all;
 108             break;
 109           case omp_atv_all:
 110           case omp_atv_cgroup:
 111           case omp_atv_pteam:
 112           case omp_atv_thread:
 113             data.access = traits[i].value;
 114             break;
 115           default:
 116             return omp_null_allocator;
 117           }
 118         break;
 119       case omp_atk_pool_size:
 120         if (traits[i].value == omp_atv_default)
 121           data.pool_size = ~(uintptr_t) 0;
 122         else
 123           data.pool_size = traits[i].value;
 124         break;
 125       case omp_atk_fallback:
 126         switch (traits[i].value)
 127           {
 128           case omp_atv_default:
 129             data.fallback = omp_atv_default_mem_fb;
 130             break;
 131           case omp_atv_default_mem_fb:
 132           case omp_atv_null_fb:
 133           case omp_atv_abort_fb:
 134           case omp_atv_allocator_fb:
 135             data.fallback = traits[i].value;
 136             break;
 137           default:
 138             return omp_null_allocator;
 139           }
 140         break;
 141       case omp_atk_fb_data:
 142         data.fb_data = traits[i].value;
 143         break;
 144       case omp_atk_pinned:
 145         switch (traits[i].value)
 146           {
 147           case omp_atv_default:
 148           case omp_atv_false:
 149             data.pinned = omp_atv_false;
 150             break;
 151           case omp_atv_true:
 152             data.pinned = omp_atv_true;
 153             break;
 154           default:
 155             return omp_null_allocator;
 156           }
 157         break;
 158       case omp_atk_partition:
 159         switch (traits[i].value)
 160           {
 161           case omp_atv_default:
 162             data.partition = omp_atv_environment;
 163             break;
 164           case omp_atv_environment:
 165           case omp_atv_nearest:
 166           case omp_atv_blocked:
 167           case omp_atv_interleaved:
 168             data.partition = traits[i].value;
 169             break;
 170           default:
 171             return omp_null_allocator;
 172           }
 173         break;
 174       default:
 175         return omp_null_allocator;
 176       }
 177
 178   if (data.alignment < sizeof (void *))
 179     data.alignment = sizeof (void *);
 180
 181   /* No support for these so far (for hbw will use memkind).  */
 182   if (data.pinned || data.memspace == omp_high_bw_mem_space)
 183     return omp_null_allocator;
 184
 185   ret = gomp_malloc (sizeof (struct omp_allocator_data));
 186   *ret = data;
 187 #ifndef HAVE_SYNC_BUILTINS
 188   gomp_mutex_init (&ret->lock);
 189 #endif
 190   return (omp_allocator_handle_t) ret;
 191 }
 192
 193 void
 194 omp_destroy_allocator (omp_allocator_handle_t allocator)
 195 {
 196   if (allocator != omp_null_allocator)
 197     {
 198 #ifndef HAVE_SYNC_BUILTINS
 199       gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
 200 #endif
 201       free ((void *) allocator);
 202     }
 203 }
 204
 205 ialias (omp_init_allocator)
 206 ialias (omp_destroy_allocator)
 207
 208 static void *
 209 omp_aligned_alloc (size_t alignment, size_t size,
 210                    omp_allocator_handle_t allocator)
 211 {
 212   struct omp_allocator_data *allocator_data;
 213   size_t new_size;
 214   void *ptr, *ret;
 215
 216   if (__builtin_expect (size == 0, 0))
 217     return NULL;
 218
 219 retry:
 220   if (allocator == omp_null_allocator)
 221     {
 222       struct gomp_thread *thr = gomp_thread ();
 223       if (thr->ts.def_allocator == omp_null_allocator)
 224         thr->ts.def_allocator = gomp_def_allocator;
 225       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 226     }
 227
 228   if (allocator > omp_max_predefined_alloc)
 229     {
 230       allocator_data = (struct omp_allocator_data *) allocator;
 231       if (alignment < allocator_data->alignment)
 232         alignment = allocator_data->alignment;
 233     }
 234   else
 235     {
 236       allocator_data = NULL;
 237       if (alignment < sizeof (void *))
 238         alignment = sizeof (void *);
 239     }
 240
 241   new_size = sizeof (struct omp_mem_header);
 242   if (alignment > sizeof (void *))
 243     new_size += alignment - sizeof (void *);
 244   if (__builtin_add_overflow (size, new_size, &new_size))
 245     goto fail;
 246
 247   if (__builtin_expect (allocator_data
 248                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 249     {
 250       uintptr_t used_pool_size;
 251       if (new_size > allocator_data->pool_size)
 252         goto fail;
 253 #ifdef HAVE_SYNC_BUILTINS
 254       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 255                                         MEMMODEL_RELAXED);
 256       do
 257         {
 258           uintptr_t new_pool_size;
 259           if (__builtin_add_overflow (used_pool_size, new_size,
 260                                       &new_pool_size)
 261               || new_pool_size > allocator_data->pool_size)
 262             goto fail;
 263           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 264                                            &used_pool_size, new_pool_size,
 265                                            true, MEMMODEL_RELAXED,
 266                                            MEMMODEL_RELAXED))
 267             break;
 268         }
 269       while (1);
 270 #else
 271       gomp_mutex_lock (&allocator_data->lock);
 272       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 273                                   &used_pool_size)
 274           || used_pool_size > allocator_data->pool_size)
 275         {
 276           gomp_mutex_unlock (&allocator_data->lock);
 277           goto fail;
 278         }
 279       allocator_data->used_pool_size = used_pool_size;
 280       gomp_mutex_unlock (&allocator_data->lock);
 281 #endif
 282       ptr = malloc (new_size);
 283       if (ptr == NULL)
 284         {
 285 #ifdef HAVE_SYNC_BUILTINS
 286           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 287                               MEMMODEL_RELAXED);
 288 #else
 289           gomp_mutex_lock (&allocator_data->lock);
 290           allocator_data->used_pool_size -= new_size;
 291           gomp_mutex_unlock (&allocator_data->lock);
 292 #endif
 293           goto fail;
 294         }
 295     }
 296   else
 297     {
 298       ptr = malloc (new_size);
 299       if (ptr == NULL)
 300         goto fail;
 301     }
 302
 303   if (alignment > sizeof (void *))
 304     ret = (void *) (((uintptr_t) ptr
 305                      + sizeof (struct omp_mem_header)
 306                      + alignment - sizeof (void *)) & ~(alignment - 1));
 307   else
 308     ret = (char *) ptr + sizeof (struct omp_mem_header);
 309   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 310   ((struct omp_mem_header *) ret)[-1].size = new_size;
 311   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 312   return ret;
 313
 314 fail:
 315   if (allocator_data)
 316     {
 317       switch (allocator_data->fallback)
 318         {
 319         case omp_atv_default_mem_fb:
 320           if (alignment > sizeof (void *)
 321               || (allocator_data
 322                   && allocator_data->pool_size < ~(uintptr_t) 0))
 323             {
 324               allocator = omp_default_mem_alloc;
 325               goto retry;
 326             }
 327           /* Otherwise, we've already performed default mem allocation
 328              and if that failed, it won't succeed again (unless it was
 329              intermitent.  Return NULL then, as that is the fallback.  */
 330           break;
 331         case omp_atv_null_fb:
 332           break;
 333         default:
 334         case omp_atv_abort_fb:
 335           gomp_fatal ("Out of memory allocating %lu bytes",
 336                       (unsigned long) size);
 337         case omp_atv_allocator_fb:
 338           allocator = allocator_data->fb_data;
 339           goto retry;
 340         }
 341     }
 342   return NULL;
 343 }
 344
 345 void *
 346 omp_alloc (size_t size, omp_allocator_handle_t allocator)
 347 {
 348   return omp_aligned_alloc (1, size, allocator);
 349 }
 350
 351 /* Like omp_aligned_alloc, but apply on top of that:
 352    "For allocations that arise from this ... the null_fb value of the
 353    fallback allocator trait behaves as if the abort_fb had been specified."  */
 354
 355 void *
 356 GOMP_alloc (size_t alignment, size_t size, uintptr_t allocator)
 357 {
 358   void *ret = omp_aligned_alloc (alignment, size,
 359                                  (omp_allocator_handle_t) allocator);
 360   if (__builtin_expect (ret == NULL, 0) && size)
 361     gomp_fatal ("Out of memory allocating %lu bytes",
 362                 (unsigned long) size);
 363   return ret;
 364 }
 365
 366 void
 367 omp_free (void *ptr, omp_allocator_handle_t allocator)
 368 {
 369   struct omp_mem_header *data;
 370
 371   if (ptr == NULL)
 372     return;
 373   (void) allocator;
 374   data = &((struct omp_mem_header *) ptr)[-1];
 375   if (data->allocator > omp_max_predefined_alloc)
 376     {
 377       struct omp_allocator_data *allocator_data
 378         = (struct omp_allocator_data *) (data->allocator);
 379       if (allocator_data->pool_size < ~(uintptr_t) 0)
 380         {
 381 #ifdef HAVE_SYNC_BUILTINS
 382           __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
 383                               MEMMODEL_RELAXED);
 384 #else
 385           gomp_mutex_lock (&allocator_data->lock);
 386           allocator_data->used_pool_size -= data->size;
 387           gomp_mutex_unlock (&allocator_data->lock);
 388 #endif
 389         }
 390     }
 391   free (data->ptr);
 392 }
 393
 394 ialias (omp_free)
 395
 396 void
 397 GOMP_free (void *ptr, uintptr_t allocator)
 398 {
 399   return omp_free (ptr, (omp_allocator_handle_t) allocator);
 400 }