1 /* Copyright (C) 2020-2024 Free Software Foundation, Inc.
2 Contributed by Jakub Jelinek <jakub@redhat.com>.
4 This file is part of the GNU Offloading and Multi Processing Library
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
26 /* This file contains wrappers for the system allocation routines. Most
27 places in the OpenMP API do not make any provision for failure, so in
28 general we cannot allow memory allocation to fail. */
34 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
38 /* Keeping track whether a Fortran scalar allocatable/pointer has been
39 allocated via 'omp allocators'/'omp allocate'. */
41 struct fort_alloc_splay_tree_key_s
{
45 typedef struct fort_alloc_splay_tree_node_s
*fort_alloc_splay_tree_node
;
46 typedef struct fort_alloc_splay_tree_s
*fort_alloc_splay_tree
;
47 typedef struct fort_alloc_splay_tree_key_s
*fort_alloc_splay_tree_key
;
50 fort_alloc_splay_compare (fort_alloc_splay_tree_key x
, fort_alloc_splay_tree_key y
)
58 #define splay_tree_prefix fort_alloc
59 #define splay_tree_static
60 #include "splay-tree.h"
62 #define splay_tree_prefix fort_alloc
63 #define splay_tree_static
65 #include "splay-tree.h"
67 static struct fort_alloc_splay_tree_s fort_alloc_scalars
;
69 /* Add pointer as being alloced by GOMP_alloc. */
71 GOMP_add_alloc (void *ptr
)
75 fort_alloc_splay_tree_node item
;
76 item
= gomp_malloc (sizeof (struct splay_tree_node_s
));
80 fort_alloc_splay_tree_insert (&fort_alloc_scalars
, item
);
83 /* Remove pointer, either called by FREE or by REALLOC,
84 either of them can change the allocation status. */
86 GOMP_is_alloc (void *ptr
)
88 struct fort_alloc_splay_tree_key_s needle
;
89 fort_alloc_splay_tree_node n
;
91 n
= fort_alloc_splay_tree_lookup_node (&fort_alloc_scalars
, &needle
);
94 fort_alloc_splay_tree_remove (&fort_alloc_scalars
, &n
->key
);
101 #define omp_max_predefined_alloc omp_thread_mem_alloc
103 /* These macros may be overridden in config/<target>/allocator.c.
104 The defaults (no override) are to return NULL for pinned memory requests
105 and pass through to the regular OS calls otherwise.
106 The following definitions (ab)use comma operators to avoid unused
108 #ifndef MEMSPACE_ALLOC
109 #define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
110 (PIN ? NULL : malloc (((void)(MEMSPACE), (SIZE))))
112 #ifndef MEMSPACE_CALLOC
113 #define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
114 (PIN ? NULL : calloc (1, (((void)(MEMSPACE), (SIZE)))))
116 #ifndef MEMSPACE_REALLOC
117 #define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
118 ((PIN) || (OLDPIN) ? NULL \
119 : realloc (ADDR, (((void)(MEMSPACE), (void)(OLDSIZE), (SIZE)))))
121 #ifndef MEMSPACE_FREE
122 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
123 if (PIN) free (((void)(MEMSPACE), (void)(SIZE), (ADDR)))
125 #ifndef MEMSPACE_VALIDATE
126 #define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \
127 (PIN ? 0 : ((void)(MEMSPACE), (void)(ACCESS), 1))
130 /* Map the predefined allocators to the correct memory space.
131 The index to this table is the omp_allocator_handle_t enum value.
132 When the user calls omp_alloc with a predefined allocator this
133 table determines what memory they get. */
134 static const omp_memspace_handle_t predefined_alloc_mapping
[] = {
135 omp_default_mem_space
, /* omp_null_allocator doesn't actually use this. */
136 omp_default_mem_space
, /* omp_default_mem_alloc. */
137 omp_large_cap_mem_space
, /* omp_large_cap_mem_alloc. */
138 omp_const_mem_space
, /* omp_const_mem_alloc. */
139 omp_high_bw_mem_space
, /* omp_high_bw_mem_alloc. */
140 omp_low_lat_mem_space
, /* omp_low_lat_mem_alloc. */
141 omp_low_lat_mem_space
, /* omp_cgroup_mem_alloc (implementation defined). */
142 omp_low_lat_mem_space
, /* omp_pteam_mem_alloc (implementation defined). */
143 omp_low_lat_mem_space
, /* omp_thread_mem_alloc (implementation defined). */
146 #define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
147 _Static_assert (ARRAY_SIZE (predefined_alloc_mapping
)
148 == omp_max_predefined_alloc
+ 1,
149 "predefined_alloc_mapping must match omp_memspace_handle_t");
151 enum gomp_numa_memkind_kind
153 GOMP_MEMKIND_NONE
= 0,
154 #define GOMP_MEMKIND_KINDS \
155 GOMP_MEMKIND_KIND (HBW_INTERLEAVE), \
156 GOMP_MEMKIND_KIND (HBW_PREFERRED), \
157 GOMP_MEMKIND_KIND (DAX_KMEM_ALL), \
158 GOMP_MEMKIND_KIND (DAX_KMEM), \
159 GOMP_MEMKIND_KIND (INTERLEAVE), \
160 GOMP_MEMKIND_KIND (DEFAULT)
161 #define GOMP_MEMKIND_KIND(kind) GOMP_MEMKIND_##kind
163 #undef GOMP_MEMKIND_KIND
165 GOMP_MEMKIND_LIBNUMA
= GOMP_MEMKIND_COUNT
168 struct omp_allocator_data
170 omp_memspace_handle_t memspace
;
171 omp_uintptr_t alignment
;
172 omp_uintptr_t pool_size
;
173 omp_uintptr_t used_pool_size
;
174 omp_allocator_handle_t fb_data
;
175 unsigned int sync_hint
: 8;
176 unsigned int access
: 8;
177 unsigned int fallback
: 8;
178 unsigned int pinned
: 1;
179 unsigned int partition
: 7;
180 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
181 unsigned int memkind
: 8;
183 #ifndef HAVE_SYNC_BUILTINS
188 struct omp_mem_header
192 omp_allocator_handle_t allocator
;
196 struct gomp_libnuma_data
199 void *(*numa_alloc_local
) (size_t);
200 void *(*numa_realloc
) (void *, size_t, size_t);
201 void (*numa_free
) (void *, size_t);
204 struct gomp_memkind_data
206 void *memkind_handle
;
207 void *(*memkind_malloc
) (void *, size_t);
208 void *(*memkind_calloc
) (void *, size_t, size_t);
209 void *(*memkind_realloc
) (void *, void *, size_t);
210 void (*memkind_free
) (void *, void *);
211 int (*memkind_check_available
) (void *);
212 void **kinds
[GOMP_MEMKIND_COUNT
];
215 #ifdef LIBGOMP_USE_LIBNUMA
216 static struct gomp_libnuma_data
*libnuma_data
;
217 static pthread_once_t libnuma_data_once
= PTHREAD_ONCE_INIT
;
220 gomp_init_libnuma (void)
222 void *handle
= dlopen ("libnuma.so.1", RTLD_LAZY
);
223 struct gomp_libnuma_data
*data
;
225 data
= calloc (1, sizeof (struct gomp_libnuma_data
));
234 int (*numa_available
) (void);
236 = (__typeof (numa_available
)) dlsym (handle
, "numa_available");
237 if (!numa_available
|| numa_available () != 0)
245 __atomic_store_n (&libnuma_data
, data
, MEMMODEL_RELEASE
);
248 data
->numa_handle
= handle
;
249 data
->numa_alloc_local
250 = (__typeof (data
->numa_alloc_local
)) dlsym (handle
, "numa_alloc_local");
252 = (__typeof (data
->numa_realloc
)) dlsym (handle
, "numa_realloc");
254 = (__typeof (data
->numa_free
)) dlsym (handle
, "numa_free");
255 __atomic_store_n (&libnuma_data
, data
, MEMMODEL_RELEASE
);
258 static struct gomp_libnuma_data
*
259 gomp_get_libnuma (void)
261 struct gomp_libnuma_data
*data
262 = __atomic_load_n (&libnuma_data
, MEMMODEL_ACQUIRE
);
265 pthread_once (&libnuma_data_once
, gomp_init_libnuma
);
266 return __atomic_load_n (&libnuma_data
, MEMMODEL_ACQUIRE
);
270 #ifdef LIBGOMP_USE_MEMKIND
271 static struct gomp_memkind_data
*memkind_data
;
272 static pthread_once_t memkind_data_once
= PTHREAD_ONCE_INIT
;
275 gomp_init_memkind (void)
277 void *handle
= dlopen ("libmemkind.so.0", RTLD_LAZY
);
278 struct gomp_memkind_data
*data
;
280 static const char *kinds
[] = {
282 #define GOMP_MEMKIND_KIND(kind) "MEMKIND_" #kind
284 #undef GOMP_MEMKIND_KIND
287 data
= calloc (1, sizeof (struct gomp_memkind_data
));
296 __atomic_store_n (&memkind_data
, data
, MEMMODEL_RELEASE
);
299 data
->memkind_handle
= handle
;
301 = (__typeof (data
->memkind_malloc
)) dlsym (handle
, "memkind_malloc");
303 = (__typeof (data
->memkind_calloc
)) dlsym (handle
, "memkind_calloc");
304 data
->memkind_realloc
305 = (__typeof (data
->memkind_realloc
)) dlsym (handle
, "memkind_realloc");
307 = (__typeof (data
->memkind_free
)) dlsym (handle
, "memkind_free");
308 data
->memkind_check_available
309 = (__typeof (data
->memkind_check_available
))
310 dlsym (handle
, "memkind_check_available");
311 if (data
->memkind_malloc
312 && data
->memkind_calloc
313 && data
->memkind_realloc
314 && data
->memkind_free
315 && data
->memkind_check_available
)
316 for (i
= 1; i
< GOMP_MEMKIND_COUNT
; ++i
)
318 data
->kinds
[i
] = (void **) dlsym (handle
, kinds
[i
]);
319 if (data
->kinds
[i
] && data
->memkind_check_available (*data
->kinds
[i
]))
320 data
->kinds
[i
] = NULL
;
322 __atomic_store_n (&memkind_data
, data
, MEMMODEL_RELEASE
);
325 static struct gomp_memkind_data
*
326 gomp_get_memkind (void)
328 struct gomp_memkind_data
*data
329 = __atomic_load_n (&memkind_data
, MEMMODEL_ACQUIRE
);
332 pthread_once (&memkind_data_once
, gomp_init_memkind
);
333 return __atomic_load_n (&memkind_data
, MEMMODEL_ACQUIRE
);
337 omp_allocator_handle_t
338 omp_init_allocator (omp_memspace_handle_t memspace
, int ntraits
,
339 const omp_alloctrait_t traits
[])
341 struct omp_allocator_data data
342 = { memspace
, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended
, omp_atv_all
,
343 omp_atv_default_mem_fb
, omp_atv_false
, omp_atv_environment
,
344 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
348 struct omp_allocator_data
*ret
;
351 if (memspace
> omp_low_lat_mem_space
)
352 return omp_null_allocator
;
353 for (i
= 0; i
< ntraits
; i
++)
354 switch (traits
[i
].key
)
356 case omp_atk_sync_hint
:
357 switch (traits
[i
].value
)
359 case omp_atv_default
:
360 data
.sync_hint
= omp_atv_contended
;
362 case omp_atv_contended
:
363 case omp_atv_uncontended
:
364 case omp_atv_serialized
:
365 case omp_atv_private
:
366 data
.sync_hint
= traits
[i
].value
;
369 return omp_null_allocator
;
372 case omp_atk_alignment
:
373 if (traits
[i
].value
== omp_atv_default
)
378 if ((traits
[i
].value
& (traits
[i
].value
- 1)) != 0
380 return omp_null_allocator
;
381 data
.alignment
= traits
[i
].value
;
384 switch (traits
[i
].value
)
386 case omp_atv_default
:
387 data
.access
= omp_atv_all
;
393 data
.access
= traits
[i
].value
;
396 return omp_null_allocator
;
399 case omp_atk_pool_size
:
400 if (traits
[i
].value
== omp_atv_default
)
401 data
.pool_size
= ~(uintptr_t) 0;
403 data
.pool_size
= traits
[i
].value
;
405 case omp_atk_fallback
:
406 switch (traits
[i
].value
)
408 case omp_atv_default
:
409 data
.fallback
= omp_atv_default_mem_fb
;
411 case omp_atv_default_mem_fb
:
412 case omp_atv_null_fb
:
413 case omp_atv_abort_fb
:
414 case omp_atv_allocator_fb
:
415 data
.fallback
= traits
[i
].value
;
418 return omp_null_allocator
;
421 case omp_atk_fb_data
:
422 data
.fb_data
= traits
[i
].value
;
425 switch (traits
[i
].value
)
427 case omp_atv_default
:
429 data
.pinned
= omp_atv_false
;
432 data
.pinned
= omp_atv_true
;
435 return omp_null_allocator
;
438 case omp_atk_partition
:
439 switch (traits
[i
].value
)
441 case omp_atv_default
:
442 data
.partition
= omp_atv_environment
;
444 case omp_atv_environment
:
445 case omp_atv_nearest
:
446 case omp_atv_blocked
:
447 case omp_atv_interleaved
:
448 data
.partition
= traits
[i
].value
;
451 return omp_null_allocator
;
455 return omp_null_allocator
;
458 if (data
.alignment
< sizeof (void *))
459 data
.alignment
= sizeof (void *);
463 #ifdef LIBGOMP_USE_MEMKIND
464 case omp_high_bw_mem_space
:
465 struct gomp_memkind_data
*memkind_data
;
466 memkind_data
= gomp_get_memkind ();
467 if (data
.partition
== omp_atv_interleaved
468 && memkind_data
->kinds
[GOMP_MEMKIND_HBW_INTERLEAVE
])
470 data
.memkind
= GOMP_MEMKIND_HBW_INTERLEAVE
;
473 else if (memkind_data
->kinds
[GOMP_MEMKIND_HBW_PREFERRED
])
475 data
.memkind
= GOMP_MEMKIND_HBW_PREFERRED
;
479 case omp_large_cap_mem_space
:
480 memkind_data
= gomp_get_memkind ();
481 if (memkind_data
->kinds
[GOMP_MEMKIND_DAX_KMEM_ALL
])
482 data
.memkind
= GOMP_MEMKIND_DAX_KMEM_ALL
;
483 else if (memkind_data
->kinds
[GOMP_MEMKIND_DAX_KMEM
])
484 data
.memkind
= GOMP_MEMKIND_DAX_KMEM
;
488 #ifdef LIBGOMP_USE_MEMKIND
489 if (data
.partition
== omp_atv_interleaved
)
491 memkind_data
= gomp_get_memkind ();
492 if (memkind_data
->kinds
[GOMP_MEMKIND_INTERLEAVE
])
493 data
.memkind
= GOMP_MEMKIND_INTERLEAVE
;
499 #ifdef LIBGOMP_USE_LIBNUMA
500 if (data
.memkind
== GOMP_MEMKIND_NONE
&& data
.partition
== omp_atv_nearest
)
502 libnuma_data
= gomp_get_libnuma ();
503 if (libnuma_data
->numa_alloc_local
!= NULL
)
504 data
.memkind
= GOMP_MEMKIND_LIBNUMA
;
508 /* Reject unsupported memory spaces. */
509 if (!MEMSPACE_VALIDATE (data
.memspace
, data
.access
, data
.pinned
))
510 return omp_null_allocator
;
512 ret
= gomp_malloc (sizeof (struct omp_allocator_data
));
514 #ifndef HAVE_SYNC_BUILTINS
515 gomp_mutex_init (&ret
->lock
);
517 return (omp_allocator_handle_t
) ret
;
521 omp_destroy_allocator (omp_allocator_handle_t allocator
)
523 if (allocator
!= omp_null_allocator
)
525 #ifndef HAVE_SYNC_BUILTINS
526 gomp_mutex_destroy (&((struct omp_allocator_data
*) allocator
)->lock
);
528 free ((void *) allocator
);
532 ialias (omp_init_allocator
)
533 ialias (omp_destroy_allocator
)
536 omp_aligned_alloc (size_t alignment
, size_t size
,
537 omp_allocator_handle_t allocator
)
539 struct omp_allocator_data
*allocator_data
;
540 size_t new_size
, new_alignment
;
542 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
543 enum gomp_numa_memkind_kind memkind
;
546 if (__builtin_expect (size
== 0, 0))
550 new_alignment
= alignment
;
551 if (allocator
== omp_null_allocator
)
553 struct gomp_thread
*thr
= gomp_thread ();
554 if (thr
->ts
.def_allocator
== omp_null_allocator
)
555 thr
->ts
.def_allocator
= gomp_def_allocator
;
556 allocator
= (omp_allocator_handle_t
) thr
->ts
.def_allocator
;
559 if (allocator
> omp_max_predefined_alloc
)
561 allocator_data
= (struct omp_allocator_data
*) allocator
;
562 if (new_alignment
< allocator_data
->alignment
)
563 new_alignment
= allocator_data
->alignment
;
564 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
565 memkind
= allocator_data
->memkind
;
570 allocator_data
= NULL
;
571 if (new_alignment
< sizeof (void *))
572 new_alignment
= sizeof (void *);
573 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
574 memkind
= GOMP_MEMKIND_NONE
;
576 #ifdef LIBGOMP_USE_MEMKIND
577 if (allocator
== omp_high_bw_mem_alloc
)
578 memkind
= GOMP_MEMKIND_HBW_PREFERRED
;
579 else if (allocator
== omp_large_cap_mem_alloc
)
580 memkind
= GOMP_MEMKIND_DAX_KMEM_ALL
;
583 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
584 if (!memkind_data
->kinds
[memkind
])
585 memkind
= GOMP_MEMKIND_NONE
;
590 new_size
= sizeof (struct omp_mem_header
);
591 if (new_alignment
> sizeof (void *))
592 new_size
+= new_alignment
- sizeof (void *);
593 if (__builtin_add_overflow (size
, new_size
, &new_size
))
595 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
596 if (allocator
== omp_low_lat_mem_alloc
)
600 if (__builtin_expect (allocator_data
601 && allocator_data
->pool_size
< ~(uintptr_t) 0, 0))
603 uintptr_t used_pool_size
;
604 if (new_size
> allocator_data
->pool_size
)
606 #ifdef HAVE_SYNC_BUILTINS
607 used_pool_size
= __atomic_load_n (&allocator_data
->used_pool_size
,
611 uintptr_t new_pool_size
;
612 if (__builtin_add_overflow (used_pool_size
, new_size
,
614 || new_pool_size
> allocator_data
->pool_size
)
616 if (__atomic_compare_exchange_n (&allocator_data
->used_pool_size
,
617 &used_pool_size
, new_pool_size
,
618 true, MEMMODEL_RELAXED
,
624 gomp_mutex_lock (&allocator_data
->lock
);
625 if (__builtin_add_overflow (allocator_data
->used_pool_size
, new_size
,
627 || used_pool_size
> allocator_data
->pool_size
)
629 gomp_mutex_unlock (&allocator_data
->lock
);
632 allocator_data
->used_pool_size
= used_pool_size
;
633 gomp_mutex_unlock (&allocator_data
->lock
);
635 #ifdef LIBGOMP_USE_LIBNUMA
636 if (memkind
== GOMP_MEMKIND_LIBNUMA
)
637 ptr
= libnuma_data
->numa_alloc_local (new_size
);
638 # ifdef LIBGOMP_USE_MEMKIND
642 #ifdef LIBGOMP_USE_MEMKIND
645 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
646 void *kind
= *memkind_data
->kinds
[memkind
];
647 ptr
= memkind_data
->memkind_malloc (kind
, new_size
);
651 ptr
= MEMSPACE_ALLOC (allocator_data
->memspace
, new_size
,
652 allocator_data
->pinned
);
655 #ifdef HAVE_SYNC_BUILTINS
656 __atomic_add_fetch (&allocator_data
->used_pool_size
, -new_size
,
659 gomp_mutex_lock (&allocator_data
->lock
);
660 allocator_data
->used_pool_size
-= new_size
;
661 gomp_mutex_unlock (&allocator_data
->lock
);
668 #ifdef LIBGOMP_USE_LIBNUMA
669 if (memkind
== GOMP_MEMKIND_LIBNUMA
)
670 ptr
= libnuma_data
->numa_alloc_local (new_size
);
671 # ifdef LIBGOMP_USE_MEMKIND
675 #ifdef LIBGOMP_USE_MEMKIND
678 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
679 void *kind
= *memkind_data
->kinds
[memkind
];
680 ptr
= memkind_data
->memkind_malloc (kind
, new_size
);
685 omp_memspace_handle_t memspace
;
686 memspace
= (allocator_data
687 ? allocator_data
->memspace
688 : predefined_alloc_mapping
[allocator
]);
689 ptr
= MEMSPACE_ALLOC (memspace
, new_size
,
690 allocator_data
&& allocator_data
->pinned
);
696 if (new_alignment
> sizeof (void *))
697 ret
= (void *) (((uintptr_t) ptr
698 + sizeof (struct omp_mem_header
)
699 + new_alignment
- sizeof (void *))
700 & ~(new_alignment
- 1));
702 ret
= (char *) ptr
+ sizeof (struct omp_mem_header
);
703 ((struct omp_mem_header
*) ret
)[-1].ptr
= ptr
;
704 ((struct omp_mem_header
*) ret
)[-1].size
= new_size
;
705 ((struct omp_mem_header
*) ret
)[-1].allocator
= allocator
;
709 int fallback
= (allocator_data
710 ? allocator_data
->fallback
711 : allocator
== omp_default_mem_alloc
713 : omp_atv_default_mem_fb
);
716 case omp_atv_default_mem_fb
:
717 allocator
= omp_default_mem_alloc
;
719 case omp_atv_null_fb
:
722 case omp_atv_abort_fb
:
723 gomp_fatal ("Out of memory allocating %lu bytes",
724 (unsigned long) size
);
725 case omp_atv_allocator_fb
:
726 allocator
= allocator_data
->fb_data
;
732 ialias (omp_aligned_alloc
)
735 omp_alloc (size_t size
, omp_allocator_handle_t allocator
)
737 return ialias_call (omp_aligned_alloc
) (1, size
, allocator
);
740 /* Like omp_aligned_alloc, but apply on top of that:
741 "For allocations that arise from this ... the null_fb value of the
742 fallback allocator trait behaves as if the abort_fb had been specified." */
745 GOMP_alloc (size_t alignment
, size_t size
, uintptr_t allocator
)
748 = ialias_call (omp_aligned_alloc
) (alignment
, size
,
749 (omp_allocator_handle_t
) allocator
);
750 if (__builtin_expect (ret
== NULL
, 0) && size
)
751 gomp_fatal ("Out of memory allocating %lu bytes",
752 (unsigned long) size
);
757 omp_free (void *ptr
, omp_allocator_handle_t allocator
)
759 struct omp_mem_header
*data
;
760 omp_memspace_handle_t memspace
= omp_default_mem_space
;
766 data
= &((struct omp_mem_header
*) ptr
)[-1];
767 if (data
->allocator
> omp_max_predefined_alloc
)
769 struct omp_allocator_data
*allocator_data
770 = (struct omp_allocator_data
*) (data
->allocator
);
771 if (allocator_data
->pool_size
< ~(uintptr_t) 0)
773 #ifdef HAVE_SYNC_BUILTINS
774 __atomic_add_fetch (&allocator_data
->used_pool_size
, -data
->size
,
777 gomp_mutex_lock (&allocator_data
->lock
);
778 allocator_data
->used_pool_size
-= data
->size
;
779 gomp_mutex_unlock (&allocator_data
->lock
);
782 #ifdef LIBGOMP_USE_LIBNUMA
783 if (allocator_data
->memkind
== GOMP_MEMKIND_LIBNUMA
)
785 libnuma_data
->numa_free (data
->ptr
, data
->size
);
788 # ifdef LIBGOMP_USE_MEMKIND
792 #ifdef LIBGOMP_USE_MEMKIND
793 if (allocator_data
->memkind
)
795 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
796 void *kind
= *memkind_data
->kinds
[allocator_data
->memkind
];
797 memkind_data
->memkind_free (kind
, data
->ptr
);
802 memspace
= allocator_data
->memspace
;
803 pinned
= allocator_data
->pinned
;
807 #ifdef LIBGOMP_USE_MEMKIND
808 enum gomp_numa_memkind_kind memkind
= GOMP_MEMKIND_NONE
;
809 if (data
->allocator
== omp_high_bw_mem_alloc
)
810 memkind
= GOMP_MEMKIND_HBW_PREFERRED
;
811 else if (data
->allocator
== omp_large_cap_mem_alloc
)
812 memkind
= GOMP_MEMKIND_DAX_KMEM_ALL
;
815 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
816 if (memkind_data
->kinds
[memkind
])
818 void *kind
= *memkind_data
->kinds
[memkind
];
819 memkind_data
->memkind_free (kind
, data
->ptr
);
825 memspace
= predefined_alloc_mapping
[data
->allocator
];
828 MEMSPACE_FREE (memspace
, data
->ptr
, data
->size
, pinned
);
834 GOMP_free (void *ptr
, uintptr_t allocator
)
836 return ialias_call (omp_free
) (ptr
, (omp_allocator_handle_t
) allocator
);
840 omp_aligned_calloc (size_t alignment
, size_t nmemb
, size_t size
,
841 omp_allocator_handle_t allocator
)
843 struct omp_allocator_data
*allocator_data
;
844 size_t new_size
, size_temp
, new_alignment
;
846 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
847 enum gomp_numa_memkind_kind memkind
;
850 if (__builtin_expect (size
== 0 || nmemb
== 0, 0))
854 new_alignment
= alignment
;
855 if (allocator
== omp_null_allocator
)
857 struct gomp_thread
*thr
= gomp_thread ();
858 if (thr
->ts
.def_allocator
== omp_null_allocator
)
859 thr
->ts
.def_allocator
= gomp_def_allocator
;
860 allocator
= (omp_allocator_handle_t
) thr
->ts
.def_allocator
;
863 if (allocator
> omp_max_predefined_alloc
)
865 allocator_data
= (struct omp_allocator_data
*) allocator
;
866 if (new_alignment
< allocator_data
->alignment
)
867 new_alignment
= allocator_data
->alignment
;
868 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
869 memkind
= allocator_data
->memkind
;
874 allocator_data
= NULL
;
875 if (new_alignment
< sizeof (void *))
876 new_alignment
= sizeof (void *);
877 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
878 memkind
= GOMP_MEMKIND_NONE
;
880 #ifdef LIBGOMP_USE_MEMKIND
881 if (allocator
== omp_high_bw_mem_alloc
)
882 memkind
= GOMP_MEMKIND_HBW_PREFERRED
;
883 else if (allocator
== omp_large_cap_mem_alloc
)
884 memkind
= GOMP_MEMKIND_DAX_KMEM_ALL
;
887 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
888 if (!memkind_data
->kinds
[memkind
])
889 memkind
= GOMP_MEMKIND_NONE
;
894 new_size
= sizeof (struct omp_mem_header
);
895 if (new_alignment
> sizeof (void *))
896 new_size
+= new_alignment
- sizeof (void *);
897 if (__builtin_mul_overflow (size
, nmemb
, &size_temp
))
899 if (__builtin_add_overflow (size_temp
, new_size
, &new_size
))
901 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
902 if (allocator
== omp_low_lat_mem_alloc
)
906 if (__builtin_expect (allocator_data
907 && allocator_data
->pool_size
< ~(uintptr_t) 0, 0))
909 uintptr_t used_pool_size
;
910 if (new_size
> allocator_data
->pool_size
)
912 #ifdef HAVE_SYNC_BUILTINS
913 used_pool_size
= __atomic_load_n (&allocator_data
->used_pool_size
,
917 uintptr_t new_pool_size
;
918 if (__builtin_add_overflow (used_pool_size
, new_size
,
920 || new_pool_size
> allocator_data
->pool_size
)
922 if (__atomic_compare_exchange_n (&allocator_data
->used_pool_size
,
923 &used_pool_size
, new_pool_size
,
924 true, MEMMODEL_RELAXED
,
930 gomp_mutex_lock (&allocator_data
->lock
);
931 if (__builtin_add_overflow (allocator_data
->used_pool_size
, new_size
,
933 || used_pool_size
> allocator_data
->pool_size
)
935 gomp_mutex_unlock (&allocator_data
->lock
);
938 allocator_data
->used_pool_size
= used_pool_size
;
939 gomp_mutex_unlock (&allocator_data
->lock
);
941 #ifdef LIBGOMP_USE_LIBNUMA
942 if (memkind
== GOMP_MEMKIND_LIBNUMA
)
943 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
944 memory that is initialized to zero. */
945 ptr
= libnuma_data
->numa_alloc_local (new_size
);
946 # ifdef LIBGOMP_USE_MEMKIND
950 #ifdef LIBGOMP_USE_MEMKIND
953 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
954 void *kind
= *memkind_data
->kinds
[memkind
];
955 ptr
= memkind_data
->memkind_calloc (kind
, 1, new_size
);
959 ptr
= MEMSPACE_CALLOC (allocator_data
->memspace
, new_size
,
960 allocator_data
->pinned
);
963 #ifdef HAVE_SYNC_BUILTINS
964 __atomic_add_fetch (&allocator_data
->used_pool_size
, -new_size
,
967 gomp_mutex_lock (&allocator_data
->lock
);
968 allocator_data
->used_pool_size
-= new_size
;
969 gomp_mutex_unlock (&allocator_data
->lock
);
976 #ifdef LIBGOMP_USE_LIBNUMA
977 if (memkind
== GOMP_MEMKIND_LIBNUMA
)
978 /* numa_alloc_local uses mmap with MAP_ANONYMOUS, returning
979 memory that is initialized to zero. */
980 ptr
= libnuma_data
->numa_alloc_local (new_size
);
981 # ifdef LIBGOMP_USE_MEMKIND
985 #ifdef LIBGOMP_USE_MEMKIND
988 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
989 void *kind
= *memkind_data
->kinds
[memkind
];
990 ptr
= memkind_data
->memkind_calloc (kind
, 1, new_size
);
995 omp_memspace_handle_t memspace
;
996 memspace
= (allocator_data
997 ? allocator_data
->memspace
998 : predefined_alloc_mapping
[allocator
]);
999 ptr
= MEMSPACE_CALLOC (memspace
, new_size
,
1000 allocator_data
&& allocator_data
->pinned
);
1006 if (new_alignment
> sizeof (void *))
1007 ret
= (void *) (((uintptr_t) ptr
1008 + sizeof (struct omp_mem_header
)
1009 + new_alignment
- sizeof (void *))
1010 & ~(new_alignment
- 1));
1012 ret
= (char *) ptr
+ sizeof (struct omp_mem_header
);
1013 ((struct omp_mem_header
*) ret
)[-1].ptr
= ptr
;
1014 ((struct omp_mem_header
*) ret
)[-1].size
= new_size
;
1015 ((struct omp_mem_header
*) ret
)[-1].allocator
= allocator
;
1019 int fallback
= (allocator_data
1020 ? allocator_data
->fallback
1021 : allocator
== omp_default_mem_alloc
1023 : omp_atv_default_mem_fb
);
1026 case omp_atv_default_mem_fb
:
1027 allocator
= omp_default_mem_alloc
;
1029 case omp_atv_null_fb
:
1032 case omp_atv_abort_fb
:
1033 gomp_fatal ("Out of memory allocating %lu bytes",
1034 (unsigned long) (size
* nmemb
));
1035 case omp_atv_allocator_fb
:
1036 allocator
= allocator_data
->fb_data
;
1042 ialias (omp_aligned_calloc
)
1045 omp_calloc (size_t nmemb
, size_t size
, omp_allocator_handle_t allocator
)
1047 return ialias_call (omp_aligned_calloc
) (1, nmemb
, size
, allocator
);
1051 omp_realloc (void *ptr
, size_t size
, omp_allocator_handle_t allocator
,
1052 omp_allocator_handle_t free_allocator
)
1054 struct omp_allocator_data
*allocator_data
, *free_allocator_data
;
1055 size_t new_size
, old_size
, new_alignment
, old_alignment
;
1056 void *new_ptr
, *ret
;
1057 struct omp_mem_header
*data
;
1058 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1059 enum gomp_numa_memkind_kind memkind
, free_memkind
;
1062 if (__builtin_expect (ptr
== NULL
, 0))
1063 return ialias_call (omp_aligned_alloc
) (1, size
, allocator
);
1065 if (__builtin_expect (size
== 0, 0))
1067 ialias_call (omp_free
) (ptr
, free_allocator
);
1071 data
= &((struct omp_mem_header
*) ptr
)[-1];
1072 free_allocator
= data
->allocator
;
1075 new_alignment
= sizeof (void *);
1076 if (allocator
== omp_null_allocator
)
1077 allocator
= free_allocator
;
1079 if (allocator
> omp_max_predefined_alloc
)
1081 allocator_data
= (struct omp_allocator_data
*) allocator
;
1082 if (new_alignment
< allocator_data
->alignment
)
1083 new_alignment
= allocator_data
->alignment
;
1084 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1085 memkind
= allocator_data
->memkind
;
1090 allocator_data
= NULL
;
1091 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1092 memkind
= GOMP_MEMKIND_NONE
;
1094 #ifdef LIBGOMP_USE_MEMKIND
1095 if (allocator
== omp_high_bw_mem_alloc
)
1096 memkind
= GOMP_MEMKIND_HBW_PREFERRED
;
1097 else if (allocator
== omp_large_cap_mem_alloc
)
1098 memkind
= GOMP_MEMKIND_DAX_KMEM_ALL
;
1101 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
1102 if (!memkind_data
->kinds
[memkind
])
1103 memkind
= GOMP_MEMKIND_NONE
;
1107 if (free_allocator
> omp_max_predefined_alloc
)
1109 free_allocator_data
= (struct omp_allocator_data
*) free_allocator
;
1110 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1111 free_memkind
= free_allocator_data
->memkind
;
1116 free_allocator_data
= NULL
;
1117 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1118 free_memkind
= GOMP_MEMKIND_NONE
;
1120 #ifdef LIBGOMP_USE_MEMKIND
1121 if (free_allocator
== omp_high_bw_mem_alloc
)
1122 free_memkind
= GOMP_MEMKIND_HBW_PREFERRED
;
1123 else if (free_allocator
== omp_large_cap_mem_alloc
)
1124 free_memkind
= GOMP_MEMKIND_DAX_KMEM_ALL
;
1127 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
1128 if (!memkind_data
->kinds
[free_memkind
])
1129 free_memkind
= GOMP_MEMKIND_NONE
;
1133 old_alignment
= (uintptr_t) ptr
- (uintptr_t) (data
->ptr
);
1135 new_size
= sizeof (struct omp_mem_header
);
1136 if (new_alignment
> sizeof (void *))
1137 new_size
+= new_alignment
- sizeof (void *);
1138 if (__builtin_add_overflow (size
, new_size
, &new_size
))
1140 old_size
= data
->size
;
1141 #ifdef OMP_LOW_LAT_MEM_ALLOC_INVALID
1142 if (allocator
== omp_low_lat_mem_alloc
)
1146 if (__builtin_expect (allocator_data
1147 && allocator_data
->pool_size
< ~(uintptr_t) 0, 0))
1149 uintptr_t used_pool_size
;
1150 size_t prev_size
= 0;
1151 /* Check if we can use realloc. Don't use it if extra alignment
1152 was used previously or newly, because realloc might return a pointer
1153 with different alignment and then we'd need to memmove the data
1155 if (free_allocator_data
1156 && free_allocator_data
== allocator_data
1157 && new_alignment
== sizeof (void *)
1158 && old_alignment
== sizeof (struct omp_mem_header
))
1159 prev_size
= old_size
;
1160 if (new_size
> prev_size
1161 && new_size
- prev_size
> allocator_data
->pool_size
)
1163 #ifdef HAVE_SYNC_BUILTINS
1164 used_pool_size
= __atomic_load_n (&allocator_data
->used_pool_size
,
1168 uintptr_t new_pool_size
;
1169 if (new_size
> prev_size
)
1171 if (__builtin_add_overflow (used_pool_size
, new_size
- prev_size
,
1173 || new_pool_size
> allocator_data
->pool_size
)
1177 new_pool_size
= used_pool_size
+ new_size
- prev_size
;
1178 if (__atomic_compare_exchange_n (&allocator_data
->used_pool_size
,
1179 &used_pool_size
, new_pool_size
,
1180 true, MEMMODEL_RELAXED
,
1186 gomp_mutex_lock (&allocator_data
->lock
);
1187 if (new_size
> prev_size
)
1189 if (__builtin_add_overflow (allocator_data
->used_pool_size
,
1190 new_size
- prev_size
,
1192 || used_pool_size
> allocator_data
->pool_size
)
1194 gomp_mutex_unlock (&allocator_data
->lock
);
1199 used_pool_size
= (allocator_data
->used_pool_size
1200 + new_size
- prev_size
);
1201 allocator_data
->used_pool_size
= used_pool_size
;
1202 gomp_mutex_unlock (&allocator_data
->lock
);
1204 #ifdef LIBGOMP_USE_LIBNUMA
1205 if (memkind
== GOMP_MEMKIND_LIBNUMA
)
1208 new_ptr
= libnuma_data
->numa_realloc (data
->ptr
, data
->size
,
1211 new_ptr
= libnuma_data
->numa_alloc_local (new_size
);
1213 # ifdef LIBGOMP_USE_MEMKIND
1217 #ifdef LIBGOMP_USE_MEMKIND
1220 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
1221 void *kind
= *memkind_data
->kinds
[memkind
];
1223 new_ptr
= memkind_data
->memkind_realloc (kind
, data
->ptr
,
1226 new_ptr
= memkind_data
->memkind_malloc (kind
, new_size
);
1231 new_ptr
= MEMSPACE_REALLOC (allocator_data
->memspace
, data
->ptr
,
1232 data
->size
, new_size
,
1233 (free_allocator_data
1234 && free_allocator_data
->pinned
),
1235 allocator_data
->pinned
);
1237 new_ptr
= MEMSPACE_ALLOC (allocator_data
->memspace
, new_size
,
1238 allocator_data
->pinned
);
1239 if (new_ptr
== NULL
)
1241 #ifdef HAVE_SYNC_BUILTINS
1242 __atomic_add_fetch (&allocator_data
->used_pool_size
,
1243 prev_size
- new_size
,
1246 gomp_mutex_lock (&allocator_data
->lock
);
1247 allocator_data
->used_pool_size
-= new_size
- prev_size
;
1248 gomp_mutex_unlock (&allocator_data
->lock
);
1254 ret
= (char *) new_ptr
+ sizeof (struct omp_mem_header
);
1255 ((struct omp_mem_header
*) ret
)[-1].ptr
= new_ptr
;
1256 ((struct omp_mem_header
*) ret
)[-1].size
= new_size
;
1257 ((struct omp_mem_header
*) ret
)[-1].allocator
= allocator
;
1261 else if (new_alignment
== sizeof (void *)
1262 && old_alignment
== sizeof (struct omp_mem_header
)
1263 #if defined(LIBGOMP_USE_MEMKIND) || defined(LIBGOMP_USE_LIBNUMA)
1264 && memkind
== free_memkind
1266 && (free_allocator_data
== NULL
1267 || free_allocator_data
->pool_size
== ~(uintptr_t) 0))
1269 #ifdef LIBGOMP_USE_LIBNUMA
1270 if (memkind
== GOMP_MEMKIND_LIBNUMA
)
1271 new_ptr
= libnuma_data
->numa_realloc (data
->ptr
, data
->size
, new_size
);
1272 # ifdef LIBGOMP_USE_MEMKIND
1276 #ifdef LIBGOMP_USE_MEMKIND
1279 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
1280 void *kind
= *memkind_data
->kinds
[memkind
];
1281 new_ptr
= memkind_data
->memkind_realloc (kind
, data
->ptr
,
1287 omp_memspace_handle_t memspace
;
1288 memspace
= (allocator_data
1289 ? allocator_data
->memspace
1290 : predefined_alloc_mapping
[allocator
]);
1291 new_ptr
= MEMSPACE_REALLOC (memspace
, data
->ptr
, data
->size
, new_size
,
1292 (free_allocator_data
1293 && free_allocator_data
->pinned
),
1294 allocator_data
&& allocator_data
->pinned
);
1296 if (new_ptr
== NULL
)
1299 ret
= (char *) new_ptr
+ sizeof (struct omp_mem_header
);
1300 ((struct omp_mem_header
*) ret
)[-1].ptr
= new_ptr
;
1301 ((struct omp_mem_header
*) ret
)[-1].size
= new_size
;
1302 ((struct omp_mem_header
*) ret
)[-1].allocator
= allocator
;
1307 #ifdef LIBGOMP_USE_LIBNUMA
1308 if (memkind
== GOMP_MEMKIND_LIBNUMA
)
1309 new_ptr
= libnuma_data
->numa_alloc_local (new_size
);
1310 # ifdef LIBGOMP_USE_MEMKIND
1314 #ifdef LIBGOMP_USE_MEMKIND
1317 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
1318 void *kind
= *memkind_data
->kinds
[memkind
];
1319 new_ptr
= memkind_data
->memkind_malloc (kind
, new_size
);
1324 omp_memspace_handle_t memspace
;
1325 memspace
= (allocator_data
1326 ? allocator_data
->memspace
1327 : predefined_alloc_mapping
[allocator
]);
1328 new_ptr
= MEMSPACE_ALLOC (memspace
, new_size
,
1329 allocator_data
&& allocator_data
->pinned
);
1331 if (new_ptr
== NULL
)
1335 if (new_alignment
> sizeof (void *))
1336 ret
= (void *) (((uintptr_t) new_ptr
1337 + sizeof (struct omp_mem_header
)
1338 + new_alignment
- sizeof (void *))
1339 & ~(new_alignment
- 1));
1341 ret
= (char *) new_ptr
+ sizeof (struct omp_mem_header
);
1342 ((struct omp_mem_header
*) ret
)[-1].ptr
= new_ptr
;
1343 ((struct omp_mem_header
*) ret
)[-1].size
= new_size
;
1344 ((struct omp_mem_header
*) ret
)[-1].allocator
= allocator
;
1345 if (old_size
- old_alignment
< size
)
1346 size
= old_size
- old_alignment
;
1347 memcpy (ret
, ptr
, size
);
1348 if (__builtin_expect (free_allocator_data
1349 && free_allocator_data
->pool_size
< ~(uintptr_t) 0, 0))
1351 #ifdef HAVE_SYNC_BUILTINS
1352 __atomic_add_fetch (&free_allocator_data
->used_pool_size
, -data
->size
,
1355 gomp_mutex_lock (&free_allocator_data
->lock
);
1356 free_allocator_data
->used_pool_size
-= data
->size
;
1357 gomp_mutex_unlock (&free_allocator_data
->lock
);
1360 #ifdef LIBGOMP_USE_LIBNUMA
1361 if (free_memkind
== GOMP_MEMKIND_LIBNUMA
)
1363 libnuma_data
->numa_free (data
->ptr
, data
->size
);
1366 # ifdef LIBGOMP_USE_MEMKIND
1370 #ifdef LIBGOMP_USE_MEMKIND
1373 struct gomp_memkind_data
*memkind_data
= gomp_get_memkind ();
1374 void *kind
= *memkind_data
->kinds
[free_memkind
];
1375 memkind_data
->memkind_free (kind
, data
->ptr
);
1380 omp_memspace_handle_t was_memspace
;
1381 was_memspace
= (free_allocator_data
1382 ? free_allocator_data
->memspace
1383 : predefined_alloc_mapping
[free_allocator
]);
1384 int was_pinned
= (free_allocator_data
&& free_allocator_data
->pinned
);
1385 MEMSPACE_FREE (was_memspace
, data
->ptr
, data
->size
, was_pinned
);
1390 int fallback
= (allocator_data
1391 ? allocator_data
->fallback
1392 : allocator
== omp_default_mem_alloc
1394 : omp_atv_default_mem_fb
);
1397 case omp_atv_default_mem_fb
:
1398 allocator
= omp_default_mem_alloc
;
1400 case omp_atv_null_fb
:
1403 case omp_atv_abort_fb
:
1404 gomp_fatal ("Out of memory allocating %lu bytes",
1405 (unsigned long) size
);
1406 case omp_atv_allocator_fb
:
1407 allocator
= allocator_data
->fb_data
;