Remove powerpc, sparc fdim inlines (bug 22987).
[glibc.git] / malloc / arena.c
blob37183cfb6ab5d0735cc82759626670aff3832cd0
1 /* Malloc implementation for multiple threads without lock contention.
2 Copyright (C) 2001-2018 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Wolfram Gloger <wg@malloc.de>, 2001.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public License as
8 published by the Free Software Foundation; either version 2.1 of the
9 License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; see the file COPYING.LIB. If
18 not, see <http://www.gnu.org/licenses/>. */
20 #include <stdbool.h>
22 #if HAVE_TUNABLES
23 # define TUNABLE_NAMESPACE malloc
24 #endif
25 #include <elf/dl-tunables.h>
27 /* Compile-time constants. */
29 #define HEAP_MIN_SIZE (32 * 1024)
30 #ifndef HEAP_MAX_SIZE
31 # ifdef DEFAULT_MMAP_THRESHOLD_MAX
32 # define HEAP_MAX_SIZE (2 * DEFAULT_MMAP_THRESHOLD_MAX)
33 # else
34 # define HEAP_MAX_SIZE (1024 * 1024) /* must be a power of two */
35 # endif
36 #endif
38 /* HEAP_MIN_SIZE and HEAP_MAX_SIZE limit the size of mmap()ed heaps
39 that are dynamically created for multi-threaded programs. The
40 maximum size must be a power of two, for fast determination of
41 which heap belongs to a chunk. It should be much larger than the
42 mmap threshold, so that requests with a size just below that
43 threshold can be fulfilled without creating too many heaps. */
45 /***************************************************************************/
47 #define top(ar_ptr) ((ar_ptr)->top)
49 /* A heap is a single contiguous memory region holding (coalesceable)
50 malloc_chunks. It is allocated with mmap() and always starts at an
51 address aligned to HEAP_MAX_SIZE. */
53 typedef struct _heap_info
55 mstate ar_ptr; /* Arena for this heap. */
56 struct _heap_info *prev; /* Previous heap. */
57 size_t size; /* Current size in bytes. */
58 size_t mprotect_size; /* Size in bytes that has been mprotected
59 PROT_READ|PROT_WRITE. */
60 /* Make sure the following data is properly aligned, particularly
61 that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of
62 MALLOC_ALIGNMENT. */
63 char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK];
64 } heap_info;
66 /* Get a compile-time error if the heap_info padding is not correct
67 to make alignment work as expected in sYSMALLOc. */
68 extern int sanity_check_heap_info_alignment[(sizeof (heap_info)
69 + 2 * SIZE_SZ) % MALLOC_ALIGNMENT
70 ? -1 : 1];
72 /* Thread specific data. */
74 static __thread mstate thread_arena attribute_tls_model_ie;
76 /* Arena free list. free_list_lock synchronizes access to the
77 free_list variable below, and the next_free and attached_threads
78 members of struct malloc_state objects. No other locks must be
79 acquired after free_list_lock has been acquired. */
81 __libc_lock_define_initialized (static, free_list_lock);
82 static size_t narenas = 1;
83 static mstate free_list;
85 /* list_lock prevents concurrent writes to the next member of struct
86 malloc_state objects.
88 Read access to the next member is supposed to synchronize with the
89 atomic_write_barrier and the write to the next member in
90 _int_new_arena. This suffers from data races; see the FIXME
91 comments in _int_new_arena and reused_arena.
93 list_lock also prevents concurrent forks. At the time list_lock is
94 acquired, no arena lock must have been acquired, but it is
95 permitted to acquire arena locks subsequently, while list_lock is
96 acquired. */
97 __libc_lock_define_initialized (static, list_lock);
99 /* Already initialized? */
100 int __malloc_initialized = -1;
102 /**************************************************************************/
105 /* arena_get() acquires an arena and locks the corresponding mutex.
106 First, try the one last locked successfully by this thread. (This
107 is the common case and handled with a macro for speed.) Then, loop
108 once over the circularly linked list of arenas. If no arena is
109 readily available, create a new one. In this latter case, `size'
110 is just a hint as to how much memory will be required immediately
111 in the new arena. */
113 #define arena_get(ptr, size) do { \
114 ptr = thread_arena; \
115 arena_lock (ptr, size); \
116 } while (0)
118 #define arena_lock(ptr, size) do { \
119 if (ptr) \
120 __libc_lock_lock (ptr->mutex); \
121 else \
122 ptr = arena_get2 ((size), NULL); \
123 } while (0)
125 /* find the heap and corresponding arena for a given ptr */
127 #define heap_for_ptr(ptr) \
128 ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1)))
129 #define arena_for_chunk(ptr) \
130 (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr)
133 /**************************************************************************/
135 /* atfork support. */
137 /* The following three functions are called around fork from a
138 multi-threaded process. We do not use the general fork handler
139 mechanism to make sure that our handlers are the last ones being
140 called, so that other fork handlers can use the malloc
141 subsystem. */
143 void
144 __malloc_fork_lock_parent (void)
146 if (__malloc_initialized < 1)
147 return;
149 /* We do not acquire free_list_lock here because we completely
150 reconstruct free_list in __malloc_fork_unlock_child. */
152 __libc_lock_lock (list_lock);
154 for (mstate ar_ptr = &main_arena;; )
156 __libc_lock_lock (ar_ptr->mutex);
157 ar_ptr = ar_ptr->next;
158 if (ar_ptr == &main_arena)
159 break;
163 void
164 __malloc_fork_unlock_parent (void)
166 if (__malloc_initialized < 1)
167 return;
169 for (mstate ar_ptr = &main_arena;; )
171 __libc_lock_unlock (ar_ptr->mutex);
172 ar_ptr = ar_ptr->next;
173 if (ar_ptr == &main_arena)
174 break;
176 __libc_lock_unlock (list_lock);
179 void
180 __malloc_fork_unlock_child (void)
182 if (__malloc_initialized < 1)
183 return;
185 /* Push all arenas to the free list, except thread_arena, which is
186 attached to the current thread. */
187 __libc_lock_init (free_list_lock);
188 if (thread_arena != NULL)
189 thread_arena->attached_threads = 1;
190 free_list = NULL;
191 for (mstate ar_ptr = &main_arena;; )
193 __libc_lock_init (ar_ptr->mutex);
194 if (ar_ptr != thread_arena)
196 /* This arena is no longer attached to any thread. */
197 ar_ptr->attached_threads = 0;
198 ar_ptr->next_free = free_list;
199 free_list = ar_ptr;
201 ar_ptr = ar_ptr->next;
202 if (ar_ptr == &main_arena)
203 break;
206 __libc_lock_init (list_lock);
209 #if HAVE_TUNABLES
210 static inline int do_set_mallopt_check (int32_t value);
211 void
212 TUNABLE_CALLBACK (set_mallopt_check) (tunable_val_t *valp)
214 int32_t value = (int32_t) valp->numval;
215 if (value != 0)
216 __malloc_check_init ();
219 # define TUNABLE_CALLBACK_FNDECL(__name, __type) \
220 static inline int do_ ## __name (__type value); \
221 void \
222 TUNABLE_CALLBACK (__name) (tunable_val_t *valp) \
224 __type value = (__type) (valp)->numval; \
225 do_ ## __name (value); \
228 TUNABLE_CALLBACK_FNDECL (set_mmap_threshold, size_t)
229 TUNABLE_CALLBACK_FNDECL (set_mmaps_max, int32_t)
230 TUNABLE_CALLBACK_FNDECL (set_top_pad, size_t)
231 TUNABLE_CALLBACK_FNDECL (set_perturb_byte, int32_t)
232 TUNABLE_CALLBACK_FNDECL (set_trim_threshold, size_t)
233 TUNABLE_CALLBACK_FNDECL (set_arena_max, size_t)
234 TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t)
235 #if USE_TCACHE
236 TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t)
237 TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t)
238 TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t)
239 #endif
240 #else
241 /* Initialization routine. */
242 #include <string.h>
243 extern char **_environ;
245 static char *
246 next_env_entry (char ***position)
248 char **current = *position;
249 char *result = NULL;
251 while (*current != NULL)
253 if (__builtin_expect ((*current)[0] == 'M', 0)
254 && (*current)[1] == 'A'
255 && (*current)[2] == 'L'
256 && (*current)[3] == 'L'
257 && (*current)[4] == 'O'
258 && (*current)[5] == 'C'
259 && (*current)[6] == '_')
261 result = &(*current)[7];
263 /* Save current position for next visit. */
264 *position = ++current;
266 break;
269 ++current;
272 return result;
274 #endif
277 #ifdef SHARED
278 static void *
279 __failing_morecore (ptrdiff_t d)
281 return (void *) MORECORE_FAILURE;
284 extern struct dl_open_hook *_dl_open_hook;
285 libc_hidden_proto (_dl_open_hook);
286 #endif
288 static void
289 ptmalloc_init (void)
291 if (__malloc_initialized >= 0)
292 return;
294 __malloc_initialized = 0;
296 #ifdef SHARED
297 /* In case this libc copy is in a non-default namespace, never use brk.
298 Likewise if dlopened from statically linked program. */
299 Dl_info di;
300 struct link_map *l;
302 if (_dl_open_hook != NULL
303 || (_dl_addr (ptmalloc_init, &di, &l, NULL) != 0
304 && l->l_ns != LM_ID_BASE))
305 __morecore = __failing_morecore;
306 #endif
308 thread_arena = &main_arena;
310 malloc_init_state (&main_arena);
312 #if HAVE_TUNABLES
313 TUNABLE_GET (check, int32_t, TUNABLE_CALLBACK (set_mallopt_check));
314 TUNABLE_GET (top_pad, size_t, TUNABLE_CALLBACK (set_top_pad));
315 TUNABLE_GET (perturb, int32_t, TUNABLE_CALLBACK (set_perturb_byte));
316 TUNABLE_GET (mmap_threshold, size_t, TUNABLE_CALLBACK (set_mmap_threshold));
317 TUNABLE_GET (trim_threshold, size_t, TUNABLE_CALLBACK (set_trim_threshold));
318 TUNABLE_GET (mmap_max, int32_t, TUNABLE_CALLBACK (set_mmaps_max));
319 TUNABLE_GET (arena_max, size_t, TUNABLE_CALLBACK (set_arena_max));
320 TUNABLE_GET (arena_test, size_t, TUNABLE_CALLBACK (set_arena_test));
321 # if USE_TCACHE
322 TUNABLE_GET (tcache_max, size_t, TUNABLE_CALLBACK (set_tcache_max));
323 TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK (set_tcache_count));
324 TUNABLE_GET (tcache_unsorted_limit, size_t,
325 TUNABLE_CALLBACK (set_tcache_unsorted_limit));
326 # endif
327 #else
328 const char *s = NULL;
329 if (__glibc_likely (_environ != NULL))
331 char **runp = _environ;
332 char *envline;
334 while (__builtin_expect ((envline = next_env_entry (&runp)) != NULL,
337 size_t len = strcspn (envline, "=");
339 if (envline[len] != '=')
340 /* This is a "MALLOC_" variable at the end of the string
341 without a '=' character. Ignore it since otherwise we
342 will access invalid memory below. */
343 continue;
345 switch (len)
347 case 6:
348 if (memcmp (envline, "CHECK_", 6) == 0)
349 s = &envline[7];
350 break;
351 case 8:
352 if (!__builtin_expect (__libc_enable_secure, 0))
354 if (memcmp (envline, "TOP_PAD_", 8) == 0)
355 __libc_mallopt (M_TOP_PAD, atoi (&envline[9]));
356 else if (memcmp (envline, "PERTURB_", 8) == 0)
357 __libc_mallopt (M_PERTURB, atoi (&envline[9]));
359 break;
360 case 9:
361 if (!__builtin_expect (__libc_enable_secure, 0))
363 if (memcmp (envline, "MMAP_MAX_", 9) == 0)
364 __libc_mallopt (M_MMAP_MAX, atoi (&envline[10]));
365 else if (memcmp (envline, "ARENA_MAX", 9) == 0)
366 __libc_mallopt (M_ARENA_MAX, atoi (&envline[10]));
368 break;
369 case 10:
370 if (!__builtin_expect (__libc_enable_secure, 0))
372 if (memcmp (envline, "ARENA_TEST", 10) == 0)
373 __libc_mallopt (M_ARENA_TEST, atoi (&envline[11]));
375 break;
376 case 15:
377 if (!__builtin_expect (__libc_enable_secure, 0))
379 if (memcmp (envline, "TRIM_THRESHOLD_", 15) == 0)
380 __libc_mallopt (M_TRIM_THRESHOLD, atoi (&envline[16]));
381 else if (memcmp (envline, "MMAP_THRESHOLD_", 15) == 0)
382 __libc_mallopt (M_MMAP_THRESHOLD, atoi (&envline[16]));
384 break;
385 default:
386 break;
390 if (s && s[0] != '\0' && s[0] != '0')
391 __malloc_check_init ();
392 #endif
394 #if HAVE_MALLOC_INIT_HOOK
395 void (*hook) (void) = atomic_forced_read (__malloc_initialize_hook);
396 if (hook != NULL)
397 (*hook)();
398 #endif
399 __malloc_initialized = 1;
402 /* Managing heaps and arenas (for concurrent threads) */
404 #if MALLOC_DEBUG > 1
406 /* Print the complete contents of a single heap to stderr. */
408 static void
409 dump_heap (heap_info *heap)
411 char *ptr;
412 mchunkptr p;
414 fprintf (stderr, "Heap %p, size %10lx:\n", heap, (long) heap->size);
415 ptr = (heap->ar_ptr != (mstate) (heap + 1)) ?
416 (char *) (heap + 1) : (char *) (heap + 1) + sizeof (struct malloc_state);
417 p = (mchunkptr) (((unsigned long) ptr + MALLOC_ALIGN_MASK) &
418 ~MALLOC_ALIGN_MASK);
419 for (;; )
421 fprintf (stderr, "chunk %p size %10lx", p, (long) p->size);
422 if (p == top (heap->ar_ptr))
424 fprintf (stderr, " (top)\n");
425 break;
427 else if (p->size == (0 | PREV_INUSE))
429 fprintf (stderr, " (fence)\n");
430 break;
432 fprintf (stderr, "\n");
433 p = next_chunk (p);
436 #endif /* MALLOC_DEBUG > 1 */
438 /* If consecutive mmap (0, HEAP_MAX_SIZE << 1, ...) calls return decreasing
439 addresses as opposed to increasing, new_heap would badly fragment the
440 address space. In that case remember the second HEAP_MAX_SIZE part
441 aligned to HEAP_MAX_SIZE from last mmap (0, HEAP_MAX_SIZE << 1, ...)
442 call (if it is already aligned) and try to reuse it next time. We need
443 no locking for it, as kernel ensures the atomicity for us - worst case
444 we'll call mmap (addr, HEAP_MAX_SIZE, ...) for some value of addr in
445 multiple threads, but only one will succeed. */
446 static char *aligned_heap_area;
448 /* Create a new heap. size is automatically rounded up to a multiple
449 of the page size. */
451 static heap_info *
452 new_heap (size_t size, size_t top_pad)
454 size_t pagesize = GLRO (dl_pagesize);
455 char *p1, *p2;
456 unsigned long ul;
457 heap_info *h;
459 if (size + top_pad < HEAP_MIN_SIZE)
460 size = HEAP_MIN_SIZE;
461 else if (size + top_pad <= HEAP_MAX_SIZE)
462 size += top_pad;
463 else if (size > HEAP_MAX_SIZE)
464 return 0;
465 else
466 size = HEAP_MAX_SIZE;
467 size = ALIGN_UP (size, pagesize);
469 /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed.
470 No swap space needs to be reserved for the following large
471 mapping (on Linux, this is the case for all non-writable mappings
472 anyway). */
473 p2 = MAP_FAILED;
474 if (aligned_heap_area)
476 p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
477 MAP_NORESERVE);
478 aligned_heap_area = NULL;
479 if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)))
481 __munmap (p2, HEAP_MAX_SIZE);
482 p2 = MAP_FAILED;
485 if (p2 == MAP_FAILED)
487 p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE);
488 if (p1 != MAP_FAILED)
490 p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1))
491 & ~(HEAP_MAX_SIZE - 1));
492 ul = p2 - p1;
493 if (ul)
494 __munmap (p1, ul);
495 else
496 aligned_heap_area = p2 + HEAP_MAX_SIZE;
497 __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
499 else
501 /* Try to take the chance that an allocation of only HEAP_MAX_SIZE
502 is already aligned. */
503 p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
504 if (p2 == MAP_FAILED)
505 return 0;
507 if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))
509 __munmap (p2, HEAP_MAX_SIZE);
510 return 0;
514 if (__mprotect (p2, size, PROT_READ | PROT_WRITE) != 0)
516 __munmap (p2, HEAP_MAX_SIZE);
517 return 0;
519 h = (heap_info *) p2;
520 h->size = size;
521 h->mprotect_size = size;
522 LIBC_PROBE (memory_heap_new, 2, h, h->size);
523 return h;
526 /* Grow a heap. size is automatically rounded up to a
527 multiple of the page size. */
529 static int
530 grow_heap (heap_info *h, long diff)
532 size_t pagesize = GLRO (dl_pagesize);
533 long new_size;
535 diff = ALIGN_UP (diff, pagesize);
536 new_size = (long) h->size + diff;
537 if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE)
538 return -1;
540 if ((unsigned long) new_size > h->mprotect_size)
542 if (__mprotect ((char *) h + h->mprotect_size,
543 (unsigned long) new_size - h->mprotect_size,
544 PROT_READ | PROT_WRITE) != 0)
545 return -2;
547 h->mprotect_size = new_size;
550 h->size = new_size;
551 LIBC_PROBE (memory_heap_more, 2, h, h->size);
552 return 0;
555 /* Shrink a heap. */
557 static int
558 shrink_heap (heap_info *h, long diff)
560 long new_size;
562 new_size = (long) h->size - diff;
563 if (new_size < (long) sizeof (*h))
564 return -1;
566 /* Try to re-map the extra heap space freshly to save memory, and make it
567 inaccessible. See malloc-sysdep.h to know when this is true. */
568 if (__glibc_unlikely (check_may_shrink_heap ()))
570 if ((char *) MMAP ((char *) h + new_size, diff, PROT_NONE,
571 MAP_FIXED) == (char *) MAP_FAILED)
572 return -2;
574 h->mprotect_size = new_size;
576 else
577 __madvise ((char *) h + new_size, diff, MADV_DONTNEED);
578 /*fprintf(stderr, "shrink %p %08lx\n", h, new_size);*/
580 h->size = new_size;
581 LIBC_PROBE (memory_heap_less, 2, h, h->size);
582 return 0;
585 /* Delete a heap. */
587 #define delete_heap(heap) \
588 do { \
589 if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area) \
590 aligned_heap_area = NULL; \
591 __munmap ((char *) (heap), HEAP_MAX_SIZE); \
592 } while (0)
594 static int
595 heap_trim (heap_info *heap, size_t pad)
597 mstate ar_ptr = heap->ar_ptr;
598 unsigned long pagesz = GLRO (dl_pagesize);
599 mchunkptr top_chunk = top (ar_ptr), p, bck, fwd;
600 heap_info *prev_heap;
601 long new_size, top_size, top_area, extra, prev_size, misalign;
603 /* Can this heap go away completely? */
604 while (top_chunk == chunk_at_offset (heap, sizeof (*heap)))
606 prev_heap = heap->prev;
607 prev_size = prev_heap->size - (MINSIZE - 2 * SIZE_SZ);
608 p = chunk_at_offset (prev_heap, prev_size);
609 /* fencepost must be properly aligned. */
610 misalign = ((long) p) & MALLOC_ALIGN_MASK;
611 p = chunk_at_offset (prev_heap, prev_size - misalign);
612 assert (chunksize_nomask (p) == (0 | PREV_INUSE)); /* must be fencepost */
613 p = prev_chunk (p);
614 new_size = chunksize (p) + (MINSIZE - 2 * SIZE_SZ) + misalign;
615 assert (new_size > 0 && new_size < (long) (2 * MINSIZE));
616 if (!prev_inuse (p))
617 new_size += prev_size (p);
618 assert (new_size > 0 && new_size < HEAP_MAX_SIZE);
619 if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz)
620 break;
621 ar_ptr->system_mem -= heap->size;
622 LIBC_PROBE (memory_heap_free, 2, heap, heap->size);
623 delete_heap (heap);
624 heap = prev_heap;
625 if (!prev_inuse (p)) /* consolidate backward */
627 p = prev_chunk (p);
628 unlink (ar_ptr, p, bck, fwd);
630 assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0);
631 assert (((char *) p + new_size) == ((char *) heap + heap->size));
632 top (ar_ptr) = top_chunk = p;
633 set_head (top_chunk, new_size | PREV_INUSE);
634 /*check_chunk(ar_ptr, top_chunk);*/
637 /* Uses similar logic for per-thread arenas as the main arena with systrim
638 and _int_free by preserving the top pad and rounding down to the nearest
639 page. */
640 top_size = chunksize (top_chunk);
641 if ((unsigned long)(top_size) <
642 (unsigned long)(mp_.trim_threshold))
643 return 0;
645 top_area = top_size - MINSIZE - 1;
646 if (top_area < 0 || (size_t) top_area <= pad)
647 return 0;
649 /* Release in pagesize units and round down to the nearest page. */
650 extra = ALIGN_DOWN(top_area - pad, pagesz);
651 if (extra == 0)
652 return 0;
654 /* Try to shrink. */
655 if (shrink_heap (heap, extra) != 0)
656 return 0;
658 ar_ptr->system_mem -= extra;
660 /* Success. Adjust top accordingly. */
661 set_head (top_chunk, (top_size - extra) | PREV_INUSE);
662 /*check_chunk(ar_ptr, top_chunk);*/
663 return 1;
666 /* Create a new arena with initial size "size". */
668 /* If REPLACED_ARENA is not NULL, detach it from this thread. Must be
669 called while free_list_lock is held. */
670 static void
671 detach_arena (mstate replaced_arena)
673 if (replaced_arena != NULL)
675 assert (replaced_arena->attached_threads > 0);
676 /* The current implementation only detaches from main_arena in
677 case of allocation failure. This means that it is likely not
678 beneficial to put the arena on free_list even if the
679 reference count reaches zero. */
680 --replaced_arena->attached_threads;
684 static mstate
685 _int_new_arena (size_t size)
687 mstate a;
688 heap_info *h;
689 char *ptr;
690 unsigned long misalign;
692 h = new_heap (size + (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT),
693 mp_.top_pad);
694 if (!h)
696 /* Maybe size is too large to fit in a single heap. So, just try
697 to create a minimally-sized arena and let _int_malloc() attempt
698 to deal with the large request via mmap_chunk(). */
699 h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad);
700 if (!h)
701 return 0;
703 a = h->ar_ptr = (mstate) (h + 1);
704 malloc_init_state (a);
705 a->attached_threads = 1;
706 /*a->next = NULL;*/
707 a->system_mem = a->max_system_mem = h->size;
709 /* Set up the top chunk, with proper alignment. */
710 ptr = (char *) (a + 1);
711 misalign = (unsigned long) chunk2mem (ptr) & MALLOC_ALIGN_MASK;
712 if (misalign > 0)
713 ptr += MALLOC_ALIGNMENT - misalign;
714 top (a) = (mchunkptr) ptr;
715 set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE);
717 LIBC_PROBE (memory_arena_new, 2, a, size);
718 mstate replaced_arena = thread_arena;
719 thread_arena = a;
720 __libc_lock_init (a->mutex);
722 __libc_lock_lock (list_lock);
724 /* Add the new arena to the global list. */
725 a->next = main_arena.next;
726 /* FIXME: The barrier is an attempt to synchronize with read access
727 in reused_arena, which does not acquire list_lock while
728 traversing the list. */
729 atomic_write_barrier ();
730 main_arena.next = a;
732 __libc_lock_unlock (list_lock);
734 __libc_lock_lock (free_list_lock);
735 detach_arena (replaced_arena);
736 __libc_lock_unlock (free_list_lock);
738 /* Lock this arena. NB: Another thread may have been attached to
739 this arena because the arena is now accessible from the
740 main_arena.next list and could have been picked by reused_arena.
741 This can only happen for the last arena created (before the arena
742 limit is reached). At this point, some arena has to be attached
743 to two threads. We could acquire the arena lock before list_lock
744 to make it less likely that reused_arena picks this new arena,
745 but this could result in a deadlock with
746 __malloc_fork_lock_parent. */
748 __libc_lock_lock (a->mutex);
750 return a;
754 /* Remove an arena from free_list. */
755 static mstate
756 get_free_list (void)
758 mstate replaced_arena = thread_arena;
759 mstate result = free_list;
760 if (result != NULL)
762 __libc_lock_lock (free_list_lock);
763 result = free_list;
764 if (result != NULL)
766 free_list = result->next_free;
768 /* The arena will be attached to this thread. */
769 assert (result->attached_threads == 0);
770 result->attached_threads = 1;
772 detach_arena (replaced_arena);
774 __libc_lock_unlock (free_list_lock);
776 if (result != NULL)
778 LIBC_PROBE (memory_arena_reuse_free_list, 1, result);
779 __libc_lock_lock (result->mutex);
780 thread_arena = result;
784 return result;
787 /* Remove the arena from the free list (if it is present).
788 free_list_lock must have been acquired by the caller. */
789 static void
790 remove_from_free_list (mstate arena)
792 mstate *previous = &free_list;
793 for (mstate p = free_list; p != NULL; p = p->next_free)
795 assert (p->attached_threads == 0);
796 if (p == arena)
798 /* Remove the requested arena from the list. */
799 *previous = p->next_free;
800 break;
802 else
803 previous = &p->next_free;
807 /* Lock and return an arena that can be reused for memory allocation.
808 Avoid AVOID_ARENA as we have already failed to allocate memory in
809 it and it is currently locked. */
810 static mstate
811 reused_arena (mstate avoid_arena)
813 mstate result;
814 /* FIXME: Access to next_to_use suffers from data races. */
815 static mstate next_to_use;
816 if (next_to_use == NULL)
817 next_to_use = &main_arena;
819 /* Iterate over all arenas (including those linked from
820 free_list). */
821 result = next_to_use;
824 if (!__libc_lock_trylock (result->mutex))
825 goto out;
827 /* FIXME: This is a data race, see _int_new_arena. */
828 result = result->next;
830 while (result != next_to_use);
832 /* Avoid AVOID_ARENA as we have already failed to allocate memory
833 in that arena and it is currently locked. */
834 if (result == avoid_arena)
835 result = result->next;
837 /* No arena available without contention. Wait for the next in line. */
838 LIBC_PROBE (memory_arena_reuse_wait, 3, &result->mutex, result, avoid_arena);
839 __libc_lock_lock (result->mutex);
841 out:
842 /* Attach the arena to the current thread. */
844 /* Update the arena thread attachment counters. */
845 mstate replaced_arena = thread_arena;
846 __libc_lock_lock (free_list_lock);
847 detach_arena (replaced_arena);
849 /* We may have picked up an arena on the free list. We need to
850 preserve the invariant that no arena on the free list has a
851 positive attached_threads counter (otherwise,
852 arena_thread_freeres cannot use the counter to determine if the
853 arena needs to be put on the free list). We unconditionally
854 remove the selected arena from the free list. The caller of
855 reused_arena checked the free list and observed it to be empty,
856 so the list is very short. */
857 remove_from_free_list (result);
859 ++result->attached_threads;
861 __libc_lock_unlock (free_list_lock);
864 LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena);
865 thread_arena = result;
866 next_to_use = result->next;
868 return result;
871 static mstate
872 arena_get2 (size_t size, mstate avoid_arena)
874 mstate a;
876 static size_t narenas_limit;
878 a = get_free_list ();
879 if (a == NULL)
881 /* Nothing immediately available, so generate a new arena. */
882 if (narenas_limit == 0)
884 if (mp_.arena_max != 0)
885 narenas_limit = mp_.arena_max;
886 else if (narenas > mp_.arena_test)
888 int n = __get_nprocs ();
890 if (n >= 1)
891 narenas_limit = NARENAS_FROM_NCORES (n);
892 else
893 /* We have no information about the system. Assume two
894 cores. */
895 narenas_limit = NARENAS_FROM_NCORES (2);
898 repeat:;
899 size_t n = narenas;
900 /* NB: the following depends on the fact that (size_t)0 - 1 is a
901 very large number and that the underflow is OK. If arena_max
902 is set the value of arena_test is irrelevant. If arena_test
903 is set but narenas is not yet larger or equal to arena_test
904 narenas_limit is 0. There is no possibility for narenas to
905 be too big for the test to always fail since there is not
906 enough address space to create that many arenas. */
907 if (__glibc_unlikely (n <= narenas_limit - 1))
909 if (catomic_compare_and_exchange_bool_acq (&narenas, n + 1, n))
910 goto repeat;
911 a = _int_new_arena (size);
912 if (__glibc_unlikely (a == NULL))
913 catomic_decrement (&narenas);
915 else
916 a = reused_arena (avoid_arena);
918 return a;
921 /* If we don't have the main arena, then maybe the failure is due to running
922 out of mmapped areas, so we can try allocating on the main arena.
923 Otherwise, it is likely that sbrk() has failed and there is still a chance
924 to mmap(), so try one of the other arenas. */
925 static mstate
926 arena_get_retry (mstate ar_ptr, size_t bytes)
928 LIBC_PROBE (memory_arena_retry, 2, bytes, ar_ptr);
929 if (ar_ptr != &main_arena)
931 __libc_lock_unlock (ar_ptr->mutex);
932 ar_ptr = &main_arena;
933 __libc_lock_lock (ar_ptr->mutex);
935 else
937 __libc_lock_unlock (ar_ptr->mutex);
938 ar_ptr = arena_get2 (bytes, ar_ptr);
941 return ar_ptr;
944 static void __attribute__ ((section ("__libc_thread_freeres_fn")))
945 arena_thread_freeres (void)
947 /* Shut down the thread cache first. This could deallocate data for
948 the thread arena, so do this before we put the arena on the free
949 list. */
950 tcache_thread_shutdown ();
952 mstate a = thread_arena;
953 thread_arena = NULL;
955 if (a != NULL)
957 __libc_lock_lock (free_list_lock);
958 /* If this was the last attached thread for this arena, put the
959 arena on the free list. */
960 assert (a->attached_threads > 0);
961 if (--a->attached_threads == 0)
963 a->next_free = free_list;
964 free_list = a;
966 __libc_lock_unlock (free_list_lock);
969 text_set_element (__libc_thread_subfreeres, arena_thread_freeres);
972 * Local variables:
973 * c-basic-offset: 2
974 * End: