1 /* Malloc implementation for multiple threads without lock contention.
2 Copyright (C) 1996-2013 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Wolfram Gloger <wg@malloc.de>
5 and Doug Lea <dl@cs.oswego.edu>, 2001.
7 The GNU C Library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public License as
9 published by the Free Software Foundation; either version 2.1 of the
10 License, or (at your option) any later version.
12 The GNU C Library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
17 You should have received a copy of the GNU Lesser General Public
18 License along with the GNU C Library; see the file COPYING.LIB. If
19 not, see <http://www.gnu.org/licenses/>. */
22 This is a version (aka ptmalloc2) of malloc/free/realloc written by
23 Doug Lea and adapted to multiple threads/arenas by Wolfram Gloger.
25 There have been substantial changesmade after the integration into
26 glibc in all parts of the code. Do not look for much commonality
27 with the ptmalloc2 version.
29 * Version ptmalloc2-20011215
31 VERSION 2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee)
35 In order to compile this implementation, a Makefile is provided with
36 the ptmalloc2 distribution, which has pre-defined targets for some
37 popular systems (e.g. "make posix" for Posix threads). All that is
38 typically required with regard to compiler flags is the selection of
39 the thread package via defining one out of USE_PTHREADS, USE_THR or
40 USE_SPROC. Check the thread-m.h file for what effects this has.
41 Many/most systems will additionally require USE_TSD_DATA_HACK to be
42 defined, so this is the default for "make posix".
44 * Why use this malloc?
46 This is not the fastest, most space-conserving, most portable, or
47 most tunable malloc ever written. However it is among the fastest
48 while also being among the most space-conserving, portable and tunable.
49 Consistent balance across these factors results in a good general-purpose
50 allocator for malloc-intensive programs.
52 The main properties of the algorithms are:
53 * For large (>= 512 bytes) requests, it is a pure best-fit allocator,
54 with ties normally decided via FIFO (i.e. least recently used).
55 * For small (<= 64 bytes by default) requests, it is a caching
56 allocator, that maintains pools of quickly recycled chunks.
57 * In between, and for combinations of large and small requests, it does
58 the best it can trying to meet both goals at once.
59 * For very large requests (>= 128KB by default), it relies on system
60 memory mapping facilities, if supported.
62 For a longer but slightly out of date high-level description, see
63 http://gee.cs.oswego.edu/dl/html/malloc.html
65 You may already by default be using a C library containing a malloc
66 that is based on some version of this malloc (for example in
67 linux). You might still want to use the one in this file in order to
68 customize settings or to avoid overheads associated with library
71 * Contents, described in more detail in "description of public routines" below.
73 Standard (ANSI/SVID/...) functions:
75 calloc(size_t n_elements, size_t element_size);
77 realloc(void* p, size_t n);
78 memalign(size_t alignment, size_t n);
81 mallopt(int parameter_number, int parameter_value)
84 independent_calloc(size_t n_elements, size_t size, void* chunks[]);
85 independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
88 malloc_trim(size_t pad);
89 malloc_usable_size(void* p);
94 Supported pointer representation: 4 or 8 bytes
95 Supported size_t representation: 4 or 8 bytes
96 Note that size_t is allowed to be 4 bytes even if pointers are 8.
97 You can adjust this by defining INTERNAL_SIZE_T
99 Alignment: 2 * sizeof(size_t) (default)
100 (i.e., 8 byte alignment with 4byte size_t). This suffices for
101 nearly all current machines and C compilers. However, you can
102 define MALLOC_ALIGNMENT to be wider than this if necessary.
104 Minimum overhead per allocated chunk: 4 or 8 bytes
105 Each malloced chunk has a hidden word of overhead holding size
106 and status information.
108 Minimum allocated size: 4-byte ptrs: 16 bytes (including 4 overhead)
109 8-byte ptrs: 24/32 bytes (including, 4/8 overhead)
111 When a chunk is freed, 12 (for 4byte ptrs) or 20 (for 8 byte
112 ptrs but 4 byte size) or 24 (for 8/8) additional bytes are
113 needed; 4 (8) for a trailing size field and 8 (16) bytes for
114 free list pointers. Thus, the minimum allocatable size is
117 Even a request for zero bytes (i.e., malloc(0)) returns a
118 pointer to something of the minimum allocatable size.
120 The maximum overhead wastage (i.e., number of extra bytes
121 allocated than were requested in malloc) is less than or equal
122 to the minimum size, except for requests >= mmap_threshold that
123 are serviced via mmap(), where the worst case wastage is 2 *
124 sizeof(size_t) bytes plus the remainder from a system page (the
125 minimal mmap unit); typically 4096 or 8192 bytes.
127 Maximum allocated size: 4-byte size_t: 2^32 minus about two pages
128 8-byte size_t: 2^64 minus about two pages
130 It is assumed that (possibly signed) size_t values suffice to
131 represent chunk sizes. `Possibly signed' is due to the fact
132 that `size_t' may be defined on a system as either a signed or
133 an unsigned type. The ISO C standard says that it must be
134 unsigned, but a few systems are known not to adhere to this.
135 Additionally, even when size_t is unsigned, sbrk (which is by
136 default used to obtain memory from system) accepts signed
137 arguments, and may not be able to handle size_t-wide arguments
138 with negative sign bit. Generally, values that would
139 appear as negative after accounting for overhead and alignment
140 are supported only via mmap(), which does not have this
143 Requests for sizes outside the allowed range will perform an optional
144 failure action and then return null. (Requests may also
145 also fail because a system is out of memory.)
147 Thread-safety: thread-safe
149 Compliance: I believe it is compliant with the 1997 Single Unix Specification
150 Also SVID/XPG, ANSI C, and probably others as well.
152 * Synopsis of compile-time options:
154 People have reported using previous versions of this malloc on all
155 versions of Unix, sometimes by tweaking some of the defines
156 below. It has been tested most extensively on Solaris and Linux.
157 People also report using it in stand-alone embedded systems.
159 The implementation is in straight, hand-tuned ANSI C. It is not
160 at all modular. (Sorry!) It uses a lot of macros. To be at all
161 usable, this code should be compiled using an optimizing compiler
162 (for example gcc -O3) that can simplify expressions and control
163 paths. (FAQ: some macros import variables as arguments rather than
164 declare locals because people reported that some debuggers
165 otherwise get confused.)
169 Compilation Environment options:
173 Changing default word sizes:
175 INTERNAL_SIZE_T size_t
176 MALLOC_ALIGNMENT MAX (2 * sizeof(INTERNAL_SIZE_T),
177 __alignof__ (long double))
179 Configuration and functionality options:
181 USE_PUBLIC_MALLOC_WRAPPERS NOT defined
182 USE_MALLOC_LOCK NOT defined
183 MALLOC_DEBUG NOT defined
184 REALLOC_ZERO_BYTES_FREES 1
187 Options for customizing MORECORE:
191 MORECORE_CONTIGUOUS 1
192 MORECORE_CANNOT_TRIM NOT defined
194 MMAP_AS_MORECORE_SIZE (1024 * 1024)
196 Tuning options that are also dynamically changeable via mallopt:
198 DEFAULT_MXFAST 64 (for 32bit), 128 (for 64bit)
199 DEFAULT_TRIM_THRESHOLD 128 * 1024
201 DEFAULT_MMAP_THRESHOLD 128 * 1024
202 DEFAULT_MMAP_MAX 65536
204 There are several other #defined constants and macros that you
205 probably don't want to touch unless you are extending or adapting malloc. */
208 void* is the pointer type that malloc should say it returns
215 #include <stddef.h> /* for size_t */
216 #include <stdlib.h> /* for getenv(), abort() */
217 #include <unistd.h> /* for __libc_enable_secure */
219 #include <malloc-machine.h>
220 #include <malloc-sysdep.h>
224 #include <bits/wordsize.h>
225 #include <sys/sysinfo.h>
227 #include <ldsodefs.h>
230 #include <stdio.h> /* needed for malloc_stats */
233 #include <shlib-compat.h>
238 /* For va_arg, va_start, va_end. */
245 Because freed chunks may be overwritten with bookkeeping fields, this
246 malloc will often die when freed memory is overwritten by user
247 programs. This can be very effective (albeit in an annoying way)
248 in helping track down dangling pointers.
250 If you compile with -DMALLOC_DEBUG, a number of assertion checks are
251 enabled that will catch more memory errors. You probably won't be
252 able to make much sense of the actual assertion errors, but they
253 should help you locate incorrectly overwritten memory. The checking
254 is fairly extensive, and will slow down execution
255 noticeably. Calling malloc_stats or mallinfo with MALLOC_DEBUG set
256 will attempt to check every non-mmapped allocated and free chunk in
257 the course of computing the summmaries. (By nature, mmapped regions
258 cannot be checked very much automatically.)
260 Setting MALLOC_DEBUG may also be helpful if you are trying to modify
261 this code. The assertions in the check routines spell out in more
262 detail the assumptions and invariants underlying the algorithms.
264 Setting MALLOC_DEBUG does NOT provide an automated mechanism for
265 checking that all accesses to malloced memory stay within their
266 bounds. However, there are several add-ons and adaptations of this
267 or other mallocs available that do this.
271 # define assert(expr) ((void) 0)
273 # define assert(expr) \
276 : __malloc_assert (__STRING (expr), __FILE__, __LINE__, __func__))
278 extern const char *__progname
;
281 __malloc_assert (const char *assertion
, const char *file
, unsigned int line
,
282 const char *function
)
284 (void) __fxprintf (NULL
, "%s%s%s:%u: %s%sAssertion `%s' failed.\n",
285 __progname
, __progname
[0] ? ": " : "",
287 function
? function
: "", function
? ": " : "",
296 INTERNAL_SIZE_T is the word-size used for internal bookkeeping
299 The default version is the same as size_t.
301 While not strictly necessary, it is best to define this as an
302 unsigned type, even if size_t is a signed type. This may avoid some
303 artificial size limitations on some systems.
305 On a 64-bit machine, you may be able to reduce malloc overhead by
306 defining INTERNAL_SIZE_T to be a 32 bit `unsigned int' at the
307 expense of not being able to handle more than 2^32 of malloced
308 space. If this limitation is acceptable, you are encouraged to set
309 this unless you are on a platform requiring 16byte alignments. In
310 this case the alignment requirements turn out to negate any
311 potential advantages of decreasing size_t word size.
313 Implementors: Beware of the possible combinations of:
314 - INTERNAL_SIZE_T might be signed or unsigned, might be 32 or 64 bits,
315 and might be the same width as int or as long
316 - size_t might have different width and signedness as INTERNAL_SIZE_T
317 - int and long might be 32 or 64 bits, and might be the same width
318 To deal with this, most comparisons and difference computations
319 among INTERNAL_SIZE_Ts should cast them to unsigned long, being
320 aware of the fact that casting an unsigned int to a wider long does
321 not sign-extend. (This also makes checking for negative numbers
322 awkward.) Some of these casts result in harmless compiler warnings
326 #ifndef INTERNAL_SIZE_T
327 #define INTERNAL_SIZE_T size_t
330 /* The corresponding word size */
331 #define SIZE_SZ (sizeof(INTERNAL_SIZE_T))
335 MALLOC_ALIGNMENT is the minimum alignment for malloc'ed chunks.
336 It must be a power of two at least 2 * SIZE_SZ, even on machines
337 for which smaller alignments would suffice. It may be defined as
338 larger than this though. Note however that code and data structures
339 are optimized for the case of 8-byte alignment.
343 #ifndef MALLOC_ALIGNMENT
344 # if !SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_16)
345 /* This is the correct definition when there is no past ABI to constrain it.
347 Among configurations with a past ABI constraint, it differs from
348 2*SIZE_SZ only on powerpc32. For the time being, changing this is
349 causing more compatibility problems due to malloc_get_state and
350 malloc_set_state than will returning blocks not adequately aligned for
351 long double objects under -mlong-double-128. */
353 # define MALLOC_ALIGNMENT (2 * SIZE_SZ < __alignof__ (long double) \
354 ? __alignof__ (long double) : 2 * SIZE_SZ)
356 # define MALLOC_ALIGNMENT (2 * SIZE_SZ)
360 /* The corresponding bit mask value */
361 #define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
366 REALLOC_ZERO_BYTES_FREES should be set if a call to
367 realloc with zero bytes should be the same as a call to free.
368 This is required by the C standard. Otherwise, since this malloc
369 returns a unique pointer for malloc(0), so does realloc(p, 0).
372 #ifndef REALLOC_ZERO_BYTES_FREES
373 #define REALLOC_ZERO_BYTES_FREES 1
377 TRIM_FASTBINS controls whether free() of a very small chunk can
378 immediately lead to trimming. Setting to true (1) can reduce memory
379 footprint, but will almost always slow down programs that use a lot
382 Define this only if you are willing to give up some speed to more
383 aggressively reduce system-level memory footprint when releasing
384 memory in programs that use many small chunks. You can get
385 essentially the same effect by setting MXFAST to 0, but this can
386 lead to even greater slowdowns in programs using many small chunks.
387 TRIM_FASTBINS is an in-between compile-time option, that disables
388 only those chunks bordering topmost memory from being placed in
392 #ifndef TRIM_FASTBINS
393 #define TRIM_FASTBINS 0
397 /* Definition for getting more memory from the OS. */
398 #define MORECORE (*__morecore)
399 #define MORECORE_FAILURE 0
400 void * __default_morecore (ptrdiff_t);
401 void *(*__morecore
)(ptrdiff_t) = __default_morecore
;
407 /* Force a value to be in a register and stop the compiler referring
408 to the source (mostly memory location) again. */
409 #define force_reg(val) \
410 ({ __typeof (val) _v; asm ("" : "=r" (_v) : "0" (val)); _v; })
414 MORECORE-related declarations. By default, rely on sbrk
419 MORECORE is the name of the routine to call to obtain more memory
420 from the system. See below for general guidance on writing
421 alternative MORECORE functions, as well as a version for WIN32 and a
422 sample version for pre-OSX macos.
426 #define MORECORE sbrk
430 MORECORE_FAILURE is the value returned upon failure of MORECORE
431 as well as mmap. Since it cannot be an otherwise valid memory address,
432 and must reflect values of standard sys calls, you probably ought not
436 #ifndef MORECORE_FAILURE
437 #define MORECORE_FAILURE (-1)
441 If MORECORE_CONTIGUOUS is true, take advantage of fact that
442 consecutive calls to MORECORE with positive arguments always return
443 contiguous increasing addresses. This is true of unix sbrk. Even
444 if not defined, when regions happen to be contiguous, malloc will
445 permit allocations spanning regions obtained from different
446 calls. But defining this when applicable enables some stronger
447 consistency checks and space efficiencies.
450 #ifndef MORECORE_CONTIGUOUS
451 #define MORECORE_CONTIGUOUS 1
455 Define MORECORE_CANNOT_TRIM if your version of MORECORE
456 cannot release space back to the system when given negative
457 arguments. This is generally necessary only if you are using
458 a hand-crafted MORECORE function that cannot handle negative arguments.
461 /* #define MORECORE_CANNOT_TRIM */
463 /* MORECORE_CLEARS (default 1)
464 The degree to which the routine mapped to MORECORE zeroes out
465 memory: never (0), only for newly allocated space (1) or always
466 (2). The distinction between (1) and (2) is necessary because on
467 some systems, if the application first decrements and then
468 increments the break value, the contents of the reallocated space
472 #ifndef MORECORE_CLEARS
473 #define MORECORE_CLEARS 1
478 MMAP_AS_MORECORE_SIZE is the minimum mmap size argument to use if
479 sbrk fails, and mmap is used as a backup. The value must be a
480 multiple of page size. This backup strategy generally applies only
481 when systems have "holes" in address space, so sbrk cannot perform
482 contiguous expansion, but there is still space available on system.
483 On systems for which this is known to be useful (i.e. most linux
484 kernels), this occurs only when programs allocate huge amounts of
485 memory. Between this, and the fact that mmap regions tend to be
486 limited, the size should be large, to avoid too many mmap calls and
487 thus avoid running out of kernel resources. */
489 #ifndef MMAP_AS_MORECORE_SIZE
490 #define MMAP_AS_MORECORE_SIZE (1024 * 1024)
494 Define HAVE_MREMAP to make realloc() use mremap() to re-allocate
499 #define HAVE_MREMAP 0
504 This version of malloc supports the standard SVID/XPG mallinfo
505 routine that returns a struct containing usage properties and
506 statistics. It should work on any SVID/XPG compliant system that has
507 a /usr/include/malloc.h defining struct mallinfo. (If you'd like to
508 install such a thing yourself, cut out the preliminary declarations
509 as described above and below and save them in a malloc.h file. But
510 there's no compelling reason to bother to do this.)
512 The main declaration needed is the mallinfo struct that is returned
513 (by-copy) by mallinfo(). The SVID/XPG malloinfo struct contains a
514 bunch of fields that are not even meaningful in this version of
515 malloc. These fields are are instead filled by mallinfo() with
516 other numbers that might be of interest.
520 /* ---------- description of public routines ------------ */
524 Returns a pointer to a newly allocated chunk of at least n bytes, or null
525 if no space is available. Additionally, on failure, errno is
526 set to ENOMEM on ANSI C systems.
528 If n is zero, malloc returns a minumum-sized chunk. (The minimum
529 size is 16 bytes on most 32bit systems, and 24 or 32 bytes on 64bit
530 systems.) On most systems, size_t is an unsigned type, so calls
531 with negative arguments are interpreted as requests for huge amounts
532 of space, which will often fail. The maximum supported value of n
533 differs across systems, but is in all cases less than the maximum
534 representable value of a size_t.
536 void* __libc_malloc(size_t);
537 libc_hidden_proto (__libc_malloc
)
541 Releases the chunk of memory pointed to by p, that had been previously
542 allocated using malloc or a related routine such as realloc.
543 It has no effect if p is null. It can have arbitrary (i.e., bad!)
544 effects if p has already been freed.
546 Unless disabled (using mallopt), freeing very large spaces will
547 when possible, automatically trigger operations that give
548 back unused memory to the system, thus reducing program footprint.
550 void __libc_free(void*);
551 libc_hidden_proto (__libc_free
)
554 calloc(size_t n_elements, size_t element_size);
555 Returns a pointer to n_elements * element_size bytes, with all locations
558 void* __libc_calloc(size_t, size_t);
561 realloc(void* p, size_t n)
562 Returns a pointer to a chunk of size n that contains the same data
563 as does chunk p up to the minimum of (n, p's size) bytes, or null
564 if no space is available.
566 The returned pointer may or may not be the same as p. The algorithm
567 prefers extending p when possible, otherwise it employs the
568 equivalent of a malloc-copy-free sequence.
570 If p is null, realloc is equivalent to malloc.
572 If space is not available, realloc returns null, errno is set (if on
573 ANSI) and p is NOT freed.
575 if n is for fewer bytes than already held by p, the newly unused
576 space is lopped off and freed if possible. Unless the #define
577 REALLOC_ZERO_BYTES_FREES is set, realloc with a size argument of
578 zero (re)allocates a minimum-sized chunk.
580 Large chunks that were internally obtained via mmap will always
581 be reallocated using malloc-copy-free sequences unless
582 the system supports MREMAP (currently only linux).
584 The old unix realloc convention of allowing the last-free'd chunk
585 to be used as an argument to realloc is not supported.
587 void* __libc_realloc(void*, size_t);
588 libc_hidden_proto (__libc_realloc
)
591 memalign(size_t alignment, size_t n);
592 Returns a pointer to a newly allocated chunk of n bytes, aligned
593 in accord with the alignment argument.
595 The alignment argument should be a power of two. If the argument is
596 not a power of two, the nearest greater power is used.
597 8-byte alignment is guaranteed by normal malloc calls, so don't
598 bother calling memalign with an argument of 8 or less.
600 Overreliance on memalign is a sure way to fragment space.
602 void* __libc_memalign(size_t, size_t);
603 libc_hidden_proto (__libc_memalign
)
607 Equivalent to memalign(pagesize, n), where pagesize is the page
608 size of the system. If the pagesize is unknown, 4096 is used.
610 void* __libc_valloc(size_t);
615 mallopt(int parameter_number, int parameter_value)
616 Sets tunable parameters The format is to provide a
617 (parameter-number, parameter-value) pair. mallopt then sets the
618 corresponding parameter to the argument value if it can (i.e., so
619 long as the value is meaningful), and returns 1 if successful else
620 0. SVID/XPG/ANSI defines four standard param numbers for mallopt,
621 normally defined in malloc.h. Only one of these (M_MXFAST) is used
622 in this malloc. The others (M_NLBLKS, M_GRAIN, M_KEEP) don't apply,
623 so setting them has no effect. But this malloc also supports four
624 other options in mallopt. See below for details. Briefly, supported
625 parameters are as follows (listed defaults are for "typical"
628 Symbol param # default allowed param values
629 M_MXFAST 1 64 0-80 (0 disables fastbins)
630 M_TRIM_THRESHOLD -1 128*1024 any (-1U disables trimming)
632 M_MMAP_THRESHOLD -3 128*1024 any (or 0 if no MMAP support)
633 M_MMAP_MAX -4 65536 any (0 disables use of mmap)
635 int __libc_mallopt(int, int);
636 libc_hidden_proto (__libc_mallopt
)
641 Returns (by copy) a struct containing various summary statistics:
643 arena: current total non-mmapped bytes allocated from system
644 ordblks: the number of free chunks
645 smblks: the number of fastbin blocks (i.e., small chunks that
646 have been freed but not use resused or consolidated)
647 hblks: current number of mmapped regions
648 hblkhd: total bytes held in mmapped regions
649 usmblks: the maximum total allocated space. This will be greater
650 than current total if trimming has occurred.
651 fsmblks: total bytes held in fastbin blocks
652 uordblks: current total allocated space (normal or mmapped)
653 fordblks: total free space
654 keepcost: the maximum number of bytes that could ideally be released
655 back to system via malloc_trim. ("ideally" means that
656 it ignores page restrictions etc.)
658 Because these fields are ints, but internal bookkeeping may
659 be kept as longs, the reported values may wrap around zero and
662 struct mallinfo
__libc_mallinfo(void);
667 Equivalent to valloc(minimum-page-that-holds(n)), that is,
668 round up n to nearest pagesize.
670 void* __libc_pvalloc(size_t);
673 malloc_trim(size_t pad);
675 If possible, gives memory back to the system (via negative
676 arguments to sbrk) if there is unused memory at the `high' end of
677 the malloc pool. You can call this after freeing large blocks of
678 memory to potentially reduce the system-level memory requirements
679 of a program. However, it cannot guarantee to reduce memory. Under
680 some allocation patterns, some large free blocks of memory will be
681 locked between two used chunks, so they cannot be given back to
684 The `pad' argument to malloc_trim represents the amount of free
685 trailing space to leave untrimmed. If this argument is zero,
686 only the minimum amount of memory to maintain internal data
687 structures will be left (one page or less). Non-zero arguments
688 can be supplied to maintain enough trailing space to service
689 future expected allocations without having to re-obtain memory
692 Malloc_trim returns 1 if it actually released any memory, else 0.
693 On systems that do not support "negative sbrks", it will always
696 int __malloc_trim(size_t);
699 malloc_usable_size(void* p);
701 Returns the number of bytes you can actually use in
702 an allocated chunk, which may be more than you requested (although
703 often not) due to alignment and minimum size constraints.
704 You can use this many bytes without worrying about
705 overwriting other allocated objects. This is not a particularly great
706 programming practice. malloc_usable_size can be more useful in
707 debugging and assertions, for example:
710 assert(malloc_usable_size(p) >= 256);
713 size_t __malloc_usable_size(void*);
717 Prints on stderr the amount of space obtained from the system (both
718 via sbrk and mmap), the maximum amount (which may be more than
719 current if malloc_trim and/or munmap got called), and the current
720 number of bytes allocated via malloc (or realloc, etc) but not yet
721 freed. Note that this is the number of bytes allocated, not the
722 number requested. It will be larger than the number requested
723 because of alignment and bookkeeping overhead. Because it includes
724 alignment wastage as being in use, this figure may be greater than
725 zero even when no user-level chunks are allocated.
727 The reported current and maximum system memory can be inaccurate if
728 a program makes other calls to system memory allocation functions
729 (normally sbrk) outside of malloc.
731 malloc_stats prints only the most commonly interesting statistics.
732 More information can be obtained by calling mallinfo.
735 void __malloc_stats(void);
738 malloc_get_state(void);
740 Returns the state of all malloc variables in an opaque data
743 void* __malloc_get_state(void);
746 malloc_set_state(void* state);
748 Restore the state of all malloc variables from data obtained with
751 int __malloc_set_state(void*);
754 posix_memalign(void **memptr, size_t alignment, size_t size);
756 POSIX wrapper like memalign(), checking for validity of size.
758 int __posix_memalign(void **, size_t, size_t);
760 /* mallopt tuning options */
763 M_MXFAST is the maximum request size used for "fastbins", special bins
764 that hold returned chunks without consolidating their spaces. This
765 enables future requests for chunks of the same size to be handled
766 very quickly, but can increase fragmentation, and thus increase the
767 overall memory footprint of a program.
769 This malloc manages fastbins very conservatively yet still
770 efficiently, so fragmentation is rarely a problem for values less
771 than or equal to the default. The maximum supported value of MXFAST
772 is 80. You wouldn't want it any higher than this anyway. Fastbins
773 are designed especially for use with many small structs, objects or
774 strings -- the default handles structs/objects/arrays with sizes up
775 to 8 4byte fields, or small strings representing words, tokens,
776 etc. Using fastbins for larger objects normally worsens
777 fragmentation without improving speed.
779 M_MXFAST is set in REQUEST size units. It is internally used in
780 chunksize units, which adds padding and alignment. You can reduce
781 M_MXFAST to 0 to disable all use of fastbins. This causes the malloc
782 algorithm to be a closer approximation of fifo-best-fit in all cases,
783 not just for larger requests, but will generally cause it to be
788 /* M_MXFAST is a standard SVID/XPG tuning option, usually listed in malloc.h */
793 #ifndef DEFAULT_MXFAST
794 #define DEFAULT_MXFAST (64 * SIZE_SZ / 4)
799 M_TRIM_THRESHOLD is the maximum amount of unused top-most memory
800 to keep before releasing via malloc_trim in free().
802 Automatic trimming is mainly useful in long-lived programs.
803 Because trimming via sbrk can be slow on some systems, and can
804 sometimes be wasteful (in cases where programs immediately
805 afterward allocate more large chunks) the value should be high
806 enough so that your overall system performance would improve by
807 releasing this much memory.
809 The trim threshold and the mmap control parameters (see below)
810 can be traded off with one another. Trimming and mmapping are
811 two different ways of releasing unused memory back to the
812 system. Between these two, it is often possible to keep
813 system-level demands of a long-lived program down to a bare
814 minimum. For example, in one test suite of sessions measuring
815 the XF86 X server on Linux, using a trim threshold of 128K and a
816 mmap threshold of 192K led to near-minimal long term resource
819 If you are using this malloc in a long-lived program, it should
820 pay to experiment with these values. As a rough guide, you
821 might set to a value close to the average size of a process
822 (program) running on your system. Releasing this much memory
823 would allow such a process to run in memory. Generally, it's
824 worth it to tune for trimming rather tham memory mapping when a
825 program undergoes phases where several large chunks are
826 allocated and released in ways that can reuse each other's
827 storage, perhaps mixed with phases where there are no such
828 chunks at all. And in well-behaved long-lived programs,
829 controlling release of large blocks via trimming versus mapping
832 However, in most programs, these parameters serve mainly as
833 protection against the system-level effects of carrying around
834 massive amounts of unneeded memory. Since frequent calls to
835 sbrk, mmap, and munmap otherwise degrade performance, the default
836 parameters are set to relatively high values that serve only as
839 The trim value It must be greater than page size to have any useful
840 effect. To disable trimming completely, you can set to
843 Trim settings interact with fastbin (MXFAST) settings: Unless
844 TRIM_FASTBINS is defined, automatic trimming never takes place upon
845 freeing a chunk with size less than or equal to MXFAST. Trimming is
846 instead delayed until subsequent freeing of larger chunks. However,
847 you can still force an attempted trim by calling malloc_trim.
849 Also, trimming is not generally possible in cases where
850 the main arena is obtained via mmap.
852 Note that the trick some people use of mallocing a huge space and
853 then freeing it at program startup, in an attempt to reserve system
854 memory, doesn't have the intended effect under automatic trimming,
855 since that memory will immediately be returned to the system.
858 #define M_TRIM_THRESHOLD -1
860 #ifndef DEFAULT_TRIM_THRESHOLD
861 #define DEFAULT_TRIM_THRESHOLD (128 * 1024)
865 M_TOP_PAD is the amount of extra `padding' space to allocate or
866 retain whenever sbrk is called. It is used in two ways internally:
868 * When sbrk is called to extend the top of the arena to satisfy
869 a new malloc request, this much padding is added to the sbrk
872 * When malloc_trim is called automatically from free(),
873 it is used as the `pad' argument.
875 In both cases, the actual amount of padding is rounded
876 so that the end of the arena is always a system page boundary.
878 The main reason for using padding is to avoid calling sbrk so
879 often. Having even a small pad greatly reduces the likelihood
880 that nearly every malloc request during program start-up (or
881 after trimming) will invoke sbrk, which needlessly wastes
884 Automatic rounding-up to page-size units is normally sufficient
885 to avoid measurable overhead, so the default is 0. However, in
886 systems where sbrk is relatively slow, it can pay to increase
887 this value, at the expense of carrying around more memory than
893 #ifndef DEFAULT_TOP_PAD
894 #define DEFAULT_TOP_PAD (0)
898 MMAP_THRESHOLD_MAX and _MIN are the bounds on the dynamically
899 adjusted MMAP_THRESHOLD.
902 #ifndef DEFAULT_MMAP_THRESHOLD_MIN
903 #define DEFAULT_MMAP_THRESHOLD_MIN (128 * 1024)
906 #ifndef DEFAULT_MMAP_THRESHOLD_MAX
907 /* For 32-bit platforms we cannot increase the maximum mmap
908 threshold much because it is also the minimum value for the
909 maximum heap size and its alignment. Going above 512k (i.e., 1M
910 for new heaps) wastes too much address space. */
911 # if __WORDSIZE == 32
912 # define DEFAULT_MMAP_THRESHOLD_MAX (512 * 1024)
914 # define DEFAULT_MMAP_THRESHOLD_MAX (4 * 1024 * 1024 * sizeof(long))
919 M_MMAP_THRESHOLD is the request size threshold for using mmap()
920 to service a request. Requests of at least this size that cannot
921 be allocated using already-existing space will be serviced via mmap.
922 (If enough normal freed space already exists it is used instead.)
924 Using mmap segregates relatively large chunks of memory so that
925 they can be individually obtained and released from the host
926 system. A request serviced through mmap is never reused by any
927 other request (at least not directly; the system may just so
928 happen to remap successive requests to the same locations).
930 Segregating space in this way has the benefits that:
932 1. Mmapped space can ALWAYS be individually released back
933 to the system, which helps keep the system level memory
934 demands of a long-lived program low.
935 2. Mapped memory can never become `locked' between
936 other chunks, as can happen with normally allocated chunks, which
937 means that even trimming via malloc_trim would not release them.
938 3. On some systems with "holes" in address spaces, mmap can obtain
939 memory that sbrk cannot.
941 However, it has the disadvantages that:
943 1. The space cannot be reclaimed, consolidated, and then
944 used to service later requests, as happens with normal chunks.
945 2. It can lead to more wastage because of mmap page alignment
947 3. It causes malloc performance to be more dependent on host
948 system memory management support routines which may vary in
949 implementation quality and may impose arbitrary
950 limitations. Generally, servicing a request via normal
951 malloc steps is faster than going through a system's mmap.
953 The advantages of mmap nearly always outweigh disadvantages for
954 "large" chunks, but the value of "large" varies across systems. The
955 default is an empirically derived value that works well in most
960 The above was written in 2001. Since then the world has changed a lot.
961 Memory got bigger. Applications got bigger. The virtual address space
962 layout in 32 bit linux changed.
964 In the new situation, brk() and mmap space is shared and there are no
965 artificial limits on brk size imposed by the kernel. What is more,
966 applications have started using transient allocations larger than the
967 128Kb as was imagined in 2001.
969 The price for mmap is also high now; each time glibc mmaps from the
970 kernel, the kernel is forced to zero out the memory it gives to the
971 application. Zeroing memory is expensive and eats a lot of cache and
972 memory bandwidth. This has nothing to do with the efficiency of the
973 virtual memory system, by doing mmap the kernel just has no choice but
976 In 2001, the kernel had a maximum size for brk() which was about 800
977 megabytes on 32 bit x86, at that point brk() would hit the first
978 mmaped shared libaries and couldn't expand anymore. With current 2.6
979 kernels, the VA space layout is different and brk() and mmap
980 both can span the entire heap at will.
982 Rather than using a static threshold for the brk/mmap tradeoff,
983 we are now using a simple dynamic one. The goal is still to avoid
984 fragmentation. The old goals we kept are
985 1) try to get the long lived large allocations to use mmap()
986 2) really large allocations should always use mmap()
987 and we're adding now:
988 3) transient allocations should use brk() to avoid forcing the kernel
989 having to zero memory over and over again
991 The implementation works with a sliding threshold, which is by default
992 limited to go between 128Kb and 32Mb (64Mb for 64 bitmachines) and starts
993 out at 128Kb as per the 2001 default.
995 This allows us to satisfy requirement 1) under the assumption that long
996 lived allocations are made early in the process' lifespan, before it has
997 started doing dynamic allocations of the same size (which will
998 increase the threshold).
1000 The upperbound on the threshold satisfies requirement 2)
1002 The threshold goes up in value when the application frees memory that was
1003 allocated with the mmap allocator. The idea is that once the application
1004 starts freeing memory of a certain size, it's highly probable that this is
1005 a size the application uses for transient allocations. This estimator
1006 is there to satisfy the new third requirement.
1010 #define M_MMAP_THRESHOLD -3
1012 #ifndef DEFAULT_MMAP_THRESHOLD
1013 #define DEFAULT_MMAP_THRESHOLD DEFAULT_MMAP_THRESHOLD_MIN
1017 M_MMAP_MAX is the maximum number of requests to simultaneously
1018 service using mmap. This parameter exists because
1019 some systems have a limited number of internal tables for
1020 use by mmap, and using more than a few of them may degrade
1023 The default is set to a value that serves only as a safeguard.
1024 Setting to 0 disables use of mmap for servicing large requests.
1027 #define M_MMAP_MAX -4
1029 #ifndef DEFAULT_MMAP_MAX
1030 #define DEFAULT_MMAP_MAX (65536)
1035 #ifndef RETURN_ADDRESS
1036 #define RETURN_ADDRESS(X_) (NULL)
1039 /* On some platforms we can compile internal, not exported functions better.
1040 Let the environment provide a macro and define it to be empty if it
1041 is not available. */
1042 #ifndef internal_function
1043 # define internal_function
1046 /* Forward declarations. */
1047 struct malloc_chunk
;
1048 typedef struct malloc_chunk
* mchunkptr
;
1050 /* Internal routines. */
1052 static void* _int_malloc(mstate
, size_t);
1053 static void _int_free(mstate
, mchunkptr
, int);
1054 static void* _int_realloc(mstate
, mchunkptr
, INTERNAL_SIZE_T
,
1056 static void* _int_memalign(mstate
, size_t, size_t);
1057 static void* _int_valloc(mstate
, size_t);
1058 static void* _int_pvalloc(mstate
, size_t);
1059 static void malloc_printerr(int action
, const char *str
, void *ptr
);
1061 static void* internal_function
mem2mem_check(void *p
, size_t sz
);
1062 static int internal_function
top_check(void);
1063 static void internal_function
munmap_chunk(mchunkptr p
);
1065 static mchunkptr internal_function
mremap_chunk(mchunkptr p
, size_t new_size
);
1068 static void* malloc_check(size_t sz
, const void *caller
);
1069 static void free_check(void* mem
, const void *caller
);
1070 static void* realloc_check(void* oldmem
, size_t bytes
,
1071 const void *caller
);
1072 static void* memalign_check(size_t alignment
, size_t bytes
,
1073 const void *caller
);
1075 static void* malloc_atfork(size_t sz
, const void *caller
);
1076 static void free_atfork(void* mem
, const void *caller
);
1080 /* ------------- Optional versions of memcopy ---------------- */
1084 Note: memcpy is ONLY invoked with non-overlapping regions,
1085 so the (usually slower) memmove is not needed.
1088 #define MALLOC_COPY(dest, src, nbytes) memcpy(dest, src, nbytes)
1089 #define MALLOC_ZERO(dest, nbytes) memset(dest, 0, nbytes)
1092 /* ------------------ MMAP support ------------------ */
1096 #include <sys/mman.h>
1098 #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
1099 # define MAP_ANONYMOUS MAP_ANON
1102 #ifndef MAP_NORESERVE
1103 # define MAP_NORESERVE 0
1106 #define MMAP(addr, size, prot, flags) \
1107 __mmap((addr), (size), (prot), (flags)|MAP_ANONYMOUS|MAP_PRIVATE, -1, 0)
1111 ----------------------- Chunk representations -----------------------
1116 This struct declaration is misleading (but accurate and necessary).
1117 It declares a "view" into memory allowing access to necessary
1118 fields at known offsets from a given base. See explanation below.
1121 struct malloc_chunk
{
1123 INTERNAL_SIZE_T prev_size
; /* Size of previous chunk (if free). */
1124 INTERNAL_SIZE_T size
; /* Size in bytes, including overhead. */
1126 struct malloc_chunk
* fd
; /* double links -- used only if free. */
1127 struct malloc_chunk
* bk
;
1129 /* Only used for large blocks: pointer to next larger size. */
1130 struct malloc_chunk
* fd_nextsize
; /* double links -- used only if free. */
1131 struct malloc_chunk
* bk_nextsize
;
1136 malloc_chunk details:
1138 (The following includes lightly edited explanations by Colin Plumb.)
1140 Chunks of memory are maintained using a `boundary tag' method as
1141 described in e.g., Knuth or Standish. (See the paper by Paul
1142 Wilson ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a
1143 survey of such techniques.) Sizes of free chunks are stored both
1144 in the front of each chunk and at the end. This makes
1145 consolidating fragmented chunks into bigger chunks very fast. The
1146 size fields also hold bits representing whether chunks are free or
1149 An allocated chunk looks like this:
1152 chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1153 | Size of previous chunk, if allocated | |
1154 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1155 | Size of chunk, in bytes |M|P|
1156 mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1157 | User data starts here... .
1159 . (malloc_usable_size() bytes) .
1161 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1163 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1166 Where "chunk" is the front of the chunk for the purpose of most of
1167 the malloc code, but "mem" is the pointer that is returned to the
1168 user. "Nextchunk" is the beginning of the next contiguous chunk.
1170 Chunks always begin on even word boundaries, so the mem portion
1171 (which is returned to the user) is also on an even word boundary, and
1172 thus at least double-word aligned.
1174 Free chunks are stored in circular doubly-linked lists, and look like this:
1176 chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1177 | Size of previous chunk |
1178 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1179 `head:' | Size of chunk, in bytes |P|
1180 mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1181 | Forward pointer to next chunk in list |
1182 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1183 | Back pointer to previous chunk in list |
1184 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1185 | Unused space (may be 0 bytes long) .
1188 nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1189 `foot:' | Size of chunk, in bytes |
1190 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1192 The P (PREV_INUSE) bit, stored in the unused low-order bit of the
1193 chunk size (which is always a multiple of two words), is an in-use
1194 bit for the *previous* chunk. If that bit is *clear*, then the
1195 word before the current chunk size contains the previous chunk
1196 size, and can be used to find the front of the previous chunk.
1197 The very first chunk allocated always has this bit set,
1198 preventing access to non-existent (or non-owned) memory. If
1199 prev_inuse is set for any given chunk, then you CANNOT determine
1200 the size of the previous chunk, and might even get a memory
1201 addressing fault when trying to do so.
1203 Note that the `foot' of the current chunk is actually represented
1204 as the prev_size of the NEXT chunk. This makes it easier to
1205 deal with alignments etc but can be very confusing when trying
1206 to extend or adapt this code.
1208 The two exceptions to all this are
1210 1. The special chunk `top' doesn't bother using the
1211 trailing size field since there is no next contiguous chunk
1212 that would have to index off it. After initialization, `top'
1213 is forced to always exist. If it would become less than
1214 MINSIZE bytes long, it is replenished.
1216 2. Chunks allocated via mmap, which have the second-lowest-order
1217 bit M (IS_MMAPPED) set in their size fields. Because they are
1218 allocated one-by-one, each must contain its own trailing size field.
1223 ---------- Size and alignment checks and conversions ----------
1226 /* conversion from malloc headers to user pointers, and back */
1228 #define chunk2mem(p) ((void*)((char*)(p) + 2*SIZE_SZ))
1229 #define mem2chunk(mem) ((mchunkptr)((char*)(mem) - 2*SIZE_SZ))
1231 /* The smallest possible chunk */
1232 #define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
1234 /* The smallest size we can malloc is an aligned minimal chunk */
1237 (unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
1239 /* Check if m has acceptable alignment */
1241 #define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0)
1243 #define misaligned_chunk(p) \
1244 ((uintptr_t)(MALLOC_ALIGNMENT == 2 * SIZE_SZ ? (p) : chunk2mem (p)) \
1245 & MALLOC_ALIGN_MASK)
1249 Check if a request is so large that it would wrap around zero when
1250 padded and aligned. To simplify some other code, the bound is made
1251 low enough so that adding MINSIZE will also not wrap around zero.
1254 #define REQUEST_OUT_OF_RANGE(req) \
1255 ((unsigned long)(req) >= \
1256 (unsigned long)(INTERNAL_SIZE_T)(-2 * MINSIZE))
1258 /* pad request bytes into a usable size -- internal version */
1260 #define request2size(req) \
1261 (((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
1263 ((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
1265 /* Same, except also perform argument check */
1267 #define checked_request2size(req, sz) \
1268 if (REQUEST_OUT_OF_RANGE(req)) { \
1269 __set_errno (ENOMEM); \
1272 (sz) = request2size(req);
1275 --------------- Physical chunk operations ---------------
1279 /* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
1280 #define PREV_INUSE 0x1
1282 /* extract inuse bit of previous chunk */
1283 #define prev_inuse(p) ((p)->size & PREV_INUSE)
1286 /* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
1287 #define IS_MMAPPED 0x2
1289 /* check for mmap()'ed chunk */
1290 #define chunk_is_mmapped(p) ((p)->size & IS_MMAPPED)
1293 /* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
1294 from a non-main arena. This is only set immediately before handing
1295 the chunk to the user, if necessary. */
1296 #define NON_MAIN_ARENA 0x4
1298 /* check for chunk from non-main arena */
1299 #define chunk_non_main_arena(p) ((p)->size & NON_MAIN_ARENA)
1303 Bits to mask off when extracting size
1305 Note: IS_MMAPPED is intentionally not masked off from size field in
1306 macros for which mmapped chunks should never be seen. This should
1307 cause helpful core dumps to occur if it is tried by accident by
1308 people extending or adapting this malloc.
1310 #define SIZE_BITS (PREV_INUSE|IS_MMAPPED|NON_MAIN_ARENA)
1312 /* Get size, ignoring use bits */
1313 #define chunksize(p) ((p)->size & ~(SIZE_BITS))
1316 /* Ptr to next physical malloc_chunk. */
1317 #define next_chunk(p) ((mchunkptr)( ((char*)(p)) + ((p)->size & ~SIZE_BITS) ))
1319 /* Ptr to previous physical malloc_chunk */
1320 #define prev_chunk(p) ((mchunkptr)( ((char*)(p)) - ((p)->prev_size) ))
1322 /* Treat space at ptr + offset as a chunk */
1323 #define chunk_at_offset(p, s) ((mchunkptr)(((char*)(p)) + (s)))
1325 /* extract p's inuse bit */
1327 ((((mchunkptr)(((char*)(p))+((p)->size & ~SIZE_BITS)))->size) & PREV_INUSE)
1329 /* set/clear chunk as being inuse without otherwise disturbing */
1330 #define set_inuse(p)\
1331 ((mchunkptr)(((char*)(p)) + ((p)->size & ~SIZE_BITS)))->size |= PREV_INUSE
1333 #define clear_inuse(p)\
1334 ((mchunkptr)(((char*)(p)) + ((p)->size & ~SIZE_BITS)))->size &= ~(PREV_INUSE)
1337 /* check/set/clear inuse bits in known places */
1338 #define inuse_bit_at_offset(p, s)\
1339 (((mchunkptr)(((char*)(p)) + (s)))->size & PREV_INUSE)
1341 #define set_inuse_bit_at_offset(p, s)\
1342 (((mchunkptr)(((char*)(p)) + (s)))->size |= PREV_INUSE)
1344 #define clear_inuse_bit_at_offset(p, s)\
1345 (((mchunkptr)(((char*)(p)) + (s)))->size &= ~(PREV_INUSE))
1348 /* Set size at head, without disturbing its use bit */
1349 #define set_head_size(p, s) ((p)->size = (((p)->size & SIZE_BITS) | (s)))
1351 /* Set size/use field */
1352 #define set_head(p, s) ((p)->size = (s))
1354 /* Set size at footer (only when chunk is not in use) */
1355 #define set_foot(p, s) (((mchunkptr)((char*)(p) + (s)))->prev_size = (s))
1359 -------------------- Internal data structures --------------------
1361 All internal state is held in an instance of malloc_state defined
1362 below. There are no other static variables, except in two optional
1364 * If USE_MALLOC_LOCK is defined, the mALLOC_MUTEx declared above.
1365 * If mmap doesn't support MAP_ANONYMOUS, a dummy file descriptor
1368 Beware of lots of tricks that minimize the total bookkeeping space
1369 requirements. The result is a little over 1K bytes (for 4byte
1370 pointers and size_t.)
1376 An array of bin headers for free chunks. Each bin is doubly
1377 linked. The bins are approximately proportionally (log) spaced.
1378 There are a lot of these bins (128). This may look excessive, but
1379 works very well in practice. Most bins hold sizes that are
1380 unusual as malloc request sizes, but are more usual for fragments
1381 and consolidated sets of chunks, which is what these bins hold, so
1382 they can be found quickly. All procedures maintain the invariant
1383 that no consolidated chunk physically borders another one, so each
1384 chunk in a list is known to be preceeded and followed by either
1385 inuse chunks or the ends of memory.
1387 Chunks in bins are kept in size order, with ties going to the
1388 approximately least recently used chunk. Ordering isn't needed
1389 for the small bins, which all contain the same-sized chunks, but
1390 facilitates best-fit allocation for larger chunks. These lists
1391 are just sequential. Keeping them in order almost never requires
1392 enough traversal to warrant using fancier ordered data
1395 Chunks of the same size are linked with the most
1396 recently freed at the front, and allocations are taken from the
1397 back. This results in LRU (FIFO) allocation order, which tends
1398 to give each chunk an equal opportunity to be consolidated with
1399 adjacent freed chunks, resulting in larger free chunks and less
1402 To simplify use in double-linked lists, each bin header acts
1403 as a malloc_chunk. This avoids special-casing for headers.
1404 But to conserve space and improve locality, we allocate
1405 only the fd/bk pointers of bins, and then use repositioning tricks
1406 to treat these as the fields of a malloc_chunk*.
1409 typedef struct malloc_chunk
* mbinptr
;
1411 /* addressing -- note that bin_at(0) does not exist */
1412 #define bin_at(m, i) \
1413 (mbinptr) (((char *) &((m)->bins[((i) - 1) * 2])) \
1414 - offsetof (struct malloc_chunk, fd))
1416 /* analog of ++bin */
1417 #define next_bin(b) ((mbinptr)((char*)(b) + (sizeof(mchunkptr)<<1)))
1419 /* Reminders about list directionality within bins */
1420 #define first(b) ((b)->fd)
1421 #define last(b) ((b)->bk)
1423 /* Take a chunk off a bin list */
1424 #define unlink(P, BK, FD) { \
1427 if (__builtin_expect (FD->bk != P || BK->fd != P, 0)) \
1428 malloc_printerr (check_action, "corrupted double-linked list", P); \
1432 if (!in_smallbin_range (P->size) \
1433 && __builtin_expect (P->fd_nextsize != NULL, 0)) { \
1434 assert (P->fd_nextsize->bk_nextsize == P); \
1435 assert (P->bk_nextsize->fd_nextsize == P); \
1436 if (FD->fd_nextsize == NULL) { \
1437 if (P->fd_nextsize == P) \
1438 FD->fd_nextsize = FD->bk_nextsize = FD; \
1440 FD->fd_nextsize = P->fd_nextsize; \
1441 FD->bk_nextsize = P->bk_nextsize; \
1442 P->fd_nextsize->bk_nextsize = FD; \
1443 P->bk_nextsize->fd_nextsize = FD; \
1446 P->fd_nextsize->bk_nextsize = P->bk_nextsize; \
1447 P->bk_nextsize->fd_nextsize = P->fd_nextsize; \
1456 Bins for sizes < 512 bytes contain chunks of all the same size, spaced
1457 8 bytes apart. Larger bins are approximately logarithmically spaced:
1463 4 bins of size 32768
1464 2 bins of size 262144
1465 1 bin of size what's left
1467 There is actually a little bit of slop in the numbers in bin_index
1468 for the sake of speed. This makes no difference elsewhere.
1470 The bins top out around 1MB because we expect to service large
1473 Bin 0 does not exist. Bin 1 is the unordered list; if that would be
1474 a valid chunk size the small bins are bumped up one.
1478 #define NSMALLBINS 64
1479 #define SMALLBIN_WIDTH MALLOC_ALIGNMENT
1480 #define SMALLBIN_CORRECTION (MALLOC_ALIGNMENT > 2 * SIZE_SZ)
1481 #define MIN_LARGE_SIZE ((NSMALLBINS - SMALLBIN_CORRECTION) * SMALLBIN_WIDTH)
1483 #define in_smallbin_range(sz) \
1484 ((unsigned long)(sz) < (unsigned long)MIN_LARGE_SIZE)
1486 #define smallbin_index(sz) \
1487 ((SMALLBIN_WIDTH == 16 ? (((unsigned)(sz)) >> 4) : (((unsigned)(sz)) >> 3)) \
1488 + SMALLBIN_CORRECTION)
1490 #define largebin_index_32(sz) \
1491 (((((unsigned long)(sz)) >> 6) <= 38)? 56 + (((unsigned long)(sz)) >> 6): \
1492 ((((unsigned long)(sz)) >> 9) <= 20)? 91 + (((unsigned long)(sz)) >> 9): \
1493 ((((unsigned long)(sz)) >> 12) <= 10)? 110 + (((unsigned long)(sz)) >> 12): \
1494 ((((unsigned long)(sz)) >> 15) <= 4)? 119 + (((unsigned long)(sz)) >> 15): \
1495 ((((unsigned long)(sz)) >> 18) <= 2)? 124 + (((unsigned long)(sz)) >> 18): \
1498 #define largebin_index_32_big(sz) \
1499 (((((unsigned long)(sz)) >> 6) <= 45)? 49 + (((unsigned long)(sz)) >> 6): \
1500 ((((unsigned long)(sz)) >> 9) <= 20)? 91 + (((unsigned long)(sz)) >> 9): \
1501 ((((unsigned long)(sz)) >> 12) <= 10)? 110 + (((unsigned long)(sz)) >> 12): \
1502 ((((unsigned long)(sz)) >> 15) <= 4)? 119 + (((unsigned long)(sz)) >> 15): \
1503 ((((unsigned long)(sz)) >> 18) <= 2)? 124 + (((unsigned long)(sz)) >> 18): \
1506 // XXX It remains to be seen whether it is good to keep the widths of
1507 // XXX the buckets the same or whether it should be scaled by a factor
1508 // XXX of two as well.
1509 #define largebin_index_64(sz) \
1510 (((((unsigned long)(sz)) >> 6) <= 48)? 48 + (((unsigned long)(sz)) >> 6): \
1511 ((((unsigned long)(sz)) >> 9) <= 20)? 91 + (((unsigned long)(sz)) >> 9): \
1512 ((((unsigned long)(sz)) >> 12) <= 10)? 110 + (((unsigned long)(sz)) >> 12): \
1513 ((((unsigned long)(sz)) >> 15) <= 4)? 119 + (((unsigned long)(sz)) >> 15): \
1514 ((((unsigned long)(sz)) >> 18) <= 2)? 124 + (((unsigned long)(sz)) >> 18): \
1517 #define largebin_index(sz) \
1518 (SIZE_SZ == 8 ? largebin_index_64 (sz) \
1519 : MALLOC_ALIGNMENT == 16 ? largebin_index_32_big (sz) \
1520 : largebin_index_32 (sz))
1522 #define bin_index(sz) \
1523 ((in_smallbin_range(sz)) ? smallbin_index(sz) : largebin_index(sz))
1529 All remainders from chunk splits, as well as all returned chunks,
1530 are first placed in the "unsorted" bin. They are then placed
1531 in regular bins after malloc gives them ONE chance to be used before
1532 binning. So, basically, the unsorted_chunks list acts as a queue,
1533 with chunks being placed on it in free (and malloc_consolidate),
1534 and taken off (to be either used or placed in bins) in malloc.
1536 The NON_MAIN_ARENA flag is never set for unsorted chunks, so it
1537 does not have to be taken into account in size comparisons.
1540 /* The otherwise unindexable 1-bin is used to hold unsorted chunks. */
1541 #define unsorted_chunks(M) (bin_at(M, 1))
1546 The top-most available chunk (i.e., the one bordering the end of
1547 available memory) is treated specially. It is never included in
1548 any bin, is used only if no other chunk is available, and is
1549 released back to the system if it is very large (see
1550 M_TRIM_THRESHOLD). Because top initially
1551 points to its own bin with initial zero size, thus forcing
1552 extension on the first malloc request, we avoid having any special
1553 code in malloc to check whether it even exists yet. But we still
1554 need to do so when getting memory from system, so we make
1555 initial_top treat the bin as a legal but unusable chunk during the
1556 interval between initialization and the first call to
1557 sysmalloc. (This is somewhat delicate, since it relies on
1558 the 2 preceding words to be zero during this interval as well.)
1561 /* Conveniently, the unsorted bin can be used as dummy top on first call */
1562 #define initial_top(M) (unsorted_chunks(M))
1567 To help compensate for the large number of bins, a one-level index
1568 structure is used for bin-by-bin searching. `binmap' is a
1569 bitvector recording whether bins are definitely empty so they can
1570 be skipped over during during traversals. The bits are NOT always
1571 cleared as soon as bins are empty, but instead only
1572 when they are noticed to be empty during traversal in malloc.
1575 /* Conservatively use 32 bits per map word, even if on 64bit system */
1576 #define BINMAPSHIFT 5
1577 #define BITSPERMAP (1U << BINMAPSHIFT)
1578 #define BINMAPSIZE (NBINS / BITSPERMAP)
1580 #define idx2block(i) ((i) >> BINMAPSHIFT)
1581 #define idx2bit(i) ((1U << ((i) & ((1U << BINMAPSHIFT)-1))))
1583 #define mark_bin(m,i) ((m)->binmap[idx2block(i)] |= idx2bit(i))
1584 #define unmark_bin(m,i) ((m)->binmap[idx2block(i)] &= ~(idx2bit(i)))
1585 #define get_binmap(m,i) ((m)->binmap[idx2block(i)] & idx2bit(i))
1590 An array of lists holding recently freed small chunks. Fastbins
1591 are not doubly linked. It is faster to single-link them, and
1592 since chunks are never removed from the middles of these lists,
1593 double linking is not necessary. Also, unlike regular bins, they
1594 are not even processed in FIFO order (they use faster LIFO) since
1595 ordering doesn't much matter in the transient contexts in which
1596 fastbins are normally used.
1598 Chunks in fastbins keep their inuse bit set, so they cannot
1599 be consolidated with other free chunks. malloc_consolidate
1600 releases all chunks in fastbins and consolidates them with
1604 typedef struct malloc_chunk
* mfastbinptr
;
1605 #define fastbin(ar_ptr, idx) ((ar_ptr)->fastbinsY[idx])
1607 /* offset 2 to use otherwise unindexable first 2 bins */
1608 #define fastbin_index(sz) \
1609 ((((unsigned int)(sz)) >> (SIZE_SZ == 8 ? 4 : 3)) - 2)
1612 /* The maximum fastbin request size we support */
1613 #define MAX_FAST_SIZE (80 * SIZE_SZ / 4)
1615 #define NFASTBINS (fastbin_index(request2size(MAX_FAST_SIZE))+1)
1618 FASTBIN_CONSOLIDATION_THRESHOLD is the size of a chunk in free()
1619 that triggers automatic consolidation of possibly-surrounding
1620 fastbin chunks. This is a heuristic, so the exact value should not
1621 matter too much. It is defined at half the default trim threshold as a
1622 compromise heuristic to only attempt consolidation if it is likely
1623 to lead to trimming. However, it is not dynamically tunable, since
1624 consolidation reduces fragmentation surrounding large chunks even
1625 if trimming is not used.
1628 #define FASTBIN_CONSOLIDATION_THRESHOLD (65536UL)
1631 Since the lowest 2 bits in max_fast don't matter in size comparisons,
1632 they are used as flags.
1636 FASTCHUNKS_BIT held in max_fast indicates that there are probably
1637 some fastbin chunks. It is set true on entering a chunk into any
1638 fastbin, and cleared only in malloc_consolidate.
1640 The truth value is inverted so that have_fastchunks will be true
1641 upon startup (since statics are zero-filled), simplifying
1642 initialization checks.
1645 #define FASTCHUNKS_BIT (1U)
1647 #define have_fastchunks(M) (((M)->flags & FASTCHUNKS_BIT) == 0)
1648 #define clear_fastchunks(M) catomic_or (&(M)->flags, FASTCHUNKS_BIT)
1649 #define set_fastchunks(M) catomic_and (&(M)->flags, ~FASTCHUNKS_BIT)
1652 NONCONTIGUOUS_BIT indicates that MORECORE does not return contiguous
1653 regions. Otherwise, contiguity is exploited in merging together,
1654 when possible, results from consecutive MORECORE calls.
1656 The initial value comes from MORECORE_CONTIGUOUS, but is
1657 changed dynamically if mmap is ever used as an sbrk substitute.
1660 #define NONCONTIGUOUS_BIT (2U)
1662 #define contiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) == 0)
1663 #define noncontiguous(M) (((M)->flags & NONCONTIGUOUS_BIT) != 0)
1664 #define set_noncontiguous(M) ((M)->flags |= NONCONTIGUOUS_BIT)
1665 #define set_contiguous(M) ((M)->flags &= ~NONCONTIGUOUS_BIT)
1668 Set value of max_fast.
1669 Use impossibly small value if 0.
1670 Precondition: there are no existing fastbin chunks.
1671 Setting the value clears fastchunk bit but preserves noncontiguous bit.
1674 #define set_max_fast(s) \
1675 global_max_fast = (((s) == 0) \
1676 ? SMALLBIN_WIDTH: ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK))
1677 #define get_max_fast() global_max_fast
1681 ----------- Internal state representation and initialization -----------
1684 struct malloc_state
{
1685 /* Serialize access. */
1688 /* Flags (formerly in max_fast). */
1692 /* Statistics for locking. Only used if THREAD_STATS is defined. */
1693 long stat_lock_direct
, stat_lock_loop
, stat_lock_wait
;
1697 mfastbinptr fastbinsY
[NFASTBINS
];
1699 /* Base of the topmost chunk -- not otherwise kept in a bin */
1702 /* The remainder from the most recent split of a small request */
1703 mchunkptr last_remainder
;
1705 /* Normal bins packed as described above */
1706 mchunkptr bins
[NBINS
* 2 - 2];
1708 /* Bitmap of bins */
1709 unsigned int binmap
[BINMAPSIZE
];
1712 struct malloc_state
*next
;
1715 /* Linked list for free arenas. */
1716 struct malloc_state
*next_free
;
1719 /* Memory allocated from the system in this arena. */
1720 INTERNAL_SIZE_T system_mem
;
1721 INTERNAL_SIZE_T max_system_mem
;
1725 /* Tunable parameters */
1726 unsigned long trim_threshold
;
1727 INTERNAL_SIZE_T top_pad
;
1728 INTERNAL_SIZE_T mmap_threshold
;
1730 INTERNAL_SIZE_T arena_test
;
1731 INTERNAL_SIZE_T arena_max
;
1734 /* Memory map support */
1738 /* the mmap_threshold is dynamic, until the user sets
1739 it manually, at which point we need to disable any
1740 dynamic behavior. */
1741 int no_dyn_threshold
;
1744 INTERNAL_SIZE_T mmapped_mem
;
1745 /*INTERNAL_SIZE_T sbrked_mem;*/
1746 /*INTERNAL_SIZE_T max_sbrked_mem;*/
1747 INTERNAL_SIZE_T max_mmapped_mem
;
1748 INTERNAL_SIZE_T max_total_mem
; /* only kept for NO_THREADS */
1750 /* First address handed out by MORECORE/sbrk. */
1754 /* There are several instances of this struct ("arenas") in this
1755 malloc. If you are adapting this malloc in a way that does NOT use
1756 a static or mmapped malloc_state, you MUST explicitly zero-fill it
1757 before using. This malloc relies on the property that malloc_state
1758 is initialized to all zeroes (as is true of C statics). */
1760 static struct malloc_state main_arena
=
1762 .mutex
= MUTEX_INITIALIZER
,
1766 /* There is only one instance of the malloc parameters. */
1768 static struct malloc_par mp_
=
1770 .top_pad
= DEFAULT_TOP_PAD
,
1771 .n_mmaps_max
= DEFAULT_MMAP_MAX
,
1772 .mmap_threshold
= DEFAULT_MMAP_THRESHOLD
,
1773 .trim_threshold
= DEFAULT_TRIM_THRESHOLD
,
1775 # define NARENAS_FROM_NCORES(n) ((n) * (sizeof(long) == 4 ? 2 : 8))
1776 .arena_test
= NARENAS_FROM_NCORES (1)
1782 /* Non public mallopt parameters. */
1783 #define M_ARENA_TEST -7
1784 #define M_ARENA_MAX -8
1788 /* Maximum size of memory handled in fastbins. */
1789 static INTERNAL_SIZE_T global_max_fast
;
1792 Initialize a malloc_state struct.
1794 This is called only from within malloc_consolidate, which needs
1795 be called in the same contexts anyway. It is never called directly
1796 outside of malloc_consolidate because some optimizing compilers try
1797 to inline it at all call points, which turns out not to be an
1798 optimization at all. (Inlining it in malloc_consolidate is fine though.)
1801 static void malloc_init_state(mstate av
)
1806 /* Establish circular links for normal bins */
1807 for (i
= 1; i
< NBINS
; ++i
) {
1809 bin
->fd
= bin
->bk
= bin
;
1812 #if MORECORE_CONTIGUOUS
1813 if (av
!= &main_arena
)
1815 set_noncontiguous(av
);
1816 if (av
== &main_arena
)
1817 set_max_fast(DEFAULT_MXFAST
);
1818 av
->flags
|= FASTCHUNKS_BIT
;
1820 av
->top
= initial_top(av
);
1824 Other internal utilities operating on mstates
1827 static void* sysmalloc(INTERNAL_SIZE_T
, mstate
);
1828 static int systrim(size_t, mstate
);
1829 static void malloc_consolidate(mstate
);
1832 /* -------------- Early definitions for debugging hooks ---------------- */
1834 /* Define and initialize the hook variables. These weak definitions must
1835 appear before any use of the variables in a function (arena.c uses one). */
1836 #ifndef weak_variable
1837 /* In GNU libc we want the hook variables to be weak definitions to
1838 avoid a problem with Emacs. */
1839 # define weak_variable weak_function
1842 /* Forward declarations. */
1843 static void* malloc_hook_ini (size_t sz
,
1844 const void *caller
) __THROW
;
1845 static void* realloc_hook_ini (void* ptr
, size_t sz
,
1846 const void *caller
) __THROW
;
1847 static void* memalign_hook_ini (size_t alignment
, size_t sz
,
1848 const void *caller
) __THROW
;
1850 void weak_variable (*__malloc_initialize_hook
) (void) = NULL
;
1851 void weak_variable (*__free_hook
) (void *__ptr
,
1852 const void *) = NULL
;
1853 void *weak_variable (*__malloc_hook
)
1854 (size_t __size
, const void *) = malloc_hook_ini
;
1855 void *weak_variable (*__realloc_hook
)
1856 (void *__ptr
, size_t __size
, const void *)
1858 void *weak_variable (*__memalign_hook
)
1859 (size_t __alignment
, size_t __size
, const void *)
1860 = memalign_hook_ini
;
1861 void weak_variable (*__after_morecore_hook
) (void) = NULL
;
1864 /* ---------------- Error behavior ------------------------------------ */
1866 #ifndef DEFAULT_CHECK_ACTION
1867 #define DEFAULT_CHECK_ACTION 3
1870 static int check_action
= DEFAULT_CHECK_ACTION
;
1873 /* ------------------ Testing support ----------------------------------*/
1875 static int perturb_byte
;
1877 #define alloc_perturb(p, n) memset (p, (perturb_byte ^ 0xff) & 0xff, n)
1878 #define free_perturb(p, n) memset (p, perturb_byte & 0xff, n)
1881 #include <stap-probe.h>
1883 /* ------------------- Support for multiple arenas -------------------- */
1889 These routines make a number of assertions about the states
1890 of data structures that should be true at all times. If any
1891 are not true, it's very likely that a user program has somehow
1892 trashed memory. (It's also possible that there is a coding error
1893 in malloc. In which case, please report it!)
1898 #define check_chunk(A,P)
1899 #define check_free_chunk(A,P)
1900 #define check_inuse_chunk(A,P)
1901 #define check_remalloced_chunk(A,P,N)
1902 #define check_malloced_chunk(A,P,N)
1903 #define check_malloc_state(A)
1907 #define check_chunk(A,P) do_check_chunk(A,P)
1908 #define check_free_chunk(A,P) do_check_free_chunk(A,P)
1909 #define check_inuse_chunk(A,P) do_check_inuse_chunk(A,P)
1910 #define check_remalloced_chunk(A,P,N) do_check_remalloced_chunk(A,P,N)
1911 #define check_malloced_chunk(A,P,N) do_check_malloced_chunk(A,P,N)
1912 #define check_malloc_state(A) do_check_malloc_state(A)
1915 Properties of all chunks
1918 static void do_check_chunk(mstate av
, mchunkptr p
)
1920 unsigned long sz
= chunksize(p
);
1921 /* min and max possible addresses assuming contiguous allocation */
1922 char* max_address
= (char*)(av
->top
) + chunksize(av
->top
);
1923 char* min_address
= max_address
- av
->system_mem
;
1925 if (!chunk_is_mmapped(p
)) {
1927 /* Has legal address ... */
1929 if (contiguous(av
)) {
1930 assert(((char*)p
) >= min_address
);
1931 assert(((char*)p
+ sz
) <= ((char*)(av
->top
)));
1935 /* top size is always at least MINSIZE */
1936 assert((unsigned long)(sz
) >= MINSIZE
);
1937 /* top predecessor always marked inuse */
1938 assert(prev_inuse(p
));
1943 /* address is outside main heap */
1944 if (contiguous(av
) && av
->top
!= initial_top(av
)) {
1945 assert(((char*)p
) < min_address
|| ((char*)p
) >= max_address
);
1947 /* chunk is page-aligned */
1948 assert(((p
->prev_size
+ sz
) & (GLRO(dl_pagesize
)-1)) == 0);
1949 /* mem is aligned */
1950 assert(aligned_OK(chunk2mem(p
)));
1955 Properties of free chunks
1958 static void do_check_free_chunk(mstate av
, mchunkptr p
)
1960 INTERNAL_SIZE_T sz
= p
->size
& ~(PREV_INUSE
|NON_MAIN_ARENA
);
1961 mchunkptr next
= chunk_at_offset(p
, sz
);
1963 do_check_chunk(av
, p
);
1965 /* Chunk must claim to be free ... */
1967 assert (!chunk_is_mmapped(p
));
1969 /* Unless a special marker, must have OK fields */
1970 if ((unsigned long)(sz
) >= MINSIZE
)
1972 assert((sz
& MALLOC_ALIGN_MASK
) == 0);
1973 assert(aligned_OK(chunk2mem(p
)));
1974 /* ... matching footer field */
1975 assert(next
->prev_size
== sz
);
1976 /* ... and is fully consolidated */
1977 assert(prev_inuse(p
));
1978 assert (next
== av
->top
|| inuse(next
));
1980 /* ... and has minimally sane links */
1981 assert(p
->fd
->bk
== p
);
1982 assert(p
->bk
->fd
== p
);
1984 else /* markers are always of size SIZE_SZ */
1985 assert(sz
== SIZE_SZ
);
1989 Properties of inuse chunks
1992 static void do_check_inuse_chunk(mstate av
, mchunkptr p
)
1996 do_check_chunk(av
, p
);
1998 if (chunk_is_mmapped(p
))
1999 return; /* mmapped chunks have no next/prev */
2001 /* Check whether it claims to be in use ... */
2004 next
= next_chunk(p
);
2006 /* ... and is surrounded by OK chunks.
2007 Since more things can be checked with free chunks than inuse ones,
2008 if an inuse chunk borders them and debug is on, it's worth doing them.
2010 if (!prev_inuse(p
)) {
2011 /* Note that we cannot even look at prev unless it is not inuse */
2012 mchunkptr prv
= prev_chunk(p
);
2013 assert(next_chunk(prv
) == p
);
2014 do_check_free_chunk(av
, prv
);
2017 if (next
== av
->top
) {
2018 assert(prev_inuse(next
));
2019 assert(chunksize(next
) >= MINSIZE
);
2021 else if (!inuse(next
))
2022 do_check_free_chunk(av
, next
);
2026 Properties of chunks recycled from fastbins
2029 static void do_check_remalloced_chunk(mstate av
, mchunkptr p
, INTERNAL_SIZE_T s
)
2031 INTERNAL_SIZE_T sz
= p
->size
& ~(PREV_INUSE
|NON_MAIN_ARENA
);
2033 if (!chunk_is_mmapped(p
)) {
2034 assert(av
== arena_for_chunk(p
));
2035 if (chunk_non_main_arena(p
))
2036 assert(av
!= &main_arena
);
2038 assert(av
== &main_arena
);
2041 do_check_inuse_chunk(av
, p
);
2043 /* Legal size ... */
2044 assert((sz
& MALLOC_ALIGN_MASK
) == 0);
2045 assert((unsigned long)(sz
) >= MINSIZE
);
2046 /* ... and alignment */
2047 assert(aligned_OK(chunk2mem(p
)));
2048 /* chunk is less than MINSIZE more than request */
2049 assert((long)(sz
) - (long)(s
) >= 0);
2050 assert((long)(sz
) - (long)(s
+ MINSIZE
) < 0);
2054 Properties of nonrecycled chunks at the point they are malloced
2057 static void do_check_malloced_chunk(mstate av
, mchunkptr p
, INTERNAL_SIZE_T s
)
2059 /* same as recycled case ... */
2060 do_check_remalloced_chunk(av
, p
, s
);
2063 ... plus, must obey implementation invariant that prev_inuse is
2064 always true of any allocated chunk; i.e., that each allocated
2065 chunk borders either a previously allocated and still in-use
2066 chunk, or the base of its memory arena. This is ensured
2067 by making all allocations from the `lowest' part of any found
2068 chunk. This does not necessarily hold however for chunks
2069 recycled via fastbins.
2072 assert(prev_inuse(p
));
2077 Properties of malloc_state.
2079 This may be useful for debugging malloc, as well as detecting user
2080 programmer errors that somehow write into malloc_state.
2082 If you are extending or experimenting with this malloc, you can
2083 probably figure out how to hack this routine to print out or
2084 display chunk addresses, sizes, bins, and other instrumentation.
2087 static void do_check_malloc_state(mstate av
)
2094 INTERNAL_SIZE_T size
;
2095 unsigned long total
= 0;
2098 /* internal size_t must be no wider than pointer type */
2099 assert(sizeof(INTERNAL_SIZE_T
) <= sizeof(char*));
2101 /* alignment is a power of 2 */
2102 assert((MALLOC_ALIGNMENT
& (MALLOC_ALIGNMENT
-1)) == 0);
2104 /* cannot run remaining checks until fully initialized */
2105 if (av
->top
== 0 || av
->top
== initial_top(av
))
2108 /* pagesize is a power of 2 */
2109 assert((GLRO(dl_pagesize
) & (GLRO(dl_pagesize
)-1)) == 0);
2111 /* A contiguous main_arena is consistent with sbrk_base. */
2112 if (av
== &main_arena
&& contiguous(av
))
2113 assert((char*)mp_
.sbrk_base
+ av
->system_mem
==
2114 (char*)av
->top
+ chunksize(av
->top
));
2116 /* properties of fastbins */
2118 /* max_fast is in allowed range */
2119 assert((get_max_fast () & ~1) <= request2size(MAX_FAST_SIZE
));
2121 max_fast_bin
= fastbin_index(get_max_fast ());
2123 for (i
= 0; i
< NFASTBINS
; ++i
) {
2124 p
= fastbin (av
, i
);
2126 /* The following test can only be performed for the main arena.
2127 While mallopt calls malloc_consolidate to get rid of all fast
2128 bins (especially those larger than the new maximum) this does
2129 only happen for the main arena. Trying to do this for any
2130 other arena would mean those arenas have to be locked and
2131 malloc_consolidate be called for them. This is excessive. And
2132 even if this is acceptable to somebody it still cannot solve
2133 the problem completely since if the arena is locked a
2134 concurrent malloc call might create a new arena which then
2135 could use the newly invalid fast bins. */
2137 /* all bins past max_fast are empty */
2138 if (av
== &main_arena
&& i
> max_fast_bin
)
2142 /* each chunk claims to be inuse */
2143 do_check_inuse_chunk(av
, p
);
2144 total
+= chunksize(p
);
2145 /* chunk belongs in this bin */
2146 assert(fastbin_index(chunksize(p
)) == i
);
2152 assert(have_fastchunks(av
));
2153 else if (!have_fastchunks(av
))
2156 /* check normal bins */
2157 for (i
= 1; i
< NBINS
; ++i
) {
2160 /* binmap is accurate (except for bin 1 == unsorted_chunks) */
2162 unsigned int binbit
= get_binmap(av
,i
);
2163 int empty
= last(b
) == b
;
2170 for (p
= last(b
); p
!= b
; p
= p
->bk
) {
2171 /* each chunk claims to be free */
2172 do_check_free_chunk(av
, p
);
2173 size
= chunksize(p
);
2176 /* chunk belongs in bin */
2177 idx
= bin_index(size
);
2179 /* lists are sorted */
2180 assert(p
->bk
== b
||
2181 (unsigned long)chunksize(p
->bk
) >= (unsigned long)chunksize(p
));
2183 if (!in_smallbin_range(size
))
2185 if (p
->fd_nextsize
!= NULL
)
2187 if (p
->fd_nextsize
== p
)
2188 assert (p
->bk_nextsize
== p
);
2191 if (p
->fd_nextsize
== first (b
))
2192 assert (chunksize (p
) < chunksize (p
->fd_nextsize
));
2194 assert (chunksize (p
) > chunksize (p
->fd_nextsize
));
2197 assert (chunksize (p
) > chunksize (p
->bk_nextsize
));
2199 assert (chunksize (p
) < chunksize (p
->bk_nextsize
));
2203 assert (p
->bk_nextsize
== NULL
);
2205 } else if (!in_smallbin_range(size
))
2206 assert (p
->fd_nextsize
== NULL
&& p
->bk_nextsize
== NULL
);
2207 /* chunk is followed by a legal chain of inuse chunks */
2208 for (q
= next_chunk(p
);
2209 (q
!= av
->top
&& inuse(q
) &&
2210 (unsigned long)(chunksize(q
)) >= MINSIZE
);
2212 do_check_inuse_chunk(av
, q
);
2216 /* top chunk is OK */
2217 check_chunk(av
, av
->top
);
2219 /* sanity checks for statistics */
2221 assert(mp_
.n_mmaps
<= mp_
.max_n_mmaps
);
2223 assert((unsigned long)(av
->system_mem
) <=
2224 (unsigned long)(av
->max_system_mem
));
2226 assert((unsigned long)(mp_
.mmapped_mem
) <=
2227 (unsigned long)(mp_
.max_mmapped_mem
));
2232 /* ----------------- Support for debugging hooks -------------------- */
2236 /* ----------- Routines dealing with system allocation -------------- */
2239 sysmalloc handles malloc cases requiring more memory from the system.
2240 On entry, it is assumed that av->top does not have enough
2241 space to service request for nb bytes, thus requiring that av->top
2242 be extended or replaced.
2245 static void* sysmalloc(INTERNAL_SIZE_T nb
, mstate av
)
2247 mchunkptr old_top
; /* incoming value of av->top */
2248 INTERNAL_SIZE_T old_size
; /* its size */
2249 char* old_end
; /* its end address */
2251 long size
; /* arg to first MORECORE or mmap call */
2252 char* brk
; /* return value from MORECORE */
2254 long correction
; /* arg to 2nd MORECORE call */
2255 char* snd_brk
; /* 2nd return val */
2257 INTERNAL_SIZE_T front_misalign
; /* unusable bytes at front of new space */
2258 INTERNAL_SIZE_T end_misalign
; /* partial page left at end of new space */
2259 char* aligned_brk
; /* aligned offset into brk */
2261 mchunkptr p
; /* the allocated/returned chunk */
2262 mchunkptr remainder
; /* remainder from allocation */
2263 unsigned long remainder_size
; /* its size */
2265 unsigned long sum
; /* for updating stats */
2267 size_t pagemask
= GLRO(dl_pagesize
) - 1;
2268 bool tried_mmap
= false;
2272 If have mmap, and the request size meets the mmap threshold, and
2273 the system supports mmap, and there are few enough currently
2274 allocated mmapped regions, try to directly map this request
2275 rather than expanding top.
2278 if ((unsigned long)(nb
) >= (unsigned long)(mp_
.mmap_threshold
) &&
2279 (mp_
.n_mmaps
< mp_
.n_mmaps_max
)) {
2281 char* mm
; /* return value from mmap call*/
2285 Round up size to nearest page. For mmapped chunks, the overhead
2286 is one SIZE_SZ unit larger than for normal chunks, because there
2287 is no following chunk whose prev_size field could be used.
2289 See the front_misalign handling below, for glibc there is no
2290 need for further alignments unless we have have high alignment.
2292 if (MALLOC_ALIGNMENT
== 2 * SIZE_SZ
)
2293 size
= (nb
+ SIZE_SZ
+ pagemask
) & ~pagemask
;
2295 size
= (nb
+ SIZE_SZ
+ MALLOC_ALIGN_MASK
+ pagemask
) & ~pagemask
;
2298 /* Don't try if size wraps around 0 */
2299 if ((unsigned long)(size
) > (unsigned long)(nb
)) {
2301 mm
= (char*)(MMAP(0, size
, PROT_READ
|PROT_WRITE
, 0));
2303 if (mm
!= MAP_FAILED
) {
2306 The offset to the start of the mmapped region is stored
2307 in the prev_size field of the chunk. This allows us to adjust
2308 returned start address to meet alignment requirements here
2309 and in memalign(), and still be able to compute proper
2310 address argument for later munmap in free() and realloc().
2313 if (MALLOC_ALIGNMENT
== 2 * SIZE_SZ
)
2315 /* For glibc, chunk2mem increases the address by 2*SIZE_SZ and
2316 MALLOC_ALIGN_MASK is 2*SIZE_SZ-1. Each mmap'ed area is page
2317 aligned and therefore definitely MALLOC_ALIGN_MASK-aligned. */
2318 assert (((INTERNAL_SIZE_T
)chunk2mem(mm
) & MALLOC_ALIGN_MASK
) == 0);
2322 front_misalign
= (INTERNAL_SIZE_T
)chunk2mem(mm
) & MALLOC_ALIGN_MASK
;
2323 if (front_misalign
> 0) {
2324 correction
= MALLOC_ALIGNMENT
- front_misalign
;
2325 p
= (mchunkptr
)(mm
+ correction
);
2326 p
->prev_size
= correction
;
2327 set_head(p
, (size
- correction
) |IS_MMAPPED
);
2332 set_head(p
, size
|IS_MMAPPED
);
2335 /* update statistics */
2337 if (++mp_
.n_mmaps
> mp_
.max_n_mmaps
)
2338 mp_
.max_n_mmaps
= mp_
.n_mmaps
;
2340 sum
= mp_
.mmapped_mem
+= size
;
2341 if (sum
> (unsigned long)(mp_
.max_mmapped_mem
))
2342 mp_
.max_mmapped_mem
= sum
;
2346 return chunk2mem(p
);
2351 /* Record incoming configuration of top */
2354 old_size
= chunksize(old_top
);
2355 old_end
= (char*)(chunk_at_offset(old_top
, old_size
));
2357 brk
= snd_brk
= (char*)(MORECORE_FAILURE
);
2360 If not the first time through, we require old_size to be
2361 at least MINSIZE and to have prev_inuse set.
2364 assert((old_top
== initial_top(av
) && old_size
== 0) ||
2365 ((unsigned long) (old_size
) >= MINSIZE
&&
2366 prev_inuse(old_top
) &&
2367 ((unsigned long)old_end
& pagemask
) == 0));
2369 /* Precondition: not enough current space to satisfy nb request */
2370 assert((unsigned long)(old_size
) < (unsigned long)(nb
+ MINSIZE
));
2373 if (av
!= &main_arena
) {
2375 heap_info
*old_heap
, *heap
;
2376 size_t old_heap_size
;
2378 /* First try to extend the current heap. */
2379 old_heap
= heap_for_ptr(old_top
);
2380 old_heap_size
= old_heap
->size
;
2381 if ((long) (MINSIZE
+ nb
- old_size
) > 0
2382 && grow_heap(old_heap
, MINSIZE
+ nb
- old_size
) == 0) {
2383 av
->system_mem
+= old_heap
->size
- old_heap_size
;
2384 arena_mem
+= old_heap
->size
- old_heap_size
;
2385 set_head(old_top
, (((char *)old_heap
+ old_heap
->size
) - (char *)old_top
)
2388 else if ((heap
= new_heap(nb
+ (MINSIZE
+ sizeof(*heap
)), mp_
.top_pad
))) {
2389 /* Use a newly allocated heap. */
2391 heap
->prev
= old_heap
;
2392 av
->system_mem
+= heap
->size
;
2393 arena_mem
+= heap
->size
;
2394 /* Set up the new top. */
2395 top(av
) = chunk_at_offset(heap
, sizeof(*heap
));
2396 set_head(top(av
), (heap
->size
- sizeof(*heap
)) | PREV_INUSE
);
2398 /* Setup fencepost and free the old top chunk with a multiple of
2399 MALLOC_ALIGNMENT in size. */
2400 /* The fencepost takes at least MINSIZE bytes, because it might
2401 become the top chunk again later. Note that a footer is set
2402 up, too, although the chunk is marked in use. */
2403 old_size
= (old_size
- MINSIZE
) & ~MALLOC_ALIGN_MASK
;
2404 set_head(chunk_at_offset(old_top
, old_size
+ 2*SIZE_SZ
), 0|PREV_INUSE
);
2405 if (old_size
>= MINSIZE
) {
2406 set_head(chunk_at_offset(old_top
, old_size
), (2*SIZE_SZ
)|PREV_INUSE
);
2407 set_foot(chunk_at_offset(old_top
, old_size
), (2*SIZE_SZ
));
2408 set_head(old_top
, old_size
|PREV_INUSE
|NON_MAIN_ARENA
);
2409 _int_free(av
, old_top
, 1);
2411 set_head(old_top
, (old_size
+ 2*SIZE_SZ
)|PREV_INUSE
);
2412 set_foot(old_top
, (old_size
+ 2*SIZE_SZ
));
2415 else if (!tried_mmap
)
2416 /* We can at least try to use to mmap memory. */
2419 } else { /* av == main_arena */
2422 /* Request enough space for nb + pad + overhead */
2424 size
= nb
+ mp_
.top_pad
+ MINSIZE
;
2427 If contiguous, we can subtract out existing space that we hope to
2428 combine with new space. We add it back later only if
2429 we don't actually get contiguous space.
2436 Round to a multiple of page size.
2437 If MORECORE is not contiguous, this ensures that we only call it
2438 with whole-page arguments. And if MORECORE is contiguous and
2439 this is not first time through, this preserves page-alignment of
2440 previous calls. Otherwise, we correct to page-align below.
2443 size
= (size
+ pagemask
) & ~pagemask
;
2446 Don't try to call MORECORE if argument is so big as to appear
2447 negative. Note that since mmap takes size_t arg, it may succeed
2448 below even if we cannot call MORECORE.
2452 brk
= (char*)(MORECORE(size
));
2454 if (brk
!= (char*)(MORECORE_FAILURE
)) {
2455 /* Call the `morecore' hook if necessary. */
2456 void (*hook
) (void) = force_reg (__after_morecore_hook
);
2457 if (__builtin_expect (hook
!= NULL
, 0))
2461 If have mmap, try using it as a backup when MORECORE fails or
2462 cannot be used. This is worth doing on systems that have "holes" in
2463 address space, so sbrk cannot extend to give contiguous space, but
2464 space is available elsewhere. Note that we ignore mmap max count
2465 and threshold limits, since the space will not be used as a
2466 segregated mmap region.
2469 /* Cannot merge with old top, so add its size back in */
2471 size
= (size
+ old_size
+ pagemask
) & ~pagemask
;
2473 /* If we are relying on mmap as backup, then use larger units */
2474 if ((unsigned long)(size
) < (unsigned long)(MMAP_AS_MORECORE_SIZE
))
2475 size
= MMAP_AS_MORECORE_SIZE
;
2477 /* Don't try if size wraps around 0 */
2478 if ((unsigned long)(size
) > (unsigned long)(nb
)) {
2480 char *mbrk
= (char*)(MMAP(0, size
, PROT_READ
|PROT_WRITE
, 0));
2482 if (mbrk
!= MAP_FAILED
) {
2484 /* We do not need, and cannot use, another sbrk call to find end */
2486 snd_brk
= brk
+ size
;
2489 Record that we no longer have a contiguous sbrk region.
2490 After the first time mmap is used as backup, we do not
2491 ever rely on contiguous space since this could incorrectly
2494 set_noncontiguous(av
);
2499 if (brk
!= (char*)(MORECORE_FAILURE
)) {
2500 if (mp_
.sbrk_base
== 0)
2501 mp_
.sbrk_base
= brk
;
2502 av
->system_mem
+= size
;
2505 If MORECORE extends previous space, we can likewise extend top size.
2508 if (brk
== old_end
&& snd_brk
== (char*)(MORECORE_FAILURE
))
2509 set_head(old_top
, (size
+ old_size
) | PREV_INUSE
);
2511 else if (contiguous(av
) && old_size
&& brk
< old_end
) {
2512 /* Oops! Someone else killed our space.. Can't touch anything. */
2513 malloc_printerr (3, "break adjusted to free malloc space", brk
);
2517 Otherwise, make adjustments:
2519 * If the first time through or noncontiguous, we need to call sbrk
2520 just to find out where the end of memory lies.
2522 * We need to ensure that all returned chunks from malloc will meet
2525 * If there was an intervening foreign sbrk, we need to adjust sbrk
2526 request size to account for fact that we will not be able to
2527 combine new space with existing space in old_top.
2529 * Almost all systems internally allocate whole pages at a time, in
2530 which case we might as well use the whole last page of request.
2531 So we allocate enough more memory to hit a page boundary now,
2532 which in turn causes future contiguous calls to page-align.
2541 /* handle contiguous cases */
2542 if (contiguous(av
)) {
2544 /* Count foreign sbrk as system_mem. */
2546 av
->system_mem
+= brk
- old_end
;
2548 /* Guarantee alignment of first new chunk made from this space */
2550 front_misalign
= (INTERNAL_SIZE_T
)chunk2mem(brk
) & MALLOC_ALIGN_MASK
;
2551 if (front_misalign
> 0) {
2554 Skip over some bytes to arrive at an aligned position.
2555 We don't need to specially mark these wasted front bytes.
2556 They will never be accessed anyway because
2557 prev_inuse of av->top (and any chunk created from its start)
2558 is always true after initialization.
2561 correction
= MALLOC_ALIGNMENT
- front_misalign
;
2562 aligned_brk
+= correction
;
2566 If this isn't adjacent to existing space, then we will not
2567 be able to merge with old_top space, so must add to 2nd request.
2570 correction
+= old_size
;
2572 /* Extend the end address to hit a page boundary */
2573 end_misalign
= (INTERNAL_SIZE_T
)(brk
+ size
+ correction
);
2574 correction
+= ((end_misalign
+ pagemask
) & ~pagemask
) - end_misalign
;
2576 assert(correction
>= 0);
2577 snd_brk
= (char*)(MORECORE(correction
));
2580 If can't allocate correction, try to at least find out current
2581 brk. It might be enough to proceed without failing.
2583 Note that if second sbrk did NOT fail, we assume that space
2584 is contiguous with first sbrk. This is a safe assumption unless
2585 program is multithreaded but doesn't use locks and a foreign sbrk
2586 occurred between our first and second calls.
2589 if (snd_brk
== (char*)(MORECORE_FAILURE
)) {
2591 snd_brk
= (char*)(MORECORE(0));
2593 /* Call the `morecore' hook if necessary. */
2594 void (*hook
) (void) = force_reg (__after_morecore_hook
);
2595 if (__builtin_expect (hook
!= NULL
, 0))
2600 /* handle non-contiguous cases */
2602 if (MALLOC_ALIGNMENT
== 2 * SIZE_SZ
)
2603 /* MORECORE/mmap must correctly align */
2604 assert(((unsigned long)chunk2mem(brk
) & MALLOC_ALIGN_MASK
) == 0);
2606 front_misalign
= (INTERNAL_SIZE_T
)chunk2mem(brk
) & MALLOC_ALIGN_MASK
;
2607 if (front_misalign
> 0) {
2610 Skip over some bytes to arrive at an aligned position.
2611 We don't need to specially mark these wasted front bytes.
2612 They will never be accessed anyway because
2613 prev_inuse of av->top (and any chunk created from its start)
2614 is always true after initialization.
2617 aligned_brk
+= MALLOC_ALIGNMENT
- front_misalign
;
2621 /* Find out current end of memory */
2622 if (snd_brk
== (char*)(MORECORE_FAILURE
)) {
2623 snd_brk
= (char*)(MORECORE(0));
2627 /* Adjust top based on results of second sbrk */
2628 if (snd_brk
!= (char*)(MORECORE_FAILURE
)) {
2629 av
->top
= (mchunkptr
)aligned_brk
;
2630 set_head(av
->top
, (snd_brk
- aligned_brk
+ correction
) | PREV_INUSE
);
2631 av
->system_mem
+= correction
;
2634 If not the first time through, we either have a
2635 gap due to foreign sbrk or a non-contiguous region. Insert a
2636 double fencepost at old_top to prevent consolidation with space
2637 we don't own. These fenceposts are artificial chunks that are
2638 marked as inuse and are in any case too small to use. We need
2639 two to make sizes and alignments work out.
2642 if (old_size
!= 0) {
2644 Shrink old_top to insert fenceposts, keeping size a
2645 multiple of MALLOC_ALIGNMENT. We know there is at least
2646 enough space in old_top to do this.
2648 old_size
= (old_size
- 4*SIZE_SZ
) & ~MALLOC_ALIGN_MASK
;
2649 set_head(old_top
, old_size
| PREV_INUSE
);
2652 Note that the following assignments completely overwrite
2653 old_top when old_size was previously MINSIZE. This is
2654 intentional. We need the fencepost, even if old_top otherwise gets
2657 chunk_at_offset(old_top
, old_size
)->size
=
2658 (2*SIZE_SZ
)|PREV_INUSE
;
2660 chunk_at_offset(old_top
, old_size
+ 2*SIZE_SZ
)->size
=
2661 (2*SIZE_SZ
)|PREV_INUSE
;
2663 /* If possible, release the rest. */
2664 if (old_size
>= MINSIZE
) {
2665 _int_free(av
, old_top
, 1);
2673 } /* if (av != &main_arena) */
2675 if ((unsigned long)av
->system_mem
> (unsigned long)(av
->max_system_mem
))
2676 av
->max_system_mem
= av
->system_mem
;
2677 check_malloc_state(av
);
2679 /* finally, do the allocation */
2681 size
= chunksize(p
);
2683 /* check that one of the above allocation paths succeeded */
2684 if ((unsigned long)(size
) >= (unsigned long)(nb
+ MINSIZE
)) {
2685 remainder_size
= size
- nb
;
2686 remainder
= chunk_at_offset(p
, nb
);
2687 av
->top
= remainder
;
2688 set_head(p
, nb
| PREV_INUSE
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
2689 set_head(remainder
, remainder_size
| PREV_INUSE
);
2690 check_malloced_chunk(av
, p
, nb
);
2691 return chunk2mem(p
);
2694 /* catch all failure paths */
2695 __set_errno (ENOMEM
);
2701 systrim is an inverse of sorts to sysmalloc. It gives memory back
2702 to the system (via negative arguments to sbrk) if there is unused
2703 memory at the `high' end of the malloc pool. It is called
2704 automatically by free() when top space exceeds the trim
2705 threshold. It is also called by the public malloc_trim routine. It
2706 returns 1 if it actually released any memory, else 0.
2709 static int systrim(size_t pad
, mstate av
)
2711 long top_size
; /* Amount of top-most memory */
2712 long extra
; /* Amount to release */
2713 long released
; /* Amount actually released */
2714 char* current_brk
; /* address returned by pre-check sbrk call */
2715 char* new_brk
; /* address returned by post-check sbrk call */
2718 pagesz
= GLRO(dl_pagesize
);
2719 top_size
= chunksize(av
->top
);
2721 /* Release in pagesize units, keeping at least one page */
2722 extra
= (top_size
- pad
- MINSIZE
- 1) & ~(pagesz
- 1);
2727 Only proceed if end of memory is where we last set it.
2728 This avoids problems if there were foreign sbrk calls.
2730 current_brk
= (char*)(MORECORE(0));
2731 if (current_brk
== (char*)(av
->top
) + top_size
) {
2734 Attempt to release memory. We ignore MORECORE return value,
2735 and instead call again to find out where new end of memory is.
2736 This avoids problems if first call releases less than we asked,
2737 of if failure somehow altered brk value. (We could still
2738 encounter problems if it altered brk in some very bad way,
2739 but the only thing we can do is adjust anyway, which will cause
2740 some downstream failure.)
2744 /* Call the `morecore' hook if necessary. */
2745 void (*hook
) (void) = force_reg (__after_morecore_hook
);
2746 if (__builtin_expect (hook
!= NULL
, 0))
2748 new_brk
= (char*)(MORECORE(0));
2750 if (new_brk
!= (char*)MORECORE_FAILURE
) {
2751 released
= (long)(current_brk
- new_brk
);
2753 if (released
!= 0) {
2754 /* Success. Adjust top. */
2755 av
->system_mem
-= released
;
2756 set_head(av
->top
, (top_size
- released
) | PREV_INUSE
);
2757 check_malloc_state(av
);
2768 munmap_chunk(mchunkptr p
)
2770 INTERNAL_SIZE_T size
= chunksize(p
);
2772 assert (chunk_is_mmapped(p
));
2774 uintptr_t block
= (uintptr_t) p
- p
->prev_size
;
2775 size_t total_size
= p
->prev_size
+ size
;
2776 /* Unfortunately we have to do the compilers job by hand here. Normally
2777 we would test BLOCK and TOTAL-SIZE separately for compliance with the
2778 page size. But gcc does not recognize the optimization possibility
2779 (in the moment at least) so we combine the two values into one before
2781 if (__builtin_expect (((block
| total_size
) & (GLRO(dl_pagesize
) - 1)) != 0, 0))
2783 malloc_printerr (check_action
, "munmap_chunk(): invalid pointer",
2789 mp_
.mmapped_mem
-= total_size
;
2791 /* If munmap failed the process virtual memory address space is in a
2792 bad shape. Just leave the block hanging around, the process will
2793 terminate shortly anyway since not much can be done. */
2794 __munmap((char *)block
, total_size
);
2801 mremap_chunk(mchunkptr p
, size_t new_size
)
2803 size_t page_mask
= GLRO(dl_pagesize
) - 1;
2804 INTERNAL_SIZE_T offset
= p
->prev_size
;
2805 INTERNAL_SIZE_T size
= chunksize(p
);
2808 assert (chunk_is_mmapped(p
));
2809 assert(((size
+ offset
) & (GLRO(dl_pagesize
)-1)) == 0);
2811 /* Note the extra SIZE_SZ overhead as in mmap_chunk(). */
2812 new_size
= (new_size
+ offset
+ SIZE_SZ
+ page_mask
) & ~page_mask
;
2814 /* No need to remap if the number of pages does not change. */
2815 if (size
+ offset
== new_size
)
2818 cp
= (char *)__mremap((char *)p
- offset
, size
+ offset
, new_size
,
2821 if (cp
== MAP_FAILED
) return 0;
2823 p
= (mchunkptr
)(cp
+ offset
);
2825 assert(aligned_OK(chunk2mem(p
)));
2827 assert((p
->prev_size
== offset
));
2828 set_head(p
, (new_size
- offset
)|IS_MMAPPED
);
2830 mp_
.mmapped_mem
-= size
+ offset
;
2831 mp_
.mmapped_mem
+= new_size
;
2832 if ((unsigned long)mp_
.mmapped_mem
> (unsigned long)mp_
.max_mmapped_mem
)
2833 mp_
.max_mmapped_mem
= mp_
.mmapped_mem
;
2837 #endif /* HAVE_MREMAP */
2839 /*------------------------ Public wrappers. --------------------------------*/
2842 __libc_malloc(size_t bytes
)
2847 void *(*hook
) (size_t, const void *)
2848 = force_reg (__malloc_hook
);
2849 if (__builtin_expect (hook
!= NULL
, 0))
2850 return (*hook
)(bytes
, RETURN_ADDRESS (0));
2852 arena_lookup(ar_ptr
);
2854 arena_lock(ar_ptr
, bytes
);
2857 victim
= _int_malloc(ar_ptr
, bytes
);
2859 ar_ptr
= arena_get_retry(ar_ptr
, bytes
);
2860 if (__builtin_expect(ar_ptr
!= NULL
, 1)) {
2861 victim
= _int_malloc(ar_ptr
, bytes
);
2862 (void)mutex_unlock(&ar_ptr
->mutex
);
2865 (void)mutex_unlock(&ar_ptr
->mutex
);
2866 assert(!victim
|| chunk_is_mmapped(mem2chunk(victim
)) ||
2867 ar_ptr
== arena_for_chunk(mem2chunk(victim
)));
2870 libc_hidden_def(__libc_malloc
)
2873 __libc_free(void* mem
)
2876 mchunkptr p
; /* chunk corresponding to mem */
2878 void (*hook
) (void *, const void *)
2879 = force_reg (__free_hook
);
2880 if (__builtin_expect (hook
!= NULL
, 0)) {
2881 (*hook
)(mem
, RETURN_ADDRESS (0));
2885 if (mem
== 0) /* free(0) has no effect */
2890 if (chunk_is_mmapped(p
)) /* release mmapped memory. */
2892 /* see if the dynamic brk/mmap threshold needs adjusting */
2893 if (!mp_
.no_dyn_threshold
2894 && p
->size
> mp_
.mmap_threshold
2895 && p
->size
<= DEFAULT_MMAP_THRESHOLD_MAX
)
2897 mp_
.mmap_threshold
= chunksize (p
);
2898 mp_
.trim_threshold
= 2 * mp_
.mmap_threshold
;
2899 LIBC_PROBE (memory_mallopt_free_dyn_thresholds
, 2,
2900 mp_
.mmap_threshold
, mp_
.trim_threshold
);
2906 ar_ptr
= arena_for_chunk(p
);
2907 _int_free(ar_ptr
, p
, 0);
2909 libc_hidden_def (__libc_free
)
2912 __libc_realloc(void* oldmem
, size_t bytes
)
2915 INTERNAL_SIZE_T nb
; /* padded request size */
2917 void* newp
; /* chunk to return */
2919 void *(*hook
) (void *, size_t, const void *) =
2920 force_reg (__realloc_hook
);
2921 if (__builtin_expect (hook
!= NULL
, 0))
2922 return (*hook
)(oldmem
, bytes
, RETURN_ADDRESS (0));
2924 #if REALLOC_ZERO_BYTES_FREES
2925 if (bytes
== 0 && oldmem
!= NULL
) { __libc_free(oldmem
); return 0; }
2928 /* realloc of null is supposed to be same as malloc */
2929 if (oldmem
== 0) return __libc_malloc(bytes
);
2931 /* chunk corresponding to oldmem */
2932 const mchunkptr oldp
= mem2chunk(oldmem
);
2934 const INTERNAL_SIZE_T oldsize
= chunksize(oldp
);
2936 /* Little security check which won't hurt performance: the
2937 allocator never wrapps around at the end of the address space.
2938 Therefore we can exclude some size values which might appear
2939 here by accident or by "design" from some intruder. */
2940 if (__builtin_expect ((uintptr_t) oldp
> (uintptr_t) -oldsize
, 0)
2941 || __builtin_expect (misaligned_chunk (oldp
), 0))
2943 malloc_printerr (check_action
, "realloc(): invalid pointer", oldmem
);
2947 checked_request2size(bytes
, nb
);
2949 if (chunk_is_mmapped(oldp
))
2954 newp
= mremap_chunk(oldp
, nb
);
2955 if(newp
) return chunk2mem(newp
);
2957 /* Note the extra SIZE_SZ overhead. */
2958 if(oldsize
- SIZE_SZ
>= nb
) return oldmem
; /* do nothing */
2959 /* Must alloc, copy, free. */
2960 newmem
= __libc_malloc(bytes
);
2961 if (newmem
== 0) return 0; /* propagate failure */
2962 MALLOC_COPY(newmem
, oldmem
, oldsize
- 2*SIZE_SZ
);
2967 ar_ptr
= arena_for_chunk(oldp
);
2969 if(!mutex_trylock(&ar_ptr
->mutex
))
2970 ++(ar_ptr
->stat_lock_direct
);
2972 (void)mutex_lock(&ar_ptr
->mutex
);
2973 ++(ar_ptr
->stat_lock_wait
);
2976 (void)mutex_lock(&ar_ptr
->mutex
);
2979 #if !defined PER_THREAD
2980 LIBC_PROBE (memory_arena_reuse_realloc
, 1, ar_ptr
);
2981 /* As in malloc(), remember this arena for the next allocation. */
2982 tsd_setspecific(arena_key
, (void *)ar_ptr
);
2985 newp
= _int_realloc(ar_ptr
, oldp
, oldsize
, nb
);
2987 (void)mutex_unlock(&ar_ptr
->mutex
);
2988 assert(!newp
|| chunk_is_mmapped(mem2chunk(newp
)) ||
2989 ar_ptr
== arena_for_chunk(mem2chunk(newp
)));
2993 /* Try harder to allocate memory in other arenas. */
2994 newp
= __libc_malloc(bytes
);
2997 MALLOC_COPY (newp
, oldmem
, oldsize
- SIZE_SZ
);
2998 _int_free(ar_ptr
, oldp
, 0);
3004 libc_hidden_def (__libc_realloc
)
3007 __libc_memalign(size_t alignment
, size_t bytes
)
3012 void *(*hook
) (size_t, size_t, const void *) =
3013 force_reg (__memalign_hook
);
3014 if (__builtin_expect (hook
!= NULL
, 0))
3015 return (*hook
)(alignment
, bytes
, RETURN_ADDRESS (0));
3017 /* If need less alignment than we give anyway, just relay to malloc */
3018 if (alignment
<= MALLOC_ALIGNMENT
) return __libc_malloc(bytes
);
3020 /* Otherwise, ensure that it is at least a minimum chunk size */
3021 if (alignment
< MINSIZE
) alignment
= MINSIZE
;
3023 /* Check for overflow. */
3024 if (bytes
> SIZE_MAX
- alignment
- MINSIZE
)
3026 __set_errno (ENOMEM
);
3030 arena_get(ar_ptr
, bytes
+ alignment
+ MINSIZE
);
3033 p
= _int_memalign(ar_ptr
, alignment
, bytes
);
3035 ar_ptr
= arena_get_retry (ar_ptr
, bytes
);
3036 if (__builtin_expect(ar_ptr
!= NULL
, 1)) {
3037 p
= _int_memalign(ar_ptr
, alignment
, bytes
);
3038 (void)mutex_unlock(&ar_ptr
->mutex
);
3041 (void)mutex_unlock(&ar_ptr
->mutex
);
3042 assert(!p
|| chunk_is_mmapped(mem2chunk(p
)) ||
3043 ar_ptr
== arena_for_chunk(mem2chunk(p
)));
3047 weak_alias (__libc_memalign
, aligned_alloc
)
3048 libc_hidden_def (__libc_memalign
)
3051 __libc_valloc(size_t bytes
)
3056 if(__malloc_initialized
< 0)
3059 size_t pagesz
= GLRO(dl_pagesize
);
3061 /* Check for overflow. */
3062 if (bytes
> SIZE_MAX
- pagesz
- MINSIZE
)
3064 __set_errno (ENOMEM
);
3068 void *(*hook
) (size_t, size_t, const void *) =
3069 force_reg (__memalign_hook
);
3070 if (__builtin_expect (hook
!= NULL
, 0))
3071 return (*hook
)(pagesz
, bytes
, RETURN_ADDRESS (0));
3073 arena_get(ar_ptr
, bytes
+ pagesz
+ MINSIZE
);
3076 p
= _int_valloc(ar_ptr
, bytes
);
3078 ar_ptr
= arena_get_retry (ar_ptr
, bytes
);
3079 if (__builtin_expect(ar_ptr
!= NULL
, 1)) {
3080 p
= _int_memalign(ar_ptr
, pagesz
, bytes
);
3081 (void)mutex_unlock(&ar_ptr
->mutex
);
3084 (void)mutex_unlock (&ar_ptr
->mutex
);
3085 assert(!p
|| chunk_is_mmapped(mem2chunk(p
)) ||
3086 ar_ptr
== arena_for_chunk(mem2chunk(p
)));
3092 __libc_pvalloc(size_t bytes
)
3097 if(__malloc_initialized
< 0)
3100 size_t pagesz
= GLRO(dl_pagesize
);
3101 size_t page_mask
= GLRO(dl_pagesize
) - 1;
3102 size_t rounded_bytes
= (bytes
+ page_mask
) & ~(page_mask
);
3104 /* Check for overflow. */
3105 if (bytes
> SIZE_MAX
- 2*pagesz
- MINSIZE
)
3107 __set_errno (ENOMEM
);
3111 void *(*hook
) (size_t, size_t, const void *) =
3112 force_reg (__memalign_hook
);
3113 if (__builtin_expect (hook
!= NULL
, 0))
3114 return (*hook
)(pagesz
, rounded_bytes
, RETURN_ADDRESS (0));
3116 arena_get(ar_ptr
, bytes
+ 2*pagesz
+ MINSIZE
);
3117 p
= _int_pvalloc(ar_ptr
, bytes
);
3119 ar_ptr
= arena_get_retry (ar_ptr
, bytes
+ 2*pagesz
+ MINSIZE
);
3120 if (__builtin_expect(ar_ptr
!= NULL
, 1)) {
3121 p
= _int_memalign(ar_ptr
, pagesz
, rounded_bytes
);
3122 (void)mutex_unlock(&ar_ptr
->mutex
);
3125 (void)mutex_unlock(&ar_ptr
->mutex
);
3126 assert(!p
|| chunk_is_mmapped(mem2chunk(p
)) ||
3127 ar_ptr
== arena_for_chunk(mem2chunk(p
)));
3133 __libc_calloc(size_t n
, size_t elem_size
)
3136 mchunkptr oldtop
, p
;
3137 INTERNAL_SIZE_T bytes
, sz
, csz
, oldtopsize
;
3139 unsigned long clearsize
;
3140 unsigned long nclears
;
3143 /* size_t is unsigned so the behavior on overflow is defined. */
3144 bytes
= n
* elem_size
;
3145 #define HALF_INTERNAL_SIZE_T \
3146 (((INTERNAL_SIZE_T) 1) << (8 * sizeof (INTERNAL_SIZE_T) / 2))
3147 if (__builtin_expect ((n
| elem_size
) >= HALF_INTERNAL_SIZE_T
, 0)) {
3148 if (elem_size
!= 0 && bytes
/ elem_size
!= n
) {
3149 __set_errno (ENOMEM
);
3154 void *(*hook
) (size_t, const void *) =
3155 force_reg (__malloc_hook
);
3156 if (__builtin_expect (hook
!= NULL
, 0)) {
3158 mem
= (*hook
)(sz
, RETURN_ADDRESS (0));
3161 return memset(mem
, 0, sz
);
3170 /* Check if we hand out the top chunk, in which case there may be no
3174 oldtopsize
= chunksize(top(av
));
3175 #if MORECORE_CLEARS < 2
3176 /* Only newly allocated memory is guaranteed to be cleared. */
3177 if (av
== &main_arena
&&
3178 oldtopsize
< mp_
.sbrk_base
+ av
->max_system_mem
- (char *)oldtop
)
3179 oldtopsize
= (mp_
.sbrk_base
+ av
->max_system_mem
- (char *)oldtop
);
3181 if (av
!= &main_arena
)
3183 heap_info
*heap
= heap_for_ptr (oldtop
);
3184 if (oldtopsize
< (char *) heap
+ heap
->mprotect_size
- (char *) oldtop
)
3185 oldtopsize
= (char *) heap
+ heap
->mprotect_size
- (char *) oldtop
;
3188 mem
= _int_malloc(av
, sz
);
3191 assert(!mem
|| chunk_is_mmapped(mem2chunk(mem
)) ||
3192 av
== arena_for_chunk(mem2chunk(mem
)));
3195 av
= arena_get_retry (av
, sz
);
3196 if (__builtin_expect(av
!= NULL
, 1)) {
3197 mem
= _int_malloc(av
, sz
);
3198 (void)mutex_unlock(&av
->mutex
);
3200 if (mem
== 0) return 0;
3202 (void)mutex_unlock(&av
->mutex
);
3205 /* Two optional cases in which clearing not necessary */
3206 if (chunk_is_mmapped (p
))
3208 if (__builtin_expect (perturb_byte
, 0))
3209 MALLOC_ZERO (mem
, sz
);
3216 if (perturb_byte
== 0 && (p
== oldtop
&& csz
> oldtopsize
)) {
3217 /* clear only the bytes from non-freshly-sbrked memory */
3222 /* Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that
3223 contents have an odd number of INTERNAL_SIZE_T-sized words;
3225 d
= (INTERNAL_SIZE_T
*)mem
;
3226 clearsize
= csz
- SIZE_SZ
;
3227 nclears
= clearsize
/ sizeof(INTERNAL_SIZE_T
);
3228 assert(nclears
>= 3);
3231 MALLOC_ZERO(d
, clearsize
);
3255 ------------------------------ malloc ------------------------------
3259 _int_malloc(mstate av
, size_t bytes
)
3261 INTERNAL_SIZE_T nb
; /* normalized request size */
3262 unsigned int idx
; /* associated bin index */
3263 mbinptr bin
; /* associated bin */
3265 mchunkptr victim
; /* inspected/selected chunk */
3266 INTERNAL_SIZE_T size
; /* its size */
3267 int victim_index
; /* its bin index */
3269 mchunkptr remainder
; /* remainder from a split */
3270 unsigned long remainder_size
; /* its size */
3272 unsigned int block
; /* bit map traverser */
3273 unsigned int bit
; /* bit map traverser */
3274 unsigned int map
; /* current word of binmap */
3276 mchunkptr fwd
; /* misc temp for linking */
3277 mchunkptr bck
; /* misc temp for linking */
3279 const char *errstr
= NULL
;
3282 Convert request size to internal form by adding SIZE_SZ bytes
3283 overhead plus possibly more to obtain necessary alignment and/or
3284 to obtain a size of at least MINSIZE, the smallest allocatable
3285 size. Also, checked_request2size traps (returning 0) request sizes
3286 that are so large that they wrap around zero when padded and
3290 checked_request2size(bytes
, nb
);
3293 If the size qualifies as a fastbin, first check corresponding bin.
3294 This code is safe to execute even if av is not yet initialized, so we
3295 can try it without checking, which saves some time on this fast path.
3298 if ((unsigned long)(nb
) <= (unsigned long)(get_max_fast ())) {
3299 idx
= fastbin_index(nb
);
3300 mfastbinptr
* fb
= &fastbin (av
, idx
);
3308 while ((pp
= catomic_compare_and_exchange_val_acq (fb
, victim
->fd
, victim
))
3311 if (__builtin_expect (fastbin_index (chunksize (victim
)) != idx
, 0))
3313 errstr
= "malloc(): memory corruption (fast)";
3315 malloc_printerr (check_action
, errstr
, chunk2mem (victim
));
3318 check_remalloced_chunk(av
, victim
, nb
);
3319 void *p
= chunk2mem(victim
);
3320 if (__builtin_expect (perturb_byte
, 0))
3321 alloc_perturb (p
, bytes
);
3327 If a small request, check regular bin. Since these "smallbins"
3328 hold one size each, no searching within bins is necessary.
3329 (For a large request, we need to wait until unsorted chunks are
3330 processed to find best fit. But for small ones, fits are exact
3331 anyway, so we can check now, which is faster.)
3334 if (in_smallbin_range(nb
)) {
3335 idx
= smallbin_index(nb
);
3336 bin
= bin_at(av
,idx
);
3338 if ( (victim
= last(bin
)) != bin
) {
3339 if (victim
== 0) /* initialization check */
3340 malloc_consolidate(av
);
3343 if (__builtin_expect (bck
->fd
!= victim
, 0))
3345 errstr
= "malloc(): smallbin double linked list corrupted";
3348 set_inuse_bit_at_offset(victim
, nb
);
3352 if (av
!= &main_arena
)
3353 victim
->size
|= NON_MAIN_ARENA
;
3354 check_malloced_chunk(av
, victim
, nb
);
3355 void *p
= chunk2mem(victim
);
3356 if (__builtin_expect (perturb_byte
, 0))
3357 alloc_perturb (p
, bytes
);
3364 If this is a large request, consolidate fastbins before continuing.
3365 While it might look excessive to kill all fastbins before
3366 even seeing if there is space available, this avoids
3367 fragmentation problems normally associated with fastbins.
3368 Also, in practice, programs tend to have runs of either small or
3369 large requests, but less often mixtures, so consolidation is not
3370 invoked all that often in most programs. And the programs that
3371 it is called frequently in otherwise tend to fragment.
3375 idx
= largebin_index(nb
);
3376 if (have_fastchunks(av
))
3377 malloc_consolidate(av
);
3381 Process recently freed or remaindered chunks, taking one only if
3382 it is exact fit, or, if this a small request, the chunk is remainder from
3383 the most recent non-exact fit. Place other traversed chunks in
3384 bins. Note that this step is the only place in any routine where
3385 chunks are placed in bins.
3387 The outer loop here is needed because we might not realize until
3388 near the end of malloc that we should have consolidated, so must
3389 do so and retry. This happens at most once, and only when we would
3390 otherwise need to expand memory to service a "small" request.
3396 while ( (victim
= unsorted_chunks(av
)->bk
) != unsorted_chunks(av
)) {
3398 if (__builtin_expect (victim
->size
<= 2 * SIZE_SZ
, 0)
3399 || __builtin_expect (victim
->size
> av
->system_mem
, 0))
3400 malloc_printerr (check_action
, "malloc(): memory corruption",
3401 chunk2mem (victim
));
3402 size
= chunksize(victim
);
3405 If a small request, try to use last remainder if it is the
3406 only chunk in unsorted bin. This helps promote locality for
3407 runs of consecutive small requests. This is the only
3408 exception to best-fit, and applies only when there is
3409 no exact fit for a small chunk.
3412 if (in_smallbin_range(nb
) &&
3413 bck
== unsorted_chunks(av
) &&
3414 victim
== av
->last_remainder
&&
3415 (unsigned long)(size
) > (unsigned long)(nb
+ MINSIZE
)) {
3417 /* split and reattach remainder */
3418 remainder_size
= size
- nb
;
3419 remainder
= chunk_at_offset(victim
, nb
);
3420 unsorted_chunks(av
)->bk
= unsorted_chunks(av
)->fd
= remainder
;
3421 av
->last_remainder
= remainder
;
3422 remainder
->bk
= remainder
->fd
= unsorted_chunks(av
);
3423 if (!in_smallbin_range(remainder_size
))
3425 remainder
->fd_nextsize
= NULL
;
3426 remainder
->bk_nextsize
= NULL
;
3429 set_head(victim
, nb
| PREV_INUSE
|
3430 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
3431 set_head(remainder
, remainder_size
| PREV_INUSE
);
3432 set_foot(remainder
, remainder_size
);
3434 check_malloced_chunk(av
, victim
, nb
);
3435 void *p
= chunk2mem(victim
);
3436 if (__builtin_expect (perturb_byte
, 0))
3437 alloc_perturb (p
, bytes
);
3441 /* remove from unsorted list */
3442 unsorted_chunks(av
)->bk
= bck
;
3443 bck
->fd
= unsorted_chunks(av
);
3445 /* Take now instead of binning if exact fit */
3448 set_inuse_bit_at_offset(victim
, size
);
3449 if (av
!= &main_arena
)
3450 victim
->size
|= NON_MAIN_ARENA
;
3451 check_malloced_chunk(av
, victim
, nb
);
3452 void *p
= chunk2mem(victim
);
3453 if (__builtin_expect (perturb_byte
, 0))
3454 alloc_perturb (p
, bytes
);
3458 /* place chunk in bin */
3460 if (in_smallbin_range(size
)) {
3461 victim_index
= smallbin_index(size
);
3462 bck
= bin_at(av
, victim_index
);
3466 victim_index
= largebin_index(size
);
3467 bck
= bin_at(av
, victim_index
);
3470 /* maintain large bins in sorted order */
3472 /* Or with inuse bit to speed comparisons */
3474 /* if smaller than smallest, bypass loop below */
3475 assert((bck
->bk
->size
& NON_MAIN_ARENA
) == 0);
3476 if ((unsigned long)(size
) < (unsigned long)(bck
->bk
->size
)) {
3480 victim
->fd_nextsize
= fwd
->fd
;
3481 victim
->bk_nextsize
= fwd
->fd
->bk_nextsize
;
3482 fwd
->fd
->bk_nextsize
= victim
->bk_nextsize
->fd_nextsize
= victim
;
3485 assert((fwd
->size
& NON_MAIN_ARENA
) == 0);
3486 while ((unsigned long) size
< fwd
->size
)
3488 fwd
= fwd
->fd_nextsize
;
3489 assert((fwd
->size
& NON_MAIN_ARENA
) == 0);
3492 if ((unsigned long) size
== (unsigned long) fwd
->size
)
3493 /* Always insert in the second position. */
3497 victim
->fd_nextsize
= fwd
;
3498 victim
->bk_nextsize
= fwd
->bk_nextsize
;
3499 fwd
->bk_nextsize
= victim
;
3500 victim
->bk_nextsize
->fd_nextsize
= victim
;
3505 victim
->fd_nextsize
= victim
->bk_nextsize
= victim
;
3508 mark_bin(av
, victim_index
);
3514 #define MAX_ITERS 10000
3515 if (++iters
>= MAX_ITERS
)
3520 If a large request, scan through the chunks of current bin in
3521 sorted order to find smallest that fits. Use the skip list for this.
3524 if (!in_smallbin_range(nb
)) {
3525 bin
= bin_at(av
, idx
);
3527 /* skip scan if empty or largest chunk is too small */
3528 if ((victim
= first(bin
)) != bin
&&
3529 (unsigned long)(victim
->size
) >= (unsigned long)(nb
)) {
3531 victim
= victim
->bk_nextsize
;
3532 while (((unsigned long)(size
= chunksize(victim
)) <
3533 (unsigned long)(nb
)))
3534 victim
= victim
->bk_nextsize
;
3536 /* Avoid removing the first entry for a size so that the skip
3537 list does not have to be rerouted. */
3538 if (victim
!= last(bin
) && victim
->size
== victim
->fd
->size
)
3539 victim
= victim
->fd
;
3541 remainder_size
= size
- nb
;
3542 unlink(victim
, bck
, fwd
);
3545 if (remainder_size
< MINSIZE
) {
3546 set_inuse_bit_at_offset(victim
, size
);
3547 if (av
!= &main_arena
)
3548 victim
->size
|= NON_MAIN_ARENA
;
3552 remainder
= chunk_at_offset(victim
, nb
);
3553 /* We cannot assume the unsorted list is empty and therefore
3554 have to perform a complete insert here. */
3555 bck
= unsorted_chunks(av
);
3557 if (__builtin_expect (fwd
->bk
!= bck
, 0))
3559 errstr
= "malloc(): corrupted unsorted chunks";
3562 remainder
->bk
= bck
;
3563 remainder
->fd
= fwd
;
3564 bck
->fd
= remainder
;
3565 fwd
->bk
= remainder
;
3566 if (!in_smallbin_range(remainder_size
))
3568 remainder
->fd_nextsize
= NULL
;
3569 remainder
->bk_nextsize
= NULL
;
3571 set_head(victim
, nb
| PREV_INUSE
|
3572 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
3573 set_head(remainder
, remainder_size
| PREV_INUSE
);
3574 set_foot(remainder
, remainder_size
);
3576 check_malloced_chunk(av
, victim
, nb
);
3577 void *p
= chunk2mem(victim
);
3578 if (__builtin_expect (perturb_byte
, 0))
3579 alloc_perturb (p
, bytes
);
3585 Search for a chunk by scanning bins, starting with next largest
3586 bin. This search is strictly by best-fit; i.e., the smallest
3587 (with ties going to approximately the least recently used) chunk
3588 that fits is selected.
3590 The bitmap avoids needing to check that most blocks are nonempty.
3591 The particular case of skipping all bins during warm-up phases
3592 when no chunks have been returned yet is faster than it might look.
3596 bin
= bin_at(av
,idx
);
3597 block
= idx2block(idx
);
3598 map
= av
->binmap
[block
];
3603 /* Skip rest of block if there are no more set bits in this block. */
3604 if (bit
> map
|| bit
== 0) {
3606 if (++block
>= BINMAPSIZE
) /* out of bins */
3608 } while ( (map
= av
->binmap
[block
]) == 0);
3610 bin
= bin_at(av
, (block
<< BINMAPSHIFT
));
3614 /* Advance to bin with set bit. There must be one. */
3615 while ((bit
& map
) == 0) {
3616 bin
= next_bin(bin
);
3621 /* Inspect the bin. It is likely to be non-empty */
3624 /* If a false alarm (empty bin), clear the bit. */
3625 if (victim
== bin
) {
3626 av
->binmap
[block
] = map
&= ~bit
; /* Write through */
3627 bin
= next_bin(bin
);
3632 size
= chunksize(victim
);
3634 /* We know the first chunk in this bin is big enough to use. */
3635 assert((unsigned long)(size
) >= (unsigned long)(nb
));
3637 remainder_size
= size
- nb
;
3640 unlink(victim
, bck
, fwd
);
3643 if (remainder_size
< MINSIZE
) {
3644 set_inuse_bit_at_offset(victim
, size
);
3645 if (av
!= &main_arena
)
3646 victim
->size
|= NON_MAIN_ARENA
;
3651 remainder
= chunk_at_offset(victim
, nb
);
3653 /* We cannot assume the unsorted list is empty and therefore
3654 have to perform a complete insert here. */
3655 bck
= unsorted_chunks(av
);
3657 if (__builtin_expect (fwd
->bk
!= bck
, 0))
3659 errstr
= "malloc(): corrupted unsorted chunks 2";
3662 remainder
->bk
= bck
;
3663 remainder
->fd
= fwd
;
3664 bck
->fd
= remainder
;
3665 fwd
->bk
= remainder
;
3667 /* advertise as last remainder */
3668 if (in_smallbin_range(nb
))
3669 av
->last_remainder
= remainder
;
3670 if (!in_smallbin_range(remainder_size
))
3672 remainder
->fd_nextsize
= NULL
;
3673 remainder
->bk_nextsize
= NULL
;
3675 set_head(victim
, nb
| PREV_INUSE
|
3676 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
3677 set_head(remainder
, remainder_size
| PREV_INUSE
);
3678 set_foot(remainder
, remainder_size
);
3680 check_malloced_chunk(av
, victim
, nb
);
3681 void *p
= chunk2mem(victim
);
3682 if (__builtin_expect (perturb_byte
, 0))
3683 alloc_perturb (p
, bytes
);
3690 If large enough, split off the chunk bordering the end of memory
3691 (held in av->top). Note that this is in accord with the best-fit
3692 search rule. In effect, av->top is treated as larger (and thus
3693 less well fitting) than any other available chunk since it can
3694 be extended to be as large as necessary (up to system
3697 We require that av->top always exists (i.e., has size >=
3698 MINSIZE) after initialization, so if it would otherwise be
3699 exhausted by current request, it is replenished. (The main
3700 reason for ensuring it exists is that we may need MINSIZE space
3701 to put in fenceposts in sysmalloc.)
3705 size
= chunksize(victim
);
3707 if ((unsigned long)(size
) >= (unsigned long)(nb
+ MINSIZE
)) {
3708 remainder_size
= size
- nb
;
3709 remainder
= chunk_at_offset(victim
, nb
);
3710 av
->top
= remainder
;
3711 set_head(victim
, nb
| PREV_INUSE
|
3712 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
3713 set_head(remainder
, remainder_size
| PREV_INUSE
);
3715 check_malloced_chunk(av
, victim
, nb
);
3716 void *p
= chunk2mem(victim
);
3717 if (__builtin_expect (perturb_byte
, 0))
3718 alloc_perturb (p
, bytes
);
3722 /* When we are using atomic ops to free fast chunks we can get
3723 here for all block sizes. */
3724 else if (have_fastchunks(av
)) {
3725 malloc_consolidate(av
);
3726 /* restore original bin index */
3727 if (in_smallbin_range(nb
))
3728 idx
= smallbin_index(nb
);
3730 idx
= largebin_index(nb
);
3734 Otherwise, relay to handle system-dependent cases
3737 void *p
= sysmalloc(nb
, av
);
3738 if (p
!= NULL
&& __builtin_expect (perturb_byte
, 0))
3739 alloc_perturb (p
, bytes
);
3746 ------------------------------ free ------------------------------
3750 _int_free(mstate av
, mchunkptr p
, int have_lock
)
3752 INTERNAL_SIZE_T size
; /* its size */
3753 mfastbinptr
* fb
; /* associated fastbin */
3754 mchunkptr nextchunk
; /* next contiguous chunk */
3755 INTERNAL_SIZE_T nextsize
; /* its size */
3756 int nextinuse
; /* true if nextchunk is used */
3757 INTERNAL_SIZE_T prevsize
; /* size of previous contiguous chunk */
3758 mchunkptr bck
; /* misc temp for linking */
3759 mchunkptr fwd
; /* misc temp for linking */
3761 const char *errstr
= NULL
;
3764 size
= chunksize(p
);
3766 /* Little security check which won't hurt performance: the
3767 allocator never wrapps around at the end of the address space.
3768 Therefore we can exclude some size values which might appear
3769 here by accident or by "design" from some intruder. */
3770 if (__builtin_expect ((uintptr_t) p
> (uintptr_t) -size
, 0)
3771 || __builtin_expect (misaligned_chunk (p
), 0))
3773 errstr
= "free(): invalid pointer";
3775 if (! have_lock
&& locked
)
3776 (void)mutex_unlock(&av
->mutex
);
3777 malloc_printerr (check_action
, errstr
, chunk2mem(p
));
3780 /* We know that each chunk is at least MINSIZE bytes in size or a
3781 multiple of MALLOC_ALIGNMENT. */
3782 if (__builtin_expect (size
< MINSIZE
|| !aligned_OK (size
), 0))
3784 errstr
= "free(): invalid size";
3788 check_inuse_chunk(av
, p
);
3791 If eligible, place chunk on a fastbin so it can be found
3792 and used quickly in malloc.
3795 if ((unsigned long)(size
) <= (unsigned long)(get_max_fast ())
3799 If TRIM_FASTBINS set, don't place chunks
3800 bordering top into fastbins
3802 && (chunk_at_offset(p
, size
) != av
->top
)
3806 if (__builtin_expect (chunk_at_offset (p
, size
)->size
<= 2 * SIZE_SZ
, 0)
3807 || __builtin_expect (chunksize (chunk_at_offset (p
, size
))
3808 >= av
->system_mem
, 0))
3810 /* We might not have a lock at this point and concurrent modifications
3811 of system_mem might have let to a false positive. Redo the test
3812 after getting the lock. */
3814 || ({ assert (locked
== 0);
3815 mutex_lock(&av
->mutex
);
3817 chunk_at_offset (p
, size
)->size
<= 2 * SIZE_SZ
3818 || chunksize (chunk_at_offset (p
, size
)) >= av
->system_mem
;
3821 errstr
= "free(): invalid next size (fast)";
3826 (void)mutex_unlock(&av
->mutex
);
3831 if (__builtin_expect (perturb_byte
, 0))
3832 free_perturb (chunk2mem(p
), size
- 2 * SIZE_SZ
);
3835 unsigned int idx
= fastbin_index(size
);
3836 fb
= &fastbin (av
, idx
);
3839 mchunkptr old
= *fb
;
3840 unsigned int old_idx
= ~0u;
3843 /* Another simple check: make sure the top of the bin is not the
3844 record we are going to add (i.e., double free). */
3845 if (__builtin_expect (old
== p
, 0))
3847 errstr
= "double free or corruption (fasttop)";
3851 old_idx
= fastbin_index(chunksize(old
));
3854 while ((old
= catomic_compare_and_exchange_val_rel (fb
, p
, fd
)) != fd
);
3856 if (fd
!= NULL
&& __builtin_expect (old_idx
!= idx
, 0))
3858 errstr
= "invalid fastbin entry (free)";
3864 Consolidate other non-mmapped chunks as they arrive.
3867 else if (!chunk_is_mmapped(p
)) {
3870 if(!mutex_trylock(&av
->mutex
))
3871 ++(av
->stat_lock_direct
);
3873 (void)mutex_lock(&av
->mutex
);
3874 ++(av
->stat_lock_wait
);
3877 (void)mutex_lock(&av
->mutex
);
3882 nextchunk
= chunk_at_offset(p
, size
);
3884 /* Lightweight tests: check whether the block is already the
3886 if (__builtin_expect (p
== av
->top
, 0))
3888 errstr
= "double free or corruption (top)";
3891 /* Or whether the next chunk is beyond the boundaries of the arena. */
3892 if (__builtin_expect (contiguous (av
)
3893 && (char *) nextchunk
3894 >= ((char *) av
->top
+ chunksize(av
->top
)), 0))
3896 errstr
= "double free or corruption (out)";
3899 /* Or whether the block is actually not marked used. */
3900 if (__builtin_expect (!prev_inuse(nextchunk
), 0))
3902 errstr
= "double free or corruption (!prev)";
3906 nextsize
= chunksize(nextchunk
);
3907 if (__builtin_expect (nextchunk
->size
<= 2 * SIZE_SZ
, 0)
3908 || __builtin_expect (nextsize
>= av
->system_mem
, 0))
3910 errstr
= "free(): invalid next size (normal)";
3914 if (__builtin_expect (perturb_byte
, 0))
3915 free_perturb (chunk2mem(p
), size
- 2 * SIZE_SZ
);
3917 /* consolidate backward */
3918 if (!prev_inuse(p
)) {
3919 prevsize
= p
->prev_size
;
3921 p
= chunk_at_offset(p
, -((long) prevsize
));
3922 unlink(p
, bck
, fwd
);
3925 if (nextchunk
!= av
->top
) {
3926 /* get and clear inuse bit */
3927 nextinuse
= inuse_bit_at_offset(nextchunk
, nextsize
);
3929 /* consolidate forward */
3931 unlink(nextchunk
, bck
, fwd
);
3934 clear_inuse_bit_at_offset(nextchunk
, 0);
3937 Place the chunk in unsorted chunk list. Chunks are
3938 not placed into regular bins until after they have
3939 been given one chance to be used in malloc.
3942 bck
= unsorted_chunks(av
);
3944 if (__builtin_expect (fwd
->bk
!= bck
, 0))
3946 errstr
= "free(): corrupted unsorted chunks";
3951 if (!in_smallbin_range(size
))
3953 p
->fd_nextsize
= NULL
;
3954 p
->bk_nextsize
= NULL
;
3959 set_head(p
, size
| PREV_INUSE
);
3962 check_free_chunk(av
, p
);
3966 If the chunk borders the current high end of memory,
3967 consolidate into top
3972 set_head(p
, size
| PREV_INUSE
);
3978 If freeing a large space, consolidate possibly-surrounding
3979 chunks. Then, if the total unused topmost memory exceeds trim
3980 threshold, ask malloc_trim to reduce top.
3982 Unless max_fast is 0, we don't know if there are fastbins
3983 bordering top, so we cannot tell for sure whether threshold
3984 has been reached unless fastbins are consolidated. But we
3985 don't want to consolidate on each free. As a compromise,
3986 consolidation is performed if FASTBIN_CONSOLIDATION_THRESHOLD
3990 if ((unsigned long)(size
) >= FASTBIN_CONSOLIDATION_THRESHOLD
) {
3991 if (have_fastchunks(av
))
3992 malloc_consolidate(av
);
3994 if (av
== &main_arena
) {
3995 #ifndef MORECORE_CANNOT_TRIM
3996 if ((unsigned long)(chunksize(av
->top
)) >=
3997 (unsigned long)(mp_
.trim_threshold
))
3998 systrim(mp_
.top_pad
, av
);
4001 /* Always try heap_trim(), even if the top chunk is not
4002 large, because the corresponding heap might go away. */
4003 heap_info
*heap
= heap_for_ptr(top(av
));
4005 assert(heap
->ar_ptr
== av
);
4006 heap_trim(heap
, mp_
.top_pad
);
4012 (void)mutex_unlock(&av
->mutex
);
4016 If the chunk was allocated via mmap, release via munmap().
4025 ------------------------- malloc_consolidate -------------------------
4027 malloc_consolidate is a specialized version of free() that tears
4028 down chunks held in fastbins. Free itself cannot be used for this
4029 purpose since, among other things, it might place chunks back onto
4030 fastbins. So, instead, we need to use a minor variant of the same
4033 Also, because this routine needs to be called the first time through
4034 malloc anyway, it turns out to be the perfect place to trigger
4035 initialization code.
4038 static void malloc_consolidate(mstate av
)
4040 mfastbinptr
* fb
; /* current fastbin being consolidated */
4041 mfastbinptr
* maxfb
; /* last fastbin (for loop control) */
4042 mchunkptr p
; /* current chunk being consolidated */
4043 mchunkptr nextp
; /* next chunk to consolidate */
4044 mchunkptr unsorted_bin
; /* bin header */
4045 mchunkptr first_unsorted
; /* chunk to link to */
4047 /* These have same use as in free() */
4048 mchunkptr nextchunk
;
4049 INTERNAL_SIZE_T size
;
4050 INTERNAL_SIZE_T nextsize
;
4051 INTERNAL_SIZE_T prevsize
;
4057 If max_fast is 0, we know that av hasn't
4058 yet been initialized, in which case do so below
4061 if (get_max_fast () != 0) {
4062 clear_fastchunks(av
);
4064 unsorted_bin
= unsorted_chunks(av
);
4067 Remove each chunk from fast bin and consolidate it, placing it
4068 then in unsorted bin. Among other reasons for doing this,
4069 placing in unsorted bin avoids needing to calculate actual bins
4070 until malloc is sure that chunks aren't immediately going to be
4074 maxfb
= &fastbin (av
, NFASTBINS
- 1);
4075 fb
= &fastbin (av
, 0);
4077 p
= atomic_exchange_acq (fb
, 0);
4080 check_inuse_chunk(av
, p
);
4083 /* Slightly streamlined version of consolidation code in free() */
4084 size
= p
->size
& ~(PREV_INUSE
|NON_MAIN_ARENA
);
4085 nextchunk
= chunk_at_offset(p
, size
);
4086 nextsize
= chunksize(nextchunk
);
4088 if (!prev_inuse(p
)) {
4089 prevsize
= p
->prev_size
;
4091 p
= chunk_at_offset(p
, -((long) prevsize
));
4092 unlink(p
, bck
, fwd
);
4095 if (nextchunk
!= av
->top
) {
4096 nextinuse
= inuse_bit_at_offset(nextchunk
, nextsize
);
4100 unlink(nextchunk
, bck
, fwd
);
4102 clear_inuse_bit_at_offset(nextchunk
, 0);
4104 first_unsorted
= unsorted_bin
->fd
;
4105 unsorted_bin
->fd
= p
;
4106 first_unsorted
->bk
= p
;
4108 if (!in_smallbin_range (size
)) {
4109 p
->fd_nextsize
= NULL
;
4110 p
->bk_nextsize
= NULL
;
4113 set_head(p
, size
| PREV_INUSE
);
4114 p
->bk
= unsorted_bin
;
4115 p
->fd
= first_unsorted
;
4121 set_head(p
, size
| PREV_INUSE
);
4125 } while ( (p
= nextp
) != 0);
4128 } while (fb
++ != maxfb
);
4131 malloc_init_state(av
);
4132 check_malloc_state(av
);
4137 ------------------------------ realloc ------------------------------
4141 _int_realloc(mstate av
, mchunkptr oldp
, INTERNAL_SIZE_T oldsize
,
4144 mchunkptr newp
; /* chunk to return */
4145 INTERNAL_SIZE_T newsize
; /* its size */
4146 void* newmem
; /* corresponding user mem */
4148 mchunkptr next
; /* next contiguous chunk after oldp */
4150 mchunkptr remainder
; /* extra space at end of newp */
4151 unsigned long remainder_size
; /* its size */
4153 mchunkptr bck
; /* misc temp for linking */
4154 mchunkptr fwd
; /* misc temp for linking */
4156 unsigned long copysize
; /* bytes to copy */
4157 unsigned int ncopies
; /* INTERNAL_SIZE_T words to copy */
4158 INTERNAL_SIZE_T
* s
; /* copy source */
4159 INTERNAL_SIZE_T
* d
; /* copy destination */
4161 const char *errstr
= NULL
;
4164 if (__builtin_expect (oldp
->size
<= 2 * SIZE_SZ
, 0)
4165 || __builtin_expect (oldsize
>= av
->system_mem
, 0))
4167 errstr
= "realloc(): invalid old size";
4169 malloc_printerr (check_action
, errstr
, chunk2mem(oldp
));
4173 check_inuse_chunk(av
, oldp
);
4175 /* All callers already filter out mmap'ed chunks. */
4176 assert (!chunk_is_mmapped(oldp
));
4178 next
= chunk_at_offset(oldp
, oldsize
);
4179 INTERNAL_SIZE_T nextsize
= chunksize(next
);
4180 if (__builtin_expect (next
->size
<= 2 * SIZE_SZ
, 0)
4181 || __builtin_expect (nextsize
>= av
->system_mem
, 0))
4183 errstr
= "realloc(): invalid next size";
4187 if ((unsigned long)(oldsize
) >= (unsigned long)(nb
)) {
4188 /* already big enough; split below */
4194 /* Try to expand forward into top */
4195 if (next
== av
->top
&&
4196 (unsigned long)(newsize
= oldsize
+ nextsize
) >=
4197 (unsigned long)(nb
+ MINSIZE
)) {
4198 set_head_size(oldp
, nb
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4199 av
->top
= chunk_at_offset(oldp
, nb
);
4200 set_head(av
->top
, (newsize
- nb
) | PREV_INUSE
);
4201 check_inuse_chunk(av
, oldp
);
4202 return chunk2mem(oldp
);
4205 /* Try to expand forward into next chunk; split off remainder below */
4206 else if (next
!= av
->top
&&
4208 (unsigned long)(newsize
= oldsize
+ nextsize
) >=
4209 (unsigned long)(nb
)) {
4211 unlink(next
, bck
, fwd
);
4214 /* allocate, copy, free */
4216 newmem
= _int_malloc(av
, nb
- MALLOC_ALIGN_MASK
);
4218 return 0; /* propagate failure */
4220 newp
= mem2chunk(newmem
);
4221 newsize
= chunksize(newp
);
4224 Avoid copy if newp is next chunk after oldp.
4232 Unroll copy of <= 36 bytes (72 if 8byte sizes)
4233 We know that contents have an odd number of
4234 INTERNAL_SIZE_T-sized words; minimally 3.
4237 copysize
= oldsize
- SIZE_SZ
;
4238 s
= (INTERNAL_SIZE_T
*)(chunk2mem(oldp
));
4239 d
= (INTERNAL_SIZE_T
*)(newmem
);
4240 ncopies
= copysize
/ sizeof(INTERNAL_SIZE_T
);
4241 assert(ncopies
>= 3);
4244 MALLOC_COPY(d
, s
, copysize
);
4264 _int_free(av
, oldp
, 1);
4265 check_inuse_chunk(av
, newp
);
4266 return chunk2mem(newp
);
4271 /* If possible, free extra space in old or extended chunk */
4273 assert((unsigned long)(newsize
) >= (unsigned long)(nb
));
4275 remainder_size
= newsize
- nb
;
4277 if (remainder_size
< MINSIZE
) { /* not enough extra to split off */
4278 set_head_size(newp
, newsize
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4279 set_inuse_bit_at_offset(newp
, newsize
);
4281 else { /* split remainder */
4282 remainder
= chunk_at_offset(newp
, nb
);
4283 set_head_size(newp
, nb
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4284 set_head(remainder
, remainder_size
| PREV_INUSE
|
4285 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4286 /* Mark remainder as inuse so free() won't complain */
4287 set_inuse_bit_at_offset(remainder
, remainder_size
);
4288 _int_free(av
, remainder
, 1);
4291 check_inuse_chunk(av
, newp
);
4292 return chunk2mem(newp
);
4296 ------------------------------ memalign ------------------------------
4300 _int_memalign(mstate av
, size_t alignment
, size_t bytes
)
4302 INTERNAL_SIZE_T nb
; /* padded request size */
4303 char* m
; /* memory returned by malloc call */
4304 mchunkptr p
; /* corresponding chunk */
4305 char* brk
; /* alignment point within p */
4306 mchunkptr newp
; /* chunk to return */
4307 INTERNAL_SIZE_T newsize
; /* its size */
4308 INTERNAL_SIZE_T leadsize
; /* leading space before alignment point */
4309 mchunkptr remainder
; /* spare room at end to split off */
4310 unsigned long remainder_size
; /* its size */
4311 INTERNAL_SIZE_T size
;
4313 /* If need less alignment than we give anyway, just relay to malloc */
4315 if (alignment
<= MALLOC_ALIGNMENT
) return _int_malloc(av
, bytes
);
4317 /* Otherwise, ensure that it is at least a minimum chunk size */
4319 if (alignment
< MINSIZE
) alignment
= MINSIZE
;
4321 /* Make sure alignment is power of 2 (in case MINSIZE is not). */
4322 if ((alignment
& (alignment
- 1)) != 0) {
4323 size_t a
= MALLOC_ALIGNMENT
* 2;
4324 while ((unsigned long)a
< (unsigned long)alignment
) a
<<= 1;
4328 checked_request2size(bytes
, nb
);
4331 Strategy: find a spot within that chunk that meets the alignment
4332 request, and then possibly free the leading and trailing space.
4336 /* Call malloc with worst case padding to hit alignment. */
4338 m
= (char*)(_int_malloc(av
, nb
+ alignment
+ MINSIZE
));
4340 if (m
== 0) return 0; /* propagate failure */
4344 if ((((unsigned long)(m
)) % alignment
) != 0) { /* misaligned */
4347 Find an aligned spot inside chunk. Since we need to give back
4348 leading space in a chunk of at least MINSIZE, if the first
4349 calculation places us at a spot with less than MINSIZE leader,
4350 we can move to the next aligned spot -- we've allocated enough
4351 total room so that this is always possible.
4354 brk
= (char*)mem2chunk(((unsigned long)(m
+ alignment
- 1)) &
4355 -((signed long) alignment
));
4356 if ((unsigned long)(brk
- (char*)(p
)) < MINSIZE
)
4359 newp
= (mchunkptr
)brk
;
4360 leadsize
= brk
- (char*)(p
);
4361 newsize
= chunksize(p
) - leadsize
;
4363 /* For mmapped chunks, just adjust offset */
4364 if (chunk_is_mmapped(p
)) {
4365 newp
->prev_size
= p
->prev_size
+ leadsize
;
4366 set_head(newp
, newsize
|IS_MMAPPED
);
4367 return chunk2mem(newp
);
4370 /* Otherwise, give back leader, use the rest */
4371 set_head(newp
, newsize
| PREV_INUSE
|
4372 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4373 set_inuse_bit_at_offset(newp
, newsize
);
4374 set_head_size(p
, leadsize
| (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4375 _int_free(av
, p
, 1);
4378 assert (newsize
>= nb
&&
4379 (((unsigned long)(chunk2mem(p
))) % alignment
) == 0);
4382 /* Also give back spare room at the end */
4383 if (!chunk_is_mmapped(p
)) {
4384 size
= chunksize(p
);
4385 if ((unsigned long)(size
) > (unsigned long)(nb
+ MINSIZE
)) {
4386 remainder_size
= size
- nb
;
4387 remainder
= chunk_at_offset(p
, nb
);
4388 set_head(remainder
, remainder_size
| PREV_INUSE
|
4389 (av
!= &main_arena
? NON_MAIN_ARENA
: 0));
4390 set_head_size(p
, nb
);
4391 _int_free(av
, remainder
, 1);
4395 check_inuse_chunk(av
, p
);
4396 return chunk2mem(p
);
4401 ------------------------------ valloc ------------------------------
4405 _int_valloc(mstate av
, size_t bytes
)
4407 /* Ensure initialization/consolidation */
4408 if (have_fastchunks(av
)) malloc_consolidate(av
);
4409 return _int_memalign(av
, GLRO(dl_pagesize
), bytes
);
4413 ------------------------------ pvalloc ------------------------------
4418 _int_pvalloc(mstate av
, size_t bytes
)
4422 /* Ensure initialization/consolidation */
4423 if (have_fastchunks(av
)) malloc_consolidate(av
);
4424 pagesz
= GLRO(dl_pagesize
);
4425 return _int_memalign(av
, pagesz
, (bytes
+ pagesz
- 1) & ~(pagesz
- 1));
4430 ------------------------------ malloc_trim ------------------------------
4433 static int mtrim(mstate av
, size_t pad
)
4435 /* Ensure initialization/consolidation */
4436 malloc_consolidate (av
);
4438 const size_t ps
= GLRO(dl_pagesize
);
4439 int psindex
= bin_index (ps
);
4440 const size_t psm1
= ps
- 1;
4443 for (int i
= 1; i
< NBINS
; ++i
)
4444 if (i
== 1 || i
>= psindex
)
4446 mbinptr bin
= bin_at (av
, i
);
4448 for (mchunkptr p
= last (bin
); p
!= bin
; p
= p
->bk
)
4450 INTERNAL_SIZE_T size
= chunksize (p
);
4452 if (size
> psm1
+ sizeof (struct malloc_chunk
))
4454 /* See whether the chunk contains at least one unused page. */
4455 char *paligned_mem
= (char *) (((uintptr_t) p
4456 + sizeof (struct malloc_chunk
)
4459 assert ((char *) chunk2mem (p
) + 4 * SIZE_SZ
<= paligned_mem
);
4460 assert ((char *) p
+ size
> paligned_mem
);
4462 /* This is the size we could potentially free. */
4463 size
-= paligned_mem
- (char *) p
;
4468 /* When debugging we simulate destroying the memory
4470 memset (paligned_mem
, 0x89, size
& ~psm1
);
4472 __madvise (paligned_mem
, size
& ~psm1
, MADV_DONTNEED
);
4480 #ifndef MORECORE_CANNOT_TRIM
4481 return result
| (av
== &main_arena
? systrim (pad
, av
) : 0);
4489 __malloc_trim(size_t s
)
4493 if(__malloc_initialized
< 0)
4496 mstate ar_ptr
= &main_arena
;
4499 (void) mutex_lock (&ar_ptr
->mutex
);
4500 result
|= mtrim (ar_ptr
, s
);
4501 (void) mutex_unlock (&ar_ptr
->mutex
);
4503 ar_ptr
= ar_ptr
->next
;
4505 while (ar_ptr
!= &main_arena
);
4512 ------------------------- malloc_usable_size -------------------------
4522 if (__builtin_expect(using_malloc_checking
== 1, 0))
4523 return malloc_check_get_size(p
);
4524 if (chunk_is_mmapped(p
))
4525 return chunksize(p
) - 2*SIZE_SZ
;
4527 return chunksize(p
) - SIZE_SZ
;
4534 __malloc_usable_size(void* m
)
4538 result
= musable(m
);
4543 ------------------------------ mallinfo ------------------------------
4544 Accumulate malloc statistics for arena AV into M.
4548 int_mallinfo(mstate av
, struct mallinfo
*m
)
4553 INTERNAL_SIZE_T avail
;
4554 INTERNAL_SIZE_T fastavail
;
4558 /* Ensure initialization */
4559 if (av
->top
== 0) malloc_consolidate(av
);
4561 check_malloc_state(av
);
4563 /* Account for top */
4564 avail
= chunksize(av
->top
);
4565 nblocks
= 1; /* top always exists */
4567 /* traverse fastbins */
4571 for (i
= 0; i
< NFASTBINS
; ++i
) {
4572 for (p
= fastbin (av
, i
); p
!= 0; p
= p
->fd
) {
4574 fastavail
+= chunksize(p
);
4580 /* traverse regular bins */
4581 for (i
= 1; i
< NBINS
; ++i
) {
4583 for (p
= last(b
); p
!= b
; p
= p
->bk
) {
4585 avail
+= chunksize(p
);
4589 m
->smblks
+= nfastblocks
;
4590 m
->ordblks
+= nblocks
;
4591 m
->fordblks
+= avail
;
4592 m
->uordblks
+= av
->system_mem
- avail
;
4593 m
->arena
+= av
->system_mem
;
4594 m
->fsmblks
+= fastavail
;
4595 if (av
== &main_arena
)
4597 m
->hblks
= mp_
.n_mmaps
;
4598 m
->hblkhd
= mp_
.mmapped_mem
;
4599 m
->usmblks
= mp_
.max_total_mem
;
4600 m
->keepcost
= chunksize(av
->top
);
4605 struct mallinfo
__libc_mallinfo()
4610 if(__malloc_initialized
< 0)
4613 memset(&m
, 0, sizeof (m
));
4614 ar_ptr
= &main_arena
;
4616 (void)mutex_lock(&ar_ptr
->mutex
);
4617 int_mallinfo(ar_ptr
, &m
);
4618 (void)mutex_unlock(&ar_ptr
->mutex
);
4620 ar_ptr
= ar_ptr
->next
;
4621 } while (ar_ptr
!= &main_arena
);
4627 ------------------------------ malloc_stats ------------------------------
4631 __malloc_stats (void)
4635 unsigned int in_use_b
= mp_
.mmapped_mem
, system_b
= in_use_b
;
4637 long stat_lock_direct
= 0, stat_lock_loop
= 0, stat_lock_wait
= 0;
4640 if(__malloc_initialized
< 0)
4642 _IO_flockfile (stderr
);
4643 int old_flags2
= ((_IO_FILE
*) stderr
)->_flags2
;
4644 ((_IO_FILE
*) stderr
)->_flags2
|= _IO_FLAGS2_NOTCANCEL
;
4645 for (i
=0, ar_ptr
= &main_arena
;; i
++) {
4648 memset(&mi
, 0, sizeof(mi
));
4649 (void)mutex_lock(&ar_ptr
->mutex
);
4650 int_mallinfo(ar_ptr
, &mi
);
4651 fprintf(stderr
, "Arena %d:\n", i
);
4652 fprintf(stderr
, "system bytes = %10u\n", (unsigned int)mi
.arena
);
4653 fprintf(stderr
, "in use bytes = %10u\n", (unsigned int)mi
.uordblks
);
4654 #if MALLOC_DEBUG > 1
4656 dump_heap(heap_for_ptr(top(ar_ptr
)));
4658 system_b
+= mi
.arena
;
4659 in_use_b
+= mi
.uordblks
;
4661 stat_lock_direct
+= ar_ptr
->stat_lock_direct
;
4662 stat_lock_loop
+= ar_ptr
->stat_lock_loop
;
4663 stat_lock_wait
+= ar_ptr
->stat_lock_wait
;
4665 (void)mutex_unlock(&ar_ptr
->mutex
);
4666 ar_ptr
= ar_ptr
->next
;
4667 if(ar_ptr
== &main_arena
) break;
4669 fprintf(stderr
, "Total (incl. mmap):\n");
4670 fprintf(stderr
, "system bytes = %10u\n", system_b
);
4671 fprintf(stderr
, "in use bytes = %10u\n", in_use_b
);
4672 fprintf(stderr
, "max mmap regions = %10u\n", (unsigned int)mp_
.max_n_mmaps
);
4673 fprintf(stderr
, "max mmap bytes = %10lu\n",
4674 (unsigned long)mp_
.max_mmapped_mem
);
4676 fprintf(stderr
, "heaps created = %10d\n", stat_n_heaps
);
4677 fprintf(stderr
, "locked directly = %10ld\n", stat_lock_direct
);
4678 fprintf(stderr
, "locked in loop = %10ld\n", stat_lock_loop
);
4679 fprintf(stderr
, "locked waiting = %10ld\n", stat_lock_wait
);
4680 fprintf(stderr
, "locked total = %10ld\n",
4681 stat_lock_direct
+ stat_lock_loop
+ stat_lock_wait
);
4683 ((_IO_FILE
*) stderr
)->_flags2
|= old_flags2
;
4684 _IO_funlockfile (stderr
);
4689 ------------------------------ mallopt ------------------------------
4692 int __libc_mallopt(int param_number
, int value
)
4694 mstate av
= &main_arena
;
4697 if(__malloc_initialized
< 0)
4699 (void)mutex_lock(&av
->mutex
);
4700 /* Ensure initialization/consolidation */
4701 malloc_consolidate(av
);
4703 LIBC_PROBE (memory_mallopt
, 2, param_number
, value
);
4705 switch(param_number
) {
4707 if (value
>= 0 && value
<= MAX_FAST_SIZE
)
4709 LIBC_PROBE (memory_mallopt_mxfast
, 2, value
, get_max_fast ());
4710 set_max_fast(value
);
4716 case M_TRIM_THRESHOLD
:
4717 LIBC_PROBE (memory_mallopt_trim_threshold
, 3, value
,
4718 mp_
.trim_threshold
, mp_
.no_dyn_threshold
);
4719 mp_
.trim_threshold
= value
;
4720 mp_
.no_dyn_threshold
= 1;
4724 LIBC_PROBE (memory_mallopt_top_pad
, 3, value
,
4725 mp_
.top_pad
, mp_
.no_dyn_threshold
);
4726 mp_
.top_pad
= value
;
4727 mp_
.no_dyn_threshold
= 1;
4730 case M_MMAP_THRESHOLD
:
4731 /* Forbid setting the threshold too high. */
4732 if((unsigned long)value
> HEAP_MAX_SIZE
/2)
4736 LIBC_PROBE (memory_mallopt_mmap_threshold
, 3, value
,
4737 mp_
.mmap_threshold
, mp_
.no_dyn_threshold
);
4738 mp_
.mmap_threshold
= value
;
4739 mp_
.no_dyn_threshold
= 1;
4744 LIBC_PROBE (memory_mallopt_mmap_max
, 3, value
,
4745 mp_
.n_mmaps_max
, mp_
.no_dyn_threshold
);
4746 mp_
.n_mmaps_max
= value
;
4747 mp_
.no_dyn_threshold
= 1;
4750 case M_CHECK_ACTION
:
4751 LIBC_PROBE (memory_mallopt_check_action
, 2, value
, check_action
);
4752 check_action
= value
;
4756 LIBC_PROBE (memory_mallopt_perturb
, 2, value
, perturb_byte
);
4757 perturb_byte
= value
;
4764 LIBC_PROBE (memory_mallopt_arena_test
, 2, value
, mp_
.arena_test
);
4765 mp_
.arena_test
= value
;
4772 LIBC_PROBE (memory_mallopt_arena_max
, 2, value
, mp_
.arena_max
);
4773 mp_
.arena_max
= value
;
4778 (void)mutex_unlock(&av
->mutex
);
4781 libc_hidden_def (__libc_mallopt
)
4785 -------------------- Alternative MORECORE functions --------------------
4790 General Requirements for MORECORE.
4792 The MORECORE function must have the following properties:
4794 If MORECORE_CONTIGUOUS is false:
4796 * MORECORE must allocate in multiples of pagesize. It will
4797 only be called with arguments that are multiples of pagesize.
4799 * MORECORE(0) must return an address that is at least
4800 MALLOC_ALIGNMENT aligned. (Page-aligning always suffices.)
4802 else (i.e. If MORECORE_CONTIGUOUS is true):
4804 * Consecutive calls to MORECORE with positive arguments
4805 return increasing addresses, indicating that space has been
4806 contiguously extended.
4808 * MORECORE need not allocate in multiples of pagesize.
4809 Calls to MORECORE need not have args of multiples of pagesize.
4811 * MORECORE need not page-align.
4815 * MORECORE may allocate more memory than requested. (Or even less,
4816 but this will generally result in a malloc failure.)
4818 * MORECORE must not allocate memory when given argument zero, but
4819 instead return one past the end address of memory from previous
4820 nonzero call. This malloc does NOT call MORECORE(0)
4821 until at least one call with positive arguments is made, so
4822 the initial value returned is not important.
4824 * Even though consecutive calls to MORECORE need not return contiguous
4825 addresses, it must be OK for malloc'ed chunks to span multiple
4826 regions in those cases where they do happen to be contiguous.
4828 * MORECORE need not handle negative arguments -- it may instead
4829 just return MORECORE_FAILURE when given negative arguments.
4830 Negative arguments are always multiples of pagesize. MORECORE
4831 must not misinterpret negative args as large positive unsigned
4832 args. You can suppress all such calls from even occurring by defining
4833 MORECORE_CANNOT_TRIM,
4835 There is some variation across systems about the type of the
4836 argument to sbrk/MORECORE. If size_t is unsigned, then it cannot
4837 actually be size_t, because sbrk supports negative args, so it is
4838 normally the signed type of the same width as size_t (sometimes
4839 declared as "intptr_t", and sometimes "ptrdiff_t"). It doesn't much
4840 matter though. Internally, we use "long" as arguments, which should
4841 work across all reasonable possibilities.
4843 Additionally, if MORECORE ever returns failure for a positive
4844 request, then mmap is used as a noncontiguous system allocator. This
4845 is a useful backup strategy for systems with holes in address spaces
4846 -- in this case sbrk cannot contiguously expand the heap, but mmap
4847 may be able to map noncontiguous space.
4849 If you'd like mmap to ALWAYS be used, you can define MORECORE to be
4850 a function that always returns MORECORE_FAILURE.
4852 If you are using this malloc with something other than sbrk (or its
4853 emulation) to supply memory regions, you probably want to set
4854 MORECORE_CONTIGUOUS as false. As an example, here is a custom
4855 allocator kindly contributed for pre-OSX macOS. It uses virtually
4856 but not necessarily physically contiguous non-paged memory (locked
4857 in, present and won't get swapped out). You can use it by
4858 uncommenting this section, adding some #includes, and setting up the
4859 appropriate defines above:
4861 #define MORECORE osMoreCore
4862 #define MORECORE_CONTIGUOUS 0
4864 There is also a shutdown routine that should somehow be called for
4865 cleanup upon program exit.
4867 #define MAX_POOL_ENTRIES 100
4868 #define MINIMUM_MORECORE_SIZE (64 * 1024)
4869 static int next_os_pool;
4870 void *our_os_pools[MAX_POOL_ENTRIES];
4872 void *osMoreCore(int size)
4875 static void *sbrk_top = 0;
4879 if (size < MINIMUM_MORECORE_SIZE)
4880 size = MINIMUM_MORECORE_SIZE;
4881 if (CurrentExecutionLevel() == kTaskLevel)
4882 ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
4885 return (void *) MORECORE_FAILURE;
4887 // save ptrs so they can be freed during cleanup
4888 our_os_pools[next_os_pool] = ptr;
4890 ptr = (void *) ((((unsigned long) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
4891 sbrk_top = (char *) ptr + size;
4896 // we don't currently support shrink behavior
4897 return (void *) MORECORE_FAILURE;
4905 // cleanup any allocated memory pools
4906 // called as last thing before shutting down driver
4908 void osCleanupMem(void)
4912 for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
4915 PoolDeallocate(*ptr);
4925 extern char **__libc_argv attribute_hidden
;
4928 malloc_printerr(int action
, const char *str
, void *ptr
)
4930 if ((action
& 5) == 5)
4931 __libc_message (action
& 2, "%s\n", str
);
4932 else if (action
& 1)
4934 char buf
[2 * sizeof (uintptr_t) + 1];
4936 buf
[sizeof (buf
) - 1] = '\0';
4937 char *cp
= _itoa_word ((uintptr_t) ptr
, &buf
[sizeof (buf
) - 1], 16, 0);
4941 __libc_message (action
& 2, "*** Error in `%s': %s: 0x%s ***\n",
4942 __libc_argv
[0] ?: "<unknown>", str
, cp
);
4944 else if (action
& 2)
4948 #include <sys/param.h>
4950 /* We need a wrapper function for one of the additions of POSIX. */
4952 __posix_memalign (void **memptr
, size_t alignment
, size_t size
)
4956 /* Test whether the SIZE argument is valid. It must be a power of
4957 two multiple of sizeof (void *). */
4958 if (alignment
% sizeof (void *) != 0
4959 || !powerof2 (alignment
/ sizeof (void *)) != 0
4963 /* Call the hook here, so that caller is posix_memalign's caller
4964 and not posix_memalign itself. */
4965 void *(*hook
) (size_t, size_t, const void *) =
4966 force_reg (__memalign_hook
);
4967 if (__builtin_expect (hook
!= NULL
, 0))
4968 mem
= (*hook
)(alignment
, size
, RETURN_ADDRESS (0));
4970 mem
= __libc_memalign (alignment
, size
);
4979 weak_alias (__posix_memalign
, posix_memalign
)
4983 malloc_info (int options
, FILE *fp
)
4985 /* For now, at least. */
4990 size_t total_nblocks
= 0;
4991 size_t total_nfastblocks
= 0;
4992 size_t total_avail
= 0;
4993 size_t total_fastavail
= 0;
4994 size_t total_system
= 0;
4995 size_t total_max_system
= 0;
4996 size_t total_aspace
= 0;
4997 size_t total_aspace_mprotect
= 0;
4999 void mi_arena (mstate ar_ptr
)
5001 fprintf (fp
, "<heap nr=\"%d\">\n<sizes>\n", n
++);
5004 size_t nfastblocks
= 0;
5006 size_t fastavail
= 0;
5013 } sizes
[NFASTBINS
+ NBINS
- 1];
5014 #define nsizes (sizeof (sizes) / sizeof (sizes[0]))
5016 mutex_lock (&ar_ptr
->mutex
);
5018 for (size_t i
= 0; i
< NFASTBINS
; ++i
)
5020 mchunkptr p
= fastbin (ar_ptr
, i
);
5023 size_t nthissize
= 0;
5024 size_t thissize
= chunksize (p
);
5032 fastavail
+= nthissize
* thissize
;
5033 nfastblocks
+= nthissize
;
5034 sizes
[i
].from
= thissize
- (MALLOC_ALIGNMENT
- 1);
5035 sizes
[i
].to
= thissize
;
5036 sizes
[i
].count
= nthissize
;
5039 sizes
[i
].from
= sizes
[i
].to
= sizes
[i
].count
= 0;
5041 sizes
[i
].total
= sizes
[i
].count
* sizes
[i
].to
;
5044 mbinptr bin
= bin_at (ar_ptr
, 1);
5045 struct malloc_chunk
*r
= bin
->fd
;
5050 ++sizes
[NFASTBINS
].count
;
5051 sizes
[NFASTBINS
].total
+= r
->size
;
5052 sizes
[NFASTBINS
].from
= MIN (sizes
[NFASTBINS
].from
, r
->size
);
5053 sizes
[NFASTBINS
].to
= MAX (sizes
[NFASTBINS
].to
, r
->size
);
5056 nblocks
+= sizes
[NFASTBINS
].count
;
5057 avail
+= sizes
[NFASTBINS
].total
;
5060 for (size_t i
= 2; i
< NBINS
; ++i
)
5062 bin
= bin_at (ar_ptr
, i
);
5064 sizes
[NFASTBINS
- 1 + i
].from
= ~((size_t) 0);
5065 sizes
[NFASTBINS
- 1 + i
].to
= sizes
[NFASTBINS
- 1 + i
].total
5066 = sizes
[NFASTBINS
- 1 + i
].count
= 0;
5071 ++sizes
[NFASTBINS
- 1 + i
].count
;
5072 sizes
[NFASTBINS
- 1 + i
].total
+= r
->size
;
5073 sizes
[NFASTBINS
- 1 + i
].from
5074 = MIN (sizes
[NFASTBINS
- 1 + i
].from
, r
->size
);
5075 sizes
[NFASTBINS
- 1 + i
].to
= MAX (sizes
[NFASTBINS
- 1 + i
].to
,
5081 if (sizes
[NFASTBINS
- 1 + i
].count
== 0)
5082 sizes
[NFASTBINS
- 1 + i
].from
= 0;
5083 nblocks
+= sizes
[NFASTBINS
- 1 + i
].count
;
5084 avail
+= sizes
[NFASTBINS
- 1 + i
].total
;
5087 mutex_unlock (&ar_ptr
->mutex
);
5089 total_nfastblocks
+= nfastblocks
;
5090 total_fastavail
+= fastavail
;
5092 total_nblocks
+= nblocks
;
5093 total_avail
+= avail
;
5095 for (size_t i
= 0; i
< nsizes
; ++i
)
5096 if (sizes
[i
].count
!= 0 && i
!= NFASTBINS
)
5098 <size from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5099 sizes
[i
].from
, sizes
[i
].to
, sizes
[i
].total
, sizes
[i
].count
);
5101 if (sizes
[NFASTBINS
].count
!= 0)
5103 <unsorted from=\"%zu\" to=\"%zu\" total=\"%zu\" count=\"%zu\"/>\n",
5104 sizes
[NFASTBINS
].from
, sizes
[NFASTBINS
].to
,
5105 sizes
[NFASTBINS
].total
, sizes
[NFASTBINS
].count
);
5107 total_system
+= ar_ptr
->system_mem
;
5108 total_max_system
+= ar_ptr
->max_system_mem
;
5111 "</sizes>\n<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5112 "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5113 "<system type=\"current\" size=\"%zu\"/>\n"
5114 "<system type=\"max\" size=\"%zu\"/>\n",
5115 nfastblocks
, fastavail
, nblocks
, avail
,
5116 ar_ptr
->system_mem
, ar_ptr
->max_system_mem
);
5118 if (ar_ptr
!= &main_arena
)
5120 heap_info
*heap
= heap_for_ptr(top(ar_ptr
));
5122 "<aspace type=\"total\" size=\"%zu\"/>\n"
5123 "<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5124 heap
->size
, heap
->mprotect_size
);
5125 total_aspace
+= heap
->size
;
5126 total_aspace_mprotect
+= heap
->mprotect_size
;
5131 "<aspace type=\"total\" size=\"%zu\"/>\n"
5132 "<aspace type=\"mprotect\" size=\"%zu\"/>\n",
5133 ar_ptr
->system_mem
, ar_ptr
->system_mem
);
5134 total_aspace
+= ar_ptr
->system_mem
;
5135 total_aspace_mprotect
+= ar_ptr
->system_mem
;
5138 fputs ("</heap>\n", fp
);
5141 if(__malloc_initialized
< 0)
5144 fputs ("<malloc version=\"1\">\n", fp
);
5146 /* Iterate over all arenas currently in use. */
5147 mstate ar_ptr
= &main_arena
;
5151 ar_ptr
= ar_ptr
->next
;
5153 while (ar_ptr
!= &main_arena
);
5156 "<total type=\"fast\" count=\"%zu\" size=\"%zu\"/>\n"
5157 "<total type=\"rest\" count=\"%zu\" size=\"%zu\"/>\n"
5158 "<system type=\"current\" size=\"%zu\"/>\n"
5159 "<system type=\"max\" size=\"%zu\"/>\n"
5160 "<aspace type=\"total\" size=\"%zu\"/>\n"
5161 "<aspace type=\"mprotect\" size=\"%zu\"/>\n"
5163 total_nfastblocks
, total_fastavail
, total_nblocks
, total_avail
,
5164 total_system
, total_max_system
,
5165 total_aspace
, total_aspace_mprotect
);
5171 strong_alias (__libc_calloc
, __calloc
) weak_alias (__libc_calloc
, calloc
)
5172 strong_alias (__libc_free
, __cfree
) weak_alias (__libc_free
, cfree
)
5173 strong_alias (__libc_free
, __free
) strong_alias (__libc_free
, free
)
5174 strong_alias (__libc_malloc
, __malloc
) strong_alias (__libc_malloc
, malloc
)
5175 strong_alias (__libc_memalign
, __memalign
)
5176 weak_alias (__libc_memalign
, memalign
)
5177 strong_alias (__libc_realloc
, __realloc
) strong_alias (__libc_realloc
, realloc
)
5178 strong_alias (__libc_valloc
, __valloc
) weak_alias (__libc_valloc
, valloc
)
5179 strong_alias (__libc_pvalloc
, __pvalloc
) weak_alias (__libc_pvalloc
, pvalloc
)
5180 strong_alias (__libc_mallinfo
, __mallinfo
)
5181 weak_alias (__libc_mallinfo
, mallinfo
)
5182 strong_alias (__libc_mallopt
, __mallopt
) weak_alias (__libc_mallopt
, mallopt
)
5184 weak_alias (__malloc_stats
, malloc_stats
)
5185 weak_alias (__malloc_usable_size
, malloc_usable_size
)
5186 weak_alias (__malloc_trim
, malloc_trim
)
5187 weak_alias (__malloc_get_state
, malloc_get_state
)
5188 weak_alias (__malloc_set_state
, malloc_set_state
)
5191 /* ------------------------------------------------------------
5194 [see ftp://g.oswego.edu/pub/misc/malloc.c for the history of dlmalloc]