4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2016 Joyent, Inc.
26 #include <sys/types.h>
27 #include <sys/t_lock.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/tuneable.h>
31 #include <sys/systm.h>
36 #include <sys/cmn_err.h>
37 #include <sys/debug.h>
38 #include <sys/dumphdr.h>
39 #include <sys/bootconf.h>
41 #include <vm/seg_kmem.h>
44 #include <vm/vm_dep.h>
45 #include <vm/faultcode.h>
46 #include <sys/promif.h>
47 #include <vm/seg_kp.h>
48 #include <sys/bitmap.h>
49 #include <sys/mem_cage.h>
52 #include <sys/ivintr.h>
53 #include <sys/panic.h>
57 * seg_kmem is the primary kernel memory segment driver. It
58 * maps the kernel heap [kernelheap, ekernelheap), module text,
59 * and all memory which was allocated before the VM was initialized
62 * Pages which belong to seg_kmem are hashed into &kvp vnode at
63 * an offset equal to (uoff_t)virt_addr, and have p_lckcnt >= 1.
64 * They must never be paged out since segkmem_fault() is a no-op to
65 * prevent recursive faults.
67 * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
68 * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86
69 * supports relocation the #ifdef kludges can be removed.
71 * seg_kmem pages may be subject to relocation by page_relocate(),
72 * provided that the HAT supports it; if this is so, segkmem_reloc
73 * will be set to a nonzero value. All boot time allocated memory as
74 * well as static memory is considered off limits to relocation.
75 * Pages are "relocatable" if p_state does not have P_NORELOC set, so
76 * we request P_NORELOC pages for memory that isn't safe to relocate.
78 * The kernel heap is logically divided up into four pieces:
80 * heap32_arena is for allocations that require 32-bit absolute
81 * virtual addresses (e.g. code that uses 32-bit pointers/offsets).
83 * heap_core is for allocations that require 2GB *relative*
84 * offsets; in other words all memory from heap_core is within
85 * 2GB of all other memory from the same arena. This is a requirement
86 * of the addressing modes of some processors in supervisor code.
88 * heap_arena is the general heap arena.
90 * static_arena is the static memory arena. Allocations from it
91 * are not subject to relocation so it is safe to use the memory
92 * physical address as well as the virtual address (e.g. the VA to
93 * PA translations are static). Caches may import from static_arena;
94 * all other static memory allocations should use static_alloc_arena.
96 * On some platforms which have limited virtual address space, seg_kmem
97 * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
98 * segkp_bitmap is non-NULL, and each bit represents a page of virtual
99 * address space which is actually seg_kp mapped.
102 extern ulong_t
*segkp_bitmap
; /* Is set if segkp is from the kernel heap */
104 char *kernelheap
; /* start of primary kernel heap */
105 char *ekernelheap
; /* end of primary kernel heap */
106 struct seg kvseg
; /* primary kernel heap segment */
107 struct seg kvseg_core
; /* "core" kernel heap segment */
108 struct seg kzioseg
; /* Segment for zio mappings */
109 vmem_t
*heap_arena
; /* primary kernel heap arena */
110 vmem_t
*heap_core_arena
; /* core kernel heap arena */
111 char *heap_core_base
; /* start of core kernel heap arena */
112 char *heap_lp_base
; /* start of kernel large page heap arena */
113 char *heap_lp_end
; /* end of kernel large page heap arena */
114 vmem_t
*hat_memload_arena
; /* HAT translation data */
115 struct seg kvseg32
; /* 32-bit kernel heap segment */
116 vmem_t
*heap32_arena
; /* 32-bit kernel heap arena */
117 vmem_t
*heaptext_arena
; /* heaptext arena */
118 struct as kas
; /* kernel address space */
119 int segkmem_reloc
; /* enable/disable relocatable segkmem pages */
120 vmem_t
*static_arena
; /* arena for caches to import static memory */
121 vmem_t
*static_alloc_arena
; /* arena for allocating static memory */
122 vmem_t
*zio_arena
= NULL
; /* arena for allocating zio memory */
123 vmem_t
*zio_alloc_arena
= NULL
; /* arena for allocating zio memory */
126 * seg_kmem driver can map part of the kernel heap with large pages.
127 * Currently this functionality is implemented for sparc platforms only.
129 * The large page size "segkmem_lpsize" for kernel heap is selected in the
130 * platform specific code. It can also be modified via /etc/system file.
131 * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
132 * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
133 * match segkmem_lpsize.
135 * At boot time we carve from kernel heap arena a range of virtual addresses
136 * that will be used for large page mappings. This range [heap_lp_base,
137 * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
138 * create "kmem_lp_arena" that caches memory already backed up by large
139 * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
142 size_t segkmem_lpsize
;
143 static uint_t segkmem_lpshift
= PAGESHIFT
;
144 int segkmem_lpszc
= 0;
146 size_t segkmem_kmemlp_quantum
= 0x400000; /* 4MB */
147 size_t segkmem_heaplp_quantum
;
148 vmem_t
*heap_lp_arena
;
149 static vmem_t
*kmem_lp_arena
;
150 static vmem_t
*segkmem_ppa_arena
;
151 static segkmem_lpcb_t segkmem_lpcb
;
154 * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
155 * consumed by the large page heap. By default this parameter is set to 1/8 of
156 * physmem but can be adjusted through /etc/system either directly or
157 * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
158 * we allow for large page heap.
160 size_t segkmem_kmemlp_max
;
161 static uint_t segkmem_kmemlp_pcnt
;
164 * Getting large pages for kernel heap could be problematic due to
165 * physical memory fragmentation. That's why we allow to preallocate
166 * "segkmem_kmemlp_min" bytes at boot time.
168 static size_t segkmem_kmemlp_min
;
171 * Throttling is used to avoid expensive tries to allocate large pages
172 * for kernel heap when a lot of succesive attempts to do so fail.
174 static ulong_t segkmem_lpthrottle_max
= 0x400000;
175 static ulong_t segkmem_lpthrottle_start
= 0x40;
176 static ulong_t segkmem_use_lpthrottle
= 1;
179 * Freed pages accumulate on a garbage list until segkmem is ready,
180 * at which point we call segkmem_gc() to free it all.
182 typedef struct segkmem_gc_list
{
183 struct segkmem_gc_list
*gc_next
;
188 static segkmem_gc_list_t
*segkmem_gc_list
;
191 * Allocations from the hat_memload arena add VM_MEMLOAD to their
192 * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
193 * to take steps to prevent infinite recursion. HAT allocations also
194 * must be non-relocatable to prevent recursive page faults.
197 hat_memload_alloc(vmem_t
*vmp
, size_t size
, int flags
)
199 flags
|= (VM_MEMLOAD
| VM_NORELOC
);
200 return (segkmem_alloc(vmp
, size
, flags
));
204 * Allocations from static_arena arena (or any other arena that uses
205 * segkmem_alloc_permanent()) require non-relocatable (permanently
206 * wired) memory pages, since these pages are referenced by physical
207 * as well as virtual address.
210 segkmem_alloc_permanent(vmem_t
*vmp
, size_t size
, int flags
)
212 return (segkmem_alloc(vmp
, size
, flags
| VM_NORELOC
));
216 * Initialize kernel heap boundaries.
229 vmem_t
*heaptext_parent
;
230 size_t heap_lp_size
= 0;
232 size_t kmem64_sz
= kmem64_aligned_end
- kmem64_base
;
235 kernelheap
= heap_start
;
236 ekernelheap
= heap_end
;
239 heap_lp_size
= (((uintptr_t)heap_end
- (uintptr_t)heap_start
) / 4);
241 * Bias heap_lp start address by kmem64_sz to reduce collisions
242 * in 4M kernel TSB between kmem64 area and heap_lp
244 kmem64_sz
= P2ROUNDUP(kmem64_sz
, MMU_PAGESIZE256M
);
245 if (kmem64_sz
<= heap_lp_size
/ 2)
246 heap_lp_size
-= kmem64_sz
;
247 heap_lp_base
= ekernelheap
- heap_lp_size
;
248 heap_lp_end
= heap_lp_base
+ heap_lp_size
;
252 * If this platform has a 'core' heap area, then the space for
253 * overflow module text should be carved out of the end of that
254 * heap. Otherwise, it gets carved out of the general purpose
257 core_size
= (uintptr_t)core_end
- (uintptr_t)core_start
;
259 ASSERT(core_size
>= HEAPTEXT_SIZE
);
260 textbase
= (uintptr_t)core_end
- HEAPTEXT_SIZE
;
261 core_size
-= HEAPTEXT_SIZE
;
265 ekernelheap
-= HEAPTEXT_SIZE
;
266 textbase
= (uintptr_t)ekernelheap
;
270 heap_size
= (uintptr_t)ekernelheap
- (uintptr_t)kernelheap
;
271 heap_arena
= vmem_init("heap", kernelheap
, heap_size
, PAGESIZE
,
272 segkmem_alloc
, segkmem_free
);
275 heap_core_arena
= vmem_create("heap_core", core_start
,
276 core_size
, PAGESIZE
, NULL
, NULL
, NULL
, 0, VM_SLEEP
);
277 heap_core_base
= core_start
;
279 heap_core_arena
= heap_arena
;
280 heap_core_base
= kernelheap
;
284 * reserve space for the large page heap. If large pages for kernel
285 * heap is enabled large page heap arean will be created later in the
286 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
287 * range will be returned back to the heap_arena.
290 (void) vmem_xalloc(heap_arena
, heap_lp_size
, PAGESIZE
, 0, 0,
291 heap_lp_base
, heap_lp_end
,
292 VM_NOSLEEP
| VM_BESTFIT
| VM_PANIC
);
296 * Remove the already-spoken-for memory range [kernelheap, first_avail).
298 (void) vmem_xalloc(heap_arena
, first_avail
- kernelheap
, PAGESIZE
,
299 0, 0, kernelheap
, first_avail
, VM_NOSLEEP
| VM_BESTFIT
| VM_PANIC
);
302 heap32_arena
= vmem_create("heap32", (void *)SYSBASE32
,
303 SYSLIMIT32
- SYSBASE32
- HEAPTEXT_SIZE
, PAGESIZE
, NULL
,
304 NULL
, NULL
, 0, VM_SLEEP
);
306 * Prom claims the physical and virtual resources used by panicbuf
307 * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
308 * reserved interrupt vector data structures from 32-bit heap.
310 (void) vmem_xalloc(heap32_arena
, PANICBUFSIZE
, PAGESIZE
, 0, 0,
311 panicbuf
, panicbuf
+ PANICBUFSIZE
,
312 VM_NOSLEEP
| VM_BESTFIT
| VM_PANIC
);
314 (void) vmem_xalloc(heap32_arena
, IVSIZE
, PAGESIZE
, 0, 0,
315 intr_vec_table
, (caddr_t
)intr_vec_table
+ IVSIZE
,
316 VM_NOSLEEP
| VM_BESTFIT
| VM_PANIC
);
318 textbase
= SYSLIMIT32
- HEAPTEXT_SIZE
;
319 heaptext_parent
= NULL
;
321 heap32_arena
= heap_core_arena
;
322 heaptext_parent
= heap_core_arena
;
325 heaptext_arena
= vmem_create("heaptext", (void *)textbase
,
326 HEAPTEXT_SIZE
, PAGESIZE
, NULL
, NULL
, heaptext_parent
, 0, VM_SLEEP
);
329 * Create a set of arenas for memory with static translations
330 * (e.g. VA -> PA translations cannot change). Since using
331 * kernel pages by physical address implies it isn't safe to
332 * walk across page boundaries, the static_arena quantum must
333 * be PAGESIZE. Any kmem caches that require static memory
334 * should source from static_arena, while direct allocations
335 * should only use static_alloc_arena.
337 static_arena
= vmem_create("static", NULL
, 0, PAGESIZE
,
338 segkmem_alloc_permanent
, segkmem_free
, heap_arena
, 0, VM_SLEEP
);
339 static_alloc_arena
= vmem_create("static_alloc", NULL
, 0,
340 sizeof (uint64_t), vmem_alloc
, vmem_free
, static_arena
,
344 * Create an arena for translation data (ptes, hmes, or hblks).
345 * We need an arena for this because hat_memload() is essential
346 * to vmem_populate() (see comments in kernel/os/vmem.c).
348 * Note: any kmem cache that allocates from hat_memload_arena
349 * must be created as a KMC_NOHASH cache (i.e. no external slab
350 * and bufctl structures to allocate) so that slab creation doesn't
351 * require anything more than a single vmem_alloc().
353 hat_memload_arena
= vmem_create("hat_memload", NULL
, 0, PAGESIZE
,
354 hat_memload_alloc
, segkmem_free
, heap_arena
, 0,
355 VM_SLEEP
| VMC_POPULATOR
| VMC_DUMPSAFE
);
359 boot_mapin(caddr_t addr
, size_t size
)
365 if (page_resv(btop(size
), KM_NOSLEEP
) == 0)
366 panic("boot_mapin: page_resv failed");
368 for (eaddr
= addr
+ size
; addr
< eaddr
; addr
+= PAGESIZE
) {
369 pfnum
= va_to_pfn(addr
);
370 if (pfnum
== PFN_INVALID
)
372 if ((pp
= page_numtopp_nolock(pfnum
)) == NULL
)
373 panic("boot_mapin(): No pp for pfnum = %lx", pfnum
);
376 * must break up any large pages that may have constituent
377 * pages being utilized for BOP_ALLOC()'s before calling
378 * page_numtopp().The locking code (ie. page_reclaim())
382 page_boot_demote(pp
);
384 pp
= page_numtopp(pfnum
, SE_EXCL
);
385 if (pp
== NULL
|| PP_ISFREE(pp
))
386 panic("boot_alloc: pp is NULL or free");
389 * If the cage is on but doesn't yet contain this page,
390 * mark it as non-relocatable.
392 if (kcage_on
&& !PP_ISNORELOC(pp
)) {
394 PLCNT_XFER_NORELOC(pp
);
397 (void) page_hashin(pp
, &kvp
, (uoff_t
)(uintptr_t)addr
, false);
408 * Get pages from boot and hash them into the kernel's vp.
409 * Used after page structs have been allocated, but before segkmem is ready.
412 boot_alloc(void *inaddr
, size_t size
, uint_t align
)
414 caddr_t addr
= inaddr
;
417 prom_panic("boot_alloc: attempt to allocate memory after "
420 size
= ptob(btopr(size
));
422 if (bop_alloc_chunk(addr
, size
, align
) != (caddr_t
)addr
)
423 panic("boot_alloc: bop_alloc_chunk failed");
425 if (BOP_ALLOC(bootops
, addr
, size
, align
) != addr
)
426 panic("boot_alloc: BOP_ALLOC failed");
428 boot_mapin((caddr_t
)addr
, size
);
435 panic("segkmem_badop");
438 #define SEGKMEM_BADOP(t) (t(*)())segkmem_badop
442 segkmem_fault(struct hat
*hat
, struct seg
*seg
, caddr_t addr
, size_t size
,
443 enum fault_type type
, enum seg_rw rw
)
448 struct vnode
*vp
= seg
->s_data
;
450 ASSERT(RW_READ_HELD(&seg
->s_as
->a_lock
));
452 if (seg
->s_as
!= &kas
|| size
> seg
->s_size
||
453 addr
< seg
->s_base
|| addr
+ size
> seg
->s_base
+ seg
->s_size
)
454 panic("segkmem_fault: bad args");
457 * If it is one of segkp pages, call segkp_fault.
459 if (segkp_bitmap
&& seg
== &kvseg
&&
460 BT_TEST(segkp_bitmap
, btop((uintptr_t)(addr
- seg
->s_base
))))
461 return (segop_fault(hat
, segkp
, addr
, size
, type
, rw
));
463 if (rw
!= S_READ
&& rw
!= S_WRITE
&& rw
!= S_OTHER
)
464 return (FC_NOSUPPORT
);
466 npages
= btopr(size
);
469 case F_SOFTLOCK
: /* lock down already-loaded translations */
470 for (pg
= 0; pg
< npages
; pg
++) {
471 pp
= page_lookup(vp
, (uoff_t
)(uintptr_t)addr
,
475 * Hmm, no page. Does a kernel mapping
478 if (!hat_probe(kas
.a_hat
, addr
)) {
481 pp
= page_find(vp
, (uoff_t
)
493 hat_reserve(seg
->s_as
, addr
, size
);
497 pp
= page_find(vp
, (uoff_t
)(uintptr_t)addr
);
504 return (FC_NOSUPPORT
);
510 segkmem_setprot(struct seg
*seg
, caddr_t addr
, size_t size
, uint_t prot
)
512 ASSERT(RW_LOCK_HELD(&seg
->s_as
->a_lock
));
514 if (seg
->s_as
!= &kas
|| size
> seg
->s_size
||
515 addr
< seg
->s_base
|| addr
+ size
> seg
->s_base
+ seg
->s_size
)
516 panic("segkmem_setprot: bad args");
519 * If it is one of segkp pages, call segkp.
521 if (segkp_bitmap
&& seg
== &kvseg
&&
522 BT_TEST(segkp_bitmap
, btop((uintptr_t)(addr
- seg
->s_base
))))
523 return (segop_setprot(segkp
, addr
, size
, prot
));
526 hat_unload(kas
.a_hat
, addr
, size
, HAT_UNLOAD
);
528 hat_chgprot(kas
.a_hat
, addr
, size
, prot
);
533 * This is a dummy segkmem function overloaded to call segkp
534 * when segkp is under the heap.
538 segkmem_checkprot(struct seg
*seg
, caddr_t addr
, size_t size
, uint_t prot
)
540 ASSERT(RW_LOCK_HELD(&seg
->s_as
->a_lock
));
542 if (seg
->s_as
!= &kas
)
546 * If it is one of segkp pages, call into segkp.
548 if (segkp_bitmap
&& seg
== &kvseg
&&
549 BT_TEST(segkp_bitmap
, btop((uintptr_t)(addr
- seg
->s_base
))))
550 return (segop_checkprot(segkp
, addr
, size
, prot
));
557 * This is a dummy segkmem function overloaded to call segkp
558 * when segkp is under the heap.
562 segkmem_kluster(struct seg
*seg
, caddr_t addr
, ssize_t delta
)
564 ASSERT(RW_LOCK_HELD(&seg
->s_as
->a_lock
));
566 if (seg
->s_as
!= &kas
)
570 * If it is one of segkp pages, call into segkp.
572 if (segkp_bitmap
&& seg
== &kvseg
&&
573 BT_TEST(segkp_bitmap
, btop((uintptr_t)(addr
- seg
->s_base
))))
574 return (segop_kluster(segkp
, addr
, delta
));
581 segkmem_xdump_range(void *arg
, void *start
, size_t size
)
584 caddr_t addr
= start
;
585 caddr_t addr_end
= addr
+ size
;
587 while (addr
< addr_end
) {
588 pfn_t pfn
= hat_getpfnum(kas
.a_hat
, addr
);
589 if (pfn
!= PFN_INVALID
&& pfn
<= physmax
&& pf_is_memory(pfn
))
590 dump_addpage(as
, addr
, pfn
);
592 dump_timeleft
= dump_timeout
;
597 segkmem_dump_range(void *arg
, void *start
, size_t size
)
599 caddr_t addr
= start
;
600 caddr_t addr_end
= addr
+ size
;
603 * If we are about to start dumping the range of addresses we
604 * carved out of the kernel heap for the large page heap walk
605 * heap_lp_arena to find what segments are actually populated
607 if (SEGKMEM_USE_LARGEPAGES
&&
608 addr
== heap_lp_base
&& addr_end
== heap_lp_end
&&
609 vmem_size(heap_lp_arena
, VMEM_ALLOC
) < size
) {
610 vmem_walk(heap_lp_arena
, VMEM_ALLOC
| VMEM_REENTRANT
,
611 segkmem_xdump_range
, arg
);
613 segkmem_xdump_range(arg
, start
, size
);
618 segkmem_dump(struct seg
*seg
)
621 * The kernel's heap_arena (represented by kvseg) is a very large
622 * VA space, most of which is typically unused. To speed up dumping
623 * we use vmem_walk() to quickly find the pieces of heap_arena that
624 * are actually in use. We do the same for heap32_arena and
627 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
628 * may ultimately need to allocate memory. Reentrant walks are
629 * necessarily imperfect snapshots. The kernel heap continues
630 * to change during a live crash dump, for example. For a normal
631 * crash dump, however, we know that there won't be any other threads
632 * messing with the heap. Therefore, at worst, we may fail to dump
633 * the pages that get allocated by the act of dumping; but we will
634 * always dump every page that was allocated when the walk began.
636 * The other segkmem segments are dense (fully populated), so there's
637 * no need to use this technique when dumping them.
639 * Note: when adding special dump handling for any new sparsely-
640 * populated segments, be sure to add similar handling to the ::kgrep
644 vmem_walk(heap_arena
, VMEM_ALLOC
| VMEM_REENTRANT
,
645 segkmem_dump_range
, seg
->s_as
);
647 vmem_walk(heaptext_arena
, VMEM_ALLOC
| VMEM_REENTRANT
,
648 segkmem_dump_range
, seg
->s_as
);
650 } else if (seg
== &kvseg_core
) {
651 vmem_walk(heap_core_arena
, VMEM_ALLOC
| VMEM_REENTRANT
,
652 segkmem_dump_range
, seg
->s_as
);
653 } else if (seg
== &kvseg32
) {
654 vmem_walk(heap32_arena
, VMEM_ALLOC
| VMEM_REENTRANT
,
655 segkmem_dump_range
, seg
->s_as
);
656 vmem_walk(heaptext_arena
, VMEM_ALLOC
| VMEM_REENTRANT
,
657 segkmem_dump_range
, seg
->s_as
);
658 } else if (seg
== &kzioseg
) {
660 * We don't want to dump pages attached to kzioseg since they
661 * contain file data from ZFS. If this page's segment is
662 * kzioseg return instead of writing it to the dump device.
666 segkmem_dump_range(seg
->s_as
, seg
->s_base
, seg
->s_size
);
671 * lock/unlock kmem pages over a given range [addr, addr+len).
672 * Returns a shadow list of pages in ppp. If there are holes
673 * in the range (e.g. some of the kernel mappings do not have
674 * underlying page_ts) returns ENOTSUP so that as_pagelock()
675 * will handle the range via as_fault(F_SOFTLOCK).
679 segkmem_pagelock(struct seg
*seg
, caddr_t addr
, size_t len
,
680 page_t
***ppp
, enum lock_type type
, enum seg_rw rw
)
682 page_t
**pplist
, *pp
;
686 struct vnode
*vp
= seg
->s_data
;
691 * If it is one of segkp pages, call into segkp.
693 if (segkp_bitmap
&& seg
== &kvseg
&&
694 BT_TEST(segkp_bitmap
, btop((uintptr_t)(addr
- seg
->s_base
))))
695 return (segop_pagelock(segkp
, addr
, len
, ppp
, type
, rw
));
698 nb
= sizeof (page_t
*) * npages
;
700 if (type
== L_PAGEUNLOCK
) {
702 ASSERT(pplist
!= NULL
);
704 for (pg
= 0; pg
< npages
; pg
++) {
708 kmem_free(pplist
, nb
);
712 ASSERT(type
== L_PAGELOCK
);
714 pplist
= kmem_alloc(nb
, KM_NOSLEEP
);
715 if (pplist
== NULL
) {
717 return (ENOTSUP
); /* take the slow path */
720 for (pg
= 0; pg
< npages
; pg
++) {
721 pp
= page_lookup(vp
, (uoff_t
)(uintptr_t)addr
, SE_SHARED
);
724 page_unlock(pplist
[pg
]);
725 kmem_free(pplist
, nb
);
738 * This is a dummy segkmem function overloaded to call segkp
739 * when segkp is under the heap.
743 segkmem_getmemid(struct seg
*seg
, caddr_t addr
, memid_t
*memidp
)
745 ASSERT(RW_LOCK_HELD(&seg
->s_as
->a_lock
));
747 if (seg
->s_as
!= &kas
)
751 * If it is one of segkp pages, call into segkp.
753 if (segkp_bitmap
&& seg
== &kvseg
&&
754 BT_TEST(segkp_bitmap
, btop((uintptr_t)(addr
- seg
->s_base
))))
755 return (segop_getmemid(segkp
, addr
, memidp
));
763 segkmem_capable(struct seg
*seg
, segcapability_t capability
)
765 if (capability
== S_CAPABILITY_NOMINFLT
)
770 const struct seg_ops segkmem_ops
= {
771 .dup
= SEGKMEM_BADOP(int),
772 .unmap
= SEGKMEM_BADOP(int),
773 .free
= SEGKMEM_BADOP(void),
774 .fault
= segkmem_fault
,
775 .faulta
= SEGKMEM_BADOP(faultcode_t
),
776 .setprot
= segkmem_setprot
,
777 .checkprot
= segkmem_checkprot
,
778 .kluster
= segkmem_kluster
,
779 .sync
= SEGKMEM_BADOP(int),
780 .incore
= SEGKMEM_BADOP(size_t),
781 .lockop
= SEGKMEM_BADOP(int),
782 .getprot
= SEGKMEM_BADOP(int),
783 .getoffset
= SEGKMEM_BADOP(uoff_t
),
784 .gettype
= SEGKMEM_BADOP(int),
785 .getvp
= SEGKMEM_BADOP(int),
786 .advise
= SEGKMEM_BADOP(int),
787 .dump
= segkmem_dump
,
788 .pagelock
= segkmem_pagelock
,
789 .setpagesize
= SEGKMEM_BADOP(int),
790 .getmemid
= segkmem_getmemid
,
791 .capable
= segkmem_capable
,
795 segkmem_zio_create(struct seg
*seg
)
797 ASSERT(seg
->s_as
== &kas
&& RW_WRITE_HELD(&kas
.a_lock
));
798 seg
->s_ops
= &segkmem_ops
;
800 kas
.a_size
+= seg
->s_size
;
805 segkmem_create(struct seg
*seg
)
807 ASSERT(seg
->s_as
== &kas
&& RW_WRITE_HELD(&kas
.a_lock
));
808 seg
->s_ops
= &segkmem_ops
;
810 kas
.a_size
+= seg
->s_size
;
816 segkmem_page_create(void *addr
, size_t size
, int vmflag
, void *arg
)
820 struct vnode
*vp
= arg
;
828 if (segkmem_reloc
== 0 || (vmflag
& VM_NORELOC
))
829 pgflags
|= PG_NORELOC
;
830 if ((vmflag
& VM_NOSLEEP
) == 0)
832 if (vmflag
& VM_PANIC
)
834 if (vmflag
& VM_PUSHPAGE
)
835 pgflags
|= PG_PUSHPAGE
;
836 if (vmflag
& VM_NORMALPRI
) {
837 ASSERT(vmflag
& VM_NOSLEEP
);
838 pgflags
|= PG_NORMALPRI
;
841 return (page_create_va(vp
, (uoff_t
)(uintptr_t)addr
, size
,
842 pgflags
, &kseg
, addr
));
846 * Allocate pages to back the virtual address range [addr, addr + size).
847 * If addr is NULL, allocate the virtual address space as well.
850 segkmem_xalloc(vmem_t
*vmp
, void *inaddr
, size_t size
, int vmflag
, uint_t attr
,
851 page_t
*(*page_create_func
)(void *, size_t, int, void *), void *pcarg
)
854 caddr_t addr
= inaddr
;
855 pgcnt_t npages
= btopr(size
);
858 if (inaddr
== NULL
&& (addr
= vmem_alloc(vmp
, size
, vmflag
)) == NULL
)
861 ASSERT(((uintptr_t)addr
& PAGEOFFSET
) == 0);
863 if (page_resv(npages
, vmflag
& VM_KMFLAGS
) == 0) {
865 vmem_free(vmp
, addr
, size
);
869 ppl
= page_create_func(addr
, size
, vmflag
, pcarg
);
872 vmem_free(vmp
, addr
, size
);
878 * Under certain conditions, we need to let the HAT layer know
879 * that it cannot safely allocate memory. Allocations from
880 * the hat_memload vmem arena always need this, to prevent
881 * infinite recursion.
883 * In addition, the x86 hat cannot safely do memory
884 * allocations while in vmem_populate(), because there
885 * is no simple bound on its usage.
887 if (vmflag
& VM_MEMLOAD
)
888 allocflag
= HAT_NO_KALLOC
;
890 else if (vmem_is_populator())
891 allocflag
= HAT_NO_KALLOC
;
896 while (ppl
!= NULL
) {
899 ASSERT(page_iolock_assert(pp
));
900 ASSERT(PAGE_EXCL(pp
));
902 hat_memload(kas
.a_hat
, (caddr_t
)(uintptr_t)pp
->p_offset
, pp
,
903 (PROT_ALL
& ~PROT_USER
) | HAT_NOSYNC
| attr
,
904 HAT_LOAD_LOCK
| allocflag
);
909 if (vmflag
& SEGKMEM_SHARELOCKED
)
920 segkmem_alloc_vn(vmem_t
*vmp
, size_t size
, int vmflag
, struct vnode
*vp
)
923 segkmem_gc_list_t
*gcp
, **prev_gcpp
;
927 if (kvseg
.s_base
== NULL
) {
929 if (bootops
->bsys_alloc
== NULL
)
930 halt("Memory allocation between bop_alloc() and "
935 * There's not a lot of memory to go around during boot,
936 * so recycle it if we can.
938 for (prev_gcpp
= &segkmem_gc_list
; (gcp
= *prev_gcpp
) != NULL
;
939 prev_gcpp
= &gcp
->gc_next
) {
940 if (gcp
->gc_arena
== vmp
&& gcp
->gc_size
== size
) {
941 *prev_gcpp
= gcp
->gc_next
;
946 addr
= vmem_alloc(vmp
, size
, vmflag
| VM_PANIC
);
947 if (boot_alloc(addr
, size
, BO_NO_ALIGN
) != addr
)
948 panic("segkmem_alloc: boot_alloc failed");
951 return (segkmem_xalloc(vmp
, NULL
, size
, vmflag
, 0,
952 segkmem_page_create
, vp
));
956 segkmem_alloc(vmem_t
*vmp
, size_t size
, int vmflag
)
958 return (segkmem_alloc_vn(vmp
, size
, vmflag
, &kvp
));
962 segkmem_zio_alloc(vmem_t
*vmp
, size_t size
, int vmflag
)
964 return (segkmem_alloc_vn(vmp
, size
, vmflag
, &zvp
));
968 * Any changes to this routine must also be carried over to
969 * devmap_free_pages() in the seg_dev driver. This is because
970 * we currently don't have a special kernel segment for non-paged
971 * kernel memory that is exported by drivers to user space.
974 segkmem_free_vn(vmem_t
*vmp
, void *inaddr
, size_t size
, struct vnode
*vp
,
975 void (*func
)(page_t
*))
978 caddr_t addr
= inaddr
;
980 pgcnt_t npages
= btopr(size
);
982 ASSERT(((uintptr_t)addr
& PAGEOFFSET
) == 0);
985 if (kvseg
.s_base
== NULL
) {
986 segkmem_gc_list_t
*gc
= inaddr
;
989 gc
->gc_next
= segkmem_gc_list
;
990 segkmem_gc_list
= gc
;
994 hat_unload(kas
.a_hat
, addr
, size
, HAT_UNLOAD_UNLOCK
);
996 for (eaddr
= addr
+ size
; addr
< eaddr
; addr
+= PAGESIZE
) {
998 pp
= page_find(vp
, (uoff_t
)(uintptr_t)addr
);
1000 panic("segkmem_free: page not found");
1001 if (!page_tryupgrade(pp
)) {
1003 * Some other thread has a sharelock. Wait for
1004 * it to drop the lock so we can free this page.
1007 pp
= page_lookup(vp
, (uoff_t
)(uintptr_t)addr
,
1011 pp
= page_lookup(vp
, (uoff_t
)(uintptr_t)addr
, SE_EXCL
);
1014 panic("segkmem_free: page not found");
1015 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1020 page_destroy(pp
, 0);
1023 page_unresv(npages
);
1026 vmem_free(vmp
, inaddr
, size
);
1031 segkmem_xfree(vmem_t
*vmp
, void *inaddr
, size_t size
, void (*func
)(page_t
*))
1033 segkmem_free_vn(vmp
, inaddr
, size
, &kvp
, func
);
1037 segkmem_free(vmem_t
*vmp
, void *inaddr
, size_t size
)
1039 segkmem_free_vn(vmp
, inaddr
, size
, &kvp
, NULL
);
1043 segkmem_zio_free(vmem_t
*vmp
, void *inaddr
, size_t size
)
1045 segkmem_free_vn(vmp
, inaddr
, size
, &zvp
, NULL
);
1051 ASSERT(kvseg
.s_base
!= NULL
);
1052 while (segkmem_gc_list
!= NULL
) {
1053 segkmem_gc_list_t
*gc
= segkmem_gc_list
;
1054 segkmem_gc_list
= gc
->gc_next
;
1055 segkmem_free(gc
->gc_arena
, gc
, gc
->gc_size
);
1060 * Legacy entry points from here to end of file.
1063 segkmem_mapin(struct seg
*seg
, void *addr
, size_t size
, uint_t vprot
,
1064 pfn_t pfn
, uint_t flags
)
1066 hat_unload(seg
->s_as
->a_hat
, addr
, size
, HAT_UNLOAD_UNLOCK
);
1067 hat_devload(seg
->s_as
->a_hat
, addr
, size
, pfn
, vprot
,
1068 flags
| HAT_LOAD_LOCK
);
1072 segkmem_mapout(struct seg
*seg
, void *addr
, size_t size
)
1074 hat_unload(seg
->s_as
->a_hat
, addr
, size
, HAT_UNLOAD_UNLOCK
);
1078 kmem_getpages(pgcnt_t npages
, int kmflag
)
1080 return (kmem_alloc(ptob(npages
), kmflag
));
1084 kmem_freepages(void *addr
, pgcnt_t npages
)
1086 kmem_free(addr
, ptob(npages
));
1090 * segkmem_page_create_large() allocates a large page to be used for the kmem
1091 * caches. If kpr is enabled we ask for a relocatable page unless requested
1092 * otherwise. If kpr is disabled we have to ask for a non-reloc page
1095 segkmem_page_create_large(void *addr
, size_t size
, int vmflag
, void *arg
)
1101 if (segkmem_reloc
== 0 || (vmflag
& VM_NORELOC
))
1102 pgflags
|= PG_NORELOC
;
1103 if (!(vmflag
& VM_NOSLEEP
))
1105 if (vmflag
& VM_PUSHPAGE
)
1106 pgflags
|= PG_PUSHPAGE
;
1107 if (vmflag
& VM_NORMALPRI
)
1108 pgflags
|= PG_NORMALPRI
;
1110 return (page_create_va_large(&kvp
, (uoff_t
)(uintptr_t)addr
, size
,
1111 pgflags
, &kvseg
, addr
, arg
));
1115 * Allocate a large page to back the virtual address range
1116 * [addr, addr + size). If addr is NULL, allocate the virtual address
1120 segkmem_xalloc_lp(vmem_t
*vmp
, void *inaddr
, size_t size
, int vmflag
,
1121 uint_t attr
, page_t
*(*page_create_func
)(void *, size_t, int, void *),
1124 caddr_t addr
= inaddr
, pa
;
1125 size_t lpsize
= segkmem_lpsize
;
1126 pgcnt_t npages
= btopr(size
);
1127 pgcnt_t nbpages
= btop(lpsize
);
1128 pgcnt_t nlpages
= size
>> segkmem_lpshift
;
1129 size_t ppasize
= nbpages
* sizeof (page_t
*);
1130 page_t
*pp
, *rootpp
, **ppa
, *pplist
= NULL
;
1133 vmflag
|= VM_NOSLEEP
;
1135 if (page_resv(npages
, vmflag
& VM_KMFLAGS
) == 0) {
1140 * allocate an array we need for hat_memload_array.
1141 * we use a separate arena to avoid recursion.
1142 * we will not need this array when hat_memload_array learns pp++
1144 if ((ppa
= vmem_alloc(segkmem_ppa_arena
, ppasize
, vmflag
)) == NULL
) {
1145 goto fail_array_alloc
;
1148 if (inaddr
== NULL
&& (addr
= vmem_alloc(vmp
, size
, vmflag
)) == NULL
)
1149 goto fail_vmem_alloc
;
1151 ASSERT(((uintptr_t)addr
& (lpsize
- 1)) == 0);
1153 /* create all the pages */
1154 for (pa
= addr
, i
= 0; i
< nlpages
; i
++, pa
+= lpsize
) {
1155 if ((pp
= page_create_func(pa
, lpsize
, vmflag
, pcarg
)) == NULL
)
1156 goto fail_page_create
;
1157 page_list_concat(&pplist
, &pp
);
1160 /* at this point we have all the resource to complete the request */
1161 while ((rootpp
= pplist
) != NULL
) {
1162 for (i
= 0; i
< nbpages
; i
++) {
1163 ASSERT(pplist
!= NULL
);
1165 page_sub(&pplist
, pp
);
1166 ASSERT(page_iolock_assert(pp
));
1171 * Load the locked entry. It's OK to preload the entry into the
1172 * TSB since we now support large mappings in the kernel TSB.
1174 hat_memload_array(kas
.a_hat
,
1175 (caddr_t
)(uintptr_t)rootpp
->p_offset
, lpsize
,
1176 ppa
, (PROT_ALL
& ~PROT_USER
) | HAT_NOSYNC
| attr
,
1179 for (--i
; i
>= 0; --i
) {
1180 ppa
[i
]->p_lckcnt
= 1;
1181 page_unlock(ppa
[i
]);
1185 vmem_free(segkmem_ppa_arena
, ppa
, ppasize
);
1189 while ((rootpp
= pplist
) != NULL
) {
1190 for (i
= 0, pp
= pplist
; i
< nbpages
; i
++, pp
= pplist
) {
1192 page_sub(&pplist
, pp
);
1193 ASSERT(page_iolock_assert(pp
));
1196 page_destroy_pages(rootpp
);
1200 vmem_free(vmp
, addr
, size
);
1203 vmem_free(segkmem_ppa_arena
, ppa
, ppasize
);
1206 page_unresv(npages
);
1212 segkmem_free_one_lp(caddr_t addr
, size_t size
)
1214 page_t
*pp
, *rootpp
= NULL
;
1215 pgcnt_t pgs_left
= btopr(size
);
1217 ASSERT(size
== segkmem_lpsize
);
1219 hat_unload(kas
.a_hat
, addr
, size
, HAT_UNLOAD_UNLOCK
);
1221 for (; pgs_left
> 0; addr
+= PAGESIZE
, pgs_left
--) {
1222 pp
= page_lookup(&kvp
, (uoff_t
)(uintptr_t)addr
, SE_EXCL
);
1224 panic("segkmem_free_one_lp: page not found");
1225 ASSERT(PAGE_EXCL(pp
));
1230 ASSERT(rootpp
!= NULL
);
1231 page_destroy_pages(rootpp
);
1233 /* page_unresv() is done by the caller */
1237 * This function is called to import new spans into the vmem arenas like
1238 * kmem_default_arena and kmem_oversize_arena. It first tries to import
1239 * spans from large page arena - kmem_lp_arena. In order to do this it might
1240 * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1241 * it was not able to satisfy the upgraded request it then calls regular
1242 * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1246 segkmem_alloc_lp(vmem_t
*vmp
, size_t *sizep
, size_t align
, int vmflag
)
1249 kthread_t
*t
= curthread
;
1250 segkmem_lpcb_t
*lpcb
= &segkmem_lpcb
;
1252 ASSERT(sizep
!= NULL
);
1256 if (lpcb
->lp_uselp
&& !(t
->t_flag
& T_PANIC
) &&
1257 !(vmflag
& SEGKMEM_SHARELOCKED
)) {
1259 size_t kmemlp_qnt
= segkmem_kmemlp_quantum
;
1260 size_t asize
= P2ROUNDUP(size
, kmemlp_qnt
);
1262 ulong_t
*lpthrtp
= &lpcb
->lp_throttle
;
1263 ulong_t lpthrt
= *lpthrtp
;
1267 ASSERT(kmem_lp_arena
!= NULL
);
1268 ASSERT(asize
>= size
);
1271 /* try to update the throttle value */
1272 lpthrt
= atomic_inc_ulong_nv(lpthrtp
);
1273 if (lpthrt
>= segkmem_lpthrottle_max
) {
1274 lpthrt
= atomic_cas_ulong(lpthrtp
, lpthrt
,
1275 segkmem_lpthrottle_max
/ 4);
1279 * when we get above throttle start do an exponential
1280 * backoff at trying large pages and reaping
1282 if (lpthrt
> segkmem_lpthrottle_start
&&
1284 lpcb
->allocs_throttled
++;
1288 return (segkmem_alloc(vmp
, size
, vmflag
));
1292 if (!(vmflag
& VM_NOSLEEP
) &&
1293 segkmem_heaplp_quantum
>= (8 * kmemlp_qnt
) &&
1294 vmem_size(kmem_lp_arena
, VMEM_FREE
) <= kmemlp_qnt
&&
1295 asize
< (segkmem_heaplp_quantum
- kmemlp_qnt
)) {
1298 * we are low on free memory in kmem_lp_arena
1299 * we let only one guy to allocate heap_lp
1300 * quantum size chunk that everybody is going to
1303 mutex_enter(&lpcb
->lp_lock
);
1305 if (lpcb
->lp_wait
) {
1307 /* we are not the first one - wait */
1308 cv_wait(&lpcb
->lp_cv
, &lpcb
->lp_lock
);
1309 if (vmem_size(kmem_lp_arena
, VMEM_FREE
) <
1313 } else if (vmem_size(kmem_lp_arena
, VMEM_FREE
) <=
1317 * we are the first one, make sure we import
1320 if (asize
== kmemlp_qnt
)
1321 asize
+= kmemlp_qnt
;
1326 mutex_exit(&lpcb
->lp_lock
);
1330 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1331 * large pages are not available. In that case this allocation
1332 * attempt will fail and we will retry allocation with small
1333 * pages. We also do not want to panic if this allocation fails
1334 * because we are going to retry.
1337 addr
= vmem_alloc(kmem_lp_arena
, asize
,
1338 (vmflag
| VM_ABORT
) & ~VM_PANIC
);
1341 mutex_enter(&lpcb
->lp_lock
);
1342 ASSERT(lpcb
->lp_wait
!= 0);
1344 cv_broadcast(&lpcb
->lp_cv
);
1345 mutex_exit(&lpcb
->lp_lock
);
1355 if (vmflag
& VM_NOSLEEP
)
1356 lpcb
->nosleep_allocs_failed
++;
1358 lpcb
->sleep_allocs_failed
++;
1359 lpcb
->alloc_bytes_failed
+= size
;
1361 /* if large page throttling is not started yet do it */
1362 if (segkmem_use_lpthrottle
&& lpthrt
== 0) {
1363 lpthrt
= atomic_cas_ulong(lpthrtp
, lpthrt
, 1);
1366 return (segkmem_alloc(vmp
, size
, vmflag
));
1370 segkmem_free_lp(vmem_t
*vmp
, void *inaddr
, size_t size
)
1372 if (kmem_lp_arena
== NULL
|| !IS_KMEM_VA_LARGEPAGE((caddr_t
)inaddr
)) {
1373 segkmem_free(vmp
, inaddr
, size
);
1375 vmem_free(kmem_lp_arena
, inaddr
, size
);
1380 * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1381 * into kmem_lp arena. In the process it maps the imported segment with
1385 segkmem_alloc_lpi(vmem_t
*vmp
, size_t size
, int vmflag
)
1387 segkmem_lpcb_t
*lpcb
= &segkmem_lpcb
;
1391 ASSERT(vmp
== heap_lp_arena
);
1393 /* do not allow large page heap grow beyound limits */
1394 if (vmem_size(vmp
, VMEM_ALLOC
) >= segkmem_kmemlp_max
) {
1395 lpcb
->allocs_limited
++;
1399 addr
= segkmem_xalloc_lp(vmp
, NULL
, size
, vmflag
, 0,
1400 segkmem_page_create_large
, NULL
);
1405 * segkmem_free_lpi() returns virtual memory back into large page heap arena
1406 * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1407 * large pages used to map it.
1410 segkmem_free_lpi(vmem_t
*vmp
, void *inaddr
, size_t size
)
1412 pgcnt_t nlpages
= size
>> segkmem_lpshift
;
1413 size_t lpsize
= segkmem_lpsize
;
1414 caddr_t addr
= inaddr
;
1415 pgcnt_t npages
= btopr(size
);
1418 ASSERT(vmp
== heap_lp_arena
);
1419 ASSERT(IS_KMEM_VA_LARGEPAGE(addr
));
1420 ASSERT(((uintptr_t)inaddr
& (lpsize
- 1)) == 0);
1422 for (i
= 0; i
< nlpages
; i
++) {
1423 segkmem_free_one_lp(addr
, lpsize
);
1427 page_unresv(npages
);
1429 vmem_free(vmp
, inaddr
, size
);
1433 * This function is called at system boot time by kmem_init right after
1434 * /etc/system file has been read. It checks based on hardware configuration
1435 * and /etc/system settings if system is going to use large pages. The
1436 * initialiazation necessary to actually start using large pages
1437 * happens later in the process after segkmem_heap_lp_init() is called.
1442 int use_large_pages
= 0;
1446 size_t memtotal
= physmem
* PAGESIZE
;
1448 if (heap_lp_base
== NULL
) {
1449 segkmem_lpsize
= PAGESIZE
;
1453 /* get a platform dependent value of large page size for kernel heap */
1454 segkmem_lpsize
= get_segkmem_lpsize(segkmem_lpsize
);
1456 if (segkmem_lpsize
<= PAGESIZE
) {
1458 * put virtual space reserved for the large page kernel
1459 * back to the regular heap
1461 vmem_xfree(heap_arena
, heap_lp_base
,
1462 heap_lp_end
- heap_lp_base
);
1463 heap_lp_base
= NULL
;
1465 segkmem_lpsize
= PAGESIZE
;
1469 /* set heap_lp quantum if necessary */
1470 if (segkmem_heaplp_quantum
== 0 || !ISP2(segkmem_heaplp_quantum
) ||
1471 P2PHASE(segkmem_heaplp_quantum
, segkmem_lpsize
)) {
1472 segkmem_heaplp_quantum
= segkmem_lpsize
;
1475 /* set kmem_lp quantum if necessary */
1476 if (segkmem_kmemlp_quantum
== 0 || !ISP2(segkmem_kmemlp_quantum
) ||
1477 segkmem_kmemlp_quantum
> segkmem_heaplp_quantum
) {
1478 segkmem_kmemlp_quantum
= segkmem_heaplp_quantum
;
1481 /* set total amount of memory allowed for large page kernel heap */
1482 if (segkmem_kmemlp_max
== 0) {
1483 if (segkmem_kmemlp_pcnt
== 0 || segkmem_kmemlp_pcnt
> 100)
1484 segkmem_kmemlp_pcnt
= 12;
1485 segkmem_kmemlp_max
= (memtotal
* segkmem_kmemlp_pcnt
) / 100;
1487 segkmem_kmemlp_max
= P2ROUNDUP(segkmem_kmemlp_max
,
1488 segkmem_heaplp_quantum
);
1490 /* fix lp kmem preallocation request if necesssary */
1491 if (segkmem_kmemlp_min
) {
1492 segkmem_kmemlp_min
= P2ROUNDUP(segkmem_kmemlp_min
,
1493 segkmem_heaplp_quantum
);
1494 if (segkmem_kmemlp_min
> segkmem_kmemlp_max
)
1495 segkmem_kmemlp_min
= segkmem_kmemlp_max
;
1498 use_large_pages
= 1;
1499 segkmem_lpszc
= page_szc(segkmem_lpsize
);
1500 segkmem_lpshift
= page_get_shift(segkmem_lpszc
);
1503 return (use_large_pages
);
1507 segkmem_zio_init(void *zio_mem_base
, size_t zio_mem_size
)
1509 ASSERT(zio_mem_base
!= NULL
);
1510 ASSERT(zio_mem_size
!= 0);
1513 * To reduce VA space fragmentation, we set up quantum caches for the
1514 * smaller sizes; we chose 32k because that translates to 128k VA
1515 * slabs, which matches nicely with the common 128k zio_data bufs.
1517 zio_arena
= vmem_create("zfs_file_data", zio_mem_base
, zio_mem_size
,
1518 PAGESIZE
, NULL
, NULL
, NULL
, 32 * 1024, VM_SLEEP
);
1520 zio_alloc_arena
= vmem_create("zfs_file_data_buf", NULL
, 0, PAGESIZE
,
1521 segkmem_zio_alloc
, segkmem_zio_free
, zio_arena
, 0, VM_SLEEP
);
1523 ASSERT(zio_arena
!= NULL
);
1524 ASSERT(zio_alloc_arena
!= NULL
);
1531 segkmem_alloc_ppa(vmem_t
*vmp
, size_t size
, int vmflag
)
1533 size_t ppaquantum
= btopr(segkmem_lpsize
) * sizeof (page_t
*);
1536 if (ppaquantum
<= PAGESIZE
)
1537 return (segkmem_alloc(vmp
, size
, vmflag
));
1539 ASSERT((size
& (ppaquantum
- 1)) == 0);
1541 addr
= vmem_xalloc(vmp
, size
, ppaquantum
, 0, 0, NULL
, NULL
, vmflag
);
1542 if (addr
!= NULL
&& segkmem_xalloc(vmp
, addr
, size
, vmflag
, 0,
1543 segkmem_page_create
, NULL
) == NULL
) {
1544 vmem_xfree(vmp
, addr
, size
);
1552 segkmem_free_ppa(vmem_t
*vmp
, void *addr
, size_t size
)
1554 size_t ppaquantum
= btopr(segkmem_lpsize
) * sizeof (page_t
*);
1556 ASSERT(addr
!= NULL
);
1558 if (ppaquantum
<= PAGESIZE
) {
1559 segkmem_free(vmp
, addr
, size
);
1561 segkmem_free(NULL
, addr
, size
);
1562 vmem_xfree(vmp
, addr
, size
);
1567 segkmem_heap_lp_init()
1569 segkmem_lpcb_t
*lpcb
= &segkmem_lpcb
;
1570 size_t heap_lp_size
= heap_lp_end
- heap_lp_base
;
1571 size_t lpsize
= segkmem_lpsize
;
1575 if (segkmem_lpsize
<= PAGESIZE
) {
1576 ASSERT(heap_lp_base
== NULL
);
1577 ASSERT(heap_lp_end
== NULL
);
1581 ASSERT(segkmem_heaplp_quantum
>= lpsize
);
1582 ASSERT((segkmem_heaplp_quantum
& (lpsize
- 1)) == 0);
1583 ASSERT(lpcb
->lp_uselp
== 0);
1584 ASSERT(heap_lp_base
!= NULL
);
1585 ASSERT(heap_lp_end
!= NULL
);
1586 ASSERT(heap_lp_base
< heap_lp_end
);
1587 ASSERT(heap_lp_arena
== NULL
);
1588 ASSERT(((uintptr_t)heap_lp_base
& (lpsize
- 1)) == 0);
1589 ASSERT(((uintptr_t)heap_lp_end
& (lpsize
- 1)) == 0);
1591 /* create large page heap arena */
1592 heap_lp_arena
= vmem_create("heap_lp", heap_lp_base
, heap_lp_size
,
1593 segkmem_heaplp_quantum
, NULL
, NULL
, NULL
, 0, VM_SLEEP
);
1595 ASSERT(heap_lp_arena
!= NULL
);
1597 /* This arena caches memory already mapped by large pages */
1598 kmem_lp_arena
= vmem_create("kmem_lp", NULL
, 0, segkmem_kmemlp_quantum
,
1599 segkmem_alloc_lpi
, segkmem_free_lpi
, heap_lp_arena
, 0, VM_SLEEP
);
1601 ASSERT(kmem_lp_arena
!= NULL
);
1603 mutex_init(&lpcb
->lp_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
1604 cv_init(&lpcb
->lp_cv
, NULL
, CV_DEFAULT
, NULL
);
1607 * this arena is used for the array of page_t pointers necessary
1608 * to call hat_mem_load_array
1610 ppaquantum
= btopr(lpsize
) * sizeof (page_t
*);
1611 segkmem_ppa_arena
= vmem_create("segkmem_ppa", NULL
, 0, ppaquantum
,
1612 segkmem_alloc_ppa
, segkmem_free_ppa
, heap_arena
, ppaquantum
,
1615 ASSERT(segkmem_ppa_arena
!= NULL
);
1617 /* prealloacate some memory for the lp kernel heap */
1618 if (segkmem_kmemlp_min
) {
1620 ASSERT(P2PHASE(segkmem_kmemlp_min
,
1621 segkmem_heaplp_quantum
) == 0);
1623 if ((addr
= segkmem_alloc_lpi(heap_lp_arena
,
1624 segkmem_kmemlp_min
, VM_SLEEP
)) != NULL
) {
1626 addr
= vmem_add(kmem_lp_arena
, addr
,
1627 segkmem_kmemlp_min
, VM_SLEEP
);
1628 ASSERT(addr
!= NULL
);