?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
5285 pass in cpu_pause_func via pause_cpus
[unleashed.git] / usr / src / uts / common / os / mem_config.c
blob3571747e9c8813c79ff1a1e2a0cfc05050a8ef5c
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/types.h>
27 #include <sys/cmn_err.h>
28 #include <sys/vmem.h>
29 #include <sys/kmem.h>
30 #include <sys/systm.h>
31 #include <sys/machsystm.h> /* for page_freelist_coalesce() */
32 #include <sys/errno.h>
33 #include <sys/memnode.h>
34 #include <sys/memlist.h>
35 #include <sys/memlist_impl.h>
36 #include <sys/tuneable.h>
37 #include <sys/proc.h>
38 #include <sys/disp.h>
39 #include <sys/debug.h>
40 #include <sys/vm.h>
41 #include <sys/callb.h>
42 #include <sys/memlist_plat.h> /* for installed_top_size() */
43 #include <sys/condvar_impl.h> /* for CV_HAS_WAITERS() */
44 #include <sys/dumphdr.h> /* for dump_resize() */
45 #include <sys/atomic.h> /* for use in stats collection */
46 #include <sys/rwlock.h>
47 #include <sys/cpuvar.h>
48 #include <vm/seg_kmem.h>
49 #include <vm/seg_kpm.h>
50 #include <vm/page.h>
51 #include <vm/vm_dep.h>
52 #define SUNDDI_IMPL /* so sunddi.h will not redefine splx() et al */
53 #include <sys/sunddi.h>
54 #include <sys/mem_config.h>
55 #include <sys/mem_cage.h>
56 #include <sys/lgrp.h>
57 #include <sys/ddi.h>
58 #include <sys/modctl.h>
59
60 extern struct memlist *phys_avail;
61
62 extern uint_t page_ctrs_adjust(int);
63 void page_ctrs_cleanup(void);
64 static void kphysm_setup_post_add(pgcnt_t);
65 static int kphysm_setup_pre_del(pgcnt_t);
66 static void kphysm_setup_post_del(pgcnt_t, int);
67
68 static int kphysm_split_memseg(pfn_t base, pgcnt_t npgs);
69
70 static int delspan_reserve(pfn_t, pgcnt_t);
71 static void delspan_unreserve(pfn_t, pgcnt_t);
72
73 kmutex_t memseg_lists_lock;
74 struct memseg *memseg_va_avail;
75 struct memseg *memseg_alloc(void);
76 static struct memseg *memseg_delete_junk;
77 static struct memseg *memseg_edit_junk;
78 void memseg_remap_init(void);
79 static void memseg_remap_to_dummy(struct memseg *);
80 static void kphysm_addmem_error_undospan(pfn_t, pgcnt_t);
81 static struct memseg *memseg_reuse(pgcnt_t);
82
83 static struct kmem_cache *memseg_cache;
84
85 /*
86 * Interfaces to manage externally allocated
87 * page_t memory (metadata) for a memseg.
88 */
89 #pragma weak memseg_alloc_meta
90 #pragma weak memseg_free_meta
91 #pragma weak memseg_get_metapfn
92 #pragma weak memseg_remap_meta
93
94 extern int ppvm_enable;
95 extern page_t *ppvm_base;
96 extern int memseg_alloc_meta(pfn_t, pgcnt_t, void **, pgcnt_t *);
97 extern void memseg_free_meta(void *, pgcnt_t);
98 extern pfn_t memseg_get_metapfn(void *, pgcnt_t);
99 extern void memseg_remap_meta(struct memseg *);
100 static int memseg_is_dynamic(struct memseg *);
101 static int memseg_includes_meta(struct memseg *);
102 pfn_t memseg_get_start(struct memseg *);
103 static void memseg_cpu_vm_flush(void);
104
105 int meta_alloc_enable;
106
107 #ifdef DEBUG
108 static int memseg_debug;
109 #define MEMSEG_DEBUG(args...) if (memseg_debug) printf(args)
110 #else
111 #define MEMSEG_DEBUG(...)
112 #endif
113
114 /*
115 * Add a chunk of memory to the system.
116 * base: starting PAGESIZE page of new memory.
117 * npgs: length in PAGESIZE pages.
118 *
119 * Adding mem this way doesn't increase the size of the hash tables;
120 * growing them would be too hard. This should be OK, but adding memory
121 * dynamically most likely means more hash misses, since the tables will
122 * be smaller than they otherwise would be.
123 */
124 int
125 kphysm_add_memory_dynamic(pfn_t base, pgcnt_t npgs)
126 {
127 page_t *pp;
128 page_t *opp, *oepp, *segpp;
129 struct memseg *seg;
130 uint64_t avmem;
131 pfn_t pfn;
132 pfn_t pt_base = base;
133 pgcnt_t tpgs = npgs;
134 pgcnt_t metapgs = 0;
135 int exhausted;
136 pfn_t pnum;
137 int mnode;
138 caddr_t vaddr;
139 int reuse;
140 int mlret;
141 int rv;
142 int flags;
143 int meta_alloc = 0;
144 void *mapva;
145 void *metabase = (void *)base;
146 pgcnt_t nkpmpgs = 0;
147 offset_t kpm_pages_off;
148
149 cmn_err(CE_CONT,
150 "?kphysm_add_memory_dynamic: adding %ldK at 0x%" PRIx64 "\n",
151 npgs << (PAGESHIFT - 10), (uint64_t)base << PAGESHIFT);
152
153 /*
154 * Add this span in the delete list to prevent interactions.
155 */
156 if (!delspan_reserve(base, npgs)) {
157 return (KPHYSM_ESPAN);
158 }
159 /*
160 * Check to see if any of the memory span has been added
161 * by trying an add to the installed memory list. This
162 * forms the interlocking process for add.
163 */
164
165 memlist_write_lock();
166
167 mlret = memlist_add_span((uint64_t)(pt_base) << PAGESHIFT,
168 (uint64_t)(tpgs) << PAGESHIFT, &phys_install);
169
170 if (mlret == MEML_SPANOP_OK)
171 installed_top_size(phys_install, &physmax, &physinstalled);
172
173 memlist_write_unlock();
174
175 if (mlret != MEML_SPANOP_OK) {
176 if (mlret == MEML_SPANOP_EALLOC) {
177 delspan_unreserve(pt_base, tpgs);
178 return (KPHYSM_ERESOURCE);
179 } else if (mlret == MEML_SPANOP_ESPAN) {
180 delspan_unreserve(pt_base, tpgs);
181 return (KPHYSM_ESPAN);
182 } else {
183 delspan_unreserve(pt_base, tpgs);
184 return (KPHYSM_ERESOURCE);
185 }
186 }
187
188 if (meta_alloc_enable) {
189 /*
190 * Allocate the page_t's from existing memory;
191 * if that fails, allocate from the incoming memory.
192 */
193 rv = memseg_alloc_meta(base, npgs, &metabase, &metapgs);
194 if (rv == KPHYSM_OK) {
195 ASSERT(metapgs);
196 ASSERT(btopr(npgs * sizeof (page_t)) <= metapgs);
197 meta_alloc = 1;
198 goto mapalloc;
199 }
200 }
201
202 /*
203 * We store the page_t's for this new memory in the first
204 * few pages of the chunk. Here, we go and get'em ...
205 */
206
207 /*
208 * The expression after the '-' gives the number of pages
209 * that will fit in the new memory based on a requirement
210 * of (PAGESIZE + sizeof (page_t)) bytes per page.
211 */
212 metapgs = npgs - (((uint64_t)(npgs) << PAGESHIFT) /
213 (PAGESIZE + sizeof (page_t)));
214
215 npgs -= metapgs;
216 base += metapgs;
217
218 ASSERT(btopr(npgs * sizeof (page_t)) <= metapgs);
219
220 exhausted = (metapgs == 0 || npgs == 0);
221
222 if (kpm_enable && !exhausted) {
223 pgcnt_t start, end, nkpmpgs_prelim;
224 size_t ptsz;
225
226 /*
227 * A viable kpm large page mapping must not overlap two
228 * dynamic memsegs. Therefore the total size is checked
229 * to be at least kpm_pgsz and also whether start and end
230 * points are at least kpm_pgsz aligned.
231 */
232 if (ptokpmp(tpgs) < 1 || pmodkpmp(pt_base) ||
233 pmodkpmp(base + npgs)) {
234
235 kphysm_addmem_error_undospan(pt_base, tpgs);
236
237 /*
238 * There is no specific error code for violating
239 * kpm granularity constraints.
240 */
241 return (KPHYSM_ENOTVIABLE);
242 }
243
244 start = kpmptop(ptokpmp(base));
245 end = kpmptop(ptokpmp(base + npgs));
246 nkpmpgs_prelim = ptokpmp(end - start);
247 ptsz = npgs * sizeof (page_t);
248 metapgs = btopr(ptsz + nkpmpgs_prelim * KPMPAGE_T_SZ);
249 exhausted = (tpgs <= metapgs);
250 if (!exhausted) {
251 npgs = tpgs - metapgs;
252 base = pt_base + metapgs;
253
254 /* final nkpmpgs */
255 start = kpmptop(ptokpmp(base));
256 nkpmpgs = ptokpmp(end - start);
257 kpm_pages_off = ptsz +
258 (nkpmpgs_prelim - nkpmpgs) * KPMPAGE_T_SZ;
259 }
260 }
261
262 /*
263 * Is memory area supplied too small?
264 */
265 if (exhausted) {
266 kphysm_addmem_error_undospan(pt_base, tpgs);
267 /*
268 * There is no specific error code for 'too small'.
269 */
270 return (KPHYSM_ERESOURCE);
271 }
272
273 mapalloc:
274 /*
275 * We may re-use a previously allocated VA space for the page_ts
276 * eventually, but we need to initialize and lock the pages first.
277 */
278
279 /*
280 * Get an address in the kernel address map, map
281 * the page_t pages and see if we can touch them.
282 */
283
284 mapva = vmem_alloc(heap_arena, ptob(metapgs), VM_NOSLEEP);
285 if (mapva == NULL) {
286 cmn_err(CE_WARN, "kphysm_add_memory_dynamic:"
287 " Can't allocate VA for page_ts");
288
289 if (meta_alloc)
290 memseg_free_meta(metabase, metapgs);
291 kphysm_addmem_error_undospan(pt_base, tpgs);
292
293 return (KPHYSM_ERESOURCE);
294 }
295 pp = mapva;
296
297 if (physmax < (pt_base + tpgs))
298 physmax = (pt_base + tpgs);
299
300 /*
301 * In the remapping code we map one page at a time so we must do
302 * the same here to match mapping sizes.
303 */
304 pfn = pt_base;
305 vaddr = (caddr_t)pp;
306 for (pnum = 0; pnum < metapgs; pnum++) {
307 if (meta_alloc)
308 pfn = memseg_get_metapfn(metabase, (pgcnt_t)pnum);
309 hat_devload(kas.a_hat, vaddr, ptob(1), pfn,
310 PROT_READ | PROT_WRITE,
311 HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
312 pfn++;
313 vaddr += ptob(1);
314 }
315
316 if (ddi_peek32((dev_info_t *)NULL,
317 (int32_t *)pp, (int32_t *)0) == DDI_FAILURE) {
318
319 cmn_err(CE_WARN, "kphysm_add_memory_dynamic:"
320 " Can't access pp array at 0x%p [phys 0x%lx]",
321 (void *)pp, pt_base);
322
323 hat_unload(kas.a_hat, (caddr_t)pp, ptob(metapgs),
324 HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);
325
326 vmem_free(heap_arena, mapva, ptob(metapgs));
327 if (meta_alloc)
328 memseg_free_meta(metabase, metapgs);
329 kphysm_addmem_error_undospan(pt_base, tpgs);
330
331 return (KPHYSM_EFAULT);
332 }
333
334 /*
335 * Add this memory slice to its memory node translation.
336 *
337 * Note that right now, each node may have only one slice;
338 * this may change with COD or in larger SSM systems with
339 * nested latency groups, so we must not assume that the
340 * node does not yet exist.
341 *
342 * Note that there may be multiple memory nodes associated with
343 * a single lgrp node on x86 systems.
344 */
345 pnum = pt_base + tpgs - 1;
346 mem_node_add_range(pt_base, pnum);
347
348 /*
349 * Allocate or resize page counters as necessary to accommodate
350 * the increase in memory pages.
351 */
352 mnode = PFN_2_MEM_NODE(pnum);
353 PAGE_CTRS_ADJUST(base, npgs, rv);
354 if (rv) {
355
356 mem_node_del_range(pt_base, pnum);
357
358 /* cleanup the page counters */
359 page_ctrs_cleanup();
360
361 hat_unload(kas.a_hat, (caddr_t)pp, ptob(metapgs),
362 HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);
363
364 vmem_free(heap_arena, mapva, ptob(metapgs));
365 if (meta_alloc)
366 memseg_free_meta(metabase, metapgs);
367 kphysm_addmem_error_undospan(pt_base, tpgs);
368
369 return (KPHYSM_ERESOURCE);
370 }
371
372 /*
373 * Update the phys_avail memory list.
374 * The phys_install list was done at the start.
375 */
376
377 memlist_write_lock();
378
379 mlret = memlist_add_span((uint64_t)(base) << PAGESHIFT,
380 (uint64_t)(npgs) << PAGESHIFT, &phys_avail);
381 ASSERT(mlret == MEML_SPANOP_OK);
382
383 memlist_write_unlock();
384
385 /* See if we can find a memseg to re-use. */
386 if (meta_alloc) {
387 seg = memseg_reuse(0);
388 reuse = 1; /* force unmapping of temp mapva */
389 flags = MEMSEG_DYNAMIC | MEMSEG_META_ALLOC;
390 /*
391 * There is a 1:1 fixed relationship between a pfn
392 * and a page_t VA. The pfn is used as an index into
393 * the ppvm_base page_t table in order to calculate
394 * the page_t base address for a given pfn range.
395 */
396 segpp = ppvm_base + base;
397 } else {
398 seg = memseg_reuse(metapgs);
399 reuse = (seg != NULL);
400 flags = MEMSEG_DYNAMIC | MEMSEG_META_INCL;
401 segpp = pp;
402 }
403
404 /*
405 * Initialize the memseg structure representing this memory
406 * and add it to the existing list of memsegs. Do some basic
407 * initialization and add the memory to the system.
408 * In order to prevent lock deadlocks, the add_physmem()
409 * code is repeated here, but split into several stages.
410 *
411 * If a memseg is reused, invalidate memseg pointers in
412 * all cpu vm caches. We need to do this this since the check
413 * pp >= seg->pages && pp < seg->epages
414 * used in various places is not atomic and so the first compare
415 * can happen before reuse and the second compare after reuse.
416 * The invalidation ensures that a memseg is not deferenced while
417 * it's page/pfn pointers are changing.
418 */
419 if (seg == NULL) {
420 seg = memseg_alloc();
421 ASSERT(seg != NULL);
422 seg->msegflags = flags;
423 MEMSEG_DEBUG("memseg_get: alloc seg=0x%p, pages=0x%p",
424 (void *)seg, (void *)(seg->pages));
425 seg->pages = segpp;
426 } else {
427 ASSERT(seg->msegflags == flags);
428 ASSERT(seg->pages_base == seg->pages_end);
429 MEMSEG_DEBUG("memseg_get: reuse seg=0x%p, pages=0x%p",
430 (void *)seg, (void *)(seg->pages));
431 if (meta_alloc) {
432 memseg_cpu_vm_flush();
433 seg->pages = segpp;
434 }
435 }
436
437 seg->epages = seg->pages + npgs;
438 seg->pages_base = base;
439 seg->pages_end = base + npgs;
440
441 /*
442 * Initialize metadata. The page_ts are set to locked state
443 * ready to be freed.
444 */
445 bzero((caddr_t)pp, ptob(metapgs));
446
447 pfn = seg->pages_base;
448 /* Save the original pp base in case we reuse a memseg. */
449 opp = pp;
450 oepp = opp + npgs;
451 for (pp = opp; pp < oepp; pp++) {
452 pp->p_pagenum = pfn;
453 pfn++;
454 page_iolock_init(pp);
455 while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM))
456 continue;
457 pp->p_offset = (u_offset_t)-1;
458 }
459
460 if (reuse) {
461 /* Remap our page_ts to the re-used memseg VA space. */
462 pfn = pt_base;
463 vaddr = (caddr_t)seg->pages;
464 for (pnum = 0; pnum < metapgs; pnum++) {
465 if (meta_alloc)
466 pfn = memseg_get_metapfn(metabase,
467 (pgcnt_t)pnum);
468 hat_devload(kas.a_hat, vaddr, ptob(1), pfn,
469 PROT_READ | PROT_WRITE,
470 HAT_LOAD_REMAP | HAT_LOAD | HAT_LOAD_NOCONSIST);
471 pfn++;
472 vaddr += ptob(1);
473 }
474
475 hat_unload(kas.a_hat, (caddr_t)opp, ptob(metapgs),
476 HAT_UNLOAD_UNMAP|HAT_UNLOAD_UNLOCK);
477
478 vmem_free(heap_arena, mapva, ptob(metapgs));
479 }
480
481 hat_kpm_addmem_mseg_update(seg, nkpmpgs, kpm_pages_off);
482
483 memsegs_lock(1);
484
485 /*
486 * The new memseg is inserted at the beginning of the list.
487 * Not only does this save searching for the tail, but in the
488 * case of a re-used memseg, it solves the problem of what
489 * happens if some process has still got a pointer to the
490 * memseg and follows the next pointer to continue traversing
491 * the memsegs list.
492 */
493
494 hat_kpm_addmem_mseg_insert(seg);
495
496 seg->next = memsegs;
497 membar_producer();
498
499 hat_kpm_addmem_memsegs_update(seg);
500
501 memsegs = seg;
502
503 build_pfn_hash();
504
505 total_pages += npgs;
506
507 /*
508 * Recalculate the paging parameters now total_pages has changed.
509 * This will also cause the clock hands to be reset before next use.
510 */
511 setupclock(1);
512
513 memsegs_unlock(1);
514
515 PLCNT_MODIFY_MAX(seg->pages_base, (long)npgs);
516
517 /*
518 * Free the pages outside the lock to avoid locking loops.
519 */
520 for (pp = seg->pages; pp < seg->epages; pp++) {
521 page_free(pp, 1);
522 }
523
524 /*
525 * Now that we've updated the appropriate memory lists we
526 * need to reset a number of globals, since we've increased memory.
527 * Several have already been updated for us as noted above. The
528 * globals we're interested in at this point are:
529 * physmax - highest page frame number.
530 * physinstalled - number of pages currently installed (done earlier)
531 * maxmem - max free pages in the system
532 * physmem - physical memory pages available
533 * availrmem - real memory available
534 */
535
536 mutex_enter(&freemem_lock);
537 maxmem += npgs;
538 physmem += npgs;
539 availrmem += npgs;
540 availrmem_initial += npgs;
541
542 mutex_exit(&freemem_lock);
543
544 dump_resize();
545
546 page_freelist_coalesce_all(mnode);
547
548 kphysm_setup_post_add(npgs);
549
550 cmn_err(CE_CONT, "?kphysm_add_memory_dynamic: mem = %ldK "
551 "(0x%" PRIx64 ")\n",
552 physinstalled << (PAGESHIFT - 10),
553 (uint64_t)physinstalled << PAGESHIFT);
554
555 avmem = (uint64_t)freemem << PAGESHIFT;
556 cmn_err(CE_CONT, "?kphysm_add_memory_dynamic: "
557 "avail mem = %" PRId64 "\n", avmem);
558
559 /*
560 * Update lgroup generation number on single lgroup systems
561 */
562 if (nlgrps == 1)
563 lgrp_config(LGRP_CONFIG_GEN_UPDATE, 0, 0);
564
565 /*
566 * Inform DDI of update
567 */
568 ddi_mem_update((uint64_t)(pt_base) << PAGESHIFT,
569 (uint64_t)(tpgs) << PAGESHIFT);
570
571 delspan_unreserve(pt_base, tpgs);
572
573 return (KPHYSM_OK); /* Successfully added system memory */
574 }
575
576 /*
577 * There are various error conditions in kphysm_add_memory_dynamic()
578 * which require a rollback of already changed global state.
579 */
580 static void
581 kphysm_addmem_error_undospan(pfn_t pt_base, pgcnt_t tpgs)
582 {
583 int mlret;
584
585 /* Unreserve memory span. */
586 memlist_write_lock();
587
588 mlret = memlist_delete_span(
589 (uint64_t)(pt_base) << PAGESHIFT,
590 (uint64_t)(tpgs) << PAGESHIFT, &phys_install);
591
592 ASSERT(mlret == MEML_SPANOP_OK);
593 phys_install_has_changed();
594 installed_top_size(phys_install, &physmax, &physinstalled);
595
596 memlist_write_unlock();
597 delspan_unreserve(pt_base, tpgs);
598 }
599
600 /*
601 * Only return an available memseg of exactly the right size
602 * if size is required.
603 * When the meta data area has it's own virtual address space
604 * we will need to manage this more carefully and do best fit
605 * allocations, possibly splitting an available area.
606 */
607 struct memseg *
608 memseg_reuse(pgcnt_t metapgs)
609 {
610 int type;
611 struct memseg **segpp, *seg;
612
613 mutex_enter(&memseg_lists_lock);
614
615 segpp = &memseg_va_avail;
616 for (; (seg = *segpp) != NULL; segpp = &seg->lnext) {
617 caddr_t end;
618
619 /*
620 * Make sure we are reusing the right segment type.
621 */
622 type = metapgs ? MEMSEG_META_INCL : MEMSEG_META_ALLOC;
623
624 if ((seg->msegflags & (MEMSEG_META_INCL | MEMSEG_META_ALLOC))
625 != type)
626 continue;
627
628 if (kpm_enable)
629 end = hat_kpm_mseg_reuse(seg);
630 else
631 end = (caddr_t)seg->epages;
632
633 /*
634 * Check for the right size if it is provided.
635 */
636 if (!metapgs || btopr(end - (caddr_t)seg->pages) == metapgs) {
637 *segpp = seg->lnext;
638 seg->lnext = NULL;
639 break;
640 }
641 }
642 mutex_exit(&memseg_lists_lock);
643
644 return (seg);
645 }
646
647 static uint_t handle_gen;
648
649 struct memdelspan {
650 struct memdelspan *mds_next;
651 pfn_t mds_base;
652 pgcnt_t mds_npgs;
653 uint_t *mds_bitmap;
654 uint_t *mds_bitmap_retired;
655 };
656
657 #define NBPBMW (sizeof (uint_t) * NBBY)
658 #define MDS_BITMAPBYTES(MDSP) \
659 ((((MDSP)->mds_npgs + NBPBMW - 1) / NBPBMW) * sizeof (uint_t))
660
661 struct transit_list {
662 struct transit_list *trl_next;
663 struct memdelspan *trl_spans;
664 int trl_collect;
665 };
666
667 struct transit_list_head {
668 kmutex_t trh_lock;
669 struct transit_list *trh_head;
670 };
671
672 static struct transit_list_head transit_list_head;
673
674 struct mem_handle;
675 static void transit_list_collect(struct mem_handle *, int);
676 static void transit_list_insert(struct transit_list *);
677 static void transit_list_remove(struct transit_list *);
678
679 #ifdef DEBUG
680 #define MEM_DEL_STATS
681 #endif /* DEBUG */
682
683 #ifdef MEM_DEL_STATS
684 static int mem_del_stat_print = 0;
685 struct mem_del_stat {
686 uint_t nloop;
687 uint_t need_free;
688 uint_t free_loop;
689 uint_t free_low;
690 uint_t free_failed;
691 uint_t ncheck;
692 uint_t nopaget;
693 uint_t lockfail;
694 uint_t nfree;
695 uint_t nreloc;
696 uint_t nrelocfail;
697 uint_t already_done;
698 uint_t first_notfree;
699 uint_t npplocked;
700 uint_t nlockreloc;
701 uint_t nnorepl;
702 uint_t nmodreloc;
703 uint_t ndestroy;
704 uint_t nputpage;
705 uint_t nnoreclaim;
706 uint_t ndelay;
707 uint_t demotefail;
708 uint64_t nticks_total;
709 uint64_t nticks_pgrp;
710 uint_t retired;
711 uint_t toxic;
712 uint_t failing;
713 uint_t modtoxic;
714 uint_t npplkdtoxic;
715 uint_t gptlmodfail;
716 uint_t gptllckfail;
717 };
718 /*
719 * The stat values are only incremented in the delete thread
720 * so no locking or atomic required.
721 */
722 #define MDSTAT_INCR(MHP, FLD) (MHP)->mh_delstat.FLD++
723 #define MDSTAT_TOTAL(MHP, ntck) ((MHP)->mh_delstat.nticks_total += (ntck))
724 #define MDSTAT_PGRP(MHP, ntck) ((MHP)->mh_delstat.nticks_pgrp += (ntck))
725 static void mem_del_stat_print_func(struct mem_handle *);
726 #define MDSTAT_PRINT(MHP) mem_del_stat_print_func((MHP))
727 #else /* MEM_DEL_STATS */
728 #define MDSTAT_INCR(MHP, FLD)
729 #define MDSTAT_TOTAL(MHP, ntck)
730 #define MDSTAT_PGRP(MHP, ntck)
731 #define MDSTAT_PRINT(MHP)
732 #endif /* MEM_DEL_STATS */
733
734 typedef enum mhnd_state {MHND_FREE = 0, MHND_INIT, MHND_STARTING,
735 MHND_RUNNING, MHND_DONE, MHND_RELEASE} mhnd_state_t;
736
737 /*
738 * mh_mutex must be taken to examine or change mh_exthandle and mh_state.
739 * The mutex may not be required for other fields, dependent on mh_state.
740 */
741 struct mem_handle {
742 kmutex_t mh_mutex;
743 struct mem_handle *mh_next;
744 memhandle_t mh_exthandle;
745 mhnd_state_t mh_state;
746 struct transit_list mh_transit;
747 pgcnt_t mh_phys_pages;
748 pgcnt_t mh_vm_pages;
749 pgcnt_t mh_hold_todo;
750 void (*mh_delete_complete)(void *, int error);
751 void *mh_delete_complete_arg;
752 volatile uint_t mh_cancel;
753 volatile uint_t mh_dr_aio_cleanup_cancel;
754 volatile uint_t mh_aio_cleanup_done;
755 kcondvar_t mh_cv;
756 kthread_id_t mh_thread_id;
757 page_t *mh_deleted; /* link through p_next */
758 #ifdef MEM_DEL_STATS
759 struct mem_del_stat mh_delstat;
760 #endif /* MEM_DEL_STATS */
761 };
762
763 static struct mem_handle *mem_handle_head;
764 static kmutex_t mem_handle_list_mutex;
765
766 static struct mem_handle *
767 kphysm_allocate_mem_handle()
768 {
769 struct mem_handle *mhp;
770
771 mhp = kmem_zalloc(sizeof (struct mem_handle), KM_SLEEP);
772 mutex_init(&mhp->mh_mutex, NULL, MUTEX_DEFAULT, NULL);
773 mutex_enter(&mem_handle_list_mutex);
774 mutex_enter(&mhp->mh_mutex);
775 /* handle_gen is protected by list mutex. */
776 mhp->mh_exthandle = (memhandle_t)(uintptr_t)(++handle_gen);
777 mhp->mh_next = mem_handle_head;
778 mem_handle_head = mhp;
779 mutex_exit(&mem_handle_list_mutex);
780
781 return (mhp);
782 }
783
784 static void
785 kphysm_free_mem_handle(struct mem_handle *mhp)
786 {
787 struct mem_handle **mhpp;
788
789 ASSERT(mutex_owned(&mhp->mh_mutex));
790 ASSERT(mhp->mh_state == MHND_FREE);
791 /*
792 * Exit the mutex to preserve locking order. This is OK
793 * here as once in the FREE state, the handle cannot
794 * be found by a lookup.
795 */
796 mutex_exit(&mhp->mh_mutex);
797
798 mutex_enter(&mem_handle_list_mutex);
799 mhpp = &mem_handle_head;
800 while (*mhpp != NULL && *mhpp != mhp)
801 mhpp = &(*mhpp)->mh_next;
802 ASSERT(*mhpp == mhp);
803 /*
804 * No need to lock the handle (mh_mutex) as only
805 * mh_next changing and this is the only thread that
806 * can be referncing mhp.
807 */
808 *mhpp = mhp->mh_next;
809 mutex_exit(&mem_handle_list_mutex);
810
811 mutex_destroy(&mhp->mh_mutex);
812 kmem_free(mhp, sizeof (struct mem_handle));
813 }
814
815 /*
816 * This function finds the internal mem_handle corresponding to an
817 * external handle and returns it with the mh_mutex held.
818 */
819 static struct mem_handle *
820 kphysm_lookup_mem_handle(memhandle_t handle)
821 {
822 struct mem_handle *mhp;
823
824 mutex_enter(&mem_handle_list_mutex);
825 for (mhp = mem_handle_head; mhp != NULL; mhp = mhp->mh_next) {
826 if (mhp->mh_exthandle == handle) {
827 mutex_enter(&mhp->mh_mutex);
828 /*
829 * The state of the handle could have been changed
830 * by kphysm_del_release() while waiting for mh_mutex.
831 */
832 if (mhp->mh_state == MHND_FREE) {
833 mutex_exit(&mhp->mh_mutex);
834 continue;
835 }
836 break;
837 }
838 }
839 mutex_exit(&mem_handle_list_mutex);
840 return (mhp);
841 }
842
843 int
844 kphysm_del_gethandle(memhandle_t *xmhp)
845 {
846 struct mem_handle *mhp;
847
848 mhp = kphysm_allocate_mem_handle();
849 /*
850 * The handle is allocated using KM_SLEEP, so cannot fail.
851 * If the implementation is changed, the correct error to return
852 * here would be KPHYSM_ENOHANDLES.
853 */
854 ASSERT(mhp->mh_state == MHND_FREE);
855 mhp->mh_state = MHND_INIT;
856 *xmhp = mhp->mh_exthandle;
857 mutex_exit(&mhp->mh_mutex);
858 return (KPHYSM_OK);
859 }
860
861 static int
862 overlapping(pfn_t b1, pgcnt_t l1, pfn_t b2, pgcnt_t l2)
863 {
864 pfn_t e1, e2;
865
866 e1 = b1 + l1;
867 e2 = b2 + l2;
868
869 return (!(b2 >= e1 || b1 >= e2));
870 }
871
872 static int can_remove_pgs(pgcnt_t);
873
874 static struct memdelspan *
875 span_to_install(pfn_t base, pgcnt_t npgs)
876 {
877 struct memdelspan *mdsp;
878 struct memdelspan *mdsp_new;
879 uint64_t address, size, thislen;
880 struct memlist *mlp;
881
882 mdsp_new = NULL;
883
884 address = (uint64_t)base << PAGESHIFT;
885 size = (uint64_t)npgs << PAGESHIFT;
886 while (size != 0) {
887 memlist_read_lock();
888 for (mlp = phys_install; mlp != NULL; mlp = mlp->ml_next) {
889 if (address >= (mlp->ml_address + mlp->ml_size))
890 continue;
891 if ((address + size) > mlp->ml_address)
892 break;
893 }
894 if (mlp == NULL) {
895 address += size;
896 size = 0;
897 thislen = 0;
898 } else {
899 if (address < mlp->ml_address) {
900 size -= (mlp->ml_address - address);
901 address = mlp->ml_address;
902 }
903 ASSERT(address >= mlp->ml_address);
904 if ((address + size) >
905 (mlp->ml_address + mlp->ml_size)) {
906 thislen =
907 mlp->ml_size - (address - mlp->ml_address);
908 } else {
909 thislen = size;
910 }
911 }
912 memlist_read_unlock();
913 /* TODO: phys_install could change now */
914 if (thislen == 0)
915 continue;
916 mdsp = kmem_zalloc(sizeof (struct memdelspan), KM_SLEEP);
917 mdsp->mds_base = btop(address);
918 mdsp->mds_npgs = btop(thislen);
919 mdsp->mds_next = mdsp_new;
920 mdsp_new = mdsp;
921 address += thislen;
922 size -= thislen;
923 }
924 return (mdsp_new);
925 }
926
927 static void
928 free_delspans(struct memdelspan *mdsp)
929 {
930 struct memdelspan *amdsp;
931
932 while ((amdsp = mdsp) != NULL) {
933 mdsp = amdsp->mds_next;
934 kmem_free(amdsp, sizeof (struct memdelspan));
935 }
936 }
937
938 /*
939 * Concatenate lists. No list ordering is required.
940 */
941
942 static void
943 delspan_concat(struct memdelspan **mdspp, struct memdelspan *mdsp)
944 {
945 while (*mdspp != NULL)
946 mdspp = &(*mdspp)->mds_next;
947
948 *mdspp = mdsp;
949 }
950
951 /*
952 * Given a new list of delspans, check there is no overlap with
953 * all existing span activity (add or delete) and then concatenate
954 * the new spans to the given list.
955 * Return 1 for OK, 0 if overlapping.
956 */
957 static int
958 delspan_insert(
959 struct transit_list *my_tlp,
960 struct memdelspan *mdsp_new)
961 {
962 struct transit_list_head *trh;
963 struct transit_list *tlp;
964 int ret;
965
966 trh = &transit_list_head;
967
968 ASSERT(my_tlp != NULL);
969 ASSERT(mdsp_new != NULL);
970
971 ret = 1;
972 mutex_enter(&trh->trh_lock);
973 /* ASSERT(my_tlp->trl_spans == NULL || tlp_in_list(trh, my_tlp)); */
974 for (tlp = trh->trh_head; tlp != NULL; tlp = tlp->trl_next) {
975 struct memdelspan *mdsp;
976
977 for (mdsp = tlp->trl_spans; mdsp != NULL;
978 mdsp = mdsp->mds_next) {
979 struct memdelspan *nmdsp;
980
981 for (nmdsp = mdsp_new; nmdsp != NULL;
982 nmdsp = nmdsp->mds_next) {
983 if (overlapping(mdsp->mds_base, mdsp->mds_npgs,
984 nmdsp->mds_base, nmdsp->mds_npgs)) {
985 ret = 0;
986 goto done;
987 }
988 }
989 }
990 }
991 done:
992 if (ret != 0) {
993 if (my_tlp->trl_spans == NULL)
994 transit_list_insert(my_tlp);
995 delspan_concat(&my_tlp->trl_spans, mdsp_new);
996 }
997 mutex_exit(&trh->trh_lock);
998 return (ret);
999 }
1000
1001 static void
1002 delspan_remove(
1003 struct transit_list *my_tlp,
1004 pfn_t base,
1005 pgcnt_t npgs)
1006 {
1007 struct transit_list_head *trh;
1008 struct memdelspan *mdsp;
1009
1010 trh = &transit_list_head;
1011
1012 ASSERT(my_tlp != NULL);
1013
1014 mutex_enter(&trh->trh_lock);
1015 if ((mdsp = my_tlp->trl_spans) != NULL) {
1016 if (npgs == 0) {
1017 my_tlp->trl_spans = NULL;
1018 free_delspans(mdsp);
1019 transit_list_remove(my_tlp);
1020 } else {
1021 struct memdelspan **prv;
1022
1023 prv = &my_tlp->trl_spans;
1024 while (mdsp != NULL) {
1025 pfn_t p_end;
1026
1027 p_end = mdsp->mds_base + mdsp->mds_npgs;
1028 if (mdsp->mds_base >= base &&
1029 p_end <= (base + npgs)) {
1030 *prv = mdsp->mds_next;
1031 mdsp->mds_next = NULL;
1032 free_delspans(mdsp);
1033 } else {
1034 prv = &mdsp->mds_next;
1035 }
1036 mdsp = *prv;
1037 }
1038 if (my_tlp->trl_spans == NULL)
1039 transit_list_remove(my_tlp);
1040 }
1041 }
1042 mutex_exit(&trh->trh_lock);
1043 }
1044
1045 /*
1046 * Reserve interface for add to stop delete before add finished.
1047 * This list is only accessed through the delspan_insert/remove
1048 * functions and so is fully protected by the mutex in struct transit_list.
1049 */
1050
1051 static struct transit_list reserve_transit;
1052
1053 static int
1054 delspan_reserve(pfn_t base, pgcnt_t npgs)
1055 {
1056 struct memdelspan *mdsp;
1057 int ret;
1058
1059 mdsp = kmem_zalloc(sizeof (struct memdelspan), KM_SLEEP);
1060 mdsp->mds_base = base;
1061 mdsp->mds_npgs = npgs;
1062 if ((ret = delspan_insert(&reserve_transit, mdsp)) == 0) {
1063 free_delspans(mdsp);
1064 }
1065 return (ret);
1066 }
1067
1068 static void
1069 delspan_unreserve(pfn_t base, pgcnt_t npgs)
1070 {
1071 delspan_remove(&reserve_transit, base, npgs);
1072 }
1073
1074 /*
1075 * Return whether memseg was created by kphysm_add_memory_dynamic().
1076 */
1077 static int
1078 memseg_is_dynamic(struct memseg *seg)
1079 {
1080 return (seg->msegflags & MEMSEG_DYNAMIC);
1081 }
1082
1083 int
1084 kphysm_del_span(
1085 memhandle_t handle,
1086 pfn_t base,
1087 pgcnt_t npgs)
1088 {
1089 struct mem_handle *mhp;
1090 struct memseg *seg;
1091 struct memdelspan *mdsp;
1092 struct memdelspan *mdsp_new;
1093 pgcnt_t phys_pages, vm_pages;
1094 pfn_t p_end;
1095 page_t *pp;
1096 int ret;
1097
1098 mhp = kphysm_lookup_mem_handle(handle);
1099 if (mhp == NULL) {
1100 return (KPHYSM_EHANDLE);
1101 }
1102 if (mhp->mh_state != MHND_INIT) {
1103 mutex_exit(&mhp->mh_mutex);
1104 return (KPHYSM_ESEQUENCE);
1105 }
1106
1107 /*
1108 * Intersect the span with the installed memory list (phys_install).
1109 */
1110 mdsp_new = span_to_install(base, npgs);
1111 if (mdsp_new == NULL) {
1112 /*
1113 * No physical memory in this range. Is this an
1114 * error? If an attempt to start the delete is made
1115 * for OK returns from del_span such as this, start will
1116 * return an error.
1117 * Could return KPHYSM_ENOWORK.
1118 */
1119 /*
1120 * It is assumed that there are no error returns
1121 * from span_to_install() due to kmem_alloc failure.
1122 */
1123 mutex_exit(&mhp->mh_mutex);
1124 return (KPHYSM_OK);
1125 }
1126 /*
1127 * Does this span overlap an existing span?
1128 */
1129 if (delspan_insert(&mhp->mh_transit, mdsp_new) == 0) {
1130 /*
1131 * Differentiate between already on list for this handle
1132 * (KPHYSM_EDUP) and busy elsewhere (KPHYSM_EBUSY).
1133 */
1134 ret = KPHYSM_EBUSY;
1135 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
1136 mdsp = mdsp->mds_next) {
1137 if (overlapping(mdsp->mds_base, mdsp->mds_npgs,
1138 base, npgs)) {
1139 ret = KPHYSM_EDUP;
1140 break;
1141 }
1142 }
1143 mutex_exit(&mhp->mh_mutex);
1144 free_delspans(mdsp_new);
1145 return (ret);
1146 }
1147 /*
1148 * At this point the spans in mdsp_new have been inserted into the
1149 * list of spans for this handle and thereby to the global list of
1150 * spans being processed. Each of these spans must now be checked
1151 * for relocatability. As a side-effect segments in the memseg list
1152 * may be split.
1153 *
1154 * Note that mdsp_new can no longer be used as it is now part of
1155 * a larger list. Select elements of this larger list based
1156 * on base and npgs.
1157 */
1158 restart:
1159 phys_pages = 0;
1160 vm_pages = 0;
1161 ret = KPHYSM_OK;
1162 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
1163 mdsp = mdsp->mds_next) {
1164 pgcnt_t pages_checked;
1165
1166 if (!overlapping(mdsp->mds_base, mdsp->mds_npgs, base, npgs)) {
1167 continue;
1168 }
1169 p_end = mdsp->mds_base + mdsp->mds_npgs;
1170 /*
1171 * The pages_checked count is a hack. All pages should be
1172 * checked for relocatability. Those not covered by memsegs
1173 * should be tested with arch_kphysm_del_span_ok().
1174 */
1175 pages_checked = 0;
1176 for (seg = memsegs; seg; seg = seg->next) {
1177 pfn_t mseg_start;
1178
1179 if (seg->pages_base >= p_end ||
1180 seg->pages_end <= mdsp->mds_base) {
1181 /* Span and memseg don't overlap. */
1182 continue;
1183 }
1184 mseg_start = memseg_get_start(seg);
1185 /* Check that segment is suitable for delete. */
1186 if (memseg_includes_meta(seg)) {
1187 /*
1188 * Check that this segment is completely
1189 * within the span.
1190 */
1191 if (mseg_start < mdsp->mds_base ||
1192 seg->pages_end > p_end) {
1193 ret = KPHYSM_EBUSY;
1194 break;
1195 }
1196 pages_checked += seg->pages_end - mseg_start;
1197 } else {
1198 /*
1199 * If this segment is larger than the span,
1200 * try to split it. After the split, it
1201 * is necessary to restart.
1202 */
1203 if (seg->pages_base < mdsp->mds_base ||
1204 seg->pages_end > p_end) {
1205 pfn_t abase;
1206 pgcnt_t anpgs;
1207 int s_ret;
1208
1209 /* Split required. */
1210 if (mdsp->mds_base < seg->pages_base)
1211 abase = seg->pages_base;
1212 else
1213 abase = mdsp->mds_base;
1214 if (p_end > seg->pages_end)
1215 anpgs = seg->pages_end - abase;
1216 else
1217 anpgs = p_end - abase;
1218 s_ret = kphysm_split_memseg(abase,
1219 anpgs);
1220 if (s_ret == 0) {
1221 /* Split failed. */
1222 ret = KPHYSM_ERESOURCE;
1223 break;
1224 }
1225 goto restart;
1226 }
1227 pages_checked +=
1228 seg->pages_end - seg->pages_base;
1229 }
1230 /*
1231 * The memseg is wholly within the delete span.
1232 * The individual pages can now be checked.
1233 */
1234 /* Cage test. */
1235 for (pp = seg->pages; pp < seg->epages; pp++) {
1236 if (PP_ISNORELOC(pp)) {
1237 ret = KPHYSM_ENONRELOC;
1238 break;
1239 }
1240 }
1241 if (ret != KPHYSM_OK) {
1242 break;
1243 }
1244 phys_pages += (seg->pages_end - mseg_start);
1245 vm_pages += MSEG_NPAGES(seg);
1246 }
1247 if (ret != KPHYSM_OK)
1248 break;
1249 if (pages_checked != mdsp->mds_npgs) {
1250 ret = KPHYSM_ENONRELOC;
1251 break;
1252 }
1253 }
1254
1255 if (ret == KPHYSM_OK) {
1256 mhp->mh_phys_pages += phys_pages;
1257 mhp->mh_vm_pages += vm_pages;
1258 } else {
1259 /*
1260 * Keep holding the mh_mutex to prevent it going away.
1261 */
1262 delspan_remove(&mhp->mh_transit, base, npgs);
1263 }
1264 mutex_exit(&mhp->mh_mutex);
1265 return (ret);
1266 }
1267
1268 int
1269 kphysm_del_span_query(
1270 pfn_t base,
1271 pgcnt_t npgs,
1272 memquery_t *mqp)
1273 {
1274 struct memdelspan *mdsp;
1275 struct memdelspan *mdsp_new;
1276 int done_first_nonreloc;
1277
1278 mqp->phys_pages = 0;
1279 mqp->managed = 0;
1280 mqp->nonrelocatable = 0;
1281 mqp->first_nonrelocatable = 0;
1282 mqp->last_nonrelocatable = 0;
1283
1284 mdsp_new = span_to_install(base, npgs);
1285 /*
1286 * It is OK to proceed here if mdsp_new == NULL.
1287 */
1288 done_first_nonreloc = 0;
1289 for (mdsp = mdsp_new; mdsp != NULL; mdsp = mdsp->mds_next) {
1290 pfn_t sbase;
1291 pgcnt_t snpgs;
1292
1293 mqp->phys_pages += mdsp->mds_npgs;
1294 sbase = mdsp->mds_base;
1295 snpgs = mdsp->mds_npgs;
1296 while (snpgs != 0) {
1297 struct memseg *lseg, *seg;
1298 pfn_t p_end;
1299 page_t *pp;
1300 pfn_t mseg_start;
1301
1302 p_end = sbase + snpgs;
1303 /*
1304 * Find the lowest addressed memseg that starts
1305 * after sbase and account for it.
1306 * This is to catch dynamic memsegs whose start
1307 * is hidden.
1308 */
1309 seg = NULL;
1310 for (lseg = memsegs; lseg != NULL; lseg = lseg->next) {
1311 if ((lseg->pages_base >= sbase) ||
1312 (lseg->pages_base < p_end &&
1313 lseg->pages_end > sbase)) {
1314 if (seg == NULL ||
1315 seg->pages_base > lseg->pages_base)
1316 seg = lseg;
1317 }
1318 }
1319 if (seg != NULL) {
1320 mseg_start = memseg_get_start(seg);
1321 /*
1322 * Now have the full extent of the memseg so
1323 * do the range check.
1324 */
1325 if (mseg_start >= p_end ||
1326 seg->pages_end <= sbase) {
1327 /* Span does not overlap memseg. */
1328 seg = NULL;
1329 }
1330 }
1331 /*
1332 * Account for gap either before the segment if
1333 * there is one or to the end of the span.
1334 */
1335 if (seg == NULL || mseg_start > sbase) {
1336 pfn_t a_end;
1337
1338 a_end = (seg == NULL) ? p_end : mseg_start;
1339 /*
1340 * Check with arch layer for relocatability.
1341 */
1342 if (arch_kphysm_del_span_ok(sbase,
1343 (a_end - sbase))) {
1344 /*
1345 * No non-relocatble pages in this
1346 * area, avoid the fine-grained
1347 * test.
1348 */
1349 snpgs -= (a_end - sbase);
1350 sbase = a_end;
1351 }
1352 while (sbase < a_end) {
1353 if (!arch_kphysm_del_span_ok(sbase,
1354 1)) {
1355 mqp->nonrelocatable++;
1356 if (!done_first_nonreloc) {
1357 mqp->
1358 first_nonrelocatable
1359 = sbase;
1360 done_first_nonreloc = 1;
1361 }
1362 mqp->last_nonrelocatable =
1363 sbase;
1364 }
1365 sbase++;
1366 snpgs--;
1367 }
1368 }
1369 if (seg != NULL) {
1370 ASSERT(mseg_start <= sbase);
1371 if (seg->pages_base != mseg_start &&
1372 seg->pages_base > sbase) {
1373 pgcnt_t skip_pgs;
1374
1375 /*
1376 * Skip the page_t area of a
1377 * dynamic memseg.
1378 */
1379 skip_pgs = seg->pages_base - sbase;
1380 if (snpgs <= skip_pgs) {
1381 sbase += snpgs;
1382 snpgs = 0;
1383 continue;
1384 }
1385 snpgs -= skip_pgs;
1386 sbase += skip_pgs;
1387 }
1388 ASSERT(snpgs != 0);
1389 ASSERT(seg->pages_base <= sbase);
1390 /*
1391 * The individual pages can now be checked.
1392 */
1393 for (pp = seg->pages +
1394 (sbase - seg->pages_base);
1395 snpgs != 0 && pp < seg->epages; pp++) {
1396 mqp->managed++;
1397 if (PP_ISNORELOC(pp)) {
1398 mqp->nonrelocatable++;
1399 if (!done_first_nonreloc) {
1400 mqp->
1401 first_nonrelocatable
1402 = sbase;
1403 done_first_nonreloc = 1;
1404 }
1405 mqp->last_nonrelocatable =
1406 sbase;
1407 }
1408 sbase++;
1409 snpgs--;
1410 }
1411 }
1412 }
1413 }
1414
1415 free_delspans(mdsp_new);
1416
1417 return (KPHYSM_OK);
1418 }
1419
1420 /*
1421 * This release function can be called at any stage as follows:
1422 * _gethandle only called
1423 * _span(s) only called
1424 * _start called but failed
1425 * delete thread exited
1426 */
1427 int
1428 kphysm_del_release(memhandle_t handle)
1429 {
1430 struct mem_handle *mhp;
1431
1432 mhp = kphysm_lookup_mem_handle(handle);
1433 if (mhp == NULL) {
1434 return (KPHYSM_EHANDLE);
1435 }
1436 switch (mhp->mh_state) {
1437 case MHND_STARTING:
1438 case MHND_RUNNING:
1439 mutex_exit(&mhp->mh_mutex);
1440 return (KPHYSM_ENOTFINISHED);
1441 case MHND_FREE:
1442 ASSERT(mhp->mh_state != MHND_FREE);
1443 mutex_exit(&mhp->mh_mutex);
1444 return (KPHYSM_EHANDLE);
1445 case MHND_INIT:
1446 break;
1447 case MHND_DONE:
1448 break;
1449 case MHND_RELEASE:
1450 mutex_exit(&mhp->mh_mutex);
1451 return (KPHYSM_ESEQUENCE);
1452 default:
1453 #ifdef DEBUG
1454 cmn_err(CE_WARN, "kphysm_del_release(0x%p) state corrupt %d",
1455 (void *)mhp, mhp->mh_state);
1456 #endif /* DEBUG */
1457 mutex_exit(&mhp->mh_mutex);
1458 return (KPHYSM_EHANDLE);
1459 }
1460 /*
1461 * Set state so that we can wait if necessary.
1462 * Also this means that we have read/write access to all
1463 * fields except mh_exthandle and mh_state.
1464 */
1465 mhp->mh_state = MHND_RELEASE;
1466 /*
1467 * The mem_handle cannot be de-allocated by any other operation
1468 * now, so no need to hold mh_mutex.
1469 */
1470 mutex_exit(&mhp->mh_mutex);
1471
1472 delspan_remove(&mhp->mh_transit, 0, 0);
1473 mhp->mh_phys_pages = 0;
1474 mhp->mh_vm_pages = 0;
1475 mhp->mh_hold_todo = 0;
1476 mhp->mh_delete_complete = NULL;
1477 mhp->mh_delete_complete_arg = NULL;
1478 mhp->mh_cancel = 0;
1479
1480 mutex_enter(&mhp->mh_mutex);
1481 ASSERT(mhp->mh_state == MHND_RELEASE);
1482 mhp->mh_state = MHND_FREE;
1483
1484 kphysm_free_mem_handle(mhp);
1485
1486 return (KPHYSM_OK);
1487 }
1488
1489 /*
1490 * This cancel function can only be called with the thread running.
1491 */
1492 int
1493 kphysm_del_cancel(memhandle_t handle)
1494 {
1495 struct mem_handle *mhp;
1496
1497 mhp = kphysm_lookup_mem_handle(handle);
1498 if (mhp == NULL) {
1499 return (KPHYSM_EHANDLE);
1500 }
1501 if (mhp->mh_state != MHND_STARTING && mhp->mh_state != MHND_RUNNING) {
1502 mutex_exit(&mhp->mh_mutex);
1503 return (KPHYSM_ENOTRUNNING);
1504 }
1505 /*
1506 * Set the cancel flag and wake the delete thread up.
1507 * The thread may be waiting on I/O, so the effect of the cancel
1508 * may be delayed.
1509 */
1510 if (mhp->mh_cancel == 0) {
1511 mhp->mh_cancel = KPHYSM_ECANCELLED;
1512 cv_signal(&mhp->mh_cv);
1513 }
1514 mutex_exit(&mhp->mh_mutex);
1515 return (KPHYSM_OK);
1516 }
1517
1518 int
1519 kphysm_del_status(
1520 memhandle_t handle,
1521 memdelstat_t *mdstp)
1522 {
1523 struct mem_handle *mhp;
1524
1525 mhp = kphysm_lookup_mem_handle(handle);
1526 if (mhp == NULL) {
1527 return (KPHYSM_EHANDLE);
1528 }
1529 /*
1530 * Calling kphysm_del_status() is allowed before the delete
1531 * is started to allow for status display.
1532 */
1533 if (mhp->mh_state != MHND_INIT && mhp->mh_state != MHND_STARTING &&
1534 mhp->mh_state != MHND_RUNNING) {
1535 mutex_exit(&mhp->mh_mutex);
1536 return (KPHYSM_ENOTRUNNING);
1537 }
1538 mdstp->phys_pages = mhp->mh_phys_pages;
1539 mdstp->managed = mhp->mh_vm_pages;
1540 mdstp->collected = mhp->mh_vm_pages - mhp->mh_hold_todo;
1541 mutex_exit(&mhp->mh_mutex);
1542 return (KPHYSM_OK);
1543 }
1544
1545 static int mem_delete_additional_pages = 100;
1546
1547 static int
1548 can_remove_pgs(pgcnt_t npgs)
1549 {
1550 /*
1551 * If all pageable pages were paged out, freemem would
1552 * equal availrmem. There is a minimum requirement for
1553 * availrmem.
1554 */
1555 if ((availrmem - (tune.t_minarmem + mem_delete_additional_pages))
1556 < npgs)
1557 return (0);
1558 /* TODO: check swap space, etc. */
1559 return (1);
1560 }
1561
1562 static int
1563 get_availrmem(pgcnt_t npgs)
1564 {
1565 int ret;
1566
1567 mutex_enter(&freemem_lock);
1568 ret = can_remove_pgs(npgs);
1569 if (ret != 0)
1570 availrmem -= npgs;
1571 mutex_exit(&freemem_lock);
1572 return (ret);
1573 }
1574
1575 static void
1576 put_availrmem(pgcnt_t npgs)
1577 {
1578 mutex_enter(&freemem_lock);
1579 availrmem += npgs;
1580 mutex_exit(&freemem_lock);
1581 }
1582
1583 #define FREEMEM_INCR 100
1584 static pgcnt_t freemem_incr = FREEMEM_INCR;
1585 #define DEL_FREE_WAIT_FRAC 4
1586 #define DEL_FREE_WAIT_TICKS ((hz+DEL_FREE_WAIT_FRAC-1)/DEL_FREE_WAIT_FRAC)
1587
1588 #define DEL_BUSY_WAIT_FRAC 20
1589 #define DEL_BUSY_WAIT_TICKS ((hz+DEL_BUSY_WAIT_FRAC-1)/DEL_BUSY_WAIT_FRAC)
1590
1591 static void kphysm_del_cleanup(struct mem_handle *);
1592
1593 static void page_delete_collect(page_t *, struct mem_handle *);
1594
1595 static pgcnt_t
1596 delthr_get_freemem(struct mem_handle *mhp)
1597 {
1598 pgcnt_t free_get;
1599 int ret;
1600
1601 ASSERT(MUTEX_HELD(&mhp->mh_mutex));
1602
1603 MDSTAT_INCR(mhp, need_free);
1604 /*
1605 * Get up to freemem_incr pages.
1606 */
1607 free_get = freemem_incr;
1608 if (free_get > mhp->mh_hold_todo)
1609 free_get = mhp->mh_hold_todo;
1610 /*
1611 * Take free_get pages away from freemem,
1612 * waiting if necessary.
1613 */
1614
1615 while (!mhp->mh_cancel) {
1616 mutex_exit(&mhp->mh_mutex);
1617 MDSTAT_INCR(mhp, free_loop);
1618 /*
1619 * Duplicate test from page_create_throttle()
1620 * but don't override with !PG_WAIT.
1621 */
1622 if (freemem < (free_get + throttlefree)) {
1623 MDSTAT_INCR(mhp, free_low);
1624 ret = 0;
1625 } else {
1626 ret = page_create_wait(free_get, 0);
1627 if (ret == 0) {
1628 /* EMPTY */
1629 MDSTAT_INCR(mhp, free_failed);
1630 }
1631 }
1632 if (ret != 0) {
1633 mutex_enter(&mhp->mh_mutex);
1634 return (free_get);
1635 }
1636
1637 /*
1638 * Put pressure on pageout.
1639 */
1640 page_needfree(free_get);
1641 cv_signal(&proc_pageout->p_cv);
1642
1643 mutex_enter(&mhp->mh_mutex);
1644 (void) cv_reltimedwait(&mhp->mh_cv, &mhp->mh_mutex,
1645 DEL_FREE_WAIT_TICKS, TR_CLOCK_TICK);
1646 mutex_exit(&mhp->mh_mutex);
1647 page_needfree(-(spgcnt_t)free_get);
1648
1649 mutex_enter(&mhp->mh_mutex);
1650 }
1651 return (0);
1652 }
1653
1654 #define DR_AIO_CLEANUP_DELAY 25000 /* 0.025secs, in usec */
1655 #define DR_AIO_CLEANUP_MAXLOOPS_NODELAY 100
1656 /*
1657 * This function is run as a helper thread for delete_memory_thread.
1658 * It is needed in order to force kaio cleanup, so that pages used in kaio
1659 * will be unlocked and subsequently relocated by delete_memory_thread.
1660 * The address of the delete_memory_threads's mem_handle is passed in to
1661 * this thread function, and is used to set the mh_aio_cleanup_done member
1662 * prior to calling thread_exit().
1663 */
1664 static void
1665 dr_aio_cleanup_thread(caddr_t amhp)
1666 {
1667 proc_t *procp;
1668 int (*aio_cleanup_dr_delete_memory)(proc_t *);
1669 int cleaned;
1670 int n = 0;
1671 struct mem_handle *mhp;
1672 volatile uint_t *pcancel;
1673
1674 mhp = (struct mem_handle *)amhp;
1675 ASSERT(mhp != NULL);
1676 pcancel = &mhp->mh_dr_aio_cleanup_cancel;
1677 if (modload("sys", "kaio") == -1) {
1678 mhp->mh_aio_cleanup_done = 1;
1679 cmn_err(CE_WARN, "dr_aio_cleanup_thread: cannot load kaio");
1680 thread_exit();
1681 }
1682 aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
1683 modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
1684 if (aio_cleanup_dr_delete_memory == NULL) {
1685 mhp->mh_aio_cleanup_done = 1;
1686 cmn_err(CE_WARN,
1687 "aio_cleanup_dr_delete_memory not found in kaio");
1688 thread_exit();
1689 }
1690 do {
1691 cleaned = 0;
1692 mutex_enter(&pidlock);
1693 for (procp = practive; (*pcancel == 0) && (procp != NULL);
1694 procp = procp->p_next) {
1695 mutex_enter(&procp->p_lock);
1696 if (procp->p_aio != NULL) {
1697 /* cleanup proc's outstanding kaio */
1698 cleaned +=
1699 (*aio_cleanup_dr_delete_memory)(procp);
1700 }
1701 mutex_exit(&procp->p_lock);
1702 }
1703 mutex_exit(&pidlock);
1704 if ((*pcancel == 0) &&
1705 (!cleaned || (++n == DR_AIO_CLEANUP_MAXLOOPS_NODELAY))) {
1706 /* delay a bit before retrying all procs again */
1707 delay(drv_usectohz(DR_AIO_CLEANUP_DELAY));
1708 n = 0;
1709 }
1710 } while (*pcancel == 0);
1711 mhp->mh_aio_cleanup_done = 1;
1712 thread_exit();
1713 }
1714
1715 static void
1716 delete_memory_thread(caddr_t amhp)
1717 {
1718 struct mem_handle *mhp;
1719 struct memdelspan *mdsp;
1720 callb_cpr_t cprinfo;
1721 page_t *pp_targ;
1722 spgcnt_t freemem_left;
1723 void (*del_complete_funcp)(void *, int error);
1724 void *del_complete_arg;
1725 int comp_code;
1726 int ret;
1727 int first_scan;
1728 uint_t szc;
1729 #ifdef MEM_DEL_STATS
1730 uint64_t start_total, ntick_total;
1731 uint64_t start_pgrp, ntick_pgrp;
1732 #endif /* MEM_DEL_STATS */
1733
1734 mhp = (struct mem_handle *)amhp;
1735
1736 #ifdef MEM_DEL_STATS
1737 start_total = ddi_get_lbolt();
1738 #endif /* MEM_DEL_STATS */
1739
1740 CALLB_CPR_INIT(&cprinfo, &mhp->mh_mutex,
1741 callb_generic_cpr, "memdel");
1742
1743 mutex_enter(&mhp->mh_mutex);
1744 ASSERT(mhp->mh_state == MHND_STARTING);
1745
1746 mhp->mh_state = MHND_RUNNING;
1747 mhp->mh_thread_id = curthread;
1748
1749 mhp->mh_hold_todo = mhp->mh_vm_pages;
1750 mutex_exit(&mhp->mh_mutex);
1751
1752 /* Allocate the remap pages now, if necessary. */
1753 memseg_remap_init();
1754
1755 /*
1756 * Subtract from availrmem now if possible as availrmem
1757 * may not be available by the end of the delete.
1758 */
1759 if (!get_availrmem(mhp->mh_vm_pages)) {
1760 comp_code = KPHYSM_ENOTVIABLE;
1761 mutex_enter(&mhp->mh_mutex);
1762 goto early_exit;
1763 }
1764
1765 ret = kphysm_setup_pre_del(mhp->mh_vm_pages);
1766
1767 mutex_enter(&mhp->mh_mutex);
1768
1769 if (ret != 0) {
1770 mhp->mh_cancel = KPHYSM_EREFUSED;
1771 goto refused;
1772 }
1773
1774 transit_list_collect(mhp, 1);
1775
1776 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
1777 mdsp = mdsp->mds_next) {
1778 ASSERT(mdsp->mds_bitmap == NULL);
1779 mdsp->mds_bitmap = kmem_zalloc(MDS_BITMAPBYTES(mdsp), KM_SLEEP);
1780 mdsp->mds_bitmap_retired = kmem_zalloc(MDS_BITMAPBYTES(mdsp),
1781 KM_SLEEP);
1782 }
1783
1784 first_scan = 1;
1785 freemem_left = 0;
1786 /*
1787 * Start dr_aio_cleanup_thread, which periodically iterates
1788 * through the process list and invokes aio cleanup. This
1789 * is needed in order to avoid a deadly embrace between the
1790 * delete_memory_thread (waiting on writer lock for page, with the
1791 * exclusive-wanted bit set), kaio read request threads (waiting for a
1792 * reader lock on the same page that is wanted by the
1793 * delete_memory_thread), and threads waiting for kaio completion
1794 * (blocked on spt_amp->lock).
1795 */
1796 mhp->mh_dr_aio_cleanup_cancel = 0;
1797 mhp->mh_aio_cleanup_done = 0;
1798 (void) thread_create(NULL, 0, dr_aio_cleanup_thread,
1799 (caddr_t)mhp, 0, &p0, TS_RUN, maxclsyspri - 1);
1800 while ((mhp->mh_hold_todo != 0) && (mhp->mh_cancel == 0)) {
1801 pgcnt_t collected;
1802
1803 MDSTAT_INCR(mhp, nloop);
1804 collected = 0;
1805 for (mdsp = mhp->mh_transit.trl_spans; (mdsp != NULL) &&
1806 (mhp->mh_cancel == 0); mdsp = mdsp->mds_next) {
1807 pfn_t pfn, p_end;
1808
1809 p_end = mdsp->mds_base + mdsp->mds_npgs;
1810 for (pfn = mdsp->mds_base; (pfn < p_end) &&
1811 (mhp->mh_cancel == 0); pfn++) {
1812 page_t *pp, *tpp, *tpp_targ;
1813 pgcnt_t bit;
1814 struct vnode *vp;
1815 u_offset_t offset;
1816 int mod, result;
1817 spgcnt_t pgcnt;
1818
1819 bit = pfn - mdsp->mds_base;
1820 if ((mdsp->mds_bitmap[bit / NBPBMW] &
1821 (1 << (bit % NBPBMW))) != 0) {
1822 MDSTAT_INCR(mhp, already_done);
1823 continue;
1824 }
1825 if (freemem_left == 0) {
1826 freemem_left += delthr_get_freemem(mhp);
1827 if (freemem_left == 0)
1828 break;
1829 }
1830
1831 /*
1832 * Release mh_mutex - some of this
1833 * stuff takes some time (eg PUTPAGE).
1834 */
1835
1836 mutex_exit(&mhp->mh_mutex);
1837 MDSTAT_INCR(mhp, ncheck);
1838
1839 pp = page_numtopp_nolock(pfn);
1840 if (pp == NULL) {
1841 /*
1842 * Not covered by a page_t - will
1843 * be dealt with elsewhere.
1844 */
1845 MDSTAT_INCR(mhp, nopaget);
1846 mutex_enter(&mhp->mh_mutex);
1847 mdsp->mds_bitmap[bit / NBPBMW] |=
1848 (1 << (bit % NBPBMW));
1849 continue;
1850 }
1851
1852 if (!page_try_reclaim_lock(pp, SE_EXCL,
1853 SE_EXCL_WANTED | SE_RETIRED)) {
1854 /*
1855 * Page in use elsewhere. Skip it.
1856 */
1857 MDSTAT_INCR(mhp, lockfail);
1858 mutex_enter(&mhp->mh_mutex);
1859 continue;
1860 }
1861 /*
1862 * See if the cage expanded into the delete.
1863 * This can happen as we have to allow the
1864 * cage to expand.
1865 */
1866 if (PP_ISNORELOC(pp)) {
1867 page_unlock(pp);
1868 mutex_enter(&mhp->mh_mutex);
1869 mhp->mh_cancel = KPHYSM_ENONRELOC;
1870 break;
1871 }
1872 if (PP_RETIRED(pp)) {
1873 /*
1874 * Page has been retired and is
1875 * not part of the cage so we
1876 * can now do the accounting for
1877 * it.
1878 */
1879 MDSTAT_INCR(mhp, retired);
1880 mutex_enter(&mhp->mh_mutex);
1881 mdsp->mds_bitmap[bit / NBPBMW]
1882 |= (1 << (bit % NBPBMW));
1883 mdsp->mds_bitmap_retired[bit /
1884 NBPBMW] |=
1885 (1 << (bit % NBPBMW));
1886 mhp->mh_hold_todo--;
1887 continue;
1888 }
1889 ASSERT(freemem_left != 0);
1890 if (PP_ISFREE(pp)) {
1891 /*
1892 * Like page_reclaim() only 'freemem'
1893 * processing is already done.
1894 */
1895 MDSTAT_INCR(mhp, nfree);
1896 free_page_collect:
1897 if (PP_ISAGED(pp)) {
1898 page_list_sub(pp,
1899 PG_FREE_LIST);
1900 } else {
1901 page_list_sub(pp,
1902 PG_CACHE_LIST);
1903 }
1904 PP_CLRFREE(pp);
1905 PP_CLRAGED(pp);
1906 collected++;
1907 mutex_enter(&mhp->mh_mutex);
1908 page_delete_collect(pp, mhp);
1909 mdsp->mds_bitmap[bit / NBPBMW] |=
1910 (1 << (bit % NBPBMW));
1911 freemem_left--;
1912 continue;
1913 }
1914 ASSERT(pp->p_vnode != NULL);
1915 if (first_scan) {
1916 MDSTAT_INCR(mhp, first_notfree);
1917 page_unlock(pp);
1918 mutex_enter(&mhp->mh_mutex);
1919 continue;
1920 }
1921 /*
1922 * Keep stats on pages encountered that
1923 * are marked for retirement.
1924 */
1925 if (PP_TOXIC(pp)) {
1926 MDSTAT_INCR(mhp, toxic);
1927 } else if (PP_PR_REQ(pp)) {
1928 MDSTAT_INCR(mhp, failing);
1929 }
1930 /*
1931 * In certain cases below, special exceptions
1932 * are made for pages that are toxic. This
1933 * is because the current meaning of toxic
1934 * is that an uncorrectable error has been
1935 * previously associated with the page.
1936 */
1937 if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
1938 if (!PP_TOXIC(pp)) {
1939 /*
1940 * Must relocate locked in
1941 * memory pages.
1942 */
1943 #ifdef MEM_DEL_STATS
1944 start_pgrp = ddi_get_lbolt();
1945 #endif /* MEM_DEL_STATS */
1946 /*
1947 * Lock all constituent pages
1948 * of a large page to ensure
1949 * that p_szc won't change.
1950 */
1951 if (!group_page_trylock(pp,
1952 SE_EXCL)) {
1953 MDSTAT_INCR(mhp,
1954 gptllckfail);
1955 page_unlock(pp);
1956 mutex_enter(
1957 &mhp->mh_mutex);
1958 continue;
1959 }
1960 MDSTAT_INCR(mhp, npplocked);
1961 pp_targ =
1962 page_get_replacement_page(
1963 pp, NULL, 0);
1964 if (pp_targ != NULL) {
1965 #ifdef MEM_DEL_STATS
1966 ntick_pgrp =
1967 (uint64_t)
1968 ddi_get_lbolt() -
1969 start_pgrp;
1970 #endif /* MEM_DEL_STATS */
1971 MDSTAT_PGRP(mhp,
1972 ntick_pgrp);
1973 MDSTAT_INCR(mhp,
1974 nlockreloc);
1975 goto reloc;
1976 }
1977 group_page_unlock(pp);
1978 page_unlock(pp);
1979 #ifdef MEM_DEL_STATS
1980 ntick_pgrp =
1981 (uint64_t)ddi_get_lbolt() -
1982 start_pgrp;
1983 #endif /* MEM_DEL_STATS */
1984 MDSTAT_PGRP(mhp, ntick_pgrp);
1985 MDSTAT_INCR(mhp, nnorepl);
1986 mutex_enter(&mhp->mh_mutex);
1987 continue;
1988 } else {
1989 /*
1990 * Cannot do anything about
1991 * this page because it is
1992 * toxic.
1993 */
1994 MDSTAT_INCR(mhp, npplkdtoxic);
1995 page_unlock(pp);
1996 mutex_enter(&mhp->mh_mutex);
1997 continue;
1998 }
1999 }
2000 /*
2001 * Unload the mappings and check if mod bit
2002 * is set.
2003 */
2004 ASSERT(!PP_ISKAS(pp));
2005 (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
2006 mod = hat_ismod(pp);
2007
2008 #ifdef MEM_DEL_STATS
2009 start_pgrp = ddi_get_lbolt();
2010 #endif /* MEM_DEL_STATS */
2011 if (mod && !PP_TOXIC(pp)) {
2012 /*
2013 * Lock all constituent pages
2014 * of a large page to ensure
2015 * that p_szc won't change.
2016 */
2017 if (!group_page_trylock(pp, SE_EXCL)) {
2018 MDSTAT_INCR(mhp, gptlmodfail);
2019 page_unlock(pp);
2020 mutex_enter(&mhp->mh_mutex);
2021 continue;
2022 }
2023 pp_targ = page_get_replacement_page(pp,
2024 NULL, 0);
2025 if (pp_targ != NULL) {
2026 MDSTAT_INCR(mhp, nmodreloc);
2027 #ifdef MEM_DEL_STATS
2028 ntick_pgrp =
2029 (uint64_t)ddi_get_lbolt() -
2030 start_pgrp;
2031 #endif /* MEM_DEL_STATS */
2032 MDSTAT_PGRP(mhp, ntick_pgrp);
2033 goto reloc;
2034 }
2035 group_page_unlock(pp);
2036 }
2037
2038 if (!page_try_demote_pages(pp)) {
2039 MDSTAT_INCR(mhp, demotefail);
2040 page_unlock(pp);
2041 #ifdef MEM_DEL_STATS
2042 ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2043 start_pgrp;
2044 #endif /* MEM_DEL_STATS */
2045 MDSTAT_PGRP(mhp, ntick_pgrp);
2046 mutex_enter(&mhp->mh_mutex);
2047 continue;
2048 }
2049
2050 /*
2051 * Regular 'page-out'.
2052 */
2053 if (!mod) {
2054 MDSTAT_INCR(mhp, ndestroy);
2055 page_destroy(pp, 1);
2056 /*
2057 * page_destroy was called with
2058 * dontfree. As long as p_lckcnt
2059 * and p_cowcnt are both zero, the
2060 * only additional action of
2061 * page_destroy with !dontfree is to
2062 * call page_free, so we can collect
2063 * the page here.
2064 */
2065 collected++;
2066 #ifdef MEM_DEL_STATS
2067 ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2068 start_pgrp;
2069 #endif /* MEM_DEL_STATS */
2070 MDSTAT_PGRP(mhp, ntick_pgrp);
2071 mutex_enter(&mhp->mh_mutex);
2072 page_delete_collect(pp, mhp);
2073 mdsp->mds_bitmap[bit / NBPBMW] |=
2074 (1 << (bit % NBPBMW));
2075 continue;
2076 }
2077 /*
2078 * The page is toxic and the mod bit is
2079 * set, we cannot do anything here to deal
2080 * with it.
2081 */
2082 if (PP_TOXIC(pp)) {
2083 page_unlock(pp);
2084 #ifdef MEM_DEL_STATS
2085 ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2086 start_pgrp;
2087 #endif /* MEM_DEL_STATS */
2088 MDSTAT_PGRP(mhp, ntick_pgrp);
2089 MDSTAT_INCR(mhp, modtoxic);
2090 mutex_enter(&mhp->mh_mutex);
2091 continue;
2092 }
2093 MDSTAT_INCR(mhp, nputpage);
2094 vp = pp->p_vnode;
2095 offset = pp->p_offset;
2096 VN_HOLD(vp);
2097 page_unlock(pp);
2098 (void) VOP_PUTPAGE(vp, offset, PAGESIZE,
2099 B_INVAL|B_FORCE, kcred, NULL);
2100 VN_RELE(vp);
2101 #ifdef MEM_DEL_STATS
2102 ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2103 start_pgrp;
2104 #endif /* MEM_DEL_STATS */
2105 MDSTAT_PGRP(mhp, ntick_pgrp);
2106 /*
2107 * Try to get the page back immediately
2108 * so that it can be collected.
2109 */
2110 pp = page_numtopp_nolock(pfn);
2111 if (pp == NULL) {
2112 MDSTAT_INCR(mhp, nnoreclaim);
2113 /*
2114 * This should not happen as this
2115 * thread is deleting the page.
2116 * If this code is generalized, this
2117 * becomes a reality.
2118 */
2119 #ifdef DEBUG
2120 cmn_err(CE_WARN,
2121 "delete_memory_thread(0x%p) "
2122 "pfn 0x%lx has no page_t",
2123 (void *)mhp, pfn);
2124 #endif /* DEBUG */
2125 mutex_enter(&mhp->mh_mutex);
2126 continue;
2127 }
2128 if (page_try_reclaim_lock(pp, SE_EXCL,
2129 SE_EXCL_WANTED | SE_RETIRED)) {
2130 if (PP_ISFREE(pp)) {
2131 goto free_page_collect;
2132 }
2133 page_unlock(pp);
2134 }
2135 MDSTAT_INCR(mhp, nnoreclaim);
2136 mutex_enter(&mhp->mh_mutex);
2137 continue;
2138
2139 reloc:
2140 /*
2141 * Got some freemem and a target
2142 * page, so move the data to avoid
2143 * I/O and lock problems.
2144 */
2145 ASSERT(!page_iolock_assert(pp));
2146 MDSTAT_INCR(mhp, nreloc);
2147 /*
2148 * page_relocate() will return pgcnt: the
2149 * number of consecutive pages relocated.
2150 * If it is successful, pp will be a
2151 * linked list of the page structs that
2152 * were relocated. If page_relocate() is
2153 * unsuccessful, pp will be unmodified.
2154 */
2155 #ifdef MEM_DEL_STATS
2156 start_pgrp = ddi_get_lbolt();
2157 #endif /* MEM_DEL_STATS */
2158 result = page_relocate(&pp, &pp_targ, 0, 0,
2159 &pgcnt, NULL);
2160 #ifdef MEM_DEL_STATS
2161 ntick_pgrp = (uint64_t)ddi_get_lbolt() -
2162 start_pgrp;
2163 #endif /* MEM_DEL_STATS */
2164 MDSTAT_PGRP(mhp, ntick_pgrp);
2165 if (result != 0) {
2166 MDSTAT_INCR(mhp, nrelocfail);
2167 /*
2168 * We did not succeed. We need
2169 * to give the pp_targ pages back.
2170 * page_free(pp_targ, 1) without
2171 * the freemem accounting.
2172 */
2173 group_page_unlock(pp);
2174 page_free_replacement_page(pp_targ);
2175 page_unlock(pp);
2176 mutex_enter(&mhp->mh_mutex);
2177 continue;
2178 }
2179
2180 /*
2181 * We will then collect pgcnt pages.
2182 */
2183 ASSERT(pgcnt > 0);
2184 mutex_enter(&mhp->mh_mutex);
2185 /*
2186 * We need to make sure freemem_left is
2187 * large enough.
2188 */
2189 while ((freemem_left < pgcnt) &&
2190 (!mhp->mh_cancel)) {
2191 freemem_left +=
2192 delthr_get_freemem(mhp);
2193 }
2194
2195 /*
2196 * Do not proceed if mh_cancel is set.
2197 */
2198 if (mhp->mh_cancel) {
2199 while (pp_targ != NULL) {
2200 /*
2201 * Unlink and unlock each page.
2202 */
2203 tpp_targ = pp_targ;
2204 page_sub(&pp_targ, tpp_targ);
2205 page_unlock(tpp_targ);
2206 }
2207 /*
2208 * We need to give the pp pages back.
2209 * page_free(pp, 1) without the
2210 * freemem accounting.
2211 */
2212 page_free_replacement_page(pp);
2213 break;
2214 }
2215
2216 /* Now remove pgcnt from freemem_left */
2217 freemem_left -= pgcnt;
2218 ASSERT(freemem_left >= 0);
2219 szc = pp->p_szc;
2220 while (pp != NULL) {
2221 /*
2222 * pp and pp_targ were passed back as
2223 * a linked list of pages.
2224 * Unlink and unlock each page.
2225 */
2226 tpp_targ = pp_targ;
2227 page_sub(&pp_targ, tpp_targ);
2228 page_unlock(tpp_targ);
2229 /*
2230 * The original page is now free
2231 * so remove it from the linked
2232 * list and collect it.
2233 */
2234 tpp = pp;
2235 page_sub(&pp, tpp);
2236 pfn = page_pptonum(tpp);
2237 collected++;
2238 ASSERT(PAGE_EXCL(tpp));
2239 ASSERT(tpp->p_vnode == NULL);
2240 ASSERT(!hat_page_is_mapped(tpp));
2241 ASSERT(tpp->p_szc == szc);
2242 tpp->p_szc = 0;
2243 page_delete_collect(tpp, mhp);
2244 bit = pfn - mdsp->mds_base;
2245 mdsp->mds_bitmap[bit / NBPBMW] |=
2246 (1 << (bit % NBPBMW));
2247 }
2248 ASSERT(pp_targ == NULL);
2249 }
2250 }
2251 first_scan = 0;
2252 if ((mhp->mh_cancel == 0) && (mhp->mh_hold_todo != 0) &&
2253 (collected == 0)) {
2254 /*
2255 * This code is needed as we cannot wait
2256 * for a page to be locked OR the delete to
2257 * be cancelled. Also, we must delay so
2258 * that other threads get a chance to run
2259 * on our cpu, otherwise page locks may be
2260 * held indefinitely by those threads.
2261 */
2262 MDSTAT_INCR(mhp, ndelay);
2263 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2264 (void) cv_reltimedwait(&mhp->mh_cv, &mhp->mh_mutex,
2265 DEL_BUSY_WAIT_TICKS, TR_CLOCK_TICK);
2266 CALLB_CPR_SAFE_END(&cprinfo, &mhp->mh_mutex);
2267 }
2268 }
2269 /* stop the dr aio cleanup thread */
2270 mhp->mh_dr_aio_cleanup_cancel = 1;
2271 transit_list_collect(mhp, 0);
2272 if (freemem_left != 0) {
2273 /* Return any surplus. */
2274 page_create_putback(freemem_left);
2275 freemem_left = 0;
2276 }
2277 #ifdef MEM_DEL_STATS
2278 ntick_total = (uint64_t)ddi_get_lbolt() - start_total;
2279 #endif /* MEM_DEL_STATS */
2280 MDSTAT_TOTAL(mhp, ntick_total);
2281 MDSTAT_PRINT(mhp);
2282
2283 /*
2284 * If the memory delete was cancelled, exclusive-wanted bits must
2285 * be cleared. If there are retired pages being deleted, they need
2286 * to be unretired.
2287 */
2288 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2289 mdsp = mdsp->mds_next) {
2290 pfn_t pfn, p_end;
2291
2292 p_end = mdsp->mds_base + mdsp->mds_npgs;
2293 for (pfn = mdsp->mds_base; pfn < p_end; pfn++) {
2294 page_t *pp;
2295 pgcnt_t bit;
2296
2297 bit = pfn - mdsp->mds_base;
2298 if (mhp->mh_cancel) {
2299 pp = page_numtopp_nolock(pfn);
2300 if (pp != NULL) {
2301 if ((mdsp->mds_bitmap[bit / NBPBMW] &
2302 (1 << (bit % NBPBMW))) == 0) {
2303 page_lock_clr_exclwanted(pp);
2304 }
2305 }
2306 } else {
2307 pp = NULL;
2308 }
2309 if ((mdsp->mds_bitmap_retired[bit / NBPBMW] &
2310 (1 << (bit % NBPBMW))) != 0) {
2311 /* do we already have pp? */
2312 if (pp == NULL) {
2313 pp = page_numtopp_nolock(pfn);
2314 }
2315 ASSERT(pp != NULL);
2316 ASSERT(PP_RETIRED(pp));
2317 if (mhp->mh_cancel != 0) {
2318 page_unlock(pp);
2319 /*
2320 * To satisfy ASSERT below in
2321 * cancel code.
2322 */
2323 mhp->mh_hold_todo++;
2324 } else {
2325 (void) page_unretire_pp(pp,
2326 PR_UNR_CLEAN);
2327 }
2328 }
2329 }
2330 }
2331 /*
2332 * Free retired page bitmap and collected page bitmap
2333 */
2334 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2335 mdsp = mdsp->mds_next) {
2336 ASSERT(mdsp->mds_bitmap_retired != NULL);
2337 kmem_free(mdsp->mds_bitmap_retired, MDS_BITMAPBYTES(mdsp));
2338 mdsp->mds_bitmap_retired = NULL; /* Paranoia. */
2339 ASSERT(mdsp->mds_bitmap != NULL);
2340 kmem_free(mdsp->mds_bitmap, MDS_BITMAPBYTES(mdsp));
2341 mdsp->mds_bitmap = NULL; /* Paranoia. */
2342 }
2343
2344 /* wait for our dr aio cancel thread to exit */
2345 while (!(mhp->mh_aio_cleanup_done)) {
2346 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2347 delay(drv_usectohz(DR_AIO_CLEANUP_DELAY));
2348 CALLB_CPR_SAFE_END(&cprinfo, &mhp->mh_mutex);
2349 }
2350 refused:
2351 if (mhp->mh_cancel != 0) {
2352 page_t *pp;
2353
2354 comp_code = mhp->mh_cancel;
2355 /*
2356 * Go through list of deleted pages (mh_deleted) freeing
2357 * them.
2358 */
2359 while ((pp = mhp->mh_deleted) != NULL) {
2360 mhp->mh_deleted = pp->p_next;
2361 mhp->mh_hold_todo++;
2362 mutex_exit(&mhp->mh_mutex);
2363 /* Restore p_next. */
2364 pp->p_next = pp->p_prev;
2365 if (PP_ISFREE(pp)) {
2366 cmn_err(CE_PANIC,
2367 "page %p is free",
2368 (void *)pp);
2369 }
2370 page_free(pp, 1);
2371 mutex_enter(&mhp->mh_mutex);
2372 }
2373 ASSERT(mhp->mh_hold_todo == mhp->mh_vm_pages);
2374
2375 mutex_exit(&mhp->mh_mutex);
2376 put_availrmem(mhp->mh_vm_pages);
2377 mutex_enter(&mhp->mh_mutex);
2378
2379 goto t_exit;
2380 }
2381
2382 /*
2383 * All the pages are no longer in use and are exclusively locked.
2384 */
2385
2386 mhp->mh_deleted = NULL;
2387
2388 kphysm_del_cleanup(mhp);
2389
2390 /*
2391 * mem_node_del_range needs to be after kphysm_del_cleanup so
2392 * that the mem_node_config[] will remain intact for the cleanup.
2393 */
2394 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2395 mdsp = mdsp->mds_next) {
2396 mem_node_del_range(mdsp->mds_base,
2397 mdsp->mds_base + mdsp->mds_npgs - 1);
2398 }
2399 /* cleanup the page counters */
2400 page_ctrs_cleanup();
2401
2402 comp_code = KPHYSM_OK;
2403
2404 t_exit:
2405 mutex_exit(&mhp->mh_mutex);
2406 kphysm_setup_post_del(mhp->mh_vm_pages,
2407 (comp_code == KPHYSM_OK) ? 0 : 1);
2408 mutex_enter(&mhp->mh_mutex);
2409
2410 early_exit:
2411 /* mhp->mh_mutex exited by CALLB_CPR_EXIT() */
2412 mhp->mh_state = MHND_DONE;
2413 del_complete_funcp = mhp->mh_delete_complete;
2414 del_complete_arg = mhp->mh_delete_complete_arg;
2415 CALLB_CPR_EXIT(&cprinfo);
2416 (*del_complete_funcp)(del_complete_arg, comp_code);
2417 thread_exit();
2418 /*NOTREACHED*/
2419 }
2420
2421 /*
2422 * Start the delete of the memory from the system.
2423 */
2424 int
2425 kphysm_del_start(
2426 memhandle_t handle,
2427 void (*complete)(void *, int),
2428 void *complete_arg)
2429 {
2430 struct mem_handle *mhp;
2431
2432 mhp = kphysm_lookup_mem_handle(handle);
2433 if (mhp == NULL) {
2434 return (KPHYSM_EHANDLE);
2435 }
2436 switch (mhp->mh_state) {
2437 case MHND_FREE:
2438 ASSERT(mhp->mh_state != MHND_FREE);
2439 mutex_exit(&mhp->mh_mutex);
2440 return (KPHYSM_EHANDLE);
2441 case MHND_INIT:
2442 break;
2443 case MHND_STARTING:
2444 case MHND_RUNNING:
2445 mutex_exit(&mhp->mh_mutex);
2446 return (KPHYSM_ESEQUENCE);
2447 case MHND_DONE:
2448 mutex_exit(&mhp->mh_mutex);
2449 return (KPHYSM_ESEQUENCE);
2450 case MHND_RELEASE:
2451 mutex_exit(&mhp->mh_mutex);
2452 return (KPHYSM_ESEQUENCE);
2453 default:
2454 #ifdef DEBUG
2455 cmn_err(CE_WARN, "kphysm_del_start(0x%p) state corrupt %d",
2456 (void *)mhp, mhp->mh_state);
2457 #endif /* DEBUG */
2458 mutex_exit(&mhp->mh_mutex);
2459 return (KPHYSM_EHANDLE);
2460 }
2461
2462 if (mhp->mh_transit.trl_spans == NULL) {
2463 mutex_exit(&mhp->mh_mutex);
2464 return (KPHYSM_ENOWORK);
2465 }
2466
2467 ASSERT(complete != NULL);
2468 mhp->mh_delete_complete = complete;
2469 mhp->mh_delete_complete_arg = complete_arg;
2470 mhp->mh_state = MHND_STARTING;
2471 /*
2472 * Release the mutex in case thread_create sleeps.
2473 */
2474 mutex_exit(&mhp->mh_mutex);
2475
2476 /*
2477 * The "obvious" process for this thread is pageout (proc_pageout)
2478 * but this gives the thread too much power over freemem
2479 * which results in freemem starvation.
2480 */
2481 (void) thread_create(NULL, 0, delete_memory_thread, mhp, 0, &p0,
2482 TS_RUN, maxclsyspri - 1);
2483
2484 return (KPHYSM_OK);
2485 }
2486
2487 static kmutex_t pp_dummy_lock; /* Protects init. of pp_dummy. */
2488 static caddr_t pp_dummy;
2489 static pgcnt_t pp_dummy_npages;
2490 static pfn_t *pp_dummy_pfn; /* Array of dummy pfns. */
2491
2492 static void
2493 memseg_remap_init_pages(page_t *pages, page_t *epages)
2494 {
2495 page_t *pp;
2496
2497 for (pp = pages; pp < epages; pp++) {
2498 pp->p_pagenum = PFN_INVALID; /* XXXX */
2499 pp->p_offset = (u_offset_t)-1;
2500 page_iolock_init(pp);
2501 while (!page_lock(pp, SE_EXCL, (kmutex_t *)NULL, P_RECLAIM))
2502 continue;
2503 page_lock_delete(pp);
2504 }
2505 }
2506
2507 void
2508 memseg_remap_init()
2509 {
2510 mutex_enter(&pp_dummy_lock);
2511 if (pp_dummy == NULL) {
2512 uint_t dpages;
2513 int i;
2514
2515 /*
2516 * dpages starts off as the size of the structure and
2517 * ends up as the minimum number of pages that will
2518 * hold a whole number of page_t structures.
2519 */
2520 dpages = sizeof (page_t);
2521 ASSERT(dpages != 0);
2522 ASSERT(dpages <= MMU_PAGESIZE);
2523
2524 while ((dpages & 1) == 0)
2525 dpages >>= 1;
2526
2527 pp_dummy_npages = dpages;
2528 /*
2529 * Allocate pp_dummy pages directly from static_arena,
2530 * since these are whole page allocations and are
2531 * referenced by physical address. This also has the
2532 * nice fringe benefit of hiding the memory from
2533 * ::findleaks since it doesn't deal well with allocated
2534 * kernel heap memory that doesn't have any mappings.
2535 */
2536 pp_dummy = vmem_xalloc(static_arena, ptob(pp_dummy_npages),
2537 PAGESIZE, 0, 0, NULL, NULL, VM_SLEEP);
2538 bzero(pp_dummy, ptob(pp_dummy_npages));
2539 ASSERT(((uintptr_t)pp_dummy & MMU_PAGEOFFSET) == 0);
2540 pp_dummy_pfn = kmem_alloc(sizeof (*pp_dummy_pfn) *
2541 pp_dummy_npages, KM_SLEEP);
2542 for (i = 0; i < pp_dummy_npages; i++) {
2543 pp_dummy_pfn[i] = hat_getpfnum(kas.a_hat,
2544 &pp_dummy[MMU_PAGESIZE * i]);
2545 ASSERT(pp_dummy_pfn[i] != PFN_INVALID);
2546 }
2547 /*
2548 * Initialize the page_t's to a known 'deleted' state
2549 * that matches the state of deleted pages.
2550 */
2551 memseg_remap_init_pages((page_t *)pp_dummy,
2552 (page_t *)(pp_dummy + ptob(pp_dummy_npages)));
2553 /* Remove kmem mappings for the pages for safety. */
2554 hat_unload(kas.a_hat, pp_dummy, ptob(pp_dummy_npages),
2555 HAT_UNLOAD_UNLOCK);
2556 /* Leave pp_dummy pointer set as flag that init is done. */
2557 }
2558 mutex_exit(&pp_dummy_lock);
2559 }
2560
2561 /*
2562 * Remap a page-aglined range of page_t's to dummy pages.
2563 */
2564 void
2565 remap_to_dummy(caddr_t va, pgcnt_t metapgs)
2566 {
2567 int phase;
2568
2569 ASSERT(IS_P2ALIGNED((uint64_t)(uintptr_t)va, PAGESIZE));
2570
2571 /*
2572 * We may start remapping at a non-zero page offset
2573 * within the dummy pages since the low/high ends
2574 * of the outgoing pp's could be shared by other
2575 * memsegs (see memseg_remap_meta).
2576 */
2577 phase = btop((uint64_t)(uintptr_t)va) % pp_dummy_npages;
2578 /*CONSTCOND*/
2579 ASSERT(PAGESIZE % sizeof (page_t) || phase == 0);
2580
2581 while (metapgs != 0) {
2582 pgcnt_t n;
2583 int i, j;
2584
2585 n = pp_dummy_npages;
2586 if (n > metapgs)
2587 n = metapgs;
2588 for (i = 0; i < n; i++) {
2589 j = (i + phase) % pp_dummy_npages;
2590 hat_devload(kas.a_hat, va, ptob(1), pp_dummy_pfn[j],
2591 PROT_READ,
2592 HAT_LOAD | HAT_LOAD_NOCONSIST |
2593 HAT_LOAD_REMAP);
2594 va += ptob(1);
2595 }
2596 metapgs -= n;
2597 }
2598 }
2599
2600 static void
2601 memseg_remap_to_dummy(struct memseg *seg)
2602 {
2603 caddr_t pp;
2604 pgcnt_t metapgs;
2605
2606 ASSERT(memseg_is_dynamic(seg));
2607 ASSERT(pp_dummy != NULL);
2608
2609
2610 if (!memseg_includes_meta(seg)) {
2611 memseg_remap_meta(seg);
2612 return;
2613 }
2614
2615 pp = (caddr_t)seg->pages;
2616 metapgs = seg->pages_base - memseg_get_start(seg);
2617 ASSERT(metapgs != 0);
2618
2619 seg->pages_end = seg->pages_base;
2620
2621 remap_to_dummy(pp, metapgs);
2622 }
2623
2624 /*
2625 * Transition all the deleted pages to the deleted state so that
2626 * page_lock will not wait. The page_lock_delete call will
2627 * also wake up any waiters.
2628 */
2629 static void
2630 memseg_lock_delete_all(struct memseg *seg)
2631 {
2632 page_t *pp;
2633
2634 for (pp = seg->pages; pp < seg->epages; pp++) {
2635 pp->p_pagenum = PFN_INVALID; /* XXXX */
2636 page_lock_delete(pp);
2637 }
2638 }
2639
2640 static void
2641 kphysm_del_cleanup(struct mem_handle *mhp)
2642 {
2643 struct memdelspan *mdsp;
2644 struct memseg *seg;
2645 struct memseg **segpp;
2646 struct memseg *seglist;
2647 pfn_t p_end;
2648 uint64_t avmem;
2649 pgcnt_t avpgs;
2650 pgcnt_t npgs;
2651
2652 avpgs = mhp->mh_vm_pages;
2653
2654 memsegs_lock(1);
2655
2656 /*
2657 * remove from main segment list.
2658 */
2659 npgs = 0;
2660 seglist = NULL;
2661 for (mdsp = mhp->mh_transit.trl_spans; mdsp != NULL;
2662 mdsp = mdsp->mds_next) {
2663 p_end = mdsp->mds_base + mdsp->mds_npgs;
2664 for (segpp = &memsegs; (seg = *segpp) != NULL; ) {
2665 if (seg->pages_base >= p_end ||
2666 seg->pages_end <= mdsp->mds_base) {
2667 /* Span and memseg don't overlap. */
2668 segpp = &((*segpp)->next);
2669 continue;
2670 }
2671 ASSERT(seg->pages_base >= mdsp->mds_base);
2672 ASSERT(seg->pages_end <= p_end);
2673
2674 PLCNT_MODIFY_MAX(seg->pages_base,
2675 seg->pages_base - seg->pages_end);
2676
2677 /* Hide the memseg from future scans. */
2678 hat_kpm_delmem_mseg_update(seg, segpp);
2679 *segpp = seg->next;
2680 membar_producer(); /* TODO: Needed? */
2681 npgs += MSEG_NPAGES(seg);
2682
2683 /*
2684 * Leave the deleted segment's next pointer intact
2685 * in case a memsegs scanning loop is walking this
2686 * segment concurrently.
2687 */
2688 seg->lnext = seglist;
2689 seglist = seg;
2690 }
2691 }
2692
2693 build_pfn_hash();
2694
2695 ASSERT(npgs < total_pages);
2696 total_pages -= npgs;
2697
2698 /*
2699 * Recalculate the paging parameters now total_pages has changed.
2700 * This will also cause the clock hands to be reset before next use.
2701 */
2702 setupclock(1);
2703
2704 memsegs_unlock(1);
2705
2706 mutex_exit(&mhp->mh_mutex);
2707
2708 while ((seg = seglist) != NULL) {
2709 pfn_t mseg_start;
2710 pfn_t mseg_base, mseg_end;
2711 pgcnt_t mseg_npgs;
2712 int mlret;
2713
2714 seglist = seg->lnext;
2715
2716 /*
2717 * Put the page_t's into the deleted state to stop
2718 * cv_wait()s on the pages. When we remap, the dummy
2719 * page_t's will be in the same state.
2720 */
2721 memseg_lock_delete_all(seg);
2722 /*
2723 * Collect up information based on pages_base and pages_end
2724 * early so that we can flag early that the memseg has been
2725 * deleted by setting pages_end == pages_base.
2726 */
2727 mseg_base = seg->pages_base;
2728 mseg_end = seg->pages_end;
2729 mseg_npgs = MSEG_NPAGES(seg);
2730 mseg_start = memseg_get_start(seg);
2731
2732 if (memseg_is_dynamic(seg)) {
2733 /* Remap the meta data to our special dummy area. */
2734 memseg_remap_to_dummy(seg);
2735
2736 mutex_enter(&memseg_lists_lock);
2737 seg->lnext = memseg_va_avail;
2738 memseg_va_avail = seg;
2739 mutex_exit(&memseg_lists_lock);
2740 } else {
2741 /*
2742 * For memory whose page_ts were allocated
2743 * at boot, we need to find a new use for
2744 * the page_t memory.
2745 * For the moment, just leak it.
2746 * (It is held in the memseg_delete_junk list.)
2747 */
2748 seg->pages_end = seg->pages_base;
2749
2750 mutex_enter(&memseg_lists_lock);
2751 seg->lnext = memseg_delete_junk;
2752 memseg_delete_junk = seg;
2753 mutex_exit(&memseg_lists_lock);
2754 }
2755
2756 /* Must not use seg now as it could be re-used. */
2757
2758 memlist_write_lock();
2759
2760 mlret = memlist_delete_span(
2761 (uint64_t)(mseg_base) << PAGESHIFT,
2762 (uint64_t)(mseg_npgs) << PAGESHIFT,
2763 &phys_avail);
2764 ASSERT(mlret == MEML_SPANOP_OK);
2765
2766 mlret = memlist_delete_span(
2767 (uint64_t)(mseg_start) << PAGESHIFT,
2768 (uint64_t)(mseg_end - mseg_start) <<
2769 PAGESHIFT,
2770 &phys_install);
2771 ASSERT(mlret == MEML_SPANOP_OK);
2772 phys_install_has_changed();
2773
2774 memlist_write_unlock();
2775 }
2776
2777 memlist_read_lock();
2778 installed_top_size(phys_install, &physmax, &physinstalled);
2779 memlist_read_unlock();
2780
2781 mutex_enter(&freemem_lock);
2782 maxmem -= avpgs;
2783 physmem -= avpgs;
2784 /* availrmem is adjusted during the delete. */
2785 availrmem_initial -= avpgs;
2786
2787 mutex_exit(&freemem_lock);
2788
2789 dump_resize();
2790
2791 cmn_err(CE_CONT, "?kphysm_delete: mem = %ldK "
2792 "(0x%" PRIx64 ")\n",
2793 physinstalled << (PAGESHIFT - 10),
2794 (uint64_t)physinstalled << PAGESHIFT);
2795
2796 avmem = (uint64_t)freemem << PAGESHIFT;
2797 cmn_err(CE_CONT, "?kphysm_delete: "
2798 "avail mem = %" PRId64 "\n", avmem);
2799
2800 /*
2801 * Update lgroup generation number on single lgroup systems
2802 */
2803 if (nlgrps == 1)
2804 lgrp_config(LGRP_CONFIG_GEN_UPDATE, 0, 0);
2805
2806 /* Successfully deleted system memory */
2807 mutex_enter(&mhp->mh_mutex);
2808 }
2809
2810 static uint_t mdel_nullvp_waiter;
2811
2812 static void
2813 page_delete_collect(
2814 page_t *pp,
2815 struct mem_handle *mhp)
2816 {
2817 if (pp->p_vnode) {
2818 page_hashout(pp, (kmutex_t *)NULL);
2819 /* do not do PP_SETAGED(pp); */
2820 } else {
2821 kmutex_t *sep;
2822
2823 sep = page_se_mutex(pp);
2824 mutex_enter(sep);
2825 if (CV_HAS_WAITERS(&pp->p_cv)) {
2826 mdel_nullvp_waiter++;
2827 cv_broadcast(&pp->p_cv);
2828 }
2829 mutex_exit(sep);
2830 }
2831 ASSERT(pp->p_next == pp->p_prev);
2832 ASSERT(pp->p_next == NULL || pp->p_next == pp);
2833 pp->p_next = mhp->mh_deleted;
2834 mhp->mh_deleted = pp;
2835 ASSERT(mhp->mh_hold_todo != 0);
2836 mhp->mh_hold_todo--;
2837 }
2838
2839 static void
2840 transit_list_collect(struct mem_handle *mhp, int v)
2841 {
2842 struct transit_list_head *trh;
2843
2844 trh = &transit_list_head;
2845 mutex_enter(&trh->trh_lock);
2846 mhp->mh_transit.trl_collect = v;
2847 mutex_exit(&trh->trh_lock);
2848 }
2849
2850 static void
2851 transit_list_insert(struct transit_list *tlp)
2852 {
2853 struct transit_list_head *trh;
2854
2855 trh = &transit_list_head;
2856 ASSERT(MUTEX_HELD(&trh->trh_lock));
2857 tlp->trl_next = trh->trh_head;
2858 trh->trh_head = tlp;
2859 }
2860
2861 static void
2862 transit_list_remove(struct transit_list *tlp)
2863 {
2864 struct transit_list_head *trh;
2865 struct transit_list **tlpp;
2866
2867 trh = &transit_list_head;
2868 tlpp = &trh->trh_head;
2869 ASSERT(MUTEX_HELD(&trh->trh_lock));
2870 while (*tlpp != NULL && *tlpp != tlp)
2871 tlpp = &(*tlpp)->trl_next;
2872 ASSERT(*tlpp != NULL);
2873 if (*tlpp == tlp)
2874 *tlpp = tlp->trl_next;
2875 tlp->trl_next = NULL;
2876 }
2877
2878 static struct transit_list *
2879 pfnum_to_transit_list(struct transit_list_head *trh, pfn_t pfnum)
2880 {
2881 struct transit_list *tlp;
2882
2883 for (tlp = trh->trh_head; tlp != NULL; tlp = tlp->trl_next) {
2884 struct memdelspan *mdsp;
2885
2886 for (mdsp = tlp->trl_spans; mdsp != NULL;
2887 mdsp = mdsp->mds_next) {
2888 if (pfnum >= mdsp->mds_base &&
2889 pfnum < (mdsp->mds_base + mdsp->mds_npgs)) {
2890 return (tlp);
2891 }
2892 }
2893 }
2894 return (NULL);
2895 }
2896
2897 int
2898 pfn_is_being_deleted(pfn_t pfnum)
2899 {
2900 struct transit_list_head *trh;
2901 struct transit_list *tlp;
2902 int ret;
2903
2904 trh = &transit_list_head;
2905 if (trh->trh_head == NULL)
2906 return (0);
2907
2908 mutex_enter(&trh->trh_lock);
2909 tlp = pfnum_to_transit_list(trh, pfnum);
2910 ret = (tlp != NULL && tlp->trl_collect);
2911 mutex_exit(&trh->trh_lock);
2912
2913 return (ret);
2914 }
2915
2916 #ifdef MEM_DEL_STATS
2917 extern int hz;
2918 static void
2919 mem_del_stat_print_func(struct mem_handle *mhp)
2920 {
2921 uint64_t tmp;
2922
2923 if (mem_del_stat_print) {
2924 printf("memory delete loop %x/%x, statistics%s\n",
2925 (uint_t)mhp->mh_transit.trl_spans->mds_base,
2926 (uint_t)mhp->mh_transit.trl_spans->mds_npgs,
2927 (mhp->mh_cancel ? " (cancelled)" : ""));
2928 printf("\t%8u nloop\n", mhp->mh_delstat.nloop);
2929 printf("\t%8u need_free\n", mhp->mh_delstat.need_free);
2930 printf("\t%8u free_loop\n", mhp->mh_delstat.free_loop);
2931 printf("\t%8u free_low\n", mhp->mh_delstat.free_low);
2932 printf("\t%8u free_failed\n", mhp->mh_delstat.free_failed);
2933 printf("\t%8u ncheck\n", mhp->mh_delstat.ncheck);
2934 printf("\t%8u nopaget\n", mhp->mh_delstat.nopaget);
2935 printf("\t%8u lockfail\n", mhp->mh_delstat.lockfail);
2936 printf("\t%8u nfree\n", mhp->mh_delstat.nfree);
2937 printf("\t%8u nreloc\n", mhp->mh_delstat.nreloc);
2938 printf("\t%8u nrelocfail\n", mhp->mh_delstat.nrelocfail);
2939 printf("\t%8u already_done\n", mhp->mh_delstat.already_done);
2940 printf("\t%8u first_notfree\n", mhp->mh_delstat.first_notfree);
2941 printf("\t%8u npplocked\n", mhp->mh_delstat.npplocked);
2942 printf("\t%8u nlockreloc\n", mhp->mh_delstat.nlockreloc);
2943 printf("\t%8u nnorepl\n", mhp->mh_delstat.nnorepl);
2944 printf("\t%8u nmodreloc\n", mhp->mh_delstat.nmodreloc);
2945 printf("\t%8u ndestroy\n", mhp->mh_delstat.ndestroy);
2946 printf("\t%8u nputpage\n", mhp->mh_delstat.nputpage);
2947 printf("\t%8u nnoreclaim\n", mhp->mh_delstat.nnoreclaim);
2948 printf("\t%8u ndelay\n", mhp->mh_delstat.ndelay);
2949 printf("\t%8u demotefail\n", mhp->mh_delstat.demotefail);
2950 printf("\t%8u retired\n", mhp->mh_delstat.retired);
2951 printf("\t%8u toxic\n", mhp->mh_delstat.toxic);
2952 printf("\t%8u failing\n", mhp->mh_delstat.failing);
2953 printf("\t%8u modtoxic\n", mhp->mh_delstat.modtoxic);
2954 printf("\t%8u npplkdtoxic\n", mhp->mh_delstat.npplkdtoxic);
2955 printf("\t%8u gptlmodfail\n", mhp->mh_delstat.gptlmodfail);
2956 printf("\t%8u gptllckfail\n", mhp->mh_delstat.gptllckfail);
2957 tmp = mhp->mh_delstat.nticks_total / hz; /* seconds */
2958 printf(
2959 "\t%"PRIu64" nticks_total - %"PRIu64" min %"PRIu64" sec\n",
2960 mhp->mh_delstat.nticks_total, tmp / 60, tmp % 60);
2961
2962 tmp = mhp->mh_delstat.nticks_pgrp / hz; /* seconds */
2963 printf(
2964 "\t%"PRIu64" nticks_pgrp - %"PRIu64" min %"PRIu64" sec\n",
2965 mhp->mh_delstat.nticks_pgrp, tmp / 60, tmp % 60);
2966 }
2967 }
2968 #endif /* MEM_DEL_STATS */
2969
2970 struct mem_callback {
2971 kphysm_setup_vector_t *vec;
2972 void *arg;
2973 };
2974
2975 #define NMEMCALLBACKS 100
2976
2977 static struct mem_callback mem_callbacks[NMEMCALLBACKS];
2978 static uint_t nmemcallbacks;
2979 static krwlock_t mem_callback_rwlock;
2980
2981 int
2982 kphysm_setup_func_register(kphysm_setup_vector_t *vec, void *arg)
2983 {
2984 uint_t i, found;
2985
2986 /*
2987 * This test will become more complicated when the version must
2988 * change.
2989 */
2990 if (vec->version != KPHYSM_SETUP_VECTOR_VERSION)
2991 return (EINVAL);
2992
2993 if (vec->post_add == NULL || vec->pre_del == NULL ||
2994 vec->post_del == NULL)
2995 return (EINVAL);
2996
2997 rw_enter(&mem_callback_rwlock, RW_WRITER);
2998 for (i = 0, found = 0; i < nmemcallbacks; i++) {
2999 if (mem_callbacks[i].vec == NULL && found == 0)
3000 found = i + 1;
3001 if (mem_callbacks[i].vec == vec &&
3002 mem_callbacks[i].arg == arg) {
3003 #ifdef DEBUG
3004 /* Catch this in DEBUG kernels. */
3005 cmn_err(CE_WARN, "kphysm_setup_func_register"
3006 "(0x%p, 0x%p) duplicate registration from 0x%p",
3007 (void *)vec, arg, (void *)caller());
3008 #endif /* DEBUG */
3009 rw_exit(&mem_callback_rwlock);
3010 return (EEXIST);
3011 }
3012 }
3013 if (found != 0) {
3014 i = found - 1;
3015 } else {
3016 ASSERT(nmemcallbacks < NMEMCALLBACKS);
3017 if (nmemcallbacks == NMEMCALLBACKS) {
3018 rw_exit(&mem_callback_rwlock);
3019 return (ENOMEM);
3020 }
3021 i = nmemcallbacks++;
3022 }
3023 mem_callbacks[i].vec = vec;
3024 mem_callbacks[i].arg = arg;
3025 rw_exit(&mem_callback_rwlock);
3026 return (0);
3027 }
3028
3029 void
3030 kphysm_setup_func_unregister(kphysm_setup_vector_t *vec, void *arg)
3031 {
3032 uint_t i;
3033
3034 rw_enter(&mem_callback_rwlock, RW_WRITER);
3035 for (i = 0; i < nmemcallbacks; i++) {
3036 if (mem_callbacks[i].vec == vec &&
3037 mem_callbacks[i].arg == arg) {
3038 mem_callbacks[i].vec = NULL;
3039 mem_callbacks[i].arg = NULL;
3040 if (i == (nmemcallbacks - 1))
3041 nmemcallbacks--;
3042 break;
3043 }
3044 }
3045 rw_exit(&mem_callback_rwlock);
3046 }
3047
3048 static void
3049 kphysm_setup_post_add(pgcnt_t delta_pages)
3050 {
3051 uint_t i;
3052
3053 rw_enter(&mem_callback_rwlock, RW_READER);
3054 for (i = 0; i < nmemcallbacks; i++) {
3055 if (mem_callbacks[i].vec != NULL) {
3056 (*mem_callbacks[i].vec->post_add)
3057 (mem_callbacks[i].arg, delta_pages);
3058 }
3059 }
3060 rw_exit(&mem_callback_rwlock);
3061 }
3062
3063 /*
3064 * Note the locking between pre_del and post_del: The reader lock is held
3065 * between the two calls to stop the set of functions from changing.
3066 */
3067
3068 static int
3069 kphysm_setup_pre_del(pgcnt_t delta_pages)
3070 {
3071 uint_t i;
3072 int ret;
3073 int aret;
3074
3075 ret = 0;
3076 rw_enter(&mem_callback_rwlock, RW_READER);
3077 for (i = 0; i < nmemcallbacks; i++) {
3078 if (mem_callbacks[i].vec != NULL) {
3079 aret = (*mem_callbacks[i].vec->pre_del)
3080 (mem_callbacks[i].arg, delta_pages);
3081 ret |= aret;
3082 }
3083 }
3084
3085 return (ret);
3086 }
3087
3088 static void
3089 kphysm_setup_post_del(pgcnt_t delta_pages, int cancelled)
3090 {
3091 uint_t i;
3092
3093 for (i = 0; i < nmemcallbacks; i++) {
3094 if (mem_callbacks[i].vec != NULL) {
3095 (*mem_callbacks[i].vec->post_del)
3096 (mem_callbacks[i].arg, delta_pages, cancelled);
3097 }
3098 }
3099 rw_exit(&mem_callback_rwlock);
3100 }
3101
3102 static int
3103 kphysm_split_memseg(
3104 pfn_t base,
3105 pgcnt_t npgs)
3106 {
3107 struct memseg *seg;
3108 struct memseg **segpp;
3109 pgcnt_t size_low, size_high;
3110 struct memseg *seg_low, *seg_mid, *seg_high;
3111
3112 /*
3113 * Lock the memsegs list against other updates now
3114 */
3115 memsegs_lock(1);
3116
3117 /*
3118 * Find boot time memseg that wholly covers this area.
3119 */
3120
3121 /* First find the memseg with page 'base' in it. */
3122 for (segpp = &memsegs; (seg = *segpp) != NULL;
3123 segpp = &((*segpp)->next)) {
3124 if (base >= seg->pages_base && base < seg->pages_end)
3125 break;
3126 }
3127 if (seg == NULL) {
3128 memsegs_unlock(1);
3129 return (0);
3130 }
3131 if (memseg_includes_meta(seg)) {
3132 memsegs_unlock(1);
3133 return (0);
3134 }
3135 if ((base + npgs) > seg->pages_end) {
3136 memsegs_unlock(1);
3137 return (0);
3138 }
3139
3140 /*
3141 * Work out the size of the two segments that will
3142 * surround the new segment, one for low address
3143 * and one for high.
3144 */
3145 ASSERT(base >= seg->pages_base);
3146 size_low = base - seg->pages_base;
3147 ASSERT(seg->pages_end >= (base + npgs));
3148 size_high = seg->pages_end - (base + npgs);
3149
3150 /*
3151 * Sanity check.
3152 */
3153 if ((size_low + size_high) == 0) {
3154 memsegs_unlock(1);
3155 return (0);
3156 }
3157
3158 /*
3159 * Allocate the new structures. The old memseg will not be freed
3160 * as there may be a reference to it.
3161 */
3162 seg_low = NULL;
3163 seg_high = NULL;
3164
3165 if (size_low != 0)
3166 seg_low = memseg_alloc();
3167
3168 seg_mid = memseg_alloc();
3169
3170 if (size_high != 0)
3171 seg_high = memseg_alloc();
3172
3173 /*
3174 * All allocation done now.
3175 */
3176 if (size_low != 0) {
3177 seg_low->pages = seg->pages;
3178 seg_low->epages = seg_low->pages + size_low;
3179 seg_low->pages_base = seg->pages_base;
3180 seg_low->pages_end = seg_low->pages_base + size_low;
3181 seg_low->next = seg_mid;
3182 seg_low->msegflags = seg->msegflags;
3183 }
3184 if (size_high != 0) {
3185 seg_high->pages = seg->epages - size_high;
3186 seg_high->epages = seg_high->pages + size_high;
3187 seg_high->pages_base = seg->pages_end - size_high;
3188 seg_high->pages_end = seg_high->pages_base + size_high;
3189 seg_high->next = seg->next;
3190 seg_high->msegflags = seg->msegflags;
3191 }
3192
3193 seg_mid->pages = seg->pages + size_low;
3194 seg_mid->pages_base = seg->pages_base + size_low;
3195 seg_mid->epages = seg->epages - size_high;
3196 seg_mid->pages_end = seg->pages_end - size_high;
3197 seg_mid->next = (seg_high != NULL) ? seg_high : seg->next;
3198 seg_mid->msegflags = seg->msegflags;
3199
3200 /*
3201 * Update hat_kpm specific info of all involved memsegs and
3202 * allow hat_kpm specific global chain updates.
3203 */
3204 hat_kpm_split_mseg_update(seg, segpp, seg_low, seg_mid, seg_high);
3205
3206 /*
3207 * At this point we have two equivalent memseg sub-chains,
3208 * seg and seg_low/seg_mid/seg_high, which both chain on to
3209 * the same place in the global chain. By re-writing the pointer
3210 * in the previous element we switch atomically from using the old
3211 * (seg) to the new.
3212 */
3213 *segpp = (seg_low != NULL) ? seg_low : seg_mid;
3214
3215 membar_enter();
3216
3217 build_pfn_hash();
3218 memsegs_unlock(1);
3219
3220 /*
3221 * We leave the old segment, 'seg', intact as there may be
3222 * references to it. Also, as the value of total_pages has not
3223 * changed and the memsegs list is effectively the same when
3224 * accessed via the old or the new pointer, we do not have to
3225 * cause pageout_scanner() to re-evaluate its hand pointers.
3226 *
3227 * We currently do not re-use or reclaim the page_t memory.
3228 * If we do, then this may have to change.
3229 */
3230
3231 mutex_enter(&memseg_lists_lock);
3232 seg->lnext = memseg_edit_junk;
3233 memseg_edit_junk = seg;
3234 mutex_exit(&memseg_lists_lock);
3235
3236 return (1);
3237 }
3238
3239 /*
3240 * The sfmmu hat layer (e.g.) accesses some parts of the memseg
3241 * structure using physical addresses. Therefore a kmem_cache is
3242 * used with KMC_NOHASH to avoid page crossings within a memseg
3243 * structure. KMC_NOHASH requires that no external (outside of
3244 * slab) information is allowed. This, in turn, implies that the
3245 * cache's slabsize must be exactly a single page, since per-slab
3246 * information (e.g. the freelist for the slab) is kept at the
3247 * end of the slab, where it is easy to locate. Should be changed
3248 * when a more obvious kmem_cache interface/flag will become
3249 * available.
3250 */
3251 void
3252 mem_config_init()
3253 {
3254 memseg_cache = kmem_cache_create("memseg_cache", sizeof (struct memseg),
3255 0, NULL, NULL, NULL, NULL, static_arena, KMC_NOHASH);
3256 }
3257
3258 struct memseg *
3259 memseg_alloc()
3260 {
3261 struct memseg *seg;
3262
3263 seg = kmem_cache_alloc(memseg_cache, KM_SLEEP);
3264 bzero(seg, sizeof (struct memseg));
3265
3266 return (seg);
3267 }
3268
3269 /*
3270 * Return whether the page_t memory for this memseg
3271 * is included in the memseg itself.
3272 */
3273 static int
3274 memseg_includes_meta(struct memseg *seg)
3275 {
3276 return (seg->msegflags & MEMSEG_META_INCL);
3277 }
3278
3279 pfn_t
3280 memseg_get_start(struct memseg *seg)
3281 {
3282 pfn_t pt_start;
3283
3284 if (memseg_includes_meta(seg)) {
3285 pt_start = hat_getpfnum(kas.a_hat, (caddr_t)seg->pages);
3286
3287 /* Meta data is required to be at the beginning */
3288 ASSERT(pt_start < seg->pages_base);
3289 } else
3290 pt_start = seg->pages_base;
3291
3292 return (pt_start);
3293 }
3294
3295 /*
3296 * Invalidate memseg pointers in cpu private vm data caches.
3297 */
3298 static void
3299 memseg_cpu_vm_flush()
3300 {
3301 cpu_t *cp;
3302 vm_cpu_data_t *vc;
3303
3304 mutex_enter(&cpu_lock);
3305 pause_cpus(NULL, NULL);
3306
3307 cp = cpu_list;
3308 do {
3309 vc = cp->cpu_vm_data;
3310 vc->vc_pnum_memseg = NULL;
3311 vc->vc_pnext_memseg = NULL;
3312
3313 } while ((cp = cp->cpu_next) != cpu_list);
3314
3315 start_cpus();
3316 mutex_exit(&cpu_lock);
3317 }
Unleashed OS