?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
uts: make emu10k non-verbose
[unleashed.git] / kernel / vm / seg_spt.c
blob0f812a3eaba03ed2435d667707e655d1f7e5296b
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2015, Joyent, Inc. All rights reserved.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 #include <sys/param.h>
28 #include <sys/user.h>
29 #include <sys/mman.h>
30 #include <sys/kmem.h>
31 #include <sys/sysmacros.h>
32 #include <sys/cmn_err.h>
33 #include <sys/systm.h>
34 #include <sys/tuneable.h>
35 #include <vm/hat.h>
36 #include <vm/seg.h>
37 #include <vm/as.h>
38 #include <vm/anon.h>
39 #include <vm/page.h>
40 #include <sys/buf.h>
41 #include <sys/swap.h>
42 #include <sys/atomic.h>
43 #include <vm/seg_spt.h>
44 #include <sys/debug.h>
45 #include <sys/vtrace.h>
46 #include <sys/shm.h>
47 #include <sys/shm_impl.h>
48 #include <sys/lgrp.h>
49 #include <sys/vmsystm.h>
50 #include <sys/policy.h>
51 #include <sys/project.h>
52 #include <sys/tnf_probe.h>
53 #include <sys/zone.h>
54
55 #define SEGSPTADDR (caddr_t)0x0
56
57 /*
58 * # pages used for spt
59 */
60 size_t spt_used;
61
62 /*
63 * segspt_minfree is the memory left for system after ISM
64 * locked its pages; it is set up to 5% of availrmem in
65 * sptcreate when ISM is created. ISM should not use more
66 * than ~90% of availrmem; if it does, then the performance
67 * of the system may decrease. Machines with large memories may
68 * be able to use up more memory for ISM so we set the default
69 * segspt_minfree to 5% (which gives ISM max 95% of availrmem.
70 * If somebody wants even more memory for ISM (risking hanging
71 * the system) they can patch the segspt_minfree to smaller number.
72 */
73 pgcnt_t segspt_minfree = 0;
74
75 static int segspt_create(struct seg *seg, caddr_t argsp);
76 static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize);
77 static void segspt_free(struct seg *seg);
78 static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len);
79 static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr);
80
81 static void
82 segspt_badop()
83 {
84 panic("segspt_badop called");
85 /*NOTREACHED*/
86 }
87
88 #define SEGSPT_BADOP(t) (t(*)())segspt_badop
89
90 static const struct seg_ops segspt_ops = {
91 .dup = SEGSPT_BADOP(int),
92 .unmap = segspt_unmap,
93 .free = segspt_free,
94 .fault = SEGSPT_BADOP(int),
95 .faulta = SEGSPT_BADOP(faultcode_t),
96 .setprot = SEGSPT_BADOP(int),
97 .checkprot = SEGSPT_BADOP(int),
98 .kluster = SEGSPT_BADOP(int),
99 .sync = SEGSPT_BADOP(int),
100 .incore = SEGSPT_BADOP(size_t),
101 .lockop = SEGSPT_BADOP(int),
102 .getprot = SEGSPT_BADOP(int),
103 .getoffset = SEGSPT_BADOP(uoff_t),
104 .gettype = SEGSPT_BADOP(int),
105 .getvp = SEGSPT_BADOP(int),
106 .advise = SEGSPT_BADOP(int),
107 .dump = SEGSPT_BADOP(void),
108 .pagelock = SEGSPT_BADOP(int),
109 .setpagesize = SEGSPT_BADOP(int),
110 .getmemid = SEGSPT_BADOP(int),
111 .getpolicy = segspt_getpolicy,
112 .capable = SEGSPT_BADOP(int),
113 };
114
115 static int segspt_shmdup(struct seg *seg, struct seg *newseg);
116 static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize);
117 static void segspt_shmfree(struct seg *seg);
118 static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg,
119 caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw);
120 static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr);
121 static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr,
122 register size_t len, register uint_t prot);
123 static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size,
124 uint_t prot);
125 static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta);
126 static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len,
127 register char *vec);
128 static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len,
129 int attr, uint_t flags);
130 static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
131 int attr, int op, ulong_t *lockmap, size_t pos);
132 static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len,
133 uint_t *protv);
134 static uoff_t segspt_shmgetoffset(struct seg *seg, caddr_t addr);
135 static int segspt_shmgettype(struct seg *seg, caddr_t addr);
136 static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
137 static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len,
138 uint_t behav);
139 static int segspt_shmpagelock(struct seg *, caddr_t, size_t,
140 struct page ***, enum lock_type, enum seg_rw);
141 static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *);
142 static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t);
143
144 const struct seg_ops segspt_shmops = {
145 .dup = segspt_shmdup,
146 .unmap = segspt_shmunmap,
147 .free = segspt_shmfree,
148 .fault = segspt_shmfault,
149 .faulta = segspt_shmfaulta,
150 .setprot = segspt_shmsetprot,
151 .checkprot = segspt_shmcheckprot,
152 .kluster = segspt_shmkluster,
153 .sync = segspt_shmsync,
154 .incore = segspt_shmincore,
155 .lockop = segspt_shmlockop,
156 .getprot = segspt_shmgetprot,
157 .getoffset = segspt_shmgetoffset,
158 .gettype = segspt_shmgettype,
159 .getvp = segspt_shmgetvp,
160 .advise = segspt_shmadvise,
161 .pagelock = segspt_shmpagelock,
162 .getmemid = segspt_shmgetmemid,
163 .getpolicy = segspt_shmgetpolicy,
164 };
165
166 static void segspt_purge(struct seg *seg);
167 static int segspt_reclaim(void *, caddr_t, size_t, struct page **,
168 enum seg_rw, int);
169 static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len,
170 page_t **ppa);
171
172
173
174 /*ARGSUSED*/
175 int
176 sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp,
177 uint_t prot, uint_t flags, uint_t share_szc)
178 {
179 int err;
180 struct as *newas;
181 struct segspt_crargs sptcargs;
182
183 if (segspt_minfree == 0) /* leave min 5% of availrmem for */
184 segspt_minfree = availrmem/20; /* for the system */
185
186 if (!hat_supported(HAT_SHARED_PT, NULL))
187 return (EINVAL);
188
189 /*
190 * get a new as for this shared memory segment
191 */
192 newas = as_alloc();
193 newas->a_proc = NULL;
194 sptcargs.amp = amp;
195 sptcargs.prot = prot;
196 sptcargs.flags = flags;
197 sptcargs.szc = share_szc;
198 /*
199 * create a shared page table (spt) segment
200 */
201
202 if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) {
203 as_free(newas);
204 return (err);
205 }
206 *sptseg = sptcargs.seg_spt;
207 return (0);
208 }
209
210 void
211 sptdestroy(struct as *as, struct anon_map *amp)
212 {
213 (void) as_unmap(as, SEGSPTADDR, amp->size);
214 as_free(as);
215 }
216
217 /*
218 * called from seg_free().
219 * free (i.e., unlock, unmap, return to free list)
220 * all the pages in the given seg.
221 */
222 void
223 segspt_free(struct seg *seg)
224 {
225 struct spt_data *sptd = (struct spt_data *)seg->s_data;
226
227 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
228
229 if (sptd != NULL) {
230 if (sptd->spt_realsize)
231 segspt_free_pages(seg, seg->s_base, sptd->spt_realsize);
232
233 if (sptd->spt_ppa_lckcnt) {
234 kmem_free(sptd->spt_ppa_lckcnt,
235 sizeof (*sptd->spt_ppa_lckcnt)
236 * btopr(sptd->spt_amp->size));
237 }
238 kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp));
239 cv_destroy(&sptd->spt_cv);
240 mutex_destroy(&sptd->spt_lock);
241 kmem_free(sptd, sizeof (*sptd));
242 }
243 }
244
245 /*ARGSUSED*/
246 static int
247 segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr,
248 uint_t flags)
249 {
250 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
251
252 return (0);
253 }
254
255 /*ARGSUSED*/
256 static size_t
257 segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec)
258 {
259 caddr_t eo_seg;
260 pgcnt_t npages;
261 struct shm_data *shmd = (struct shm_data *)seg->s_data;
262 struct seg *sptseg;
263 struct spt_data *sptd;
264
265 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
266 sptseg = shmd->shm_sptseg;
267 sptd = sptseg->s_data;
268
269 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
270 eo_seg = addr + len;
271 while (addr < eo_seg) {
272 /* page exists, and it's locked. */
273 *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED |
274 SEG_PAGE_ANON;
275 addr += PAGESIZE;
276 }
277 return (len);
278 } else {
279 struct anon_map *amp = shmd->shm_amp;
280 struct anon *ap;
281 page_t *pp;
282 pgcnt_t anon_index;
283 struct vnode *vp;
284 uoff_t off;
285 ulong_t i;
286 int ret;
287 anon_sync_obj_t cookie;
288
289 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
290 anon_index = seg_page(seg, addr);
291 npages = btopr(len);
292 if (anon_index + npages > btopr(shmd->shm_amp->size)) {
293 return (EINVAL);
294 }
295 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
296 for (i = 0; i < npages; i++, anon_index++) {
297 ret = 0;
298 anon_array_enter(amp, anon_index, &cookie);
299 ap = anon_get_ptr(amp->ahp, anon_index);
300 if (ap != NULL) {
301 swap_xlate(ap, &vp, &off);
302 anon_array_exit(&cookie);
303 pp = page_lookup_nowait(vp, off, SE_SHARED);
304 if (pp != NULL) {
305 ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON;
306 page_unlock(pp);
307 }
308 } else {
309 anon_array_exit(&cookie);
310 }
311 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
312 ret |= SEG_PAGE_LOCKED;
313 }
314 *vec++ = (char)ret;
315 }
316 ANON_LOCK_EXIT(&amp->a_rwlock);
317 return (len);
318 }
319 }
320
321 static int
322 segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize)
323 {
324 size_t share_size;
325
326 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
327
328 /*
329 * seg.s_size may have been rounded up to the largest page size
330 * in shmat().
331 * XXX This should be cleanedup. sptdestroy should take a length
332 * argument which should be the same as sptcreate. Then
333 * this rounding would not be needed (or is done in shm.c)
334 * Only the check for full segment will be needed.
335 *
336 * XXX -- shouldn't raddr == 0 always? These tests don't seem
337 * to be useful at all.
338 */
339 share_size = page_get_pagesize(seg->s_szc);
340 ssize = P2ROUNDUP(ssize, share_size);
341
342 if (raddr == seg->s_base && ssize == seg->s_size) {
343 seg_free(seg);
344 return (0);
345 } else
346 return (EINVAL);
347 }
348
349 int
350 segspt_create(struct seg *seg, caddr_t argsp)
351 {
352 int err;
353 caddr_t addr = seg->s_base;
354 struct spt_data *sptd;
355 struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp;
356 struct anon_map *amp = sptcargs->amp;
357 struct kshmid *sp = amp->a_sp;
358 struct cred *cred = CRED();
359 ulong_t i, j, anon_index = 0;
360 pgcnt_t npages = btopr(amp->size);
361 struct vnode *vp;
362 page_t **ppa;
363 uint_t hat_flags;
364 size_t pgsz;
365 pgcnt_t pgcnt;
366 caddr_t a;
367 pgcnt_t pidx;
368 size_t sz;
369 proc_t *procp = curproc;
370 rctl_qty_t lockedbytes = 0;
371 kproject_t *proj;
372
373 /*
374 * We are holding the a_lock on the underlying dummy as,
375 * so we can make calls to the HAT layer.
376 */
377 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
378 ASSERT(sp != NULL);
379
380 if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
381 if (err = anon_swap_adjust(npages))
382 return (err);
383 }
384 err = ENOMEM;
385
386 if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL)
387 goto out1;
388
389 if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
390 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages),
391 KM_NOSLEEP)) == NULL)
392 goto out2;
393 }
394
395 mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL);
396
397 if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL)
398 goto out3;
399
400 seg->s_ops = &segspt_ops;
401 sptd->spt_vp = vp;
402 sptd->spt_amp = amp;
403 sptd->spt_prot = sptcargs->prot;
404 sptd->spt_flags = sptcargs->flags;
405 seg->s_data = (caddr_t)sptd;
406 sptd->spt_ppa = NULL;
407 sptd->spt_ppa_lckcnt = NULL;
408 seg->s_szc = sptcargs->szc;
409 cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL);
410 sptd->spt_gen = 0;
411
412 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
413 if (seg->s_szc > amp->a_szc) {
414 amp->a_szc = seg->s_szc;
415 }
416 ANON_LOCK_EXIT(&amp->a_rwlock);
417
418 /*
419 * Set policy to affect initial allocation of pages in
420 * anon_map_createpages()
421 */
422 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index,
423 NULL, 0, ptob(npages));
424
425 if (sptcargs->flags & SHM_PAGEABLE) {
426 size_t share_sz;
427 pgcnt_t new_npgs, more_pgs;
428 struct anon_hdr *nahp;
429 zone_t *zone;
430
431 share_sz = page_get_pagesize(seg->s_szc);
432 if (!IS_P2ALIGNED(amp->size, share_sz)) {
433 /*
434 * We are rounding up the size of the anon array
435 * on 4 M boundary because we always create 4 M
436 * of page(s) when locking, faulting pages and we
437 * don't have to check for all corner cases e.g.
438 * if there is enough space to allocate 4 M
439 * page.
440 */
441 new_npgs = btop(P2ROUNDUP(amp->size, share_sz));
442 more_pgs = new_npgs - npages;
443
444 /*
445 * The zone will never be NULL, as a fully created
446 * shm always has an owning zone.
447 */
448 zone = sp->shm_perm.ipc_zone_ref.zref_zone;
449 ASSERT(zone != NULL);
450 if (anon_resv_zone(ptob(more_pgs), zone) == 0) {
451 err = ENOMEM;
452 goto out4;
453 }
454
455 nahp = anon_create(new_npgs, ANON_SLEEP);
456 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
457 (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages,
458 ANON_SLEEP);
459 anon_release(amp->ahp, npages);
460 amp->ahp = nahp;
461 ASSERT(amp->swresv == ptob(npages));
462 amp->swresv = amp->size = ptob(new_npgs);
463 ANON_LOCK_EXIT(&amp->a_rwlock);
464 npages = new_npgs;
465 }
466
467 sptd->spt_ppa_lckcnt = kmem_zalloc(npages *
468 sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP);
469 sptd->spt_pcachecnt = 0;
470 sptd->spt_realsize = ptob(npages);
471 sptcargs->seg_spt = seg;
472 return (0);
473 }
474
475 /*
476 * get array of pages for each anon slot in amp
477 */
478 if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa,
479 seg, addr, S_CREATE, cred)) != 0)
480 goto out4;
481
482 mutex_enter(&sp->shm_mlock);
483
484 /* May be partially locked, so, count bytes to charge for locking */
485 for (i = 0; i < npages; i++)
486 if (ppa[i]->p_lckcnt == 0)
487 lockedbytes += PAGESIZE;
488
489 proj = sp->shm_perm.ipc_proj;
490
491 if (lockedbytes > 0) {
492 mutex_enter(&procp->p_lock);
493 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) {
494 mutex_exit(&procp->p_lock);
495 mutex_exit(&sp->shm_mlock);
496 for (i = 0; i < npages; i++)
497 page_unlock(ppa[i]);
498 err = ENOMEM;
499 goto out4;
500 }
501 mutex_exit(&procp->p_lock);
502 }
503
504 /*
505 * addr is initial address corresponding to the first page on ppa list
506 */
507 for (i = 0; i < npages; i++) {
508 /* attempt to lock all pages */
509 if (page_pp_lock(ppa[i], 0, 1) == 0) {
510 /*
511 * if unable to lock any page, unlock all
512 * of them and return error
513 */
514 for (j = 0; j < i; j++)
515 page_pp_unlock(ppa[j], 0, 1);
516 for (i = 0; i < npages; i++)
517 page_unlock(ppa[i]);
518 rctl_decr_locked_mem(NULL, proj, lockedbytes, 0);
519 mutex_exit(&sp->shm_mlock);
520 err = ENOMEM;
521 goto out4;
522 }
523 }
524 mutex_exit(&sp->shm_mlock);
525
526 /*
527 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
528 * for the entire life of the segment. For example platforms
529 * that do not support Dynamic Reconfiguration.
530 */
531 hat_flags = HAT_LOAD_SHARE;
532 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
533 hat_flags |= HAT_LOAD_LOCK;
534
535 /*
536 * Load translations one lare page at a time
537 * to make sure we don't create mappings bigger than
538 * segment's size code in case underlying pages
539 * are shared with segvn's segment that uses bigger
540 * size code than we do.
541 */
542 pgsz = page_get_pagesize(seg->s_szc);
543 pgcnt = page_get_pagecnt(seg->s_szc);
544 for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) {
545 sz = MIN(pgsz, ptob(npages - pidx));
546 hat_memload_array(seg->s_as->a_hat, a, sz,
547 &ppa[pidx], sptd->spt_prot, hat_flags);
548 }
549
550 /*
551 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
552 * we will leave the pages locked SE_SHARED for the life
553 * of the ISM segment. This will prevent any calls to
554 * hat_pageunload() on this ISM segment for those platforms.
555 */
556 if (!(hat_flags & HAT_LOAD_LOCK)) {
557 /*
558 * On platforms that support HAT_DYNAMIC_ISM_UNMAP,
559 * we no longer need to hold the SE_SHARED lock on the pages,
560 * since L_PAGELOCK and F_SOFTLOCK calls will grab the
561 * SE_SHARED lock on the pages as necessary.
562 */
563 for (i = 0; i < npages; i++)
564 page_unlock(ppa[i]);
565 }
566 sptd->spt_pcachecnt = 0;
567 kmem_free(ppa, ((sizeof (page_t *)) * npages));
568 sptd->spt_realsize = ptob(npages);
569 atomic_add_long(&spt_used, npages);
570 sptcargs->seg_spt = seg;
571 return (0);
572
573 out4:
574 seg->s_data = NULL;
575 kmem_free(vp, sizeof (*vp));
576 cv_destroy(&sptd->spt_cv);
577 out3:
578 mutex_destroy(&sptd->spt_lock);
579 if ((sptcargs->flags & SHM_PAGEABLE) == 0)
580 kmem_free(ppa, (sizeof (*ppa) * npages));
581 out2:
582 kmem_free(sptd, sizeof (*sptd));
583 out1:
584 if ((sptcargs->flags & SHM_PAGEABLE) == 0)
585 anon_swap_restore(npages);
586 return (err);
587 }
588
589 /*ARGSUSED*/
590 void
591 segspt_free_pages(struct seg *seg, caddr_t addr, size_t len)
592 {
593 struct page *pp;
594 struct spt_data *sptd = (struct spt_data *)seg->s_data;
595 pgcnt_t npages;
596 ulong_t anon_idx;
597 struct anon_map *amp;
598 struct anon *ap;
599 struct vnode *vp;
600 uoff_t off;
601 uint_t hat_flags;
602 int root = 0;
603 pgcnt_t pgs, curnpgs = 0;
604 page_t *rootpp;
605 rctl_qty_t unlocked_bytes = 0;
606 kproject_t *proj;
607 kshmid_t *sp;
608
609 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
610
611 len = P2ROUNDUP(len, PAGESIZE);
612
613 npages = btop(len);
614
615 hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP;
616 if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) ||
617 (sptd->spt_flags & SHM_PAGEABLE)) {
618 hat_flags = HAT_UNLOAD_UNMAP;
619 }
620
621 hat_unload(seg->s_as->a_hat, addr, len, hat_flags);
622
623 amp = sptd->spt_amp;
624 if (sptd->spt_flags & SHM_PAGEABLE)
625 npages = btop(amp->size);
626
627 ASSERT(amp != NULL);
628
629 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
630 sp = amp->a_sp;
631 proj = sp->shm_perm.ipc_proj;
632 mutex_enter(&sp->shm_mlock);
633 }
634 for (anon_idx = 0; anon_idx < npages; anon_idx++) {
635 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
636 if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
637 panic("segspt_free_pages: null app");
638 /*NOTREACHED*/
639 }
640 } else {
641 if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx))
642 == NULL)
643 continue;
644 }
645 ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0);
646 swap_xlate(ap, &vp, &off);
647
648 /*
649 * If this platform supports HAT_DYNAMIC_ISM_UNMAP,
650 * the pages won't be having SE_SHARED lock at this
651 * point.
652 *
653 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
654 * the pages are still held SE_SHARED locked from the
655 * original segspt_create()
656 *
657 * Our goal is to get SE_EXCL lock on each page, remove
658 * permanent lock on it and invalidate the page.
659 */
660 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
661 if (hat_flags == HAT_UNLOAD_UNMAP)
662 pp = page_lookup(vp, off, SE_EXCL);
663 else {
664 if ((pp = page_find(vp, off)) == NULL) {
665 panic("segspt_free_pages: "
666 "page not locked");
667 /*NOTREACHED*/
668 }
669 if (!page_tryupgrade(pp)) {
670 page_unlock(pp);
671 pp = page_lookup(vp, off, SE_EXCL);
672 }
673 }
674 if (pp == NULL) {
675 panic("segspt_free_pages: "
676 "page not in the system");
677 /*NOTREACHED*/
678 }
679 ASSERT(pp->p_lckcnt > 0);
680 page_pp_unlock(pp, 0, 1);
681 if (pp->p_lckcnt == 0)
682 unlocked_bytes += PAGESIZE;
683 } else {
684 if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL)
685 continue;
686 }
687 /*
688 * It's logical to invalidate the pages here as in most cases
689 * these were created by segspt.
690 */
691 if (pp->p_szc != 0) {
692 if (root == 0) {
693 ASSERT(curnpgs == 0);
694 root = 1;
695 rootpp = pp;
696 pgs = curnpgs = page_get_pagecnt(pp->p_szc);
697 ASSERT(pgs > 1);
698 ASSERT(IS_P2ALIGNED(pgs, pgs));
699 ASSERT(!(page_pptonum(pp) & (pgs - 1)));
700 curnpgs--;
701 } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) {
702 ASSERT(curnpgs == 1);
703 ASSERT(page_pptonum(pp) ==
704 page_pptonum(rootpp) + (pgs - 1));
705 page_destroy_pages(rootpp);
706 root = 0;
707 curnpgs = 0;
708 } else {
709 ASSERT(curnpgs > 1);
710 ASSERT(page_pptonum(pp) ==
711 page_pptonum(rootpp) + (pgs - curnpgs));
712 curnpgs--;
713 }
714 } else {
715 if (root != 0 || curnpgs != 0) {
716 panic("segspt_free_pages: bad large page");
717 /*NOTREACHED*/
718 }
719 /*
720 * Before destroying the pages, we need to take care
721 * of the rctl locked memory accounting. For that
722 * we need to calculte the unlocked_bytes.
723 */
724 if (pp->p_lckcnt > 0)
725 unlocked_bytes += PAGESIZE;
726
727 VN_DISPOSE(pp, B_INVAL, 0, kcred);
728 }
729 }
730 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
731 if (unlocked_bytes > 0)
732 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
733 mutex_exit(&sp->shm_mlock);
734 }
735 if (root != 0 || curnpgs != 0) {
736 panic("segspt_free_pages: bad large page");
737 /*NOTREACHED*/
738 }
739
740 /*
741 * mark that pages have been released
742 */
743 sptd->spt_realsize = 0;
744
745 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
746 atomic_add_long(&spt_used, -npages);
747 anon_swap_restore(npages);
748 }
749 }
750
751 /*
752 * Get memory allocation policy info for specified address in given segment
753 */
754 static lgrp_mem_policy_info_t *
755 segspt_getpolicy(struct seg *seg, caddr_t addr)
756 {
757 struct anon_map *amp;
758 ulong_t anon_index;
759 lgrp_mem_policy_info_t *policy_info;
760 struct spt_data *spt_data;
761
762 ASSERT(seg != NULL);
763
764 /*
765 * Get anon_map from segspt
766 *
767 * Assume that no lock needs to be held on anon_map, since
768 * it should be protected by its reference count which must be
769 * nonzero for an existing segment
770 * Need to grab readers lock on policy tree though
771 */
772 spt_data = (struct spt_data *)seg->s_data;
773 if (spt_data == NULL)
774 return (NULL);
775 amp = spt_data->spt_amp;
776 ASSERT(amp->refcnt != 0);
777
778 /*
779 * Get policy info
780 *
781 * Assume starting anon index of 0
782 */
783 anon_index = seg_page(seg, addr);
784 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
785
786 return (policy_info);
787 }
788
789 /*
790 * DISM only.
791 * Return locked pages over a given range.
792 *
793 * We will cache all DISM locked pages and save the pplist for the
794 * entire segment in the ppa field of the underlying DISM segment structure.
795 * Later, during a call to segspt_reclaim() we will use this ppa array
796 * to page_unlock() all of the pages and then we will free this ppa list.
797 */
798 /*ARGSUSED*/
799 static int
800 segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len,
801 struct page ***ppp, enum lock_type type, enum seg_rw rw)
802 {
803 struct shm_data *shmd = (struct shm_data *)seg->s_data;
804 struct seg *sptseg = shmd->shm_sptseg;
805 struct spt_data *sptd = sptseg->s_data;
806 pgcnt_t pg_idx, npages, tot_npages, npgs;
807 struct page **pplist, **pl, **ppa, *pp;
808 struct anon_map *amp;
809 spgcnt_t an_idx;
810 int ret = ENOTSUP;
811 uint_t pl_built = 0;
812 struct anon *ap;
813 struct vnode *vp;
814 uoff_t off;
815 pgcnt_t claim_availrmem = 0;
816 uint_t szc;
817
818 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
819 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
820
821 /*
822 * We want to lock/unlock the entire ISM segment. Therefore,
823 * we will be using the underlying sptseg and it's base address
824 * and length for the caching arguments.
825 */
826 ASSERT(sptseg);
827 ASSERT(sptd);
828
829 pg_idx = seg_page(seg, addr);
830 npages = btopr(len);
831
832 /*
833 * check if the request is larger than number of pages covered
834 * by amp
835 */
836 if (pg_idx + npages > btopr(sptd->spt_amp->size)) {
837 *ppp = NULL;
838 return (ENOTSUP);
839 }
840
841 if (type == L_PAGEUNLOCK) {
842 ASSERT(sptd->spt_ppa != NULL);
843
844 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
845 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
846
847 /*
848 * If someone is blocked while unmapping, we purge
849 * segment page cache and thus reclaim pplist synchronously
850 * without waiting for seg_pasync_thread. This speeds up
851 * unmapping in cases where munmap(2) is called, while
852 * raw async i/o is still in progress or where a thread
853 * exits on data fault in a multithreaded application.
854 */
855 if ((sptd->spt_flags & DISM_PPA_CHANGED) ||
856 (AS_ISUNMAPWAIT(seg->s_as) &&
857 shmd->shm_softlockcnt > 0)) {
858 segspt_purge(seg);
859 }
860 return (0);
861 }
862
863 /* The L_PAGELOCK case ... */
864
865 if (sptd->spt_flags & DISM_PPA_CHANGED) {
866 segspt_purge(seg);
867 /*
868 * for DISM ppa needs to be rebuild since
869 * number of locked pages could be changed
870 */
871 *ppp = NULL;
872 return (ENOTSUP);
873 }
874
875 /*
876 * First try to find pages in segment page cache, without
877 * holding the segment lock.
878 */
879 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
880 S_WRITE, SEGP_FORCE_WIRED);
881 if (pplist != NULL) {
882 ASSERT(sptd->spt_ppa != NULL);
883 ASSERT(sptd->spt_ppa == pplist);
884 ppa = sptd->spt_ppa;
885 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
886 if (ppa[an_idx] == NULL) {
887 seg_pinactive(seg, NULL, seg->s_base,
888 sptd->spt_amp->size, ppa,
889 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
890 *ppp = NULL;
891 return (ENOTSUP);
892 }
893 if ((szc = ppa[an_idx]->p_szc) != 0) {
894 npgs = page_get_pagecnt(szc);
895 an_idx = P2ROUNDUP(an_idx + 1, npgs);
896 } else {
897 an_idx++;
898 }
899 }
900 /*
901 * Since we cache the entire DISM segment, we want to
902 * set ppp to point to the first slot that corresponds
903 * to the requested addr, i.e. pg_idx.
904 */
905 *ppp = &(sptd->spt_ppa[pg_idx]);
906 return (0);
907 }
908
909 mutex_enter(&sptd->spt_lock);
910 /*
911 * try to find pages in segment page cache with mutex
912 */
913 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
914 S_WRITE, SEGP_FORCE_WIRED);
915 if (pplist != NULL) {
916 ASSERT(sptd->spt_ppa != NULL);
917 ASSERT(sptd->spt_ppa == pplist);
918 ppa = sptd->spt_ppa;
919 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
920 if (ppa[an_idx] == NULL) {
921 mutex_exit(&sptd->spt_lock);
922 seg_pinactive(seg, NULL, seg->s_base,
923 sptd->spt_amp->size, ppa,
924 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
925 *ppp = NULL;
926 return (ENOTSUP);
927 }
928 if ((szc = ppa[an_idx]->p_szc) != 0) {
929 npgs = page_get_pagecnt(szc);
930 an_idx = P2ROUNDUP(an_idx + 1, npgs);
931 } else {
932 an_idx++;
933 }
934 }
935 /*
936 * Since we cache the entire DISM segment, we want to
937 * set ppp to point to the first slot that corresponds
938 * to the requested addr, i.e. pg_idx.
939 */
940 mutex_exit(&sptd->spt_lock);
941 *ppp = &(sptd->spt_ppa[pg_idx]);
942 return (0);
943 }
944 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
945 SEGP_FORCE_WIRED) == SEGP_FAIL) {
946 mutex_exit(&sptd->spt_lock);
947 *ppp = NULL;
948 return (ENOTSUP);
949 }
950
951 /*
952 * No need to worry about protections because DISM pages are always rw.
953 */
954 pl = pplist = NULL;
955 amp = sptd->spt_amp;
956
957 /*
958 * Do we need to build the ppa array?
959 */
960 if (sptd->spt_ppa == NULL) {
961 pgcnt_t lpg_cnt = 0;
962
963 pl_built = 1;
964 tot_npages = btopr(sptd->spt_amp->size);
965
966 ASSERT(sptd->spt_pcachecnt == 0);
967 pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP);
968 pl = pplist;
969
970 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
971 for (an_idx = 0; an_idx < tot_npages; ) {
972 ap = anon_get_ptr(amp->ahp, an_idx);
973 /*
974 * Cache only mlocked pages. For large pages
975 * if one (constituent) page is mlocked
976 * all pages for that large page
977 * are cached also. This is for quick
978 * lookups of ppa array;
979 */
980 if ((ap != NULL) && (lpg_cnt != 0 ||
981 (sptd->spt_ppa_lckcnt[an_idx] != 0))) {
982
983 swap_xlate(ap, &vp, &off);
984 pp = page_lookup(vp, off, SE_SHARED);
985 ASSERT(pp != NULL);
986 if (lpg_cnt == 0) {
987 lpg_cnt++;
988 /*
989 * For a small page, we are done --
990 * lpg_count is reset to 0 below.
991 *
992 * For a large page, we are guaranteed
993 * to find the anon structures of all
994 * constituent pages and a non-zero
995 * lpg_cnt ensures that we don't test
996 * for mlock for these. We are done
997 * when lpg_count reaches (npgs + 1).
998 * If we are not the first constituent
999 * page, restart at the first one.
1000 */
1001 npgs = page_get_pagecnt(pp->p_szc);
1002 if (!IS_P2ALIGNED(an_idx, npgs)) {
1003 an_idx = P2ALIGN(an_idx, npgs);
1004 page_unlock(pp);
1005 continue;
1006 }
1007 }
1008 if (++lpg_cnt > npgs)
1009 lpg_cnt = 0;
1010
1011 /*
1012 * availrmem is decremented only
1013 * for unlocked pages
1014 */
1015 if (sptd->spt_ppa_lckcnt[an_idx] == 0)
1016 claim_availrmem++;
1017 pplist[an_idx] = pp;
1018 }
1019 an_idx++;
1020 }
1021 ANON_LOCK_EXIT(&amp->a_rwlock);
1022
1023 if (claim_availrmem) {
1024 mutex_enter(&freemem_lock);
1025 if (availrmem < tune.t_minarmem + claim_availrmem) {
1026 mutex_exit(&freemem_lock);
1027 ret = ENOTSUP;
1028 claim_availrmem = 0;
1029 goto insert_fail;
1030 } else {
1031 availrmem -= claim_availrmem;
1032 }
1033 mutex_exit(&freemem_lock);
1034 }
1035
1036 sptd->spt_ppa = pl;
1037 } else {
1038 /*
1039 * We already have a valid ppa[].
1040 */
1041 pl = sptd->spt_ppa;
1042 }
1043
1044 ASSERT(pl != NULL);
1045
1046 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1047 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
1048 segspt_reclaim);
1049 if (ret == SEGP_FAIL) {
1050 /*
1051 * seg_pinsert failed. We return
1052 * ENOTSUP, so that the as_pagelock() code will
1053 * then try the slower F_SOFTLOCK path.
1054 */
1055 if (pl_built) {
1056 /*
1057 * No one else has referenced the ppa[].
1058 * We created it and we need to destroy it.
1059 */
1060 sptd->spt_ppa = NULL;
1061 }
1062 ret = ENOTSUP;
1063 goto insert_fail;
1064 }
1065
1066 /*
1067 * In either case, we increment softlockcnt on the 'real' segment.
1068 */
1069 sptd->spt_pcachecnt++;
1070 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1071
1072 ppa = sptd->spt_ppa;
1073 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
1074 if (ppa[an_idx] == NULL) {
1075 mutex_exit(&sptd->spt_lock);
1076 seg_pinactive(seg, NULL, seg->s_base,
1077 sptd->spt_amp->size,
1078 pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
1079 *ppp = NULL;
1080 return (ENOTSUP);
1081 }
1082 if ((szc = ppa[an_idx]->p_szc) != 0) {
1083 npgs = page_get_pagecnt(szc);
1084 an_idx = P2ROUNDUP(an_idx + 1, npgs);
1085 } else {
1086 an_idx++;
1087 }
1088 }
1089 /*
1090 * We can now drop the sptd->spt_lock since the ppa[]
1091 * exists and we have incremented pacachecnt.
1092 */
1093 mutex_exit(&sptd->spt_lock);
1094
1095 /*
1096 * Since we cache the entire segment, we want to
1097 * set ppp to point to the first slot that corresponds
1098 * to the requested addr, i.e. pg_idx.
1099 */
1100 *ppp = &(sptd->spt_ppa[pg_idx]);
1101 return (0);
1102
1103 insert_fail:
1104 /*
1105 * We will only reach this code if we tried and failed.
1106 *
1107 * And we can drop the lock on the dummy seg, once we've failed
1108 * to set up a new ppa[].
1109 */
1110 mutex_exit(&sptd->spt_lock);
1111
1112 if (pl_built) {
1113 if (claim_availrmem) {
1114 mutex_enter(&freemem_lock);
1115 availrmem += claim_availrmem;
1116 mutex_exit(&freemem_lock);
1117 }
1118
1119 /*
1120 * We created pl and we need to destroy it.
1121 */
1122 pplist = pl;
1123 for (an_idx = 0; an_idx < tot_npages; an_idx++) {
1124 if (pplist[an_idx] != NULL)
1125 page_unlock(pplist[an_idx]);
1126 }
1127 kmem_free(pl, sizeof (page_t *) * tot_npages);
1128 }
1129
1130 if (shmd->shm_softlockcnt <= 0) {
1131 if (AS_ISUNMAPWAIT(seg->s_as)) {
1132 mutex_enter(&seg->s_as->a_contents);
1133 if (AS_ISUNMAPWAIT(seg->s_as)) {
1134 AS_CLRUNMAPWAIT(seg->s_as);
1135 cv_broadcast(&seg->s_as->a_cv);
1136 }
1137 mutex_exit(&seg->s_as->a_contents);
1138 }
1139 }
1140 *ppp = NULL;
1141 return (ret);
1142 }
1143
1144
1145
1146 /*
1147 * return locked pages over a given range.
1148 *
1149 * We will cache the entire ISM segment and save the pplist for the
1150 * entire segment in the ppa field of the underlying ISM segment structure.
1151 * Later, during a call to segspt_reclaim() we will use this ppa array
1152 * to page_unlock() all of the pages and then we will free this ppa list.
1153 */
1154 /*ARGSUSED*/
1155 static int
1156 segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len,
1157 struct page ***ppp, enum lock_type type, enum seg_rw rw)
1158 {
1159 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1160 struct seg *sptseg = shmd->shm_sptseg;
1161 struct spt_data *sptd = sptseg->s_data;
1162 pgcnt_t np, page_index, npages;
1163 caddr_t a, spt_base;
1164 struct page **pplist, **pl, *pp;
1165 struct anon_map *amp;
1166 ulong_t anon_index;
1167 int ret = ENOTSUP;
1168 uint_t pl_built = 0;
1169 struct anon *ap;
1170 struct vnode *vp;
1171 uoff_t off;
1172
1173 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1174 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
1175
1176
1177 /*
1178 * We want to lock/unlock the entire ISM segment. Therefore,
1179 * we will be using the underlying sptseg and it's base address
1180 * and length for the caching arguments.
1181 */
1182 ASSERT(sptseg);
1183 ASSERT(sptd);
1184
1185 if (sptd->spt_flags & SHM_PAGEABLE) {
1186 return (segspt_dismpagelock(seg, addr, len, ppp, type, rw));
1187 }
1188
1189 page_index = seg_page(seg, addr);
1190 npages = btopr(len);
1191
1192 /*
1193 * check if the request is larger than number of pages covered
1194 * by amp
1195 */
1196 if (page_index + npages > btopr(sptd->spt_amp->size)) {
1197 *ppp = NULL;
1198 return (ENOTSUP);
1199 }
1200
1201 if (type == L_PAGEUNLOCK) {
1202
1203 ASSERT(sptd->spt_ppa != NULL);
1204
1205 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
1206 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
1207
1208 /*
1209 * If someone is blocked while unmapping, we purge
1210 * segment page cache and thus reclaim pplist synchronously
1211 * without waiting for seg_pasync_thread. This speeds up
1212 * unmapping in cases where munmap(2) is called, while
1213 * raw async i/o is still in progress or where a thread
1214 * exits on data fault in a multithreaded application.
1215 */
1216 if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
1217 segspt_purge(seg);
1218 }
1219 return (0);
1220 }
1221
1222 /* The L_PAGELOCK case... */
1223
1224 /*
1225 * First try to find pages in segment page cache, without
1226 * holding the segment lock.
1227 */
1228 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1229 S_WRITE, SEGP_FORCE_WIRED);
1230 if (pplist != NULL) {
1231 ASSERT(sptd->spt_ppa == pplist);
1232 ASSERT(sptd->spt_ppa[page_index]);
1233 /*
1234 * Since we cache the entire ISM segment, we want to
1235 * set ppp to point to the first slot that corresponds
1236 * to the requested addr, i.e. page_index.
1237 */
1238 *ppp = &(sptd->spt_ppa[page_index]);
1239 return (0);
1240 }
1241
1242 mutex_enter(&sptd->spt_lock);
1243
1244 /*
1245 * try to find pages in segment page cache
1246 */
1247 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1248 S_WRITE, SEGP_FORCE_WIRED);
1249 if (pplist != NULL) {
1250 ASSERT(sptd->spt_ppa == pplist);
1251 /*
1252 * Since we cache the entire segment, we want to
1253 * set ppp to point to the first slot that corresponds
1254 * to the requested addr, i.e. page_index.
1255 */
1256 mutex_exit(&sptd->spt_lock);
1257 *ppp = &(sptd->spt_ppa[page_index]);
1258 return (0);
1259 }
1260
1261 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
1262 SEGP_FORCE_WIRED) == SEGP_FAIL) {
1263 mutex_exit(&sptd->spt_lock);
1264 *ppp = NULL;
1265 return (ENOTSUP);
1266 }
1267
1268 /*
1269 * No need to worry about protections because ISM pages
1270 * are always rw.
1271 */
1272 pl = pplist = NULL;
1273
1274 /*
1275 * Do we need to build the ppa array?
1276 */
1277 if (sptd->spt_ppa == NULL) {
1278 ASSERT(sptd->spt_ppa == pplist);
1279
1280 spt_base = sptseg->s_base;
1281 pl_built = 1;
1282
1283 /*
1284 * availrmem is decremented once during anon_swap_adjust()
1285 * and is incremented during the anon_unresv(), which is
1286 * called from shm_rm_amp() when the segment is destroyed.
1287 */
1288 amp = sptd->spt_amp;
1289 ASSERT(amp != NULL);
1290
1291 /* pcachecnt is protected by sptd->spt_lock */
1292 ASSERT(sptd->spt_pcachecnt == 0);
1293 pplist = kmem_zalloc(sizeof (page_t *)
1294 * btopr(sptd->spt_amp->size), KM_SLEEP);
1295 pl = pplist;
1296
1297 anon_index = seg_page(sptseg, spt_base);
1298
1299 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
1300 for (a = spt_base; a < (spt_base + sptd->spt_amp->size);
1301 a += PAGESIZE, anon_index++, pplist++) {
1302 ap = anon_get_ptr(amp->ahp, anon_index);
1303 ASSERT(ap != NULL);
1304 swap_xlate(ap, &vp, &off);
1305 pp = page_lookup(vp, off, SE_SHARED);
1306 ASSERT(pp != NULL);
1307 *pplist = pp;
1308 }
1309 ANON_LOCK_EXIT(&amp->a_rwlock);
1310
1311 if (a < (spt_base + sptd->spt_amp->size)) {
1312 ret = ENOTSUP;
1313 goto insert_fail;
1314 }
1315 sptd->spt_ppa = pl;
1316 } else {
1317 /*
1318 * We already have a valid ppa[].
1319 */
1320 pl = sptd->spt_ppa;
1321 }
1322
1323 ASSERT(pl != NULL);
1324
1325 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1326 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
1327 segspt_reclaim);
1328 if (ret == SEGP_FAIL) {
1329 /*
1330 * seg_pinsert failed. We return
1331 * ENOTSUP, so that the as_pagelock() code will
1332 * then try the slower F_SOFTLOCK path.
1333 */
1334 if (pl_built) {
1335 /*
1336 * No one else has referenced the ppa[].
1337 * We created it and we need to destroy it.
1338 */
1339 sptd->spt_ppa = NULL;
1340 }
1341 ret = ENOTSUP;
1342 goto insert_fail;
1343 }
1344
1345 /*
1346 * In either case, we increment softlockcnt on the 'real' segment.
1347 */
1348 sptd->spt_pcachecnt++;
1349 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1350
1351 /*
1352 * We can now drop the sptd->spt_lock since the ppa[]
1353 * exists and we have incremented pacachecnt.
1354 */
1355 mutex_exit(&sptd->spt_lock);
1356
1357 /*
1358 * Since we cache the entire segment, we want to
1359 * set ppp to point to the first slot that corresponds
1360 * to the requested addr, i.e. page_index.
1361 */
1362 *ppp = &(sptd->spt_ppa[page_index]);
1363 return (0);
1364
1365 insert_fail:
1366 /*
1367 * We will only reach this code if we tried and failed.
1368 *
1369 * And we can drop the lock on the dummy seg, once we've failed
1370 * to set up a new ppa[].
1371 */
1372 mutex_exit(&sptd->spt_lock);
1373
1374 if (pl_built) {
1375 /*
1376 * We created pl and we need to destroy it.
1377 */
1378 pplist = pl;
1379 np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT);
1380 while (np) {
1381 page_unlock(*pplist);
1382 np--;
1383 pplist++;
1384 }
1385 kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size));
1386 }
1387 if (shmd->shm_softlockcnt <= 0) {
1388 if (AS_ISUNMAPWAIT(seg->s_as)) {
1389 mutex_enter(&seg->s_as->a_contents);
1390 if (AS_ISUNMAPWAIT(seg->s_as)) {
1391 AS_CLRUNMAPWAIT(seg->s_as);
1392 cv_broadcast(&seg->s_as->a_cv);
1393 }
1394 mutex_exit(&seg->s_as->a_contents);
1395 }
1396 }
1397 *ppp = NULL;
1398 return (ret);
1399 }
1400
1401 /*
1402 * purge any cached pages in the I/O page cache
1403 */
1404 static void
1405 segspt_purge(struct seg *seg)
1406 {
1407 seg_ppurge(seg, NULL, SEGP_FORCE_WIRED);
1408 }
1409
1410 static int
1411 segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist,
1412 enum seg_rw rw, int async)
1413 {
1414 struct seg *seg = (struct seg *)ptag;
1415 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1416 struct seg *sptseg;
1417 struct spt_data *sptd;
1418 pgcnt_t npages, i, free_availrmem = 0;
1419 int done = 0;
1420
1421 sptseg = shmd->shm_sptseg;
1422 sptd = sptseg->s_data;
1423 npages = (len >> PAGESHIFT);
1424 ASSERT(npages);
1425 ASSERT(sptd->spt_pcachecnt != 0);
1426 ASSERT(sptd->spt_ppa == pplist);
1427 ASSERT(npages == btopr(sptd->spt_amp->size));
1428 ASSERT(async || AS_LOCK_HELD(seg->s_as));
1429
1430 /*
1431 * Acquire the lock on the dummy seg and destroy the
1432 * ppa array IF this is the last pcachecnt.
1433 */
1434 mutex_enter(&sptd->spt_lock);
1435 if (--sptd->spt_pcachecnt == 0) {
1436 for (i = 0; i < npages; i++) {
1437 if (pplist[i] == NULL) {
1438 continue;
1439 }
1440 if (rw == S_WRITE) {
1441 hat_setrefmod(pplist[i]);
1442 } else {
1443 hat_setref(pplist[i]);
1444 }
1445 if ((sptd->spt_flags & SHM_PAGEABLE) &&
1446 (sptd->spt_ppa_lckcnt[i] == 0))
1447 free_availrmem++;
1448 page_unlock(pplist[i]);
1449 }
1450 if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) {
1451 mutex_enter(&freemem_lock);
1452 availrmem += free_availrmem;
1453 mutex_exit(&freemem_lock);
1454 }
1455 /*
1456 * Since we want to cach/uncache the entire ISM segment,
1457 * we will track the pplist in a segspt specific field
1458 * ppa, that is initialized at the time we add an entry to
1459 * the cache.
1460 */
1461 ASSERT(sptd->spt_pcachecnt == 0);
1462 kmem_free(pplist, sizeof (page_t *) * npages);
1463 sptd->spt_ppa = NULL;
1464 sptd->spt_flags &= ~DISM_PPA_CHANGED;
1465 sptd->spt_gen++;
1466 cv_broadcast(&sptd->spt_cv);
1467 done = 1;
1468 }
1469 mutex_exit(&sptd->spt_lock);
1470
1471 /*
1472 * If we are pcache async thread or called via seg_ppurge_wiredpp() we
1473 * may not hold AS lock (in this case async argument is not 0). This
1474 * means if softlockcnt drops to 0 after the decrement below address
1475 * space may get freed. We can't allow it since after softlock
1476 * derement to 0 we still need to access as structure for possible
1477 * wakeup of unmap waiters. To prevent the disappearance of as we take
1478 * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes
1479 * this mutex as a barrier to make sure this routine completes before
1480 * segment is freed.
1481 *
1482 * The second complication we have to deal with in async case is a
1483 * possibility of missed wake up of unmap wait thread. When we don't
1484 * hold as lock here we may take a_contents lock before unmap wait
1485 * thread that was first to see softlockcnt was still not 0. As a
1486 * result we'll fail to wake up an unmap wait thread. To avoid this
1487 * race we set nounmapwait flag in as structure if we drop softlockcnt
1488 * to 0 if async is not 0. unmapwait thread
1489 * will not block if this flag is set.
1490 */
1491 if (async)
1492 mutex_enter(&shmd->shm_segfree_syncmtx);
1493
1494 /*
1495 * Now decrement softlockcnt.
1496 */
1497 ASSERT(shmd->shm_softlockcnt > 0);
1498 atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1499
1500 if (shmd->shm_softlockcnt <= 0) {
1501 if (async || AS_ISUNMAPWAIT(seg->s_as)) {
1502 mutex_enter(&seg->s_as->a_contents);
1503 if (async)
1504 AS_SETNOUNMAPWAIT(seg->s_as);
1505 if (AS_ISUNMAPWAIT(seg->s_as)) {
1506 AS_CLRUNMAPWAIT(seg->s_as);
1507 cv_broadcast(&seg->s_as->a_cv);
1508 }
1509 mutex_exit(&seg->s_as->a_contents);
1510 }
1511 }
1512
1513 if (async)
1514 mutex_exit(&shmd->shm_segfree_syncmtx);
1515
1516 return (done);
1517 }
1518
1519 /*
1520 * Do a F_SOFTUNLOCK call over the range requested.
1521 * The range must have already been F_SOFTLOCK'ed.
1522 *
1523 * The calls to acquire and release the anon map lock mutex were
1524 * removed in order to avoid a deadly embrace during a DR
1525 * memory delete operation. (Eg. DR blocks while waiting for a
1526 * exclusive lock on a page that is being used for kaio; the
1527 * thread that will complete the kaio and call segspt_softunlock
1528 * blocks on the anon map lock; another thread holding the anon
1529 * map lock blocks on another page lock via the segspt_shmfault
1530 * -> page_lookup -> page_lookup_create -> page_lock_es code flow.)
1531 *
1532 * The appropriateness of the removal is based upon the following:
1533 * 1. If we are holding a segment's reader lock and the page is held
1534 * shared, then the corresponding element in anonmap which points to
1535 * anon struct cannot change and there is no need to acquire the
1536 * anonymous map lock.
1537 * 2. Threads in segspt_softunlock have a reader lock on the segment
1538 * and already have the shared page lock, so we are guaranteed that
1539 * the anon map slot cannot change and therefore can call anon_get_ptr()
1540 * without grabbing the anonymous map lock.
1541 * 3. Threads that softlock a shared page break copy-on-write, even if
1542 * its a read. Thus cow faults can be ignored with respect to soft
1543 * unlocking, since the breaking of cow means that the anon slot(s) will
1544 * not be shared.
1545 */
1546 static void
1547 segspt_softunlock(struct seg *seg, caddr_t sptseg_addr,
1548 size_t len, enum seg_rw rw)
1549 {
1550 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1551 struct seg *sptseg;
1552 struct spt_data *sptd;
1553 page_t *pp;
1554 caddr_t adr;
1555 struct vnode *vp;
1556 uoff_t offset;
1557 ulong_t anon_index;
1558 struct anon_map *amp; /* XXX - for locknest */
1559 struct anon *ap = NULL;
1560 pgcnt_t npages;
1561
1562 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1563
1564 sptseg = shmd->shm_sptseg;
1565 sptd = sptseg->s_data;
1566
1567 /*
1568 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
1569 * and therefore their pages are SE_SHARED locked
1570 * for the entire life of the segment.
1571 */
1572 if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) &&
1573 ((sptd->spt_flags & SHM_PAGEABLE) == 0)) {
1574 goto softlock_decrement;
1575 }
1576
1577 /*
1578 * Any thread is free to do a page_find and
1579 * page_unlock() on the pages within this seg.
1580 *
1581 * We are already holding the as->a_lock on the user's
1582 * real segment, but we need to hold the a_lock on the
1583 * underlying dummy as. This is mostly to satisfy the
1584 * underlying HAT layer.
1585 */
1586 AS_LOCK_ENTER(sptseg->s_as, RW_READER);
1587 hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len);
1588 AS_LOCK_EXIT(sptseg->s_as);
1589
1590 amp = sptd->spt_amp;
1591 ASSERT(amp != NULL);
1592 anon_index = seg_page(sptseg, sptseg_addr);
1593
1594 for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) {
1595 ap = anon_get_ptr(amp->ahp, anon_index++);
1596 ASSERT(ap != NULL);
1597 swap_xlate(ap, &vp, &offset);
1598
1599 /*
1600 * Use page_find() instead of page_lookup() to
1601 * find the page since we know that it has a
1602 * "shared" lock.
1603 */
1604 pp = page_find(vp, offset);
1605 ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1));
1606 if (pp == NULL) {
1607 panic("segspt_softunlock: "
1608 "addr %p, ap %p, vp %p, off %llx",
1609 (void *)adr, (void *)ap, (void *)vp, offset);
1610 /*NOTREACHED*/
1611 }
1612
1613 if (rw == S_WRITE) {
1614 hat_setrefmod(pp);
1615 } else if (rw != S_OTHER) {
1616 hat_setref(pp);
1617 }
1618 page_unlock(pp);
1619 }
1620
1621 softlock_decrement:
1622 npages = btopr(len);
1623 ASSERT(shmd->shm_softlockcnt >= npages);
1624 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages);
1625 if (shmd->shm_softlockcnt == 0) {
1626 /*
1627 * All SOFTLOCKS are gone. Wakeup any waiting
1628 * unmappers so they can try again to unmap.
1629 * Check for waiters first without the mutex
1630 * held so we don't always grab the mutex on
1631 * softunlocks.
1632 */
1633 if (AS_ISUNMAPWAIT(seg->s_as)) {
1634 mutex_enter(&seg->s_as->a_contents);
1635 if (AS_ISUNMAPWAIT(seg->s_as)) {
1636 AS_CLRUNMAPWAIT(seg->s_as);
1637 cv_broadcast(&seg->s_as->a_cv);
1638 }
1639 mutex_exit(&seg->s_as->a_contents);
1640 }
1641 }
1642 }
1643
1644 int
1645 segspt_shmattach(struct seg *seg, caddr_t *argsp)
1646 {
1647 struct shm_data *shmd_arg = (struct shm_data *)argsp;
1648 struct shm_data *shmd;
1649 struct anon_map *shm_amp = shmd_arg->shm_amp;
1650 struct spt_data *sptd;
1651 int error = 0;
1652
1653 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1654
1655 shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP);
1656 if (shmd == NULL)
1657 return (ENOMEM);
1658
1659 shmd->shm_sptas = shmd_arg->shm_sptas;
1660 shmd->shm_amp = shm_amp;
1661 shmd->shm_sptseg = shmd_arg->shm_sptseg;
1662
1663 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0,
1664 NULL, 0, seg->s_size);
1665
1666 mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL);
1667
1668 seg->s_data = (void *)shmd;
1669 seg->s_ops = &segspt_shmops;
1670 seg->s_szc = shmd->shm_sptseg->s_szc;
1671 sptd = shmd->shm_sptseg->s_data;
1672
1673 if (sptd->spt_flags & SHM_PAGEABLE) {
1674 if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size),
1675 KM_NOSLEEP)) == NULL) {
1676 seg->s_data = NULL;
1677 kmem_free(shmd, (sizeof (*shmd)));
1678 return (ENOMEM);
1679 }
1680 shmd->shm_lckpgs = 0;
1681 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) {
1682 if ((error = hat_share(seg->s_as->a_hat, seg->s_base,
1683 shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
1684 seg->s_size, seg->s_szc)) != 0) {
1685 kmem_free(shmd->shm_vpage,
1686 btopr(shm_amp->size));
1687 }
1688 }
1689 } else {
1690 error = hat_share(seg->s_as->a_hat, seg->s_base,
1691 shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
1692 seg->s_size, seg->s_szc);
1693 }
1694 if (error) {
1695 seg->s_szc = 0;
1696 seg->s_data = NULL;
1697 kmem_free(shmd, (sizeof (*shmd)));
1698 } else {
1699 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
1700 shm_amp->refcnt++;
1701 ANON_LOCK_EXIT(&shm_amp->a_rwlock);
1702 }
1703 return (error);
1704 }
1705
1706 int
1707 segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize)
1708 {
1709 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1710 int reclaim = 1;
1711
1712 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1713 retry:
1714 if (shmd->shm_softlockcnt > 0) {
1715 if (reclaim == 1) {
1716 segspt_purge(seg);
1717 reclaim = 0;
1718 goto retry;
1719 }
1720 return (EAGAIN);
1721 }
1722
1723 if (ssize != seg->s_size) {
1724 #ifdef DEBUG
1725 cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n",
1726 ssize, seg->s_size);
1727 #endif
1728 return (EINVAL);
1729 }
1730
1731 (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK,
1732 NULL, 0);
1733 hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc);
1734
1735 seg_free(seg);
1736
1737 return (0);
1738 }
1739
1740 void
1741 segspt_shmfree(struct seg *seg)
1742 {
1743 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1744 struct anon_map *shm_amp = shmd->shm_amp;
1745
1746 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1747
1748 (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0,
1749 MC_UNLOCK, NULL, 0);
1750
1751 /*
1752 * Need to increment refcnt when attaching
1753 * and decrement when detaching because of dup().
1754 */
1755 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
1756 shm_amp->refcnt--;
1757 ANON_LOCK_EXIT(&shm_amp->a_rwlock);
1758
1759 if (shmd->shm_vpage) { /* only for DISM */
1760 kmem_free(shmd->shm_vpage, btopr(shm_amp->size));
1761 shmd->shm_vpage = NULL;
1762 }
1763
1764 /*
1765 * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's
1766 * still working with this segment without holding as lock.
1767 */
1768 ASSERT(shmd->shm_softlockcnt == 0);
1769 mutex_enter(&shmd->shm_segfree_syncmtx);
1770 mutex_destroy(&shmd->shm_segfree_syncmtx);
1771
1772 kmem_free(shmd, sizeof (*shmd));
1773 }
1774
1775 /*ARGSUSED*/
1776 int
1777 segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
1778 {
1779 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1780
1781 /*
1782 * Shared page table is more than shared mapping.
1783 * Individual process sharing page tables can't change prot
1784 * because there is only one set of page tables.
1785 * This will be allowed after private page table is
1786 * supported.
1787 */
1788 /* need to return correct status error? */
1789 return (0);
1790 }
1791
1792
1793 faultcode_t
1794 segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr,
1795 size_t len, enum fault_type type, enum seg_rw rw)
1796 {
1797 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1798 struct seg *sptseg = shmd->shm_sptseg;
1799 struct as *curspt = shmd->shm_sptas;
1800 struct spt_data *sptd = sptseg->s_data;
1801 pgcnt_t npages;
1802 size_t size;
1803 caddr_t segspt_addr, shm_addr;
1804 page_t **ppa;
1805 int i;
1806 ulong_t an_idx = 0;
1807 int err = 0;
1808 int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL);
1809 size_t pgsz;
1810 pgcnt_t pgcnt;
1811 caddr_t a;
1812 pgcnt_t pidx;
1813
1814 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1815
1816 /*
1817 * Because of the way spt is implemented
1818 * the realsize of the segment does not have to be
1819 * equal to the segment size itself. The segment size is
1820 * often in multiples of a page size larger than PAGESIZE.
1821 * The realsize is rounded up to the nearest PAGESIZE
1822 * based on what the user requested. This is a bit of
1823 * ungliness that is historical but not easily fixed
1824 * without re-designing the higher levels of ISM.
1825 */
1826 ASSERT(addr >= seg->s_base);
1827 if (((addr + len) - seg->s_base) > sptd->spt_realsize)
1828 return (FC_NOMAP);
1829 /*
1830 * For all of the following cases except F_PROT, we need to
1831 * make any necessary adjustments to addr and len
1832 * and get all of the necessary page_t's into an array called ppa[].
1833 *
1834 * The code in shmat() forces base addr and len of ISM segment
1835 * to be aligned to largest page size supported. Therefore,
1836 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
1837 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
1838 * in large pagesize chunks, or else we will screw up the HAT
1839 * layer by calling hat_memload_array() with differing page sizes
1840 * over a given virtual range.
1841 */
1842 pgsz = page_get_pagesize(sptseg->s_szc);
1843 pgcnt = page_get_pagecnt(sptseg->s_szc);
1844 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
1845 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
1846 npages = btopr(size);
1847
1848 /*
1849 * Now we need to convert from addr in segshm to addr in segspt.
1850 */
1851 an_idx = seg_page(seg, shm_addr);
1852 segspt_addr = sptseg->s_base + ptob(an_idx);
1853
1854 ASSERT((segspt_addr + ptob(npages)) <=
1855 (sptseg->s_base + sptd->spt_realsize));
1856 ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size));
1857
1858 switch (type) {
1859
1860 case F_SOFTLOCK:
1861
1862 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
1863 /*
1864 * Fall through to the F_INVAL case to load up the hat layer
1865 * entries with the HAT_LOAD_LOCK flag.
1866 */
1867 /* FALLTHRU */
1868 case F_INVAL:
1869
1870 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
1871 return (FC_NOMAP);
1872
1873 ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);
1874
1875 err = spt_anon_getpages(sptseg, segspt_addr, size, ppa);
1876 if (err != 0) {
1877 if (type == F_SOFTLOCK) {
1878 atomic_add_long((ulong_t *)(
1879 &(shmd->shm_softlockcnt)), -npages);
1880 }
1881 goto dism_err;
1882 }
1883 AS_LOCK_ENTER(sptseg->s_as, RW_READER);
1884 a = segspt_addr;
1885 pidx = 0;
1886 if (type == F_SOFTLOCK) {
1887
1888 /*
1889 * Load up the translation keeping it
1890 * locked and don't unlock the page.
1891 */
1892 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
1893 hat_memload_array(sptseg->s_as->a_hat,
1894 a, pgsz, &ppa[pidx], sptd->spt_prot,
1895 HAT_LOAD_LOCK | HAT_LOAD_SHARE);
1896 }
1897 } else {
1898 /*
1899 * Migrate pages marked for migration
1900 */
1901 if (lgrp_optimizations())
1902 page_migrate(seg, shm_addr, ppa, npages);
1903
1904 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
1905 hat_memload_array(sptseg->s_as->a_hat,
1906 a, pgsz, &ppa[pidx],
1907 sptd->spt_prot,
1908 HAT_LOAD_SHARE);
1909 }
1910
1911 /*
1912 * And now drop the SE_SHARED lock(s).
1913 */
1914 if (dyn_ism_unmap) {
1915 for (i = 0; i < npages; i++) {
1916 page_unlock(ppa[i]);
1917 }
1918 }
1919 }
1920
1921 if (!dyn_ism_unmap) {
1922 if (hat_share(seg->s_as->a_hat, shm_addr,
1923 curspt->a_hat, segspt_addr, ptob(npages),
1924 seg->s_szc) != 0) {
1925 panic("hat_share err in DISM fault");
1926 /* NOTREACHED */
1927 }
1928 if (type == F_INVAL) {
1929 for (i = 0; i < npages; i++) {
1930 page_unlock(ppa[i]);
1931 }
1932 }
1933 }
1934 AS_LOCK_EXIT(sptseg->s_as);
1935 dism_err:
1936 kmem_free(ppa, npages * sizeof (page_t *));
1937 return (err);
1938
1939 case F_SOFTUNLOCK:
1940
1941 /*
1942 * This is a bit ugly, we pass in the real seg pointer,
1943 * but the segspt_addr is the virtual address within the
1944 * dummy seg.
1945 */
1946 segspt_softunlock(seg, segspt_addr, size, rw);
1947 return (0);
1948
1949 case F_PROT:
1950
1951 /*
1952 * This takes care of the unusual case where a user
1953 * allocates a stack in shared memory and a register
1954 * window overflow is written to that stack page before
1955 * it is otherwise modified.
1956 *
1957 * We can get away with this because ISM segments are
1958 * always rw. Other than this unusual case, there
1959 * should be no instances of protection violations.
1960 */
1961 return (0);
1962
1963 default:
1964 #ifdef DEBUG
1965 panic("segspt_dismfault default type?");
1966 #else
1967 return (FC_NOMAP);
1968 #endif
1969 }
1970 }
1971
1972
1973 faultcode_t
1974 segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr,
1975 size_t len, enum fault_type type, enum seg_rw rw)
1976 {
1977 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1978 struct seg *sptseg = shmd->shm_sptseg;
1979 struct as *curspt = shmd->shm_sptas;
1980 struct spt_data *sptd = sptseg->s_data;
1981 pgcnt_t npages;
1982 size_t size;
1983 caddr_t sptseg_addr, shm_addr;
1984 page_t *pp, **ppa;
1985 int i;
1986 uoff_t offset;
1987 ulong_t anon_index = 0;
1988 struct vnode *vp;
1989 struct anon_map *amp; /* XXX - for locknest */
1990 struct anon *ap = NULL;
1991 size_t pgsz;
1992 pgcnt_t pgcnt;
1993 caddr_t a;
1994 pgcnt_t pidx;
1995 size_t sz;
1996
1997
1998 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1999
2000 if (sptd->spt_flags & SHM_PAGEABLE) {
2001 return (segspt_dismfault(hat, seg, addr, len, type, rw));
2002 }
2003
2004 /*
2005 * Because of the way spt is implemented
2006 * the realsize of the segment does not have to be
2007 * equal to the segment size itself. The segment size is
2008 * often in multiples of a page size larger than PAGESIZE.
2009 * The realsize is rounded up to the nearest PAGESIZE
2010 * based on what the user requested. This is a bit of
2011 * ungliness that is historical but not easily fixed
2012 * without re-designing the higher levels of ISM.
2013 */
2014 ASSERT(addr >= seg->s_base);
2015 if (((addr + len) - seg->s_base) > sptd->spt_realsize)
2016 return (FC_NOMAP);
2017 /*
2018 * For all of the following cases except F_PROT, we need to
2019 * make any necessary adjustments to addr and len
2020 * and get all of the necessary page_t's into an array called ppa[].
2021 *
2022 * The code in shmat() forces base addr and len of ISM segment
2023 * to be aligned to largest page size supported. Therefore,
2024 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
2025 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
2026 * in large pagesize chunks, or else we will screw up the HAT
2027 * layer by calling hat_memload_array() with differing page sizes
2028 * over a given virtual range.
2029 */
2030 pgsz = page_get_pagesize(sptseg->s_szc);
2031 pgcnt = page_get_pagecnt(sptseg->s_szc);
2032 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
2033 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
2034 npages = btopr(size);
2035
2036 /*
2037 * Now we need to convert from addr in segshm to addr in segspt.
2038 */
2039 anon_index = seg_page(seg, shm_addr);
2040 sptseg_addr = sptseg->s_base + ptob(anon_index);
2041
2042 /*
2043 * And now we may have to adjust npages downward if we have
2044 * exceeded the realsize of the segment or initial anon
2045 * allocations.
2046 */
2047 if ((sptseg_addr + ptob(npages)) >
2048 (sptseg->s_base + sptd->spt_realsize))
2049 size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr;
2050
2051 npages = btopr(size);
2052
2053 ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size));
2054 ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0);
2055
2056 switch (type) {
2057
2058 case F_SOFTLOCK:
2059
2060 /*
2061 * availrmem is decremented once during anon_swap_adjust()
2062 * and is incremented during the anon_unresv(), which is
2063 * called from shm_rm_amp() when the segment is destroyed.
2064 */
2065 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
2066 /*
2067 * Some platforms assume that ISM pages are SE_SHARED
2068 * locked for the entire life of the segment.
2069 */
2070 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
2071 return (0);
2072 /*
2073 * Fall through to the F_INVAL case to load up the hat layer
2074 * entries with the HAT_LOAD_LOCK flag.
2075 */
2076
2077 /* FALLTHRU */
2078 case F_INVAL:
2079
2080 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
2081 return (FC_NOMAP);
2082
2083 /*
2084 * Some platforms that do NOT support DYNAMIC_ISM_UNMAP
2085 * may still rely on this call to hat_share(). That
2086 * would imply that those hat's can fault on a
2087 * HAT_LOAD_LOCK translation, which would seem
2088 * contradictory.
2089 */
2090 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) {
2091 if (hat_share(seg->s_as->a_hat, seg->s_base,
2092 curspt->a_hat, sptseg->s_base,
2093 sptseg->s_size, sptseg->s_szc) != 0) {
2094 panic("hat_share error in ISM fault");
2095 /*NOTREACHED*/
2096 }
2097 return (0);
2098 }
2099 ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP);
2100
2101 /*
2102 * I see no need to lock the real seg,
2103 * here, because all of our work will be on the underlying
2104 * dummy seg.
2105 *
2106 * sptseg_addr and npages now account for large pages.
2107 */
2108 amp = sptd->spt_amp;
2109 ASSERT(amp != NULL);
2110 anon_index = seg_page(sptseg, sptseg_addr);
2111
2112 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2113 for (i = 0; i < npages; i++) {
2114 ap = anon_get_ptr(amp->ahp, anon_index++);
2115 ASSERT(ap != NULL);
2116 swap_xlate(ap, &vp, &offset);
2117 pp = page_lookup(vp, offset, SE_SHARED);
2118 ASSERT(pp != NULL);
2119 ppa[i] = pp;
2120 }
2121 ANON_LOCK_EXIT(&amp->a_rwlock);
2122 ASSERT(i == npages);
2123
2124 /*
2125 * We are already holding the as->a_lock on the user's
2126 * real segment, but we need to hold the a_lock on the
2127 * underlying dummy as. This is mostly to satisfy the
2128 * underlying HAT layer.
2129 */
2130 AS_LOCK_ENTER(sptseg->s_as, RW_READER);
2131 a = sptseg_addr;
2132 pidx = 0;
2133 if (type == F_SOFTLOCK) {
2134 /*
2135 * Load up the translation keeping it
2136 * locked and don't unlock the page.
2137 */
2138 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
2139 sz = MIN(pgsz, ptob(npages - pidx));
2140 hat_memload_array(sptseg->s_as->a_hat, a,
2141 sz, &ppa[pidx], sptd->spt_prot,
2142 HAT_LOAD_LOCK | HAT_LOAD_SHARE);
2143 }
2144 } else {
2145 /*
2146 * Migrate pages marked for migration.
2147 */
2148 if (lgrp_optimizations())
2149 page_migrate(seg, shm_addr, ppa, npages);
2150
2151 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
2152 sz = MIN(pgsz, ptob(npages - pidx));
2153 hat_memload_array(sptseg->s_as->a_hat,
2154 a, sz, &ppa[pidx],
2155 sptd->spt_prot, HAT_LOAD_SHARE);
2156 }
2157
2158 /*
2159 * And now drop the SE_SHARED lock(s).
2160 */
2161 for (i = 0; i < npages; i++)
2162 page_unlock(ppa[i]);
2163 }
2164 AS_LOCK_EXIT(sptseg->s_as);
2165
2166 kmem_free(ppa, sizeof (page_t *) * npages);
2167 return (0);
2168 case F_SOFTUNLOCK:
2169
2170 /*
2171 * This is a bit ugly, we pass in the real seg pointer,
2172 * but the sptseg_addr is the virtual address within the
2173 * dummy seg.
2174 */
2175 segspt_softunlock(seg, sptseg_addr, ptob(npages), rw);
2176 return (0);
2177
2178 case F_PROT:
2179
2180 /*
2181 * This takes care of the unusual case where a user
2182 * allocates a stack in shared memory and a register
2183 * window overflow is written to that stack page before
2184 * it is otherwise modified.
2185 *
2186 * We can get away with this because ISM segments are
2187 * always rw. Other than this unusual case, there
2188 * should be no instances of protection violations.
2189 */
2190 return (0);
2191
2192 default:
2193 #ifdef DEBUG
2194 cmn_err(CE_WARN, "segspt_shmfault default type?");
2195 #endif
2196 return (FC_NOMAP);
2197 }
2198 }
2199
2200 /*ARGSUSED*/
2201 static faultcode_t
2202 segspt_shmfaulta(struct seg *seg, caddr_t addr)
2203 {
2204 return (0);
2205 }
2206
2207 /*ARGSUSED*/
2208 static int
2209 segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta)
2210 {
2211 return (0);
2212 }
2213
2214 /*
2215 * duplicate the shared page tables
2216 */
2217 int
2218 segspt_shmdup(struct seg *seg, struct seg *newseg)
2219 {
2220 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2221 struct anon_map *amp = shmd->shm_amp;
2222 struct shm_data *shmd_new;
2223 struct seg *spt_seg = shmd->shm_sptseg;
2224 struct spt_data *sptd = spt_seg->s_data;
2225 int error = 0;
2226
2227 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
2228
2229 shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP);
2230 newseg->s_data = (void *)shmd_new;
2231 shmd_new->shm_sptas = shmd->shm_sptas;
2232 shmd_new->shm_amp = amp;
2233 shmd_new->shm_sptseg = shmd->shm_sptseg;
2234 newseg->s_ops = &segspt_shmops;
2235 newseg->s_szc = seg->s_szc;
2236 ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc);
2237
2238 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
2239 amp->refcnt++;
2240 ANON_LOCK_EXIT(&amp->a_rwlock);
2241
2242 if (sptd->spt_flags & SHM_PAGEABLE) {
2243 shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP);
2244 shmd_new->shm_lckpgs = 0;
2245 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) {
2246 if ((error = hat_share(newseg->s_as->a_hat,
2247 newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR,
2248 seg->s_size, seg->s_szc)) != 0) {
2249 kmem_free(shmd_new->shm_vpage,
2250 btopr(amp->size));
2251 }
2252 }
2253 return (error);
2254 } else {
2255 return (hat_share(newseg->s_as->a_hat, newseg->s_base,
2256 shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size,
2257 seg->s_szc));
2258
2259 }
2260 }
2261
2262 /*ARGSUSED*/
2263 int
2264 segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
2265 {
2266 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2267 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2268
2269 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2270
2271 /*
2272 * ISM segment is always rw.
2273 */
2274 return (((sptd->spt_prot & prot) != prot) ? EACCES : 0);
2275 }
2276
2277 /*
2278 * Return an array of locked large pages, for empty slots allocate
2279 * private zero-filled anon pages.
2280 */
2281 static int
2282 spt_anon_getpages(
2283 struct seg *sptseg,
2284 caddr_t sptaddr,
2285 size_t len,
2286 page_t *ppa[])
2287 {
2288 struct spt_data *sptd = sptseg->s_data;
2289 struct anon_map *amp = sptd->spt_amp;
2290 enum seg_rw rw = sptd->spt_prot;
2291 uint_t szc = sptseg->s_szc;
2292 size_t pg_sz, share_sz = page_get_pagesize(szc);
2293 pgcnt_t lp_npgs;
2294 caddr_t lp_addr, e_sptaddr;
2295 uint_t vpprot, ppa_szc = 0;
2296 struct vpage *vpage = NULL;
2297 ulong_t j, ppa_idx;
2298 int err, ierr = 0;
2299 pgcnt_t an_idx;
2300 anon_sync_obj_t cookie;
2301 int anon_locked = 0;
2302 pgcnt_t amp_pgs;
2303
2304
2305 ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz));
2306 ASSERT(len != 0);
2307
2308 pg_sz = share_sz;
2309 lp_npgs = btop(pg_sz);
2310 lp_addr = sptaddr;
2311 e_sptaddr = sptaddr + len;
2312 an_idx = seg_page(sptseg, sptaddr);
2313 ppa_idx = 0;
2314
2315 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2316
2317 amp_pgs = page_get_pagecnt(amp->a_szc);
2318
2319 /*CONSTCOND*/
2320 while (1) {
2321 for (; lp_addr < e_sptaddr;
2322 an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) {
2323
2324 /*
2325 * If we're currently locked, and we get to a new
2326 * page, unlock our current anon chunk.
2327 */
2328 if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) {
2329 anon_array_exit(&cookie);
2330 anon_locked = 0;
2331 }
2332 if (!anon_locked) {
2333 anon_array_enter(amp, an_idx, &cookie);
2334 anon_locked = 1;
2335 }
2336 ppa_szc = (uint_t)-1;
2337 ierr = anon_map_getpages(amp, an_idx, szc, sptseg,
2338 lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx],
2339 &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred);
2340
2341 if (ierr != 0) {
2342 if (ierr > 0) {
2343 err = FC_MAKE_ERR(ierr);
2344 goto lpgs_err;
2345 }
2346 break;
2347 }
2348 }
2349 if (lp_addr == e_sptaddr) {
2350 break;
2351 }
2352 ASSERT(lp_addr < e_sptaddr);
2353
2354 /*
2355 * ierr == -1 means we failed to allocate a large page.
2356 * so do a size down operation.
2357 *
2358 * ierr == -2 means some other process that privately shares
2359 * pages with this process has allocated a larger page and we
2360 * need to retry with larger pages. So do a size up
2361 * operation. This relies on the fact that large pages are
2362 * never partially shared i.e. if we share any constituent
2363 * page of a large page with another process we must share the
2364 * entire large page. Note this cannot happen for SOFTLOCK
2365 * case, unless current address (lpaddr) is at the beginning
2366 * of the next page size boundary because the other process
2367 * couldn't have relocated locked pages.
2368 */
2369 ASSERT(ierr == -1 || ierr == -2);
2370 if (segvn_anypgsz) {
2371 ASSERT(ierr == -2 || szc != 0);
2372 ASSERT(ierr == -1 || szc < sptseg->s_szc);
2373 szc = (ierr == -1) ? szc - 1 : szc + 1;
2374 } else {
2375 /*
2376 * For faults and segvn_anypgsz == 0
2377 * we need to be careful not to loop forever
2378 * if existing page is found with szc other
2379 * than 0 or seg->s_szc. This could be due
2380 * to page relocations on behalf of DR or
2381 * more likely large page creation. For this
2382 * case simply re-size to existing page's szc
2383 * if returned by anon_map_getpages().
2384 */
2385 if (ppa_szc == (uint_t)-1) {
2386 szc = (ierr == -1) ? 0 : sptseg->s_szc;
2387 } else {
2388 ASSERT(ppa_szc <= sptseg->s_szc);
2389 ASSERT(ierr == -2 || ppa_szc < szc);
2390 ASSERT(ierr == -1 || ppa_szc > szc);
2391 szc = ppa_szc;
2392 }
2393 }
2394 pg_sz = page_get_pagesize(szc);
2395 lp_npgs = btop(pg_sz);
2396 ASSERT(IS_P2ALIGNED(lp_addr, pg_sz));
2397 }
2398 if (anon_locked) {
2399 anon_array_exit(&cookie);
2400 }
2401 ANON_LOCK_EXIT(&amp->a_rwlock);
2402 return (0);
2403
2404 lpgs_err:
2405 if (anon_locked) {
2406 anon_array_exit(&cookie);
2407 }
2408 ANON_LOCK_EXIT(&amp->a_rwlock);
2409 for (j = 0; j < ppa_idx; j++)
2410 page_unlock(ppa[j]);
2411 return (err);
2412 }
2413
2414 /*
2415 * count the number of bytes in a set of spt pages that are currently not
2416 * locked
2417 */
2418 static rctl_qty_t
2419 spt_unlockedbytes(pgcnt_t npages, page_t **ppa)
2420 {
2421 ulong_t i;
2422 rctl_qty_t unlocked = 0;
2423
2424 for (i = 0; i < npages; i++) {
2425 if (ppa[i]->p_lckcnt == 0)
2426 unlocked += PAGESIZE;
2427 }
2428 return (unlocked);
2429 }
2430
2431 extern u_longlong_t randtick(void);
2432 /* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */
2433 #define NLCK (NCPU_P2)
2434 /* Random number with a range [0, n-1], n must be power of two */
2435 #define RAND_P2(n) \
2436 ((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1))
2437
2438 int
2439 spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2440 page_t **ppa, ulong_t *lockmap, size_t pos,
2441 rctl_qty_t *locked)
2442 {
2443 struct shm_data *shmd = seg->s_data;
2444 struct spt_data *sptd = shmd->shm_sptseg->s_data;
2445 ulong_t i;
2446 int kernel;
2447 pgcnt_t nlck = 0;
2448 int rv = 0;
2449 int use_reserved = 1;
2450
2451 /* return the number of bytes actually locked */
2452 *locked = 0;
2453
2454 /*
2455 * To avoid contention on freemem_lock, availrmem and pages_locked
2456 * global counters are updated only every nlck locked pages instead of
2457 * every time. Reserve nlck locks up front and deduct from this
2458 * reservation for each page that requires a lock. When the reservation
2459 * is consumed, reserve again. nlck is randomized, so the competing
2460 * threads do not fall into a cyclic lock contention pattern. When
2461 * memory is low, the lock ahead is disabled, and instead page_pp_lock()
2462 * is used to lock pages.
2463 */
2464 for (i = 0; i < npages; anon_index++, pos++, i++) {
2465 if (nlck == 0 && use_reserved == 1) {
2466 nlck = NLCK + RAND_P2(NLCK);
2467 /* if fewer loops left, decrease nlck */
2468 nlck = MIN(nlck, npages - i);
2469 /*
2470 * Reserve nlck locks up front and deduct from this
2471 * reservation for each page that requires a lock. When
2472 * the reservation is consumed, reserve again.
2473 */
2474 mutex_enter(&freemem_lock);
2475 if ((availrmem - nlck) < pages_pp_maximum) {
2476 /* Do not do advance memory reserves */
2477 use_reserved = 0;
2478 } else {
2479 availrmem -= nlck;
2480 pages_locked += nlck;
2481 }
2482 mutex_exit(&freemem_lock);
2483 }
2484 if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) {
2485 if (sptd->spt_ppa_lckcnt[anon_index] <
2486 (ushort_t)DISM_LOCK_MAX) {
2487 if (++sptd->spt_ppa_lckcnt[anon_index] ==
2488 (ushort_t)DISM_LOCK_MAX) {
2489 cmn_err(CE_WARN,
2490 "DISM page lock limit "
2491 "reached on DISM offset 0x%lx\n",
2492 anon_index << PAGESHIFT);
2493 }
2494 kernel = (sptd->spt_ppa &&
2495 sptd->spt_ppa[anon_index]);
2496 if (!page_pp_lock(ppa[i], 0, kernel ||
2497 use_reserved)) {
2498 sptd->spt_ppa_lckcnt[anon_index]--;
2499 rv = EAGAIN;
2500 break;
2501 }
2502 /* if this is a newly locked page, count it */
2503 if (ppa[i]->p_lckcnt == 1) {
2504 if (kernel == 0 && use_reserved == 1)
2505 nlck--;
2506 *locked += PAGESIZE;
2507 }
2508 shmd->shm_lckpgs++;
2509 shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED;
2510 if (lockmap != NULL)
2511 BT_SET(lockmap, pos);
2512 }
2513 }
2514 }
2515 /* Return unused lock reservation */
2516 if (nlck != 0 && use_reserved == 1) {
2517 mutex_enter(&freemem_lock);
2518 availrmem += nlck;
2519 pages_locked -= nlck;
2520 mutex_exit(&freemem_lock);
2521 }
2522
2523 return (rv);
2524 }
2525
2526 int
2527 spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2528 rctl_qty_t *unlocked)
2529 {
2530 struct shm_data *shmd = seg->s_data;
2531 struct spt_data *sptd = shmd->shm_sptseg->s_data;
2532 struct anon_map *amp = sptd->spt_amp;
2533 struct anon *ap;
2534 struct vnode *vp;
2535 uoff_t off;
2536 struct page *pp;
2537 int kernel;
2538 anon_sync_obj_t cookie;
2539 ulong_t i;
2540 pgcnt_t nlck = 0;
2541 pgcnt_t nlck_limit = NLCK;
2542
2543 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2544 for (i = 0; i < npages; i++, anon_index++) {
2545 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
2546 anon_array_enter(amp, anon_index, &cookie);
2547 ap = anon_get_ptr(amp->ahp, anon_index);
2548 ASSERT(ap);
2549
2550 swap_xlate(ap, &vp, &off);
2551 anon_array_exit(&cookie);
2552 pp = page_lookup(vp, off, SE_SHARED);
2553 ASSERT(pp);
2554 /*
2555 * availrmem is decremented only for pages which are not
2556 * in seg pcache, for pages in seg pcache availrmem was
2557 * decremented in _dismpagelock()
2558 */
2559 kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]);
2560 ASSERT(pp->p_lckcnt > 0);
2561
2562 /*
2563 * lock page but do not change availrmem, we do it
2564 * ourselves every nlck loops.
2565 */
2566 page_pp_unlock(pp, 0, 1);
2567 if (pp->p_lckcnt == 0) {
2568 if (kernel == 0)
2569 nlck++;
2570 *unlocked += PAGESIZE;
2571 }
2572 page_unlock(pp);
2573 shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED;
2574 sptd->spt_ppa_lckcnt[anon_index]--;
2575 shmd->shm_lckpgs--;
2576 }
2577
2578 /*
2579 * To reduce freemem_lock contention, do not update availrmem
2580 * until at least NLCK pages have been unlocked.
2581 * 1. No need to update if nlck is zero
2582 * 2. Always update if the last iteration
2583 */
2584 if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) {
2585 mutex_enter(&freemem_lock);
2586 availrmem += nlck;
2587 pages_locked -= nlck;
2588 mutex_exit(&freemem_lock);
2589 nlck = 0;
2590 nlck_limit = NLCK + RAND_P2(NLCK);
2591 }
2592 }
2593 ANON_LOCK_EXIT(&amp->a_rwlock);
2594
2595 return (0);
2596 }
2597
2598 /*ARGSUSED*/
2599 static int
2600 segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
2601 int attr, int op, ulong_t *lockmap, size_t pos)
2602 {
2603 struct shm_data *shmd = seg->s_data;
2604 struct seg *sptseg = shmd->shm_sptseg;
2605 struct spt_data *sptd = sptseg->s_data;
2606 struct kshmid *sp = sptd->spt_amp->a_sp;
2607 pgcnt_t npages, a_npages;
2608 page_t **ppa;
2609 pgcnt_t an_idx, a_an_idx, ppa_idx;
2610 caddr_t spt_addr, a_addr; /* spt and aligned address */
2611 size_t a_len; /* aligned len */
2612 size_t share_sz;
2613 ulong_t i;
2614 int sts = 0;
2615 rctl_qty_t unlocked = 0;
2616 rctl_qty_t locked = 0;
2617 struct proc *p = curproc;
2618 kproject_t *proj;
2619
2620 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2621 ASSERT(sp != NULL);
2622
2623 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
2624 return (0);
2625 }
2626
2627 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2628 an_idx = seg_page(seg, addr);
2629 npages = btopr(len);
2630
2631 if (an_idx + npages > btopr(shmd->shm_amp->size)) {
2632 return (ENOMEM);
2633 }
2634
2635 /*
2636 * A shm's project never changes, so no lock needed.
2637 * The shm has a hold on the project, so it will not go away.
2638 * Since we have a mapping to shm within this zone, we know
2639 * that the zone will not go away.
2640 */
2641 proj = sp->shm_perm.ipc_proj;
2642
2643 if (op == MC_LOCK) {
2644
2645 /*
2646 * Need to align addr and size request if they are not
2647 * aligned so we can always allocate large page(s) however
2648 * we only lock what was requested in initial request.
2649 */
2650 share_sz = page_get_pagesize(sptseg->s_szc);
2651 a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz);
2652 a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)),
2653 share_sz);
2654 a_npages = btop(a_len);
2655 a_an_idx = seg_page(seg, a_addr);
2656 spt_addr = sptseg->s_base + ptob(a_an_idx);
2657 ppa_idx = an_idx - a_an_idx;
2658
2659 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages),
2660 KM_NOSLEEP)) == NULL) {
2661 return (ENOMEM);
2662 }
2663
2664 /*
2665 * Don't cache any new pages for IO and
2666 * flush any cached pages.
2667 */
2668 mutex_enter(&sptd->spt_lock);
2669 if (sptd->spt_ppa != NULL)
2670 sptd->spt_flags |= DISM_PPA_CHANGED;
2671
2672 sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa);
2673 if (sts != 0) {
2674 mutex_exit(&sptd->spt_lock);
2675 kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
2676 return (sts);
2677 }
2678
2679 mutex_enter(&sp->shm_mlock);
2680 /* enforce locked memory rctl */
2681 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]);
2682
2683 mutex_enter(&p->p_lock);
2684 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) {
2685 mutex_exit(&p->p_lock);
2686 sts = EAGAIN;
2687 } else {
2688 mutex_exit(&p->p_lock);
2689 sts = spt_lockpages(seg, an_idx, npages,
2690 &ppa[ppa_idx], lockmap, pos, &locked);
2691
2692 /*
2693 * correct locked count if not all pages could be
2694 * locked
2695 */
2696 if ((unlocked - locked) > 0) {
2697 rctl_decr_locked_mem(NULL, proj,
2698 (unlocked - locked), 0);
2699 }
2700 }
2701 /*
2702 * unlock pages
2703 */
2704 for (i = 0; i < a_npages; i++)
2705 page_unlock(ppa[i]);
2706 if (sptd->spt_ppa != NULL)
2707 sptd->spt_flags |= DISM_PPA_CHANGED;
2708 mutex_exit(&sp->shm_mlock);
2709 mutex_exit(&sptd->spt_lock);
2710
2711 kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
2712
2713 } else if (op == MC_UNLOCK) { /* unlock */
2714 page_t **ppa;
2715
2716 mutex_enter(&sptd->spt_lock);
2717 if (shmd->shm_lckpgs == 0) {
2718 mutex_exit(&sptd->spt_lock);
2719 return (0);
2720 }
2721 /*
2722 * Don't cache new IO pages.
2723 */
2724 if (sptd->spt_ppa != NULL)
2725 sptd->spt_flags |= DISM_PPA_CHANGED;
2726
2727 mutex_enter(&sp->shm_mlock);
2728 sts = spt_unlockpages(seg, an_idx, npages, &unlocked);
2729 if ((ppa = sptd->spt_ppa) != NULL)
2730 sptd->spt_flags |= DISM_PPA_CHANGED;
2731 mutex_exit(&sptd->spt_lock);
2732
2733 rctl_decr_locked_mem(NULL, proj, unlocked, 0);
2734 mutex_exit(&sp->shm_mlock);
2735
2736 if (ppa != NULL)
2737 seg_ppurge_wiredpp(ppa);
2738 }
2739 return (sts);
2740 }
2741
2742 /*ARGSUSED*/
2743 int
2744 segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2745 {
2746 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2747 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2748 spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1;
2749
2750 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2751
2752 /*
2753 * ISM segment is always rw.
2754 */
2755 while (--pgno >= 0)
2756 *protv++ = sptd->spt_prot;
2757 return (0);
2758 }
2759
2760 /*ARGSUSED*/
2761 uoff_t
2762 segspt_shmgetoffset(struct seg *seg, caddr_t addr)
2763 {
2764 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2765
2766 /* Offset does not matter in ISM memory */
2767
2768 return (0);
2769 }
2770
2771 /* ARGSUSED */
2772 int
2773 segspt_shmgettype(struct seg *seg, caddr_t addr)
2774 {
2775 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2776 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2777
2778 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2779
2780 /*
2781 * The shared memory mapping is always MAP_SHARED, SWAP is only
2782 * reserved for DISM
2783 */
2784 return (MAP_SHARED |
2785 ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE));
2786 }
2787
2788 /*ARGSUSED*/
2789 int
2790 segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
2791 {
2792 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2793 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2794
2795 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2796
2797 *vpp = sptd->spt_vp;
2798 return (0);
2799 }
2800
2801 /*
2802 * We need to wait for pending IO to complete to a DISM segment in order for
2803 * pages to get kicked out of the seg_pcache. 120 seconds should be more
2804 * than enough time to wait.
2805 */
2806 static clock_t spt_pcache_wait = 120;
2807
2808 /*ARGSUSED*/
2809 static int
2810 segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2811 {
2812 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2813 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2814 struct anon_map *amp;
2815 pgcnt_t pg_idx;
2816 ushort_t gen;
2817 clock_t end_lbolt;
2818 int writer;
2819 page_t **ppa;
2820
2821 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2822
2823 if (behav == MADV_FREE || behav == MADV_PURGE) {
2824 if ((sptd->spt_flags & SHM_PAGEABLE) == 0)
2825 return (0);
2826
2827 amp = sptd->spt_amp;
2828 pg_idx = seg_page(seg, addr);
2829
2830 mutex_enter(&sptd->spt_lock);
2831 if ((ppa = sptd->spt_ppa) == NULL) {
2832 mutex_exit(&sptd->spt_lock);
2833 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2834 (void) anon_disclaim(amp, pg_idx, len, behav, NULL);
2835 ANON_LOCK_EXIT(&amp->a_rwlock);
2836 return (0);
2837 }
2838
2839 sptd->spt_flags |= DISM_PPA_CHANGED;
2840 gen = sptd->spt_gen;
2841
2842 mutex_exit(&sptd->spt_lock);
2843
2844 /*
2845 * Purge all DISM cached pages
2846 */
2847 seg_ppurge_wiredpp(ppa);
2848
2849 /*
2850 * Drop the AS_LOCK so that other threads can grab it
2851 * in the as_pageunlock path and hopefully get the segment
2852 * kicked out of the seg_pcache. We bump the shm_softlockcnt
2853 * to keep this segment resident.
2854 */
2855 writer = AS_WRITE_HELD(seg->s_as);
2856 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
2857 AS_LOCK_EXIT(seg->s_as);
2858
2859 mutex_enter(&sptd->spt_lock);
2860
2861 end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait);
2862
2863 /*
2864 * Try to wait for pages to get kicked out of the seg_pcache.
2865 */
2866 while (sptd->spt_gen == gen &&
2867 (sptd->spt_flags & DISM_PPA_CHANGED) &&
2868 ddi_get_lbolt() < end_lbolt) {
2869 if (!cv_timedwait_sig(&sptd->spt_cv,
2870 &sptd->spt_lock, end_lbolt)) {
2871 break;
2872 }
2873 }
2874
2875 mutex_exit(&sptd->spt_lock);
2876
2877 /* Regrab the AS_LOCK and release our hold on the segment */
2878 AS_LOCK_ENTER(seg->s_as, writer ? RW_WRITER : RW_READER);
2879 atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
2880 if (shmd->shm_softlockcnt <= 0) {
2881 if (AS_ISUNMAPWAIT(seg->s_as)) {
2882 mutex_enter(&seg->s_as->a_contents);
2883 if (AS_ISUNMAPWAIT(seg->s_as)) {
2884 AS_CLRUNMAPWAIT(seg->s_as);
2885 cv_broadcast(&seg->s_as->a_cv);
2886 }
2887 mutex_exit(&seg->s_as->a_contents);
2888 }
2889 }
2890
2891 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2892 (void) anon_disclaim(amp, pg_idx, len, behav, NULL);
2893 ANON_LOCK_EXIT(&amp->a_rwlock);
2894 } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP ||
2895 behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) {
2896 int already_set;
2897 ulong_t anon_index;
2898 lgrp_mem_policy_t policy;
2899 caddr_t shm_addr;
2900 size_t share_size;
2901 size_t size;
2902 struct seg *sptseg = shmd->shm_sptseg;
2903 caddr_t sptseg_addr;
2904
2905 /*
2906 * Align address and length to page size of underlying segment
2907 */
2908 share_size = page_get_pagesize(shmd->shm_sptseg->s_szc);
2909 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size);
2910 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)),
2911 share_size);
2912
2913 amp = shmd->shm_amp;
2914 anon_index = seg_page(seg, shm_addr);
2915
2916 /*
2917 * And now we may have to adjust size downward if we have
2918 * exceeded the realsize of the segment or initial anon
2919 * allocations.
2920 */
2921 sptseg_addr = sptseg->s_base + ptob(anon_index);
2922 if ((sptseg_addr + size) >
2923 (sptseg->s_base + sptd->spt_realsize))
2924 size = (sptseg->s_base + sptd->spt_realsize) -
2925 sptseg_addr;
2926
2927 /*
2928 * Set memory allocation policy for this segment
2929 */
2930 policy = lgrp_madv_to_policy(behav, len, MAP_SHARED);
2931 already_set = lgrp_shm_policy_set(policy, amp, anon_index,
2932 NULL, 0, len);
2933
2934 /*
2935 * If random memory allocation policy set already,
2936 * don't bother reapplying it.
2937 */
2938 if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy))
2939 return (0);
2940
2941 /*
2942 * Mark any existing pages in the given range for
2943 * migration, flushing the I/O page cache, and using
2944 * underlying segment to calculate anon index and get
2945 * anonmap and vnode pointer from
2946 */
2947 if (shmd->shm_softlockcnt > 0)
2948 segspt_purge(seg);
2949
2950 page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0);
2951 }
2952
2953 return (0);
2954 }
2955
2956 /*
2957 * get a memory ID for an addr in a given segment
2958 */
2959 static int
2960 segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
2961 {
2962 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2963 struct anon *ap;
2964 size_t anon_index;
2965 struct anon_map *amp = shmd->shm_amp;
2966 struct spt_data *sptd = shmd->shm_sptseg->s_data;
2967 struct seg *sptseg = shmd->shm_sptseg;
2968 anon_sync_obj_t cookie;
2969
2970 anon_index = seg_page(seg, addr);
2971
2972 if (addr > (seg->s_base + sptd->spt_realsize)) {
2973 return (EFAULT);
2974 }
2975
2976 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2977 anon_array_enter(amp, anon_index, &cookie);
2978 ap = anon_get_ptr(amp->ahp, anon_index);
2979 if (ap == NULL) {
2980 struct page *pp;
2981 caddr_t spt_addr = sptseg->s_base + ptob(anon_index);
2982
2983 pp = anon_zero(sptseg, spt_addr, &ap, kcred);
2984 if (pp == NULL) {
2985 anon_array_exit(&cookie);
2986 ANON_LOCK_EXIT(&amp->a_rwlock);
2987 return (ENOMEM);
2988 }
2989 (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP);
2990 page_unlock(pp);
2991 }
2992 anon_array_exit(&cookie);
2993 ANON_LOCK_EXIT(&amp->a_rwlock);
2994 memidp->val[0] = (uintptr_t)ap;
2995 memidp->val[1] = (uintptr_t)addr & PAGEOFFSET;
2996 return (0);
2997 }
2998
2999 /*
3000 * Get memory allocation policy info for specified address in given segment
3001 */
3002 static lgrp_mem_policy_info_t *
3003 segspt_shmgetpolicy(struct seg *seg, caddr_t addr)
3004 {
3005 struct anon_map *amp;
3006 ulong_t anon_index;
3007 lgrp_mem_policy_info_t *policy_info;
3008 struct shm_data *shm_data;
3009
3010 ASSERT(seg != NULL);
3011
3012 /*
3013 * Get anon_map from segshm
3014 *
3015 * Assume that no lock needs to be held on anon_map, since
3016 * it should be protected by its reference count which must be
3017 * nonzero for an existing segment
3018 * Need to grab readers lock on policy tree though
3019 */
3020 shm_data = (struct shm_data *)seg->s_data;
3021 if (shm_data == NULL)
3022 return (NULL);
3023 amp = shm_data->shm_amp;
3024 ASSERT(amp->refcnt != 0);
3025
3026 /*
3027 * Get policy info
3028 *
3029 * Assume starting anon index of 0
3030 */
3031 anon_index = seg_page(seg, addr);
3032 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
3033
3034 return (policy_info);
3035 }
Unleashed OS