?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
tcp: fix ISN generator to increment based on time again
[unleashed.git] / kernel / vm / seg_spt.c
blobfc8374327662475d9775f93a5a9f62554b35914a
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2015, Joyent, Inc. All rights reserved.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 #include <sys/param.h>
28 #include <sys/user.h>
29 #include <sys/mman.h>
30 #include <sys/kmem.h>
31 #include <sys/sysmacros.h>
32 #include <sys/cmn_err.h>
33 #include <sys/systm.h>
34 #include <sys/tuneable.h>
35 #include <vm/hat.h>
36 #include <vm/seg.h>
37 #include <vm/as.h>
38 #include <vm/anon.h>
39 #include <vm/page.h>
40 #include <sys/buf.h>
41 #include <sys/swap.h>
42 #include <sys/atomic.h>
43 #include <vm/seg_spt.h>
44 #include <sys/debug.h>
45 #include <sys/vtrace.h>
46 #include <sys/shm.h>
47 #include <sys/shm_impl.h>
48 #include <sys/lgrp.h>
49 #include <sys/vmsystm.h>
50 #include <sys/policy.h>
51 #include <sys/project.h>
52 #include <sys/tnf_probe.h>
53 #include <sys/zone.h>
54
55 #define SEGSPTADDR (caddr_t)0x0
56
57 /*
58 * # pages used for spt
59 */
60 size_t spt_used;
61
62 /*
63 * segspt_minfree is the memory left for system after ISM
64 * locked its pages; it is set up to 5% of availrmem in
65 * sptcreate when ISM is created. ISM should not use more
66 * than ~90% of availrmem; if it does, then the performance
67 * of the system may decrease. Machines with large memories may
68 * be able to use up more memory for ISM so we set the default
69 * segspt_minfree to 5% (which gives ISM max 95% of availrmem.
70 * If somebody wants even more memory for ISM (risking hanging
71 * the system) they can patch the segspt_minfree to smaller number.
72 */
73 pgcnt_t segspt_minfree = 0;
74
75 static int segspt_create(struct seg *seg, caddr_t argsp);
76 static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize);
77 static void segspt_free(struct seg *seg);
78 static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len);
79 static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr);
80
81 static void
82 segspt_badop()
83 {
84 panic("segspt_badop called");
85 /*NOTREACHED*/
86 }
87
88 #define SEGSPT_BADOP(t) (t(*)())segspt_badop
89
90 static const struct seg_ops segspt_ops = {
91 .dup = SEGSPT_BADOP(int),
92 .unmap = segspt_unmap,
93 .free = segspt_free,
94 .fault = SEGSPT_BADOP(int),
95 .faulta = SEGSPT_BADOP(faultcode_t),
96 .setprot = SEGSPT_BADOP(int),
97 .checkprot = SEGSPT_BADOP(int),
98 .kluster = SEGSPT_BADOP(int),
99 .sync = SEGSPT_BADOP(int),
100 .incore = SEGSPT_BADOP(size_t),
101 .lockop = SEGSPT_BADOP(int),
102 .getprot = SEGSPT_BADOP(int),
103 .getoffset = SEGSPT_BADOP(uoff_t),
104 .gettype = SEGSPT_BADOP(int),
105 .getvp = SEGSPT_BADOP(int),
106 .advise = SEGSPT_BADOP(int),
107 .dump = SEGSPT_BADOP(void),
108 .pagelock = SEGSPT_BADOP(int),
109 .setpagesize = SEGSPT_BADOP(int),
110 .getmemid = SEGSPT_BADOP(int),
111 .getpolicy = segspt_getpolicy,
112 .capable = SEGSPT_BADOP(int),
113 };
114
115 static int segspt_shmdup(struct seg *seg, struct seg *newseg);
116 static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize);
117 static void segspt_shmfree(struct seg *seg);
118 static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg,
119 caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw);
120 static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr);
121 static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr,
122 register size_t len, register uint_t prot);
123 static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size,
124 uint_t prot);
125 static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta);
126 static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len,
127 register char *vec);
128 static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len,
129 int attr, uint_t flags);
130 static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
131 int attr, int op, ulong_t *lockmap, size_t pos);
132 static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len,
133 uint_t *protv);
134 static uoff_t segspt_shmgetoffset(struct seg *seg, caddr_t addr);
135 static int segspt_shmgettype(struct seg *seg, caddr_t addr);
136 static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
137 static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len,
138 uint_t behav);
139 static int segspt_shmpagelock(struct seg *, caddr_t, size_t,
140 struct page ***, enum lock_type, enum seg_rw);
141 static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *);
142 static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t);
143
144 const struct seg_ops segspt_shmops = {
145 .dup = segspt_shmdup,
146 .unmap = segspt_shmunmap,
147 .free = segspt_shmfree,
148 .fault = segspt_shmfault,
149 .faulta = segspt_shmfaulta,
150 .setprot = segspt_shmsetprot,
151 .checkprot = segspt_shmcheckprot,
152 .kluster = segspt_shmkluster,
153 .sync = segspt_shmsync,
154 .incore = segspt_shmincore,
155 .lockop = segspt_shmlockop,
156 .getprot = segspt_shmgetprot,
157 .getoffset = segspt_shmgetoffset,
158 .gettype = segspt_shmgettype,
159 .getvp = segspt_shmgetvp,
160 .advise = segspt_shmadvise,
161 .pagelock = segspt_shmpagelock,
162 .getmemid = segspt_shmgetmemid,
163 .getpolicy = segspt_shmgetpolicy,
164 };
165
166 static void segspt_purge(struct seg *seg);
167 static int segspt_reclaim(void *, caddr_t, size_t, struct page **,
168 enum seg_rw, int);
169 static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len,
170 page_t **ppa);
171
172
173
174 /*ARGSUSED*/
175 int
176 sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp,
177 uint_t prot, uint_t flags, uint_t share_szc)
178 {
179 int err;
180 struct as *newas;
181 struct segspt_crargs sptcargs;
182
183 if (segspt_minfree == 0) /* leave min 5% of availrmem for */
184 segspt_minfree = availrmem/20; /* for the system */
185
186 if (!hat_supported(HAT_SHARED_PT, NULL))
187 return (EINVAL);
188
189 /*
190 * get a new as for this shared memory segment
191 */
192 newas = as_alloc();
193 newas->a_proc = NULL;
194 sptcargs.amp = amp;
195 sptcargs.prot = prot;
196 sptcargs.flags = flags;
197 sptcargs.szc = share_szc;
198 /*
199 * create a shared page table (spt) segment
200 */
201
202 if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) {
203 as_free(newas);
204 return (err);
205 }
206 *sptseg = sptcargs.seg_spt;
207 return (0);
208 }
209
210 void
211 sptdestroy(struct as *as, struct anon_map *amp)
212 {
213 (void) as_unmap(as, SEGSPTADDR, amp->size);
214 as_free(as);
215 }
216
217 /*
218 * called from seg_free().
219 * free (i.e., unlock, unmap, return to free list)
220 * all the pages in the given seg.
221 */
222 void
223 segspt_free(struct seg *seg)
224 {
225 struct spt_data *sptd = (struct spt_data *)seg->s_data;
226
227 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
228
229 if (sptd != NULL) {
230 if (sptd->spt_realsize)
231 segspt_free_pages(seg, seg->s_base, sptd->spt_realsize);
232
233 if (sptd->spt_ppa_lckcnt) {
234 kmem_free(sptd->spt_ppa_lckcnt,
235 sizeof (*sptd->spt_ppa_lckcnt)
236 * btopr(sptd->spt_amp->size));
237 }
238 kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp));
239 cv_destroy(&sptd->spt_cv);
240 mutex_destroy(&sptd->spt_lock);
241 kmem_free(sptd, sizeof (*sptd));
242 }
243 }
244
245 /*ARGSUSED*/
246 static int
247 segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr,
248 uint_t flags)
249 {
250 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
251
252 return (0);
253 }
254
255 /*ARGSUSED*/
256 static size_t
257 segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec)
258 {
259 caddr_t eo_seg;
260 pgcnt_t npages;
261 struct shm_data *shmd = (struct shm_data *)seg->s_data;
262 struct seg *sptseg;
263 struct spt_data *sptd;
264
265 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
266 sptseg = shmd->shm_sptseg;
267 sptd = sptseg->s_data;
268
269 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
270 eo_seg = addr + len;
271 while (addr < eo_seg) {
272 /* page exists, and it's locked. */
273 *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED |
274 SEG_PAGE_ANON;
275 addr += PAGESIZE;
276 }
277 return (len);
278 } else {
279 struct anon_map *amp = shmd->shm_amp;
280 struct anon *ap;
281 page_t *pp;
282 pgcnt_t anon_index;
283 struct vnode *vp;
284 uoff_t off;
285 ulong_t i;
286 int ret;
287 anon_sync_obj_t cookie;
288
289 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
290 anon_index = seg_page(seg, addr);
291 npages = btopr(len);
292 if (anon_index + npages > btopr(shmd->shm_amp->size)) {
293 return (EINVAL);
294 }
295 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
296 for (i = 0; i < npages; i++, anon_index++) {
297 ret = 0;
298 anon_array_enter(amp, anon_index, &cookie);
299 ap = anon_get_ptr(amp->ahp, anon_index);
300 if (ap != NULL) {
301 swap_xlate(ap, &vp, &off);
302 anon_array_exit(&cookie);
303 pp = page_lookup_nowait(&vp->v_object, off,
304 SE_SHARED);
305 if (pp != NULL) {
306 ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON;
307 page_unlock(pp);
308 }
309 } else {
310 anon_array_exit(&cookie);
311 }
312 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
313 ret |= SEG_PAGE_LOCKED;
314 }
315 *vec++ = (char)ret;
316 }
317 ANON_LOCK_EXIT(&amp->a_rwlock);
318 return (len);
319 }
320 }
321
322 static int
323 segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize)
324 {
325 size_t share_size;
326
327 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
328
329 /*
330 * seg.s_size may have been rounded up to the largest page size
331 * in shmat().
332 * XXX This should be cleanedup. sptdestroy should take a length
333 * argument which should be the same as sptcreate. Then
334 * this rounding would not be needed (or is done in shm.c)
335 * Only the check for full segment will be needed.
336 *
337 * XXX -- shouldn't raddr == 0 always? These tests don't seem
338 * to be useful at all.
339 */
340 share_size = page_get_pagesize(seg->s_szc);
341 ssize = P2ROUNDUP(ssize, share_size);
342
343 if (raddr == seg->s_base && ssize == seg->s_size) {
344 seg_free(seg);
345 return (0);
346 } else
347 return (EINVAL);
348 }
349
350 int
351 segspt_create(struct seg *seg, caddr_t argsp)
352 {
353 int err;
354 caddr_t addr = seg->s_base;
355 struct spt_data *sptd;
356 struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp;
357 struct anon_map *amp = sptcargs->amp;
358 struct kshmid *sp = amp->a_sp;
359 struct cred *cred = CRED();
360 ulong_t i, j, anon_index = 0;
361 pgcnt_t npages = btopr(amp->size);
362 struct vnode *vp;
363 page_t **ppa;
364 uint_t hat_flags;
365 size_t pgsz;
366 pgcnt_t pgcnt;
367 caddr_t a;
368 pgcnt_t pidx;
369 size_t sz;
370 proc_t *procp = curproc;
371 rctl_qty_t lockedbytes = 0;
372 kproject_t *proj;
373
374 /*
375 * We are holding the a_lock on the underlying dummy as,
376 * so we can make calls to the HAT layer.
377 */
378 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
379 ASSERT(sp != NULL);
380
381 if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
382 if (err = anon_swap_adjust(npages))
383 return (err);
384 }
385 err = ENOMEM;
386
387 if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL)
388 goto out1;
389
390 if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
391 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages),
392 KM_NOSLEEP)) == NULL)
393 goto out2;
394 }
395
396 mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL);
397
398 if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL)
399 goto out3;
400
401 seg->s_ops = &segspt_ops;
402 sptd->spt_vp = vp;
403 sptd->spt_amp = amp;
404 sptd->spt_prot = sptcargs->prot;
405 sptd->spt_flags = sptcargs->flags;
406 seg->s_data = (caddr_t)sptd;
407 sptd->spt_ppa = NULL;
408 sptd->spt_ppa_lckcnt = NULL;
409 seg->s_szc = sptcargs->szc;
410 cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL);
411 sptd->spt_gen = 0;
412
413 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
414 if (seg->s_szc > amp->a_szc) {
415 amp->a_szc = seg->s_szc;
416 }
417 ANON_LOCK_EXIT(&amp->a_rwlock);
418
419 /*
420 * Set policy to affect initial allocation of pages in
421 * anon_map_createpages()
422 */
423 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index,
424 NULL, 0, ptob(npages));
425
426 if (sptcargs->flags & SHM_PAGEABLE) {
427 size_t share_sz;
428 pgcnt_t new_npgs, more_pgs;
429 struct anon_hdr *nahp;
430 zone_t *zone;
431
432 share_sz = page_get_pagesize(seg->s_szc);
433 if (!IS_P2ALIGNED(amp->size, share_sz)) {
434 /*
435 * We are rounding up the size of the anon array
436 * on 4 M boundary because we always create 4 M
437 * of page(s) when locking, faulting pages and we
438 * don't have to check for all corner cases e.g.
439 * if there is enough space to allocate 4 M
440 * page.
441 */
442 new_npgs = btop(P2ROUNDUP(amp->size, share_sz));
443 more_pgs = new_npgs - npages;
444
445 /*
446 * The zone will never be NULL, as a fully created
447 * shm always has an owning zone.
448 */
449 zone = sp->shm_perm.ipc_zone_ref.zref_zone;
450 ASSERT(zone != NULL);
451 if (anon_resv_zone(ptob(more_pgs), zone) == 0) {
452 err = ENOMEM;
453 goto out4;
454 }
455
456 nahp = anon_create(new_npgs, ANON_SLEEP);
457 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
458 (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages,
459 ANON_SLEEP);
460 anon_release(amp->ahp, npages);
461 amp->ahp = nahp;
462 ASSERT(amp->swresv == ptob(npages));
463 amp->swresv = amp->size = ptob(new_npgs);
464 ANON_LOCK_EXIT(&amp->a_rwlock);
465 npages = new_npgs;
466 }
467
468 sptd->spt_ppa_lckcnt = kmem_zalloc(npages *
469 sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP);
470 sptd->spt_pcachecnt = 0;
471 sptd->spt_realsize = ptob(npages);
472 sptcargs->seg_spt = seg;
473 return (0);
474 }
475
476 /*
477 * get array of pages for each anon slot in amp
478 */
479 if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa,
480 seg, addr, S_CREATE, cred)) != 0)
481 goto out4;
482
483 mutex_enter(&sp->shm_mlock);
484
485 /* May be partially locked, so, count bytes to charge for locking */
486 for (i = 0; i < npages; i++)
487 if (ppa[i]->p_lckcnt == 0)
488 lockedbytes += PAGESIZE;
489
490 proj = sp->shm_perm.ipc_proj;
491
492 if (lockedbytes > 0) {
493 mutex_enter(&procp->p_lock);
494 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) {
495 mutex_exit(&procp->p_lock);
496 mutex_exit(&sp->shm_mlock);
497 for (i = 0; i < npages; i++)
498 page_unlock(ppa[i]);
499 err = ENOMEM;
500 goto out4;
501 }
502 mutex_exit(&procp->p_lock);
503 }
504
505 /*
506 * addr is initial address corresponding to the first page on ppa list
507 */
508 for (i = 0; i < npages; i++) {
509 /* attempt to lock all pages */
510 if (page_pp_lock(ppa[i], 0, 1) == 0) {
511 /*
512 * if unable to lock any page, unlock all
513 * of them and return error
514 */
515 for (j = 0; j < i; j++)
516 page_pp_unlock(ppa[j], 0, 1);
517 for (i = 0; i < npages; i++)
518 page_unlock(ppa[i]);
519 rctl_decr_locked_mem(NULL, proj, lockedbytes, 0);
520 mutex_exit(&sp->shm_mlock);
521 err = ENOMEM;
522 goto out4;
523 }
524 }
525 mutex_exit(&sp->shm_mlock);
526
527 /*
528 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
529 * for the entire life of the segment. For example platforms
530 * that do not support Dynamic Reconfiguration.
531 */
532 hat_flags = HAT_LOAD_SHARE;
533 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
534 hat_flags |= HAT_LOAD_LOCK;
535
536 /*
537 * Load translations one lare page at a time
538 * to make sure we don't create mappings bigger than
539 * segment's size code in case underlying pages
540 * are shared with segvn's segment that uses bigger
541 * size code than we do.
542 */
543 pgsz = page_get_pagesize(seg->s_szc);
544 pgcnt = page_get_pagecnt(seg->s_szc);
545 for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) {
546 sz = MIN(pgsz, ptob(npages - pidx));
547 hat_memload_array(seg->s_as->a_hat, a, sz,
548 &ppa[pidx], sptd->spt_prot, hat_flags);
549 }
550
551 /*
552 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
553 * we will leave the pages locked SE_SHARED for the life
554 * of the ISM segment. This will prevent any calls to
555 * hat_pageunload() on this ISM segment for those platforms.
556 */
557 if (!(hat_flags & HAT_LOAD_LOCK)) {
558 /*
559 * On platforms that support HAT_DYNAMIC_ISM_UNMAP,
560 * we no longer need to hold the SE_SHARED lock on the pages,
561 * since L_PAGELOCK and F_SOFTLOCK calls will grab the
562 * SE_SHARED lock on the pages as necessary.
563 */
564 for (i = 0; i < npages; i++)
565 page_unlock(ppa[i]);
566 }
567 sptd->spt_pcachecnt = 0;
568 kmem_free(ppa, ((sizeof (page_t *)) * npages));
569 sptd->spt_realsize = ptob(npages);
570 atomic_add_long(&spt_used, npages);
571 sptcargs->seg_spt = seg;
572 return (0);
573
574 out4:
575 seg->s_data = NULL;
576 kmem_free(vp, sizeof (*vp));
577 cv_destroy(&sptd->spt_cv);
578 out3:
579 mutex_destroy(&sptd->spt_lock);
580 if ((sptcargs->flags & SHM_PAGEABLE) == 0)
581 kmem_free(ppa, (sizeof (*ppa) * npages));
582 out2:
583 kmem_free(sptd, sizeof (*sptd));
584 out1:
585 if ((sptcargs->flags & SHM_PAGEABLE) == 0)
586 anon_swap_restore(npages);
587 return (err);
588 }
589
590 /*ARGSUSED*/
591 void
592 segspt_free_pages(struct seg *seg, caddr_t addr, size_t len)
593 {
594 struct page *pp;
595 struct spt_data *sptd = (struct spt_data *)seg->s_data;
596 pgcnt_t npages;
597 ulong_t anon_idx;
598 struct anon_map *amp;
599 struct anon *ap;
600 struct vnode *vp;
601 uoff_t off;
602 uint_t hat_flags;
603 int root = 0;
604 pgcnt_t pgs, curnpgs = 0;
605 page_t *rootpp;
606 rctl_qty_t unlocked_bytes = 0;
607 kproject_t *proj;
608 kshmid_t *sp;
609
610 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
611
612 len = P2ROUNDUP(len, PAGESIZE);
613
614 npages = btop(len);
615
616 hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP;
617 if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) ||
618 (sptd->spt_flags & SHM_PAGEABLE)) {
619 hat_flags = HAT_UNLOAD_UNMAP;
620 }
621
622 hat_unload(seg->s_as->a_hat, addr, len, hat_flags);
623
624 amp = sptd->spt_amp;
625 if (sptd->spt_flags & SHM_PAGEABLE)
626 npages = btop(amp->size);
627
628 ASSERT(amp != NULL);
629
630 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
631 sp = amp->a_sp;
632 proj = sp->shm_perm.ipc_proj;
633 mutex_enter(&sp->shm_mlock);
634 }
635 for (anon_idx = 0; anon_idx < npages; anon_idx++) {
636 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
637 if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
638 panic("segspt_free_pages: null app");
639 /*NOTREACHED*/
640 }
641 } else {
642 if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx))
643 == NULL)
644 continue;
645 }
646 ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0);
647 swap_xlate(ap, &vp, &off);
648
649 /*
650 * If this platform supports HAT_DYNAMIC_ISM_UNMAP,
651 * the pages won't be having SE_SHARED lock at this
652 * point.
653 *
654 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
655 * the pages are still held SE_SHARED locked from the
656 * original segspt_create()
657 *
658 * Our goal is to get SE_EXCL lock on each page, remove
659 * permanent lock on it and invalidate the page.
660 */
661 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
662 if (hat_flags == HAT_UNLOAD_UNMAP)
663 pp = page_lookup(&vp->v_object, off, SE_EXCL);
664 else {
665 if ((pp = page_find(&vp->v_object, off)) == NULL) {
666 panic("segspt_free_pages: "
667 "page not locked");
668 /*NOTREACHED*/
669 }
670 if (!page_tryupgrade(pp)) {
671 page_unlock(pp);
672 pp = page_lookup(&vp->v_object, off,
673 SE_EXCL);
674 }
675 }
676 if (pp == NULL) {
677 panic("segspt_free_pages: "
678 "page not in the system");
679 /*NOTREACHED*/
680 }
681 ASSERT(pp->p_lckcnt > 0);
682 page_pp_unlock(pp, 0, 1);
683 if (pp->p_lckcnt == 0)
684 unlocked_bytes += PAGESIZE;
685 } else {
686 if ((pp = page_lookup(&vp->v_object, off, SE_EXCL)) == NULL)
687 continue;
688 }
689 /*
690 * It's logical to invalidate the pages here as in most cases
691 * these were created by segspt.
692 */
693 if (pp->p_szc != 0) {
694 if (root == 0) {
695 ASSERT(curnpgs == 0);
696 root = 1;
697 rootpp = pp;
698 pgs = curnpgs = page_get_pagecnt(pp->p_szc);
699 ASSERT(pgs > 1);
700 ASSERT(IS_P2ALIGNED(pgs, pgs));
701 ASSERT(!(page_pptonum(pp) & (pgs - 1)));
702 curnpgs--;
703 } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) {
704 ASSERT(curnpgs == 1);
705 ASSERT(page_pptonum(pp) ==
706 page_pptonum(rootpp) + (pgs - 1));
707 page_destroy_pages(rootpp);
708 root = 0;
709 curnpgs = 0;
710 } else {
711 ASSERT(curnpgs > 1);
712 ASSERT(page_pptonum(pp) ==
713 page_pptonum(rootpp) + (pgs - curnpgs));
714 curnpgs--;
715 }
716 } else {
717 if (root != 0 || curnpgs != 0) {
718 panic("segspt_free_pages: bad large page");
719 /*NOTREACHED*/
720 }
721 /*
722 * Before destroying the pages, we need to take care
723 * of the rctl locked memory accounting. For that
724 * we need to calculte the unlocked_bytes.
725 */
726 if (pp->p_lckcnt > 0)
727 unlocked_bytes += PAGESIZE;
728
729 VN_DISPOSE(pp, B_INVAL, 0, kcred);
730 }
731 }
732 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
733 if (unlocked_bytes > 0)
734 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
735 mutex_exit(&sp->shm_mlock);
736 }
737 if (root != 0 || curnpgs != 0) {
738 panic("segspt_free_pages: bad large page");
739 /*NOTREACHED*/
740 }
741
742 /*
743 * mark that pages have been released
744 */
745 sptd->spt_realsize = 0;
746
747 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
748 atomic_add_long(&spt_used, -npages);
749 anon_swap_restore(npages);
750 }
751 }
752
753 /*
754 * Get memory allocation policy info for specified address in given segment
755 */
756 static lgrp_mem_policy_info_t *
757 segspt_getpolicy(struct seg *seg, caddr_t addr)
758 {
759 struct anon_map *amp;
760 ulong_t anon_index;
761 lgrp_mem_policy_info_t *policy_info;
762 struct spt_data *spt_data;
763
764 ASSERT(seg != NULL);
765
766 /*
767 * Get anon_map from segspt
768 *
769 * Assume that no lock needs to be held on anon_map, since
770 * it should be protected by its reference count which must be
771 * nonzero for an existing segment
772 * Need to grab readers lock on policy tree though
773 */
774 spt_data = (struct spt_data *)seg->s_data;
775 if (spt_data == NULL)
776 return (NULL);
777 amp = spt_data->spt_amp;
778 ASSERT(amp->refcnt != 0);
779
780 /*
781 * Get policy info
782 *
783 * Assume starting anon index of 0
784 */
785 anon_index = seg_page(seg, addr);
786 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
787
788 return (policy_info);
789 }
790
791 /*
792 * DISM only.
793 * Return locked pages over a given range.
794 *
795 * We will cache all DISM locked pages and save the pplist for the
796 * entire segment in the ppa field of the underlying DISM segment structure.
797 * Later, during a call to segspt_reclaim() we will use this ppa array
798 * to page_unlock() all of the pages and then we will free this ppa list.
799 */
800 /*ARGSUSED*/
801 static int
802 segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len,
803 struct page ***ppp, enum lock_type type, enum seg_rw rw)
804 {
805 struct shm_data *shmd = (struct shm_data *)seg->s_data;
806 struct seg *sptseg = shmd->shm_sptseg;
807 struct spt_data *sptd = sptseg->s_data;
808 pgcnt_t pg_idx, npages, tot_npages, npgs;
809 struct page **pplist, **pl, **ppa, *pp;
810 struct anon_map *amp;
811 spgcnt_t an_idx;
812 int ret = ENOTSUP;
813 uint_t pl_built = 0;
814 struct anon *ap;
815 struct vnode *vp;
816 uoff_t off;
817 pgcnt_t claim_availrmem = 0;
818 uint_t szc;
819
820 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
821 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
822
823 /*
824 * We want to lock/unlock the entire ISM segment. Therefore,
825 * we will be using the underlying sptseg and it's base address
826 * and length for the caching arguments.
827 */
828 ASSERT(sptseg);
829 ASSERT(sptd);
830
831 pg_idx = seg_page(seg, addr);
832 npages = btopr(len);
833
834 /*
835 * check if the request is larger than number of pages covered
836 * by amp
837 */
838 if (pg_idx + npages > btopr(sptd->spt_amp->size)) {
839 *ppp = NULL;
840 return (ENOTSUP);
841 }
842
843 if (type == L_PAGEUNLOCK) {
844 ASSERT(sptd->spt_ppa != NULL);
845
846 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
847 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
848
849 /*
850 * If someone is blocked while unmapping, we purge
851 * segment page cache and thus reclaim pplist synchronously
852 * without waiting for seg_pasync_thread. This speeds up
853 * unmapping in cases where munmap(2) is called, while
854 * raw async i/o is still in progress or where a thread
855 * exits on data fault in a multithreaded application.
856 */
857 if ((sptd->spt_flags & DISM_PPA_CHANGED) ||
858 (AS_ISUNMAPWAIT(seg->s_as) &&
859 shmd->shm_softlockcnt > 0)) {
860 segspt_purge(seg);
861 }
862 return (0);
863 }
864
865 /* The L_PAGELOCK case ... */
866
867 if (sptd->spt_flags & DISM_PPA_CHANGED) {
868 segspt_purge(seg);
869 /*
870 * for DISM ppa needs to be rebuild since
871 * number of locked pages could be changed
872 */
873 *ppp = NULL;
874 return (ENOTSUP);
875 }
876
877 /*
878 * First try to find pages in segment page cache, without
879 * holding the segment lock.
880 */
881 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
882 S_WRITE, SEGP_FORCE_WIRED);
883 if (pplist != NULL) {
884 ASSERT(sptd->spt_ppa != NULL);
885 ASSERT(sptd->spt_ppa == pplist);
886 ppa = sptd->spt_ppa;
887 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
888 if (ppa[an_idx] == NULL) {
889 seg_pinactive(seg, NULL, seg->s_base,
890 sptd->spt_amp->size, ppa,
891 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
892 *ppp = NULL;
893 return (ENOTSUP);
894 }
895 if ((szc = ppa[an_idx]->p_szc) != 0) {
896 npgs = page_get_pagecnt(szc);
897 an_idx = P2ROUNDUP(an_idx + 1, npgs);
898 } else {
899 an_idx++;
900 }
901 }
902 /*
903 * Since we cache the entire DISM segment, we want to
904 * set ppp to point to the first slot that corresponds
905 * to the requested addr, i.e. pg_idx.
906 */
907 *ppp = &(sptd->spt_ppa[pg_idx]);
908 return (0);
909 }
910
911 mutex_enter(&sptd->spt_lock);
912 /*
913 * try to find pages in segment page cache with mutex
914 */
915 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
916 S_WRITE, SEGP_FORCE_WIRED);
917 if (pplist != NULL) {
918 ASSERT(sptd->spt_ppa != NULL);
919 ASSERT(sptd->spt_ppa == pplist);
920 ppa = sptd->spt_ppa;
921 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
922 if (ppa[an_idx] == NULL) {
923 mutex_exit(&sptd->spt_lock);
924 seg_pinactive(seg, NULL, seg->s_base,
925 sptd->spt_amp->size, ppa,
926 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
927 *ppp = NULL;
928 return (ENOTSUP);
929 }
930 if ((szc = ppa[an_idx]->p_szc) != 0) {
931 npgs = page_get_pagecnt(szc);
932 an_idx = P2ROUNDUP(an_idx + 1, npgs);
933 } else {
934 an_idx++;
935 }
936 }
937 /*
938 * Since we cache the entire DISM segment, we want to
939 * set ppp to point to the first slot that corresponds
940 * to the requested addr, i.e. pg_idx.
941 */
942 mutex_exit(&sptd->spt_lock);
943 *ppp = &(sptd->spt_ppa[pg_idx]);
944 return (0);
945 }
946 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
947 SEGP_FORCE_WIRED) == SEGP_FAIL) {
948 mutex_exit(&sptd->spt_lock);
949 *ppp = NULL;
950 return (ENOTSUP);
951 }
952
953 /*
954 * No need to worry about protections because DISM pages are always rw.
955 */
956 pl = pplist = NULL;
957 amp = sptd->spt_amp;
958
959 /*
960 * Do we need to build the ppa array?
961 */
962 if (sptd->spt_ppa == NULL) {
963 pgcnt_t lpg_cnt = 0;
964
965 pl_built = 1;
966 tot_npages = btopr(sptd->spt_amp->size);
967
968 ASSERT(sptd->spt_pcachecnt == 0);
969 pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP);
970 pl = pplist;
971
972 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
973 for (an_idx = 0; an_idx < tot_npages; ) {
974 ap = anon_get_ptr(amp->ahp, an_idx);
975 /*
976 * Cache only mlocked pages. For large pages
977 * if one (constituent) page is mlocked
978 * all pages for that large page
979 * are cached also. This is for quick
980 * lookups of ppa array;
981 */
982 if ((ap != NULL) && (lpg_cnt != 0 ||
983 (sptd->spt_ppa_lckcnt[an_idx] != 0))) {
984
985 swap_xlate(ap, &vp, &off);
986 pp = page_lookup(&vp->v_object, off,
987 SE_SHARED);
988 ASSERT(pp != NULL);
989 if (lpg_cnt == 0) {
990 lpg_cnt++;
991 /*
992 * For a small page, we are done --
993 * lpg_count is reset to 0 below.
994 *
995 * For a large page, we are guaranteed
996 * to find the anon structures of all
997 * constituent pages and a non-zero
998 * lpg_cnt ensures that we don't test
999 * for mlock for these. We are done
1000 * when lpg_count reaches (npgs + 1).
1001 * If we are not the first constituent
1002 * page, restart at the first one.
1003 */
1004 npgs = page_get_pagecnt(pp->p_szc);
1005 if (!IS_P2ALIGNED(an_idx, npgs)) {
1006 an_idx = P2ALIGN(an_idx, npgs);
1007 page_unlock(pp);
1008 continue;
1009 }
1010 }
1011 if (++lpg_cnt > npgs)
1012 lpg_cnt = 0;
1013
1014 /*
1015 * availrmem is decremented only
1016 * for unlocked pages
1017 */
1018 if (sptd->spt_ppa_lckcnt[an_idx] == 0)
1019 claim_availrmem++;
1020 pplist[an_idx] = pp;
1021 }
1022 an_idx++;
1023 }
1024 ANON_LOCK_EXIT(&amp->a_rwlock);
1025
1026 if (claim_availrmem) {
1027 mutex_enter(&freemem_lock);
1028 if (availrmem < tune.t_minarmem + claim_availrmem) {
1029 mutex_exit(&freemem_lock);
1030 ret = ENOTSUP;
1031 claim_availrmem = 0;
1032 goto insert_fail;
1033 } else {
1034 availrmem -= claim_availrmem;
1035 }
1036 mutex_exit(&freemem_lock);
1037 }
1038
1039 sptd->spt_ppa = pl;
1040 } else {
1041 /*
1042 * We already have a valid ppa[].
1043 */
1044 pl = sptd->spt_ppa;
1045 }
1046
1047 ASSERT(pl != NULL);
1048
1049 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1050 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
1051 segspt_reclaim);
1052 if (ret == SEGP_FAIL) {
1053 /*
1054 * seg_pinsert failed. We return
1055 * ENOTSUP, so that the as_pagelock() code will
1056 * then try the slower F_SOFTLOCK path.
1057 */
1058 if (pl_built) {
1059 /*
1060 * No one else has referenced the ppa[].
1061 * We created it and we need to destroy it.
1062 */
1063 sptd->spt_ppa = NULL;
1064 }
1065 ret = ENOTSUP;
1066 goto insert_fail;
1067 }
1068
1069 /*
1070 * In either case, we increment softlockcnt on the 'real' segment.
1071 */
1072 sptd->spt_pcachecnt++;
1073 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1074
1075 ppa = sptd->spt_ppa;
1076 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
1077 if (ppa[an_idx] == NULL) {
1078 mutex_exit(&sptd->spt_lock);
1079 seg_pinactive(seg, NULL, seg->s_base,
1080 sptd->spt_amp->size,
1081 pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
1082 *ppp = NULL;
1083 return (ENOTSUP);
1084 }
1085 if ((szc = ppa[an_idx]->p_szc) != 0) {
1086 npgs = page_get_pagecnt(szc);
1087 an_idx = P2ROUNDUP(an_idx + 1, npgs);
1088 } else {
1089 an_idx++;
1090 }
1091 }
1092 /*
1093 * We can now drop the sptd->spt_lock since the ppa[]
1094 * exists and we have incremented pacachecnt.
1095 */
1096 mutex_exit(&sptd->spt_lock);
1097
1098 /*
1099 * Since we cache the entire segment, we want to
1100 * set ppp to point to the first slot that corresponds
1101 * to the requested addr, i.e. pg_idx.
1102 */
1103 *ppp = &(sptd->spt_ppa[pg_idx]);
1104 return (0);
1105
1106 insert_fail:
1107 /*
1108 * We will only reach this code if we tried and failed.
1109 *
1110 * And we can drop the lock on the dummy seg, once we've failed
1111 * to set up a new ppa[].
1112 */
1113 mutex_exit(&sptd->spt_lock);
1114
1115 if (pl_built) {
1116 if (claim_availrmem) {
1117 mutex_enter(&freemem_lock);
1118 availrmem += claim_availrmem;
1119 mutex_exit(&freemem_lock);
1120 }
1121
1122 /*
1123 * We created pl and we need to destroy it.
1124 */
1125 pplist = pl;
1126 for (an_idx = 0; an_idx < tot_npages; an_idx++) {
1127 if (pplist[an_idx] != NULL)
1128 page_unlock(pplist[an_idx]);
1129 }
1130 kmem_free(pl, sizeof (page_t *) * tot_npages);
1131 }
1132
1133 if (shmd->shm_softlockcnt <= 0) {
1134 if (AS_ISUNMAPWAIT(seg->s_as)) {
1135 mutex_enter(&seg->s_as->a_contents);
1136 if (AS_ISUNMAPWAIT(seg->s_as)) {
1137 AS_CLRUNMAPWAIT(seg->s_as);
1138 cv_broadcast(&seg->s_as->a_cv);
1139 }
1140 mutex_exit(&seg->s_as->a_contents);
1141 }
1142 }
1143 *ppp = NULL;
1144 return (ret);
1145 }
1146
1147
1148
1149 /*
1150 * return locked pages over a given range.
1151 *
1152 * We will cache the entire ISM segment and save the pplist for the
1153 * entire segment in the ppa field of the underlying ISM segment structure.
1154 * Later, during a call to segspt_reclaim() we will use this ppa array
1155 * to page_unlock() all of the pages and then we will free this ppa list.
1156 */
1157 /*ARGSUSED*/
1158 static int
1159 segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len,
1160 struct page ***ppp, enum lock_type type, enum seg_rw rw)
1161 {
1162 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1163 struct seg *sptseg = shmd->shm_sptseg;
1164 struct spt_data *sptd = sptseg->s_data;
1165 pgcnt_t np, page_index, npages;
1166 caddr_t a, spt_base;
1167 struct page **pplist, **pl, *pp;
1168 struct anon_map *amp;
1169 ulong_t anon_index;
1170 int ret = ENOTSUP;
1171 uint_t pl_built = 0;
1172 struct anon *ap;
1173 struct vnode *vp;
1174 uoff_t off;
1175
1176 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1177 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
1178
1179
1180 /*
1181 * We want to lock/unlock the entire ISM segment. Therefore,
1182 * we will be using the underlying sptseg and it's base address
1183 * and length for the caching arguments.
1184 */
1185 ASSERT(sptseg);
1186 ASSERT(sptd);
1187
1188 if (sptd->spt_flags & SHM_PAGEABLE) {
1189 return (segspt_dismpagelock(seg, addr, len, ppp, type, rw));
1190 }
1191
1192 page_index = seg_page(seg, addr);
1193 npages = btopr(len);
1194
1195 /*
1196 * check if the request is larger than number of pages covered
1197 * by amp
1198 */
1199 if (page_index + npages > btopr(sptd->spt_amp->size)) {
1200 *ppp = NULL;
1201 return (ENOTSUP);
1202 }
1203
1204 if (type == L_PAGEUNLOCK) {
1205
1206 ASSERT(sptd->spt_ppa != NULL);
1207
1208 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
1209 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
1210
1211 /*
1212 * If someone is blocked while unmapping, we purge
1213 * segment page cache and thus reclaim pplist synchronously
1214 * without waiting for seg_pasync_thread. This speeds up
1215 * unmapping in cases where munmap(2) is called, while
1216 * raw async i/o is still in progress or where a thread
1217 * exits on data fault in a multithreaded application.
1218 */
1219 if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
1220 segspt_purge(seg);
1221 }
1222 return (0);
1223 }
1224
1225 /* The L_PAGELOCK case... */
1226
1227 /*
1228 * First try to find pages in segment page cache, without
1229 * holding the segment lock.
1230 */
1231 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1232 S_WRITE, SEGP_FORCE_WIRED);
1233 if (pplist != NULL) {
1234 ASSERT(sptd->spt_ppa == pplist);
1235 ASSERT(sptd->spt_ppa[page_index]);
1236 /*
1237 * Since we cache the entire ISM segment, we want to
1238 * set ppp to point to the first slot that corresponds
1239 * to the requested addr, i.e. page_index.
1240 */
1241 *ppp = &(sptd->spt_ppa[page_index]);
1242 return (0);
1243 }
1244
1245 mutex_enter(&sptd->spt_lock);
1246
1247 /*
1248 * try to find pages in segment page cache
1249 */
1250 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1251 S_WRITE, SEGP_FORCE_WIRED);
1252 if (pplist != NULL) {
1253 ASSERT(sptd->spt_ppa == pplist);
1254 /*
1255 * Since we cache the entire segment, we want to
1256 * set ppp to point to the first slot that corresponds
1257 * to the requested addr, i.e. page_index.
1258 */
1259 mutex_exit(&sptd->spt_lock);
1260 *ppp = &(sptd->spt_ppa[page_index]);
1261 return (0);
1262 }
1263
1264 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
1265 SEGP_FORCE_WIRED) == SEGP_FAIL) {
1266 mutex_exit(&sptd->spt_lock);
1267 *ppp = NULL;
1268 return (ENOTSUP);
1269 }
1270
1271 /*
1272 * No need to worry about protections because ISM pages
1273 * are always rw.
1274 */
1275 pl = pplist = NULL;
1276
1277 /*
1278 * Do we need to build the ppa array?
1279 */
1280 if (sptd->spt_ppa == NULL) {
1281 ASSERT(sptd->spt_ppa == pplist);
1282
1283 spt_base = sptseg->s_base;
1284 pl_built = 1;
1285
1286 /*
1287 * availrmem is decremented once during anon_swap_adjust()
1288 * and is incremented during the anon_unresv(), which is
1289 * called from shm_rm_amp() when the segment is destroyed.
1290 */
1291 amp = sptd->spt_amp;
1292 ASSERT(amp != NULL);
1293
1294 /* pcachecnt is protected by sptd->spt_lock */
1295 ASSERT(sptd->spt_pcachecnt == 0);
1296 pplist = kmem_zalloc(sizeof (page_t *)
1297 * btopr(sptd->spt_amp->size), KM_SLEEP);
1298 pl = pplist;
1299
1300 anon_index = seg_page(sptseg, spt_base);
1301
1302 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
1303 for (a = spt_base; a < (spt_base + sptd->spt_amp->size);
1304 a += PAGESIZE, anon_index++, pplist++) {
1305 ap = anon_get_ptr(amp->ahp, anon_index);
1306 ASSERT(ap != NULL);
1307 swap_xlate(ap, &vp, &off);
1308 pp = page_lookup(&vp->v_object, off, SE_SHARED);
1309 ASSERT(pp != NULL);
1310 *pplist = pp;
1311 }
1312 ANON_LOCK_EXIT(&amp->a_rwlock);
1313
1314 if (a < (spt_base + sptd->spt_amp->size)) {
1315 ret = ENOTSUP;
1316 goto insert_fail;
1317 }
1318 sptd->spt_ppa = pl;
1319 } else {
1320 /*
1321 * We already have a valid ppa[].
1322 */
1323 pl = sptd->spt_ppa;
1324 }
1325
1326 ASSERT(pl != NULL);
1327
1328 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1329 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
1330 segspt_reclaim);
1331 if (ret == SEGP_FAIL) {
1332 /*
1333 * seg_pinsert failed. We return
1334 * ENOTSUP, so that the as_pagelock() code will
1335 * then try the slower F_SOFTLOCK path.
1336 */
1337 if (pl_built) {
1338 /*
1339 * No one else has referenced the ppa[].
1340 * We created it and we need to destroy it.
1341 */
1342 sptd->spt_ppa = NULL;
1343 }
1344 ret = ENOTSUP;
1345 goto insert_fail;
1346 }
1347
1348 /*
1349 * In either case, we increment softlockcnt on the 'real' segment.
1350 */
1351 sptd->spt_pcachecnt++;
1352 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1353
1354 /*
1355 * We can now drop the sptd->spt_lock since the ppa[]
1356 * exists and we have incremented pacachecnt.
1357 */
1358 mutex_exit(&sptd->spt_lock);
1359
1360 /*
1361 * Since we cache the entire segment, we want to
1362 * set ppp to point to the first slot that corresponds
1363 * to the requested addr, i.e. page_index.
1364 */
1365 *ppp = &(sptd->spt_ppa[page_index]);
1366 return (0);
1367
1368 insert_fail:
1369 /*
1370 * We will only reach this code if we tried and failed.
1371 *
1372 * And we can drop the lock on the dummy seg, once we've failed
1373 * to set up a new ppa[].
1374 */
1375 mutex_exit(&sptd->spt_lock);
1376
1377 if (pl_built) {
1378 /*
1379 * We created pl and we need to destroy it.
1380 */
1381 pplist = pl;
1382 np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT);
1383 while (np) {
1384 page_unlock(*pplist);
1385 np--;
1386 pplist++;
1387 }
1388 kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size));
1389 }
1390 if (shmd->shm_softlockcnt <= 0) {
1391 if (AS_ISUNMAPWAIT(seg->s_as)) {
1392 mutex_enter(&seg->s_as->a_contents);
1393 if (AS_ISUNMAPWAIT(seg->s_as)) {
1394 AS_CLRUNMAPWAIT(seg->s_as);
1395 cv_broadcast(&seg->s_as->a_cv);
1396 }
1397 mutex_exit(&seg->s_as->a_contents);
1398 }
1399 }
1400 *ppp = NULL;
1401 return (ret);
1402 }
1403
1404 /*
1405 * purge any cached pages in the I/O page cache
1406 */
1407 static void
1408 segspt_purge(struct seg *seg)
1409 {
1410 seg_ppurge(seg, NULL, SEGP_FORCE_WIRED);
1411 }
1412
1413 static int
1414 segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist,
1415 enum seg_rw rw, int async)
1416 {
1417 struct seg *seg = (struct seg *)ptag;
1418 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1419 struct seg *sptseg;
1420 struct spt_data *sptd;
1421 pgcnt_t npages, i, free_availrmem = 0;
1422 int done = 0;
1423
1424 sptseg = shmd->shm_sptseg;
1425 sptd = sptseg->s_data;
1426 npages = (len >> PAGESHIFT);
1427 ASSERT(npages);
1428 ASSERT(sptd->spt_pcachecnt != 0);
1429 ASSERT(sptd->spt_ppa == pplist);
1430 ASSERT(npages == btopr(sptd->spt_amp->size));
1431 ASSERT(async || AS_LOCK_HELD(seg->s_as));
1432
1433 /*
1434 * Acquire the lock on the dummy seg and destroy the
1435 * ppa array IF this is the last pcachecnt.
1436 */
1437 mutex_enter(&sptd->spt_lock);
1438 if (--sptd->spt_pcachecnt == 0) {
1439 for (i = 0; i < npages; i++) {
1440 if (pplist[i] == NULL) {
1441 continue;
1442 }
1443 if (rw == S_WRITE) {
1444 hat_setrefmod(pplist[i]);
1445 } else {
1446 hat_setref(pplist[i]);
1447 }
1448 if ((sptd->spt_flags & SHM_PAGEABLE) &&
1449 (sptd->spt_ppa_lckcnt[i] == 0))
1450 free_availrmem++;
1451 page_unlock(pplist[i]);
1452 }
1453 if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) {
1454 mutex_enter(&freemem_lock);
1455 availrmem += free_availrmem;
1456 mutex_exit(&freemem_lock);
1457 }
1458 /*
1459 * Since we want to cach/uncache the entire ISM segment,
1460 * we will track the pplist in a segspt specific field
1461 * ppa, that is initialized at the time we add an entry to
1462 * the cache.
1463 */
1464 ASSERT(sptd->spt_pcachecnt == 0);
1465 kmem_free(pplist, sizeof (page_t *) * npages);
1466 sptd->spt_ppa = NULL;
1467 sptd->spt_flags &= ~DISM_PPA_CHANGED;
1468 sptd->spt_gen++;
1469 cv_broadcast(&sptd->spt_cv);
1470 done = 1;
1471 }
1472 mutex_exit(&sptd->spt_lock);
1473
1474 /*
1475 * If we are pcache async thread or called via seg_ppurge_wiredpp() we
1476 * may not hold AS lock (in this case async argument is not 0). This
1477 * means if softlockcnt drops to 0 after the decrement below address
1478 * space may get freed. We can't allow it since after softlock
1479 * derement to 0 we still need to access as structure for possible
1480 * wakeup of unmap waiters. To prevent the disappearance of as we take
1481 * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes
1482 * this mutex as a barrier to make sure this routine completes before
1483 * segment is freed.
1484 *
1485 * The second complication we have to deal with in async case is a
1486 * possibility of missed wake up of unmap wait thread. When we don't
1487 * hold as lock here we may take a_contents lock before unmap wait
1488 * thread that was first to see softlockcnt was still not 0. As a
1489 * result we'll fail to wake up an unmap wait thread. To avoid this
1490 * race we set nounmapwait flag in as structure if we drop softlockcnt
1491 * to 0 if async is not 0. unmapwait thread
1492 * will not block if this flag is set.
1493 */
1494 if (async)
1495 mutex_enter(&shmd->shm_segfree_syncmtx);
1496
1497 /*
1498 * Now decrement softlockcnt.
1499 */
1500 ASSERT(shmd->shm_softlockcnt > 0);
1501 atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1502
1503 if (shmd->shm_softlockcnt <= 0) {
1504 if (async || AS_ISUNMAPWAIT(seg->s_as)) {
1505 mutex_enter(&seg->s_as->a_contents);
1506 if (async)
1507 AS_SETNOUNMAPWAIT(seg->s_as);
1508 if (AS_ISUNMAPWAIT(seg->s_as)) {
1509 AS_CLRUNMAPWAIT(seg->s_as);
1510 cv_broadcast(&seg->s_as->a_cv);
1511 }
1512 mutex_exit(&seg->s_as->a_contents);
1513 }
1514 }
1515
1516 if (async)
1517 mutex_exit(&shmd->shm_segfree_syncmtx);
1518
1519 return (done);
1520 }
1521
1522 /*
1523 * Do a F_SOFTUNLOCK call over the range requested.
1524 * The range must have already been F_SOFTLOCK'ed.
1525 *
1526 * The calls to acquire and release the anon map lock mutex were
1527 * removed in order to avoid a deadly embrace during a DR
1528 * memory delete operation. (Eg. DR blocks while waiting for a
1529 * exclusive lock on a page that is being used for kaio; the
1530 * thread that will complete the kaio and call segspt_softunlock
1531 * blocks on the anon map lock; another thread holding the anon
1532 * map lock blocks on another page lock via the segspt_shmfault
1533 * -> page_lookup -> page_lookup_create -> page_lock_es code flow.)
1534 *
1535 * The appropriateness of the removal is based upon the following:
1536 * 1. If we are holding a segment's reader lock and the page is held
1537 * shared, then the corresponding element in anonmap which points to
1538 * anon struct cannot change and there is no need to acquire the
1539 * anonymous map lock.
1540 * 2. Threads in segspt_softunlock have a reader lock on the segment
1541 * and already have the shared page lock, so we are guaranteed that
1542 * the anon map slot cannot change and therefore can call anon_get_ptr()
1543 * without grabbing the anonymous map lock.
1544 * 3. Threads that softlock a shared page break copy-on-write, even if
1545 * its a read. Thus cow faults can be ignored with respect to soft
1546 * unlocking, since the breaking of cow means that the anon slot(s) will
1547 * not be shared.
1548 */
1549 static void
1550 segspt_softunlock(struct seg *seg, caddr_t sptseg_addr,
1551 size_t len, enum seg_rw rw)
1552 {
1553 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1554 struct seg *sptseg;
1555 struct spt_data *sptd;
1556 page_t *pp;
1557 caddr_t adr;
1558 struct vnode *vp;
1559 uoff_t offset;
1560 ulong_t anon_index;
1561 struct anon_map *amp; /* XXX - for locknest */
1562 struct anon *ap = NULL;
1563 pgcnt_t npages;
1564
1565 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1566
1567 sptseg = shmd->shm_sptseg;
1568 sptd = sptseg->s_data;
1569
1570 /*
1571 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
1572 * and therefore their pages are SE_SHARED locked
1573 * for the entire life of the segment.
1574 */
1575 if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) &&
1576 ((sptd->spt_flags & SHM_PAGEABLE) == 0)) {
1577 goto softlock_decrement;
1578 }
1579
1580 /*
1581 * Any thread is free to do a page_find and
1582 * page_unlock() on the pages within this seg.
1583 *
1584 * We are already holding the as->a_lock on the user's
1585 * real segment, but we need to hold the a_lock on the
1586 * underlying dummy as. This is mostly to satisfy the
1587 * underlying HAT layer.
1588 */
1589 AS_LOCK_ENTER(sptseg->s_as, RW_READER);
1590 hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len);
1591 AS_LOCK_EXIT(sptseg->s_as);
1592
1593 amp = sptd->spt_amp;
1594 ASSERT(amp != NULL);
1595 anon_index = seg_page(sptseg, sptseg_addr);
1596
1597 for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) {
1598 ap = anon_get_ptr(amp->ahp, anon_index++);
1599 ASSERT(ap != NULL);
1600 swap_xlate(ap, &vp, &offset);
1601
1602 /*
1603 * Use page_find() instead of page_lookup() to
1604 * find the page since we know that it has a
1605 * "shared" lock.
1606 */
1607 pp = page_find(&vp->v_object, offset);
1608 ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1));
1609 if (pp == NULL) {
1610 panic("segspt_softunlock: "
1611 "addr %p, ap %p, vp %p, off %llx",
1612 (void *)adr, (void *)ap, (void *)vp, offset);
1613 /*NOTREACHED*/
1614 }
1615
1616 if (rw == S_WRITE) {
1617 hat_setrefmod(pp);
1618 } else if (rw != S_OTHER) {
1619 hat_setref(pp);
1620 }
1621 page_unlock(pp);
1622 }
1623
1624 softlock_decrement:
1625 npages = btopr(len);
1626 ASSERT(shmd->shm_softlockcnt >= npages);
1627 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages);
1628 if (shmd->shm_softlockcnt == 0) {
1629 /*
1630 * All SOFTLOCKS are gone. Wakeup any waiting
1631 * unmappers so they can try again to unmap.
1632 * Check for waiters first without the mutex
1633 * held so we don't always grab the mutex on
1634 * softunlocks.
1635 */
1636 if (AS_ISUNMAPWAIT(seg->s_as)) {
1637 mutex_enter(&seg->s_as->a_contents);
1638 if (AS_ISUNMAPWAIT(seg->s_as)) {
1639 AS_CLRUNMAPWAIT(seg->s_as);
1640 cv_broadcast(&seg->s_as->a_cv);
1641 }
1642 mutex_exit(&seg->s_as->a_contents);
1643 }
1644 }
1645 }
1646
1647 int
1648 segspt_shmattach(struct seg *seg, caddr_t *argsp)
1649 {
1650 struct shm_data *shmd_arg = (struct shm_data *)argsp;
1651 struct shm_data *shmd;
1652 struct anon_map *shm_amp = shmd_arg->shm_amp;
1653 struct spt_data *sptd;
1654 int error = 0;
1655
1656 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1657
1658 shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP);
1659 if (shmd == NULL)
1660 return (ENOMEM);
1661
1662 shmd->shm_sptas = shmd_arg->shm_sptas;
1663 shmd->shm_amp = shm_amp;
1664 shmd->shm_sptseg = shmd_arg->shm_sptseg;
1665
1666 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0,
1667 NULL, 0, seg->s_size);
1668
1669 mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL);
1670
1671 seg->s_data = (void *)shmd;
1672 seg->s_ops = &segspt_shmops;
1673 seg->s_szc = shmd->shm_sptseg->s_szc;
1674 sptd = shmd->shm_sptseg->s_data;
1675
1676 if (sptd->spt_flags & SHM_PAGEABLE) {
1677 if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size),
1678 KM_NOSLEEP)) == NULL) {
1679 seg->s_data = NULL;
1680 kmem_free(shmd, (sizeof (*shmd)));
1681 return (ENOMEM);
1682 }
1683 shmd->shm_lckpgs = 0;
1684 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) {
1685 if ((error = hat_share(seg->s_as->a_hat, seg->s_base,
1686 shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
1687 seg->s_size, seg->s_szc)) != 0) {
1688 kmem_free(shmd->shm_vpage,
1689 btopr(shm_amp->size));
1690 }
1691 }
1692 } else {
1693 error = hat_share(seg->s_as->a_hat, seg->s_base,
1694 shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
1695 seg->s_size, seg->s_szc);
1696 }
1697 if (error) {
1698 seg->s_szc = 0;
1699 seg->s_data = NULL;
1700 kmem_free(shmd, (sizeof (*shmd)));
1701 } else {
1702 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
1703 shm_amp->refcnt++;
1704 ANON_LOCK_EXIT(&shm_amp->a_rwlock);
1705 }
1706 return (error);
1707 }
1708
1709 int
1710 segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize)
1711 {
1712 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1713 int reclaim = 1;
1714
1715 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1716 retry:
1717 if (shmd->shm_softlockcnt > 0) {
1718 if (reclaim == 1) {
1719 segspt_purge(seg);
1720 reclaim = 0;
1721 goto retry;
1722 }
1723 return (EAGAIN);
1724 }
1725
1726 if (ssize != seg->s_size) {
1727 #ifdef DEBUG
1728 cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n",
1729 ssize, seg->s_size);
1730 #endif
1731 return (EINVAL);
1732 }
1733
1734 (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK,
1735 NULL, 0);
1736 hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc);
1737
1738 seg_free(seg);
1739
1740 return (0);
1741 }
1742
1743 void
1744 segspt_shmfree(struct seg *seg)
1745 {
1746 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1747 struct anon_map *shm_amp = shmd->shm_amp;
1748
1749 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1750
1751 (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0,
1752 MC_UNLOCK, NULL, 0);
1753
1754 /*
1755 * Need to increment refcnt when attaching
1756 * and decrement when detaching because of dup().
1757 */
1758 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
1759 shm_amp->refcnt--;
1760 ANON_LOCK_EXIT(&shm_amp->a_rwlock);
1761
1762 if (shmd->shm_vpage) { /* only for DISM */
1763 kmem_free(shmd->shm_vpage, btopr(shm_amp->size));
1764 shmd->shm_vpage = NULL;
1765 }
1766
1767 /*
1768 * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's
1769 * still working with this segment without holding as lock.
1770 */
1771 ASSERT(shmd->shm_softlockcnt == 0);
1772 mutex_enter(&shmd->shm_segfree_syncmtx);
1773 mutex_destroy(&shmd->shm_segfree_syncmtx);
1774
1775 kmem_free(shmd, sizeof (*shmd));
1776 }
1777
1778 /*ARGSUSED*/
1779 int
1780 segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
1781 {
1782 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1783
1784 /*
1785 * Shared page table is more than shared mapping.
1786 * Individual process sharing page tables can't change prot
1787 * because there is only one set of page tables.
1788 * This will be allowed after private page table is
1789 * supported.
1790 */
1791 /* need to return correct status error? */
1792 return (0);
1793 }
1794
1795
1796 faultcode_t
1797 segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr,
1798 size_t len, enum fault_type type, enum seg_rw rw)
1799 {
1800 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1801 struct seg *sptseg = shmd->shm_sptseg;
1802 struct as *curspt = shmd->shm_sptas;
1803 struct spt_data *sptd = sptseg->s_data;
1804 pgcnt_t npages;
1805 size_t size;
1806 caddr_t segspt_addr, shm_addr;
1807 page_t **ppa;
1808 int i;
1809 ulong_t an_idx = 0;
1810 int err = 0;
1811 int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL);
1812 size_t pgsz;
1813 pgcnt_t pgcnt;
1814 caddr_t a;
1815 pgcnt_t pidx;
1816
1817 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1818
1819 /*
1820 * Because of the way spt is implemented
1821 * the realsize of the segment does not have to be
1822 * equal to the segment size itself. The segment size is
1823 * often in multiples of a page size larger than PAGESIZE.
1824 * The realsize is rounded up to the nearest PAGESIZE
1825 * based on what the user requested. This is a bit of
1826 * ungliness that is historical but not easily fixed
1827 * without re-designing the higher levels of ISM.
1828 */
1829 ASSERT(addr >= seg->s_base);
1830 if (((addr + len) - seg->s_base) > sptd->spt_realsize)
1831 return (FC_NOMAP);
1832 /*
1833 * For all of the following cases except F_PROT, we need to
1834 * make any necessary adjustments to addr and len
1835 * and get all of the necessary page_t's into an array called ppa[].
1836 *
1837 * The code in shmat() forces base addr and len of ISM segment
1838 * to be aligned to largest page size supported. Therefore,
1839 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
1840 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
1841 * in large pagesize chunks, or else we will screw up the HAT
1842 * layer by calling hat_memload_array() with differing page sizes
1843 * over a given virtual range.
1844 */
1845 pgsz = page_get_pagesize(sptseg->s_szc);
1846 pgcnt = page_get_pagecnt(sptseg->s_szc);
1847 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
1848 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
1849 npages = btopr(size);
1850
1851 /*
1852 * Now we need to convert from addr in segshm to addr in segspt.
1853 */
1854 an_idx = seg_page(seg, shm_addr);
1855 segspt_addr = sptseg->s_base + ptob(an_idx);
1856
1857 ASSERT((segspt_addr + ptob(npages)) <=
1858 (sptseg->s_base + sptd->spt_realsize));
1859 ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size));
1860
1861 switch (type) {
1862
1863 case F_SOFTLOCK:
1864
1865 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
1866 /*
1867 * Fall through to the F_INVAL case to load up the hat layer
1868 * entries with the HAT_LOAD_LOCK flag.
1869 */
1870 /* FALLTHRU */
1871 case F_INVAL:
1872
1873 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
1874 return (FC_NOMAP);
1875
1876 ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);
1877
1878 err = spt_anon_getpages(sptseg, segspt_addr, size, ppa);
1879 if (err != 0) {
1880 if (type == F_SOFTLOCK) {
1881 atomic_add_long((ulong_t *)(
1882 &(shmd->shm_softlockcnt)), -npages);
1883 }
1884 goto dism_err;
1885 }
1886 AS_LOCK_ENTER(sptseg->s_as, RW_READER);
1887 a = segspt_addr;
1888 pidx = 0;
1889 if (type == F_SOFTLOCK) {
1890
1891 /*
1892 * Load up the translation keeping it
1893 * locked and don't unlock the page.
1894 */
1895 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
1896 hat_memload_array(sptseg->s_as->a_hat,
1897 a, pgsz, &ppa[pidx], sptd->spt_prot,
1898 HAT_LOAD_LOCK | HAT_LOAD_SHARE);
1899 }
1900 } else {
1901 /*
1902 * Migrate pages marked for migration
1903 */
1904 if (lgrp_optimizations())
1905 page_migrate(seg, shm_addr, ppa, npages);
1906
1907 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
1908 hat_memload_array(sptseg->s_as->a_hat,
1909 a, pgsz, &ppa[pidx],
1910 sptd->spt_prot,
1911 HAT_LOAD_SHARE);
1912 }
1913
1914 /*
1915 * And now drop the SE_SHARED lock(s).
1916 */
1917 if (dyn_ism_unmap) {
1918 for (i = 0; i < npages; i++) {
1919 page_unlock(ppa[i]);
1920 }
1921 }
1922 }
1923
1924 if (!dyn_ism_unmap) {
1925 if (hat_share(seg->s_as->a_hat, shm_addr,
1926 curspt->a_hat, segspt_addr, ptob(npages),
1927 seg->s_szc) != 0) {
1928 panic("hat_share err in DISM fault");
1929 /* NOTREACHED */
1930 }
1931 if (type == F_INVAL) {
1932 for (i = 0; i < npages; i++) {
1933 page_unlock(ppa[i]);
1934 }
1935 }
1936 }
1937 AS_LOCK_EXIT(sptseg->s_as);
1938 dism_err:
1939 kmem_free(ppa, npages * sizeof (page_t *));
1940 return (err);
1941
1942 case F_SOFTUNLOCK:
1943
1944 /*
1945 * This is a bit ugly, we pass in the real seg pointer,
1946 * but the segspt_addr is the virtual address within the
1947 * dummy seg.
1948 */
1949 segspt_softunlock(seg, segspt_addr, size, rw);
1950 return (0);
1951
1952 case F_PROT:
1953
1954 /*
1955 * This takes care of the unusual case where a user
1956 * allocates a stack in shared memory and a register
1957 * window overflow is written to that stack page before
1958 * it is otherwise modified.
1959 *
1960 * We can get away with this because ISM segments are
1961 * always rw. Other than this unusual case, there
1962 * should be no instances of protection violations.
1963 */
1964 return (0);
1965
1966 default:
1967 #ifdef DEBUG
1968 panic("segspt_dismfault default type?");
1969 #else
1970 return (FC_NOMAP);
1971 #endif
1972 }
1973 }
1974
1975
1976 faultcode_t
1977 segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr,
1978 size_t len, enum fault_type type, enum seg_rw rw)
1979 {
1980 struct shm_data *shmd = (struct shm_data *)seg->s_data;
1981 struct seg *sptseg = shmd->shm_sptseg;
1982 struct as *curspt = shmd->shm_sptas;
1983 struct spt_data *sptd = sptseg->s_data;
1984 pgcnt_t npages;
1985 size_t size;
1986 caddr_t sptseg_addr, shm_addr;
1987 page_t *pp, **ppa;
1988 int i;
1989 uoff_t offset;
1990 ulong_t anon_index = 0;
1991 struct vnode *vp;
1992 struct anon_map *amp; /* XXX - for locknest */
1993 struct anon *ap = NULL;
1994 size_t pgsz;
1995 pgcnt_t pgcnt;
1996 caddr_t a;
1997 pgcnt_t pidx;
1998 size_t sz;
1999
2000
2001 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2002
2003 if (sptd->spt_flags & SHM_PAGEABLE) {
2004 return (segspt_dismfault(hat, seg, addr, len, type, rw));
2005 }
2006
2007 /*
2008 * Because of the way spt is implemented
2009 * the realsize of the segment does not have to be
2010 * equal to the segment size itself. The segment size is
2011 * often in multiples of a page size larger than PAGESIZE.
2012 * The realsize is rounded up to the nearest PAGESIZE
2013 * based on what the user requested. This is a bit of
2014 * ungliness that is historical but not easily fixed
2015 * without re-designing the higher levels of ISM.
2016 */
2017 ASSERT(addr >= seg->s_base);
2018 if (((addr + len) - seg->s_base) > sptd->spt_realsize)
2019 return (FC_NOMAP);
2020 /*
2021 * For all of the following cases except F_PROT, we need to
2022 * make any necessary adjustments to addr and len
2023 * and get all of the necessary page_t's into an array called ppa[].
2024 *
2025 * The code in shmat() forces base addr and len of ISM segment
2026 * to be aligned to largest page size supported. Therefore,
2027 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
2028 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
2029 * in large pagesize chunks, or else we will screw up the HAT
2030 * layer by calling hat_memload_array() with differing page sizes
2031 * over a given virtual range.
2032 */
2033 pgsz = page_get_pagesize(sptseg->s_szc);
2034 pgcnt = page_get_pagecnt(sptseg->s_szc);
2035 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
2036 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
2037 npages = btopr(size);
2038
2039 /*
2040 * Now we need to convert from addr in segshm to addr in segspt.
2041 */
2042 anon_index = seg_page(seg, shm_addr);
2043 sptseg_addr = sptseg->s_base + ptob(anon_index);
2044
2045 /*
2046 * And now we may have to adjust npages downward if we have
2047 * exceeded the realsize of the segment or initial anon
2048 * allocations.
2049 */
2050 if ((sptseg_addr + ptob(npages)) >
2051 (sptseg->s_base + sptd->spt_realsize))
2052 size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr;
2053
2054 npages = btopr(size);
2055
2056 ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size));
2057 ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0);
2058
2059 switch (type) {
2060
2061 case F_SOFTLOCK:
2062
2063 /*
2064 * availrmem is decremented once during anon_swap_adjust()
2065 * and is incremented during the anon_unresv(), which is
2066 * called from shm_rm_amp() when the segment is destroyed.
2067 */
2068 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
2069 /*
2070 * Some platforms assume that ISM pages are SE_SHARED
2071 * locked for the entire life of the segment.
2072 */
2073 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
2074 return (0);
2075 /*
2076 * Fall through to the F_INVAL case to load up the hat layer
2077 * entries with the HAT_LOAD_LOCK flag.
2078 */
2079
2080 /* FALLTHRU */
2081 case F_INVAL:
2082
2083 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
2084 return (FC_NOMAP);
2085
2086 /*
2087 * Some platforms that do NOT support DYNAMIC_ISM_UNMAP
2088 * may still rely on this call to hat_share(). That
2089 * would imply that those hat's can fault on a
2090 * HAT_LOAD_LOCK translation, which would seem
2091 * contradictory.
2092 */
2093 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) {
2094 if (hat_share(seg->s_as->a_hat, seg->s_base,
2095 curspt->a_hat, sptseg->s_base,
2096 sptseg->s_size, sptseg->s_szc) != 0) {
2097 panic("hat_share error in ISM fault");
2098 /*NOTREACHED*/
2099 }
2100 return (0);
2101 }
2102 ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP);
2103
2104 /*
2105 * I see no need to lock the real seg,
2106 * here, because all of our work will be on the underlying
2107 * dummy seg.
2108 *
2109 * sptseg_addr and npages now account for large pages.
2110 */
2111 amp = sptd->spt_amp;
2112 ASSERT(amp != NULL);
2113 anon_index = seg_page(sptseg, sptseg_addr);
2114
2115 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2116 for (i = 0; i < npages; i++) {
2117 ap = anon_get_ptr(amp->ahp, anon_index++);
2118 ASSERT(ap != NULL);
2119 swap_xlate(ap, &vp, &offset);
2120 pp = page_lookup(&vp->v_object, offset, SE_SHARED);
2121 ASSERT(pp != NULL);
2122 ppa[i] = pp;
2123 }
2124 ANON_LOCK_EXIT(&amp->a_rwlock);
2125 ASSERT(i == npages);
2126
2127 /*
2128 * We are already holding the as->a_lock on the user's
2129 * real segment, but we need to hold the a_lock on the
2130 * underlying dummy as. This is mostly to satisfy the
2131 * underlying HAT layer.
2132 */
2133 AS_LOCK_ENTER(sptseg->s_as, RW_READER);
2134 a = sptseg_addr;
2135 pidx = 0;
2136 if (type == F_SOFTLOCK) {
2137 /*
2138 * Load up the translation keeping it
2139 * locked and don't unlock the page.
2140 */
2141 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
2142 sz = MIN(pgsz, ptob(npages - pidx));
2143 hat_memload_array(sptseg->s_as->a_hat, a,
2144 sz, &ppa[pidx], sptd->spt_prot,
2145 HAT_LOAD_LOCK | HAT_LOAD_SHARE);
2146 }
2147 } else {
2148 /*
2149 * Migrate pages marked for migration.
2150 */
2151 if (lgrp_optimizations())
2152 page_migrate(seg, shm_addr, ppa, npages);
2153
2154 for (; pidx < npages; a += pgsz, pidx += pgcnt) {
2155 sz = MIN(pgsz, ptob(npages - pidx));
2156 hat_memload_array(sptseg->s_as->a_hat,
2157 a, sz, &ppa[pidx],
2158 sptd->spt_prot, HAT_LOAD_SHARE);
2159 }
2160
2161 /*
2162 * And now drop the SE_SHARED lock(s).
2163 */
2164 for (i = 0; i < npages; i++)
2165 page_unlock(ppa[i]);
2166 }
2167 AS_LOCK_EXIT(sptseg->s_as);
2168
2169 kmem_free(ppa, sizeof (page_t *) * npages);
2170 return (0);
2171 case F_SOFTUNLOCK:
2172
2173 /*
2174 * This is a bit ugly, we pass in the real seg pointer,
2175 * but the sptseg_addr is the virtual address within the
2176 * dummy seg.
2177 */
2178 segspt_softunlock(seg, sptseg_addr, ptob(npages), rw);
2179 return (0);
2180
2181 case F_PROT:
2182
2183 /*
2184 * This takes care of the unusual case where a user
2185 * allocates a stack in shared memory and a register
2186 * window overflow is written to that stack page before
2187 * it is otherwise modified.
2188 *
2189 * We can get away with this because ISM segments are
2190 * always rw. Other than this unusual case, there
2191 * should be no instances of protection violations.
2192 */
2193 return (0);
2194
2195 default:
2196 #ifdef DEBUG
2197 cmn_err(CE_WARN, "segspt_shmfault default type?");
2198 #endif
2199 return (FC_NOMAP);
2200 }
2201 }
2202
2203 /*ARGSUSED*/
2204 static faultcode_t
2205 segspt_shmfaulta(struct seg *seg, caddr_t addr)
2206 {
2207 return (0);
2208 }
2209
2210 /*ARGSUSED*/
2211 static int
2212 segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta)
2213 {
2214 return (0);
2215 }
2216
2217 /*
2218 * duplicate the shared page tables
2219 */
2220 int
2221 segspt_shmdup(struct seg *seg, struct seg *newseg)
2222 {
2223 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2224 struct anon_map *amp = shmd->shm_amp;
2225 struct shm_data *shmd_new;
2226 struct seg *spt_seg = shmd->shm_sptseg;
2227 struct spt_data *sptd = spt_seg->s_data;
2228 int error = 0;
2229
2230 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
2231
2232 shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP);
2233 newseg->s_data = (void *)shmd_new;
2234 shmd_new->shm_sptas = shmd->shm_sptas;
2235 shmd_new->shm_amp = amp;
2236 shmd_new->shm_sptseg = shmd->shm_sptseg;
2237 newseg->s_ops = &segspt_shmops;
2238 newseg->s_szc = seg->s_szc;
2239 ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc);
2240
2241 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
2242 amp->refcnt++;
2243 ANON_LOCK_EXIT(&amp->a_rwlock);
2244
2245 if (sptd->spt_flags & SHM_PAGEABLE) {
2246 shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP);
2247 shmd_new->shm_lckpgs = 0;
2248 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) {
2249 if ((error = hat_share(newseg->s_as->a_hat,
2250 newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR,
2251 seg->s_size, seg->s_szc)) != 0) {
2252 kmem_free(shmd_new->shm_vpage,
2253 btopr(amp->size));
2254 }
2255 }
2256 return (error);
2257 } else {
2258 return (hat_share(newseg->s_as->a_hat, newseg->s_base,
2259 shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size,
2260 seg->s_szc));
2261
2262 }
2263 }
2264
2265 /*ARGSUSED*/
2266 int
2267 segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
2268 {
2269 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2270 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2271
2272 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2273
2274 /*
2275 * ISM segment is always rw.
2276 */
2277 return (((sptd->spt_prot & prot) != prot) ? EACCES : 0);
2278 }
2279
2280 /*
2281 * Return an array of locked large pages, for empty slots allocate
2282 * private zero-filled anon pages.
2283 */
2284 static int
2285 spt_anon_getpages(
2286 struct seg *sptseg,
2287 caddr_t sptaddr,
2288 size_t len,
2289 page_t *ppa[])
2290 {
2291 struct spt_data *sptd = sptseg->s_data;
2292 struct anon_map *amp = sptd->spt_amp;
2293 enum seg_rw rw = sptd->spt_prot;
2294 uint_t szc = sptseg->s_szc;
2295 size_t pg_sz, share_sz = page_get_pagesize(szc);
2296 pgcnt_t lp_npgs;
2297 caddr_t lp_addr, e_sptaddr;
2298 uint_t vpprot, ppa_szc = 0;
2299 struct vpage *vpage = NULL;
2300 ulong_t j, ppa_idx;
2301 int err, ierr = 0;
2302 pgcnt_t an_idx;
2303 anon_sync_obj_t cookie;
2304 int anon_locked = 0;
2305 pgcnt_t amp_pgs;
2306
2307
2308 ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz));
2309 ASSERT(len != 0);
2310
2311 pg_sz = share_sz;
2312 lp_npgs = btop(pg_sz);
2313 lp_addr = sptaddr;
2314 e_sptaddr = sptaddr + len;
2315 an_idx = seg_page(sptseg, sptaddr);
2316 ppa_idx = 0;
2317
2318 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2319
2320 amp_pgs = page_get_pagecnt(amp->a_szc);
2321
2322 /*CONSTCOND*/
2323 while (1) {
2324 for (; lp_addr < e_sptaddr;
2325 an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) {
2326
2327 /*
2328 * If we're currently locked, and we get to a new
2329 * page, unlock our current anon chunk.
2330 */
2331 if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) {
2332 anon_array_exit(&cookie);
2333 anon_locked = 0;
2334 }
2335 if (!anon_locked) {
2336 anon_array_enter(amp, an_idx, &cookie);
2337 anon_locked = 1;
2338 }
2339 ppa_szc = (uint_t)-1;
2340 ierr = anon_map_getpages(amp, an_idx, szc, sptseg,
2341 lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx],
2342 &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred);
2343
2344 if (ierr != 0) {
2345 if (ierr > 0) {
2346 err = FC_MAKE_ERR(ierr);
2347 goto lpgs_err;
2348 }
2349 break;
2350 }
2351 }
2352 if (lp_addr == e_sptaddr) {
2353 break;
2354 }
2355 ASSERT(lp_addr < e_sptaddr);
2356
2357 /*
2358 * ierr == -1 means we failed to allocate a large page.
2359 * so do a size down operation.
2360 *
2361 * ierr == -2 means some other process that privately shares
2362 * pages with this process has allocated a larger page and we
2363 * need to retry with larger pages. So do a size up
2364 * operation. This relies on the fact that large pages are
2365 * never partially shared i.e. if we share any constituent
2366 * page of a large page with another process we must share the
2367 * entire large page. Note this cannot happen for SOFTLOCK
2368 * case, unless current address (lpaddr) is at the beginning
2369 * of the next page size boundary because the other process
2370 * couldn't have relocated locked pages.
2371 */
2372 ASSERT(ierr == -1 || ierr == -2);
2373 if (segvn_anypgsz) {
2374 ASSERT(ierr == -2 || szc != 0);
2375 ASSERT(ierr == -1 || szc < sptseg->s_szc);
2376 szc = (ierr == -1) ? szc - 1 : szc + 1;
2377 } else {
2378 /*
2379 * For faults and segvn_anypgsz == 0
2380 * we need to be careful not to loop forever
2381 * if existing page is found with szc other
2382 * than 0 or seg->s_szc. This could be due
2383 * to page relocations on behalf of DR or
2384 * more likely large page creation. For this
2385 * case simply re-size to existing page's szc
2386 * if returned by anon_map_getpages().
2387 */
2388 if (ppa_szc == (uint_t)-1) {
2389 szc = (ierr == -1) ? 0 : sptseg->s_szc;
2390 } else {
2391 ASSERT(ppa_szc <= sptseg->s_szc);
2392 ASSERT(ierr == -2 || ppa_szc < szc);
2393 ASSERT(ierr == -1 || ppa_szc > szc);
2394 szc = ppa_szc;
2395 }
2396 }
2397 pg_sz = page_get_pagesize(szc);
2398 lp_npgs = btop(pg_sz);
2399 ASSERT(IS_P2ALIGNED(lp_addr, pg_sz));
2400 }
2401 if (anon_locked) {
2402 anon_array_exit(&cookie);
2403 }
2404 ANON_LOCK_EXIT(&amp->a_rwlock);
2405 return (0);
2406
2407 lpgs_err:
2408 if (anon_locked) {
2409 anon_array_exit(&cookie);
2410 }
2411 ANON_LOCK_EXIT(&amp->a_rwlock);
2412 for (j = 0; j < ppa_idx; j++)
2413 page_unlock(ppa[j]);
2414 return (err);
2415 }
2416
2417 /*
2418 * count the number of bytes in a set of spt pages that are currently not
2419 * locked
2420 */
2421 static rctl_qty_t
2422 spt_unlockedbytes(pgcnt_t npages, page_t **ppa)
2423 {
2424 ulong_t i;
2425 rctl_qty_t unlocked = 0;
2426
2427 for (i = 0; i < npages; i++) {
2428 if (ppa[i]->p_lckcnt == 0)
2429 unlocked += PAGESIZE;
2430 }
2431 return (unlocked);
2432 }
2433
2434 extern u_longlong_t randtick(void);
2435 /* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */
2436 #define NLCK (NCPU_P2)
2437 /* Random number with a range [0, n-1], n must be power of two */
2438 #define RAND_P2(n) \
2439 ((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1))
2440
2441 int
2442 spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2443 page_t **ppa, ulong_t *lockmap, size_t pos,
2444 rctl_qty_t *locked)
2445 {
2446 struct shm_data *shmd = seg->s_data;
2447 struct spt_data *sptd = shmd->shm_sptseg->s_data;
2448 ulong_t i;
2449 int kernel;
2450 pgcnt_t nlck = 0;
2451 int rv = 0;
2452 int use_reserved = 1;
2453
2454 /* return the number of bytes actually locked */
2455 *locked = 0;
2456
2457 /*
2458 * To avoid contention on freemem_lock, availrmem and pages_locked
2459 * global counters are updated only every nlck locked pages instead of
2460 * every time. Reserve nlck locks up front and deduct from this
2461 * reservation for each page that requires a lock. When the reservation
2462 * is consumed, reserve again. nlck is randomized, so the competing
2463 * threads do not fall into a cyclic lock contention pattern. When
2464 * memory is low, the lock ahead is disabled, and instead page_pp_lock()
2465 * is used to lock pages.
2466 */
2467 for (i = 0; i < npages; anon_index++, pos++, i++) {
2468 if (nlck == 0 && use_reserved == 1) {
2469 nlck = NLCK + RAND_P2(NLCK);
2470 /* if fewer loops left, decrease nlck */
2471 nlck = MIN(nlck, npages - i);
2472 /*
2473 * Reserve nlck locks up front and deduct from this
2474 * reservation for each page that requires a lock. When
2475 * the reservation is consumed, reserve again.
2476 */
2477 mutex_enter(&freemem_lock);
2478 if ((availrmem - nlck) < pages_pp_maximum) {
2479 /* Do not do advance memory reserves */
2480 use_reserved = 0;
2481 } else {
2482 availrmem -= nlck;
2483 pages_locked += nlck;
2484 }
2485 mutex_exit(&freemem_lock);
2486 }
2487 if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) {
2488 if (sptd->spt_ppa_lckcnt[anon_index] <
2489 (ushort_t)DISM_LOCK_MAX) {
2490 if (++sptd->spt_ppa_lckcnt[anon_index] ==
2491 (ushort_t)DISM_LOCK_MAX) {
2492 cmn_err(CE_WARN,
2493 "DISM page lock limit "
2494 "reached on DISM offset 0x%lx\n",
2495 anon_index << PAGESHIFT);
2496 }
2497 kernel = (sptd->spt_ppa &&
2498 sptd->spt_ppa[anon_index]);
2499 if (!page_pp_lock(ppa[i], 0, kernel ||
2500 use_reserved)) {
2501 sptd->spt_ppa_lckcnt[anon_index]--;
2502 rv = EAGAIN;
2503 break;
2504 }
2505 /* if this is a newly locked page, count it */
2506 if (ppa[i]->p_lckcnt == 1) {
2507 if (kernel == 0 && use_reserved == 1)
2508 nlck--;
2509 *locked += PAGESIZE;
2510 }
2511 shmd->shm_lckpgs++;
2512 shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED;
2513 if (lockmap != NULL)
2514 BT_SET(lockmap, pos);
2515 }
2516 }
2517 }
2518 /* Return unused lock reservation */
2519 if (nlck != 0 && use_reserved == 1) {
2520 mutex_enter(&freemem_lock);
2521 availrmem += nlck;
2522 pages_locked -= nlck;
2523 mutex_exit(&freemem_lock);
2524 }
2525
2526 return (rv);
2527 }
2528
2529 int
2530 spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2531 rctl_qty_t *unlocked)
2532 {
2533 struct shm_data *shmd = seg->s_data;
2534 struct spt_data *sptd = shmd->shm_sptseg->s_data;
2535 struct anon_map *amp = sptd->spt_amp;
2536 struct anon *ap;
2537 struct vnode *vp;
2538 uoff_t off;
2539 struct page *pp;
2540 int kernel;
2541 anon_sync_obj_t cookie;
2542 ulong_t i;
2543 pgcnt_t nlck = 0;
2544 pgcnt_t nlck_limit = NLCK;
2545
2546 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2547 for (i = 0; i < npages; i++, anon_index++) {
2548 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
2549 anon_array_enter(amp, anon_index, &cookie);
2550 ap = anon_get_ptr(amp->ahp, anon_index);
2551 ASSERT(ap);
2552
2553 swap_xlate(ap, &vp, &off);
2554 anon_array_exit(&cookie);
2555 pp = page_lookup(&vp->v_object, off, SE_SHARED);
2556 ASSERT(pp);
2557 /*
2558 * availrmem is decremented only for pages which are not
2559 * in seg pcache, for pages in seg pcache availrmem was
2560 * decremented in _dismpagelock()
2561 */
2562 kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]);
2563 ASSERT(pp->p_lckcnt > 0);
2564
2565 /*
2566 * lock page but do not change availrmem, we do it
2567 * ourselves every nlck loops.
2568 */
2569 page_pp_unlock(pp, 0, 1);
2570 if (pp->p_lckcnt == 0) {
2571 if (kernel == 0)
2572 nlck++;
2573 *unlocked += PAGESIZE;
2574 }
2575 page_unlock(pp);
2576 shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED;
2577 sptd->spt_ppa_lckcnt[anon_index]--;
2578 shmd->shm_lckpgs--;
2579 }
2580
2581 /*
2582 * To reduce freemem_lock contention, do not update availrmem
2583 * until at least NLCK pages have been unlocked.
2584 * 1. No need to update if nlck is zero
2585 * 2. Always update if the last iteration
2586 */
2587 if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) {
2588 mutex_enter(&freemem_lock);
2589 availrmem += nlck;
2590 pages_locked -= nlck;
2591 mutex_exit(&freemem_lock);
2592 nlck = 0;
2593 nlck_limit = NLCK + RAND_P2(NLCK);
2594 }
2595 }
2596 ANON_LOCK_EXIT(&amp->a_rwlock);
2597
2598 return (0);
2599 }
2600
2601 /*ARGSUSED*/
2602 static int
2603 segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
2604 int attr, int op, ulong_t *lockmap, size_t pos)
2605 {
2606 struct shm_data *shmd = seg->s_data;
2607 struct seg *sptseg = shmd->shm_sptseg;
2608 struct spt_data *sptd = sptseg->s_data;
2609 struct kshmid *sp = sptd->spt_amp->a_sp;
2610 pgcnt_t npages, a_npages;
2611 page_t **ppa;
2612 pgcnt_t an_idx, a_an_idx, ppa_idx;
2613 caddr_t spt_addr, a_addr; /* spt and aligned address */
2614 size_t a_len; /* aligned len */
2615 size_t share_sz;
2616 ulong_t i;
2617 int sts = 0;
2618 rctl_qty_t unlocked = 0;
2619 rctl_qty_t locked = 0;
2620 struct proc *p = curproc;
2621 kproject_t *proj;
2622
2623 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2624 ASSERT(sp != NULL);
2625
2626 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
2627 return (0);
2628 }
2629
2630 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2631 an_idx = seg_page(seg, addr);
2632 npages = btopr(len);
2633
2634 if (an_idx + npages > btopr(shmd->shm_amp->size)) {
2635 return (ENOMEM);
2636 }
2637
2638 /*
2639 * A shm's project never changes, so no lock needed.
2640 * The shm has a hold on the project, so it will not go away.
2641 * Since we have a mapping to shm within this zone, we know
2642 * that the zone will not go away.
2643 */
2644 proj = sp->shm_perm.ipc_proj;
2645
2646 if (op == MC_LOCK) {
2647
2648 /*
2649 * Need to align addr and size request if they are not
2650 * aligned so we can always allocate large page(s) however
2651 * we only lock what was requested in initial request.
2652 */
2653 share_sz = page_get_pagesize(sptseg->s_szc);
2654 a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz);
2655 a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)),
2656 share_sz);
2657 a_npages = btop(a_len);
2658 a_an_idx = seg_page(seg, a_addr);
2659 spt_addr = sptseg->s_base + ptob(a_an_idx);
2660 ppa_idx = an_idx - a_an_idx;
2661
2662 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages),
2663 KM_NOSLEEP)) == NULL) {
2664 return (ENOMEM);
2665 }
2666
2667 /*
2668 * Don't cache any new pages for IO and
2669 * flush any cached pages.
2670 */
2671 mutex_enter(&sptd->spt_lock);
2672 if (sptd->spt_ppa != NULL)
2673 sptd->spt_flags |= DISM_PPA_CHANGED;
2674
2675 sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa);
2676 if (sts != 0) {
2677 mutex_exit(&sptd->spt_lock);
2678 kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
2679 return (sts);
2680 }
2681
2682 mutex_enter(&sp->shm_mlock);
2683 /* enforce locked memory rctl */
2684 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]);
2685
2686 mutex_enter(&p->p_lock);
2687 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) {
2688 mutex_exit(&p->p_lock);
2689 sts = EAGAIN;
2690 } else {
2691 mutex_exit(&p->p_lock);
2692 sts = spt_lockpages(seg, an_idx, npages,
2693 &ppa[ppa_idx], lockmap, pos, &locked);
2694
2695 /*
2696 * correct locked count if not all pages could be
2697 * locked
2698 */
2699 if ((unlocked - locked) > 0) {
2700 rctl_decr_locked_mem(NULL, proj,
2701 (unlocked - locked), 0);
2702 }
2703 }
2704 /*
2705 * unlock pages
2706 */
2707 for (i = 0; i < a_npages; i++)
2708 page_unlock(ppa[i]);
2709 if (sptd->spt_ppa != NULL)
2710 sptd->spt_flags |= DISM_PPA_CHANGED;
2711 mutex_exit(&sp->shm_mlock);
2712 mutex_exit(&sptd->spt_lock);
2713
2714 kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
2715
2716 } else if (op == MC_UNLOCK) { /* unlock */
2717 page_t **ppa;
2718
2719 mutex_enter(&sptd->spt_lock);
2720 if (shmd->shm_lckpgs == 0) {
2721 mutex_exit(&sptd->spt_lock);
2722 return (0);
2723 }
2724 /*
2725 * Don't cache new IO pages.
2726 */
2727 if (sptd->spt_ppa != NULL)
2728 sptd->spt_flags |= DISM_PPA_CHANGED;
2729
2730 mutex_enter(&sp->shm_mlock);
2731 sts = spt_unlockpages(seg, an_idx, npages, &unlocked);
2732 if ((ppa = sptd->spt_ppa) != NULL)
2733 sptd->spt_flags |= DISM_PPA_CHANGED;
2734 mutex_exit(&sptd->spt_lock);
2735
2736 rctl_decr_locked_mem(NULL, proj, unlocked, 0);
2737 mutex_exit(&sp->shm_mlock);
2738
2739 if (ppa != NULL)
2740 seg_ppurge_wiredpp(ppa);
2741 }
2742 return (sts);
2743 }
2744
2745 /*ARGSUSED*/
2746 int
2747 segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2748 {
2749 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2750 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2751 spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1;
2752
2753 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2754
2755 /*
2756 * ISM segment is always rw.
2757 */
2758 while (--pgno >= 0)
2759 *protv++ = sptd->spt_prot;
2760 return (0);
2761 }
2762
2763 /*ARGSUSED*/
2764 uoff_t
2765 segspt_shmgetoffset(struct seg *seg, caddr_t addr)
2766 {
2767 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2768
2769 /* Offset does not matter in ISM memory */
2770
2771 return (0);
2772 }
2773
2774 /* ARGSUSED */
2775 int
2776 segspt_shmgettype(struct seg *seg, caddr_t addr)
2777 {
2778 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2779 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2780
2781 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2782
2783 /*
2784 * The shared memory mapping is always MAP_SHARED, SWAP is only
2785 * reserved for DISM
2786 */
2787 return (MAP_SHARED |
2788 ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE));
2789 }
2790
2791 /*ARGSUSED*/
2792 int
2793 segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
2794 {
2795 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2796 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2797
2798 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2799
2800 *vpp = sptd->spt_vp;
2801 return (0);
2802 }
2803
2804 /*
2805 * We need to wait for pending IO to complete to a DISM segment in order for
2806 * pages to get kicked out of the seg_pcache. 120 seconds should be more
2807 * than enough time to wait.
2808 */
2809 static clock_t spt_pcache_wait = 120;
2810
2811 /*ARGSUSED*/
2812 static int
2813 segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2814 {
2815 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2816 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2817 struct anon_map *amp;
2818 pgcnt_t pg_idx;
2819 ushort_t gen;
2820 clock_t end_lbolt;
2821 int writer;
2822 page_t **ppa;
2823
2824 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2825
2826 if (behav == MADV_FREE || behav == MADV_PURGE) {
2827 if ((sptd->spt_flags & SHM_PAGEABLE) == 0)
2828 return (0);
2829
2830 amp = sptd->spt_amp;
2831 pg_idx = seg_page(seg, addr);
2832
2833 mutex_enter(&sptd->spt_lock);
2834 if ((ppa = sptd->spt_ppa) == NULL) {
2835 mutex_exit(&sptd->spt_lock);
2836 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2837 (void) anon_disclaim(amp, pg_idx, len, behav, NULL);
2838 ANON_LOCK_EXIT(&amp->a_rwlock);
2839 return (0);
2840 }
2841
2842 sptd->spt_flags |= DISM_PPA_CHANGED;
2843 gen = sptd->spt_gen;
2844
2845 mutex_exit(&sptd->spt_lock);
2846
2847 /*
2848 * Purge all DISM cached pages
2849 */
2850 seg_ppurge_wiredpp(ppa);
2851
2852 /*
2853 * Drop the AS_LOCK so that other threads can grab it
2854 * in the as_pageunlock path and hopefully get the segment
2855 * kicked out of the seg_pcache. We bump the shm_softlockcnt
2856 * to keep this segment resident.
2857 */
2858 writer = AS_WRITE_HELD(seg->s_as);
2859 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
2860 AS_LOCK_EXIT(seg->s_as);
2861
2862 mutex_enter(&sptd->spt_lock);
2863
2864 end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait);
2865
2866 /*
2867 * Try to wait for pages to get kicked out of the seg_pcache.
2868 */
2869 while (sptd->spt_gen == gen &&
2870 (sptd->spt_flags & DISM_PPA_CHANGED) &&
2871 ddi_get_lbolt() < end_lbolt) {
2872 if (!cv_timedwait_sig(&sptd->spt_cv,
2873 &sptd->spt_lock, end_lbolt)) {
2874 break;
2875 }
2876 }
2877
2878 mutex_exit(&sptd->spt_lock);
2879
2880 /* Regrab the AS_LOCK and release our hold on the segment */
2881 AS_LOCK_ENTER(seg->s_as, writer ? RW_WRITER : RW_READER);
2882 atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
2883 if (shmd->shm_softlockcnt <= 0) {
2884 if (AS_ISUNMAPWAIT(seg->s_as)) {
2885 mutex_enter(&seg->s_as->a_contents);
2886 if (AS_ISUNMAPWAIT(seg->s_as)) {
2887 AS_CLRUNMAPWAIT(seg->s_as);
2888 cv_broadcast(&seg->s_as->a_cv);
2889 }
2890 mutex_exit(&seg->s_as->a_contents);
2891 }
2892 }
2893
2894 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2895 (void) anon_disclaim(amp, pg_idx, len, behav, NULL);
2896 ANON_LOCK_EXIT(&amp->a_rwlock);
2897 } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP ||
2898 behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) {
2899 int already_set;
2900 ulong_t anon_index;
2901 lgrp_mem_policy_t policy;
2902 caddr_t shm_addr;
2903 size_t share_size;
2904 size_t size;
2905 struct seg *sptseg = shmd->shm_sptseg;
2906 caddr_t sptseg_addr;
2907
2908 /*
2909 * Align address and length to page size of underlying segment
2910 */
2911 share_size = page_get_pagesize(shmd->shm_sptseg->s_szc);
2912 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size);
2913 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)),
2914 share_size);
2915
2916 amp = shmd->shm_amp;
2917 anon_index = seg_page(seg, shm_addr);
2918
2919 /*
2920 * And now we may have to adjust size downward if we have
2921 * exceeded the realsize of the segment or initial anon
2922 * allocations.
2923 */
2924 sptseg_addr = sptseg->s_base + ptob(anon_index);
2925 if ((sptseg_addr + size) >
2926 (sptseg->s_base + sptd->spt_realsize))
2927 size = (sptseg->s_base + sptd->spt_realsize) -
2928 sptseg_addr;
2929
2930 /*
2931 * Set memory allocation policy for this segment
2932 */
2933 policy = lgrp_madv_to_policy(behav, len, MAP_SHARED);
2934 already_set = lgrp_shm_policy_set(policy, amp, anon_index,
2935 NULL, 0, len);
2936
2937 /*
2938 * If random memory allocation policy set already,
2939 * don't bother reapplying it.
2940 */
2941 if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy))
2942 return (0);
2943
2944 /*
2945 * Mark any existing pages in the given range for
2946 * migration, flushing the I/O page cache, and using
2947 * underlying segment to calculate anon index and get
2948 * anonmap and vnode pointer from
2949 */
2950 if (shmd->shm_softlockcnt > 0)
2951 segspt_purge(seg);
2952
2953 page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0);
2954 }
2955
2956 return (0);
2957 }
2958
2959 /*
2960 * get a memory ID for an addr in a given segment
2961 */
2962 static int
2963 segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
2964 {
2965 struct shm_data *shmd = (struct shm_data *)seg->s_data;
2966 struct anon *ap;
2967 size_t anon_index;
2968 struct anon_map *amp = shmd->shm_amp;
2969 struct spt_data *sptd = shmd->shm_sptseg->s_data;
2970 struct seg *sptseg = shmd->shm_sptseg;
2971 anon_sync_obj_t cookie;
2972
2973 anon_index = seg_page(seg, addr);
2974
2975 if (addr > (seg->s_base + sptd->spt_realsize)) {
2976 return (EFAULT);
2977 }
2978
2979 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2980 anon_array_enter(amp, anon_index, &cookie);
2981 ap = anon_get_ptr(amp->ahp, anon_index);
2982 if (ap == NULL) {
2983 struct page *pp;
2984 caddr_t spt_addr = sptseg->s_base + ptob(anon_index);
2985
2986 pp = anon_zero(sptseg, spt_addr, &ap, kcred);
2987 if (pp == NULL) {
2988 anon_array_exit(&cookie);
2989 ANON_LOCK_EXIT(&amp->a_rwlock);
2990 return (ENOMEM);
2991 }
2992 (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP);
2993 page_unlock(pp);
2994 }
2995 anon_array_exit(&cookie);
2996 ANON_LOCK_EXIT(&amp->a_rwlock);
2997 memidp->val[0] = (uintptr_t)ap;
2998 memidp->val[1] = (uintptr_t)addr & PAGEOFFSET;
2999 return (0);
3000 }
3001
3002 /*
3003 * Get memory allocation policy info for specified address in given segment
3004 */
3005 static lgrp_mem_policy_info_t *
3006 segspt_shmgetpolicy(struct seg *seg, caddr_t addr)
3007 {
3008 struct anon_map *amp;
3009 ulong_t anon_index;
3010 lgrp_mem_policy_info_t *policy_info;
3011 struct shm_data *shm_data;
3012
3013 ASSERT(seg != NULL);
3014
3015 /*
3016 * Get anon_map from segshm
3017 *
3018 * Assume that no lock needs to be held on anon_map, since
3019 * it should be protected by its reference count which must be
3020 * nonzero for an existing segment
3021 * Need to grab readers lock on policy tree though
3022 */
3023 shm_data = (struct shm_data *)seg->s_data;
3024 if (shm_data == NULL)
3025 return (NULL);
3026 amp = shm_data->shm_amp;
3027 ASSERT(amp->refcnt != 0);
3028
3029 /*
3030 * Get policy info
3031 *
3032 * Assume starting anon index of 0
3033 */
3034 anon_index = seg_page(seg, addr);
3035 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
3036
3037 return (policy_info);
3038 }
Unleashed OS