2 * Copyright (c) 1994 Adam Glass and Charles Hannum. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * 3. All advertising materials mentioning features or use of this software
13 * must display the following acknowledgement:
14 * This product includes software developed by Adam Glass and Charles
16 * 4. The names of the authors may not be used to endorse or promote products
17 * derived from this software without specific prior written permission.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #include "opt_sysvipc.h"
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/sysproto.h>
36 #include <sys/kernel.h>
37 #include <sys/sysctl.h>
40 #include <sys/malloc.h>
43 #include <sys/sysent.h>
47 #include <vm/vm_param.h>
50 #include <vm/vm_object.h>
51 #include <vm/vm_map.h>
52 #include <vm/vm_page.h>
53 #include <vm/vm_pager.h>
55 static MALLOC_DEFINE(M_SHM
, "shm", "SVID compatible shared memory segments");
57 static int shmget_allocate_segment (struct proc
*p
, struct shmget_args
*uap
, int mode
);
58 static int shmget_existing (struct proc
*p
, struct shmget_args
*uap
, int mode
, int segnum
);
60 #define SHMSEG_FREE 0x0200
61 #define SHMSEG_REMOVED 0x0400
62 #define SHMSEG_ALLOCATED 0x0800
63 #define SHMSEG_WANTED 0x1000
65 static int shm_last_free
, shm_committed
, shmalloced
;
67 static struct shmid_ds
*shmsegs
;
68 static struct lwkt_token shm_token
= LWKT_TOKEN_INITIALIZER(shm_token
);
71 /* vm_offset_t kva; */
72 vm_object_t shm_object
;
80 static void shm_deallocate_segment (struct shmid_ds
*);
81 static int shm_find_segment_by_key (key_t
);
82 static struct shmid_ds
*shm_find_segment_by_shmid (int);
83 static int shm_delete_mapping (struct vmspace
*vm
, struct shmmap_state
*);
84 static void shmrealloc (void);
85 static void shminit (void *);
100 struct shminfo shminfo
= {
108 static int shm_allow_removed
;
109 static int shm_use_phys
= 1;
111 TUNABLE_LONG("kern.ipc.shmmin", &shminfo
.shmmin
);
112 TUNABLE_LONG("kern.ipc.shmmni", &shminfo
.shmmni
);
113 TUNABLE_LONG("kern.ipc.shmseg", &shminfo
.shmseg
);
114 TUNABLE_LONG("kern.ipc.shmmaxpgs", &shminfo
.shmall
);
115 TUNABLE_INT("kern.ipc.shm_use_phys", &shm_use_phys
);
117 SYSCTL_LONG(_kern_ipc
, OID_AUTO
, shmmax
, CTLFLAG_RW
, &shminfo
.shmmax
, 0,
118 "Max shared memory segment size");
119 SYSCTL_LONG(_kern_ipc
, OID_AUTO
, shmmin
, CTLFLAG_RW
, &shminfo
.shmmin
, 0,
120 "Min shared memory segment size");
121 SYSCTL_LONG(_kern_ipc
, OID_AUTO
, shmmni
, CTLFLAG_RD
, &shminfo
.shmmni
, 0,
122 "Max number of shared memory identifiers");
123 SYSCTL_LONG(_kern_ipc
, OID_AUTO
, shmseg
, CTLFLAG_RW
, &shminfo
.shmseg
, 0,
124 "Max shared memory segments per process");
125 SYSCTL_LONG(_kern_ipc
, OID_AUTO
, shmall
, CTLFLAG_RW
, &shminfo
.shmall
, 0,
126 "Max pages of shared memory");
127 SYSCTL_INT(_kern_ipc
, OID_AUTO
, shm_use_phys
, CTLFLAG_RW
, &shm_use_phys
, 0,
128 "Use phys pager allocation instead of swap pager allocation");
129 SYSCTL_INT(_kern_ipc
, OID_AUTO
, shm_allow_removed
, CTLFLAG_RW
,
130 &shm_allow_removed
, 0,
131 "Enable/Disable attachment to attached segments marked for removal");
134 shm_find_segment_by_key(key_t key
)
138 for (i
= 0; i
< shmalloced
; i
++) {
139 if ((shmsegs
[i
].shm_perm
.mode
& SHMSEG_ALLOCATED
) &&
140 shmsegs
[i
].shm_perm
.key
== key
)
146 static struct shmid_ds
*
147 shm_find_segment_by_shmid(int shmid
)
150 struct shmid_ds
*shmseg
;
152 segnum
= IPCID_TO_IX(shmid
);
153 if (segnum
< 0 || segnum
>= shmalloced
)
155 shmseg
= &shmsegs
[segnum
];
156 if ((shmseg
->shm_perm
.mode
& SHMSEG_ALLOCATED
) == 0 ||
157 (!shm_allow_removed
&&
158 (shmseg
->shm_perm
.mode
& SHMSEG_REMOVED
) != 0) ||
159 shmseg
->shm_perm
.seq
!= IPCID_TO_SEQ(shmid
)) {
166 shm_deallocate_segment(struct shmid_ds
*shmseg
)
168 struct shm_handle
*shm_handle
;
171 shm_handle
= shmseg
->shm_internal
;
172 vm_object_deallocate(shm_handle
->shm_object
);
173 kfree((caddr_t
)shm_handle
, M_SHM
);
174 shmseg
->shm_internal
= NULL
;
175 size
= round_page(shmseg
->shm_segsz
);
176 shm_committed
-= btoc(size
);
178 shmseg
->shm_perm
.mode
= SHMSEG_FREE
;
182 shm_delete_mapping(struct vmspace
*vm
, struct shmmap_state
*shmmap_s
)
184 struct shmid_ds
*shmseg
;
188 segnum
= IPCID_TO_IX(shmmap_s
->shmid
);
189 shmseg
= &shmsegs
[segnum
];
190 size
= round_page(shmseg
->shm_segsz
);
191 result
= vm_map_remove(&vm
->vm_map
, shmmap_s
->va
, shmmap_s
->va
+ size
);
192 if (result
!= KERN_SUCCESS
)
194 shmmap_s
->shmid
= -1;
195 shmseg
->shm_dtime
= time_second
;
196 if ((--shmseg
->shm_nattch
<= 0) &&
197 (shmseg
->shm_perm
.mode
& SHMSEG_REMOVED
)) {
198 shm_deallocate_segment(shmseg
);
199 shm_last_free
= segnum
;
208 sys_shmdt(struct shmdt_args
*uap
)
210 struct thread
*td
= curthread
;
211 struct proc
*p
= td
->td_proc
;
212 struct shmmap_state
*shmmap_s
;
216 if (!jail_sysvipc_allowed
&& td
->td_ucred
->cr_prison
!= NULL
)
219 lwkt_gettoken(&shm_token
);
220 shmmap_s
= (struct shmmap_state
*)p
->p_vmspace
->vm_shm
;
221 if (shmmap_s
== NULL
) {
225 for (i
= 0; i
< shminfo
.shmseg
; i
++, shmmap_s
++) {
226 if (shmmap_s
->shmid
!= -1 &&
227 shmmap_s
->va
== (vm_offset_t
)uap
->shmaddr
)
230 if (i
== shminfo
.shmseg
)
233 error
= shm_delete_mapping(p
->p_vmspace
, shmmap_s
);
235 lwkt_reltoken(&shm_token
);
244 sys_shmat(struct shmat_args
*uap
)
246 struct thread
*td
= curthread
;
247 struct proc
*p
= td
->td_proc
;
250 struct shmid_ds
*shmseg
;
251 struct shmmap_state
*shmmap_s
= NULL
;
252 struct shm_handle
*shm_handle
;
253 vm_offset_t attach_va
;
259 if (!jail_sysvipc_allowed
&& td
->td_ucred
->cr_prison
!= NULL
)
262 lwkt_gettoken(&shm_token
);
264 shmmap_s
= (struct shmmap_state
*)p
->p_vmspace
->vm_shm
;
265 if (shmmap_s
== NULL
) {
266 size
= shminfo
.shmseg
* sizeof(struct shmmap_state
);
267 shmmap_s
= kmalloc(size
, M_SHM
, M_WAITOK
);
268 for (i
= 0; i
< shminfo
.shmseg
; i
++)
269 shmmap_s
[i
].shmid
= -1;
270 if (p
->p_vmspace
->vm_shm
!= NULL
) {
271 kfree(shmmap_s
, M_SHM
);
274 p
->p_vmspace
->vm_shm
= (caddr_t
)shmmap_s
;
276 shmseg
= shm_find_segment_by_shmid(uap
->shmid
);
277 if (shmseg
== NULL
) {
281 error
= ipcperm(p
, &shmseg
->shm_perm
,
282 (uap
->shmflg
& SHM_RDONLY
) ? IPC_R
: IPC_R
|IPC_W
);
285 for (i
= 0; i
< shminfo
.shmseg
; i
++) {
286 if (shmmap_s
->shmid
== -1)
290 if (i
>= shminfo
.shmseg
) {
294 size
= round_page(shmseg
->shm_segsz
);
295 #ifdef VM_PROT_READ_IS_EXEC
296 prot
= VM_PROT_READ
| VM_PROT_EXECUTE
;
300 if ((uap
->shmflg
& SHM_RDONLY
) == 0)
301 prot
|= VM_PROT_WRITE
;
302 flags
= MAP_ANON
| MAP_SHARED
;
305 if (uap
->shmflg
& SHM_RND
) {
306 attach_va
= (vm_offset_t
)uap
->shmaddr
& ~(SHMLBA
-1);
307 } else if (((vm_offset_t
)uap
->shmaddr
& (SHMLBA
-1)) == 0) {
308 attach_va
= (vm_offset_t
)uap
->shmaddr
;
315 * This is just a hint to vm_map_find() about where to put it.
317 attach_va
= round_page((vm_offset_t
)p
->p_vmspace
->vm_taddr
+
322 * Handle alignment. For large memory maps it is possible
323 * that the MMU can optimize the page table so align anything
324 * that is a multiple of SEG_SIZE to SEG_SIZE.
326 if ((flags
& MAP_FIXED
) == 0 && (size
& SEG_MASK
) == 0)
331 shm_handle
= shmseg
->shm_internal
;
332 vm_object_hold(shm_handle
->shm_object
);
333 vm_object_chain_wait(shm_handle
->shm_object
, 0);
334 vm_object_reference_locked(shm_handle
->shm_object
);
335 rv
= vm_map_find(&p
->p_vmspace
->vm_map
,
336 shm_handle
->shm_object
, NULL
,
339 ((flags
& MAP_FIXED
) ? 0 : 1),
340 VM_MAPTYPE_NORMAL
, VM_SUBSYS_SHMEM
,
342 vm_object_drop(shm_handle
->shm_object
);
343 if (rv
!= KERN_SUCCESS
) {
344 vm_object_deallocate(shm_handle
->shm_object
);
348 vm_map_inherit(&p
->p_vmspace
->vm_map
,
349 attach_va
, attach_va
+ size
, VM_INHERIT_SHARE
);
351 KKASSERT(shmmap_s
->shmid
== -1);
352 shmmap_s
->va
= attach_va
;
353 shmmap_s
->shmid
= uap
->shmid
;
354 shmseg
->shm_lpid
= p
->p_pid
;
355 shmseg
->shm_atime
= time_second
;
356 shmseg
->shm_nattch
++;
357 uap
->sysmsg_resultp
= (void *)attach_va
;
360 lwkt_reltoken(&shm_token
);
369 sys_shmctl(struct shmctl_args
*uap
)
371 struct thread
*td
= curthread
;
372 struct proc
*p
= td
->td_proc
;
374 struct shmid_ds inbuf
;
375 struct shmid_ds
*shmseg
;
377 if (!jail_sysvipc_allowed
&& td
->td_ucred
->cr_prison
!= NULL
)
380 lwkt_gettoken(&shm_token
);
381 shmseg
= shm_find_segment_by_shmid(uap
->shmid
);
382 if (shmseg
== NULL
) {
389 error
= ipcperm(p
, &shmseg
->shm_perm
, IPC_R
);
391 error
= copyout(shmseg
, uap
->buf
, sizeof(inbuf
));
394 error
= ipcperm(p
, &shmseg
->shm_perm
, IPC_M
);
396 error
= copyin(uap
->buf
, &inbuf
, sizeof(inbuf
));
398 shmseg
->shm_perm
.uid
= inbuf
.shm_perm
.uid
;
399 shmseg
->shm_perm
.gid
= inbuf
.shm_perm
.gid
;
400 shmseg
->shm_perm
.mode
=
401 (shmseg
->shm_perm
.mode
& ~ACCESSPERMS
) |
402 (inbuf
.shm_perm
.mode
& ACCESSPERMS
);
403 shmseg
->shm_ctime
= time_second
;
407 error
= ipcperm(p
, &shmseg
->shm_perm
, IPC_M
);
409 shmseg
->shm_perm
.key
= IPC_PRIVATE
;
410 shmseg
->shm_perm
.mode
|= SHMSEG_REMOVED
;
411 if (shmseg
->shm_nattch
<= 0) {
412 shm_deallocate_segment(shmseg
);
413 shm_last_free
= IPCID_TO_IX(uap
->shmid
);
426 lwkt_reltoken(&shm_token
);
432 shmget_existing(struct proc
*p
, struct shmget_args
*uap
, int mode
, int segnum
)
434 struct shmid_ds
*shmseg
;
437 shmseg
= &shmsegs
[segnum
];
438 if (shmseg
->shm_perm
.mode
& SHMSEG_REMOVED
) {
440 * This segment is in the process of being allocated. Wait
441 * until it's done, and look the key up again (in case the
442 * allocation failed or it was freed).
444 shmseg
->shm_perm
.mode
|= SHMSEG_WANTED
;
445 error
= tsleep((caddr_t
)shmseg
, PCATCH
, "shmget", 0);
450 if ((uap
->shmflg
& (IPC_CREAT
| IPC_EXCL
)) == (IPC_CREAT
| IPC_EXCL
))
452 error
= ipcperm(p
, &shmseg
->shm_perm
, mode
);
455 if (uap
->size
&& uap
->size
> shmseg
->shm_segsz
)
457 uap
->sysmsg_result
= IXSEQ_TO_IPCID(segnum
, shmseg
->shm_perm
);
462 shmget_allocate_segment(struct proc
*p
, struct shmget_args
*uap
, int mode
)
464 int i
, segnum
, shmid
;
466 struct ucred
*cred
= p
->p_ucred
;
467 struct shmid_ds
*shmseg
;
468 struct shm_handle
*shm_handle
;
470 if (uap
->size
< shminfo
.shmmin
|| uap
->size
> shminfo
.shmmax
)
472 if (shm_nused
>= shminfo
.shmmni
) /* any shmids left? */
474 size
= round_page(uap
->size
);
475 if (shm_committed
+ btoc(size
) > shminfo
.shmall
)
477 if (shm_last_free
< 0) {
478 shmrealloc(); /* maybe expand the shmsegs[] array */
479 for (i
= 0; i
< shmalloced
; i
++) {
480 if (shmsegs
[i
].shm_perm
.mode
& SHMSEG_FREE
)
487 segnum
= shm_last_free
;
490 shmseg
= &shmsegs
[segnum
];
492 * In case we sleep in malloc(), mark the segment present but deleted
493 * so that noone else tries to create the same key.
495 shmseg
->shm_perm
.mode
= SHMSEG_ALLOCATED
| SHMSEG_REMOVED
;
496 shmseg
->shm_perm
.key
= uap
->key
;
497 shmseg
->shm_perm
.seq
= (shmseg
->shm_perm
.seq
+ 1) & 0x7fff;
498 shm_handle
= kmalloc(sizeof(struct shm_handle
), M_SHM
, M_WAITOK
);
499 shmid
= IXSEQ_TO_IPCID(segnum
, shmseg
->shm_perm
);
502 * We make sure that we have allocated a pager before we need
506 shm_handle
->shm_object
=
507 phys_pager_alloc(NULL
, size
, VM_PROT_DEFAULT
, 0);
509 shm_handle
->shm_object
=
510 swap_pager_alloc(NULL
, size
, VM_PROT_DEFAULT
, 0);
512 vm_object_clear_flag(shm_handle
->shm_object
, OBJ_ONEMAPPING
);
513 vm_object_set_flag(shm_handle
->shm_object
, OBJ_NOSPLIT
);
515 shmseg
->shm_internal
= shm_handle
;
516 shmseg
->shm_perm
.cuid
= shmseg
->shm_perm
.uid
= cred
->cr_uid
;
517 shmseg
->shm_perm
.cgid
= shmseg
->shm_perm
.gid
= cred
->cr_gid
;
518 shmseg
->shm_perm
.mode
= (shmseg
->shm_perm
.mode
& SHMSEG_WANTED
) |
519 (mode
& ACCESSPERMS
) | SHMSEG_ALLOCATED
;
520 shmseg
->shm_segsz
= uap
->size
;
521 shmseg
->shm_cpid
= p
->p_pid
;
522 shmseg
->shm_lpid
= shmseg
->shm_nattch
= 0;
523 shmseg
->shm_atime
= shmseg
->shm_dtime
= 0;
524 shmseg
->shm_ctime
= time_second
;
525 shm_committed
+= btoc(size
);
529 * If a physical mapping is desired and we have a ton of free pages
530 * we pre-allocate the pages here in order to avoid on-the-fly
531 * allocation later. This has a big effect on database warm-up
532 * times since DFly supports concurrent page faults coming from the
533 * same VM object for pages which already exist.
535 * This can hang the kernel for a while so only do it if shm_use_phys
536 * is set to 2 or higher.
538 if (shm_use_phys
> 1) {
539 vm_pindex_t pi
, pmax
;
542 pmax
= round_page(shmseg
->shm_segsz
) >> PAGE_SHIFT
;
543 vm_object_hold(shm_handle
->shm_object
);
544 if (pmax
> vmstats
.v_free_count
)
545 pmax
= vmstats
.v_free_count
;
546 for (pi
= 0; pi
< pmax
; ++pi
) {
547 m
= vm_page_grab(shm_handle
->shm_object
, pi
,
548 VM_ALLOC_SYSTEM
| VM_ALLOC_NULL_OK
|
552 vm_pager_get_page(shm_handle
->shm_object
, &m
, 1);
557 vm_object_drop(shm_handle
->shm_object
);
560 if (shmseg
->shm_perm
.mode
& SHMSEG_WANTED
) {
562 * Somebody else wanted this key while we were asleep. Wake
565 shmseg
->shm_perm
.mode
&= ~SHMSEG_WANTED
;
566 wakeup((caddr_t
)shmseg
);
568 uap
->sysmsg_result
= shmid
;
576 sys_shmget(struct shmget_args
*uap
)
578 struct thread
*td
= curthread
;
579 struct proc
*p
= td
->td_proc
;
580 int segnum
, mode
, error
;
582 if (!jail_sysvipc_allowed
&& td
->td_ucred
->cr_prison
!= NULL
)
585 mode
= uap
->shmflg
& ACCESSPERMS
;
587 lwkt_gettoken(&shm_token
);
589 if (uap
->key
!= IPC_PRIVATE
) {
591 segnum
= shm_find_segment_by_key(uap
->key
);
593 error
= shmget_existing(p
, uap
, mode
, segnum
);
598 if ((uap
->shmflg
& IPC_CREAT
) == 0) {
603 error
= shmget_allocate_segment(p
, uap
, mode
);
605 lwkt_reltoken(&shm_token
);
611 shmfork(struct proc
*p1
, struct proc
*p2
)
613 struct shmmap_state
*shmmap_s
;
617 lwkt_gettoken(&shm_token
);
618 size
= shminfo
.shmseg
* sizeof(struct shmmap_state
);
619 shmmap_s
= kmalloc(size
, M_SHM
, M_WAITOK
);
620 bcopy((caddr_t
)p1
->p_vmspace
->vm_shm
, (caddr_t
)shmmap_s
, size
);
621 p2
->p_vmspace
->vm_shm
= (caddr_t
)shmmap_s
;
622 for (i
= 0; i
< shminfo
.shmseg
; i
++, shmmap_s
++) {
623 if (shmmap_s
->shmid
!= -1)
624 shmsegs
[IPCID_TO_IX(shmmap_s
->shmid
)].shm_nattch
++;
626 lwkt_reltoken(&shm_token
);
630 shmexit(struct vmspace
*vm
)
632 struct shmmap_state
*base
, *shm
;
635 if ((base
= (struct shmmap_state
*)vm
->vm_shm
) != NULL
) {
637 lwkt_gettoken(&shm_token
);
638 for (i
= 0, shm
= base
; i
< shminfo
.shmseg
; i
++, shm
++) {
639 if (shm
->shmid
!= -1)
640 shm_delete_mapping(vm
, shm
);
643 lwkt_reltoken(&shm_token
);
651 struct shmid_ds
*newsegs
;
653 if (shmalloced
>= shminfo
.shmmni
)
656 newsegs
= kmalloc(shminfo
.shmmni
* sizeof(*newsegs
), M_SHM
, M_WAITOK
);
657 for (i
= 0; i
< shmalloced
; i
++)
658 bcopy(&shmsegs
[i
], &newsegs
[i
], sizeof(newsegs
[0]));
659 for (; i
< shminfo
.shmmni
; i
++) {
660 shmsegs
[i
].shm_perm
.mode
= SHMSEG_FREE
;
661 shmsegs
[i
].shm_perm
.seq
= 0;
663 kfree(shmsegs
, M_SHM
);
665 shmalloced
= shminfo
.shmmni
;
674 * If not overridden by a tunable set the maximum shm to
675 * 2/3 of main memory.
677 if (shminfo
.shmall
== 0)
678 shminfo
.shmall
= (size_t)vmstats
.v_page_count
* 2 / 3;
680 shminfo
.shmmax
= shminfo
.shmall
* PAGE_SIZE
;
681 shmalloced
= shminfo
.shmmni
;
682 shmsegs
= kmalloc(shmalloced
* sizeof(shmsegs
[0]), M_SHM
, M_WAITOK
);
683 for (i
= 0; i
< shmalloced
; i
++) {
684 shmsegs
[i
].shm_perm
.mode
= SHMSEG_FREE
;
685 shmsegs
[i
].shm_perm
.seq
= 0;
691 SYSINIT(sysv_shm
, SI_SUB_SYSV_SHM
, SI_ORDER_FIRST
, shminit
, NULL
);