3 * Copyright (C) 1992, 1993 Krishna Balasubramanian
4 * Many improvements/fixes by Bruno Haible.
5 * Replaced `struct shm_desc' by `struct vm_area_struct', July 1994.
6 * Fixed the shm swap deallocation (shm_unuse()), August 1998 Andrea Arcangeli.
8 * /proc/sysvipc/shm support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
9 * BIGMEM support, Andrea Arcangeli <andrea@suse.de>
10 * SMP thread shm, Jean-Luc Boyard <jean-luc.boyard@siemens.fr>
11 * HIGHMEM support, Ingo Molnar <mingo@redhat.com>
12 * Make shmmax, shmall, shmmni sysctl'able, Christoph Rohland <cr@sap.com>
13 * Shared /dev/zero support, Kanoj Sarcar <kanoj@sgi.com>
14 * Move the mm functionality over to mm/shmem.c, Christoph Rohland <cr@sap.com>
16 * support for audit of ipc object properties and permission changes
17 * Dustin Kirkland <dustin.kirkland@us.ibm.com>
20 #include <linux/slab.h>
22 #include <linux/hugetlb.h>
23 #include <linux/shm.h>
24 #include <linux/init.h>
25 #include <linux/file.h>
26 #include <linux/mman.h>
27 #include <linux/shmem_fs.h>
28 #include <linux/security.h>
29 #include <linux/syscalls.h>
30 #include <linux/audit.h>
31 #include <linux/capability.h>
32 #include <linux/ptrace.h>
33 #include <linux/seq_file.h>
34 #include <linux/mutex.h>
36 #include <asm/uaccess.h>
40 static struct file_operations shm_file_operations
;
41 static struct vm_operations_struct shm_vm_ops
;
43 static struct ipc_ids shm_ids
;
45 #define shm_lock(id) ((struct shmid_kernel*)ipc_lock(&shm_ids,id))
46 #define shm_unlock(shp) ipc_unlock(&(shp)->shm_perm)
47 #define shm_get(id) ((struct shmid_kernel*)ipc_get(&shm_ids,id))
48 #define shm_buildid(id, seq) \
49 ipc_buildid(&shm_ids, id, seq)
51 static int newseg (key_t key
, int shmflg
, size_t size
);
52 static void shm_open (struct vm_area_struct
*shmd
);
53 static void shm_close (struct vm_area_struct
*shmd
);
55 static int sysvipc_shm_proc_show(struct seq_file
*s
, void *it
);
58 size_t shm_ctlmax
= SHMMAX
;
59 size_t shm_ctlall
= SHMALL
;
60 int shm_ctlmni
= SHMMNI
;
62 static int shm_tot
; /* total number of shared memory pages */
64 void __init
shm_init (void)
66 ipc_init_ids(&shm_ids
, 1);
67 ipc_init_proc_interface("sysvipc/shm",
68 " key shmid perms size cpid lpid nattch uid gid cuid cgid atime dtime ctime\n",
70 sysvipc_shm_proc_show
);
73 static inline int shm_checkid(struct shmid_kernel
*s
, int id
)
75 if (ipc_checkid(&shm_ids
,&s
->shm_perm
,id
))
80 static inline struct shmid_kernel
*shm_rmid(int id
)
82 return (struct shmid_kernel
*)ipc_rmid(&shm_ids
,id
);
85 static inline int shm_addid(struct shmid_kernel
*shp
)
87 return ipc_addid(&shm_ids
, &shp
->shm_perm
, shm_ctlmni
);
92 static inline void shm_inc (int id
) {
93 struct shmid_kernel
*shp
;
97 shp
->shm_atim
= get_seconds();
98 shp
->shm_lprid
= current
->tgid
;
103 /* This is called by fork, once for every shm attach. */
104 static void shm_open (struct vm_area_struct
*shmd
)
106 shm_inc (shmd
->vm_file
->f_dentry
->d_inode
->i_ino
);
110 * shm_destroy - free the struct shmid_kernel
112 * @shp: struct to free
114 * It has to be called with shp and shm_ids.mutex locked,
115 * but returns with shp unlocked and freed.
117 static void shm_destroy (struct shmid_kernel
*shp
)
119 shm_tot
-= (shp
->shm_segsz
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
122 if (!is_file_hugepages(shp
->shm_file
))
123 shmem_lock(shp
->shm_file
, 0, shp
->mlock_user
);
125 user_shm_unlock(shp
->shm_file
->f_dentry
->d_inode
->i_size
,
127 fput (shp
->shm_file
);
128 security_shm_free(shp
);
133 * remove the attach descriptor shmd.
134 * free memory for segment if it is marked destroyed.
135 * The descriptor has already been removed from the current->mm->mmap list
136 * and will later be kfree()d.
138 static void shm_close (struct vm_area_struct
*shmd
)
140 struct file
* file
= shmd
->vm_file
;
141 int id
= file
->f_dentry
->d_inode
->i_ino
;
142 struct shmid_kernel
*shp
;
144 mutex_lock(&shm_ids
.mutex
);
145 /* remove from the list of attaches of the shm segment */
148 shp
->shm_lprid
= current
->tgid
;
149 shp
->shm_dtim
= get_seconds();
151 if(shp
->shm_nattch
== 0 &&
152 shp
->shm_perm
.mode
& SHM_DEST
)
156 mutex_unlock(&shm_ids
.mutex
);
159 static int shm_mmap(struct file
* file
, struct vm_area_struct
* vma
)
163 ret
= shmem_mmap(file
, vma
);
165 vma
->vm_ops
= &shm_vm_ops
;
166 if (!(vma
->vm_flags
& VM_WRITE
))
167 vma
->vm_flags
&= ~VM_MAYWRITE
;
168 shm_inc(file
->f_dentry
->d_inode
->i_ino
);
174 static struct file_operations shm_file_operations
= {
177 .get_unmapped_area
= shmem_get_unmapped_area
,
181 static struct vm_operations_struct shm_vm_ops
= {
182 .open
= shm_open
, /* callback for a new vm-area open */
183 .close
= shm_close
, /* callback for when the vm-area is released */
184 .nopage
= shmem_nopage
,
185 #if defined(CONFIG_NUMA) && defined(CONFIG_SHMEM)
186 .set_policy
= shmem_set_policy
,
187 .get_policy
= shmem_get_policy
,
191 static int newseg (key_t key
, int shmflg
, size_t size
)
194 struct shmid_kernel
*shp
;
195 int numpages
= (size
+ PAGE_SIZE
-1) >> PAGE_SHIFT
;
200 if (size
< SHMMIN
|| size
> shm_ctlmax
)
203 if (shm_tot
+ numpages
>= shm_ctlall
)
206 shp
= ipc_rcu_alloc(sizeof(*shp
));
210 shp
->shm_perm
.key
= key
;
211 shp
->shm_perm
.mode
= (shmflg
& S_IRWXUGO
);
212 shp
->mlock_user
= NULL
;
214 shp
->shm_perm
.security
= NULL
;
215 error
= security_shm_alloc(shp
);
221 if (shmflg
& SHM_HUGETLB
) {
222 /* hugetlb_zero_setup takes care of mlock user accounting */
223 file
= hugetlb_zero_setup(size
);
224 shp
->mlock_user
= current
->user
;
226 int acctflag
= VM_ACCOUNT
;
228 * Do not allow no accounting for OVERCOMMIT_NEVER, even
231 if ((shmflg
& SHM_NORESERVE
) &&
232 sysctl_overcommit_memory
!= OVERCOMMIT_NEVER
)
234 sprintf (name
, "SYSV%08x", key
);
235 file
= shmem_file_setup(name
, size
, acctflag
);
237 error
= PTR_ERR(file
);
246 shp
->shm_cprid
= current
->tgid
;
248 shp
->shm_atim
= shp
->shm_dtim
= 0;
249 shp
->shm_ctim
= get_seconds();
250 shp
->shm_segsz
= size
;
252 shp
->id
= shm_buildid(id
,shp
->shm_perm
.seq
);
253 shp
->shm_file
= file
;
254 file
->f_dentry
->d_inode
->i_ino
= shp
->id
;
256 /* Hugetlb ops would have already been assigned. */
257 if (!(shmflg
& SHM_HUGETLB
))
258 file
->f_op
= &shm_file_operations
;
267 security_shm_free(shp
);
272 asmlinkage
long sys_shmget (key_t key
, size_t size
, int shmflg
)
274 struct shmid_kernel
*shp
;
277 mutex_lock(&shm_ids
.mutex
);
278 if (key
== IPC_PRIVATE
) {
279 err
= newseg(key
, shmflg
, size
);
280 } else if ((id
= ipc_findkey(&shm_ids
, key
)) == -1) {
281 if (!(shmflg
& IPC_CREAT
))
284 err
= newseg(key
, shmflg
, size
);
285 } else if ((shmflg
& IPC_CREAT
) && (shmflg
& IPC_EXCL
)) {
290 if (shp
->shm_segsz
< size
)
292 else if (ipcperms(&shp
->shm_perm
, shmflg
))
295 int shmid
= shm_buildid(id
, shp
->shm_perm
.seq
);
296 err
= security_shm_associate(shp
, shmflg
);
302 mutex_unlock(&shm_ids
.mutex
);
307 static inline unsigned long copy_shmid_to_user(void __user
*buf
, struct shmid64_ds
*in
, int version
)
311 return copy_to_user(buf
, in
, sizeof(*in
));
316 ipc64_perm_to_ipc_perm(&in
->shm_perm
, &out
.shm_perm
);
317 out
.shm_segsz
= in
->shm_segsz
;
318 out
.shm_atime
= in
->shm_atime
;
319 out
.shm_dtime
= in
->shm_dtime
;
320 out
.shm_ctime
= in
->shm_ctime
;
321 out
.shm_cpid
= in
->shm_cpid
;
322 out
.shm_lpid
= in
->shm_lpid
;
323 out
.shm_nattch
= in
->shm_nattch
;
325 return copy_to_user(buf
, &out
, sizeof(out
));
338 static inline unsigned long copy_shmid_from_user(struct shm_setbuf
*out
, void __user
*buf
, int version
)
343 struct shmid64_ds tbuf
;
345 if (copy_from_user(&tbuf
, buf
, sizeof(tbuf
)))
348 out
->uid
= tbuf
.shm_perm
.uid
;
349 out
->gid
= tbuf
.shm_perm
.gid
;
350 out
->mode
= tbuf
.shm_perm
.mode
;
356 struct shmid_ds tbuf_old
;
358 if (copy_from_user(&tbuf_old
, buf
, sizeof(tbuf_old
)))
361 out
->uid
= tbuf_old
.shm_perm
.uid
;
362 out
->gid
= tbuf_old
.shm_perm
.gid
;
363 out
->mode
= tbuf_old
.shm_perm
.mode
;
372 static inline unsigned long copy_shminfo_to_user(void __user
*buf
, struct shminfo64
*in
, int version
)
376 return copy_to_user(buf
, in
, sizeof(*in
));
381 if(in
->shmmax
> INT_MAX
)
382 out
.shmmax
= INT_MAX
;
384 out
.shmmax
= (int)in
->shmmax
;
386 out
.shmmin
= in
->shmmin
;
387 out
.shmmni
= in
->shmmni
;
388 out
.shmseg
= in
->shmseg
;
389 out
.shmall
= in
->shmall
;
391 return copy_to_user(buf
, &out
, sizeof(out
));
398 static void shm_get_stat(unsigned long *rss
, unsigned long *swp
)
405 for (i
= 0; i
<= shm_ids
.max_id
; i
++) {
406 struct shmid_kernel
*shp
;
413 inode
= shp
->shm_file
->f_dentry
->d_inode
;
415 if (is_file_hugepages(shp
->shm_file
)) {
416 struct address_space
*mapping
= inode
->i_mapping
;
417 *rss
+= (HPAGE_SIZE
/PAGE_SIZE
)*mapping
->nrpages
;
419 struct shmem_inode_info
*info
= SHMEM_I(inode
);
420 spin_lock(&info
->lock
);
421 *rss
+= inode
->i_mapping
->nrpages
;
422 *swp
+= info
->swapped
;
423 spin_unlock(&info
->lock
);
428 asmlinkage
long sys_shmctl (int shmid
, int cmd
, struct shmid_ds __user
*buf
)
430 struct shm_setbuf setbuf
;
431 struct shmid_kernel
*shp
;
434 if (cmd
< 0 || shmid
< 0) {
439 version
= ipc_parse_version(&cmd
);
441 switch (cmd
) { /* replace with proc interface ? */
444 struct shminfo64 shminfo
;
446 err
= security_shm_shmctl(NULL
, cmd
);
450 memset(&shminfo
,0,sizeof(shminfo
));
451 shminfo
.shmmni
= shminfo
.shmseg
= shm_ctlmni
;
452 shminfo
.shmmax
= shm_ctlmax
;
453 shminfo
.shmall
= shm_ctlall
;
455 shminfo
.shmmin
= SHMMIN
;
456 if(copy_shminfo_to_user (buf
, &shminfo
, version
))
458 /* reading a integer is always atomic */
466 struct shm_info shm_info
;
468 err
= security_shm_shmctl(NULL
, cmd
);
472 memset(&shm_info
,0,sizeof(shm_info
));
473 mutex_lock(&shm_ids
.mutex
);
474 shm_info
.used_ids
= shm_ids
.in_use
;
475 shm_get_stat (&shm_info
.shm_rss
, &shm_info
.shm_swp
);
476 shm_info
.shm_tot
= shm_tot
;
477 shm_info
.swap_attempts
= 0;
478 shm_info
.swap_successes
= 0;
479 err
= shm_ids
.max_id
;
480 mutex_unlock(&shm_ids
.mutex
);
481 if(copy_to_user (buf
, &shm_info
, sizeof(shm_info
))) {
486 err
= err
< 0 ? 0 : err
;
492 struct shmid64_ds tbuf
;
494 memset(&tbuf
, 0, sizeof(tbuf
));
495 shp
= shm_lock(shmid
);
499 } else if(cmd
==SHM_STAT
) {
501 if (shmid
> shm_ids
.max_id
)
503 result
= shm_buildid(shmid
, shp
->shm_perm
.seq
);
505 err
= shm_checkid(shp
,shmid
);
511 if (ipcperms (&shp
->shm_perm
, S_IRUGO
))
513 err
= security_shm_shmctl(shp
, cmd
);
516 kernel_to_ipc64_perm(&shp
->shm_perm
, &tbuf
.shm_perm
);
517 tbuf
.shm_segsz
= shp
->shm_segsz
;
518 tbuf
.shm_atime
= shp
->shm_atim
;
519 tbuf
.shm_dtime
= shp
->shm_dtim
;
520 tbuf
.shm_ctime
= shp
->shm_ctim
;
521 tbuf
.shm_cpid
= shp
->shm_cprid
;
522 tbuf
.shm_lpid
= shp
->shm_lprid
;
523 if (!is_file_hugepages(shp
->shm_file
))
524 tbuf
.shm_nattch
= shp
->shm_nattch
;
526 tbuf
.shm_nattch
= file_count(shp
->shm_file
) - 1;
528 if(copy_shmid_to_user (buf
, &tbuf
, version
))
537 shp
= shm_lock(shmid
);
542 err
= shm_checkid(shp
,shmid
);
546 err
= audit_ipc_obj(&(shp
->shm_perm
));
550 if (!capable(CAP_IPC_LOCK
)) {
552 if (current
->euid
!= shp
->shm_perm
.uid
&&
553 current
->euid
!= shp
->shm_perm
.cuid
)
555 if (cmd
== SHM_LOCK
&&
556 !current
->signal
->rlim
[RLIMIT_MEMLOCK
].rlim_cur
)
560 err
= security_shm_shmctl(shp
, cmd
);
565 struct user_struct
* user
= current
->user
;
566 if (!is_file_hugepages(shp
->shm_file
)) {
567 err
= shmem_lock(shp
->shm_file
, 1, user
);
569 shp
->shm_perm
.mode
|= SHM_LOCKED
;
570 shp
->mlock_user
= user
;
573 } else if (!is_file_hugepages(shp
->shm_file
)) {
574 shmem_lock(shp
->shm_file
, 0, shp
->mlock_user
);
575 shp
->shm_perm
.mode
&= ~SHM_LOCKED
;
576 shp
->mlock_user
= NULL
;
584 * We cannot simply remove the file. The SVID states
585 * that the block remains until the last person
586 * detaches from it, then is deleted. A shmat() on
587 * an RMID segment is legal in older Linux and if
588 * we change it apps break...
590 * Instead we set a destroyed flag, and then blow
591 * the name away when the usage hits zero.
593 mutex_lock(&shm_ids
.mutex
);
594 shp
= shm_lock(shmid
);
598 err
= shm_checkid(shp
, shmid
);
602 err
= audit_ipc_obj(&(shp
->shm_perm
));
606 if (current
->euid
!= shp
->shm_perm
.uid
&&
607 current
->euid
!= shp
->shm_perm
.cuid
&&
608 !capable(CAP_SYS_ADMIN
)) {
613 err
= security_shm_shmctl(shp
, cmd
);
617 if (shp
->shm_nattch
){
618 shp
->shm_perm
.mode
|= SHM_DEST
;
619 /* Do not find it any more */
620 shp
->shm_perm
.key
= IPC_PRIVATE
;
624 mutex_unlock(&shm_ids
.mutex
);
630 if (copy_shmid_from_user (&setbuf
, buf
, version
)) {
634 mutex_lock(&shm_ids
.mutex
);
635 shp
= shm_lock(shmid
);
639 err
= shm_checkid(shp
,shmid
);
642 err
= audit_ipc_obj(&(shp
->shm_perm
));
645 err
= audit_ipc_set_perm(0, setbuf
.uid
, setbuf
.gid
, setbuf
.mode
);
649 if (current
->euid
!= shp
->shm_perm
.uid
&&
650 current
->euid
!= shp
->shm_perm
.cuid
&&
651 !capable(CAP_SYS_ADMIN
)) {
655 err
= security_shm_shmctl(shp
, cmd
);
659 shp
->shm_perm
.uid
= setbuf
.uid
;
660 shp
->shm_perm
.gid
= setbuf
.gid
;
661 shp
->shm_perm
.mode
= (shp
->shm_perm
.mode
& ~S_IRWXUGO
)
662 | (setbuf
.mode
& S_IRWXUGO
);
663 shp
->shm_ctim
= get_seconds();
676 mutex_unlock(&shm_ids
.mutex
);
685 * Fix shmaddr, allocate descriptor, map shm, add attach descriptor to lists.
687 * NOTE! Despite the name, this is NOT a direct system call entrypoint. The
688 * "raddr" thing points to kernel space, and there has to be a wrapper around
691 long do_shmat(int shmid
, char __user
*shmaddr
, int shmflg
, ulong
*raddr
)
693 struct shmid_kernel
*shp
;
706 } else if ((addr
= (ulong
)shmaddr
)) {
707 if (addr
& (SHMLBA
-1)) {
708 if (shmflg
& SHM_RND
)
709 addr
&= ~(SHMLBA
-1); /* round down */
711 #ifndef __ARCH_FORCE_SHMLBA
712 if (addr
& ~PAGE_MASK
)
716 flags
= MAP_SHARED
| MAP_FIXED
;
718 if ((shmflg
& SHM_REMAP
))
724 if (shmflg
& SHM_RDONLY
) {
728 prot
= PROT_READ
| PROT_WRITE
;
729 acc_mode
= S_IRUGO
| S_IWUGO
;
731 if (shmflg
& SHM_EXEC
) {
737 * We cannot rely on the fs check since SYSV IPC does have an
738 * additional creator id...
740 shp
= shm_lock(shmid
);
745 err
= shm_checkid(shp
,shmid
);
750 if (ipcperms(&shp
->shm_perm
, acc_mode
)) {
756 err
= security_shm_shmat(shp
, shmaddr
, shmflg
);
762 file
= shp
->shm_file
;
763 size
= i_size_read(file
->f_dentry
->d_inode
);
767 down_write(¤t
->mm
->mmap_sem
);
768 if (addr
&& !(shmflg
& SHM_REMAP
)) {
769 user_addr
= ERR_PTR(-EINVAL
);
770 if (find_vma_intersection(current
->mm
, addr
, addr
+ size
))
773 * If shm segment goes below stack, make sure there is some
774 * space left for the stack to grow (at least 4 pages).
776 if (addr
< current
->mm
->start_stack
&&
777 addr
> current
->mm
->start_stack
- size
- PAGE_SIZE
* 5)
781 user_addr
= (void*) do_mmap (file
, addr
, size
, prot
, flags
, 0);
784 up_write(¤t
->mm
->mmap_sem
);
786 mutex_lock(&shm_ids
.mutex
);
787 shp
= shm_lock(shmid
);
790 if(shp
->shm_nattch
== 0 &&
791 shp
->shm_perm
.mode
& SHM_DEST
)
795 mutex_unlock(&shm_ids
.mutex
);
797 *raddr
= (unsigned long) user_addr
;
799 if (IS_ERR(user_addr
))
800 err
= PTR_ERR(user_addr
);
805 asmlinkage
long sys_shmat(int shmid
, char __user
*shmaddr
, int shmflg
)
810 err
= do_shmat(shmid
, shmaddr
, shmflg
, &ret
);
813 force_successful_syscall_return();
818 * detach and kill segment if marked destroyed.
819 * The work is done in shm_close.
821 asmlinkage
long sys_shmdt(char __user
*shmaddr
)
823 struct mm_struct
*mm
= current
->mm
;
824 struct vm_area_struct
*vma
, *next
;
825 unsigned long addr
= (unsigned long)shmaddr
;
827 int retval
= -EINVAL
;
829 if (addr
& ~PAGE_MASK
)
832 down_write(&mm
->mmap_sem
);
835 * This function tries to be smart and unmap shm segments that
836 * were modified by partial mlock or munmap calls:
837 * - It first determines the size of the shm segment that should be
838 * unmapped: It searches for a vma that is backed by shm and that
839 * started at address shmaddr. It records it's size and then unmaps
841 * - Then it unmaps all shm vmas that started at shmaddr and that
842 * are within the initially determined size.
843 * Errors from do_munmap are ignored: the function only fails if
844 * it's called with invalid parameters or if it's called to unmap
845 * a part of a vma. Both calls in this function are for full vmas,
846 * the parameters are directly copied from the vma itself and always
847 * valid - therefore do_munmap cannot fail. (famous last words?)
850 * If it had been mremap()'d, the starting address would not
851 * match the usual checks anyway. So assume all vma's are
852 * above the starting address given.
854 vma
= find_vma(mm
, addr
);
860 * Check if the starting address would match, i.e. it's
861 * a fragment created by mprotect() and/or munmap(), or it
862 * otherwise it starts at this address with no hassles.
864 if ((vma
->vm_ops
== &shm_vm_ops
|| is_vm_hugetlb_page(vma
)) &&
865 (vma
->vm_start
- addr
)/PAGE_SIZE
== vma
->vm_pgoff
) {
868 size
= vma
->vm_file
->f_dentry
->d_inode
->i_size
;
869 do_munmap(mm
, vma
->vm_start
, vma
->vm_end
- vma
->vm_start
);
871 * We discovered the size of the shm segment, so
872 * break out of here and fall through to the next
873 * loop that uses the size information to stop
874 * searching for matching vma's.
884 * We need look no further than the maximum address a fragment
885 * could possibly have landed at. Also cast things to loff_t to
886 * prevent overflows and make comparisions vs. equal-width types.
888 size
= PAGE_ALIGN(size
);
889 while (vma
&& (loff_t
)(vma
->vm_end
- addr
) <= size
) {
892 /* finding a matching vma now does not alter retval */
893 if ((vma
->vm_ops
== &shm_vm_ops
|| is_vm_hugetlb_page(vma
)) &&
894 (vma
->vm_start
- addr
)/PAGE_SIZE
== vma
->vm_pgoff
)
896 do_munmap(mm
, vma
->vm_start
, vma
->vm_end
- vma
->vm_start
);
900 up_write(&mm
->mmap_sem
);
904 #ifdef CONFIG_PROC_FS
905 static int sysvipc_shm_proc_show(struct seq_file
*s
, void *it
)
907 struct shmid_kernel
*shp
= it
;
910 #define SMALL_STRING "%10d %10d %4o %10u %5u %5u %5d %5u %5u %5u %5u %10lu %10lu %10lu\n"
911 #define BIG_STRING "%10d %10d %4o %21u %5u %5u %5d %5u %5u %5u %5u %10lu %10lu %10lu\n"
913 if (sizeof(size_t) <= sizeof(int))
914 format
= SMALL_STRING
;
917 return seq_printf(s
, format
,
924 is_file_hugepages(shp
->shm_file
) ? (file_count(shp
->shm_file
) - 1) : shp
->shm_nattch
,