4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
14 #include <linux/mount.h>
15 #include <linux/pipe_fs_i.h>
16 #include <linux/uio.h>
17 #include <linux/highmem.h>
18 #include <linux/pagemap.h>
20 #include <asm/uaccess.h>
21 #include <asm/ioctls.h>
24 * We use a start+len construction, which provides full use of the
26 * -- Florian Coosmann (FGC)
28 * Reads with count = 0 should always return 0.
29 * -- Julian Bradfield 1999-06-07.
31 * FIFOs and Pipes now generate SIGIO for both readers and writers.
32 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
34 * pipe_read & write cleanup
35 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
38 /* Drop the inode semaphore and wait for a pipe event, atomically */
39 void pipe_wait(struct pipe_inode_info
*pipe
)
44 * Pipes are system-local resources, so sleeping on them
45 * is considered a noninteractive wait:
47 prepare_to_wait(&pipe
->wait
, &wait
,
48 TASK_INTERRUPTIBLE
| TASK_NONINTERACTIVE
);
50 mutex_unlock(&pipe
->inode
->i_mutex
);
52 finish_wait(&pipe
->wait
, &wait
);
54 mutex_lock(&pipe
->inode
->i_mutex
);
58 pipe_iov_copy_from_user(void *to
, struct iovec
*iov
, unsigned long len
)
65 copy
= min_t(unsigned long, len
, iov
->iov_len
);
67 if (copy_from_user(to
, iov
->iov_base
, copy
))
71 iov
->iov_base
+= copy
;
78 pipe_iov_copy_to_user(struct iovec
*iov
, const void *from
, unsigned long len
)
85 copy
= min_t(unsigned long, len
, iov
->iov_len
);
87 if (copy_to_user(iov
->iov_base
, from
, copy
))
91 iov
->iov_base
+= copy
;
97 static void anon_pipe_buf_release(struct pipe_inode_info
*pipe
,
98 struct pipe_buffer
*buf
)
100 struct page
*page
= buf
->page
;
102 buf
->flags
&= ~PIPE_BUF_FLAG_STOLEN
;
105 * If nobody else uses this page, and we don't already have a
106 * temporary page, let's keep track of it as a one-deep
107 * allocation cache. (Otherwise just release our reference to it)
109 if (page_count(page
) == 1 && !pipe
->tmp_page
)
110 pipe
->tmp_page
= page
;
112 page_cache_release(page
);
115 static void * anon_pipe_buf_map(struct file
*file
, struct pipe_inode_info
*pipe
,
116 struct pipe_buffer
*buf
)
118 return kmap(buf
->page
);
121 static void anon_pipe_buf_unmap(struct pipe_inode_info
*pipe
,
122 struct pipe_buffer
*buf
)
127 static int anon_pipe_buf_steal(struct pipe_inode_info
*pipe
,
128 struct pipe_buffer
*buf
)
130 buf
->flags
|= PIPE_BUF_FLAG_STOLEN
;
134 static void anon_pipe_buf_get(struct pipe_inode_info
*info
,
135 struct pipe_buffer
*buf
)
137 page_cache_get(buf
->page
);
140 static struct pipe_buf_operations anon_pipe_buf_ops
= {
142 .map
= anon_pipe_buf_map
,
143 .unmap
= anon_pipe_buf_unmap
,
144 .release
= anon_pipe_buf_release
,
145 .steal
= anon_pipe_buf_steal
,
146 .get
= anon_pipe_buf_get
,
150 pipe_readv(struct file
*filp
, const struct iovec
*_iov
,
151 unsigned long nr_segs
, loff_t
*ppos
)
153 struct inode
*inode
= filp
->f_dentry
->d_inode
;
154 struct pipe_inode_info
*pipe
;
157 struct iovec
*iov
= (struct iovec
*)_iov
;
160 total_len
= iov_length(iov
, nr_segs
);
161 /* Null read succeeds. */
162 if (unlikely(total_len
== 0))
167 mutex_lock(&inode
->i_mutex
);
168 pipe
= inode
->i_pipe
;
170 int bufs
= pipe
->nrbufs
;
172 int curbuf
= pipe
->curbuf
;
173 struct pipe_buffer
*buf
= pipe
->bufs
+ curbuf
;
174 struct pipe_buf_operations
*ops
= buf
->ops
;
176 size_t chars
= buf
->len
;
179 if (chars
> total_len
)
182 addr
= ops
->map(filp
, pipe
, buf
);
188 error
= pipe_iov_copy_to_user(iov
, addr
+ buf
->offset
, chars
);
189 ops
->unmap(pipe
, buf
);
190 if (unlikely(error
)) {
196 buf
->offset
+= chars
;
200 ops
->release(pipe
, buf
);
201 curbuf
= (curbuf
+ 1) & (PIPE_BUFFERS
-1);
202 pipe
->curbuf
= curbuf
;
203 pipe
->nrbufs
= --bufs
;
208 break; /* common path: read succeeded */
210 if (bufs
) /* More to do? */
214 if (!pipe
->waiting_writers
) {
215 /* syscall merging: Usually we must not sleep
216 * if O_NONBLOCK is set, or if we got some data.
217 * But if a writer sleeps in kernel space, then
218 * we can wait for that data without violating POSIX.
222 if (filp
->f_flags
& O_NONBLOCK
) {
227 if (signal_pending(current
)) {
233 wake_up_interruptible_sync(&pipe
->wait
);
234 kill_fasync(&pipe
->fasync_writers
, SIGIO
, POLL_OUT
);
238 mutex_unlock(&inode
->i_mutex
);
240 /* Signal writers asynchronously that there is more room. */
242 wake_up_interruptible(&pipe
->wait
);
243 kill_fasync(&pipe
->fasync_writers
, SIGIO
, POLL_OUT
);
251 pipe_read(struct file
*filp
, char __user
*buf
, size_t count
, loff_t
*ppos
)
253 struct iovec iov
= { .iov_base
= buf
, .iov_len
= count
};
255 return pipe_readv(filp
, &iov
, 1, ppos
);
259 pipe_writev(struct file
*filp
, const struct iovec
*_iov
,
260 unsigned long nr_segs
, loff_t
*ppos
)
262 struct inode
*inode
= filp
->f_dentry
->d_inode
;
263 struct pipe_inode_info
*pipe
;
266 struct iovec
*iov
= (struct iovec
*)_iov
;
270 total_len
= iov_length(iov
, nr_segs
);
271 /* Null write succeeds. */
272 if (unlikely(total_len
== 0))
277 mutex_lock(&inode
->i_mutex
);
278 pipe
= inode
->i_pipe
;
280 if (!pipe
->readers
) {
281 send_sig(SIGPIPE
, current
, 0);
286 /* We try to merge small writes */
287 chars
= total_len
& (PAGE_SIZE
-1); /* size of the last buffer */
288 if (pipe
->nrbufs
&& chars
!= 0) {
289 int lastbuf
= (pipe
->curbuf
+ pipe
->nrbufs
- 1) &
291 struct pipe_buffer
*buf
= pipe
->bufs
+ lastbuf
;
292 struct pipe_buf_operations
*ops
= buf
->ops
;
293 int offset
= buf
->offset
+ buf
->len
;
295 if (ops
->can_merge
&& offset
+ chars
<= PAGE_SIZE
) {
299 addr
= ops
->map(filp
, pipe
, buf
);
301 error
= PTR_ERR(addr
);
304 error
= pipe_iov_copy_from_user(offset
+ addr
, iov
,
306 ops
->unmap(pipe
, buf
);
322 if (!pipe
->readers
) {
323 send_sig(SIGPIPE
, current
, 0);
329 if (bufs
< PIPE_BUFFERS
) {
330 int newbuf
= (pipe
->curbuf
+ bufs
) & (PIPE_BUFFERS
-1);
331 struct pipe_buffer
*buf
= pipe
->bufs
+ newbuf
;
332 struct page
*page
= pipe
->tmp_page
;
336 page
= alloc_page(GFP_HIGHUSER
);
337 if (unlikely(!page
)) {
338 ret
= ret
? : -ENOMEM
;
341 pipe
->tmp_page
= page
;
343 /* Always wake up, even if the copy fails. Otherwise
344 * we lock up (O_NONBLOCK-)readers that sleep due to
346 * FIXME! Is this really true?
350 if (chars
> total_len
)
353 error
= pipe_iov_copy_from_user(kmap(page
), iov
, chars
);
355 if (unlikely(error
)) {
362 /* Insert it into the buffer array */
364 buf
->ops
= &anon_pipe_buf_ops
;
367 pipe
->nrbufs
= ++bufs
;
368 pipe
->tmp_page
= NULL
;
374 if (bufs
< PIPE_BUFFERS
)
376 if (filp
->f_flags
& O_NONBLOCK
) {
381 if (signal_pending(current
)) {
387 wake_up_interruptible_sync(&pipe
->wait
);
388 kill_fasync(&pipe
->fasync_readers
, SIGIO
, POLL_IN
);
391 pipe
->waiting_writers
++;
393 pipe
->waiting_writers
--;
396 mutex_unlock(&inode
->i_mutex
);
398 wake_up_interruptible(&pipe
->wait
);
399 kill_fasync(&pipe
->fasync_readers
, SIGIO
, POLL_IN
);
402 file_update_time(filp
);
407 pipe_write(struct file
*filp
, const char __user
*buf
,
408 size_t count
, loff_t
*ppos
)
410 struct iovec iov
= { .iov_base
= (void __user
*)buf
, .iov_len
= count
};
412 return pipe_writev(filp
, &iov
, 1, ppos
);
416 bad_pipe_r(struct file
*filp
, char __user
*buf
, size_t count
, loff_t
*ppos
)
422 bad_pipe_w(struct file
*filp
, const char __user
*buf
, size_t count
,
429 pipe_ioctl(struct inode
*pino
, struct file
*filp
,
430 unsigned int cmd
, unsigned long arg
)
432 struct inode
*inode
= filp
->f_dentry
->d_inode
;
433 struct pipe_inode_info
*pipe
;
434 int count
, buf
, nrbufs
;
438 mutex_lock(&inode
->i_mutex
);
439 pipe
= inode
->i_pipe
;
442 nrbufs
= pipe
->nrbufs
;
443 while (--nrbufs
>= 0) {
444 count
+= pipe
->bufs
[buf
].len
;
445 buf
= (buf
+1) & (PIPE_BUFFERS
-1);
447 mutex_unlock(&inode
->i_mutex
);
449 return put_user(count
, (int __user
*)arg
);
455 /* No kernel lock held - fine */
457 pipe_poll(struct file
*filp
, poll_table
*wait
)
460 struct inode
*inode
= filp
->f_dentry
->d_inode
;
461 struct pipe_inode_info
*pipe
= inode
->i_pipe
;
464 poll_wait(filp
, &pipe
->wait
, wait
);
466 /* Reading only -- no need for acquiring the semaphore. */
467 nrbufs
= pipe
->nrbufs
;
469 if (filp
->f_mode
& FMODE_READ
) {
470 mask
= (nrbufs
> 0) ? POLLIN
| POLLRDNORM
: 0;
471 if (!pipe
->writers
&& filp
->f_version
!= pipe
->w_counter
)
475 if (filp
->f_mode
& FMODE_WRITE
) {
476 mask
|= (nrbufs
< PIPE_BUFFERS
) ? POLLOUT
| POLLWRNORM
: 0;
478 * Most Unices do not set POLLERR for FIFOs but on Linux they
479 * behave exactly like pipes for poll().
489 pipe_release(struct inode
*inode
, int decr
, int decw
)
491 struct pipe_inode_info
*pipe
;
493 mutex_lock(&inode
->i_mutex
);
494 pipe
= inode
->i_pipe
;
495 pipe
->readers
-= decr
;
496 pipe
->writers
-= decw
;
498 if (!pipe
->readers
&& !pipe
->writers
) {
499 free_pipe_info(inode
);
501 wake_up_interruptible(&pipe
->wait
);
502 kill_fasync(&pipe
->fasync_readers
, SIGIO
, POLL_IN
);
503 kill_fasync(&pipe
->fasync_writers
, SIGIO
, POLL_OUT
);
505 mutex_unlock(&inode
->i_mutex
);
511 pipe_read_fasync(int fd
, struct file
*filp
, int on
)
513 struct inode
*inode
= filp
->f_dentry
->d_inode
;
516 mutex_lock(&inode
->i_mutex
);
517 retval
= fasync_helper(fd
, filp
, on
, &inode
->i_pipe
->fasync_readers
);
518 mutex_unlock(&inode
->i_mutex
);
528 pipe_write_fasync(int fd
, struct file
*filp
, int on
)
530 struct inode
*inode
= filp
->f_dentry
->d_inode
;
533 mutex_lock(&inode
->i_mutex
);
534 retval
= fasync_helper(fd
, filp
, on
, &inode
->i_pipe
->fasync_writers
);
535 mutex_unlock(&inode
->i_mutex
);
545 pipe_rdwr_fasync(int fd
, struct file
*filp
, int on
)
547 struct inode
*inode
= filp
->f_dentry
->d_inode
;
548 struct pipe_inode_info
*pipe
= inode
->i_pipe
;
551 mutex_lock(&inode
->i_mutex
);
553 retval
= fasync_helper(fd
, filp
, on
, &pipe
->fasync_readers
);
556 retval
= fasync_helper(fd
, filp
, on
, &pipe
->fasync_writers
);
558 mutex_unlock(&inode
->i_mutex
);
568 pipe_read_release(struct inode
*inode
, struct file
*filp
)
570 pipe_read_fasync(-1, filp
, 0);
571 return pipe_release(inode
, 1, 0);
575 pipe_write_release(struct inode
*inode
, struct file
*filp
)
577 pipe_write_fasync(-1, filp
, 0);
578 return pipe_release(inode
, 0, 1);
582 pipe_rdwr_release(struct inode
*inode
, struct file
*filp
)
586 pipe_rdwr_fasync(-1, filp
, 0);
587 decr
= (filp
->f_mode
& FMODE_READ
) != 0;
588 decw
= (filp
->f_mode
& FMODE_WRITE
) != 0;
589 return pipe_release(inode
, decr
, decw
);
593 pipe_read_open(struct inode
*inode
, struct file
*filp
)
595 /* We could have perhaps used atomic_t, but this and friends
596 below are the only places. So it doesn't seem worthwhile. */
597 mutex_lock(&inode
->i_mutex
);
598 inode
->i_pipe
->readers
++;
599 mutex_unlock(&inode
->i_mutex
);
605 pipe_write_open(struct inode
*inode
, struct file
*filp
)
607 mutex_lock(&inode
->i_mutex
);
608 inode
->i_pipe
->writers
++;
609 mutex_unlock(&inode
->i_mutex
);
615 pipe_rdwr_open(struct inode
*inode
, struct file
*filp
)
617 mutex_lock(&inode
->i_mutex
);
618 if (filp
->f_mode
& FMODE_READ
)
619 inode
->i_pipe
->readers
++;
620 if (filp
->f_mode
& FMODE_WRITE
)
621 inode
->i_pipe
->writers
++;
622 mutex_unlock(&inode
->i_mutex
);
628 * The file_operations structs are not static because they
629 * are also used in linux/fs/fifo.c to do operations on FIFOs.
631 const struct file_operations read_fifo_fops
= {
638 .open
= pipe_read_open
,
639 .release
= pipe_read_release
,
640 .fasync
= pipe_read_fasync
,
643 const struct file_operations write_fifo_fops
= {
647 .writev
= pipe_writev
,
650 .open
= pipe_write_open
,
651 .release
= pipe_write_release
,
652 .fasync
= pipe_write_fasync
,
655 const struct file_operations rdwr_fifo_fops
= {
660 .writev
= pipe_writev
,
663 .open
= pipe_rdwr_open
,
664 .release
= pipe_rdwr_release
,
665 .fasync
= pipe_rdwr_fasync
,
668 static struct file_operations read_pipe_fops
= {
675 .open
= pipe_read_open
,
676 .release
= pipe_read_release
,
677 .fasync
= pipe_read_fasync
,
680 static struct file_operations write_pipe_fops
= {
684 .writev
= pipe_writev
,
687 .open
= pipe_write_open
,
688 .release
= pipe_write_release
,
689 .fasync
= pipe_write_fasync
,
692 static struct file_operations rdwr_pipe_fops
= {
697 .writev
= pipe_writev
,
700 .open
= pipe_rdwr_open
,
701 .release
= pipe_rdwr_release
,
702 .fasync
= pipe_rdwr_fasync
,
705 struct pipe_inode_info
* alloc_pipe_info(struct inode
*inode
)
707 struct pipe_inode_info
*pipe
;
709 pipe
= kzalloc(sizeof(struct pipe_inode_info
), GFP_KERNEL
);
711 init_waitqueue_head(&pipe
->wait
);
712 pipe
->r_counter
= pipe
->w_counter
= 1;
719 void __free_pipe_info(struct pipe_inode_info
*pipe
)
723 for (i
= 0; i
< PIPE_BUFFERS
; i
++) {
724 struct pipe_buffer
*buf
= pipe
->bufs
+ i
;
726 buf
->ops
->release(pipe
, buf
);
729 __free_page(pipe
->tmp_page
);
733 void free_pipe_info(struct inode
*inode
)
735 __free_pipe_info(inode
->i_pipe
);
736 inode
->i_pipe
= NULL
;
739 static struct vfsmount
*pipe_mnt __read_mostly
;
740 static int pipefs_delete_dentry(struct dentry
*dentry
)
745 static struct dentry_operations pipefs_dentry_operations
= {
746 .d_delete
= pipefs_delete_dentry
,
749 static struct inode
* get_pipe_inode(void)
751 struct inode
*inode
= new_inode(pipe_mnt
->mnt_sb
);
752 struct pipe_inode_info
*pipe
;
757 pipe
= alloc_pipe_info(inode
);
760 inode
->i_pipe
= pipe
;
762 pipe
->readers
= pipe
->writers
= 1;
763 inode
->i_fop
= &rdwr_pipe_fops
;
766 * Mark the inode dirty from the very beginning,
767 * that way it will never be moved to the dirty
768 * list because "mark_inode_dirty()" will think
769 * that it already _is_ on the dirty list.
771 inode
->i_state
= I_DIRTY
;
772 inode
->i_mode
= S_IFIFO
| S_IRUSR
| S_IWUSR
;
773 inode
->i_uid
= current
->fsuid
;
774 inode
->i_gid
= current
->fsgid
;
775 inode
->i_atime
= inode
->i_mtime
= inode
->i_ctime
= CURRENT_TIME
;
776 inode
->i_blksize
= PAGE_SIZE
;
791 struct dentry
*dentry
;
792 struct inode
* inode
;
793 struct file
*f1
, *f2
;
798 f1
= get_empty_filp();
802 f2
= get_empty_filp();
806 inode
= get_pipe_inode();
810 error
= get_unused_fd();
812 goto close_f12_inode
;
815 error
= get_unused_fd();
817 goto close_f12_inode_i
;
821 sprintf(name
, "[%lu]", inode
->i_ino
);
823 this.len
= strlen(name
);
824 this.hash
= inode
->i_ino
; /* will go */
825 dentry
= d_alloc(pipe_mnt
->mnt_sb
->s_root
, &this);
827 goto close_f12_inode_i_j
;
829 dentry
->d_op
= &pipefs_dentry_operations
;
830 d_add(dentry
, inode
);
831 f1
->f_vfsmnt
= f2
->f_vfsmnt
= mntget(mntget(pipe_mnt
));
832 f1
->f_dentry
= f2
->f_dentry
= dget(dentry
);
833 f1
->f_mapping
= f2
->f_mapping
= inode
->i_mapping
;
836 f1
->f_pos
= f2
->f_pos
= 0;
837 f1
->f_flags
= O_RDONLY
;
838 f1
->f_op
= &read_pipe_fops
;
839 f1
->f_mode
= FMODE_READ
;
843 f2
->f_flags
= O_WRONLY
;
844 f2
->f_op
= &write_pipe_fops
;
845 f2
->f_mode
= FMODE_WRITE
;
860 free_pipe_info(inode
);
871 * pipefs should _never_ be mounted by userland - too much of security hassle,
872 * no real gain from having the whole whorehouse mounted. So we don't need
873 * any operations on the root directory. However, we need a non-trivial
874 * d_name - pipe: will go nicely and kill the special-casing in procfs.
877 static struct super_block
*
878 pipefs_get_sb(struct file_system_type
*fs_type
, int flags
,
879 const char *dev_name
, void *data
)
881 return get_sb_pseudo(fs_type
, "pipe:", NULL
, PIPEFS_MAGIC
);
884 static struct file_system_type pipe_fs_type
= {
886 .get_sb
= pipefs_get_sb
,
887 .kill_sb
= kill_anon_super
,
890 static int __init
init_pipe_fs(void)
892 int err
= register_filesystem(&pipe_fs_type
);
895 pipe_mnt
= kern_mount(&pipe_fs_type
);
896 if (IS_ERR(pipe_mnt
)) {
897 err
= PTR_ERR(pipe_mnt
);
898 unregister_filesystem(&pipe_fs_type
);
904 static void __exit
exit_pipe_fs(void)
906 unregister_filesystem(&pipe_fs_type
);
910 fs_initcall(init_pipe_fs
);
911 module_exit(exit_pipe_fs
);