Luca's patch ported
[cbs-scheduler.git] / fs / pipe.c
blob8a0fc0e3e406784aa10e44fde5d21399da3be508
1 /*
2 * linux/fs/pipe.c
4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
5 */
7 #include <linux/mm.h>
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/fs.h>
14 #include <linux/mount.h>
15 #include <linux/pipe_fs_i.h>
16 #include <linux/uio.h>
17 #include <linux/highmem.h>
18 #include <linux/pagemap.h>
19 #include <linux/audit.h>
20 #include <linux/syscalls.h>
22 #include <asm/uaccess.h>
23 #include <asm/ioctls.h>
26 * We use a start+len construction, which provides full use of the
27 * allocated memory.
28 * -- Florian Coosmann (FGC)
30 * Reads with count = 0 should always return 0.
31 * -- Julian Bradfield 1999-06-07.
33 * FIFOs and Pipes now generate SIGIO for both readers and writers.
34 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
36 * pipe_read & write cleanup
37 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
40 /* Drop the inode semaphore and wait for a pipe event, atomically */
41 void pipe_wait(struct pipe_inode_info *pipe)
43 DEFINE_WAIT(wait);
46 * Pipes are system-local resources, so sleeping on them
47 * is considered a noninteractive wait:
49 prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE);
50 if (pipe->inode)
51 mutex_unlock(&pipe->inode->i_mutex);
52 schedule();
53 finish_wait(&pipe->wait, &wait);
54 if (pipe->inode)
55 mutex_lock(&pipe->inode->i_mutex);
58 static int
59 pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len,
60 int atomic)
62 unsigned long copy;
64 while (len > 0) {
65 while (!iov->iov_len)
66 iov++;
67 copy = min_t(unsigned long, len, iov->iov_len);
69 if (atomic) {
70 if (__copy_from_user_inatomic(to, iov->iov_base, copy))
71 return -EFAULT;
72 } else {
73 if (copy_from_user(to, iov->iov_base, copy))
74 return -EFAULT;
76 to += copy;
77 len -= copy;
78 iov->iov_base += copy;
79 iov->iov_len -= copy;
81 return 0;
84 static int
85 pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len,
86 int atomic)
88 unsigned long copy;
90 while (len > 0) {
91 while (!iov->iov_len)
92 iov++;
93 copy = min_t(unsigned long, len, iov->iov_len);
95 if (atomic) {
96 if (__copy_to_user_inatomic(iov->iov_base, from, copy))
97 return -EFAULT;
98 } else {
99 if (copy_to_user(iov->iov_base, from, copy))
100 return -EFAULT;
102 from += copy;
103 len -= copy;
104 iov->iov_base += copy;
105 iov->iov_len -= copy;
107 return 0;
111 * Attempt to pre-fault in the user memory, so we can use atomic copies.
112 * Returns the number of bytes not faulted in.
114 static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len)
116 while (!iov->iov_len)
117 iov++;
119 while (len > 0) {
120 unsigned long this_len;
122 this_len = min_t(unsigned long, len, iov->iov_len);
123 if (fault_in_pages_writeable(iov->iov_base, this_len))
124 break;
126 len -= this_len;
127 iov++;
130 return len;
134 * Pre-fault in the user memory, so we can use atomic copies.
136 static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len)
138 while (!iov->iov_len)
139 iov++;
141 while (len > 0) {
142 unsigned long this_len;
144 this_len = min_t(unsigned long, len, iov->iov_len);
145 fault_in_pages_readable(iov->iov_base, this_len);
146 len -= this_len;
147 iov++;
151 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
152 struct pipe_buffer *buf)
154 struct page *page = buf->page;
157 * If nobody else uses this page, and we don't already have a
158 * temporary page, let's keep track of it as a one-deep
159 * allocation cache. (Otherwise just release our reference to it)
161 if (page_count(page) == 1 && !pipe->tmp_page)
162 pipe->tmp_page = page;
163 else
164 page_cache_release(page);
168 * generic_pipe_buf_map - virtually map a pipe buffer
169 * @pipe: the pipe that the buffer belongs to
170 * @buf: the buffer that should be mapped
171 * @atomic: whether to use an atomic map
173 * Description:
174 * This function returns a kernel virtual address mapping for the
175 * pipe_buffer passed in @buf. If @atomic is set, an atomic map is provided
176 * and the caller has to be careful not to fault before calling
177 * the unmap function.
179 * Note that this function occupies KM_USER0 if @atomic != 0.
181 void *generic_pipe_buf_map(struct pipe_inode_info *pipe,
182 struct pipe_buffer *buf, int atomic)
184 if (atomic) {
185 buf->flags |= PIPE_BUF_FLAG_ATOMIC;
186 return kmap_atomic(buf->page, KM_USER0);
189 return kmap(buf->page);
193 * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer
194 * @pipe: the pipe that the buffer belongs to
195 * @buf: the buffer that should be unmapped
196 * @map_data: the data that the mapping function returned
198 * Description:
199 * This function undoes the mapping that ->map() provided.
201 void generic_pipe_buf_unmap(struct pipe_inode_info *pipe,
202 struct pipe_buffer *buf, void *map_data)
204 if (buf->flags & PIPE_BUF_FLAG_ATOMIC) {
205 buf->flags &= ~PIPE_BUF_FLAG_ATOMIC;
206 kunmap_atomic(map_data, KM_USER0);
207 } else
208 kunmap(buf->page);
212 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer
213 * @pipe: the pipe that the buffer belongs to
214 * @buf: the buffer to attempt to steal
216 * Description:
217 * This function attempts to steal the &struct page attached to
218 * @buf. If successful, this function returns 0 and returns with
219 * the page locked. The caller may then reuse the page for whatever
220 * he wishes; the typical use is insertion into a different file
221 * page cache.
223 int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
224 struct pipe_buffer *buf)
226 struct page *page = buf->page;
229 * A reference of one is golden, that means that the owner of this
230 * page is the only one holding a reference to it. lock the page
231 * and return OK.
233 if (page_count(page) == 1) {
234 lock_page(page);
235 return 0;
238 return 1;
242 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
243 * @pipe: the pipe that the buffer belongs to
244 * @buf: the buffer to get a reference to
246 * Description:
247 * This function grabs an extra reference to @buf. It's used in
248 * in the tee() system call, when we duplicate the buffers in one
249 * pipe into another.
251 void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
253 page_cache_get(buf->page);
257 * generic_pipe_buf_confirm - verify contents of the pipe buffer
258 * @info: the pipe that the buffer belongs to
259 * @buf: the buffer to confirm
261 * Description:
262 * This function does nothing, because the generic pipe code uses
263 * pages that are always good when inserted into the pipe.
265 int generic_pipe_buf_confirm(struct pipe_inode_info *info,
266 struct pipe_buffer *buf)
268 return 0;
271 static const struct pipe_buf_operations anon_pipe_buf_ops = {
272 .can_merge = 1,
273 .map = generic_pipe_buf_map,
274 .unmap = generic_pipe_buf_unmap,
275 .confirm = generic_pipe_buf_confirm,
276 .release = anon_pipe_buf_release,
277 .steal = generic_pipe_buf_steal,
278 .get = generic_pipe_buf_get,
281 static ssize_t
282 pipe_read(struct kiocb *iocb, const struct iovec *_iov,
283 unsigned long nr_segs, loff_t pos)
285 struct file *filp = iocb->ki_filp;
286 struct inode *inode = filp->f_path.dentry->d_inode;
287 struct pipe_inode_info *pipe;
288 int do_wakeup;
289 ssize_t ret;
290 struct iovec *iov = (struct iovec *)_iov;
291 size_t total_len;
293 total_len = iov_length(iov, nr_segs);
294 /* Null read succeeds. */
295 if (unlikely(total_len == 0))
296 return 0;
298 do_wakeup = 0;
299 ret = 0;
300 mutex_lock(&inode->i_mutex);
301 pipe = inode->i_pipe;
302 for (;;) {
303 int bufs = pipe->nrbufs;
304 if (bufs) {
305 int curbuf = pipe->curbuf;
306 struct pipe_buffer *buf = pipe->bufs + curbuf;
307 const struct pipe_buf_operations *ops = buf->ops;
308 void *addr;
309 size_t chars = buf->len;
310 int error, atomic;
312 if (chars > total_len)
313 chars = total_len;
315 error = ops->confirm(pipe, buf);
316 if (error) {
317 if (!ret)
318 error = ret;
319 break;
322 atomic = !iov_fault_in_pages_write(iov, chars);
323 redo:
324 addr = ops->map(pipe, buf, atomic);
325 error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic);
326 ops->unmap(pipe, buf, addr);
327 if (unlikely(error)) {
329 * Just retry with the slow path if we failed.
331 if (atomic) {
332 atomic = 0;
333 goto redo;
335 if (!ret)
336 ret = error;
337 break;
339 ret += chars;
340 buf->offset += chars;
341 buf->len -= chars;
342 if (!buf->len) {
343 buf->ops = NULL;
344 ops->release(pipe, buf);
345 curbuf = (curbuf + 1) & (PIPE_BUFFERS-1);
346 pipe->curbuf = curbuf;
347 pipe->nrbufs = --bufs;
348 do_wakeup = 1;
350 total_len -= chars;
351 if (!total_len)
352 break; /* common path: read succeeded */
354 if (bufs) /* More to do? */
355 continue;
356 if (!pipe->writers)
357 break;
358 if (!pipe->waiting_writers) {
359 /* syscall merging: Usually we must not sleep
360 * if O_NONBLOCK is set, or if we got some data.
361 * But if a writer sleeps in kernel space, then
362 * we can wait for that data without violating POSIX.
364 if (ret)
365 break;
366 if (filp->f_flags & O_NONBLOCK) {
367 ret = -EAGAIN;
368 break;
371 if (signal_pending(current)) {
372 if (!ret)
373 ret = -ERESTARTSYS;
374 break;
376 if (do_wakeup) {
377 wake_up_interruptible_sync(&pipe->wait);
378 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
380 pipe_wait(pipe);
382 mutex_unlock(&inode->i_mutex);
384 /* Signal writers asynchronously that there is more room. */
385 if (do_wakeup) {
386 wake_up_interruptible_sync(&pipe->wait);
387 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
390 * Hack: we turn off atime updates for -RT kernels.
391 * Who uses them on pipes anyway?
393 #ifndef CONFIG_PREEMPT_RT
394 if (ret > 0)
395 file_accessed(filp);
396 #endif
397 return ret;
400 static ssize_t
401 pipe_write(struct kiocb *iocb, const struct iovec *_iov,
402 unsigned long nr_segs, loff_t ppos)
404 struct file *filp = iocb->ki_filp;
405 struct inode *inode = filp->f_path.dentry->d_inode;
406 struct pipe_inode_info *pipe;
407 ssize_t ret;
408 int do_wakeup;
409 struct iovec *iov = (struct iovec *)_iov;
410 size_t total_len;
411 ssize_t chars;
413 total_len = iov_length(iov, nr_segs);
414 /* Null write succeeds. */
415 if (unlikely(total_len == 0))
416 return 0;
418 do_wakeup = 0;
419 ret = 0;
420 mutex_lock(&inode->i_mutex);
421 pipe = inode->i_pipe;
423 if (!pipe->readers) {
424 send_sig(SIGPIPE, current, 0);
425 ret = -EPIPE;
426 goto out;
429 /* We try to merge small writes */
430 chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
431 if (pipe->nrbufs && chars != 0) {
432 int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) &
433 (PIPE_BUFFERS-1);
434 struct pipe_buffer *buf = pipe->bufs + lastbuf;
435 const struct pipe_buf_operations *ops = buf->ops;
436 int offset = buf->offset + buf->len;
438 if (ops->can_merge && offset + chars <= PAGE_SIZE) {
439 int error, atomic = 1;
440 void *addr;
442 error = ops->confirm(pipe, buf);
443 if (error)
444 goto out;
446 iov_fault_in_pages_read(iov, chars);
447 redo1:
448 addr = ops->map(pipe, buf, atomic);
449 error = pipe_iov_copy_from_user(offset + addr, iov,
450 chars, atomic);
451 ops->unmap(pipe, buf, addr);
452 ret = error;
453 do_wakeup = 1;
454 if (error) {
455 if (atomic) {
456 atomic = 0;
457 goto redo1;
459 goto out;
461 buf->len += chars;
462 total_len -= chars;
463 ret = chars;
464 if (!total_len)
465 goto out;
469 for (;;) {
470 int bufs;
472 if (!pipe->readers) {
473 send_sig(SIGPIPE, current, 0);
474 if (!ret)
475 ret = -EPIPE;
476 break;
478 bufs = pipe->nrbufs;
479 if (bufs < PIPE_BUFFERS) {
480 int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1);
481 struct pipe_buffer *buf = pipe->bufs + newbuf;
482 struct page *page = pipe->tmp_page;
483 char *src;
484 int error, atomic = 1;
486 if (!page) {
487 page = alloc_page(GFP_HIGHUSER);
488 if (unlikely(!page)) {
489 ret = ret ? : -ENOMEM;
490 break;
492 pipe->tmp_page = page;
494 /* Always wake up, even if the copy fails. Otherwise
495 * we lock up (O_NONBLOCK-)readers that sleep due to
496 * syscall merging.
497 * FIXME! Is this really true?
499 do_wakeup = 1;
500 chars = PAGE_SIZE;
501 if (chars > total_len)
502 chars = total_len;
504 iov_fault_in_pages_read(iov, chars);
505 redo2:
506 if (atomic)
507 src = kmap_atomic(page, KM_USER0);
508 else
509 src = kmap(page);
511 error = pipe_iov_copy_from_user(src, iov, chars,
512 atomic);
513 if (atomic)
514 kunmap_atomic(src, KM_USER0);
515 else
516 kunmap(page);
518 if (unlikely(error)) {
519 if (atomic) {
520 atomic = 0;
521 goto redo2;
523 if (!ret)
524 ret = error;
525 break;
527 ret += chars;
529 /* Insert it into the buffer array */
530 buf->page = page;
531 buf->ops = &anon_pipe_buf_ops;
532 buf->offset = 0;
533 buf->len = chars;
534 pipe->nrbufs = ++bufs;
535 pipe->tmp_page = NULL;
537 total_len -= chars;
538 if (!total_len)
539 break;
541 if (bufs < PIPE_BUFFERS)
542 continue;
543 if (filp->f_flags & O_NONBLOCK) {
544 if (!ret)
545 ret = -EAGAIN;
546 break;
548 if (signal_pending(current)) {
549 if (!ret)
550 ret = -ERESTARTSYS;
551 break;
553 if (do_wakeup) {
554 wake_up_interruptible_sync(&pipe->wait);
555 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
556 do_wakeup = 0;
558 pipe->waiting_writers++;
559 pipe_wait(pipe);
560 pipe->waiting_writers--;
562 out:
563 mutex_unlock(&inode->i_mutex);
564 if (do_wakeup) {
565 wake_up_interruptible_sync(&pipe->wait);
566 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
569 * Hack: we turn off atime updates for -RT kernels.
570 * Who uses them on pipes anyway?
572 #ifndef CONFIG_PREEMPT_RT
573 if (ret > 0)
574 file_update_time(filp);
575 #endif
576 return ret;
579 static ssize_t
580 bad_pipe_r(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
582 return -EBADF;
585 static ssize_t
586 bad_pipe_w(struct file *filp, const char __user *buf, size_t count,
587 loff_t *ppos)
589 return -EBADF;
592 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
594 struct inode *inode = filp->f_path.dentry->d_inode;
595 struct pipe_inode_info *pipe;
596 int count, buf, nrbufs;
598 switch (cmd) {
599 case FIONREAD:
600 mutex_lock(&inode->i_mutex);
601 pipe = inode->i_pipe;
602 count = 0;
603 buf = pipe->curbuf;
604 nrbufs = pipe->nrbufs;
605 while (--nrbufs >= 0) {
606 count += pipe->bufs[buf].len;
607 buf = (buf+1) & (PIPE_BUFFERS-1);
609 mutex_unlock(&inode->i_mutex);
611 return put_user(count, (int __user *)arg);
612 default:
613 return -EINVAL;
617 /* No kernel lock held - fine */
618 static unsigned int
619 pipe_poll(struct file *filp, poll_table *wait)
621 unsigned int mask;
622 struct inode *inode = filp->f_path.dentry->d_inode;
623 struct pipe_inode_info *pipe = inode->i_pipe;
624 int nrbufs;
626 poll_wait(filp, &pipe->wait, wait);
628 /* Reading only -- no need for acquiring the semaphore. */
629 nrbufs = pipe->nrbufs;
630 mask = 0;
631 if (filp->f_mode & FMODE_READ) {
632 mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0;
633 if (!pipe->writers && filp->f_version != pipe->w_counter)
634 mask |= POLLHUP;
637 if (filp->f_mode & FMODE_WRITE) {
638 mask |= (nrbufs < PIPE_BUFFERS) ? POLLOUT | POLLWRNORM : 0;
640 * Most Unices do not set POLLERR for FIFOs but on Linux they
641 * behave exactly like pipes for poll().
643 if (!pipe->readers)
644 mask |= POLLERR;
647 return mask;
650 static int
651 pipe_release(struct inode *inode, int decr, int decw)
653 struct pipe_inode_info *pipe;
655 mutex_lock(&inode->i_mutex);
656 pipe = inode->i_pipe;
657 pipe->readers -= decr;
658 pipe->writers -= decw;
660 if (!pipe->readers && !pipe->writers) {
661 free_pipe_info(inode);
662 } else {
663 wake_up_interruptible_sync(&pipe->wait);
664 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
665 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
667 mutex_unlock(&inode->i_mutex);
669 return 0;
672 static int
673 pipe_read_fasync(int fd, struct file *filp, int on)
675 struct inode *inode = filp->f_path.dentry->d_inode;
676 int retval;
678 mutex_lock(&inode->i_mutex);
679 retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_readers);
680 mutex_unlock(&inode->i_mutex);
682 if (retval < 0)
683 return retval;
685 return 0;
689 static int
690 pipe_write_fasync(int fd, struct file *filp, int on)
692 struct inode *inode = filp->f_path.dentry->d_inode;
693 int retval;
695 mutex_lock(&inode->i_mutex);
696 retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_writers);
697 mutex_unlock(&inode->i_mutex);
699 if (retval < 0)
700 return retval;
702 return 0;
706 static int
707 pipe_rdwr_fasync(int fd, struct file *filp, int on)
709 struct inode *inode = filp->f_path.dentry->d_inode;
710 struct pipe_inode_info *pipe = inode->i_pipe;
711 int retval;
713 mutex_lock(&inode->i_mutex);
714 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
715 if (retval >= 0) {
716 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
717 if (retval < 0) /* this can happen only if on == T */
718 fasync_helper(-1, filp, 0, &pipe->fasync_readers);
720 mutex_unlock(&inode->i_mutex);
722 if (retval < 0)
723 return retval;
725 return 0;
729 static int
730 pipe_read_release(struct inode *inode, struct file *filp)
732 return pipe_release(inode, 1, 0);
735 static int
736 pipe_write_release(struct inode *inode, struct file *filp)
738 return pipe_release(inode, 0, 1);
741 static int
742 pipe_rdwr_release(struct inode *inode, struct file *filp)
744 int decr, decw;
746 decr = (filp->f_mode & FMODE_READ) != 0;
747 decw = (filp->f_mode & FMODE_WRITE) != 0;
748 return pipe_release(inode, decr, decw);
751 static int
752 pipe_read_open(struct inode *inode, struct file *filp)
754 /* We could have perhaps used atomic_t, but this and friends
755 below are the only places. So it doesn't seem worthwhile. */
756 mutex_lock(&inode->i_mutex);
757 inode->i_pipe->readers++;
758 mutex_unlock(&inode->i_mutex);
760 return 0;
763 static int
764 pipe_write_open(struct inode *inode, struct file *filp)
766 mutex_lock(&inode->i_mutex);
767 inode->i_pipe->writers++;
768 mutex_unlock(&inode->i_mutex);
770 return 0;
773 static int
774 pipe_rdwr_open(struct inode *inode, struct file *filp)
776 mutex_lock(&inode->i_mutex);
777 if (filp->f_mode & FMODE_READ)
778 inode->i_pipe->readers++;
779 if (filp->f_mode & FMODE_WRITE)
780 inode->i_pipe->writers++;
781 mutex_unlock(&inode->i_mutex);
783 return 0;
787 * The file_operations structs are not static because they
788 * are also used in linux/fs/fifo.c to do operations on FIFOs.
790 * Pipes reuse fifos' file_operations structs.
792 const struct file_operations read_pipefifo_fops = {
793 .llseek = no_llseek,
794 .read = do_sync_read,
795 .aio_read = pipe_read,
796 .write = bad_pipe_w,
797 .poll = pipe_poll,
798 .unlocked_ioctl = pipe_ioctl,
799 .open = pipe_read_open,
800 .release = pipe_read_release,
801 .fasync = pipe_read_fasync,
804 const struct file_operations write_pipefifo_fops = {
805 .llseek = no_llseek,
806 .read = bad_pipe_r,
807 .write = do_sync_write,
808 .aio_write = pipe_write,
809 .poll = pipe_poll,
810 .unlocked_ioctl = pipe_ioctl,
811 .open = pipe_write_open,
812 .release = pipe_write_release,
813 .fasync = pipe_write_fasync,
816 const struct file_operations rdwr_pipefifo_fops = {
817 .llseek = no_llseek,
818 .read = do_sync_read,
819 .aio_read = pipe_read,
820 .write = do_sync_write,
821 .aio_write = pipe_write,
822 .poll = pipe_poll,
823 .unlocked_ioctl = pipe_ioctl,
824 .open = pipe_rdwr_open,
825 .release = pipe_rdwr_release,
826 .fasync = pipe_rdwr_fasync,
829 struct pipe_inode_info * alloc_pipe_info(struct inode *inode)
831 struct pipe_inode_info *pipe;
833 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
834 if (pipe) {
835 init_waitqueue_head(&pipe->wait);
836 pipe->r_counter = pipe->w_counter = 1;
837 pipe->inode = inode;
840 return pipe;
843 void __free_pipe_info(struct pipe_inode_info *pipe)
845 int i;
847 for (i = 0; i < PIPE_BUFFERS; i++) {
848 struct pipe_buffer *buf = pipe->bufs + i;
849 if (buf->ops)
850 buf->ops->release(pipe, buf);
852 if (pipe->tmp_page)
853 __free_page(pipe->tmp_page);
854 kfree(pipe);
857 void free_pipe_info(struct inode *inode)
859 __free_pipe_info(inode->i_pipe);
860 inode->i_pipe = NULL;
863 static struct vfsmount *pipe_mnt __read_mostly;
864 static int pipefs_delete_dentry(struct dentry *dentry)
867 * At creation time, we pretended this dentry was hashed
868 * (by clearing DCACHE_UNHASHED bit in d_flags)
869 * At delete time, we restore the truth : not hashed.
870 * (so that dput() can proceed correctly)
872 dentry->d_flags |= DCACHE_UNHASHED;
873 return 0;
877 * pipefs_dname() is called from d_path().
879 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
881 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
882 dentry->d_inode->i_ino);
885 static struct dentry_operations pipefs_dentry_operations = {
886 .d_delete = pipefs_delete_dentry,
887 .d_dname = pipefs_dname,
890 static struct inode * get_pipe_inode(void)
892 struct inode *inode = new_inode(pipe_mnt->mnt_sb);
893 struct pipe_inode_info *pipe;
895 if (!inode)
896 goto fail_inode;
898 pipe = alloc_pipe_info(inode);
899 if (!pipe)
900 goto fail_iput;
901 inode->i_pipe = pipe;
903 pipe->readers = pipe->writers = 1;
904 inode->i_fop = &rdwr_pipefifo_fops;
907 * Mark the inode dirty from the very beginning,
908 * that way it will never be moved to the dirty
909 * list because "mark_inode_dirty()" will think
910 * that it already _is_ on the dirty list.
912 inode->i_state = I_DIRTY;
913 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
914 inode->i_uid = current_fsuid();
915 inode->i_gid = current_fsgid();
916 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
918 return inode;
920 fail_iput:
921 iput(inode);
923 fail_inode:
924 return NULL;
927 struct file *create_write_pipe(int flags)
929 int err;
930 struct inode *inode;
931 struct file *f;
932 struct dentry *dentry;
933 struct qstr name = { .name = "" };
935 err = -ENFILE;
936 inode = get_pipe_inode();
937 if (!inode)
938 goto err;
940 err = -ENOMEM;
941 dentry = d_alloc(pipe_mnt->mnt_sb->s_root, &name);
942 if (!dentry)
943 goto err_inode;
945 dentry->d_op = &pipefs_dentry_operations;
947 * We dont want to publish this dentry into global dentry hash table.
948 * We pretend dentry is already hashed, by unsetting DCACHE_UNHASHED
949 * This permits a working /proc/$pid/fd/XXX on pipes
951 dentry->d_flags &= ~DCACHE_UNHASHED;
952 d_instantiate(dentry, inode);
954 err = -ENFILE;
955 f = alloc_file(pipe_mnt, dentry, FMODE_WRITE, &write_pipefifo_fops);
956 if (!f)
957 goto err_dentry;
958 f->f_mapping = inode->i_mapping;
960 f->f_flags = O_WRONLY | (flags & O_NONBLOCK);
961 f->f_version = 0;
963 return f;
965 err_dentry:
966 free_pipe_info(inode);
967 dput(dentry);
968 return ERR_PTR(err);
970 err_inode:
971 free_pipe_info(inode);
972 iput(inode);
973 err:
974 return ERR_PTR(err);
977 void free_write_pipe(struct file *f)
979 free_pipe_info(f->f_dentry->d_inode);
980 path_put(&f->f_path);
981 put_filp(f);
984 struct file *create_read_pipe(struct file *wrf, int flags)
986 struct file *f = get_empty_filp();
987 if (!f)
988 return ERR_PTR(-ENFILE);
990 /* Grab pipe from the writer */
991 f->f_path = wrf->f_path;
992 path_get(&wrf->f_path);
993 f->f_mapping = wrf->f_path.dentry->d_inode->i_mapping;
995 f->f_pos = 0;
996 f->f_flags = O_RDONLY | (flags & O_NONBLOCK);
997 f->f_op = &read_pipefifo_fops;
998 f->f_mode = FMODE_READ;
999 f->f_version = 0;
1001 return f;
1004 int do_pipe_flags(int *fd, int flags)
1006 struct file *fw, *fr;
1007 int error;
1008 int fdw, fdr;
1010 if (flags & ~(O_CLOEXEC | O_NONBLOCK))
1011 return -EINVAL;
1013 fw = create_write_pipe(flags);
1014 if (IS_ERR(fw))
1015 return PTR_ERR(fw);
1016 fr = create_read_pipe(fw, flags);
1017 error = PTR_ERR(fr);
1018 if (IS_ERR(fr))
1019 goto err_write_pipe;
1021 error = get_unused_fd_flags(flags);
1022 if (error < 0)
1023 goto err_read_pipe;
1024 fdr = error;
1026 error = get_unused_fd_flags(flags);
1027 if (error < 0)
1028 goto err_fdr;
1029 fdw = error;
1031 audit_fd_pair(fdr, fdw);
1032 fd_install(fdr, fr);
1033 fd_install(fdw, fw);
1034 fd[0] = fdr;
1035 fd[1] = fdw;
1037 return 0;
1039 err_fdr:
1040 put_unused_fd(fdr);
1041 err_read_pipe:
1042 path_put(&fr->f_path);
1043 put_filp(fr);
1044 err_write_pipe:
1045 free_write_pipe(fw);
1046 return error;
1049 int do_pipe(int *fd)
1051 return do_pipe_flags(fd, 0);
1055 * sys_pipe() is the normal C calling standard for creating
1056 * a pipe. It's not the way Unix traditionally does this, though.
1058 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
1060 int fd[2];
1061 int error;
1063 error = do_pipe_flags(fd, flags);
1064 if (!error) {
1065 if (copy_to_user(fildes, fd, sizeof(fd))) {
1066 sys_close(fd[0]);
1067 sys_close(fd[1]);
1068 error = -EFAULT;
1071 return error;
1074 SYSCALL_DEFINE1(pipe, int __user *, fildes)
1076 return sys_pipe2(fildes, 0);
1080 * pipefs should _never_ be mounted by userland - too much of security hassle,
1081 * no real gain from having the whole whorehouse mounted. So we don't need
1082 * any operations on the root directory. However, we need a non-trivial
1083 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1085 static int pipefs_get_sb(struct file_system_type *fs_type,
1086 int flags, const char *dev_name, void *data,
1087 struct vfsmount *mnt)
1089 return get_sb_pseudo(fs_type, "pipe:", NULL, PIPEFS_MAGIC, mnt);
1092 static struct file_system_type pipe_fs_type = {
1093 .name = "pipefs",
1094 .get_sb = pipefs_get_sb,
1095 .kill_sb = kill_anon_super,
1098 static int __init init_pipe_fs(void)
1100 int err = register_filesystem(&pipe_fs_type);
1102 if (!err) {
1103 pipe_mnt = kern_mount(&pipe_fs_type);
1104 if (IS_ERR(pipe_mnt)) {
1105 err = PTR_ERR(pipe_mnt);
1106 unregister_filesystem(&pipe_fs_type);
1109 return err;
1112 static void __exit exit_pipe_fs(void)
1114 unregister_filesystem(&pipe_fs_type);
1115 mntput(pipe_mnt);
1118 fs_initcall(init_pipe_fs);
1119 module_exit(exit_pipe_fs);