2 * Copyright (c) 1996 John S. Dyson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice immediately at the beginning of the file, without modification,
10 * this list of conditions, and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Absolutely no warranty of function or purpose is made by the author
16 * 4. Modifications may be freely made to this file if the above conditions
19 * $FreeBSD: src/sys/kern/sys_pipe.c,v 1.60.2.13 2002/08/05 15:05:15 des Exp $
20 * $DragonFly: src/sys/kern/sys_pipe.c,v 1.32 2005/09/02 07:16:58 hsu Exp $
24 * This file contains a high-performance replacement for the socket-based
25 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support
26 * all features of sockets, but does do everything that pipes normally
31 * This code has two modes of operation, a small write mode and a large
32 * write mode. The small write mode acts like conventional pipes with
33 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the
34 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT
35 * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and
36 * the receiving process can copy it directly from the pages in the sending
39 * If the sending process receives a signal, it is possible that it will
40 * go away, and certainly its address space can change, because control
41 * is returned back to the user-mode side. In that case, the pipe code
42 * arranges to copy the buffer supplied by the user process, to a pageable
43 * kernel buffer, and the receiving process will grab the data from the
44 * pageable kernel buffer. Since signals don't happen all that often,
45 * the copy operation is normally eliminated.
47 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
48 * happen for small transfers so that the system will not spend all of
49 * its time context switching. PIPE_SIZE is constrained by the
50 * amount of kernel virtual memory.
53 #include <sys/param.h>
54 #include <sys/systm.h>
55 #include <sys/kernel.h>
57 #include <sys/fcntl.h>
59 #include <sys/filedesc.h>
60 #include <sys/filio.h>
61 #include <sys/ttycom.h>
64 #include <sys/select.h>
65 #include <sys/signalvar.h>
66 #include <sys/sysproto.h>
68 #include <sys/vnode.h>
70 #include <sys/event.h>
71 #include <sys/globaldata.h>
72 #include <sys/module.h>
73 #include <sys/malloc.h>
74 #include <sys/sysctl.h>
75 #include <sys/socket.h>
78 #include <vm/vm_param.h>
80 #include <vm/vm_object.h>
81 #include <vm/vm_kern.h>
82 #include <vm/vm_extern.h>
84 #include <vm/vm_map.h>
85 #include <vm/vm_page.h>
86 #include <vm/vm_zone.h>
88 #include <sys/file2.h>
90 #include <machine/cpufunc.h>
93 * interfaces to the outside world
95 static int pipe_read (struct file
*fp
, struct uio
*uio
,
96 struct ucred
*cred
, int flags
, struct thread
*td
);
97 static int pipe_write (struct file
*fp
, struct uio
*uio
,
98 struct ucred
*cred
, int flags
, struct thread
*td
);
99 static int pipe_close (struct file
*fp
, struct thread
*td
);
100 static int pipe_shutdown (struct file
*fp
, int how
, struct thread
*td
);
101 static int pipe_poll (struct file
*fp
, int events
, struct ucred
*cred
,
103 static int pipe_kqfilter (struct file
*fp
, struct knote
*kn
);
104 static int pipe_stat (struct file
*fp
, struct stat
*sb
, struct thread
*td
);
105 static int pipe_ioctl (struct file
*fp
, u_long cmd
, caddr_t data
, struct thread
*td
);
107 static struct fileops pipeops
= {
110 pipe_read
, pipe_write
, pipe_ioctl
, pipe_poll
, pipe_kqfilter
,
111 pipe_stat
, pipe_close
, pipe_shutdown
114 static void filt_pipedetach(struct knote
*kn
);
115 static int filt_piperead(struct knote
*kn
, long hint
);
116 static int filt_pipewrite(struct knote
*kn
, long hint
);
118 static struct filterops pipe_rfiltops
=
119 { 1, NULL
, filt_pipedetach
, filt_piperead
};
120 static struct filterops pipe_wfiltops
=
121 { 1, NULL
, filt_pipedetach
, filt_pipewrite
};
123 MALLOC_DEFINE(M_PIPE
, "pipe", "pipe structures");
126 * Default pipe buffer size(s), this can be kind-of large now because pipe
127 * space is pageable. The pipe code will try to maintain locality of
128 * reference for performance reasons, so small amounts of outstanding I/O
129 * will not wipe the cache.
131 #define MINPIPESIZE (PIPE_SIZE/3)
132 #define MAXPIPESIZE (2*PIPE_SIZE/3)
135 * Maximum amount of kva for pipes -- this is kind-of a soft limit, but
136 * is there so that on large systems, we don't exhaust it.
138 #define MAXPIPEKVA (8*1024*1024)
141 * Limit for direct transfers, we cannot, of course limit
142 * the amount of kva for pipes in general though.
144 #define LIMITPIPEKVA (16*1024*1024)
147 * Limit the number of "big" pipes
149 #define LIMITBIGPIPES 32
150 #define PIPEQ_MAX_CACHE 16 /* per-cpu pipe structure cache */
152 static int pipe_maxbig
= LIMITBIGPIPES
;
153 static int pipe_maxcache
= PIPEQ_MAX_CACHE
;
154 static int pipe_nbig
;
155 static int pipe_bcache_alloc
;
156 static int pipe_bkmem_alloc
;
157 static int pipe_dwrite_enable
= 1; /* 0:copy, 1:kmem/sfbuf 2:force */
158 static int pipe_dwrite_sfbuf
= 1; /* 0:kmem_map 1:sfbufs 2:sfbufs_dmap */
159 /* 3:sfbuf_dmap w/ forced invlpg */
161 SYSCTL_NODE(_kern
, OID_AUTO
, pipe
, CTLFLAG_RW
, 0, "Pipe operation");
162 SYSCTL_INT(_kern_pipe
, OID_AUTO
, nbig
,
163 CTLFLAG_RD
, &pipe_nbig
, 0, "numer of big pipes allocated");
164 SYSCTL_INT(_kern_pipe
, OID_AUTO
, maxcache
,
165 CTLFLAG_RW
, &pipe_maxcache
, 0, "max pipes cached per-cpu");
166 SYSCTL_INT(_kern_pipe
, OID_AUTO
, maxbig
,
167 CTLFLAG_RW
, &pipe_maxbig
, 0, "max number of big pipes");
168 SYSCTL_INT(_kern_pipe
, OID_AUTO
, dwrite_enable
,
169 CTLFLAG_RW
, &pipe_dwrite_enable
, 0, "1:enable/2:force direct writes");
170 SYSCTL_INT(_kern_pipe
, OID_AUTO
, dwrite_sfbuf
,
171 CTLFLAG_RW
, &pipe_dwrite_sfbuf
, 0,
172 "(if dwrite_enable) 0:kmem 1:sfbuf 2:sfbuf_dmap 3:sfbuf_dmap_forceinvlpg");
173 #if !defined(NO_PIPE_SYSCTL_STATS)
174 SYSCTL_INT(_kern_pipe
, OID_AUTO
, bcache_alloc
,
175 CTLFLAG_RW
, &pipe_bcache_alloc
, 0, "pipe buffer from pcpu cache");
176 SYSCTL_INT(_kern_pipe
, OID_AUTO
, bkmem_alloc
,
177 CTLFLAG_RW
, &pipe_bkmem_alloc
, 0, "pipe buffer from kmem");
180 static void pipeclose (struct pipe
*cpipe
);
181 static void pipe_free_kmem (struct pipe
*cpipe
);
182 static int pipe_create (struct pipe
**cpipep
);
183 static __inline
int pipelock (struct pipe
*cpipe
, int catch);
184 static __inline
void pipeunlock (struct pipe
*cpipe
);
185 static __inline
void pipeselwakeup (struct pipe
*cpipe
);
186 #ifndef PIPE_NODIRECT
187 static int pipe_build_write_buffer (struct pipe
*wpipe
, struct uio
*uio
);
188 static int pipe_direct_write (struct pipe
*wpipe
, struct uio
*uio
);
189 static void pipe_clone_write_buffer (struct pipe
*wpipe
);
191 static int pipespace (struct pipe
*cpipe
, int size
);
194 * The pipe system call for the DTYPE_PIPE type of pipes
196 * pipe_ARgs(int dummy)
201 pipe(struct pipe_args
*uap
)
203 struct thread
*td
= curthread
;
204 struct proc
*p
= td
->td_proc
;
205 struct filedesc
*fdp
;
206 struct file
*rf
, *wf
;
207 struct pipe
*rpipe
, *wpipe
;
213 rpipe
= wpipe
= NULL
;
214 if (pipe_create(&rpipe
) || pipe_create(&wpipe
)) {
220 rpipe
->pipe_state
|= PIPE_DIRECTOK
;
221 wpipe
->pipe_state
|= PIPE_DIRECTOK
;
224 * Select the direct-map features to use for this pipe. Since the
225 * sysctl's can change on the fly we record the settings when the
228 * Generally speaking the system will default to what we consider
229 * to be the best-balanced and most stable option. Right now this
230 * is SFBUF1. Modes 2 and 3 are considered experiemental at the
233 wpipe
->pipe_feature
= PIPE_COPY
;
234 if (pipe_dwrite_enable
) {
235 switch(pipe_dwrite_sfbuf
) {
237 wpipe
->pipe_feature
= PIPE_KMEM
;
240 wpipe
->pipe_feature
= PIPE_SFBUF1
;
244 wpipe
->pipe_feature
= PIPE_SFBUF2
;
248 rpipe
->pipe_feature
= wpipe
->pipe_feature
;
250 error
= falloc(p
, &rf
, &fd1
);
256 uap
->sysmsg_fds
[0] = fd1
;
259 * Warning: once we've gotten past allocation of the fd for the
260 * read-side, we can only drop the read side via fdrop() in order
261 * to avoid races against processes which manage to dup() the read
262 * side while we are blocked trying to allocate the write side.
264 rf
->f_type
= DTYPE_PIPE
;
265 rf
->f_flag
= FREAD
| FWRITE
;
266 rf
->f_ops
= &pipeops
;
268 error
= falloc(p
, &wf
, &fd2
);
270 if (fdp
->fd_files
[fd1
].fp
== rf
) {
275 /* rpipe has been closed by fdrop(). */
279 wf
->f_type
= DTYPE_PIPE
;
280 wf
->f_flag
= FREAD
| FWRITE
;
281 wf
->f_ops
= &pipeops
;
283 uap
->sysmsg_fds
[1] = fd2
;
285 rpipe
->pipe_peer
= wpipe
;
286 wpipe
->pipe_peer
= rpipe
;
294 * Allocate kva for pipe circular buffer, the space is pageable
295 * This routine will 'realloc' the size of a pipe safely, if it fails
296 * it will retain the old buffer.
297 * If it fails it will return ENOMEM.
300 pipespace(struct pipe
*cpipe
, int size
)
302 struct vm_object
*object
;
306 npages
= round_page(size
) / PAGE_SIZE
;
307 object
= cpipe
->pipe_buffer
.object
;
310 * [re]create the object if necessary and reserve space for it
311 * in the kernel_map. The object and memory are pageable. On
312 * success, free the old resources before assigning the new
315 if (object
== NULL
|| object
->size
!= npages
) {
316 object
= vm_object_allocate(OBJT_DEFAULT
, npages
);
317 buffer
= (caddr_t
) vm_map_min(kernel_map
);
319 error
= vm_map_find(kernel_map
, object
, 0,
320 (vm_offset_t
*) &buffer
, size
, 1,
321 VM_PROT_ALL
, VM_PROT_ALL
, 0);
323 if (error
!= KERN_SUCCESS
) {
324 vm_object_deallocate(object
);
327 pipe_free_kmem(cpipe
);
328 cpipe
->pipe_buffer
.object
= object
;
329 cpipe
->pipe_buffer
.buffer
= buffer
;
330 cpipe
->pipe_buffer
.size
= size
;
335 cpipe
->pipe_buffer
.in
= 0;
336 cpipe
->pipe_buffer
.out
= 0;
337 cpipe
->pipe_buffer
.cnt
= 0;
342 * Initialize and allocate VM and memory for pipe, pulling the pipe from
343 * our per-cpu cache if possible. For now make sure it is sized for the
344 * smaller PIPE_SIZE default.
348 struct pipe
**cpipep
;
350 globaldata_t gd
= mycpu
;
354 if ((cpipe
= gd
->gd_pipeq
) != NULL
) {
355 gd
->gd_pipeq
= cpipe
->pipe_peer
;
357 cpipe
->pipe_peer
= NULL
;
359 cpipe
= malloc(sizeof(struct pipe
), M_PIPE
, M_WAITOK
|M_ZERO
);
362 if ((error
= pipespace(cpipe
, PIPE_SIZE
)) != 0)
364 vfs_timestamp(&cpipe
->pipe_ctime
);
365 cpipe
->pipe_atime
= cpipe
->pipe_ctime
;
366 cpipe
->pipe_mtime
= cpipe
->pipe_ctime
;
372 * lock a pipe for I/O, blocking other access
375 pipelock(cpipe
, catch)
381 while (cpipe
->pipe_state
& PIPE_LOCK
) {
382 cpipe
->pipe_state
|= PIPE_LWANT
;
383 error
= tsleep(cpipe
, (catch ? PCATCH
: 0), "pipelk", 0);
387 cpipe
->pipe_state
|= PIPE_LOCK
;
392 * unlock a pipe I/O lock
399 cpipe
->pipe_state
&= ~PIPE_LOCK
;
400 if (cpipe
->pipe_state
& PIPE_LWANT
) {
401 cpipe
->pipe_state
&= ~PIPE_LWANT
;
411 if (cpipe
->pipe_state
& PIPE_SEL
) {
412 cpipe
->pipe_state
&= ~PIPE_SEL
;
413 selwakeup(&cpipe
->pipe_sel
);
415 if ((cpipe
->pipe_state
& PIPE_ASYNC
) && cpipe
->pipe_sigio
)
416 pgsigio(cpipe
->pipe_sigio
, SIGIO
, 0);
417 KNOTE(&cpipe
->pipe_sel
.si_note
, 0);
422 pipe_read(struct file
*fp
, struct uio
*uio
, struct ucred
*cred
,
423 int flags
, struct thread
*td
)
425 struct pipe
*rpipe
= (struct pipe
*) fp
->f_data
;
431 error
= pipelock(rpipe
, 1);
435 while (uio
->uio_resid
) {
438 if (rpipe
->pipe_buffer
.cnt
> 0) {
440 * normal pipe buffer receive
442 size
= rpipe
->pipe_buffer
.size
- rpipe
->pipe_buffer
.out
;
443 if (size
> rpipe
->pipe_buffer
.cnt
)
444 size
= rpipe
->pipe_buffer
.cnt
;
445 if (size
> (u_int
) uio
->uio_resid
)
446 size
= (u_int
) uio
->uio_resid
;
448 error
= uiomove(&rpipe
->pipe_buffer
.buffer
[rpipe
->pipe_buffer
.out
],
453 rpipe
->pipe_buffer
.out
+= size
;
454 if (rpipe
->pipe_buffer
.out
>= rpipe
->pipe_buffer
.size
)
455 rpipe
->pipe_buffer
.out
= 0;
457 rpipe
->pipe_buffer
.cnt
-= size
;
460 * If there is no more to read in the pipe, reset
461 * its pointers to the beginning. This improves
464 if (rpipe
->pipe_buffer
.cnt
== 0) {
465 rpipe
->pipe_buffer
.in
= 0;
466 rpipe
->pipe_buffer
.out
= 0;
469 #ifndef PIPE_NODIRECT
470 } else if (rpipe
->pipe_kva
&&
471 rpipe
->pipe_feature
== PIPE_KMEM
&&
472 (rpipe
->pipe_state
& (PIPE_DIRECTW
|PIPE_DIRECTIP
))
476 * Direct copy using source-side kva mapping
478 size
= rpipe
->pipe_map
.xio_bytes
-
479 rpipe
->pipe_buffer
.out
;
480 if (size
> (u_int
)uio
->uio_resid
)
481 size
= (u_int
)uio
->uio_resid
;
482 va
= (caddr_t
)rpipe
->pipe_kva
+
483 xio_kvaoffset(&rpipe
->pipe_map
, rpipe
->pipe_buffer
.out
);
484 error
= uiomove(va
, size
, uio
);
488 rpipe
->pipe_buffer
.out
+= size
;
489 if (rpipe
->pipe_buffer
.out
== rpipe
->pipe_map
.xio_bytes
) {
490 rpipe
->pipe_state
|= PIPE_DIRECTIP
;
491 rpipe
->pipe_state
&= ~PIPE_DIRECTW
;
492 /* reset out index for copy mode */
493 rpipe
->pipe_buffer
.out
= 0;
496 } else if (rpipe
->pipe_buffer
.out
!= rpipe
->pipe_map
.xio_bytes
&&
498 rpipe
->pipe_feature
== PIPE_SFBUF2
&&
499 (rpipe
->pipe_state
& (PIPE_DIRECTW
|PIPE_DIRECTIP
))
503 * Direct copy, bypassing a kernel buffer. We cannot
504 * mess with the direct-write buffer until
505 * PIPE_DIRECTIP is cleared. In order to prevent
506 * the pipe_write code from racing itself in
507 * direct_write, we set DIRECTIP when we clear
508 * DIRECTW after we have exhausted the buffer.
510 if (pipe_dwrite_sfbuf
== 3)
511 rpipe
->pipe_kvamask
= 0;
512 pmap_qenter2(rpipe
->pipe_kva
, rpipe
->pipe_map
.xio_pages
,
513 rpipe
->pipe_map
.xio_npages
,
514 &rpipe
->pipe_kvamask
);
515 size
= rpipe
->pipe_map
.xio_bytes
-
516 rpipe
->pipe_buffer
.out
;
517 if (size
> (u_int
)uio
->uio_resid
)
518 size
= (u_int
)uio
->uio_resid
;
519 va
= (caddr_t
)rpipe
->pipe_kva
+ xio_kvaoffset(&rpipe
->pipe_map
, rpipe
->pipe_buffer
.out
);
520 error
= uiomove(va
, size
, uio
);
524 rpipe
->pipe_buffer
.out
+= size
;
525 if (rpipe
->pipe_buffer
.out
== rpipe
->pipe_map
.xio_bytes
) {
526 rpipe
->pipe_state
|= PIPE_DIRECTIP
;
527 rpipe
->pipe_state
&= ~PIPE_DIRECTW
;
528 /* reset out index for copy mode */
529 rpipe
->pipe_buffer
.out
= 0;
532 } else if (rpipe
->pipe_buffer
.out
!= rpipe
->pipe_map
.xio_bytes
&&
533 rpipe
->pipe_feature
== PIPE_SFBUF1
&&
534 (rpipe
->pipe_state
& (PIPE_DIRECTW
|PIPE_DIRECTIP
))
538 * Direct copy, bypassing a kernel buffer. We cannot
539 * mess with the direct-write buffer until
540 * PIPE_DIRECTIP is cleared. In order to prevent
541 * the pipe_write code from racing itself in
542 * direct_write, we set DIRECTIP when we clear
543 * DIRECTW after we have exhausted the buffer.
545 error
= xio_uio_copy(&rpipe
->pipe_map
, rpipe
->pipe_buffer
.out
, uio
, &size
);
549 rpipe
->pipe_buffer
.out
+= size
;
550 if (rpipe
->pipe_buffer
.out
== rpipe
->pipe_map
.xio_bytes
) {
551 rpipe
->pipe_state
|= PIPE_DIRECTIP
;
552 rpipe
->pipe_state
&= ~PIPE_DIRECTW
;
553 /* reset out index for copy mode */
554 rpipe
->pipe_buffer
.out
= 0;
560 * detect EOF condition
561 * read returns 0 on EOF, no need to set error
563 if (rpipe
->pipe_state
& PIPE_EOF
)
567 * If the "write-side" has been blocked, wake it up now.
569 if (rpipe
->pipe_state
& PIPE_WANTW
) {
570 rpipe
->pipe_state
&= ~PIPE_WANTW
;
575 * Break if some data was read.
581 * Unlock the pipe buffer for our remaining
582 * processing. We will either break out with an
583 * error or we will sleep and relock to loop.
588 * Handle non-blocking mode operation or
589 * wait for more data.
591 if (fp
->f_flag
& FNONBLOCK
) {
594 rpipe
->pipe_state
|= PIPE_WANTR
;
595 if ((error
= tsleep(rpipe
, PCATCH
|PNORESCHED
,
596 "piperd", 0)) == 0) {
597 error
= pipelock(rpipe
, 1);
607 vfs_timestamp(&rpipe
->pipe_atime
);
612 * PIPE_WANT processing only makes sense if pipe_busy is 0.
614 if ((rpipe
->pipe_busy
== 0) && (rpipe
->pipe_state
& PIPE_WANT
)) {
615 rpipe
->pipe_state
&= ~(PIPE_WANT
|PIPE_WANTW
);
617 } else if (rpipe
->pipe_buffer
.cnt
< MINPIPESIZE
) {
619 * Handle write blocking hysteresis.
621 if (rpipe
->pipe_state
& PIPE_WANTW
) {
622 rpipe
->pipe_state
&= ~PIPE_WANTW
;
627 if ((rpipe
->pipe_buffer
.size
- rpipe
->pipe_buffer
.cnt
) >= PIPE_BUF
)
628 pipeselwakeup(rpipe
);
632 #ifndef PIPE_NODIRECT
634 * Map the sending processes' buffer into kernel space and wire it.
635 * This is similar to a physical write operation.
638 pipe_build_write_buffer(wpipe
, uio
)
645 size
= (u_int
) uio
->uio_iov
->iov_len
;
646 if (size
> wpipe
->pipe_buffer
.size
)
647 size
= wpipe
->pipe_buffer
.size
;
649 if (uio
->uio_segflg
== UIO_SYSSPACE
) {
650 error
= xio_init_kbuf(&wpipe
->pipe_map
, uio
->uio_iov
->iov_base
,
653 error
= xio_init_ubuf(&wpipe
->pipe_map
, uio
->uio_iov
->iov_base
,
656 wpipe
->pipe_buffer
.out
= 0;
661 * Create a kernel map for KMEM and SFBUF2 copy modes. SFBUF2 will
662 * map the pages on the target while KMEM maps the pages now.
664 switch(wpipe
->pipe_feature
) {
667 if (wpipe
->pipe_kva
== NULL
) {
669 kmem_alloc_nofault(kernel_map
, XIO_INTERNAL_SIZE
);
670 wpipe
->pipe_kvamask
= 0;
672 if (wpipe
->pipe_feature
== PIPE_KMEM
) {
673 pmap_qenter(wpipe
->pipe_kva
, wpipe
->pipe_map
.xio_pages
,
674 wpipe
->pipe_map
.xio_npages
);
682 * And update the uio data. The XIO might have loaded fewer bytes
683 * then requested so reload 'size'.
685 size
= wpipe
->pipe_map
.xio_bytes
;
686 uio
->uio_iov
->iov_len
-= size
;
687 uio
->uio_iov
->iov_base
+= size
;
688 if (uio
->uio_iov
->iov_len
== 0)
690 uio
->uio_resid
-= size
;
691 uio
->uio_offset
+= size
;
696 * In the case of a signal, the writing process might go away. This
697 * code copies the data into the circular buffer so that the source
698 * pages can be freed without loss of data.
700 * Note that in direct mode pipe_buffer.out is used to track the
701 * XIO offset. We are converting the direct mode into buffered mode
702 * which changes the meaning of pipe_buffer.out.
705 pipe_clone_write_buffer(wpipe
)
711 offset
= wpipe
->pipe_buffer
.out
;
712 size
= wpipe
->pipe_map
.xio_bytes
- offset
;
714 KKASSERT(size
<= wpipe
->pipe_buffer
.size
);
716 wpipe
->pipe_buffer
.in
= size
;
717 wpipe
->pipe_buffer
.out
= 0;
718 wpipe
->pipe_buffer
.cnt
= size
;
719 wpipe
->pipe_state
&= ~(PIPE_DIRECTW
| PIPE_DIRECTIP
);
721 xio_copy_xtok(&wpipe
->pipe_map
, offset
, wpipe
->pipe_buffer
.buffer
, size
);
722 xio_release(&wpipe
->pipe_map
);
723 if (wpipe
->pipe_kva
) {
724 pmap_qremove(wpipe
->pipe_kva
, XIO_INTERNAL_PAGES
);
725 kmem_free(kernel_map
, wpipe
->pipe_kva
, XIO_INTERNAL_SIZE
);
726 wpipe
->pipe_kva
= NULL
;
731 * This implements the pipe buffer write mechanism. Note that only
732 * a direct write OR a normal pipe write can be pending at any given time.
733 * If there are any characters in the pipe buffer, the direct write will
734 * be deferred until the receiving process grabs all of the bytes from
735 * the pipe buffer. Then the direct mapping write is set-up.
738 pipe_direct_write(wpipe
, uio
)
745 while (wpipe
->pipe_state
& (PIPE_DIRECTW
|PIPE_DIRECTIP
)) {
746 if (wpipe
->pipe_state
& PIPE_WANTR
) {
747 wpipe
->pipe_state
&= ~PIPE_WANTR
;
750 wpipe
->pipe_state
|= PIPE_WANTW
;
751 error
= tsleep(wpipe
, PCATCH
, "pipdww", 0);
754 if (wpipe
->pipe_state
& PIPE_EOF
) {
759 KKASSERT(wpipe
->pipe_map
.xio_bytes
== 0);
760 if (wpipe
->pipe_buffer
.cnt
> 0) {
761 if (wpipe
->pipe_state
& PIPE_WANTR
) {
762 wpipe
->pipe_state
&= ~PIPE_WANTR
;
766 wpipe
->pipe_state
|= PIPE_WANTW
;
767 error
= tsleep(wpipe
, PCATCH
, "pipdwc", 0);
770 if (wpipe
->pipe_state
& PIPE_EOF
) {
778 * Build our direct-write buffer
780 wpipe
->pipe_state
|= PIPE_DIRECTW
| PIPE_DIRECTIP
;
781 error
= pipe_build_write_buffer(wpipe
, uio
);
784 wpipe
->pipe_state
&= ~PIPE_DIRECTIP
;
787 * Wait until the receiver has snarfed the data. Since we are likely
788 * going to sleep we optimize the case and yield synchronously,
789 * possibly avoiding the tsleep().
792 while (!error
&& (wpipe
->pipe_state
& PIPE_DIRECTW
)) {
793 if (wpipe
->pipe_state
& PIPE_EOF
) {
795 xio_release(&wpipe
->pipe_map
);
796 if (wpipe
->pipe_kva
) {
797 pmap_qremove(wpipe
->pipe_kva
, XIO_INTERNAL_PAGES
);
798 kmem_free(kernel_map
, wpipe
->pipe_kva
, XIO_INTERNAL_SIZE
);
799 wpipe
->pipe_kva
= NULL
;
802 pipeselwakeup(wpipe
);
806 if (wpipe
->pipe_state
& PIPE_WANTR
) {
807 wpipe
->pipe_state
&= ~PIPE_WANTR
;
810 pipeselwakeup(wpipe
);
811 error
= tsleep(wpipe
, PCATCH
|PNORESCHED
, "pipdwt", 0);
814 if (wpipe
->pipe_state
& PIPE_DIRECTW
) {
816 * this bit of trickery substitutes a kernel buffer for
817 * the process that might be going away.
819 pipe_clone_write_buffer(wpipe
);
820 KKASSERT((wpipe
->pipe_state
& PIPE_DIRECTIP
) == 0);
823 * note: The pipe_kva mapping is not qremove'd here. For
824 * legacy PIPE_KMEM mode this constitutes an improvement
825 * over the original FreeBSD-4 algorithm. For PIPE_SFBUF2
826 * mode the kva mapping must not be removed to get the
829 * For testing purposes we will give the original algorithm
830 * the benefit of the doubt 'what it could have been', and
831 * keep the optimization.
833 KKASSERT(wpipe
->pipe_state
& PIPE_DIRECTIP
);
834 xio_release(&wpipe
->pipe_map
);
835 wpipe
->pipe_state
&= ~PIPE_DIRECTIP
;
841 * Direct-write error, clear the direct write flags.
844 wpipe
->pipe_state
&= ~(PIPE_DIRECTW
| PIPE_DIRECTIP
);
848 * General error, wakeup the other side if it happens to be sleeping.
857 pipe_write(struct file
*fp
, struct uio
*uio
, struct ucred
*cred
,
858 int flags
, struct thread
*td
)
862 struct pipe
*wpipe
, *rpipe
;
864 rpipe
= (struct pipe
*) fp
->f_data
;
865 wpipe
= rpipe
->pipe_peer
;
868 * detect loss of pipe read side, issue SIGPIPE if lost.
870 if ((wpipe
== NULL
) || (wpipe
->pipe_state
& PIPE_EOF
)) {
876 * If it is advantageous to resize the pipe buffer, do
879 if ((uio
->uio_resid
> PIPE_SIZE
) &&
880 (pipe_nbig
< pipe_maxbig
) &&
881 (wpipe
->pipe_state
& (PIPE_DIRECTW
|PIPE_DIRECTIP
)) == 0 &&
882 (wpipe
->pipe_buffer
.size
<= PIPE_SIZE
) &&
883 (wpipe
->pipe_buffer
.cnt
== 0)) {
885 if ((error
= pipelock(wpipe
,1)) == 0) {
886 if (pipespace(wpipe
, BIG_PIPE_SIZE
) == 0)
893 * If an early error occured unbusy and return, waking up any pending
898 if ((wpipe
->pipe_busy
== 0) &&
899 (wpipe
->pipe_state
& PIPE_WANT
)) {
900 wpipe
->pipe_state
&= ~(PIPE_WANT
| PIPE_WANTR
);
906 KASSERT(wpipe
->pipe_buffer
.buffer
!= NULL
, ("pipe buffer gone"));
908 orig_resid
= uio
->uio_resid
;
910 while (uio
->uio_resid
) {
913 #ifndef PIPE_NODIRECT
915 * If the transfer is large, we can gain performance if
916 * we do process-to-process copies directly.
917 * If the write is non-blocking, we don't use the
918 * direct write mechanism.
920 * The direct write mechanism will detect the reader going
923 if ((uio
->uio_iov
->iov_len
>= PIPE_MINDIRECT
||
924 pipe_dwrite_enable
> 1) &&
925 (fp
->f_flag
& FNONBLOCK
) == 0 &&
926 pipe_dwrite_enable
) {
927 error
= pipe_direct_write( wpipe
, uio
);
935 * Pipe buffered writes cannot be coincidental with
936 * direct writes. We wait until the currently executing
937 * direct write is completed before we start filling the
938 * pipe buffer. We break out if a signal occurs or the
942 while (wpipe
->pipe_state
& (PIPE_DIRECTW
|PIPE_DIRECTIP
)) {
943 if (wpipe
->pipe_state
& PIPE_WANTR
) {
944 wpipe
->pipe_state
&= ~PIPE_WANTR
;
947 error
= tsleep(wpipe
, PCATCH
, "pipbww", 0);
948 if (wpipe
->pipe_state
& PIPE_EOF
)
953 if (wpipe
->pipe_state
& PIPE_EOF
) {
958 space
= wpipe
->pipe_buffer
.size
- wpipe
->pipe_buffer
.cnt
;
960 /* Writes of size <= PIPE_BUF must be atomic. */
961 if ((space
< uio
->uio_resid
) && (orig_resid
<= PIPE_BUF
))
965 * Write to fill, read size handles write hysteresis. Also
966 * additional restrictions can cause select-based non-blocking
970 if ((error
= pipelock(wpipe
,1)) == 0) {
971 int size
; /* Transfer size */
972 int segsize
; /* first segment to transfer */
975 * It is possible for a direct write to
976 * slip in on us... handle it here...
978 if (wpipe
->pipe_state
& (PIPE_DIRECTW
|PIPE_DIRECTIP
)) {
983 * If a process blocked in uiomove, our
984 * value for space might be bad.
986 * XXX will we be ok if the reader has gone
989 if (space
> wpipe
->pipe_buffer
.size
-
990 wpipe
->pipe_buffer
.cnt
) {
996 * Transfer size is minimum of uio transfer
997 * and free space in pipe buffer.
999 if (space
> uio
->uio_resid
)
1000 size
= uio
->uio_resid
;
1004 * First segment to transfer is minimum of
1005 * transfer size and contiguous space in
1006 * pipe buffer. If first segment to transfer
1007 * is less than the transfer size, we've got
1008 * a wraparound in the buffer.
1010 segsize
= wpipe
->pipe_buffer
.size
-
1011 wpipe
->pipe_buffer
.in
;
1015 /* Transfer first segment */
1017 error
= uiomove(&wpipe
->pipe_buffer
.buffer
[wpipe
->pipe_buffer
.in
],
1020 if (error
== 0 && segsize
< size
) {
1022 * Transfer remaining part now, to
1023 * support atomic writes. Wraparound
1026 if (wpipe
->pipe_buffer
.in
+ segsize
!=
1027 wpipe
->pipe_buffer
.size
)
1028 panic("Expected pipe buffer wraparound disappeared");
1030 error
= uiomove(&wpipe
->pipe_buffer
.buffer
[0],
1031 size
- segsize
, uio
);
1034 wpipe
->pipe_buffer
.in
+= size
;
1035 if (wpipe
->pipe_buffer
.in
>=
1036 wpipe
->pipe_buffer
.size
) {
1037 if (wpipe
->pipe_buffer
.in
!= size
- segsize
+ wpipe
->pipe_buffer
.size
)
1038 panic("Expected wraparound bad");
1039 wpipe
->pipe_buffer
.in
= size
- segsize
;
1042 wpipe
->pipe_buffer
.cnt
+= size
;
1043 if (wpipe
->pipe_buffer
.cnt
> wpipe
->pipe_buffer
.size
)
1044 panic("Pipe buffer overflow");
1054 * If the "read-side" has been blocked, wake it up now
1055 * and yield to let it drain synchronously rather
1058 if (wpipe
->pipe_state
& PIPE_WANTR
) {
1059 wpipe
->pipe_state
&= ~PIPE_WANTR
;
1064 * don't block on non-blocking I/O
1066 if (fp
->f_flag
& FNONBLOCK
) {
1072 * We have no more space and have something to offer,
1073 * wake up select/poll.
1075 pipeselwakeup(wpipe
);
1077 wpipe
->pipe_state
|= PIPE_WANTW
;
1078 error
= tsleep(wpipe
, PCATCH
|PNORESCHED
, "pipewr", 0);
1082 * If read side wants to go away, we just issue a signal
1085 if (wpipe
->pipe_state
& PIPE_EOF
) {
1094 if ((wpipe
->pipe_busy
== 0) && (wpipe
->pipe_state
& PIPE_WANT
)) {
1095 wpipe
->pipe_state
&= ~(PIPE_WANT
| PIPE_WANTR
);
1097 } else if (wpipe
->pipe_buffer
.cnt
> 0) {
1099 * If we have put any characters in the buffer, we wake up
1102 if (wpipe
->pipe_state
& PIPE_WANTR
) {
1103 wpipe
->pipe_state
&= ~PIPE_WANTR
;
1109 * Don't return EPIPE if I/O was successful
1111 if ((wpipe
->pipe_buffer
.cnt
== 0) &&
1112 (uio
->uio_resid
== 0) &&
1118 vfs_timestamp(&wpipe
->pipe_mtime
);
1121 * We have something to offer,
1122 * wake up select/poll.
1124 if (wpipe
->pipe_buffer
.cnt
)
1125 pipeselwakeup(wpipe
);
1131 * we implement a very minimal set of ioctls for compatibility with sockets.
1134 pipe_ioctl(struct file
*fp
, u_long cmd
, caddr_t data
, struct thread
*td
)
1136 struct pipe
*mpipe
= (struct pipe
*)fp
->f_data
;
1145 mpipe
->pipe_state
|= PIPE_ASYNC
;
1147 mpipe
->pipe_state
&= ~PIPE_ASYNC
;
1152 if (mpipe
->pipe_state
& PIPE_DIRECTW
) {
1153 *(int *)data
= mpipe
->pipe_map
.xio_bytes
-
1154 mpipe
->pipe_buffer
.out
;
1156 *(int *)data
= mpipe
->pipe_buffer
.cnt
;
1161 return (fsetown(*(int *)data
, &mpipe
->pipe_sigio
));
1164 *(int *)data
= fgetown(mpipe
->pipe_sigio
);
1167 /* This is deprecated, FIOSETOWN should be used instead. */
1169 return (fsetown(-(*(int *)data
), &mpipe
->pipe_sigio
));
1171 /* This is deprecated, FIOGETOWN should be used instead. */
1173 *(int *)data
= -fgetown(mpipe
->pipe_sigio
);
1181 pipe_poll(struct file
*fp
, int events
, struct ucred
*cred
, struct thread
*td
)
1183 struct pipe
*rpipe
= (struct pipe
*)fp
->f_data
;
1187 wpipe
= rpipe
->pipe_peer
;
1188 if (events
& (POLLIN
| POLLRDNORM
))
1189 if ((rpipe
->pipe_state
& PIPE_DIRECTW
) ||
1190 (rpipe
->pipe_buffer
.cnt
> 0) ||
1191 (rpipe
->pipe_state
& PIPE_EOF
))
1192 revents
|= events
& (POLLIN
| POLLRDNORM
);
1194 if (events
& (POLLOUT
| POLLWRNORM
))
1195 if (wpipe
== NULL
|| (wpipe
->pipe_state
& PIPE_EOF
) ||
1196 (((wpipe
->pipe_state
& PIPE_DIRECTW
) == 0) &&
1197 (wpipe
->pipe_buffer
.size
- wpipe
->pipe_buffer
.cnt
) >= PIPE_BUF
))
1198 revents
|= events
& (POLLOUT
| POLLWRNORM
);
1200 if ((rpipe
->pipe_state
& PIPE_EOF
) ||
1202 (wpipe
->pipe_state
& PIPE_EOF
))
1206 if (events
& (POLLIN
| POLLRDNORM
)) {
1207 selrecord(td
, &rpipe
->pipe_sel
);
1208 rpipe
->pipe_state
|= PIPE_SEL
;
1211 if (events
& (POLLOUT
| POLLWRNORM
)) {
1212 selrecord(td
, &wpipe
->pipe_sel
);
1213 wpipe
->pipe_state
|= PIPE_SEL
;
1221 pipe_stat(struct file
*fp
, struct stat
*ub
, struct thread
*td
)
1223 struct pipe
*pipe
= (struct pipe
*)fp
->f_data
;
1225 bzero((caddr_t
)ub
, sizeof(*ub
));
1226 ub
->st_mode
= S_IFIFO
;
1227 ub
->st_blksize
= pipe
->pipe_buffer
.size
;
1228 ub
->st_size
= pipe
->pipe_buffer
.cnt
;
1229 if (ub
->st_size
== 0 && (pipe
->pipe_state
& PIPE_DIRECTW
)) {
1230 ub
->st_size
= pipe
->pipe_map
.xio_bytes
-
1231 pipe
->pipe_buffer
.out
;
1233 ub
->st_blocks
= (ub
->st_size
+ ub
->st_blksize
- 1) / ub
->st_blksize
;
1234 ub
->st_atimespec
= pipe
->pipe_atime
;
1235 ub
->st_mtimespec
= pipe
->pipe_mtime
;
1236 ub
->st_ctimespec
= pipe
->pipe_ctime
;
1238 * Left as 0: st_dev, st_ino, st_nlink, st_uid, st_gid, st_rdev,
1240 * XXX (st_dev, st_ino) should be unique.
1247 pipe_close(struct file
*fp
, struct thread
*td
)
1249 struct pipe
*cpipe
= (struct pipe
*)fp
->f_data
;
1251 fp
->f_ops
= &badfileops
;
1253 funsetown(cpipe
->pipe_sigio
);
1259 * Shutdown one or both directions of a full-duplex pipe.
1263 pipe_shutdown(struct file
*fp
, int how
, struct thread
*td
)
1265 struct pipe
*rpipe
= (struct pipe
*)fp
->f_data
;
1273 rpipe
->pipe_state
|= PIPE_EOF
;
1274 pipeselwakeup(rpipe
);
1275 if (rpipe
->pipe_busy
)
1283 if (rpipe
&& (wpipe
= rpipe
->pipe_peer
) != NULL
) {
1284 wpipe
->pipe_state
|= PIPE_EOF
;
1285 pipeselwakeup(wpipe
);
1286 if (wpipe
->pipe_busy
)
1295 pipe_free_kmem(struct pipe
*cpipe
)
1297 if (cpipe
->pipe_buffer
.buffer
!= NULL
) {
1298 if (cpipe
->pipe_buffer
.size
> PIPE_SIZE
)
1300 kmem_free(kernel_map
,
1301 (vm_offset_t
)cpipe
->pipe_buffer
.buffer
,
1302 cpipe
->pipe_buffer
.size
);
1303 cpipe
->pipe_buffer
.buffer
= NULL
;
1304 cpipe
->pipe_buffer
.object
= NULL
;
1306 #ifndef PIPE_NODIRECT
1307 KKASSERT(cpipe
->pipe_map
.xio_bytes
== 0 &&
1308 cpipe
->pipe_map
.xio_offset
== 0 &&
1309 cpipe
->pipe_map
.xio_npages
== 0);
1317 pipeclose(struct pipe
*cpipe
)
1325 pipeselwakeup(cpipe
);
1328 * If the other side is blocked, wake it up saying that
1329 * we want to close it down.
1331 while (cpipe
->pipe_busy
) {
1333 cpipe
->pipe_state
|= PIPE_WANT
| PIPE_EOF
;
1334 tsleep(cpipe
, 0, "pipecl", 0);
1338 * Disconnect from peer
1340 if ((ppipe
= cpipe
->pipe_peer
) != NULL
) {
1341 pipeselwakeup(ppipe
);
1343 ppipe
->pipe_state
|= PIPE_EOF
;
1345 KNOTE(&ppipe
->pipe_sel
.si_note
, 0);
1346 ppipe
->pipe_peer
= NULL
;
1349 if (cpipe
->pipe_kva
) {
1350 pmap_qremove(cpipe
->pipe_kva
, XIO_INTERNAL_PAGES
);
1351 kmem_free(kernel_map
, cpipe
->pipe_kva
, XIO_INTERNAL_SIZE
);
1352 cpipe
->pipe_kva
= NULL
;
1356 * free or cache resources
1359 if (gd
->gd_pipeqcount
>= pipe_maxcache
||
1360 cpipe
->pipe_buffer
.size
!= PIPE_SIZE
1362 pipe_free_kmem(cpipe
);
1363 free(cpipe
, M_PIPE
);
1365 KKASSERT(cpipe
->pipe_map
.xio_npages
== 0 &&
1366 cpipe
->pipe_map
.xio_bytes
== 0 &&
1367 cpipe
->pipe_map
.xio_offset
== 0);
1368 cpipe
->pipe_state
= 0;
1369 cpipe
->pipe_busy
= 0;
1370 cpipe
->pipe_peer
= gd
->gd_pipeq
;
1371 gd
->gd_pipeq
= cpipe
;
1372 ++gd
->gd_pipeqcount
;
1378 pipe_kqfilter(struct file
*fp
, struct knote
*kn
)
1380 struct pipe
*cpipe
= (struct pipe
*)kn
->kn_fp
->f_data
;
1382 switch (kn
->kn_filter
) {
1384 kn
->kn_fop
= &pipe_rfiltops
;
1387 kn
->kn_fop
= &pipe_wfiltops
;
1388 cpipe
= cpipe
->pipe_peer
;
1390 /* other end of pipe has been closed */
1396 kn
->kn_hook
= (caddr_t
)cpipe
;
1398 SLIST_INSERT_HEAD(&cpipe
->pipe_sel
.si_note
, kn
, kn_selnext
);
1403 filt_pipedetach(struct knote
*kn
)
1405 struct pipe
*cpipe
= (struct pipe
*)kn
->kn_hook
;
1407 SLIST_REMOVE(&cpipe
->pipe_sel
.si_note
, kn
, knote
, kn_selnext
);
1412 filt_piperead(struct knote
*kn
, long hint
)
1414 struct pipe
*rpipe
= (struct pipe
*)kn
->kn_fp
->f_data
;
1415 struct pipe
*wpipe
= rpipe
->pipe_peer
;
1417 kn
->kn_data
= rpipe
->pipe_buffer
.cnt
;
1418 if ((kn
->kn_data
== 0) && (rpipe
->pipe_state
& PIPE_DIRECTW
)) {
1419 kn
->kn_data
= rpipe
->pipe_map
.xio_bytes
-
1420 rpipe
->pipe_buffer
.out
;
1423 if ((rpipe
->pipe_state
& PIPE_EOF
) ||
1424 (wpipe
== NULL
) || (wpipe
->pipe_state
& PIPE_EOF
)) {
1425 kn
->kn_flags
|= EV_EOF
;
1428 return (kn
->kn_data
> 0);
1433 filt_pipewrite(struct knote
*kn
, long hint
)
1435 struct pipe
*rpipe
= (struct pipe
*)kn
->kn_fp
->f_data
;
1436 struct pipe
*wpipe
= rpipe
->pipe_peer
;
1438 if ((wpipe
== NULL
) || (wpipe
->pipe_state
& PIPE_EOF
)) {
1440 kn
->kn_flags
|= EV_EOF
;
1443 kn
->kn_data
= wpipe
->pipe_buffer
.size
- wpipe
->pipe_buffer
.cnt
;
1444 if (wpipe
->pipe_state
& PIPE_DIRECTW
)
1447 return (kn
->kn_data
>= PIPE_BUF
);