2 * Copyright (c) 2005 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 1982, 1986, 1988, 1990, 1993
4 * The Regents of the University of California. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Neither the name of the University nor the names of its contributors
15 * may be used to endorse or promote products derived from this software
16 * without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93
31 * $FreeBSD: src/sys/kern/uipc_socket2.c,v 1.55.2.17 2002/08/31 19:04:55 dwmalone Exp $
34 #include "opt_param.h"
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/domain.h>
38 #include <sys/file.h> /* for maxfiles */
39 #include <sys/kernel.h>
41 #include <sys/malloc.h>
43 #include <sys/protosw.h>
44 #include <sys/resourcevar.h>
46 #include <sys/socket.h>
47 #include <sys/socketvar.h>
49 #include <sys/thread2.h>
50 #include <sys/msgport2.h>
53 * Routines to add and remove data from an mbuf queue.
55 * The routines sbappend() or sbappendrecord() are normally called to
56 * append new mbufs to a socket buffer. sbappendrecord() differs from
57 * sbappend() in that data supplied is treated as the beginning of a new
58 * record. sbappend() only begins a new record if the last mbuf in the
59 * sockbuf is marked M_EOR.
61 * To place a sender's address, optional access rights, and data in a
62 * socket receive buffer, sbappendaddr() or sbappendcontrol() should be
63 * used. These functions also begin a new record.
65 * Reliable protocols may use the socket send buffer to hold data
66 * awaiting acknowledgement. Data is normally copied from a socket
67 * send buffer in a protocol with m_copym for output to a peer,
68 * and then removing the data from the socket buffer with sbdrop()
69 * or sbdroprecord() when the data is acknowledged by the peer.
73 * Append mbuf chain m to the last record in the socket buffer sb.
74 * The additional space associated the mbuf chain is recorded in sb.
75 * Empty mbufs are discarded and mbufs are compacted where possible.
77 * If M_EOR is set in the first or last mbuf of the last record, the
78 * mbuf chain is appended as a new record. M_EOR is usually just set
79 * in the last mbuf of the last record's mbuf chain (see sbcompress()),
80 * but this may be changed in the future since there is no real need
81 * to propogate the flag any more.
84 sbappend(struct sockbuf
*sb
, struct mbuf
*m
)
91 n
= sb
->sb_lastrecord
;
93 if (n
->m_flags
& M_EOR
) {
94 sbappendrecord(sb
, m
);
100 if (n
->m_flags
& M_EOR
) {
101 sbappendrecord(sb
, m
);
105 sbcompress(sb
, m
, n
);
110 * sbappendstream() is an optimized form of sbappend() for protocols
111 * such as TCP that only have one record in the socket buffer, are
112 * not PR_ATOMIC, nor allow MT_CONTROL data. A protocol that uses
113 * sbappendstream() must use sbappendstream() exclusively.
116 sbappendstream(struct sockbuf
*sb
, struct mbuf
*m
)
119 KKASSERT(m
->m_nextpkt
== NULL
);
120 sbcompress(sb
, m
, sb
->sb_lastmbuf
);
126 _sbcheck(struct sockbuf
*sb
)
129 struct mbuf
*n
= NULL
;
130 u_long len
= 0, mbcnt
= 0;
132 for (m
= sb
->sb_mb
; m
; m
= n
) {
134 if (n
== NULL
&& sb
->sb_lastrecord
!= m
) {
135 kprintf("sockbuf %p mismatched lastrecord %p vs %p\n", sb
, sb
->sb_lastrecord
, m
);
138 for (; m
; m
= m
->m_next
) {
141 if (m
->m_flags
& M_EXT
) /*XXX*/ /* pretty sure this is bogus */
142 mbcnt
+= m
->m_ext
.ext_size
;
143 if (n
== NULL
&& m
->m_next
== NULL
) {
144 if (sb
->sb_lastmbuf
!= m
) {
145 kprintf("sockbuf %p mismatched lastmbuf %p vs %p\n", sb
, sb
->sb_lastmbuf
, m
);
151 if (sb
->sb_mb
== NULL
) {
152 if (sb
->sb_lastrecord
!= NULL
) {
153 kprintf("sockbuf %p is empty, lastrecord not NULL: %p\n",
154 sb
, sb
->sb_lastrecord
);
157 if (sb
->sb_lastmbuf
!= NULL
) {
158 kprintf("sockbuf %p is empty, lastmbuf not NULL: %p\n",
159 sb
, sb
->sb_lastmbuf
);
163 if (len
!= sb
->sb_cc
|| mbcnt
!= sb
->sb_mbcnt
) {
164 kprintf("sockbuf %p cc %ld != %ld || mbcnt %ld != %ld\n",
165 sb
, len
, sb
->sb_cc
, mbcnt
, sb
->sb_mbcnt
);
173 * Same as sbappend(), except the mbuf chain begins a new record.
176 sbappendrecord(struct sockbuf
*sb
, struct mbuf
*m0
)
178 struct mbuf
*firstmbuf
;
179 struct mbuf
*secondmbuf
;
188 * Break the first mbuf off from the rest of the mbuf chain.
191 secondmbuf
= m0
->m_next
;
195 * Insert the first mbuf of the m0 mbuf chain as the last record of
196 * the sockbuf. Note this permits zero length records! Keep the
197 * sockbuf state consistent.
199 if (sb
->sb_mb
== NULL
)
200 sb
->sb_mb
= firstmbuf
;
202 sb
->sb_lastrecord
->m_nextpkt
= firstmbuf
;
203 sb
->sb_lastrecord
= firstmbuf
; /* update hint for new last record */
204 sb
->sb_lastmbuf
= firstmbuf
; /* update hint for new last mbuf */
207 * propagate the EOR flag so sbcompress() can pick it up
209 if ((firstmbuf
->m_flags
& M_EOR
) && (secondmbuf
!= NULL
)) {
210 firstmbuf
->m_flags
&= ~M_EOR
;
211 secondmbuf
->m_flags
|= M_EOR
;
215 * The succeeding call to sbcompress() omits accounting for
216 * the first mbuf, so do it here.
218 sballoc(sb
, firstmbuf
);
220 /* Compact the rest of the mbuf chain in after the first mbuf. */
221 sbcompress(sb
, secondmbuf
, firstmbuf
);
225 * Append address and data, and optionally, control (ancillary) data
226 * to the receive queue of a socket. If present,
227 * m0 must include a packet header with total length.
228 * Returns 0 if insufficient mbufs.
231 sbappendaddr(struct sockbuf
*sb
, const struct sockaddr
*asa
, struct mbuf
*m0
,
232 struct mbuf
*control
)
238 mbuftrackid(control
, 20);
239 if (m0
&& (m0
->m_flags
& M_PKTHDR
) == 0)
240 panic("sbappendaddr");
243 for (n
= control
; n
; n
= n
->m_next
) {
244 if (n
->m_next
== NULL
) /* keep pointer to last control buf */
247 if (asa
->sa_len
> MLEN
)
249 MGET(m
, M_NOWAIT
, MT_SONAME
);
252 KKASSERT(m
->m_nextpkt
== NULL
);
253 m
->m_len
= asa
->sa_len
;
254 bcopy(asa
, mtod(m
, caddr_t
), asa
->sa_len
);
256 n
->m_next
= m0
; /* concatenate data to control */
260 for (n
= m
; n
; n
= n
->m_next
)
263 if (sb
->sb_mb
== NULL
)
266 sb
->sb_lastrecord
->m_nextpkt
= m
;
267 sb
->sb_lastrecord
= m
;
270 * Propogate M_EOR to the last mbuf and calculate sb_lastmbuf
271 * so sbappend() can find it.
275 m
->m_flags
&= ~M_EOR
;
279 m
->m_flags
|= eor
& M_EOR
;
286 * Append control information followed by data. Both the control and data
290 sbappendcontrol(struct sockbuf
*sb
, struct mbuf
*m0
, struct mbuf
*control
)
293 u_int length
, cmbcnt
, m0mbcnt
;
296 KASSERT(control
!= NULL
, ("sbappendcontrol"));
297 KKASSERT(control
->m_nextpkt
== NULL
);
301 mbuftrackid(control
, 22);
303 length
= m_countm(control
, &n
, &cmbcnt
) + m_countm(m0
, NULL
, &m0mbcnt
);
305 KKASSERT(m0
!= NULL
);
307 n
->m_next
= m0
; /* concatenate data to control */
309 if (sb
->sb_mb
== NULL
)
312 sb
->sb_lastrecord
->m_nextpkt
= control
;
313 sb
->sb_lastrecord
= control
;
316 * Propogate M_EOR to the last mbuf and calculate sb_lastmbuf
317 * so sbappend() can find it.
321 m0
->m_flags
&= ~M_EOR
;
325 m0
->m_flags
|= eor
& M_EOR
;
326 sb
->sb_lastmbuf
= m0
;
329 sb
->sb_mbcnt
+= cmbcnt
+ m0mbcnt
;
335 * Compress mbuf chain m into the socket buffer sb following mbuf tailm.
336 * If tailm is null, the buffer is presumed empty. Also, as a side-effect,
337 * increment the sockbuf counts for each mbuf in the chain.
340 sbcompress(struct sockbuf
*sb
, struct mbuf
*m
, struct mbuf
*tailm
)
343 struct mbuf
*free_chain
= NULL
;
351 eor
|= m
->m_flags
& M_EOR
;
353 * Disregard empty mbufs as long as we don't encounter
354 * an end-of-record or there is a trailing mbuf of
355 * the same type to propagate the EOR flag to.
357 * Defer the m_free() call because it can block and break
358 * the atomicy of the sockbuf.
362 (((o
= m
->m_next
) || (o
= tailm
)) &&
363 o
->m_type
== m
->m_type
))) {
365 m
->m_next
= free_chain
;
372 * See if we can coalesce with preceding mbuf. Never try
373 * to coalesce a mbuf representing an end-of-record or
374 * a mbuf locked by userland for reading.
376 if (tailm
&& !(tailm
->m_flags
& (M_EOR
| M_SOLOCKED
)) &&
378 m
->m_len
<= MCLBYTES
/ 4 && /* XXX: Don't copy too much */
379 m
->m_len
<= M_TRAILINGSPACE(tailm
) &&
380 tailm
->m_type
== m
->m_type
) {
383 bcopy(mtod(m
, caddr_t
),
384 mtod(tailm
, caddr_t
) + tailm
->m_len
,
386 tailm
->m_len
+= m
->m_len
;
388 sb
->sb_cc
+= m
->m_len
; /* update sb counter */
391 * Fix the wrongly updated mbcnt_prealloc
394 if (m
->m_flags
& M_EXT
)
395 mbcnt_sz
+= m
->m_ext
.ext_size
;
396 atomic_subtract_long(&sb
->sb_mbcnt_prealloc
, mbcnt_sz
);
399 m
->m_next
= free_chain
;
405 /* Insert whole mbuf. */
407 KASSERT(sb
->sb_mb
== NULL
,
408 ("sbcompress: sb_mb not NULL"));
409 sb
->sb_mb
= m
; /* only mbuf in sockbuf */
410 sb
->sb_lastrecord
= m
; /* new last record */
412 tailm
->m_next
= m
; /* tack m on following tailm */
414 sb
->sb_lastmbuf
= m
; /* update last mbuf hint */
416 tailm
= m
; /* just inserted mbuf becomes the new tail */
417 m
= m
->m_next
; /* advance to next mbuf */
418 tailm
->m_next
= NULL
; /* split inserted mbuf off from chain */
420 /* update sb counters for just added mbuf */
423 /* clear EOR on intermediate mbufs */
424 tailm
->m_flags
&= ~M_EOR
;
428 * Propogate EOR to the last mbuf
432 tailm
->m_flags
|= eor
;
434 kprintf("semi-panic: sbcompress");
438 * Clean up any defered frees.
441 free_chain
= m_free(free_chain
);
447 * Free all mbufs in a sockbuf.
448 * Check that all resources are reclaimed.
451 sbflush(struct sockbuf
*sb
)
453 while (sb
->sb_mbcnt
) {
455 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty:
456 * we would loop forever. Panic instead.
458 if (!sb
->sb_cc
&& (sb
->sb_mb
== NULL
|| sb
->sb_mb
->m_len
))
460 sbdrop(sb
, (int)sb
->sb_cc
);
462 KASSERT(!(sb
->sb_cc
|| sb
->sb_mb
|| sb
->sb_mbcnt
|| sb
->sb_lastmbuf
),
463 ("sbflush: cc %ld || mb %p || mbcnt %ld || lastmbuf %p",
464 sb
->sb_cc
, sb
->sb_mb
, sb
->sb_mbcnt
, sb
->sb_lastmbuf
));
468 * Drop data from (the front of) a sockbuf. If the current record is
469 * exhausted this routine will move onto the next one and continue dropping
473 sbdrop(struct sockbuf
*sb
, int len
)
476 struct mbuf
*free_chain
= NULL
;
482 while (m
&& len
> 0) {
483 if (m
->m_len
> len
) {
487 atomic_subtract_long(&sb
->sb_cc_prealloc
, len
);
491 m
= sbunlinkmbuf(sb
, m
, &free_chain
);
492 if (m
== NULL
&& len
)
497 * Remove any trailing 0-length mbufs in the current record. If
498 * the last record for which data was removed is now empty, m will be
501 while (m
&& m
->m_len
== 0) {
502 m
= sbunlinkmbuf(sb
, m
, &free_chain
);
511 * Drop a record off the front of a sockbuf and move the next record
514 * Must be called while holding a critical section.
517 sbdroprecord(struct sockbuf
*sb
)
525 if ((sb
->sb_mb
= m
->m_nextpkt
) == NULL
) {
526 sb
->sb_lastrecord
= NULL
;
527 sb
->sb_lastmbuf
= NULL
;
530 for (n
= m
; n
; n
= n
->m_next
)
538 * Drop the first mbuf off the sockbuf and move the next mbuf to the front.
539 * Currently only the head mbuf of the sockbuf may be dropped this way.
541 * The next mbuf in the same record as the mbuf being removed is returned
542 * or NULL if the record is exhausted. Note that other records may remain
543 * in the sockbuf when NULL is returned.
545 * Must be called while holding a critical section.
548 sbunlinkmbuf(struct sockbuf
*sb
, struct mbuf
*m
, struct mbuf
**free_chain
)
552 KKASSERT(sb
->sb_mb
== m
);
557 if (sb
->sb_lastrecord
== m
)
558 sb
->sb_lastrecord
= n
;
559 KKASSERT(sb
->sb_lastmbuf
!= m
);
560 n
->m_nextpkt
= m
->m_nextpkt
;
562 sb
->sb_mb
= m
->m_nextpkt
;
563 if (sb
->sb_lastrecord
== m
) {
564 KKASSERT(sb
->sb_mb
== NULL
);
565 sb
->sb_lastrecord
= NULL
;
567 if (sb
->sb_mb
== NULL
)
568 sb
->sb_lastmbuf
= NULL
;
572 m
->m_next
= *free_chain
;
581 * Create a "control" mbuf containing the specified data
582 * with the specified type for presentation on a socket buffer.
585 sbcreatecontrol(const void *p
, size_t size
, int type
, int level
)
590 if (CMSG_SPACE(size
) > MCLBYTES
)
592 m
= m_getl(CMSG_SPACE(size
), M_NOWAIT
, MT_CONTROL
, 0, NULL
);
595 m
->m_len
= CMSG_SPACE(size
);
596 cp
= mtod(m
, struct cmsghdr
*);
598 memcpy(CMSG_DATA(cp
), p
, size
);
599 cp
->cmsg_len
= CMSG_LEN(size
);
600 cp
->cmsg_level
= level
;
601 cp
->cmsg_type
= type
;