2 * Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * $FreeBSD: src/sys/netipsec/ipsec_mbuf.c,v 1.5.2.2 2003/03/28 20:32:53 sam Exp $
30 * IPsec-specific mbuf routines.
33 #include "opt_param.h"
35 #include <sys/param.h>
36 #include <sys/systm.h>
38 #include <sys/socket.h>
40 #include <net/route.h>
41 #include <netinet/in.h>
43 #include <netproto/ipsec/ipsec.h>
46 * Create a writable copy of the mbuf chain. While doing this
47 * we compact the chain with a goal of producing a chain with
48 * at most two mbufs. The second mbuf in this chain is likely
49 * to be a cluster. The primary purpose of this work is to create
50 * a writable packet for encryption, compression, etc. The
51 * secondary goal is to linearize the data so the data can be
52 * passed to crypto hardware in the most efficient manner possible.
55 m_clone(struct mbuf
*m0
)
57 struct mbuf
*m
, *mprev
;
58 struct mbuf
*n
, *mfirst
, *mlast
;
61 KASSERT(m0
!= NULL
, ("m_clone: null mbuf"));
64 for (m
= m0
; m
!= NULL
; m
= mprev
->m_next
) {
66 * Regular mbufs are ignored unless there's a cluster
67 * in front of it that we can use to coalesce. We do
68 * the latter mainly so later clusters can be coalesced
69 * also w/o having to handle them specially (i.e. convert
70 * mbuf+cluster -> cluster). This optimization is heavily
71 * influenced by the assumption that we're running over
72 * Ethernet where MCLBYTES is large enough that the max
73 * packet size will permit lots of coalescing into a
74 * single cluster. This in turn permits efficient
75 * crypto operations, especially when using hardware.
77 if ((m
->m_flags
& M_EXT
) == 0) {
78 if (mprev
&& (mprev
->m_flags
& M_EXT
) &&
79 m
->m_len
<= M_TRAILINGSPACE(mprev
)) {
80 /* XXX: this ignores mbuf types */
81 memcpy(mtod(mprev
, caddr_t
) + mprev
->m_len
,
82 mtod(m
, caddr_t
), m
->m_len
);
83 mprev
->m_len
+= m
->m_len
;
84 mprev
->m_next
= m
->m_next
; /* unlink from chain */
85 m_free(m
); /* reclaim mbuf */
86 newipsecstat
.ips_mbcoalesced
++;
93 * Writable mbufs are left alone (for now). Note
94 * that for 4.x systems it's not possible to identify
95 * whether or not mbufs with external buffers are
96 * writable unless they use clusters.
98 if (M_EXT_WRITABLE(m
)) {
104 * Not writable, replace with a copy or coalesce with
105 * the previous mbuf if possible (since we have to copy
106 * it anyway, we try to reduce the number of mbufs and
107 * clusters so that future work is easier).
109 KASSERT(m
->m_flags
& M_EXT
,
110 ("m_clone: m_flags 0x%x", m
->m_flags
));
111 /* NB: we only coalesce into a cluster or larger */
112 if (mprev
!= NULL
&& (mprev
->m_flags
& M_EXT
) &&
113 m
->m_len
<= M_TRAILINGSPACE(mprev
)) {
114 /* XXX: this ignores mbuf types */
115 memcpy(mtod(mprev
, caddr_t
) + mprev
->m_len
,
116 mtod(m
, caddr_t
), m
->m_len
);
117 mprev
->m_len
+= m
->m_len
;
118 mprev
->m_next
= m
->m_next
; /* unlink from chain */
119 m_free(m
); /* reclaim mbuf */
120 newipsecstat
.ips_clcoalesced
++;
125 * Allocate new space to hold the copy...
127 /* XXX why can M_PKTHDR be set past the first mbuf? */
128 if (mprev
== NULL
&& (m
->m_flags
& M_PKTHDR
)) {
130 * NB: if a packet header is present we must
131 * allocate the mbuf separately from any cluster
132 * because M_MOVE_PKTHDR will smash the data
133 * pointer and drop the M_EXT marker.
135 MGETHDR(n
, M_NOWAIT
, m
->m_type
);
142 if ((n
->m_flags
& M_EXT
) == 0) {
148 n
= m_getcl(M_NOWAIT
, m
->m_type
, m
->m_flags
);
155 * ... and copy the data. We deal with jumbo mbufs
156 * (i.e. m_len > MCLBYTES) by splitting them into
157 * clusters. We could just malloc a buffer and make
158 * it external but too many device drivers don't know
159 * how to break up the non-contiguous memory when
167 int cc
= min(len
, MCLBYTES
);
168 memcpy(mtod(n
, caddr_t
), mtod(m
, caddr_t
) + off
, cc
);
173 newipsecstat
.ips_clcopied
++;
180 n
= m_getcl(M_NOWAIT
, m
->m_type
, m
->m_flags
);
187 n
->m_next
= m
->m_next
;
189 m0
= mfirst
; /* new head of chain */
191 mprev
->m_next
= mfirst
; /* replace old mbuf */
192 m_free(m
); /* release old mbuf */
199 * Make space for a new header of length hlen at skip bytes
200 * into the packet. When doing this we allocate new mbufs only
201 * when absolutely necessary. The mbuf where the new header
202 * is to go is returned together with an offset into the mbuf.
203 * If NULL is returned then the mbuf chain may have been modified;
204 * the caller is assumed to always free the chain.
207 m_makespace(struct mbuf
*m0
, int skip
, int hlen
, int *off
)
212 KASSERT(m0
!= NULL
, ("m_dmakespace: null mbuf"));
213 KASSERT(hlen
< MHLEN
, ("m_makespace: hlen too big: %u", hlen
));
215 for (m
= m0
; m
&& skip
> m
->m_len
; m
= m
->m_next
)
220 * At this point skip is the offset into the mbuf m
221 * where the new header should be placed. Figure out
222 * if there's space to insert the new header. If so,
223 * and copying the remainder makese sense then do so.
224 * Otherwise insert a new mbuf in the chain, splitting
225 * the contents of m as needed.
227 remain
= m
->m_len
- skip
; /* data to move */
228 if (hlen
> M_TRAILINGSPACE(m
)) {
231 /* XXX code doesn't handle clusters XXX */
232 KASSERT(remain
< MLEN
,
233 ("m_makespace: remainder too big: %u", remain
));
235 * Not enough space in m, split the contents
236 * of m, inserting new mbufs as required.
238 * NB: this ignores mbuf types.
240 MGET(n
, M_NOWAIT
, MT_DATA
);
243 n
->m_next
= m
->m_next
; /* splice new mbuf */
245 newipsecstat
.ips_mbinserted
++;
246 if (hlen
<= M_TRAILINGSPACE(m
) + remain
) {
248 * New header fits in the old mbuf if we copy
249 * the remainder; just do the copy to the new
250 * mbuf and we're good to go.
252 memcpy(mtod(n
, caddr_t
),
253 mtod(m
, caddr_t
) + skip
, remain
);
255 m
->m_len
= skip
+ hlen
;
259 * No space in the old mbuf for the new header.
260 * Make space in the new mbuf and check the
261 * remainder'd data fits too. If not then we
262 * must allocate an additional mbuf (yech).
265 if (remain
+ hlen
> M_TRAILINGSPACE(n
)) {
268 MGET(n2
, M_NOWAIT
, MT_DATA
);
269 /* NB: new mbuf is on chain, let caller free */
273 memcpy(mtod(n2
, caddr_t
),
274 mtod(m
, caddr_t
) + skip
, remain
);
276 /* splice in second mbuf */
277 n2
->m_next
= n
->m_next
;
279 newipsecstat
.ips_mbinserted
++;
281 memcpy(mtod(n
, caddr_t
) + hlen
,
282 mtod(m
, caddr_t
) + skip
, remain
);
287 m
= n
; /* header is at front ... */
288 *off
= 0; /* ... of new mbuf */
292 * Copy the remainder to the back of the mbuf
293 * so there's space to write the new header.
295 /* XXX can this be memcpy? does it handle overlap? */
296 bcopy(mtod(m
, caddr_t
) + skip
,
297 mtod(m
, caddr_t
) + skip
+ hlen
, remain
);
301 m0
->m_pkthdr
.len
+= hlen
; /* adjust packet length */
306 * m_pad(m, n) pads <m> with <n> bytes at the end. The packet header
307 * length is updated, and a pointer to the first byte of the padding
308 * (which is guaranteed to be all in one mbuf) is returned.
311 m_pad(struct mbuf
*m
, int n
)
313 struct mbuf
*m0
, *m1
;
317 if (n
<= 0) { /* No stupid arguments. */
318 DPRINTF(("m_pad: pad length invalid (%d)\n", n
));
323 len
= m
->m_pkthdr
.len
;
327 while (m0
->m_len
< len
) {
328 KASSERT(m0
->m_next
!= NULL
, ("m_pad: m0 null, len %u m_len %u", len
, m0
->m_len
));/*XXX*/
333 if (m0
->m_len
!= len
) {
334 DPRINTF(("m_pad: length mismatch (should be %d instead of %d)\n",
335 m
->m_pkthdr
.len
, m
->m_pkthdr
.len
+ m0
->m_len
- len
));
341 /* Check for zero-length trailing mbufs, and find the last one. */
342 for (m1
= m0
; m1
->m_next
; m1
= m1
->m_next
) {
343 if (m1
->m_next
->m_len
!= 0) {
344 DPRINTF(("m_pad: length mismatch (should be %d "
347 m
->m_pkthdr
.len
+ m1
->m_next
->m_len
));
356 if (pad
> M_TRAILINGSPACE(m0
)) {
357 /* Add an mbuf to the chain. */
358 MGET(m1
, M_NOWAIT
, MT_DATA
);
361 DPRINTF(("m_pad: unable to get extra mbuf\n"));
370 retval
= m0
->m_data
+ m0
->m_len
;
372 m
->m_pkthdr
.len
+= pad
;
378 * Remove hlen data at offset skip in the packet. This is used by
379 * the protocols strip protocol headers and associated data (e.g. IV,
380 * authenticator) on input.
383 m_striphdr(struct mbuf
*m
, int skip
, int hlen
)
388 /* Find beginning of header */
389 m1
= m_getptr(m
, skip
, &roff
);
393 /* Remove the header and associated data from the mbuf. */
395 /* The header was at the beginning of the mbuf */
396 newipsecstat
.ips_input_front
++;
398 if ((m1
->m_flags
& M_PKTHDR
) == 0)
399 m
->m_pkthdr
.len
-= hlen
;
400 } else if (roff
+ hlen
>= m1
->m_len
) {
404 * Part or all of the header is at the end of this mbuf,
405 * so first let's remove the remainder of the header from
406 * the beginning of the remainder of the mbuf chain, if any.
408 newipsecstat
.ips_input_end
++;
409 if (roff
+ hlen
> m1
->m_len
) {
410 /* Adjust the next mbuf by the remainder */
411 m_adj(m1
->m_next
, roff
+ hlen
- m1
->m_len
);
413 /* The second mbuf is guaranteed not to have a pkthdr... */
414 m
->m_pkthdr
.len
-= (roff
+ hlen
- m1
->m_len
);
417 /* Now, let's unlink the mbuf chain for a second...*/
421 /* ...and trim the end of the first part of the chain...sick */
422 m_adj(m1
, -(m1
->m_len
- roff
));
423 if ((m1
->m_flags
& M_PKTHDR
) == 0)
424 m
->m_pkthdr
.len
-= (m1
->m_len
- roff
);
426 /* Finally, let's relink */
430 * The header lies in the "middle" of the mbuf; copy
431 * the remainder of the mbuf down over the header.
433 newipsecstat
.ips_input_middle
++;
434 bcopy(mtod(m1
, u_char
*) + roff
+ hlen
,
435 mtod(m1
, u_char
*) + roff
,
436 m1
->m_len
- (roff
+ hlen
));
438 m
->m_pkthdr
.len
-= hlen
;
444 * Diagnostic routine to check mbuf alignment as required by the
445 * crypto device drivers (that use DMA).
448 m_checkalignment(const char* where
, struct mbuf
*m0
, int off
, int len
)
451 struct mbuf
*m
= m_getptr(m0
, off
, &roff
);
456 kprintf("%s (off %u len %u): ", where
, off
, len
);
457 addr
= mtod(m
, caddr_t
) + roff
;
461 if (((uintptr_t) addr
) & 3) {
462 kprintf("addr misaligned %p,", addr
);
469 if (len
&& (mlen
& 3)) {
470 kprintf("len mismatch %u,", mlen
);
474 addr
= m
? mtod(m
, caddr_t
) : NULL
;
475 } while (m
&& len
> 0);
476 for (m
= m0
; m
; m
= m
->m_next
)
477 kprintf(" [%p:%u]", mtod(m
, caddr_t
), m
->m_len
);