sys/netproto/ipsec/ipsec_mbuf.c

   1 /*-
   2  * Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD: src/sys/netipsec/ipsec_mbuf.c,v 1.5.2.2 2003/03/28 20:32:53 sam Exp $
  27  */
  28
  29 /*
  30  * IPsec-specific mbuf routines.
  31  */
  32
  33 #include "opt_param.h"
  34
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/mbuf.h>
  38 #include <sys/socket.h>
  39
  40 #include <net/route.h>
  41 #include <netinet/in.h>
  42
  43 #include <netproto/ipsec/ipsec.h>
  44
  45 /*
  46  * Create a writable copy of the mbuf chain.  While doing this
  47  * we compact the chain with a goal of producing a chain with
  48  * at most two mbufs.  The second mbuf in this chain is likely
  49  * to be a cluster.  The primary purpose of this work is to create
  50  * a writable packet for encryption, compression, etc.  The
  51  * secondary goal is to linearize the data so the data can be
  52  * passed to crypto hardware in the most efficient manner possible.
  53  */
  54 struct mbuf *
  55 m_clone(struct mbuf *m0)
  56 {
  57         struct mbuf *m, *mprev;
  58         struct mbuf *n, *mfirst, *mlast;
  59         int len, off;
  60
  61         KASSERT(m0 != NULL, ("m_clone: null mbuf"));
  62
  63         mprev = NULL;
  64         for (m = m0; m != NULL; m = mprev->m_next) {
  65                 /*
  66                  * Regular mbufs are ignored unless there's a cluster
  67                  * in front of it that we can use to coalesce.  We do
  68                  * the latter mainly so later clusters can be coalesced
  69                  * also w/o having to handle them specially (i.e. convert
  70                  * mbuf+cluster -> cluster).  This optimization is heavily
  71                  * influenced by the assumption that we're running over
  72                  * Ethernet where MCLBYTES is large enough that the max
  73                  * packet size will permit lots of coalescing into a
  74                  * single cluster.  This in turn permits efficient
  75                  * crypto operations, especially when using hardware.
  76                  */
  77                 if ((m->m_flags & M_EXT) == 0) {
  78                         if (mprev && (mprev->m_flags & M_EXT) &&
  79                             m->m_len <= M_TRAILINGSPACE(mprev)) {
  80                                 /* XXX: this ignores mbuf types */
  81                                 memcpy(mtod(mprev, caddr_t) + mprev->m_len,
  82                                        mtod(m, caddr_t), m->m_len);
  83                                 mprev->m_len += m->m_len;
  84                                 mprev->m_next = m->m_next;      /* unlink from chain */
  85                                 m_free(m);                      /* reclaim mbuf */
  86                                 newipsecstat.ips_mbcoalesced++;
  87                         } else {
  88                                 mprev = m;
  89                         }
  90                         continue;
  91                 }
  92                 /*
  93                  * Writable mbufs are left alone (for now).  Note
  94                  * that for 4.x systems it's not possible to identify
  95                  * whether or not mbufs with external buffers are
  96                  * writable unless they use clusters.
  97                  */
  98                 if (M_EXT_WRITABLE(m)) {
  99                         mprev = m;
 100                         continue;
 101                 }
 102
 103                 /*
 104                  * Not writable, replace with a copy or coalesce with
 105                  * the previous mbuf if possible (since we have to copy
 106                  * it anyway, we try to reduce the number of mbufs and
 107                  * clusters so that future work is easier).
 108                  */
 109                 KASSERT(m->m_flags & M_EXT,
 110                         ("m_clone: m_flags 0x%x", m->m_flags));
 111                 /* NB: we only coalesce into a cluster or larger */
 112                 if (mprev != NULL && (mprev->m_flags & M_EXT) &&
 113                     m->m_len <= M_TRAILINGSPACE(mprev)) {
 114                         /* XXX: this ignores mbuf types */
 115                         memcpy(mtod(mprev, caddr_t) + mprev->m_len,
 116                                mtod(m, caddr_t), m->m_len);
 117                         mprev->m_len += m->m_len;
 118                         mprev->m_next = m->m_next;      /* unlink from chain */
 119                         m_free(m);                      /* reclaim mbuf */
 120                         newipsecstat.ips_clcoalesced++;
 121                         continue;
 122                 }
 123
 124                 /*
 125                  * Allocate new space to hold the copy...
 126                  */
 127                 /* XXX why can M_PKTHDR be set past the first mbuf? */
 128                 if (mprev == NULL && (m->m_flags & M_PKTHDR)) {
 129                         /*
 130                          * NB: if a packet header is present we must
 131                          * allocate the mbuf separately from any cluster
 132                          * because M_MOVE_PKTHDR will smash the data
 133                          * pointer and drop the M_EXT marker.
 134                          */
 135                         MGETHDR(n, M_NOWAIT, m->m_type);
 136                         if (n == NULL) {
 137                                 m_freem(m0);
 138                                 return (NULL);
 139                         }
 140                         M_MOVE_PKTHDR(n, m);
 141                         MCLGET(n, M_NOWAIT);
 142                         if ((n->m_flags & M_EXT) == 0) {
 143                                 m_free(n);
 144                                 m_freem(m0);
 145                                 return (NULL);
 146                         }
 147                 } else {
 148                         n = m_getcl(M_NOWAIT, m->m_type, m->m_flags);
 149                         if (n == NULL) {
 150                                 m_freem(m0);
 151                                 return (NULL);
 152                         }
 153                 }
 154                 /*
 155                  * ... and copy the data.  We deal with jumbo mbufs
 156                  * (i.e. m_len > MCLBYTES) by splitting them into
 157                  * clusters.  We could just malloc a buffer and make
 158                  * it external but too many device drivers don't know
 159                  * how to break up the non-contiguous memory when
 160                  * doing DMA.
 161                  */
 162                 len = m->m_len;
 163                 off = 0;
 164                 mfirst = n;
 165                 mlast = NULL;
 166                 for (;;) {
 167                         int cc = min(len, MCLBYTES);
 168                         memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
 169                         n->m_len = cc;
 170                         if (mlast != NULL)
 171                                 mlast->m_next = n;
 172                         mlast = n;
 173                         newipsecstat.ips_clcopied++;
 174
 175                         len -= cc;
 176                         if (len <= 0)
 177                                 break;
 178                         off += cc;
 179
 180                         n = m_getcl(M_NOWAIT, m->m_type, m->m_flags);
 181                         if (n == NULL) {
 182                                 m_freem(mfirst);
 183                                 m_freem(m0);
 184                                 return (NULL);
 185                         }
 186                 }
 187                 n->m_next = m->m_next;
 188                 if (mprev == NULL)
 189                         m0 = mfirst;            /* new head of chain */
 190                 else
 191                         mprev->m_next = mfirst; /* replace old mbuf */
 192                 m_free(m);                      /* release old mbuf */
 193                 mprev = mfirst;
 194         }
 195         return (m0);
 196 }
 197
 198 /*
 199  * Make space for a new header of length hlen at skip bytes
 200  * into the packet.  When doing this we allocate new mbufs only
 201  * when absolutely necessary.  The mbuf where the new header
 202  * is to go is returned together with an offset into the mbuf.
 203  * If NULL is returned then the mbuf chain may have been modified;
 204  * the caller is assumed to always free the chain.
 205  */
 206 struct mbuf *
 207 m_makespace(struct mbuf *m0, int skip, int hlen, int *off)
 208 {
 209         struct mbuf *m;
 210         unsigned remain;
 211
 212         KASSERT(m0 != NULL, ("m_dmakespace: null mbuf"));
 213         KASSERT(hlen < MHLEN, ("m_makespace: hlen too big: %u", hlen));
 214
 215         for (m = m0; m && skip > m->m_len; m = m->m_next)
 216                 skip -= m->m_len;
 217         if (m == NULL)
 218                 return (NULL);
 219         /*
 220          * At this point skip is the offset into the mbuf m
 221          * where the new header should be placed.  Figure out
 222          * if there's space to insert the new header.  If so,
 223          * and copying the remainder makese sense then do so.
 224          * Otherwise insert a new mbuf in the chain, splitting
 225          * the contents of m as needed.
 226          */
 227         remain = m->m_len - skip;               /* data to move */
 228         if (hlen > M_TRAILINGSPACE(m)) {
 229                 struct mbuf *n;
 230
 231                 /* XXX code doesn't handle clusters XXX */
 232                 KASSERT(remain < MLEN,
 233                         ("m_makespace: remainder too big: %u", remain));
 234                 /*
 235                  * Not enough space in m, split the contents
 236                  * of m, inserting new mbufs as required.
 237                  *
 238                  * NB: this ignores mbuf types.
 239                  */
 240                 MGET(n, M_NOWAIT, MT_DATA);
 241                 if (n == NULL)
 242                         return (NULL);
 243                 n->m_next = m->m_next;          /* splice new mbuf */
 244                 m->m_next = n;
 245                 newipsecstat.ips_mbinserted++;
 246                 if (hlen <= M_TRAILINGSPACE(m) + remain) {
 247                         /*
 248                          * New header fits in the old mbuf if we copy
 249                          * the remainder; just do the copy to the new
 250                          * mbuf and we're good to go.
 251                          */
 252                         memcpy(mtod(n, caddr_t),
 253                                mtod(m, caddr_t) + skip, remain);
 254                         n->m_len = remain;
 255                         m->m_len = skip + hlen;
 256                         *off = skip;
 257                 } else {
 258                         /*
 259                          * No space in the old mbuf for the new header.
 260                          * Make space in the new mbuf and check the
 261                          * remainder'd data fits too.  If not then we
 262                          * must allocate an additional mbuf (yech).
 263                          */
 264                         n->m_len = 0;
 265                         if (remain + hlen > M_TRAILINGSPACE(n)) {
 266                                 struct mbuf *n2;
 267
 268                                 MGET(n2, M_NOWAIT, MT_DATA);
 269                                 /* NB: new mbuf is on chain, let caller free */
 270                                 if (n2 == NULL)
 271                                         return (NULL);
 272                                 n2->m_len = 0;
 273                                 memcpy(mtod(n2, caddr_t),
 274                                        mtod(m, caddr_t) + skip, remain);
 275                                 n2->m_len = remain;
 276                                 /* splice in second mbuf */
 277                                 n2->m_next = n->m_next;
 278                                 n->m_next = n2;
 279                                 newipsecstat.ips_mbinserted++;
 280                         } else {
 281                                 memcpy(mtod(n, caddr_t) + hlen,
 282                                        mtod(m, caddr_t) + skip, remain);
 283                                 n->m_len += remain;
 284                         }
 285                         m->m_len -= remain;
 286                         n->m_len += hlen;
 287                         m = n;                  /* header is at front ... */
 288                         *off = 0;               /* ... of new mbuf */
 289                 }
 290         } else {
 291                 /*
 292                  * Copy the remainder to the back of the mbuf
 293                  * so there's space to write the new header.
 294                  */
 295                 /* XXX can this be memcpy? does it handle overlap? */
 296                 bcopy(mtod(m, caddr_t) + skip,
 297                       mtod(m, caddr_t) + skip + hlen, remain);
 298                 m->m_len += hlen;
 299                 *off = skip;
 300         }
 301         m0->m_pkthdr.len += hlen;               /* adjust packet length */
 302         return m;
 303 }
 304
 305 /*
 306  * m_pad(m, n) pads <m> with <n> bytes at the end. The packet header
 307  * length is updated, and a pointer to the first byte of the padding
 308  * (which is guaranteed to be all in one mbuf) is returned.
 309  */
 310 caddr_t
 311 m_pad(struct mbuf *m, int n)
 312 {
 313         struct mbuf *m0, *m1;
 314         int len, pad;
 315         caddr_t retval;
 316
 317         if (n <= 0) {  /* No stupid arguments. */
 318                 DPRINTF(("m_pad: pad length invalid (%d)\n", n));
 319                 m_freem(m);
 320                 return NULL;
 321         }
 322
 323         len = m->m_pkthdr.len;
 324         pad = n;
 325         m0 = m;
 326
 327         while (m0->m_len < len) {
 328 KASSERT(m0->m_next != NULL, ("m_pad: m0 null, len %u m_len %u", len, m0->m_len));/*XXX*/
 329                 len -= m0->m_len;
 330                 m0 = m0->m_next;
 331         }
 332
 333         if (m0->m_len != len) {
 334                 DPRINTF(("m_pad: length mismatch (should be %d instead of %d)\n",
 335                     m->m_pkthdr.len, m->m_pkthdr.len + m0->m_len - len));
 336
 337                 m_freem(m);
 338                 return NULL;
 339         }
 340
 341         /* Check for zero-length trailing mbufs, and find the last one. */
 342         for (m1 = m0; m1->m_next; m1 = m1->m_next) {
 343                 if (m1->m_next->m_len != 0) {
 344                         DPRINTF(("m_pad: length mismatch (should be %d "
 345                             "instead of %d)\n",
 346                             m->m_pkthdr.len,
 347                             m->m_pkthdr.len + m1->m_next->m_len));
 348
 349                         m_freem(m);
 350                         return NULL;
 351                 }
 352
 353                 m0 = m1->m_next;
 354         }
 355
 356         if (pad > M_TRAILINGSPACE(m0)) {
 357                 /* Add an mbuf to the chain. */
 358                 MGET(m1, M_NOWAIT, MT_DATA);
 359                 if (m1 == NULL) {
 360                         m_freem(m0);
 361                         DPRINTF(("m_pad: unable to get extra mbuf\n"));
 362                         return NULL;
 363                 }
 364
 365                 m0->m_next = m1;
 366                 m0 = m1;
 367                 m0->m_len = 0;
 368         }
 369
 370         retval = m0->m_data + m0->m_len;
 371         m0->m_len += pad;
 372         m->m_pkthdr.len += pad;
 373
 374         return retval;
 375 }
 376
 377 /*
 378  * Remove hlen data at offset skip in the packet.  This is used by
 379  * the protocols strip protocol headers and associated data (e.g. IV,
 380  * authenticator) on input.
 381  */
 382 int
 383 m_striphdr(struct mbuf *m, int skip, int hlen)
 384 {
 385         struct mbuf *m1;
 386         int roff;
 387
 388         /* Find beginning of header */
 389         m1 = m_getptr(m, skip, &roff);
 390         if (m1 == NULL)
 391                 return (EINVAL);
 392
 393         /* Remove the header and associated data from the mbuf. */
 394         if (roff == 0) {
 395                 /* The header was at the beginning of the mbuf */
 396                 newipsecstat.ips_input_front++;
 397                 m_adj(m1, hlen);
 398                 if ((m1->m_flags & M_PKTHDR) == 0)
 399                         m->m_pkthdr.len -= hlen;
 400         } else if (roff + hlen >= m1->m_len) {
 401                 struct mbuf *mo;
 402
 403                 /*
 404                  * Part or all of the header is at the end of this mbuf,
 405                  * so first let's remove the remainder of the header from
 406                  * the beginning of the remainder of the mbuf chain, if any.
 407                  */
 408                 newipsecstat.ips_input_end++;
 409                 if (roff + hlen > m1->m_len) {
 410                         /* Adjust the next mbuf by the remainder */
 411                         m_adj(m1->m_next, roff + hlen - m1->m_len);
 412
 413                         /* The second mbuf is guaranteed not to have a pkthdr... */
 414                         m->m_pkthdr.len -= (roff + hlen - m1->m_len);
 415                 }
 416
 417                 /* Now, let's unlink the mbuf chain for a second...*/
 418                 mo = m1->m_next;
 419                 m1->m_next = NULL;
 420
 421                 /* ...and trim the end of the first part of the chain...sick */
 422                 m_adj(m1, -(m1->m_len - roff));
 423                 if ((m1->m_flags & M_PKTHDR) == 0)
 424                         m->m_pkthdr.len -= (m1->m_len - roff);
 425
 426                 /* Finally, let's relink */
 427                 m1->m_next = mo;
 428         } else {
 429                 /*
 430                  * The header lies in the "middle" of the mbuf; copy
 431                  * the remainder of the mbuf down over the header.
 432                  */
 433                 newipsecstat.ips_input_middle++;
 434                 bcopy(mtod(m1, u_char *) + roff + hlen,
 435                       mtod(m1, u_char *) + roff,
 436                       m1->m_len - (roff + hlen));
 437                 m1->m_len -= hlen;
 438                 m->m_pkthdr.len -= hlen;
 439         }
 440         return (0);
 441 }
 442
 443 /*
 444  * Diagnostic routine to check mbuf alignment as required by the
 445  * crypto device drivers (that use DMA).
 446  */
 447 void
 448 m_checkalignment(const char* where, struct mbuf *m0, int off, int len)
 449 {
 450         int roff;
 451         struct mbuf *m = m_getptr(m0, off, &roff);
 452         caddr_t addr;
 453
 454         if (m == NULL)
 455                 return;
 456         kprintf("%s (off %u len %u): ", where, off, len);
 457         addr = mtod(m, caddr_t) + roff;
 458         do {
 459                 int mlen;
 460
 461                 if (((uintptr_t) addr) & 3) {
 462                         kprintf("addr misaligned %p,", addr);
 463                         break;
 464                 }
 465                 mlen = m->m_len;
 466                 if (mlen > len)
 467                         mlen = len;
 468                 len -= mlen;
 469                 if (len && (mlen & 3)) {
 470                         kprintf("len mismatch %u,", mlen);
 471                         break;
 472                 }
 473                 m = m->m_next;
 474                 addr = m ? mtod(m, caddr_t) : NULL;
 475         } while (m && len > 0);
 476         for (m = m0; m; m = m->m_next)
 477                 kprintf(" [%p:%u]", mtod(m, caddr_t), m->m_len);
 478         kprintf("\n");
 479 }