sys/netproto/ipsec/ipsec_mbuf.c

   1 /*-
   2  * Copyright (c) 2002, 2003 Sam Leffler, Errno Consulting
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD: src/sys/netipsec/ipsec_mbuf.c,v 1.5.2.2 2003/03/28 20:32:53 sam Exp $
  27  * $DragonFly: src/sys/netproto/ipsec/ipsec_mbuf.c,v 1.7 2006/12/22 23:57:54 swildner Exp $
  28  */
  29
  30 /*
  31  * IPsec-specific mbuf routines.
  32  */
  33
  34 #include "opt_param.h"
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/mbuf.h>
  39 #include <sys/socket.h>
  40
  41 #include <net/route.h>
  42 #include <netinet/in.h>
  43
  44 #include <netproto/ipsec/ipsec.h>
  45
  46 extern  struct mbuf *m_getptr(struct mbuf *, int, int *);
  47
  48 /*
  49  * Create a writable copy of the mbuf chain.  While doing this
  50  * we compact the chain with a goal of producing a chain with
  51  * at most two mbufs.  The second mbuf in this chain is likely
  52  * to be a cluster.  The primary purpose of this work is to create
  53  * a writable packet for encryption, compression, etc.  The
  54  * secondary goal is to linearize the data so the data can be
  55  * passed to crypto hardware in the most efficient manner possible.
  56  */
  57 struct mbuf *
  58 m_clone(struct mbuf *m0)
  59 {
  60         struct mbuf *m, *mprev;
  61         struct mbuf *n, *mfirst, *mlast;
  62         int len, off;
  63
  64         KASSERT(m0 != NULL, ("m_clone: null mbuf"));
  65
  66         mprev = NULL;
  67         for (m = m0; m != NULL; m = mprev->m_next) {
  68                 /*
  69                  * Regular mbufs are ignored unless there's a cluster
  70                  * in front of it that we can use to coalesce.  We do
  71                  * the latter mainly so later clusters can be coalesced
  72                  * also w/o having to handle them specially (i.e. convert
  73                  * mbuf+cluster -> cluster).  This optimization is heavily
  74                  * influenced by the assumption that we're running over
  75                  * Ethernet where MCLBYTES is large enough that the max
  76                  * packet size will permit lots of coalescing into a
  77                  * single cluster.  This in turn permits efficient
  78                  * crypto operations, especially when using hardware.
  79                  */
  80                 if ((m->m_flags & M_EXT) == 0) {
  81                         if (mprev && (mprev->m_flags & M_EXT) &&
  82                             m->m_len <= M_TRAILINGSPACE(mprev)) {
  83                                 /* XXX: this ignores mbuf types */
  84                                 memcpy(mtod(mprev, caddr_t) + mprev->m_len,
  85                                        mtod(m, caddr_t), m->m_len);
  86                                 mprev->m_len += m->m_len;
  87                                 mprev->m_next = m->m_next;      /* unlink from chain */
  88                                 m_free(m);                      /* reclaim mbuf */
  89                                 newipsecstat.ips_mbcoalesced++;
  90                         } else {
  91                                 mprev = m;
  92                         }
  93                         continue;
  94                 }
  95                 /*
  96                  * Writable mbufs are left alone (for now).  Note
  97                  * that for 4.x systems it's not possible to identify
  98                  * whether or not mbufs with external buffers are
  99                  * writable unless they use clusters.
 100                  */
 101                 if (M_EXT_WRITABLE(m)) {
 102                         mprev = m;
 103                         continue;
 104                 }
 105
 106                 /*
 107                  * Not writable, replace with a copy or coalesce with
 108                  * the previous mbuf if possible (since we have to copy
 109                  * it anyway, we try to reduce the number of mbufs and
 110                  * clusters so that future work is easier).
 111                  */
 112                 KASSERT(m->m_flags & M_EXT,
 113                         ("m_clone: m_flags 0x%x", m->m_flags));
 114                 /* NB: we only coalesce into a cluster or larger */
 115                 if (mprev != NULL && (mprev->m_flags & M_EXT) &&
 116                     m->m_len <= M_TRAILINGSPACE(mprev)) {
 117                         /* XXX: this ignores mbuf types */
 118                         memcpy(mtod(mprev, caddr_t) + mprev->m_len,
 119                                mtod(m, caddr_t), m->m_len);
 120                         mprev->m_len += m->m_len;
 121                         mprev->m_next = m->m_next;      /* unlink from chain */
 122                         m_free(m);                      /* reclaim mbuf */
 123                         newipsecstat.ips_clcoalesced++;
 124                         continue;
 125                 }
 126
 127                 /*
 128                  * Allocate new space to hold the copy...
 129                  */
 130                 /* XXX why can M_PKTHDR be set past the first mbuf? */
 131                 if (mprev == NULL && (m->m_flags & M_PKTHDR)) {
 132                         /*
 133                          * NB: if a packet header is present we must
 134                          * allocate the mbuf separately from any cluster
 135                          * because M_MOVE_PKTHDR will smash the data
 136                          * pointer and drop the M_EXT marker.
 137                          */
 138                         MGETHDR(n, MB_DONTWAIT, m->m_type);
 139                         if (n == NULL) {
 140                                 m_freem(m0);
 141                                 return (NULL);
 142                         }
 143                         M_MOVE_PKTHDR(n, m);
 144                         MCLGET(n, MB_DONTWAIT);
 145                         if ((n->m_flags & M_EXT) == 0) {
 146                                 m_free(n);
 147                                 m_freem(m0);
 148                                 return (NULL);
 149                         }
 150                 } else {
 151                         n = m_getcl(MB_DONTWAIT, m->m_type, m->m_flags);
 152                         if (n == NULL) {
 153                                 m_freem(m0);
 154                                 return (NULL);
 155                         }
 156                 }
 157                 /*
 158                  * ... and copy the data.  We deal with jumbo mbufs
 159                  * (i.e. m_len > MCLBYTES) by splitting them into
 160                  * clusters.  We could just malloc a buffer and make
 161                  * it external but too many device drivers don't know
 162                  * how to break up the non-contiguous memory when
 163                  * doing DMA.
 164                  */
 165                 len = m->m_len;
 166                 off = 0;
 167                 mfirst = n;
 168                 mlast = NULL;
 169                 for (;;) {
 170                         int cc = min(len, MCLBYTES);
 171                         memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
 172                         n->m_len = cc;
 173                         if (mlast != NULL)
 174                                 mlast->m_next = n;
 175                         mlast = n;
 176                         newipsecstat.ips_clcopied++;
 177
 178                         len -= cc;
 179                         if (len <= 0)
 180                                 break;
 181                         off += cc;
 182
 183                         n = m_getcl(MB_DONTWAIT, m->m_type, m->m_flags);
 184                         if (n == NULL) {
 185                                 m_freem(mfirst);
 186                                 m_freem(m0);
 187                                 return (NULL);
 188                         }
 189                 }
 190                 n->m_next = m->m_next;
 191                 if (mprev == NULL)
 192                         m0 = mfirst;            /* new head of chain */
 193                 else
 194                         mprev->m_next = mfirst; /* replace old mbuf */
 195                 m_free(m);                      /* release old mbuf */
 196                 mprev = mfirst;
 197         }
 198         return (m0);
 199 }
 200
 201 /*
 202  * Make space for a new header of length hlen at skip bytes
 203  * into the packet.  When doing this we allocate new mbufs only
 204  * when absolutely necessary.  The mbuf where the new header
 205  * is to go is returned together with an offset into the mbuf.
 206  * If NULL is returned then the mbuf chain may have been modified;
 207  * the caller is assumed to always free the chain.
 208  */
 209 struct mbuf *
 210 m_makespace(struct mbuf *m0, int skip, int hlen, int *off)
 211 {
 212         struct mbuf *m;
 213         unsigned remain;
 214
 215         KASSERT(m0 != NULL, ("m_dmakespace: null mbuf"));
 216         KASSERT(hlen < MHLEN, ("m_makespace: hlen too big: %u", hlen));
 217
 218         for (m = m0; m && skip > m->m_len; m = m->m_next)
 219                 skip -= m->m_len;
 220         if (m == NULL)
 221                 return (NULL);
 222         /*
 223          * At this point skip is the offset into the mbuf m
 224          * where the new header should be placed.  Figure out
 225          * if there's space to insert the new header.  If so,
 226          * and copying the remainder makese sense then do so.
 227          * Otherwise insert a new mbuf in the chain, splitting
 228          * the contents of m as needed.
 229          */
 230         remain = m->m_len - skip;               /* data to move */
 231         if (hlen > M_TRAILINGSPACE(m)) {
 232                 struct mbuf *n;
 233
 234                 /* XXX code doesn't handle clusters XXX */
 235                 KASSERT(remain < MLEN,
 236                         ("m_makespace: remainder too big: %u", remain));
 237                 /*
 238                  * Not enough space in m, split the contents
 239                  * of m, inserting new mbufs as required.
 240                  *
 241                  * NB: this ignores mbuf types.
 242                  */
 243                 MGET(n, MB_DONTWAIT, MT_DATA);
 244                 if (n == NULL)
 245                         return (NULL);
 246                 n->m_next = m->m_next;          /* splice new mbuf */
 247                 m->m_next = n;
 248                 newipsecstat.ips_mbinserted++;
 249                 if (hlen <= M_TRAILINGSPACE(m) + remain) {
 250                         /*
 251                          * New header fits in the old mbuf if we copy
 252                          * the remainder; just do the copy to the new
 253                          * mbuf and we're good to go.
 254                          */
 255                         memcpy(mtod(n, caddr_t),
 256                                mtod(m, caddr_t) + skip, remain);
 257                         n->m_len = remain;
 258                         m->m_len = skip + hlen;
 259                         *off = skip;
 260                 } else {
 261                         /*
 262                          * No space in the old mbuf for the new header.
 263                          * Make space in the new mbuf and check the
 264                          * remainder'd data fits too.  If not then we
 265                          * must allocate an additional mbuf (yech).
 266                          */
 267                         n->m_len = 0;
 268                         if (remain + hlen > M_TRAILINGSPACE(n)) {
 269                                 struct mbuf *n2;
 270
 271                                 MGET(n2, MB_DONTWAIT, MT_DATA);
 272                                 /* NB: new mbuf is on chain, let caller free */
 273                                 if (n2 == NULL)
 274                                         return (NULL);
 275                                 n2->m_len = 0;
 276                                 memcpy(mtod(n2, caddr_t),
 277                                        mtod(m, caddr_t) + skip, remain);
 278                                 n2->m_len = remain;
 279                                 /* splice in second mbuf */
 280                                 n2->m_next = n->m_next;
 281                                 n->m_next = n2;
 282                                 newipsecstat.ips_mbinserted++;
 283                         } else {
 284                                 memcpy(mtod(n, caddr_t) + hlen,
 285                                        mtod(m, caddr_t) + skip, remain);
 286                                 n->m_len += remain;
 287                         }
 288                         m->m_len -= remain;
 289                         n->m_len += hlen;
 290                         m = n;                  /* header is at front ... */
 291                         *off = 0;               /* ... of new mbuf */
 292                 }
 293         } else {
 294                 /*
 295                  * Copy the remainder to the back of the mbuf
 296                  * so there's space to write the new header.
 297                  */
 298                 /* XXX can this be memcpy? does it handle overlap? */
 299                 ovbcopy(mtod(m, caddr_t) + skip,
 300                         mtod(m, caddr_t) + skip + hlen, remain);
 301                 m->m_len += hlen;
 302                 *off = skip;
 303         }
 304         m0->m_pkthdr.len += hlen;               /* adjust packet length */
 305         return m;
 306 }
 307
 308 /*
 309  * m_pad(m, n) pads <m> with <n> bytes at the end. The packet header
 310  * length is updated, and a pointer to the first byte of the padding
 311  * (which is guaranteed to be all in one mbuf) is returned.
 312  */
 313 caddr_t
 314 m_pad(struct mbuf *m, int n)
 315 {
 316         struct mbuf *m0, *m1;
 317         int len, pad;
 318         caddr_t retval;
 319
 320         if (n <= 0) {  /* No stupid arguments. */
 321                 DPRINTF(("m_pad: pad length invalid (%d)\n", n));
 322                 m_freem(m);
 323                 return NULL;
 324         }
 325
 326         len = m->m_pkthdr.len;
 327         pad = n;
 328         m0 = m;
 329
 330         while (m0->m_len < len) {
 331 KASSERT(m0->m_next != NULL, ("m_pad: m0 null, len %u m_len %u", len, m0->m_len));/*XXX*/
 332                 len -= m0->m_len;
 333                 m0 = m0->m_next;
 334         }
 335
 336         if (m0->m_len != len) {
 337                 DPRINTF(("m_pad: length mismatch (should be %d instead of %d)\n",
 338                     m->m_pkthdr.len, m->m_pkthdr.len + m0->m_len - len));
 339
 340                 m_freem(m);
 341                 return NULL;
 342         }
 343
 344         /* Check for zero-length trailing mbufs, and find the last one. */
 345         for (m1 = m0; m1->m_next; m1 = m1->m_next) {
 346                 if (m1->m_next->m_len != 0) {
 347                         DPRINTF(("m_pad: length mismatch (should be %d "
 348                             "instead of %d)\n",
 349                             m->m_pkthdr.len,
 350                             m->m_pkthdr.len + m1->m_next->m_len));
 351
 352                         m_freem(m);
 353                         return NULL;
 354                 }
 355
 356                 m0 = m1->m_next;
 357         }
 358
 359         if (pad > M_TRAILINGSPACE(m0)) {
 360                 /* Add an mbuf to the chain. */
 361                 MGET(m1, MB_DONTWAIT, MT_DATA);
 362                 if (m1 == 0) {
 363                         m_freem(m0);
 364                         DPRINTF(("m_pad: unable to get extra mbuf\n"));
 365                         return NULL;
 366                 }
 367
 368                 m0->m_next = m1;
 369                 m0 = m1;
 370                 m0->m_len = 0;
 371         }
 372
 373         retval = m0->m_data + m0->m_len;
 374         m0->m_len += pad;
 375         m->m_pkthdr.len += pad;
 376
 377         return retval;
 378 }
 379
 380 /*
 381  * Remove hlen data at offset skip in the packet.  This is used by
 382  * the protocols strip protocol headers and associated data (e.g. IV,
 383  * authenticator) on input.
 384  */
 385 int
 386 m_striphdr(struct mbuf *m, int skip, int hlen)
 387 {
 388         struct mbuf *m1;
 389         int roff;
 390
 391         /* Find beginning of header */
 392         m1 = m_getptr(m, skip, &roff);
 393         if (m1 == NULL)
 394                 return (EINVAL);
 395
 396         /* Remove the header and associated data from the mbuf. */
 397         if (roff == 0) {
 398                 /* The header was at the beginning of the mbuf */
 399                 newipsecstat.ips_input_front++;
 400                 m_adj(m1, hlen);
 401                 if ((m1->m_flags & M_PKTHDR) == 0)
 402                         m->m_pkthdr.len -= hlen;
 403         } else if (roff + hlen >= m1->m_len) {
 404                 struct mbuf *mo;
 405
 406                 /*
 407                  * Part or all of the header is at the end of this mbuf,
 408                  * so first let's remove the remainder of the header from
 409                  * the beginning of the remainder of the mbuf chain, if any.
 410                  */
 411                 newipsecstat.ips_input_end++;
 412                 if (roff + hlen > m1->m_len) {
 413                         /* Adjust the next mbuf by the remainder */
 414                         m_adj(m1->m_next, roff + hlen - m1->m_len);
 415
 416                         /* The second mbuf is guaranteed not to have a pkthdr... */
 417                         m->m_pkthdr.len -= (roff + hlen - m1->m_len);
 418                 }
 419
 420                 /* Now, let's unlink the mbuf chain for a second...*/
 421                 mo = m1->m_next;
 422                 m1->m_next = NULL;
 423
 424                 /* ...and trim the end of the first part of the chain...sick */
 425                 m_adj(m1, -(m1->m_len - roff));
 426                 if ((m1->m_flags & M_PKTHDR) == 0)
 427                         m->m_pkthdr.len -= (m1->m_len - roff);
 428
 429                 /* Finally, let's relink */
 430                 m1->m_next = mo;
 431         } else {
 432                 /*
 433                  * The header lies in the "middle" of the mbuf; copy
 434                  * the remainder of the mbuf down over the header.
 435                  */
 436                 newipsecstat.ips_input_middle++;
 437                 bcopy(mtod(m1, u_char *) + roff + hlen,
 438                       mtod(m1, u_char *) + roff,
 439                       m1->m_len - (roff + hlen));
 440                 m1->m_len -= hlen;
 441                 m->m_pkthdr.len -= hlen;
 442         }
 443         return (0);
 444 }
 445
 446 /*
 447  * Diagnostic routine to check mbuf alignment as required by the
 448  * crypto device drivers (that use DMA).
 449  */
 450 void
 451 m_checkalignment(const char* where, struct mbuf *m0, int off, int len)
 452 {
 453         int roff;
 454         struct mbuf *m = m_getptr(m0, off, &roff);
 455         caddr_t addr;
 456
 457         if (m == NULL)
 458                 return;
 459         kprintf("%s (off %u len %u): ", where, off, len);
 460         addr = mtod(m, caddr_t) + roff;
 461         do {
 462                 int mlen;
 463
 464                 if (((uintptr_t) addr) & 3) {
 465                         kprintf("addr misaligned %p,", addr);
 466                         break;
 467                 }
 468                 mlen = m->m_len;
 469                 if (mlen > len)
 470                         mlen = len;
 471                 len -= mlen;
 472                 if (len && (mlen & 3)) {
 473                         kprintf("len mismatch %u,", mlen);
 474                         break;
 475                 }
 476                 m = m->m_next;
 477                 addr = m ? mtod(m, caddr_t) : NULL;
 478         } while (m && len > 0);
 479         for (m = m0; m; m = m->m_next)
 480                 kprintf(" [%p:%u]", mtod(m, caddr_t), m->m_len);
 481         kprintf("\n");
 482 }