net/core/skbuff.c

   1 /*
   2  *      Routines having to do with the 'struct sk_buff' memory handlers.
   3  *
   4  *      Authors:        Alan Cox <iiitac@pyr.swan.ac.uk>
   5  *                      Florian La Roche <rzsfl@rz.uni-sb.de>
   6  *
   7  *      Version:        $Id: skbuff.c,v 1.75 2000/12/08 17:15:53 davem Exp $
   8  *
   9  *      Fixes:
  10  *              Alan Cox        :       Fixed the worst of the load balancer bugs.
  11  *              Dave Platt      :       Interrupt stacking fix.
  12  *      Richard Kooijman        :       Timestamp fixes.
  13  *              Alan Cox        :       Changed buffer format.
  14  *              Alan Cox        :       destructor hook for AF_UNIX etc.
  15  *              Linus Torvalds  :       Better skb_clone.
  16  *              Alan Cox        :       Added skb_copy.
  17  *              Alan Cox        :       Added all the changed routines Linus
  18  *                                      only put in the headers
  19  *              Ray VanTassle   :       Fixed --skb->lock in free
  20  *              Alan Cox        :       skb_copy copy arp field
  21  *              Andi Kleen      :       slabified it.
  22  *
  23  *      NOTE:
  24  *              The __skb_ routines should be called with interrupts
  25  *      disabled, or you better be *real* sure that the operation is atomic
  26  *      with respect to whatever list is being frobbed (e.g. via lock_sock()
  27  *      or via disabling bottom half handlers, etc).
  28  *
  29  *      This program is free software; you can redistribute it and/or
  30  *      modify it under the terms of the GNU General Public License
  31  *      as published by the Free Software Foundation; either version
  32  *      2 of the License, or (at your option) any later version.
  33  */
  34
  35 /*
  36  *      The functions in this file will not compile correctly with gcc 2.4.x
  37  */
  38
  39 #include <linux/config.h>
  40 #include <linux/types.h>
  41 #include <linux/kernel.h>
  42 #include <linux/sched.h>
  43 #include <linux/mm.h>
  44 #include <linux/interrupt.h>
  45 #include <linux/in.h>
  46 #include <linux/inet.h>
  47 #include <linux/malloc.h>
  48 #include <linux/netdevice.h>
  49 #include <linux/string.h>
  50 #include <linux/skbuff.h>
  51 #include <linux/slab.h>
  52 #include <linux/cache.h>
  53 #include <linux/init.h>
  54
  55 #include <net/ip.h>
  56 #include <net/protocol.h>
  57 #include <net/dst.h>
  58 #include <net/tcp.h>
  59 #include <net/udp.h>
  60 #include <net/sock.h>
  61
  62 #include <asm/uaccess.h>
  63 #include <asm/system.h>
  64
  65 int sysctl_hot_list_len = 128;
  66
  67 static kmem_cache_t *skbuff_head_cache;
  68
  69 static union {
  70         struct sk_buff_head     list;
  71         char                    pad[SMP_CACHE_BYTES];
  72 } skb_head_pool[NR_CPUS];
  73
  74 /*
  75  *      Keep out-of-line to prevent kernel bloat.
  76  *      __builtin_return_address is not used because it is not always
  77  *      reliable.
  78  */
  79
  80 /**
  81  *      skb_over_panic  -       private function
  82  *      @skb: buffer
  83  *      @sz: size
  84  *      @here: address
  85  *
  86  *      Out of line support code for skb_put(). Not user callable.
  87  */
  88
  89 void skb_over_panic(struct sk_buff *skb, int sz, void *here)
  90 {
  91         printk("skput:over: %p:%d put:%d dev:%s",
  92                 here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>");
  93         BUG();
  94 }
  95
  96 /**
  97  *      skb_under_panic -       private function
  98  *      @skb: buffer
  99  *      @sz: size
 100  *      @here: address
 101  *
 102  *      Out of line support code for skb_push(). Not user callable.
 103  */
 104
 105
 106 void skb_under_panic(struct sk_buff *skb, int sz, void *here)
 107 {
 108         printk("skput:under: %p:%d put:%d dev:%s",
 109                 here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>");
 110         BUG();
 111 }
 112
 113 static __inline__ struct sk_buff *skb_head_from_pool(void)
 114 {
 115         struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list;
 116
 117         if (skb_queue_len(list)) {
 118                 struct sk_buff *skb;
 119                 unsigned long flags;
 120
 121                 local_irq_save(flags);
 122                 skb = __skb_dequeue(list);
 123                 local_irq_restore(flags);
 124                 return skb;
 125         }
 126         return NULL;
 127 }
 128
 129 static __inline__ void skb_head_to_pool(struct sk_buff *skb)
 130 {
 131         struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list;
 132
 133         if (skb_queue_len(list) < sysctl_hot_list_len) {
 134                 unsigned long flags;
 135
 136                 local_irq_save(flags);
 137                 __skb_queue_head(list, skb);
 138                 local_irq_restore(flags);
 139
 140                 return;
 141         }
 142         kmem_cache_free(skbuff_head_cache, skb);
 143 }
 144
 145
 146 /*      Allocate a new skbuff. We do this ourselves so we can fill in a few
 147  *      'private' fields and also do memory statistics to find all the
 148  *      [BEEP] leaks.
 149  *
 150  */
 151
 152 /**
 153  *      alloc_skb       -       allocate a network buffer
 154  *      @size: size to allocate
 155  *      @gfp_mask: allocation mask
 156  *
 157  *      Allocate a new &sk_buff. The returned buffer has no headroom and a
 158  *      tail room of size bytes. The object has a reference count of one.
 159  *      The return is the buffer. On a failure the return is %NULL.
 160  *
 161  *      Buffers may only be allocated from interrupts using a @gfp_mask of
 162  *      %GFP_ATOMIC.
 163  */
 164
 165 struct sk_buff *alloc_skb(unsigned int size,int gfp_mask)
 166 {
 167         struct sk_buff *skb;
 168         u8 *data;
 169
 170         if (in_interrupt() && (gfp_mask & __GFP_WAIT)) {
 171                 static int count = 0;
 172                 if (++count < 5) {
 173                         printk(KERN_ERR "alloc_skb called nonatomically "
 174                                "from interrupt %p\n", NET_CALLER(size));
 175                         BUG();
 176                 }
 177                 gfp_mask &= ~__GFP_WAIT;
 178         }
 179
 180         /* Get the HEAD */
 181         skb = skb_head_from_pool();
 182         if (skb == NULL) {
 183                 skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
 184                 if (skb == NULL)
 185                         goto nohead;
 186         }
 187
 188         /* Get the DATA. Size must match skb_add_mtu(). */
 189         size = ((size + 15) & ~15);
 190         data = kmalloc(size + sizeof(atomic_t), gfp_mask);
 191         if (data == NULL)
 192                 goto nodata;
 193
 194         /* XXX: does not include slab overhead */
 195         skb->truesize = size + sizeof(struct sk_buff);
 196
 197         /* Load the data pointers. */
 198         skb->head = data;
 199         skb->data = data;
 200         skb->tail = data;
 201         skb->end = data + size;
 202
 203         /* Set up other state */
 204         skb->len = 0;
 205         skb->cloned = 0;
 206
 207         atomic_set(&skb->users, 1);
 208         atomic_set(skb_datarefp(skb), 1);
 209         return skb;
 210
 211 nodata:
 212         skb_head_to_pool(skb);
 213 nohead:
 214         return NULL;
 215 }
 216
 217
 218 /*
 219  *      Slab constructor for a skb head.
 220  */
 221 static inline void skb_headerinit(void *p, kmem_cache_t *cache,
 222                                   unsigned long flags)
 223 {
 224         struct sk_buff *skb = p;
 225
 226         skb->next = NULL;
 227         skb->prev = NULL;
 228         skb->list = NULL;
 229         skb->sk = NULL;
 230         skb->stamp.tv_sec=0;    /* No idea about time */
 231         skb->dev = NULL;
 232         skb->dst = NULL;
 233         memset(skb->cb, 0, sizeof(skb->cb));
 234         skb->pkt_type = PACKET_HOST;    /* Default type */
 235         skb->ip_summed = 0;
 236         skb->priority = 0;
 237         skb->security = 0;      /* By default packets are insecure */
 238         skb->destructor = NULL;
 239
 240 #ifdef CONFIG_NETFILTER
 241         skb->nfmark = skb->nfcache = 0;
 242         skb->nfct = NULL;
 243 #ifdef CONFIG_NETFILTER_DEBUG
 244         skb->nf_debug = 0;
 245 #endif
 246 #endif
 247 #ifdef CONFIG_NET_SCHED
 248         skb->tc_index = 0;
 249 #endif
 250 }
 251
 252 /*
 253  *      Free an skbuff by memory without cleaning the state.
 254  */
 255 void kfree_skbmem(struct sk_buff *skb)
 256 {
 257         if (!skb->cloned || atomic_dec_and_test(skb_datarefp(skb)))
 258                 kfree(skb->head);
 259
 260         skb_head_to_pool(skb);
 261 }
 262
 263 /**
 264  *      __kfree_skb - private function
 265  *      @skb: buffer
 266  *
 267  *      Free an sk_buff. Release anything attached to the buffer.
 268  *      Clean the state. This is an internal helper function. Users should
 269  *      always call kfree_skb
 270  */
 271
 272 void __kfree_skb(struct sk_buff *skb)
 273 {
 274         if (skb->list) {
 275                 printk(KERN_WARNING "Warning: kfree_skb passed an skb still "
 276                        "on a list (from %p).\n", NET_CALLER(skb));
 277                 BUG();
 278         }
 279
 280         dst_release(skb->dst);
 281         if(skb->destructor) {
 282                 if (in_irq()) {
 283                         printk(KERN_WARNING "Warning: kfree_skb on hard IRQ %p\n",
 284                                 NET_CALLER(skb));
 285                 }
 286                 skb->destructor(skb);
 287         }
 288 #ifdef CONFIG_NETFILTER
 289         nf_conntrack_put(skb->nfct);
 290 #endif
 291         skb_headerinit(skb, NULL, 0);  /* clean state */
 292         kfree_skbmem(skb);
 293 }
 294
 295 /**
 296  *      skb_clone       -       duplicate an sk_buff
 297  *      @skb: buffer to clone
 298  *      @gfp_mask: allocation priority
 299  *
 300  *      Duplicate an &sk_buff. The new one is not owned by a socket. Both
 301  *      copies share the same packet data but not structure. The new
 302  *      buffer has a reference count of 1. If the allocation fails the
 303  *      function returns %NULL otherwise the new buffer is returned.
 304  *
 305  *      If this function is called from an interrupt gfp_mask() must be
 306  *      %GFP_ATOMIC.
 307  */
 308
 309 struct sk_buff *skb_clone(struct sk_buff *skb, int gfp_mask)
 310 {
 311         struct sk_buff *n;
 312
 313         n = skb_head_from_pool();
 314         if (!n) {
 315                 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
 316                 if (!n)
 317                         return NULL;
 318         }
 319
 320         memcpy(n, skb, sizeof(*n));
 321         atomic_inc(skb_datarefp(skb));
 322         skb->cloned = 1;
 323
 324         dst_clone(n->dst);
 325         n->cloned = 1;
 326         n->next = n->prev = NULL;
 327         n->list = NULL;
 328         n->sk = NULL;
 329         atomic_set(&n->users, 1);
 330         n->destructor = NULL;
 331 #ifdef CONFIG_NETFILTER
 332         nf_conntrack_get(skb->nfct);
 333 #endif
 334         return n;
 335 }
 336
 337 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
 338 {
 339         /*
 340          *      Shift between the two data areas in bytes
 341          */
 342         unsigned long offset = new->data - old->data;
 343
 344         new->list=NULL;
 345         new->sk=NULL;
 346         new->dev=old->dev;
 347         new->priority=old->priority;
 348         new->protocol=old->protocol;
 349         new->dst=dst_clone(old->dst);
 350         new->h.raw=old->h.raw+offset;
 351         new->nh.raw=old->nh.raw+offset;
 352         new->mac.raw=old->mac.raw+offset;
 353         memcpy(new->cb, old->cb, sizeof(old->cb));
 354         new->used=old->used;
 355         atomic_set(&new->users, 1);
 356         new->pkt_type=old->pkt_type;
 357         new->stamp=old->stamp;
 358         new->destructor = NULL;
 359         new->security=old->security;
 360 #ifdef CONFIG_NETFILTER
 361         new->nfmark=old->nfmark;
 362         new->nfcache=old->nfcache;
 363         new->nfct=old->nfct;
 364         nf_conntrack_get(new->nfct);
 365 #ifdef CONFIG_NETFILTER_DEBUG
 366         new->nf_debug=old->nf_debug;
 367 #endif
 368 #endif
 369 #ifdef CONFIG_NET_SCHED
 370         new->tc_index = old->tc_index;
 371 #endif
 372 }
 373
 374 /**
 375  *      skb_copy        -       copy an sk_buff
 376  *      @skb: buffer to copy
 377  *      @gfp_mask: allocation priority
 378  *
 379  *      Make a copy of both an &sk_buff and its data. This is used when the
 380  *      caller wishes to modify the data and needs a private copy of the
 381  *      data to alter. Returns %NULL on failure or the pointer to the buffer
 382  *      on success. The returned buffer has a reference count of 1.
 383  *
 384  *      You must pass %GFP_ATOMIC as the allocation priority if this function
 385  *      is called from an interrupt.
 386  */
 387
 388 struct sk_buff *skb_copy(const struct sk_buff *skb, int gfp_mask)
 389 {
 390         struct sk_buff *n;
 391
 392         /*
 393          *      Allocate the copy buffer
 394          */
 395
 396         n=alloc_skb(skb->end - skb->head, gfp_mask);
 397         if(n==NULL)
 398                 return NULL;
 399
 400         /* Set the data pointer */
 401         skb_reserve(n,skb->data-skb->head);
 402         /* Set the tail pointer and length */
 403         skb_put(n,skb->len);
 404         /* Copy the bytes */
 405         memcpy(n->head,skb->head,skb->end-skb->head);
 406         n->csum = skb->csum;
 407         copy_skb_header(n, skb);
 408
 409         return n;
 410 }
 411
 412 /**
 413  *      skb_copy_expand -       copy and expand sk_buff
 414  *      @skb: buffer to copy
 415  *      @newheadroom: new free bytes at head
 416  *      @newtailroom: new free bytes at tail
 417  *      @gfp_mask: allocation priority
 418  *
 419  *      Make a copy of both an &sk_buff and its data and while doing so
 420  *      allocate additional space.
 421  *
 422  *      This is used when the caller wishes to modify the data and needs a
 423  *      private copy of the data to alter as well as more space for new fields.
 424  *      Returns %NULL on failure or the pointer to the buffer
 425  *      on success. The returned buffer has a reference count of 1.
 426  *
 427  *      You must pass %GFP_ATOMIC as the allocation priority if this function
 428  *      is called from an interrupt.
 429  */
 430
 431
 432 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
 433                                 int newheadroom,
 434                                 int newtailroom,
 435                                 int gfp_mask)
 436 {
 437         struct sk_buff *n;
 438
 439         /*
 440          *      Allocate the copy buffer
 441          */
 442
 443         n=alloc_skb(newheadroom + (skb->tail - skb->data) + newtailroom,
 444                     gfp_mask);
 445         if(n==NULL)
 446                 return NULL;
 447
 448         skb_reserve(n,newheadroom);
 449
 450         /* Set the tail pointer and length */
 451         skb_put(n,skb->len);
 452
 453         /* Copy the data only. */
 454         memcpy(n->data, skb->data, skb->len);
 455
 456         copy_skb_header(n, skb);
 457         return n;
 458 }
 459
 460 #if 0
 461 /*
 462  *      Tune the memory allocator for a new MTU size.
 463  */
 464 void skb_add_mtu(int mtu)
 465 {
 466         /* Must match allocation in alloc_skb */
 467         mtu = ((mtu + 15) & ~15) + sizeof(atomic_t);
 468
 469         kmem_add_cache_size(mtu);
 470 }
 471 #endif
 472
 473 void __init skb_init(void)
 474 {
 475         int i;
 476
 477         skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
 478                                               sizeof(struct sk_buff),
 479                                               0,
 480                                               SLAB_HWCACHE_ALIGN,
 481                                               skb_headerinit, NULL);
 482         if (!skbuff_head_cache)
 483                 panic("cannot create skbuff cache");
 484
 485         for (i=0; i<NR_CPUS; i++)
 486                 skb_queue_head_init(&skb_head_pool[i].list);
 487 }