Staging: rtl8192e: use DEFINE_PCI_DEVICE_TABLE
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / net / core / dev.c
blob0dd54a69dace255fcdf54732d982e8c521c574a5
1 /*
2 * NET3 Protocol independent device support routines.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
75 #include <asm/uaccess.h>
76 #include <asm/system.h>
77 #include <linux/bitops.h>
78 #include <linux/capability.h>
79 #include <linux/cpu.h>
80 #include <linux/types.h>
81 #include <linux/kernel.h>
82 #include <linux/hash.h>
83 #include <linux/slab.h>
84 #include <linux/sched.h>
85 #include <linux/mutex.h>
86 #include <linux/string.h>
87 #include <linux/mm.h>
88 #include <linux/socket.h>
89 #include <linux/sockios.h>
90 #include <linux/errno.h>
91 #include <linux/interrupt.h>
92 #include <linux/if_ether.h>
93 #include <linux/netdevice.h>
94 #include <linux/etherdevice.h>
95 #include <linux/ethtool.h>
96 #include <linux/notifier.h>
97 #include <linux/skbuff.h>
98 #include <net/net_namespace.h>
99 #include <net/sock.h>
100 #include <linux/rtnetlink.h>
101 #include <linux/proc_fs.h>
102 #include <linux/seq_file.h>
103 #include <linux/stat.h>
104 #include <net/dst.h>
105 #include <net/pkt_sched.h>
106 #include <net/checksum.h>
107 #include <net/xfrm.h>
108 #include <linux/highmem.h>
109 #include <linux/init.h>
110 #include <linux/kmod.h>
111 #include <linux/module.h>
112 #include <linux/netpoll.h>
113 #include <linux/rcupdate.h>
114 #include <linux/delay.h>
115 #include <net/wext.h>
116 #include <net/iw_handler.h>
117 #include <asm/current.h>
118 #include <linux/audit.h>
119 #include <linux/dmaengine.h>
120 #include <linux/err.h>
121 #include <linux/ctype.h>
122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h>
124 #include <linux/ip.h>
125 #include <net/ip.h>
126 #include <linux/ipv6.h>
127 #include <linux/in.h>
128 #include <linux/jhash.h>
129 #include <linux/random.h>
130 #include <trace/events/napi.h>
131 #include <trace/events/net.h>
132 #include <trace/events/skb.h>
133 #include <linux/pci.h>
134 #include <linux/inetdevice.h>
136 #include "net-sysfs.h"
138 /* Instead of increasing this, you should create a hash table. */
139 #define MAX_GRO_SKBS 8
141 /* This should be increased if a protocol with a bigger head is added. */
142 #define GRO_MAX_HEAD (MAX_HEADER + 128)
145 * The list of packet types we will receive (as opposed to discard)
146 * and the routines to invoke.
148 * Why 16. Because with 16 the only overlap we get on a hash of the
149 * low nibble of the protocol value is RARP/SNAP/X.25.
151 * NOTE: That is no longer true with the addition of VLAN tags. Not
152 * sure which should go first, but I bet it won't make much
153 * difference if we are running VLANs. The good news is that
154 * this protocol won't be in the list unless compiled in, so
155 * the average user (w/out VLANs) will not be adversely affected.
156 * --BLG
158 * 0800 IP
159 * 8100 802.1Q VLAN
160 * 0001 802.3
161 * 0002 AX.25
162 * 0004 802.2
163 * 8035 RARP
164 * 0005 SNAP
165 * 0805 X.25
166 * 0806 ARP
167 * 8137 IPX
168 * 0009 Localtalk
169 * 86DD IPv6
172 #define PTYPE_HASH_SIZE (16)
173 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
175 static DEFINE_SPINLOCK(ptype_lock);
176 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
177 static struct list_head ptype_all __read_mostly; /* Taps */
180 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
181 * semaphore.
183 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
185 * Writers must hold the rtnl semaphore while they loop through the
186 * dev_base_head list, and hold dev_base_lock for writing when they do the
187 * actual updates. This allows pure readers to access the list even
188 * while a writer is preparing to update it.
190 * To put it another way, dev_base_lock is held for writing only to
191 * protect against pure readers; the rtnl semaphore provides the
192 * protection against other writers.
194 * See, for example usages, register_netdevice() and
195 * unregister_netdevice(), which must be called with the rtnl
196 * semaphore held.
198 DEFINE_RWLOCK(dev_base_lock);
199 EXPORT_SYMBOL(dev_base_lock);
201 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
203 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
204 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
207 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
209 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
212 static inline void rps_lock(struct softnet_data *sd)
214 #ifdef CONFIG_RPS
215 spin_lock(&sd->input_pkt_queue.lock);
216 #endif
219 static inline void rps_unlock(struct softnet_data *sd)
221 #ifdef CONFIG_RPS
222 spin_unlock(&sd->input_pkt_queue.lock);
223 #endif
226 /* Device list insertion */
227 static int list_netdevice(struct net_device *dev)
229 struct net *net = dev_net(dev);
231 ASSERT_RTNL();
233 write_lock_bh(&dev_base_lock);
234 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
235 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
236 hlist_add_head_rcu(&dev->index_hlist,
237 dev_index_hash(net, dev->ifindex));
238 write_unlock_bh(&dev_base_lock);
239 return 0;
242 /* Device list removal
243 * caller must respect a RCU grace period before freeing/reusing dev
245 static void unlist_netdevice(struct net_device *dev)
247 ASSERT_RTNL();
249 /* Unlink dev from the device chain */
250 write_lock_bh(&dev_base_lock);
251 list_del_rcu(&dev->dev_list);
252 hlist_del_rcu(&dev->name_hlist);
253 hlist_del_rcu(&dev->index_hlist);
254 write_unlock_bh(&dev_base_lock);
258 * Our notifier list
261 static RAW_NOTIFIER_HEAD(netdev_chain);
264 * Device drivers call our routines to queue packets here. We empty the
265 * queue in the local softnet handler.
268 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
269 EXPORT_PER_CPU_SYMBOL(softnet_data);
271 #ifdef CONFIG_LOCKDEP
273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
274 * according to dev->type
276 static const unsigned short netdev_lock_type[] =
277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
289 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
290 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
291 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
292 ARPHRD_VOID, ARPHRD_NONE};
294 static const char *const netdev_lock_name[] =
295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
307 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
308 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
309 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
310 "_xmit_VOID", "_xmit_NONE"};
312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
315 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
317 int i;
319 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
320 if (netdev_lock_type[i] == dev_type)
321 return i;
322 /* the last key is used by default */
323 return ARRAY_SIZE(netdev_lock_type) - 1;
326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
327 unsigned short dev_type)
329 int i;
331 i = netdev_lock_pos(dev_type);
332 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
333 netdev_lock_name[i]);
336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
338 int i;
340 i = netdev_lock_pos(dev->type);
341 lockdep_set_class_and_name(&dev->addr_list_lock,
342 &netdev_addr_lock_key[i],
343 netdev_lock_name[i]);
345 #else
346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
347 unsigned short dev_type)
350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
353 #endif
355 /*******************************************************************************
357 Protocol management and registration routines
359 *******************************************************************************/
362 * Add a protocol ID to the list. Now that the input handler is
363 * smarter we can dispense with all the messy stuff that used to be
364 * here.
366 * BEWARE!!! Protocol handlers, mangling input packets,
367 * MUST BE last in hash buckets and checking protocol handlers
368 * MUST start from promiscuous ptype_all chain in net_bh.
369 * It is true now, do not change it.
370 * Explanation follows: if protocol handler, mangling packet, will
371 * be the first on list, it is not able to sense, that packet
372 * is cloned and should be copied-on-write, so that it will
373 * change it and subsequent readers will get broken packet.
374 * --ANK (980803)
377 static inline struct list_head *ptype_head(const struct packet_type *pt)
379 if (pt->type == htons(ETH_P_ALL))
380 return &ptype_all;
381 else
382 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
386 * dev_add_pack - add packet handler
387 * @pt: packet type declaration
389 * Add a protocol handler to the networking stack. The passed &packet_type
390 * is linked into kernel lists and may not be freed until it has been
391 * removed from the kernel lists.
393 * This call does not sleep therefore it can not
394 * guarantee all CPU's that are in middle of receiving packets
395 * will see the new packet type (until the next received packet).
398 void dev_add_pack(struct packet_type *pt)
400 struct list_head *head = ptype_head(pt);
402 spin_lock(&ptype_lock);
403 list_add_rcu(&pt->list, head);
404 spin_unlock(&ptype_lock);
406 EXPORT_SYMBOL(dev_add_pack);
409 * __dev_remove_pack - remove packet handler
410 * @pt: packet type declaration
412 * Remove a protocol handler that was previously added to the kernel
413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
414 * from the kernel lists and can be freed or reused once this function
415 * returns.
417 * The packet type might still be in use by receivers
418 * and must not be freed until after all the CPU's have gone
419 * through a quiescent state.
421 void __dev_remove_pack(struct packet_type *pt)
423 struct list_head *head = ptype_head(pt);
424 struct packet_type *pt1;
426 spin_lock(&ptype_lock);
428 list_for_each_entry(pt1, head, list) {
429 if (pt == pt1) {
430 list_del_rcu(&pt->list);
431 goto out;
435 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
436 out:
437 spin_unlock(&ptype_lock);
439 EXPORT_SYMBOL(__dev_remove_pack);
442 * dev_remove_pack - remove packet handler
443 * @pt: packet type declaration
445 * Remove a protocol handler that was previously added to the kernel
446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
447 * from the kernel lists and can be freed or reused once this function
448 * returns.
450 * This call sleeps to guarantee that no CPU is looking at the packet
451 * type after return.
453 void dev_remove_pack(struct packet_type *pt)
455 __dev_remove_pack(pt);
457 synchronize_net();
459 EXPORT_SYMBOL(dev_remove_pack);
461 /******************************************************************************
463 Device Boot-time Settings Routines
465 *******************************************************************************/
467 /* Boot time configuration table */
468 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
471 * netdev_boot_setup_add - add new setup entry
472 * @name: name of the device
473 * @map: configured settings for the device
475 * Adds new setup entry to the dev_boot_setup list. The function
476 * returns 0 on error and 1 on success. This is a generic routine to
477 * all netdevices.
479 static int netdev_boot_setup_add(char *name, struct ifmap *map)
481 struct netdev_boot_setup *s;
482 int i;
484 s = dev_boot_setup;
485 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
486 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
487 memset(s[i].name, 0, sizeof(s[i].name));
488 strlcpy(s[i].name, name, IFNAMSIZ);
489 memcpy(&s[i].map, map, sizeof(s[i].map));
490 break;
494 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
498 * netdev_boot_setup_check - check boot time settings
499 * @dev: the netdevice
501 * Check boot time settings for the device.
502 * The found settings are set for the device to be used
503 * later in the device probing.
504 * Returns 0 if no settings found, 1 if they are.
506 int netdev_boot_setup_check(struct net_device *dev)
508 struct netdev_boot_setup *s = dev_boot_setup;
509 int i;
511 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
512 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
513 !strcmp(dev->name, s[i].name)) {
514 dev->irq = s[i].map.irq;
515 dev->base_addr = s[i].map.base_addr;
516 dev->mem_start = s[i].map.mem_start;
517 dev->mem_end = s[i].map.mem_end;
518 return 1;
521 return 0;
523 EXPORT_SYMBOL(netdev_boot_setup_check);
527 * netdev_boot_base - get address from boot time settings
528 * @prefix: prefix for network device
529 * @unit: id for network device
531 * Check boot time settings for the base address of device.
532 * The found settings are set for the device to be used
533 * later in the device probing.
534 * Returns 0 if no settings found.
536 unsigned long netdev_boot_base(const char *prefix, int unit)
538 const struct netdev_boot_setup *s = dev_boot_setup;
539 char name[IFNAMSIZ];
540 int i;
542 sprintf(name, "%s%d", prefix, unit);
545 * If device already registered then return base of 1
546 * to indicate not to probe for this interface
548 if (__dev_get_by_name(&init_net, name))
549 return 1;
551 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
552 if (!strcmp(name, s[i].name))
553 return s[i].map.base_addr;
554 return 0;
558 * Saves at boot time configured settings for any netdevice.
560 int __init netdev_boot_setup(char *str)
562 int ints[5];
563 struct ifmap map;
565 str = get_options(str, ARRAY_SIZE(ints), ints);
566 if (!str || !*str)
567 return 0;
569 /* Save settings */
570 memset(&map, 0, sizeof(map));
571 if (ints[0] > 0)
572 map.irq = ints[1];
573 if (ints[0] > 1)
574 map.base_addr = ints[2];
575 if (ints[0] > 2)
576 map.mem_start = ints[3];
577 if (ints[0] > 3)
578 map.mem_end = ints[4];
580 /* Add new entry to the list */
581 return netdev_boot_setup_add(str, &map);
584 __setup("netdev=", netdev_boot_setup);
586 /*******************************************************************************
588 Device Interface Subroutines
590 *******************************************************************************/
593 * __dev_get_by_name - find a device by its name
594 * @net: the applicable net namespace
595 * @name: name to find
597 * Find an interface by name. Must be called under RTNL semaphore
598 * or @dev_base_lock. If the name is found a pointer to the device
599 * is returned. If the name is not found then %NULL is returned. The
600 * reference counters are not incremented so the caller must be
601 * careful with locks.
604 struct net_device *__dev_get_by_name(struct net *net, const char *name)
606 struct hlist_node *p;
607 struct net_device *dev;
608 struct hlist_head *head = dev_name_hash(net, name);
610 hlist_for_each_entry(dev, p, head, name_hlist)
611 if (!strncmp(dev->name, name, IFNAMSIZ))
612 return dev;
614 return NULL;
616 EXPORT_SYMBOL(__dev_get_by_name);
619 * dev_get_by_name_rcu - find a device by its name
620 * @net: the applicable net namespace
621 * @name: name to find
623 * Find an interface by name.
624 * If the name is found a pointer to the device is returned.
625 * If the name is not found then %NULL is returned.
626 * The reference counters are not incremented so the caller must be
627 * careful with locks. The caller must hold RCU lock.
630 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
632 struct hlist_node *p;
633 struct net_device *dev;
634 struct hlist_head *head = dev_name_hash(net, name);
636 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
637 if (!strncmp(dev->name, name, IFNAMSIZ))
638 return dev;
640 return NULL;
642 EXPORT_SYMBOL(dev_get_by_name_rcu);
645 * dev_get_by_name - find a device by its name
646 * @net: the applicable net namespace
647 * @name: name to find
649 * Find an interface by name. This can be called from any
650 * context and does its own locking. The returned handle has
651 * the usage count incremented and the caller must use dev_put() to
652 * release it when it is no longer needed. %NULL is returned if no
653 * matching device is found.
656 struct net_device *dev_get_by_name(struct net *net, const char *name)
658 struct net_device *dev;
660 rcu_read_lock();
661 dev = dev_get_by_name_rcu(net, name);
662 if (dev)
663 dev_hold(dev);
664 rcu_read_unlock();
665 return dev;
667 EXPORT_SYMBOL(dev_get_by_name);
670 * __dev_get_by_index - find a device by its ifindex
671 * @net: the applicable net namespace
672 * @ifindex: index of device
674 * Search for an interface by index. Returns %NULL if the device
675 * is not found or a pointer to the device. The device has not
676 * had its reference counter increased so the caller must be careful
677 * about locking. The caller must hold either the RTNL semaphore
678 * or @dev_base_lock.
681 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
683 struct hlist_node *p;
684 struct net_device *dev;
685 struct hlist_head *head = dev_index_hash(net, ifindex);
687 hlist_for_each_entry(dev, p, head, index_hlist)
688 if (dev->ifindex == ifindex)
689 return dev;
691 return NULL;
693 EXPORT_SYMBOL(__dev_get_by_index);
696 * dev_get_by_index_rcu - find a device by its ifindex
697 * @net: the applicable net namespace
698 * @ifindex: index of device
700 * Search for an interface by index. Returns %NULL if the device
701 * is not found or a pointer to the device. The device has not
702 * had its reference counter increased so the caller must be careful
703 * about locking. The caller must hold RCU lock.
706 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
708 struct hlist_node *p;
709 struct net_device *dev;
710 struct hlist_head *head = dev_index_hash(net, ifindex);
712 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
713 if (dev->ifindex == ifindex)
714 return dev;
716 return NULL;
718 EXPORT_SYMBOL(dev_get_by_index_rcu);
722 * dev_get_by_index - find a device by its ifindex
723 * @net: the applicable net namespace
724 * @ifindex: index of device
726 * Search for an interface by index. Returns NULL if the device
727 * is not found or a pointer to the device. The device returned has
728 * had a reference added and the pointer is safe until the user calls
729 * dev_put to indicate they have finished with it.
732 struct net_device *dev_get_by_index(struct net *net, int ifindex)
734 struct net_device *dev;
736 rcu_read_lock();
737 dev = dev_get_by_index_rcu(net, ifindex);
738 if (dev)
739 dev_hold(dev);
740 rcu_read_unlock();
741 return dev;
743 EXPORT_SYMBOL(dev_get_by_index);
746 * dev_getbyhwaddr - find a device by its hardware address
747 * @net: the applicable net namespace
748 * @type: media type of device
749 * @ha: hardware address
751 * Search for an interface by MAC address. Returns NULL if the device
752 * is not found or a pointer to the device. The caller must hold the
753 * rtnl semaphore. The returned device has not had its ref count increased
754 * and the caller must therefore be careful about locking
756 * BUGS:
757 * If the API was consistent this would be __dev_get_by_hwaddr
760 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)
762 struct net_device *dev;
764 ASSERT_RTNL();
766 for_each_netdev(net, dev)
767 if (dev->type == type &&
768 !memcmp(dev->dev_addr, ha, dev->addr_len))
769 return dev;
771 return NULL;
773 EXPORT_SYMBOL(dev_getbyhwaddr);
775 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
777 struct net_device *dev;
779 ASSERT_RTNL();
780 for_each_netdev(net, dev)
781 if (dev->type == type)
782 return dev;
784 return NULL;
786 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
788 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
790 struct net_device *dev, *ret = NULL;
792 rcu_read_lock();
793 for_each_netdev_rcu(net, dev)
794 if (dev->type == type) {
795 dev_hold(dev);
796 ret = dev;
797 break;
799 rcu_read_unlock();
800 return ret;
802 EXPORT_SYMBOL(dev_getfirstbyhwtype);
805 * dev_get_by_flags_rcu - find any device with given flags
806 * @net: the applicable net namespace
807 * @if_flags: IFF_* values
808 * @mask: bitmask of bits in if_flags to check
810 * Search for any interface with the given flags. Returns NULL if a device
811 * is not found or a pointer to the device. Must be called inside
812 * rcu_read_lock(), and result refcount is unchanged.
815 struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
816 unsigned short mask)
818 struct net_device *dev, *ret;
820 ret = NULL;
821 for_each_netdev_rcu(net, dev) {
822 if (((dev->flags ^ if_flags) & mask) == 0) {
823 ret = dev;
824 break;
827 return ret;
829 EXPORT_SYMBOL(dev_get_by_flags_rcu);
832 * dev_valid_name - check if name is okay for network device
833 * @name: name string
835 * Network device names need to be valid file names to
836 * to allow sysfs to work. We also disallow any kind of
837 * whitespace.
839 int dev_valid_name(const char *name)
841 if (*name == '\0')
842 return 0;
843 if (strlen(name) >= IFNAMSIZ)
844 return 0;
845 if (!strcmp(name, ".") || !strcmp(name, ".."))
846 return 0;
848 while (*name) {
849 if (*name == '/' || isspace(*name))
850 return 0;
851 name++;
853 return 1;
855 EXPORT_SYMBOL(dev_valid_name);
858 * __dev_alloc_name - allocate a name for a device
859 * @net: network namespace to allocate the device name in
860 * @name: name format string
861 * @buf: scratch buffer and result name string
863 * Passed a format string - eg "lt%d" it will try and find a suitable
864 * id. It scans list of devices to build up a free map, then chooses
865 * the first empty slot. The caller must hold the dev_base or rtnl lock
866 * while allocating the name and adding the device in order to avoid
867 * duplicates.
868 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
869 * Returns the number of the unit assigned or a negative errno code.
872 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
874 int i = 0;
875 const char *p;
876 const int max_netdevices = 8*PAGE_SIZE;
877 unsigned long *inuse;
878 struct net_device *d;
880 p = strnchr(name, IFNAMSIZ-1, '%');
881 if (p) {
883 * Verify the string as this thing may have come from
884 * the user. There must be either one "%d" and no other "%"
885 * characters.
887 if (p[1] != 'd' || strchr(p + 2, '%'))
888 return -EINVAL;
890 /* Use one page as a bit array of possible slots */
891 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
892 if (!inuse)
893 return -ENOMEM;
895 for_each_netdev(net, d) {
896 if (!sscanf(d->name, name, &i))
897 continue;
898 if (i < 0 || i >= max_netdevices)
899 continue;
901 /* avoid cases where sscanf is not exact inverse of printf */
902 snprintf(buf, IFNAMSIZ, name, i);
903 if (!strncmp(buf, d->name, IFNAMSIZ))
904 set_bit(i, inuse);
907 i = find_first_zero_bit(inuse, max_netdevices);
908 free_page((unsigned long) inuse);
911 if (buf != name)
912 snprintf(buf, IFNAMSIZ, name, i);
913 if (!__dev_get_by_name(net, buf))
914 return i;
916 /* It is possible to run out of possible slots
917 * when the name is long and there isn't enough space left
918 * for the digits, or if all bits are used.
920 return -ENFILE;
924 * dev_alloc_name - allocate a name for a device
925 * @dev: device
926 * @name: name format string
928 * Passed a format string - eg "lt%d" it will try and find a suitable
929 * id. It scans list of devices to build up a free map, then chooses
930 * the first empty slot. The caller must hold the dev_base or rtnl lock
931 * while allocating the name and adding the device in order to avoid
932 * duplicates.
933 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
934 * Returns the number of the unit assigned or a negative errno code.
937 int dev_alloc_name(struct net_device *dev, const char *name)
939 char buf[IFNAMSIZ];
940 struct net *net;
941 int ret;
943 BUG_ON(!dev_net(dev));
944 net = dev_net(dev);
945 ret = __dev_alloc_name(net, name, buf);
946 if (ret >= 0)
947 strlcpy(dev->name, buf, IFNAMSIZ);
948 return ret;
950 EXPORT_SYMBOL(dev_alloc_name);
952 static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
954 struct net *net;
956 BUG_ON(!dev_net(dev));
957 net = dev_net(dev);
959 if (!dev_valid_name(name))
960 return -EINVAL;
962 if (fmt && strchr(name, '%'))
963 return dev_alloc_name(dev, name);
964 else if (__dev_get_by_name(net, name))
965 return -EEXIST;
966 else if (dev->name != name)
967 strlcpy(dev->name, name, IFNAMSIZ);
969 return 0;
973 * dev_change_name - change name of a device
974 * @dev: device
975 * @newname: name (or format string) must be at least IFNAMSIZ
977 * Change name of a device, can pass format strings "eth%d".
978 * for wildcarding.
980 int dev_change_name(struct net_device *dev, const char *newname)
982 char oldname[IFNAMSIZ];
983 int err = 0;
984 int ret;
985 struct net *net;
987 ASSERT_RTNL();
988 BUG_ON(!dev_net(dev));
990 net = dev_net(dev);
991 if (dev->flags & IFF_UP)
992 return -EBUSY;
994 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
995 return 0;
997 memcpy(oldname, dev->name, IFNAMSIZ);
999 err = dev_get_valid_name(dev, newname, 1);
1000 if (err < 0)
1001 return err;
1003 rollback:
1004 ret = device_rename(&dev->dev, dev->name);
1005 if (ret) {
1006 memcpy(dev->name, oldname, IFNAMSIZ);
1007 return ret;
1010 write_lock_bh(&dev_base_lock);
1011 hlist_del(&dev->name_hlist);
1012 write_unlock_bh(&dev_base_lock);
1014 synchronize_rcu();
1016 write_lock_bh(&dev_base_lock);
1017 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1018 write_unlock_bh(&dev_base_lock);
1020 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1021 ret = notifier_to_errno(ret);
1023 if (ret) {
1024 /* err >= 0 after dev_alloc_name() or stores the first errno */
1025 if (err >= 0) {
1026 err = ret;
1027 memcpy(dev->name, oldname, IFNAMSIZ);
1028 goto rollback;
1029 } else {
1030 printk(KERN_ERR
1031 "%s: name change rollback failed: %d.\n",
1032 dev->name, ret);
1036 return err;
1040 * dev_set_alias - change ifalias of a device
1041 * @dev: device
1042 * @alias: name up to IFALIASZ
1043 * @len: limit of bytes to copy from info
1045 * Set ifalias for a device,
1047 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1049 ASSERT_RTNL();
1051 if (len >= IFALIASZ)
1052 return -EINVAL;
1054 if (!len) {
1055 if (dev->ifalias) {
1056 kfree(dev->ifalias);
1057 dev->ifalias = NULL;
1059 return 0;
1062 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1063 if (!dev->ifalias)
1064 return -ENOMEM;
1066 strlcpy(dev->ifalias, alias, len+1);
1067 return len;
1072 * netdev_features_change - device changes features
1073 * @dev: device to cause notification
1075 * Called to indicate a device has changed features.
1077 void netdev_features_change(struct net_device *dev)
1079 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1081 EXPORT_SYMBOL(netdev_features_change);
1084 * netdev_state_change - device changes state
1085 * @dev: device to cause notification
1087 * Called to indicate a device has changed state. This function calls
1088 * the notifier chains for netdev_chain and sends a NEWLINK message
1089 * to the routing socket.
1091 void netdev_state_change(struct net_device *dev)
1093 if (dev->flags & IFF_UP) {
1094 call_netdevice_notifiers(NETDEV_CHANGE, dev);
1095 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1098 EXPORT_SYMBOL(netdev_state_change);
1100 int netdev_bonding_change(struct net_device *dev, unsigned long event)
1102 return call_netdevice_notifiers(event, dev);
1104 EXPORT_SYMBOL(netdev_bonding_change);
1107 * dev_load - load a network module
1108 * @net: the applicable net namespace
1109 * @name: name of interface
1111 * If a network interface is not present and the process has suitable
1112 * privileges this function loads the module. If module loading is not
1113 * available in this kernel then it becomes a nop.
1116 void dev_load(struct net *net, const char *name)
1118 struct net_device *dev;
1120 rcu_read_lock();
1121 dev = dev_get_by_name_rcu(net, name);
1122 rcu_read_unlock();
1124 if (!dev && capable(CAP_NET_ADMIN))
1125 request_module("%s", name);
1127 EXPORT_SYMBOL(dev_load);
1129 static int __dev_open(struct net_device *dev)
1131 const struct net_device_ops *ops = dev->netdev_ops;
1132 int ret;
1134 ASSERT_RTNL();
1137 * Is it even present?
1139 if (!netif_device_present(dev))
1140 return -ENODEV;
1142 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1143 ret = notifier_to_errno(ret);
1144 if (ret)
1145 return ret;
1148 * Call device private open method
1150 set_bit(__LINK_STATE_START, &dev->state);
1152 if (ops->ndo_validate_addr)
1153 ret = ops->ndo_validate_addr(dev);
1155 if (!ret && ops->ndo_open)
1156 ret = ops->ndo_open(dev);
1159 * If it went open OK then:
1162 if (ret)
1163 clear_bit(__LINK_STATE_START, &dev->state);
1164 else {
1166 * Set the flags.
1168 dev->flags |= IFF_UP;
1171 * Enable NET_DMA
1173 net_dmaengine_get();
1176 * Initialize multicasting status
1178 dev_set_rx_mode(dev);
1181 * Wakeup transmit queue engine
1183 dev_activate(dev);
1186 return ret;
1190 * dev_open - prepare an interface for use.
1191 * @dev: device to open
1193 * Takes a device from down to up state. The device's private open
1194 * function is invoked and then the multicast lists are loaded. Finally
1195 * the device is moved into the up state and a %NETDEV_UP message is
1196 * sent to the netdev notifier chain.
1198 * Calling this function on an active interface is a nop. On a failure
1199 * a negative errno code is returned.
1201 int dev_open(struct net_device *dev)
1203 int ret;
1206 * Is it already up?
1208 if (dev->flags & IFF_UP)
1209 return 0;
1212 * Open device
1214 ret = __dev_open(dev);
1215 if (ret < 0)
1216 return ret;
1219 * ... and announce new interface.
1221 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1222 call_netdevice_notifiers(NETDEV_UP, dev);
1224 return ret;
1226 EXPORT_SYMBOL(dev_open);
1228 static int __dev_close(struct net_device *dev)
1230 const struct net_device_ops *ops = dev->netdev_ops;
1232 ASSERT_RTNL();
1233 might_sleep();
1236 * Tell people we are going down, so that they can
1237 * prepare to death, when device is still operating.
1239 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1241 clear_bit(__LINK_STATE_START, &dev->state);
1243 /* Synchronize to scheduled poll. We cannot touch poll list,
1244 * it can be even on different cpu. So just clear netif_running().
1246 * dev->stop() will invoke napi_disable() on all of it's
1247 * napi_struct instances on this device.
1249 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1251 dev_deactivate(dev);
1254 * Call the device specific close. This cannot fail.
1255 * Only if device is UP
1257 * We allow it to be called even after a DETACH hot-plug
1258 * event.
1260 if (ops->ndo_stop)
1261 ops->ndo_stop(dev);
1264 * Device is now down.
1267 dev->flags &= ~IFF_UP;
1270 * Shutdown NET_DMA
1272 net_dmaengine_put();
1274 return 0;
1278 * dev_close - shutdown an interface.
1279 * @dev: device to shutdown
1281 * This function moves an active device into down state. A
1282 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1283 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1284 * chain.
1286 int dev_close(struct net_device *dev)
1288 if (!(dev->flags & IFF_UP))
1289 return 0;
1291 __dev_close(dev);
1294 * Tell people we are down
1296 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1297 call_netdevice_notifiers(NETDEV_DOWN, dev);
1299 return 0;
1301 EXPORT_SYMBOL(dev_close);
1305 * dev_disable_lro - disable Large Receive Offload on a device
1306 * @dev: device
1308 * Disable Large Receive Offload (LRO) on a net device. Must be
1309 * called under RTNL. This is needed if received packets may be
1310 * forwarded to another interface.
1312 void dev_disable_lro(struct net_device *dev)
1314 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1315 dev->ethtool_ops->set_flags) {
1316 u32 flags = dev->ethtool_ops->get_flags(dev);
1317 if (flags & ETH_FLAG_LRO) {
1318 flags &= ~ETH_FLAG_LRO;
1319 dev->ethtool_ops->set_flags(dev, flags);
1322 WARN_ON(dev->features & NETIF_F_LRO);
1324 EXPORT_SYMBOL(dev_disable_lro);
1327 static int dev_boot_phase = 1;
1330 * Device change register/unregister. These are not inline or static
1331 * as we export them to the world.
1335 * register_netdevice_notifier - register a network notifier block
1336 * @nb: notifier
1338 * Register a notifier to be called when network device events occur.
1339 * The notifier passed is linked into the kernel structures and must
1340 * not be reused until it has been unregistered. A negative errno code
1341 * is returned on a failure.
1343 * When registered all registration and up events are replayed
1344 * to the new notifier to allow device to have a race free
1345 * view of the network device list.
1348 int register_netdevice_notifier(struct notifier_block *nb)
1350 struct net_device *dev;
1351 struct net_device *last;
1352 struct net *net;
1353 int err;
1355 rtnl_lock();
1356 err = raw_notifier_chain_register(&netdev_chain, nb);
1357 if (err)
1358 goto unlock;
1359 if (dev_boot_phase)
1360 goto unlock;
1361 for_each_net(net) {
1362 for_each_netdev(net, dev) {
1363 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1364 err = notifier_to_errno(err);
1365 if (err)
1366 goto rollback;
1368 if (!(dev->flags & IFF_UP))
1369 continue;
1371 nb->notifier_call(nb, NETDEV_UP, dev);
1375 unlock:
1376 rtnl_unlock();
1377 return err;
1379 rollback:
1380 last = dev;
1381 for_each_net(net) {
1382 for_each_netdev(net, dev) {
1383 if (dev == last)
1384 break;
1386 if (dev->flags & IFF_UP) {
1387 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1388 nb->notifier_call(nb, NETDEV_DOWN, dev);
1390 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1391 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
1395 raw_notifier_chain_unregister(&netdev_chain, nb);
1396 goto unlock;
1398 EXPORT_SYMBOL(register_netdevice_notifier);
1401 * unregister_netdevice_notifier - unregister a network notifier block
1402 * @nb: notifier
1404 * Unregister a notifier previously registered by
1405 * register_netdevice_notifier(). The notifier is unlinked into the
1406 * kernel structures and may then be reused. A negative errno code
1407 * is returned on a failure.
1410 int unregister_netdevice_notifier(struct notifier_block *nb)
1412 int err;
1414 rtnl_lock();
1415 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1416 rtnl_unlock();
1417 return err;
1419 EXPORT_SYMBOL(unregister_netdevice_notifier);
1422 * call_netdevice_notifiers - call all network notifier blocks
1423 * @val: value passed unmodified to notifier function
1424 * @dev: net_device pointer passed unmodified to notifier function
1426 * Call all network notifier blocks. Parameters and return value
1427 * are as for raw_notifier_call_chain().
1430 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1432 ASSERT_RTNL();
1433 return raw_notifier_call_chain(&netdev_chain, val, dev);
1436 /* When > 0 there are consumers of rx skb time stamps */
1437 static atomic_t netstamp_needed = ATOMIC_INIT(0);
1439 void net_enable_timestamp(void)
1441 atomic_inc(&netstamp_needed);
1443 EXPORT_SYMBOL(net_enable_timestamp);
1445 void net_disable_timestamp(void)
1447 atomic_dec(&netstamp_needed);
1449 EXPORT_SYMBOL(net_disable_timestamp);
1451 static inline void net_timestamp_set(struct sk_buff *skb)
1453 if (atomic_read(&netstamp_needed))
1454 __net_timestamp(skb);
1455 else
1456 skb->tstamp.tv64 = 0;
1459 static inline void net_timestamp_check(struct sk_buff *skb)
1461 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1462 __net_timestamp(skb);
1466 * dev_forward_skb - loopback an skb to another netif
1468 * @dev: destination network device
1469 * @skb: buffer to forward
1471 * return values:
1472 * NET_RX_SUCCESS (no congestion)
1473 * NET_RX_DROP (packet was dropped, but freed)
1475 * dev_forward_skb can be used for injecting an skb from the
1476 * start_xmit function of one device into the receive queue
1477 * of another device.
1479 * The receiving device may be in another namespace, so
1480 * we have to clear all information in the skb that could
1481 * impact namespace isolation.
1483 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1485 skb_orphan(skb);
1486 nf_reset(skb);
1488 if (unlikely(!(dev->flags & IFF_UP) ||
1489 (skb->len > (dev->mtu + dev->hard_header_len + VLAN_HLEN)))) {
1490 atomic_long_inc(&dev->rx_dropped);
1491 kfree_skb(skb);
1492 return NET_RX_DROP;
1494 skb_set_dev(skb, dev);
1495 skb->tstamp.tv64 = 0;
1496 skb->pkt_type = PACKET_HOST;
1497 skb->protocol = eth_type_trans(skb, dev);
1498 return netif_rx(skb);
1500 EXPORT_SYMBOL_GPL(dev_forward_skb);
1503 * Support routine. Sends outgoing frames to any network
1504 * taps currently in use.
1507 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1509 struct packet_type *ptype;
1511 #ifdef CONFIG_NET_CLS_ACT
1512 if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
1513 net_timestamp_set(skb);
1514 #else
1515 net_timestamp_set(skb);
1516 #endif
1518 rcu_read_lock();
1519 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1520 /* Never send packets back to the socket
1521 * they originated from - MvS (miquels@drinkel.ow.org)
1523 if ((ptype->dev == dev || !ptype->dev) &&
1524 (ptype->af_packet_priv == NULL ||
1525 (struct sock *)ptype->af_packet_priv != skb->sk)) {
1526 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
1527 if (!skb2)
1528 break;
1530 /* skb->nh should be correctly
1531 set by sender, so that the second statement is
1532 just protection against buggy protocols.
1534 skb_reset_mac_header(skb2);
1536 if (skb_network_header(skb2) < skb2->data ||
1537 skb2->network_header > skb2->tail) {
1538 if (net_ratelimit())
1539 printk(KERN_CRIT "protocol %04x is "
1540 "buggy, dev %s\n",
1541 ntohs(skb2->protocol),
1542 dev->name);
1543 skb_reset_network_header(skb2);
1546 skb2->transport_header = skb2->network_header;
1547 skb2->pkt_type = PACKET_OUTGOING;
1548 ptype->func(skb2, skb->dev, ptype, skb->dev);
1551 rcu_read_unlock();
1555 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1556 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1558 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
1560 if (txq < 1 || txq > dev->num_tx_queues)
1561 return -EINVAL;
1563 if (dev->reg_state == NETREG_REGISTERED) {
1564 ASSERT_RTNL();
1566 if (txq < dev->real_num_tx_queues)
1567 qdisc_reset_all_tx_gt(dev, txq);
1570 dev->real_num_tx_queues = txq;
1571 return 0;
1573 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
1575 #ifdef CONFIG_RPS
1577 * netif_set_real_num_rx_queues - set actual number of RX queues used
1578 * @dev: Network device
1579 * @rxq: Actual number of RX queues
1581 * This must be called either with the rtnl_lock held or before
1582 * registration of the net device. Returns 0 on success, or a
1583 * negative error code. If called before registration, it always
1584 * succeeds.
1586 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1588 int rc;
1590 if (rxq < 1 || rxq > dev->num_rx_queues)
1591 return -EINVAL;
1593 if (dev->reg_state == NETREG_REGISTERED) {
1594 ASSERT_RTNL();
1596 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1597 rxq);
1598 if (rc)
1599 return rc;
1602 dev->real_num_rx_queues = rxq;
1603 return 0;
1605 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1606 #endif
1608 static inline void __netif_reschedule(struct Qdisc *q)
1610 struct softnet_data *sd;
1611 unsigned long flags;
1613 local_irq_save(flags);
1614 sd = &__get_cpu_var(softnet_data);
1615 q->next_sched = NULL;
1616 *sd->output_queue_tailp = q;
1617 sd->output_queue_tailp = &q->next_sched;
1618 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1619 local_irq_restore(flags);
1622 void __netif_schedule(struct Qdisc *q)
1624 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1625 __netif_reschedule(q);
1627 EXPORT_SYMBOL(__netif_schedule);
1629 void dev_kfree_skb_irq(struct sk_buff *skb)
1631 if (atomic_dec_and_test(&skb->users)) {
1632 struct softnet_data *sd;
1633 unsigned long flags;
1635 local_irq_save(flags);
1636 sd = &__get_cpu_var(softnet_data);
1637 skb->next = sd->completion_queue;
1638 sd->completion_queue = skb;
1639 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1640 local_irq_restore(flags);
1643 EXPORT_SYMBOL(dev_kfree_skb_irq);
1645 void dev_kfree_skb_any(struct sk_buff *skb)
1647 if (in_irq() || irqs_disabled())
1648 dev_kfree_skb_irq(skb);
1649 else
1650 dev_kfree_skb(skb);
1652 EXPORT_SYMBOL(dev_kfree_skb_any);
1656 * netif_device_detach - mark device as removed
1657 * @dev: network device
1659 * Mark device as removed from system and therefore no longer available.
1661 void netif_device_detach(struct net_device *dev)
1663 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1664 netif_running(dev)) {
1665 netif_tx_stop_all_queues(dev);
1668 EXPORT_SYMBOL(netif_device_detach);
1671 * netif_device_attach - mark device as attached
1672 * @dev: network device
1674 * Mark device as attached from system and restart if needed.
1676 void netif_device_attach(struct net_device *dev)
1678 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1679 netif_running(dev)) {
1680 netif_tx_wake_all_queues(dev);
1681 __netdev_watchdog_up(dev);
1684 EXPORT_SYMBOL(netif_device_attach);
1686 static bool can_checksum_protocol(unsigned long features, __be16 protocol)
1688 return ((features & NETIF_F_NO_CSUM) ||
1689 ((features & NETIF_F_V4_CSUM) &&
1690 protocol == htons(ETH_P_IP)) ||
1691 ((features & NETIF_F_V6_CSUM) &&
1692 protocol == htons(ETH_P_IPV6)) ||
1693 ((features & NETIF_F_FCOE_CRC) &&
1694 protocol == htons(ETH_P_FCOE)));
1697 static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)
1699 __be16 protocol = skb->protocol;
1700 int features = dev->features;
1702 if (vlan_tx_tag_present(skb)) {
1703 features &= dev->vlan_features;
1704 } else if (protocol == htons(ETH_P_8021Q)) {
1705 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1706 protocol = veh->h_vlan_encapsulated_proto;
1707 features &= dev->vlan_features;
1710 return can_checksum_protocol(features, protocol);
1714 * skb_dev_set -- assign a new device to a buffer
1715 * @skb: buffer for the new device
1716 * @dev: network device
1718 * If an skb is owned by a device already, we have to reset
1719 * all data private to the namespace a device belongs to
1720 * before assigning it a new device.
1722 #ifdef CONFIG_NET_NS
1723 void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1725 skb_dst_drop(skb);
1726 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1727 secpath_reset(skb);
1728 nf_reset(skb);
1729 skb_init_secmark(skb);
1730 skb->mark = 0;
1731 skb->priority = 0;
1732 skb->nf_trace = 0;
1733 skb->ipvs_property = 0;
1734 #ifdef CONFIG_NET_SCHED
1735 skb->tc_index = 0;
1736 #endif
1738 skb->dev = dev;
1740 EXPORT_SYMBOL(skb_set_dev);
1741 #endif /* CONFIG_NET_NS */
1744 * Invalidate hardware checksum when packet is to be mangled, and
1745 * complete checksum manually on outgoing path.
1747 int skb_checksum_help(struct sk_buff *skb)
1749 __wsum csum;
1750 int ret = 0, offset;
1752 if (skb->ip_summed == CHECKSUM_COMPLETE)
1753 goto out_set_summed;
1755 if (unlikely(skb_shinfo(skb)->gso_size)) {
1756 /* Let GSO fix up the checksum. */
1757 goto out_set_summed;
1760 offset = skb->csum_start - skb_headroom(skb);
1761 BUG_ON(offset >= skb_headlen(skb));
1762 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1764 offset += skb->csum_offset;
1765 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1767 if (skb_cloned(skb) &&
1768 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
1769 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1770 if (ret)
1771 goto out;
1774 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
1775 out_set_summed:
1776 skb->ip_summed = CHECKSUM_NONE;
1777 out:
1778 return ret;
1780 EXPORT_SYMBOL(skb_checksum_help);
1783 * skb_gso_segment - Perform segmentation on skb.
1784 * @skb: buffer to segment
1785 * @features: features for the output path (see dev->features)
1787 * This function segments the given skb and returns a list of segments.
1789 * It may return NULL if the skb requires no segmentation. This is
1790 * only possible when GSO is used for verifying header integrity.
1792 struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
1794 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1795 struct packet_type *ptype;
1796 __be16 type = skb->protocol;
1797 int err;
1799 if (type == htons(ETH_P_8021Q)) {
1800 struct vlan_ethhdr *veh;
1802 if (unlikely(!pskb_may_pull(skb, VLAN_ETH_HLEN)))
1803 return ERR_PTR(-EINVAL);
1805 veh = (struct vlan_ethhdr *)skb->data;
1806 type = veh->h_vlan_encapsulated_proto;
1809 skb_reset_mac_header(skb);
1810 skb->mac_len = skb->network_header - skb->mac_header;
1811 __skb_pull(skb, skb->mac_len);
1813 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1814 struct net_device *dev = skb->dev;
1815 struct ethtool_drvinfo info = {};
1817 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1818 dev->ethtool_ops->get_drvinfo(dev, &info);
1820 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "
1821 "ip_summed=%d",
1822 info.driver, dev ? dev->features : 0L,
1823 skb->sk ? skb->sk->sk_route_caps : 0L,
1824 skb->len, skb->data_len, skb->ip_summed);
1826 if (skb_header_cloned(skb) &&
1827 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1828 return ERR_PTR(err);
1831 rcu_read_lock();
1832 list_for_each_entry_rcu(ptype,
1833 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
1834 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
1835 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1836 err = ptype->gso_send_check(skb);
1837 segs = ERR_PTR(err);
1838 if (err || skb_gso_ok(skb, features))
1839 break;
1840 __skb_push(skb, (skb->data -
1841 skb_network_header(skb)));
1843 segs = ptype->gso_segment(skb, features);
1844 break;
1847 rcu_read_unlock();
1849 __skb_push(skb, skb->data - skb_mac_header(skb));
1851 return segs;
1853 EXPORT_SYMBOL(skb_gso_segment);
1855 /* Take action when hardware reception checksum errors are detected. */
1856 #ifdef CONFIG_BUG
1857 void netdev_rx_csum_fault(struct net_device *dev)
1859 if (net_ratelimit()) {
1860 printk(KERN_ERR "%s: hw csum failure.\n",
1861 dev ? dev->name : "<unknown>");
1862 dump_stack();
1865 EXPORT_SYMBOL(netdev_rx_csum_fault);
1866 #endif
1868 /* Actually, we should eliminate this check as soon as we know, that:
1869 * 1. IOMMU is present and allows to map all the memory.
1870 * 2. No high memory really exists on this machine.
1873 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1875 #ifdef CONFIG_HIGHMEM
1876 int i;
1877 if (!(dev->features & NETIF_F_HIGHDMA)) {
1878 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1879 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1880 return 1;
1883 if (PCI_DMA_BUS_IS_PHYS) {
1884 struct device *pdev = dev->dev.parent;
1886 if (!pdev)
1887 return 0;
1888 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1889 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1890 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1891 return 1;
1894 #endif
1895 return 0;
1898 struct dev_gso_cb {
1899 void (*destructor)(struct sk_buff *skb);
1902 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1904 static void dev_gso_skb_destructor(struct sk_buff *skb)
1906 struct dev_gso_cb *cb;
1908 do {
1909 struct sk_buff *nskb = skb->next;
1911 skb->next = nskb->next;
1912 nskb->next = NULL;
1913 kfree_skb(nskb);
1914 } while (skb->next);
1916 cb = DEV_GSO_CB(skb);
1917 if (cb->destructor)
1918 cb->destructor(skb);
1922 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1923 * @skb: buffer to segment
1925 * This function segments the given skb and stores the list of segments
1926 * in skb->next.
1928 static int dev_gso_segment(struct sk_buff *skb)
1930 struct net_device *dev = skb->dev;
1931 struct sk_buff *segs;
1932 int features = dev->features & ~(illegal_highdma(dev, skb) ?
1933 NETIF_F_SG : 0);
1935 segs = skb_gso_segment(skb, features);
1937 /* Verifying header integrity only. */
1938 if (!segs)
1939 return 0;
1941 if (IS_ERR(segs))
1942 return PTR_ERR(segs);
1944 skb->next = segs;
1945 DEV_GSO_CB(skb)->destructor = skb->destructor;
1946 skb->destructor = dev_gso_skb_destructor;
1948 return 0;
1952 * Try to orphan skb early, right before transmission by the device.
1953 * We cannot orphan skb if tx timestamp is requested or the sk-reference
1954 * is needed on driver level for other reasons, e.g. see net/can/raw.c
1956 static inline void skb_orphan_try(struct sk_buff *skb)
1958 struct sock *sk = skb->sk;
1960 if (sk && !skb_shinfo(skb)->tx_flags) {
1961 /* skb_tx_hash() wont be able to get sk.
1962 * We copy sk_hash into skb->rxhash
1964 if (!skb->rxhash)
1965 skb->rxhash = sk->sk_hash;
1966 skb_orphan(skb);
1971 * Returns true if either:
1972 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
1973 * 2. skb is fragmented and the device does not support SG, or if
1974 * at least one of fragments is in highmem and device does not
1975 * support DMA from it.
1977 static inline int skb_needs_linearize(struct sk_buff *skb,
1978 struct net_device *dev)
1980 int features = dev->features;
1982 if (skb->protocol == htons(ETH_P_8021Q) || vlan_tx_tag_present(skb))
1983 features &= dev->vlan_features;
1985 return skb_is_nonlinear(skb) &&
1986 ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
1987 (skb_shinfo(skb)->nr_frags && (!(features & NETIF_F_SG) ||
1988 illegal_highdma(dev, skb))));
1991 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1992 struct netdev_queue *txq)
1994 const struct net_device_ops *ops = dev->netdev_ops;
1995 int rc = NETDEV_TX_OK;
1997 if (likely(!skb->next)) {
1998 if (!list_empty(&ptype_all))
1999 dev_queue_xmit_nit(skb, dev);
2002 * If device doesnt need skb->dst, release it right now while
2003 * its hot in this cpu cache
2005 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2006 skb_dst_drop(skb);
2008 skb_orphan_try(skb);
2010 if (vlan_tx_tag_present(skb) &&
2011 !(dev->features & NETIF_F_HW_VLAN_TX)) {
2012 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2013 if (unlikely(!skb))
2014 goto out;
2016 skb->vlan_tci = 0;
2019 if (netif_needs_gso(dev, skb)) {
2020 if (unlikely(dev_gso_segment(skb)))
2021 goto out_kfree_skb;
2022 if (skb->next)
2023 goto gso;
2024 } else {
2025 if (skb_needs_linearize(skb, dev) &&
2026 __skb_linearize(skb))
2027 goto out_kfree_skb;
2029 /* If packet is not checksummed and device does not
2030 * support checksumming for this protocol, complete
2031 * checksumming here.
2033 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2034 skb_set_transport_header(skb, skb->csum_start -
2035 skb_headroom(skb));
2036 if (!dev_can_checksum(dev, skb) &&
2037 skb_checksum_help(skb))
2038 goto out_kfree_skb;
2042 rc = ops->ndo_start_xmit(skb, dev);
2043 trace_net_dev_xmit(skb, rc);
2044 if (rc == NETDEV_TX_OK)
2045 txq_trans_update(txq);
2046 return rc;
2049 gso:
2050 do {
2051 struct sk_buff *nskb = skb->next;
2053 skb->next = nskb->next;
2054 nskb->next = NULL;
2057 * If device doesnt need nskb->dst, release it right now while
2058 * its hot in this cpu cache
2060 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2061 skb_dst_drop(nskb);
2063 rc = ops->ndo_start_xmit(nskb, dev);
2064 trace_net_dev_xmit(nskb, rc);
2065 if (unlikely(rc != NETDEV_TX_OK)) {
2066 if (rc & ~NETDEV_TX_MASK)
2067 goto out_kfree_gso_skb;
2068 nskb->next = skb->next;
2069 skb->next = nskb;
2070 return rc;
2072 txq_trans_update(txq);
2073 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
2074 return NETDEV_TX_BUSY;
2075 } while (skb->next);
2077 out_kfree_gso_skb:
2078 if (likely(skb->next == NULL))
2079 skb->destructor = DEV_GSO_CB(skb)->destructor;
2080 out_kfree_skb:
2081 kfree_skb(skb);
2082 out:
2083 return rc;
2086 static u32 hashrnd __read_mostly;
2088 u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
2090 u32 hash;
2092 if (skb_rx_queue_recorded(skb)) {
2093 hash = skb_get_rx_queue(skb);
2094 while (unlikely(hash >= dev->real_num_tx_queues))
2095 hash -= dev->real_num_tx_queues;
2096 return hash;
2099 if (skb->sk && skb->sk->sk_hash)
2100 hash = skb->sk->sk_hash;
2101 else
2102 hash = (__force u16) skb->protocol ^ skb->rxhash;
2103 hash = jhash_1word(hash, hashrnd);
2105 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
2107 EXPORT_SYMBOL(skb_tx_hash);
2109 static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2111 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2112 if (net_ratelimit()) {
2113 pr_warning("%s selects TX queue %d, but "
2114 "real number of TX queues is %d\n",
2115 dev->name, queue_index, dev->real_num_tx_queues);
2117 return 0;
2119 return queue_index;
2122 static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2123 struct sk_buff *skb)
2125 int queue_index;
2126 const struct net_device_ops *ops = dev->netdev_ops;
2128 if (ops->ndo_select_queue) {
2129 queue_index = ops->ndo_select_queue(dev, skb);
2130 queue_index = dev_cap_txqueue(dev, queue_index);
2131 } else {
2132 struct sock *sk = skb->sk;
2133 queue_index = sk_tx_queue_get(sk);
2134 if (queue_index < 0 || queue_index >= dev->real_num_tx_queues) {
2136 queue_index = 0;
2137 if (dev->real_num_tx_queues > 1)
2138 queue_index = skb_tx_hash(dev, skb);
2140 if (sk) {
2141 struct dst_entry *dst = rcu_dereference_check(sk->sk_dst_cache, 1);
2143 if (dst && skb_dst(skb) == dst)
2144 sk_tx_queue_set(sk, queue_index);
2149 skb_set_queue_mapping(skb, queue_index);
2150 return netdev_get_tx_queue(dev, queue_index);
2153 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2154 struct net_device *dev,
2155 struct netdev_queue *txq)
2157 spinlock_t *root_lock = qdisc_lock(q);
2158 bool contended = qdisc_is_running(q);
2159 int rc;
2162 * Heuristic to force contended enqueues to serialize on a
2163 * separate lock before trying to get qdisc main lock.
2164 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2165 * and dequeue packets faster.
2167 if (unlikely(contended))
2168 spin_lock(&q->busylock);
2170 spin_lock(root_lock);
2171 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2172 kfree_skb(skb);
2173 rc = NET_XMIT_DROP;
2174 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2175 qdisc_run_begin(q)) {
2177 * This is a work-conserving queue; there are no old skbs
2178 * waiting to be sent out; and the qdisc is not running -
2179 * xmit the skb directly.
2181 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2182 skb_dst_force(skb);
2183 __qdisc_update_bstats(q, skb->len);
2184 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2185 if (unlikely(contended)) {
2186 spin_unlock(&q->busylock);
2187 contended = false;
2189 __qdisc_run(q);
2190 } else
2191 qdisc_run_end(q);
2193 rc = NET_XMIT_SUCCESS;
2194 } else {
2195 skb_dst_force(skb);
2196 rc = qdisc_enqueue_root(skb, q);
2197 if (qdisc_run_begin(q)) {
2198 if (unlikely(contended)) {
2199 spin_unlock(&q->busylock);
2200 contended = false;
2202 __qdisc_run(q);
2205 spin_unlock(root_lock);
2206 if (unlikely(contended))
2207 spin_unlock(&q->busylock);
2208 return rc;
2211 static DEFINE_PER_CPU(int, xmit_recursion);
2212 #define RECURSION_LIMIT 10
2215 * dev_queue_xmit - transmit a buffer
2216 * @skb: buffer to transmit
2218 * Queue a buffer for transmission to a network device. The caller must
2219 * have set the device and priority and built the buffer before calling
2220 * this function. The function can be called from an interrupt.
2222 * A negative errno code is returned on a failure. A success does not
2223 * guarantee the frame will be transmitted as it may be dropped due
2224 * to congestion or traffic shaping.
2226 * -----------------------------------------------------------------------------------
2227 * I notice this method can also return errors from the queue disciplines,
2228 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2229 * be positive.
2231 * Regardless of the return value, the skb is consumed, so it is currently
2232 * difficult to retry a send to this method. (You can bump the ref count
2233 * before sending to hold a reference for retry if you are careful.)
2235 * When calling this method, interrupts MUST be enabled. This is because
2236 * the BH enable code must have IRQs enabled so that it will not deadlock.
2237 * --BLG
2239 int dev_queue_xmit(struct sk_buff *skb)
2241 struct net_device *dev = skb->dev;
2242 struct netdev_queue *txq;
2243 struct Qdisc *q;
2244 int rc = -ENOMEM;
2246 /* Disable soft irqs for various locks below. Also
2247 * stops preemption for RCU.
2249 rcu_read_lock_bh();
2251 txq = dev_pick_tx(dev, skb);
2252 q = rcu_dereference_bh(txq->qdisc);
2254 #ifdef CONFIG_NET_CLS_ACT
2255 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2256 #endif
2257 trace_net_dev_queue(skb);
2258 if (q->enqueue) {
2259 rc = __dev_xmit_skb(skb, q, dev, txq);
2260 goto out;
2263 /* The device has no queue. Common case for software devices:
2264 loopback, all the sorts of tunnels...
2266 Really, it is unlikely that netif_tx_lock protection is necessary
2267 here. (f.e. loopback and IP tunnels are clean ignoring statistics
2268 counters.)
2269 However, it is possible, that they rely on protection
2270 made by us here.
2272 Check this and shot the lock. It is not prone from deadlocks.
2273 Either shot noqueue qdisc, it is even simpler 8)
2275 if (dev->flags & IFF_UP) {
2276 int cpu = smp_processor_id(); /* ok because BHs are off */
2278 if (txq->xmit_lock_owner != cpu) {
2280 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2281 goto recursion_alert;
2283 HARD_TX_LOCK(dev, txq, cpu);
2285 if (!netif_tx_queue_stopped(txq)) {
2286 __this_cpu_inc(xmit_recursion);
2287 rc = dev_hard_start_xmit(skb, dev, txq);
2288 __this_cpu_dec(xmit_recursion);
2289 if (dev_xmit_complete(rc)) {
2290 HARD_TX_UNLOCK(dev, txq);
2291 goto out;
2294 HARD_TX_UNLOCK(dev, txq);
2295 if (net_ratelimit())
2296 printk(KERN_CRIT "Virtual device %s asks to "
2297 "queue packet!\n", dev->name);
2298 } else {
2299 /* Recursion is detected! It is possible,
2300 * unfortunately
2302 recursion_alert:
2303 if (net_ratelimit())
2304 printk(KERN_CRIT "Dead loop on virtual device "
2305 "%s, fix it urgently!\n", dev->name);
2309 rc = -ENETDOWN;
2310 rcu_read_unlock_bh();
2312 kfree_skb(skb);
2313 return rc;
2314 out:
2315 rcu_read_unlock_bh();
2316 return rc;
2318 EXPORT_SYMBOL(dev_queue_xmit);
2321 /*=======================================================================
2322 Receiver routines
2323 =======================================================================*/
2325 int netdev_max_backlog __read_mostly = 1000;
2326 int netdev_tstamp_prequeue __read_mostly = 1;
2327 int netdev_budget __read_mostly = 300;
2328 int weight_p __read_mostly = 64; /* old backlog weight */
2330 /* Called with irq disabled */
2331 static inline void ____napi_schedule(struct softnet_data *sd,
2332 struct napi_struct *napi)
2334 list_add_tail(&napi->poll_list, &sd->poll_list);
2335 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2339 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2340 * and src/dst port numbers. Returns a non-zero hash number on success
2341 * and 0 on failure.
2343 __u32 __skb_get_rxhash(struct sk_buff *skb)
2345 int nhoff, hash = 0, poff;
2346 struct ipv6hdr *ip6;
2347 struct iphdr *ip;
2348 u8 ip_proto;
2349 u32 addr1, addr2, ihl;
2350 union {
2351 u32 v32;
2352 u16 v16[2];
2353 } ports;
2355 nhoff = skb_network_offset(skb);
2357 switch (skb->protocol) {
2358 case __constant_htons(ETH_P_IP):
2359 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2360 goto done;
2362 ip = (struct iphdr *) (skb->data + nhoff);
2363 if (ip->frag_off & htons(IP_MF | IP_OFFSET))
2364 ip_proto = 0;
2365 else
2366 ip_proto = ip->protocol;
2367 addr1 = (__force u32) ip->saddr;
2368 addr2 = (__force u32) ip->daddr;
2369 ihl = ip->ihl;
2370 break;
2371 case __constant_htons(ETH_P_IPV6):
2372 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2373 goto done;
2375 ip6 = (struct ipv6hdr *) (skb->data + nhoff);
2376 ip_proto = ip6->nexthdr;
2377 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2378 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2379 ihl = (40 >> 2);
2380 break;
2381 default:
2382 goto done;
2385 ports.v32 = 0;
2386 poff = proto_ports_offset(ip_proto);
2387 if (poff >= 0) {
2388 nhoff += ihl * 4 + poff;
2389 if (pskb_may_pull(skb, nhoff + 4)) {
2390 ports.v32 = * (__force u32 *) (skb->data + nhoff);
2391 if (ports.v16[1] < ports.v16[0])
2392 swap(ports.v16[0], ports.v16[1]);
2396 /* get a consistent hash (same value on both flow directions) */
2397 if (addr2 < addr1)
2398 swap(addr1, addr2);
2400 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2401 if (!hash)
2402 hash = 1;
2404 done:
2405 return hash;
2407 EXPORT_SYMBOL(__skb_get_rxhash);
2409 #ifdef CONFIG_RPS
2411 /* One global table that all flow-based protocols share. */
2412 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2413 EXPORT_SYMBOL(rps_sock_flow_table);
2416 * get_rps_cpu is called from netif_receive_skb and returns the target
2417 * CPU from the RPS map of the receiving queue for a given skb.
2418 * rcu_read_lock must be held on entry.
2420 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2421 struct rps_dev_flow **rflowp)
2423 struct netdev_rx_queue *rxqueue;
2424 struct rps_map *map;
2425 struct rps_dev_flow_table *flow_table;
2426 struct rps_sock_flow_table *sock_flow_table;
2427 int cpu = -1;
2428 u16 tcpu;
2430 if (skb_rx_queue_recorded(skb)) {
2431 u16 index = skb_get_rx_queue(skb);
2432 if (unlikely(index >= dev->real_num_rx_queues)) {
2433 WARN_ONCE(dev->real_num_rx_queues > 1,
2434 "%s received packet on queue %u, but number "
2435 "of RX queues is %u\n",
2436 dev->name, index, dev->real_num_rx_queues);
2437 goto done;
2439 rxqueue = dev->_rx + index;
2440 } else
2441 rxqueue = dev->_rx;
2443 map = rcu_dereference(rxqueue->rps_map);
2444 if (map) {
2445 if (map->len == 1) {
2446 tcpu = map->cpus[0];
2447 if (cpu_online(tcpu))
2448 cpu = tcpu;
2449 goto done;
2451 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) {
2452 goto done;
2455 skb_reset_network_header(skb);
2456 if (!skb_get_rxhash(skb))
2457 goto done;
2459 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2460 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2461 if (flow_table && sock_flow_table) {
2462 u16 next_cpu;
2463 struct rps_dev_flow *rflow;
2465 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2466 tcpu = rflow->cpu;
2468 next_cpu = sock_flow_table->ents[skb->rxhash &
2469 sock_flow_table->mask];
2472 * If the desired CPU (where last recvmsg was done) is
2473 * different from current CPU (one in the rx-queue flow
2474 * table entry), switch if one of the following holds:
2475 * - Current CPU is unset (equal to RPS_NO_CPU).
2476 * - Current CPU is offline.
2477 * - The current CPU's queue tail has advanced beyond the
2478 * last packet that was enqueued using this table entry.
2479 * This guarantees that all previous packets for the flow
2480 * have been dequeued, thus preserving in order delivery.
2482 if (unlikely(tcpu != next_cpu) &&
2483 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2484 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
2485 rflow->last_qtail)) >= 0)) {
2486 tcpu = rflow->cpu = next_cpu;
2487 if (tcpu != RPS_NO_CPU)
2488 rflow->last_qtail = per_cpu(softnet_data,
2489 tcpu).input_queue_head;
2491 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2492 *rflowp = rflow;
2493 cpu = tcpu;
2494 goto done;
2498 if (map) {
2499 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
2501 if (cpu_online(tcpu)) {
2502 cpu = tcpu;
2503 goto done;
2507 done:
2508 return cpu;
2511 /* Called from hardirq (IPI) context */
2512 static void rps_trigger_softirq(void *data)
2514 struct softnet_data *sd = data;
2516 ____napi_schedule(sd, &sd->backlog);
2517 sd->received_rps++;
2520 #endif /* CONFIG_RPS */
2523 * Check if this softnet_data structure is another cpu one
2524 * If yes, queue it to our IPI list and return 1
2525 * If no, return 0
2527 static int rps_ipi_queued(struct softnet_data *sd)
2529 #ifdef CONFIG_RPS
2530 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2532 if (sd != mysd) {
2533 sd->rps_ipi_next = mysd->rps_ipi_list;
2534 mysd->rps_ipi_list = sd;
2536 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2537 return 1;
2539 #endif /* CONFIG_RPS */
2540 return 0;
2544 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2545 * queue (may be a remote CPU queue).
2547 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2548 unsigned int *qtail)
2550 struct softnet_data *sd;
2551 unsigned long flags;
2553 sd = &per_cpu(softnet_data, cpu);
2555 local_irq_save(flags);
2557 rps_lock(sd);
2558 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2559 if (skb_queue_len(&sd->input_pkt_queue)) {
2560 enqueue:
2561 __skb_queue_tail(&sd->input_pkt_queue, skb);
2562 input_queue_tail_incr_save(sd, qtail);
2563 rps_unlock(sd);
2564 local_irq_restore(flags);
2565 return NET_RX_SUCCESS;
2568 /* Schedule NAPI for backlog device
2569 * We can use non atomic operation since we own the queue lock
2571 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
2572 if (!rps_ipi_queued(sd))
2573 ____napi_schedule(sd, &sd->backlog);
2575 goto enqueue;
2578 sd->dropped++;
2579 rps_unlock(sd);
2581 local_irq_restore(flags);
2583 atomic_long_inc(&skb->dev->rx_dropped);
2584 kfree_skb(skb);
2585 return NET_RX_DROP;
2589 * netif_rx - post buffer to the network code
2590 * @skb: buffer to post
2592 * This function receives a packet from a device driver and queues it for
2593 * the upper (protocol) levels to process. It always succeeds. The buffer
2594 * may be dropped during processing for congestion control or by the
2595 * protocol layers.
2597 * return values:
2598 * NET_RX_SUCCESS (no congestion)
2599 * NET_RX_DROP (packet was dropped)
2603 int netif_rx(struct sk_buff *skb)
2605 int ret;
2607 /* if netpoll wants it, pretend we never saw it */
2608 if (netpoll_rx(skb))
2609 return NET_RX_DROP;
2611 if (netdev_tstamp_prequeue)
2612 net_timestamp_check(skb);
2614 trace_netif_rx(skb);
2615 #ifdef CONFIG_RPS
2617 struct rps_dev_flow voidflow, *rflow = &voidflow;
2618 int cpu;
2620 preempt_disable();
2621 rcu_read_lock();
2623 cpu = get_rps_cpu(skb->dev, skb, &rflow);
2624 if (cpu < 0)
2625 cpu = smp_processor_id();
2627 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2629 rcu_read_unlock();
2630 preempt_enable();
2632 #else
2634 unsigned int qtail;
2635 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2636 put_cpu();
2638 #endif
2639 return ret;
2641 EXPORT_SYMBOL(netif_rx);
2643 int netif_rx_ni(struct sk_buff *skb)
2645 int err;
2647 preempt_disable();
2648 err = netif_rx(skb);
2649 if (local_softirq_pending())
2650 do_softirq();
2651 preempt_enable();
2653 return err;
2655 EXPORT_SYMBOL(netif_rx_ni);
2657 static void net_tx_action(struct softirq_action *h)
2659 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2661 if (sd->completion_queue) {
2662 struct sk_buff *clist;
2664 local_irq_disable();
2665 clist = sd->completion_queue;
2666 sd->completion_queue = NULL;
2667 local_irq_enable();
2669 while (clist) {
2670 struct sk_buff *skb = clist;
2671 clist = clist->next;
2673 WARN_ON(atomic_read(&skb->users));
2674 trace_kfree_skb(skb, net_tx_action);
2675 __kfree_skb(skb);
2679 if (sd->output_queue) {
2680 struct Qdisc *head;
2682 local_irq_disable();
2683 head = sd->output_queue;
2684 sd->output_queue = NULL;
2685 sd->output_queue_tailp = &sd->output_queue;
2686 local_irq_enable();
2688 while (head) {
2689 struct Qdisc *q = head;
2690 spinlock_t *root_lock;
2692 head = head->next_sched;
2694 root_lock = qdisc_lock(q);
2695 if (spin_trylock(root_lock)) {
2696 smp_mb__before_clear_bit();
2697 clear_bit(__QDISC_STATE_SCHED,
2698 &q->state);
2699 qdisc_run(q);
2700 spin_unlock(root_lock);
2701 } else {
2702 if (!test_bit(__QDISC_STATE_DEACTIVATED,
2703 &q->state)) {
2704 __netif_reschedule(q);
2705 } else {
2706 smp_mb__before_clear_bit();
2707 clear_bit(__QDISC_STATE_SCHED,
2708 &q->state);
2715 static inline int deliver_skb(struct sk_buff *skb,
2716 struct packet_type *pt_prev,
2717 struct net_device *orig_dev)
2719 atomic_inc(&skb->users);
2720 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2723 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2724 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
2725 /* This hook is defined here for ATM LANE */
2726 int (*br_fdb_test_addr_hook)(struct net_device *dev,
2727 unsigned char *addr) __read_mostly;
2728 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
2729 #endif
2731 #ifdef CONFIG_NET_CLS_ACT
2732 /* TODO: Maybe we should just force sch_ingress to be compiled in
2733 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2734 * a compare and 2 stores extra right now if we dont have it on
2735 * but have CONFIG_NET_CLS_ACT
2736 * NOTE: This doesnt stop any functionality; if you dont have
2737 * the ingress scheduler, you just cant add policies on ingress.
2740 static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
2742 struct net_device *dev = skb->dev;
2743 u32 ttl = G_TC_RTTL(skb->tc_verd);
2744 int result = TC_ACT_OK;
2745 struct Qdisc *q;
2747 if (unlikely(MAX_RED_LOOP < ttl++)) {
2748 if (net_ratelimit())
2749 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
2750 skb->skb_iif, dev->ifindex);
2751 return TC_ACT_SHOT;
2754 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2755 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2757 q = rxq->qdisc;
2758 if (q != &noop_qdisc) {
2759 spin_lock(qdisc_lock(q));
2760 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2761 result = qdisc_enqueue_root(skb, q);
2762 spin_unlock(qdisc_lock(q));
2765 return result;
2768 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2769 struct packet_type **pt_prev,
2770 int *ret, struct net_device *orig_dev)
2772 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
2774 if (!rxq || rxq->qdisc == &noop_qdisc)
2775 goto out;
2777 if (*pt_prev) {
2778 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2779 *pt_prev = NULL;
2782 switch (ing_filter(skb, rxq)) {
2783 case TC_ACT_SHOT:
2784 case TC_ACT_STOLEN:
2785 kfree_skb(skb);
2786 return NULL;
2789 out:
2790 skb->tc_verd = 0;
2791 return skb;
2793 #endif
2796 * netdev_rx_handler_register - register receive handler
2797 * @dev: device to register a handler for
2798 * @rx_handler: receive handler to register
2799 * @rx_handler_data: data pointer that is used by rx handler
2801 * Register a receive hander for a device. This handler will then be
2802 * called from __netif_receive_skb. A negative errno code is returned
2803 * on a failure.
2805 * The caller must hold the rtnl_mutex.
2807 int netdev_rx_handler_register(struct net_device *dev,
2808 rx_handler_func_t *rx_handler,
2809 void *rx_handler_data)
2811 ASSERT_RTNL();
2813 if (dev->rx_handler)
2814 return -EBUSY;
2816 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
2817 rcu_assign_pointer(dev->rx_handler, rx_handler);
2819 return 0;
2821 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
2824 * netdev_rx_handler_unregister - unregister receive handler
2825 * @dev: device to unregister a handler from
2827 * Unregister a receive hander from a device.
2829 * The caller must hold the rtnl_mutex.
2831 void netdev_rx_handler_unregister(struct net_device *dev)
2834 ASSERT_RTNL();
2835 rcu_assign_pointer(dev->rx_handler, NULL);
2836 rcu_assign_pointer(dev->rx_handler_data, NULL);
2838 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
2840 static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,
2841 struct net_device *master)
2843 if (skb->pkt_type == PACKET_HOST) {
2844 u16 *dest = (u16 *) eth_hdr(skb)->h_dest;
2846 memcpy(dest, master->dev_addr, ETH_ALEN);
2850 /* On bonding slaves other than the currently active slave, suppress
2851 * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
2852 * ARP on active-backup slaves with arp_validate enabled.
2854 int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)
2856 struct net_device *dev = skb->dev;
2858 if (master->priv_flags & IFF_MASTER_ARPMON)
2859 dev->last_rx = jiffies;
2861 if ((master->priv_flags & IFF_MASTER_ALB) &&
2862 (master->priv_flags & IFF_BRIDGE_PORT)) {
2863 /* Do address unmangle. The local destination address
2864 * will be always the one master has. Provides the right
2865 * functionality in a bridge.
2867 skb_bond_set_mac_by_master(skb, master);
2870 if (dev->priv_flags & IFF_SLAVE_INACTIVE) {
2871 if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&
2872 skb->protocol == __cpu_to_be16(ETH_P_ARP))
2873 return 0;
2875 if (master->priv_flags & IFF_MASTER_ALB) {
2876 if (skb->pkt_type != PACKET_BROADCAST &&
2877 skb->pkt_type != PACKET_MULTICAST)
2878 return 0;
2880 if (master->priv_flags & IFF_MASTER_8023AD &&
2881 skb->protocol == __cpu_to_be16(ETH_P_SLOW))
2882 return 0;
2884 return 1;
2886 return 0;
2888 EXPORT_SYMBOL(__skb_bond_should_drop);
2890 static int __netif_receive_skb(struct sk_buff *skb)
2892 struct packet_type *ptype, *pt_prev;
2893 rx_handler_func_t *rx_handler;
2894 struct net_device *orig_dev;
2895 struct net_device *master;
2896 struct net_device *null_or_orig;
2897 struct net_device *orig_or_bond;
2898 int ret = NET_RX_DROP;
2899 __be16 type;
2901 if (!netdev_tstamp_prequeue)
2902 net_timestamp_check(skb);
2904 trace_netif_receive_skb(skb);
2906 /* if we've gotten here through NAPI, check netpoll */
2907 if (netpoll_receive_skb(skb))
2908 return NET_RX_DROP;
2910 if (!skb->skb_iif)
2911 skb->skb_iif = skb->dev->ifindex;
2914 * bonding note: skbs received on inactive slaves should only
2915 * be delivered to pkt handlers that are exact matches. Also
2916 * the deliver_no_wcard flag will be set. If packet handlers
2917 * are sensitive to duplicate packets these skbs will need to
2918 * be dropped at the handler.
2920 null_or_orig = NULL;
2921 orig_dev = skb->dev;
2922 master = ACCESS_ONCE(orig_dev->master);
2923 if (skb->deliver_no_wcard)
2924 null_or_orig = orig_dev;
2925 else if (master) {
2926 if (skb_bond_should_drop(skb, master)) {
2927 skb->deliver_no_wcard = 1;
2928 null_or_orig = orig_dev; /* deliver only exact match */
2929 } else
2930 skb->dev = master;
2933 __this_cpu_inc(softnet_data.processed);
2934 skb_reset_network_header(skb);
2935 skb_reset_transport_header(skb);
2936 skb->mac_len = skb->network_header - skb->mac_header;
2938 pt_prev = NULL;
2940 rcu_read_lock();
2942 #ifdef CONFIG_NET_CLS_ACT
2943 if (skb->tc_verd & TC_NCLS) {
2944 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
2945 goto ncls;
2947 #endif
2949 list_for_each_entry_rcu(ptype, &ptype_all, list) {
2950 if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2951 ptype->dev == orig_dev) {
2952 if (pt_prev)
2953 ret = deliver_skb(skb, pt_prev, orig_dev);
2954 pt_prev = ptype;
2958 #ifdef CONFIG_NET_CLS_ACT
2959 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
2960 if (!skb)
2961 goto out;
2962 ncls:
2963 #endif
2965 /* Handle special case of bridge or macvlan */
2966 rx_handler = rcu_dereference(skb->dev->rx_handler);
2967 if (rx_handler) {
2968 if (pt_prev) {
2969 ret = deliver_skb(skb, pt_prev, orig_dev);
2970 pt_prev = NULL;
2972 skb = rx_handler(skb);
2973 if (!skb)
2974 goto out;
2977 if (vlan_tx_tag_present(skb)) {
2978 if (pt_prev) {
2979 ret = deliver_skb(skb, pt_prev, orig_dev);
2980 pt_prev = NULL;
2982 if (vlan_hwaccel_do_receive(&skb)) {
2983 ret = __netif_receive_skb(skb);
2984 goto out;
2985 } else if (unlikely(!skb))
2986 goto out;
2990 * Make sure frames received on VLAN interfaces stacked on
2991 * bonding interfaces still make their way to any base bonding
2992 * device that may have registered for a specific ptype. The
2993 * handler may have to adjust skb->dev and orig_dev.
2995 orig_or_bond = orig_dev;
2996 if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&
2997 (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {
2998 orig_or_bond = vlan_dev_real_dev(skb->dev);
3001 type = skb->protocol;
3002 list_for_each_entry_rcu(ptype,
3003 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3004 if (ptype->type == type && (ptype->dev == null_or_orig ||
3005 ptype->dev == skb->dev || ptype->dev == orig_dev ||
3006 ptype->dev == orig_or_bond)) {
3007 if (pt_prev)
3008 ret = deliver_skb(skb, pt_prev, orig_dev);
3009 pt_prev = ptype;
3013 if (pt_prev) {
3014 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3015 } else {
3016 atomic_long_inc(&skb->dev->rx_dropped);
3017 kfree_skb(skb);
3018 /* Jamal, now you will not able to escape explaining
3019 * me how you were going to use this. :-)
3021 ret = NET_RX_DROP;
3024 out:
3025 rcu_read_unlock();
3026 return ret;
3030 * netif_receive_skb - process receive buffer from network
3031 * @skb: buffer to process
3033 * netif_receive_skb() is the main receive data processing function.
3034 * It always succeeds. The buffer may be dropped during processing
3035 * for congestion control or by the protocol layers.
3037 * This function may only be called from softirq context and interrupts
3038 * should be enabled.
3040 * Return values (usually ignored):
3041 * NET_RX_SUCCESS: no congestion
3042 * NET_RX_DROP: packet was dropped
3044 int netif_receive_skb(struct sk_buff *skb)
3046 if (netdev_tstamp_prequeue)
3047 net_timestamp_check(skb);
3049 if (skb_defer_rx_timestamp(skb))
3050 return NET_RX_SUCCESS;
3052 #ifdef CONFIG_RPS
3054 struct rps_dev_flow voidflow, *rflow = &voidflow;
3055 int cpu, ret;
3057 rcu_read_lock();
3059 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3061 if (cpu >= 0) {
3062 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3063 rcu_read_unlock();
3064 } else {
3065 rcu_read_unlock();
3066 ret = __netif_receive_skb(skb);
3069 return ret;
3071 #else
3072 return __netif_receive_skb(skb);
3073 #endif
3075 EXPORT_SYMBOL(netif_receive_skb);
3077 /* Network device is going away, flush any packets still pending
3078 * Called with irqs disabled.
3080 static void flush_backlog(void *arg)
3082 struct net_device *dev = arg;
3083 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3084 struct sk_buff *skb, *tmp;
3086 rps_lock(sd);
3087 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3088 if (skb->dev == dev) {
3089 __skb_unlink(skb, &sd->input_pkt_queue);
3090 kfree_skb(skb);
3091 input_queue_head_incr(sd);
3094 rps_unlock(sd);
3096 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3097 if (skb->dev == dev) {
3098 __skb_unlink(skb, &sd->process_queue);
3099 kfree_skb(skb);
3100 input_queue_head_incr(sd);
3105 static int napi_gro_complete(struct sk_buff *skb)
3107 struct packet_type *ptype;
3108 __be16 type = skb->protocol;
3109 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3110 int err = -ENOENT;
3112 if (NAPI_GRO_CB(skb)->count == 1) {
3113 skb_shinfo(skb)->gso_size = 0;
3114 goto out;
3117 rcu_read_lock();
3118 list_for_each_entry_rcu(ptype, head, list) {
3119 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3120 continue;
3122 err = ptype->gro_complete(skb);
3123 break;
3125 rcu_read_unlock();
3127 if (err) {
3128 WARN_ON(&ptype->list == head);
3129 kfree_skb(skb);
3130 return NET_RX_SUCCESS;
3133 out:
3134 return netif_receive_skb(skb);
3137 inline void napi_gro_flush(struct napi_struct *napi)
3139 struct sk_buff *skb, *next;
3141 for (skb = napi->gro_list; skb; skb = next) {
3142 next = skb->next;
3143 skb->next = NULL;
3144 napi_gro_complete(skb);
3147 napi->gro_count = 0;
3148 napi->gro_list = NULL;
3150 EXPORT_SYMBOL(napi_gro_flush);
3152 enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3154 struct sk_buff **pp = NULL;
3155 struct packet_type *ptype;
3156 __be16 type = skb->protocol;
3157 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3158 int same_flow;
3159 int mac_len;
3160 enum gro_result ret;
3162 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3163 goto normal;
3165 if (skb_is_gso(skb) || skb_has_frag_list(skb))
3166 goto normal;
3168 rcu_read_lock();
3169 list_for_each_entry_rcu(ptype, head, list) {
3170 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3171 continue;
3173 skb_set_network_header(skb, skb_gro_offset(skb));
3174 mac_len = skb->network_header - skb->mac_header;
3175 skb->mac_len = mac_len;
3176 NAPI_GRO_CB(skb)->same_flow = 0;
3177 NAPI_GRO_CB(skb)->flush = 0;
3178 NAPI_GRO_CB(skb)->free = 0;
3180 pp = ptype->gro_receive(&napi->gro_list, skb);
3181 break;
3183 rcu_read_unlock();
3185 if (&ptype->list == head)
3186 goto normal;
3188 same_flow = NAPI_GRO_CB(skb)->same_flow;
3189 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3191 if (pp) {
3192 struct sk_buff *nskb = *pp;
3194 *pp = nskb->next;
3195 nskb->next = NULL;
3196 napi_gro_complete(nskb);
3197 napi->gro_count--;
3200 if (same_flow)
3201 goto ok;
3203 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3204 goto normal;
3206 napi->gro_count++;
3207 NAPI_GRO_CB(skb)->count = 1;
3208 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3209 skb->next = napi->gro_list;
3210 napi->gro_list = skb;
3211 ret = GRO_HELD;
3213 pull:
3214 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3215 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3217 BUG_ON(skb->end - skb->tail < grow);
3219 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3221 skb->tail += grow;
3222 skb->data_len -= grow;
3224 skb_shinfo(skb)->frags[0].page_offset += grow;
3225 skb_shinfo(skb)->frags[0].size -= grow;
3227 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3228 put_page(skb_shinfo(skb)->frags[0].page);
3229 memmove(skb_shinfo(skb)->frags,
3230 skb_shinfo(skb)->frags + 1,
3231 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
3236 return ret;
3238 normal:
3239 ret = GRO_NORMAL;
3240 goto pull;
3242 EXPORT_SYMBOL(dev_gro_receive);
3244 static inline gro_result_t
3245 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3247 struct sk_buff *p;
3249 for (p = napi->gro_list; p; p = p->next) {
3250 unsigned long diffs;
3252 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3253 diffs |= p->vlan_tci ^ skb->vlan_tci;
3254 diffs |= compare_ether_header(skb_mac_header(p),
3255 skb_gro_mac_header(skb));
3256 NAPI_GRO_CB(p)->same_flow = !diffs;
3257 NAPI_GRO_CB(p)->flush = 0;
3260 return dev_gro_receive(napi, skb);
3263 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
3265 switch (ret) {
3266 case GRO_NORMAL:
3267 if (netif_receive_skb(skb))
3268 ret = GRO_DROP;
3269 break;
3271 case GRO_DROP:
3272 case GRO_MERGED_FREE:
3273 kfree_skb(skb);
3274 break;
3276 case GRO_HELD:
3277 case GRO_MERGED:
3278 break;
3281 return ret;
3283 EXPORT_SYMBOL(napi_skb_finish);
3285 void skb_gro_reset_offset(struct sk_buff *skb)
3287 NAPI_GRO_CB(skb)->data_offset = 0;
3288 NAPI_GRO_CB(skb)->frag0 = NULL;
3289 NAPI_GRO_CB(skb)->frag0_len = 0;
3291 if (skb->mac_header == skb->tail &&
3292 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
3293 NAPI_GRO_CB(skb)->frag0 =
3294 page_address(skb_shinfo(skb)->frags[0].page) +
3295 skb_shinfo(skb)->frags[0].page_offset;
3296 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3299 EXPORT_SYMBOL(skb_gro_reset_offset);
3301 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3303 skb_gro_reset_offset(skb);
3305 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
3307 EXPORT_SYMBOL(napi_gro_receive);
3309 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3311 __skb_pull(skb, skb_headlen(skb));
3312 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
3313 skb->vlan_tci = 0;
3315 napi->skb = skb;
3318 struct sk_buff *napi_get_frags(struct napi_struct *napi)
3320 struct sk_buff *skb = napi->skb;
3322 if (!skb) {
3323 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3324 if (skb)
3325 napi->skb = skb;
3327 return skb;
3329 EXPORT_SYMBOL(napi_get_frags);
3331 gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3332 gro_result_t ret)
3334 switch (ret) {
3335 case GRO_NORMAL:
3336 case GRO_HELD:
3337 skb->protocol = eth_type_trans(skb, skb->dev);
3339 if (ret == GRO_HELD)
3340 skb_gro_pull(skb, -ETH_HLEN);
3341 else if (netif_receive_skb(skb))
3342 ret = GRO_DROP;
3343 break;
3345 case GRO_DROP:
3346 case GRO_MERGED_FREE:
3347 napi_reuse_skb(napi, skb);
3348 break;
3350 case GRO_MERGED:
3351 break;
3354 return ret;
3356 EXPORT_SYMBOL(napi_frags_finish);
3358 struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3360 struct sk_buff *skb = napi->skb;
3361 struct ethhdr *eth;
3362 unsigned int hlen;
3363 unsigned int off;
3365 napi->skb = NULL;
3367 skb_reset_mac_header(skb);
3368 skb_gro_reset_offset(skb);
3370 off = skb_gro_offset(skb);
3371 hlen = off + sizeof(*eth);
3372 eth = skb_gro_header_fast(skb, off);
3373 if (skb_gro_header_hard(skb, hlen)) {
3374 eth = skb_gro_header_slow(skb, hlen, off);
3375 if (unlikely(!eth)) {
3376 napi_reuse_skb(napi, skb);
3377 skb = NULL;
3378 goto out;
3382 skb_gro_pull(skb, sizeof(*eth));
3385 * This works because the only protocols we care about don't require
3386 * special handling. We'll fix it up properly at the end.
3388 skb->protocol = eth->h_proto;
3390 out:
3391 return skb;
3393 EXPORT_SYMBOL(napi_frags_skb);
3395 gro_result_t napi_gro_frags(struct napi_struct *napi)
3397 struct sk_buff *skb = napi_frags_skb(napi);
3399 if (!skb)
3400 return GRO_DROP;
3402 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
3404 EXPORT_SYMBOL(napi_gro_frags);
3407 * net_rps_action sends any pending IPI's for rps.
3408 * Note: called with local irq disabled, but exits with local irq enabled.
3410 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3412 #ifdef CONFIG_RPS
3413 struct softnet_data *remsd = sd->rps_ipi_list;
3415 if (remsd) {
3416 sd->rps_ipi_list = NULL;
3418 local_irq_enable();
3420 /* Send pending IPI's to kick RPS processing on remote cpus. */
3421 while (remsd) {
3422 struct softnet_data *next = remsd->rps_ipi_next;
3424 if (cpu_online(remsd->cpu))
3425 __smp_call_function_single(remsd->cpu,
3426 &remsd->csd, 0);
3427 remsd = next;
3429 } else
3430 #endif
3431 local_irq_enable();
3434 static int process_backlog(struct napi_struct *napi, int quota)
3436 int work = 0;
3437 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
3439 #ifdef CONFIG_RPS
3440 /* Check if we have pending ipi, its better to send them now,
3441 * not waiting net_rx_action() end.
3443 if (sd->rps_ipi_list) {
3444 local_irq_disable();
3445 net_rps_action_and_irq_enable(sd);
3447 #endif
3448 napi->weight = weight_p;
3449 local_irq_disable();
3450 while (work < quota) {
3451 struct sk_buff *skb;
3452 unsigned int qlen;
3454 while ((skb = __skb_dequeue(&sd->process_queue))) {
3455 local_irq_enable();
3456 __netif_receive_skb(skb);
3457 local_irq_disable();
3458 input_queue_head_incr(sd);
3459 if (++work >= quota) {
3460 local_irq_enable();
3461 return work;
3465 rps_lock(sd);
3466 qlen = skb_queue_len(&sd->input_pkt_queue);
3467 if (qlen)
3468 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3469 &sd->process_queue);
3471 if (qlen < quota - work) {
3473 * Inline a custom version of __napi_complete().
3474 * only current cpu owns and manipulates this napi,
3475 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3476 * we can use a plain write instead of clear_bit(),
3477 * and we dont need an smp_mb() memory barrier.
3479 list_del(&napi->poll_list);
3480 napi->state = 0;
3482 quota = work + qlen;
3484 rps_unlock(sd);
3486 local_irq_enable();
3488 return work;
3492 * __napi_schedule - schedule for receive
3493 * @n: entry to schedule
3495 * The entry's receive function will be scheduled to run
3497 void __napi_schedule(struct napi_struct *n)
3499 unsigned long flags;
3501 local_irq_save(flags);
3502 ____napi_schedule(&__get_cpu_var(softnet_data), n);
3503 local_irq_restore(flags);
3505 EXPORT_SYMBOL(__napi_schedule);
3507 void __napi_complete(struct napi_struct *n)
3509 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3510 BUG_ON(n->gro_list);
3512 list_del(&n->poll_list);
3513 smp_mb__before_clear_bit();
3514 clear_bit(NAPI_STATE_SCHED, &n->state);
3516 EXPORT_SYMBOL(__napi_complete);
3518 void napi_complete(struct napi_struct *n)
3520 unsigned long flags;
3523 * don't let napi dequeue from the cpu poll list
3524 * just in case its running on a different cpu
3526 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3527 return;
3529 napi_gro_flush(n);
3530 local_irq_save(flags);
3531 __napi_complete(n);
3532 local_irq_restore(flags);
3534 EXPORT_SYMBOL(napi_complete);
3536 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3537 int (*poll)(struct napi_struct *, int), int weight)
3539 INIT_LIST_HEAD(&napi->poll_list);
3540 napi->gro_count = 0;
3541 napi->gro_list = NULL;
3542 napi->skb = NULL;
3543 napi->poll = poll;
3544 napi->weight = weight;
3545 list_add(&napi->dev_list, &dev->napi_list);
3546 napi->dev = dev;
3547 #ifdef CONFIG_NETPOLL
3548 spin_lock_init(&napi->poll_lock);
3549 napi->poll_owner = -1;
3550 #endif
3551 set_bit(NAPI_STATE_SCHED, &napi->state);
3553 EXPORT_SYMBOL(netif_napi_add);
3555 void netif_napi_del(struct napi_struct *napi)
3557 struct sk_buff *skb, *next;
3559 list_del_init(&napi->dev_list);
3560 napi_free_frags(napi);
3562 for (skb = napi->gro_list; skb; skb = next) {
3563 next = skb->next;
3564 skb->next = NULL;
3565 kfree_skb(skb);
3568 napi->gro_list = NULL;
3569 napi->gro_count = 0;
3571 EXPORT_SYMBOL(netif_napi_del);
3573 static void net_rx_action(struct softirq_action *h)
3575 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3576 unsigned long time_limit = jiffies + 2;
3577 int budget = netdev_budget;
3578 void *have;
3580 local_irq_disable();
3582 while (!list_empty(&sd->poll_list)) {
3583 struct napi_struct *n;
3584 int work, weight;
3586 /* If softirq window is exhuasted then punt.
3587 * Allow this to run for 2 jiffies since which will allow
3588 * an average latency of 1.5/HZ.
3590 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
3591 goto softnet_break;
3593 local_irq_enable();
3595 /* Even though interrupts have been re-enabled, this
3596 * access is safe because interrupts can only add new
3597 * entries to the tail of this list, and only ->poll()
3598 * calls can remove this head entry from the list.
3600 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
3602 have = netpoll_poll_lock(n);
3604 weight = n->weight;
3606 /* This NAPI_STATE_SCHED test is for avoiding a race
3607 * with netpoll's poll_napi(). Only the entity which
3608 * obtains the lock and sees NAPI_STATE_SCHED set will
3609 * actually make the ->poll() call. Therefore we avoid
3610 * accidently calling ->poll() when NAPI is not scheduled.
3612 work = 0;
3613 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
3614 work = n->poll(n, weight);
3615 trace_napi_poll(n);
3618 WARN_ON_ONCE(work > weight);
3620 budget -= work;
3622 local_irq_disable();
3624 /* Drivers must not modify the NAPI state if they
3625 * consume the entire weight. In such cases this code
3626 * still "owns" the NAPI instance and therefore can
3627 * move the instance around on the list at-will.
3629 if (unlikely(work == weight)) {
3630 if (unlikely(napi_disable_pending(n))) {
3631 local_irq_enable();
3632 napi_complete(n);
3633 local_irq_disable();
3634 } else
3635 list_move_tail(&n->poll_list, &sd->poll_list);
3638 netpoll_poll_unlock(have);
3640 out:
3641 net_rps_action_and_irq_enable(sd);
3643 #ifdef CONFIG_NET_DMA
3645 * There may not be any more sk_buffs coming right now, so push
3646 * any pending DMA copies to hardware
3648 dma_issue_pending_all();
3649 #endif
3651 return;
3653 softnet_break:
3654 sd->time_squeeze++;
3655 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3656 goto out;
3659 static gifconf_func_t *gifconf_list[NPROTO];
3662 * register_gifconf - register a SIOCGIF handler
3663 * @family: Address family
3664 * @gifconf: Function handler
3666 * Register protocol dependent address dumping routines. The handler
3667 * that is passed must not be freed or reused until it has been replaced
3668 * by another handler.
3670 int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
3672 if (family >= NPROTO)
3673 return -EINVAL;
3674 gifconf_list[family] = gifconf;
3675 return 0;
3677 EXPORT_SYMBOL(register_gifconf);
3681 * Map an interface index to its name (SIOCGIFNAME)
3685 * We need this ioctl for efficient implementation of the
3686 * if_indextoname() function required by the IPv6 API. Without
3687 * it, we would have to search all the interfaces to find a
3688 * match. --pb
3691 static int dev_ifname(struct net *net, struct ifreq __user *arg)
3693 struct net_device *dev;
3694 struct ifreq ifr;
3697 * Fetch the caller's info block.
3700 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3701 return -EFAULT;
3703 rcu_read_lock();
3704 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
3705 if (!dev) {
3706 rcu_read_unlock();
3707 return -ENODEV;
3710 strcpy(ifr.ifr_name, dev->name);
3711 rcu_read_unlock();
3713 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3714 return -EFAULT;
3715 return 0;
3719 * Perform a SIOCGIFCONF call. This structure will change
3720 * size eventually, and there is nothing I can do about it.
3721 * Thus we will need a 'compatibility mode'.
3724 static int dev_ifconf(struct net *net, char __user *arg)
3726 struct ifconf ifc;
3727 struct net_device *dev;
3728 char __user *pos;
3729 int len;
3730 int total;
3731 int i;
3734 * Fetch the caller's info block.
3737 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3738 return -EFAULT;
3740 pos = ifc.ifc_buf;
3741 len = ifc.ifc_len;
3744 * Loop over the interfaces, and write an info block for each.
3747 total = 0;
3748 for_each_netdev(net, dev) {
3749 for (i = 0; i < NPROTO; i++) {
3750 if (gifconf_list[i]) {
3751 int done;
3752 if (!pos)
3753 done = gifconf_list[i](dev, NULL, 0);
3754 else
3755 done = gifconf_list[i](dev, pos + total,
3756 len - total);
3757 if (done < 0)
3758 return -EFAULT;
3759 total += done;
3765 * All done. Write the updated control block back to the caller.
3767 ifc.ifc_len = total;
3770 * Both BSD and Solaris return 0 here, so we do too.
3772 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3775 #ifdef CONFIG_PROC_FS
3777 * This is invoked by the /proc filesystem handler to display a device
3778 * in detail.
3780 void *dev_seq_start(struct seq_file *seq, loff_t *pos)
3781 __acquires(RCU)
3783 struct net *net = seq_file_net(seq);
3784 loff_t off;
3785 struct net_device *dev;
3787 rcu_read_lock();
3788 if (!*pos)
3789 return SEQ_START_TOKEN;
3791 off = 1;
3792 for_each_netdev_rcu(net, dev)
3793 if (off++ == *pos)
3794 return dev;
3796 return NULL;
3799 void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3801 struct net_device *dev = (v == SEQ_START_TOKEN) ?
3802 first_net_device(seq_file_net(seq)) :
3803 next_net_device((struct net_device *)v);
3805 ++*pos;
3806 return rcu_dereference(dev);
3809 void dev_seq_stop(struct seq_file *seq, void *v)
3810 __releases(RCU)
3812 rcu_read_unlock();
3815 static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
3817 struct rtnl_link_stats64 temp;
3818 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
3820 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
3821 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
3822 dev->name, stats->rx_bytes, stats->rx_packets,
3823 stats->rx_errors,
3824 stats->rx_dropped + stats->rx_missed_errors,
3825 stats->rx_fifo_errors,
3826 stats->rx_length_errors + stats->rx_over_errors +
3827 stats->rx_crc_errors + stats->rx_frame_errors,
3828 stats->rx_compressed, stats->multicast,
3829 stats->tx_bytes, stats->tx_packets,
3830 stats->tx_errors, stats->tx_dropped,
3831 stats->tx_fifo_errors, stats->collisions,
3832 stats->tx_carrier_errors +
3833 stats->tx_aborted_errors +
3834 stats->tx_window_errors +
3835 stats->tx_heartbeat_errors,
3836 stats->tx_compressed);
3840 * Called from the PROCfs module. This now uses the new arbitrary sized
3841 * /proc/net interface to create /proc/net/dev
3843 static int dev_seq_show(struct seq_file *seq, void *v)
3845 if (v == SEQ_START_TOKEN)
3846 seq_puts(seq, "Inter-| Receive "
3847 " | Transmit\n"
3848 " face |bytes packets errs drop fifo frame "
3849 "compressed multicast|bytes packets errs "
3850 "drop fifo colls carrier compressed\n");
3851 else
3852 dev_seq_printf_stats(seq, v);
3853 return 0;
3856 static struct softnet_data *softnet_get_online(loff_t *pos)
3858 struct softnet_data *sd = NULL;
3860 while (*pos < nr_cpu_ids)
3861 if (cpu_online(*pos)) {
3862 sd = &per_cpu(softnet_data, *pos);
3863 break;
3864 } else
3865 ++*pos;
3866 return sd;
3869 static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
3871 return softnet_get_online(pos);
3874 static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3876 ++*pos;
3877 return softnet_get_online(pos);
3880 static void softnet_seq_stop(struct seq_file *seq, void *v)
3884 static int softnet_seq_show(struct seq_file *seq, void *v)
3886 struct softnet_data *sd = v;
3888 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
3889 sd->processed, sd->dropped, sd->time_squeeze, 0,
3890 0, 0, 0, 0, /* was fastroute */
3891 sd->cpu_collision, sd->received_rps);
3892 return 0;
3895 static const struct seq_operations dev_seq_ops = {
3896 .start = dev_seq_start,
3897 .next = dev_seq_next,
3898 .stop = dev_seq_stop,
3899 .show = dev_seq_show,
3902 static int dev_seq_open(struct inode *inode, struct file *file)
3904 return seq_open_net(inode, file, &dev_seq_ops,
3905 sizeof(struct seq_net_private));
3908 static const struct file_operations dev_seq_fops = {
3909 .owner = THIS_MODULE,
3910 .open = dev_seq_open,
3911 .read = seq_read,
3912 .llseek = seq_lseek,
3913 .release = seq_release_net,
3916 static const struct seq_operations softnet_seq_ops = {
3917 .start = softnet_seq_start,
3918 .next = softnet_seq_next,
3919 .stop = softnet_seq_stop,
3920 .show = softnet_seq_show,
3923 static int softnet_seq_open(struct inode *inode, struct file *file)
3925 return seq_open(file, &softnet_seq_ops);
3928 static const struct file_operations softnet_seq_fops = {
3929 .owner = THIS_MODULE,
3930 .open = softnet_seq_open,
3931 .read = seq_read,
3932 .llseek = seq_lseek,
3933 .release = seq_release,
3936 static void *ptype_get_idx(loff_t pos)
3938 struct packet_type *pt = NULL;
3939 loff_t i = 0;
3940 int t;
3942 list_for_each_entry_rcu(pt, &ptype_all, list) {
3943 if (i == pos)
3944 return pt;
3945 ++i;
3948 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
3949 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
3950 if (i == pos)
3951 return pt;
3952 ++i;
3955 return NULL;
3958 static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
3959 __acquires(RCU)
3961 rcu_read_lock();
3962 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
3965 static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3967 struct packet_type *pt;
3968 struct list_head *nxt;
3969 int hash;
3971 ++*pos;
3972 if (v == SEQ_START_TOKEN)
3973 return ptype_get_idx(0);
3975 pt = v;
3976 nxt = pt->list.next;
3977 if (pt->type == htons(ETH_P_ALL)) {
3978 if (nxt != &ptype_all)
3979 goto found;
3980 hash = 0;
3981 nxt = ptype_base[0].next;
3982 } else
3983 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
3985 while (nxt == &ptype_base[hash]) {
3986 if (++hash >= PTYPE_HASH_SIZE)
3987 return NULL;
3988 nxt = ptype_base[hash].next;
3990 found:
3991 return list_entry(nxt, struct packet_type, list);
3994 static void ptype_seq_stop(struct seq_file *seq, void *v)
3995 __releases(RCU)
3997 rcu_read_unlock();
4000 static int ptype_seq_show(struct seq_file *seq, void *v)
4002 struct packet_type *pt = v;
4004 if (v == SEQ_START_TOKEN)
4005 seq_puts(seq, "Type Device Function\n");
4006 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
4007 if (pt->type == htons(ETH_P_ALL))
4008 seq_puts(seq, "ALL ");
4009 else
4010 seq_printf(seq, "%04x", ntohs(pt->type));
4012 seq_printf(seq, " %-8s %pF\n",
4013 pt->dev ? pt->dev->name : "", pt->func);
4016 return 0;
4019 static const struct seq_operations ptype_seq_ops = {
4020 .start = ptype_seq_start,
4021 .next = ptype_seq_next,
4022 .stop = ptype_seq_stop,
4023 .show = ptype_seq_show,
4026 static int ptype_seq_open(struct inode *inode, struct file *file)
4028 return seq_open_net(inode, file, &ptype_seq_ops,
4029 sizeof(struct seq_net_private));
4032 static const struct file_operations ptype_seq_fops = {
4033 .owner = THIS_MODULE,
4034 .open = ptype_seq_open,
4035 .read = seq_read,
4036 .llseek = seq_lseek,
4037 .release = seq_release_net,
4041 static int __net_init dev_proc_net_init(struct net *net)
4043 int rc = -ENOMEM;
4045 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
4046 goto out;
4047 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
4048 goto out_dev;
4049 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
4050 goto out_softnet;
4052 if (wext_proc_init(net))
4053 goto out_ptype;
4054 rc = 0;
4055 out:
4056 return rc;
4057 out_ptype:
4058 proc_net_remove(net, "ptype");
4059 out_softnet:
4060 proc_net_remove(net, "softnet_stat");
4061 out_dev:
4062 proc_net_remove(net, "dev");
4063 goto out;
4066 static void __net_exit dev_proc_net_exit(struct net *net)
4068 wext_proc_exit(net);
4070 proc_net_remove(net, "ptype");
4071 proc_net_remove(net, "softnet_stat");
4072 proc_net_remove(net, "dev");
4075 static struct pernet_operations __net_initdata dev_proc_ops = {
4076 .init = dev_proc_net_init,
4077 .exit = dev_proc_net_exit,
4080 static int __init dev_proc_init(void)
4082 return register_pernet_subsys(&dev_proc_ops);
4084 #else
4085 #define dev_proc_init() 0
4086 #endif /* CONFIG_PROC_FS */
4090 * netdev_set_master - set up master/slave pair
4091 * @slave: slave device
4092 * @master: new master device
4094 * Changes the master device of the slave. Pass %NULL to break the
4095 * bonding. The caller must hold the RTNL semaphore. On a failure
4096 * a negative errno code is returned. On success the reference counts
4097 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the
4098 * function returns zero.
4100 int netdev_set_master(struct net_device *slave, struct net_device *master)
4102 struct net_device *old = slave->master;
4104 ASSERT_RTNL();
4106 if (master) {
4107 if (old)
4108 return -EBUSY;
4109 dev_hold(master);
4112 slave->master = master;
4114 if (old) {
4115 synchronize_net();
4116 dev_put(old);
4118 if (master)
4119 slave->flags |= IFF_SLAVE;
4120 else
4121 slave->flags &= ~IFF_SLAVE;
4123 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4124 return 0;
4126 EXPORT_SYMBOL(netdev_set_master);
4128 static void dev_change_rx_flags(struct net_device *dev, int flags)
4130 const struct net_device_ops *ops = dev->netdev_ops;
4132 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4133 ops->ndo_change_rx_flags(dev, flags);
4136 static int __dev_set_promiscuity(struct net_device *dev, int inc)
4138 unsigned short old_flags = dev->flags;
4139 uid_t uid;
4140 gid_t gid;
4142 ASSERT_RTNL();
4144 dev->flags |= IFF_PROMISC;
4145 dev->promiscuity += inc;
4146 if (dev->promiscuity == 0) {
4148 * Avoid overflow.
4149 * If inc causes overflow, untouch promisc and return error.
4151 if (inc < 0)
4152 dev->flags &= ~IFF_PROMISC;
4153 else {
4154 dev->promiscuity -= inc;
4155 printk(KERN_WARNING "%s: promiscuity touches roof, "
4156 "set promiscuity failed, promiscuity feature "
4157 "of device might be broken.\n", dev->name);
4158 return -EOVERFLOW;
4161 if (dev->flags != old_flags) {
4162 printk(KERN_INFO "device %s %s promiscuous mode\n",
4163 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4164 "left");
4165 if (audit_enabled) {
4166 current_uid_gid(&uid, &gid);
4167 audit_log(current->audit_context, GFP_ATOMIC,
4168 AUDIT_ANOM_PROMISCUOUS,
4169 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4170 dev->name, (dev->flags & IFF_PROMISC),
4171 (old_flags & IFF_PROMISC),
4172 audit_get_loginuid(current),
4173 uid, gid,
4174 audit_get_sessionid(current));
4177 dev_change_rx_flags(dev, IFF_PROMISC);
4179 return 0;
4183 * dev_set_promiscuity - update promiscuity count on a device
4184 * @dev: device
4185 * @inc: modifier
4187 * Add or remove promiscuity from a device. While the count in the device
4188 * remains above zero the interface remains promiscuous. Once it hits zero
4189 * the device reverts back to normal filtering operation. A negative inc
4190 * value is used to drop promiscuity on the device.
4191 * Return 0 if successful or a negative errno code on error.
4193 int dev_set_promiscuity(struct net_device *dev, int inc)
4195 unsigned short old_flags = dev->flags;
4196 int err;
4198 err = __dev_set_promiscuity(dev, inc);
4199 if (err < 0)
4200 return err;
4201 if (dev->flags != old_flags)
4202 dev_set_rx_mode(dev);
4203 return err;
4205 EXPORT_SYMBOL(dev_set_promiscuity);
4208 * dev_set_allmulti - update allmulti count on a device
4209 * @dev: device
4210 * @inc: modifier
4212 * Add or remove reception of all multicast frames to a device. While the
4213 * count in the device remains above zero the interface remains listening
4214 * to all interfaces. Once it hits zero the device reverts back to normal
4215 * filtering operation. A negative @inc value is used to drop the counter
4216 * when releasing a resource needing all multicasts.
4217 * Return 0 if successful or a negative errno code on error.
4220 int dev_set_allmulti(struct net_device *dev, int inc)
4222 unsigned short old_flags = dev->flags;
4224 ASSERT_RTNL();
4226 dev->flags |= IFF_ALLMULTI;
4227 dev->allmulti += inc;
4228 if (dev->allmulti == 0) {
4230 * Avoid overflow.
4231 * If inc causes overflow, untouch allmulti and return error.
4233 if (inc < 0)
4234 dev->flags &= ~IFF_ALLMULTI;
4235 else {
4236 dev->allmulti -= inc;
4237 printk(KERN_WARNING "%s: allmulti touches roof, "
4238 "set allmulti failed, allmulti feature of "
4239 "device might be broken.\n", dev->name);
4240 return -EOVERFLOW;
4243 if (dev->flags ^ old_flags) {
4244 dev_change_rx_flags(dev, IFF_ALLMULTI);
4245 dev_set_rx_mode(dev);
4247 return 0;
4249 EXPORT_SYMBOL(dev_set_allmulti);
4252 * Upload unicast and multicast address lists to device and
4253 * configure RX filtering. When the device doesn't support unicast
4254 * filtering it is put in promiscuous mode while unicast addresses
4255 * are present.
4257 void __dev_set_rx_mode(struct net_device *dev)
4259 const struct net_device_ops *ops = dev->netdev_ops;
4261 /* dev_open will call this function so the list will stay sane. */
4262 if (!(dev->flags&IFF_UP))
4263 return;
4265 if (!netif_device_present(dev))
4266 return;
4268 if (ops->ndo_set_rx_mode)
4269 ops->ndo_set_rx_mode(dev);
4270 else {
4271 /* Unicast addresses changes may only happen under the rtnl,
4272 * therefore calling __dev_set_promiscuity here is safe.
4274 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4275 __dev_set_promiscuity(dev, 1);
4276 dev->uc_promisc = 1;
4277 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4278 __dev_set_promiscuity(dev, -1);
4279 dev->uc_promisc = 0;
4282 if (ops->ndo_set_multicast_list)
4283 ops->ndo_set_multicast_list(dev);
4287 void dev_set_rx_mode(struct net_device *dev)
4289 netif_addr_lock_bh(dev);
4290 __dev_set_rx_mode(dev);
4291 netif_addr_unlock_bh(dev);
4295 * dev_get_flags - get flags reported to userspace
4296 * @dev: device
4298 * Get the combination of flag bits exported through APIs to userspace.
4300 unsigned dev_get_flags(const struct net_device *dev)
4302 unsigned flags;
4304 flags = (dev->flags & ~(IFF_PROMISC |
4305 IFF_ALLMULTI |
4306 IFF_RUNNING |
4307 IFF_LOWER_UP |
4308 IFF_DORMANT)) |
4309 (dev->gflags & (IFF_PROMISC |
4310 IFF_ALLMULTI));
4312 if (netif_running(dev)) {
4313 if (netif_oper_up(dev))
4314 flags |= IFF_RUNNING;
4315 if (netif_carrier_ok(dev))
4316 flags |= IFF_LOWER_UP;
4317 if (netif_dormant(dev))
4318 flags |= IFF_DORMANT;
4321 return flags;
4323 EXPORT_SYMBOL(dev_get_flags);
4325 int __dev_change_flags(struct net_device *dev, unsigned int flags)
4327 int old_flags = dev->flags;
4328 int ret;
4330 ASSERT_RTNL();
4333 * Set the flags on our device.
4336 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4337 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4338 IFF_AUTOMEDIA)) |
4339 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4340 IFF_ALLMULTI));
4343 * Load in the correct multicast list now the flags have changed.
4346 if ((old_flags ^ flags) & IFF_MULTICAST)
4347 dev_change_rx_flags(dev, IFF_MULTICAST);
4349 dev_set_rx_mode(dev);
4352 * Have we downed the interface. We handle IFF_UP ourselves
4353 * according to user attempts to set it, rather than blindly
4354 * setting it.
4357 ret = 0;
4358 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
4359 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
4361 if (!ret)
4362 dev_set_rx_mode(dev);
4365 if ((flags ^ dev->gflags) & IFF_PROMISC) {
4366 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4368 dev->gflags ^= IFF_PROMISC;
4369 dev_set_promiscuity(dev, inc);
4372 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4373 is important. Some (broken) drivers set IFF_PROMISC, when
4374 IFF_ALLMULTI is requested not asking us and not reporting.
4376 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4377 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4379 dev->gflags ^= IFF_ALLMULTI;
4380 dev_set_allmulti(dev, inc);
4383 return ret;
4386 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4388 unsigned int changes = dev->flags ^ old_flags;
4390 if (changes & IFF_UP) {
4391 if (dev->flags & IFF_UP)
4392 call_netdevice_notifiers(NETDEV_UP, dev);
4393 else
4394 call_netdevice_notifiers(NETDEV_DOWN, dev);
4397 if (dev->flags & IFF_UP &&
4398 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4399 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4403 * dev_change_flags - change device settings
4404 * @dev: device
4405 * @flags: device state flags
4407 * Change settings on device based state flags. The flags are
4408 * in the userspace exported format.
4410 int dev_change_flags(struct net_device *dev, unsigned flags)
4412 int ret, changes;
4413 int old_flags = dev->flags;
4415 ret = __dev_change_flags(dev, flags);
4416 if (ret < 0)
4417 return ret;
4419 changes = old_flags ^ dev->flags;
4420 if (changes)
4421 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4423 __dev_notify_flags(dev, old_flags);
4424 return ret;
4426 EXPORT_SYMBOL(dev_change_flags);
4429 * dev_set_mtu - Change maximum transfer unit
4430 * @dev: device
4431 * @new_mtu: new transfer unit
4433 * Change the maximum transfer size of the network device.
4435 int dev_set_mtu(struct net_device *dev, int new_mtu)
4437 const struct net_device_ops *ops = dev->netdev_ops;
4438 int err;
4440 if (new_mtu == dev->mtu)
4441 return 0;
4443 /* MTU must be positive. */
4444 if (new_mtu < 0)
4445 return -EINVAL;
4447 if (!netif_device_present(dev))
4448 return -ENODEV;
4450 err = 0;
4451 if (ops->ndo_change_mtu)
4452 err = ops->ndo_change_mtu(dev, new_mtu);
4453 else
4454 dev->mtu = new_mtu;
4456 if (!err && dev->flags & IFF_UP)
4457 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4458 return err;
4460 EXPORT_SYMBOL(dev_set_mtu);
4463 * dev_set_mac_address - Change Media Access Control Address
4464 * @dev: device
4465 * @sa: new address
4467 * Change the hardware (MAC) address of the device
4469 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4471 const struct net_device_ops *ops = dev->netdev_ops;
4472 int err;
4474 if (!ops->ndo_set_mac_address)
4475 return -EOPNOTSUPP;
4476 if (sa->sa_family != dev->type)
4477 return -EINVAL;
4478 if (!netif_device_present(dev))
4479 return -ENODEV;
4480 err = ops->ndo_set_mac_address(dev, sa);
4481 if (!err)
4482 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4483 return err;
4485 EXPORT_SYMBOL(dev_set_mac_address);
4488 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
4490 static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
4492 int err;
4493 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
4495 if (!dev)
4496 return -ENODEV;
4498 switch (cmd) {
4499 case SIOCGIFFLAGS: /* Get interface flags */
4500 ifr->ifr_flags = (short) dev_get_flags(dev);
4501 return 0;
4503 case SIOCGIFMETRIC: /* Get the metric on the interface
4504 (currently unused) */
4505 ifr->ifr_metric = 0;
4506 return 0;
4508 case SIOCGIFMTU: /* Get the MTU of a device */
4509 ifr->ifr_mtu = dev->mtu;
4510 return 0;
4512 case SIOCGIFHWADDR:
4513 if (!dev->addr_len)
4514 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4515 else
4516 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4517 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4518 ifr->ifr_hwaddr.sa_family = dev->type;
4519 return 0;
4521 case SIOCGIFSLAVE:
4522 err = -EINVAL;
4523 break;
4525 case SIOCGIFMAP:
4526 ifr->ifr_map.mem_start = dev->mem_start;
4527 ifr->ifr_map.mem_end = dev->mem_end;
4528 ifr->ifr_map.base_addr = dev->base_addr;
4529 ifr->ifr_map.irq = dev->irq;
4530 ifr->ifr_map.dma = dev->dma;
4531 ifr->ifr_map.port = dev->if_port;
4532 return 0;
4534 case SIOCGIFINDEX:
4535 ifr->ifr_ifindex = dev->ifindex;
4536 return 0;
4538 case SIOCGIFTXQLEN:
4539 ifr->ifr_qlen = dev->tx_queue_len;
4540 return 0;
4542 default:
4543 /* dev_ioctl() should ensure this case
4544 * is never reached
4546 WARN_ON(1);
4547 err = -EINVAL;
4548 break;
4551 return err;
4555 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4557 static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4559 int err;
4560 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
4561 const struct net_device_ops *ops;
4563 if (!dev)
4564 return -ENODEV;
4566 ops = dev->netdev_ops;
4568 switch (cmd) {
4569 case SIOCSIFFLAGS: /* Set interface flags */
4570 return dev_change_flags(dev, ifr->ifr_flags);
4572 case SIOCSIFMETRIC: /* Set the metric on the interface
4573 (currently unused) */
4574 return -EOPNOTSUPP;
4576 case SIOCSIFMTU: /* Set the MTU of a device */
4577 return dev_set_mtu(dev, ifr->ifr_mtu);
4579 case SIOCSIFHWADDR:
4580 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
4582 case SIOCSIFHWBROADCAST:
4583 if (ifr->ifr_hwaddr.sa_family != dev->type)
4584 return -EINVAL;
4585 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4586 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4587 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4588 return 0;
4590 case SIOCSIFMAP:
4591 if (ops->ndo_set_config) {
4592 if (!netif_device_present(dev))
4593 return -ENODEV;
4594 return ops->ndo_set_config(dev, &ifr->ifr_map);
4596 return -EOPNOTSUPP;
4598 case SIOCADDMULTI:
4599 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4600 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4601 return -EINVAL;
4602 if (!netif_device_present(dev))
4603 return -ENODEV;
4604 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
4606 case SIOCDELMULTI:
4607 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4608 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4609 return -EINVAL;
4610 if (!netif_device_present(dev))
4611 return -ENODEV;
4612 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
4614 case SIOCSIFTXQLEN:
4615 if (ifr->ifr_qlen < 0)
4616 return -EINVAL;
4617 dev->tx_queue_len = ifr->ifr_qlen;
4618 return 0;
4620 case SIOCSIFNAME:
4621 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4622 return dev_change_name(dev, ifr->ifr_newname);
4625 * Unknown or private ioctl
4627 default:
4628 if ((cmd >= SIOCDEVPRIVATE &&
4629 cmd <= SIOCDEVPRIVATE + 15) ||
4630 cmd == SIOCBONDENSLAVE ||
4631 cmd == SIOCBONDRELEASE ||
4632 cmd == SIOCBONDSETHWADDR ||
4633 cmd == SIOCBONDSLAVEINFOQUERY ||
4634 cmd == SIOCBONDINFOQUERY ||
4635 cmd == SIOCBONDCHANGEACTIVE ||
4636 cmd == SIOCGMIIPHY ||
4637 cmd == SIOCGMIIREG ||
4638 cmd == SIOCSMIIREG ||
4639 cmd == SIOCBRADDIF ||
4640 cmd == SIOCBRDELIF ||
4641 cmd == SIOCSHWTSTAMP ||
4642 cmd == SIOCWANDEV) {
4643 err = -EOPNOTSUPP;
4644 if (ops->ndo_do_ioctl) {
4645 if (netif_device_present(dev))
4646 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4647 else
4648 err = -ENODEV;
4650 } else
4651 err = -EINVAL;
4654 return err;
4658 * This function handles all "interface"-type I/O control requests. The actual
4659 * 'doing' part of this is dev_ifsioc above.
4663 * dev_ioctl - network device ioctl
4664 * @net: the applicable net namespace
4665 * @cmd: command to issue
4666 * @arg: pointer to a struct ifreq in user space
4668 * Issue ioctl functions to devices. This is normally called by the
4669 * user space syscall interfaces but can sometimes be useful for
4670 * other purposes. The return value is the return from the syscall if
4671 * positive or a negative errno code on error.
4674 int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
4676 struct ifreq ifr;
4677 int ret;
4678 char *colon;
4680 /* One special case: SIOCGIFCONF takes ifconf argument
4681 and requires shared lock, because it sleeps writing
4682 to user space.
4685 if (cmd == SIOCGIFCONF) {
4686 rtnl_lock();
4687 ret = dev_ifconf(net, (char __user *) arg);
4688 rtnl_unlock();
4689 return ret;
4691 if (cmd == SIOCGIFNAME)
4692 return dev_ifname(net, (struct ifreq __user *)arg);
4694 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4695 return -EFAULT;
4697 ifr.ifr_name[IFNAMSIZ-1] = 0;
4699 colon = strchr(ifr.ifr_name, ':');
4700 if (colon)
4701 *colon = 0;
4704 * See which interface the caller is talking about.
4707 switch (cmd) {
4709 * These ioctl calls:
4710 * - can be done by all.
4711 * - atomic and do not require locking.
4712 * - return a value
4714 case SIOCGIFFLAGS:
4715 case SIOCGIFMETRIC:
4716 case SIOCGIFMTU:
4717 case SIOCGIFHWADDR:
4718 case SIOCGIFSLAVE:
4719 case SIOCGIFMAP:
4720 case SIOCGIFINDEX:
4721 case SIOCGIFTXQLEN:
4722 dev_load(net, ifr.ifr_name);
4723 rcu_read_lock();
4724 ret = dev_ifsioc_locked(net, &ifr, cmd);
4725 rcu_read_unlock();
4726 if (!ret) {
4727 if (colon)
4728 *colon = ':';
4729 if (copy_to_user(arg, &ifr,
4730 sizeof(struct ifreq)))
4731 ret = -EFAULT;
4733 return ret;
4735 case SIOCETHTOOL:
4736 dev_load(net, ifr.ifr_name);
4737 rtnl_lock();
4738 ret = dev_ethtool(net, &ifr);
4739 rtnl_unlock();
4740 if (!ret) {
4741 if (colon)
4742 *colon = ':';
4743 if (copy_to_user(arg, &ifr,
4744 sizeof(struct ifreq)))
4745 ret = -EFAULT;
4747 return ret;
4750 * These ioctl calls:
4751 * - require superuser power.
4752 * - require strict serialization.
4753 * - return a value
4755 case SIOCGMIIPHY:
4756 case SIOCGMIIREG:
4757 case SIOCSIFNAME:
4758 if (!capable(CAP_NET_ADMIN))
4759 return -EPERM;
4760 dev_load(net, ifr.ifr_name);
4761 rtnl_lock();
4762 ret = dev_ifsioc(net, &ifr, cmd);
4763 rtnl_unlock();
4764 if (!ret) {
4765 if (colon)
4766 *colon = ':';
4767 if (copy_to_user(arg, &ifr,
4768 sizeof(struct ifreq)))
4769 ret = -EFAULT;
4771 return ret;
4774 * These ioctl calls:
4775 * - require superuser power.
4776 * - require strict serialization.
4777 * - do not return a value
4779 case SIOCSIFFLAGS:
4780 case SIOCSIFMETRIC:
4781 case SIOCSIFMTU:
4782 case SIOCSIFMAP:
4783 case SIOCSIFHWADDR:
4784 case SIOCSIFSLAVE:
4785 case SIOCADDMULTI:
4786 case SIOCDELMULTI:
4787 case SIOCSIFHWBROADCAST:
4788 case SIOCSIFTXQLEN:
4789 case SIOCSMIIREG:
4790 case SIOCBONDENSLAVE:
4791 case SIOCBONDRELEASE:
4792 case SIOCBONDSETHWADDR:
4793 case SIOCBONDCHANGEACTIVE:
4794 case SIOCBRADDIF:
4795 case SIOCBRDELIF:
4796 case SIOCSHWTSTAMP:
4797 if (!capable(CAP_NET_ADMIN))
4798 return -EPERM;
4799 /* fall through */
4800 case SIOCBONDSLAVEINFOQUERY:
4801 case SIOCBONDINFOQUERY:
4802 dev_load(net, ifr.ifr_name);
4803 rtnl_lock();
4804 ret = dev_ifsioc(net, &ifr, cmd);
4805 rtnl_unlock();
4806 return ret;
4808 case SIOCGIFMEM:
4809 /* Get the per device memory space. We can add this but
4810 * currently do not support it */
4811 case SIOCSIFMEM:
4812 /* Set the per device memory buffer space.
4813 * Not applicable in our case */
4814 case SIOCSIFLINK:
4815 return -EINVAL;
4818 * Unknown or private ioctl.
4820 default:
4821 if (cmd == SIOCWANDEV ||
4822 (cmd >= SIOCDEVPRIVATE &&
4823 cmd <= SIOCDEVPRIVATE + 15)) {
4824 dev_load(net, ifr.ifr_name);
4825 rtnl_lock();
4826 ret = dev_ifsioc(net, &ifr, cmd);
4827 rtnl_unlock();
4828 if (!ret && copy_to_user(arg, &ifr,
4829 sizeof(struct ifreq)))
4830 ret = -EFAULT;
4831 return ret;
4833 /* Take care of Wireless Extensions */
4834 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
4835 return wext_handle_ioctl(net, &ifr, cmd, arg);
4836 return -EINVAL;
4842 * dev_new_index - allocate an ifindex
4843 * @net: the applicable net namespace
4845 * Returns a suitable unique value for a new device interface
4846 * number. The caller must hold the rtnl semaphore or the
4847 * dev_base_lock to be sure it remains unique.
4849 static int dev_new_index(struct net *net)
4851 static int ifindex;
4852 for (;;) {
4853 if (++ifindex <= 0)
4854 ifindex = 1;
4855 if (!__dev_get_by_index(net, ifindex))
4856 return ifindex;
4860 /* Delayed registration/unregisteration */
4861 static LIST_HEAD(net_todo_list);
4863 static void net_set_todo(struct net_device *dev)
4865 list_add_tail(&dev->todo_list, &net_todo_list);
4868 static void rollback_registered_many(struct list_head *head)
4870 struct net_device *dev, *tmp;
4872 BUG_ON(dev_boot_phase);
4873 ASSERT_RTNL();
4875 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
4876 /* Some devices call without registering
4877 * for initialization unwind. Remove those
4878 * devices and proceed with the remaining.
4880 if (dev->reg_state == NETREG_UNINITIALIZED) {
4881 pr_debug("unregister_netdevice: device %s/%p never "
4882 "was registered\n", dev->name, dev);
4884 WARN_ON(1);
4885 list_del(&dev->unreg_list);
4886 continue;
4889 BUG_ON(dev->reg_state != NETREG_REGISTERED);
4891 /* If device is running, close it first. */
4892 dev_close(dev);
4894 /* And unlink it from device chain. */
4895 unlist_netdevice(dev);
4897 dev->reg_state = NETREG_UNREGISTERING;
4900 synchronize_net();
4902 list_for_each_entry(dev, head, unreg_list) {
4903 /* Shutdown queueing discipline. */
4904 dev_shutdown(dev);
4907 /* Notify protocols, that we are about to destroy
4908 this device. They should clean all the things.
4910 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4912 if (!dev->rtnl_link_ops ||
4913 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
4914 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
4917 * Flush the unicast and multicast chains
4919 dev_uc_flush(dev);
4920 dev_mc_flush(dev);
4922 if (dev->netdev_ops->ndo_uninit)
4923 dev->netdev_ops->ndo_uninit(dev);
4925 /* Notifier chain MUST detach us from master device. */
4926 WARN_ON(dev->master);
4928 /* Remove entries from kobject tree */
4929 netdev_unregister_kobject(dev);
4932 /* Process any work delayed until the end of the batch */
4933 dev = list_first_entry(head, struct net_device, unreg_list);
4934 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
4936 rcu_barrier();
4938 list_for_each_entry(dev, head, unreg_list)
4939 dev_put(dev);
4942 static void rollback_registered(struct net_device *dev)
4944 LIST_HEAD(single);
4946 list_add(&dev->unreg_list, &single);
4947 rollback_registered_many(&single);
4950 unsigned long netdev_fix_features(unsigned long features, const char *name)
4952 /* Fix illegal SG+CSUM combinations. */
4953 if ((features & NETIF_F_SG) &&
4954 !(features & NETIF_F_ALL_CSUM)) {
4955 if (name)
4956 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
4957 "checksum feature.\n", name);
4958 features &= ~NETIF_F_SG;
4961 /* TSO requires that SG is present as well. */
4962 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
4963 if (name)
4964 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
4965 "SG feature.\n", name);
4966 features &= ~NETIF_F_TSO;
4969 if (features & NETIF_F_UFO) {
4970 if (!(features & NETIF_F_GEN_CSUM)) {
4971 if (name)
4972 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4973 "since no NETIF_F_HW_CSUM feature.\n",
4974 name);
4975 features &= ~NETIF_F_UFO;
4978 if (!(features & NETIF_F_SG)) {
4979 if (name)
4980 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4981 "since no NETIF_F_SG feature.\n", name);
4982 features &= ~NETIF_F_UFO;
4986 return features;
4988 EXPORT_SYMBOL(netdev_fix_features);
4991 * netif_stacked_transfer_operstate - transfer operstate
4992 * @rootdev: the root or lower level device to transfer state from
4993 * @dev: the device to transfer operstate to
4995 * Transfer operational state from root to device. This is normally
4996 * called when a stacking relationship exists between the root
4997 * device and the device(a leaf device).
4999 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5000 struct net_device *dev)
5002 if (rootdev->operstate == IF_OPER_DORMANT)
5003 netif_dormant_on(dev);
5004 else
5005 netif_dormant_off(dev);
5007 if (netif_carrier_ok(rootdev)) {
5008 if (!netif_carrier_ok(dev))
5009 netif_carrier_on(dev);
5010 } else {
5011 if (netif_carrier_ok(dev))
5012 netif_carrier_off(dev);
5015 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5017 static int netif_alloc_rx_queues(struct net_device *dev)
5019 #ifdef CONFIG_RPS
5020 unsigned int i, count = dev->num_rx_queues;
5021 struct netdev_rx_queue *rx;
5023 BUG_ON(count < 1);
5025 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5026 if (!rx) {
5027 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5028 return -ENOMEM;
5030 dev->_rx = rx;
5033 * Set a pointer to first element in the array which holds the
5034 * reference count.
5036 for (i = 0; i < count; i++)
5037 rx[i].first = rx;
5038 #endif
5039 return 0;
5042 static int netif_alloc_netdev_queues(struct net_device *dev)
5044 unsigned int count = dev->num_tx_queues;
5045 struct netdev_queue *tx;
5047 BUG_ON(count < 1);
5049 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5050 if (!tx) {
5051 pr_err("netdev: Unable to allocate %u tx queues.\n",
5052 count);
5053 return -ENOMEM;
5055 dev->_tx = tx;
5056 return 0;
5059 static void netdev_init_one_queue(struct net_device *dev,
5060 struct netdev_queue *queue,
5061 void *_unused)
5063 queue->dev = dev;
5065 /* Initialize queue lock */
5066 spin_lock_init(&queue->_xmit_lock);
5067 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5068 queue->xmit_lock_owner = -1;
5071 static void netdev_init_queues(struct net_device *dev)
5073 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5074 spin_lock_init(&dev->tx_global_lock);
5078 * register_netdevice - register a network device
5079 * @dev: device to register
5081 * Take a completed network device structure and add it to the kernel
5082 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5083 * chain. 0 is returned on success. A negative errno code is returned
5084 * on a failure to set up the device, or if the name is a duplicate.
5086 * Callers must hold the rtnl semaphore. You may want
5087 * register_netdev() instead of this.
5089 * BUGS:
5090 * The locking appears insufficient to guarantee two parallel registers
5091 * will not get the same name.
5094 int register_netdevice(struct net_device *dev)
5096 int ret;
5097 struct net *net = dev_net(dev);
5099 BUG_ON(dev_boot_phase);
5100 ASSERT_RTNL();
5102 might_sleep();
5104 /* When net_device's are persistent, this will be fatal. */
5105 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5106 BUG_ON(!net);
5108 spin_lock_init(&dev->addr_list_lock);
5109 netdev_set_addr_lockdep_class(dev);
5111 dev->iflink = -1;
5113 ret = netif_alloc_rx_queues(dev);
5114 if (ret)
5115 goto out;
5117 ret = netif_alloc_netdev_queues(dev);
5118 if (ret)
5119 goto out;
5121 netdev_init_queues(dev);
5123 /* Init, if this function is available */
5124 if (dev->netdev_ops->ndo_init) {
5125 ret = dev->netdev_ops->ndo_init(dev);
5126 if (ret) {
5127 if (ret > 0)
5128 ret = -EIO;
5129 goto out;
5133 ret = dev_get_valid_name(dev, dev->name, 0);
5134 if (ret)
5135 goto err_uninit;
5137 dev->ifindex = dev_new_index(net);
5138 if (dev->iflink == -1)
5139 dev->iflink = dev->ifindex;
5141 /* Fix illegal checksum combinations */
5142 if ((dev->features & NETIF_F_HW_CSUM) &&
5143 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5144 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
5145 dev->name);
5146 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5149 if ((dev->features & NETIF_F_NO_CSUM) &&
5150 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5151 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
5152 dev->name);
5153 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5156 dev->features = netdev_fix_features(dev->features, dev->name);
5158 /* Enable software GSO if SG is supported. */
5159 if (dev->features & NETIF_F_SG)
5160 dev->features |= NETIF_F_GSO;
5162 /* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
5163 * vlan_dev_init() will do the dev->features check, so these features
5164 * are enabled only if supported by underlying device.
5166 dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);
5168 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5169 ret = notifier_to_errno(ret);
5170 if (ret)
5171 goto err_uninit;
5173 ret = netdev_register_kobject(dev);
5174 if (ret)
5175 goto err_uninit;
5176 dev->reg_state = NETREG_REGISTERED;
5179 * Default initial state at registry is that the
5180 * device is present.
5183 set_bit(__LINK_STATE_PRESENT, &dev->state);
5185 dev_init_scheduler(dev);
5186 dev_hold(dev);
5187 list_netdevice(dev);
5189 /* Notify protocols, that a new device appeared. */
5190 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5191 ret = notifier_to_errno(ret);
5192 if (ret) {
5193 rollback_registered(dev);
5194 dev->reg_state = NETREG_UNREGISTERED;
5197 * Prevent userspace races by waiting until the network
5198 * device is fully setup before sending notifications.
5200 if (!dev->rtnl_link_ops ||
5201 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5202 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5204 out:
5205 return ret;
5207 err_uninit:
5208 if (dev->netdev_ops->ndo_uninit)
5209 dev->netdev_ops->ndo_uninit(dev);
5210 goto out;
5212 EXPORT_SYMBOL(register_netdevice);
5215 * init_dummy_netdev - init a dummy network device for NAPI
5216 * @dev: device to init
5218 * This takes a network device structure and initialize the minimum
5219 * amount of fields so it can be used to schedule NAPI polls without
5220 * registering a full blown interface. This is to be used by drivers
5221 * that need to tie several hardware interfaces to a single NAPI
5222 * poll scheduler due to HW limitations.
5224 int init_dummy_netdev(struct net_device *dev)
5226 /* Clear everything. Note we don't initialize spinlocks
5227 * are they aren't supposed to be taken by any of the
5228 * NAPI code and this dummy netdev is supposed to be
5229 * only ever used for NAPI polls
5231 memset(dev, 0, sizeof(struct net_device));
5233 /* make sure we BUG if trying to hit standard
5234 * register/unregister code path
5236 dev->reg_state = NETREG_DUMMY;
5238 /* NAPI wants this */
5239 INIT_LIST_HEAD(&dev->napi_list);
5241 /* a dummy interface is started by default */
5242 set_bit(__LINK_STATE_PRESENT, &dev->state);
5243 set_bit(__LINK_STATE_START, &dev->state);
5245 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5246 * because users of this 'device' dont need to change
5247 * its refcount.
5250 return 0;
5252 EXPORT_SYMBOL_GPL(init_dummy_netdev);
5256 * register_netdev - register a network device
5257 * @dev: device to register
5259 * Take a completed network device structure and add it to the kernel
5260 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5261 * chain. 0 is returned on success. A negative errno code is returned
5262 * on a failure to set up the device, or if the name is a duplicate.
5264 * This is a wrapper around register_netdevice that takes the rtnl semaphore
5265 * and expands the device name if you passed a format string to
5266 * alloc_netdev.
5268 int register_netdev(struct net_device *dev)
5270 int err;
5272 rtnl_lock();
5275 * If the name is a format string the caller wants us to do a
5276 * name allocation.
5278 if (strchr(dev->name, '%')) {
5279 err = dev_alloc_name(dev, dev->name);
5280 if (err < 0)
5281 goto out;
5284 err = register_netdevice(dev);
5285 out:
5286 rtnl_unlock();
5287 return err;
5289 EXPORT_SYMBOL(register_netdev);
5291 int netdev_refcnt_read(const struct net_device *dev)
5293 int i, refcnt = 0;
5295 for_each_possible_cpu(i)
5296 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5297 return refcnt;
5299 EXPORT_SYMBOL(netdev_refcnt_read);
5302 * netdev_wait_allrefs - wait until all references are gone.
5304 * This is called when unregistering network devices.
5306 * Any protocol or device that holds a reference should register
5307 * for netdevice notification, and cleanup and put back the
5308 * reference if they receive an UNREGISTER event.
5309 * We can get stuck here if buggy protocols don't correctly
5310 * call dev_put.
5312 static void netdev_wait_allrefs(struct net_device *dev)
5314 unsigned long rebroadcast_time, warning_time;
5315 int refcnt;
5317 linkwatch_forget_dev(dev);
5319 rebroadcast_time = warning_time = jiffies;
5320 refcnt = netdev_refcnt_read(dev);
5322 while (refcnt != 0) {
5323 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5324 rtnl_lock();
5326 /* Rebroadcast unregister notification */
5327 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5328 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
5329 * should have already handle it the first time */
5331 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5332 &dev->state)) {
5333 /* We must not have linkwatch events
5334 * pending on unregister. If this
5335 * happens, we simply run the queue
5336 * unscheduled, resulting in a noop
5337 * for this device.
5339 linkwatch_run_queue();
5342 __rtnl_unlock();
5344 rebroadcast_time = jiffies;
5347 msleep(250);
5349 refcnt = netdev_refcnt_read(dev);
5351 if (time_after(jiffies, warning_time + 10 * HZ)) {
5352 printk(KERN_EMERG "unregister_netdevice: "
5353 "waiting for %s to become free. Usage "
5354 "count = %d\n",
5355 dev->name, refcnt);
5356 warning_time = jiffies;
5361 /* The sequence is:
5363 * rtnl_lock();
5364 * ...
5365 * register_netdevice(x1);
5366 * register_netdevice(x2);
5367 * ...
5368 * unregister_netdevice(y1);
5369 * unregister_netdevice(y2);
5370 * ...
5371 * rtnl_unlock();
5372 * free_netdev(y1);
5373 * free_netdev(y2);
5375 * We are invoked by rtnl_unlock().
5376 * This allows us to deal with problems:
5377 * 1) We can delete sysfs objects which invoke hotplug
5378 * without deadlocking with linkwatch via keventd.
5379 * 2) Since we run with the RTNL semaphore not held, we can sleep
5380 * safely in order to wait for the netdev refcnt to drop to zero.
5382 * We must not return until all unregister events added during
5383 * the interval the lock was held have been completed.
5385 void netdev_run_todo(void)
5387 struct list_head list;
5389 /* Snapshot list, allow later requests */
5390 list_replace_init(&net_todo_list, &list);
5392 __rtnl_unlock();
5394 while (!list_empty(&list)) {
5395 struct net_device *dev
5396 = list_first_entry(&list, struct net_device, todo_list);
5397 list_del(&dev->todo_list);
5399 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5400 printk(KERN_ERR "network todo '%s' but state %d\n",
5401 dev->name, dev->reg_state);
5402 dump_stack();
5403 continue;
5406 dev->reg_state = NETREG_UNREGISTERED;
5408 on_each_cpu(flush_backlog, dev, 1);
5410 netdev_wait_allrefs(dev);
5412 /* paranoia */
5413 BUG_ON(netdev_refcnt_read(dev));
5414 WARN_ON(rcu_dereference_raw(dev->ip_ptr));
5415 WARN_ON(rcu_dereference_raw(dev->ip6_ptr));
5416 WARN_ON(dev->dn_ptr);
5418 if (dev->destructor)
5419 dev->destructor(dev);
5421 /* Free network device */
5422 kobject_put(&dev->dev.kobj);
5427 * dev_txq_stats_fold - fold tx_queues stats
5428 * @dev: device to get statistics from
5429 * @stats: struct rtnl_link_stats64 to hold results
5431 void dev_txq_stats_fold(const struct net_device *dev,
5432 struct rtnl_link_stats64 *stats)
5434 u64 tx_bytes = 0, tx_packets = 0, tx_dropped = 0;
5435 unsigned int i;
5436 struct netdev_queue *txq;
5438 for (i = 0; i < dev->num_tx_queues; i++) {
5439 txq = netdev_get_tx_queue(dev, i);
5440 spin_lock_bh(&txq->_xmit_lock);
5441 tx_bytes += txq->tx_bytes;
5442 tx_packets += txq->tx_packets;
5443 tx_dropped += txq->tx_dropped;
5444 spin_unlock_bh(&txq->_xmit_lock);
5446 if (tx_bytes || tx_packets || tx_dropped) {
5447 stats->tx_bytes = tx_bytes;
5448 stats->tx_packets = tx_packets;
5449 stats->tx_dropped = tx_dropped;
5452 EXPORT_SYMBOL(dev_txq_stats_fold);
5454 /* Convert net_device_stats to rtnl_link_stats64. They have the same
5455 * fields in the same order, with only the type differing.
5457 static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5458 const struct net_device_stats *netdev_stats)
5460 #if BITS_PER_LONG == 64
5461 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5462 memcpy(stats64, netdev_stats, sizeof(*stats64));
5463 #else
5464 size_t i, n = sizeof(*stats64) / sizeof(u64);
5465 const unsigned long *src = (const unsigned long *)netdev_stats;
5466 u64 *dst = (u64 *)stats64;
5468 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5469 sizeof(*stats64) / sizeof(u64));
5470 for (i = 0; i < n; i++)
5471 dst[i] = src[i];
5472 #endif
5476 * dev_get_stats - get network device statistics
5477 * @dev: device to get statistics from
5478 * @storage: place to store stats
5480 * Get network statistics from device. Return @storage.
5481 * The device driver may provide its own method by setting
5482 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5483 * otherwise the internal statistics structure is used.
5485 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5486 struct rtnl_link_stats64 *storage)
5488 const struct net_device_ops *ops = dev->netdev_ops;
5490 if (ops->ndo_get_stats64) {
5491 memset(storage, 0, sizeof(*storage));
5492 ops->ndo_get_stats64(dev, storage);
5493 } else if (ops->ndo_get_stats) {
5494 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
5495 } else {
5496 netdev_stats_to_stats64(storage, &dev->stats);
5497 dev_txq_stats_fold(dev, storage);
5499 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
5500 return storage;
5502 EXPORT_SYMBOL(dev_get_stats);
5504 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
5506 struct netdev_queue *queue = dev_ingress_queue(dev);
5508 #ifdef CONFIG_NET_CLS_ACT
5509 if (queue)
5510 return queue;
5511 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5512 if (!queue)
5513 return NULL;
5514 netdev_init_one_queue(dev, queue, NULL);
5515 queue->qdisc = &noop_qdisc;
5516 queue->qdisc_sleeping = &noop_qdisc;
5517 rcu_assign_pointer(dev->ingress_queue, queue);
5518 #endif
5519 return queue;
5523 * alloc_netdev_mq - allocate network device
5524 * @sizeof_priv: size of private data to allocate space for
5525 * @name: device name format string
5526 * @setup: callback to initialize device
5527 * @queue_count: the number of subqueues to allocate
5529 * Allocates a struct net_device with private data area for driver use
5530 * and performs basic initialization. Also allocates subquue structs
5531 * for each queue on the device at the end of the netdevice.
5533 struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
5534 void (*setup)(struct net_device *), unsigned int queue_count)
5536 struct net_device *dev;
5537 size_t alloc_size;
5538 struct net_device *p;
5540 BUG_ON(strlen(name) >= sizeof(dev->name));
5542 if (queue_count < 1) {
5543 pr_err("alloc_netdev: Unable to allocate device "
5544 "with zero queues.\n");
5545 return NULL;
5548 alloc_size = sizeof(struct net_device);
5549 if (sizeof_priv) {
5550 /* ensure 32-byte alignment of private area */
5551 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5552 alloc_size += sizeof_priv;
5554 /* ensure 32-byte alignment of whole construct */
5555 alloc_size += NETDEV_ALIGN - 1;
5557 p = kzalloc(alloc_size, GFP_KERNEL);
5558 if (!p) {
5559 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
5560 return NULL;
5563 dev = PTR_ALIGN(p, NETDEV_ALIGN);
5564 dev->padded = (char *)dev - (char *)p;
5566 dev->pcpu_refcnt = alloc_percpu(int);
5567 if (!dev->pcpu_refcnt)
5568 goto free_p;
5570 if (dev_addr_init(dev))
5571 goto free_pcpu;
5573 dev_mc_init(dev);
5574 dev_uc_init(dev);
5576 dev_net_set(dev, &init_net);
5578 dev->num_tx_queues = queue_count;
5579 dev->real_num_tx_queues = queue_count;
5581 #ifdef CONFIG_RPS
5582 dev->num_rx_queues = queue_count;
5583 dev->real_num_rx_queues = queue_count;
5584 #endif
5586 dev->gso_max_size = GSO_MAX_SIZE;
5588 INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
5589 dev->ethtool_ntuple_list.count = 0;
5590 INIT_LIST_HEAD(&dev->napi_list);
5591 INIT_LIST_HEAD(&dev->unreg_list);
5592 INIT_LIST_HEAD(&dev->link_watch_list);
5593 dev->priv_flags = IFF_XMIT_DST_RELEASE;
5594 setup(dev);
5595 strcpy(dev->name, name);
5596 return dev;
5598 free_pcpu:
5599 free_percpu(dev->pcpu_refcnt);
5600 free_p:
5601 kfree(p);
5602 return NULL;
5604 EXPORT_SYMBOL(alloc_netdev_mq);
5607 * free_netdev - free network device
5608 * @dev: device
5610 * This function does the last stage of destroying an allocated device
5611 * interface. The reference to the device object is released.
5612 * If this is the last reference then it will be freed.
5614 void free_netdev(struct net_device *dev)
5616 struct napi_struct *p, *n;
5618 release_net(dev_net(dev));
5620 kfree(dev->_tx);
5622 kfree(rcu_dereference_raw(dev->ingress_queue));
5624 /* Flush device addresses */
5625 dev_addr_flush(dev);
5627 /* Clear ethtool n-tuple list */
5628 ethtool_ntuple_flush(dev);
5630 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5631 netif_napi_del(p);
5633 free_percpu(dev->pcpu_refcnt);
5634 dev->pcpu_refcnt = NULL;
5636 /* Compatibility with error handling in drivers */
5637 if (dev->reg_state == NETREG_UNINITIALIZED) {
5638 kfree((char *)dev - dev->padded);
5639 return;
5642 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5643 dev->reg_state = NETREG_RELEASED;
5645 /* will free via device release */
5646 put_device(&dev->dev);
5648 EXPORT_SYMBOL(free_netdev);
5651 * synchronize_net - Synchronize with packet receive processing
5653 * Wait for packets currently being received to be done.
5654 * Does not block later packets from starting.
5656 void synchronize_net(void)
5658 might_sleep();
5659 synchronize_rcu();
5661 EXPORT_SYMBOL(synchronize_net);
5664 * unregister_netdevice_queue - remove device from the kernel
5665 * @dev: device
5666 * @head: list
5668 * This function shuts down a device interface and removes it
5669 * from the kernel tables.
5670 * If head not NULL, device is queued to be unregistered later.
5672 * Callers must hold the rtnl semaphore. You may want
5673 * unregister_netdev() instead of this.
5676 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
5678 ASSERT_RTNL();
5680 if (head) {
5681 list_move_tail(&dev->unreg_list, head);
5682 } else {
5683 rollback_registered(dev);
5684 /* Finish processing unregister after unlock */
5685 net_set_todo(dev);
5688 EXPORT_SYMBOL(unregister_netdevice_queue);
5691 * unregister_netdevice_many - unregister many devices
5692 * @head: list of devices
5694 void unregister_netdevice_many(struct list_head *head)
5696 struct net_device *dev;
5698 if (!list_empty(head)) {
5699 rollback_registered_many(head);
5700 list_for_each_entry(dev, head, unreg_list)
5701 net_set_todo(dev);
5704 EXPORT_SYMBOL(unregister_netdevice_many);
5707 * unregister_netdev - remove device from the kernel
5708 * @dev: device
5710 * This function shuts down a device interface and removes it
5711 * from the kernel tables.
5713 * This is just a wrapper for unregister_netdevice that takes
5714 * the rtnl semaphore. In general you want to use this and not
5715 * unregister_netdevice.
5717 void unregister_netdev(struct net_device *dev)
5719 rtnl_lock();
5720 unregister_netdevice(dev);
5721 rtnl_unlock();
5723 EXPORT_SYMBOL(unregister_netdev);
5726 * dev_change_net_namespace - move device to different nethost namespace
5727 * @dev: device
5728 * @net: network namespace
5729 * @pat: If not NULL name pattern to try if the current device name
5730 * is already taken in the destination network namespace.
5732 * This function shuts down a device interface and moves it
5733 * to a new network namespace. On success 0 is returned, on
5734 * a failure a netagive errno code is returned.
5736 * Callers must hold the rtnl semaphore.
5739 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5741 int err;
5743 ASSERT_RTNL();
5745 /* Don't allow namespace local devices to be moved. */
5746 err = -EINVAL;
5747 if (dev->features & NETIF_F_NETNS_LOCAL)
5748 goto out;
5750 /* Ensure the device has been registrered */
5751 err = -EINVAL;
5752 if (dev->reg_state != NETREG_REGISTERED)
5753 goto out;
5755 /* Get out if there is nothing todo */
5756 err = 0;
5757 if (net_eq(dev_net(dev), net))
5758 goto out;
5760 /* Pick the destination device name, and ensure
5761 * we can use it in the destination network namespace.
5763 err = -EEXIST;
5764 if (__dev_get_by_name(net, dev->name)) {
5765 /* We get here if we can't use the current device name */
5766 if (!pat)
5767 goto out;
5768 if (dev_get_valid_name(dev, pat, 1))
5769 goto out;
5773 * And now a mini version of register_netdevice unregister_netdevice.
5776 /* If device is running close it first. */
5777 dev_close(dev);
5779 /* And unlink it from device chain */
5780 err = -ENODEV;
5781 unlist_netdevice(dev);
5783 synchronize_net();
5785 /* Shutdown queueing discipline. */
5786 dev_shutdown(dev);
5788 /* Notify protocols, that we are about to destroy
5789 this device. They should clean all the things.
5791 Note that dev->reg_state stays at NETREG_REGISTERED.
5792 This is wanted because this way 8021q and macvlan know
5793 the device is just moving and can keep their slaves up.
5795 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5796 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5799 * Flush the unicast and multicast chains
5801 dev_uc_flush(dev);
5802 dev_mc_flush(dev);
5804 /* Actually switch the network namespace */
5805 dev_net_set(dev, net);
5807 /* If there is an ifindex conflict assign a new one */
5808 if (__dev_get_by_index(net, dev->ifindex)) {
5809 int iflink = (dev->iflink == dev->ifindex);
5810 dev->ifindex = dev_new_index(net);
5811 if (iflink)
5812 dev->iflink = dev->ifindex;
5815 /* Fixup kobjects */
5816 err = device_rename(&dev->dev, dev->name);
5817 WARN_ON(err);
5819 /* Add the device back in the hashes */
5820 list_netdevice(dev);
5822 /* Notify protocols, that a new device appeared. */
5823 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5826 * Prevent userspace races by waiting until the network
5827 * device is fully setup before sending notifications.
5829 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5831 synchronize_net();
5832 err = 0;
5833 out:
5834 return err;
5836 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
5838 static int dev_cpu_callback(struct notifier_block *nfb,
5839 unsigned long action,
5840 void *ocpu)
5842 struct sk_buff **list_skb;
5843 struct sk_buff *skb;
5844 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5845 struct softnet_data *sd, *oldsd;
5847 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
5848 return NOTIFY_OK;
5850 local_irq_disable();
5851 cpu = smp_processor_id();
5852 sd = &per_cpu(softnet_data, cpu);
5853 oldsd = &per_cpu(softnet_data, oldcpu);
5855 /* Find end of our completion_queue. */
5856 list_skb = &sd->completion_queue;
5857 while (*list_skb)
5858 list_skb = &(*list_skb)->next;
5859 /* Append completion queue from offline CPU. */
5860 *list_skb = oldsd->completion_queue;
5861 oldsd->completion_queue = NULL;
5863 /* Append output queue from offline CPU. */
5864 if (oldsd->output_queue) {
5865 *sd->output_queue_tailp = oldsd->output_queue;
5866 sd->output_queue_tailp = oldsd->output_queue_tailp;
5867 oldsd->output_queue = NULL;
5868 oldsd->output_queue_tailp = &oldsd->output_queue;
5871 raise_softirq_irqoff(NET_TX_SOFTIRQ);
5872 local_irq_enable();
5874 /* Process offline CPU's input_pkt_queue */
5875 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
5876 netif_rx(skb);
5877 input_queue_head_incr(oldsd);
5879 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
5880 netif_rx(skb);
5881 input_queue_head_incr(oldsd);
5884 return NOTIFY_OK;
5889 * netdev_increment_features - increment feature set by one
5890 * @all: current feature set
5891 * @one: new feature set
5892 * @mask: mask feature set
5894 * Computes a new feature set after adding a device with feature set
5895 * @one to the master device with current feature set @all. Will not
5896 * enable anything that is off in @mask. Returns the new feature set.
5898 unsigned long netdev_increment_features(unsigned long all, unsigned long one,
5899 unsigned long mask)
5901 /* If device needs checksumming, downgrade to it. */
5902 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
5903 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
5904 else if (mask & NETIF_F_ALL_CSUM) {
5905 /* If one device supports v4/v6 checksumming, set for all. */
5906 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
5907 !(all & NETIF_F_GEN_CSUM)) {
5908 all &= ~NETIF_F_ALL_CSUM;
5909 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
5912 /* If one device supports hw checksumming, set for all. */
5913 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
5914 all &= ~NETIF_F_ALL_CSUM;
5915 all |= NETIF_F_HW_CSUM;
5919 one |= NETIF_F_ALL_CSUM;
5921 one |= all & NETIF_F_ONE_FOR_ALL;
5922 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;
5923 all |= one & mask & NETIF_F_ONE_FOR_ALL;
5925 return all;
5927 EXPORT_SYMBOL(netdev_increment_features);
5929 static struct hlist_head *netdev_create_hash(void)
5931 int i;
5932 struct hlist_head *hash;
5934 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
5935 if (hash != NULL)
5936 for (i = 0; i < NETDEV_HASHENTRIES; i++)
5937 INIT_HLIST_HEAD(&hash[i]);
5939 return hash;
5942 /* Initialize per network namespace state */
5943 static int __net_init netdev_init(struct net *net)
5945 INIT_LIST_HEAD(&net->dev_base_head);
5947 net->dev_name_head = netdev_create_hash();
5948 if (net->dev_name_head == NULL)
5949 goto err_name;
5951 net->dev_index_head = netdev_create_hash();
5952 if (net->dev_index_head == NULL)
5953 goto err_idx;
5955 return 0;
5957 err_idx:
5958 kfree(net->dev_name_head);
5959 err_name:
5960 return -ENOMEM;
5964 * netdev_drivername - network driver for the device
5965 * @dev: network device
5966 * @buffer: buffer for resulting name
5967 * @len: size of buffer
5969 * Determine network driver for device.
5971 char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
5973 const struct device_driver *driver;
5974 const struct device *parent;
5976 if (len <= 0 || !buffer)
5977 return buffer;
5978 buffer[0] = 0;
5980 parent = dev->dev.parent;
5982 if (!parent)
5983 return buffer;
5985 driver = parent->driver;
5986 if (driver && driver->name)
5987 strlcpy(buffer, driver->name, len);
5988 return buffer;
5991 static int __netdev_printk(const char *level, const struct net_device *dev,
5992 struct va_format *vaf)
5994 int r;
5996 if (dev && dev->dev.parent)
5997 r = dev_printk(level, dev->dev.parent, "%s: %pV",
5998 netdev_name(dev), vaf);
5999 else if (dev)
6000 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6001 else
6002 r = printk("%s(NULL net_device): %pV", level, vaf);
6004 return r;
6007 int netdev_printk(const char *level, const struct net_device *dev,
6008 const char *format, ...)
6010 struct va_format vaf;
6011 va_list args;
6012 int r;
6014 va_start(args, format);
6016 vaf.fmt = format;
6017 vaf.va = &args;
6019 r = __netdev_printk(level, dev, &vaf);
6020 va_end(args);
6022 return r;
6024 EXPORT_SYMBOL(netdev_printk);
6026 #define define_netdev_printk_level(func, level) \
6027 int func(const struct net_device *dev, const char *fmt, ...) \
6029 int r; \
6030 struct va_format vaf; \
6031 va_list args; \
6033 va_start(args, fmt); \
6035 vaf.fmt = fmt; \
6036 vaf.va = &args; \
6038 r = __netdev_printk(level, dev, &vaf); \
6039 va_end(args); \
6041 return r; \
6043 EXPORT_SYMBOL(func);
6045 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6046 define_netdev_printk_level(netdev_alert, KERN_ALERT);
6047 define_netdev_printk_level(netdev_crit, KERN_CRIT);
6048 define_netdev_printk_level(netdev_err, KERN_ERR);
6049 define_netdev_printk_level(netdev_warn, KERN_WARNING);
6050 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6051 define_netdev_printk_level(netdev_info, KERN_INFO);
6053 static void __net_exit netdev_exit(struct net *net)
6055 kfree(net->dev_name_head);
6056 kfree(net->dev_index_head);
6059 static struct pernet_operations __net_initdata netdev_net_ops = {
6060 .init = netdev_init,
6061 .exit = netdev_exit,
6064 static void __net_exit default_device_exit(struct net *net)
6066 struct net_device *dev, *aux;
6068 * Push all migratable network devices back to the
6069 * initial network namespace
6071 rtnl_lock();
6072 for_each_netdev_safe(net, dev, aux) {
6073 int err;
6074 char fb_name[IFNAMSIZ];
6076 /* Ignore unmoveable devices (i.e. loopback) */
6077 if (dev->features & NETIF_F_NETNS_LOCAL)
6078 continue;
6080 /* Leave virtual devices for the generic cleanup */
6081 if (dev->rtnl_link_ops)
6082 continue;
6084 /* Push remaing network devices to init_net */
6085 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6086 err = dev_change_net_namespace(dev, &init_net, fb_name);
6087 if (err) {
6088 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
6089 __func__, dev->name, err);
6090 BUG();
6093 rtnl_unlock();
6096 static void __net_exit default_device_exit_batch(struct list_head *net_list)
6098 /* At exit all network devices most be removed from a network
6099 * namespace. Do this in the reverse order of registeration.
6100 * Do this across as many network namespaces as possible to
6101 * improve batching efficiency.
6103 struct net_device *dev;
6104 struct net *net;
6105 LIST_HEAD(dev_kill_list);
6107 rtnl_lock();
6108 list_for_each_entry(net, net_list, exit_list) {
6109 for_each_netdev_reverse(net, dev) {
6110 if (dev->rtnl_link_ops)
6111 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6112 else
6113 unregister_netdevice_queue(dev, &dev_kill_list);
6116 unregister_netdevice_many(&dev_kill_list);
6117 rtnl_unlock();
6120 static struct pernet_operations __net_initdata default_device_ops = {
6121 .exit = default_device_exit,
6122 .exit_batch = default_device_exit_batch,
6126 * Initialize the DEV module. At boot time this walks the device list and
6127 * unhooks any devices that fail to initialise (normally hardware not
6128 * present) and leaves us with a valid list of present and active devices.
6133 * This is called single threaded during boot, so no need
6134 * to take the rtnl semaphore.
6136 static int __init net_dev_init(void)
6138 int i, rc = -ENOMEM;
6140 BUG_ON(!dev_boot_phase);
6142 if (dev_proc_init())
6143 goto out;
6145 if (netdev_kobject_init())
6146 goto out;
6148 INIT_LIST_HEAD(&ptype_all);
6149 for (i = 0; i < PTYPE_HASH_SIZE; i++)
6150 INIT_LIST_HEAD(&ptype_base[i]);
6152 if (register_pernet_subsys(&netdev_net_ops))
6153 goto out;
6156 * Initialise the packet receive queues.
6159 for_each_possible_cpu(i) {
6160 struct softnet_data *sd = &per_cpu(softnet_data, i);
6162 memset(sd, 0, sizeof(*sd));
6163 skb_queue_head_init(&sd->input_pkt_queue);
6164 skb_queue_head_init(&sd->process_queue);
6165 sd->completion_queue = NULL;
6166 INIT_LIST_HEAD(&sd->poll_list);
6167 sd->output_queue = NULL;
6168 sd->output_queue_tailp = &sd->output_queue;
6169 #ifdef CONFIG_RPS
6170 sd->csd.func = rps_trigger_softirq;
6171 sd->csd.info = sd;
6172 sd->csd.flags = 0;
6173 sd->cpu = i;
6174 #endif
6176 sd->backlog.poll = process_backlog;
6177 sd->backlog.weight = weight_p;
6178 sd->backlog.gro_list = NULL;
6179 sd->backlog.gro_count = 0;
6182 dev_boot_phase = 0;
6184 /* The loopback device is special if any other network devices
6185 * is present in a network namespace the loopback device must
6186 * be present. Since we now dynamically allocate and free the
6187 * loopback device ensure this invariant is maintained by
6188 * keeping the loopback device as the first device on the
6189 * list of network devices. Ensuring the loopback devices
6190 * is the first device that appears and the last network device
6191 * that disappears.
6193 if (register_pernet_device(&loopback_net_ops))
6194 goto out;
6196 if (register_pernet_device(&default_device_ops))
6197 goto out;
6199 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6200 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
6202 hotcpu_notifier(dev_cpu_callback, 0);
6203 dst_init();
6204 dev_mcast_init();
6205 rc = 0;
6206 out:
6207 return rc;
6210 subsys_initcall(net_dev_init);
6212 static int __init initialize_hashrnd(void)
6214 get_random_bytes(&hashrnd, sizeof(hashrnd));
6215 return 0;
6218 late_initcall_sync(initialize_hashrnd);